#!/usr/bin/env python3
import re, sys, os, json, time, requests
DB_PATH = os.environ.get("DB_PATH", "/root/search.db")
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}
SITE_NAME = "\u6e58\u6f6d\u7ecf\u6d4e\u6280\u672f\u5f00\u53d1\u533a-\u901a\u77e5\u516c\u544a"
GROUP = "\u6e58\u6f6d"
PER_PAGE = 15
MAX_PAGES = 5
BASE_URL = "http://xtjkq.xiangtan.gov.cn/6046/6054/6059"
FULL_DOMAIN = "http://xtjkq.xiangtan.gov.cn"

def get_list_url(page):
    if page == 1:
        return BASE_URL + "/index.htm"
    else:
        return BASE_URL + "/index_%d.htm" % (page - 1)

def fetch_page(url, encoding="gb2312"):
    r = requests.get(url, headers=HEADERS, timeout=30, allow_redirects=True)
    r.encoding = encoding
    return r.text

def resolve_url(base_url, rel_url):
    if rel_url.startswith("http"):
        return rel_url
    if rel_url.startswith("/"):
        return FULL_DOMAIN + rel_url
    # Handle ../../../ path relative to base
    base_dir = base_url.rsplit("/", 1)[0] if "/" in base_url else base_url
    parts = base_dir.split("/")
    while rel_url.startswith("../"):
        rel_url = rel_url[3:]
        if parts:
            parts.pop()
    parts.append(rel_url)
    return "/".join(parts)

def extract_list_items(html, page=1):
    items = []
    m = re.search(r'<tbody>(.*?)</tbody>', html, re.DOTALL)
    if not m:
        return items
    tbody = m.group(1)
    for tr in re.findall(r'<tr>(.*?)</tr>', tbody, re.DOTALL):
        td = re.findall(r'<td>(.*?)</td>', tr, re.DOTALL)
        if len(td) >= 3:
            am = re.search(r'<a[^>]*title=(["\'])([^"\'<>]+)\1[^>]*href=(["\'])([^"\'<>]+)\3[^>]*>', td[1], re.DOTALL)
            if am:
                title = am.group(2).strip()
                rel_url = am.group(4).strip()
                date_str = td[2].strip()
                full_url = BASE_URL + "/" + rel_url
                items.append({"url": full_url, "title": title, "date": date_str})
    return items

def extract_content(html, source_url):
    title = ""
    m = re.search(r'<h2>(.*?)</h2>', html, re.DOTALL)
    if m:
        title = m.group(1).strip()
    else:
        m = re.search(r'<title>(.*?)</title>', html, re.DOTALL)
        if m:
            title = m.group(1).strip()
    pub_date = ""
    m = re.search(r'\u53d1\u5e03\u65f6\u95f4[\uff1a:]\s*(\d{4}-\d{1,2}-\d{1,2})', html)
    if m:
        pub_date = m.group(1).strip()
    content = ""
    attachments = []
    content_html = ""
    m = re.search(r'<div[^>]*class="xl-xqnr"[^>]*>(.*?)</div>\s*(?:<table|<div[^>]*id="div_div")', html, re.DOTALL)
    if m:
        content_html = m.group(1)
    if not content_html:
        m = re.search(r'<div[^>]*class="xl-xqnr"[^>]*>(.*?)</div>', html, re.DOTALL)
        if m:
            content_html = m.group(1)
    if content_html:
        content_html = re.sub(r'<wbr>', '', content_html)
        for am in re.finditer(r'<a[^>]*href=(["\'])([^"\']*\.(?:pdf|doc|docx|xls|xlsx|rar|zip))\1[^>]*>(.*?)</a>', content_html, re.I | re.DOTALL):
            link_url = resolve_url(source_url, am.group(2))
            link_text = re.sub(r'<[^>]+>', '', am.group(3)).strip()
            attachments.append({"name": link_text or link_url.split("/")[-1], "url": link_url})
        parts = []
        for elem in re.finditer(r'<p[^>]*>(.*?)</p>|<table[^>]*>(.*?)</table>', content_html, re.DOTALL | re.I):
            if elem.group(0).startswith('<table'):
                parts.append(elem.group(0))
            else:
                p_text = elem.group(1)
                p_text = re.sub(r'<[^>]+>', '', p_text)
                p_text = re.sub(r'&[nN][bB][sS][pP];', ' ', p_text)
                p_text = re.sub(r'\u3000', '', p_text)
                p_text = p_text.strip()
                if p_text:
                    parts.append(p_text)
        content = "\n\n".join(parts)
    return {"title": title, "content": content, "date": pub_date, "url": source_url,
            "attachments": attachments, "site_name": SITE_NAME, "group": GROUP}

def crawl(test_mode=False, max_pages=MAX_PAGES):
    import sqlite3
    all_items = []
    seen_urls = set()
    for page in range(1, max_pages + 1):
        url = get_list_url(page)
        print("[Page %d/%d] %s" % (page, max_pages, url))
        html = fetch_page(url)
        items = extract_list_items(html, page)
        print("  Found %d items" % len(items))
        for item in items:
            if item["url"] not in seen_urls:
                seen_urls.add(item["url"])
                all_items.append(item)
        if test_mode and len(all_items) >= 3:
            break
    print("\nTotal unique items: %d" % len(all_items))
    if test_mode:
        for item in all_items[:3]:
            print("\n=== Detail: %s ===" % item["title"][:50])
            html = fetch_page(item["url"])
            result = extract_content(html, item["url"])
            print("  Title: %s" % result["title"])
            print("  Date: %s" % result["date"])
            print("  Content (%d chars): %s" % (len(result["content"]), result["content"][:200]))
            print("  Attachments: %d" % len(result["attachments"]))
            for att in result["attachments"]:
                print("    - %s: %s" % (att["name"], att["url"][:80]))
        return
    conn = sqlite3.connect(DB_PATH, timeout=60)
    conn.execute("PRAGMA busy_timeout=30000")
    c = conn.cursor()
    inserted = 0
    existing = 0
    total = len(all_items)
    for idx, item in enumerate(all_items):
        c.execute("SELECT id FROM gov_raw WHERE page_url = ?", (item["url"],))
        if c.fetchone():
            existing += 1
            continue
        html = fetch_page(item["url"])
        result = extract_content(html, item["url"])
        attachments_json = json.dumps(result["attachments"], ensure_ascii=False) if result["attachments"] else ""
        summary = result["title"] + " " + SITE_NAME
        if result["content"]:
            summary += " " + result["content"][:200]
        c.execute("INSERT INTO gov_raw (title, content, summary, publish_date, page_url, site_name, attachments, group_name, source_url) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
            (result["title"], result["content"], summary, result["date"] or item["date"], result["url"], result["site_name"], attachments_json, result["group"], result["url"]))
        inserted += 1
        if inserted % 10 == 0:
            conn.commit()
            print("  Progress: %d/%d inserted..." % (inserted, total))
    conn.commit()
    conn.close()
    print("\nDone. Inserted: %d, Existing: %d (total: %d)" % (inserted, existing, total))

if __name__ == "__main__":
    if "--test" in sys.argv:
        crawl(test_mode=True)
    else:
        crawl(test_mode=False, max_pages=MAX_PAGES)