#!/usr/bin/env python3
import re, sys, os, json, time, requests
DB_PATH = os.environ.get("DB_PATH", "/root/search.db")
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}
SITE_NAME = "\u6d59\u6c5f\u653f\u52a1\u670d\u52a1\u7f51\uff08\u6e29\u5dde\u5e02\u5e73\u9633\u53bf\uff09-\u5efa\u8bbe\u9879\u76ee\u73af\u5883\u5f71\u54cd\u8bc4\u4ef7\u4fe1\u606f\u516c\u793a"
GROUP = "\u5e73\u9633"
PER_PAGE = 9
MAX_PAGES = 1
BASE_URL = "http://wzpy.zjzwfw.gov.cn"
COL_URL = "http://wzpy.zjzwfw.gov.cn/col/col1460372/index.html"

def fetch_page(url, encoding="utf-8"):
    r = requests.get(url, headers=HEADERS, timeout=30, allow_redirects=True)
    r.encoding = encoding
    return r.text

def resolve_url(base_url, rel_url):
    if rel_url.startswith("http"):
        return rel_url
    if rel_url.startswith("/"):
        return BASE_URL + rel_url
    base_dir = base_url.rsplit("/", 1)[0] if "/" in base_url else base_url
    parts = base_dir.split("/")
    while rel_url.startswith("../"):
        rel_url = rel_url[3:]
        if parts:
            parts.pop()
    parts.append(rel_url)
    return "/".join(parts)

def extract_list_items(html):
    items = []
    # Each item is in <div style="height:80px..."> with <a> link and date
    pattern = r'<div[^>]*style="height:80px[^"]*"[^>]*>.*?<a[^>]*href=(["\'])([^"\']+)\1[^>]*title=(["\'])([^"\']+)\3[^>]*>.*?</a>.*?<div[^>]*style="color:\s*#[Bb]4[Bb]4[Bb]4[^"]*"[^>]*>(\d{4}-\d{1,2}-\d{1,2})</div>'
    for m in re.finditer(pattern, html, re.DOTALL):
        rel_url = m.group(2).strip()
        title = m.group(4).strip()
        date_str = m.group(5).strip()
        full_url = resolve_url(COL_URL, rel_url)
        if title:
            items.append({"url": full_url, "title": title, "date": date_str})
    return items

def extract_content(html, source_url):
    title = ""
    m = re.search(r'<td[^>]*style="color:#fff;font-size:30px[^"]*"[^>]*>([^<]+)</td>', html, re.DOTALL)
    if m:
        title = m.group(1).strip()
    else:
        m = re.search(r'<title>(.*?)</title>', html, re.DOTALL)
        if m:
            title = m.group(1).strip()
    
    pub_date = ""
    m = re.search(r'(\d{4}-\d{1,2}-\d{1,2})\s*\d{2}:\d{2}', html)
    if m:
        pub_date = m.group(1).strip()
    
    content = ""
    attachments = []
    content_html = ""
    
    m = re.search(r'<div id="zoom"[^>]*>(.*?)</div>', html, re.DOTALL)
    if m:
        content_html = m.group(1)
    
    if content_html:
        # Extract attachments
        for am in re.finditer(r'<a[^>]*href=(["\'])([^"\']*\.(?:pdf|doc|docx|xls|xlsx|rar|zip))\1[^>]*>(.*?)</a>', content_html, re.I | re.DOTALL):
            link_url = resolve_url(source_url, am.group(2))
            link_text = re.sub(r'<[^>]+>', '', am.group(3)).strip()
            attachments.append({"name": link_text or link_url.split("/")[-1], "url": link_url})
        
        parts = []
        for elem in re.finditer(r'<p[^>]*>(.*?)</p>|<table[^>]*>(.*?)</table>', content_html, re.DOTALL | re.I):
            if elem.group(0).startswith('<table'):
                parts.append(elem.group(0))
            else:
                p_text = elem.group(1)
                p_text = re.sub(r'<[^>]+>', '', p_text)
                p_text = re.sub(r'&[nN][bB][sS][pP];', ' ', p_text)
                p_text = re.sub(r'\u3000', '', p_text)
                p_text = p_text.strip()
                if p_text:
                    parts.append(p_text)
        content = "\n\n".join(parts)
    
    return {"title": title, "content": content, "date": pub_date, "url": source_url,
            "attachments": attachments, "site_name": SITE_NAME, "group": GROUP}

def crawl(test_mode=False):
    import sqlite3
    all_items = []
    seen_urls = set()
    
    html = fetch_page(COL_URL)
    items = extract_list_items(html)
    print("[Page 1] %s" % COL_URL)
    print("  Found %d items" % len(items))
    for item in items:
        if item["url"] not in seen_urls:
            seen_urls.add(item["url"])
            all_items.append(item)
    
    print("\nTotal unique items: %d" % len(all_items))
    
    if test_mode:
        for item in all_items[:3]:
            print("\n=== Detail: %s ===" % item["title"][:50])
            html = fetch_page(item["url"])
            result = extract_content(html, item["url"])
            print("  Title: %s" % result["title"])
            print("  Date: %s" % result["date"])
            print("  Content (%d chars): %s" % (len(result["content"]), result["content"][:200]))
            print("  Attachments: %d" % len(result["attachments"]))
            for att in result["attachments"]:
                print("    - %s: %s" % (att["name"], att["url"][:80]))
        return
    
    conn = sqlite3.connect(DB_PATH, timeout=60)
    conn.execute("PRAGMA busy_timeout=30000")
    c = conn.cursor()
    inserted = 0
    existing = 0
    total = len(all_items)
    
    for idx, item in enumerate(all_items):
        c.execute("SELECT id FROM gov_raw WHERE page_url = ?", (item["url"],))
        if c.fetchone():
            existing += 1
            continue
        html = fetch_page(item["url"])
        result = extract_content(html, item["url"])
        attachments_json = json.dumps(result["attachments"], ensure_ascii=False) if result["attachments"] else ""
        summary = result["title"] + " " + SITE_NAME
        if result["content"]:
            summary += " " + result["content"][:200]
        c.execute("INSERT INTO gov_raw (title, content, summary, publish_date, page_url, site_name, attachments, group_name, source_url) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
            (result["title"], result["content"], summary, result["date"] or item["date"], result["url"], result["site_name"], attachments_json, result["group"], result["url"]))
        inserted += 1
        if inserted % 5 == 0:
            conn.commit()
            print("  Progress: %d/%d inserted..." % (inserted, total))
    conn.commit()
    conn.close()
    print("\nDone. Inserted: %d, Existing: %d (total: %d)" % (inserted, existing, total))

if __name__ == "__main__":
    if "--test" in sys.argv:
        crawl(test_mode=True)
    else:
        crawl(test_mode=False)