#!/usr/bin/env python3
import sys, os, re, subprocess, sqlite3

SITE_NAME = "薛城区-生态环境分局(水土保持等)"
SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")

SUB_COLS = [
    ("968", "水土保持"),
    ("969", "空气质量"),
    ("970", "饮用水水质"),
]

def curl(url):
    r = subprocess.run(["curl", "-s", "-L", "--max-time", "15", url], capture_output=True, text=True, timeout=20)
    return r.stdout

def insert_db(items):
    if not items:
        return
    db = sqlite3.connect(SEARCH_DB, timeout=60)
    db.execute("PRAGMA journal_mode=WAL")
    db.execute("PRAGMA busy_timeout=8000")
    db.execute("PRAGMA synchronous=NORMAL")
    ok = skip = 0
    for item in items:
        try:
            db.execute("""INSERT OR IGNORE INTO gov_raw
                (site_name, source_url, page_url, title, publish_date, summary, content, status, attachments)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
                (SITE_NAME,
                 item.get("source_url",""), item.get("url",""),
                 item.get("title","")[:500], item.get("publish_date","")[:10],
                 item.get("summary",item.get("title",""))[:500],
                 item.get("content",""), "active", item.get("attachments","")))
            if db.total_changes > 0:
                ok += 1
            else:
                skip += 1
        except:
            skip += 1
    db.commit()
    try:
        db.execute("""INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary)
            SELECT r.id, r.title, r.site_name, r.summary
            FROM gov_raw r WHERE r.site_name=? AND r.id NOT IN (SELECT rowid FROM gov_search)""", (SITE_NAME,))
        db.commit()
    except:
        pass
    db.close()
    print(f"  [DB] 新增: {ok}, 跳过: {skip}")
    return ok

def fetch_list(cid, page):
    html = curl(f"http://www.xuecheng.gov.cn/govsearch/searPageNewXueCheng.jsp?siteid=44&classinfoid={cid}&page={page}")
    items = []
    for m in re.finditer(r'<li>(.*?)</li>', html, re.DOTALL):
        li = m.group(1)
        href = re.search(r'href="(http[^"]*t202[^"]*\.html)"', li)
        title = re.search(r'title="([^"]*)"', li)
        date = re.search(r'<b[^>]*>(\d{4}[-]?\d{1,2}[-]?\d{1,2})</b>', li)
        if href and title:
            items.append({"url": href.group(1), "title": title.group(1).strip(),
                          "date": date.group(1) if date else ""})
    return items

def fetch_detail(url):
    html = curl(url)
    title = None
    tt = re.search(r'<meta[^>]*name="ArticleTitle"[^>]*content="([^"]*)"', html)
    if tt:
        title = tt.group(1).strip()
    if not title:
        tt = re.search(r'<title>([^<]+)</title>', html)
        if tt:
            title = re.sub(r'[-–—]\s*薛城区政府信息公开$', '', tt.group(1)).strip()
    date = ""
    dp = re.search(r'<meta[^>]*name="PubDate"[^>]*content="([^"]*)"', html)
    if dp:
        date = dp.group(1)[:10]
    content = ""
    m = re.search(r'<div[^>]*class="[^"]*(?:TRS_UEDITOR|trs_paper|trs_web)[^"]*"[^>]*>(.*?)</div>\s*</div>', html, re.DOTALL)
    if m:
        inner = m.group(1)
        inner = re.sub(r'<style[^>]*>.*?</style>', '', inner, flags=re.DOTALL)
        inner = re.sub(r'<script[^>]*>.*?</script>', '', inner, flags=re.DOTALL)
        texts = re.findall(r'<p[^>]*>(.*?)</p>', inner, re.DOTALL)
        paras = []
        for p in texts:
            t = re.sub(r'<[^>]+>', '', p).strip()
            t = re.sub(r'&nbsp;', ' ', t)
            t = re.sub(r'\s+', ' ', t)
            if t:
                paras.append(t)
        content = '\n\n'.join(paras)
    return {"title": title, "content": content, "date": date}

total_ok = 0
total_fail = 0
for cid, name in SUB_COLS:
    print(f"\n=== {name} (CID={cid}) ===")
    html = curl(f"http://www.xuecheng.gov.cn/govsearch/searPageNewXueCheng.jsp?siteid=44&classinfoid={cid}&page=1")
    m = re.search(r'm_nRecordCount = (\d+)', html)
    total = int(m.group(1)) if m else 0
    pages = (total + 19) // 20
    print(f"Total: {total} records, ~{pages} pages")
    
    all_items = []
    for pn in range(1, pages + 1):
        items = fetch_list(cid, pn)
        seen = {i["url"] for i in all_items}
        new = [i for i in items if i["url"] not in seen]
        all_items.extend(new)
        print(f"  Page {pn}/{pages}: {len(items)} items (new {len(new)})")
    
    db_items = []
    for i, item in enumerate(all_items):
        print(f"  [{i+1}/{len(all_items)}] {item['title'][:50]}...", end=" ")
        detail = fetch_detail(item["url"])
        if detail and detail.get("content"):
            db_items.append({
                "title": detail.get("title") or item["title"],
                "source_url": item["url"], "url": item["url"],
                "publish_date": detail.get("date") or item["date"][:10],
                "summary": (detail.get("title") or item["title"])[:200],
                "content": detail["content"],
                "attachments": "",
            })
            total_ok += 1
            print(f"✅ {len(detail['content'])}字")
        else:
            total_fail += 1
            print("❌ 空正文")
    
    ok = insert_db(db_items)
    print(f"  {name}: {ok} 条入库")

print(f"\n{'='*50}")
print(f"总计: 成功 {total_ok}, 失败 {total_fail}")