#!/usr/bin/env python3
import os
"""吉安市政府网站群通用爬虫 - 处理所有AJAX动态加载站点"""
import requests, re, sqlite3, time, json, sys
from datetime import datetime, timedelta

SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
CUTOFF = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
H = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

# 站点配置
SITES = [
    {"name": "泰和县人民政府-政府采购", "domain": "www.jxth.gov.cn", "catid": "107854", "ajax_type": "9_xxgk", "num": 50},
    {"name": "永丰县-基础建设", "domain": "www.jxyongfeng.gov.cn", "catid": "134486", "ajax_type": "5_xxgk", "num": 50},
    {"name": "吉水县-污染防治", "domain": "www.jishui.gov.cn", "catid": "165920", "ajax_type": "4_xxgk", "num": 35},
    {"name": "遂川县-部门项目", "domain": "www.suichuan.gov.cn", "catid": "166018", "ajax_type": "8_xxgk", "num": 50},
    {"name": "峡江县-生态环境", "domain": "www.xiajiang.gov.cn", "catid": "165729", "ajax_type": "7_xxgk", "num": 35},
    {"name": "新干县-基础建设", "domain": "www.xingan.gov.cn", "catid": "116418", "ajax_type": "6_xxgk", "num": 50},
]

def fetch_api(domain, page, catid, ajax_type, num=50):
    """用curl调API（避免requests库header差异导致结果不一致）"""
    import subprocess
    # Extract site_id from ajax_type (e.g., "6_xxgk" -> 6)
    site_id = ajax_type.split("_")[0] if "_" in ajax_type else "9"
    url = f"http://{domain}/api-ajax_list-{page}.html"
    data_str = (f"ajax_type%5B%5D={ajax_type}&ajax_type%5B%5D={catid}&ajax_type%5B%5D={site_id}&ajax_type%5B%5D=xxgk"
                f"&ajax_type%5B%5D=Y-m-d&ajax_type%5B%5D={num}&ajax_type%5B%5D=20"
                f"&ajax_type%5B%5D%5B%5D=is_top+DESC&ajax_type%5B%5D%5B%5D=displayorder+DESC"
                f"&ajax_type%5B%5D%5B%5D=inputtime+DESC&ajax_type%5B%5D=&is_ds=1")
    for retry in range(3):
        try:
            cmd = ["curl", "-s", url, "-A", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
                   "-H", "X-Requested-With: XMLHttpRequest", "--data", data_str, "--max-time", "30"]
            result = subprocess.run(cmd, capture_output=True, text=True, timeout=35)
            if result.returncode == 0 and result.stdout:
                return json.loads(result.stdout)
        except:
            time.sleep(2)
    return None

def fetch(url):
    for retry in range(2):
        try:
            r = requests.get(url, headers=H, timeout=30)
            if r.status_code == 200: r.encoding = "utf-8"; return r.text
        except: time.sleep(2)
    return None

def extract_content(html):
    patterns = ['id="zoom"', 'class="xxgk_content"', 'class="zwxxgk_box"',
                'class="zwxxgk_bd"', 'class="nr_content"', 'class="content"',
                'class="articlecon"', 'class="info-content"', 'class="fmy_content"',
                'class="cont"', 'class="fullscreen-layout-padding__content"']
    for p in patterns:
        i = html.find(p)
        if i < 0: continue
        ds = html.rfind("<div", 0, i)
        if ds < 0: continue
        s = html[ds:]; d = 0
        for j in range(len(s)):
            if s[j:j+4] == "<div" and (j+4 >= len(s) or s[j+4] in " >\n\r\t"): d += 1
            elif s[j:j+6] == "</div>":
                d -= 1
                if d == 0:
                    gt = s.find(">", 0, j)
                    c = s[gt+1:j] if gt > 0 else s[7:j]
                    c = re.sub(r'<(script|style)[^>]*>.*?</\1>', '', c, flags=re.DOTALL|re.I)
                    c = re.sub(r'\s*(style|class|align|lang|dir)="[^"]*"', '', c)
                    c = re.sub(r'\n\s*\n+', '\n', c)
                    if c.strip(): return c.strip()
    return ""

def extract(html):
    t = (re.search(r'<meta[^>]*name="ArticleTitle"[^>]*content="([^"]*)"', html) or [None, ""])[1]
    if not t:
        t = re.sub(r'_[^_]+$', '', (re.search(r'<title>(.*?)<', html) or [None,""])[1]).strip()
    pd = (re.search(r'<meta[^>]*name="PubDate"[^>]*content="(\d{4}-\d{2}-\d{2})', html) or [None, ""])[1]
    if not pd: pd = (re.findall(r'(\d{4}-\d{2}-\d{2})', html) or [""])[0]
    return t, pd, extract_content(html)

def crawl_one(site, max_pages):
    name = site["name"]
    domain = site["domain"]
    catid = site["catid"]
    ajax_type = site["ajax_type"]
    num = site.get("num", 50)
    print(f"\n{'='*50}\n{name} (catid={catid}, {domain})")
    print(f"{'='*50}")

    # Use session with cookies for WAF bypass
    sess = requests.Session()
    sess.headers.update(H)
    base_url = f"http://{domain}"

    # Get initial cookies
    try:
        sess.get(base_url, timeout=15)
    except:
        pass

    conn = sqlite3.connect(SEARCH_DB, timeout=60)

    conn.execute("PRAGMA busy_timeout=60000")
# QC20260925 去掉整站清空再重灌(抢锁+中途死掉会清空整站; page_url 有 UNIQUE 索引，插入本就幂等)     conn.execute("DELETE FROM gov_raw WHERE site_name=?", (name,))
    conn.commit()

    items = []
    for pg in range(1, max_pages+1):
        d = fetch_api(domain, pg, catid, ajax_type, num)
        if not d or not d.get("data"):
            print(f"  -> 第{pg}页无数据"); break
        data = d["data"]
        print(f"第{pg}页: {len(data)} 条 (共{d.get('total','?')}条)")
        for item in data:
            items.append({
                "url": f"http://{domain}/xxgk-show-{item['id']}.html",
                "title": item["title"],
                "date": item.get("inputtime", ""),
                "summary": item.get("description", ""),
                "serial": item.get("serial", "")
            })
        if len(data) < 20:
            break
        time.sleep(0.3)

    print(f"\n共 {len(items)} 条")
    new = skip = no_body = 0
    for i, item in enumerate(items):
        html = None
        for retry in range(3):
            try:
                r = sess.get(item["url"], timeout=30)
                if r.status_code == 200:
                    r.encoding = "utf-8"
                    html = r.text
                    break
                elif r.status_code == 403:
                    sess.get(base_url, timeout=15)
                    time.sleep(2)
            except:
                time.sleep(2)

        if not html: print(f"  ? {item['title'][:40]}..."); skip+=1; continue
        if "404" in str(html)[:500] and "Not Found" in str(html)[:500]:
            print(f"  - 404: {item['title'][:40]}"); skip+=1; continue
        if "环境异常" in str(html)[:1000]:
            print(f"  - WAF: {item['title'][:40]}"); skip+=1; continue
        t, pd, content = extract(html)
        title = t or item["title"]
        date = pd or item["date"]
        if not content:
            no_body += 1
            content = item.get("summary", "")
        try:
            conn.execute("""INSERT OR IGNORE INTO gov_raw(site_name,source_url,page_url,title,publish_date,content,summary,date_rank,category)
                VALUES(?,?,?,?,?,?,?,?,?)""",
                (name, item["url"], item["url"], title, date, content, title, 0, "政府公告"))
            if conn.total_changes: new+=1; print(f"  +{title[:40]} ({date})")
            else: skip+=1
        except Exception as e: print(f"  ? DB: {e}"); skip+=1
        if i%10==0: conn.commit()
        time.sleep(0.3)
    conn.commit()

    # 清理超3年的旧数据
    deleted = conn.execute("DELETE FROM gov_raw WHERE site_name=? AND publish_date < ?", (name, CUTOFF)).rowcount
    if deleted:
        print(f"  清理超3年: {deleted} 条")
    conn.commit()

    # FTS
    try:
# QC20260925 去掉整站清空再重灌(抢锁+中途死掉会清空整站; page_url 有 UNIQUE 索引，插入本就幂等)         conn.execute("DELETE FROM gov_search WHERE site_name=?",(name,))
        conn.execute("INSERT OR REPLACE INTO gov_search(rowid,title,site_name,summary) SELECT rowid,title,site_name,summary FROM gov_raw WHERE site_name=?",(name,))
        conn.commit()
    except Exception as e: print(f"? FTS: {e}")
    conn.close()
    print(f"\n{name}: 新增{new}, 空正文{no_body}, 跳过{skip}")
    return {"name": name, "new": new, "no_body": no_body, "skip": skip}

if __name__=="__main__":
    mp = int(sys.argv[1]) if len(sys.argv) > 1 else 10
    site_idx = int(sys.argv[2]) if len(sys.argv) > 2 else None
    results = []
    for i, s in enumerate(SITES):
        if site_idx is not None and i != site_idx:
            continue
        r = crawl_one(s, mp)
        results.append(r)
        time.sleep(1)

    print(f"\n\n{'='*50}")
    print("全部完成")
    for r in results:
        print(f"  {r['name']}: 新增{r['new']}, 空正文{r['no_body']}, 跳过{r['skip']}")