#!/usr/bin/env python3
"""河间市-征集调查"""
import requests, re, sqlite3, time
from datetime import datetime, timedelta
import os

SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
NAME = "河间市-征集调查"
BASE = "https://hejian.gov.cn"
LIST_PATH = "/hejian/c100280/listDisplaySon.shtml"
CUTOFF = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
H = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

def fetch(url):
    for retry in range(2):
        try:
            r = requests.get(url, headers=H, timeout=30)
            if r.status_code == 200: r.encoding = "utf-8"; return r.text
        except: time.sleep(2)
    return None

def parse_list(html):
    items = []
    dates = re.findall(r'<span[^>]*class="date"[^>]*>(\d{4}-\d{2}-\d{2})</span>', html)
    di = 0
    for m in re.finditer(r'<a\s+href="(/hejian/c100280/\d+/[^"]+\.shtml)"[^>]*>\s*(.*?)\s*</a>', html, re.DOTALL):
        url = BASE + m.group(1)
        title = re.sub(r'<[^>]+>', '', m.group(2)).strip()
        if not title: continue
        date = dates[di] if di < len(dates) else ""
        di += 1
        items.append({"url": url, "title": title, "date": date})
    return items

def extract_content(html):
    """深度计数法提取正文"""
    for p in ['class="articlecon"', 'class="nr_content"', 'class="content"']:
        i = html.find(p)
        if i < 0: continue
        ds = html.rfind("<div", 0, i)
        if ds < 0: continue
        s = html[ds:]; d = 0
        for j in range(len(s)):
            if s[j:j+4] == "<div" and (j+4 >= len(s) or s[j+4] in " >\n\r\t"): d += 1
            elif s[j:j+6] == "</div>":
                d -= 1
                if d == 0:
                    gt = s.find(">", 0, j)
                    c = s[gt+1:j] if gt > 0 else s[7:j]
                    c = re.sub(r'<(script|style)[^>]*>.*?</\1>', '', c, flags=re.DOTALL|re.I)
                    c = re.sub(r'\s*(style|class|align|lang|dir)="[^"]*"', '', c)
                    c = re.sub(r'\n\s*\n+', '\n', c)
                    if c.strip(): return c.strip()
    return ""

def extract(html):
    t = re.sub(r'_河间市人民政府.*$', '', (re.search(r'<title>(.*?)<', html) or [None, ""])[1]).strip()
    d = (re.findall(r'发布日期[：:]\s*(\d{4}-\d{2}-\d{2})', html) or [""])[0]
    if not d: d = (re.findall(r'(\d{4}-\d{2}-\d{2})', html) or [""])[0]
    c = extract_content(html)
    return t, d, c

def run(max_pages=5):
    print(f"\n{'='*50}\n🚀 {NAME}\n{'='*50}")
    conn = sqlite3.connect(SEARCH_DB, timeout=60)
    conn.execute("PRAGMA busy_timeout=60000")
# QC20260925 去掉整站清空再重灌(抢锁+中途死掉会清空整站; page_url 有 UNIQUE 索引，插入本就幂等)     conn.execute("DELETE FROM gov_raw WHERE site_name=?", (NAME,))
    conn.commit()
    print("🧹 清理旧数据")

    items = []
    for pg in range(1, max_pages+1):
        url = BASE + LIST_PATH if pg==1 else BASE + LIST_PATH + f"?page={pg}"
        html = fetch(url)
        if not html: print(f"  ⚠️ 第{pg}页取不到"); break
        more = parse_list(html)
        if not more: print(f"  → 第{pg}页无条目"); break
        print(f"📋 第{pg}页: {len(more)} 条")
        items.extend(more)
        time.sleep(0.5)

    print(f"\n📊 共 {len(items)} 条")
    new=skip=no_body=0
    for i,item in enumerate(items,1):
        if item["date"] and item["date"]<CUTOFF: skip+=1; continue
        html=fetch(item["url"])
        if not html: print(f"  ⚠️ 详情取不到: {item['title'][:30]}"); skip+=1; continue
        title,date,content=extract(html)
        if not title: title=item["title"]
        if not date: date=item["date"]
        text_len=len(re.sub(r'<[^>]+>','',content).strip()) if content else 0
        if text_len<10: print(f"  ⚠️ 空正文: {title[:30]}"); no_body+=1; continue
        try:
            dr=0
            try: dr=int(datetime.strptime(date,"%Y-%m-%d").timestamp())
            except: dr=int(time.time())
            conn.execute("INSERT OR IGNORE INTO gov_raw(site_name,source_url,page_url,title,publish_date,content,summary,date_rank,category) VALUES(?,?,?,?,?,?,?,?,?)",
                (NAME,item["url"],item["url"],title,date,content,title,dr,"政府公告"))
            if conn.total_changes: new+=1
            else: skip+=1
        except Exception as e: print(f"  ❌ DB: {e}"); skip+=1
        if i%10==0: conn.commit(); print(f"  ...{i}/{len(items)}")
        time.sleep(0.3)
    conn.commit()
    try:
# QC20260925 去掉整站清空再重灌(抢锁+中途死掉会清空整站; page_url 有 UNIQUE 索引，插入本就幂等)         conn.execute("DELETE FROM gov_search WHERE site_name=?",(NAME,))
        conn.execute("INSERT OR REPLACE INTO gov_search(rowid,title,site_name,summary) SELECT rowid,title,site_name,summary FROM gov_raw WHERE site_name=?",(NAME,))
        conn.commit()
    except Exception as e: print(f"⚠️ FTS: {e}")
    conn.close()
    print(f"\n✅ {NAME}: 新增{new}, 空正文{no_body}, 跳过{skip}")

if __name__=="__main__":
    import sys; mp=int(sys.argv[1]) if len(sys.argv)>1 else 5; run(mp)