#!/usr/bin/env python3
import os
"""仁怀市政府通知公告爬虫 - rh.gov.cn"""
import re, time, json, urllib.request, urllib.error, sqlite3, ssl
from datetime import datetime, timedelta
from bs4 import BeautifulSoup



import sys as _SYS
_MAX_PG = int(_SYS.argv[1]) if len(_SYS.argv) > 1 and _SYS.argv[1].isdigit() else None
if _MAX_PG is not None:
    print('[AutoPg] max_pages=' + str(_MAX_PG))
# END AUTO PAGES
BASE_URL = "https://www.rh.gov.cn"
SITE_NAME = "仁怀市-通知公告"
LIST_TMPL = BASE_URL + "/xwzx/tzgg/index%s.html"
CUTOFF_DATE = (datetime.now() - timedelta(days=365*3)).strftime("%Y-%m-%d")
MAX_PAGES = 100
DELAY = 1.0
DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")

ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

def fetch(url, retries=3):
    for i in range(retries):
        try:
            req = urllib.request.Request(url, headers=HEADERS)
            with urllib.request.urlopen(req, context=ctx, timeout=20) as r:
                return r.read().decode("utf-8", errors="replace")
        except Exception as e:
            if i < retries - 1:
                time.sleep(DELAY * (i+1))
                continue
            print("  [WARN] " + str(e)[:80], flush=True)
            return None

def parse_list(html):
    items = []
    soup = BeautifulSoup(html, "html.parser")
    ul = soup.find("ul", class_="NewsList")
    if not ul:
        return items
    for li in ul.find_all("li"):
        a = li.find("a")
        date_span = li.find("span")
        if a and a.get("href") and date_span:
            url = a["href"].strip()
            if not url.startswith("http"):
                url = BASE_URL + url
            title = a.get("title") or a.get_text(strip=True)
            date_text = date_span.get_text(strip=True)
            m = re.search(r"(\d{4}-\d{2}-\d{2})", date_text)
            date = m.group(1) if m else ""
            if date >= CUTOFF_DATE:
                items.append((url, title, date))
    return items

def parse_detail(html, url):
    soup = BeautifulSoup(html, "html.parser")
    
    title_el = soup.select_one("div.Article_bt")
    title = title_el.get_text(strip=True) if title_el else ""
    
    meta_date = soup.find("meta", attrs={"name": "PubDate"})
    date = meta_date["content"][:10] if meta_date and meta_date.get("content") else ""
    
    meta_source = soup.find("meta", attrs={"name": "ContentSource"})
    source = meta_source["content"] if meta_source and meta_source.get("content") else ""
    
    content_div = soup.select_one("div.Article_zw font#Zoom, div.Article_zw")
    content = ""
    if content_div:
        for el in content_div.find_all(["script", "style"]):
            el.decompose()
        content = str(content_div)
    
    return title, date, source, content

def save(items):
    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()
    n = 0
    for url, title, date, source, content in items:
        try:
            summary = "来源：" + source if source else ""
            c.execute("""
                INSERT OR IGNORE INTO gov_raw 
                (page_url, title, publish_date, site_name, source_url, summary, content)
                VALUES (?, ?, ?, ?, ?, ?, ?)
            """, (url, title, date, SITE_NAME, url, summary, content))
            if c.rowcount > 0:
                n += 1
        except Exception as e:
            print("  [DB] " + str(e)[:60], flush=True)
    conn.commit()
    conn.close()
    return n

def main():
    print("[rh] 仁怀市-通知公告 - 截止: " + CUTOFF_DATE, flush=True)
    
    all_items = []
    for page in range(min(MAX_PAGES, _MAX_PG or MAX_PAGES)):
        if page == 0:
            url = LIST_TMPL % ""
        else:
            url = LIST_TMPL % ("_" + str(page))
        
        html = fetch(url)
        if not html:
            print("[rh] 列表 " + str(page+1) + ": 获取失败", flush=True)
            continue
        
        items = parse_list(html)
        if not items:
            print("[rh] 列表 " + str(page+1) + ": 空 - 结束", flush=True)
            break
        
        dates = sorted(set(d for _,_,d in items))
        oldest = min(dates)
        print("[rh] 列表 " + str(page+1) + ": " + str(len(items)) + "条, " + oldest + " ~ " + max(dates), flush=True)
        
        all_items.extend(items)
        
        if oldest < CUTOFF_DATE:
            print("[rh] 截止: " + oldest + " < " + CUTOFF_DATE, flush=True)
            break
        time.sleep(DELAY)
    
    print("\n[rh] 共 " + str(len(all_items)) + " 条待抓详情", flush=True)
    
    # Detail fetch + save combined
    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()
    saved = 0
    for i, (url, title, date) in enumerate(all_items, 1):
        print("[rh] " + str(i) + "/" + str(len(all_items)) + ": " + title[:30] + "...", flush=True)
        html = fetch(url)
        if not html:
            continue
        dt, dd, ds, dc = parse_detail(html, url)
        final_title = dt or title
        final_date = dd or date
        cl = len(dc) if dc else 0
        summary = "来源：" + ds if ds else ""
        try:
            c.execute("""
                INSERT OR IGNORE INTO gov_raw 
                (page_url, title, publish_date, site_name, source_url, summary, content)
                VALUES (?, ?, ?, ?, ?, ?, ?)
            """, (url, final_title, final_date, SITE_NAME, url, summary, dc))
            if c.rowcount > 0:
                saved += 1
        except Exception as e:
            print("  [DB] " + str(e)[:60], flush=True)
        time.sleep(DELAY)
    
    conn.commit()
    conn.close()
    print("\n[rh] ✅ 入库 " + str(saved) + " 条", flush=True)
    
    # FTS rebuild
    conn = sqlite3.connect(DB_PATH, timeout=60)
    try:
        conn.execute("INSERT INTO gov_search(gov_search,rowid,title,site_name,summary) VALUES('rebuild',0,'','','')")
    except:
        pass
    conn.commit()
    conn.close()
    print("[rh] ✅ 完成", flush=True)

if __name__ == "__main__":
    main()