#!/usr/bin/env python3
import os
"""
宿州经开区-公示公告 爬虫
站点: kfq.ahsz.gov.cn
栏目: 公示公告
"""
import requests, re, sqlite3, time, sys, json, warnings
warnings.filterwarnings('ignore')
from datetime import datetime, timedelta

SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "宿州经开区-公示公告"
DOMAIN = "kfq.ahsz.gov.cn"
COLUMN_ID = "111328164"

CUTOFF = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

LIST_URLS = [
    f"https://{DOMAIN}/zxzx/gsgg/index.html",  # Page 1
]

def fetch_list_page(page):
    """Get list page items"""
    if page == 1:
        url = LIST_URLS[0]
    else:
        url = f"https://{DOMAIN}/content/column/{COLUMN_ID}?pageIndex={page}"
    
    for retry in range(3):
        try:
            r = requests.get(url, headers=HEADERS, timeout=30)
            if r.status_code == 200:
                r.encoding = "utf-8"
                html = r.text
                
                items = []
                for li in re.findall(r'<li[^>]*>.*?</li>', html, re.DOTALL):
                    m = re.search(r'href="([^"]+)"[^>]*title="([^"]*)"', li)
                    if not m:
                        continue
                    dm = re.search(r'class="right date"[^>]*>([^<]*)<', li)
                    date = dm.group(1).strip() if dm else ""
                    url_str = m.group(1)
                    if not url_str.startswith("http"):
                        url_str = "https://" + DOMAIN + url_str
                    items.append({
                        "url": url_str,
                        "title": m.group(2).strip(),
                        "date": date,
                    })
                
                # Total count from JS
                cm = re.search(r'total:\s*(\d+)', html)
                total = int(cm.group(1)) if cm else 0
                return items, total
        except Exception as e:
            if retry < 2:
                time.sleep(2)
    return [], 0


def extract_content(html):
    """提取正文 - 从 wenzhang 容器，清理无关元素"""
    # Try wenzhang container first
    idx = html.find('class="wenzhang')
    if idx < 0:
        idx = html.find('class="wenzhang')
    if idx < 0:
        return ""
    
    # Find the opening div
    ds = html.rfind("<div", 0, idx)
    if ds < 0:
        return ""
    
    # Depth counting to find matching </div>
    s = html[ds:]
    d = 0
    for j in range(len(s)):
        if s[j:j+4] == "<div" and (j+4 >= len(s) or s[j+4] in " >\n\r\t"):
            d += 1
        elif s[j:j+6] == "</div>":
            d -= 1
            if d == 0:
                gt = s.find(">", 0, j)
                c = s[gt+1:j] if gt > 0 else s[7:j]
                c = re.sub(r'<(script|style)[^>]*>.*?</\1>', '', c, flags=re.DOTALL|re.I)
                # Strip redundant header (title + metadata)
                c = re.sub(r'<h1\s+class="newstitle">.*?</h1>', '', c, flags=re.DOTALL)
                c = re.sub(r'<div\s+class="newsinfo">.*?</div>', '', c, flags=re.DOTALL)
                # Strip wzbot (share buttons, print, etc.)
                c = re.sub(r'<div\s+class="wzbot[^"]*"[^>]*>.*?</div>\s*</div>', '', c, flags=re.DOTALL)
                text = re.sub(r'<[^>]+>', '', c).strip()
                if len(text) > 50:
                    return c.strip()
    return ""


def extract_meta(html):
    """Extract title and date from meta tags"""
    title = ""
    tm = re.search(r'<meta[^>]*name="ArticleTitle"[^>]*content="([^"]*)"', html)
    if tm:
        title = tm.group(1)
    if not title:
        ttm = re.search(r'<title>(.*?)<', html)
        if ttm:
            title = ttm.group(1).strip()
    
    date = ""
    pm = re.search(r'<meta[^>]*name="PubDate"[^>]*content="([^"]*)"', html)
    if pm:
        d = pm.group(1).strip()
        cm = re.match(r'(\d{4})-(\d{1,2})-(\d{1,2})', d)
        if cm:
            date = f"{cm.group(1)}-{cm.group(2).zfill(2)}-{cm.group(3).zfill(2)}"
    if not date:
        dm = re.search(r'(\d{4}-\d{2}-\d{2})', html)
        if dm:
            date = dm.group(1)
    return title, date


def insert_article(url, title, date, content):
    db = sqlite3.connect(SEARCH_DB, timeout=60)
    db.execute("PRAGMA journal_mode=WAL")
    try:
        db.execute(
            "INSERT OR IGNORE INTO gov_raw(site_name,source_url,page_url,title,publish_date,content,summary,status,category) VALUES(?,?,?,?,?,?,?,?,?)",
            (SITE_NAME, url, url, title, date, content, title[:200], "active", "公示公告"))
        affected = db.total_changes
        db.commit()
        if affected:
            db.execute(
                "INSERT OR REPLACE INTO gov_search(rowid,title,site_name,summary) SELECT r.id,r.title,r.site_name,r.summary FROM gov_raw r WHERE r.page_url=? AND r.id NOT IN (SELECT rowid FROM gov_search)",
                (url,))
            db.commit()
        db.close()
        return affected
    except Exception as e:
        db.close()
        print(f"  DB error: {e}")
        return 0


def run(max_pages=50):
    print(f"\n{'='*50}")
    print(f"🚀 {SITE_NAME}")
    print(f"{'='*50}")

    items, total = fetch_list_page(1)
    if not items:
        print("❌ 列表页无数据")
        return
    print(f"  共{total}条")
    total_pages = min(max_pages, (total + 19) // 20)
    
    all_items = list(items)
    for pg in range(2, total_pages + 1):
        its, _ = fetch_list_page(pg)
        if not its:
            print(f"  第{pg}页: 空 -> 结束")
            break
        all_items.extend(its)
        dates = [it["date"] for it in its if it["date"]]
        if dates:
            print(f"  第{pg}页: {len(its)} 条 ({min(dates)} ~ {max(dates)})")
            if min(dates) < CUTOFF and max(dates) < CUTOFF:
                print(f"  全部早于{CUTOFF}，停止翻页")
                break
        else:
            print(f"  第{pg}页: {len(its)} 条")
        time.sleep(0.2)

    print(f"\n📊 API共 {len(all_items)} 条")
    new = skip = no_body = err = 0
    for i, item in enumerate(all_items, 1):
        if item["date"] and item["date"] < CUTOFF:
            skip += 1
            continue

        for retry in range(3):
            try:
                r = requests.get(item["url"], headers=HEADERS, timeout=30)
                if r.status_code == 200:
                    r.encoding = "utf-8"
                    html = r.text
                    break
            except:
                if retry < 2:
                    time.sleep(2)
                html = None
        
        if not html:
            err += 1
            continue

        pt, pd = extract_meta(html)
        real_title = pt if pt else item["title"]
        real_date = pd if pd else item["date"]

        if real_date and real_date < CUTOFF:
            skip += 1
            continue

        content = extract_content(html)
        text_len = len(re.sub(r'<[^>]+>', '', content).strip()) if content else 0
        if text_len < 10:
            no_body += 1
            continue

        if insert_article(item["url"], real_title, real_date, content):
            new += 1
        else:
            skip += 1

        if i % 5 == 0:
            print(f"  ...{i}/{len(all_items)}")
        time.sleep(0.2)

    # FTS sync
    print(f"\n📊 FTS同步...")
    db = sqlite3.connect(SEARCH_DB, timeout=60)
    # QC20260926 去掉手写 gov_search 整站删除(抢锁源; FTS 由 gov_raw 触发器维护) 
    # db.execute("DELETE FROM gov_search WHERE site_name=?", (SITE_NAME,))
    db.execute("INSERT OR REPLACE INTO gov_search(rowid,title,site_name,summary) SELECT rowid,title,site_name,summary FROM gov_raw WHERE site_name=?", (SITE_NAME,))
    db.commit()
    c = db.execute("SELECT COUNT(*) FROM gov_search WHERE site_name=?", (SITE_NAME,))
    cnt = c.fetchone()[0]
    db.close()
    print(f"  FTS: {cnt} 条")
    print(f"\n✅ {SITE_NAME}: 新增{new}, 跳过{skip}, 空正文{no_body}, 错误{err}, FTS共{cnt}条")


if __name__ == "__main__":
    mp = int(sys.argv[1]) if len(sys.argv) > 1 else 50
    run(mp)