#!/usr/bin/env python3
"""上饶市人民政府-公示公告 爬虫 (静态分页 zwgk_xxgklist_N.shtml)
"""
import os, sqlite3, re, time, requests, json
from datetime import datetime, date
from bs4 import BeautifulSoup
from concurrent.futures import ThreadPoolExecutor, as_completed

BASE = "https://www.zgsr.gov.cn"
LIST_URL = "https://www.zgsr.gov.cn/zgsr/gsgg/zwgk_xxgklist.shtml"
PAGE_URL = "https://www.zgsr.gov.cn/zgsr/gsgg/zwgk_xxgklist_{}.shtml"
DB_PATH = os.environ.get("DB_PATH", "/root/search.db")
SITE_NAME = "上饶市人民政府-公示公告"
MAX_WORKERS = 6
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
    "Referer": "https://www.zgsr.gov.cn/",
}


def fetch(url, timeout=30):
    for att in range(3):
        try:
            r = requests.get(url, timeout=timeout, headers=HEADERS)
            r.encoding = "utf-8"
            if r.status_code == 200:
                return r
            time.sleep(2)
        except:
            if att < 2:
                time.sleep(2)
    return None


def parse_list_page(html):
    soup = BeautifulSoup(html, "html.parser")
    items = []
    for li in soup.select("ul#doclist li"):
        a_tag = li.select_one("div.zcwj-wj > a[href]")
        date_tag = li.select_one("span.span_date")
        if a_tag and a_tag.get("href"):
            href = a_tag["href"].strip()
            full_url = BASE + href if href.startswith("/") else href
            date_str = date_tag.get_text(strip=True) if date_tag else ""
            items.append((full_url, date_str))
    return items


def collect_urls(max_pages=5):
    all_urls = []
    for pn in range(1, max_pages + 1):
        url = LIST_URL if pn == 1 else PAGE_URL.format(pn)
        print(f"  列表页 {pn}: {url}", flush=True)
        r = fetch(url)
        if not r:
            print(f"  第{pn}页获取失败", flush=True)
            continue
        items = parse_list_page(r.text)
        if not items:
            print(f"  第{pn}页无数据，停止", flush=True)
            break
        print(f"    找到 {len(items)} 条", flush=True)
        all_urls.extend(items)
    print(f"共收集 {len(all_urls)} 条URL", flush=True)
    return all_urls


def extract_detail(url):
    r = fetch(url)
    if not r:
        return None
    soup = BeautifulSoup(r.text, "html.parser")
    # 标题
    title_tag = soup.select_one("div.zwtit")
    title = title_tag.get_text(strip=True) if title_tag else ""
    if not title:
        ucap = soup.find("ucaptitle")
        title = ucap.get_text(strip=True) if ucap else "无标题"
    # 日期
    publish_date = ""
    for li in soup.select("ul.ztfl li"):
        text = li.get_text(" ", strip=True)
        if "成文日期" in text:
            date_span = li.select_one("span:last-child")
            if date_span:
                publish_date = date_span.get_text(strip=True)
            break
    # 正文
    content_html = ""
    content_div = soup.select_one("div.zwcon.clearfix#zoomcon")
    if content_div:
        ucap = content_div.find("UCAPCONTENT")
        if ucap:
            content_parts = []
            for p in ucap.find_all("p"):
                content_parts.append(str(p))
            content_html = "\n".join(content_parts)
        else:
            content_html = str(content_div)
    # 附件
    attachments = []
    appendix_div = soup.select_one("div.article-appendixs")
    if appendix_div:
        for a in appendix_div.select("a[href]"):
            href = a["href"].strip()
            fname = a.get_text(strip=True)
            if href and fname:
                full_href = BASE + href if href.startswith("/") else href
                attachments.append({"name": fname, "url": full_href})
    return {
        "title": title,
        "publish_date": publish_date,
        "content": content_html,
        "attachments": attachments,
    }


def process_item(args):
    url, date_str = args
    detail = extract_detail(url)
    if not detail:
        return None
    title = detail["title"] or "无标题"
    publish_date = detail["publish_date"] or date_str
    content = detail["content"]
    attachments = detail["attachments"]
    # 空内容回退
    if not content or len(content.strip()) < 20:
        content = f'<p><a href="{url}">{title}</a></p>'
    # 附加附件链接
    if attachments:
        for att in attachments:
            content += f"\n\n[附件: {att['name']}]({att['url']})"
    return {
        "title": title,
        "page_url": url,
        "content": content,
        "publish_date": publish_date,
        "summary": (content[:200] if content else "").replace("\n", " ").strip(),
        "site_name": SITE_NAME,
        "category": "公示公告",
        "attachments": attachments,
    }


def save_to_db(items):
    db = sqlite3.connect(DB_PATH, timeout=60)
    inserted = 0
    updated = 0
    for item in items:
        if not item:
            continue
        att_json = json.dumps(item["attachments"], ensure_ascii=False) if item["attachments"] else ""
        try:
            db.execute("""
                INSERT OR REPLACE INTO gov_raw
                (title, page_url, content, publish_date, summary, site_name, category, attachments)
                VALUES (?,?,?,?,?,?,?,?)
            """, (
                item["title"], item["page_url"], item["content"],
                item["publish_date"], item["summary"],
                item["site_name"], item["category"], att_json
            ))
            inserted += 1
        except sqlite3.IntegrityError:
            updated += 1
    db.commit()
    db.close()
    return inserted, updated


def main(max_pages=5):
    print(f"=== {SITE_NAME} 爬虫 ===", flush=True)
    print(f"爬取前 {max_pages} 页", flush=True)
    urls = collect_urls(max_pages)
    print(f"开始爬取详情 ({len(urls)} 条)...", flush=True)
    results = []
    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as pool:
        futures = {pool.submit(process_item, u): u for u in urls}
        for f in as_completed(futures):
            try:
                res = f.result()
                if res:
                    results.append(res)
                    print(f"  ✅ {res['title'][:40]}...", flush=True)
                else:
                    url = futures[f][0]
                    print(f"  ❌ {url}", flush=True)
            except Exception as e:
                url = futures[f][0]
                print(f"  ❌ {url}: {e}", flush=True)
    print(f"详情提取完成: {len(results)}/{len(urls)}", flush=True)
    inserted, updated = save_to_db(results)
    print(f"入库: 新增{inserted} / 更新{updated}", flush=True)
    print("=== 完成 ===", flush=True)
    return inserted, updated, len(results)


if __name__ == "__main__":
    import sys
    _pages_args = [int(a.split('=', 1)[1]) for a in sys.argv if a.startswith('--pages=')]
    _pages_args = _pages_args or [int(a) for a in sys.argv[1:] if a.isdigit()]
    pages = _pages_args[0] if _pages_args else 5
    main(max_pages=pages)