#!/usr/bin/env python3
"""
政府网站群批量爬虫: 方城/镇江经开/泰安高新 (结构相似: meta ArticleTitle+PubDate + zoom/content 容器)
用法: python3 crawl_gov_group.py --site=fangcheng|zjna|taian [--pages=N]
"""
import os, sys, re, time, json
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
HEADERS = {"User-Agent": UA, "Accept": "text/html,application/xhtml+xml", "Accept-Language": "zh-CN,zh;q=0.9"}

SITES = {
    "fangcheng": {
        "name": "方城县生态环境局-公示公告",
        "list": "https://www.fangcheng.gov.cn/nyssthjjfcfj/gsgg/",
        "link_pat": r'href="([^"]*/\d{4}/\d{2}-\d{2}/\d+\.html)"[^>]*>([^<]{4,80})</a>',
        "page_url": "https://www.fangcheng.gov.cn/nyssthjjfcfj/gsgg/index_{}.html",
        "containers": ["#contents", "div.newscontents", "div.content", "div.zoom", "#zoom", "div.view", "div.article"],
        "category": "公示公告",
    },
    "zjna": {
        "name": "镇江经开区-通告公示",
        "list": "https://www.zjna.gov.cn/zjna/tggs/xxgk_lists.shtml",
        "link_pat": r'href="(/zjna/[a-z]+/\d{6}/[0-9a-f]{32}\.shtml)"[^>]*>([^<]{4,80})</a>',
        "page_url": "https://www.zjna.gov.cn/zjna/tggs/xxgk_lists_{}.shtml",
        "containers": ["#zoom", "div.zoom", "div.content", "div.view", "div.article", "#content"],
        "category": "通告公示",
    },
    "taian": {
        "name": "泰安高新区-通知公告",
        "list": "https://gxq.taian.gov.cn/col/col47833/index.html",
        "link_pat": r'href="([^"]*art_47833_\d+\.html)"[^>]*>([^<]{4,80})</a>',
        "page_url": "https://gxq.taian.gov.cn/col/col47833/index_{}.html",
        "containers": ["#zoom", "div.zoom", "div.content", "div.view", "div.article"],
        "category": "通知公告",
    },
}

def parse_pages_arg():
    for a in sys.argv[1:]:
        if a.startswith("--pages="):
            return int(a.split("=")[1])
        if a.isdigit():
            return int(a)
    return None

def get_site():
    for a in sys.argv[1:]:
        if a.startswith("--site="):
            return a.split("=")[1]
    return None

def clean_title(title):
    import html as html_mod
    title = html_mod.unescape(title or "")
    title = re.sub(r"&middot;|&nbsp;", "", title)
    title = re.sub(r"[\u200b\ufeff]", "", title)
    title = re.sub(r"<[^>]+>", " ", title)
    title = re.sub(r"^[•·]\s*", "", title)
    title = re.sub(r"\s+", " ", title)
    return title.strip()

def fetch(url, session):
    try:
        r = session.get(url, timeout=20)
        r.encoding = "utf-8"
        return r.text if r.status_code == 200 else ""
    except Exception:
        return ""

def parse_list(html, cfg, list_url):
    items = []
    for m in re.finditer(cfg["link_pat"], html):
        href, title = m.group(1), m.group(2).strip()
        if not title or len(title) < 4:
            continue
        if any(x in title for x in ["首页", "邮箱", "搜索", "无障碍", "地图", "网站群", "目录"]):
            continue
        url = urljoin(list_url, href)
        dm = re.search(r'/(\d{4})/(\d{2})-(\d{2})/', href)
        date = ""
        if dm:
            date = f"{dm.group(1)}-{dm.group(2)}-{dm.group(3)}"
        items.append({"title": clean_title(title), "url": url, "date": date})
    return items

def extract_detail(html, page_url, cfg):
    soup = BeautifulSoup(html, "html.parser")
    title = ""
    m = re.search(r'<meta[^>]*name=["\']ArticleTitle["\'][^>]*content=["\']([^"\']*)["\']', html, re.I)
    if m:
        title = clean_title(m.group(1))
    if not title:
        h1 = soup.find("h1")
        if h1:
            title = clean_title(h1.get_text(strip=True))
    if not title:
        t = soup.find("title")
        if t:
            title = clean_title(t.get_text(strip=True).split("|")[0])

    date = ""
    m = re.search(r'<meta[^>]*name=["\']PubDate["\'][^>]*content=["\'](\d{4}-\d{2}-\d{2})', html, re.I)
    if m:
        date = m.group(1)
    if not date:
        for kw in ["发布时间", "发布日期", "成文日期"]:
            m = re.search(r'%s[：:]\s*(\d{4}-\d{2}-\d{2})' % kw, html)
            if m:
                date = m.group(1)
                break

    body_el = None
    for sel in cfg["containers"]:
        body_el = soup.select_one(sel)
        if body_el and body_el.get_text(strip=True):
            break

    content = ""
    attachments = []
    if body_el:
        for a in body_el.find_all("a", href=True):
            href = a["href"].strip()
            if re.search(r'\.(pdf|doc|docx|xls|xlsx|zip|rar|wps)$', href.lower()) or "download" in href.lower():
                full = urljoin(page_url, href)
                name = a.get_text(strip=True) or href.split("/")[-1]
                attachments.append({"name": name, "url": full})
        parts = []
        for el in body_el.find_all(["p", "table", "img"]):
            if el.name == "p":
                if el.find_parent("table") or el.find("table") or el.find("p"):
                    continue
                for a in el.find_all("a", href=True):
                    href = a["href"].strip()
                    if re.search(r'\.(pdf|doc|docx|xls|xlsx|zip|rar|wps)$', href.lower()) or "download" in href.lower():
                        a["href"] = urljoin(page_url, href)
                txt = re.sub(r"\s+", " ", el.get_text(" ", strip=True))
                if txt:
                    parts.append(f"<p>{txt}</p>")
            elif el.name == "table":
                parts.append(str(el))
            elif el.name == "img":
                src = el.get("src", "")
                if src:
                    parts.append(f'<p><img src="{urljoin(page_url, src)}"></p>')
        for att in attachments:
            parts.append(f'<p><a href="{att["url"]}">{att["name"]}</a></p>')
        content = "\n".join(parts)
        if not content:
            txt = body_el.get_text(" ", strip=True)
            if txt:
                content = f"<p>{txt}</p>"
    return title, date, content, attachments

def main():
    site_key = get_site()
    if site_key not in SITES:
        print("用法: python3 crawl_gov_group.py --site=fangcheng|zjna|taian [--pages=N]")
        return
    cfg = SITES[site_key]
    max_pages = parse_pages_arg()
    if max_pages is None:
        max_pages = 40
    print(f"[AutoPg] max_pages={max_pages} (SITE={cfg['name']})")
    session = requests.Session()
    session.headers.update(HEADERS)

    all_items = []
    seen = set()
    for pg in range(1, max_pages + 1):
        if pg == 1:
            url = cfg["list"]
        else:
            url = cfg["page_url"].format(pg)
        html = fetch(url, session)
        if not html or len(html) < 300:
            print(f"  Page {pg}: 空/失败, 停止")
            break
        items = parse_list(html, cfg, cfg["list"])
        new_items = [x for x in items if x["url"] not in seen]
        for x in new_items:
            seen.add(x["url"])
        all_items.extend(new_items)
        print(f"  Page {pg}: {len(items)}条(新{len(new_items)}) 累计{len(all_items)}")
        if not new_items:
            break
        time.sleep(0.5)

    print(f"\n共 {len(all_items)} 条, 抓详情...")
    results = []
    for i, item in enumerate(all_items):
        html = fetch(item["url"], session)
        title, date, content, attachments = extract_detail(html, item["url"], cfg) if html else (None, None, None, None)
        if not title:
            title = item["title"]
        if not date:
            date = item["date"]
        if not content:
            print(f"  [WARN] 空正文: {title[:40]}")
        results.append({"site_name": cfg["name"], "title": title, "pub_date": date, "content": content,
                        "source_url": item["url"], "url": item["url"], "category": cfg["category"],
                        "attachments": json.dumps(attachments, ensure_ascii=False) if attachments else ""})
        if (i + 1) % 20 == 0:
            print(f"  [{i+1}/{len(all_items)}] {title[:40]}")
        time.sleep(0.3)

    push_to_searchdb(results, batch_label=cfg["name"])
    dates = [r["pub_date"] for r in results if r["pub_date"]]
    print(f"\n站点: {cfg['name']} 采集: {len(results)} 日期: {min(dates) if dates else '-'} ~ {max(dates) if dates else '-'}")

if __name__ == "__main__":
    main()