#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
江门市生态环境局 蓬江/江海分局 - 工作动态 (www.jiangmen.gov.cn/bmpd/jmssthjj/pcfj/jhfj/gzdt/)
列表: TRS 静态分页 index_N.html (首页 index.html, 共75页, 23条/页, P76=404 自动停)
      条目 <a href=".../content/post_ID.html">标题</a> + 日期
详情: meta ArticleTitle/PubDate + div.content 正文 (保留 <p>/<table>)
      附件 <a href="/attachment/0/x/x/ID.pdf"> 绝对 URL
样本: https://www.jiangmen.gov.cn/bmpd/jmssthjj/pcfj/jhfj/gzdt/content/post_3534708.html
用法: python3 crawl_jm_jhfj_gzdt.py [--pages=N | N] [--sync]
"""
import requests
import re
import sys
import os
import json
import html as html_mod
from bs4 import BeautifulSoup

SITE_NAME = "江门市江海分局-工作动态"
BASE_URL = "https://www.jiangmen.gov.cn"
LIST_PATH = "/bmpd/jmssthjj/pcfj/jhfj/gzdt"
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Referer": "https://www.jiangmen.gov.cn/bmpd/jmssthjj/pcfj/jhfj/gzdt/",
}
CUTOFF = "2023-01-01"

QUALITY_DB = os.path.join(os.path.dirname(os.path.abspath(__file__)), "quality_results.db")
SEARCH_DB = "/root/search.db"
DOMAIN = "www.jiangmen.gov.cn"


def push_to_searchdb(records, batch_label):
    import sqlite3
    conn = sqlite3.connect(QUALITY_DB, timeout=60)
    conn.execute("PRAGMA busy_timeout=300000")
    cur = conn.cursor()
    cur.execute("""CREATE TABLE IF NOT EXISTS crawl_results (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        site_id INTEGER DEFAULT 0,
        title TEXT, url TEXT, content TEXT,
        publish_date TEXT, summary TEXT,
        domain TEXT, category TEXT DEFAULT '',
        content_hash TEXT, crawl_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
        industry TEXT DEFAULT '', group_name TEXT DEFAULT '',
        script_name TEXT DEFAULT '', attachments TEXT DEFAULT '[]',
        UNIQUE(domain, url)
    )""")
    n_new = 0
    for r in records:
        try:
            cur.execute(
                "INSERT OR IGNORE INTO crawl_results (domain, title, url, content, publish_date, summary, industry, group_name, script_name, attachments) VALUES (?,?,?,?,?,?,?,?,?,?)",
                (r.get("domain", ""), r.get("title", ""), r.get("url", ""),
                 r.get("content", ""), r.get("pub_date", ""), r.get("summary", ""),
                 r.get("industry", ""), r.get("group_name", ""),
                 r.get("script_name", ""), r.get("attachments", "[]")),
            )
            n_new += cur.rowcount
        except Exception as e:
            print(f"  [push] ERR {r.get('url','')}: {e}")
    conn.commit()
    conn.close()
    print(f"[push_to_searchdb] new={n_new} skip={len(records)-n_new}")
    return n_new


def sync_to_server():
    import sqlite3
    try:
        conn = sqlite3.connect(QUALITY_DB, timeout=60)
        conn.execute("PRAGMA busy_timeout=300000")
        rows = conn.execute(
            "SELECT title, url, content, publish_date, summary, industry, group_name, script_name, attachments FROM crawl_results WHERE domain=? ORDER BY id",
            (DOMAIN,),
        ).fetchall()
        conn.close()
    except Exception as e:
        print(f"[sync] local read ERR: {e}")
        return
    if not rows:
        print("[sync] 本地没有数据")
        return
    dst = sqlite3.connect(SEARCH_DB, timeout=300)
    dst.execute("PRAGMA journal_mode=WAL")
    dst.execute("PRAGMA busy_timeout=300000")
    upd = ins = 0
    try:
        for r in rows:
            title, url, content, pub_date, summary, ind, grp, sn, att = r
            cur = dst.execute("SELECT id FROM gov_raw WHERE page_url=?", (url,))
            row = cur.fetchone()
            try:
                if row:
                    dst.execute(
                        "UPDATE gov_raw SET title=?, content=?, publish_date=?, summary=?, industry=?, group_name=?, script_name=?, attachments=? WHERE id=?",
                        (title, (content or "")[:500000], pub_date or "", (summary or "")[:300],
                         ind or "", grp or "", sn or "", att or "[]", row[0]))
                    upd += 1
                else:
                    dst.execute(
                        "INSERT INTO gov_raw (title, page_url, content, publish_date, summary, industry, group_name, script_name, attachments, site_name, source_url) VALUES (?,?,?,?,?,?,?,?,?,?,?)",
                        (title, url, (content or "")[:500000], pub_date or "", (summary or "")[:300],
                         ind or "", grp or "", sn or "", att or "[]", SITE_NAME, url))
                    ins += 1
            except Exception as e:
                print(f"  [sync] ERR {url}: {e}")
        dst.commit()
        dst.execute(
            "INSERT INTO gov_search(rowid,title,site_name,summary) "
            "SELECT r.id,r.title,r.site_name,r.summary FROM gov_raw r "
            "WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?",
            (SITE_NAME,))
        dst.commit()
        total = dst.execute("SELECT COUNT(*) FROM gov_raw WHERE site_name=?", (SITE_NAME,)).fetchone()[0]
    finally:
        dst.close()
    print(f"[sync] OK updated={upd} inserted={ins} DB共{total}条")


def clean_title(t):
    t = re.sub(r"&middot;|&nbsp;|&#160;|\u200b|\ufeff", "", t or "")
    t = re.sub(r"^[•·]\s*", "", t)
    return re.sub(r"\s+", " ", t).strip()


def fetch_list(page):
    if page == 1:
        url = f"{BASE_URL}{LIST_PATH}/"
    else:
        url = f"{BASE_URL}{LIST_PATH}/index_{page}.html"
    try:
        r = requests.get(url, headers=HEADERS, timeout=30)
        if r.status_code == 404:
            return [], False
        r.raise_for_status()
        r.encoding = "utf-8"
        soup = BeautifulSoup(r.text, "html.parser")
    except Exception as e:
        print(f"  [list] P{page} ERR: {e}")
        return [], False
    items = []
    for a in soup.find_all("a", href=True):
        href = a["href"].strip()
        m = re.search(r"content/post_(\d+)\.html$", href)
        if not m:
            continue
        # 必须限定 jhfj 栏目路径，防页脚「关于我们/联系方式/声明」混入
        if "jhfj/gzdt/" not in href and "pcfj/jhfj/gzdt/" not in href:
            continue
        title = clean_title(a.get_text(strip=True))
        if not title:
            continue
        # 找日期（a 的父级 li 内 span/时间）
        pub_date = ""
        p = a.find_parent("li")
        if p:
            m2 = re.search(r"(\d{4}-\d{2}-\d{2})", p.get_text())
            if m2:
                pub_date = m2.group(1)
        if not pub_date:
            m2 = re.search(r"(\d{4}-\d{2}-\d{2})", a.get_text())
            if m2:
                pub_date = m2.group(1)
        full_url = href if href.startswith("http") else BASE_URL + (href if href.startswith("/") else "/" + href)
        items.append({"title": title, "url": full_url, "pub_date": pub_date})
    # 去重
    seen = set()
    uniq = []
    for it in items:
        if it["url"] in seen:
            continue
        seen.add(it["url"])
        uniq.append(it)
    return uniq, True


def parse_detail(url):
    try:
        r = requests.get(url, headers=HEADERS, timeout=30)
        r.raise_for_status()
        r.encoding = "utf-8"
        soup = BeautifulSoup(r.text, "html.parser")
    except Exception as e:
        print(f"  [detail] {url} ERR: {e}")
        return None

    title = ""
    m = soup.find("meta", attrs={"name": "ArticleTitle"})
    if m and m.get("content"):
        title = clean_title(m["content"])
    if not title:
        h = soup.find("h1")
        if h:
            title = clean_title(h.get_text())
    if not title:
        m = re.search(r"<title>(.*?)</title>", r.text, re.S)
        if m:
            title = clean_title(re.sub(r"_\s*江门市生态环境局$", "", m.group(1).strip()))

    pub_date = ""
    m = soup.find("meta", attrs={"name": "PubDate"})
    if m and m.get("content"):
        pub_date = m["content"].strip()[:10]

    body_el = soup.select_one("div.content") or soup.select_one("div.TRS_UEDITOR") or soup.select_one("div.view")
    if body_el is None:
        for el in soup.find_all(class_=re.compile("TRS_UEDITOR|article-content|news-content")):
            body_el = el
            break
    if body_el is None:
        return {"title": title, "content": "", "pub_date": pub_date, "attachments": []}

    # 附件
    atts = []
    for a in body_el.find_all("a", href=True):
        href = a["href"].strip()
        if re.search(r"\.(pdf|doc|docx|xls|xlsx|zip|rar|wps|7z)$", href, re.I) or "/attachment/" in href:
            abs_url = href if href.startswith("http") else ("https:" + href if href.startswith("//") else BASE_URL + (href if href.startswith("/") else "/" + href))
            name = clean_title(a.get_text(strip=True)) or href.split("/")[-1]
            if not any(x["url"] == abs_url for x in atts):
                atts.append({"name": name, "url": abs_url})
    page_att = soup.select_one("div.attachment") or soup.select_one("div.attachments")
    if page_att:
        for a in page_att.find_all("a", href=True):
            href = a["href"].strip()
            if href:
                abs_url = href if href.startswith("http") else ("https:" + href if href.startswith("//") else BASE_URL + (href if href.startswith("/") else "/" + href))
                name = clean_title(a.get_text(strip=True)) or href.split("/")[-1]
                if not any(x["url"] == abs_url for x in atts):
                    atts.append({"name": name, "url": abs_url})

    # 附件段落从正文移除
    for a in body_el.find_all("a", href=True):
        href = a["href"].strip()
        if re.search(r"\.(pdf|doc|docx|xls|xlsx|zip|rar|wps|7z)$", href, re.I) or "/attachment/" in href:
            p = a.find_parent("p")
            if p:
                p.decompose()
    # 绝对化
    for a in body_el.find_all("a", href=True):
        href = a["href"].strip()
        if href.startswith("/") or href.startswith("./") or (not href.startswith("http") and not href.startswith("javascript")):
            a["href"] = ("https:" + href if href.startswith("//") else BASE_URL + (href if href.startswith("/") else "/" + href))
    for img in body_el.find_all("img"):
        src = img.get("src", "")
        if src and not src.startswith("http"):
            img["src"] = ("https:" + src if src.startswith("//") else BASE_URL + (src if src.startswith("/") else "/" + src))
    # 二维码/分享清理
    for el in body_el.select("div.share, div.ewm, div.qrcode, div.jzgov-qrcode-div, div.app-share"):
        el.decompose()
    for el in body_el.find_all(string=re.compile("扫一扫|分享到|打印本页")):
        p = el.find_parent("p") or el.find_parent("div")
        if p:
            p.decompose()

    body_html = str(body_el.decode_contents())
    body_html = re.sub(r"<script[^>]*>.*?</script>", "", body_html, flags=re.S | re.I)
    body_html = re.sub(r"<style[^>]*>.*?</style>", "", body_html, flags=re.S | re.I)
    body_html = re.sub(r"<p[^>]*>", "<p>", body_html)
    body_html = re.sub(r"<p>\s*</p>", "", body_html)
    body_html = re.sub(r'\s*<div id="docAppendix".*$', "", body_html, flags=re.S)
    body_html = re.sub(r'\s*<div class="video".*$', "", body_html, flags=re.S)

    if atts:
        has_att = any(a["url"] in body_html for a in atts)
        if not has_att:
            att_html = "".join(f'<p><a href="{a["url"]}">{html_mod.escape(a["name"])}</a></p>' for a in atts)
            body_html = body_html.rstrip() + "\n" + att_html

    text_len = len(re.sub(r"<[^>]+>", "", body_html).strip())
    has_img = "<img" in body_html
    if text_len < 5 and not has_img and not atts:
        return {"title": title, "content": "", "pub_date": pub_date, "attachments": atts}
    return {"title": title, "content": body_html, "pub_date": pub_date, "attachments": atts}


def main():
    max_pages = None
    for a in sys.argv[1:]:
        if a.startswith("--pages="):
            try:
                max_pages = int(a.split("=", 1)[1])
            except ValueError:
                pass
        elif a.isdigit():
            max_pages = int(a)
    print(f"[AutoPg] max_pages={max_pages}")

    valid = []
    for pg in range(1, 100):
        if max_pages and pg > max_pages:
            break
        items, has_next = fetch_list(pg)
        print(f"[list] P{pg}: items={len(items)}")
        if not items:
            break
        for it in items:
            if it["pub_date"] and it["pub_date"] < CUTOFF:
                print(f"  [cutoff] {it['pub_date']} {it['title'][:40]}")
                continue
            det = parse_detail(it["url"])
            if det is None:
                print(f"  [fail] {it['title'][:50]}")
                det = {"title": it["title"], "content": "", "pub_date": it["pub_date"], "attachments": []}
            rec = {
                "domain": DOMAIN,
                "site_name": SITE_NAME,
                "title": det["title"] or it["title"],
                "url": it["url"],
                "content": det["content"],
                "pub_date": det["pub_date"] or it["pub_date"],
                "summary": re.sub(r"<[^>]+>", "", det["content"])[:200],
                "industry": "",
                "group_name": "广东",
                "script_name": os.path.basename(sys.argv[0]),
                "attachments": json.dumps(det["attachments"], ensure_ascii=False) if det["attachments"] else "[]",
            }
            valid.append(rec)
        if not has_next:
            break
    print(f"[total] valid={len(valid)}")
    if valid:
        push_to_searchdb(valid, os.path.basename(sys.argv[0]))
        if "--sync" in sys.argv:
            sync_to_server()


if __name__ == "__main__":
    main()