#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
梧州临港经济区管理委员会 - 通知公告 (www.wuzhou.gov.cn/wzlgjjqglwyh/zfxxgk/fdzdgk_lg/wjzl_lg/tzgg/)
CMS: 广西政府集约化平台 (TRS) - 与藤县 tengxian 同款
列表: <ul class="more-list"><li><span>日期</span><a href="./tXXX.shtml" title="完整标题">标题</a></li>
      分页 createPageHTML(总页数, 当前, "index","shtml","总条数") → index_N.shtml (本栏目单页12条)
详情: meta ArticleTitle / meta PubDate / div.article-con > div.trs_editor_view (正文 HTML)
特殊: 纯 PDF 附件公告(无详情页) → 正文 = 标题 + URL 内嵌段; 外链(rsj.wuzhou.gov.cn等)跳过
用法: python3 crawl_wuzhou_lg_tzgg.py [--pages=N | N] [--sync] [--sync-only]
"""
import re
import sys
import os
import json
import html as html_mod
from datetime import datetime
from urllib.parse import urljoin
from concurrent.futures import ThreadPoolExecutor, as_completed
import requests
import sqlite3

SITE_NAME = "梧州临港经济区管委会-通知公告"
BASE_URL = "http://www.wuzhou.gov.cn"
LIST_URL = "http://www.wuzhou.gov.cn/wzlgjjqglwyh/zfxxgk/fdzdgk_lg/wjzl_lg/tzgg/"
LIST_BASE = LIST_URL  # 相对链接基于栏目目录
DOMAIN = "www.wuzhou.gov.cn"
GROUP = "广西"
CUTOFF = "2020-01-01"  # 全量抓取
DB_PATH = os.environ.get("DB_PATH", "/root/search.db")
QUALITY_DB = os.path.join(os.path.dirname(os.path.abspath(__file__)), "quality_results.db")
MAX_WORKERS = 5

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Referer": LIST_URL,
}


def clean_title(t):
    t = re.sub(r"&middot;|&nbsp;|&#160;|\u200b|\ufeff", "", t or "")
    t = re.sub(r"^[•·]\s*", "", t)
    return re.sub(r"\s+", " ", t).strip()


def normalize_date(s):
    m = re.search(r"(\d{4})[年\-/. ](\d{1,2})[月\-/. ](\d{1,2})", s or "")
    if m:
        y, mo, d = int(m.group(1)), int(m.group(2)), int(m.group(3))
        return f"{y:04d}-{mo:02d}-{d:02d}"
    return ""


def fetch(url, retries=3):
    for i in range(retries):
        try:
            resp = requests.get(url, timeout=30, headers=HEADERS)
            resp.encoding = "utf-8"
            if resp.status_code == 200:
                return resp.text
        except Exception as e:
            if i < retries - 1:
                import time
                time.sleep(2)
    return None


def get_page_count(html):
    m = re.search(r"createPageHTML\s*\(\s*(\d+)", html or "")
    return int(m.group(1)) if m else 1


def extract_list_items(html, base_url):
    """提取 ul.more-list 中的条目（PC/移动端双份去重）。返回 [{title,url,pub_date,href_raw}]"""
    items = []
    if not html:
        return items
    seen = set()
    for m in re.finditer(
        r'<li>\s*<span>([^<]*)</span>\s*<a[^>]*href="([^"]+)"[^>]*title="([^"]*)"[^>]*>(.*?)</a>',
        html, re.DOTALL):
        date_s, href, title_attr = m.group(1).strip(), m.group(2).strip(), m.group(3).strip()
        pub_date = normalize_date(date_s)
        title = clean_title(title_attr) or clean_title(re.sub(r"<[^>]+>", "", m.group(4)))
        if href in seen:
            continue
        seen.add(href)
        detail_url = urljoin(base_url, href) if not href.startswith("http") else href
        items.append({"title": title, "url": detail_url, "pub_date": pub_date, "href_raw": href})
    return items


def parse_detail(url):
    html_text = fetch(url)
    if not html_text:
        return None

    title = ""
    m = re.search(r'<meta\s+name="ArticleTitle"\s+content="([^"]*)"', html_text)
    if m:
        title = clean_title(m.group(1))
    if not title:
        m = re.search(r'<div\s+id="NewsArticleTitle"[^>]*>(.*?)</div>', html_text, re.DOTALL)
        if m:
            title = clean_title(re.sub(r"<[^>]+>", "", m.group(1)))
    if not title:
        m = re.search(r"<title>(.*?)</title>", html_text, re.DOTALL)
        if m:
            title = clean_title(re.sub(r"[-_—]\s*.*$", "", m.group(1).strip()))

    pub_date = ""
    m = re.search(r'<meta\s+name="PubDate"\s+content="([^"]*)"', html_text)
    if m:
        pub_date = normalize_date(m.group(1))
    if not pub_date:
        m = re.search(r'<div\s+id="NewsArticlePubDay"[^>]*>(.*?)</div>', html_text, re.DOTALL)
        if m:
            pub_date = normalize_date(re.sub(r"<[^>]+>", "", m.group(1)))

    body_el = None
    m = re.search(r'<div\s+class="trs_editor_view[^"]*"[^>]*>(.*)', html_text, re.DOTALL)
    if m:
        # 正文到 article-con 结束
        raw = m.group(1)
        end = raw.find("</div>")
        # trs_editor_view 内部有嵌套 div，截取到正文区自然结束：找 article-con 的闭合
        m2 = re.search(r'(.*?)</div>\s*<!--\s*正文e\s*-->', raw, re.DOTALL)
        if m2:
            body_html = m2.group(1)
        else:
            # 截取到下一个明显的尾部标记
            cut = re.search(r'</div>\s*</div>\s*<!--|</div>\s*</div>\s*<div class="(footer|share)', raw)
            body_html = raw[: cut.start() + 6] if cut else raw[: end + 6]
    else:
        body_html = ""

    body_html = re.sub(r"<script[^>]*>.*?</script>", "", body_html, flags=re.S | re.I)
    body_html = re.sub(r"<style[^>]*>.*?</style>", "", body_html, flags=re.S | re.I)

    # 附件：正文内 + 独立附件区 div#downloadfile（TRS 模板"文件下载"区在正文容器外）
    atts = []
    att_zone = ""
    m_zone = re.search(r'<div\s+id="(?:downloadfile|article-file|relfile)"[^>]*>(.*?)</div>', html_text, re.DOTALL)
    if m_zone:
        att_zone = m_zone.group(1)
    att_sources = body_html + "\n" + att_zone
    for m in re.finditer(r'<a[^>]*href="([^"]+)"[^>]*>([^<]{0,120})</a>', att_sources):
        href, name = m.group(1).strip(), clean_title(m.group(2))
        if re.search(r"\.(pdf|doc|docx|xls|xlsx|zip|rar|wps|7z|et|dps)$", href, re.I) or "/document/download" in href:
            abs_url = href if href.startswith("http") else (BASE_URL + href if href.startswith("/") else urljoin(LIST_BASE, href))
            if not any(a["url"] == abs_url for a in atts):
                atts.append({"name": name or href.split("/")[-1], "url": abs_url})

    # 正文内链接/图片绝对化
    def abs_link(m):
        href = m.group(1).strip()
        if href.startswith("http") or href.startswith("javascript") or href.startswith("#"):
            return m.group(0)
        if href.startswith("//"):
            return m.group(0).replace(href, "http:" + href)
        return m.group(0).replace(href, BASE_URL + href if href.startswith("/") else urljoin(LIST_BASE, href))

    body_html = re.sub(r'href="([^"]+)"', abs_link, body_html)
    body_html = re.sub(r'src="([^"]+)"', abs_link, body_html)

    # 清理二维码/分享/打印/附件区包装
    body_html = re.sub(r'<div[^>]*class="[^"]*(share|ewm|qrcode|print|bshare)[^"]*"[^>]*>.*?</div>', "", body_html, flags=re.S | re.I)
    body_html = re.sub(r'<p>\s*</p>', "", body_html)
    body_html = re.sub(r'<p[^>]*>', "<p>", body_html)

    if atts:
        has_att = any(a["url"] in body_html for a in atts)
        if not has_att:
            att_html = "".join(f'<p><a href="{a["url"]}">{html_mod.escape(a["name"])}</a></p>' for a in atts)
            body_html = body_html.rstrip() + "\n" + att_html

    text_len = len(re.sub(r"<[^>]+>", "", body_html).strip())
    has_img = "<img" in body_html
    has_video = "<iframe" in body_html or "video" in body_html
    if text_len < 5 and not has_img and not has_video and not atts:
        return {"title": title, "content": "", "pub_date": pub_date, "attachments": []}
    return {"title": title, "content": body_html, "pub_date": pub_date, "attachments": atts}


def push_local(records, script_name):
    """本地 quality_results.db（幂等）"""
    conn = sqlite3.connect(QUALITY_DB, timeout=60)
    conn.execute("PRAGMA busy_timeout=300000")
    cur = conn.cursor()
    cur.execute("""CREATE TABLE IF NOT EXISTS crawl_results (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        site_id INTEGER DEFAULT 0,
        title TEXT, url TEXT, content TEXT,
        publish_date TEXT, summary TEXT,
        domain TEXT, category TEXT DEFAULT '',
        content_hash TEXT, crawl_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
        industry TEXT DEFAULT '', group_name TEXT DEFAULT '',
        script_name TEXT DEFAULT '', attachments TEXT DEFAULT '[]',
        UNIQUE(domain, url)
    )""")
    n_new = 0
    for r in records:
        try:
            cur.execute(
                "INSERT OR IGNORE INTO crawl_results (domain, title, url, content, publish_date, summary, industry, group_name, script_name, attachments) VALUES (?,?,?,?,?,?,?,?,?,?)",
                (r.get("domain", ""), r.get("title", ""), r.get("url", ""),
                 r.get("content", ""), r.get("pub_date", ""), r.get("summary", ""),
                 r.get("industry", ""), r.get("group_name", ""),
                 r.get("script_name", ""), r.get("attachments", "[]")),
            )
            n_new += cur.rowcount
        except Exception as e:
            print(f"  [push] ERR {r.get('url','')}: {e}")
    conn.commit()
    conn.close()
    print(f"[push_local] new={n_new} skip={len(records)-n_new}")
    return n_new


def sync_to_server():
    """quality_results.db → /root/search.db gov_raw + gov_search"""
    try:
        conn = sqlite3.connect(QUALITY_DB, timeout=60)
        conn.execute("PRAGMA busy_timeout=300000")
        rows = conn.execute(
            "SELECT title, url, content, publish_date, summary, industry, group_name, script_name, attachments FROM crawl_results WHERE domain=? ORDER BY id",
            (DOMAIN,),
        ).fetchall()
        conn.close()
    except Exception as e:
        print(f"[sync] local read ERR: {e}")
        return
    if not rows:
        print("[sync] 本地没有数据")
        return
    dst = sqlite3.connect(DB_PATH, timeout=300)
    dst.execute("PRAGMA journal_mode=WAL")
    dst.execute("PRAGMA busy_timeout=300000")
    upd = ins = 0
    try:
        for r in rows:
            title, url, content, pub_date, summary, ind, grp, sn, att = r
            cur = dst.execute("SELECT id FROM gov_raw WHERE page_url=?", (url,))
            row = cur.fetchone()
            try:
                if row:
                    dst.execute(
                        "UPDATE gov_raw SET title=?, content=?, publish_date=?, summary=?, industry=?, group_name=?, script_name=?, attachments=? WHERE id=?",
                        (title, (content or "")[:500000], pub_date or "", (summary or "")[:300],
                         ind or "", grp or "", sn or "", att or "[]", row[0]))
                    upd += 1
                else:
                    dst.execute(
                        "INSERT INTO gov_raw (title, page_url, content, publish_date, summary, industry, group_name, script_name, attachments, site_name, source_url) VALUES (?,?,?,?,?,?,?,?,?,?,?)",
                        (title, url, (content or "")[:500000], pub_date or "", (summary or "")[:300],
                         ind or "", grp or "", sn or "", att or "[]", SITE_NAME, url))
                    ins += 1
            except Exception as e:
                print(f"  [sync] ERR {url}: {e}")
        dst.commit()
        dst.execute(
            "INSERT INTO gov_search(rowid,title,site_name,summary) "
            "SELECT r.id,r.title,r.site_name,r.summary FROM gov_raw r "
            "WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?",
            (SITE_NAME,))
        dst.commit()
        total = dst.execute("SELECT COUNT(*) FROM gov_raw WHERE site_name=?", (SITE_NAME,)).fetchone()[0]
    finally:
        dst.close()
    print(f"[sync] OK updated={upd} inserted={ins} DB共{total}条")


def main():
    sync_only = "--sync-only" in sys.argv
    max_pages = None
    do_sync = "--sync" in sys.argv
    for a in sys.argv[1:]:
        if a.startswith("--pages="):
            try:
                max_pages = int(a.split("=", 1)[1])
            except ValueError:
                pass
        elif a.isdigit():
            max_pages = int(a)
    print(f"[WuzhouLG] max_pages={max_pages} sync_only={sync_only}")
    if sync_only:
        sync_to_server()
        return

    html_text = fetch(LIST_URL)
    if not html_text:
        print("[list] 无法获取列表页")
        return
    total_pages = get_page_count(html_text)
    if max_pages and total_pages > max_pages:
        total_pages = max_pages
    print(f"[list] 总页数={total_pages}")

    all_items = []
    for pg in range(total_pages):
        if pg == 0:
            page_html = html_text
        else:
            page_url = f"{LIST_BASE}index_{pg}.shtml"
            page_html = fetch(page_url)
            if not page_html:
                print(f"[list] P{pg+1} 获取失败")
                break
        items = extract_list_items(page_html, LIST_BASE)
        # 过滤: 外链(其他子域)跳过, 只保留本站 t*.shtml 详情 或 本站纯PDF公告
        kept = []
        for it in items:
            if it["url"].startswith("http") and DOMAIN not in it["url"]:
                print(f"  [skip外链] {it['title'][:40]} -> {it['url']}")
                continue
            if re.search(r"\.(pdf|doc|docx|xls|xlsx|zip|rar)$", it["href_raw"], re.I):
                it["is_attachment"] = True
            else:
                it["is_attachment"] = False
            kept.append(it)
        print(f"[list] P{pg+1}: 原始{len(items)}条 保留{len(kept)}条")
        all_items.extend(kept)

    seen = set()
    uniq = []
    for it in all_items:
        if it["url"] in seen:
            continue
        seen.add(it["url"])
        uniq.append(it)
    print(f"[total] 列表去重后 {len(uniq)} 条")

    valid = []
    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
        fut_map = {executor.submit(parse_detail, it["url"]): it for it in uniq if not it.get("is_attachment")}
        for fut in as_completed(fut_map):
            it = fut_map[fut]
            det = fut.result()
            if det is None:
                print(f"  [fail] {it['title'][:40]}")
                det = {"title": it["title"], "content": "", "pub_date": it["pub_date"], "attachments": []}
            if det["pub_date"] and det["pub_date"] < CUTOFF:
                print(f"  [cutoff] {det['pub_date']} {det['title'][:40]}")
                continue
            rec = {
                "domain": DOMAIN,
                "site_name": SITE_NAME,
                "title": det["title"] or it["title"],
                "url": it["url"],
                "content": det["content"],
                "pub_date": det["pub_date"] or it["pub_date"],
                "summary": re.sub(r"<[^>]+>", "", det["content"])[:200],
                "industry": "",
                "group_name": GROUP,
                "script_name": os.path.basename(sys.argv[0]),
                "attachments": json.dumps(det["attachments"], ensure_ascii=False) if det["attachments"] else "[]",
            }
            valid.append(rec)

    # 纯 PDF 附件公告：正文 = 标题 + URL 内嵌段
    for it in uniq:
        if it.get("is_attachment"):
            att_url = it["url"]
            att_name = it["title"] or att_url.split("/")[-1]
            body = f'<p>{html_mod.escape(it["title"])}</p>\n<p><a href="{att_url}">{html_mod.escape(att_name)}</a></p>'
            valid.append({
                "domain": DOMAIN,
                "site_name": SITE_NAME,
                "title": it["title"],
                "url": att_url,
                "content": body,
                "pub_date": it["pub_date"],
                "summary": it["title"],
                "industry": "",
                "group_name": GROUP,
                "script_name": os.path.basename(sys.argv[0]),
                "attachments": json.dumps([{"name": att_name, "url": att_url}], ensure_ascii=False),
            })

    print(f"[total] valid={len(valid)}")
    if valid:
        push_local(valid, os.path.basename(sys.argv[0]))
        if do_sync:
            sync_to_server()


if __name__ == "__main__":
    main()