#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
crawl_thx_xxgk.py - 太湖县-通知公告
https://www.thx.gov.cn/grassroots/column/2000002251?catId=1041651
列表: grassroots/column, /grassroots/2000002251/{id}.html
分页: 单页(非增量)
详情: title + .content_main
"""
import sys
import os
import re
import time
import html as html_lib
import urllib.parse
import sqlite3
import json

try:
    from bs4 import BeautifulSoup
except ImportError:
    BeautifulSoup = None

# ---------------- config ----------------
BASE_URL = "https://www.thx.gov.cn"
LIST_URL_P1 = "https://www.thx.gov.cn/grassroots/column/2000002251?catId=1041651"
LIST_URL_PN = "https://www.thx.gov.cn/grassroots/column/2000002251?catId=1041651"
SITE_NAME = "太湖县-通知公告"
GROUP_NAME = "安徽"
SCRIPT_NAME = "crawl_thx_xxgk.py"
DB_PATH = os.environ.get("DB_PATH", "/mnt/data/search.db")
JSONL_PATH = os.environ.get("JSONL_PATH", "")
ENC = "utf-8"
DTITLE_MODE = "title"
DTITLE_SEL = ""
DCONTENT_SEL = ".xxgk_content"
LIST_LINK_RE = r'<a[^>]*href="(https?://[^"]*grassroots/2000002251/\d+\.html)"[^>]*>(.*?)</a>'

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9",
}

_MAX_PAGES = 1
for i, a in enumerate(sys.argv):
    if a.startswith("--pages="):
        try:
            _MAX_PAGES = int(a.split("=", 1)[1])
        except ValueError:
            pass
    elif a == "--pages" and i + 1 < len(sys.argv):
        try:
            _MAX_PAGES = int(sys.argv[i + 1])
        except ValueError:
            pass

_PAGES = _MAX_PAGES if _MAX_PAGES >= 1 else 1


def clean_title(t):
    if not t:
        return ""
    t = html_lib.unescape(t)
    t = t.replace("\u200b", "").replace("\u200c", "").replace("\u200d", "").replace("\ufeff", "")
    t = re.sub(r"^\s*(?:&middot;|·|\u00b7)?\s*(?:&nbsp;|\u00a0)?\s*", "", t)
    t = re.sub(r"\s+", " ", t)
    return t.strip()


def fetch_list(session, page):
    if page == 1:
        url = LIST_URL_P1
    else:
        url = LIST_URL_PN.format(page=page)
    r = session.get(url, headers=HEADERS, timeout=30, verify=False)
    if ENC == "gbk":
        try:
            return r.content.decode("gbk")
        except Exception:
            return r.content.decode("gb18030", errors="replace")
    r.encoding = "utf-8"
    return r.text


def parse_list(html_text):
    items = []
    for m in re.finditer(LIST_LINK_RE, html_text, re.S | re.I):
        href = m.group(1)
        title = clean_title(m.group(2))
        date = m.group(3) if m.lastindex and m.lastindex >= 3 and m.group(3) else ""
        if not title or len(title) < 4:
            continue
        abs_url = urllib.parse.urljoin(LIST_URL_P1, href)
        if abs_url.startswith("//"):
            abs_url = "http:" + abs_url
        items.append((abs_url, title, date))
    return items


def parse_detail(html_text, page_url):
    title = ""
    date = ""
    content_html = ""
    if BeautifulSoup:
        soup = BeautifulSoup(html_text, "html.parser")
        if DTITLE_MODE.startswith("meta:"):
            name = DTITLE_MODE.split(":", 1)[1]
            mt = soup.find("meta", attrs={"name": re.compile(name, re.I)})
            if mt and mt.get("content"):
                title = clean_title(mt["content"])
        elif DTITLE_MODE == "title":
            t = soup.find("title")
            if t:
                raw = t.get_text(strip=True)
                for sep in ["_", "-", "—", "|"]:
                    if sep in raw:
                        raw = raw.split(sep)[0]
                        break
                title = clean_title(raw)
        elif DTITLE_MODE == "selector" and DTITLE_SEL:
            el = soup.select_one(DTITLE_SEL)
            if el:
                title = clean_title(el.get_text(" ", strip=True))
        elif DTITLE_MODE == "firstline":
            if DCONTENT_SEL:
                if DCONTENT_SEL.startswith("table:"):
                    el = soup.find("table", attrs={"ergodic": DCONTENT_SEL.split(":", 1)[1]})
                else:
                    el = soup.select_one(DCONTENT_SEL)
                if el:
                    txt = el.get_text(" ", strip=True)
                    lines = [l.strip() for l in txt.split("\n") if len(l.strip()) > 10]
                    if lines:
                        title = clean_title(lines[0])
        if DCONTENT_SEL:
            if DCONTENT_SEL.startswith("table:"):
                el = soup.find("table", attrs={"ergodic": DCONTENT_SEL.split(":", 1)[1]})
                if el:
                    trs = el.find_all("tr")
                    if len(trs) > 1:
                        body_rows = []
                        for tr in trs[1:]:
                            txt = tr.get_text(" ", strip=True)
                            if re.search(r"发布(?:时间|日期)|信息来源|字号|分享到", txt) and not re.search(r"[，。；]", txt[-3:]):
                                continue
                            body_rows.append(str(tr))
                        content_html = "".join(body_rows) if body_rows else "".join(str(tr) for tr in trs[1:])
                    else:
                        content_html = "".join(str(c) for c in el.contents)
            else:
                el = soup.select_one(DCONTENT_SEL)
                if el:
                    content_html = "".join(str(c) for c in el.contents)
        md = soup.find("meta", attrs={"name": re.compile(r"PubDate|publishdate", re.I)})
        if md and md.get("content"):
            dm = re.match(r"(\d{4})-(\d{2})-(\d{2})", md["content"].strip())
            if dm:
                date = dm.group(0)
        if not date:
            dm = re.search(r"(\d{4})-(\d{2})-(\d{2})", html_text)
            if dm:
                date = dm.group(0)
    return title, date, content_html


def html_to_text(content, page_url):
    if not content:
        return "", 0, []
    content = re.sub(r"<!--.*?-->", "", content, flags=re.S)
    content = re.sub(r"<meta[^>]*>", "", content, flags=re.I)
    content = re.sub(r"<script[^>]*>.*?</script>", "", content, flags=re.S | re.I)
    content = re.sub(r"<style[^>]*>.*?</style>", "", content, flags=re.S | re.I)

    attachments = []
    link_protect = {}

    # 清理常见装饰性元素
    content = re.sub(
        r'<img[^>]*src="[^"]*fileTypeImages/icon_[a-z]+\.gif"[^>]*/?>', "", content,
        flags=re.I,
    )
    content = re.sub(r'<img[^>]*src="[^"]*/images/20200907-wz-banner\.png"[^>]*/?>', "", content, flags=re.I)
    # 字号切换按钮
    content = re.sub(r'【字号：[^】]*】', "", content)
    content = re.sub(r'发布(?:时间|日期)[:：][^<]{0,30}', "", content)

    if BeautifulSoup:
        try:
            _soup = BeautifulSoup(content, "html.parser")
            for a in _soup.find_all("a", href=True):
                if a.get("appendix") or a.get("data-appendix") or re.search(r"\.(pdf|docx?|xlsx?|zip|rar)", a.get("href", ""), re.I):
                    href = a["href"]
                    txt = a.get_text(strip=True) or a.get("_title") or os.path.basename(href.split("?")[0])
                    abs_url = urllib.parse.urljoin(page_url, href)
                    attachments.append((abs_url, txt))
                    key = f"__ATTACH__{len(link_protect)}__"
                    link_protect[key] = f'<p><a href="{abs_url}" target="_blank">{txt}</a></p>'
                    a.replace_with(key)
            for p in _soup.find_all("p"):
                imgs = p.find_all("img")
                if imgs and not p.get_text(strip=True):
                    srcs = []
                    for img in imgs:
                        src = img.get("src") or img.get("oldsrc") or ""
                        if src:
                            abs_url = urllib.parse.urljoin(page_url, src)
                            srcs.append(f'<a href="{abs_url}" target="_blank">{os.path.basename(src.split("?")[0]) or "图片"}</a>')
                    if srcs:
                        key = f"__IMG__{len(link_protect)}__"
                        link_protect[key] = "<p>" + "<br/>".join(srcs) + "</p>"
                        p.replace_with(key)
            content = str(_soup)
        except Exception:
            pass

    def _link_repl(m):
        href = m.group(1)
        inner = m.group(2)
        txt = re.sub(r"<[^>]+>", "", inner)
        txt = html_lib.unescape(txt).strip()
        txt = re.sub(r"^附件[:：]\s*", "", txt).strip()
        if not txt:
            txt = os.path.basename(href.split("?")[0]) or "附件"
        abs_url = urllib.parse.urljoin(page_url, href)
        attachments.append((abs_url, txt))
        key = f"__LINK__{len(link_protect)}__"
        link_protect[key] = f'<a href="{abs_url}" target="_blank">{txt}</a>'
        return key
    content = re.sub(r'<a\s[^>]*href="([^"]+)"[^>]*>(.*?)</a>', _link_repl, content, flags=re.S | re.I)

    if BeautifulSoup:
        try:
            _soup = BeautifulSoup(content, "html.parser")
            for tag in _soup.find_all(True):
                for attr in ("style", "class", "lang", "dir", "align", "valign", "width", "height", "border", "cellpadding", "cellspacing"):
                    tag.attrs.pop(attr, None)
            content = str(_soup)
        except Exception:
            pass

    table_protect = []
    def _tbl_repl(m):
        table_protect.append(m.group(0))
        return f"__TBL__{len(table_protect)-1}__"
    content = re.sub(r"<table[^>]*>.*?</table>", _tbl_repl, content, flags=re.S | re.I)
    has_table = 1 if re.search(r"<table[^>]*>", content, re.I) else 0

    content = re.sub(r"</p>", "</p>\n\n", content, flags=re.I)
    content = re.sub(r"<br\s*/?>", "\n", content, flags=re.I)
    content = re.sub(r"</?(?:span|font|o:p|st1?:[a-z]+)\b[^>]*>", "", content, flags=re.I)

    parts = []
    for block in content.split("\n\n"):
        b = block.strip()
        if not b:
            continue
        plain = re.sub(r"<[^>]+>", "", b)
        plain = html_lib.unescape(plain)
        plain = plain.replace("\xa0", " ").replace("&nbsp;", " ")
        plain = re.sub(r"[\s\u200b\u200c\u200d\ufeff]+", "", plain)
        if not plain:
            continue
        for k, v in link_protect.items():
            b = b.replace(k, v)
        for i, tbl in enumerate(table_protect):
            b = b.replace(f"__TBL__{i}__", tbl)
        parts.append(b.strip())

    out = "\n\n".join(parts)
    for k, v in link_protect.items():
        out = out.replace(k, v)
    for i, tbl in enumerate(table_protect):
        out = out.replace(f"__TBL__{i}__", tbl)

    out = re.sub(r"__LINK__\d+__", "", out)
    out = re.sub(r"__TBL__\d+__", "", out)
    out = re.sub(r"__IMG__\d+__", "", out)
    out = re.sub(r"__ATTACH__\d+__", "", out)

    out = html_lib.unescape(out)
    out = re.sub(r"[ \t]+\n", "\n", out)
    out = re.sub(r"\n{3,}", "\n\n", out)
    out = re.sub(r"[ \t]{2,}", " ", out)

    seen = set()
    final = []
    for p in out.split("\n\n"):
        key = re.sub(r"\s+", "", re.sub(r"<[^>]+>", "", p))
        if not key:
            continue
        if key in seen:
            continue
        seen.add(key)
        final.append(p)
    out = "\n\n".join(final).strip()

    return out, has_table, attachments


def store_record(conn, url, title, content, date, has_table):
    cur = conn.execute("SELECT id FROM gov_raw WHERE page_url=? AND site_name=?", (url, SITE_NAME))
    if cur.fetchone():
        return False
    try:
        industry = "other"
        try:
            from crawler_lib import classify_industry
            industry = classify_industry(title)
        except Exception:
            pass
        cur = conn.execute(
            "INSERT OR IGNORE INTO gov_raw "
            "(page_url, source_url, title, content, publish_date, site_name, group_name, script_name, has_table, industry) "
            "VALUES (?,?,?,?,?,?,?,?,?,?)",
            (url, url, title, content, date, SITE_NAME, GROUP_NAME, SCRIPT_NAME, has_table, industry),
        )
        if cur.rowcount > 0:
            summary = re.sub(r"<[^>]+>", "", content)
            summary = html_lib.unescape(summary)
            summary = re.sub(r"\s+", " ", summary).strip()[:200]
            conn.execute(
                "INSERT OR IGNORE INTO gov_search (rowid, title, site_name, summary) VALUES (?,?,?,?)",
                (cur.lastrowid, title, SITE_NAME, summary),
            )
            return True
        return False
    except sqlite3.OperationalError as e:
        if "locked" in str(e):
            time.sleep(3)
            return store_record(conn, url, title, content, date, has_table)
        raise


def fetch_detail(session, url):
    r = session.get(url, headers=HEADERS, timeout=30, verify=False)
    if r.status_code == 200:
        if ENC == "gbk":
            try:
                return r.content.decode("gbk")
            except Exception:
                return r.content.decode("gb18030", errors="replace")
        r.encoding = "utf-8"
        return r.text
    raise RuntimeError(f"HTTP {r.status_code}")


def main():
    import requests
    s = requests.Session()
    s.headers.update(HEADERS)
    conn = None
    if not JSONL_PATH:
        conn = sqlite3.connect(DB_PATH, timeout=60)
        conn.execute("PRAGMA busy_timeout=60000")
        conn.execute("PRAGMA journal_mode=WAL")

    new_count = 0
    skip_count = 0
    seen_urls = set()
    jsonl_rows = []

    for page in range(1, _PAGES + 1):
        try:
            html_text = fetch_list(s, page)
        except Exception as e:
            print(f"Page {page} fetch error: {e}", flush=True)
            break
        items = parse_list(html_text)
        print(f"Page {page}: found {len(items)} items", flush=True)
        if not items:
            print("Empty page, stop pagination", flush=True)
            break
        for abs_url, title, date in items:
            if abs_url in seen_urls:
                continue
            seen_urls.add(abs_url)
            try:
                dhtml = fetch_detail(s, abs_url)
            except Exception as e:
                print(f"  detail error {abs_url}: {e}", flush=True)
                skip_count += 1
                continue
            dtitle, ddate, content_html = parse_detail(dhtml, abs_url)
            if not dtitle:
                dtitle = title
            if not ddate:
                ddate = date
            content, has_table, atts = html_to_text(content_html, abs_url)
            plain_len = len(re.sub(r"<[^>]+>", "", content).strip())
            if plain_len < 10 and not content_html:
                print(f"  skip empty: {dtitle[:30]}", flush=True)
                skip_count += 1
                continue
            if JSONL_PATH:
                jsonl_rows.append({
                    "page_url": abs_url, "title": dtitle, "content": content,
                    "publish_date": ddate, "has_table": has_table,
                    "site_name": SITE_NAME, "group_name": GROUP_NAME,
                    "script_name": SCRIPT_NAME,
                })
                print(f"  JSONL + {dtitle[:40]} ({ddate})", flush=True)
                new_count += 1
                continue
            ok = store_record(conn, abs_url, dtitle, content, ddate, has_table)
            if ok:
                new_count += 1
                print(f"  + {dtitle[:40]} ({ddate})", flush=True)
            else:
                skip_count += 1
            time.sleep(0.25)
        time.sleep(0.5)

    if JSONL_PATH:
        with open(JSONL_PATH, "w", encoding="utf-8") as f:
            for row in jsonl_rows:
                f.write(json.dumps(row, ensure_ascii=False) + "\n")
        print(f"JSONL written: {JSONL_PATH} ({len(jsonl_rows)} rows)", flush=True)

    if conn is not None:
        conn.commit()
        conn.close()
    print(f"新增: {new_count}", flush=True)
    print(f"跳过: {skip_count}", flush=True)
    print(f"=== {SITE_NAME} done ===", flush=True)


if __name__ == "__main__":
    main()