#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
彭泽县生态环境局 - 重大建设项目 (www.pengze.gov.cn/sthjj/fdzdgknr_171512/01/)
CMS: TRS WCM (meta ArticleTitle/PubDate/ContentSource, trs_editor_view 正文)
列表: div.box_list > ul.pd20.clearfix > li > a.xxgklist[title] + span 日期 (15条/页)
分页: createPageHTML(11, 0, "index", "html") -> index.html / index_{N}.html (11页)
详情: div#mart.xl-text > div.trs_editor_view (TRS_UEDITOR Word转换, span逐字包裹)
      标题 meta ArticleTitle 完整, 附件 appendix 内嵌
"""
import sys
import os
import re
import time
import html as html_lib
import urllib.parse
import sqlite3

try:
    from bs4 import BeautifulSoup
except ImportError:
    BeautifulSoup = None

# ---------------- config ----------------
BASE_URL = "https://www.pengze.gov.cn"
LIST_PATH = "/sthjj/fdzdgknr_171512/01/"
LIST_URL = BASE_URL + LIST_PATH
SITE_NAME = "彭泽县重大建设项目"
GROUP_NAME = "江西"
SCRIPT_NAME = "crawl_pengze_zdxm.py"
DB_PATH = "/mnt/data/search.db"

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9",
}

# default: daily incremental = 1 page
_MAX_PAGES = 1
for i, a in enumerate(sys.argv):
    if a.startswith("--pages="):
        try:
            _MAX_PAGES = int(a.split("=", 1)[1])
        except ValueError:
            pass
    elif a == "--pages" and i + 1 < len(sys.argv):
        try:
            _MAX_PAGES = int(sys.argv[i + 1])
        except ValueError:
            pass

_PAGES = _MAX_PAGES if _MAX_PAGES >= 1 else 1


# ---------------- helpers ----------------
def clean_title(t):
    """Strip &middot;&nbsp; entity prefixes + unescape all entities + strip zero-width."""
    if not t:
        return ""
    t = html_lib.unescape(t)
    t = t.replace("\u200b", "").replace("\u200c", "").replace("\u200d", "").replace("\ufeff", "")
    t = re.sub(r"\s+", " ", t)
    return t.strip()


def fetch(url, session=None, timeout=30, retries=3):
    import requests
    last = None
    for i in range(retries):
        try:
            r = (session or requests).get(url, headers=HEADERS, timeout=timeout, verify=False)
            if r.status_code == 200:
                return r
            last = r
        except Exception as e:
            last = e
        time.sleep(1.5 * (i + 1))
    if isinstance(last, Exception):
        raise last
    return last


def html_to_text(content, page_url):
    """Convert detail-content inner HTML to stored format:
    - keep <table> HTML intact
    - \n\n between <p> blocks
    - attachment <a> links embedded with absolute URL
    - strip <script>/<style>/noise
    """
    if not content:
        return "", 0, []
    content = re.sub(r"^\s*<div\s+class=[\"']?(?:detail-content|TRS_Editor|trs_editor_view|xl-text|lhx_news)[\"']?[^>]*>", "", content, flags=re.I)
    content = re.sub(r"</div>\s*$", "", content)
    content = re.sub(r"</p>\s*$", "", content)
    content = re.sub(r"<!--.*?-->", "", content, flags=re.S)
    content = re.sub(r"<script[^>]*>.*?</script>", "", content, flags=re.S | re.I)
    content = re.sub(r"<style[^>]*>.*?</style>", "", content, flags=re.S | re.I)

    table_protect = []
    def _tbl_repl(m):
        table_protect.append(m.group(0))
        return f"__TBL__{len(table_protect)-1}__"
    content2 = re.sub(r"<table[^>]*>.*?</table>", _tbl_repl, content, flags=re.S | re.I)
    has_table = 1 if re.search(r"<table[^>]*>", content, re.I) else 0

    attachments = []
    link_protect = {}
    def _link_repl(m):
        href = m.group(1)
        inner = m.group(2)
        txt = re.sub(r"<[^>]+>", "", inner)
        txt = html_lib.unescape(txt).strip()
        txt = re.sub(r"^附件[:：]\s*", "", txt).strip()
        if not txt:
            txt = os.path.basename(href.split("?")[0]) or "附件"
        abs_url = urllib.parse.urljoin(page_url, href)
        attachments.append((abs_url, txt))
        key = f"__LINK__{len(link_protect)}__"
        link_protect[key] = f'<a href="{abs_url}" target="_blank">{txt}</a>'
        return key
    content2 = re.sub(r'<a\s[^>]*href="([^"]+)"[^>]*>(.*?)</a>', _link_repl, content2, flags=re.S | re.I)

    content2 = re.sub(r"</p>", "</p>\n\n", content2, flags=re.I)
    content2 = re.sub(r"<br\s*/?>", "\n", content2, flags=re.I)
    # strip Word-conversion inline tags (span/font) but keep their text
    content2 = re.sub(r"</?(?:span|font|o:p|st1?:[a-z]+)\b[^>]*>", "", content2, flags=re.I)

    parts = []
    for block in content2.split("\n\n"):
        b = block.strip()
        if not b:
            continue
        plain = re.sub(r"<[^>]+>", "", b)
        plain = html_lib.unescape(plain)
        plain = plain.replace("\xa0", " ").replace("&nbsp;", " ")
        plain = re.sub(r"[\s\u200b\u200c\u200d\ufeff]+", "", plain)
        if not plain:
            continue
        for k, v in link_protect.items():
            b = b.replace(k, v)
        for i, tbl in enumerate(table_protect):
            b = b.replace(f"__TBL__{i}__", tbl)
        parts.append(b.strip())

    out = "\n\n".join(parts)
    for k, v in link_protect.items():
        out = out.replace(k, v)
    for i, tbl in enumerate(table_protect):
        out = out.replace(f"__TBL__{i}__", tbl)

    out = re.sub(r"__LINK__\d+__", "", out)
    out = re.sub(r"__TBL__\d+__", "", out)

    out = html_lib.unescape(out)
    out = re.sub(r"[ \t]+\n", "\n", out)
    out = re.sub(r"\n{3,}", "\n\n", out)
    out = re.sub(r"[ \t]{2,}", " ", out)

    # dedupe paragraphs
    seen = set()
    final = []
    for p in out.split("\n\n"):
        key = re.sub(r"\s+", "", re.sub(r"<[^>]+>", "", p))
        if not key:
            continue
        if key in seen:
            continue
        seen.add(key)
        final.append(p)
    out = "\n\n".join(final).strip()

    # if no attachment links captured but content had some, append (normalize &amp; vs &)
    if attachments:
        appended = []
        for abs_url, txt in attachments:
            if abs_url.replace("&", "&amp;") not in out and abs_url not in out:
                appended.append(f'<p><a href="{abs_url}" target="_blank">{txt}</a></p>')
        if appended:
            out = (out + "\n\n" + "\n\n".join(appended)).strip()

    return out, has_table, attachments


def parse_list(html_text, page_url):
    """Extract (url, title, date) from list page."""
    items = []
    if BeautifulSoup:
        soup = BeautifulSoup(html_text, "html.parser")
        box = soup.find("div", class_="box_list")
        if box:
            for a in box.find_all("a", class_="xxgklist", href=True):
                href = a.get("href", "").strip()
                abs_url = urllib.parse.urljoin(LIST_URL, href)
                title = a.get("title") or a.get_text(" ", strip=True)
                title = clean_title(title)
                date = ""
                li = a.find_parent("li")
                if li:
                    span = li.find("span")
                    if span:
                        date = span.get_text(strip=True)
                if not re.match(r"^20\d{2}-\d{2}-\d{2}$", date):
                    date = ""
                items.append((abs_url, title, date))
    if not items:
        # regex fallback
        for m in re.finditer(r'<a\s+class=\'xxgklist\'[^>]*href="([^"]+)"[^>]*title="([^"]*)"[^>]*>(.*?)</a>\s*<span>([^<]*)</span>', html_text, re.S):
            abs_url = urllib.parse.urljoin(LIST_URL, m.group(1))
            items.append((abs_url, clean_title(m.group(2) or m.group(3)), m.group(4).strip()))
    return items


def parse_detail(html_text, page_url):
    """Extract title, date, content from detail page."""
    title = ""
    date = ""
    content_html = ""
    if BeautifulSoup:
        soup = BeautifulSoup(html_text, "html.parser")
        m = soup.find("meta", attrs={"name": re.compile(r"ArticleTitle", re.I)})
        if m and m.get("content"):
            title = clean_title(m["content"])
        m = soup.find("meta", attrs={"name": re.compile(r"PubDate", re.I)})
        if m and m.get("content"):
            date = m["content"].strip()[:10]
        mart = soup.find("div", id="mart")
        if mart:
            te = mart.find("div", class_=re.compile(r"trs_editor_view", re.I))
            if te:
                content_html = str(te)
    if not content_html:
        m = re.search(r'name="ArticleTitle"\s+content="([^"]*)"', html_text)
        if m:
            title = clean_title(m.group(1))
        m = re.search(r'name="PubDate"\s+content="([^"]*)"', html_text)
        if m:
            date = m.group(1).strip()[:10]
        idx = html_text.find('id="mart"')
        if idx != -1:
            te = re.search(r'<div[^>]*class="trs_editor_view[^"]*"[^>]*>', html_text[idx:])
            if te:
                start = idx + te.start()
                seg = html_text[start:]
                depth = 0
                end = -1
                for mm in re.finditer(r"<div\b[^>]*>|</div>", seg):
                    if mm.group(0).startswith("</div>"):
                        depth -= 1
                        if depth == 0:
                            end = mm.end()
                            break
                    else:
                        depth += 1
                if end > 0:
                    content_html = seg[:end]
    return title, date, content_html


def store_record(conn, url, title, content, date, has_table):
    cur = conn.execute(
        "SELECT id FROM gov_raw WHERE page_url=?",
        (url,),
    )
    if cur.fetchone():
        return False
    try:
        cur = conn.execute(
            "INSERT OR IGNORE INTO gov_raw "
            "(page_url, source_url, title, content, publish_date, site_name, group_name, script_name, has_table, industry) "
            "VALUES (?,?,?,?,?,?,?,?,?,?)",
            (url, url, title, content, date, SITE_NAME, GROUP_NAME, SCRIPT_NAME, has_table, "other"),
        )
        if cur.rowcount > 0:
            summary = re.sub(r"<[^>]+>", "", content)
            summary = html_lib.unescape(summary)
            summary = re.sub(r"\s+", " ", summary).strip()[:200]
            conn.execute(
                "INSERT OR IGNORE INTO gov_search (rowid, title, site_name, summary) VALUES (?,?,?,?)",
                (cur.lastrowid, title, SITE_NAME, summary),
            )
            return True
        return False
    except sqlite3.OperationalError as e:
        if "locked" in str(e):
            time.sleep(3)
            return store_record(conn, url, title, content, date, has_table)
        raise


def main():
    import requests
    s = requests.Session()
    s.headers.update(HEADERS)
    conn = sqlite3.connect(DB_PATH, timeout=60)
    conn.execute("PRAGMA busy_timeout=60000")
    conn.execute("PRAGMA journal_mode=WAL")

    new_count = 0
    skip_count = 0
    seen_urls = set()

    for page in range(1, _PAGES + 1):
        if page == 1:
            url = LIST_URL
        else:
            url = f"{LIST_URL}index_{page-1}.html"
        try:
            r = fetch(url, session=s)
            html_text = r.content.decode("utf-8", errors="replace")
        except Exception as e:
            print(f"Page {page} fetch error: {e}", flush=True)
            break
        items = parse_list(html_text, url)
        print(f"Page {page}: found {len(items)} items", flush=True)
        if not items and page > 1:
            print("Empty page, stop pagination", flush=True)
            break
        for abs_url, title, date in items:
            if abs_url in seen_urls:
                continue
            seen_urls.add(abs_url)
            try:
                r = fetch(abs_url, session=s)
                dhtml = r.content.decode("utf-8", errors="replace")
            except Exception as e:
                print(f"  detail error {abs_url}: {e}", flush=True)
                skip_count += 1
                continue
            dtitle, ddate, content_html = parse_detail(dhtml, abs_url)
            if not dtitle:
                dtitle = title
            if not ddate:
                ddate = date
            content, has_table, _ = html_to_text(content_html, abs_url)
            plain_len = len(re.sub(r"<[^>]+>", "", content).strip())
            if plain_len < 10:
                print(f"  skip empty: {dtitle[:30]}", flush=True)
                skip_count += 1
                continue
            ok = store_record(conn, abs_url, dtitle, content, ddate, has_table)
            if ok:
                new_count += 1
                print(f"  + {dtitle[:40]} ({ddate})", flush=True)
            else:
                skip_count += 1
            time.sleep(0.3)
        time.sleep(0.5)

    conn.commit()
    conn.close()
    print(f"新增: {new_count}", flush=True)
    print(f"跳过: {skip_count}", flush=True)
    print(f"=== {SITE_NAME} done ===", flush=True)


if __name__ == "__main__":
    main()