#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""垦利区人民政府 - 通知公告 (col39604) 爬虫
CMS: Hanweb 大汉版通 (JPage dataproxy 动态分页)
列表: /col/col39604/index.html  首屏内嵌 <datastore> 45条(3组×15条/页)
     翻页 POST /module/web/jpage/dataproxy.jsp?startrecord=N&endrecord=M&perpage=15
     totalRecord=290, perPage=15 -> 共20页 (290/15=19.3)
详情: /art/YYYY/M/D/art_39604_{id}.html
     meta ArticleTitle / pubdate + div#zoom (ContentStart~ContentEnd) 正文
附件: 正文内 <a href="/module/download/downfile.jsp?..."> (相对路径, urljoin 详情页URL)
"""
import sys, re, time, sqlite3, ssl, urllib.request, urllib.parse
from urllib.parse import urljoin
from bs4 import BeautifulSoup

SITE_NAME = "垦利区人民政府-通知公告"
SCRIPT_NAME = "crawl_kenli_tzgg.py"
GROUP_NAME = "山东"
BASE_URL = "http://www.kenli.gov.cn"
LIST_URL = BASE_URL + "/col/col39604/index.html"
PROXY_URL = "/module/web/jpage/dataproxy.jsp"
COLUMNID = "39604"
UNITID = "127350"
WEBID = "274"
WEBNAME = "垦利区人民政府"
DB_PATH = "/mnt/data/search.db"
PER_PAGE = 15
GROUP_RECORDS = 45  # PER_PAGE * groupSize(3)

ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}


def http_get(url, timeout=30, retries=3):
    for i in range(retries):
        try:
            req = urllib.request.Request(url, headers=HEADERS)
            resp = urllib.request.urlopen(req, timeout=timeout, context=ctx)
            body = resp.read()
            if len(body) < 300 and b"404" in body[:100]:
                return ""
            return body.decode("utf-8", errors="replace")
        except Exception as e:
            if i == retries - 1:
                print(f"  [WARN] 请求失败 {url}: {e}", file=sys.stderr)
                return ""
            time.sleep(1.2)
    return ""


def http_post(url, data, timeout=30, retries=3):
    for i in range(retries):
        try:
            body = urllib.parse.urlencode(data).encode("utf-8")
            req = urllib.request.Request(url, data=body, headers={
                **HEADERS, "Referer": LIST_URL,
                "Content-Type": "application/x-www-form-urlencoded"})
            resp = urllib.request.urlopen(req, timeout=timeout, context=ctx)
            return resp.read().decode("utf-8", errors="replace")
        except Exception as e:
            if i == retries - 1:
                print(f"  [WARN] POST失败 {url}: {e}", file=sys.stderr)
                return ""
            time.sleep(1.2)
    return ""


def clean_title(title):
    """strip &middot;&nbsp; 实体前缀 和省略号截断后缀"""
    title = title.replace("&middot;", "").replace("&nbsp;", "")
    title = title.replace("\u200b", "").replace("\ufeff", "")  # 零宽空格/BOM
    title = re.sub(r"^[\s\xa0·\u00b7]+", "", title)
    title = re.sub(r"\s*\.{3,}\s*$", "", title)
    return title.strip()


def parse_datastore_records(html):
    """从 <datastore><record><![CDATA[<li>...]]></record> 提取 (url, title, date)"""
    items = []
    for m in re.finditer(r"<record><!\[CDATA\[(.*?)\]\]></record>", html, re.S):
        cdata = m.group(1)
        a = re.search(r'<a[^>]*href="([^"]+)"[^>]*title="([^"]*)"', cdata)
        if not a:
            a = re.search(r"<a[^>]*href='([^']+)'[^>]*title='([^']*)'", cdata)
        if not a:
            continue
        href = a.group(1)
        title = clean_title(a.group(2))
        url = urljoin(BASE_URL, href)
        dm = re.search(r"<span>(\d{4}-\d{2}-\d{2})</span>", cdata)
        date = dm.group(1) if dm else ""
        if title and "art_" in url:
            items.append((url, title, date))
    return items


_init_records = None


def fetch_page(page):
    """第page页 (1-based) 列表条目。1-3页在首屏 datastore, 4+页 POST dataproxy 按组取。"""
    global _init_records
    if page <= 3:
        if _init_records is None:
            html = http_get(LIST_URL)
            if not html:
                return []
            _init_records = parse_datastore_records(html)
        recs = _init_records
        return recs[(page - 1) * PER_PAGE: page * PER_PAGE]
    # 4+ 页: 每3页一组, 组 g 覆盖记录 [(g-1)*45+1, g*45]
    group = (page - 1) // 3 + 1
    start = (group - 1) * GROUP_RECORDS + 1
    end = start + GROUP_RECORDS - 1
    data = {
        "col": 1, "webid": WEBID, "path": BASE_URL + "/",
        "columnid": COLUMNID, "sourceContentType": 1, "unitid": UNITID,
        "webname": WEBNAME, "permissiontype": 0,
    }
    xml = http_post(BASE_URL + PROXY_URL + f"?startrecord={start}&endrecord={end}&perpage={PER_PAGE}", data)
    if not xml:
        return []
    recs = parse_datastore_records(xml)
    idx = (page - 1) % 3  # 0,1,2 页内偏移
    return recs[idx * PER_PAGE: (idx + 1) * PER_PAGE]


def extract_detail(html, page_url):
    """提取 (title, date, content_html, attachments)"""
    # 标题
    title = ""
    mt = re.search(r'<meta\s+name="ArticleTitle"\s+content="([^"]+)"', html)
    if mt:
        title = clean_title(mt.group(1))
    # 日期
    date = ""
    mp = re.search(r'<meta\s+name="pubdate"\s+content="(\d{4}-\d{2}-\d{2})', html)
    if mp:
        date = mp.group(1)

    soup = BeautifulSoup(html, "html.parser")
    zoom = soup.find("div", id="zoom")
    if not zoom:
        return title, date, "", []

    # 正文区域: 正则切片 ContentStart~ContentEnd 之间原始HTML, 独立解析
    # (BS4 html.parser 会把 <meta name="ContentStart"/> 后的 <p> 嵌套进 meta 内部,
    #  直接遍历 zoom.children 会漏掉正文 —— 必须切片段再解析)
    body_html = ""
    m_start = re.search(r'<meta\s+name="ContentStart"\s*/?>', html)
    m_end = re.search(r'<meta\s+name="ContentEnd"\s*/?>', html)
    if m_start and m_end and m_end.start() > m_start.end():
        body_html = html[m_start.end():m_end.start()]
    frag = BeautifulSoup(body_html, "html.parser") if body_html.strip() else zoom
    attachments = []
    for a in frag.find_all("a", href=True):
        href = a["href"]
        txt = a.get_text(strip=True)
        if (re.search(r"(?i)\.(pdf|doc|docx|xls|xlsx|zip|rar|wps|et|ofd|png|jpg)(\?|$)", href)
                or "download" in href.lower() or "attach" in href.lower() or "file" in href.lower() or txt.startswith("附件")):
            abs_url = urljoin(page_url, href)
            if not abs_url.startswith(("http://", "https://")):
                abs_url = "http:" + abs_url if abs_url.startswith("//") else abs_url
            name = txt if txt else abs_url.split("/")[-1]
            attachments.append((a, name, abs_url))
    for a, name, abs_url in attachments:
        new_a = frag.new_tag("a", href=abs_url, target="_blank")
        new_a.string = name
        if a.parent is not None:
            # 清理附件前文件类型小图标 (img 在 a 之前的同级)
            p = a.parent
            idx_a = None
            for ci, child in enumerate(p.contents):
                if child is a:
                    idx_a = ci
                    break
            if idx_a is not None:
                for child in list(p.contents[:idx_a]):
                    if getattr(child, "name", None) == "img":
                        child.decompose()
        a.replace_with(new_a)
    attached_hrefs = {abs_url for _, _, abs_url in attachments}

    body_els = [el for el in frag.children if getattr(el, "name", None)]

    parts = []
    for el in body_els:
        if el.name == "table":
            parts.append(str(el))
            continue
        if el.name in ("p", "div", "h1", "h2", "h3", "ul", "ol", "li", "blockquote"):
            txt = el.get_text("", strip=True)
            if not txt and not el.find("img"):
                continue
            inner_tables = el.find_all("table")
            el_has_attach = any(
                urljoin(page_url, x["href"]) in attached_hrefs or x["href"] in attached_hrefs
                for x in el.find_all("a", href=True))
            if inner_tables or el_has_attach or el.find("img"):
                parts.append(str(el))
                continue
            parts.append(txt)
        else:
            txt = el.get_text("", strip=True)
            if txt:
                parts.append(txt)

    # 段落去重 (key 先剥标签)
    seen = set()
    final_parts = []
    for p in parts:
        key = re.sub(r"\s+", "", re.sub(r"<[^>]+>", "", p))
        if key and key not in seen:
            seen.add(key)
            final_parts.append(p)

    content = "\n\n".join(final_parts)
    content_unescaped = content.replace("&amp;", "&")
    for _, name, abs_url in attachments:
        if abs_url not in content and abs_url not in content_unescaped:
            content += f'\n\n<p><a href="{abs_url}" target="_blank">{name}</a></p>'

    content = re.sub(r"打印本页[^\n]*", "", content)
    content = re.sub(r"\n{3,}", "\n\n", content).strip()
    return title, date, content, attachments


def main():
    max_pages = 1
    args = sys.argv[1:]
    i = 0
    while i < len(args):
        a = args[i]
        if a.startswith("--pages="):
            max_pages = int(a.split("=", 1)[1])
        elif a == "--pages" and i + 1 < len(args):
            max_pages = int(args[i + 1])
            i += 1
        i += 1

    conn = sqlite3.connect(DB_PATH, timeout=60)
    conn.execute("PRAGMA busy_timeout=60000")
    conn.execute("PRAGMA journal_mode=WAL")
    c = conn.cursor()
    total_new = 0
    total_dup = 0
    total_skip = 0

    fail_streak = 0
    for page in range(1, max_pages + 1):
        items = fetch_page(page)
        if not items:
            fail_streak += 1
            print(f"  [WARN] 第{page}页无数据 (fail_streak={fail_streak})", file=sys.stderr)
            if fail_streak >= 2 and page > 1:
                break
            continue
        fail_streak = 0
        print(f"  第{page}页: 找到 {len(items)} 条")
        for url, title, date in items:
            try:
                c.execute("SELECT id FROM gov_raw WHERE page_url=?", (url,))
                if c.fetchone():
                    total_dup += 1
                    continue
            except sqlite3.OperationalError:
                time.sleep(3)
                continue
            dhtml = http_get(url)
            if not dhtml:
                total_skip += 1
                continue
            d_title, d_date, content, attachments = extract_detail(dhtml, url)
            if not content or len(re.sub(r"<[^>]+>", "", content).strip()) < 10:
                total_skip += 1  # 纯图片型/空正文
                continue
            if not d_title:
                d_title = title
            if not d_date:
                d_date = date
            summary = re.sub(r"<[^>]+>", "", content)[:200]
            summary = re.sub(r"\s+", " ", summary).strip()
            has_table = 1 if "<table" in content else 0
            try:
                cur = c.execute(
                    "INSERT INTO gov_raw (title, summary, content, page_url, source_url, publish_date, site_name, script_name, group_name, has_table, date_rank) VALUES (?,?,?,?,?,?,?,?,?,?,?)",
                    (d_title, summary, content, url, url, d_date, SITE_NAME, SCRIPT_NAME, GROUP_NAME, has_table, 0))
                rid = cur.lastrowid
                c.execute("INSERT OR IGNORE INTO gov_search (rowid, title, site_name, summary) VALUES (?,?,?,?)",
                          (rid, d_title, SITE_NAME, summary))
                conn.commit()
                total_new += 1
                print(f"    [{d_date}] {d_title[:45]}")
            except sqlite3.OperationalError as e:
                if "locked" in str(e):
                    conn.rollback()
                    time.sleep(8)
                    try:
                        cur = c.execute(
                            "INSERT INTO gov_raw (title, summary, content, page_url, source_url, publish_date, site_name, script_name, group_name, has_table, date_rank) VALUES (?,?,?,?,?,?,?,?,?,?,?)",
                            (d_title, summary, content, url, url, d_date, SITE_NAME, SCRIPT_NAME, GROUP_NAME, has_table, 0))
                        rid = cur.lastrowid
                        c.execute("INSERT OR IGNORE INTO gov_search (rowid, title, site_name, summary) VALUES (?,?,?,?)",
                                  (rid, d_title, SITE_NAME, summary))
                        conn.commit()
                        total_new += 1
                        print(f"    [{d_date}] {d_title[:45]} (重试成功)")
                    except Exception:
                        conn.rollback()
                        total_skip += 1
                else:
                    total_skip += 1
            except sqlite3.IntegrityError:
                total_dup += 1
            time.sleep(0.4)

    conn.close()
    print(f"\n新增: {total_new}  重复: {total_dup}  过短/404: {total_skip}  总计: {total_new+total_dup+total_skip}")


if __name__ == "__main__":
    main()