#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
茂县人民政府 — 通知公告 (mxrmzf/c100053)
https://maoxian.gov.cn/mxrmzf/c100053/nav_list.shtml
UCAP CMS (数融平台, 同阿坝州站群 maerkang 模板), nav_list_{N}.shtml 分页, 20条/页
实际只有 50 页 = 1000 条 (2024-03-04 ~ 2026-08-07, 全在 3 年窗口内, 无需 CUTOFF 截断)
列表: <div class="nav_list_list_container"><ul><li><a href="/mxrmzf/c100053/YYYYMM/32hex.shtml" title="标题"><p>标题</p><span>YYYY-MM-DD</span></a></li>
详情: meta ArticleTitle/PubDate; 正文 <div class="common_detail"> (嵌套p去重, 保留<p>段落HTML); 附件 files/*.pdf
"""
import os, re, sys, time
import requests
from bs4 import BeautifulSoup
from datetime import datetime, timedelta
from urllib.parse import urljoin

DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
BASE_URL = "https://maoxian.gov.cn"
LIST_PATH = "/mxrmzf/c100053/nav_list.shtml"
SITE_NAME = "茂县人民政府-通知公告"
THRESHOLD = (datetime.now() - timedelta(days=3 * 365)).strftime("%Y-%m-%d")
MAX_PAGES = 50

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}
session = requests.Session()
session.headers.update(HEADERS)
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
import sqlite3


def store_item(title, date_str, content, page_url):
    try:
        conn = sqlite3.connect(DB_PATH, timeout=10)
        c = conn.cursor()
        summary = (content.strip()[:200] if content.strip() else '')
        date_rank = 0
        if date_str:
            try:
                date_rank = int(datetime.strptime(date_str[:10], "%Y-%m-%d").timestamp())
            except Exception:
                pass
        c.execute("""INSERT OR IGNORE INTO gov_raw (title, publish_date, content, page_url, source_url, site_name, summary, date_rank)
                     VALUES (?,?,?,?,?,?,?,?)""",
                  (title.strip(), date_str, content.strip(), page_url.strip(), page_url.strip(), SITE_NAME, summary, date_rank))
        a = c.rowcount
        conn.commit()
        conn.close()
        return a
    except Exception as e:
        print(f"[ERROR] 写入失败: {e}")
        return 0


def get_page_url(page):
    if page == 1:
        return BASE_URL + LIST_PATH
    return f"{BASE_URL}/mxrmzf/c100053/nav_list_{page}.shtml"


def parse_list_page(html):
    """解析列表页，返回 [(title, date_str, detail_url), ...]"""
    items = []
    soup = BeautifulSoup(html, "html.parser")
    lc = soup.find("div", class_="nav_list_list_container")
    if not lc:
        lc = soup.find("div", class_="nav_list_list")
    if not lc:
        return items

    for a in lc.find_all("a", href=True):
        href = a["href"]
        # 标题优先取 title 属性（完整）；回退文本
        title = (a.get("title") or "").strip() or a.get_text(strip=True)
        if not title or len(title) < 5:
            continue
        if "nav_list" in href or "jump.shtml" in href:
            continue

        # 日期从 span 提取
        date_str = ""
        sp = a.find("span")
        if sp:
            dm = re.search(r"(\d{4}-\d{2}-\d{2})", sp.get_text())
            if dm:
                date_str = dm.group(1)
        if not date_str:
            dm = re.search(r"(\d{4}-\d{2}-\d{2})", title)
            if dm:
                date_str = dm.group(1)

        full_url = href if href.startswith("http") else urljoin(BASE_URL, href)
        items.append((title, date_str, full_url))

    return items


def parse_detail_page(html, page_url):
    """解析详情页，返回 (title, date_str, content)"""
    soup = BeautifulSoup(html, "html.parser")

    # 标题: meta ArticleTitle → h1 → title
    title = ""
    mt = soup.find("meta", attrs={"name": "ArticleTitle"})
    if mt and mt.get("content"):
        title = mt["content"].strip()
    if not title:
        h1 = soup.find("h1")
        if h1:
            title = h1.get_text(strip=True)
    if not title:
        tt = soup.find("title")
        if tt:
            title = tt.get_text(strip=True).replace("_茂县人民政府", "").replace("_通知公告", "")

    # 日期: meta PubDate → 页面内首个日期
    date_str = ""
    mp = soup.find("meta", attrs={"name": "PubDate"})
    if mp and mp.get("content"):
        dm = re.search(r"(\d{4}-\d{2}-\d{2})", mp["content"])
        if dm:
            date_str = dm.group(1)
    if not date_str:
        detail_div = soup.find("div", class_="detail")
        if detail_div:
            dm = re.search(r"(\d{4}-\d{2}-\d{2})", detail_div.get_text())
            if dm:
                date_str = dm.group(1)
    if not date_str:
        for m in re.finditer(r"(\d{4}-\d{2}-\d{2})", html[:3000]):
            date_str = m.group(1)
            break

    # 正文: common_detail (保留 <p> 段落 HTML + 表格)
    content = ""
    cd = soup.find("div", class_="common_detail")
    if cd:
        parts = []
        for el in cd.find_all(["p", "table", "img"]):
            if el.name == "p":
                if el.find("p") or el.find("table"):
                    continue  # 跳过外层包裹 p
                # 保留段落内附件链接
                html_p = str(el)
                txt = re.sub(r"<[^>]+>", "", html_p)
                txt = re.sub(r"[ \t\u3000]+", " ", txt)
                txt = re.sub(r"\s+", " ", txt).strip()
                if txt:
                    parts.append(f"<p>{txt}</p>")
            elif el.name == "table":
                rows = []
                for tr in el.find_all("tr"):
                    cells = [td.get_text(strip=True) for td in tr.find_all(["td", "th"])]
                    if any(cells):
                        rows.append(" | ".join(cells))
                if rows:
                    parts.append("<table>\n" + "\n".join(f"<tr><td>{r}</td></tr>" for r in rows) + "\n</table>")
            elif el.name == "img":
                src = el.get("src", "")
                if src:
                    full_src = urljoin(page_url, src)
                    parts.append(f'<p><img src="{full_src}"></p>')
        content = "\n".join(parts)

    # 附件 (files/*.pdf 等, 绝对化 — 用 page_url 作基址! href 是相对详情页的路径)
    atts = []
    for a in soup.find_all("a", href=True):
        h = a["href"]
        if re.search(r"\.(pdf|doc|docx|xls|xlsx|zip|rar|caj|wps|et)$", h, re.I):
            t = a.get_text(strip=True) or os.path.basename(h.split("?")[0])
            fu = h if h.startswith("http") else urljoin(page_url, h)
            atts.append(f'<p><a href="{fu}">{t}</a></p>')
    if atts:
        if content:
            content += "\n" + "\n".join(atts)
        else:
            content = "\n".join(atts)

    return title, date_str, content


def main():
    import argparse
    ap = argparse.ArgumentParser()
    ap.add_argument("--pages", type=int, default=MAX_PAGES)
    ap.add_argument("--start", type=int, default=1)
    args = ap.parse_args()
    print(f"[INFO] 开始爬取: {SITE_NAME}")
    print(f"[INFO] 阈值: {THRESHOLD}")
    total_new = 0
    total_skip = 0
    total_fail = 0
    stopped_early = False

    for page in range(args.start, min(args.start + args.pages, MAX_PAGES + 1)):
        if stopped_early:
            break
        page_url = get_page_url(page)
        print(f"\n=== 第{page}页: {page_url}", flush=True)
        if page > 1:
            time.sleep(1.2)

        try:
            r = session.get(page_url, timeout=20, verify=False)
            r.encoding = "utf-8"
        except Exception as e:
            print(f"[ERROR] 获取列表页失败: {e}")
            total_fail += 1
            continue

        if r.status_code != 200:
            print(f"[INFO] HTTP {r.status_code}, 停止")
            break

        items = parse_list_page(r.text)
        if not items:
            print(f"[INFO] 第{page}页无数据，停止")
            break

        print(f"  找到 {len(items)} 条", flush=True)
        page_new = 0
        for title, date_str, detail_url in items:
            if date_str and date_str < THRESHOLD:
                print(f"  [跳过窗口外] {date_str} {title[:30]}")
                continue
            try:
                r2 = session.get(detail_url, timeout=20, verify=False)
                r2.encoding = "utf-8"
            except Exception as e:
                print(f"[ERROR] 详情页失败: {e}")
                total_fail += 1
                continue
            if r2.status_code != 200:
                print(f"[WARN] 详情页 HTTP {r2.status_code}: {detail_url}")
                total_fail += 1
                continue
            d_title, d_date, content = parse_detail_page(r2.text, detail_url)
            if not d_title:
                d_title = title
            if not d_date:
                d_date = date_str
            n = store_item(d_title, d_date, content, detail_url)
            page_new += n
            if n:
                total_new += 1
            else:
                total_skip += 1
            time.sleep(0.5)
        print(f"  第{page}页: 新增 {page_new}", flush=True)

    print(f"\n[DONE] 新增 {total_new}, 跳过 {total_skip}, 失败 {total_fail}")


if __name__ == "__main__":
    main()