#!/usr/bin/env python3
"""马尔康市人民政府 — 通知公告
http://www.maerkang.gov.cn/maerkang/c100053/nav_list.shtml
UCAP CMS (数融平台), nav_list_{N}.shtml 分页, 20条/页
"""
import os, sys, re, time
import requests
from bs4 import BeautifulSoup
from datetime import datetime, timedelta
from urllib.parse import urljoin

DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
BASE_URL = "http://www.maerkang.gov.cn"
LIST_PATH = "/maerkang/c100053/nav_list.shtml"
SITE_NAME = "马尔康市人民政府-通知公告"
THRESHOLD = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
MAX_PAGES = 50

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}
session = requests.Session()
session.headers.update(HEADERS)
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
import sqlite3


def store_item(title, date_str, content, page_url):
    try:
        conn = sqlite3.connect(DB_PATH, timeout=10)
        c = conn.cursor()
        summary = (content.strip()[:200] if content.strip() else '')
        date_rank = 0
        if date_str:
            try:
                date_rank = int(datetime.strptime(date_str[:10], "%Y-%m-%d").timestamp())
            except:
                pass
        c.execute("""INSERT OR IGNORE INTO gov_raw (title, publish_date, content, page_url, source_url, site_name, summary, date_rank)
                     VALUES (?,?,?,?,?,?,?,?)""",
                  (title.strip(), date_str, content.strip(), page_url.strip(), page_url.strip(), SITE_NAME, summary, date_rank))
        a = c.rowcount
        conn.commit()
        conn.close()
        return a
    except Exception as e:
        print(f"[ERROR] 写入失败: {e}")
        return 0


def get_page_url(page):
    if page == 1:
        return BASE_URL + LIST_PATH
    return f"{BASE_URL}/maerkang/c100053/nav_list_{page}.shtml"


def parse_list_page(html):
    """解析列表页，返回 [(title, date_str, detail_url), ...]"""
    items = []
    soup = BeautifulSoup(html, "html.parser")
    lc = soup.find("div", class_="nav_list_list_container")
    if not lc:
        lc = soup.find("div", class_="nav_list_list")
    if not lc:
        return items

    for a in lc.find_all("a", href=True):
        href = a["href"]
        title = a.get_text(strip=True)

        if not title or len(title) < 5:
            continue
        if "nav_list" in href or "jump.shtml" in href:
            continue

        # Extract date from title text (date appears at end or embedded)
        date_str = ""
        dm = re.search(r"(\d{4}-\d{2}-\d{2})", title)
        if dm:
            date_str = dm.group(1)

        full_url = href if href.startswith("http") else urljoin(BASE_URL, href)
        items.append((title, date_str, full_url))

    return items


def parse_detail_page(html, page_url):
    """解析详情页，返回 (title, date_str, content)"""
    soup = BeautifulSoup(html, "html.parser")

    # Title from <h1>
    title = ""
    h1 = soup.find("h1")
    if h1:
        title = h1.get_text(strip=True)
    if not title:
        tt = soup.find("title")
        if tt:
            title = tt.get_text(strip=True).replace("_马尔康市人民政府", "").replace("_通知公告", "")

    # Date from detail div: "发布时间：2026-07-07"
    date_str = ""
    detail_div = soup.find("div", class_="detail")
    if detail_div:
        dm = re.search(r"(\d{4}-\d{2}-\d{2})", detail_div.get_text())
        if dm:
            date_str = dm.group(1)
    if not date_str:
        for m in re.finditer(r"(\d{4}-\d{2}-\d{2})", html[:3000]):
            date_str = m.group(1)
            break

    # Content from common_detail
    content = ""
    cd = soup.find("div", class_="common_detail")
    if cd:
        parts = []

        # Collect inner p/text elements - skip outer <p> that wraps other <p>
        for el in cd.find_all(["p", "table", "img"]):
            if el.name == "p":
                # Skip outer wrapper <p> that contains child <p> elements
                if el.find("p") or el.find("table"):
                    continue
                txt = el.get_text(strip=True)
                if txt:
                    parts.append(txt)
            elif el.name == "table":
                rows = []
                for tr in el.find_all("tr"):
                    cells = [td.get_text(strip=True) for td in tr.find_all(["td", "th"])]
                    if any(cells):
                        rows.append(" | ".join(cells))
                if rows:
                    parts.append("\n".join(rows))
            elif el.name == "img":
                src = el.get("src", "")
                alt = el.get("alt", "")
                if src:
                    full_src = urljoin(page_url, src)
                    alt_text = f" [{alt}]" if alt else ""
                    parts.append(f"![{alt}]({full_src})")

        content = "\n\n".join(parts)

    # Attachments
    atts = []
    for a in soup.find_all("a", href=True):
        h = a["href"]
        if re.search(r"\.(pdf|doc|docx|xls|xlsx|zip|rar|caj)$", h, re.I):
            t = a.get_text(strip=True) or "附件"
            fu = h if h.startswith("http") else urljoin(BASE_URL, h)
            atts.append(f"[{t}]({fu})")
    if atts:
        if content:
            content += "\n\n---\n**附件：**\n" + "\n".join(atts)
        else:
            content = "\n".join(atts)

    return title, date_str, content


def main():
    print(f"[INFO] 开始爬取: {SITE_NAME}")
    print(f"[INFO] 阈值: {THRESHOLD}")
    total_new = 0
    total_skip = 0
    total_fail = 0
    stopped_early = False

    for page in range(1, MAX_PAGES + 1):
        if stopped_early:
            break
        page_url = get_page_url(page)
        print(f"\n=== 第{page}页: {page_url}")
        if page > 1:
            time.sleep(1.5)

        try:
            r = session.get(page_url, timeout=20, verify=False)
            r.encoding = "utf-8"
        except Exception as e:
            print(f"[ERROR] 获取列表页失败: {e}")
            total_fail += 1
            continue

        if r.status_code != 200:
            print(f"[INFO] HTTP {r.status_code}, 停止")
            break

        items = parse_list_page(r.text)
        if not items:
            print(f"[INFO] 第{page}页无数据，停止")
            break

        print(f"  找到 {len(items)} 条")

        for title, date_str, detail_url in items:
            # Check threshold
            if date_str and date_str < THRESHOLD:
                print(f"[STOP] {date_str} {title[:40]} 超阈值，停止翻页")
                stopped_early = True
                break

            time.sleep(0.3)
            try:
                rd = session.get(detail_url, timeout=20, verify=False)
                rd.encoding = "utf-8"
            except Exception as e:
                print(f"  [FAIL] {title[:30]}... 详情页获取失败: {e}")
                total_fail += 1
                continue

            if rd.status_code != 200:
                print(f"  [FAIL] {title[:30]}... HTTP {rd.status_code}")
                total_fail += 1
                continue

            dt, dd, dc = parse_detail_page(rd.text, detail_url)

            ft = dt if dt else title
            fd = dd if dd else date_str
            if not dc.strip():
                print(f"  [SKIP] {title[:30]}... 正文为空")
                total_skip += 1
                continue

            affected = store_item(ft, fd, dc, detail_url)
            if affected > 0:
                total_new += 1
                print(f"  [+] {ft[:40]} ({fd})")
            else:
                print(f"  [-] {ft[:30]}... 已存在")

        # If all items on this page are old, stop
        if stopped_early:
            break

    print(f"\n=== 完成 ===")
    print(f"新增: {total_new}, 跳过: {total_skip}, 失败: {total_fail}")


if __name__ == "__main__":
    main()