#!/usr/bin/env python3
"""邵武市人民政府 — 部门（公示公告）
https://www.shaowu.gov.cn/cms/html/swsrmzf/bm/index.html
YLCMS 炎黄网络CMS，分页 sitemanage/index.shtml
"""
import os, sys, re, time

import sys as _SYS
_MAX_PG = int(_SYS.argv[1]) if len(_SYS.argv) > 1 and _SYS.argv[1].isdigit() else None
if _MAX_PG is not None:
    print('[AutoPg] max_pages=' + str(_MAX_PG))
# END AUTO PAGES
import requests
from bs4 import BeautifulSoup
from datetime import datetime, timedelta
from urllib.parse import urljoin

DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
BASE_URL = "https://www.shaowu.gov.cn/cms/html/swsrmzf/bm/index.html"
SITE_NAME = "邵武市人民政府-部门公告"
THRESHOLD = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}
session = requests.Session()
session.headers.update(HEADERS)
session.verify = False
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
import sqlite3

def _drop_container_parts(parts):
    """剔除「容器段」：find_all(['p','div']) 会同时收下容器 <div> 与它内部的 <p>，
    导致同一内容重复（容器那份常还被 get_text(strip=True) 拍平）。
    判据：某段被列表内另一段完全包含 → 它是容器段 → 剔除。
    保护：剔除后为空则原样返回。
    """
    if not parts:
        return parts
    ps = [p for p in parts if isinstance(p, str)]
    if len(ps) != len(parts):
        return parts
    # ① 先按值去重 —— 否则两段完全相同时 a in b 与 b in a 双向成立，
    #    会把两段都判成「容器段」剔光，再触发下面的保护而原样返回（等于没修）。
    seen, uniq = set(), []
    for p in parts:
        if p not in seen:
            seen.add(p); uniq.append(p)
    # ② 再剔容器段（被其它段完全包含的那个）
    keep = [a for a in uniq
            if not (len(a) >= 40 and any(b is not a and b and b in a for b in uniq))]
    return keep if keep else uniq


def store_item(title, date_str, content, page_url):
    try:
        conn = sqlite3.connect(DB_PATH, timeout=10)
        c = conn.cursor()
        summary = (content.strip()[:200] if content.strip() else '')
        date_rank = 0
        if date_str:
            try:
                date_rank = int(datetime.strptime(date_str[:10], "%Y-%m-%d").timestamp())
            except:
                pass
        c.execute("""INSERT OR IGNORE INTO gov_raw (title, publish_date, content, page_url, source_url, site_name, summary, date_rank)
                     VALUES (?,?,?,?,?,?,?,?)""",
                  (title.strip(), date_str, content.strip(), page_url.strip(), page_url.strip(), SITE_NAME, summary, date_rank))
        a = c.rowcount; conn.commit(); conn.close(); return a
    except Exception as e:
        print(f"[ERROR] 写入失败: {e}"); return 0

def fetch(url, max_retries=3):
    for i in range(max_retries):
        try:
            r = session.get(url, timeout=15)
            r.encoding = "utf-8"
            if r.status_code == 200:
                return r.text
            print(f"[WARN] {url} 返回 {r.status_code}")
        except Exception as e:
            print(f"[WARN] {url} 请求失败: {e}")
        if i < max_retries - 1:
            time.sleep(2)
    return None

def parse_list_page(html, base_url):
    soup = BeautifulSoup(html, "html.parser")
    items = []
    div = soup.find("div", class_="mid-mj-list")
    if not div:
        print("[WARN] 未找到 mid-mj-list")
        return items
    ul = div.find("ul")
    if not ul:
        return items
    for li in ul.find_all("li"):
        a = li.find("a", href=True)
        spans = li.find_all("span")
        if not a or len(spans) < 2:
            continue
        title = a.get_text(strip=True)
        if not title:
            title = spans[0].get_text(strip=True)
        date_str = spans[-1].get_text(strip=True)
        href = a["href"]
        if not href:
            continue
        # Normalize date
        date_str = re.sub(r'\s+', ' ', date_str).strip()
        items.append((title, date_str, urljoin(base_url, href)))
    return items

def get_page_url(pg):
    if pg == 1:
        return BASE_URL
    return f"https://www.shaowu.gov.cn/cms/sitemanage/index.shtml?siteId=280409444942130000&page={pg}"

def parse_detail_page(url):
    html = fetch(url)
    if not html:
        return None, None, None
    soup = BeautifulSoup(html, "html.parser")
    
    # 标题
    title = ""
    t_div = soup.find("div", class_="content-title")
    if t_div:
        title = t_div.get_text(strip=True)
    if not title:
        t = soup.find("title")
        if t:
            title = t.get_text(strip=True)
            for suffix in ["-部门", "—部门", "_部门", "-邵武市人民政府", "_邵武市人民政府"]:
                if suffix in title:
                    title = title.split(suffix)[0].strip()
                    break
    
    # 日期
    date_str = ""
    page_text = soup.get_text()
    m = re.search(r'发布时间[：:]\s*(\d{4}[-年]\d{1,2}[-月]\d{1,2})', page_text)
    if m:
        date_str = m.group(1).replace('年', '-').replace('月', '-').replace('日', '')
    
    # 正文
    content_parts = []
    content_div = soup.find("div", id="Content")
    if content_div:
        for p in content_div.find_all(["p", "div", "table"]):
            txt = p.get_text(strip=True)
            if txt and len(txt) > 3:
                if "分享到" in txt or "微信" in txt or "微博" in txt:
                    continue
                content_parts.append(txt)
    else:
        # Fallback: use mid-content
        mc = soup.find("div", class_="mid-content")
        if mc:
            for p in mc.find_all(["p", "div"]):
                txt = p.get_text(strip=True)
                if txt and len(txt) > 5:
                    if "当前位置" in txt or "分享到" in txt or "微信" in txt:
                        continue
                    content_parts.append(txt)
    
    content_parts = _drop_container_parts(content_parts)
    content = "\n\n".join(content_parts)
    
    # 附件
    attachments = []
    for a in soup.find_all("a", href=True):
        h = a["href"].lower()
        if re.search(r'\.(pdf|doc|docx|xls|xlsx|zip|rar|ppt|pptx)$', h):
            fname = a.get_text(strip=True) or os.path.basename(a["href"])
            full_url = urljoin(url, a["href"])
            attachments.append(f"[{fname}]({full_url})")
    if attachments:
        if content:
            content += "\n\n---\n**附件：**\n" + "\n".join(attachments)
        else:
            content = "\n".join(attachments)
    
    return title, date_str, content

def main():
    print(f"[START] 邵武市人民政府-部门公告 阈值: {THRESHOLD}")
    total_new = 0
    total_skip = 0
    total_err = 0
    
    # Crawl all 52 pages
    for pg in range(1, (_MAX_PG or 52) + 1):   # QC20260925 修 off-by-one: args=1 时 range(1,1) 为空导致循环不执行
        page_url = get_page_url(pg)
        print(f"\n[PAGE] 第{pg}页: {page_url}")
        html = fetch(page_url)
        if not html:
            print(f"  [ERR] 第{pg}页获取失败")
            continue
        
        items = parse_list_page(html, page_url)
        if not items:
            print(f"  [WARN] 第{pg}页无条目")
            continue
        
        for title, date_str, detail_url in items:
            try:
                if date_str < THRESHOLD:
                    print(f"  [SKIP] {date_str} {title[:40]}")
                    total_skip += 1
                    continue
            except:
                pass
            
            t, d, c = parse_detail_page(detail_url)
            if t and c:
                st = store_item(t, d or date_str, c, detail_url)
                if st:
                    total_new += 1
                    print(f"  [OK] {d or date_str} {t[:40]}")
                else:
                    total_err += 1
                    print(f"  [DUP] {d or date_str} {t[:40]}")
            else:
                total_err += 1
                print(f"  [ERR] {d or date_str} {title[:40]} 详情页解析失败 {detail_url}")
            time.sleep(0.3)
        time.sleep(0.5)
    
    print(f"\n[DONE] 新增: {total_new}, 跳过: {total_skip}, 失败: {total_err}")

if __name__ == "__main__":
    main()