#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
crawl_dingyuan_xkgs.py - 定远县-重大建设项目许可公示 (基层政务公开专题)
URL: https://www.dingyuan.gov.cn/site/tpl/162748834?organId=161056888&type=4&action=list&pcatId=&catId=170111903
CMS: 滁州统一信息公开平台 AJAX (site/label/8888 publicInfoList)
列表API: labelName=publicInfoList&organId=161056888&catId=170111903&pageIndex=N (20条/页)
列表: ul.xxgk_navli > li.mc > a[href=/public/...]标题 + li.rq 日期
详情: https://www.dingyuan.gov.cn/public/161056888/{id}.html, div#contentbox 内 content (去元数据表后)
"""
import sys, os, re, time, html as html_lib
from datetime import datetime, timezone, timedelta

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb
import requests, urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
from bs4 import BeautifulSoup

_MAX_PG = 5
for i, a in enumerate(sys.argv):
    if a == '--pages' and i + 1 < len(sys.argv):
        _MAX_PG = int(sys.argv[i + 1])
        break

SITE_NAME = "定远县-项目许可公示"
API_URL = "https://www.dingyuan.gov.cn/chuzhou/site/label/8888"
CUTOFF = (datetime.now(timezone.utc) - timedelta(days=365 * 3)).strftime("%Y-%m-%d")
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0 Safari/537.36",
    "X-Requested-With": "XMLHttpRequest",
    "Referer": "https://www.dingyuan.gov.cn/site/tpl/162748834?organId=161056888&type=4&action=list&pcatId=&catId=170111903",
}
API_PARAMS = {
    "labelName": "publicInfoList", "siteId": "2653861", "pageSize": "20", "pageIndex": "1",
    "action": "list", "isDate": "true", "dateFormat": "yyyy-MM-dd", "length": "80",
    "organId": "161056888", "catId": "170111903", "type": "4",
}

ITEM_RE = re.compile(r'<a[^>]+href="(https://www\.dingyuan\.gov\.cn/public/\d+/\d+\.html)"[^>]*>\s*([^<]{4,140}?)\s*</a>.*?<li class="rq">(\d{4}-\d{2}-\d{2})</li>', re.S)


def fetch_list(page):
    p = dict(API_PARAMS)
    p["pageIndex"] = str(page)
    for _ in range(3):
        try:
            r = requests.get(API_URL, params=p, headers=HEADERS, timeout=25, verify=False)
            r.encoding = "utf-8"
            return r.text
        except Exception:
            time.sleep(2)
    return None


def fetch(url, timeout=25):
    for _ in range(3):
        try:
            r = requests.get(url, headers={"User-Agent": HEADERS["User-Agent"]}, timeout=timeout, verify=False)
            r.encoding = "utf-8"
            return r.text
        except Exception:
            time.sleep(2)
    return None


def parse_list(html):
    items = []
    for m in ITEM_RE.finditer(html):
        url, title, date = m.group(1), html_lib.unescape(m.group(2)).strip(), m.group(3).strip()
        if title and date:
            items.append((title, url, date))
    return items


def extract_detail_body(html):
    """div.xxgk_contnet 内正文: 去掉 xxgk_newtit 头部/二维码/分享, 保留正文表格"""
    try:
        soup = BeautifulSoup(html, "html.parser")
        d = soup.find("div", class_="xxgk_contnet") or soup.find("div", class_="contentbox minh500")
        if not d:
            return ""
        # 删除工具块
        for sel in [".xxgk_newtit", ".wzewm", ".newsinfo", ".clear", "script", "style",
                    "[class*=share]", "[class*=print]", "[class*=ewm]", "[class*=qrcode]", ".xxgk_zclist"]:
            for el in d.select(sel):
                if el:
                    el.decompose()
        # 附件链接独立成段
        for a in d.find_all("a", href=True):
            if re.search(r'\.(pdf|doc|docx|xls|xlsx|zip|rar|wps)(\?|$)', a["href"], re.I):
                a.wrap(soup.new_tag("p"))
        out = str(d)
        out = re.sub(r'<!--.*?-->', '', out, flags=re.S)
        out = re.sub(r'\s{2,}', ' ', out)
        return out.strip()
    except Exception:
        return ""


def parse_detail(url):
    html = fetch(url)
    if not html:
        return None, None
    soup = BeautifulSoup(html, "html.parser")
    title = ""
    mt = soup.find("meta", attrs={"name": re.compile("ArticleTitle", re.I)})
    if mt and mt.get("content"):
        title = html_lib.unescape(mt["content"]).strip()
    if not title:
        t = soup.find("title")
        if t:
            title = re.sub(r'[-_].*$', '', t.get_text(strip=True)).strip()
    date = ""
    pd = soup.find("meta", attrs={"name": re.compile("PubDate", re.I)})
    if pd and pd.get("content"):
        m = re.search(r'(\d{4}-\d{2}-\d{2})', pd["content"])
        if m:
            date = m.group(1)
    if not date:
        m = re.search(r'发布时间[:：]\s*(\d{4}-\d{2}-\d{2})', html)
        if m:
            date = m.group(1)
    content = extract_detail_body(html)
    # 附件: 正文后的附件链接 (<p><a>)
    if content:
        content = re.sub(r'(<a[^>]+href="(?:[^"]*\.(?:pdf|doc|docx|xls|xlsx|zip|rar|wps))"[^>]*>.*?</a>)',
                         lambda mm: f'<p>{mm.group(1)}</p>', content, flags=re.I | re.S)
    return title, content, date


def main():
    print(f"=== {SITE_NAME} ===")
    all_items = []
    for page in range(_MAX_PG):
        html = fetch_list(page + 1)
        if not html:
            print(f"  p{page+1} FAIL"); continue
        items = parse_list(html)
        if not items:
            print(f"  p{page+1} empty, stop"); break
        all_items.extend(items)
        print(f"  p{page+1}: {len(items)}")
        time.sleep(0.5)
    seen, filtered = set(), []
    for title, url, date in all_items:
        if url in seen:
            continue
        seen.add(url)
        if date and date < CUTOFF:
            continue
        filtered.append((title, url, date))
    print(f"  去重/3yr后: {len(filtered)}")
    results = []
    for idx, (title, url, date) in enumerate(filtered):
        dt, content, dd = parse_detail(url)
        ft = dt or title
        fd = date or dd
        if not content or len(content.strip()) < 10:
            print(f"  [{idx+1}] ⚠空 {title[:30]}")
            continue
        summary = re.sub(r'<[^>]+>', '', content)[:200].strip() or ft[:200]
        results.append({"site_name": SITE_NAME, "title": ft, "url": url,
                        "content": content, "summary": summary, "pub_date": fd})
        print(f"  [{idx+1}] ✅ {ft[:35]} ({len(content)}B)")
        time.sleep(0.3)
    if results:
        push_to_searchdb(results, "dingyuan_xkgs")
        print(f"入库 {len(results)}")
    print("完成")


if __name__ == "__main__":
    main()