#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
大英县人民政府 - 生态环境 (daying.gov.cn)
列表: https://www.daying.gov.cn/gongkai/kuozhan/10002.html?page=N
      栏目「生态环境」，共 30 页，20 条/页，日期倒序
      条目: <li><a href="/gongkai/show/{md5}.html" target="_blank" title="完整标题">标题</a><span>2026-09-20</span></li>
详情: /gongkai/show/{md5}.html
      <h1> 标题 / meta[name=PubDate] content="2026-09-20 17:46"
      正文 div.xqing-web-box
      附件 div.xqing-web-file > div > a[href=/UploadFiles/...]
      正文里另有纯文本「附件：1.xxx」清单（不重复采）

⚠️ 本脚本补齐「10002 生态环境」栏目。该栏目此前只由旧版脚本抓过，
   41 条历史数据（2025-12-16 ~ 2026-06-09）停在旧 site_name
   「大英县人民政府-生态环境」下 —— 这里沿用同名，历史与新增连续。
   注意 crawl_daying.py 抓的是 10043「通知公告」，crawl_daying_gk.py 抓的是
   10053「重大项目实施进展」，三者栏目不同、page_url 交集为 0。

用法:
  python3 crawl_daying_sthj.py --pages=1
  python3 crawl_daying_sthj.py --pages=5
"""
import os
import re
import sys
import time
import json
from datetime import datetime, timezone, timedelta
from urllib.parse import urljoin

import requests
import urllib3
from bs4 import BeautifulSoup

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

SITE_NAME = "大英县人民政府-生态环境"
BASE_URL = "https://www.daying.gov.cn"
LIST_URL = "https://www.daying.gov.cn/gongkai/kuozhan/10002.html"
CUTOFF = (datetime.now(timezone.utc) - timedelta(days=365 * 3)).strftime("%Y-%m-%d")
MAX_PAGES = 5
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
                  "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
}


def fetch(url):
    try:
        r = requests.get(url, headers=HEADERS, timeout=25, verify=False)
        r.encoding = "utf-8"
        return r.text
    except Exception as e:
        print("  [ERR] %s" % str(e)[:80])
        return None


def clean(s):
    return re.sub(r"\s+", " ", (s or "")).strip()


def parse_list(html):
    """ul.content-list > li ；日期 <span> 在 <a> 之后"""
    soup = BeautifulSoup(html, "html.parser")
    items = []
    for li in soup.select("ul.content-list > li"):
        a = li.find("a", href=True)
        if not a:
            continue
        href = a["href"].strip()
        if "/gongkai/show/" not in href:
            continue
        title = clean(a.get("title") or a.get_text(" ", strip=True))
        if not title or len(title) < 5:
            continue
        sp = li.find("span")
        date = clean(sp.get_text(" ", strip=True)) if sp else ""
        m = re.search(r"(\d{4}-\d{1,2}-\d{1,2})", date)
        date = m.group(1) if m else ""
        items.append({"title": title, "url": urljoin(BASE_URL, href), "date": date})
    return items


def extract_body(soup):
    """div.xqing-web-box → 段落化的 <p> 序列（勿 get_text 拍平）"""
    box = soup.find("div", class_="xqing-web-box")
    if not box:
        return ""
    for t in box.find_all(["script", "style"]):
        t.decompose()
    parts = []
    for el in box.find_all(["p", "h1", "h2", "h3", "h4", "table", "ul", "ol", "img"], recursive=True):
        if el.name == "img":
            src = el.get("src") or el.get("data-src") or ""
            if src:
                parts.append('<p><img src="%s"/></p>' % urljoin(BASE_URL, src))
            continue
        if el.name in ("ul", "ol"):
            if el.find_parent(["ul", "ol"]):
                continue
            for li in el.find_all("li", recursive=False):
                txt = clean(li.get_text(" ", strip=True))
                if txt:
                    parts.append("<p>%s</p>" % txt)
            continue
        if el.find_parent(["table", "ul", "ol"]):
            continue
        if el.name == "table":
            parts.append(re.sub(r"\s+", " ", str(el)))
            continue
        txt = clean(el.get_text(" ", strip=True))
        if txt:
            parts.append("<p>%s</p>" % txt)
    if not parts:
        txt = clean(box.get_text(" ", strip=True))
        if txt:
            parts = ["<p>%s</p>" % txt]
    out = "\n".join(parts)
    out = re.sub(r"<p>\s*</p>", "", out)
    return out.strip()


def extract_atts(soup, page_url):
    """div.xqing-web-file 附件区（在正文容器之外）"""
    atts = []
    seen = set()
    for cont in soup.select("div.xqing-web-file, div.article-file, div.fjxz"):
        for a in cont.find_all("a", href=True):
            u = a["href"].strip()
            if not u or u.startswith(("javascript:", "#", "mailto:")):
                continue
            u = urljoin(page_url, u)
            if u in seen:
                continue
            seen.add(u)
            atts.append({"name": clean(a.get_text(" ", strip=True))
                         or os.path.basename(u.split("?")[0])[:120], "url": u})
    for m in re.finditer(r'href="([^"]+\.(?:pdf|docx?|xlsx?|zip|rar|wps|7z|txt))"', str(soup), re.I):
        u = urljoin(page_url, m.group(1).strip())
        if u in seen:
            continue
        seen.add(u)
        atts.append({"name": os.path.basename(u.split("?")[0])[:120], "url": u})
    return atts


def fetch_detail(url, list_title, list_date):
    html = fetch(url)
    if not html:
        return list_title, "", list_date, []
    soup = BeautifulSoup(html, "html.parser")

    title = list_title
    h1 = soup.find("h1")
    if h1:
        t = clean(h1.get_text(" ", strip=True))
        if t:
            title = t

    date = list_date
    mp = soup.find("meta", attrs={"name": "PubDate"})
    if mp and mp.get("content"):
        m = re.search(r"(\d{4})-(\d{1,2})-(\d{1,2})", mp["content"])
        if m:
            date = "%04d-%02d-%02d" % (int(m.group(1)), int(m.group(2)), int(m.group(3)))
    if not date:
        m = re.search(r"发布时间[：:]\s*(\d{4})-(\d{1,2})-(\d{1,2})", html)
        if m:
            date = "%04d-%02d-%02d" % (int(m.group(1)), int(m.group(2)), int(m.group(3)))

    content = extract_body(soup)
    atts = extract_atts(soup, url)
    # 附件区转 <p><a href="完整地址">附件名</a></p>，多附件独立成段
    for at in atts:
        content += '\n<p><a href="%s">%s</a></p>' % (at["url"], at["name"])
    return title, content, date, atts


def main():
    pages = MAX_PAGES
    for a in sys.argv[1:]:
        m = re.match(r"--pages=(\d+)$", a)
        if m:
            pages = int(m.group(1))

    print("\n%s\n🏠 %s\n列表: %s\n截断: %s (近3年)\n页数: %d\n%s"
          % ("=" * 60, SITE_NAME, LIST_URL, CUTOFF, pages, "=" * 60))

    all_list, seen = [], set()
    for pg in range(1, pages + 1):
        url = LIST_URL if pg == 1 else "%s?page=%d" % (LIST_URL, pg)
        html = fetch(url)
        if not html:
            print("  第%d页 ❌ 获取失败" % pg)
            break
        items = parse_list(html)
        n_new = 0
        for it in items:
            if it["url"] in seen:
                continue
            seen.add(it["url"])
            all_list.append(it)
            n_new += 1
        print("  第%d页 %d 条 (新 %d)" % (pg, len(items), n_new))
        if not items:
            break
        time.sleep(0.3)

    print("\n列表合计 %d 条" % len(all_list))
    n_cut = 0
    kept = []
    for it in all_list:
        if it["date"] and it["date"] < CUTOFF:
            n_cut += 1
            continue
        kept.append(it)
    print("CUTOFF截断 %d 条 → 待抓 %d 条" % (n_cut, len(kept)))

    out = []
    for i, it in enumerate(kept):
        title, content, date, atts = fetch_detail(it["url"], it["title"], it["date"])
        if not date:
            date = it["date"]
        if not content or len(content) < 50:
            print("  [%d/%d] ⚠️ 正文过短(%d) %s" % (i + 1, len(kept), len(content or ""), title[:34]))
            continue
        summary = clean(re.sub(r"<[^>]+>", " ", content))[:300]
        out.append({
            "site_name": SITE_NAME,
            "title": title,
            "url": it["url"],
            "content": content,
            "pub_date": date,
            "summary": summary,
            "tags": SITE_NAME,
            "attachments": json.dumps(atts, ensure_ascii=False) if atts else "[]",
            "group_name": "四川",
        })
        print("  [%d/%d] ✅ %s | %s | 附件%d" % (i + 1, len(kept), date, title[:32], len(atts)))
        time.sleep(0.3)

    if out:
        push_to_searchdb(out, "daying_sthj")
    empty = sum(1 for x in out if not x["content"].strip())
    print("\n✅ 完成: %d 条 (正文空 %d / CUTOFF截断 %d)" % (len(out), empty, n_cut))


if __name__ == "__main__":
    t0 = time.time()
    main()
    print("⏱ 耗时: %.1fs" % (time.time() - t0))