#!/usr/bin/env python3
"""丰都县生态环境局-环评审批情况 (cqfd.gov.cn)
单页: /bm/sthjj/.../hpspqk/ -> ul.zsj-fr-main li
详情: div.trs_editor_view.TRS_UEDITOR 正文
"""
import sys, os, re, json, time
from datetime import datetime, timedelta
import requests
from bs4 import BeautifulSoup

BASE_URL = "http://cqfd.gov.cn"
LIST_PATH = "/bm/sthjj/zwgk_36090/fdzdgknr_36093/zdxmhjyxpj/hpspqk"
SITE = "丰都县生态环境局"
COLUMN = "环评审批情况"
PROVINCE = "重庆"

HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
session = requests.Session()
session.headers.update(HEADERS)
MARKER = "\x00P\x00"

def log(msg):
    sys.stderr.write(msg + "\n")
    sys.stderr.flush()

def fetch_list():
    url = f"{BASE_URL}{LIST_PATH}/"
    try:
        r = session.get(url, timeout=20)
        r.encoding = "utf-8"
        return r.text
    except Exception as e:
        log(f"  [ERROR] list: {e}")
        return None

def parse_list(html):
    soup = BeautifulSoup(html, "html.parser")
    items = []
    for li in soup.select("ul.zsj-fr-main li"):
        a = li.find("a", href=True)
        if not a:
            continue
        title = a.get_text(strip=True)
        href = a["href"]
        if not href.startswith("http"):
            href = BASE_URL + LIST_PATH + "/" + href.lstrip("./")
        # Date from li text
        li_text = li.get_text(strip=True)
        dm = re.search(r'(\d{4}-\d{2}-\d{2})', li_text)
        date = dm.group(1) if dm else ""
        if title and href:
            items.append({"title": title, "url": href, "date": date})
    return items

def extract_content(detail_url, title):
    try:
        r = session.get(detail_url, timeout=20)
        r.encoding = "utf-8"
    except Exception as e:
        return f"[{title}]({detail_url})", title, ""

    soup = BeautifulSoup(r.text, "html.parser")
    meta_title = soup.select_one("meta[name=ArticleTitle]")
    meta_date = soup.select_one("meta[name=PubDate]")
    if meta_title and meta_title.get("content"):
        title = meta_title["content"].strip()
    pub_date = ""
    if meta_date and meta_date.get("content"):
        pub_date = meta_date["content"].strip()[:10]

    # Try TRS_UEDITOR first, then fallback to zwxl-content
    content_el = soup.select_one("div.trs_editor_view")
    if not content_el:
        content_el = soup.select_one("div.zwxl-content")

    if not content_el:
        return f"[{title}]({detail_url})", title, pub_date

    # Save attachment links
    attach_links = []
    for a_tag in content_el.find_all("a"):
        href = a_tag.get("href", "")
        if href and not href.startswith("#") and not href.startswith("javascript"):
            if href.startswith("/"):
                href = BASE_URL + href
            elif href.startswith("./"):
                href = detail_url.rstrip(detail_url.split("/")[-1]) + href[2:]
            txt = a_tag.get_text(strip=True)
            if txt and href:
                attach_links.append(f"[{txt}]({href})")

    # Save tables
    tables_html = []
    for table in content_el.find_all("table"):
        tables_html.append(str(table))
        table.decompose()

    # Paragraph markers
    for tag in content_el.find_all(["p", "h1", "h2", "h3", "h4", "h5", "h6"]):
        tag.insert(0, MARKER)
        tag.append(MARKER)
    for br in content_el.find_all("br"):
        br.replace_with(MARKER)

    # Unwrap inline
    for tag in content_el.find_all(["span", "b", "strong", "font", "em", "i", "u", "s"]):
        tag.unwrap()

    text = content_el.get_text(separator="", strip=True)
    text = re.sub(r"\x00P\x00(\s*\x00P\x00)+", "\x00P\x00", text)
    text = text.replace("\x00P\x00", "\n\n")
    text = re.sub(r"\n{3,}", "\n\n", text)
    text = text.strip()

    for tbl_html in tables_html:
        text += f"\n\n{tbl_html}"
    for link in attach_links:
        if link not in text:
            text += f"\n\n{link}"

    if not text.strip() or len(text.strip()) < 20:
        text = f"[{title}]({detail_url})"
        if attach_links:
            text += "\n\n" + "\n".join(attach_links)

    return text, title, pub_date

def crawl_all():
    html = fetch_list()
    if not html:
        return
    items = parse_list(html)
    log(f"[LIST] {len(items)} 条")
    for idx, item in enumerate(items, 1):
        content, title, pub_date = extract_content(item["url"], item["title"])
        if pub_date:
            item["date"] = pub_date
        if title:
            item["title"] = title
        item["content"] = content
        output_item(item, idx)
    log(f"[DONE] 共爬取 {len(items)} 条")

def crawl_incremental():
    crawl_all()

def output_item(item, idx):
    record = {
        "title": item.get("title", ""),
        "page_url": item.get("url", ""),
        "publish_date": item.get("date", ""),
        "content": item.get("content", ""),
        "site_name": f"{SITE}-{COLUMN}",
        "column": COLUMN,
        "province": PROVINCE,
    }
    print(json.dumps(record, ensure_ascii=False))

if __name__ == "__main__":
    mode = sys.argv[1] if len(sys.argv) > 1 else "incremental"
    if mode == "incremental" or mode == "full":
        crawl_all()
    elif mode == "list":
        html = fetch_list()
        items = parse_list(html)
        log(f"共 {len(items)} 条")
        for it in items:
            log(f"  {it['date']} | {it['title'][:50]} | {it['url']}")
    else:
        log(f"Usage: {sys.argv[0]} [incremental|full|list]")