#!/usr/bin/env python3
"""张掖经济技术开发区-通知公告 (www.zhangye.gov.cn)
列表: index.html -> index_N.html (6页, 0-indexed via curl)
详情: div.view.TRS_UEDITOR
"""
import sys, os, re, json, subprocess, time
from datetime import datetime, timedelta
from bs4 import BeautifulSoup

BASE = "https://www.zhangye.gov.cn"
LIST_PATH = "/jjkfq/gzdt/tzgg"
SITE = "张掖经济技术开发区"
COLUMN = "通知公告"
PROVINCE = "甘肃"
TOTAL_PAGES = 6

CURL = ["curl", "-s", "-k",
    "-H", "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
    "-H", "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "-H", "Accept-Language: zh-CN,zh;q=0.9,en;q=0.8"]
MARKER = "\x00P\x00"

def log(msg):
    sys.stderr.write(msg + "\n")
    sys.stderr.flush()

def http_get(url):
    try:
        r = subprocess.run(CURL + [url], capture_output=True, text=True, timeout=30)
        if r.returncode == 0 and len(r.stdout) > 500:
            return r.stdout
        return None
    except Exception as e:
        log(f"  [ERROR] {str(e)[:60]}")
        return None

def fetch_list(page):
    if page == 1:
        url = f"{BASE}{LIST_PATH}/index.html"
    else:
        url = f"{BASE}{LIST_PATH}/index_{page-1}.html"
    return http_get(url)

def parse_list(html):
    soup = BeautifulSoup(html, "html.parser")
    items = []
    for li in soup.select("ul.news_list li"):
        a = li.find("a", href=True)
        if not a: continue
        span = a.find("span")
        date = span.get_text(strip=True) if span else ""
        if span:
            span.extract()
        title = a.get_text(strip=True)
        href = a["href"]
        if href.startswith("./"):
            href = BASE + LIST_PATH + "/" + href[2:]
        elif href.startswith("/"):
            href = BASE + href
        if title:
            items.append({"title": title.strip(), "url": href, "date": date})
    return items

def extract_content(detail_url, title):
    html = http_get(detail_url)
    if not html:
        return f"[{title}]({detail_url})", title, ""
    soup = BeautifulSoup(html, "html.parser")
    meta_title = soup.select_one("meta[name=ArticleTitle]")
    meta_date = soup.select_one("meta[name=PubDate]")
    if meta_title and meta_title.get("content"):
        title = meta_title["content"].strip()
    pub_date = ""
    if meta_date and meta_date.get("content"):
        pub_date = meta_date["content"].strip()[:10]
    content_el = soup.select_one("div.view.TRS_UEDITOR") or soup.select_one("div.view")
    if not content_el:
        return f"[{title}]({detail_url})", title, pub_date
    tables_html = []
    for table in content_el.find_all("table"):
        tables_html.append(str(table))
        table.decompose()
    for tag in content_el.find_all(["p", "h1", "h2", "h3", "h4", "h5", "h6"]):
        tag.insert(0, MARKER)
        tag.append(MARKER)
    for br in content_el.find_all("br"):
        br.replace_with(MARKER)
    for tag in content_el.find_all(["span", "b", "strong", "font", "em", "i", "u", "s"]):
        tag.unwrap()
    text = content_el.get_text(separator="", strip=True)
    text = re.sub(r"\x00P\x00(\s*\x00P\x00)+", "\x00P\x00", text)
    text = text.replace("\x00P\x00", "\n\n")
    text = re.sub(r"\n{3,}", "\n\n", text)
    text = text.strip()
    for tbl_html in tables_html:
        text += f"\n\n{tbl_html}"
    if not text.strip() or len(text.strip()) < 20:
        text = f"[{title}]({detail_url})"
    return text, title, pub_date

def crawl_all(months_back=36):
    cutoff = datetime.now() - timedelta(days=months_back * 30) if months_back else None
    all_items = []
    seen_urls = set()
    for page in range(1, TOTAL_PAGES + 1):
        html = fetch_list(page)
        if not html: break
        items = parse_list(html)
        if not items: break
        new_count = 0
        for item in items:
            if item["url"] not in seen_urls:
                if cutoff and item["date"]:
                    try:
                        d = datetime.strptime(item["date"], "%Y-%m-%d")
                        if d < cutoff: continue
                    except: pass
                seen_urls.add(item["url"])
                all_items.append(item)
                new_count += 1
        log(f"  [PAGE {page}/{TOTAL_PAGES}] {len(items)} items, +{new_count} new")
        if new_count == 0 and page > 1: break
        time.sleep(2)
    log(f"\n共 {len(all_items)} 条待爬详情")
    for idx, item in enumerate(all_items, 1):
        content, title, pub_date = extract_content(item["url"], item["title"])
        if pub_date: item["date"] = pub_date
        if title: item["title"] = title
        item["content"] = content
        output_item(item, idx)
        if idx % 10 == 0: log(f"  [PROGRESS] {idx}/{len(all_items)}")
    log(f"\n[DONE] 共爬取 {len(all_items)} 条")

def crawl_incremental():
    html = fetch_list(1)
    if not html: return
    items = parse_list(html)
    log(f"[LIST] page 1 -> {len(items)} 条")
    for idx, item in enumerate(items, 1):
        content, title, pub_date = extract_content(item["url"], item["title"])
        if pub_date: item["date"] = pub_date
        if title: item["title"] = title
        item["content"] = content
        output_item(item, idx)

def output_item(item, idx):
    print(json.dumps({
        "title": item.get("title", ""),
        "page_url": item.get("url", ""),
        "publish_date": item.get("date", ""),
        "content": item.get("content", ""),
        "site_name": f"{SITE}-{COLUMN}",
        "column": COLUMN,
        "province": PROVINCE,
    }, ensure_ascii=False))

if __name__ == "__main__":
    mode = sys.argv[1] if len(sys.argv) > 1 else "incremental"
    if mode == "incremental": crawl_incremental()
    elif mode == "full":
        months = int(sys.argv[2]) if len(sys.argv) > 2 else 36
        crawl_all(months)
    else:
        log(f"Usage: {sys.argv[0]} [incremental|full [months]]")