#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
中咨华宇 - 环评公示 (chinaeia.cn/channel.aspx?id=124)
CMS: ASP.NET (channel.aspx?id=N&page=N, 详情 content.aspx?id=N)
列表: a[href=content.aspx?id=N]
分页: ?page=2 ...
详情: 正文 div.nr3 (含 title 去后缀)
用法: python3 crawl_chinaeia_hpgongshi.py [--pages=N] [--json out.jsonl]
"""
import re, sys, os, json, time, sqlite3, requests
from datetime import datetime, timedelta
from bs4 import BeautifulSoup
from urllib.parse import urljoin

SITE_NAME = "中咨华宇-环评公示"
BASE_URL = "http://www.chinaeia.cn"
LIST_URL = "http://www.chinaeia.cn/channel.aspx?id=124"
DOMAIN = "www.chinaeia.cn"
GROUP = "北京"
CUTOFF = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
DB_PATH = os.environ.get("DB_PATH", "/root/search.db")
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"}


def log(msg):
    print(f"[{SITE_NAME}] {msg}", flush=True)


def clean_title(t):
    t = re.sub(r" | |\u200b|\ufeff", "", t or "")
    t = re.sub(r"\s+", " ", t)
    t = re.sub(r"\.\.\.|…$", "", t).strip()
    return t


def fetch(url, timeout=30):
    r = requests.get(url, headers=HEADERS, timeout=timeout)
    r.encoding = "utf-8"
    return r.text


def parse_list(html):
    soup = BeautifulSoup(html, "html.parser")
    items = []
    for a in soup.find_all("a", href=True):
        href = a.get("href", "")
        if "content.aspx?id=" not in href:
            continue
        title = clean_title(a.get_text(strip=True))
        if not title:
            continue
        pub_date = ""
        parent = a.find_parent(["li", "div"])
        if parent:
            m = re.search(r"20\d{2}[-/年]\d{1,2}[-/月]\d{1,2}", parent.get_text())
            if m:
                pub_date = m.group(0).replace("年", "-").replace("月", "-").replace("/", "-")[:10]
        items.append({"title": title, "url": urljoin(BASE_URL, href), "pub_date": pub_date})
    seen = set()
    uniq = []
    for it in items:
        if it["url"] not in seen:
            seen.add(it["url"])
            uniq.append(it)
    return uniq


def parse_detail(html):
    result = {"title": "", "publish_date": "", "content": ""}
    soup = BeautifulSoup(html, "html.parser")
    title_tag = soup.find("title")
    if title_tag:
        t = title_tag.get_text(strip=True)
        t = re.sub(r"[_\-—]\s*中咨华宇\s*$", "", t)
        result["title"] = t.strip()
    meta_date = soup.find("meta", attrs={"name": "PubDate"})
    if meta_date and meta_date.get("content"):
        result["publish_date"] = meta_date["content"][:10]
    if not result["publish_date"]:
        m = re.search(r"20\d{2}[-/年]\d{1,2}[-/月]\d{1,2}", html[:8000])
        if m:
            result["publish_date"] = m.group(0).replace("年", "-").replace("月", "-").replace("/", "-")[:10]
    content_div = soup.find("dl", class_="nr3") or soup.find("div", class_="nr3")
    if not content_div:
        content_div = soup.find("div", class_="ny_main")
    if content_div:
        result["content"] = extract_clean_text(str(content_div))
    return result


def extract_clean_text(content_html):
    """输出纯 HTML：段落 <p>、链接 <a href target=_blank>、表格保留、图片 <img>、附件独立成段。
    禁止混合格式（纯文本 + 原始 HTML 会触发 search_app HTML 分支导致换行折叠）。"""
    if not content_html:
        return ""
    soup = BeautifulSoup(content_html, "html.parser")
    for tag in soup.find_all(['script', 'style']):
        tag.decompose()
    # dl/dt/dd → p（正文 dl.nr3 结构，防 dl 内容丢失导致整页转义兜底）
    for dt in soup.find_all('dt'):
        dt.name = 'p'
    for dd in soup.find_all('dd'):
        dd.name = 'p'
    # 过滤收藏/分享等装饰图标
    for img in soup.find_all('img'):
        src0 = img.get('src', '')
        if any(k in src0 for k in ('un-collect', 'collect', 'share', 'favicon', 'qrcode', 'ewm')):
            img.decompose()
    # 链接/附件 → HTML 内嵌绝对 URL
    for a_tag in soup.find_all("a", href=True):
        href = a_tag.get("href", "").strip()
        text = a_tag.get_text(strip=True) or os.path.basename(href.split("?")[0]) or "附件"
        if not href or href.startswith(("javascript:", "#")):
            a_tag.unwrap()
            continue
        full_url = urljoin(BASE_URL, href)
        new_a = soup.new_tag("a", href=full_url, target="_blank")
        new_a.string = text
        a_tag.replace_with(new_a)
    # 剥装饰标签保留文字
    for tag in soup.find_all(['span', 'b', 'strong', 'font', 'em', 'i', 'u', 's']):
        tag.unwrap()
    # 附件区 ul/li 扁平化：收集 li 内链接，独立成段
    attach_links = []
    for li in soup.find_all('li'):
        a_in = li.find('a', href=True)
        if a_in and not li.find_parent('table'):
            href = a_in.get('href', '').strip()
            text = a_in.get_text(strip=True) or os.path.basename(href.split('?')[0]) or '附件'
            if href and not href.startswith(('javascript:', '#')):
                attach_links.append((text, urljoin(BASE_URL, href)))
    for li in soup.find_all('li'):
        if li.find('a', href=True) and not li.find_parent('table'):
            li.decompose()
    # 移除纯装饰空标签
    for tag in soup.find_all(['div', 'p']):
        if not tag.get_text(strip=True) and not tag.find(['img', 'table', 'a']):
            tag.decompose()
    from html import escape
    parts = []
    for el in soup.find_all(['table', 'p']):
        if el.name == 'table':
            parts.append(str(el))
        elif el.name == 'p' and not el.find_parent('table'):
            # 保留内部 a/img/table HTML，其余文本 escape；过滤文件图标 img
            inner = []
            for c in el.contents:
                name = getattr(c, "name", None)
                if name == 'img':
                    src0 = c.get('src', '')
                    if any(k in src0.lower() for k in ('.gif', 'icon16', 'file', 'attach')):
                        continue
                    inner.append(str(c))
                elif name in ('a', 'table', 'br'):
                    inner.append(str(c))
                elif name:
                    inner.append(str(c))
                else:
                    inner.append(escape(str(c)))
            t = "".join(inner).strip()
            if t:
                parts.append(f"<p>{t}</p>")
    # 附件链接独立成段（去重）
    seen_a = set()
    for text, url in attach_links:
        key = (text, url)
        if key in seen_a:
            continue
        seen_a.add(key)
        parts.append(f'<p><a href="{url}" target="_blank">{text}</a></p>')
    return "\n\n".join(parts) if parts else escape(content_html).strip()


def crawl(pages=1, json_out=None):
    log(f"Crawl pages={pages} cutoff={CUTOFF}")
    items_all = []
    seen = set()
    for p in range(1, pages + 1):
        url = LIST_URL if p == 1 else f"{LIST_URL}&page={p}"
        try:
            html = fetch(url)
        except Exception as e:
            log(f"P{p} ERR: {e}")
            break
        items = parse_list(html)
        new = 0
        for it in items:
            if it["url"] not in seen:
                seen.add(it["url"])
                items_all.append(it)
                new += 1
        log(f"P{p}: {len(items)} items ({new} new), total {len(items_all)}")
        if not items:
            break
        time.sleep(0.8)

    enriched = []
    for idx, it in enumerate(items_all, 1):
        if it["pub_date"] and it["pub_date"] < CUTOFF:
            continue
        try:
            html = fetch(it["url"])
            detail = parse_detail(html)
        except Exception as e:
            log(f"  detail {it['url'][-40:]} ERR: {e}")
            detail = {"title": "", "publish_date": "", "content": ""}
        enriched.append({
            "title": detail["title"] or it["title"],
            "page_url": it["url"],
            "publish_date": detail["publish_date"] or it["pub_date"],
            "content": detail["content"],
            "site_name": SITE_NAME,
            "column": "环评公示",
        })
        if idx % 10 == 0:
            log(f"  {idx}/{len(items_all)}")
        time.sleep(0.4)

    if json_out:
        with open(json_out, "w", encoding="utf-8") as f:
            for item in enriched:
                f.write(json.dumps(item, ensure_ascii=False) + "\n")
        log(f"JSONL: {len(enriched)} items -> {json_out}")
        return

    conn = sqlite3.connect(DB_PATH, timeout=60)
    conn.execute("PRAGMA busy_timeout=10000")
    c = conn.cursor()
    stored = skipped = 0
    for item in enriched:
        try:
            c.execute(
                "INSERT OR IGNORE INTO gov_raw "
                "(site_name, source_url, page_url, title, publish_date, content, summary, category, script_name)"
                " VALUES (?,?,?,?,?,?,?,?,?)",
                (item["site_name"], item["page_url"], item["page_url"],
                 item["title"], item["publish_date"], item["content"],
                 (item["content"] or "")[:500], item["column"], "crawl_chinaeia_hpgongshi.py"))
            if c.rowcount > 0:
                stored += 1
            else:
                skipped += 1
        except Exception as e:
            log(f"DB error: {e}")
            skipped += 1
    conn.commit()
    conn.close()
    log(f"Result: {stored} new, {skipped} skipped")


if __name__ == "__main__":
    args = sys.argv[1:]
    pages = 5
    json_out = None
    if "--pages=" in " ".join(args):
        pages = int(re.search(r"--pages=(\d+)", " ".join(args)).group(1))
    if "--json" in args:
        json_out = args[args.index("--json") + 1]
    crawl(pages=pages, json_out=json_out)