#!/usr/bin/env python3
"""
大连市生态环境局 - 清洁生产审核
https://epb.dl.gov.cn/col/col4737/index.html
TRS/Hanweb CMS - AJAX pagination via dataproxy.jsp
"""
import json, re, sys, time, requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin

DB_PATH = "/root/search.db"
SITE_NAME = "大连市生态环境局-清洁生产审核"
CATEGORY = "大连"
BASE_URL = "https://epb.dl.gov.cn/col/col4737/index.html"
MAX_PAGES = 5
PER_PAGE = 15

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
    "Referer": "https://epb.dl.gov.cn/",
}
INSERT_SQL = """INSERT OR IGNORE INTO gov_raw 
    (site_name, page_url, title, publish_date, summary, content, category, attachments)
    VALUES (?, ?, ?, ?, '', ?, ?, ?)"""

PROXY_URL = ("https://epb.dl.gov.cn/module/web/jpage/dataproxy.jsp"
             "?page=1&webid=33&path=https://epb.dl.gov.cn/"
             "&columnid=4737&unitid=32863"
             "&webname=%E5%A4%A7%E8%BF%9E%E5%B8%82%E7%94%9F%E6%80%81%E7%8E%AF%E5%A2%83%E5%B1%80"
             "&permissiontype=0")


def init_db():
    import sqlite3
    conn = sqlite3.connect(DB_PATH, timeout=60)
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute("PRAGMA busy_timeout=5000")
    return conn


def fetch(url, timeout=30, headers=None):
    h = headers or HEADERS
    r = requests.get(url, headers=h, timeout=timeout)
    r.encoding = "utf-8"
    return r


def table_to_markdown(table, *args, **kwargs):
    """保留 HTML 表格结构（不转 md）"""
    return str(table)

def extract_content(text):
    """Parse article HTML content (between ContentStart/End)."""
    soup = BeautifulSoup(text, "html.parser")
    parts = []

    # Process all direct children and table elements
    for child in list(soup.children):
        if child.name is None:
            continue
        tn = child.name.lower()

        if tn == "p":
            txt = child.get_text(" ", strip=True)
            if txt:
                parts.append(txt)
        elif tn == "table":
            md = table_to_markdown(child)
            if md:
                parts.append(md)
        elif tn in ("div", "center", "section"):
            # Recursively process
            for sub_p in child.find_all("p", recursive=False):
                txt = sub_p.get_text(" ", strip=True)
                if txt:
                    parts.append(txt)
            for sub_t in child.find_all("table", recursive=False):
                md = table_to_markdown(sub_t)
                if md:
                    parts.append(md)
            remaining = child.get_text(" ", strip=True)
            # Remove already-extracted text to avoid duplication
            # (Simplified: just get remaining non-p/non-table text)
        elif tn in ("script", "style", "meta", "br"):
            continue
        else:
            txt = child.get_text(" ", strip=True)
            if txt:
                parts.append(txt)

    return "\n\n".join(parts)


def get_all_items():
    """Get all list items from the proxy (returns all 89)."""
    try:
        r = requests.get(PROXY_URL, headers=HEADERS, timeout=30)
        r.encoding = "utf-8"
    except Exception as e:
        print(f"  [ERROR] Proxy: {e}", flush=True)
        return []
    if r.status_code != 200:
        print(f"  [ERROR] Proxy HTTP {r.status_code}", flush=True)
        return []
    records = re.findall(r'<record><!\[CDATA\[(.*?)\]\]></record>', r.text, re.DOTALL)
    items = []
    for rec in records:
        a_match = re.search(r'<a\s+[^>]*href="([^"]*)"[^>]*title="([^"]*)"', rec)
        if not a_match:
            continue
        href = a_match.group(1)
        title = a_match.group(2).strip()
        full_url = urljoin("https://epb.dl.gov.cn/", href)
        date_m = re.search(r'\[(\d{4}-\d{2}-\d{2})\]', rec)
        date_str = date_m.group(1) if date_m else ""
        items.append({"title": title, "url": full_url, "date": date_str})
    return items


def extract_detail(url):
    """Fetch detail page and extract full content."""
    try:
        r = fetch(url)
    except Exception as e:
        print(f"  [ERROR] {url}: {e}", flush=True)
        return None
    if r.status_code != 200:
        return None

    soup = BeautifulSoup(r.text, "html.parser")
    html = r.text

    # Title
    title = ""
    mt = soup.find("meta", attrs={"name": "ArticleTitle"})
    if mt and mt.get("content"):
        title = mt["content"].strip()

    # Date
    pub_date = ""
    md = soup.find("meta", attrs={"name": "PubDate"})
    if md and md.get("content"):
        pub_date = md["content"].strip()
    # Fallback: meta PubDate with different case
    if not pub_date:
        md = soup.find("meta", attrs={"name": "pubdate"})
        if md and md.get("content"):
            pub_date = md["content"].strip()

    # Content between markers
    content_text = ""
    cs = html.find("ContentStart")
    ce = html.find("ContentEnd")
    if cs > 0 and ce > 0:
        after = html.find(">", cs)
        raw = html[after + 1:ce]
        content_text = extract_content(raw)

    # Attachments
    attachments_list = []
    art_con = soup.select_one("div.art_con")
    if art_con:
        for a_tag in art_con.find_all("a", href=True):
            h = a_tag["href"]
            if re.search(r'\.(pdf|doc|docx|xls|xlsx|rar|zip)$', h, re.IGNORECASE):
                at = a_tag.get_text(strip=True) or h.split("/")[-1]
                fu = urljoin(url, h)
                attachments_list.append({"name": at, "url": fu})

    # Empty content fallback (PDF-only articles)
    if len(content_text.strip()) < 20:
        # Try to get PDF description from the page
        content_text = f'<p><a href="{url}">{title}</a></p>\n（本文为PDF附件）'

    return {
        "title": title,
        "content": content_text,
        "pub_date": pub_date,
        "attachments": json.dumps(attachments_list, ensure_ascii=False) if attachments_list else "",
    }


def crawl(test_mode=False, max_pages=MAX_PAGES):
    print(f"[{SITE_NAME}] Starting, max_pages={max_pages}", flush=True)
    conn = init_db()
    cur = conn.cursor()

    all_items = get_all_items()
    print(f"  Total items from proxy: {len(all_items)}", flush=True)

    if not all_items:
        print("  [ERROR] No items found", flush=True)
        conn.close()
        return 0

    # Take first N items (N = max_pages * PER_PAGE)
    max_items = max_pages * PER_PAGE
    items = all_items[:max_items]
    print(f"  Taking first {len(items)} items ({max_pages} pages)", flush=True)

    total_inserted = total_skipped = 0
    for item in items:
        title, url, date = item["title"], item["url"], item["date"]

        cur.execute("SELECT id FROM gov_raw WHERE page_url = ?", (url,))
        if cur.fetchone():
            total_skipped += 1
            continue

        detail = extract_detail(url)
        if detail is None:
            total_skipped += 1
            continue

        if detail["title"] and len(detail["title"]) > len(title):
            title = detail["title"]
        content = detail["content"]
        attachments = detail["attachments"]

        cur.execute(INSERT_SQL, (
            SITE_NAME, url, title, date or detail["pub_date"],
            content, CATEGORY, attachments,
        ))
        total_inserted += 1

        if test_mode and total_inserted >= 10:
            break
        time.sleep(0.3)

    conn.commit()
    conn.close()
    print(f"[{SITE_NAME}] Done. Inserted={total_inserted}, Skipped={total_skipped}", flush=True)
    return total_inserted


if __name__ == "__main__":
    test_mode = "--test" in sys.argv
    mp = 3 if test_mode else MAX_PAGES
    if "--max-pages" in sys.argv:
        idx = sys.argv.index("--max-pages")
        if idx + 1 < len(sys.argv):
            mp = int(sys.argv[idx + 1])
    crawl(test_mode=test_mode, max_pages=mp)