#!/usr/bin/env python3
"""Crawl tzw.nantong.gov.cn - 通州湾示范区 环评公示"""
import sys, re, json, sqlite3, urllib.request, traceback
from datetime import datetime

BASE = "https://tzw.nantong.gov.cn"
SITE_NAME = "tzwnantong_hpgs"
DB_PATH = "/root/search.db"
PER_PAGE = 10
MAX_PAGES = 30

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
    "Referer": BASE + "/tzwjhldkfsfq/hpgs/hpgs.html",
}

def fetch(url):
    req = urllib.request.Request(url, headers=headers)
    resp = urllib.request.urlopen(req, timeout=15)
    return resp.read().decode("utf-8", errors="replace")

COLUMN_ID = "28345d5c-4438-40a5-a053-4f5369150c46"
def _get_column_id_old():
    """Extract columnId from list page JS config."""
    html = fetch(BASE + "/tzwjhldkfsfq/hpgs/hpgs.html")
    m = re.search(r"columnId:'([^']+)'", html)
    if m:
        return m.group(1)
    # fallback
    m = re.search(r'columnId:"([^"]+)"', html)
    return m.group(1) if m else None

def get_articles_from_api(column_id, start, end):
    """Get article list from TrueCMS API. Returns list of {title, url, date, href}."""
    url = (f"{BASE}/truecms/messageController/getMessage.do"
           f"?callback=jq&columnId={column_id}"
           f"&startrecord={start}&endrecord={end}&perpage={PER_PAGE}")
    resp = fetch(url)
    # Parse JSONP response: callback({...})
    m = re.search(r"\{.*\}", resp)
    if not m:
        return []
    data = json.loads(m.group())
    result_xml = data.get("result", "")
    
    # Parse records from XML: <record><![CDATA[ <li>... </li>]]></record>
    articles = []
    records = re.findall(r"<record><!\[CDATA\[(.*?)\]\]></record>", result_xml)
    for rec in records:
        # Extract href, title, date
        href_m = re.search(r'href="([^"]+)"', rec)
        title_m = re.search(r'title="([^"]*)"', rec)
        date_m = re.search(r'<span[^>]*>(\d{4}-\d{2}-\d{2})', rec)
        if href_m and title_m:
            href = href_m.group(1)
            if not href.startswith("http"):
                href = BASE + href
            articles.append({
                "title": title_m.group(1),
                "url": href,
                "date": date_m.group(1) if date_m else "",
            })
    
    # Get total count
    total_m = re.search(r"<totalrecord>(\d+)</totalrecord>", result_xml)
    total = int(total_m.group(1)) if total_m else 0
    
    return articles, total


def extract_clean_text(html_chunk):
    """Extract text from HTML, preserving paragraph breaks."""
    text = re.sub(r"</p>", "\n\n", html_chunk, flags=re.I)
    text = re.sub(r"<br\s*/?>", "\n", text, flags=re.I)
    text = re.sub(r"</div>", "\n\n", text, flags=re.I)
    text = re.sub(r"</?(?:b|span|font|strong|em|u|i|a)\b[^>]*>", "", text, flags=re.I)
    text = re.sub(r"<p\b[^>]*>", "", text, flags=re.I)
    text = re.sub(r"\n{3,}", "\n\n", text)
    lines = [l.strip() for l in text.split("\n")]
    text = "\n".join(lines)
    text = re.sub(r"\n{3,}", "\n\n", text)
    return text.strip()


def extract_attachments(html):
    """Extract attachment links, return as markdown string."""
    parts = []
    links = re.findall(
        r'<a\s[^>]*href="(/truecms/attachmentController/download\.do[^"]*)"[^>]*>([^<]+)</a>',
        html, re.I
    )
    for href, name in links:
        full_url = BASE + href
        parts.append(f"[{name.strip()}]({full_url})")
    return "\n".join(parts)


def get_detail(url):
    """Get (title, date, content, source) from detail page."""
    html = fetch(url)
    
    # Title from <h1>
    title = ""
    tm = re.search(r"<h1>([^<]+)</h1>", html)
    if tm:
        title = tm.group(1).strip()
    if not title:
        tm2 = re.search(r'<title>([^<]+)</title>', html)
        if tm2:
            title = re.sub(r"\s*[-–—|_].*$", "", tm2.group(1)).strip()
    
    # Date from pp1 span
    date = ""
    dm = re.search(r"发布时间[：:]\s*(\d{4}-\d{2}-\d{2})", html)
    if dm:
        date = dm.group(1)
    if not date:
        dm2 = re.search(r'<span class="fl">(\d{4}-\d{2}-\d{2})', html)
        if dm2:
            date = dm2.group(1)
    
    # Source
    source = ""
    sm = re.search(r"来源[：:]\s*([^<]+)", html)
    if sm:
        source = sm.group(1).strip()
    
    # Extract content from div#zoom
    content = ""
    idx = html.find('id="zoom"')
    if idx < 0:
        idx = html.find('class="pp3"')
    if idx > 0:
        start = html.rfind("<div", 0, idx)
        if start < 0:
            start = idx
        pos = html.find(">", idx) + 1
        
        depth = 1
        end = pos
        while depth > 0 and end < len(html):
            no = html.find("<div", end)
            nc = html.find("</div>", end)
            if nc < 0:
                break
            if no >= 0 and no < nc:
                depth += 1
                end = html.find(">", no) + 1
            else:
                depth -= 1
                end = nc + 6
        
        chunk = html[pos:end-6] if depth == 0 else html[pos:]
        content = extract_clean_text(chunk)
    
    # Also extract attachments
    attachments = extract_attachments(html)
    if attachments:
        if content:
            content += "\n\n"
        content += attachments
    
    return title, date, content, source


def save_to_db(records):
    if not records:
        return 0
    conn = sqlite3.connect(DB_PATH)
    conn.execute("PRAGMA busy_timeout=30000")
    c = conn.cursor()
    count = 0
    for r in records:
        existing = c.execute("SELECT id FROM gov_raw WHERE source_url=?", (r["url"],)).fetchone()
        if existing:
            print(f"  SKIP: {r['title'][:40]}")
            continue
        content = (r["content"] or "")[:5000]
        c.execute("""INSERT INTO gov_raw (site_name, source_url, page_url, title, publish_date, date_rank, summary, content)
            VALUES (?,?,?,?,?,?,?,?)""",
            (SITE_NAME, r["url"], r["url"], r["title"], r["date"],
             int(datetime.strptime(r["date"], "%Y-%m-%d").timestamp()) if r["date"] else 0,
             content[:200], content))
        conn.commit()
        count += 1
        print(f"  OK: {r['title'][:50]} | {r['date']}")
    conn.close()
    return count


def main():
    print(f"=== Crawling {SITE_NAME} ===")
    
    # Get column ID
    column_id = COLUMN_ID
    print(f"Column ID: {column_id}")
    
    # Get first batch to know total
    articles, total = get_articles_from_api(column_id, 1, PER_PAGE)
    if not articles:
        print("ERROR: No articles found")
        return
    print(f"Total: {total} articles ({total // PER_PAGE + 1} pages)")
    
    all_articles = list(articles)
    
    # Fetch remaining pages
    for start in range(PER_PAGE + 1, min(total, MAX_PAGES * PER_PAGE) + 1, PER_PAGE):
        end = min(start + PER_PAGE - 1, total)
        page_num = start // PER_PAGE + 1
        print(f"\nPage {page_num} (records {start}-{end})...")
        items, _ = get_articles_from_api(column_id, start, end)
        if not items:
            print(f"  No more items, stopping")
            break
        print(f"  Got {len(items)} articles")
        all_articles.extend(items)
    
    print(f"\n=== Total: {len(all_articles)} articles ===")
    
    # Fetch details
    records = []
    for i, a in enumerate(all_articles):
        print(f"\n[{i+1}/{len(all_articles)}] {a['title'][:50]}...")
        try:
            title, date, content, source = get_detail(a["url"])
            records.append({
                "title": title or a["title"],
                "url": a["url"],
                "date": date or a["date"],
                "content": content,
                "source": source,
            })
        except Exception as e:
            print(f"  ERROR: {e}")
            traceback.print_exc()
    
    saved = save_to_db(records)
    print(f"\n=== Complete: {saved}/{len(records)} new records ===")


if __name__ == "__main__":
    main()