#!/usr/bin/env python3
"""
绍兴市生态环境局 - 建设项目审批公示 (sxepb.sx.gov.cn)
CMS: TRS/JCMS, AJAX 分页
List API: /api-gateway/jpaas-publish-server/front/page/build/unit
  pageId=1229459599, pageNo=N, 15条/页, 共770条/52页
Detail: /col/col1229459599/art/2026/art_XXXX.html
  Title: h1.ewiunfjtitle
  Date: li.fabushijian
  Content: div.wenzhangzwendnrong (文章正文内容)
  Attachments: <a> links with PDF files
IP: 203.107.60.182 (needs Host header)
"""

import urllib.request
import urllib.error
import json
import re
import sys
import os
import ssl

# === CONFIG ===
SITE_NAME = "绍兴市生态环境局-建设项目审批公示"
HOST = "sxepb.sx.gov.cn"
IP = "203.107.60.182"
BASE = f"http://{IP}"
COL_ID = "1229459599"
WEB_ID = "3001"
API_URL = (f"{BASE}/api-gateway/jpaas-publish-server/front/page/build/unit"
           f"?parseType=bulidstatic&webId={WEB_ID}"
           f"&tplSetId=1EDJzLJApOIQLd0OS4KST&pageType=column"
           f"&tagId=ajax%E5%88%86%E9%A1%B5%E5%88%97%E8%A1%A8"
           f"&pageId={COL_ID}")
DB_PATH = "/root/search.db"
CUTOFF_DATE = "2023-01-01"
PAGES_DEFAULT = 5
ROWS_PER_PAGE = 15

ssl_ctx = ssl.create_default_context()
ssl_ctx.check_hostname = False
ssl_ctx.verify_mode = ssl.CERT_NONE


def fetch_api(page_no):
    url = f"{API_URL}&pageNo={page_no}"
    req = urllib.request.Request(url, headers={
        "Host": HOST,
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
        "Referer": f"http://{HOST}/col/col{COL_ID}/index.html",
        "Accept": "application/json, text/javascript, */*",
        "X-Requested-With": "XMLHttpRequest",
    })
    try:
        resp = urllib.request.urlopen(req, timeout=20, context=ssl_ctx)
        data = json.loads(resp.read().decode("utf-8"))
        result = data.get("data", {}).get("html", "")
        return result
    except Exception as e:
        print(f"  [API ERROR] page {page_no}: {e}", file=sys.stderr)
        import traceback
        traceback.print_exc(file=sys.stderr)
        return None


def fetch_page(url):
    """Fetch a regular HTML page"""
    req = urllib.request.Request(url, headers={
        "Host": HOST,
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
    })
    try:
        resp = urllib.request.urlopen(req, timeout=20, context=ssl_ctx)
        return resp.read().decode("utf-8", errors="replace")
    except Exception as e:
        print(f"  [FETCH ERROR] {url}: {e}")
        return None


def parse_list(html):
    """Parse list API response HTML"""
    items = []
    pattern = r'<a[^>]+href="([^"]+)"[^>]*title="([^"]*)"[^>]*>.*?<span class="dwebhjcfdc">\[?(\d{4}-\d{2}-\d{2})\]?</span>'
    for m in re.finditer(pattern, html, re.DOTALL):
        href = m.group(1).strip()
        title = m.group(2).strip()
        date = m.group(3).strip()

        # Build full URL
        if href.startswith("http"):
            full_url = href
        elif href.startswith("/"):
            full_url = f"http://{HOST}{href}"
        else:
            full_url = f"http://{HOST}/col/col{COL_ID}/{href}"

        items.append({"title": title, "url": full_url, "date": date})

    return items


def parse_detail(html, url):
    """Parse detail page"""
    result = {"content": "", "attachments": []}

    # Title
    m = re.search(r'<h1[^>]*class="[^"]*ewiunfjtitle[^"]*"[^>]*>(.*?)</h1>', html, re.DOTALL)
    if m:
        result["title"] = m.group(1).strip()

    # Content - div.wenzhangzwendnrong
    m = re.search(r'<div[^>]*class="[^"]*wenzhangzwendnrong[^"]*"[^>]*>(.*?)</div>\s*<div[^>]*class="[^"]*poiiuygvncdssd[^"]*"', html, re.DOTALL)
    if not m:
        # Fallback: find the div and capture until next sibling
        m = re.search(r'<div[^>]*class="[^"]*wenzhangzwendnrong[^"]*"[^>]*>(.*?)</div>\s*', html, re.DOTALL)

    if m:
        content_html = m.group(1)
        result["content"] = html_to_markdown(content_html)

        # Extract attachments (PDF links referencing download API)
        for am in re.finditer(r'<a[^>]+href="([^"]*download[^"]*)"[^>]*>([^<]*)</a>', content_html):
            fname = am.group(2).strip()
            furl = am.group(1).strip()
            if fname and furl:
                result["attachments"].append(f"{fname}]({furl}")
    else:
        print(f"  [WARN] No content div found in {url}")

    # Date from meta
    if "date" not in result or not result.get("date"):
        m = re.search(r'meta[^>]+PubDate[^>]+content="(\d{4}-\d{2}-\d{2})"', html)
        if m:
            result["date"] = m.group(1)

    return result


def html_to_markdown(html):
    """提取正文：段落取文本，表格保留原始 HTML"""
    tables = []
    def keep(m):
        tables.append(m.group(0))
        return "\n@@TABLE_%d@@\n" % (len(tables) - 1)
    html = re.sub(r'<table[^>]*>.*?</table>', keep, html, flags=re.DOTALL)

    # Clean remaining HTML
    text = re.sub(r'<script[^>]*>.*?</script>', '', html, flags=re.DOTALL)
    text = re.sub(r'<style[^>]*>.*?</style>', '', text, flags=re.DOTALL)
    text = re.sub(r'<br\s*/?>', '\n', text)
    text = re.sub(r'</p>', '\n\n', text)
    text = re.sub(r'</div>', '\n', text)
    text = re.sub(r'<[^>]+>', '', text)
    for e, r in [('&nbsp;', ' '), ('&lt;', '<'), ('&gt;', '>'), ('&amp;', '&'),
                  ('&#39;', "'"), ('&quot;', '"'), ('&mdash;', '—'), ('&middot;', '·')]:
        text = text.replace(e, r)
    text = re.sub(r'\n{3,}', '\n\n', text)
    # Clean Chinese inter-character spaces
    text = re.sub(r'(?<=[\u4e00-\u9fff])\s+(?=[\u4e00-\u9fff])', '', text)
    # Restore tables
    for i, t in enumerate(tables):
        text = text.replace("@@TABLE_%d@@" % i, t)
    return text.strip()


def insert_to_db(items):
    import sqlite3
    conn = sqlite3.connect(DB_PATH, timeout=30)
    c = conn.cursor()
    c.execute("PRAGMA busy_timeout=30000")

    inserted = 0
    skipped = 0
    for item in items:
        attachments = "\n".join(item.get("attachments", []))
        try:
            c.execute("""
                INSERT OR IGNORE INTO gov_raw
                (title, page_url, site_name, publish_date, content, summary, attachments, source_url, date_rank)
                VALUES (?, ?, ?, ?, ?, '', ?, ?, CAST(strftime('%s', ?) AS INTEGER))
            """, (
                item.get("title", ""),
                item.get("url", ""),
                SITE_NAME,
                item.get("date", ""),
                item.get("content", ""),
                attachments,
                item.get("url", ""),
                item.get("date", ""),
            ))
            if c.rowcount > 0:
                inserted += 1
            else:
                skipped += 1
        except Exception as e:
            print(f"  [DB ERROR] {item.get('title', '')[:30]}: {e}")
            skipped += 1

    conn.commit()
    conn.close()
    return inserted, skipped


def main():
    max_pages = PAGES_DEFAULT
    for arg in sys.argv[1:]:
        if arg.isdigit():
            max_pages = int(arg)

    print(f"[INFO] {SITE_NAME} - 爬虫, pages={max_pages}")

    all_items = []
    total_new = 0
    total_old = 0

    first_html = None
    for page_no in range(1, max_pages + 1):
        print(f"\n  [Page {page_no}] Fetching API...")
        html = fetch_api(page_no)
        if not html:
            print(f"  [SKIP] Failed page {page_no}")
            continue

        if page_no == 1:
            first_html = html
        elif html == first_html:
            print(f"  [STOP] Page {page_no} same as page 1 (AJAX returns identical content)")
            break

        items = parse_list(html)
        print(f"  Found {len(items)} items")

        if not items:
            print(f"  [STOP] No items")
            break

        for item in items:
            if item["date"] < CUTOFF_DATE:
                print(f"  [STOP] Date {item['date']} < {CUTOFF_DATE}")
                break

            print(f"    {item['date']} {item['title'][:50]}...")
            detail_html = fetch_page(item["url"])
            if not detail_html:
                print(f"    [SKIP] Cannot fetch")
                total_old += 1
                continue

            detail = parse_detail(detail_html, item["url"])
            item["content"] = detail.get("content", "")
            if "title" in detail:
                item["title"] = detail["title"]
            item["attachments"] = detail.get("attachments", [])
            if "date" in detail:
                item["date"] = detail["date"]
            all_items.append(item)

        # Batch insert
        if all_items:
            new, old = insert_to_db(all_items)
            total_new += new
            total_old += old
            print(f"  [DB] +{new} new, {old} existing")
            all_items = []

    if all_items:
        new, old = insert_to_db(all_items)
        total_new += new
        total_old += old

    print(f"\n[DONE] 新增: {total_new}, 跳过: {total_old}")

    if total_new > 0:
        import sqlite3
        conn = sqlite3.connect(DB_PATH, timeout=30)
        conn.execute("SELECT 1 /* noop: gov_search 由触发器维护, 无需 rebuild */")
        conn.commit()
        conn.close()
        print("FTS rebuilt")


if __name__ == "__main__":
    main()