#!/usr/bin/env python3
"""
沾益区政府 - 行政执法信息公示 (zhanyi.gov.cn)
CMS: 自定义 PHP, GET 分页
List: /pub/special/377.html?cate=377&page=N
  Container: ol.news_list > li > a(href, title) + date text
Detail: /news/xzzfxxgs/{id}.html
  Content: div.web_con.wow.fadeInUp (p > span inline formatting)
  Title: <title> (strip site suffix)
  Date: YYYY-MM-DD from page
  No WAF, direct curl OK, ~13 items/page, 2 pages
"""
import sys
import re
import os
import sqlite3
import urllib.request
import ssl

# === CONFIG ===
SITE_NAME = "沾益区-行政执法信息公示"
BASE_URL = "https://www.zhanyi.gov.cn"
LIST_TPL = BASE_URL + "/pub/special/377.html?cate=377&page={}"
DB_PATH = "/root/search.db"
CUTOFF_DATE = "2023-01-01"
PAGES_DEFAULT = 2  # 2 pages total

ssl_ctx = ssl.create_default_context()
ssl_ctx.check_hostname = False
ssl_ctx.verify_mode = ssl.CERT_NONE


def fetch(url):
    req = urllib.request.Request(url, headers={
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
    })
    try:
        resp = urllib.request.urlopen(req, timeout=20, context=ssl_ctx)
        return resp.read().decode("utf-8", errors="replace")
    except Exception as e:
        print(f"  [FETCH ERROR] {url}: {e}")
        return None


def parse_list(html):
    """Parse list page"""
    items = []
    pattern = r'<li>.*?<a href="([^"]*)"[^>]*>([^<]+)</a>(\d{4}-\d{2}-\d{2})'
    for match in re.finditer(pattern, html, re.DOTALL):
        href = match.group(1).strip()
        title = match.group(2).strip()
        date = match.group(3).strip()
        if not href.startswith("http"):
            href = BASE_URL + href
        items.append({"url": href, "title": title, "date": date})
    return items


def parse_detail(html):
    """Extract title, date, content from detail page"""
    # Title from <title> (strip site suffix)
    title = ""
    title_match = re.search(r'<title>(.*?)</title>', html)
    if title_match:
        raw = title_match.group(1)
        # Strip suffix like "-曲靖市沾益区人民政府-..."
        title = re.sub(r'[-–—].*$', '', raw).strip()
        if not title:
            title = raw

    # Date
    date = ""
    date_match = re.search(r'(\d{4}-\d{2}-\d{2})\s*\d{2}:\d{2}', html)
    if date_match:
        date = date_match.group(1)

    # Content from div.web_con
    content = ""
    idx = html.find('class="web_con wow fadeInUp"')
    if idx < 0:
        idx = html.find('class="web_con"')
    if idx < 0:
        idx = html.find('class="wow fadeInUp"')

    if idx > 0:
        start = html.rfind("<div", 0, idx)
        if start < 0:
            start = idx
        end_markers = ['<div class="arti_attr"', '<div class="near', '<div class="share', '<div class="footer', '<div class="config']
        end = len(html)
        for marker in end_markers:
            pos = html.find(marker, idx)
            if 0 < pos < end:
                end = pos
        content_html = html[start:end]

        # Remove scripts
        text = re.sub(r'<script[^>]*>.*?</script>', '', content_html, flags=re.DOTALL)
        text = re.sub(r'<style[^>]*>.*?</style>', '', text, flags=re.DOTALL)

        # Preserve breaks
        text = re.sub(r'<br\s*/?>', '\n', text)
        text = re.sub(r'</p>', '\n\n', text)
        text = text.replace('\r', '')

        # Clean HTML entities
        text = text.replace('&nbsp;', ' ').replace('&ldquo;', '\u201c').replace('&rdquo;', '\u201d')

        # Extract tables
        tables = []
        for table_match in re.finditer(r'<table[^>]*>.*?</table>', text, re.DOTALL):
            table_html = table_match.group(0)
            rows = re.findall(r'<tr[^>]*>(.*?)</tr>', table_html, re.DOTALL)
            table_lines = []
            for row in rows:
                cells = re.findall(r'<t[dh][^>]*>(.*?)</t[dh]>', row, re.DOTALL)
                cell_texts = [re.sub(r'<[^>]+>', '', c).strip() for c in cells]
                if cell_texts and any(cell_texts):
                    table_lines.append("| " + " | ".join(cell_texts) + " |")
            if table_lines:
                if len(table_lines) >= 2:
                    hdr = table_lines[0].count("|") - 1
                    table_lines.insert(1, "|" + "---|" * hdr)
                tables.append("\n".join(table_lines))

        # Clean text
        clean = re.sub(r'<[^>]+>', '', text)
        clean = re.sub(r'[ \t]+', ' ', clean)
        clean = re.sub(r' *\n *', '\n', clean)
        clean = re.sub(r'\n{3,}', '\n\n', clean)
        clean = clean.strip()

        parts = [clean] if clean else []
        if tables:
            parts.append("\n\n".join(tables))
        content = "\n\n".join(parts)

    # Attachments
    attachments = []
    pdfs = re.findall(r'<a[^>]*href="([^"]+\.pdf)"[^>]*>([^<]*)</a>', html, re.IGNORECASE)
    for href, text in pdfs:
        t = text.strip()
        if not href.startswith("http"):
            href = BASE_URL + href
        attachments.append({"href": href, "text": t or href.split("/")[-1]})

    return {"title": title, "date": date, "content": content, "attachments": attachments}


def insert_to_db(items):
    conn = sqlite3.connect(DB_PATH, timeout=30)
    c = conn.cursor()
    inserted = 0
    skipped = 0

    for item in items:
        title = item.get("title", "")
        url = item.get("url", "")
        date = item.get("date", "")
        content = item.get("content", "")

        attach_str = ""
        for att in item.get("attachments", []):
            if attach_str:
                attach_str += "\n"
            attach_str += f"[{att['text']}]({att['href']})"

        if not content and not title:
            skipped += 1
            continue

        try:
            c.execute("""
                INSERT OR IGNORE INTO gov_raw
                (title, page_url, site_name, publish_date, content, summary, attachments, source_url, date_rank)
                VALUES (?, ?, ?, ?, ?, '', ?, ?, CAST(strftime('%s', ?) AS INTEGER))
            """, (title, url, SITE_NAME, date, content, attach_str, url, date))
            if c.rowcount > 0:
                inserted += 1
            else:
                skipped += 1
        except Exception as e:
            print(f"  [DB ERROR] {title[:30]}: {e}")
            skipped += 1

    conn.commit()
    conn.close()
    return inserted, skipped


def main():
    max_pages = PAGES_DEFAULT
    for arg in sys.argv[1:]:
        if arg.isdigit():
            max_pages = int(arg)

    print(f"[INFO] {SITE_NAME} - 爬虫, pages={max_pages}")

    all_items = []
    total_new = 0
    total_old = 0

    for page_no in range(1, max_pages + 1):
        url = LIST_TPL.format(page_no)
        print(f"\n  [Page {page_no}] {url}")
        html = fetch(url)
        if not html:
            print(f"  [SKIP] Cannot fetch page {page_no}")
            continue

        items = parse_list(html)
        print(f"  Found {len(items)} items")

        if not items:
            print(f"  [STOP] No items")
            break

        for item in items:
            if item["date"] < CUTOFF_DATE:
                print(f"  [STOP] Date {item['date']} < {CUTOFF_DATE}")
                break

            print(f"    {item['date']} {item['title'][:50]}...")
            detail_html = fetch(item["url"])
            if not detail_html:
                print(f"    [SKIP] Cannot fetch detail")
                total_old += 1
                continue

            detail = parse_detail(detail_html)
            if detail["title"]:
                item["title"] = detail["title"]
            if detail["date"]:
                item["date"] = detail["date"]
            item["content"] = detail["content"]
            item["attachments"] = detail.get("attachments", [])
            all_items.append(item)

        if all_items:
            new, old = insert_to_db(all_items)
            total_new += new
            total_old += old
            print(f"  [DB] +{new} new, {old} existing")
            all_items = []

    if all_items:
        new, old = insert_to_db(all_items)
        total_new += new
        total_old += old

    print(f"\n[DONE] 新增: {total_new}, 跳过: {total_old}")

    if total_new > 0:
        conn = sqlite3.connect(DB_PATH, timeout=30)
        conn.execute("SELECT 1 /* noop: gov_search 由触发器维护, 无需 rebuild */")
        conn.commit()
        conn.close()
        print("FTS rebuilt")


if __name__ == "__main__":
    main()