#!/usr/bin/env python3
"""
crawl_cz_sthjj.py — 郴州市生态环境局 建设项目受理情况公开 爬虫
============================================================
站点: https://sthjj.czs.gov.cn/zwgk/xxgkml/hjpj/jsxmhbspqgs/
特点: 天融信WAF防护，需 Playwright 绕过 JS 挑战
列表: UL > LI > a[href*='content_']，每页16条
详情: 文字信息（项目名称/建设地点/建设单位等）+ PDF附件
分页: default.htm, default_1.htm, ... (offset=1530 ≈ 97页)

用法:
  python3 crawl_cz_sthjj.py           # 增量爬
  python3 crawl_cz_sthjj.py --full    # 全量爬
  python3 crawl_cz_sthjj.py --test N  # 测试N条
  python3 crawl_cz_sthjj.py --sync    # 仅同步
"""

import os, re, sys, time, json, sqlite3, subprocess, ssl
from urllib.parse import urljoin
from playwright.sync_api import sync_playwright

BASE_DIR = os.path.dirname(os.path.abspath(__file__))
DB_PATH = os.path.join(BASE_DIR, "cz_sthjj_results.db")
SITE_NAME = "郴州市生态环境局"
DOMAIN = "sthjj.czs.gov.cn"
SERVER_SSH = "root@1.94.217.116"
SERVER_SEARCH_DB = "/root/search.db"

BASE_LIST = "https://sthjj.czs.gov.cn/zwgk/xxgkml/hjpj/jsxmhbspqgs/default.htm"
DETAIL_BASE = "https://sthjj.czs.gov.cn/zwgk/xxgkml/hjpj/jsxmhbspqgs/"

CHECKPOINT_FILE = os.path.join(BASE_DIR, ".cz_sthjj_chkpt")
stats = {"new": 0, "skip": 0, "errors": 0}

def save_checkpoint(page_num, idx):
    """保存断点：第page_num页的第idx条"""
    with open(CHECKPOINT_FILE, 'w') as f:
        f.write(f"{page_num}|{idx}")

def load_checkpoint():
    """读取断点，返回(page_num, idx)"""
    if os.path.exists(CHECKPOINT_FILE):
        try:
            with open(CHECKPOINT_FILE) as f:
                parts = f.read().strip().split('|')
                return int(parts[0]), int(parts[1])
        except:
            pass
    return 1, 0

def clear_checkpoint():
    if os.path.exists(CHECKPOINT_FILE):
        os.remove(CHECKPOINT_FILE)

def ensure_browser(browser_obj=None, context_obj=None):
    """检查 Playwright 连接状态，断开则重建 browser + context"""
    try:
        if browser_obj:
            browser_obj.contexts  # 触发异常如果已断
        if browser_obj and context_obj:
            context_obj.pages
        return browser_obj, context_obj
    except Exception:
        print("  🔄 Playwright 连接断开，重建 browser + context...")
    pw = sync_playwright().__enter__()
    browser_obj = pw.chromium.launch(headless=True)
    context_obj = browser_obj.new_context(
        user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
        viewport={"width": 1920, "height": 1080},
    )
    return browser_obj, context_obj

def init_db():
    conn = sqlite3.connect(DB_PATH, timeout=60)
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute("""CREATE TABLE IF NOT EXISTS crawl_results (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        site_name TEXT DEFAULT '郴州市生态环境局',
        title TEXT,
        url TEXT UNIQUE,
        content TEXT,
        publish_date TEXT,
        summary TEXT,
        crawled_at TEXT DEFAULT (datetime('now','localtime'))
    )""")
    conn.commit()
    conn.close()

def store_record(title, url, content, date, summary):
    conn = sqlite3.connect(DB_PATH, timeout=60)
    try:
        conn.execute(
            "INSERT OR IGNORE INTO crawl_results "
            "(title, url, content, publish_date, summary) "
            "VALUES (?,?,?,?,?)",
            (title, url, content, date, summary),
        )
        conn.commit()
        if conn.total_changes > 0:
            stats["new"] += 1
        else:
            stats["skip"] += 1
    except:
        stats["errors"] += 1
    finally:
        conn.close()

def esc(v):
    if v is None: return "''"
    return "'" + str(v).replace("'", "''") + "'"

def sync_to_server():
    """将本地数据直接写入 search.db（服务器本地模式）"""
    print("\n📤 同步到 search.db...")

    conn = sqlite3.connect(DB_PATH, timeout=60)
    rows = conn.execute("SELECT title, url, content, publish_date, summary FROM crawl_results ORDER BY id").fetchall()
    conn.close()

    if not rows:
        print("  本地没有数据")
        return

    dst = sqlite3.connect("/root/search.db", timeout=60)
    dst.execute("PRAGMA journal_mode=WAL")

    site_name = "郴州市生态环境局"
    new_count = 0
    for r in rows:
        title, url, content, pub_date, summary = r
        try:
            dst.execute(
                "INSERT OR IGNORE INTO gov_raw "
                "(title, page_url, content, publish_date, summary, site_name, tags) "
                "VALUES (?,?,?,?,?,?,?)",
                (title, url, (content or "")[:500000], pub_date or "",
                 (summary or "")[:300], site_name, "")
            )
            if dst.total_changes > 0:
                new_count += 1
        except Exception as e:
            print(f"  Error: {e}")

    if new_count > 0:
        dst.commit()
        # Update FTS
        dst.execute(
            "INSERT OR REPLACE INTO gov_search(rowid,title,site_name,summary) "
            "SELECT r.id,r.title,r.site_name,r.summary FROM gov_raw r "
            "WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?",
            (site_name,))
        dst.commit()

    total = dst.execute(
        "SELECT COUNT(*) FROM gov_raw WHERE site_name=?", (site_name,)).fetchone()[0]
    dst.close()

    print(f"  OK {new_count}/{len(rows)} 条同步到 search.db (DB共{total}条)")

def pw_get(url, page, browser, context, retry=2):
    """用 Playwright 页面实例获取 HTML，断连时自动重建"""
    for attempt in range(retry + 1):
        try:
            # 先检查连接
            browser, context = ensure_browser(browser, context)
            page = context.pages[0] if context.pages else context.new_page()
            page.goto(url, wait_until="networkidle", timeout=30000)
            time.sleep(2)
            return page.content(), browser, context
        except Exception as e:
            err_str = str(e)
            if "EPIPE" in err_str or "Target closed" in err_str or "Browser closed" in err_str:
                print(f"  ⚡ 断连({err_str[:60]}), 重建连接...")
                browser, context = ensure_browser(None, None)
                page = context.new_page()
            elif attempt < retry:
                print(f"  ⚠ 重试({err_str[:60]})...")
                time.sleep(3)
            else:
                return None, browser, context
    return None, browser, context

def fetch_list_page(page_num, pw_page, browser, context):
    """获取列表页，返回 [{'title','url','date'}]"""
    if page_num == 1:
        url = BASE_LIST
    else:
        url = f"https://sthjj.czs.gov.cn/zwgk/xxgkml/hjpj/jsxmhbspqgs/default_{page_num - 1}.htm"

    result, browser, context = pw_get(url, pw_page, browser, context)
    if not result:
        return None, browser, context

    items = []
    # 从整个页面HTML中提取列表项
    for m in re.finditer(
        r'<li>\s*<a\s+href="(content_\d+\.html)"[^>]*title="([^"]+)"[^>]*>.*?</a>\s*<span[^>]*>([\d-]+)',
        result, re.DOTALL
    ):
        href = m.group(1).strip()
        title = m.group(2).strip()
        date = m.group(3).strip()
        items.append({
            "url": urljoin(DETAIL_BASE, href),
            "title": title,
            "date": date,
        })

    return items, browser, context

def fetch_detail(detail_url, pw_page, browser, context):
    """抓取详情页"""
    html, browser, context = pw_get(detail_url, pw_page, browser, context)
    if not html:
        return None

    # 标题：<title> 或 ArticleTitle meta（页面有多个 h2，取 title 最可靠）
    title = ""
    tm = re.search(r'<title>(.*?)</title>', html, re.DOTALL)
    if tm:
        title = tm.group(1).strip()
    if not title:
        atm = re.search(r'<meta[^>]*name="ArticleTitle"[^>]*content="([^"]+)"', html, re.I)
        if atm:
            title = atm.group(1).strip()

    # 正文：xl-xqnr 内容区（包含表格字段 + PDF附件链接）
    content = ""
    cm = re.search(r'class="xl-xqnr">(.*?)</div>\s*</div>', html, re.DOTALL)
    if cm:
        content_raw = cm.group(1).strip()
        content_raw = re.sub(r'<script[^>]*>.*?</script>', '', content_raw, flags=re.DOTALL|re.I)
        content_raw = re.sub(r'<style[^>]*>.*?</style>', '', content_raw, flags=re.DOTALL|re.I)
        content_raw = re.sub(r'\s+style="[^"]*"', '', content_raw)
        content = content_raw.strip()
    if not content:
        # 降级：直接取页面关键文本
        text = re.sub(r'<script[^>]*>.*?</script>', '', html, flags=re.DOTALL|re.I)
        text = re.sub(r'<style[^>]*>.*?</style>', '', text, flags=re.DOTALL|re.I)
        text = re.sub(r'<[^>]+>', '\n', text)
        text = re.sub(r'\s*\n\s*\n\s*', '\n', text).strip()
        # 提取项目信息区域
        lines = text.split('\n')
        info_lines = []
        capture = False
        for i, line in enumerate(lines):
            if any(kw in line for kw in ['项目名称', '建设地点', '建设单位', '受理日期']):
                capture = True
            if capture:
                if '扫一扫' in line or '打印本页' in line or '设为首页' in line:
                    break
                info_lines.append(line.strip())
        if info_lines:
            content = '<p>' + '</p><p>'.join(info_lines[:30]) + '</p>'

    # 日期
    date = ""
    dm = re.search(r'发布时间[：:]\s*(\d{4}-\d{2}-\d{2})', html)
    if dm:
        date = dm.group(1)

    # 附件PDF
    attaches = []
    for am in re.finditer(
        r'<a[^>]*href="([^"]+\.pdf)"[^>]*>([^<]+)</a>',
        html, re.I
    ):
        url = am.group(1).strip()
        name = am.group(2).strip()
        # 补全相对路径
        if url.startswith('../../'):
            url = urljoin("https://sthjj.czs.gov.cn/", url[5:])
        elif url.startswith('../'):
            url = urljoin("https://sthjj.czs.gov.cn/", url[2:])
        elif not url.startswith('http'):
            url = urljoin(detail_url, url)
        attaches.append(f'<a href="{url}" target="_blank">{name}</a>')

    if attaches:
        content += '<br><br><strong>附件：</strong><br>' + '<br>'.join(attaches)

    # 摘要
    summary = re.sub(r'<[^>]+>', '', content)
    summary = re.sub(r'\s+', ' ', summary).strip()[:500] if summary else title

    return {
        "title": title or "",
        "content": content,
        "date": date,
        "summary": summary,
        "url": detail_url,
    }

def main():
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("--full", action="store_true", help="全量")
    parser.add_argument("--sync", action="store_true", help="仅同步")
    parser.add_argument("--test", type=int, default=0, help="测试N条")
    # Handle bare numeric arg for _MAX_PG
    import sys as _SYS2
    _MAX_PG = int(_SYS2.argv[-1]) if len(_SYS2.argv) > 1 and _SYS2.argv[-1].isdigit() else None
    if _MAX_PG is not None:
        print('[AutoPg] max_pages=' + str(_MAX_PG))
        _SYS2.argv.pop()
    args = parser.parse_args()

    init_db()

    if args.sync:
        sync_to_server()
        return

    start_time = time.time()
    pw = sync_playwright()
    browser_obj = pw.__enter__()
    browser = browser_obj.chromium.launch(headless=True)
    context = browser.new_context(
        user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
        viewport={"width": 1920, "height": 1080}
    )
    page = context.new_page()

    try:
        total_new = 0
        max_pages = 100 if args.full else 1
        if args.test:
            max_pages = 100

        # 先获取第一页
        items, browser, context = fetch_list_page(1, page, browser, context)
        if items is None:
            print("⚠️ 无法访问站点")
            return
        print(f"📃 第1页 ({len(items)}条)")

        for item in items:
            if args.test and total_new >= args.test:
                break
            if _process_item(item, page, browser, context):
                total_new += 1
            time.sleep(1)

        # 后续页面
        if args.full or args.test:
            for page_num in range(2, max_pages + 1):
                if args.test and total_new >= args.test:
                    break
                items, browser, context = fetch_list_page(page_num, page, browser, context)
                if items is None or len(items) == 0:
                    break
                print(f"📃 第{page_num}页 ({len(items)}条)")
                for item in items:
                    if args.test and total_new >= args.test:
                        break
                    if _process_item(item, page, browser, context):
                        total_new += 1
                    time.sleep(1)

    finally:
        browser.close()
        pw.__exit__(None, None, None)

    elapsed = time.time() - start_time
    print(f"\n{'='*50}")
    print(f"🏁 新增:{stats['new']} 跳过:{stats['skip']} 错误:{stats['errors']}")
    print(f"⏱️ {elapsed:.0f}s")

    if stats["new"] > 0:
        sync_to_server()

def _process_item(item, page, browser=None, context=None):
    conn = sqlite3.connect(DB_PATH, timeout=60)
    exists = conn.execute("SELECT 1 FROM crawl_results WHERE url=?", (item["url"],)).fetchone()
    conn.close()
    if exists:
        stats["skip"] += 1
        return False  # not new

    print(f"  📄 {item['title'][:40]}...", end="", flush=True)
    detail = fetch_detail(item["url"], page, browser, context)

    if detail is None:
        print(" ✗ 详情失败")
        stats["errors"] += 1
        return False

    store_record(
        detail["title"] or item["title"],
        item["url"],
        detail["content"],
        detail.get("date") or item.get("date", ""),
        detail["summary"],
    )

    clen = len(detail.get("content", ""))
    print(f" ✅ {clen}B")
    return True  # new item crawled

if __name__ == "__main__":
    main()