#!/usr/bin/env python3
"""
yanshougs.com (工程建设验收公示网) 爬虫
=============================================
WAF 封 HTTP，需 Playwright 浏览器渲染。

数据：
  - 6 个类别：水保验收(2) 环保验收(3) 环评公示(4) 水保方案(30) 水保监测(31) 其它公示(29)
  - 详情页：标题/日期/正文/建设单位/建设地点/附件

继承 base_crawler 的 DB / 同步工具函数。

用法:
  python3 crawl_yanshougs.py --test=3   # 测试3条
  python3 crawl_yanshougs.py            # 增量爬（默认第1页）
  python3 crawl_yanshougs.py --full     # 全量爬所有类别所有页
"""

import os, re, sys, time, sqlite3, subprocess, base64, asyncio, traceback

BASE_DIR = os.path.dirname(os.path.abspath(__file__))
SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")

# ─── 从 base_crawler 复用函数 ───
sys.path.insert(0, BASE_DIR)
from base_crawler import clean_noisy_html, extract_plain_text, clean_content, init_db as _base_init_db

SITE_NAME = "工程建设验收公示网"
DOMAIN = "www.yanshougs.com"

# 类别映射
CATEGORIES = {
    2: "水保验收", 3: "环保验收", 4: "环评公示",
    30: "水保方案", 31: "水保监测", 29: "其它公示",
}

stats = {"new": 0, "skip": 0, "errors": 0}


def init_db():
    """确保search.db有gov_raw和gov_search表"""
    conn = sqlite3.connect(SEARCH_DB)
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute('''CREATE TABLE IF NOT EXISTS gov_raw (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        site_name TEXT, source_url TEXT, page_url TEXT,
        title TEXT, publish_date TEXT, date_rank INTEGER DEFAULT 0,
        summary TEXT, status TEXT, category TEXT DEFAULT '',
        visits INTEGER DEFAULT 0, content TEXT DEFAULT '', tags TEXT DEFAULT ''
    )''')
    try:
        conn.execute('''CREATE VIRTUAL TABLE IF NOT EXISTS gov_search USING fts5(
            title, content, site_name,
            content='gov_raw', content_rowid='id', tokenize='unicode61'
        )''')
    except sqlite3.OperationalError:
        pass
    conn.commit()
    conn.close()


def store_record(title, url, content, date, summary, builder, location, category, attachments):
    """存储一条记录到 search.db（去重）"""
    conn = sqlite3.connect(SEARCH_DB)
    attach_str = "\n".join(attachments) if attachments else ""
    try:
        conn.execute(
            "INSERT OR IGNORE INTO gov_raw "
            "(title, content, publish_date, page_url, source_url, site_name, summary, tags, category) "
            "VALUES (?,?,?,?,?,?,?,?,?)",
            (title, content, date, url, DOMAIN, SITE_NAME, summary, attach_str, category),
        )
        conn.commit()
        if conn.total_changes > 0:
            row_id = conn.execute("SELECT id FROM gov_raw WHERE page_url=?", (url,)).fetchone()
            if row_id:
                try:
                    conn.execute(
                        "INSERT INTO gov_search(rowid, title, content, site_name) VALUES (?,?,?,?)",
                        (row_id[0], title, content, SITE_NAME)
                    )
                except sqlite3.IntegrityError:
                    pass
            stats["new"] += 1
        else:
            stats["skip"] += 1
    except:
        stats["errors"] += 1
    finally:
        conn.close()



if __name__ == "__main__":
    # 注意: main() 需要实现 Playwright 爬取逻辑后使用
    init_db()
    print(f"{SITE_NAME} - 已配置直接写入 search.db")
    print("请使用 --full 或 --test 参数执行爬取（需实现 main() 函数）")