#!/usr/bin/env python3
"""
颍州区马寨镇 — 回应关切 爬虫
site: www.yingzhou.gov.cn
栏目: /OpennessContent/showList/1232/102465/page_N.html
"""
import re, os, sqlite3, time, urllib.request, ssl

SITE_NAME = "颍州区马寨镇-回应关切"
DOMAIN = "www.yingzhou.gov.cn"
BASE = "https://www.yingzhou.gov.cn"
LIST_API = "/OpennessTarget/1232/102465/page_{page}.html"
PAGE_SIZE = 15

SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
CTX = ssl._create_unverified_context()
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/125.0.0.0 Safari/537.36"}

def http_get(url):
    req = urllib.request.Request(url, headers=HEADERS)
    try:
        resp = urllib.request.urlopen(req, timeout=30, context=CTX)
        return resp.read().decode("utf-8", errors="replace")
    except: return None

def parse_list(html):
    """解析列表页：<li><a href=... title=...>标题</a><span>日期</span></li>"""
    items = []
    for m in re.finditer(
        r'<a href="(/OpennessContent/show/\d+\.html)" title="([^"]*)"[^>]*>.*?</a>\s*<span>([^<]+)</span>',
        html
    ):
        href, title, date = m.group(1), m.group(2).strip(), m.group(3).strip()
        if title:
            items.append({
                "title": title,
                "url": BASE + href,
                "publish_date": date[:10]
            })
    return items

def get_total_pages(html):
    m = re.search(r'pagecount="(\d+)"', html)
    return int(m.group(1)) if m else 1

def fetch_detail(url):
    html = http_get(url)
    if not html: return None
    # 日期
    date = ""
    md = re.search(r'发布时间[：:]\s*(\d{4}-\d{2}-\d{2})', html)
    if md: date = md.group(1)
    # 来源
    source = ""
    ms = re.search(r'来源[：:]\s*([^<]+)', html)
    if ms: source = ms.group(1).strip()
    # 正文：div#zoom
    content = ""
    mc = re.search(r'id="zoom"[^>]*>(.*?)</div>\s*</div>', html, re.DOTALL)
    if mc:
        content = mc.group(1)
        content = re.sub(r'<script[^>]*>.*?</script>', '', content, flags=re.DOTALL|re.I)
        content = re.sub(r'<style[^>]*>.*?</style>', '', content, flags=re.DOTALL|re.I)
        content = content.strip()
    return {"title": "", "content": content, "date": date, "source": source}

def init_db():
    conn = sqlite3.connect(SEARCH_DB, timeout=60)
    conn.execute('''CREATE TABLE IF NOT EXISTS gov_raw (
        id INTEGER PRIMARY KEY AUTOINCREMENT, site_name TEXT, source_url TEXT, page_url TEXT,
        title TEXT, publish_date TEXT, date_rank INTEGER DEFAULT 0,
        summary TEXT, status TEXT, category TEXT DEFAULT '',
        visits INTEGER DEFAULT 0, content TEXT DEFAULT '', tags TEXT DEFAULT ''
    )''')
    try:
        conn.execute('''CREATE VIRTUAL TABLE IF NOT EXISTS gov_search USING fts5(
            title, content, site_name,
            content='gov_raw', content_rowid='id', tokenize='unicode61'
        )''')
    except sqlite3.OperationalError: pass
    conn.commit(); conn.close()

def main():
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("--full", action="store_true")
    parser.add_argument("--pages", type=int, default=1)
    args = parser.parse_args()
    init_db()
    conn = sqlite3.connect(SEARCH_DB, timeout=60); cur = conn.cursor()
    
    # 第1页获取总页数
    html = http_get(BASE + LIST_API.format(page=1))
    if not html: print("✗ HTTP失败"); return
    items = parse_list(html)
    total_pages = get_total_pages(html)
    pages_to_crawl = total_pages if args.full else args.pages
    mode = "全量" if args.full else f"增量({pages_to_crawl}页)"
    total = 0
    print(f"🔍 {SITE_NAME} — {mode} ({total_pages}页)")
    new_total = skip_total = 0
    for page in range(1, pages_to_crawl + 1):
        if page > 1:
            html = http_get(BASE + LIST_API.format(page=page))
            if not html: continue
            items = parse_list(html)
        if not items:
            print(f"  [p{page:2d}/{pages_to_crawl}] ✗ 无数据"); continue
        page_new = page_skip = 0
        for item in items:
            if cur.execute("SELECT 1 FROM gov_raw WHERE page_url=? AND site_name=?", (item["url"], SITE_NAME)).fetchone():
                page_skip += 1; skip_total += 1; continue
            detail = fetch_detail(item["url"])
            if detail:
                content = detail.get("content", "")
                title = item["title"]
                date = detail.get("date", item.get("publish_date", ""))
                source = detail.get("source", "")
            else:
                content = ""; title = item["title"]; date = item.get("publish_date", ""); source = ""
            if not content:
                page_skip += 1; skip_total += 1; continue
            tags = f"来源：{source}" if source else ""
            try:
                cur.execute(
                    "INSERT OR IGNORE INTO gov_raw (title, content, publish_date, page_url, source_url, site_name, tags) VALUES (?,?,?,?,?,?,?)",
                    (title, content, date, item["url"], DOMAIN, SITE_NAME, tags))
                if cur.rowcount > 0:
                    row_id = cur.lastrowid
                    try: cur.execute("INSERT OR REPLACE INTO gov_search(rowid, title, site_name) VALUES (?,?,?,?)", (row_id, title, content, SITE_NAME))
                    except sqlite3.IntegrityError: pass
                    page_new += 1; new_total += 1
            except: pass
        conn.commit()
        print(f"  [p{page:2d}/{pages_to_crawl}] ✓ {len(items)}条 (新增{page_new} 跳过{page_skip})")
        time.sleep(0.3)
    conn.close()
    print(f"\n📊 完成！新增: {new_total} | 跳过: {skip_total}")

if __name__ == "__main__": main()