#!/usr/bin/env python3
"""
南丰县人民政府 - 环境保护(col27107)
http://www.jxnf.gov.cn/col/col27107/index.html?number=C60000C60004C60001
Hanweb CMS
"""
import sys, os, re, time, json, sqlite3
from datetime import datetime, timezone, timedelta
import requests, urllib3
import os
urllib3.disable_warnings()

SITE_NAME = "南丰环境保护"
BASE_URL = "http://www.jxnf.gov.cn"
LIST_URL = "http://www.jxnf.gov.cn/col/col27107/index.html?number=C60000C60004C60001"
SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
MAX_PAGES = 5
CUTOFF = (datetime.now(timezone.utc) - timedelta(days=365*3)).strftime("%Y-%m-%d")
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/125.0.0.0"}

def fetch(url):
    try:
        r = requests.get(url, headers=HEADERS, timeout=20, verify=False)
        r.encoding = 'utf-8'
        return r.text
    except:
        return None

def parse_list(html):
    items = []
    for m in re.finditer(r'<a[^>]*href=\"(/art/\d+/\d+/\d+/art_\d+_\d+\.html)\"[^>]*title=\"([^\"]*)\"', html):
        href = BASE_URL + m.group(1)
        title = m.group(2).strip()
        # 向前找日期
        ctx = html[max(0, html.find(m.group(1))-400):html.find(m.group(1))+100]
        dm = re.search(r'(\d{4}-\d{2}-\d{2})', ctx)
        pub_date = dm.group(1) if dm else ""
        if title:
            items.append((title, href, pub_date))
    return items

def get_page_url(page):
    return LIST_URL.replace('index.html', f'index_{page}.html') if page > 1 else LIST_URL

def fetch_detail(url):
    html = fetch(url)
    if not html:
        return "", "", ""
    title = ""
    m = re.search(r'<meta[^>]*name="ArticleTitle"[^>]*content="([^\"]+)"', html)
    if m:
        title = m.group(1)
    if not title:
        m = re.search(r'<h1[^>]*>(.*?)</h1>', html, re.S)
        if m:
            title = re.sub(r'<[^>]+>', '', m.group(1)).strip()
    pub_date = ""
    m = re.search(r'<meta[^>]*name="PubDate"[^>]*content="(\d{4}-\d{1,2}-\d{1,2})', html)
    if m:
        pub_date = m.group(1)
    if not pub_date:
        m = re.search(r'(\d{4}-\d{2}-\d{2})\s*\d{2}:\d{2}', html)
        if m:
            pub_date = m.group(1)
    content = ""
    for sel in [r'<!--ZJEG_RSS\.content\.begin-->(.*?)<!--ZJEG_RSS\.content\.end-->',
                r'<meta name="ContentStart">(.*?)<meta name="ContentEnd">',
                r'<div[^>]*class="[^"]*content[^"]*"[^>]*>(.*?)</div>',
                r'<div[^>]*class="[^"]*article[^"]*"[^>]*>(.*?)</div>',
                r'<div[^>]*id="[^"]*zoom[^"]*"[^>]*>(.*?)</div>']:
        m = re.search(sel, html, re.S)
        if m and len(m.group(1)) > 100:
            content = m.group(1).strip()
            break
    if content:
        content = re.sub(r'<script[^>]*>.*?</script>', '', content, flags=re.S|re.I)
        content = re.sub(r'<style[^>]*>.*?</style>', '', content, flags=re.S|re.I)
    return title, pub_date, content

def insert_to_db(items):
    if not items:
        return
    db = sqlite3.connect(SEARCH_DB)
    db.execute("PRAGMA journal_mode=WAL")
    db.execute("PRAGMA synchronous=NORMAL")
    ok, skip = 0, 0
    for item in items:
        try:
            db.execute("INSERT OR IGNORE INTO gov_raw "
                "(site_name, source_url, page_url, title, publish_date, summary, content, status, category, tags) "
                "VALUES (?,?,?,?,?,?,?,?,?,?)", (
                item.get("site_name","")[:200], item.get("url",""),
                item.get("url",""), (item.get("title") or "")[:500],
                (item.get("pub_date") or "")[:10],
                (item.get("summary") or "")[:500],
                item.get("content",""), "active", "",
                (item.get("tags") or "")[:100],
            ))
            if db.total_changes > 0: ok += 1
            else: skip += 1
        except: skip += 1
    db.commit()
    site_name = items[0].get("site_name", "")
    db.execute("INSERT INTO gov_search(rowid, title, site_name, summary) "
        "SELECT r.id, r.title, r.site_name, r.summary FROM gov_raw r "
        "WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?", (site_name,))
    db.commit()
    db.close()
    print(f"  💾 入库: 新增{ok}, 跳过{skip}")

def main():
    incremental = len(sys.argv) > 1 and sys.argv[1] in ('1', '--incremental', '-i')
    limit = None
    if '--limit' in sys.argv:
        idx = sys.argv.index('--limit')
        limit = int(sys.argv[idx+1])
    print(f"\n爬取: {SITE_NAME}" + (" [增量]" if incremental else ""))
    results = []
    pages = 1 if incremental else MAX_PAGES
    for page in range(1, pages+1):
        url = get_page_url(page)
        html = fetch(url)
        if not html:
            print(f"  第{page}页: ❌ 请求失败")
            break
        items = parse_list(html)
        if not items:
            print(f"  第{page}页: 0 条")
            break
        print(f"  第{page}页: {len(items)} 条")
        for title, url, list_date in items:
            if limit and len(results) >= limit: break
            if list_date and list_date < CUTOFF: continue
            print(f"  {title[:45]}...", end=" ", flush=True)
            dt, date, content = fetch_detail(url)
            if not content or len(content) < 50:
                print("无正文")
                continue
            final_title = dt or title
            final_date = date or list_date
            summary = re.sub(r'<[^>]+>', '', content)[:200].strip()
            results.append({"site_name": SITE_NAME, "title": final_title,
                "url": url, "content": content, "summary": summary,
                "pub_date": final_date, "tags": SITE_NAME})
            print(f"OK ({len(content)}字)")
            time.sleep(0.3)
        if limit and len(results) >= limit: break

    if results:
        insert_to_db(results)
    print(f"\n✅ 完成! 共 {len(results)} 条")

if __name__ == "__main__":
    t0 = time.time()
    main()
    print(f"⏱ {time.time()-t0:.1f}s")