#!/usr/bin/env python3
import os
"""
阜新市清河门区人民政府 — 通知公告 (fxqhm.gov.cn)
=========================================
列表: /channel/list/12424.html → /channel/list/12424_N.html (N=2..17)
详情: /content/YYYY/NNNNNN.html
正文: div.Detailcontent_inner.f16
标题: div.DetailcontentTile
日期: span.DetailcontentTime_item (日期：YYYY-MM-DD)
"""
import sys, os, re, time
from datetime import datetime, timezone, timedelta
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

SITE_NAME = "阜新市清河门区-通知公告"
BASE_URL = "https://www.fxqhm.gov.cn"
LIST_URL = "https://www.fxqhm.gov.cn/channel/list/12424.html"
TOTAL_PAGES = 17
MAX_PAGES = 5
CUTOFF = (datetime.now(timezone.utc) - timedelta(days=365*3)).strftime("%Y-%m-%d")
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

def fetch(url):
    try:
        import requests, urllib3
        urllib3.disable_warnings()
        r = requests.get(url, headers=HEADERS, timeout=20, verify=False)
        r.encoding = 'utf-8'
        return r.text
    except Exception as e:
        print(f"⚠ fetch error: {e}")
        return None

def parse_list(html):
    """解析列表页"""
    items = []
    # <a class="list1-ul-link" href="URL">
    #   <div class="list1-ul-link-inner">Title</div>
    #   <div class="tabs2Right-part2">YYYY-MM-DD</div>
    # </a>
    pattern = r'<a[^>]*class="list1-ul-link"[^>]*href="(https://www\.fxqhm\.gov\.cn/content/\d+/\d+\.html)"[^>]*>.*?<div[^>]*class="list1-ul-link-inner"[^>]*>(.*?)</div>.*?<div[^>]*class="tabs2Right-part2"[^>]*>\s*(\d{4}-\d{2}-\d{2})'
    for m in re.finditer(pattern, html, re.DOTALL):
        url, title, date_str = m.group(1), m.group(2).strip(), m.group(3).strip()
        items.append((title, url, date_str))
    return items

def fetch_detail(url):
    """提取详情页"""
    html = fetch(url)
    if not html:
        return None, None, None
    
    # Title
    title = ""
    m = re.search(r'class="DetailcontentTile"[^>]*>(.*?)</div>', html, re.DOTALL)
    if m:
        title = re.sub(r'<[^>]+>', '', m.group(1)).strip()
    
    # Date
    date = ""
    m = re.search(r'class="DetailcontentTime_item"[^>]*>\s*日期[：:]\s*(\d{4}-\d{1,2}-\d{1,2})', html)
    if m:
        date = m.group(1)
    
    # Content
    content = ""
    m = re.search(r'<div[^>]*class="Detailcontent_inner[^"]*"[^>]*>', html)
    if m:
        tag_start = m.end()
        remaining = html[m.start():]
        depth = 0
        i = 0
        while i < len(remaining):
            if remaining[i:i+4] == '<!--':
                e = remaining.find('-->', i+4)
                i = e+3 if e > i else i+1
                continue
            if remaining[i:i+6] == '</div>':
                if depth == 0:
                    content = remaining[m.end()-m.start():i]
                    break
                depth -= 1
                i += 6
                continue
            if i+1 < len(remaining) and remaining[i:i+1] == '<':
                next_tag = remaining[i+1:i+5]
                if next_tag in ('div ', 'div>', 'div\t'):
                    depth += 1
                    i += 4
                    continue
                elif remaining[i+1:i+4] in ('div', 'div\t'):
                    i += 4
                    continue
            i += 1
    
    if content:
        content = re.sub(r'<script[^>]*>.*?</script>', '', content, flags=re.DOTALL|re.I)
        content = re.sub(r'<style[^>]*>.*?</style>', '', content, flags=re.DOTALL|re.I)
        content = re.sub(r' style="[^"]*"', '', content)
        content = re.sub(r'<!--.*?-->', '', content)
        content = re.sub(r'\n\s*\n', '\n', content)
        content = content.strip()
    
    return title, content, date

def run(args=None):
    incremental = False
    test_mode = False
    if args:
        if '1' in args: incremental = True
        if 't' in args: test_mode = True
    
    print(f"爬取: {SITE_NAME}")
    
    all_items = []
    for pg in range(1, TOTAL_PAGES + 1):
        if pg > MAX_PAGES:
            break
        if pg == 1:
            url = LIST_URL
        else:
            url = f"{BASE_URL}/channel/list/12424_{pg}.html"
        
        html = fetch(url)
        if not html:
            print(f"  第{pg}页: 无法访问")
            break
        
        items = parse_list(html)
        if not items:
            print(f"  第{pg}页: 0 条")
            if pg > 1:
                break
            continue
        
        print(f"  第{pg}页: {len(items)} 条")
        all_items.extend(items)
        
        if incremental or test_mode:
            break
    
    print(f"  共 {len(all_items)} 条列表项")
    
    results = []
    for i, (title, url, date_str) in enumerate(all_items):
        print(f"  [{i+1}/{len(all_items)}] {title[:40]}...", end=" ", flush=True)
        
        if date_str and date_str < CUTOFF:
            print("⏭ 超时范围")
            continue
        
        dt, content, _ = fetch_detail(url)
        if not content or len(content) < 50:
            print("⚠ 无正文")
            continue
        
        summary = re.sub(r'<[^>]+>', '', content)[:200].strip()
        results.append({
            "site_name": SITE_NAME, "title": dt or title,
            "url": url, "content": content,
            "summary": summary,
            "pub_date": date_str,
            "category": "通知公告", "tags": "阜新,清河门",
        })
        print(f"✅ ({len(content)}字)")
        time.sleep(0.3)
        
        if test_mode and len(results) >= 3:
            break
    
    if results:
        print(f"\n  入库 {len(results)} 条")
        if os.path.exists('/root/search.db'):
            import sqlite3
            conn = sqlite3.connect(os.getenv("SEARCH_DB", "/root/search.db"), timeout=60)
            cur = conn.cursor()
            new_count = 0
            for r in results:
                r['_type'] = 'gov'
                cur.execute("""
                    INSERT OR IGNORE INTO gov_raw (site_name, source_url, page_url, title, publish_date, summary, content, category, tags)
                    VALUES (?,?,?,?,?,?,?,?,?)
                """, (
                    r['site_name'], r.get('url',''), r.get('url',''),
                    r['title'], r.get('pub_date',''), r.get('summary',''),
                    r['content'], r.get('category',''), r.get('tags','')
                ))
                if cur.lastrowid:
                    new_count += 1
                    rowid = cur.lastrowid
                    cur.execute("""
                        INSERT OR IGNORE INTO gov_search (rowid, title, site_name, summary)
                        VALUES (?,?,?,?,?)
                    """, (rowid, r['title'], r['site_name'], r.get('summary',''), r.get('content','')))
            conn.commit()
            conn.close()
            print(f"  ✅ 本地入库 {new_count} 条 (FTS已同步)")
        else:
            push_to_searchdb(results, "fxqhm_tzgg")
    print("完成")

if __name__ == "__main__":
    run(sys.argv[1] if len(sys.argv) > 1 else None)