#!/usr/bin/env python3
"""
福建三钢（集团） — 环评公示 (fjsg.com.cn)
=======================================
列表: /sgweb/lists?topicId=116&page=N (N=1..11)
数据在页面内嵌JS变量 var cl = [...] 中，含完整内容
"""
import sys, os, re, json, time
from datetime import datetime, timezone, timedelta
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

SITE_NAME = "福建三钢-环评公示"
BASE_URL = "https://www.fjsg.com.cn"
LIST_URL = "https://www.fjsg.com.cn/sgweb/lists?topicId=116"
TOTAL_PAGES = 11
MAX_PAGES = 5
CUTOFF = (datetime.now(timezone.utc) - timedelta(days=365*3)).strftime("%Y-%m-%d")
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

def fetch(url):
    try:
        import requests, urllib3
        urllib3.disable_warnings()
        r = requests.get(url, headers=HEADERS, timeout=20, verify=False)
        r.encoding = 'utf-8'
        return r.text
    except:
        return None

def extract_items(html):
    """从JS变量 var cl = [...] 提取数据（深度匹配括号）"""
    start = html.find('var cl = ')
    if start < 0:
        return []
    start += len('var cl = ')
    depth = 0
    end = start
    for i in range(start, len(html)):
        if html[i] == '[':
            depth += 1
        elif html[i] == ']':
            depth -= 1
            if depth == 0:
                end = i + 1
                break
    try:
        items = json.loads(html[start:end])
    except:
        return []
    results = []
    for item in items:
        title = item.get("title", "").strip()
        content = item.get("content", "")
        date_str = (item.get("issue_date") or "")[:10]
        cont_id = item.get("cont_id")
        url = f"{BASE_URL}/sgweb/detail?contId={cont_id}"
        if title and content:
            results.append((title, url, content, date_str))
    return results

def run(args=None):
    incremental = False; test_mode = False
    if args:
        if '1' in args: incremental = True
        if 't' in args: test_mode = True
    
    print(f"爬取: {SITE_NAME}")
    all_items = []
    
    for pg in range(1, TOTAL_PAGES + 1):
        if pg > MAX_PAGES: break
        url = LIST_URL if pg == 1 else f"{LIST_URL}&page={pg}"
        
        html = fetch(url)
        if not html:
            print(f"  第{pg}页: 无法访问"); break
        
        items = extract_items(html)
        if not items:
            print(f"  第{pg}页: 0 条")
            if pg > 1: break
            continue
        
        print(f"  第{pg}页: {len(items)} 条")
        all_items.extend(items)
        if incremental or test_mode: break
    
    print(f"  共 {len(all_items)} 条列表项")
    
    results = []
    for i, (title, url, content, date_str) in enumerate(all_items):
        print(f"  [{i+1}/{len(all_items)}] {title[:40]}...", end=" ", flush=True)
        if date_str and date_str < CUTOFF: print("⏭ 超时范围"); continue
        
        # Clean content
        cleaned = re.sub(r'<script[^>]*>.*?</script>', '', content, flags=re.DOTALL|re.I)
        cleaned = re.sub(r'<style[^>]*>.*?</style>', '', cleaned, flags=re.DOTALL|re.I)
        cleaned = re.sub(r' style="[^"]*"', '', cleaned)
        cleaned = cleaned.strip()
        
        if len(cleaned) < 50: print("⚠ 无正文"); continue
        
        summary = re.sub(r'<[^>]+>', '', cleaned)[:200].strip()
        results.append({
            "site_name": SITE_NAME, "title": title,
            "url": url, "content": cleaned,
            "summary": summary, "pub_date": date_str,
            "category": "环评公示", "tags": "福建三钢",
        })
        print(f"✅ ({len(cleaned)}字)")
        time.sleep(0.2)
        if test_mode and len(results) >= 3: break
    
    if results:
        print(f"\n  入库 {len(results)} 条")
        if os.path.exists('/root/search.db'):
            import sqlite3
            conn = sqlite3.connect(os.getenv("SEARCH_DB", "/root/search.db"), timeout=60); cur = conn.cursor()
            new_count = 0
            for r in results:
                r['_type'] = 'gov'
                cur.execute("INSERT OR IGNORE INTO gov_raw (site_name, source_url, page_url, title, publish_date, summary, content, category, tags) VALUES (?,?,?,?,?,?,?,?,?)",
                    (r['site_name'], r.get('url',''), r.get('url',''), r['title'], r.get('pub_date',''), r.get('summary',''), r['content'], r.get('category',''), r.get('tags','')))
                if cur.lastrowid:
                    new_count += 1
                    cur.execute("INSERT OR IGNORE INTO gov_search (rowid, title, site_name, summary) VALUES (?,?,?,?)",
                        (cur.lastrowid, r['title'], r['site_name'], r.get('summary','')))
            conn.commit(); conn.close()
            print(f"  ✅ 本地入库 {new_count} 条 (FTS已同步)")
        else:
            push_to_searchdb(results, "fjsg_hjxp")
    print("完成")

if __name__ == "__main__":
    run(sys.argv[1] if len(sys.argv) > 1 else None)