#!/usr/bin/env python3
"""
德昌县人民政府 - 环境影响评价 爬虫
URL: http://www.lsdc.gov.cn/ztzl_85/zdlyxxgkzt/hjbh/hjyxpj/
CMS: TRS
"""
import re, urllib.request, urllib.parse, os, time
from urllib.parse import urljoin

BASE_URL = "http://www.lsdc.gov.cn"
LIST_URL = BASE_URL + "/ztzl_85/zdlyxxgkzt/hjbh/hjyxpj/"
SITE_NAME = "德昌县人民政府-环境影响评价"
HEADERS = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'}

SEARCH_DB = os.getenv("SEARCH_DB", "/root/temp_search.db")

def ensure_db():
    import sqlite3
    conn = sqlite3.connect(SEARCH_DB)
    conn.execute("""
        CREATE TABLE IF NOT EXISTS gov_raw (
            id INTEGER PRIMARY KEY,
            title TEXT,
            content TEXT,
            publish_date TEXT,
            page_url TEXT UNIQUE,
            source_url TEXT,
            site_name TEXT DEFAULT '',
            category TEXT DEFAULT ''
        )
    """)
    conn.commit()
    return conn

def save_item(conn, title, content, pub_date, url):
    conn.execute(
        "INSERT OR IGNORE INTO gov_raw (title, content, publish_date, page_url, source_url, site_name) VALUES (?, ?, ?, ?, ?, ?)",
        (title, content, pub_date, url, LIST_URL, SITE_NAME)
    )
    conn.commit()

def fetch_page(page_no):
    """获取列表页"""
    if page_no == 1:
        url = LIST_URL
    else:
        url = f"{LIST_URL}index_{page_no - 1}.html"
    
    req = urllib.request.Request(url, headers=HEADERS)
    resp = urllib.request.urlopen(req, timeout=30)
    html = resp.read().decode('utf-8', errors='replace')
    
    # 提取列表项
    items = []
    pattern = re.compile(
        r'<li><a href="([^"]+)"[^>]*>\s*([^<]+)\s*</a><span>\s*(\d{4}-\d{2}-\d{2})'
    )
    for m in pattern.finditer(html):
        href = m.group(1)
        title = m.group(2).strip()
        date = m.group(3)
        full_url = urljoin(LIST_URL, href)
        items.append((title, date, full_url))
    
    # 获取分页信息
    pg_m = re.search(r"createPageHTML\((\d+)", html)
    total_pages = int(pg_m.group(1)) if pg_m else 0
    
    return items, total_pages

def fetch_detail(url):
    """获取详情页"""
    req = urllib.request.Request(url, headers=HEADERS)
    resp = urllib.request.urlopen(req, timeout=30)
    html = resp.read().decode('utf-8', errors='replace')
    
    # 标题
    title_m = re.search(r'<p class="xl-title">(.*?)</p>', html, re.DOTALL)
    title = title_m.group(1).strip() if title_m else ''
    
    # 日期
    date_m = re.search(r'<span id="rq">.*?(\d{4}-\d{2}-\d{2})', html)
    date = date_m.group(1) if date_m else ''
    
    # 正文 - TRS_UEDITOR
    content_m = re.search(
        r'<div class="trs_editor_view TRS_UEDITOR[^"]*"[^>]*>(.*?)</div>\s*</div>',
        html, re.DOTALL
    )
    content = content_m.group(1).strip() if content_m else ''
    
    return title, date, content

def main():
    conn = ensure_db()
    
    first_items, total_pages = fetch_page(1)
    print(f"Total pages: {total_pages}")
    
    all_items = []
    for pn in range(1, total_pages + 1):
        if pn == 1:
            items = first_items
        else:
            items, _ = fetch_page(pn)
        print(f"Page {pn}: {len(items)} items")
        all_items.extend(items)
        time.sleep(0.3)
    
    print(f"\nTotal items from list: {len(all_items)}")
    
    for idx, (title, date, url) in enumerate(all_items, 1):
        try:
            det_title, det_date, content = fetch_detail(url)
            use_title = title or det_title
            use_date = date or det_date
            
            text = re.sub(r'<[^>]+>', '', content).strip() if content else ''
            content_ok = len(text) >= 50
            save_item(conn, use_title, content, use_date, url)
            status = "OK" if content_ok else "SHORT"
            print(f"  [{idx}/{len(all_items)}] {status} {use_date} {use_title[:60]}")
        except Exception as e:
            print(f"  [{idx}/{len(all_items)}] ERROR {url}: {e}")
        
        time.sleep(0.5)
    
    cur = conn.execute("SELECT COUNT(*) FROM gov_raw WHERE site_name=? AND content IS NOT NULL AND length(content)>50", (SITE_NAME,))
    ok_count = cur.fetchone()[0]
    total_count = conn.execute("SELECT COUNT(*) FROM gov_raw WHERE site_name=?", (SITE_NAME,)).fetchone()[0]
    
    print(f"\n{'='*50}")
    print(f"完成！总计: {total_count} 条, 有内容: {ok_count} 条")
    
    conn.close()

if __name__ == '__main__':
    main()