#!/usr/bin/env python3
"""
连云港市人民政府-公示通告 爬虫
CMS: TrueCMS (jQuery jpage AJAX, 但所有数据在初始HTML中)
列表: http://www.lyg.gov.cn/zglygzfmhwz/gsgg/gsgg.html (单页全量39条)
详情: /zglygzfmhwz/gsgg/content/UUID.html
"""
import os, re, sys, json
from urllib import request
from urllib.parse import urljoin
import ssl

SITE_NAME = "连云港市人民政府-公示通告"
LIST_URL = "http://www.lyg.gov.cn/zglygzfmhwz/gsgg/gsgg.html"
BASE_URL = "http://www.lyg.gov.cn"
DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9",
}

ssl._create_default_https_context = ssl._create_unverified_context

def fetch(url, timeout=20):
    req = request.Request(url, headers=HEADERS)
    try:
        resp = request.urlopen(req, timeout=timeout)
        return resp.read().decode('utf-8', errors='ignore')
    except Exception as e:
        print(f"  [ERROR] {url}: {e}", file=sys.stderr)
        return None

def parse_list(html):
    """Extract article links and dates from list page."""
    items = []
    for m in re.finditer(r'<li[^>]*>(.*?)</li>', html, re.DOTALL):
        li = m.group(1)
        if 'gsgg/content' not in li:
            continue
        
        a = re.search(r'<a[^>]*href="([^"]+)"[^>]*>(.*?)</a>', li, re.DOTALL)
        if not a:
            continue
        
        href = a.group(1)
        title = re.sub(r'<[^>]+>', '', a.group(2)).strip()
        
        # Date from the same li
        date = ''
        dm = re.search(r'(\d{4}-\d{2}-\d{2})', li)
        if dm:
            date = dm.group(1)
        
        full_url = urljoin(BASE_URL, href)
        items.append((title, full_url, date))
    
    return items

def parse_detail(html, url):
    """Extract content, date, attachments from detail page."""
    content = ""
    attachments = []
    
    # Content from zoom div
    m = re.search(r'<div[^>]*id="zoom"[^>]*>(.*?)</div>\s*<!--', html, re.DOTALL)
    if not m:
        m = re.search(r'<div[^>]*id="zoom"[^>]*>(.*?)</div>', html, re.DOTALL)
    if m:
        inner = m.group(1)
        # Extract paragraphs
        parts = []
        for p in re.findall(r'<p[^>]*>(.*?)</p>', inner, re.DOTALL):
            text = re.sub(r'<[^>]+>', '', p).strip()
            if text:
                parts.append(text)
        content = '\n\n'.join(parts)
    
    # Attachments in detail page
    for a_href, a_text in re.findall(r'<a[^>]*href="([^"]+\.(?:doc|docx|pdf|xls|xlsx))"[^>]*title="([^"]*)"', html):
        if len(a_text.strip()) < 3:
            continue
        full_url = urljoin(url, a_href)
        attachments.append({"title": a_text.strip(), "url": full_url})
    
    # Also attachmentController links
    for a_href, a_text in re.findall(r'<a[^>]*href="([^"]*(?:attachmentController|download)[^"]+)"[^>]*title="([^"]*)"', html):
        if len(a_text.strip()) < 3:
            continue
        full_url = urljoin(url, a_href)
        attachments.append({"title": a_text.strip(), "url": full_url})
    
    return content, attachments

def crawl():
    """Main crawl function."""
    import sqlite3
    conn = sqlite3.connect(DB_PATH, timeout=60)
    cur = conn.cursor()
    
    print(f"[列表页] {LIST_URL}", flush=True)
    html = fetch(LIST_URL)
    if not html:
        print("[失败] 无法获取列表页", flush=True)
        return
    
    items = parse_list(html)
    print(f"  找到 {len(items)} 条", flush=True)
    
    new_count = 0
    skip_count = 0
    
    for title, url, date in items:
        cur.execute("SELECT id FROM gov_raw WHERE page_url=? AND site_name=?", (url, SITE_NAME))
        if cur.fetchone():
            skip_count += 1
            continue
        
        detail_html = fetch(url)
        if not detail_html:
            print(f"  [跳过] 详情页失败: {title[:30]}", flush=True)
            skip_count += 1
            continue
        
        content, attachments = parse_detail(detail_html, url)
        attachments_json = json.dumps(attachments, ensure_ascii=False)
        
        cur.execute(
            """INSERT OR IGNORE INTO gov_raw 
               (site_name, page_url, title, content, publish_date, attachments, summary)
               VALUES (?, ?, ?, ?, ?, ?, ?)""",
            (SITE_NAME, url, title, content, date, attachments_json, content[:200] if content else "")
        )
        if cur.rowcount > 0:
            new_count += 1
            print(f"  [新增] {title[:40]} [{date}]", flush=True)
        else:
            skip_count += 1
    
    conn.commit()
    conn.close()
    print(f"\n[DONE] {SITE_NAME}: 新增={new_count}, 跳过={skip_count}", flush=True)

if __name__ == "__main__":
    crawl()