#!/usr/bin/env python3
"""
吕梁市生态环境局-公示公告 爬虫
CMS: TRS
列表: http://www.lvliang.gov.cn/llxxgk/zfxxgk/xxgkml/gbmwj/hbj/fdzdgknr_55371/gggs_55380/
分页: index_N.html (N>=2, 首页无后缀)
详情: ./YYYYMM/tYYYYMMDD_NNNNNNN.html
每页: 20条
"""
import os, re, sys, json
from urllib import request
from urllib.parse import urljoin
import ssl

SITE_NAME = "吕梁市生态环境局-公示公告"
BASE_URL = "http://www.lvliang.gov.cn/llxxgk/zfxxgk/xxgkml/gbmwj/hbj/fdzdgknr_55371/gggs_55380/"
DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9",
}

ssl._create_default_https_context = ssl._create_unverified_context

def fetch(url, timeout=20):
    req = request.Request(url, headers=HEADERS)
    try:
        resp = request.urlopen(req, timeout=timeout)
        return resp.read().decode('utf-8', errors='ignore')
    except Exception as e:
        print(f"  [ERROR] {url}: {e}", file=sys.stderr)
        return None

def parse_list(html, base):
    """Extract article links from list page."""
    items = []
    # Find all article links (relative ./202607/... paths)
    links = re.findall(r'<a[^>]*href="(\./\d{6}/[^"]+)"[^>]*>(.*?)</a>', html, re.DOTALL)
    for href, text in links:
        title = re.sub(r'<[^>]+>', '', text).strip()
        if not title or len(title) < 5:
            continue
        url = urljoin(base, href)
        items.append((title, url))
    return items

def parse_detail(html, url):
    """Extract content, date, attachments from detail page."""
    result = {"content": "", "date": "", "attachments": []}
    
    # Title from h2
    m = re.search(r'class="t"[^>]*>.*?<h2>(.*?)</h2>', html, re.DOTALL)
    if m:
        result["title"] = re.sub(r'<[^>]+>', '', m.group(1)).strip()
    
    # Date from metadata div
    m = re.search(r'更新时间[：:]\s*(\d{4}[-年]\d{1,2}[-月]\d{1,2})', html)
    if m:
        result["date"] = m.group(1).replace('年', '-').replace('月', '-').replace('日', '').strip()
    
    # Content from .n div
    m = re.search(r'class="n"[^>]*>(.*?)</div>', html, re.DOTALL)
    if m:
        content_html = m.group(1)
        # Extract text with paragraph breaks
        parts = []
        for p in re.findall(r'<p[^>]*>(.*?)</p>', content_html, re.DOTALL):
            text = re.sub(r'<[^>]+>', '', p).strip()
            if text:
                parts.append(text)
        # Extract tables as HTML (keep original structure)
        for tm in re.finditer(r'<table[^>]*>.*?</table>', content_html, re.DOTALL):
            parts.append(tm.group(0))
        result["content"] = '\n\n'.join(parts)
    
    # Attachments
    for href, text in re.findall(r'<a[^>]*href="([^"]+\.(?:pdf|doc|docx|xls|xlsx|zip|rar))"[^>]*>([^<]+)</a>', html, re.DOTALL):
        full_url = urljoin(url, href)
        result["attachments"].append({"title": text.strip(), "url": full_url})
    
    return result

def crawl(max_pages=5):
    """Main crawl function."""
    import sqlite3
    conn = sqlite3.connect(DB_PATH)
    cur = conn.cursor()
    
    new_count = 0
    skip_count = 0
    
    for page in range(1, max_pages + 1):
        if page == 1:
            list_url = BASE_URL
        else:
            list_url = BASE_URL.rstrip('/') + f'/index_{page}.html'
        
        print(f"[分页] 第{page}页: {list_url}", flush=True)
        html = fetch(list_url)
        if not html:
            print(f"  [失败] 第{page}页无法获取", flush=True)
            break
        
        items = parse_list(html, list_url)
        if not items:
            print(f"  [完成] 第{page}页无更多数据", flush=True)
            break
        
        print(f"  找到 {len(items)} 条", flush=True)
        
        for title, url in items:
            # Check if exists
            cur.execute("SELECT id FROM gov_raw WHERE page_url=? AND site_name=?", (url, SITE_NAME))
            if cur.fetchone():
                skip_count += 1
                continue
            
            # Fetch detail
            detail_html = fetch(url)
            if not detail_html:
                print(f"  [跳过] 详情页失败: {title[:30]}", flush=True)
                skip_count += 1
                continue
            
            detail = parse_detail(detail_html, url)
            content = detail.get("content", "") or ""
            date = detail.get("date", "") or ""
            attachments = json.dumps(detail.get("attachments", []), ensure_ascii=False)
            
            cur.execute(
                """INSERT OR IGNORE INTO gov_raw 
                   (site_name, page_url, title, content, publish_date, attachments, summary)
                   VALUES (?, ?, ?, ?, ?, ?, ?)""",
                (SITE_NAME, url, title, content, date, attachments, content[:200] if content else "")
            )
            if cur.rowcount > 0:
                new_count += 1
                print(f"  [新增] {title[:40]}", flush=True)
            else:
                skip_count += 1
        
        conn.commit()
    
    conn.close()
    print(f"\n[DONE] {SITE_NAME}: 新增={new_count}, 跳过={skip_count}", flush=True)

if __name__ == "__main__":
    max_p = int(sys.argv[1]) if len(sys.argv) > 1 else 5
    crawl(max_p)