#!/usr/bin/env python3
"""濮阳市生态环境局 - 环评受理公示 (sthjj.puyang.gov.cn)
TRS CMS, static pagination: /channel/list/15513_{page}.html
36 pages × 13 items = 465 records
Detail: /content/YYYY/NNNN.html
Title: h3.title, Date: span.fbsj, Content: div.detail-info
"""

import requests, re, sys, os, sqlite3, time, json
from datetime import datetime, timedelta

BASE_URL = "https://sthjj.puyang.gov.cn"
LIST_URL = BASE_URL + "/channel/list/15513.html"
SITE_NAME = "濮阳市生态环境局"
GROUP = "河南"
INDUSTRY = "环评公示"
SCRIPT_NAME = "crawl_puyang_hj.py"

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}
TIMEOUT = 20
session = requests.Session()
session.headers.update(HEADERS)

SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
CUTOFF = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")

def fetch(url):
    resp = session.get(url, timeout=TIMEOUT, verify=False)
    resp.encoding = "utf-8"
    return resp.text

def parse_list(html):
    """Parse list page — div.list-wrap > ul.list-item > li > a"""
    items = []
    pattern = r'<li>\s*<a[^>]*href="([^"]*)"[^>]*>\s*<span[^>]*class="title[^"]*"[^>]*>([^<]*)</span>\s*<span[^>]*class="date[^"]*"[^>]*>([^<]*)</span>'
    for m in re.finditer(pattern, html, re.DOTALL):
        href, title, date = m.group(1), m.group(2).strip(), m.group(3).strip()
        if not href.startswith("http"):
            href = BASE_URL + href if href.startswith("/") else BASE_URL + "/" + href
        items.append((title, href, date))
    return items

def parse_detail(html, url):
    """Parse detail page — title from h3.title, date from span.fbsj, content from div.detail-info"""
    # Title
    title = ""
    m = re.search(r'<h3[^>]*class="title"[^>]*>(.*?)</h3>', html, re.DOTALL)
    if m:
        title = re.sub(r'<[^>]+>', '', m.group(1)).strip()
    if not title:
        m = re.search(r'<title>(.*?)</title>', html, re.DOTALL)
        if m:
            title = m.group(1).replace("-濮阳市生态环境局", "").strip()

    # Date
    date_str = ""
    m = re.search(r'<span[^>]*class="fbsj"[^>]*>发表时间[：:]\s*(\d{4}[-/]\d{1,2}[-/]\d{1,2})', html)
    if m:
        date_str = m.group(1).replace("/", "-")

    # Content from div.detail-info
    content = ""
    m = re.search(r'<div[^>]*class="detail-info"[^>]*>(.*?)</div>\s*</div>', html, re.DOTALL)
    if m:
        content = m.group(1)
        content = re.sub(r'<script[^>]*>.*?</script>', '', content, flags=re.DOTALL)
        content = re.sub(r'<style[^>]*>.*?</style>', '', content, flags=re.DOTALL)
        content = content.strip()

    if not content or len(content) < 100:
        # Fallback: zoom
        m = re.search(r'<div[^>]*(?:id|class)=["\']zoom["\'](.*?)</div>\s*</div>', html, re.DOTALL)
        if m:
            content = m.group(1)
            content = re.sub(r'<script[^>]*>.*?</script>', '', content, flags=re.DOTALL)
            content = re.sub(r'<style[^>]*>.*?</style>', '', content, flags=re.DOTALL)
            content = content.strip()

    return title, date_str, content

def push_to_searchdb(items, label="puyang_hj"):
    """Push items to search.db"""
    try:
        import subprocess
        for title, url, date, site, content, group, ind, script in items:
            sql = f"""INSERT OR IGNORE INTO gov_raw (source_url, page_url, title, publish_date, site_name, content, group_name, industry, script_name)
VALUES ('{url}', '{url}', '{title.replace(chr(39), chr(39)*2)}', '{date}', '{site}', '{content.replace(chr(39), chr(39)*2)}', '{group}', '{ind}', '{script}');"""
            p = subprocess.run(["sqlite3", "-cmd", ".timeout 60000", SEARCH_DB], input=sql.encode("utf-8"), capture_output=True, timeout=10)
            if p.returncode != 0:
                print(f"[push] SQL error: {p.stderr.decode()[:100]}")
    except Exception as e:
        print(f"[push] Error: {e}")

def crawl(pages=5):
    conn = sqlite3.connect(SEARCH_DB, timeout=60)
    cursor = conn.cursor()
    
    all_count = 0
    skip_count = 0
    
    for page in range(1, pages + 1):
        url = LIST_URL if page == 1 else BASE_URL + f"/channel/list/15513_{page}.html"
        print(f"[{SCRIPT_NAME}] Page {page}/{pages}: {url}")
        
        html = fetch(url)
        items = parse_list(html)
        if not items:
            print(f"[{SCRIPT_NAME}] No items on page {page}, stopping")
            break
        
        print(f"[{SCRIPT_NAME}] Found {len(items)} items")
        
        for title, item_url, date_str in items:
            existing = cursor.execute(
                "SELECT id FROM gov_raw WHERE page_url = ?", (item_url,)
            ).fetchone()
            if existing:
                skip_count += 1
                continue
            
            try:
                detail_html = fetch(item_url)
            except Exception as e:
                print(f"[{SCRIPT_NAME}] Fetch detail failed: {item_url} - {e}")
                skip_count += 1
                continue
            
            detail_title, detail_date, content = parse_detail(detail_html, item_url)
            if not detail_title:
                detail_title = title
            if not detail_date:
                detail_date = date_str
            if not content or len(content.strip()) < 50:
                content = "正文为空"
            
            try:
                cursor.execute("""
                    INSERT INTO gov_raw (source_url, page_url, title, publish_date, site_name, content, group_name, industry)
                    VALUES (?, ?, ?, ?, ?, ?, ?, ?)
                """, (item_url, item_url, detail_title, detail_date, SITE_NAME, content, GROUP, INDUSTRY))
                conn.commit()
                all_count += 1
                print(f"[{SCRIPT_NAME}] +{all_count}: {detail_title[:40]} ({detail_date})")
            except Exception as e:
                print(f"[{SCRIPT_NAME}] Insert error: {e}")
                conn.rollback()
                skip_count += 1
            
            time.sleep(0.5)
        
        time.sleep(0.5)
    
    conn.close()
    print(f"\n[{SCRIPT_NAME}] Done. New: {all_count}, Skipped: {skip_count}")
    return all_count

if __name__ == "__main__":
    pages = 5
    if len(sys.argv) > 1 and sys.argv[1].startswith("--pages="):
        pages = int(sys.argv[1].split("=")[1])
    elif len(sys.argv) > 2 and sys.argv[1] == "--pages":
        pages = int(sys.argv[2])
    crawl(pages=pages)