#!/usr/bin/env python3
import os
"""
南平市生态环境局 — 环评审批受理公告 爬虫
YLCMS 架构，静态分页，详情页正文在 <DIV class="content" id="Content">
"""
import requests, sqlite3, re, os, sys, time, json
from datetime import datetime, timedelta

DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
BASE = "https://hbj.np.gov.cn"
LIST_URL = "/cms/sitemanage/index.shtml?siteId=100433125044680000&page={}"
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
    "Referer": BASE + "/cms/html/npshjbhj/hpspslgg/index.html",
}
CUTOFF_DATE = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")

session = requests.Session()
session.headers.update(HEADERS)


def parse_list(page):
    """解析列表页，返回 [(url, title, date_str), ...]"""
    try:
        r = session.get(BASE + LIST_URL.format(page), timeout=20)
        r.encoding = "utf-8"
    except Exception as e:
        print(f"  [ERROR] Page {page} fetch failed: {e}")
        return []
    
    items = []
    # <LI><SPAN class="list-content"><A href="...">TITLE</A></SPAN><SPAN class="list-time">DATE</SPAN></LI>
    pattern = re.compile(
        r'<A\s+href="(/cms/html/npshjbhj/\d{4}-\d{2}-\d{2}/\d+\.html)"[^>]*>([^<]*)</A>'
        r'.*?<SPAN\s+class="list-time">(\d{4}-\d{2}-\d{2})</SPAN>',
        re.DOTALL
    )
    for m in pattern.finditer(r.text):
        url, title, date_str = m.group(1), m.group(2).strip(), m.group(3)
        # Filter: skip non-article links (like "联系我们")
        if "联系" in title or not title:
            continue
        # Normalize &nbsp;
        title = title.replace("\u00a0", " ").replace("&nbsp;", " ").strip()
        # Trim very long titles
        if len(title) > 300:
            title = title[:300]
        items.append((url, title, date_str))
    return items


def parse_detail(url):
    """解析详情页，返回 (title, pub_date, content_html)"""
    try:
        r = session.get(BASE + url, timeout=20)
        r.encoding = "utf-8"
    except Exception as e:
        print(f"  [ERROR] Detail {url} failed: {e}")
        return None, None, None
    
    html = r.text
    
    # Title from meta
    m = re.search(r'content="([^"]*)"\s+name="ArticleTitle"', html, re.I)
    if not m:
        m = re.search(r'name="ArticleTitle"\s+content="([^"]*)"', html, re.I)
    title = m.group(1).strip() if m else ""
    
    # Date from meta - uses em-dash "—" not "-"
    m = re.search(r'content="([^"]*)"\s+name="PubDate"', html, re.I)
    if not m:
        m = re.search(r'name="PubDate"\s+content="([^"]*)"', html, re.I)
    pub_date = ""
    if m:
        raw = m.group(1).strip()
        # Convert em-dash to regular dash
        raw = raw.replace("\u2014", "-").replace("\u2015", "-")
        # Extract date part YYYY-MM-DD
        dm = re.search(r"(\d{4}-\d{2}-\d{2})", raw)
        if dm:
            pub_date = dm.group(1)
    
    # Content body: <DIV class="content" id="Content">...
    m = re.search(r'<DIV\s+class="content"\s+id="Content">(.*?)</DIV>', html, re.DOTALL)
    content = ""
    if m:
        content = m.group(1).strip()
    else:
        # Fallback: try to find any content
        m = re.search(r'<!--\s*正文\s*-->(.*?)<!--', html, re.DOTALL)
        if m:
            content = m.group(1).strip()
    
    return title, pub_date, content


def save_to_db(items_data):
    """批量写入search.db"""
    conn = sqlite3.connect(DB_PATH, timeout=30)
    c = conn.cursor()
    inserted = 0
    updated = 0
    for url, title, pub_date, content in items_data:
        try:
            c.execute("""
                INSERT OR REPLACE INTO gov_raw 
                (page_url, title, content, publish_date, source_url, category, site_name)
                VALUES (?, ?, ?, ?, ?, ?, ?)
            """, (
                BASE + url,
                title,
                content,
                pub_date or "",
                BASE + url,
                "环评审批受理公告",
                "南平市生态环境局",
            ))
            if c.rowcount == 1:
                inserted += 1
            else:
                updated += 1
        except Exception as e:
            print(f"  [ERROR] DB insert {url}: {e}")
    conn.commit()
    conn.close()
    return inserted, updated


def main():
    print(f"=== 南平市生态环境局 爬虫 ===")
    print(f"截止日期(近3年): {CUTOFF_DATE}")
    
    all_items = []
    seen_urls = set()
    
    # 爬列表页 (pages 1-14 total, ~3 years in first 8)
    for page in range(1, 15):
        print(f"\n[Page {page}] Fetching list...")
        items = parse_list(page)
        if not items:
            print(f"  -> No items found, stopping pagination")
            break
        
        new_count = 0
        stop = False
        for url, title, date_str in items:
            if url not in seen_urls:
                seen_urls.add(url)
                all_items.append((url, title, date_str))
                new_count += 1
            if date_str < CUTOFF_DATE:
                stop = True
        
        print(f"  -> Found {len(items)} items ({new_count} new), total queued: {len(all_items)}")
        if stop:
            print(f"  -> Reached cutoff date {CUTOFF_DATE}, stopping")
            break
        time.sleep(0.5)
    
    print(f"\n=== Total list items: {len(all_items)} ===")
    
    # 爬详情页
    detail_data = []
    for i, (url, title, date_str) in enumerate(all_items):
        print(f"  [{i+1}/{len(all_items)}] {url[:60]}...", end=" ")
        det_title, det_date, content = parse_detail(url)
        if det_title:
            title = det_title
        if det_date:
            date_str = det_date
        detail_data.append((url, title, date_str, content))
        print(f"len(content)={len(content) if content else 0}", end="")
        if not content:
            print(" [WARN empty]", end="")
        print()
        time.sleep(0.5)
    
    # 写入DB
    ins, upd = save_to_db(detail_data)
    print(f"\n=== DB write complete: {ins} inserted, {upd} updated ===")
    print(f"Total in this run: {len(detail_data)}")


if __name__ == "__main__":
    main()