#!/usr/bin/env python3
"""crawl_yangpu.py — yangpu.hainan.gov.cn huanbao"""
import subprocess, sqlite3, re, os, sys, time, random
from datetime import datetime, timedelta
from urllib.parse import urljoin
from bs4 import BeautifulSoup

DB_PATH = os.environ.get("SEARCH_DB", "/root/search.db")
SITE_NAME = "洋浦经济开发区-环境保护"
SOURCE = "洋浦经济开发区"
BASE_URL = "https://yangpu.hainan.gov.cn/yangpu/3302/list3.shtml?ddtab=true"
URL_TPL = "https://yangpu.hainan.gov.cn/yangpu/3302/list3_{i}.shtml?ddtab=true"
MAX_PAGES = 11
HEADERS = "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
CUTOFF = (datetime.now() - timedelta(days=3 * 365)).strftime("%Y-%m-%d")
seen_urls = set()

def log(msg):
    print(f"[{SITE_NAME}] {msg}", flush=True)

def fetch(url):
    result = subprocess.run(["curl", "-s", "-L", url, "-H", HEADERS], capture_output=True, text=True, timeout=25)
    if result.returncode != 0:
        log(f"fetch fail: {url[-60:]}")
        return None
    return result.stdout

def extract_list(html):
    items = []
    for m in re.finditer(r'<div class="list-right_title fon_1">\s*<a href="([^"]+)"[^>]*>(.*?)</a>\s*</div>', html, re.DOTALL):
        rel_url = m.group(1).strip()
        title = m.group(2).strip()
        full_url = urljoin(BASE_URL, rel_url)
        if full_url in seen_urls:
            continue
        seen_urls.add(full_url)
        pos = m.end()
        dm = re.search(r'发布时间：\s*\n\s*(\d{4}-\d{2}-\d{2})', html[pos:pos+300])
        date_str = dm.group(1).strip() if dm else ""
        items.append((full_url, title, date_str))
    return items

def extract_detail(html, url):
    title = ""
    content = ""
    pub_date = ""
    m = re.search(r'ArticleTitle"\s*content="([^"]+)"', html)
    if m:
        title = m.group(1).strip()
    m = re.search(r'PubDate"\s*content="([^"]+)"', html)
    if m:
        pub_date = m.group(1).strip()
    soup = BeautifulSoup(html, "html.parser")
    zoomcon = soup.find(id="zoomcon")
    if zoomcon:
        parts = []
        for p in zoomcon.find_all("p"):
            text = p.get_text(strip=True)
            if text:
                parts.append(text)
        if len(parts) < 2:
            parts = []
            for child in zoomcon.find_all(["p", "div", "span"], recursive=True):
                text = child.get_text(strip=True)
                if text:
                    parts.append(text)
            deduped = []
            for t in parts:
                if not deduped or t != deduped[-1]:
                    deduped.append(t)
            parts = deduped
        content = "\n\n".join(parts)
    return title, content, pub_date

def save_to_db(articles):
    conn = sqlite3.connect(DB_PATH, timeout=60)
    cursor = conn.cursor()
    inserted = 0
    skipped = 0
    for url, title, date, content in articles:
        try:
            cursor.execute(
                "INSERT OR IGNORE INTO gov_raw (site_name, title, page_url, publish_date, source_url, content) VALUES (?, ?, ?, ?, ?, ?)",
                (SITE_NAME, title, url, date, SOURCE, content or ""))
            if cursor.rowcount > 0:
                inserted += 1
            else:
                skipped += 1
        except Exception as e:
            log(f"db fail: {url[-50:]}: {e}")
    conn.commit()
    conn.close()
    return inserted, skipped

def main():
    pages = [BASE_URL] + [URL_TPL.replace("{i}", str(i)) for i in range(2, MAX_PAGES + 1)]
    total = []
    for page_url in pages:
        log(f"list: {page_url}")
        html = fetch(page_url)
        if not html:
            continue
        items = extract_list(html)
        if not items:
            log("  no items, stop")
            break
        log(f"  {len(items)} items")
        for url, title, date_str in items:
            if date_str < CUTOFF:
                log(f"  skip(old): {date_str} {title[:40]}")
                continue
            time.sleep(random.uniform(0.3, 0.8))
            dh = fetch(url)
            if not dh:
                continue
            dt, content, dd = extract_detail(dh, url)
            if not dt:
                dt = title
            if not dd:
                dd = date_str
            if not content:
                log(f"  no content: {dt[:40]}")
                continue
            total.append((url, dt, dd, content))
        time.sleep(0.3)
    ins, skip = save_to_db(total)
    log(f"done! new={ins} skip={skip} total={len(total)}")

if __name__ == "__main__":
    main()