#!/usr/bin/env python3
"""
crawl_yangpu.py — 洋浦经济开发区-环境保护 (yangpu.hainan.gov.cn)
CMS: 海南政府网站统一平台（TRS UEDITOR）
列表: 静态HTML分页 list3.shtml (首页) / list3_{2-11}.shtml
      7条/页, 11页 ≈ 77条
      <div class="list-right_title fon_1"><a href="URL">TITLE</a></div>
      日期: <td>发布时间：\n   YYYY-MM-DD</td>
详情: <meta ArticleTitle> / <meta PubDate> / <div id="zoomcon">UCAPCONTENT正文HTML</div>
"""

import subprocess, sqlite3, re, os, sys, time, random
from datetime import datetime, timedelta
from urllib.parse import urljoin

DB_PATH = os.environ.get("SEARCH_DB", "/root/search.db")
SITE_NAME = "洋浦经济开发区-环境保护"
SOURCE = "洋浦经济开发区"
BASE_URL = "https://yangpu.hainan.gov.cn/yangpu/3302/list3.shtml?ddtab=true"
URL_TPL = "https://yangpu.hainan.gov.cn/yangpu/3302/list3_{i}.shtml?ddtab=true"
MAX_PAGES = 11
HEADERS = "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
CUTOFF = (datetime.now() - timedelta(days=3 * 365)).strftime("%Y-%m-%d")
seen_urls = set()

def log(msg):
    print(f"[{SITE_NAME}] {msg}", flush=True)

def fetch(url):
    result = subprocess.run(
        ["curl", "-s", "-L", url, "-H", HEADERS],
        capture_output=True, text=True, timeout=25
    )
    if result.returncode != 0:
        log(f"请求失败: {url[-60:]}")
        return None
    return result.stdout

def extract_list(html):
    items = []
    # Match each article block
    for m in re.finditer(
        r'<div class="list-right_title fon_1">\s*<a href="([^"]+)"[^>]*>(.*?)</a>\s*</div>',
        html, re.DOTALL
    ):
        rel_url = m.group(1).strip()
        title = m.group(2).strip()
        full_url = urljoin(BASE_URL, rel_url)
        if full_url in seen_urls:
            continue
        seen_urls.add(full_url)
        
        # Find date after this block
        # Date is in the next table: 发布时间：\n   YYYY-MM-DD
        pos = m.end()
        date_match = re.search(r'发布时间：\s*\n\s*(\d{4}-\d{2}-\d{2})', html[pos:pos+300])
        date_str = date_match.group(1).strip() if date_match else ""
        
        items.append((full_url, title, date_str))
    return items

def extract_detail(html, url):
    title = ""
    content = ""
    pub_date = ""
    
    # Title from meta
    m = re.search(r'ArticleTitle"\s*content="([^"]+)"', html)
    if m:
        title = m.group(1).strip()
    
    # Date from meta
    m = re.search(r'PubDate"\s*content="([^"]+)"', html)
    if m:
        pub_date = m.group(1).strip()
    
    # Content from zoomcon div with depth counting
    m = re.search(r'class="con_cen line mar-t2"[^>]*id="zoomcon"[^>]*>', html)
    if m:
        start = m.end()
        depth = 0
        buf = []
        i = start
        while i < len(html):
            if html[i:i+6] == '</div>':
                if depth == 0:
                    content = ''.join(buf)
                    break
                depth -= 1
                buf.append('</div>')
                i += 6
            elif html[i:i+5] == '<div ' or html[i:i+5] == '<div>' or html[i:i+5] == '<div\n':
                depth += 1
                buf.append(html[i:i+5])
                i += 5
            else:
                buf.append(html[i])
                i += 1
    
    # Remove UCAPCONTENT tags
    content = re.sub(r'</?UCAPCONTENT>', '', content)
    
    return title, content, pub_date

def save_to_db(articles):
    conn = sqlite3.connect(DB_PATH)
    cursor = conn.cursor()
    inserted = 0
    skipped = 0
    for url, title, date, content in articles:
        try:
            cursor.execute(
                """INSERT OR IGNORE INTO gov_raw
                   (site_name, title, page_url, publish_date, source_url, content)
                   VALUES (?, ?, ?, ?, ?, ?)""",
                (SITE_NAME, title, url, date, SOURCE, content or ""),
            )
            if cursor.rowcount > 0:
                inserted += 1
            else:
                skipped += 1
        except Exception as e:
            log(f"入库失败: {url[-50:]}: {e}")
    conn.commit()
    conn.close()
    return inserted, skipped

def main():
    pages = [BASE_URL] + [URL_TPL.replace("{i}", str(i)) for i in range(2, MAX_PAGES + 1)]
    total_articles = []

    for page_url in pages:
        log(f"取列表: {page_url}")
        html = fetch(page_url)
        if not html:
            continue
        items = extract_list(html)
        if not items:
            log(f"  无内容，停止分页")
            break
        log(f"  共{len(items)}条")

        for url, title, date_str in items:
            if date_str < CUTOFF:
                log(f"  跳过(旧): {date_str} {title[:40]}")
                continue

            time.sleep(random.uniform(0.3, 0.8))
            detail_html = fetch(url)
            if not detail_html:
                continue
            d_title, content, d_date = extract_detail(detail_html, url)
            if not d_title:
                d_title = title
            if not d_date:
                d_date = date_str
            if not content:
                log(f"  无正文: {d_title[:40]}")
                continue

            total_articles.append((url, d_title, d_date, content))

        time.sleep(0.3)

    inserted, skipped = save_to_db(total_articles)
    log(f"完成！新增{inserted}条，跳过{skipped}条（总计{len(total_articles)}条）")

if __name__ == "__main__":
    main()