#!/usr/bin/env python3
"""
crawl_teda.py — 天津经济技术开发区-通知公告 (www.teda.gov.cn)
CMS: TEDA政务服务平台
列表: 静态分页 channels/29.html (首页) / channels/29_{2-10}.html
      25条/页, 约10页
      <a href="/contents/29/XXXXX.html"><li><span class=news_title>TITLE</span><span class=news_date>DATE</span></li></a>
详情: <meta ArticleTitle> / <meta PubDate> / <div class="thr_cont">正文HTML</div>
"""

import subprocess, sqlite3, re, os, sys, time, random
from datetime import datetime, timedelta
from urllib.parse import urljoin
MAX_PAGES_DEFAULT = 10

# 支持 --pages 参数
import argparse as _AP
_AP_PARSER = _AP.ArgumentParser()
_AP_PARSER.add_argument("--pages", type=int, default=0, help="限制页数")
_AP_ARGS, _ = _AP_PARSER.parse_known_args()
MAX_PAGES = _AP_ARGS.pages if _AP_ARGS.pages > 0 else MAX_PAGES_DEFAULT

DB_PATH = os.environ.get("SEARCH_DB", "/root/search.db")
SITE_NAME = "天津经济技术开发区-通知公告"
SOURCE = "天津经济技术开发区"
BASE_URL = "https://www.teda.gov.cn/channels/29.html"
URL_TPL = "https://www.teda.gov.cn/channels/29_{i}.html"
HEADERS = "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
CUTOFF = (datetime.now() - timedelta(days=3 * 365)).strftime("%Y-%m-%d")
seen_urls = set()

def log(msg):
    print(f"[{SITE_NAME}] {msg}", flush=True)

def fetch(url):
    result = subprocess.run(
        ["curl", "-s", "-L", "-m", "20", url, "-H", HEADERS],
        capture_output=True, text=True, timeout=25
    )
    if result.returncode != 0:
        log(f"请求失败: {url[-60:]}: {result.stderr[:100]}")
        return None
    return result.stdout

def extract_list(html):
    items = []
    # Match: <a href="/contents/29/XXXXX.html"><li>...<span class='news_title'>TITLE</span>...<span class="news_date">DATE</span>...</li></a>
    for m in re.finditer(
        r"<a href=['\"]/contents/29/(\d+\.html)['\"][^>]*>.*?<span class='news_title'>(.*?)</span>.*?<span class=\"news_date\">\s*(\d{4}-\d{2}-\d{2})\s*</span>",
        html, re.DOTALL
    ):
        art_name = m.group(1)
        title = m.group(2).strip()
        date_str = m.group(3).strip()
        full_url = f"https://www.teda.gov.cn/contents/29/{art_name}"
        if full_url in seen_urls:
            continue
        seen_urls.add(full_url)
        items.append((full_url, title, date_str))
    return items

def extract_detail(html, url):
    title = ""
    content = ""
    pub_date = ""

    # Title from meta
    m = re.search(r'ArticleTitle"\s*content="([^"]+)"', html)
    if m:
        title = m.group(1).strip()

    # Date from meta
    m = re.search(r'PubDate"\s*content="([^"]+)"', html)
    if m:
        pub_date = m.group(1).strip()

    # Content from thr_cont
    m = re.search(r'class="thr_cont"[^>]*>(.*?)</div>', html, re.DOTALL)
    if m:
        content = m.group(1).strip()

    # Append attachments (conattach)
    m = re.search(r'class="conattach"[^>]*>(.*?)</div>', html, re.DOTALL)
    if m:
        attach_html = m.group(1).strip()
        if attach_html:
            content = content + '\n\n<div class="conattach">' + attach_html + '</div>'

    return title, content, pub_date

def save_to_db(articles):
    conn = sqlite3.connect(DB_PATH, timeout=60)
    cursor = conn.cursor()
    inserted = 0
    skipped = 0
    for url, title, date, content in articles:
        try:
            cursor.execute(
                """INSERT OR IGNORE INTO gov_raw
                   (site_name, title, page_url, publish_date, source_url, content)
                   VALUES (?, ?, ?, ?, ?, ?)""",
                (SITE_NAME, title, url, date, SOURCE, content or ""),
            )
            if cursor.rowcount > 0:
                inserted += 1
            else:
                skipped += 1
        except Exception as e:
            log(f"入库失败: {url[-50:]}: {e}")
    conn.commit()
    conn.close()
    return inserted, skipped

def main():
    pages = [BASE_URL] + [URL_TPL.replace("{i}", str(i)) for i in range(2, MAX_PAGES + 1)]
    total_articles = []

    for page_url in pages:
        log(f"取列表: {page_url}")
        html = fetch(page_url)
        if not html:
            continue
        items = extract_list(html)
        if not items:
            log(f"  无内容，停止分页")
            break
        log(f"  共{len(items)}条")

        for url, title, date_str in items:
            if date_str < CUTOFF:
                log(f"  跳过(旧): {date_str} {title[:40]}")
                continue

            time.sleep(random.uniform(0.3, 0.8))
            detail_html = fetch(url)
            if not detail_html:
                continue
            d_title, content, d_date = extract_detail(detail_html, url)
            if not d_title:
                d_title = title
            if not d_date:
                d_date = date_str
            if not content:
                log(f"  无正文: {d_title[:40]}")
                continue

            total_articles.append((url, d_title, d_date, content))

        time.sleep(0.3)

    inserted, skipped = save_to_db(total_articles)
    log(f"完成！新增{inserted}条，跳过{skipped}条（总计{len(total_articles)}条）")

if __name__ == "__main__":
    main()