#!/usr/bin/env python3
"""
crawl_whsthj.py — 威海市生态环境局-拟环评审批公示 (col44912)
列表: POST dataproxy (XML→HTML)
详情: GET 直连
"""

import re, time, os
from datetime import datetime, timedelta
import requests

DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "威海市生态环境局-拟环评审批公示"
THREE_YEARS_AGO = datetime.now() - timedelta(days=3 * 365)
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36",
}

DATAPROXY_URL = "https://sthjj.weihai.gov.cn/module/web/jpage/dataproxy.jsp"
DETAIL_BASE = "https://sthjj.weihai.gov.cn"
import sys as _SYS
_MAX_PG = int(_SYS.argv[1]) if len(_SYS.argv) > 1 and _SYS.argv[1].isdigit() else None
if _MAX_PG is not None:
    print('[AutoPg] max_pages=' + str(_MAX_PG))
# END AUTO PAGES


def get_db():
    import sqlite3
    conn = sqlite3.connect(DB_PATH, timeout=60)
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute("PRAGMA busy_timeout=5000")
    return conn

def insert_article(conn, art):
    date_rank = 0
    if art["publish_date"] and len(art["publish_date"]) >= 10:
        try:
            date_rank = int(art["publish_date"][:10].replace("-", ""))
        except:
            pass
    conn.execute(
        """INSERT OR IGNORE INTO gov_raw
           (site_name, page_url, title, publish_date, content, source_url, date_rank)
           VALUES (?, ?, ?, ?, ?, ?, ?)""",
        (SITE_NAME, art["url"], art["title"], art["publish_date"],
         art["content"], art.get("source", SITE_NAME), date_rank)
    )

def get_list_page(page):
    """获取列表页"""
    data = {
        "page": page, "appid": 1, "webid": 105, "path": "/",
        "columnid": 44912, "unitid": 207606,
        "webname": "威海市生态环境局", "permissiontype": 0,
    }
    try:
        resp = requests.post(DATAPROXY_URL, data=data, headers=HEADERS, timeout=15)
        resp.encoding = "utf-8"
        return resp.text
    except Exception as e:
        print(f"  [ERR] page {page}: {e}")
        return ""

def parse_list_page(html):
    """解析列表XML，返回文章列表"""
    items = []
    records = re.findall(r"<record><!\[CDATA\[(.*?)\]\]></record>", html, re.DOTALL)
    for rec in records:
        # URL
        m = re.search(r'<a href=\'(/art/\d+/\d+/\d+/art_\d+_\d+\.html)\'', rec)
        if not m:
            continue
        url = DETAIL_BASE + m.group(1)

        # Title (from title attribute, truncation is in the display text)
        m = re.search(r"title='([^']*)'", rec)
        title = ""
        if m:
            title = m.group(1).strip()
        if not title:
            # fallback: display text
            m = re.search(r"target='_blank'>([^<]+)", rec)
            if m:
                title = m.group(1).strip()
        title = re.sub(r"\s+", " ", title).strip()
        if not title:
            continue

        # Date
        m = re.search(r"<span[^>]*>(\d{4}-\d{2}-\d{2})</span>", rec)
        if not m:
            continue
        pub_date = m.group(1)

        # 日期过滤
        try:
            dt = datetime.strptime(pub_date, "%Y-%m-%d")
            if dt < THREE_YEARS_AGO:
                continue
        except:
            continue

        items.append({"url": url, "title": title, "date": pub_date})

    return items

def crawl_detail(url):
    """爬取详情正文"""
    try:
        resp = requests.get(url, headers=HEADERS, timeout=20)
        resp.encoding = "utf-8"
    except:
        return "", ""
    if resp.status_code != 200:
        return "", ""

    html = resp.text

    # 正文: <div id='zoom'>
    content = ""
    m = re.search(r"<div id='zoom'[^>]*>(.*?)</div>\s*</div>\s*<div class=\"insFooter\"", html, re.DOTALL)
    if m:
        content = m.group(1).strip()
    if not content:
        m = re.search(r"<div id='zoom'[^>]*>(.*?)</div>", html, re.DOTALL)
        if m:
            content = m.group(1).strip()

    # 发布日期: 时间：2026-06-15
    pub_date = ""
    m = re.search(r"时间[：:](\d{4}-\d{2}-\d{2})", html)
    if m:
        pub_date = m.group(1)

    return content, pub_date

def main():
    print(f"\n{'='*50}")
    print(f"[{SITE_NAME}]")

    conn = get_db()
    all_items = []

    # 阶段1: 列表
    print("阶段1: 爬取列表...")
    for page in range(1, min(200, _MAX_PG or 200)):
        html = get_list_page(page)
        if not html:
            break
        items = parse_list_page(html)
        if not items:
            if page == 1:
                print("  [ERR] 第一页无数据")
                return
            break
        all_items.extend(items)
        print(f"  page {page}: {len(items)} 条 (累计 {len(all_items)})")
        time.sleep(0.3)

    print(f"\n  列表合计: {len(all_items)} 条 (近3年)")
    if not all_items:
        return

    # 阶段2: 详情
    total_inserted = 0
    print(f"阶段2: 爬取详情 ({len(all_items)} 条)...")

    for i, item in enumerate(all_items):
        content, pub_date = crawl_detail(item["url"])
        if not content or len(content) < 50:
            print(f"  [{i+1}/{len(all_items)}] ❌ 内容空: {item['title'][:40]}")
            continue

        art = {
            "title": item["title"],
            "url": item["url"],
            "publish_date": pub_date or item["date"],
            "content": content,
            "source": SITE_NAME,
        }
        insert_article(conn, art)
        total_inserted += 1

        if i % 20 == 0:
            conn.commit()
        if i % 5 == 0 and total_inserted > 0:
            print(f"  [{i+1}/{len(all_items)}] ✅ {total_inserted} 条 -- {item['title'][:40]}")

        time.sleep(0.3)

    conn.commit()
    conn.close()

    print(f"\n{'='*50}")
    print(f"✅ {SITE_NAME} 完成!")
    print(f"  列表: {len(all_items)} 条, 入库: {total_inserted} 条")
    print(f"{'='*50}")

if __name__ == "__main__":
    main()