#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""唐山海港经济开发区-项目建设爬虫 | JEECMS | requests直取"""
"""
site: www.tshg.gov.cn (唐山海港经济开发区管委会)
栏目: /tshaigang/tshgxmjs/ (项目建设)
列表: ul.list > li > a + span.date (20条/页, index.html / index_{N}.html)
详情: /tshaigang/tshgxmjs/YYYYMMDD/{id}.html, h1.top20 标题 + div.conten_box 正文
分页: index.html -> index_2.html ... (静态, 3年cutoff)
2026-08-01 修复: 加--pages支持(默认1页), 补gov_search/script_name同步, content去div包装
"""
import os
import re
import sys
import time
import sqlite3
import requests
from bs4 import BeautifulSoup
from datetime import datetime, timedelta
from urllib.parse import urljoin

DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
BASE_URL = "https://www.tshg.gov.cn/tshaigang/tshgxmjs/"
SITE_NAME = "唐山海港经济开发区-项目建设"
GROUP_NAME = "河北"
SCRIPT_NAME = os.path.basename(__file__)
CUTOFF_DATE = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9",
}

# ---- 参数解析: 兼容 --pages=N 和 --pages N ----
_MAX_PAGES = 1
i = 0
while i < len(sys.argv):
    a = sys.argv[i]
    if a.startswith("--pages="):
        _MAX_PAGES = int(a.split("=", 1)[1])
    elif a == "--pages" and i + 1 < len(sys.argv):
        _MAX_PAGES = int(sys.argv[i + 1]); i += 1
    i += 1

def get_soup(url):
    for attempt in range(3):
        try:
            r = requests.get(url, headers=HEADERS, timeout=30)
            r.encoding = 'utf-8'
            return BeautifulSoup(r.text, 'html.parser')
        except Exception as e:
            if attempt < 2:
                time.sleep(2)
            else:
                print(f"  [ERROR] {e}")
                return None

def parse_list(soup):
    items = []
    ul = soup.find('ul', class_='list')
    if not ul:
        return items
    for li in ul.find_all('li'):
        a = li.find('a')
        span = li.find('span', class_='date')
        if a and span:
            href = a.get('href', '')
            title = a.get('title', a.get_text(strip=True))
            date_str = span.get_text(strip=True)
            if href and date_str:
                full_url = urljoin(BASE_URL, href)
                items.append((title, full_url, date_str))
    return items

def get_detail(url):
    soup = get_soup(url)
    if not soup:
        return "", "", ""
    title_tag = soup.select_one('h1.top20')
    title = title_tag.get_text(strip=True) if title_tag else ""
    content_div = soup.select_one('.conten_box')
    if not content_div:
        return title, "", ""
    body = str(content_div)
    summary = re.sub(r'<[^>]+>', '', body)[:200].strip()
    return title, body, summary

def main():
    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()
    total_new = total_skip = 0
    fail_streak = 0

    for page_idx in range(1, _MAX_PAGES + 1):
        if page_idx == 1:
            url = BASE_URL + "index.html"
        else:
            url = BASE_URL + f"index_{page_idx}.html"
        print(f"\n--- 第{page_idx}页 ---")
        soup = get_soup(url)
        if not soup:
            fail_streak += 1
            if fail_streak >= 1 and page_idx > 1:
                print(f"  第{page_idx}页请求失败, 提前停止")
                break
            continue
        fail_streak = 0
        items = parse_list(soup)
        if not items:
            print("  无列表，跳过")
            break
        print(f"  {len(items)}条")
        for title, detail_url, date_str in items:
            if date_str < CUTOFF_DATE:
                print(f"  [SKIP] {date_str} 早于{CUTOFF_DATE}")
                continue
            c.execute("SELECT id FROM gov_raw WHERE page_url = ?", (detail_url,))
            if c.fetchone():
                total_skip += 1
                continue
            time.sleep(0.5)
            full_title, body, summary = get_detail(detail_url)
            if not body:
                print(f"  [EMPTY] {full_title or title[:30]}")
                continue
            final_title = full_title or title
            c.execute("""INSERT OR IGNORE INTO gov_raw 
                (title, page_url, content, site_name, publish_date, source_url, category, status, summary, group_name, script_name)
                VALUES (?,?,?,?,?,?,?,?,?,?,?)""",
                (final_title, detail_url, body, SITE_NAME, date_str, detail_url, '环评', 'published', summary, GROUP_NAME, SCRIPT_NAME))
            if c.rowcount > 0:
                rid = c.execute("SELECT id FROM gov_raw WHERE page_url = ?", (detail_url,)).fetchone()[0]
                c.execute("INSERT OR IGNORE INTO gov_search(rowid, title, site_name, summary) VALUES(?,?,?,?)",
                          (rid, final_title, SITE_NAME, summary))
                total_new += 1
                print(f"  [NEW] {final_title[:45]}... | {date_str}")
            else:
                total_skip += 1
        time.sleep(1)

    conn.commit()
    conn.close()
    print(f"\n========== 汇总 ==========")
    print(f"站点: {SITE_NAME}")
    print(f"新增: {total_new}")
    print(f"跳过: {total_skip}")
    print(f"==========================")

if __name__ == "__main__":
    main()