#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
爬虫：广州市生态环境局 - 建设项目环评审批公告
站点：https://sthjj.gz.gov.cn/hjgl/jsxm/hpspgg/
CMS：NFCMS (广东省政府网站群系统)
分页：index_N.html (N=1~100)，每页16条
结构: <div class="conts-list"><span><a href="...">标题</a></span><span>日期</span></div>
"""

import requests
import re
import sqlite3
import os
import time
from bs4 import BeautifulSoup
from urllib.parse import urljoin
MAX_PAGES_DEFAULT = 100

# 支持 --pages 参数
import argparse as _AP
_AP_PARSER = _AP.ArgumentParser()
_AP_PARSER.add_argument("--pages", type=int, default=0, help="限制页数")
_AP_ARGS, _ = _AP_PARSER.parse_known_args()
MAX_PAGES = _AP_ARGS.pages if _AP_ARGS.pages > 0 else MAX_PAGES_DEFAULT

BASE_URL = "https://sthjj.gz.gov.cn/hjgl/jsxm/hpspgg/"
SITE_NAME = "广州市-环评审批公告"
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36",
    "Referer": "https://sthjj.gz.gov.cn/",
}
DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")

def _get_db():
    conn = sqlite3.connect(DB_PATH, timeout=30)
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute("PRAGMA synchronous=NORMAL")
    conn.execute("PRAGMA busy_timeout=30000")
    return conn

def ensure_table(conn):
    conn.execute("""
        CREATE TABLE IF NOT EXISTS gov_raw (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            title TEXT NOT NULL,
            content TEXT,
            source_url TEXT NOT NULL UNIQUE,
            publish_date TEXT,
            site_name TEXT,
            created_at TEXT DEFAULT (datetime('now','localtime'))
        )
    """)
    conn.execute("CREATE INDEX IF NOT EXISTS idx_gov_raw_site ON gov_raw(site_name)")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_gov_raw_date ON gov_raw(publish_date)")
    # FTS 由 search.db 触发器 trg_gov_raw_fts_* 统一维护, 脚本不再自建 gov_fts 表
    conn.commit()

def row_exists(conn, source_url):
    cur = conn.execute("SELECT 1 FROM gov_raw WHERE source_url=?", (source_url,))
    return cur.fetchone() is not None

def insert_row(conn, title, content, source_url, publish_date):
    conn.execute(
        "INSERT OR IGNORE INTO gov_raw(title, content, source_url, publish_date, site_name) VALUES (?,?,?,?,?)",
        (title, content, source_url, publish_date, SITE_NAME)
    )
    if conn.total_changes > 0:
        return True
    return False

def extract_content(soup):
    """从详情页提取正文 - NFCMS广州站"""
    # 正文在 #logPanel 里
    div = soup.select_one("#logPanel")
    if div:
        # 处理图片
        for img in div.find_all("img"):
            src = img.get("src", "")
            if src and not src.startswith("data:"):
                img_url = urljoin(BASE_URL, src) if not src.startswith("http") else src
                img.replace_with('<img src="{}">'.format(img_url))
            else:
                img.decompose()
        # 附件链接 - 确保绝对URL
        for a_tag in div.find_all("a", href=True):
            h = a_tag["href"]
            if h and not h.startswith("http") and not h.startswith("#") and not h.startswith("javascript"):
                a_tag["href"] = urljoin(BASE_URL, h)
        return str(div)
    return ""

def get_detail(url, session, retries=3):
    """获取详情页内容"""
    for attempt in range(retries):
        try:
            r = session.get(url, headers=HEADERS, timeout=60)
            r.encoding = 'utf-8'
            if r.status_code == 200:
                soup = BeautifulSoup(r.text, 'html.parser')
                content = extract_content(soup)
                
                # 从详情页提取日期
                date_str = ""
                for p in [
                    r'(\d{4}-\d{2}-\d{2})',
                    r'发布时间[：:]\s*(\d{4}[-/\.]\d{1,2}[-/\.]\d{1,2})',
                    r'发布日期[：:]\s*(\d{4}[-/\.]\d{1,2}[-/\.]\d{1,2})',
                ]:
                    m = re.search(p, r.text)
                    if m:
                        date_str = m.group(1).replace('/', '-').replace('.', '-')
                        break
                
                return content, date_str
        except requests.RequestException as e:
            if attempt < retries - 1:
                time.sleep(2 ** attempt)
    return "", ""

def parse_list_page(url, session):
    """解析列表页，返回 [(title, detail_url, date), ...]"""
    results = []
    try:
        r = session.get(url, headers=HEADERS, timeout=60)
        r.encoding = 'utf-8'
        if r.status_code != 200:
            return results
    except Exception:
        return results

    soup = BeautifulSoup(r.text, 'html.parser')
    
    # 每条在 div.conts-list 里：<span><a>标题</a></span><span>日期</span>
    for div in soup.select('div.conts-list'):
        spans = div.find_all('span')
        if len(spans) < 2:
            continue
        a = spans[0].find('a')
        if not a or not a.get('href'):
            continue
        href = a['href']
        if 'content/post_' not in href:
            continue
        title = a.get_text(strip=True)
        if not title or len(title) < 5 or '穗好办' in title:
            continue
        
        detail_url = href if href.startswith('http') else urljoin(url, href)
        date_str = spans[1].get_text(strip=True) if len(spans) > 1 else ""
        if date_str and not re.match(r'\d{4}-\d{2}-\d{2}', date_str):
            date_str = ""
        
        results.append((title, detail_url, date_str))
    
    return results

def crawl():
    conn = _get_db()
    ensure_table(conn)
    
    session = requests.Session()
    session.headers.update(HEADERS)
    
    total_new = 0
    total_skip = 0
    
    for page in range(1, MAX_PAGES + 1):
        if page == 1:
            url = BASE_URL
        else:
            url = "https://sthjj.gz.gov.cn/hjgl/jsxm/hpspgg/index_{}.html".format(page)
        
        articles = parse_list_page(url, session)
        if not articles:
            print("  第{}页无文章，采集结束".format(page))
            break
        
        for title, detail_url, date_str in articles:
            if row_exists(conn, detail_url):
                total_skip += 1
                continue
            
            content, detail_date = get_detail(detail_url, session)
            if not detail_date and date_str:
                detail_date = date_str
            
            content_text = BeautifulSoup(content or '', 'html.parser').get_text(strip=True)
            if len(content_text) < 20:
                print("  ⚠ 内容过短: {}... 跳过".format(title[:30]))
                total_skip += 1
                continue
            
            if insert_row(conn, title, content, detail_url, detail_date):
                total_new += 1
                conn.commit()
                print("  ✓ [{}] {} ({})".format(total_new, title[:40], detail_date or '无日期'))
            else:
                total_skip += 1
        
        print("  第{}/{}页: {}条, 累计新{}条/跳过{}条".format(page, MAX_PAGES, len(articles), total_new, total_skip))
        time.sleep(1)
    
    conn.close()
    print("\n✅ 采集完成！新增{}条，跳过{}条，共{}条".format(total_new, total_skip, total_new+total_skip))

if __name__ == '__main__':
    crawl()