#!/usr/bin/env python3
"""开阳县人民政府 - 项目环评 爬虫
静态列表页，20条/页，共3页~42条
列表: <a href="full_url">title</a><span>date</span>
详情: <div class="text"> 正文
"""
import os
import sys
import re
import json
import urllib.request
import urllib.parse
from datetime import datetime, timedelta

DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "kaiyang.gov.cn-项目环评"
BASE_URL = "https://www.kaiyang.gov.cn"
CUTOFF_DATE = "2023-06-16"

HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}


def fetch_page(url):
    """获取页面HTML"""
    try:
        req = urllib.request.Request(url, headers=HEADERS)
        resp = urllib.request.urlopen(req, timeout=30)
        return resp.read().decode("utf-8", errors="replace")
    except Exception as e:
        print(f"[error] 请求失败: {url[-60:]}, {e}")
        return None


def parse_list_items(html):
    """解析列表，提取标题、URL、日期"""
    items = []
    for m in re.finditer(r'<a[^>]*href="([^"]*t2026[^"]*)"[^>]*>([\s\S]*?)</a>\s*<span[^>]*>([^<]+)</span>', html):
        title = re.sub(r'<[^>]+>', '', m.group(2)).strip()
        if title:
            items.append({
                "title": title,
                "url": m.group(1),
                "date": m.group(3).strip() if m.group(3) else "",
            })
    return items


def get_detail(url):
    """获取详情页正文HTML"""
    html = fetch_page(url)
    if not html:
        return ""

    # 正文: div.Article_zw (含TRS_Editor内容)
    m = re.search(r'<div class="Article_zw">([\s\S]*?)</div>\s*</div>\s*<div class="share_jiucuo"', html)
    if not m:
        m = re.search(r'<div class="Article_zw">([\s\S]*?)</div>\s*</div>', html)
    if m:
        content = m.group(1).strip()
        content = re.sub(r'\n\s*\n', '\n', content)
        # 补全相对链接
        content = re.sub(r'href="(?!http|/)', 'href="' + BASE_URL + '/', content)
        content = re.sub(r'src="(?!http|/|data:)', 'src="' + BASE_URL + '/', content)
        # 图片内嵌
        content = inline_images(content, referer=url)
        return content
    return ""

def inline_images(html_content, referer=None):
    """将img标签的src转为绝对URL（去掉base64内嵌）"""
    if "<img" not in html_content:
        return html_content

    def _resolve_url(img_url):
        """将相对路径解析为绝对URL"""
        if img_url.startswith("data:"):
            return img_url, None
        full_url = img_url
        if not full_url.startswith("http"):
            full_url = (BASE_URL if full_url.startswith("/") else BASE_URL + "/") + full_url
        return img_url, full_url


def crawl(incremental_days=None):
    import sqlite3

    conn = sqlite3.connect(DB_PATH)
    c = conn.cursor()
    c.execute("PRAGMA journal_mode=WAL")

    c.execute("""
        CREATE TABLE IF NOT EXISTS gov_raw (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            title TEXT,
            page_url TEXT UNIQUE,
            content TEXT,
            publish_date TEXT,
            site_name TEXT DEFAULT '',
            crawl_time TEXT DEFAULT (datetime('now', '+8 hours')),
            similar TEXT
        )
    """)
    conn.commit()

    existing_urls = set()
    for row in c.execute("SELECT page_url FROM gov_raw WHERE site_name = ?", (SITE_NAME,)):
        existing_urls.add(row[0])

    if incremental_days:
        inc_cutoff = (datetime.now() - timedelta(days=incremental_days)).strftime("%Y-%m-%d")
        print(f"[info] 增量模式: 最近 {incremental_days} 天 >= {inc_cutoff}")
    else:
        inc_cutoff = CUTOFF_DATE
        print(f"[info] 全量模式: 截止 {CUTOFF_DATE}")

    total_added = 0
    total_skipped = 0
    cutoff_reached = False
    page = 1

    while True:
        if cutoff_reached:
            break

        url = f"{BASE_URL}/zwgk/zdlyxxgk/hjbh_5777165/xmhp/"
        if page > 1:
            url = f"{BASE_URL}/zwgk/zdlyxxgk/hjbh_5777165/xmhp/index_{page}.html"

        html = fetch_page(url)
        if not html:
            print(f"[info] 第{page}页无响应，结束")
            break

        items = parse_list_items(html)
        if not items:
            print(f"[info] 第{page}页无数据，结束")
            break

        print(f"[info] 第{page}页 ({len(items)}条)...")

        for item in items:
            pub = item["date"][:10]
            if pub and pub < CUTOFF_DATE and not incremental_days:
                cutoff_reached = True
                print(f"  [info] 截止日期 {CUTOFF_DATE} (当前 {pub})，跳过后续")
                break
            if incremental_days and pub and pub < inc_cutoff:
                cutoff_reached = True
                print(f"  [info] 增量截止 {inc_cutoff} (当前 {pub})，跳过后续")
                break

            detail_url = item["url"]
            if detail_url in existing_urls:
                total_skipped += 1
                continue

            content = get_detail(detail_url)
            if not content:
                print(f"  [warn] 空正文: {item['title'][:40]}")
                total_skipped += 1
                continue

            try:
                c.execute(
                    "INSERT OR IGNORE INTO gov_raw (title, page_url, content, publish_date, site_name) VALUES (?, ?, ?, ?, ?)",
                    (item["title"], detail_url, content, pub, SITE_NAME),
                )
                if c.rowcount > 0:
                    total_added += 1
                    existing_urls.add(detail_url)
                conn.commit()
            except Exception as e:
                print(f"  [error] DB写入失败: {e}")
                conn.rollback()

        if cutoff_reached:
            break
        page += 1

    conn.close()
    print(f"[done] 完成: 新增 {total_added} 条, 跳过 {total_skipped} 条")


if __name__ == "__main__":
    args = sys.argv[1:]
    if args and args[0].isdigit():
        crawl(incremental_days=int(args[0]))
    else:
        crawl()