#!/usr/bin/env python3
"""爬取博爱县人民政府 - 建设项目环境影响评价信息

列表: <ul id="newslist"> with <li><span class="date">date</span><a href="url">title</a></li>
分页: index.html, index_1.html, ..., index_N.html (pagesize=24, 64页)
正文: <div class="content"> - 含 <p> 标签的HTML
"""

import sys, re, sqlite3, requests, urllib.parse
from datetime import datetime, timedelta
from bs4 import BeautifulSoup
import os

BASE_URL = "https://www.boai.gov.cn/zfxxgk/xxgkml/zdly/hjbh/jsxmhjyxpjxx"
DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "博爱县人民政府-建设项目环境影响评价信息"
THREE_YEARS_AGO = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}


def get_soup(url):
    r = requests.get(url, headers=HEADERS, timeout=30)
    r.encoding = "utf-8"
    return BeautifulSoup(r.text, "html.parser")


def extract_list_items(soup):
    """从列表页提取文章信息"""
    ul = soup.find("ul", id="newslist")
    if not ul: return []
    items = []
    for li in ul.find_all("li", recursive=False):
        a_tag = li.find("a")
        span = li.find("span", class_="date")
        if not a_tag: continue
        href = a_tag.get("href", "")
        title = a_tag.get_text(strip=True)
        date_str = span.get_text(strip=True) if span else ""
        items.append((title, href, date_str))
    return items


def get_pagination_info(soup):
    """获取总页数"""
    page_div = soup.find("div", id="pageDec")
    if page_div:
        pc = page_div.get("pagecount", "")
        ps = page_div.get("pagesize", "24")
        if pc and pc.isdigit():
            return (int(pc) + int(ps) - 1) // int(ps)
    return 1


def make_abs_url(base_url, src):
    if not src or src.startswith("http://") or src.startswith("https://") or src.startswith("data:"):
        return src
    parsed = urllib.parse.urlparse(base_url)
    path_dir = parsed.path.rsplit("/", 1)[0] + "/"
    if src.startswith("./"): src = src[2:]
    elif src.startswith("/"):
        return parsed.scheme + "://" + parsed.netloc + src
    return parsed.scheme + "://" + parsed.netloc + path_dir + src.lstrip("/")


def extract_content(detail_url):
    """从详情页提取正文HTML"""
    try:
        r = requests.get(detail_url, headers=HEADERS, timeout=30)
        r.encoding = "utf-8"
        soup = BeautifulSoup(r.text, "html.parser")
        div = soup.find("div", class_="content")
        if not div:
            return ""
        # 移除分享/打印等非内容元素
        for unwanted in div.find_all(["script", "style"]):
            unwanted.decompose()
        # 图片相对路径转绝对
        for img in div.find_all("img"):
            src = img.get("src", "")
            if src:
                img["src"] = make_abs_url(detail_url, src)
        return str(div)
    except Exception as e:
        print("  [ERROR] extract content: %s" % str(e))
        return ""


def main():
    is_inc = "incremental" in sys.argv
    mode = "增量" if is_inc else "全量"
    print("=== %s模式: %s ===" % (mode, SITE_NAME))

    soup = get_soup(BASE_URL + "/index.html")
    total_pages = get_pagination_info(soup)
    print("总页数: %d" % total_pages)
    max_pages = 1 if is_inc else total_pages

    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()
    ins, skip, streak = 0, 0, 0

    for pi in range(max_pages):
        pu = BASE_URL + ("/index.html" if pi == 0 else "/index_%d.html" % pi)
        print("\n--- 第%d页 ---" % (pi+1))
        soup = get_soup(pu)
        items = extract_list_items(soup)
        print("  条目数: %d" % len(items))
        if not items: break

        has_r = False
        for title, url, ds in items:
            if ds and ds < THREE_YEARS_AGO:
                skip += 1; continue
            if ds: has_r = True

            html = extract_content(url)
            if not html:
                print("  [WARN] 无正文: %s..." % title[:40])

            try:
                c.execute("INSERT OR IGNORE INTO gov_raw (site_name, page_url, title, content, publish_date, summary, date_rank) VALUES (?,?,?,?,?,?,?)",
                    (SITE_NAME, url, title.strip(), html, ds, SITE_NAME, ds))
                if c.rowcount > 0: ins += 1
                else: skip += 1
            except Exception as e:
                print("  [ERR] %s" % str(e)); skip += 1

        if not has_r:
            streak += 1
            if streak >= 2 and pi >= 4:
                print("  连续2页无新数据，停止")
                break
        else: streak = 0

    conn.commit(); conn.close()
    print("\n=== 完成 新增:%d 跳过:%d ===" % (ins, skip))


if __name__ == "__main__": main()