#!/usr/bin/env python3
"""爬取巴林左旗人民政府 - 公示公告

列表: <ul id="lb"> with <li class="clearfix"><a href="url">title</a><span>date</span></li>
分页: index.html, index_1.html, ..., index_14.html (15页, 215条)
正文: <div class="deatil-detail"> 内含 trs_editor_view
"""

import sys, re, sqlite3, requests, urllib.parse
from datetime import datetime, timedelta
from bs4 import BeautifulSoup
import os

BASE_URL = "http://www.blzq.gov.cn/zfzx/gsgg"
DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "巴林左旗人民政府-公示公告"
THREE_YEARS_AGO = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}


def get_soup(url):
    r = requests.get(url, headers=HEADERS, timeout=30)
    r.encoding = "utf-8"
    return BeautifulSoup(r.text, "html.parser")


def extract_list_items(soup):
    ul = soup.find("ul", id="lb")
    if not ul: return []
    items = []
    for li in ul.find_all("li", recursive=False):
        a_tag = li.find("a")
        span = li.find("span")
        if not a_tag: continue
        href = a_tag.get("href", "")
        title = a_tag.get("title", "") or a_tag.get_text(strip=True)
        if href.startswith("./"): href = href[1:]
        if not href.startswith("http"):
            href = BASE_URL.rstrip("/") + "/" + href.lstrip("/")
        date_str = span.get_text(strip=True) if span else ""
        items.append((title, href, date_str))
    return items


def get_pagination_info(soup):
    """从JS变量获取总页数"""
    for script in soup.find_all("script"):
        text = script.string or ""
        m = re.search(r"var\s+countPage\s*=\s*(\d+)", text)
        if m: return int(m.group(1))
    return 1


def make_abs_url(base_url, src):
    if not src or src.startswith("http://") or src.startswith("https://") or src.startswith("data:"):
        return src
    parsed = urllib.parse.urlparse(base_url)
    path_dir = parsed.path.rsplit("/", 1)[0] + "/"
    if src.startswith("./"): src = src[2:]
    elif src.startswith("/"):
        return parsed.scheme + "://" + parsed.netloc + src
    return parsed.scheme + "://" + parsed.netloc + path_dir + src.lstrip("/")


def extract_content(detail_url):
    try:
        r = requests.get(detail_url, headers=HEADERS, timeout=30)
        r.encoding = "utf-8"
        soup = BeautifulSoup(r.text, "html.parser")
        div = soup.find("div", class_="deatil-detail")
        if not div:
            div = soup.find("div", class_=lambda c: c and "trs_editor_view" in str(c))
        if not div:
            div = soup.find("div", id="article")
        if not div:
            return ""
        for img in div.find_all("img"):
            src = img.get("src", "")
            if src: img["src"] = make_abs_url(detail_url, src)
        return str(div)
    except Exception as e:
        print("  [ERROR] extract content: %s" % str(e))
        return ""


def main():
    is_inc = "incremental" in sys.argv
    mode = "增量" if is_inc else "全量"
    print("=== %s模式: %s ===" % (mode, SITE_NAME))

    soup = get_soup(BASE_URL + "/index.html")
    total_pages = get_pagination_info(soup)
    print("总页数: %d" % total_pages)
    max_pages = 1 if is_inc else total_pages

    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()
    ins, skip, streak = 0, 0, 0

    for pi in range(max_pages):
        pu = BASE_URL + ("/index.html" if pi == 0 else "/index_%d.html" % pi)
        print("\n--- 第%d页 ---" % (pi+1))
        soup = get_soup(pu)
        items = extract_list_items(soup)
        print("  条目数: %d" % len(items))
        if not items: break

        has_r = False
        for title, url, ds in items:
            if ds and ds < THREE_YEARS_AGO:
                skip += 1; continue
            if ds: has_r = True

            html = extract_content(url)
            if not html:
                print("  [WARN] 无正文: %s..." % title[:40])

            try:
                c.execute("INSERT OR IGNORE INTO gov_raw (site_name, page_url, title, content, publish_date, summary, date_rank) VALUES (?,?,?,?,?,?,?)",
                    (SITE_NAME, url, title.strip(), html, ds, SITE_NAME, ds))
                if c.rowcount > 0: ins += 1
                else: skip += 1
            except Exception as e:
                print("  [ERR] %s" % str(e)); skip += 1

        if not has_r:
            streak += 1
            if streak >= 2 and pi >= 4:
                print("  连续2页无新数据，停止")
                break
        else: streak = 0

    conn.commit(); conn.close()
    print("\n=== 完成 新增:%d 跳过:%d ===" % (ins, skip))


if __name__ == "__main__": main()