#!/usr/bin/env python3
import sys, re, sqlite3, requests, urllib.parse
from datetime import datetime, timedelta
from bs4 import BeautifulSoup
import os

BASE_URL = "http://ws.cq.gov.cn/bm/hbj/zwgk_70287/fdzdgknr_70290/zdxmyxpj/jsxmhpspqk"
DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "重庆市万盛经开区生态环境局-建设项目环评审批情况"
THREE_YEARS_AGO = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

def get_soup(url):
    r = requests.get(url, headers=HEADERS, timeout=30)
    r.encoding = "utf-8"
    return BeautifulSoup(r.text, "html.parser")

def extract_list_items(soup):
    ul = soup.find("ul", class_="zsj-fr-main")
    if not ul: return []
    items = []
    for li in ul.find_all("li", recursive=False):
        a_tag = li.find("a")
        if not a_tag: continue
        href = a_tag.get("href", "")
        title = a_tag.get("title", "") or a_tag.get_text(strip=True)
        if href.startswith("./"): href = href[1:]
        if not href.startswith("http"):
            href = BASE_URL.rstrip("/") + "/" + href.lstrip("/")
        span = li.find("span")
        date_str = span.get_text(strip=True) if span else ""
        items.append((title, href, date_str))
    return items

def get_pagination_info(soup):
    for script in soup.find_all("script"):
        text = script.string or ""
        m = re.search(r"createPage\s*\(\s*(\d+)", text)
        if m: return int(m.group(1))
    return 1

def make_abs_url(base_url, src):
    if not src or src.startswith("http://") or src.startswith("https://") or src.startswith("data:"):
        return src
    parsed = urllib.parse.urlparse(base_url)
    path_dir = parsed.path.rsplit("/", 1)[0] + "/"
    if src.startswith("./"):
        src = src[2:]
    elif src.startswith("/"):
        return parsed.scheme + "://" + parsed.netloc + src
    return parsed.scheme + "://" + parsed.netloc + path_dir + src.lstrip("/")

def extract_content(detail_url):
    try:
        r = requests.get(detail_url, headers=HEADERS, timeout=30)
        r.encoding = "utf-8"
        soup = BeautifulSoup(r.text, "html.parser")
        div = soup.find("div", class_=lambda c: c and "trs_editor_view" in c)
        if not div: div = soup.find("div", class_="zwxl-content")
        if not div: return ""
        for img in div.find_all("img"):
            src = img.get("src", "")
            if src:
                img["src"] = make_abs_url(detail_url, src)
        return str(div)
    except Exception as e:
        print("  [ERROR] extract content: %s" % str(e))
        return ""

def main():
    is_inc = "incremental" in sys.argv
    mode = "增量" if is_inc else "全量"
    print("=== %s模式: %s ===" % (mode, SITE_NAME))
    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()
    soup = get_soup(BASE_URL + "/index.html")
    total_pages = get_pagination_info(soup)
    print("总页数: %d" % total_pages)
    max_pages = 1 if is_inc else total_pages
    ins, skip, streak = 0, 0, 0
    for pi in range(max_pages):
        pu = BASE_URL + ("/index.html" if pi == 0 else "/index_%d.html" % pi)
        print("\n--- 第%d页 ---" % (pi+1))
        soup = get_soup(pu)
        items = extract_list_items(soup)
        print("  条目数: %d" % len(items))
        if not items: break
        has_r = False
        for title, url, ds in items:
            if ds and ds < THREE_YEARS_AGO: skip += 1; continue
            if ds: has_r = True
            html = extract_content(url)
            if not html: print("  [WARN] 无正文: %s..." % title[:40])
            try:
                c.execute("INSERT OR IGNORE INTO gov_raw (site_name, page_url, title, content, publish_date, summary, date_rank) VALUES (?,?,?,?,?,?,?)",
                    (SITE_NAME, url, title.strip(), html, ds, SITE_NAME, ds))
                if c.rowcount > 0: ins += 1
                else: skip += 1
            except Exception as e: print("  [ERR] %s" % str(e)); skip += 1
        if not has_r:
            streak += 1
            if streak >= 2 and pi >= 4:
                print("  连续2页无新数据，停止")
                break
        else: streak = 0
    conn.commit(); conn.close()
    print("\n=== 完成 新增:%d 跳过:%d ===" % (ins, skip))

if __name__ == "__main__": main()