#!/usr/bin/env python3
"""
古蔺县人民政府 - 生态环境保护
PowerCMS, 37页/364条
https://gulin.gov.cn/zt/jczwgkzl/jczwzdgkly/sthj1/sthjbhdc
"""
import requests
import re
import sys
import time
from urllib.parse import urljoin

DB_PATH = "/root/search.db"
BASE_URL = "https://gulin.gov.cn"
LIST_BASE = "/zt/jczwgkzl/jczwzdgkly/sthj1/sthjbhdc"
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Referer": BASE_URL + "/",
    "Accept-Language": "zh-CN,zh;q=0.9",
}
SITE_NAME = "古蔺县生态环境保护"
SITE_URL = BASE_URL + LIST_BASE
MAX_PAGES = 5  # 增量日跑


def get_list_page(session, page_no):
    """获取单页列表"""
    if page_no <= 1:
        url = BASE_URL + LIST_BASE
    else:
        url = f"{BASE_URL}{LIST_BASE}_{page_no}"

    r = session.get(url, headers=HEADERS, timeout=30, verify=False)
    r.encoding = "utf-8"
    html = r.text

    items = re.findall(
        r'<li[^>]*>.*?<span class="date">(\d{4}-\d{2}-\d{2})</span>\s*'
        r'<a href="([^"]*)"[^>]*title="[^"]*"[^>]*>(.*?)</a>',
        html, re.DOTALL
    )
    results = []
    for date_str, href, title in items:
        title = re.sub(r'^\s+|\s+$', '', title)
        full_url = urljoin(BASE_URL + "/", href)
        results.append({
            "title": title,
            "url": full_url,
            "date": date_str.strip(),
        })
    return results


def get_detail(session, url):
    """获取详情页内容"""
    try:
        r = session.get(url, headers=HEADERS, timeout=30, verify=False)
        r.encoding = "utf-8"
        html = r.text
    except Exception as e:
        return {"content": "", "error": str(e)}

    # conTxt content
    m = re.search(
        r'<div[^>]*class="conTxt"[^>]*>(.*?)</div>\s*</div>',
        html, re.DOTALL
    )
    content = m.group(1).strip() if m else ""

    if content:
        content = re.sub(r'\s+style="[^"]*"', '', content)
        content = re.sub(r'\s+class="[^"]*"', '', content)
        content = content.strip()

    # attachments
    atts = re.findall(
        r'<a[^>]*href="([^"]*\.(?:pdf|doc|docx|xls|xlsx|zip)[^"]*)"[^>]*>([^<]+)',
        html
    )
    att_html = ""
    if atts:
        att_html = '<div class="attachments"><p><strong>附件：</strong></p><ul>'
        for att_href, att_name in atts:
            att_url = urljoin(url, att_href)
            att_html += f'<li><a href="{att_url}" target="_blank">{att_name.strip()}</a></li>'
        att_html += '</ul></div>'
    if att_html:
        content += att_html

    return {"content": content, "error": ""}


def save_to_db(conn, items):
    cur = conn.cursor()
    saved = 0
    for item in items:
        title = item["title"]
        url = item["url"]
        content = item["content"]
        pub_date = item["date"]
        if not content:
            continue
        cur.execute("SELECT id FROM gov_raw WHERE page_url = ?", (url,))
        if cur.fetchone():
            continue

        summary = re.sub(r'<[^>]+>', '', content)[:200]
        content_plain = re.sub(r'<[^>]+>', '', content).strip()

        cur.execute(
            "INSERT OR IGNORE INTO gov_raw (title, page_url, content, summary, site_name, publish_date, source_url, category) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
            (title, url, content, summary, SITE_NAME, pub_date, url, "环评公示"),
        )

        saved += 1
    conn.commit()
    return saved


def main():
    import sqlite3
    full_mode = "--full" in sys.argv
    max_pages = 999 if full_mode else MAX_PAGES

    conn = sqlite3.connect(DB_PATH, timeout=60)
    session = requests.Session()

    total_saved = 0
    page_no = 1

    while page_no <= max_pages:
        items = get_list_page(session, page_no)
        if not items:
            print(f"Page {page_no}: no items, stopping")
            break
        print(f"Page {page_no}: {len(items)} items")

        # 增量检查
        if not full_mode:
            cur = conn.cursor()
            for item in items:
                cur.execute("SELECT id FROM gov_raw WHERE page_url = ?", (item["url"],))
                if cur.fetchone():
                    print(f"  Found existing record, stopping (incremental)")
                    conn.close()
                    print(f"Done. Pages: {page_no-1}, New: {total_saved}")
                    return

        # 获取详情
        for item in items:
            time.sleep(1.5)  # WAF限速: 不超过1请求/秒
            detail = get_detail(session, item["url"])
            if detail.get("error"):
                print(f"  ERROR {item['title'][:30]}: {detail['error']}")
                continue
            item["content"] = detail["content"]

        db_saved = save_to_db(conn, items)
        total_saved += db_saved
        print(f"  Saved {db_saved}")

        page_no += 1

    conn.close()
    print(f"\nDone. Pages: {page_no-1}, New: {total_saved}")


if __name__ == "__main__":
    main()