#!/usr/bin/env python3
"""
牟定县人民政府 - 其他信息 (环评相关)
VSB9 CMS, 单页无分页
https://www.mdx.gov.cn/xxgk/bmxxgk/cxzsthjjmdfj/qtxx.htm
"""
import requests
import re
import sys
import os
from datetime import datetime
from urllib.parse import urljoin

DB_PATH = "/root/search.db"
BASE_URL = "https://www.mdx.gov.cn"
LIST_URL = BASE_URL + "/xxgk/bmxxgk/cxzsthjjmdfj/qtxx.htm"
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Referer": BASE_URL + "/",
}
SITE_NAME = "牟定县生态环境信息"
SITE_URL = LIST_URL


def get_list(session):
    """获取列表页全部条目"""
    r = session.get(LIST_URL, headers=HEADERS, timeout=30, verify=False)
    r.encoding = "utf-8"
    html = r.text

    items = re.findall(
        r'<li[^>]*id="line_u8_\d+"[^>]*>.*?'
        r'<span class="date">(.*?)</span>\s*'
        r'<a href="([^"]*)"[^>]*title="([^"]*)"',
        html, re.DOTALL
    )
    results = []
    for date_str, href, title in items:
        # clean date: 2026年03月10日 -> 2026-03-10
        d = re.match(r'(\d{4})年(\d{1,2})月(\d{1,2})日', date_str.strip())
        pub_date = f"{d.group(1)}-{d.group(2).zfill(2)}-{d.group(3).zfill(2)}" if d else date_str.strip()
        full_url = urljoin(LIST_URL, href)
        results.append({
            "title": title.strip(),
            "url": full_url,
            "date": pub_date,
        })
    return results


def get_detail(session, url):
    """获取详情页内容"""
    try:
        r = session.get(url, headers=HEADERS, timeout=30, verify=False)
        r.encoding = "utf-8"
        html = r.text
    except Exception as e:
        return {"content": "", "error": str(e)}

    # content
    content_m = re.search(
        r'<div[^>]*class="v_news_content"[^>]*>(.*?)</div>\s*</div>',
        html, re.DOTALL
    )
    content = content_m.group(1).strip() if content_m else ""

    if content:
        # clean inline styles but keep structure
        content = re.sub(r'\s+style="[^"]*"', '', content)
        content = re.sub(r'\s+class="[^"]*"', '', content)
        content = content.strip()

    # extract attachments from content area
    atts = re.findall(
        r'<a[^>]*href="([^"]*\.(?:pdf|doc|docx|xls|xlsx|zip)[^"]*)"[^>]*>([^<]+)',
        html
    )
    att_html = ""
    if atts:
        att_html = '<div class="attachments"><p><strong>附件：</strong></p><ul>'
        for att_href, att_name in atts:
            att_url = urljoin(url, att_href)
            att_html += f'<li><a href="{att_url}" target="_blank">{att_name.strip()}</a></li>'
        att_html += '</ul></div>'

    if att_html:
        content += att_html

    return {"content": content, "error": ""}


def save_to_db(conn, items):
    """批量入库"""
    cur = conn.cursor()
    saved = 0
    for item in items:
        title = item["title"]
        url = item["url"]
        content = item["content"]
        pub_date = item["date"]

        if not content:
            continue

        # 去重
        cur.execute("SELECT id FROM gov_raw WHERE page_url = ?", (url,))
        if cur.fetchone():
            continue

        summary = re.sub(r'<[^>]+>', '', content)[:200]
        content_plain = re.sub(r'<[^>]+>', '', content).strip()

        cur.execute(
            "INSERT OR IGNORE INTO gov_raw (title, page_url, content, summary, site_name, publish_date, source_url, category) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
            (title, url, content, summary, SITE_NAME, pub_date, url, "环评公示"),
        )

        saved += 1

    conn.commit()
    return saved


def main():
    import sqlite3
    full_mode = "--full" in sys.argv
    conn = sqlite3.connect(DB_PATH, timeout=60)
    session = requests.Session()

    # 获取列表
    items = get_list(session)
    print(f"List items: {len(items)}")

    if not full_mode:
        # 增量：检查是否有已入库
        cur = conn.cursor()
        for item in items:
            cur.execute("SELECT id FROM gov_raw WHERE page_url = ?", (item["url"],))
            if cur.fetchone():
                print("Existing record found, skipping")
                conn.close()
                return

    saved = 0
    for item in items:
        detail = get_detail(session, item["url"])
        if detail.get("error"):
            print(f"  ERROR {item['title'][:30]}: {detail['error']}")
            continue
        item["content"] = detail["content"]

    db_saved = save_to_db(conn, items)
    conn.close()
    print(f"Done. Crawled: {len(items)}, New: {db_saved}")


if __name__ == "__main__":
    main()