#!/usr/bin/env python3
"""潜江市人民政府 - 建设项目环境影响评价公示 爬虫
   https://www.hbqj.gov.cn/xxgk/xxgkml/szfxxgkml/gysyjs/hjbh/jsxmhjyxpjgs/index.html
   CMS: TRS, 静态分页 index_N.html, 26页, 15条/页
   正文: div.view.TRS_UEDITOR.trs_paper_default.trs_word
"""
import urllib.request, urllib.parse, ssl, re, sqlite3, sys, os, time
from datetime import datetime, timedelta

DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "潜江市-建设项目环评公示"
BASE_URL = "https://www.hbqj.gov.cn"
LIST_PREFIX = "/xxgk/xxgkml/szfxxgkml/gysyjs/hjbh/jsxmhjyxpjgs/"
TOTAL_PAGES = 26  # createPageHTML(26, 0, "index", "html", ...)
ITEMS_PER_PAGE = 15
CUTOFF = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

ssl_ctx = ssl.create_default_context()
ssl_ctx.check_hostname = False
ssl_ctx.verify_mode = ssl.CERT_NONE


def fetch(url):
    """安全抓取页面"""
    try:
        req = urllib.request.Request(url, headers=HEADERS)
        r = urllib.request.urlopen(req, context=ssl_ctx, timeout=30)
        # TRS sites use utf-8, but sometimes miss charset in response
        content = r.read()
        try:
            return content.decode("utf-8")
        except:
            return content.decode("utf-8", errors="replace")
    except Exception as e:
        print(f"[warn] fetch失败: {url[:80]} -> {str(e)[:60]}")
        return ""


def parse_list(html):
    """解析列表页返回items列表"""
    items = []
    # 找 ul.info-list 区域
    ul_match = re.search(r'<ul\s+class="info-list"[^>]*>(.*?)</ul>', html, re.DOTALL)
    if not ul_match:
        return items
    ul_html = ul_match.group(1)
    for li_match in re.finditer(r'<li[^>]*>(.*?)</li>', ul_html, re.DOTALL):
        li = li_match.group(1)
        a_match = re.search(r'<a\s+href="([^"]+)"[^>]*title="([^"]*)"', li)
        if not a_match:
            continue
        href = a_match.group(1).strip()
        title = a_match.group(2).strip()
        # 跳过无效链接
        if href.startswith("javascript:") or "void" in href or href.startswith("#"):
            continue
        # 日期
        date_match = re.search(r'<span>([\d-]+)</span>', li)
        date_str = date_match.group(1) if date_match else ""
        if not href.startswith("http"):
            href = urllib.parse.urljoin(BASE_URL, href)
        items.append({"title": title, "url": href, "date": date_str})
    return items


def fetch_detail(url):
    """获取详情页正文"""
    html = fetch(url)
    if not html:
        return ""
    # 正文: div.view.TRS_UEDITOR.trs_paper_default.trs_word
    m = re.search(r'<div\s+class="view\s+TRS_UEDITOR[^"]*"[^>]*>(.*?)</div>\s*<div\s+class="article_appendix', html, re.DOTALL)
    if m:
        inner = m.group(1)
    else:
        # 回退: div#detailCont
        m = re.search(r'<div[^>]*id="detailCont"[^>]*>(.*?)</div>\s*<div\s+class="article_appendix', html, re.DOTALL)
        if m:
            inner = m.group(1)
        else:
            # 再回退: div.xl_tit_box
            m = re.search(r'<div[^>]*class="xl_tit_box[^"]*"[^>]*>(.*?)</div>\s*<div\s+class="article_appendix', html, re.DOTALL)
            if m:
                inner = m.group(1)
            else:
                return ""
    # 提取纯文本段落
    parts = []
    for p in re.findall(r'<p[^>]*>(.*?)</p>', inner, re.DOTALL):
        txt = re.sub(r'<[^>]+>', '', p).strip()
        if txt:
            parts.append(txt)
    if not parts:
        txt = re.sub(r'<[^>]+>', '', inner).strip()
        parts = [txt]
    return "\n\n".join(parts)


def main():
    INCREMENTAL = "--full" not in sys.argv
    mode = "增量" if INCREMENTAL else "全量"
    print(f"=== {SITE_NAME} === (模式: {mode}, cutoff={CUTOFF})")

    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()

    new_total = 0
    dup_total = 0
    old_total = 0
    err_total = 0

    for page in range(TOTAL_PAGES):
        if page == 0:
            url = urllib.parse.urljoin(BASE_URL, LIST_PREFIX + "index.html")
        else:
            url = urllib.parse.urljoin(BASE_URL, f"{LIST_PREFIX}index_{page}.html")

        html = fetch(url)
        if not html:
            print(f"  [page {page+1}/{TOTAL_PAGES}] 抓取失败，跳过")
            continue

        items = parse_list(html)
        page_new = 0
        for item in items:
            # 日期过滤
            if INCREMENTAL and item["date"] and item["date"] < CUTOFF:
                old_total += 1
                continue

            # 去重
            c.execute("SELECT COUNT(*) FROM gov_raw WHERE page_url=?", (item["url"],))
            if c.fetchone()[0] > 0:
                dup_total += 1
                continue

            # 获取详情
            content = fetch_detail(item["url"])
            if not content or len(content) < 50:
                print(f"  [warn] 正文过短: {item['title'][:40]}...")
                err_total += 1
                continue

            # 入库
            publish_date = item["date"] if item["date"] else datetime.now().strftime("%Y-%m-%d")
            c.execute(
                "INSERT INTO gov_raw (page_url, title, content, site_name, publish_date, source_url, category) "
                "VALUES (?, ?, ?, ?, ?, ?, ?)",
                (item["url"], item["title"], content, SITE_NAME, publish_date, item["url"], "环评公示")
            )
            page_new += 1
            new_total += 1

        conn.commit()
        print(f"  [page {page+1}/{TOTAL_PAGES}] +{page_new} new, total: {new_total}")
        time.sleep(0.5)

    conn.close()
    print(f"\n[完成] {SITE_NAME}: 新增 {new_total} 条, 旧跳过 {old_total}, 重复 {dup_total}, 错误 {err_total}")


if __name__ == "__main__":
    main()