#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""高昌区-公示公告爬虫 (2026-08-21 改版重写: zfxxgk_list.shtml → list.shtml)

改版变化:
- 列表页: /gcq/c105962/list.shtml, 分页 list_N.shtml (共63页约943条)
- 列表项: <a href="/gcq/c105962/YYYYMM/<hash>.shtml" target="_blank" title='单引号标题'>
- 详情标题: meta name="ArticleTitle" (UCAPTITLE 已移除)
- 正文: div.v_news_content (保留)
- 日期: meta PubDate content (保留)
"""
import re, os, sys, urllib.request, ssl, sqlite3
from datetime import datetime, timedelta

ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE

SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "高昌区-公示公告"
BASE = "http://gcq.tlf.gov.cn"
CATID = 105962
CUTOFF = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")

def fetch(url):
    req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
    resp = urllib.request.urlopen(req, timeout=30, context=ctx)
    return resp.read().decode("utf-8", errors="ignore")

def fetch_list(page=1):
    """新版列表: list.shtml / list_N.shtml, 返回 [(title, url, date)]"""
    if page == 1:
        url = "{}/gcq/c{}/list.shtml".format(BASE, CATID)
    else:
        url = "{}/gcq/c{}/list_{}.shtml".format(BASE, CATID, page)
    html = fetch(url)
    # 列表项: href 指向 /gcq/c105962/YYYYMM/hash.shtml, title 用单引号
    items = re.findall(
        r"<a href=\"(/gcq/c\d+/20\d{4}/[0-9a-f]+\.shtml)\"[^>]*title='([^']*)'",
        html
    )
    out = []
    for url, title in items:
        title = title.strip()
        if not title:
            continue
        # 日期从 URL 路径提取 (YYYYMM)
        m = re.search(r"/(20\d{4})/", url)
        date = ""
        if m:
            yyyymm = m.group(1)
            date = "{}-{}-01".format(yyyymm[:4], yyyymm[4:6])
        out.append((title, url, date))
    return out

def fetch_detail(url):
    full_url = url if url.startswith("http") else BASE + url
    html = fetch(full_url)
    # 标题: meta ArticleTitle
    title = ""
    m = re.search(r'<meta name="ArticleTitle" content="([^"]*)"', html)
    if m:
        title = m.group(1).strip()
    if not title:
        # 兜底 <title>标题-站名</title>
        m = re.search(r"<title>\s*([^<]+?)\s*</title>", html)
        if m:
            title = re.sub(r"[-–—|｜]\s*[^<]*$", "", m.group(1)).strip()
    content = "正文为空"
    m = re.search(r'class="v_news_content"[^>]*>(.*?)</div>', html, re.DOTALL)
    if m:
        c = m.group(1).strip()
        if len(c) > 50:
            content = c
    if content == "正文为空":
        m = re.search(r'id="vsb_content"[^>]*>(.*?)</div>', html, re.DOTALL)
        if m:
            c = m.group(1).strip()
            if len(c) > 50:
                content = c
    if content != "正文为空":
        content = re.sub(r"<UCAPCONTENT>|</UCAPCONTENT>", "", content)
        content = re.sub(r"<script[^>]*>.*?</script>", "", content, flags=re.DOTALL)
        content = re.sub(r"<iframe[^>]*>.*?</iframe>", "", content, flags=re.DOTALL)
        content = content.strip()
        if not content: content = "正文为空"
    date = ""
    m = re.search(r"发布时间[：:]\s*(\d{4}-\d{2}-\d{2})", html)
    if m: date = m.group(1)
    if not date:
        m = re.search(r'PubDate" content="(\d{4}-\d{2}-\d{2})', html)
        if m: date = m.group(1)
    title = re.sub(r"<[^>]+>", "", title).strip()
    return title, date, content

def main():
    pages = 5
    if "--pages" in sys.argv:
        i = sys.argv.index("--pages")
        try:
            pages = int(sys.argv[i+1])
        except (IndexError, ValueError):
            pass
    conn = sqlite3.connect(SEARCH_DB, timeout=60)
    cur = conn.cursor()
    conn.execute("PRAGMA journal_mode=WAL")
    total_new = 0
    for page in range(1, pages + 1):
        try:
            items = fetch_list(page)
        except Exception as e:
            print("  Page {} ERR: {}".format(page, str(e)[:80]))
            continue
        if not items:
            print("  Page {} 无列表项(可能已到末页)".format(page))
            break
        for title, url, date in items:
            page_url = url if url.startswith("http") else BASE + url
            cur.execute("SELECT id FROM gov_raw WHERE page_url=?", (page_url,))
            if cur.fetchone():
                continue
            try:
                detail_title, pub_date, content = fetch_detail(url)
                if content == "正文为空":
                    continue
                final_date = pub_date or date
                if final_date and final_date < CUTOFF:
                    continue
                cur.execute(
                    "INSERT OR IGNORE INTO gov_raw (site_name, page_url, title, publish_date, content, date_rank, category, script_name) VALUES (?,?,?,?,?,?,?,?)",
                    (SITE_NAME, page_url, detail_title or title, final_date, content,
                     int(final_date.replace("-", "")) if final_date else 0, "公示公告", "crawl_gcq.py")
                )
                if cur.rowcount > 0:
                    total_new += 1
                    print("  NEW: {} | {}".format((detail_title or title)[:60], final_date))
            except Exception as e:
                print("  ERR: {} - {}".format(title[:30], str(e)[:60]))
        conn.commit()
        print("  Page {} done".format(page))
    conn.close()
    print("高昌区-公示公告 新增: {} 条".format(total_new))

if __name__ == "__main__":
    main()