#!/usr/bin/env python3
"""crawl_zz_hb.py v2 - 郑州市生态环境局政务公开 (合并 zz_collect + 详情提取)
   自建CMS (JEECMS风格): /?a=dir&d=24&page=N 分页, 529页/10条
   详情: /D44Y/4927267.jhtml (静态HTML, 无需playwright, 2026-08-21实测服务器直连200)
   正文: div[class*=content], 标题 <title>
"""
import requests, re, sqlite3, json, os, sys, time
from urllib.parse import urljoin

DB = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "郑州市生态环境局-政务公开"
SCRIPT_NAME = "crawl_zz_hb.py"
BASE_URL = "https://public.zhengzhou.gov.cn"
LIST_URL = BASE_URL + "/?a=dir&d=24"
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"
CUTOFF = "2023-08-18"

HEADERS = {"User-Agent": UA, "Accept": "text/html,application/xhtml+xml"}

stats = {"new": 0, "skip": 0, "err": 0}


def get_list(page):
    url = LIST_URL if page == 1 else f"{LIST_URL}&page={page}"
    try:
        r = requests.get(url, headers=HEADERS, timeout=20, verify=False)
        if r.status_code != 200:
            return None
        return r.text
    except Exception as e:
        print(f"  列表页{page}错误: {e}", file=sys.stderr)
        return None


def extract_list_items(html, page_url):
    items = []
    seen = set()
    # 详情链接: 只匹配 href（标题可能跨行/在子元素），标题详情页再取
    for m in re.finditer(r'href="(https://public\.zhengzhou\.gov\.cn/D\w+/\d+\.jhtml)"', html):
        url = m.group(1)
        if url in seen:
            continue
        seen.add(url)
        items.append({"url": url, "title": ""})
    return items


def extract_detail(html, url):
    result = {"title": "", "content": "", "date": ""}
    m = re.search(r'<title[^>]*>(.*?)</title>', html, re.DOTALL)
    if m:
        result["title"] = m.group(1).strip()
    # 正文容器: 精确匹配 content-txt（实测 2026-08-21）
    for pattern in [r'<div[^>]*class="content-txt[^"]*"[^>]*>(.*?)</div>',
                    r'<div[^>]*class="[^"]*content[^"]*"[^>]*>(.*?)</div>',
                    r'<div[^>]*id="zoom"[^>]*>(.*?)</div>',
                    r'<div[^>]*class="TRS_Editor"[^>]*>(.*?)</div>',
                    r'<div[^>]*class="news_content"[^>]*>(.*?)</div>']:
        m = re.search(pattern, html, re.DOTALL)
        if m:
            inner = m.group(1)
            inner = re.sub(r'<script[^>]*>.*?</script>', '', inner, flags=re.DOTALL)
            inner = re.sub(r'<style[^>]*>.*?</style>', '', inner, flags=re.DOTALL)
            text_len = len(re.sub(r'<[^>]+>', '', inner).strip())
            if text_len >= 50:
                result["content"] = inner.strip()
                break
    # 日期
    m = re.search(r'(\d{4}-\d{2}-\d{2})', html)
    if m:
        result["date"] = m.group(1)
    return result


def main():
    max_pages = 3
    for i, a in enumerate(sys.argv):
        if a == "--pages" and i + 1 < len(sys.argv):
            max_pages = int(sys.argv[i + 1])
        elif a.startswith("--pages="):
            max_pages = int(a.split("=", 1)[1])
    print(f"=== 郑州政务公开 列表收集+详情 ===", flush=True)

    conn = sqlite3.connect(DB, timeout=30)
    cur = conn.cursor()
    t0 = time.time()

    # 先访问首页拿总页数
    first = get_list(1)
    if not first:
        print("首页获取失败")
        return
    total_pages = 1
    m = re.findall(r'\?a=dir&d=24&page=(\d+)', first)
    if m:
        total_pages = max(int(x) for x in m)
    print(f"总页数: {total_pages}, 抓前 {max_pages} 页", flush=True)

    for page in range(1, min(max_pages, total_pages) + 1):
        html = first if page == 1 else get_list(page)
        if not html:
            stats["err"] += 1
            continue
        items = extract_list_items(html, LIST_URL)
        if not items:
            print(f"  页{page} 无条目，停止")
            break
        print(f"  页{page}: {len(items)} 条", flush=True)
        for item in items:
            url = item["url"]
            cur.execute("SELECT id FROM gov_raw WHERE page_url=?", (url,))
            if cur.fetchone():
                stats["skip"] += 1
                continue
            try:
                r = requests.get(url, headers=HEADERS, timeout=20, verify=False)
                if r.status_code != 200:
                    stats["err"] += 1
                    continue
                det = extract_detail(r.text, url)
            except Exception as e:
                stats["err"] += 1
                continue
            if not det["title"]:
                stats["err"] += 1
                continue
            content = det["content"] or f'<p><a href="{url}">{det["title"]}</a></p>'
            try:
                cur.execute("""
                    INSERT OR IGNORE INTO gov_raw
                    (site_name, source_url, page_url, title, publish_date, date_rank, summary, status,
                     category, visits, content, tags, industry, attachments, group_name, has_table, script_name)
                    VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
                """, (
                    SITE_NAME, LIST_URL, url, det["title"], det["date"] or "1900-01-01",
                    det["date"] or "1900-01-01", re.sub(r'<[^>]+>', '', content)[:200],
                    "published", "政务公开", 0, content, "", "", "[]",
                    "河南", 1 if "<table" in content else 0, SCRIPT_NAME
                ))
                if cur.rowcount > 0:
                    stats["new"] += 1
                else:
                    stats["skip"] += 1
                conn.commit()
            except Exception as e:
                print(f"  DB错误: {e}", file=sys.stderr)
                stats["err"] += 1
        time.sleep(1)

    conn.close()
    print(f"完成: 新增{stats['new']} 跳过{stats['skip']} 错误{stats['err']} 用时{time.time()-t0:.0f}s", flush=True)


if __name__ == "__main__":
    import urllib3
    urllib3.disable_warnings()
    main()