#!/usr/bin/env python3
"""Crawler for 滕州市-枣庄市生态环境局滕州分局-通知公告
   http://www.tengzhou.gov.cn/zwgk/xxgkml/szbm/zzssthjjtzfj/
   TRS CMS /irs/front/search 接口 (filters 组配分类+部门)
   列表: POST /irs/front/search 71条/4页 (20条每页)
   详情: curl 直连, 正文 div.zwnr, 标题 <title>, 日期 成文时间
"""
import requests, re, sqlite3, json, sys, time
from urllib.parse import urljoin

BASE = "http://www.tengzhou.gov.cn"
SEARCH_URL = BASE + "/irs/front/search"
DB = "/root/search.db"
SITE_NAME = "滕州市生态环境分局-通知公告"
SCRIPT_NAME = "crawl_tengzhou_sthjj.py"
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"}
CUTOFF = "2023-08-18"

SEARCH_BODY = {
    "appendixType": "", "beginDateTime": "", "code": "18d72ce8cb5", "codes": "",
    "configCode": "", "dataTypeId": "2849", "endDateTime": "",
    "filters": [
        {"aggrName": "f_2024329561714", "aggrValues": "通知公告", "filterId": "1"},
        {"aggrName": "f_2024329501386", "aggrValues": "枣庄市生态环境局滕州分局", "filterId": "1"}
    ],
    "granularity": "ALL", "isSearchForced": 0, "historySearchWords": ["通知公告"],
    "orderBy": "time", "pageNo": 1, "pageSize": 20, "searchBy": "all"
}


def log(msg):
    print(f"[{SITE_NAME}] {msg}", flush=True)


def fetch_search(page):
    body = dict(SEARCH_BODY)
    body["pageNo"] = page
    r = requests.post(SEARCH_URL, json=body, headers=HEADERS, timeout=30)
    if r.status_code != 200:
        log(f"搜索接口 HTTP {r.status_code}")
        return None
    return r.json()


def parse_list(data):
    """从搜索响应提取 (url, title, date)"""
    items = []
    middle = data.get("data", {}).get("middle", {})
    for entry in middle.get("listAndBox", []):
        d = entry.get("data", {})
        url = d.get("url", "")
        title = d.get("title_no_tag") or d.get("title", "")
        t = d.get("table-2") or d.get("time", "")  # 2026-08-11 16:27:40
        date = t[:10] if t else ""
        if url and title:
            items.append({"url": url, "title": title.strip(), "date": date})
    return items


def fetch_detail(url):
    """详情页: 标题/正文/日期/文号/附件"""
    try:
        r = requests.get(url, headers=HEADERS, timeout=30)
        r.encoding = "utf-8"
        html = r.text
    except Exception as e:
        log(f"详情请求失败 {url.split('/')[-1]}: {str(e)[:50]}")
        return None
    result = {"title": "", "content": "", "date": "", "doc_no": "", "attachments": [], "has_table": False}
    # 标题: <title>滕州市政府信息公开--xxx</title>
    m = re.search(r"<title>(.*?)</title>", html, re.DOTALL)
    if m:
        t = m.group(1).strip().replace("滕州市政府信息公开--", "").replace("_滕州市政府信息公开", "").strip()
        if t:
            result["title"] = t
    # 正文: div.zwnr
    m = re.search(r'<div[^>]*class="zwnr"[^>]*>(.*?)</div>', html, re.DOTALL)
    if not m:
        m = re.search(r'<div[^>]*class="view[^"]*"[^>]*>(.*?)</div>', html, re.DOTALL)
    if m:
        inner = m.group(1)
        result["has_table"] = bool(re.search(r"<table", inner))
        inner = re.sub(r"<script[^>]*>.*?</script>", "", inner, flags=re.DOTALL)
        inner = re.sub(r"<style[^>]*>.*?</style>", "", inner, flags=re.DOTALL)
        result["content"] = inner.strip()
        # 正文内附件
        for am in re.finditer(r'<a[^>]*href="([^"]+)"[^>]*>([^<]{2,80})</a>', inner):
            href, txt = am.group(1), am.group(2).strip()
            if any(k in txt for k in ["附件", "下载", "文件"]) or re.search(r"\.(docx?|xlsx?|pdf|zip|rar)$", href, re.I):
                full = urljoin(BASE, href)
                result["attachments"].append({"name": txt, "url": full})
    # 日期: 成文时间
    m = re.search(r"成文时间[：:]\s*<[^>]*>\s*([\d]{4})年([\d]{1,2})月([\d]{1,2})日", html)
    if not m:
        m = re.search(r"([\d]{4})年([\d]{1,2})月([\d]{1,2})日", html)
    if m:
        result["date"] = f"{m.group(1)}-{int(m.group(2)):02d}-{int(m.group(3)):02d}"
    # 文号
    m = re.search(r"文\s*号[：:]\s*<[^>]*>\s*([^<\s]{3,40})", html)
    if m:
        result["doc_no"] = m.group(1).strip()
    return result


def main():
    pages = 1
    if "--pages" in sys.argv:
        i = sys.argv.index("--pages")
        if i + 1 < len(sys.argv):
            pages = int(sys.argv[i + 1])
    total_new = 0
    total_skip = 0
    total_fail = 0
    t0 = time.time()

    conn = sqlite3.connect(DB, timeout=30)
    cur = conn.cursor()

    for pg in range(1, pages + 1):
        data = fetch_search(pg)
        if not data:
            log(f"页{pg} 接口失败")
            total_fail += 1
            continue
        pager = data.get("data", {}).get("pager", {})
        total = pager.get("total", 0)
        page_count = pager.get("pageCount", 0)
        log(f"页{pg}/{page_count}: 共{total}条")
        items = parse_list(data)
        if not items:
            log(f"页{pg} 无条目，停止")
            break
        for item in items:
            cur.execute("SELECT id FROM gov_raw WHERE page_url=?", (item["url"],))
            if cur.fetchone():
                total_skip += 1
                continue
            det = fetch_detail(item["url"])
            if not det or not det["title"] or not det["content"]:
                if det and det["title"]:
                    det["content"] = f'<p><a href="{item["url"]}">{det["title"]}</a></p>'
                else:
                    log(f"  详情失败: {item['url'].split('/')[-1]}")
                    total_fail += 1
                    continue
            date = det["date"] or item["date"] or "1900-01-01"
            cur.execute("""
                INSERT OR IGNORE INTO gov_raw
                (site_name, source_url, page_url, title, publish_date, date_rank, summary, status,
                 category, visits, content, tags, industry, attachments, group_name, has_table, script_name)
                VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
            """, (
                SITE_NAME, SEARCH_URL, item["url"], det["title"], date,
                date, det["content"][:200].replace("<", "").replace(">", ""),
                "published", "通知公告", 0, det["content"],
                det["doc_no"], "", json.dumps(det["attachments"], ensure_ascii=False),
                "山东", 1 if det["has_table"] else 0, SCRIPT_NAME
            ))
            if cur.rowcount > 0:
                total_new += 1
            else:
                total_skip += 1
            conn.commit()
            if total_new % 10 == 0 and total_new > 0:
                log(f"  进度: 新增{total_new} 跳过{total_skip} 失败{total_fail} ({time.time()-t0:.0f}s)")

    conn.close()
    log(f"完成: 新增{total_new} 跳过{total_skip} 失败{total_fail} 用时{time.time()-t0:.0f}s")


if __name__ == "__main__":
    main()