#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
临沂沂河新区 — 通知公告 爬虫
site: www.leda.gov.cn (山东临沂沂河新区, VSB9 可视化建站系统)
栏目: /xwzx/tzgg.htm (通知公告, 共1224条62页, 20条/页)
列表: ul.listNewsList > li#line_u9_N > a > span标题 + font日期 (YYYY-MM-DD)
分页: 反向分页! 第N页 = tzgg/{62-N+1}.htm (第2页=tzgg/61.htm, 尾页=tzgg/1.htm)
详情: /info/2297/{id}.htm, meta ArticleTitle/PubDate + div.v_news_content 正文(表格保留)
2026-08-01 新建
"""
import re
import sys
import os
import time
import sqlite3
import requests
from urllib.parse import urljoin

SITE_NAME = "临沂沂河新区-通知公告"
GROUP_NAME = "山东"
DOMAIN = "www.leda.gov.cn"
BASE_URL = "http://www.leda.gov.cn"
LIST_URL = "http://www.leda.gov.cn/xwzx/tzgg.htm"
TOTAL_PAGES = 62             # 共1224条, 20条/页 -> 62页
SCRIPT_NAME = os.path.basename(__file__)

SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
DB_PATH = SEARCH_DB

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}
TIMEOUT = 30
session = requests.Session()
session.headers.update(HEADERS)

_MAX_PAGES = 1  # 默认1页, 可 --pages=N 覆盖

# ---- 参数解析: 兼容 --pages=N 和 --pages N ----
i = 0
while i < len(sys.argv):
    a = sys.argv[i]
    if a.startswith("--pages="):
        _MAX_PAGES = int(a.split("=", 1)[1])
    elif a == "--pages" and i + 1 < len(sys.argv):
        _MAX_PAGES = int(sys.argv[i + 1]); i += 1
    i += 1
if _MAX_PAGES > TOTAL_PAGES:
    _MAX_PAGES = TOTAL_PAGES


def fetch(url):
    try:
        resp = session.get(url, timeout=TIMEOUT)
        if resp.status_code != 200:
            return None
        return resp.content.decode("utf-8-sig", errors="replace")
    except Exception:
        return None


def clean_title(title):
    """标题清洗: strip 实体前缀 + 省略号截断后缀"""
    title = re.sub(r'^[\s\xa0·\u00b7]+', '', title or '')
    title = re.sub(r'\s*\.{3,}\s*$', '', title)
    return title.strip()


def page_url(n):
    """VSB9 反向分页: 第N页 = tzgg/{TOTAL-N+1}.htm; 第1页 = tzgg.htm"""
    if n <= 1:
        return LIST_URL
    idx = TOTAL_PAGES - n + 1
    return "http://www.leda.gov.cn/xwzx/tzgg/%d.htm" % idx


def parse_list(html):
    """解析列表: ul.listNewsList > li#line_u9_N > a > span + font"""
    items = []
    m = re.search(r'<ul[^>]*class="[^"]*listNewsList[^"]*"[^>]*>(.*?)</ul>', html, re.DOTALL)
    if not m:
        return items
    ul_seg = m.group(1) + "</ul>"  # 补回 </ul> 让最后一个li的前瞻可匹配
    for li in re.finditer(r'<li[^>]*>(.*?)(?=<li\b|</ul>)', ul_seg, re.DOTALL):
        seg = li.group(1)
        a = re.search(r'<a[^>]+href="([^"]+)"[^>]*>(.*?)</a>', seg, re.DOTALL)
        if not a:
            continue
        href = a.group(1).strip()
        inner = a.group(2)
        tm = re.search(r'<span[^>]*>(.*?)</span>', inner, re.DOTALL)
        title = clean_title(re.sub(r'<[^>]+>', '', tm.group(1))) if tm else ""
        dm = re.search(r'<font[^>]*>(.*?)</font>', inner, re.DOTALL)
        date = re.sub(r'<[^>]+>', '', dm.group(1)).strip() if dm else ""
        dm2 = re.search(r'(\d{4}-\d{2}-\d{2})', inner)
        if not date and dm2:
            date = dm2.group(1)
        if not title:
            continue
        if not href.startswith("http"):
            href = urljoin(LIST_URL, href)  # ../info/2297/xxx.htm -> 根路径
        items.append({"title": title, "url": href, "date": date})
    return items


def extract_content(html, page_url):
    """提取正文: div.v_news_content; 返回 (title, date, content, has_table, att_html)"""
    title = ""
    date = ""
    m = re.search(r'<meta\s+name="ArticleTitle"\s+content="([^"]*)"', html, re.I)
    if m:
        title = clean_title(m.group(1))
    m = re.search(r'<meta\s+name="PubDate"\s+content="([^"]*)"', html, re.I)
    if m:
        d = m.group(1).strip()
        dm = re.search(r'(\d{4}-\d{2}-\d{2})', d)
        if dm:
            date = dm.group(1)
    # 正文容器
    m = re.search(r'<div[^>]*class="[^"]*v_news_content[^"]*"[^>]*>(.*?)</div>\s*(?:<|$)', html, re.DOTALL)
    if not m:
        return title, date, "", 0, ""
    seg = m.group(1)
    # 附件提取
    atts = []
    for a in re.finditer(r'<a[^>]+href="([^"]+)"[^>]*>([^<]{1,80})</a>', seg):
        href = a.group(1).strip()
        name = a.group(2).strip()
        if re.search(r'\.(pdf|docx?|xlsx?|zip|rar|wps|et|dps)(\?|$)', href.lower()) or 'upload' in href.lower():
            abs_href = urljoin(page_url, href) if not href.startswith("http") else href
            atts.append((abs_href, name))
    atts_dedup = []
    seen_h = set()
    for h, n in atts:
        if h in seen_h:
            continue
        seen_h.add(h)
        atts_dedup.append((h, n))
    # 正文分块: 表格保留HTML, 段落 \n\n 分隔
    parts = []
    has_table = 0
    tbl_ph = []
    def _tbl_ph(m):
        tbl_ph.append(m.group(0))
        return f"\x00TBL{len(tbl_ph)-1}\x00"
    seg2 = re.sub(r'<table[^>]*>.*?</table>', _tbl_ph, seg, flags=re.DOTALL)
    has_table = 1 if tbl_ph else 0
    blocks = re.split(r'(?=<\s*p[^>]*>|<\s*div[^>]*>|<\s*tr[^>]*>)', seg2)
    for blk in blocks:
        blk = re.sub(r'\x00TBL(\d+)\x00', lambda m: tbl_ph[int(m.group(1))], blk)
        if re.search(r'<table', blk):
            parts.append(blk.strip())
            continue
        txt = re.sub(r'<br\s*/?>', '\n', blk)
        txt = re.sub(r'<[^>]+>', '', txt)
        txt = re.sub(r'&nbsp;|&ensp;|&emsp;', ' ', txt)
        txt = re.sub(r'&[a-z]+;', '', txt)
        txt = re.sub(r'[ \t\u3000]+', ' ', txt)
        txt = re.sub(r'\n+', '\n', txt)
        txt = txt.strip()
        if txt:
            parts.append(txt)
    content = "\n\n".join(parts)
    content = re.sub(r'\n{3,}', '\n\n', content)
    att_html = ""
    if atts_dedup:
        att_html = "\n".join(
            '<p><a href="%s" target="_blank">%s</a></p>' % (h, n) for h, n in atts_dedup)
    return title, date, content.strip(), has_table, att_html


def store_record(cur, item):
    """入库 gov_raw + gov_search; 返回 'new'/'skip'"""
    exists = cur.execute(
        "SELECT 1 FROM gov_raw WHERE page_url=? AND site_name=?",
        (item["url"], item["site_name"])).fetchone()
    if exists:
        return "skip"
    cur.execute(
        "INSERT OR IGNORE INTO gov_raw(page_url, source_url, title, content, publish_date, site_name, group_name, script_name, has_table) "
        "VALUES(?,?,?,?,?,?,?,?,?)",
        (item["url"], item["source_url"], item["title"], item["content"],
         item["date"], item["site_name"], item["group_name"],
         item["script_name"], item["has_table"]))
    if cur.rowcount > 0:
        rid = cur.execute("SELECT id FROM gov_raw WHERE page_url=?", (item["url"],)).fetchone()[0]
        summary = re.sub(r'<[^>]+>', '', item["content"] or "")[:200]
        cur.execute(
            "INSERT OR IGNORE INTO gov_search(rowid, title, site_name, summary) VALUES(?,?,?,?)",
            (rid, item["title"], item["site_name"], summary))
        return "new"
    return "skip"


def main():
    conn = sqlite3.connect(DB_PATH, timeout=60)
    cur = conn.cursor()
    n_new = 0
    n_skip = 0
    n_fail = 0
    fail_streak = 0
    seen_urls = set()
    for page in range(1, _MAX_PAGES + 1):
        url = page_url(page)
        html = fetch(url)
        if not html:
            fail_streak += 1
            if fail_streak >= 1 and page > 1:
                print("Page %d 请求失败, 提前停止" % page)
                break
            continue
        fail_streak = 0
        items = parse_list(html)
        if not items:
            print("Page %d 无列表数据" % page)
            break
        page_new = 0
        for it in items:
            if it["url"] in seen_urls:
                continue
            seen_urls.add(it["url"])
            detail = fetch(it["url"])
            if not detail:
                n_fail += 1
                continue
            title, date, content, has_table, att_html = extract_content(detail, it["url"])
            if not title:
                title = it["title"]
            if not date:
                date = it["date"]
            if not content and not att_html:
                n_skip += 1
                continue
            if att_html:
                content = (content + "\n\n" + att_html).strip() if content else att_html
            item = {
                "url": it["url"],
                "title": title,
                "content": content,
                "date": date,
                "site_name": SITE_NAME,
                "group_name": GROUP_NAME,
                "source_url": DOMAIN,
                "script_name": SCRIPT_NAME,
                "has_table": has_table,
            }
            r = store_record(cur, item)
            if r == "new":
                n_new += 1
                page_new += 1
            else:
                n_skip += 1
            time.sleep(0.3)
        print("Page %d: 新增 %d, 累计新增 %d" % (page, page_new, n_new))
    conn.commit()
    conn.close()
    print("完成: 总计 %d | 新增: %d | 跳过: %d | 失败: %d" % (_MAX_PAGES, n_new, n_skip, n_fail))


if __name__ == "__main__":
    main()