#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
天津环评公示平台（欣国环环保科技）— 信息公示 爬虫
site: www.tjxgh.cn (ThinkPHP)
栏目: /index.php/category/index/id/49.html (信息公示, 共48页568条, 12条/页)
列表: ul.news1 > li > a(title完整标题, class=pg-color) + span 日期 (YYYY-MM-DD)
分页: /index.php/category/index/id/49/p/{N}.html
详情: /index.php/arc/show/id/{N}.html
详情结构: div.view > h1.view-title 标题 + p.MsoNormal 段落(Word转换HTML) + 可能表格/附件
编码: 页面声明utf-8但含个别损坏字节 -> decode('utf-8', errors='replace')
2026-08-01 新建
"""
import re
import sys
import os
import time
import sqlite3
import requests
from urllib.parse import urljoin

SITE_NAME = "天津环评公示平台-信息公示"
GROUP_NAME = "天津"
DOMAIN = "www.tjxgh.cn"
BASE_URL = "http://www.tjxgh.cn"
LIST_URL = "http://www.tjxgh.cn/index.php/category/index/id/49.html"
TOTAL_PAGES = 48             # 分页显示共48页568条
SCRIPT_NAME = os.path.basename(__file__)

SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
DB_PATH = SEARCH_DB

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}
TIMEOUT = 30
session = requests.Session()
session.headers.update(HEADERS)

_MAX_PAGES = 1  # 默认1页, 可 --pages=N 覆盖

# ---- 参数解析: 兼容 --pages=N 和 --pages N ----
i = 0
while i < len(sys.argv):
    a = sys.argv[i]
    if a.startswith("--pages="):
        _MAX_PAGES = int(a.split("=", 1)[1])
    elif a == "--pages" and i + 1 < len(sys.argv):
        _MAX_PAGES = int(sys.argv[i + 1]); i += 1
    i += 1
if _MAX_PAGES > TOTAL_PAGES:
    _MAX_PAGES = TOTAL_PAGES


def fetch(url):
    try:
        resp = session.get(url, timeout=TIMEOUT)
        if resp.status_code != 200:
            return None
        # 页面声明 utf-8 但含个别损坏字节
        return resp.content.decode("utf-8", errors="replace")
    except Exception:
        return None


def clean_title(title):
    """标题清洗: strip 实体前缀 + 省略号截断后缀"""
    title = re.sub(r'^[\s\xa0·\u00b7]+', '', title or '')
    title = re.sub(r'\s*\.{3,}\s*$', '', title)
    return title.strip()


def parse_list(html):
    """解析列表: ul.news1 > li > a + span日期"""
    items = []
    m = re.search(r'<ul[^>]*class="[^"]*news1[^"]*"[^>]*>(.*?)</ul>', html, re.DOTALL)
    if not m:
        return items
    for li in re.finditer(r'<li[^>]*>(.*?)</li>', m.group(1), re.DOTALL):
        seg = li.group(1)
        a = re.search(r'<a[^>]+href="([^"]+)"[^>]*title="([^"]*)"', seg)
        if not a:
            continue
        href, title = a.group(1), clean_title(a.group(2))
        if not title:
            continue
        if not href.startswith("http"):
            href = urljoin(BASE_URL, href)
        d = re.search(r'(\d{4}-\d{2}-\d{2})', seg)
        date = d.group(1) if d else ""
        items.append({"title": title, "url": href, "date": date})
    return items


def extract_content(html, page_url):
    """提取正文: div.view > h1.view-title + MsoNormal段落; 返回 (title, date, content, has_table, att_html)"""
    title = ""
    date = ""
    # 标题 h1.view-title
    m = re.search(r'<h1[^>]*class="[^"]*view-title[^"]*"[^>]*>(.*?)</h1>', html, re.DOTALL)
    if m:
        title = clean_title(re.sub(r'<[^>]+>', '', m.group(1)))
    # 正文容器 div.view
    i1 = html.find('<div class="view">')
    i2 = html.find('<div class="page"', i1)
    i3 = html.find('<div class="footer', i1)
    ends = [x for x in [i2, i3] if x > 0]
    end = min(ends) if ends else i1 + 100000
    seg = html[i1:end] if i1 >= 0 else ""
    if not seg:
        return title, date, "", 0, ""
    # 附件提取（正文内 <a href="...pdf/doc">）
    atts = []
    for a in re.finditer(r'<a[^>]+href="([^"]+)"[^>]*>([^<]{1,80})</a>', seg):
        href = a.group(1).strip()
        name = a.group(2).strip()
        if re.search(r'\.(pdf|docx?|xlsx?|zip|rar|wps|et|dps)(\?|$)', href.lower()) or 'uploads' in href.lower():
            abs_href = urljoin(page_url, href) if not href.startswith("http") else href
            atts.append((abs_href, name))
    # 去重
    atts_dedup = []
    seen_h = set()
    for h, n in atts:
        if h in seen_h:
            continue
        seen_h.add(h)
        atts_dedup.append((h, n))
    # 正文: 按块提取, 保留表格HTML, 段落 \n\n 分隔
    parts = []
    has_table = 0
    body = seg
    # 去掉 h1 标题行
    body = re.sub(r'<h1[^>]*class="[^"]*view-title[^"]*"[^>]*>.*?</h1>', '', body, flags=re.DOTALL)
    # 逐块处理 <table> / <p> / <div> / <br>
    # 用占位符保护表格
    tbl_ph = []
    def _tbl_ph(m):
        tbl_ph.append(m.group(0))
        return f"\x00TBL{len(tbl_ph)-1}\x00"
    body = re.sub(r'<table[^>]*>.*?</table>', _tbl_ph, body, flags=re.DOTALL)
    has_table = 1 if tbl_ph else 0
    # 分块
    blocks = re.split(r'(?=<\s*p[^>]*>|<\s*div[^>]*>)', body)
    for blk in blocks:
        # 还原表格
        blk = re.sub(r'\x00TBL(\d+)\x00', lambda m: tbl_ph[int(m.group(1))], blk)
        if re.search(r'<table', blk):
            parts.append(blk.strip())
            continue
        txt = re.sub(r'<br\s*/?>', '\n', blk)
        txt = re.sub(r'<[^>]+>', '', txt)
        txt = re.sub(r'&nbsp;|&ensp;|&emsp;', ' ', txt)
        txt = re.sub(r'&[a-z]+;', '', txt)
        txt = re.sub(r'[ \t\u3000]+', ' ', txt)
        txt = re.sub(r'\n+', '\n', txt)
        txt = txt.strip()
        if txt:
            parts.append(txt)
    content = "\n\n".join(parts)
    content = re.sub(r'\n{3,}', '\n\n', content)
    # 附件转 <p><a> 名称内嵌URL
    att_html = ""
    if atts_dedup:
        att_html = "\n".join(
            '<p><a href="%s" target="_blank">%s</a></p>' % (h, n) for h, n in atts_dedup)
    return title, date, content.strip(), has_table, att_html


def store_record(cur, item):
    """入库 gov_raw + gov_search; 返回 'new'/'skip'"""
    exists = cur.execute(
        "SELECT 1 FROM gov_raw WHERE page_url=? AND site_name=?",
        (item["url"], item["site_name"])).fetchone()
    if exists:
        return "skip"
    cur.execute(
        "INSERT OR IGNORE INTO gov_raw(page_url, source_url, title, content, publish_date, site_name, group_name, script_name, has_table) "
        "VALUES(?,?,?,?,?,?,?,?,?)",
        (item["url"], item["source_url"], item["title"], item["content"],
         item["date"], item["site_name"], item["group_name"],
         item["script_name"], item["has_table"]))
    if cur.rowcount > 0:
        rid = cur.execute("SELECT id FROM gov_raw WHERE page_url=?", (item["url"],)).fetchone()[0]
        summary = re.sub(r'<[^>]+>', '', item["content"] or "")[:200]
        cur.execute(
            "INSERT OR IGNORE INTO gov_search(rowid, title, site_name, summary) VALUES(?,?,?,?)",
            (rid, item["title"], item["site_name"], summary))
        return "new"
    return "skip"


def main():
    conn = sqlite3.connect(DB_PATH, timeout=60)
    cur = conn.cursor()
    n_new = 0
    n_skip = 0
    n_fail = 0
    fail_streak = 0
    seen_urls = set()
    for page in range(1, _MAX_PAGES + 1):
        url = LIST_URL if page == 1 else \
            "http://www.tjxgh.cn/index.php/category/index/id/49/p/%d.html" % page
        html = fetch(url)
        if not html:
            fail_streak += 1
            if fail_streak >= 1 and page > 1:
                print("Page %d 请求失败, 提前停止" % page)
                break
            continue
        fail_streak = 0
        items = parse_list(html)
        if not items:
            print("Page %d 无列表数据" % page)
            break
        page_new = 0
        for it in items:
            if it["url"] in seen_urls:
                continue
            seen_urls.add(it["url"])
            detail = fetch(it["url"])
            if not detail:
                n_fail += 1
                continue
            title, date, content, has_table, att_html = extract_content(detail, it["url"])
            if not title:
                title = it["title"]
            if not date:
                date = it["date"]
            if not content and not att_html:
                n_skip += 1
                continue
            if att_html:
                content = (content + "\n\n" + att_html).strip() if content else att_html
            item = {
                "url": it["url"],
                "title": title,
                "content": content,
                "date": date,
                "site_name": SITE_NAME,
                "group_name": GROUP_NAME,
                "source_url": DOMAIN,
                "script_name": SCRIPT_NAME,
                "has_table": has_table,
            }
            r = store_record(cur, item)
            if r == "new":
                n_new += 1
                page_new += 1
            else:
                n_skip += 1
            time.sleep(0.3)
        print("Page %d: 新增 %d, 累计新增 %d" % (page, page_new, n_new))
    conn.commit()
    conn.close()
    print("完成: 总计 %d | 新增: %d | 跳过: %d | 失败: %d" % (_MAX_PAGES, n_new, n_skip, n_fail))


if __name__ == "__main__":
    main()