#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
大化瑶族自治县人民政府 — 建设项目环评审批 爬虫
site: www.gxdh.gov.cn (广西河池大化瑶族自治县)
栏目: /xxgk/zdlyxxgk/shgysyjsly/hjbhly/jsxmhpsp/ (重点领域-环境保护-建设项目环评审批)
CMS: 广西政府网站统一平台 (同田东县 gxtd.gov.cn)
列表: div#morelist > ul.more-list > li > span(日期) + a(title, href=./tXXX.shtml)
分页: createPageHTML(4,0,"index","shtml","120") → index.shtml / index_{N}.shtml
详情: {栏目目录}/t{id}.shtml
详情结构: h1 标题 + meta ArticleTitle/PubDate + div.article-con > div.trs_editor_view(正文) + div#downloadfile/#relfile(附件)
正文规则: 保留表格HTML、\n\n分段、附件转<p><a>名称内嵌URL、去装饰
"""
import re
import sys
import os
import time
import sqlite3
import copy
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin

SITE_NAME = "大化县-建设项目环评审批"
GROUP_NAME = "广西"
DOMAIN = "www.gxdh.gov.cn"
BASE_URL = "http://www.gxdh.gov.cn"
LIST_BASE = "/xxgk/zdlyxxgk/shgysyjsly/hjbhly/jsxmhpsp"
TOTAL_PAGES = 3                # createPageHTML报4页但index_4实际404, 实际3页90条
SCRIPT_NAME = os.path.basename(__file__)

SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}
TIMEOUT = 30
session = requests.Session()
session.headers.update(HEADERS)


def fetch(url):
    try:
        resp = session.get(url, timeout=TIMEOUT)
        resp.encoding = "utf-8"
        if resp.status_code != 200:
            return None
        return resp.text
    except Exception:
        return None


def clean_title(title):
    """标题清洗: strip &middot;&nbsp; 实体前缀 + 省略号截断后缀"""
    title = re.sub(r'^[\s\xa0·\u00b7]+', '', title or '')
    title = re.sub(r'\s*\.{3,}\s*$', '', title)
    return title.strip()


def parse_list(html):
    """解析列表: div#morelist > ul.more-list > li > span + a"""
    items = []
    soup = BeautifulSoup(html, "html.parser")
    morelist = soup.find("div", id="morelist")
    if not morelist:
        return items
    for ul in morelist.find_all("ul", class_="more-list"):
        for li in ul.find_all("li", recursive=False):
            a = li.find("a", href=True)
            if not a:
                continue
            href = a["href"].strip()
            title = a.get("title") or a.get_text(strip=True)
            if not title or not href:
                continue
            if href.startswith("./"):
                href = BASE_URL + LIST_BASE + "/" + href[2:]
            elif href.startswith("/"):
                href = BASE_URL + href
            elif not href.startswith("http"):
                href = BASE_URL + LIST_BASE + "/" + href
            span = li.find("span")
            date_str = span.get_text(strip=True) if span else ""
            items.append({
                "title": clean_title(title),
                "url": href,
                "publish_date": date_str[:10],
            })
    return items


def get_total_pages(html):
    """从 createPageHTML(N, 0, ...) 提取总页数"""
    m = re.search(r'createPageHTML\(\s*(\d+)\s*,', html)
    if m:
        return int(m.group(1))
    return TOTAL_PAGES


def _normalize_text(el):
    """段落文本: <br>转\n, 无分隔拼接(get_text), 压缩空白, 去空行"""
    brs = el.find_all("br")
    for br in brs:
        br.replace_with("\n")
    text = el.get_text()
    lines = [re.sub(r'[ \t\xa0\u3000]+', ' ', ln).strip() for ln in text.split("\n")]
    lines = [ln for ln in lines if ln]
    return "\n".join(lines)


def process_content_div(div, page_url):
    """正文div处理: 保留表格HTML, \n\n分段, 附件转<p><a>名称内嵌URL, 去装饰图/重复段落"""
    parts = []
    seen_texts = set()
    attachments = []  # (name, abs_url)

    for el in div.find_all(recursive=False):
        if el.name == "table":
            parts.append(str(el))
            continue
        if el.name in ("p", "div", "section", "article", "center"):
            # 附件链接提取
            file_links = []
            for a in el.find_all("a"):
                href = a.get("href")
                if not href:
                    continue
                href = str(href).strip()
                if re.search(r'\.(pdf|doc|docx|xls|xlsx|zip|rar|wps)$', href, re.I):
                    name = a.get_text(strip=True) or href.split("/")[-1]
                    if not href.startswith("http"):
                        href = urljoin(page_url, href)
                    file_links.append((a, name, href))
            for a, name, href in file_links:
                if not any(u == href for _, u in attachments):
                    attachments.append((name, href))
                a.decompose()
            # 装饰图片移除
            imgs = el.find_all("img")
            for img in imgs:
                img.decompose()
            # 段落内嵌表格
            tbls = el.find_all("table")
            if tbls:
                p_copy = copy.deepcopy(el)
                for t in p_copy.find_all("table"):
                    t.decompose()
                text = _normalize_text(p_copy)
                if text and text not in seen_texts:
                    seen_texts.add(text)
                    parts.append(text)
                for t in tbls:
                    parts.append(str(t))
            else:
                text = _normalize_text(el)
                if text and text not in seen_texts:
                    seen_texts.add(text)
                    parts.append(text)
        elif el.name == "br":
            continue
        else:
            text = _normalize_text(el)
            if text and text not in seen_texts:
                seen_texts.add(text)
                parts.append(text)

    body = "\n\n".join(parts)
    # 附件统一转 <p><a>名称内嵌URL
    for name, url in attachments:
        if 'href="%s"' % url in body:
            continue
        if body:
            body += "\n\n"
        body += '<p><a href="%s" target="_blank">%s</a></p>' % (url, name)
    return body, attachments


def extract_detail(html, page_url=""):
    """详情解析: 返回 (title, date, content, attachments)"""
    soup = BeautifulSoup(html, "html.parser")
    # 标题: meta ArticleTitle -> h1
    title = ""
    m = soup.find("meta", attrs={"name": "ArticleTitle"})
    if m and m.get("content"):
        title = clean_title(m["content"])
    if not title:
        h1 = soup.find("h1")
        if h1:
            title = clean_title(h1.get_text(strip=True))
    # 日期: meta PubDate
    date = ""
    m = soup.find("meta", attrs={"name": "PubDate"})
    if m and m.get("content"):
        date = m["content"].strip()[:10]
    if not date:
        inf = soup.find("div", class_="article-inf-left")
        if inf:
            dm = re.search(r'(\d{4}-\d{2}-\d{2})', inf.get_text())
            if dm:
                date = dm.group(1)
    # 正文: div.article-con 内的 trs_editor_view (或 article-con 本身)
    content = ""
    attachments = []
    article_con = soup.find("div", class_="article-con")
    if article_con:
        editor = article_con.find("div", class_=re.compile(r'trs_editor_view|TRS_UEDITOR|trs_paper', re.I))
        target = editor if editor else article_con
        content, attachments = process_content_div(target, page_url)
    # 附件区 downloadfile / relfile
    for aid in ("downloadfile", "relfile"):
        ddiv = soup.find("div", attrs={"id": aid})
        if ddiv:
            for a in ddiv.find_all("a"):
                href = a.get("href")
                if not href:
                    continue
                href = str(href).strip()
                if not href.startswith("http"):
                    href = urljoin(page_url, href)
                name = a.get_text(strip=True) or href.split("/")[-1]
                if not any(u == href for _, u in attachments):
                    attachments.append((name, href))
    # 附件统一拼到正文末尾
    for name, url in attachments:
        if 'href="%s"' % url in content:
            continue
        if content:
            content += "\n\n"
        content += '<p><a href="%s" target="_blank">%s</a></p>' % (url, name)
    return title, date, content, attachments


def main():
    import argparse
    parser = argparse.ArgumentParser(description=f"{SITE_NAME}爬虫")
    parser.add_argument("--pages", type=int, default=1, help="爬取页数(默认1)")
    parser.add_argument("--script-name", default=SCRIPT_NAME, help="脚本名(写库)")
    args = parser.parse_args()

    pages_to_crawl = args.pages
    script_name = args.script_name
    print(f"🔍 {SITE_NAME} — 爬取 {pages_to_crawl} 页 (共{TOTAL_PAGES}页)")

    conn = sqlite3.connect(SEARCH_DB, timeout=30)
    cur = conn.cursor()

    new_total = 0
    skip_total = 0
    total_items = 0
    fail_streak = 0

    for page in range(1, pages_to_crawl + 1):
        if page == 1:
            list_url = BASE_URL + LIST_BASE + "/index.shtml"
        else:
            list_url = "%s%s/index_%d.shtml" % (BASE_URL, LIST_BASE, page)
        html = fetch(list_url)
        if not html:
            print(f"  [p{page:2d}/{pages_to_crawl}] ✗ HTTP失败")
            fail_streak += 1
            if fail_streak >= 1 and page > 1:
                print(f"  ⏹ 第{page}页不存在(404/过报)，停止翻页")
                break
            continue
        fail_streak = 0
        if page == 1:
            tp = get_total_pages(html)
            if tp and tp < pages_to_crawl:
                pages_to_crawl = tp
                print(f"  ℹ 站点实际总页数 {tp}，本次爬取上限调整")

        items = parse_list(html)
        if not items:
            print(f"  [p{page:2d}/{pages_to_crawl}] ✗ 未解析到条目")
            continue

        page_new = 0
        page_skip = 0
        for item in items:
            total_items += 1
            exists = cur.execute(
                "SELECT 1 FROM gov_raw WHERE page_url=? AND site_name=?",
                (item["url"], SITE_NAME)
            ).fetchone()
            if exists:
                page_skip += 1
                skip_total += 1
                continue

            detail_html = fetch(item["url"])
            if detail_html:
                d_title, d_date, content, attachments = extract_detail(detail_html, item["url"])
            else:
                d_title, d_date, content, attachments = "", "", "", []

            title = d_title or item["title"]
            date = d_date or item.get("publish_date", "")
            if not content:
                page_skip += 1
                skip_total += 1
                continue

            has_table = 1 if "<table" in content else 0
            att_urls = ",".join(u for _, u in attachments)
            try:
                cur.execute(
                    "INSERT OR IGNORE INTO gov_raw "
                    "(title, content, publish_date, page_url, source_url, site_name, "
                    " group_name, script_name, has_table, attachments, status) "
                    "VALUES (?,?,?,?,?,?,?,?,?,?,?)",
                    (title, content, date, item["url"], DOMAIN, SITE_NAME,
                     GROUP_NAME, script_name, has_table, att_urls, "published")
                )
                if cur.rowcount > 0:
                    row_id = cur.lastrowid
                    try:
                        plain = re.sub(r"<[^>]+>", " ", content)
                        plain = re.sub(r"\s+", " ", plain).strip()[:200] or title
                        cur.execute(
                            "INSERT OR IGNORE INTO gov_search(rowid, title, site_name, summary) VALUES (?,?,?,?)",
                            (row_id, title, SITE_NAME, plain)
                        )
                    except Exception:
                        pass
                    page_new += 1
                    new_total += 1
            except Exception as e:
                print(f"    ! 入库异常 {item['url']}: {e}")

        conn.commit()
        print(f"  [p{page:2d}/{pages_to_crawl}] ✓ {len(items)}条 (新增{page_new} 跳过{page_skip})")
        time.sleep(0.3)

    conn.close()
    print(f"\n📊 完成！列表 {total_items} 条 | 新增: {new_total} | 跳过: {skip_total}")


if __name__ == "__main__":
    main()