#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
crawl_jsrthj_xmgs.py — 江苏润天环境科技有限公司「项目公示」栏目
http://www.jsrthj.com/article/29/1.aspx
ASP.NET 站:
- 列表: dl.news_dl > dt > a.dt_1[href=/article/show/{id}.aspx][title] + span.dt_2[YYYY-MM-DD]
- 分页: /article/29/{N}.aspx (10条/页, 共65页≈650条)
- 详情: /article/show/{id}.aspx
    标题 div.agent_tit, 日期 div.agent_data(更新时间：2026/6/24 15:58:15), 正文 div.agent_con
- 正文为纯文本公示(环评一次/报批前/验收), 无 PDF 附件;
  "附件"=正文内链接(百度网盘报告下载链接+提取码 / mee.gov.cn 公众意见表) → 内嵌URL保留
"""
import argparse
import html as html_lib
import os
import re
import sqlite3
import time
import urllib.parse

import requests

BASE = "http://www.jsrthj.com"
SITE_NAME = "江苏润天环境"
SCRIPT_NAME = "crawl_jsrthj_xmgs.py"
GROUP_NAME = "江苏"
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36",
    "Accept-Language": "zh-CN,zh;q=0.9",
}
TIMEOUT = 25


def clean_title(t):
    """标题清洗: strip ·   实体前缀与空白"""
    if not t:
        return ""
    t = t.strip()
    t = re.sub(r"·| | | ", "", t, flags=re.I)
    t = t.strip()
    t = re.sub(r"\s+", " ", t)
    return t


def html_to_text(html_seg):
    """HTML -> 文本, 保留 a 链接(内嵌URL)与表格, 去 script/style, \n\n 分段"""
    seg = html_seg
    # 记录链接(原始href, 绝对URL, 文本) — 该站"附件"即正文内链接(百度网盘报告下载/mee意见表)
    links = []

    def _link_repl(m):
        href, txt = m.group(1), m.group(2)
        txt = re.sub(r"<[^>]+>", "", txt)
        txt = html_lib.unescape(txt).strip()
        if not txt or href.startswith("#"):
            return txt or ""
        abs_url = urllib.parse.urljoin(BASE, href)
        links.append((href, abs_url, txt))
        return f"__LINK__{len(links)}__"

    seg = re.sub(r'<a\s[^>]*href="([^"]+)"[^>]*>(.*?)</a>', _link_repl, seg, flags=re.S | re.I)
    # 去 script/style
    seg = re.sub(r"<(script|style)[^>]*>.*?</\1>", "", seg, flags=re.S | re.I)
    # 段落边界: 块级标签 -> \n\n
    seg = re.sub(r"</(p|div|li|tr|h[1-6]|br)>", "\n\n", seg, flags=re.I)
    seg = re.sub(r"<(p|div|li|tr|h[1-6]|br)[^>]*>", "\n", seg, flags=re.I)
    seg = re.sub(r"<[^>]+>", "", seg)
    seg = html_lib.unescape(seg)
    # 还原链接(内嵌URL)
    for i, (href, abs_url, txt) in enumerate(links, 1):
        seg = seg.replace(f"__LINK__{i}__", f'<a href="{abs_url}" target="_blank">{txt}</a>')
    seg = re.sub(r"\r", "", seg)
    seg = re.sub(r"\n{3,}", "\n\n", seg)
    seg = re.sub(r"[ \t\u00a0]+", " ", seg)
    return seg.strip(), links


def parse_list(html_src):
    """列表页 -> [(title, art_url, pub_date)]"""
    items = re.findall(
        r'<dl class="news_dl">\s*<dt><a href="([^"]+)"[^>]*>(.*?)</a>\s*<span class="dt_2">\[(\d{4}-\d{2}-\d{2})\]</span></dt>',
        html_src, re.S)
    out = []
    for href, t, d in items:
        art_url = urllib.parse.urljoin(BASE, href)
        out.append((clean_title(t), art_url, d))
    return out


def parse_detail(html_src, art_url):
    """详情页 -> (title, publish_date, body_html, links)"""
    # 标题
    m = re.search(r'<div class="agent_tit">(.*?)</div>', html_src, re.S)
    title = clean_title(m.group(1)) if m else ""
    if not title:
        m = re.search(r"<title>(.*?)</title>", html_src, re.S)
        if m:
            title = clean_title(m.group(1).split("-")[0])
    # 日期 2026/6/24 15:58:15
    m = re.search(r"更新时间[：:]\s*(\d{4})/(\d{1,2})/(\d{1,2})", html_src)
    publish = ""
    if m:
        publish = "%s-%02d-%02d" % (m.group(1), int(m.group(2)), int(m.group(3)))
    # 正文
    m = re.search(r'<div class="agent_con">(.*?)</div>', html_src, re.S)
    body_html = ""
    if m:
        body_html = m.group(1).strip()
    return title or None, publish or None, body_html, []


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--pages", type=int, default=1, help="抓取页数")
    ap.add_argument("--db", default="/mnt/data/search.db", help="SQLite 数据库路径")
    ap.add_argument("--script-name", default=SCRIPT_NAME)
    args = ap.parse_args()

    db_path = args.db
    script_name = args.script_name
    if not db_path.startswith("/mnt/"):
        db_path = os.path.expanduser(db_path)

    conn = sqlite3.connect(db_path, timeout=60)
    cur = conn.cursor()
    new_count = 0
    dup_count = 0
    err_count = 0

    session = requests.Session()

    for page in range(1, args.pages + 1):
        url = f"{BASE}/article/29/{page}.aspx"
        try:
            r = session.get(url, headers=HEADERS, timeout=TIMEOUT)
            r.encoding = "utf-8"
        except Exception as e:
            print(f"[{page}] 请求失败 {url}: {e}")
            err_count += 1
            continue
        items = parse_list(r.text)
        print(f"[{page}] 列表 {len(items)} 条 {url}")
        if not items:
            continue
        for title, art_url, pub_date in items:
            try:
                cur.execute("SELECT COUNT(*) FROM gov_raw WHERE page_url=? AND script_name=?", (art_url, script_name))
                if cur.fetchone()[0] > 0:
                    dup_count += 1
                    continue
                dr = session.get(art_url, headers=HEADERS, timeout=TIMEOUT)
                dr.encoding = "utf-8"
                d_title, d_date, content_html, links = parse_detail(dr.text, art_url)
                if not content_html.strip():
                    print(f"  跳过空正文: {title[:30]}")
                    dup_count += 1
                    continue
                if d_title:
                    title = d_title
                if d_date:
                    pub_date = d_date
                content, _ = html_to_text(content_html)
                att_txt = "\n\n".join(a[2] for a in _) if _ else ""
                summary = content[:500] if content else ""
                cur.execute(
                    "INSERT INTO gov_raw (site_name, source_url, page_url, title, publish_date, date_rank, summary, status, category, visits, content, tags, industry, attachments, group_name, has_table, script_name) "
                    "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
                    (SITE_NAME, art_url, art_url, title, pub_date, 0, summary, "", "", 0, content, "",
                     "other", att_txt, GROUP_NAME, 1 if "<table" in content else 0, script_name),
                )
                # 2026-09-22: 先提交 gov_raw —— 库上触发器已维护 FTS，下面这条手动写入会因
                #   rowid 重复而 IntegrityError；不先 commit 会把 gov_raw 那条一并回滚（静默丢数据）
                conn.commit()
                cur.execute(
                    "INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary) VALUES (?,?,?,?)",
                    (cur.lastrowid, title, SITE_NAME, summary),
                )
                conn.commit()
                new_count += 1
                print(f"  + {pub_date} {title[:40]}")
            except Exception as e:
                err_count += 1
                print(f"  ERR {title[:30]}: {e}")
        time.sleep(0.3)

    conn.close()
    print(f"完成: 新增 {new_count} / 重复 {dup_count} / 错误 {err_count}")


if __name__ == "__main__":
    main()