#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""九江市生态环境局 建设项目环境影响评价 (sthjj.jiujiang.gov.cn)
- 列表: /zwgk_215/xzqlyx/jsxmhjyxpj/index.html (20条/页), index_N.html 翻页
- 详情: /zwgk_215/xzqlyx/jsxmhjyxpj/YYYYMM/tYYYYMMDD_ID.html
- 标题: h2.tc > a, 日期: p.info span 发布时间, 来源: p.info span
- 正文: div#article-box 内 div.trs_editor_view
用法: python3 crawl_jj_hpgs.py [--pages N] [--out FILE]
"""
import argparse, json, re, sys, time, urllib.request, ssl

HOST = "sthjj.jiujiang.gov.cn"
BASE = f"http://{HOST}"
LIST_PATH = "/zwgk_215/xzqlyx/jsxmhjyxpj"

UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
HEADERS = {
    "Host": HOST,
    "User-Agent": UA,
    "Referer": f"http://{HOST}{LIST_PATH}/",
}


def fetch(url, timeout=20):
    req = urllib.request.Request(url, headers=HEADERS)
    with urllib.request.urlopen(req, timeout=timeout) as r:
        return r.read().decode("utf-8", "replace")


def get_list(pageno):
    fname = "index.html" if pageno == 1 else f"index_{pageno-1}.html"
    html = fetch(f"{BASE}{LIST_PATH}/{fname}")
    items = re.findall(r'<li class="td01"><a href="([^"]+)" title="([^"]+)"[^>]*>.*?<span>([0-9-]+)</span>', html, re.S)
    return [(u, t.strip(), d) for u, t, d in items]


def parse_detail(html):
    m = re.search(r'<h2 class="tc">\s*<a[^>]*>(.*?)</a>', html, re.S)
    title = re.sub(r'<[^>]+>', '', m.group(1)).strip() if m else None
    m = re.search(r'发布时间[：:]\s*([0-9-]+ [0-9:]+)', html)
    date = m.group(1).strip() if m else None
    m = re.search(r'来源[：:]\s*([^<\s][^<]{0,20})', html)
    source = m.group(1).strip() if m else None
    return title, date, source


def clean_body(html):
    i = html.find('class="trs_editor_view')
    if i < 0:
        return None
    # 找到对应 div 结束（嵌套 div 计数）
    start = html.rfind('<div', 0, i)
    if start < 0:
        return None
    depth = 0
    pos = start
    seg = None
    while pos < len(html):
        nxt = html.find('<div', pos + 1)
        end = html.find('</div>', pos + 1)
        if nxt != -1 and (end == -1 or nxt < end):
            depth += 1
            pos = nxt
        elif end != -1:
            depth -= 1
            if depth == 0:
                seg = html[start:end + 6]
                break
            pos = end
        else:
            break
    if seg is None:
        seg = html[start:start + 5000]
    seg = re.sub(r'<!--.*?-->', '', seg, flags=re.S)
    seg = re.sub(r'<script.*?</script>', '', seg, flags=re.S)
    seg = re.sub(r'<style.*?</style>', '', seg, flags=re.S)
    # 相对路径转绝对
    seg = re.sub(r'(src|href)="(/[^"]+)"', lambda m: f'{m.group(1)}="http://{HOST}{m.group(2)}"' if not m.group(2).startswith('//') else f'{m.group(1)}="http:{m.group(2)}"', seg)
    # 清 inline style + 无障碍属性
    seg = re.sub(r'\sstyle="[^"]*"', '', seg)
    for attr in ['width', 'height', 'title', 'alt', 'align', 'border', 'cellspacing', 'cellpadding', 'class', 'id', 'name', 'target', 'rel']:
        seg = re.sub(rf'\s{attr}="[^"]*"', '', seg)
    seg = re.sub(r'<br\s*/?>', '<br/>', seg)
    seg = re.sub(r'<span[^>]*>|</span>', '', seg)
    seg = re.sub(r'<strong[^>]*>|</strong>', '', seg)
    seg = re.sub(r'<font[^>]*>|</font>', '', seg)
    seg = re.sub(r'<th[^>]*>', '<td>', seg)
    seg = re.sub(r'</th>', '</td>', seg)
    seg = re.sub(r'\s+', ' ', seg)
    return seg.strip()


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--pages", type=int, default=5)
    ap.add_argument("--out", default="/tmp/jj_hpgs.jsonl")
    args = ap.parse_args()

    out = open(args.out, "a", encoding="utf-8")
    seen = set()
    try:
        for line in open(args.out, encoding="utf-8"):
            d = json.loads(line)
            seen.add(d["url"])
    except FileNotFoundError:
        pass

    total_ok = 0
    for p in range(1, args.pages + 1):
        try:
            items = get_list(p)
        except Exception as e:
            print(f"[list] p{p} ERR {e}", flush=True)
            time.sleep(3)
            continue
        if not items:
            print(f"[list] p{p}: 空, 结束", flush=True)
            break
        print(f"[list] p{p}: {len(items)} 条", flush=True)
        for url, title_list, d in items:
            url = url if url.startswith("http") else f"http://{HOST}{url}"
            if url in seen:
                continue
            try:
                html = fetch(url)
                title, date, source = parse_detail(html)
                body = clean_body(html)
                if not body or len(body) < 50:
                    print(f"  [SKIP 空正文] {url[-60:]} | {title_list[:25]}", flush=True)
                    seen.add(url)
                    continue
                rec = {
                    "id": re.search(r'_(\d+)\.html', url).group(1),
                    "title": (title or title_list).strip(),
                    "date": (date or d)[:10],
                    "url": url,
                    "source": source or "九江市生态环境局",
                    "site": "jj_hpgs",
                    "body": body,
                    "attachments": [],
                    "crawl_time": time.strftime("%Y-%m-%d %H:%M:%S"),
                }
                out.write(json.dumps(rec, ensure_ascii=False) + "\n")
                out.flush()
                seen.add(url)
                total_ok += 1
            except Exception as e:
                print(f"  [ERR] {url[-60:]} {e}", flush=True)
            time.sleep(0.4)
        time.sleep(0.6)
    out.close()
    print(f"[*] 完成, 共 {total_ok} 条新写入 {args.out}", flush=True)


if __name__ == "__main__":
    main()