#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
昌吉国家高新技术产业开发区 - 环保建设 (sthj1) 爬虫
列表: https://www.cjgjgxq.cn/p172/sthj1.html + sthj1_N.html 静态分页
  首页 15 条/页, total 668 条 ≈ 45 页; 越界页返回空(0条)自动停
详情: /p172/sthj1/YYYYMMDD/NUMBER.html
  标题: <meta name="ArticleTitle"> (完整)
  日期: <meta name="PubDate">
  正文: font#Zoom (含 TRS_UEDITOR 正文 + 附件)
用法: python3 crawl_cjgjgxq_sthj1.py [--pages=N]
"""
import os, sys, re, time, json, html as html_mod
import requests
from urllib.parse import urljoin

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

BASE_URL = "http://www.cjgjgxq.cn"
LIST_BASE = "https://www.cjgjgxq.cn/p172/sthj1"
SITE_NAME = "昌吉国家高新区-环保建设"
CATEGORY = "环保建设"
MAX_PAGES = 25
CUTOFF = "2023-08-10"

UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"
HEADERS = {"User-Agent": UA, "Referer": LIST_BASE + ".html"}


def parse_args():
    pages = 0
    for a in sys.argv[1:]:
        if a.startswith("--pages="):
            pages = int(a.split("=")[1])
        elif a.isdigit():
            pages = int(a)
    return pages


def clean_title(t):
    t = html_mod.unescape(t or "")
    t = re.sub(r"[\u200b\u200e\u200f\ufeff\xa0]", "", t)
    t = re.sub(r"\s+", " ", t)
    t = re.sub(r"^[•·\-—]\s*", "", t)
    return t.strip()


def fetch_list(session, page):
    url = LIST_BASE + ".html" if page == 1 else f"{LIST_BASE}_{page}.html"
    try:
        r = session.get(url, timeout=30)
        if r.status_code != 200:
            return []
        # 源站 <meta charset> 在 body 内, HTTP 头无 charset → requests 误判 ISO-8859-1
        r.encoding = "utf-8"
        html = r.text
        items = []
        for m in re.finditer(r'<a[^>]*href="(/p172/sthj1/\d+/\d+\.html)"[^>]*>([^<]*)</a>', html):
            href, title = m.group(1), clean_title(m.group(2))
            if len(title) < 4:
                continue
            # 找该链接后的日期
            date = ""
            dm = re.search(r"(\d{4}-\d{2}-\d{2})", html[m.end():m.end()+200])
            if dm:
                date = dm.group(1)
            items.append({"title": title, "href": BASE_URL + href, "date": date})
        return items
    except Exception as e:
        print(f"    [ERR] list page {page}: {e}", flush=True)
        return []


def fetch_detail(session, url):
    try:
        r = session.get(url, headers={"User-Agent": UA, "Referer": LIST_BASE + ".html"}, timeout=30)
        # 同上: 显式 UTF-8, 否则 requests 误判 ISO-8859-1 → 中文乱码
        r.encoding = "utf-8"
        html = r.text
        title = ""
        m = re.search(r'<meta name="ArticleTitle" content="([^"]*)"', html)
        if m:
            title = clean_title(m.group(1))
        pub_date = ""
        m = re.search(r'<meta name="PubDate" content="([^"]*)"', html)
        if m:
            pub_date = m.group(1).strip()[:10]
        body = ""
        i = html.find('id="Zoom"')
        if i >= 0:
            start = html.find(">", i) + 1
            j = html.find("</font>", start)
            if j > 0:
                raw = html[start:j]
                raw = re.sub(r"<script[\s\S]*?</script>", "", raw)
                raw = re.sub(r"<style[\s\S]*?</style>", "", raw)
                raw = re.sub(r'href="//([^"]*)"', lambda m: 'href="%s://%s"' % (BASE_URL.split("://")[0], m.group(1)), raw)
                raw = re.sub(r'src="//([^"]*)"', lambda m: 'src="%s://%s"' % (BASE_URL.split("://")[0], m.group(1)), raw)
                raw = re.sub(r'href="([^"]*)"', lambda m: 'href="%s"' % (urljoin(url, m.group(1)) if not m.group(1).startswith(("http", "#", "javascript")) else m.group(1)), raw)
                raw = re.sub(r'src="([^"]*)"', lambda m: 'src="%s"' % (urljoin(url, m.group(1)) if not m.group(1).startswith(("http", "data:", "javascript")) else m.group(1)), raw)
                raw = re.sub(r'\sstyle="[^"]*"', "", raw)
                raw = re.sub(r"<span[^>]*>|</span>|<strong[^>]*>|</strong>|<b[^>]*>|</b>|<font[^>]*>|</font>", "", raw)
                # 剥开头容器 div (class="view TRS_UEDITOR...")
                raw = re.sub(r'^\s*<div[^>]*>\s*', '', raw)
                body = raw.strip()
        if not body or len(body) < 20:
            body = ""
        return title, pub_date, body
    except Exception as e:
        print(f"    [ERR] detail {url}: {e}", flush=True)
        return "", "", ""


def main():
    max_pages = parse_args() or MAX_PAGES
    print(f"[CJGJGXQ-STHJ1] SITE={SITE_NAME} max_pages={max_pages}", flush=True)
    session = requests.Session()
    session.headers.update({"User-Agent": UA})

    all_items = []
    seen = set()
    for pg in range(1, max_pages + 1):
        items = fetch_list(session, pg)
        if not items:
            print(f"  [END] page {pg} empty", flush=True)
            break
        new_count = 0
        for it in items:
            if it["href"] in seen:
                continue
            seen.add(it["href"])
            all_items.append(it)
            new_count += 1
        print(f"  Page {pg}: {len(items)}条(新{new_count}) 累计{len(all_items)}", flush=True)
        time.sleep(0.4)

    print(f"列表完成: {len(all_items)} 条, 抓详情...", flush=True)
    results = []
    for i, it in enumerate(all_items):
        d_title, d_date, body = fetch_detail(session, it["href"])
        title = d_title or it["title"]
        pub_date = d_date or it.get("date", "")
        summary = re.sub(r"<[^>]+>", " ", body) if body else title
        summary = re.sub(r"\s+", " ", summary).strip()[:300]
        atts = []
        if body:
            for m in re.finditer(r'<a[^>]*href="([^"]*\.(?:pdf|doc|docx|xls|xlsx|zip|rar|ofd|wps))"[^>]*>([^<]*)</a>', body, re.I):
                atts.append({"name": m.group(2).strip(), "url": m.group(1)})
        results.append({
            "site_name": SITE_NAME, "source_url": it["href"], "url": it["href"],
            "title": title, "pub_date": pub_date, "content": body,
            "summary": summary, "category": CATEGORY,
            "attachments": json.dumps(atts, ensure_ascii=False) if atts else "",
        })
        if (i + 1) % 10 == 0:
            print(f"  detail {i+1}/{len(all_items)}", flush=True)
        time.sleep(0.3)

    print(f"  pushing {len(results)} items...", flush=True)
    push_to_searchdb(results, batch_label=SITE_NAME)
    empty = sum(1 for r in results if not (r.get("content") or "").strip())
    print(f"  DONE. pushed={len(results)} 空正文={empty}", flush=True)


if __name__ == "__main__":
    main()