#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
crawl_shangyu_sthj.py - 绍兴市上虞区-生态环境
CMS: JCMS (Hanweb, API分页)
API: /api-gateway/jpaas-publish-server/front/page/build/unit (GET, pageNo=N)
     27条/页, 2456条, ~91页
详情: /col/col{id}/art/YYYY/art_{hash}.html
标题: <meta ArticleTitle>
日期: <meta PubDate>
正文: div#zoom > p
"""

import requests
import re
import json
import sys
import os
import time
from bs4 import BeautifulSoup

DB_PATH = "/root/search.db"
BASE_URL = "https://www.shangyu.gov.cn"
API_URL = "/api-gateway/jpaas-publish-server/front/page/build/unit"
SITE_NAME = "绍兴市上虞区-生态环境"
GROUP = "浙江"
INDUSTRY = "环境公示"
ITEMS_PER_PAGE = 27
MAX_PAGES = 1  # 假分页

API_PARAMS = {
    "parseType": "bulidstatic",
    "webId": "2328",
    "tplSetId": "9nuI0IJgTDxmXUM0arEPb",
    "pageType": "column",
    "tagId": "当前栏目列表1a",
    "pageId": "1229604608",
    "rows": "27",
}

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
}


def get_api_page(page_no, session):
    """获取列表页数据"""
    params = API_PARAMS.copy()
    params["pageNo"] = str(page_no)
    try:
        r = session.get(BASE_URL + API_URL, params=params, timeout=60, verify=False)
        r.encoding = "utf-8"
        d = r.json()
        if d.get("success") and d.get("data", {}).get("html"):
            return d["data"]["html"]
        return None
    except Exception as e:
        print(f"  API ERR:{e}")
        return None


def parse_list(html):
    """解析列表HTML"""
    items = []
    soup = BeautifulSoup(html, "html.parser")
    for tr in soup.select("tr.xxgkList"):
        a = tr.find("a", href=True)
        date_td = tr.find("td", class_="bt_time")
        if a and date_td:
            href = a["href"]
            if href.startswith("/"):
                href = BASE_URL + href
            elif not href.startswith("http"):
                href = BASE_URL + "/" + href
            title = a.get("title", "") or a.get_text(strip=True)
            date = date_td.get_text(strip=True)
            items.append((href, title, date))
    return items


def extract_detail(html, url):
    soup = BeautifulSoup(html, "html.parser")

    # Title
    title = ""
    meta_title = soup.find("meta", {"name": "ArticleTitle"})
    if meta_title:
        title = meta_title.get("content", "").strip()

    # Date
    date_str = ""
    pubdate_meta = soup.find("meta", {"name": "PubDate"})
    if pubdate_meta:
        content = pubdate_meta.get("content", "")
        m = re.search(r"(\d{4}-\d{2}-\d{2})", content)
        if m:
            date_str = m.group(1)

    # Content from div#zoom
    content = ""
    zoom = soup.find("div", id="zoom")
    if zoom:
        parts = []
        for p in zoom.find_all("p"):
            txt = p.get_text(strip=True)
            if txt:
                parts.append(txt)
        for tbl in zoom.find_all("table"):
            parts.append(str(tbl))
        if parts:
            content = "\n\n".join(parts)
        else:
            content = zoom.get_text(separator="\n").strip()

    if content:
        content = re.sub(r"&nbsp;", "", content)

    summary = content[:200] if content else ""
    return title, date_str, content, summary


def save_to_db(items_data):
    import sqlite3
    conn = sqlite3.connect(DB_PATH, timeout=10)
    c = conn.cursor()
    inserted = 0
    fts_batch = []
    for title, date_str, content, summary, page_url in items_data:
        try:
            c.execute("SELECT id FROM gov_raw WHERE page_url = ?", (page_url,))
            if c.fetchone():
                continue
            c.execute(
                """INSERT INTO gov_raw 
                   (page_url, title, site_name, publish_date, content, summary,
                    source_url, category, industry, group_name, script_name)
                   VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
                (page_url, title, SITE_NAME, date_str, content, summary,
                 page_url, "生态环境", INDUSTRY, GROUP, "crawl_shangyu_sthj.py"),
            )
            new_id = c.lastrowid
            if new_id:
                fts_batch.append((new_id, title, SITE_NAME, summary or ""))
            inserted += 1
        except Exception as e:
            print(f"  [DB ERROR] {title[:30]}: {e}")
    conn.commit()
    if fts_batch:
        for rid, title, site, summary in fts_batch:
            try:
                c.execute(
                    "INSERT OR IGNORE INTO gov_search(rowid, title, site_name, summary) VALUES (?, ?, ?, ?)",
                    (rid, title, site, summary),
                )
            except Exception as e:
                print(f"  [FTS ERROR] id={rid}: {e}")
        conn.commit()
        print(f"  FTS同步: {len(fts_batch)}条")
    conn.close()
    return inserted


def main():
    import argparse
    parser = argparse.ArgumentParser(description=f"{SITE_NAME}爬虫")
    parser.add_argument("--pages", type=int, default=1, help="爬取页数")
    parser.add_argument("--full", action="store_true", help="全量爬取")
    args = parser.parse_args()

    max_pages = MAX_PAGES if args.full else args.pages
    print(f"[{time.strftime('%H:%M:%S')}] 开始爬取 {SITE_NAME}...", flush=True)

    session = requests.Session()
    session.headers.update(HEADERS)

    all_items = []
    for pg in range(1, max_pages + 1):
        html = get_api_page(pg, session)
        if not html:
            print(f"  第{pg}页: 请求失败")
            continue
        items = parse_list(html)
        print(f"  第{pg}页: {len(items)}条", flush=True)
        if not items:
            print(f"  第{pg}页无数据，停止")
            break
        all_items.extend(items)

    if not all_items:
        print("无数据")
        return

    print(f"\n共获取 {len(all_items)} 条链接")

    batch = []
    for href, title, date in all_items:
        try:
            r = session.get(href, timeout=60, verify=False)
            r.encoding = "utf-8"
            detail_html = r.text
        except Exception as e:
            print(f"  ERR获取详情 {href[-40:]}: {e}")
            time.sleep(0.5)
            continue
        dt, dd, content, summary = extract_detail(detail_html, href)
        if not dd:
            dd = date
        batch.append((dt or title, dd, content, summary, href))
        time.sleep(0.3)

    saved = save_to_db(batch)
    print(f"入库{saved}/{len(batch)}条")

    print(f"\n===== 完成 =====")
    print(f"新增入库: {saved} 条")
    print(f"站点: {SITE_NAME}")


if __name__ == "__main__":
    main()