#!/usr/bin/env python3
"""crawl_szwz_jshp.py - 苏州市吴中区人民政府 行政审批（建设环保）"""
import sys
import os
import sqlite3
import time
import re

# Reuse extraction logic from crawl_szwz.py
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import crawl_szwz as base

BASE_URL = "http://www.szwz.gov.cn"
LIST_URL = BASE_URL + "/szwz/jshp/list{page}.shtml"

HEADERS = base.HEADERS
TIMEOUT = 20
DELAY = 0.5

DB_PATH = os.environ.get("DB_PATH", "/root/search.db")
SITE_NAME = "苏州市吴中区人民政府"
COLUMN_NAME = "行政审批"

session = base.session


def crawl():
    page = 1
    total_added = 0
    MAX_PAGES = 49  # from JS total=779 records, 16/page

    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()

    while page <= MAX_PAGES:
        if page == 1:
            url = LIST_URL.format(page="")
        else:
            url = LIST_URL.format(page=f"_{page}")
        print(f"  第{page}页...", end=" ", flush=True)
        soup = base.get_soup(url)
        if not soup:
            print("FAIL")
            break

        items = soup.select("li.ewb-info-item")
        if not items:
            print("空列表，结束")
            break

        print(f"{len(items)}条")
        for item in items:
            a_tag = item.find("a")
            if not a_tag:
                continue
            link = a_tag.get("href", "").strip()
            if not link:
                continue
            link = base.resolve_url(url, soup, link)

            list_title = a_tag.get_text(strip=True)

            c.execute("SELECT 1 FROM gov_raw WHERE page_url=?", (link,))
            if c.fetchone():
                continue

            time.sleep(DELAY)
            ext_title, pub_date, content = base.extract_detail(link)
            if not ext_title:
                ext_title = list_title
            if not content or len(content) < 20:
                content = f'<p><a href="{link}">{ext_title}</a></p>'
                print(f"    ⚠ {list_title[:40]}... - 正文为空，回退为链接")

            try:
                c.execute(
                    """INSERT OR REPLACE INTO gov_raw (page_url, title, site_name, publish_date, content, date_rank, summary, script_name) VALUES (?,?,?,?,?,?,?, 'crawl_szwz_jshp.py')""",
                    (link, ext_title, SITE_NAME, pub_date, content,
                     pub_date.replace("-", "") if pub_date else "0", ext_title[:200]),
                )
                total_added += 1
            except Exception as e:
                print(f"    ERROR: {e}")

        if page % 10 == 0:
            conn.commit()
            print(f"  [已提交 {total_added} 条]")

        page += 1
        time.sleep(DELAY)

    conn.commit()
    conn.close()
    print(f"\n完成: 新增 {total_added} 条")
    return total_added


if __name__ == "__main__":
    print(f"站点: {SITE_NAME} - {COLUMN_NAME}")
    added = crawl()
    print(f"总计新增: {added}")