#!/usr/bin/env python3
"""沿滩区人民政府 - 通知公告 爬虫 (数融CMS API)"""
import json, urllib.request, sys, re
from datetime import datetime

DB_PATH = "/root/search.db"
API_URL = "https://www.zgyt.gov.cn/queryList"
SITE_NAME = "zgyt"
CHANNEL_CODE = "tzgg2254"
WEBSITE_CODE = "ytqrmzf"
PAGE_SIZE = 100
MAX_PAGES = 5

def extract_text(html_content):
    if not html_content:
        return ""
    text = re.sub(r'<[^>]+>', " ", html_content)
    text = re.sub(r'\s+', " ", text).strip()
    return text[:200]

def crawl():
    now = datetime.now()
    import sqlite3
    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()

    c.execute("SELECT COUNT(*) FROM gov_raw WHERE site_name=?", (SITE_NAME,))
    existing = c.fetchone()[0]
    incremental = existing > 0
    max_pages = 1 if incremental else MAX_PAGES
    total_new = 0

    for page in range(1, max_pages + 1):
        payload = json.dumps({
            "current": page, "pageSize": PAGE_SIZE,
            "webSiteCode": [WEBSITE_CODE], "channelCode": [CHANNEL_CODE]
        }).encode('utf-8')
        req = urllib.request.Request(API_URL, data=payload,
            headers={'Content-Type': 'application/json'})
        try:
            resp = urllib.request.urlopen(req, timeout=30)
            data = json.loads(resp.read().decode('utf-8'))
        except Exception as e:
            print(f"第{page}页请求失败: {e}", flush=True)
            continue

        items = data.get('data', {}).get('results', [])
        if not items:
            break

        for item in items:
            src = item.get('source', {})
            title = src.get('title', '').strip()
            if not title:
                continue
            pub_date = src.get('pubDate', '')
            if pub_date:
                pub_date = pub_date.split(' ')[0]
            urls_json = src.get('urls', '{}')
            try:
                urls_obj = json.loads(urls_json) if isinstance(urls_json, str) else urls_json
                page_url = "https://www.zgyt.gov.cn" + urls_obj.get('pc', '')
            except:
                page_url = ""
            content_obj = src.get('content', {})
            content = ''
            if isinstance(content_obj, dict):
                content = content_obj.get('content', '')
            elif isinstance(content_obj, str):
                content = content_obj
            summary = extract_text(content)

            # 检查重复 (page_url 有唯一索引)
            try:
                c.execute(
                    "INSERT INTO gov_raw (site_name, page_url, title, publish_date, summary, content, category, source_url) VALUES (?,?,?,?,?,?,?,?)",
                    (SITE_NAME, page_url, title, pub_date, summary, content, '通知公告',
                     f"https://www.zgyt.gov.cn/ytqrmzf/tzgg2254/pc/list.html")
                )
                conn.commit()
                total_new += 1
            except sqlite3.IntegrityError:
                pass  # 重复跳过
            except Exception as e:
                print(f"入库失败 [{title[:30]}]: {e}", flush=True)
                conn.rollback()

        print(f"第{page}页完成，本页{len(items)}条，新增累计{total_new}条", flush=True)
    conn.close()
    print(f"\n沿滩区通知公告爬取完成，共新增 {total_new} 条", flush=True)

if __name__ == '__main__':
    crawl()