#!/usr/bin/env python3
"""双鸭山-通知公告 爬虫"""
import json, sqlite3, re, os, sys
from datetime import datetime, timezone, timedelta
from urllib.parse import urljoin

BASE = 'http://www.shuangyashan.gov.cn'
API = '/common/search/2b514013230043a2bc6ffe91cdc1c338?_isAgg=false&_isJson=true&_pageSize=20&_template=index&_rangeTimeGte=&_channelName=&page={page}'
SITE_NAME = '双鸭山通知公告'
DB = os.getenv("SEARCH_DB", "/root/search.db")

tz = timezone(timedelta(hours=8))
cutoff = datetime.now(tz) - timedelta(days=365*3)
cutoff_ts = int(cutoff.timestamp() * 1000)
print(f'Cutoff: {cutoff.isoformat()} (ts={cutoff_ts})')

count = 0
page = 1
while True:
    url = BASE + API.format(page=page)
    resp = os.popen(f'curl -s "{url}"').read()
    if not resp.strip():
        print(f'Page {page}: empty response, stop')
        break
    try:
        data = json.loads(resp)
    except:
        print(f'Page {page}: parse error, stop')
        break
    results = data.get('data', {}).get('results', [])
    if not results:
        print(f'Page {page}: no results, stop')
        break
    print(f'Page {page}: {len(results)} results')
    all_before_cutoff = True
    for r in results:
        pub_ts = r.get('publishedTime', 0)
        if pub_ts < cutoff_ts:
            continue
        all_before_cutoff = False
        title = r.get('title', '').strip()
        rel_url = r.get('url', '').strip()
        full_url = urljoin(BASE, rel_url)
        pub_str = r.get('publishedTimeStr', '')
        content_html = r.get('contentHtml', '') or r.get('content', '') or ''
        content_html = re.sub(r'<style[^>]*>.*?</style>', '', content_html, flags=re.DOTALL)
        content_html = content_html.strip()
        if content_html and not content_html.startswith('<'):
            content_html = '<p>' + content_html + '</p>'
        pub_date = pub_str[:10] if len(pub_str) >= 10 else ''
        if not pub_date:
            continue
        try:
            conn = sqlite3.connect(DB, timeout=60)
            c = conn.cursor()
            sql = 'INSERT OR REPLACE INTO gov_raw (id, title, content, publish_date, source_url, page_url, site_name, summary) VALUES (?, ?, ?, ?, ?, ?, ?, ?)'
            c.execute(sql, (None, title, content_html, pub_date, full_url, full_url, SITE_NAME, title[:200]))
            conn.commit()
            conn.close()
            count += 1
        except Exception as e:
            print(f'  DB error: {e}')
    if all_before_cutoff:
        print(f'All remaining articles before cutoff, stop')
        break
    page += 1
    if page > 200:
        break

print(f'\nDone. Total: {count} articles imported.')