#!/usr/bin/env python3
"""
鄄城县人民政府 - 通知公告 爬虫
API: POST /els-service/article/{page}/15
"""

import requests, sqlite3, re, time
from datetime import datetime, date
import os

HEADERS = {'User-Agent': 'Mozilla/5.0'}
SITE_NAME = '鄄城县人民政府-通知公告'
CUTOFF_DATE = date(2023, 6, 1)
DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
CATAS = '1574201302689386496'

def get_list(page):
    try:
        r = requests.post(f'http://www.juancheng.gov.cn/els-service/article/{page}/15', json={
            "dq": "88904", "catas": [CATAS]
        }, headers=HEADERS, timeout=15)
        data = r.json()
        if not data.get('success'): return []
        return [{'xxid':c['xxid'],'dwid':c['dwid'],'subject':c['subject'],'fwdate':c['fwdate']} for c in data['data']['contents']]
    except:
        return []

def fetch_detail(dwid, xxid):
    url = f'http://www.juancheng.gov.cn/88904/{dwid}/{xxid}.html'
    try:
        r = requests.get(url, headers=HEADERS, timeout=20)
        r.encoding = 'utf-8'
        idx = r.text.find('var memo')
        if idx < 0: return url, ''
        start = r.text.find('"', idx + 8)
        if start < 0: return url, ''
        end = start + 1
        while end < len(r.text):
            end = r.text.find('"', end)
            if end < 0: return url, ''
            if end > 0 and r.text[end-1] == '\\': end += 1; continue
            rest = r.text[end+1:end+3].strip()
            if rest == '' or rest == ';': break
            end += 1
        raw = r.text[start+1:end]
        content = raw.replace(r'\/', '/')
        try: content = content.encode().decode('unicode_escape')
        except: pass
        return url, content
    except:
        return url, ''

def main():
    conn = sqlite3.connect(DB_PATH)
    c = conn.cursor()
    total_added = 0
    no_content = 0

    for page in range(1, 20):  # Pages 1-19 for 3-year range
        print(f'Page {page}/19...', end=' ', flush=True)
        items = get_list(page)
        if not items:
            print('empty')
            continue
        page_added = 0
        for item in items:
            try: d = datetime.strptime(item['fwdate'], '%Y-%m-%d').date()
            except: d = date(1900,1,1)
            if d < CUTOFF_DATE: continue
            
            detail_url = f'http://www.juancheng.gov.cn/88904/{item["dwid"]}/{item["xxid"]}.html'
            c.execute('SELECT 1 FROM gov_raw WHERE page_url = ?', (detail_url,))
            if c.fetchone(): continue
            
            url, content = fetch_detail(item['dwid'], item['xxid'])
            if not content: no_content += 1
            
            c.execute('''INSERT OR IGNORE INTO gov_raw
                (site_name, source_url, page_url, title, publish_date, content, summary)
                VALUES (?, ?, ?, ?, ?, ?, ?)''',
                (SITE_NAME, detail_url, detail_url, item['subject'],
                 item['fwdate'], content, item['subject']))
            page_added += 1
            total_added += 1
            if total_added % 10 == 0: time.sleep(0.2)
        
        print(f'+{page_added}')
        conn.commit()
        time.sleep(0.3)

    conn.close()
    print(f'\nDone! Added: {total_added}, NoContent: {no_content}')

if __name__ == '__main__':
    main()