#!/usr/bin/env python3
import os
"""
巨野县人民政府 - 法定主动公开内容 爬虫 (修复版)
www.juye.gov.cn /fdnew/fdgk.html?catas=1567399802659016704
API: POST /els-service/article/{page}/10
- 修复 var memo 提取逻辑（正则匹配）
- 空正文时跳过插入
"""

import requests
import sqlite3
import re
import time
import json
from datetime import datetime, date

HEADERS = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'}
SITE_NAME = '巨野县人民政府-法定主动公开内容'
CUTOFF_DATE = date(2023, 6, 1)
DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")


def get_list(page):
    url = f'http://www.juye.gov.cn/els-service/article/{page}/10'
    try:
        r = requests.post(url, json={
            "catas": ["1567399802659016704"],
            "dq": "89003",
            "order": "up"
        }, headers=HEADERS, timeout=15)
        data = r.json()
        if not data.get('success'):
            return []
        items = []
        for c in data['data']['contents']:
            items.append({
                'xxid': c['xxid'],
                'dwid': c['dwid'],
                'subject': c['subject'],
                'fwdate': c['fwdate'],
            })
        return items
    except Exception as e:
        print(f'  [ERR] list page {page}: {e}')
        return []


def extract_memo_content(html):
    """Extract content from var memo = '...' or var memo = \"...\" in HTML."""
    # Try regex: var memo = "..." (handle escaped quotes)
    m = re.search(r'var\s+memo\s*=\s*"((?:[^"\\]|\\.)*)"', html, re.DOTALL)
    if m:
        raw = m.group(1)
    else:
        # Try single-quoted version
        m = re.search(r"var\s+memo\s*=\s*'((?:[^'\\]|\\.)*)'", html, re.DOTALL)
        if m:
            raw = m.group(1)
        else:
            return ''
    
    # Unescape: replace \/ with /
    raw = raw.replace(r'\/', '/')
    # Unescape: replace \\n, \\t, etc.
    try:
        raw = raw.encode().decode('unicode_escape')
    except:
        pass
    # Replace unicode escapes like \u4E00
    try:
        raw = raw.encode().decode('unicode_escape')
    except:
        pass
    
    return raw


def fetch_detail(dqid, dwid, xxid):
    url = f'http://www.juye.gov.cn/{dqid}/{dwid}/{xxid}.html'
    try:
        r = requests.get(url, headers=HEADERS, timeout=15)
        r.encoding = 'utf-8'
        
        if r.status_code != 200:
            print(f'    [WARN] {url} HTTP {r.status_code}')
            return url, ''
        
        # Extract memo content using regex
        content = extract_memo_content(r.text)
        return url, content
    except Exception as e:
        print(f'  [ERR] detail {xxid}: {e}')
        return url, ''


def main():
    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()
    total_added = 0
    no_content = 0
    total_pages = min(384, 5)  # limited from 384

    for page in range(1, total_pages + 1):
        print(f'Page {page}/{total_pages}...', end=' ', flush=True)
        items = get_list(page)
        if not items:
            print('empty')
            continue

        page_added = 0
        for item in items:
            # Date filter
            try:
                d = datetime.strptime(item['fwdate'], '%Y-%m-%d').date()
            except:
                d = date(1900, 1, 1)
            if d < CUTOFF_DATE:
                continue

            detail_url = f'http://www.juye.gov.cn/89003/{item["dwid"]}/{item["xxid"]}.html'
            
            # Check if exists
            c.execute('SELECT 1 FROM gov_raw WHERE page_url = ?', (detail_url,))
            if c.fetchone():
                continue

            # Fetch detail
            url, content = fetch_detail('89003', item['dwid'], item['xxid'])
            if not content:
                no_content += 1
                print(f'    [SKIP] 空正文: {item["subject"][:40]}')
                continue  # Skip insertion - don't store empty records

            c.execute('''INSERT OR IGNORE INTO gov_raw
                (site_name, source_url, page_url, title, publish_date, content, summary)
                VALUES (?, ?, ?, ?, ?, ?, ?)''',
                (SITE_NAME, detail_url, detail_url, item['subject'],
                 item['fwdate'], content, item['subject']))
            page_added += 1
            total_added += 1
            if total_added % 10 == 0:
                time.sleep(0.2)

        print(f'+{page_added}')
        conn.commit()
        time.sleep(0.3)

    conn.close()
    print(f'\nDone! Added: {total_added}, Skipped(empty): {no_content}')


if __name__ == '__main__':
    main()