#!/usr/bin/env python3
"""上高县-建设项目环境影响评价 爬虫"""
import json, sqlite3, os, re
from datetime import datetime, timezone, timedelta
from urllib.parse import urljoin

BASE = 'http://www.shanggao.gov.cn'
API_URL = BASE + '/searchManuscript'
SITE_NAME = '上高环评公示'
DB = os.getenv("SEARCH_DB", "/root/search.db")

tz = timezone(timedelta(hours=8))
cutoff = datetime.now(tz) - timedelta(days=365*3)
cutoff_str = cutoff.strftime('%Y-%m-%d')
print(f'Cutoff: {cutoff_str}')

count = 0
page = 1
while True:
    payload = json.dumps({
        "current": page, "pageSize": 20,
        "channelTreeIds": ["1996849911248297984"]
    })
    resp = os.popen(f'curl -s "{API_URL}" -X POST -H "Content-Type: application/json" -d \'{payload}\'').read()
    if not resp.strip():
        print(f'Page {page}: empty, stop')
        break
    
    try:
        data = json.loads(resp)
    except:
        print(f'Page {page}: parse error, stop')
        break
    
    results = data.get('data', {}).get('results', [])
    if not results:
        print(f'Page {page}: no results, stop')
        break
    
    print(f'Page {page}: {len(results)} items ({results[0]["pubDate"][:10]} ~ {results[-1]["pubDate"][:10]})')
    all_before = True
    
    for r in results:
        pub_date = r.get('pubDate', '')[:10]
        if pub_date < cutoff_str:
            continue
        all_before = False
        
        title = r.get('title', '').strip()
        content_html = r.get('content', {}).get('content', '')
        if not content_html:
            continue
        
        content_html = re.sub(r'<script[^>]*>.*?</script>', '', content_html, flags=re.DOTALL)
        content_html = re.sub(r'<style[^>]*>.*?</style>', '', content_html, flags=re.DOTALL)
        content_html = content_html.strip()
        if len(content_html) < 20:
            continue
        
        # Parse urls field (JSON string)
        page_url = ''
        urls_str = r.get('urls', '{}')
        if isinstance(urls_str, str):
            try:
                urls_data = json.loads(urls_str)
                page_url = urls_data.get('pc', '')
            except:
                pass
        
        if not page_url:
            article_id = r.get('id', '')
            page_url = f'/sgxrmzf/jsxmhjyjpj2/pc/content/{article_id}/content_{article_id}.html'
        
        full_url = urljoin(BASE, page_url)
        
        try:
            conn = sqlite3.connect(DB, timeout=60)
            c = conn.cursor()
            c.execute('INSERT OR REPLACE INTO gov_raw (id, title, content, publish_date, source_url, page_url, site_name, summary) VALUES (?,?,?,?,?,?,?,?)',
                      (None, title, content_html, pub_date, full_url, full_url, SITE_NAME, title[:200]))
            conn.commit()
            conn.close()
            count += 1
        except Exception as e:
            print(f'  DB error: {e}')
    
    if all_before:
        print(f'All before cutoff, stop')
        break
    page += 1
    if page > 30:
        break

print(f'\nDone. Total: {count} articles imported.')