#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""crawl_xiapu_ycz_tzgg.py — 霞浦县牙城镇人民政府-通知公告"""

import os, re, sqlite3, time, sys, random, urllib.parse
from datetime import datetime, timedelta
from concurrent.futures import ThreadPoolExecutor, as_completed

DB_PATH = os.environ.get('SEARCH_DB', '/root/search.db')
SITE_NAME = '霞浦县牙城镇人民政府-通知公告'
BASE_URL = 'https://www.xiapu.gov.cn/zwgk/gkzl/bmzfxxgk/xzjdbsc/yczzf/gkml_25199/tzgg/'
CUTOFF = (datetime.now() - timedelta(days=3*365)).strftime('%Y-%m-%d')
MAX_WORKERS = 5

HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
}

import requests
sess = requests.Session()
sess.headers.update(HEADERS)


def fetch_page(url=None):
    """Fetch a page"""
    target = url or BASE_URL
    for _ in range(3):
        try:
            r = sess.get(target, timeout=15, verify=False)
            r.encoding = 'utf-8'
            return r.text
        except Exception as e:
            time.sleep(1)
    return None


def parse_static_items(html):
    """Parse article items from static HTML"""
    items = []
    # Match href="./YYYYMM/tXXXXXXXX_XXXXXXX.htm"
    for m in re.finditer(r'href="(\./\d+/t\d+_\d+\.htm)"[^>]*title="([^"]*)"', html):
        url_path = m.group(1)
        title = m.group(2).strip()
        # Build full URL
        full_url = urllib.parse.urljoin(BASE_URL, url_path)
        
        # Get date from the following <span> element
        items.append({
            'url': full_url,
            'title': title,
            'date': '',  # Will extract from detail page
        })
    return items


def fetch_detail(url):
    """Fetch detail page"""
    for _ in range(3):
        try:
            resp = requests.get(url, headers=HEADERS, timeout=15, verify=False)
            resp.encoding = 'utf-8'
            return resp.text
        except Exception as e:
            time.sleep(1)
    return None


def extract_detail(html):
    """Extract title, date, and content from detail page"""
    title = ''
    pub_date = ''
    
    # Title from meta tag
    m = re.search(r'<meta name="ArticleTitle"\s+content="([^"]+)"', html)
    if m:
        title = m.group(1).strip()
    
    # Fallback: title from h1/article_title div
    if not title:
        m = re.search(r'<div class="article_title[^"]*">\s*(.*?)\s*</div>', html, re.DOTALL)
        if m:
            title = m.group(1).strip()
    
    # Date from PubDate meta
    m = re.search(r'<meta name="PubDate"\s+content="([^"]+)"', html)
    if m:
        pub_date = m.group(1).strip()[:10]
    
    # Fallback: date from article_time span
    if not pub_date:
        m = re.search(r'发布时间：(\d{4}-\d{2}-\d{2})', html)
        if m:
            pub_date = m.group(1)
    
    # Content from TRS_Editor
    content = ''
    m = re.search(r'<div class=TRS_Editor>(.*?)</div>\s*</div>', html, re.DOTALL)
    if m:
        content = m.group(1).strip()
    
    # Clean scripts and styles
    content = re.sub(r'<script[^>]*>.*?</script>', '', content, flags=re.DOTALL)
    content = re.sub(r'<style[^>]*>.*?</style>', '', content, flags=re.DOTALL)
    
    return title, pub_date, content.strip()


def process_item(item):
    """Process one article"""
    html = fetch_detail(item['url'])
    if not html:
        return None, 'FETCH_ERR'
    
    title, pub_date, content = extract_detail(html)
    
    return {
        'site_name': SITE_NAME,
        'title': title or item['title'],
        'page_url': item['url'],
        'publish_date': pub_date or item['date'],
        'source_url': item['url'],
        'content': content,
    }, None


def main():
    print(f'[{datetime.now().strftime("%H:%M:%S")}] {SITE_NAME}')
    
    html = fetch_page()
    if not html:
        print('[ERROR] Failed to fetch page')
        return
    
    items = parse_static_items(html)
    print(f'Static items: {len(items)}')
    
    # Extract dates from list page spans
    # The <span> after each <a> contains the date
    date_spans = re.findall(r'<span>(\d{4}-\d{2}-\d{2})</span>', html)
    for i, item in enumerate(items):
        if i < len(date_spans):
            item['date'] = date_spans[i]
    
    recent = [i for i in items if i['date'] >= CUTOFF]
    print(f'Within 3 years: {len(recent)} (filtered: {len(items)-len(recent)})')
    
    conn = sqlite3.connect(DB_PATH, timeout=30)
    c = conn.cursor()
    existing = set(r[0] for r in c.execute('SELECT page_url FROM gov_raw WHERE site_name=?', (SITE_NAME,)))
    
    to_fetch = [i for i in recent if i['url'] not in existing]
    print(f'New items: {len(to_fetch)}')
    
    new = err = 0
    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
        futures = {executor.submit(process_item, item): item for item in to_fetch}
        for i, future in enumerate(as_completed(futures), 1):
            item = futures[future]
            result, error = future.result()
            if error or not result:
                print(f'  [{i}/{len(to_fetch)}] {item["title"][:30]}... {error or "FAIL"}')
                err += 1
                continue
            try:
                c.execute('''INSERT OR IGNORE INTO gov_raw
                    (site_name, title, page_url, publish_date, source_url, content)
                    VALUES (?,?,?,?,?,?)''', (
                        result['site_name'], result['title'], result['page_url'],
                        result['publish_date'], result['source_url'], result['content']))
                if c.rowcount:
                    new += 1
                    print(f'  [{i}/{len(to_fetch)}] {item["title"][:30]}... OK ({len(result["content"])}B)')
                else:
                    print(f'  [{i}/{len(to_fetch)}] {item["title"][:30]}... DUP')
            except Exception as e:
                print(f'  [{i}/{len(to_fetch)}] {item["title"][:30]}... ERR: {e}')
                err += 1
            conn.commit()
    
    conn.close()
    print(f'\n=== Done === New: {new}, Errors: {err}')


if __name__ == '__main__':
    main()