#!/usr/bin/env python3
"""crawl_xceda.py - 宣城经开区管委会-公示公告"""
import requests
import re
import sqlite3
import os
import sys
from datetime import datetime, timedelta



import sys as _SYS
_MAX_PG = int(_SYS.argv[1]) if len(_SYS.argv) > 1 and _SYS.argv[1].isdigit() else None
if _MAX_PG is not None:
    print('[AutoPg] max_pages=' + str(_MAX_PG))
# END AUTO PAGES
DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "宣城经开区管委会-公示公告"
BASE_URL = "https://xceda.xuancheng.gov.cn"
LIST_URL = "https://xceda.xuancheng.gov.cn/News/showList/3245/page_{}.html"
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
}
THREE_YEARS_AGO = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
# Use 2023-06-19 as cutoff for consistency
CUTOFF_DATE = "2023-06-19"

session = requests.Session()
session.headers.update(HEADERS)

def get_total_pages():
    """Get total pages from first page"""
    try:
        r = session.get(LIST_URL.format(1), timeout=30)
        r.encoding = 'utf-8'
        m = re.search(r'pagecount="(\d+)"', r.text)
        if m:
            return int(m.group(1))
        m = re.search(r'total="(\d+)"', r.text)
        if m:
            total = int(m.group(1))
            return (total + 19) // 20  # ceil division
        return 94  # fallback
    except Exception as e:
        print(f"Error getting total pages: {e}", flush=True)
        return 94

def parse_list(html):
    """Parse list page, return list of (url, title, date)"""
    items = []
    for m in re.finditer(
        r'<li>\s*<span>\[(\d{4}-\d{2}-\d{2})\]</span>\s*<a href="(/News/show/\d+\.html)"[^>]*title="([^"]*)"',
        html
    ):
        date_str = m.group(1)
        url = BASE_URL + m.group(2)
        title = m.group(3).strip()
        
        # Filter: within 3 years
        if date_str < CUTOFF_DATE:
            continue
        
        items.append((url, title, date_str))
    return items

def fetch_detail(url):
    """Fetch detail page and extract content"""
    try:
        r = session.get(url, timeout=30)
        r.encoding = 'utf-8'
        html = r.text
        
        # Extract content from <div class="m-contnet j-fontContent" id="zoom">
        start = html.find('<div class="m-contnet j-fontContent"')
        if start < 0:
            start = html.find('class="m-contnet j-fontContent"')
            if start >= 0:
                # Find the opening <div
                prefix = html[:start]
                div_start = prefix.rfind('<div')
                if div_start >= 0:
                    start = div_start
                else:
                    return None
        
        if start < 0:
            return None
        
        # Find matching closing </div> with nesting
        depth = 0
        i = start
        in_tag = False
        while i < len(html):
            if html[i] == '<':
                in_tag = True
                tag_start = i
            elif html[i] == '>':
                if in_tag:
                    tag = html[tag_start:i+1]
                    in_tag = False
                    if tag.startswith('<!--'):
                        continue
                    if tag.startswith('</div'):
                        depth -= 1
                        if depth == 0:
                            content = html[start:i+1]
                            # Clean up empty paragraphs at start
                            content = re.sub(r'^\s*(?:<p[^>]*>\s*(?:&nbsp;|\s)*\s*</p>\s*|<br\s*/?>\s*)*', '', content)
                            # Remove h1 title if it's the same as article title (duplicate)
                            content = re.sub(r'<h1[^>]*>.*?</h1>', '', content)
                            return content.strip()
                    elif not tag.startswith('<br') and not tag.startswith('<img') and not tag.startswith('<input') and not tag.startswith('<hr'):
                        if tag.startswith('<div'):
                            depth += 1
                in_tag = False
            elif html[i] == '\n':
                if in_tag:
                    # Check for comment
                    pass
            i += 1
        return None
    except Exception as e:
        print(f"Error fetching {url}: {e}", flush=True)
        return None

def save_to_db(items):
    """Save items to database"""
    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()
    
    new_count = 0
    skip_count = 0
    error_count = 0
    
    for url, title, date_str in items:
        try:
            content = fetch_detail(url)
            if not content:
                print(f"  [SKIP] No content: {title[:40]}", flush=True)
                error_count += 1
                continue
            
            c.execute(
                "INSERT OR IGNORE INTO gov_raw (site_name, title, page_url, publish_date, content) VALUES (?, ?, ?, ?, ?)",
                (SITE_NAME, title, url, date_str, content)
            )
            if c.rowcount > 0:
                new_count += 1
                print(f"  [NEW] {title[:50]} | {date_str}", flush=True)
            else:
                skip_count += 1
        except Exception as e:
            print(f"  [ERR] {title[:40]}: {e}", flush=True)
            error_count += 1
    
    conn.commit()
    conn.close()
    return new_count, skip_count, error_count

def main():
    print(f"=== {SITE_NAME} ===", flush=True)
    print(f"Cutoff: {CUTOFF_DATE}", flush=True)
    
    total_pages = get_total_pages()
    print(f"Total pages: {total_pages}", flush=True)
    
    all_items = []
    for page in range(1, min(total_pages, _MAX_PG or total_pages)+1):
        try:
            r = session.get(LIST_URL.format(page), timeout=30)
            r.encoding = 'utf-8'
            items = parse_list(r.text)
            if not items:
                print(f"  Page {page}: no items (stopping)", flush=True)
                break
            all_items.extend(items)
            print(f"  Page {page}: {len(items)} items", flush=True)
        except Exception as e:
            print(f"  Page {page} error: {e}", flush=True)
            continue
    
    print(f"\nTotal items to process: {len(all_items)}", flush=True)
    
    if not all_items:
        print("No new items found.", flush=True)
        return
    
    new_count, skip_count, error_count = save_to_db(all_items)
    
    print(f"\n=== Summary ===", flush=True)
    print(f"New: {new_count}, Skipped: {skip_count}, Errors: {error_count}", flush=True)

if __name__ == "__main__":
    main()