#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
禄劝彝族苗族自治县人民政府-公示公告 爬虫
TRS CMS, shtml pagination
List: /zfxxgkml2/zfxxgkzd/gsgg/ (p1), index_N.shtml (N=2..10)
Detail: /c/YYYY-MM-DD/NUM.shtml
"""

import re, time, os
from urllib.parse import urljoin
import requests
from bs4 import BeautifulSoup

BASE_URL = "http://www.kmlq.gov.cn/zfxxgkml2/zfxxgkzd/gsgg"
TOTAL_PAGES = 10
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

def safe_get(url, timeout=15):
    for attempt in range(3):
        try:
            r = requests.get(url, headers=HEADERS, timeout=timeout)
            r.encoding = 'utf-8'
            return r
        except Exception as e:
            if attempt < 2:
                time.sleep(2)
            else:
                print(f"  [WARN] Failed {url}: {e}")
                return None

def strip_inline_tags(html_str):
    """Strip inline formatting tags that cause number splits."""
    return re.sub(
        r'</?(?:span|b|strong|font|em|i|u|s|sub|sup|small|mark)[^>]*>',
        '', html_str, flags=re.I
    )

def extract_content(activity_div, page_url):
    parts = []
    
    for child in activity_div.children:
        name = getattr(child, 'name', None)
        
        if name == 'table':
            parts.append(str(child))
            parts.append("\n\n")
        elif name in ('p', 'div'):
            inner_table = child.find('table')
            if inner_table:
                parts.append(str(inner_table))
                parts.append("\n\n")
            
            # Images
            for img in child.find_all('img'):
                src = img.get('src', '')
                if src:
                    parts.append(f"![image]({urljoin(page_url, src)})\n\n")
            
            # Attachments
            for a in child.find_all('a', href=True):
                h = a['href']
                if any(h.lower().endswith(e) for e in ('.doc', '.docx', '.pdf', '.xls', '.xlsx', '.zip')):
                    parts.append(f"[{a.get_text(' ', strip=True)}]({urljoin(page_url, h)})\n\n")
            
            # Clean text
            child_html = strip_inline_tags(str(child))
            clean = BeautifulSoup(child_html, 'html.parser')
            txt = clean.get_text(' ', strip=True)
            txt = re.sub(r'\s+', ' ', txt).strip() if txt else ""
            if txt:
                parts.append(txt)
                parts.append("\n\n")
        elif name == 'h1':
            txt = child.get_text(' ', strip=True)
            if txt:
                parts.append(f"# {txt}\n\n")
        elif name == 'h2':
            txt = child.get_text(' ', strip=True)
            if txt:
                parts.append(f"## {txt}\n\n")
        elif name == 'ul':
            for li in child.find_all('li', recursive=False):
                li_text = li.get_text(' ', strip=True)
                if li_text:
                    parts.append(f"- {li_text}\n")
            parts.append("\n")
        elif name == 'ol':
            for li in child.find_all('li', recursive=False):
                li_text = li.get_text(' ', strip=True)
                if li_text:
                    parts.append(f"  {li_text}\n")
            parts.append("\n")
        elif name is None:
            t = str(child).strip()
            if t and len(t) > 5:
                parts.append(t)
                parts.append("\n\n")
    
    result = ''.join(parts).strip()
    result = re.sub(r'\n{4,}', '\n\n', result)
    return result

def parse_detail(url):
    r = safe_get(url)
    if not r:
        return None, None, None
    
    soup = BeautifulSoup(r.text, "html.parser")
    
    # Title from h1 inside content div
    title = ""
    content_div = soup.find("div", class_="content")
    if content_div:
        h1 = content_div.find("h1")
        if h1:
            title = h1.get_text(strip=True)
    if not title:
        meta = soup.find("meta", attrs={"name": "ArticleTitle"})
        if meta:
            title = meta.get("content", "")
    if not title:
        title_tag = soup.find("title")
        if title_tag:
            t = title_tag.get_text(strip=True)
            title = re.sub(r'_禄劝彝族苗族自治县人民政府.*', '', t).strip()
    
    # Date
    date = ""
    activity_info = soup.find("div", class_="activity-info")
    if activity_info:
        m = re.search(r'(\d{4}[-/\.]\d{1,2}[-/\.]\d{1,2})', activity_info.get_text())
        if m:
            date = m.group(1).replace("/", "-").replace(".", "-")
    if not date:
        meta = soup.find("meta", attrs={"name": "PubDate"})
        if meta:
            d = meta.get("content", "")
            m = re.search(r'(\d{4}[-/\.]\d{1,2}[-/\.]\d{1,2})', d)
            if m:
                date = m.group(1).replace("/", "-").replace(".", "-")
    
    # Content
    content = ""
    content_div = soup.find("div", class_="content")
    if content_div:
        activity = content_div.find("div", class_="activity")
        if activity:
            content = extract_content(activity, url)
        else:
            content = extract_content(content_div, url)
    
    return title, content, date

def parse_list_page(url):
    r = safe_get(url)
    if not r:
        return []
    
    soup = BeautifulSoup(r.text, "html.parser")
    seen = set()
    items = []
    
    for a in soup.find_all("a", href=True):
        href = a["href"].strip()
        if not href.startswith("/c/") and not href.startswith(f"{BASE_URL.split('/zfxxgkml2')[0]}/c/"):
            continue
        
        title = a.get_text(strip=True)
        if len(title) < 8:
            continue
        
        # Dedup: each article appears twice (title + date link)
        if href in seen:
            continue
        seen.add(href)
        
        # Normalize
        full_url = urljoin("http://www.kmlq.gov.cn", href)
        
        # Date from URL
        m = re.search(r'/c/(\d{4}-\d{2}-\d{2})/', href)
        date = m.group(1) if m else ""
        
        items.append((title, full_url, date))
    
    return items

def main():
    import sqlite3
    
    db_paths = ["/root/search.db", "/root/gov_crawler/search.db"]
    db_path = None
    for p in db_paths:
        if os.path.exists(p):
            db_path = p
            break
    if not db_path:
        db_path = "/root/search.db"
    
    print(f"=== 禄劝彝族苗族自治县-公示公告 ===")
    
    all_items = []
    for page in range(1, TOTAL_PAGES + 1):
        if page == 1:
            url = BASE_URL + "/"
        else:
            url = f"{BASE_URL}/index_{page}.shtml"
        
        print(f"  Fetching page {page}/{TOTAL_PAGES}: {url}")
        items = parse_list_page(url)
        if not items:
            print(f"    Empty, stopping")
            break
        print(f"    Found {len(items)} items")
        all_items.extend(items)
        time.sleep(0.2)
    
    print(f"\n  Total: {len(all_items)} items\n")
    
    if not all_items:
        return
    
    db = sqlite3.connect(db_path, timeout=60)
    db.execute("PRAGMA journal_mode=WAL")
    db.execute("PRAGMA synchronous=OFF")
    
    site_name = "禄劝县-公示公告"
    inserted = 0
    skipped = 0
    errors = 0
    
    for idx, (title, href, list_date) in enumerate(all_items):
        existing = db.execute("SELECT id FROM gov_raw WHERE page_url = ?", (href,)).fetchone()
        if existing:
            skipped += 1
            continue
        
        detail_title, content, detail_date = parse_detail(href)
        if not content:
            errors += 1
            print(f"  [WARN] Empty: {title[:50]}")
            continue
        
        final_title = detail_title or title
        final_date = detail_date or list_date
        content = content[:500000] if len(content) > 500000 else content
        
        try:
            db.execute(
                "INSERT OR IGNORE INTO gov_raw (title, content, page_url, publish_date, site_name) VALUES (?, ?, ?, ?, ?)",
                (final_title, content, href, final_date, site_name)
            )
            db.commit()
            inserted += 1
        except Exception as e:
            print(f"  [DB ERROR] {e}")
            db.rollback()
        
        if (idx + 1) % 50 == 0:
            print(f"  Progress: {idx+1}/{len(all_items)} (inserted={inserted}, skipped={skipped})")
        
        time.sleep(0.15)
    
    # Bulk FTS
    print("\nRebuilding FTS...")

    
    db.close()
    
    print(f"\n========================================")
    print(f"Inserted: {inserted}/{len(all_items)} new")
    print(f"Skipped: {skipped}")
    print(f"Errors: {errors}")

if __name__ == "__main__":
    main()