#!/usr/bin/env python3
"""


乌拉特中旗-通知公告 爬虫
http://www.wltzq.gov.cn/ywdt/tzgg/
CMS: Custom TRS, static HTML pages
分页: index.html + index_N.html (N=1~23, 24页)
列表: 每个li在独立ul中, a[href^="./202"]
详情: div.trs_editor_view.TRS_UEDITOR
"""
import sys
import time
import re
import requests
import sqlite3
from bs4 import BeautifulSoup
from urllib.parse import urljoin

import sys as _SYS
_MAX_PG = int(_SYS.argv[1]) if len(_SYS.argv) > 1 and _SYS.argv[1].isdigit() else None
if _MAX_PG is not None:
    print('[AutoPg] max_pages=' + str(_MAX_PG))
# END AUTO PAGES
DB_PATH = "/mnt/data/search.db"
BASE_URL = "http://www.wltzq.gov.cn/ywdt/tzgg/"
SITE_NAME = "乌拉特中旗-通知公告"
MAX_PAGES = 24

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
}

def crawl_page(page_num):
    """Parse list page. page_num=0 -> index.html, page_num>=1 -> index_N.html"""
    if page_num == 0:
        url = BASE_URL + "index.html"
    else:
        url = f"{BASE_URL}index_{page_num}.html"
    
    r = requests.get(url, headers=HEADERS, timeout=30)
    r.encoding = 'utf-8'
    if r.status_code != 200:
        print(f"  HTTP {r.status_code}, stopping")
        return None
    
    soup = BeautifulSoup(r.text, 'html.parser')
    
    # Find content items: each li with a[href^="./202"]
    items = []
    for li in soup.find_all('li'):
        a = li.find('a', href=True)
        if not a:
            continue
        href = a['href'].strip()
        if not href.startswith('./202'):
            continue
        
        title = a.get_text(strip=True)
        if len(title) < 5:
            continue
        
        span = li.find('span')
        date = span.get_text(strip=True) if span else ''
        
        # Date is split across day/month/year spans: <span class="day">28</span>
        # <span class="month">-08</span> <span class="year">2026</span> -> 2026-08-28
        if date:
            day_el = li.find('span', class_='day')
            mon_el = li.find('span', class_='month')
            yr_el = li.find('span', class_='year')
            if day_el and mon_el and yr_el:
                dd = day_el.get_text(strip=True).zfill(2)
                mm = mon_el.get_text(strip=True).strip('-').zfill(2)
                yyyy = yr_el.get_text(strip=True)
                if dd.isdigit() and mm.isdigit() and yyyy.isdigit():
                    date = f"{yyyy}-{mm}-{dd}"
            else:
                # fallback: try to normalize a bare date string
                m = re.search(r'(\d{4})[-/年]?(\d{1,2})[-/月]?(\d{1,2})', date)
                if m:
                    date = f"{m.group(1)}-{int(m.group(2)):02d}-{int(m.group(3)):02d}"
        
        # URL fallback: ./202608/t20260828_590496.html -> 2026-08-28
        if not re.match(r'^\d{4}-\d{2}-\d{2}', date):
            um = re.search(r't(\d{4})(\d{2})(\d{2})_', href)
            if um:
                date = f"{um.group(1)}-{um.group(2)}-{um.group(3)}"
        
        full_url = urljoin(url, href)
        items.append({'title': title, 'url': full_url, 'date': date})
    
    return items


def fetch_detail(item):
    """Parse detail page"""
    r = requests.get(item['url'], headers=HEADERS, timeout=30)
    r.encoding = 'utf-8'
    soup = BeautifulSoup(r.text, 'html.parser')
    
    # Content - TRS editor
    content_div = soup.select_one('div.trs_editor_view.TRS_UEDITOR')
    if not content_div:
        content_div = soup.select_one('div.art-con')
    if not content_div:
        content_div = soup.find('div', class_=lambda x: x and 'TRS_Editor' in (x or ''))
    
    if not content_div:
        print(f"  No content for {item['url']}")
        return None
    
    for tag in content_div.find_all(['script', 'style', 'iframe']):
        tag.decompose()
    
    # Recursive content extraction - handles any nesting depth
    def extract_content(node):
        """Recursively extract content preserving paragraph breaks and table HTML"""
        parts = []
        for child in node.children:
            tag = getattr(child, 'name', None)
            if tag == 'p':
                txt = child.get_text(separator='', strip=True)
                if txt:
                    parts.append(txt)
            elif tag == 'table':
                parts.append(str(child))
            elif tag == 'div':
                sub = extract_content(child)
                if sub:
                    parts.append(sub)
            elif tag is None and isinstance(child, str):
                txt = child.strip()
                if txt and len(txt) > 3:
                    parts.append(txt)
        return '\n\n'.join(parts) if parts else ''
    
    content = extract_content(content_div)
    if not content or len(content) < 20:
        content = content_div.get_text(separator='', strip=True)
        if not content or len(content) < 20:
            return None
    
    if not content or len(content) < 20:
        print(f"  Content too short for {item['url']}")
        return None
    
    # Title
    title_tag = soup.find('title')
    full_title = title_tag.get_text(strip=True) if title_tag else ''
    full_title = re.sub(r'\s*[-_―]\s*乌拉特中旗人民政府.*', '', full_title).strip()
    if not full_title or len(full_title) < len(item['title']):
        full_title = item['title']
    
    return {
        'title': full_title,
        'content': content,
        'date': item['date'],
        'url': item['url'],
        'site_name': SITE_NAME,
    }


def save_to_db(records):
    if not records:
        return 0
    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()
    inserted = 0
    for rec in records:
        try:
            c.execute(
                "INSERT OR IGNORE INTO gov_raw (title, content, page_url, publish_date, site_name) "
                "VALUES (?, ?, ?, ?, ?)",
                (rec['title'], rec['content'], rec['url'], rec['date'], rec['site_name'])
            )
            if c.rowcount > 0:
                inserted += 1
        except Exception as e:
            print(f"  DB error: {e}")
    conn.commit()
    conn.commit()
    conn.close()
    return inserted


def main():
    import argparse

    if '_MAX_PG' in globals() and _MAX_PG is not None:
        import sys
        sys.argv = [sys.argv[0]] + ['--pages', str(_MAX_PG)] + sys.argv[2:]

    parser = argparse.ArgumentParser()
    parser.add_argument('--start', type=int, default=0, help='Start page (0-indexed)')
    parser.add_argument('--pages', type=int, default=MAX_PAGES, help='Max pages to crawl')
    args = parser.parse_args()
    
    print(f"=== {SITE_NAME} ===")
    print(f"Pages: {args.start+1} to {args.start+args.pages}")
    
    total_inserted = 0
    
    for page in range(args.start, args.start + args.pages):
        print(f"\nPage {page+1}/{MAX_PAGES}...")
        items = crawl_page(page)
        
        if items is None:
            print(f"  Stopped at page {page+1}")
            break
        
        if not items:
            print(f"  Empty page {page+1}")
            continue
        
        print(f"  Found {len(items)} items")
        
        batch = []
        for item in items:
            detail = fetch_detail(item)
            if detail:
                batch.append(detail)
            time.sleep(0.2)
        
        if batch:
            inserted = save_to_db(batch)
            total_inserted += inserted
            print(f"  Inserted {inserted}/{len(batch)} new")
        
        time.sleep(0.5)
    
    print(f"\n{'='*40}")
    print(f"Total inserted: {total_inserted}")


if __name__ == '__main__':
    main()