#!/usr/bin/env python3
"""乌审旗人民政府-通知公告"""
import re, sys, os, json, time, requests
from bs4 import BeautifulSoup

DB_PATH = os.environ.get("DB_PATH", "/root/search.db")
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9",
}
SITE_NAME = "乌审旗人民政府-通知公告"
GROUP = "内蒙古鄂尔多斯"
MAX_PAGES = 5  # ~75 items
BASE_URL = "https://www.wsq.gov.cn/zw/tzggx"
LIST_URL = "https://www.wsq.gov.cn/zw/tzggx/index.html"
TOTAL_PAGES = 14  # pageCount from JS: 14

def fetch(url, encoding="utf-8"):
    r = requests.get(url, headers=HEADERS, timeout=30, allow_redirects=True)
    r.encoding = encoding
    return r.text

def parse_date_from_url(href):
    """Extract date from URL like ./202607/t20260709_3908306.html -> 2026-07-09"""
    m = re.search(r'/t(\d{4})(\d{2})(\d{2})_', href)
    if m:
        return "{}-{}-{}".format(m.group(1), m.group(2), m.group(3))
    return ""

def extract_list(html):
    """Extract (title, url, date) from list page"""
    items = []
    soup = BeautifulSoup(html, 'html.parser')
    uls = soup.find_all('ul')
    if len(uls) < 4:
        return items
    ul = uls[3]  # 4th ul is the notification list
    for li in ul.find_all('li'):
        a = li.find('a')
        if not a:
            continue
        href = a.get('href', '').strip()
        if not href:
            continue
        spans = a.find_all('span')
        if len(spans) >= 2:
            title = spans[0].get_text(strip=True)
            date = spans[1].get_text(strip=True)
        else:
            # fallback: parse from text
            txt = a.get_text(strip=True)
            dm = re.search(r'(\d{4}-\d{1,2}-\d{1,2})', txt)
            date = dm.group(1) if dm else parse_date_from_url(href)
            title = re.sub(r'\s*\d{4}-\d{1,2}-\d{1,2}\s*$', '', txt).strip()
        items.append((title, href, date))
    return items

def resolve_url(rel_url):
    if rel_url.startswith("http"):
        return rel_url
    rel = rel_url.lstrip("./")
    return "https://www.wsq.gov.cn/zw/tzggx/" + rel

def extract_detail(html):
    """Extract body content from detail page"""
    soup = BeautifulSoup(html, 'html.parser')

    # Title
    title = ""
    t = soup.find('title')
    if t:
        title = t.get_text(strip=True)
        title = re.sub(r'_乌审旗人民政府.*$', '', title).strip()

    # Date from page
    date = ""
    dm = re.search(r'(\d{4}-\d{1,2}-\d{1,2})', html[:2000])
    if dm:
        date = dm.group(1)

    # Body - details_inner.font_medium or TRS_Editor
    body = ""
    for cls_name in ['details_inner', 'TRS_Editor', 'art-content', 'article-content', 'content']:
        el = soup.find(class_=cls_name)
        if not el:
            continue
        # Collect paragraphs and tables
        parts = []
        for child in el.find_all(['p', 'table']):
            if child.name == 'table':
                parts.append(str(child))
            elif child.name in ('p', 'div'):
                txt = child.get_text(strip=True)
                if txt:
                    parts.append(txt)
        if parts:
            body = '\n\n'.join(parts)
            break

    if not body:
        # Fallback: any div with substantial text
        for div in soup.find_all('div'):
            txt = div.get_text(strip=True)
            if len(txt) > 200:
                paras = [p.get_text(strip=True) for p in div.find_all('p') if p.get_text(strip=True)]
                if paras:
                    body = '\n\n'.join(paras)
                    break

    return title, body, date

def save_to_db(items, site_name, group):
    import sqlite3
    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()
    inserted = 0
    for title, url, date, body in items:
        try:
            c.execute("""INSERT OR REPLACE INTO gov_raw (page_url, site_name, group_name, title, publish_date, summary, content, script_name) VALUES (?, ?, ?, ?, ?, ?, ?, 'crawl_wsq.py')""",
                (url, site_name, group, title, date, body, body))
            inserted += 1
        except Exception as e:
            print("DB error: {} - {}".format(url, e))
    conn.commit()
    conn.close()
    return inserted

def main():
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('--max-pages', type=int, default=MAX_PAGES)
    args = parser.parse_args()

    max_pages = min(args.max_pages, TOTAL_PAGES)

    all_items = []
    for page in range(max_pages):
        if page == 0:
            url = LIST_URL
        else:
            url = "https://www.wsq.gov.cn/zw/tzggx/index_{}.html".format(page)
        try:
            html = fetch(url)
            items = extract_list(html)
            all_items.extend(items)
            print("Page {}: {} items".format(page + 1, len(items)))
        except Exception as e:
            print("Page {} FAILED: {}".format(page + 1, e))
        time.sleep(0.5)

    print("\nTotal list items: {}".format(len(all_items)))

    # Fetch details
    detail_items = []
    failed = 0
    for i, (title, rel_url, date) in enumerate(all_items):
        url = resolve_url(rel_url)
        try:
            html = fetch(url)
            d_title, body, d_date = extract_detail(html)
            if not d_date:
                d_date = date
            if not d_title:
                d_title = title
            detail_items.append((d_title, url, d_date, body))
            if (i + 1) % 10 == 0:
                print("  progress: {}/{}".format(i + 1, len(all_items)))
        except Exception as e:
            print("  FAILED: {} - {}".format(url, e))
            failed += 1
        time.sleep(0.3)

    print("\nDetails fetched: {}, failed: {}".format(len(detail_items), failed))

    # Save to DB
    inserted = save_to_db(detail_items, SITE_NAME, GROUP)
    print("Inserted: {}".format(inserted))

    # Summary
    print("\n=== Summary ===")
    print("{} | {} | {} | {} | {}".format(SITE_NAME, inserted, 0, failed, 0))

if __name__ == "__main__":
    main()