#!/usr/bin/env python3
"""万州经济技术开发区-公告公示"""
import re, sys, os, json, time, requests
DB_PATH = os.environ.get("DB_PATH", "/root/search.db")
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9",
}
SITE_NAME = "万州经济技术开发区-公告公示"
GROUP = "万州经开区"
PER_PAGE = 15
MAX_PAGES = 5
BASE_URL = "http://www.wz.gov.cn"
LIST_URL = "http://www.wz.gov.cn/jkq/wzjkq/zwgk_106889/gsgg/"

def fetch(url, encoding="utf-8"):
    r = requests.get(url, headers=HEADERS, timeout=30, allow_redirects=True)
    r.encoding = encoding
    return r.text

def resolve_url(rel_url):
    if rel_url.startswith("http"):
        return rel_url
    if rel_url.startswith("./"):
        return BASE_URL + "/jkq/wzjkq/zwgk_106889/gsgg/" + rel_url[2:]
    if rel_url.startswith("/"):
        return BASE_URL + rel_url
    return BASE_URL + "/" + rel_url

def extract_list(html):
    """Extract (title, url, date) from list page"""
    items = []
    # ul.dt-list > li > a > span (title) + span (date)
    pattern = r'<li>\s*<a\s+href="([^"]+)"[^>]*>\s*<span>([^<]+)</span>\s*<span>(\d{4}-\d{1,2}-\d{1,2})</span>\s*</a>\s*</li>'
    for m in re.finditer(pattern, html):
        rel_url = m.group(1).strip()
        title = m.group(2).strip()
        date = m.group(3).strip()
        items.append((title, rel_url, date))
    return items

def extract_detail(html):
    """Extract body content from detail page"""
    from bs4 import BeautifulSoup
    soup = BeautifulSoup(html, 'html.parser')

    # Title from <title>
    title_tag = soup.find('title')
    title = title_tag.get_text(strip=True) if title_tag else ""
    title = re.sub(r'_重庆市万州区人民政府$', '', title).strip()

    # Date from table-box: 发布日期 or 成文日期
    date = ""
    tb = soup.find('div', class_='table-box')
    if tb:
        for item in tb.find_all('div', class_='table-item'):
            spans = item.find_all('span')
            if len(spans) >= 2:
                label = spans[0].get_text(strip=True)
                val = spans[1].get_text(strip=True)
                if '发布' in label or '成文' in label:
                    if re.match(r'\d{4}-\d{1,2}-\d{1,2}', val):
                        date = val

    # Body from div.content > div.trs_editor_view
    body = ""
    content_div = soup.find('div', class_='content')
    if content_div:
        te = content_div.find('div', class_='trs_editor_view')
        if te:
            paras = [p.get_text(strip=True) for p in te.find_all('p') if p.get_text(strip=True)]
            body = '\n\n'.join(paras)

    return title, body, date

def save_to_db(items, site_name, group):
    import sqlite3
    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()
    inserted = 0
    for title, url, date, body in items:
        try:
            c.execute("""INSERT OR REPLACE INTO gov_raw (page_url, site_name, group_name, title, publish_date, summary, content, script_name) VALUES (?, ?, ?, ?, ?, ?, ?, 'crawl_wzjkq.py')""",
                (url, site_name, group, title, date, body, body))
            inserted += 1
        except Exception as e:
            print("DB error: {} - {}".format(url, e))
    conn.commit()
    conn.close()
    return inserted

def main():
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('--max-pages', type=int, default=MAX_PAGES)
    args = parser.parse_args()
    max_pages = args.max_pages

    all_items = []

    # Page 1
    html = fetch(LIST_URL)
    items = extract_list(html)
    all_items.extend(items)
    print("Page 1: {} items".format(len(items)))

    # Pages 2+
    for page in range(2, max_pages + 1):
        url = "{}/index_{}.html".format(LIST_URL.rstrip('/'), page - 1)
        try:
            html = fetch(url)
            items = extract_list(html)
            if not items:
                print("Page {}: empty, stopping".format(page))
                break
            all_items.extend(items)
            print("Page {}: {} items".format(page, len(items)))
        except Exception as e:
            print("Page {} error: {}".format(page, e))
            break
        time.sleep(0.5)

    print("\nTotal list items: {}".format(len(all_items)))

    # Fetch details
    detail_items = []
    failed = 0
    for i, (title, rel_url, date) in enumerate(all_items):
        url = resolve_url(rel_url)
        try:
            html = fetch(url)
            d_title, body, d_date = extract_detail(html)
            if not d_date:
                d_date = date
            if not d_title:
                d_title = title
            detail_items.append((d_title, url, d_date, body))
            if (i + 1) % 10 == 0:
                print("  progress: {}/{}".format(i + 1, len(all_items)))
        except Exception as e:
            print("  FAILED: {} - {}".format(url, e))
            failed += 1
        time.sleep(0.3)

    print("\nDetails fetched: {}, failed: {}".format(len(detail_items), failed))

    # Save to DB
    inserted = save_to_db(detail_items, SITE_NAME, GROUP)
    print("Inserted: {}".format(inserted))

    # Summary
    timeout_count = 0
    error_count = failed
    abnormal_count = 0
    print("\n=== Summary ===")
    print("{} | {} | {} | {} | {}".format(SITE_NAME, inserted, timeout_count, error_count, abnormal_count))

if __name__ == "__main__":
    main()