#!/usr/bin/env python3
"""


西塞山区人民政府 - 通知公告爬虫
CMS: TRS (拓尔思)
分页: index_N.html (1~33页, 每页约15条)
列表: div.news_list1 > ul > li > span(日期) + a[title](标题)
详情: div.view.TRS_UEDITOR > p
"""

import sys
import os
import time
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin

import sys as _SYS
_MAX_PG = int(_SYS.argv[1]) if len(_SYS.argv) > 1 and _SYS.argv[1].isdigit() else None
if _MAX_PG is not None:
    print('[AutoPg] max_pages=' + str(_MAX_PG))
# END AUTO PAGES
DB_PATH = "/mnt/data/search.db"
BASE_URL = "http://www.xisaishan.gov.cn/xxgk/fdzdgknr/tzgg/"
SITE_NAME = "西塞山区-通知公告"
START_PAGE = 1
MAX_PAGES = 35

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
                  "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}

def extract_content(view_div):
    """提取正文内容，保留表格HTML，段落用\\n\\n分隔，去掉内联span/font/b标签防日期拆分"""
    if not view_div:
        return ""
    
    parts = []
    for child in view_div.children:
        tag_name = getattr(child, 'name', None)
        if tag_name == 'p':
            # 去掉内联span/font/b标签，用空分隔符让文本连续
            txt = child.get_text(separator='', strip=True)
            if txt:
                parts.append(txt)
        elif tag_name in ('table',):
            # 表格HTML原样保留
            parts.append(str(child))
        elif tag_name is None and isinstance(child, str):
            txt = child.strip()
            if txt:
                parts.append(txt)
        elif tag_name in ('div', 'section'):
            # 递归处理
            txt = child.get_text(separator='', strip=True)
            if txt:
                parts.append(txt)
    
    return '\n\n'.join(parts)


def crawl_page(page_num):
    """爬取指定页码的列表"""
    try:
        if page_num == 1:
            url = BASE_URL
            r = requests.get(url, headers=HEADERS, timeout=30)
            r.encoding = 'utf-8'
        else:
            url = f"{BASE_URL}index_{page_num}.html"
            r = requests.get(url, headers=HEADERS, timeout=30)
            r.encoding = 'utf-8'
        
        if r.status_code != 200:
            print(f"  HTTP {r.status_code} for {url}, stopping")
            return None
        
        soup = BeautifulSoup(r.text, 'html.parser')
        
        if page_num == 1:
            items = soup.select('div.news_list1 ul li')
        else:
            items = soup.select('div#op1 > ul > li')
    except Exception as e:
        print(f"  Error fetching {url}: {e}")
        return None
    
    if not items:
        print(f"  No items found on page {page_num}")
        return []
    
    results = []
    for li in items:
        a = li.find('a')
        span = li.find('span')
        if not a or not a.get('href'):
            continue
        
        link = a['href'].strip()
        title = a.get('title', '') or a.get_text(strip=True)
        date_str = span.get_text(strip=True) if span else ''
        
        full_url = urljoin(url, link)
        
        results.append({
            'title': title,
            'url': full_url,
            'date': date_str,
            'page_num': page_num,
        })
    
    return results


def crawl_detail(item):
    """爬取详情页"""
    url = item['url']
    try:
        r = requests.get(url, headers=HEADERS, timeout=30)
        r.encoding = 'utf-8'
        if r.status_code != 200:
            print(f"  HTTP {r.status_code} for detail {url}")
            return None
    except Exception as e:
        print(f"  Error fetching detail {url}: {e}")
        return None
    
    soup = BeautifulSoup(r.text, 'html.parser')
    
    # 正文
    view = soup.select_one('div.view.TRS_UEDITOR') or soup.select_one('div.View')
    if not view:
        print(f"  No content div found for {url}")
        return None
    
    content = extract_content(view)
    if not content or len(content) < 20:
        print(f"  Content too short for {url}")
        return None
    
    # 标题优先从h1取
    h1 = soup.find('h1')
    title_h1 = h1.get_text(strip=True) if h1 else ''
    if not title_h1:
        title_h1 = item['title']
    
    return {
        'title': title_h1,
        'content': content,
        'date': item['date'],
        'url': url,
        'site_name': SITE_NAME,
    }


def save_to_db(records):
    """批量写入SQLite"""
    import sqlite3
    
    if not records:
        print("  No records to save")
        return 0
    
    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()
    
    inserted = 0
    for rec in records:
        try:
            c.execute(
                "INSERT OR IGNORE INTO gov_raw (title, content, page_url, publish_date, site_name) "
                "VALUES (?, ?, ?, ?, ?)",
                (rec['title'], rec['content'], rec['url'], rec['date'], rec['site_name'])
            )
            if c.rowcount > 0:
                inserted += 1
        except Exception as e:
            print(f"  DB error: {e}")
    
    conn.commit()
    
    # FTS sync
    conn.commit()
    conn.close()
    
    return inserted


def main():
    import argparse
    if '_MAX_PG' in globals() and _MAX_PG is not None:
        import sys
        sys.argv = [sys.argv[0]] + ['--pages', str(_MAX_PG)] + sys.argv[2:]

    parser = argparse.ArgumentParser(description='Crawl xisaishan tzgg')
    parser.add_argument('--debug', action='store_true', help='Debug mode')
    parser.add_argument('--pages', type=int, default=MAX_PAGES, help='Max pages to crawl')
    parser.add_argument('--start', type=int, default=START_PAGE, help='Start page')
    args = parser.parse_args()
    
    total_found = 0
    total_inserted = 0
    
    for page in range(args.start, args.start + args.pages):
        print(f"\nPage {page}...")
        items = crawl_page(page)
        
        if items is None:
            print(f"  No more pages (stopped at page {page})")
            break
        
        if not items:
            print(f"  Empty page {page}, skipping")
            continue
        
        total_found += len(items)
        print(f"  Found {len(items)} items")
        
        batch = []
        for item in items:
            detail = crawl_detail(item)
            if detail:
                batch.append(detail)
            time.sleep(0.5)  # 礼貌等待
        
        if batch:
            inserted = save_to_db(batch)
            total_inserted += inserted
            print(f"  Inserted {inserted}/{len(batch)} new records")
        
        if page < args.start + args.pages - 1:
            time.sleep(1)  # 翻页间隔
    
    print(f"\n{'='*50}")
    print(f"Total found: {total_found}")
    print(f"Total inserted: {total_inserted}")
    print(f"{'='*50}")


if __name__ == '__main__':
    main()