#!/usr/bin/env python3
"""
南京市溧水区人民政府-公示公告 爬虫
URL: http://www.njls.gov.cn/lsqrmzf/214/228/index_17658.html
CMS: TRS / 溧水区政府网站群
AJAX API: /igs/front/search/publish/data/list.html
列表: HTML：pages 1-3 直接加载；pages 4+ AJAX
注意: 创宇盾WAF，需Playwright渲染获取数据
"""

import re, sys, time, os, json
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin, urlencode

SITE_NAME = "溧水区-公示公告"
GROUP = "江苏"
DB_PATH = "/root/search.db"

# Playwright-based AJAX fetcher
FETCHER_SCRIPT = "/root/fetch_njls_list.py"

def debug(msg):
    print(f"  [DEBUG] {msg}", file=sys.stderr)

def parse_date(text):
    m = re.search(r'(\d{4})[-/](\d{1,2})[-/](\d{1,2})', text)
    if m:
        return f"{m.group(1)}-{int(m.group(2)):02d}-{int(m.group(3)):02d}"
    return ""

def fetch_page_via_playwright(pages):
    """
    Call Playwright script to fetch list data for specified page numbers.
    Returns list of items dicts.
    """
    import subprocess
    pages_str = ','.join(str(p) for p in pages)
    cmd = [sys.executable, FETCHER_SCRIPT, '--pages', pages_str]
    result = subprocess.run(cmd, capture_output=True, text=True, timeout=120)
    if result.returncode != 0:
        debug(f"Playwright 脚本错误: {result.stderr[:200]}")
        return []
    
    try:
        data = json.loads(result.stdout.strip())
        return data
    except json.JSONDecodeError as e:
        debug(f"JSON 解析失败: {e}, stdout: {result.stdout[:200]}")
        return []

def parse_doccontent(html_content):
    """Parse the HTML content from DOCCONTENT field."""
    if not html_content or not html_content.strip():
        return "", []
    
    soup = BeautifulSoup(html_content, 'html.parser')
    parts = []
    attachments = []
    
    for child in soup.children:
        if not child.name:
            text = str(child).strip()
            if text:
                parts.append(text)
            continue
        
        tag = child.name.lower()
        
        if tag == 'p':
            txt = child.get_text(strip=True)
            if txt and not all(c in ' \n\r\t\u3000 \xa0' for c in txt):
                parts.append(txt)
        elif tag == 'table':
            parts.append(str(child))
        elif tag in ('div', 'span'):
            txt = child.get_text(strip=True)
            if txt:
                parts.append(txt)
        elif tag == 'br':
            pass
        elif tag == 'img':
            src = child.get('src', '')
            alt = child.get('alt', '')
            if src:
                parts.append(f"![{alt}]({urljoin('http://www.njls.gov.cn', src)})")
        else:
            txt = child.get_text(strip=True)
            if txt:
                parts.append(txt)
    
    # Extract attachments
    for a in soup.find_all('a', href=True):
        href = a['href']
        text = a.text.strip()
        if re.search(r'\.(pdf|doc|docx|xls|xlsx|zip|rar|ceb|ofd|ppt|pptx)$', href.lower()):
            full_url = urljoin('http://www.njls.gov.cn', href)
            attachments.append(f"[{text}]({full_url})")
    
    # Dedup
    seen = set()
    unique_parts = []
    for p in parts:
        key = p[:100]
        if key not in seen:
            seen.add(key)
            unique_parts.append(p)
    
    content = '\n\n'.join(unique_parts)
    return content, attachments

def process_item(row, page_num):
    """Convert a JSON row from AJAX to our item format."""
    title = row.get('DOCTITLE', '')
    url = row.get('DOCPUBURL', '')
    date_raw = row.get('DOCRELTIME', '')
    pub_date = parse_date(date_raw) if date_raw else ''
    content_html = row.get('DOCCONTENT', '') or ''
    
    content, attachments = parse_doccontent(content_html)
    
    return {
        'title': title,
        'url': url,
        'content': content,
        'pub_date': pub_date,
        'attachments': attachments,
        '_page': page_num,
    }

def init_db():
    import sqlite3
    conn = sqlite3.connect(DB_PATH, timeout=60)
    conn.execute("PRAGMA busy_timeout=60000")
    c = conn.cursor()
    c.execute('''CREATE TABLE IF NOT EXISTS gov_raw (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        page_url TEXT,
        title TEXT,
        content TEXT,
        publish_date TEXT,
        site_name TEXT,
        summary TEXT,
        attachments TEXT,
        date_rank TEXT,
        created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
    )''')
    c.execute('''CREATE INDEX IF NOT EXISTS idx_gov_raw_url 
        ON gov_raw(page_url, site_name)''')
    conn.commit()
    conn.close()

def save_to_db(items):
    import sqlite3
    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()
    imported = 0
    for item in items:
        summary = item['content'][:200] if item['content'] else ''
        try:
            c.execute('''INSERT OR REPLACE INTO gov_raw (page_url, title, content, publish_date, site_name, summary, attachments, date_rank, script_name) VALUES (?, ?, ?, ?, ?, ?, ?, ?, \'crawl_njls_tzgg.py\')''', (
                item['url'],
                item['title'],
                item['content'],
                item['pub_date'],
                SITE_NAME,
                summary,
                '\n'.join(item['attachments']) if item['attachments'] else '',
                item['pub_date'] or '0000-00-00',
            ))
            imported += 1
        except Exception as e:
            debug(f"  入库失败: {item['url']} - {e}")
    conn.commit()
    conn.close()
    return imported

def main():
    import argparse
    parser = argparse.ArgumentParser(description='溧水区-公示公告 爬虫')
    parser.add_argument('--pages', type=int, default=5, help='爬取页数, 0=全部')
    parser.add_argument('--skip-db', action='store_true', help='跳过入库')
    args = parser.parse_args()
    
    init_db()
    
    print(f"===== {SITE_NAME} 爬取 =====")
    
    total_pages = 15  # dataCount=283, 20/page => 15 pages
    pages_to_fetch = total_pages if args.pages == 0 else min(args.pages, total_pages)
    print(f"总页数: {total_pages}, 将爬取: {pages_to_fetch} 页")
    
    all_items = []
    
    for page in range(1, pages_to_fetch + 1):
        print(f"\n--- 第 {page}/{pages_to_fetch} 页 ---")
        
        items = fetch_page_via_playwright([page])
        if not items:
            print(f"  第 {page} 页无数据")
            continue
        
        print(f"  列表项: {len(items)} 条")
        
        for idx, item in enumerate(items):
            print(f"  [{len(all_items)+1}] {item['title'][:40]}...")
            
            if item['content']:
                seg_count = len(item['content'].split('\n\n'))
                print(f"    seg={seg_count} | attach={'YES' if item['attachments'] else 'no'} | date={item['pub_date']}")
            else:
                print(f"    ⚠️ 空正文 | attach={'YES' if item['attachments'] else 'no'}")
            
            all_items.append(item)
    
    total = len(all_items)
    with_body = sum(1 for i in all_items if i['content'])
    total_attach = sum(len(i['attachments']) for i in all_items)
    avg_seg = sum(len(i['content'].split('\n\n')) for i in all_items if i['content']) / max(with_body, 1)
    
    print(f"\n===== {SITE_NAME} 爬取完成 =====")
    print(f"共爬取: {total} 条")
    print(f"有正文: {with_body} 条 ({with_body/total*100:.1f}%)" if total else "有正文: 0 条")
    print(f"平均段落数: {avg_seg:.1f}")
    print(f"附件数: {total_attach}")
    
    if not args.skip_db:
        # Clean old data first
        import sqlite3
        conn = sqlite3.connect(DB_PATH, timeout=60)
        c = conn.cursor()
# QC20260925 去掉整站清空再重灌(抢锁+中途死掉会清空整站; page_url 有 UNIQUE 索引，插入本就幂等)         c.execute("DELETE FROM gov_search WHERE rowid IN (SELECT id FROM gov_raw WHERE site_name=?)", (SITE_NAME,))
# QC20260925 去掉整站清空再重灌(抢锁+中途死掉会清空整站; page_url 有 UNIQUE 索引，插入本就幂等)         c.execute("DELETE FROM gov_raw WHERE site_name=?", (SITE_NAME,))
        conn.commit()
        conn.close()
        
        imported = save_to_db(all_items)
        print(f"入库: {imported} 条")
    else:
        print("入库: 已跳过")

if __name__ == '__main__':
    main()