#!/usr/bin/env python3
import os
# -*- coding: utf-8 -*-
"""
泉州市人民政府 - 通知公告 爬虫
http://www.quanzhou.gov.cn/zfb/xxgk/zfxxgkzl/gggs/
CMS: TerTon (SSR + SmartSearch API)
数据内嵌在首页JS中，API需登录认证，仅爬首页15条
"""
import requests, re, time, json, sys, os
from bs4 import BeautifulSoup
from datetime import datetime

SITE_NAME = "泉州市人民政府-通知公告"
BASE_URL = "http://www.quanzhou.gov.cn/zfb/xxgk/zfxxgkzl/gggs/"
DETAIL_BASE = "http://www.quanzhou.gov.cn/zfb/xxgk/zfxxgkzl/gggs/"
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}

def fetch(url, retries=3):
    for i in range(retries):
        try:
            r = requests.get(url, headers=HEADERS, timeout=20)
            r.encoding = 'utf-8'
            if r.status_code == 200:
                return r.text
        except Exception as e:
            if i < retries-1:
                time.sleep(2)
    return None

def parse_list():
    """从首页提取内嵌的15条数据"""
    html = fetch(BASE_URL)
    if not html:
        print("ERROR: 无法获取首页")
        return []
    
    # 提取 terton.dynamicAndStaticData 中的 list 数组
    start = html.find('terton.dynamicAndStaticData')
    if start < 0:
        print("ERROR: 未找到数据块")
        return []
    
    block = html[start:start+5000]
    dates = re.findall(r'"docreltime":\s*\'([^\']+)\'', block)
    titles = re.findall(r'"doctitle":\s*\'([^\']+)\'', block)
    urls = re.findall(r'"docpuburl":\s*\'([^\']+)\'', block)
    
    items = []
    for i in range(min(len(dates), len(titles), len(urls))):
        items.append({
            'date': dates[i],
            'title': titles[i].strip(),
            'url': urls[i].lstrip('./')
        })
    
    print(f"首页提取: {len(items)} 条")
    return items

def parse_detail(item):
    """解析详情页"""
    url = DETAIL_BASE + item['url']
    html = fetch(url)
    if not html:
        print(f"  ERROR: 无法获取 {url}")
        return {
            'title': item['title'],
            'date': item['date'],
            'content': '',
            'attachments': []
        }
    
    soup = BeautifulSoup(html, 'html.parser')
    
    # 标题
    title = item['title']
    meta_title = soup.find('meta', attrs={'name': 'ArticleTitle'})
    if meta_title and meta_title.get('content'):
        title = meta_title['content'].strip()
    
    # 日期
    date = item['date']
    meta_date = soup.find('meta', attrs={'name': 'PubDate'})
    if meta_date and meta_date.get('content'):
        date = meta_date['content'].strip()
    
    # 正文：TRS_Editor
    content = ''
    attachments = []
    
    editor = soup.find('div', class_='TRS_Editor')
    if editor:
        blocks = []
        for child in editor.children:
            if child.name == 'p':
                # 检查附件链接
                text_parts = []
                for c in child.children:
                    if isinstance(c, str):
                        t = c.strip()
                        if t:
                            text_parts.append(t)
                    elif c.name == 'a':
                        href = c.get('href', '')
                        txt = c.get_text(strip=True)
                        if href and txt:
                            full = href if href.startswith('http') else (DETAIL_BASE + href.lstrip('./'))
                            text_parts.append(f'[{txt}]({full})')
                            # 附件检测
                            if re.search(r'\.(doc|docx|pdf|xls|xlsx|rar|zip)$', href, re.I):
                                attachments.append({'name': txt, 'url': full})
                        elif txt:
                            text_parts.append(txt)
                    elif c.name in ['span', 'strong', 'em', 'b']:
                        t = c.get_text(strip=True)
                        if t:
                            text_parts.append(t)
                    elif c.name == 'br':
                        text_parts.append('\n')
                    else:
                        t = c.get_text(strip=True)
                        if t:
                            text_parts.append(t)
                text = ''.join(text_parts).strip()
                if text:
                    blocks.append(text)
            elif child.name == 'div':
                # 递归提取div内的p
                for p in child.find_all('p', recursive=False):
                    t = p.get_text(strip=True)
                    if t:
                        blocks.append(t)
            elif child.name == 'table':
                # 渲染为HTML table
                rows = []
                for tr in child.find_all('tr'):
                    cells = []
                    for td in tr.find_all(['td', 'th']):
                        tag = 'th' if td.name == 'th' else 'td'
                        ct = td.get_text(strip=True)
                        cells.append(f'<{tag}>{ct}</{tag}>')
                    if any(c for c in cells):
                        rows.append('<tr>' + ''.join(cells) + '</tr>')
                if rows:
                    blocks.append('<table>' + ''.join(rows) + '</table>')
            elif child.name == 'img':
                src = child.get('src', '')
                if src:
                    alt = child.get('alt', '')
                    full_src = src if src.startswith('http') else (DETAIL_BASE + src.lstrip('./'))
                    blocks.append(f'![{alt}]({full_src})')
        
        content = '\n\n'.join(blocks)
    else:
        # 备用
        for p in soup.find_all('p'):
            t = p.get_text(strip=True)
            if t:
                content += t + '\n\n'
        content = content.strip()
    
    # 全文搜索附件
    if not attachments:
        for a in soup.find_all('a', href=re.compile(r'\.(doc|docx|pdf|xls|xlsx|rar|zip)$', re.I)):
            href = a.get('href', '')
            txt = a.get_text(strip=True) or '附件'
            full = href if href.startswith('http') else (DETAIL_BASE + href.lstrip('./'))
            if not any(a2['url'] == full for a2 in attachments):
                attachments.append({'name': txt, 'url': full})
    
    # summary
    clean = re.sub(r'\s+', ' ', content).strip()
    summary = clean[:200]
    
    # date_rank
    date_parsed = date[:10] if date else ''
    date_rank = 0
    if date_parsed:
        try:
            dt = datetime.strptime(date_parsed, '%Y-%m-%d')
            date_rank = int(dt.timestamp())
        except:
            pass
    
    print(f"  OK: {title[:30]}... | {date_parsed} | content={len(content)}c")
    return {
        'title': title,
        'date': date_parsed,
        'content': content,
        'summary': summary,
        'attachments': json.dumps(attachments, ensure_ascii=False),
        'date_rank': date_rank,
        'source_url': url
    }

def main():
    print(f"=== {SITE_NAME} 爬虫 ===")
    
    items = parse_list()
    if not items:
        print("无数据，退出")
        return
    
    print(f"共 {len(items)} 条待爬")
    
    # DB
    if os.path.exists('/root/search.db'):
        db_path = '/root/search.db'
    else:
        db_path = '/mnt/data/search.db'
    
    import sqlite3
    conn = sqlite3.connect(db_path, timeout=60)
    conn.execute("PRAGMA busy_timeout=60000")
    c = conn.cursor()
    
    # 清理旧数据
# QC20260925 去掉整站清空再重灌(抢锁+中途死掉会清空整站; page_url 有 UNIQUE 索引，插入本就幂等)     c.execute("DELETE FROM gov_search WHERE rowid IN (SELECT id FROM gov_raw WHERE site_name=?)", (SITE_NAME,))
# QC20260925 去掉整站清空再重灌(抢锁+中途死掉会清空整站; page_url 有 UNIQUE 索引，插入本就幂等)     c.execute("DELETE FROM gov_raw WHERE site_name=?", (SITE_NAME,))
    conn.commit()
    print(f"已清理 {SITE_NAME} 旧数据")
    
    count = 0
    for item in items:
        detail = parse_detail(item)
        
        c.execute("""
            INSERT OR REPLACE INTO gov_raw (page_url, title, publish_date, content, summary, site_name, attachments, date_rank, source_url, script_name) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'crawl_quanzhou_gggs.py')
        """, (
            detail.get('source_url', ''),
            detail['title'],
            detail.get('date', ''),
            detail.get('content', ''),
            detail.get('summary', ''),
            SITE_NAME,
            detail.get('attachments', '[]'),
            detail.get('date_rank', 0),
            detail.get('source_url', '')
        ))
        count += 1
        time.sleep(1.5)
    
    conn.commit()
    print(f"入库 {count} 条")
    
    # 验证
    c.execute("SELECT COUNT(*) FROM gov_raw WHERE site_name=?", (SITE_NAME,))
    raw_count = c.fetchone()[0]
    c.execute("SELECT COUNT(*) FROM gov_search WHERE site_name=?", (SITE_NAME,))
    search_count = c.fetchone()[0]
    
    conn.close()
    print(f"验证: gov_raw={raw_count}, gov_search={search_count}")
    
    print(f"\n配置条目:")
    print(json.dumps({
        "name": SITE_NAME,
        "script": "crawl_quanzhou_gggs.py",
        "url": BASE_URL,
        "group": "福建"
    }, ensure_ascii=False, indent=2))

if __name__ == '__main__':
    main()