#!/usr/bin/env python3
"""漳州古雷港经济开发区 - 公示公告爬虫
站点: www.zhangzhou.gov.cn
栏目: zzglkfq/gsgg (公示公告)
CMS: YL CMS - 仅第一页静态HTML可访问，分页被CloudWAF封锁
策略: 每次只抓第一页(最新20条)，日跑增量检查新文章
"""

import requests
import sqlite3
import re
import sys
import os
from datetime import datetime, timedelta
from bs4 import BeautifulSoup

BASE_URL = "https://www.zhangzhou.gov.cn"
LIST_URL = BASE_URL + "/cms/html/zzglkfq/gsgg/index.html"
DB_PATH = "/root/search.db"
SITE_NAME = "zhangzhou.gov.cn"

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}

def init_db():
    conn = sqlite3.connect(DB_PATH, timeout=60)
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute("""
        CREATE TABLE IF NOT EXISTS gov_raw (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            site_name TEXT NOT NULL,
            page_url TEXT UNIQUE NOT NULL,
            title TEXT NOT NULL,
            pub_date TEXT,
            summary TEXT,
            content TEXT,
            crawl_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP
        )
    """)
    conn.execute("CREATE INDEX IF NOT EXISTS idx_gov_raw_site ON gov_raw(site_name)")
    conn.commit()
    return conn

def fetch_page(url, timeout=30):
    try:
        resp = requests.get(url, headers=HEADERS, timeout=timeout, verify=False)
        resp.encoding = 'utf-8'
        if resp.status_code == 200:
            return resp.text
        print(f"  HTTP {resp.status_code} for {url}")
        return None
    except Exception as e:
        print(f"  Error fetching {url}: {e}")
        return None

def parse_list(html):
    """解析列表页，提取文章URL、标题、日期"""
    items = []
    soup = BeautifulSoup(html, 'html.parser')
    ul = soup.find('ul', id='resources')
    if not ul:
        print("  WARNING: resources UL not found")
        return items
    
    for li in ul.find_all('li', recursive=False):
        link = li.find('a')
        date_span = li.find('span', class_='list-time')
        if not link or not link.get('href'):
            continue
        href = link.get('href', '').strip()
        if not href.startswith('http'):
            href = BASE_URL + href
        title = link.get_text(strip=True)
        pub_date = date_span.get_text(strip=True) if date_span else ''
        items.append({
            'url': href,
            'title': title,
            'pub_date': pub_date,
        })
    return items

def parse_detail(html, url):
    """解析详情页：标题、正文、附件"""
    soup = BeautifulSoup(html, 'html.parser')
    
    # 标题
    title_el = soup.find('div', class_='content-title')
    title = title_el.get_text(strip=True) if title_el else '无标题'
    
    # 正文 - id="Content" div
    content_div = soup.find('div', id='Content')
    content_html = ''
    if content_div:
        content_html = str(content_div)
    
    # 清理内容
    if content_html:
        # 移除脚本和样式
        for tag in content_div.find_all(['script', 'style']):
            tag.decompose()
        # 用正则剥离内联样式标签（b/font/span/strong等），
        # 防止"<b>2</b><b>0</b><b>2</b><b>6</b>"在BS4中保留
        # 文本节点边界导致日期变成"202\n6\n年\n6\n月\n9\n日"
        raw_html = str(content_div)
        raw_html = re.sub(r'</?(?:b|font|span|strong|em|u|i|a)[^>]*>', '', raw_html, flags=re.IGNORECASE)
        # 再解析
        content_div_clean = BeautifulSoup(raw_html, 'html.parser')
        content = content_div_clean.get_text(separator='\n\n', strip=True)
        # 合并过多空行
        content = re.sub(r'\n{4,}', '\n\n', content)
        # 清理每行首尾空白（含全角空格\u3000、\u2002等）
        content = '\n'.join(line.strip() for line in content.split('\n'))
        # 再次合并多余空行
        content = re.sub(r'\n{3,}', '\n\n', content)
    else:
        content = ''
    
    # 提取附件
    attachments = []
    if content_div:
        for a_tag in content_div.find_all('a', href=True):
            href = a_tag['href']
            if re.search(r'\.(pdf|doc|docx|xls|xlsx|rar|zip|jpg|png|gif)$', href, re.I):
                if not href.startswith('http'):
                    href = BASE_URL + href
                fname = a_tag.get_text(strip=True) or os.path.basename(href)
                full_url = href
                attachments.append({'name': fname, 'url': full_url})
    
    # 额外检查页面中的附件
    for a_tag in soup.find_all('a', href=True):
        href = a_tag['href']
        if '/attachments/' in href and re.search(r'\.(pdf|doc|docx|xls|xlsx)$', href, re.I):
            if not href.startswith('http'):
                href = BASE_URL + href
            if not any(a['url'] == href for a in attachments):
                fname = a_tag.get_text(strip=True) or os.path.basename(href)
                attachments.append({'name': fname, 'url': href})
    
    # 获取发布时间
    pub_date = ''
    date_spans = soup.find_all('span')
    for span in date_spans:
        text = span.get_text(strip=True)
        m = re.search(r'(\d{4}-\d{2}-\d{2})', text)
        if m:
            pub_date = m.group(1)
            break
    
    return {
        'title': title,
        'content': content,
        'content_html': content_html,
        'pub_date': pub_date,
        'attachments': attachments,
    }

def deduplicate(conn, items):
    """去重：排除已存在的URL"""
    cur = conn.execute("SELECT page_url FROM gov_raw WHERE site_name = ?", (SITE_NAME,))
    existing = set(row[0] for row in cur.fetchall())
    new = [it for it in items if it['url'] not in existing]
    skipped = len(items) - len(new)
    if skipped:
        print(f"  Skipped {skipped} existing items")
    return new

def main(mode='full'):
    """mode: 'full'=全量, 'incremental'=仅今天"""
    conn = init_db()
    
    print(f"[{SITE_NAME}] Fetching list page...")
    html = fetch_page(LIST_URL)
    if not html:
        print("ERROR: Could not fetch list page")
        conn.close()
        return
    
    items = parse_list(html)
    print(f"  Found {len(items)} items on first page")
    
    if not items:
        conn.close()
        return
    
    # 增量模式：只处理今天/昨天的文章
    today = datetime.now().strftime('%Y-%m-%d')
    yesterday = (datetime.now() - timedelta(days=1)).strftime('%Y-%m-%d')
    
    if mode == 'incremental':
        items = [it for it in items if it.get('pub_date') in (today, yesterday)]
        print(f"  After date filter: {len(items)} items")
        if not items:
            print(f"  No new items for {today}/{yesterday}")
            conn.close()
            return
    
    # 去重
    new_items = deduplicate(conn, items)
    print(f"  New items to crawl: {len(new_items)}")
    
    if not new_items:
        conn.close()
        return
    
    added = 0
    for item in new_items:
        print(f"  Fetching: {item['title'][:50]}...")
        detail_html = fetch_page(item['url'])
        if not detail_html:
            continue
        
        detail = parse_detail(detail_html, item['url'])
        
        # 构建正文内容（含附件链接）
        content = detail['content']
        if detail['attachments']:
            content += '\n\n---\n附件:\n'
            for att in detail['attachments']:
                content += f"\n[{att['name']}]({att['url']})"
        
        # 入库
        try:
            conn.execute("""
                INSERT OR IGNORE INTO gov_raw (site_name, source_url, page_url, title, publish_date, summary, content)
                VALUES (?, ?, ?, ?, ?, ?, ?)
            """, (
                SITE_NAME,
                item['url'],
                item['url'],
                detail['title'] or item['title'],
                detail['pub_date'] or item.get('pub_date', ''),
                item['title'][:200],
                content
            ))
            conn.commit()
            added += 1
            if added % 5 == 0:
                print(f"    Progress: {added}/{len(new_items)}")
        except Exception as e:
            print(f"    DB error: {e}")
    
    print(f"\n  Done! Added {added} new articles")
    conn.close()

if __name__ == "__main__":
    mode = sys.argv[1] if len(sys.argv) > 1 else 'full'
    main(mode)