#!/usr/bin/env python3
"""crawl_xbxxb.py — 西北在线 广告专栏"""
import os, re, sys, requests
from bs4 import BeautifulSoup
from datetime import datetime, timedelta
import sqlite3

SEARCH_DB = os.getenv("SEARCH_DB", "/mnt/data/search.db")
SITE_NAME = "西北在线-广告专栏"
CATEGORY = "广告专栏"
BASE_URL = "http://www.xbxxb.com//ggzl/"
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
CUTOFF_DATE = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")

is_incremental = len(sys.argv) >= 2
if is_incremental:
    cutoff_days = int(sys.argv[1])
    cutoff = (datetime.now() - timedelta(days=cutoff_days)).strftime("%Y-%m-%d")
else:
    cutoff = CUTOFF_DATE

MAX_PAGES = 50  # site has ~49 pages

def fetch(url):
    try:
        r = requests.get(url, headers=HEADERS, timeout=15)
        r.encoding = 'utf-8'
        return r.text
    except Exception as e:
        print("  [ERROR] fetch %s: %s" % (url, e))
        return ""

def safe_summary(text, max_len=500):
    if len(text) <= max_len:
        return text
    truncated = text[:max_len]
    if '<' in truncated:
        last_open = truncated.rfind('<')
        last_close = truncated.rfind('>')
        if last_open > last_close:
            truncated = truncated[:last_open]
    return truncated

def extract_content(html):
    """Extract content from detail page div.content."""
    m = re.search(r'<div class="content">(.*?)</div>\s*</div>', html, re.DOTALL)
    if not m:
        return ""
    txt = m.group(1)
    # Remove script/style
    txt = re.sub(r'<script[^>]*>.*?</script>', '', txt, flags=re.DOTALL)
    txt = re.sub(r'<style[^>]*>.*?</style>', '', txt, flags=re.DOTALL)
    txt = re.sub(r'<br\s*/?>', '\n', txt)
    txt = re.sub(r'</p>', '\n\n', txt)
    # Preserve table HTML
    table_placeholders = []
    def save_table(m):
        ph = '__TABLE_%d__' % len(table_placeholders)
        table_placeholders.append((ph, m.group(0)))
        return ph
    txt = re.sub(r'<table[^>]*>.*?</table>', save_table, txt, flags=re.DOTALL)
    txt = re.sub(r'<[^>]+>', '', txt)
    txt = re.sub(r'\n{3,}', '\n\n', txt)
    for ph, ht in table_placeholders:
        txt = txt.replace(ph, ht)
    return txt.strip()

def get_list_items(html):
    """Extract (url, title, date) from list page."""
    items = []
    soup = BeautifulSoup(html, 'lxml')
    list_div = soup.find('div', class_='list')
    if not list_div:
        return items
    for a in list_div.find_all('a', class_='list-item'):
        href = a.get('href', '')
        if not href.startswith('http'):
            href = 'http://www.xbxxb.com' + href
        title_tag = a.find('p', class_='list-item-tit')
        title = title_tag.get_text(strip=True) if title_tag else ''
        time_tag = a.find('p', class_='list-time')
        date = time_tag.get_text(strip=True) if time_tag else ''
        if title and date:
            items.append((href, title, date))
    return items

def main():
    conn = sqlite3.connect(SEARCH_DB, timeout=60)
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute("PRAGMA busy_timeout=10000")
    c = conn.cursor()
    total_new = 0
    total_skip = 0

    mode = "增量" if is_incremental else "全量"
    print("[%s] %s (cutoff=%s)" % (SITE_NAME, mode, cutoff))

    for page in range(1, MAX_PAGES + 1):
        url = BASE_URL + "%d.html" % page
        html = fetch(url)
        if not html:
            break

        items = get_list_items(html)
        if not items:
            print("  第%d页: 无数据（可能是最后一页）" % page)
            break

        page_new = 0
        for href, title, date_str in items:
            if date_str < cutoff:
                continue

            c.execute("SELECT id FROM gov_raw WHERE page_url = ?", (href,))
            if c.fetchone():
                total_skip += 1
                continue

            detail_html = fetch(href)
            if not detail_html:
                total_skip += 1
                continue

            content = extract_content(detail_html)
            if not content or len(content) < 20:
                total_skip += 1
                continue

            summary = safe_summary(content)

            c.execute("""INSERT INTO gov_raw (site_name, category, title, page_url, content, summary, publish_date)
                         VALUES (?, ?, ?, ?, ?, ?, ?)""",
                      (SITE_NAME, CATEGORY, title, href, content, summary, date_str))
            page_new += 1
            total_new += 1

        conn.commit()
        print("  第%d页: +%d (累计%d)" % (page, page_new, total_new))

    conn.close()
    print("\n[%s] 完成: 新增 %d, 跳过 %d" % (SITE_NAME, total_new, total_skip))

if __name__ == "__main__":
    main()