#!/usr/bin/env python3
"""
德州新闻网-商业资讯 (www.dezhoudaily.com/syzx) 爬虫
静态HTML站点，单页17条环评公示类信息
"""
import re
import requests
import sqlite3
import os
import sys

SITE_NAME = '德州新闻网'
SITE_DOMAIN = 'www.dezhoudaily.com'
LIST_URL = 'https://www.dezhoudaily.com/syzx/index.html?spm=zm1353-001.0.0.4.1DcX0h'

SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")

def fetch_list():
    """获取列表页所有条目"""
    resp = requests.get(LIST_URL, timeout=30)
    resp.encoding = 'utf-8'
    html = resp.text
    
    items = []
    # 提取标题和链接
    pattern = r'<h3><a href="([^"]+)"[^>]*>([^<]+)</a></h3>'
    matches = re.findall(pattern, html)
    
    # 提取日期
    date_pattern = r'<time[^>]*>([^<]+)</time>'
    dates = re.findall(date_pattern, html)
    
    seen = set()
    for i, (url, title) in enumerate(matches):
        title = title.strip()
        if not title or title in seen:
            continue
        seen.add(title)
        
        pub_date = ''
        if i < len(dates):
            pub_date = dates[i].strip()[:10]  # YYYY-MM-DD
        
        items.append({
            'title': title,
            'url': url,
            'publish_date': pub_date,
        })
    
    return items

def fetch_detail(url):
    """获取详情页内容和完整标题"""
    try:
        resp = requests.get(url, timeout=30)
        resp.encoding = 'utf-8'
        html = resp.text
        
        # 从detail page取完整标题
        full_title = ''
        m_title = re.search(r'<h1[^>]*>(.*?)</h1>', html)
        if m_title:
            full_title = m_title.group(1).strip()
        
        # 提取<article>内的正文
        m = re.search(r'<article[^>]*>(.*?)</article>', html, re.DOTALL)
        if not m:
            return '', full_title
        
        content = m.group(1)
        # 去掉标题和meta信息
        content = re.sub(r'<h1[^>]*>.*?</h1>', '', content, flags=re.DOTALL)
        content = re.sub(r'<div class="article-infos">.*?</div>', '', content, flags=re.DOTALL)
        
        # 保留主要正文 (article-content)
        m2 = re.search(r'<div class="article-content"[^>]*>(.*?)</div>\s*</article>', content, re.DOTALL)
        if m2:
            content = m2.group(1)
        
        content = content.strip()
        return content, full_title
    except Exception as e:
        print(f'  详情页获取失败: {url[:60]}... {e}')
        return '', ''

def init_db():
    db = sqlite3.connect(SEARCH_DB)
    db.execute('''CREATE TABLE IF NOT EXISTS gov_raw (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        site_name TEXT,
        source_url TEXT,
        page_url TEXT,
        title TEXT,
        publish_date TEXT,
        date_rank INTEGER DEFAULT 0,
        summary TEXT,
        status TEXT,
        category TEXT DEFAULT '',
        visits INTEGER DEFAULT 0,
        content TEXT DEFAULT '',
        tags TEXT DEFAULT ''
    )''')
    try:
        db.execute('''CREATE VIRTUAL TABLE IF NOT EXISTS gov_search USING fts5(
            title, content, site_name,
            content='gov_raw', content_rowid='id',
            tokenize='unicode61'
        )''')
    except sqlite3.OperationalError:
        pass
    db.commit()
    db.close()

def save_items(items):
    db = sqlite3.connect(SEARCH_DB)
    cur = db.cursor()
    new_count = 0
    total = len(items)
    
    for idx, item in enumerate(items):
        print(f'  [{idx+1}/{total}] {item["title"][:30]}... ', end='', flush=True)
        
        # 获取详情，同时获取完整标题
        content, full_title = fetch_detail(item['url'])
        if not content:
            print('无内容')
            continue
        
        # 使用完整标题替换截断标题
        title = full_title or item['title']
        
        print(f'({len(content)}字)')
        
        try:
            cur.execute('''INSERT OR IGNORE INTO gov_raw 
                (title, content, publish_date, page_url, source_url, site_name, status)
                VALUES (?, ?, ?, ?, ?, ?, ?)''', (
                title,
                content,
                item['publish_date'],
                item['url'],
                SITE_DOMAIN,
                SITE_NAME,
                'published',
            ))
            if cur.rowcount > 0:
                new_count += 1
                row_id = cur.lastrowid
                try:
                    db.execute('INSERT INTO gov_search(rowid, title, content, site_name) VALUES (?,?,?,?)',
                               (row_id, title, content, SITE_NAME))
                except sqlite3.IntegrityError:
                    pass
        except Exception as e:
            print(f'  入库失败: {e}')
    
    db.commit()
    total_rows = cur.execute('SELECT COUNT(*) FROM gov_raw WHERE site_name=?', (SITE_NAME,)).fetchone()[0]
    db.close()
    print(f'共获取 {total} 条有效记录')
    print(f'入库: 新增{new_count}, 累计{total_rows}')
    return new_count


if __name__ == '__main__':
    print(f'=== {SITE_NAME} 爬虫 ({SITE_DOMAIN}) ===')
    init_db()
    items = fetch_list()
    print(f'找到 {len(items)} 条记录')
    save_items(items)