#!/usr/bin/env python3
"""
重庆环科源博达环保科技有限公司 (www.cqbdhb.com) 爬虫
ASP.NET站点，公开公示栏目，25页共244条
"""
import re
import requests
import sqlite3
import os
import sys
import time

SITE_NAME = '重庆环科源博达'
SITE_DOMAIN = 'www.cqbdhb.com'
BASE_URL = 'http://www.cqbdhb.com'
LIST_URL = 'http://www.cqbdhb.com/nc-gcgslist.html'

DB_PATH = os.environ.get('DB_PATH', os.path.join(os.path.dirname(os.path.abspath(__file__)), 'search.db'))

HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
}


def fetch_list_page(page=1):
    """获取分页列表"""
    url = f'{LIST_URL}?page={page}' if page > 1 else LIST_URL
    try:
        resp = requests.get(url, headers=HEADERS, timeout=30)
        resp.encoding = 'utf-8'
        html = resp.text
    except Exception as e:
        print(f"  [ERROR] 列表页第{page}页请求失败: {e}")
        return [], 0, 0

    items = []
    # 只提取主列表 ArticleUL 中的条目
    main_list = re.search(r'<ul[^>]*id=[\'"]ArticleUL[\'"][^>]*>(.*?)</ul>', html, re.DOTALL)
    if not main_list:
        print("  [WARN] 未找到主列表 ArticleUL")
        return [], 0, 0
    ul_html = main_list.group(1)

    pattern = r'<li><a href="([^"]+)"[^>]*title="([^"]*)"[^>]*>.*?</a><span[^>]*class=[\'"]news_time[\'"]>([^<]+)</span></li>'
    matches = re.findall(pattern, ul_html, re.DOTALL)

    for href, title, date_str in matches:
        title = title.strip()
        date_str = date_str.strip()
        if not title:
            continue
        # 补全URL
        if href.startswith('http'):
            full_url = href
        else:
            full_url = f'{BASE_URL}/{href}' if not href.startswith('/') else f'{BASE_URL}{href}'
        items.append({
            'title': title,
            'url': full_url,
            'publish_date': date_str[:10],
        })

    # 解析总记录数和总页数
    total = 0
    total_pages = 1
    total_match = re.search(r'共<span>(\d+)</span>条', html)
    if total_match:
        total = int(total_match.group(1))
    pages_match = re.search(r'共<span>(\d+)</span>页', html)
    if pages_match:
        total_pages = int(pages_match.group(1))

    print(f"  第{page}页: 找到 {len(items)} 条 (总计 {total} 条, 共 {total_pages} 页)")
    return items, total_pages, total


def fetch_detail(url):
    """获取详情页完整标题和正文"""
    try:
        resp = requests.get(url, headers=HEADERS, timeout=30)
        resp.encoding = 'utf-8'
        html = resp.text
    except Exception as e:
        print(f"    [ERROR] 详情页请求失败 {url}: {e}")
        return None

    result = {}

    # 提取完整标题: <h1 class="contents_title">xxx</h1>
    title_match = re.search(r'<h1[^>]*class=[\'"]contents_title[\'"][^>]*>([^<]+)</h1>', html)
    if title_match:
        result['title'] = title_match.group(1).strip()

    # 提取发布日期: <span>发布时间：2026-05-18</span>
    date_match = re.search(r'发布时间[：:]?\s*([\d-]+)', html)
    if date_match:
        result['publish_date'] = date_match.group(1).strip()

    # 提取正文: <div id="ContentDiv" class="contents">...</div>
    content_match = re.search(r'<div[^>]*id="ContentDiv"[^>]*class="contents"[^>]*>(.*?)<div[^>]*id="PNDiv"', html, re.DOTALL)
    if not content_match:
        content_match = re.search(r'<div[^>]*id="ContentDiv"[^>]*class="contents"[^>]*>(.*?)</div>', html, re.DOTALL)

    if content_match:
        content_html = content_match.group(1).strip()
        # 移除标题部分的h1（已单独提取）
        content_html = re.sub(r'<h1[^>]*class="contents_title"[^>]*>.*?</h1>', '', content_html)
        # 移除分隔线
        content_html = re.sub(r'<div[^>]*class="dashed-line"[^>]*>.*?</div>', '', content_html)
        # 移除作者/点击/日期信息行
        content_html = re.sub(r'<p[^>]*class="text-center"[^>]*>.*?</p>', '', content_html)
        # 补全图片src和链接href
        content_html = re.sub(r'src="(?!http)', f'src="{BASE_URL}/', content_html)
        content_html = re.sub(r'href="(?!http)', f'href="{BASE_URL}/', content_html)
        result['content'] = content_html.strip()
    else:
        result['content'] = ''

    return result


def get_db():
    """获取数据库连接"""
    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()
    c.execute('''CREATE TABLE IF NOT EXISTS gov_raw (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        title TEXT,
        content TEXT,
        publish_date TEXT,
        url TEXT UNIQUE,
        site_name TEXT,
        source TEXT,
        attachments TEXT,
        created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
    )''')
    conn.commit()
    return conn


def save_item(conn, item):
    """保存单条数据"""
    c = conn.cursor()
    try:
        c.execute('''
            INSERT OR IGNORE INTO gov_raw (title, content, publish_date, url, site_name, source)
            VALUES (?, ?, ?, ?, ?, ?)
        ''', (
            item['title'],
            item.get('content', ''),
            item.get('publish_date', ''),
            item['url'],
            SITE_NAME,
            f'{SITE_NAME} - 公开公示',
        ))
        if c.rowcount > 0:
            print(f"    ✅ 新增: {item['title'][:50]}...")
        else:
            print(f"    ⏭️  已存在: {item['title'][:50]}...")
        conn.commit()
    except Exception as e:
        print(f"    ❌ 写入失败: {e}")


def run(max_pages=None):
    """主运行函数"""
    print(f"={'='*50}")
    print(f"  🌐 {SITE_NAME} ({SITE_DOMAIN}) 爬虫")
    print(f"={'='*50}")

    conn = get_db()

    # 先获取第一页，确认总页数
    page = 1
    items, total_pages, total = fetch_list_page(page)
    if not items:
        print("⚠️ 无法获取列表数据，退出")
        conn.close()
        return

    print(f"\n📊 总计 {total} 条, 共 {total_pages} 页")

    if max_pages and max_pages < total_pages:
        print(f"   ⚠️  限制只爬 {max_pages} 页")
        total_pages = max_pages

    # 处理第一页
    print(f"\n📄 第1页 ({len(items)} 条)")
    for idx, item in enumerate(items):
        print(f"  ▶️  [{idx+1}/{len(items)}] {item['title'][:50]}...")
        detail = fetch_detail(item['url'])
        if detail:
            item['title'] = detail.get('title', item['title'])
            item['publish_date'] = detail.get('publish_date', item['publish_date'])
            item['content'] = detail.get('content', '')
        save_item(conn, item)
        time.sleep(0.5)

    # 处理后续页
    page = 2
    while page <= total_pages:
        items, _, _ = fetch_list_page(page)
        if not items:
            break

        print(f"\n📄 第{page}页 ({len(items)} 条)")
        for idx, item in enumerate(items):
            print(f"  ▶️  [{idx+1}/{len(items)}] {item['title'][:50]}...")
            detail = fetch_detail(item['url'])
            if detail:
                item['title'] = detail.get('title', item['title'])
                item['publish_date'] = detail.get('publish_date', item['publish_date'])
                item['content'] = detail.get('content', '')
            save_item(conn, item)
            time.sleep(0.5)

        page += 1

    conn.close()

    # 统计
    conn2 = sqlite3.connect(DB_PATH, timeout=60)
    count = conn2.execute("SELECT COUNT(*) FROM gov_raw WHERE site_name=?", (SITE_NAME,)).fetchone()[0]
    conn2.close()
    print(f"\n={'='*50}")
    print(f"  ✅ 完成! {SITE_NAME} 共导入 {count} 条数据到 {DB_PATH}")
    print(f"={'='*50}")


if __name__ == '__main__':
    max_pages = None
    if len(sys.argv) > 1:
        max_pages = int(sys.argv[1])
    run(max_pages)