"""Tier 1 测试 - 先跑10个站点验证"""

import scrapy
from datetime import datetime
from urllib.parse import urlparse
from gov_crawler.items import AnnouncementItem
from utils.classifier import load_sources


class TestSpider(scrapy.Spider):
    """测试爬虫 - 先跑10个确认流程没问题"""
    name = 'test_tier1'

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        tiers = load_sources('data/sources.csv')
        self.urls_to_crawl = tiers[1][:10]
        self.logger.info(f'测试爬虫: {len(self.urls_to_crawl)} 个站点')

    def start_requests(self):
        for site in self.urls_to_crawl:
            yield scrapy.Request(
                url=site['url'],
                callback=self.parse,
                errback=self.handle_error,
                meta={
                    'site_name': site['site_name'],
                    'source_url': site['url'],
                    'domain': site['domain'],
                },
                dont_filter=True,
            )

    def parse(self, response):
        item = AnnouncementItem()
        item['site_name'] = response.meta['site_name']
        item['source_url'] = response.meta['source_url']
        item['page_url'] = response.url
        item['domain'] = response.meta['domain']
        item['crawl_time'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
        item['status'] = 'success'

        # 尝试提取标题
        title = response.css('title::text').get('')
        item['title'] = title.strip()[:200] if title else ''

        # 尝试提取正文摘要
        body_text = response.css('body').xpath('.//text()').getall()
        clean = ' '.join(t.strip() for t in body_text if t.strip())[:500]
        item['content'] = clean
        item['content_text'] = clean[:200]

        item['publish_date'] = ''
        item['error_msg'] = ''

        self.logger.info(f'✅ {item["site_name"]} - {item["title"][:60]}')
        yield item

    def handle_error(self, failure):
        request = failure.request
        item = AnnouncementItem()
        item['site_name'] = request.meta['site_name']
        item['source_url'] = request.meta['source_url']
        item['page_url'] = request.url
        item['domain'] = request.meta['domain']
        item['title'] = ''
        item['content'] = ''
        item['content_text'] = ''
        item['publish_date'] = ''
        item['crawl_time'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
        item['status'] = 'failed'
        item['error_msg'] = str(failure.value)[:200]
        self.logger.warning(f'❌ {item["site_name"]} - {item["error_msg"][:60]}')
        yield item