"""改进版Tier 1测试 - 增加对老旧政府网站的支持"""

import scrapy
from datetime import datetime
from gov_crawler.items import AnnouncementItem
from utils.classifier import load_sources


class TestV2Spider(scrapy.Spider):
    """改进版测试爬虫 - 处理更多连接问题"""
    name = 'test_tier1_v2'

    custom_settings = {
        'DOWNLOAD_TIMEOUT': 60,
        'RETRY_TIMES': 3,
        'RETRY_HTTP_CODES': [500, 502, 503, 504, 408, 429, 400, 403],
        'CONCURRENT_REQUESTS': 2,
        'CONCURRENT_REQUESTS_PER_DOMAIN': 1,
        'DOWNLOAD_DELAY': 2,
        'AUTOTHROTTLE_ENABLED': False,
        'ROBOTSTXT_OBEY': False,
        'COOKIES_ENABLED': True,
    }

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        tiers = load_sources('data/sources.csv')
        # 跳过已知问题站点，选看起来正常的
        self.urls_to_crawl = []
        for site in tiers[1]:
            url = site['url']
            # 跳过明显有问题的
            if any(x in url for x in ['showList', 'list.php', 'default.jsp', 'api.']):
                continue
            self.urls_to_crawl.append(site)
            if len(self.urls_to_crawl) >= 5:
                break
        self.logger.info(f'测试V2: {len(self.urls_to_crawl)} 个站点')

    def start_requests(self):
        for site in self.urls_to_crawl:
            yield scrapy.Request(
                url=site['url'],
                callback=self.parse,
                errback=self.handle_error,
                meta={
                    'site_name': site['site_name'],
                    'source_url': site['url'],
                    'domain': site['domain'],
                    'handle_httpstatus_list': [301, 302, 303, 307, 308, 403, 404, 500],
                },
                dont_filter=True,
            )

    def parse(self, response):
        item = AnnouncementItem()
        item['site_name'] = response.meta['site_name']
        item['source_url'] = response.meta['source_url']
        item['page_url'] = response.url
        item['domain'] = response.meta['domain']
        item['crawl_time'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')

        if response.status >= 400:
            item['status'] = 'failed'
            item['error_msg'] = f'HTTP {response.status}'
        else:
            item['status'] = 'success'
            title = response.css('title::text').get('')
            item['title'] = title.strip()[:200] if title else '页面无标题'
            
            body_text = response.css('body').xpath('.//text()').getall()
            clean = ' '.join(t.strip() for t in body_text if t.strip())[:500]
            item['content'] = clean
            item['content_text'] = clean[:200]
            item['error_msg'] = ''

            self.logger.info(f'✅ {item["site_name"]} ({response.status}) - {item["title"][:40]}')

        item['publish_date'] = ''
        yield item

    def handle_error(self, failure):
        request = failure.request
        item = AnnouncementItem()
        item['site_name'] = request.meta['site_name']
        item['source_url'] = request.meta['source_url']
        item['page_url'] = request.url
        item['domain'] = request.meta['domain']
        item['title'] = ''
        item['content'] = ''
        item['content_text'] = ''
        item['publish_date'] = ''
        item['crawl_time'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
        item['status'] = 'failed'
        item['error_msg'] = str(failure.value)[:200]
        self.logger.warning(f'❌ {item["site_name"]} - {item["error_msg"][:60]}')
        yield item