"""Tier 1 爬虫 - 处理简单静态HTML站点"""

import scrapy
from datetime import datetime
from urllib.parse import urlparse
from gov_crawler.items import AnnouncementItem
from utils.classifier import load_sources


class Tier1Spider(scrapy.Spider):
    """通用爬虫 - 处理所有简单静态站点

    策略：
    1. 对每个URL发送GET请求
    2. 尝试通用XPath/CSS选择器提取标题、日期、正文
    3. 保存结果
    """
    name = 'tier1'
    allowed_domains = []  # 动态添加

    # 常见公告列表页的选择器
    LIST_SELECTORS = [
        # 通用链接列表
        '//a[contains(@href, "list")]/@href',
        '//a[contains(@href, "showList")]/@href',
        '//a[contains(@href, "article")]/@href',
        '//a[contains(@href, "content")]/@href',
        '//a[contains(@href, "info")]/@href',
        '//a[contains(@href, "detail")]/@href',
        '//a[contains(@href, "index")]/@href',
        '//ul[@class="list"]//a/@href',
        '//div[@class="list"]//a/@href',
    ]

    # 常见标题选择器
    TITLE_SELECTORS = [
        '//title/text()',
        '//h1/text()',
        '//h2/text()',
        '//div[@class="title"]/text()',
        '//div[@class="tit"]/text()',
        '//div[@class="bt"]/text()',
        '//span[@class="title"]/text()',
        '//*[@class="news_title"]//text()',
        '//*[@class="article-title"]//text()',
        '//*[contains(@class, "content-title")]//text()',
    ]

    # 常见日期选择器
    DATE_SELECTORS = [
        '//*[contains(text(), "20\d{2}")]/text()',
        '//div[@class="time"]/text()',
        '//div[@class="date"]/text()',
        '//span[@class="time"]/text()',
        '//span[contains(@class, "date")]/text()',
        '//em[contains(text(), "20")]/text()',
        '//*[contains(@class, "time")]//text()',
        '//*[contains(@class, "date")]//text()',
        '//*[@id="pub-date"]/text()',
        '//meta[@name="PubDate"]/@content',
        '//meta[@name="publishdate"]/@content',
    ]

    # 常见正文选择器
    CONTENT_SELECTORS = [
        '//div[@class="content"]//text()',
        '//div[@class="article"]//text()',
        '//div[@class="text"]//text()',
        '//div[@class="main"]//text()',
        '//div[@id="content"]//text()',
        '//div[contains(@class, "article")]//text()',
        '//div[contains(@class, "content")]//text()',
        '//div[@class="TRS_Editor"]//text()',
        '//div[contains(@class, "TRS")]//text()',
        '//*[@class="news-content"]//text()',
        '//*[@class="article-content"]//text()',
    ]

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.tiers = load_sources('data/sources.csv')
        self.urls_to_crawl = self.tiers[1]
        self.crawled = 0
        self.total = len(self.urls_to_crawl)
        self.logger.info(f'Tier 1 爬虫启动: 共 {self.total} 个站点')

    def start_requests(self):
        for site in self.urls_to_crawl:
            self.crawled += 1
            yield scrapy.Request(
                url=site['url'],
                callback=self.parse_page,
                errback=self.handle_error,
                meta={
                    'site_name': site['site_name'],
                    'source_url': site['url'],
                    'domain': site['domain'],
                    'download_timeout': 30,
                },
                dont_filter=True,
                # 大部分政府网站使用GBK/GB2312编码
                encoding='utf-8',
            )

    def parse_page(self, response):
        """解析任意静态页面"""
        item = AnnouncementItem()
        meta = response.meta

        item['site_name'] = meta.get('site_name', '')
        item['source_url'] = meta.get('source_url', '')
        item['page_url'] = response.url
        item['domain'] = meta.get('domain', '')
        item['crawl_time'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
        item['status'] = 'success'
        item['error_msg'] = ''

        # 提取标题
        title = None
        for selector in self.TITLE_SELECTORS:
            result = response.xpath(selector).getall()
            if result:
                title = ' '.join(r.strip() for r in result if r.strip())
                if title:
                    break
        if not title:
            # 从URL提取文件名作为标题
            path = urlparse(response.url).path
            title = path.split('/')[-1] if path.split('/')[-1] else path
        item['title'] = (title or '').strip()[:500]

        # 提取日期
        date_text = ''
        import re
        for selector in self.DATE_SELECTORS:
            result = response.xpath(selector).getall()
            for r in result:
                dates = re.findall(r'20\d{2}[-/]\d{1,2}[-/]\d{1,2}', r)
                if dates:
                    date_text = dates[0]
                    break
            if date_text:
                break
        item['publish_date'] = date_text

        # 提取正文
        content_parts = []
        for selector in self.CONTENT_SELECTORS:
            result = response.xpath(selector).getall()
            if result:
                texts = [r.strip() for r in result if r.strip()]
                if texts:
                    content_parts = texts
                    break

        full_text = '\n'.join(content_parts)
        item['content'] = full_text[:2000] if full_text else ''
        item['content_text'] = full_text[:500] if full_text else ''

        # 提取页面上的链接（如果有列表页）
        self._extract_links(response)

        self.logger.info(f'[{self.crawled}/{self.total}] ✅ {item["site_name"]} - {item["title"][:50]}')
        yield item

    def _extract_links(self, response):
        """提取页面上的链接，后续可跟进"""
        # 暂不实现，后续可以跟进爬取详情页
        pass

    def handle_error(self, failure):
        """处理请求失败"""
        request = failure.request
        item = AnnouncementItem()
        item['site_name'] = request.meta.get('site_name', '')
        item['source_url'] = request.meta.get('source_url', '')
        item['page_url'] = request.url
        item['domain'] = request.meta.get('domain', '')
        item['title'] = ''
        item['publish_date'] = ''
        item['content'] = ''
        item['content_text'] = ''
        item['crawl_time'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
        item['status'] = 'failed'
        item['error_msg'] = str(failure.value)[:200] if hasattr(failure, 'value') else str(failure)

        self.logger.warning(f'[{self.crawled}/{self.total}] ❌ {item["site_name"]} - {item["error_msg"][:60]}')
        yield item