"""自定义中间件 - 处理老旧政府网站的特殊网络问题"""

from scrapy.downloadermiddlewares.retry import RetryMiddleware
from scrapy.utils.response import response_status_message
import logging

logger = logging.getLogger(__name__)


class GovRetryMiddleware(RetryMiddleware):
    """改进版重试中间件 - 处理ConnectionDone等Twisted连接错误"""

    def process_exception(self, request, exception, spider):
        # 记录连接错误详情
        error_name = type(exception).__name__
        error_msg = str(exception)[:100]

        # 对常见的连接问题自动重试
        retryable_errors = [
            'ConnectionDone',
            'ConnectionLost',
            'ConnectionRefused',
            'ConnectionClosed',
            'TimeoutError',
            'DNSLookupError',
            'TCPTimedOutError',
        ]

        if any(e in error_name for e in retryable_errors):
            retries = request.meta.get('retry_times', 0) + 1
            if retries <= self.max_retry_times:
                logger.warning(f'重试 [{retries}/{self.max_retry_times}]: {request.url[:60]} - {error_name}')
                return self._retry(request, error_name, spider)
            else:
                logger.error(f'放弃: {request.url[:60]} - {error_name} (已重试{self.max_retry_times}次)')

        return super().process_exception(request, exception, spider)