import os

BOT_NAME = 'gov_crawler'

SPIDER_MODULES = ['gov_crawler.spiders']
NEWSPIDER_MODULE = 'gov_crawler.spiders'

# Obey robots.txt
ROBOTSTXT_OBEY = False

# 并发控制 - 初始阶段保守一点
CONCURRENT_REQUESTS = 8
CONCURRENT_REQUESTS_PER_DOMAIN = 2

# 下载延迟 - 防止被封
DOWNLOAD_DELAY = 1.0
RANDOMIZE_DOWNLOAD_DELAY = True

# 超时设置
DOWNLOAD_TIMEOUT = 30

# 自动限速
AUTOTHROTTLE_ENABLED = True
AUTOTHROTTLE_START_DELAY = 1.0
AUTOTHROTTLE_MAX_DELAY = 10.0
AUTOTHROTTLE_TARGET_CONCURRENCY = 2.0

# 重试设置
RETRY_ENABLED = True
RETRY_TIMES = 2
RETRY_HTTP_CODES = [500, 502, 503, 504, 408, 429]

# Middleware
DOWNLOADER_MIDDLEWARES = {
    'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,
    'scrapy.downloadermiddlewares.retry.RetryMiddleware': None,
    'gov_crawler.middlewares_retry.GovRetryMiddleware': 90,
    'gov_crawler.middlewares.RandomUserAgentMiddleware': 400,
}

# Pipeline
ITEM_PIPELINES = {
    'gov_crawler.pipelines.JsonPipeline': 300,
    'gov_crawler.pipelines.CsvPipeline': 301,
    'gov_crawler.pipelines.NotionKeywordPipeline': 500,
}

# Notion 关键词推送
NOTION_API_KEY = os.environ.get('NOTION_API_KEY_CRAWLER', '')
NOTION_DATABASE_ID = 'YOUR_DATABASE_ID_HERE'
NOTION_KEYWORDS = ['化工', '石油', '乙烯', '丙烯', '炼化', '新材料', '新能源',
                   '搬迁', '改造', '扩建', '新建项目', '环评批复', '验收']
NOTION_PUSH_INTERVAL = 0.5  # 每条推送间隔0.5秒，防限流

# 日志
LOG_LEVEL = 'INFO'
LOG_FILE = 'crawl.log'

# 默认请求头
DEFAULT_REQUEST_HEADERS = {
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
    'Accept-Encoding': 'gzip, deflate',
}