"""URL去重管理 - 实现增量爬取"""

import json
import os
from datetime import datetime


class UrlDedup:
    """URL去重器 - 基于已完成列表"""

    def __init__(self, data_file='data/crawled_urls.json'):
        self.data_file = data_file
        self.crawled = set()  # 已爬取的详情页URL
        self.crawled_list = {}  # 已爬取的列表页 {url: last_crawl_time}
        self._load()

    def _load(self):
        if os.path.exists(self.data_file):
            with open(self.data_file, 'r') as f:
                data = json.load(f)
                self.crawled = set(data.get('detail_urls', []))
                self.crawled_list = data.get('list_urls', {})

    def save(self):
        os.makedirs(os.path.dirname(self.data_file), exist_ok=True)
        with open(self.data_file, 'w') as f:
            json.dump({
                'detail_urls': list(self.crawled),
                'list_urls': self.crawled_list,
                'updated_at': datetime.now().isoformat(),
            }, f, ensure_ascii=False, indent=2)

    def is_detail_crawled(self, url):
        """判断详情页是否已爬取"""
        return url in self.crawled

    def mark_detail_crawled(self, url):
        """标记详情页为已爬取"""
        self.crawled.add(url)

    def is_list_crawled(self, url):
        """判断列表页是否已爬取（增量用）"""
        return url in self.crawled_list

    def mark_list_crawled(self, url):
        """标记列表页为已爬取"""
        self.crawled_list[url] = datetime.now().isoformat()

    def stats(self):
        return {
            'detail_urls_crawled': len(self.crawled),
            'list_urls_crawled': len(self.crawled_list),
        }