"""URL分类工具 - 将站点分为三个等级"""

import csv
from urllib.parse import urlparse

# 无法入库的关键词
HARD_KEYWORDS = ['无法入库', 'Forbidden', 'Post', 'CONNECT']
# 可能需要特殊处理的
MEDIUM_KEYWORDS = ['Cookie', 'user-agent', '-show-', '/art/']


def classify_site(row):
    """将站点分为 Tier 1/2/3"""
    site_type = (row.get('类型', '') or '').strip()
    url = (row.get('Url', '') or '').strip()

    # Tier 3 - 最难
    for kw in HARD_KEYWORDS:
        if kw in site_type:
            return 3

    # Tier 2 - 需要特殊处理
    for kw in MEDIUM_KEYWORDS:
        if kw in site_type:
            return 2

    # 默认 Tier 1 - 简单静态页面
    return 1


def load_sources(csv_path):
    """加载CSV并分类"""
    tiers = {1: [], 2: [], 3: []}

    with open(csv_path, 'r', encoding='utf-8-sig') as f:
        reader = csv.DictReader(f)
        for row in reader:
            url = (row.get('Url', '') or '').strip()
            if not url:
                continue
            tier = classify_site(row)
            tiers[tier].append({
                'site_name': (row.get('站点名', '') or '').strip(),
                'url': url,
                'type': (row.get('类型', '') or '').strip(),
                'domain': urlparse(url).netloc,
            })

    return tiers


def print_stats(tiers):
    """打印分类统计"""
    print(f"{'='*50}")
    print(f"站点分类统计")
    print(f"{'='*50}")
    print(f"  Tier 1 (简单静态): {len(tiers[1])} 个")
    print(f"  Tier 2 (特殊处理): {len(tiers[2])} 个")
    print(f"  Tier 3 (困难):     {len(tiers[3])} 个")
    print(f"  {'='*20}")
    print(f"  总计:              {sum(len(v) for v in tiers.values())} 个")
    print()

    # 按域名去重统计
    for tier_num in [1, 2, 3]:
        domains = set(item['domain'] for item in tiers[tier_num])
        print(f"  Tier {tier_num} 唯一域名: {len(domains)}")


if __name__ == '__main__':
    tiers = load_sources('data/sources.csv')
    print_stats(tiers)