#!/usr/bin/env python3
"""分析7个政府/环评网站的结构"""

import sys, os, re, json
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import fetch_page
from urllib.parse import urljoin

BASE_DIR = os.path.dirname(os.path.abspath(__file__))

sites = [
    {
        "name": "1. 新乡生态环境局",
        "url": "https://sthjj.xinxiang.gov.cn/zwgk/public/column/6638717?type=4&catId=6720855&action=list",
        "note": "政府信息公开平台"
    },
    {
        "name": "2. 南谯区生态环境(第七师胡杨河市)",
        "url": "http://www.nqs.gov.cn/ztzl/xzzfgs/bm5/sthjj7/zhgs16",
        "note": "实际是第七师胡杨河市"
    },
    {
        "name": "3. 黄冈生态环境局",
        "url": "https://hbj.hg.gov.cn/zwgk/public/column/6636213?type=4&catId=7026845&action=list",
        "note": "政府信息公开平台"
    },
    {
        "name": "4. 信阳环评公示",
        "url": "http://58.210.182.61:10520/gongshixinxi",
        "note": "IP直连"
    },
    {
        "name": "5. 柴达木开发区(常熟经开区)",
        "url": "https://www.cedz.org/part-xinxifabu.html",
        "note": "实际是常熟经开区"
    },
    {
        "name": "6. 揭阳高新区(大南海石化工业区)",
        "url": "http://www.jieyang.gov.cn/szfjg/jysdnhshgyq/zwgk/gsgg/",
        "note": "大南海石化工业区"
    },
    {
        "name": "7. 太和县",
        "url": "https://www.taihe.gov.cn/OpennessContent/showList/317/20154/page_1.html",
        "note": "OpennessContent API"
    },
]

def analyze_list(url, name):
    print(f"\n{'='*70}")
    print(f"🔍 正在分析: {name}")
    print(f"   URL: {url}")
    print(f"{'='*70}")
    
    html = fetch_page(url)
    if not html:
        print(f"   ❌ 无法获取页面")
        return {"name": name, "error": "无法获取页面"}
    
    result = {"name": name, "url": url, "len": len(html)}
    
    # 保存HTML用于后续分析
    safe_name = re.sub(r'[^\w]', '_', name)[:30]
    path = os.path.join(BASE_DIR, f"_analyze_{safe_name}.html")
    with open(path, 'w', encoding='utf-8') as f:
        f.write(html[:50000])  # 只保存前50000字符
    result["html_path"] = path
    
    # 1. 分析列表容器
    from bs4 import BeautifulSoup
    soup = BeautifulSoup(html, 'html.parser')
    
    # 查找链接列表
    all_links = soup.find_all('a', href=True)
    print(f"   📎 页面中共 {len(all_links)} 个链接")
    
    # 查找可能包含列表的容器
    uls = soup.find_all('ul')
    print(f"   📋 页面中共 {len(uls)} 个 <ul>")
    
    # 查找li元素
    lis = soup.find_all('li')
    print(f"   📝 页面中共 {len(lis)} 个 <li>")
    
    # 查找分页相关信息
    page_info = re.findall(r'(?:page|页码|分页|当前|共\s*\d+\s*页|page_\d+|index_\d+)', html, re.I)
    if page_info:
        print(f"   📄 分页相关文本: {page_info[:10]}")
    
    # 查找日期
    dates = re.findall(r'\d{4}[-/]\d{1,2}[-/]\d{1,2}', html)
    print(f"   📅 日期字符串: {dates[:10]}")
    
    # 查找包含 href 和日期的 li 结构
    for li in lis[:50]:
        a = li.find('a', href=True)
        span = li.find('span')
        date_text = ""
        if span:
            date_text = span.get_text(strip=True)
        elif a:
            # 检查a周围
            parent = li.get_text(' ', strip=True)
            date_match = re.search(r'\d{4}[-/]\d{1,2}[-/]\d{1,2}', parent)
            if date_match:
                date_text = date_match.group()
        
        if a and len(a.get('href','')) > 5:
            href = a.get('href','')
            title = a.get('title','') or a.get_text(strip=True)
            if len(title) > 5:
                print(f"      ▶ 示例条目: title='{title[:60]}' href='{href[:80]}' date='{date_text[:20]}'")
                # 只取前3个示例
                break
    
    # 检查是否有 queryList API 或 OpennessContent
    if 'queryList' in html or '/queryList' in html:
        print(f"   🔌 检测到 queryList API 模式")
        result["mode"] = "queryList API"
    
    if 'OpennessContent/showList' in html or 'OpennessContent' in html:
        print(f"   🔌 检测到 OpennessContent API 模式")
        result["mode"] = "OpennessContent"
    
    # 检查分页URL模式
    page_patterns = re.findall(r'(page_\d+|index_\d+|PageNo=\d+|currentPage=\d+)', html, re.I)
    if page_patterns:
        print(f"   🔢 分页模式: {page_patterns[:5]}")
        result["page_pattern"] = page_patterns[:5]
    
    # 检查详情页URL
    for a in all_links[:30]:
        href = a.get('href','')
        if href and len(href) > 20 and not href.startswith('#') and 'javascript' not in href:
            print(f"      🔗 详情链接: href='{href[:100]}' title='{a.get('title','')[:50]}'")
            result["sample_detail_url"] = href
            break
    
    # 搜索容器class/id
    content_divs = soup.find_all(['div', 'ul'], class_=re.compile(r'(list|news|article|content|main|right|center)', re.I))
    print(f"   🏷️ 容器 class/id:")
    for d in content_divs[:5]:
        cls = d.get('class', [])
        id_ = d.get('id', '')
        if cls or id_:
            print(f"      <{d.name} class={cls} id={id_}> (子元素: {len(d.find_all(recursive=False))})")
    
    # 查看表格结构
    tables = soup.find_all('table')
    if tables:
        print(f"   📊 表格数量: {len(tables)}")
    
    # 查找 script 中的 JSON/API 配置
    scripts = soup.find_all('script')
    for s in scripts:
        if s.string and ('queryList' in s.string or 'ajax' in s.string or 'page' in s.string):
            print(f"   📜 脚本中含 API/分页逻辑 (片段前200字符): {s.string[:200]}")
            break
    
    result["sample_html"] = html[:3000]
    return result


def analyze_detail(detail_url, name):
    """尝试分析详情页"""
    print(f"\n   --- 尝试分析详情页: {detail_url[:80]}...")
    html = fetch_page(detail_url)
    if not html:
        print(f"   ❌ 无法获取详情页")
        return
    
    from bs4 import BeautifulSoup
    soup = BeautifulSoup(html, 'html.parser')
    
    # 查找正文容器
    for cls in ['content', 'article', 'main', 'text', 'detail', 'zoom', 'TRS_Editor', 'Custom_UnionStyle', 'news-content', 'article-content']:
        el = soup.find(['div', 'article'], class_=re.compile(cls, re.I)) or soup.find(['div', 'article'], id=re.compile(cls, re.I))
        if el:
            text_len = len(el.get_text(strip=True))
            print(f"   ✅ 正文容器: <{el.name} class={el.get('class','')} id={el.get('id','')}> 文本长度={text_len}")
            break
    
    # 查找title
    h1 = soup.find('h1')
    if h1:
        print(f"   📌 H1标题: {h1.get_text(strip=True)[:80]}")
    
    # 查找日期
    for pat in [r'发布时间[：:]\s*(\d{4}[-/]\d{1,2}[-/]\d{1,2})', 
                r'发布日期[：:]\s*(\d{4}[-/]\d{1,2}[-/]\d{1,2})',
                r'pubDate["\']?\s*[:=]\s*["\']?(\d{4}-\d{2}-\d{2})',
                r'<meta[^>]*name=["\']PubDate["\'][^>]*content=["\'](\d{4}-\d{2}-\d{2})']:
        m = re.search(pat, html, re.I)
        if m:
            print(f"   📅 详情页日期: {m.group(1)}")
            break


# ===== 执行分析 =====
all_results = []
for site in sites:
    result = analyze_list(site["url"], site["name"])
    all_results.append(result)
    
    # 如果有详情URL，尝试分析
    if result.get("sample_detail_url"):
        detail_url = result["sample_detail_url"]
        if not detail_url.startswith('http'):
            base_url = site["url"]
            detail_url = urljoin(base_url, detail_url)
        analyze_detail(detail_url, site["name"])

print("\n\n" + "="*70)
print("✅ 7站点初步分析完成!")
print("="*70)

# 保存结果
with open(os.path.join(BASE_DIR, '_analyze_results.json'), 'w', encoding='utf-8') as f:
    json.dump(all_results, f, ensure_ascii=False, indent=2)
print("结果已保存到 _analyze_results.json")