#!/usr/bin/env python3
"""快速测试9个站点 — 每站抓5条"""
import sys, os, time, re, json, hashlib, sqlite3, urllib.request, ssl, base64, subprocess
from html import unescape

QUALITY_DB = os.path.expanduser("~/gov_crawler/quality_results.db")
SERVER_SSH = "root@1.94.217.116"
SERVER_QUALITY_DB = "/root/quality_results.db"

# ── 规则库（直接内联） ──
# 从 ljobx_rules.py 和 ljobx_list_rules.py 提取相关规则
DETAIL_RULES = {}  # domain -> (title_start, title_end, content_start, content_end)
LIST_RULES = {}    # domain -> (area_start, area_end, url_must, url_forbid, pagination)

# 手动加载
sys.path.insert(0, os.path.expanduser("~/gov_crawler/utils"))
try:
    from ljobx_rules import LJOBX_RULES
    DETAIL_RULES = LJOBX_RULES
except:
    with open(os.path.expanduser("~/gov_crawler/utils/ljobx_rules.py")) as f:
        exec(f.read(), globals())
        DETAIL_RULES = LJOBX_RULES

try:
    from ljobx_list_rules import LJOBX_LIST_RULES
    LIST_RULES = LJOBX_LIST_RULES
except:
    with open(os.path.expanduser("~/gov_crawler/utils/ljobx_list_rules.py")) as f:
        exec(f.read(), globals())
        LIST_RULES = LJOBX_LIST_RULES

SITES = [
    ("淮南煤化工", "http://ahccci.huainan.gov.cn/xwzx/tzgg/index.html", "ahccci.huainan.gov.cn"),
    ("佛冈", "http://www.fogang.gov.cn/zwgk/zdlyxx/hjbh/jsxmhjyxpjxx/index.html", "www.fogang.gov.cn"),
    ("环评云", "https://www.eiacloud.com/gs/list/1", "www.eiacloud.com"),
    ("万华化学", "https://www.whchem.com/column/123/", "www.whchem.com"),
    ("泰兴", "http://www.taixing.gov.cn/col/col52053/index.html", "www.taixing.gov.cn"),
    ("新会", "http://www.xinhui.gov.cn/zwgk/zdlyxxgk/hjbhxxgk/jsxmhjyxpjxx", "www.xinhui.gov.cn"),
    ("环评爱好者", "http://www.eiafans.com/forum-64-1.html", "www.eiafans.com"),
    ("新疆环保", "http://www.xjhbcy.cn/articles/list/121", "www.xjhbcy.cn"),
    ("准东", "https://www.xjzdkfq.cn/p163/sthj1.html", "www.xjzdkfq.cn"),
]

# ── 通用工具 ──

def fetch(url, timeout=20):
    ctx = ssl._create_unverified_context()
    req = urllib.request.Request(url, headers={
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
    })
    try:
        resp = urllib.request.urlopen(req, timeout=timeout, context=ctx)
        html = resp.read().decode("utf-8", errors="replace")
        return html, resp.geturl()
    except Exception as e:
        print(f"    ⚠️ {e}")
        return None, url


def get_between(text, start, end):
    """提取 start 和 end 之间的文本"""
    if not start or not end: return ""
    s = text.find(start)
    if s == -1: return ""
    s += len(start)
    e = text.find(end, s)
    if e == -1: return text[s:]
    return text[s:e]


def extract_title_by_rules(html, domain):
    """用 ljobx 规则提取标题"""
    rule = DETAIL_RULES.get(domain)
    if not rule: return ""
    ts, te = rule[0], rule[1]
    title = get_between(html, ts, te)
    if title:
        title = re.sub(r'<[^>]+>', '', title).strip()
        title = unescape(title)
    return title


def extract_content_by_rules(html, domain):
    """用 ljobx 规则提取正文"""
    rule = DETAIL_RULES.get(domain)
    if not rule: return ""
    cs, ce = rule[2], rule[3]
    # 处理全页模式 <(*)> → </html>
    if cs == "<(*)>" and ce == "</html>":
        content = html
    else:
        content = get_between(html, cs, ce)
    # 去标签
    content = re.sub(r'<script[^>]*>.*?</script>', '', content, flags=re.DOTALL|re.I)
    content = re.sub(r'<style[^>]*>.*?</style>', '', content, flags=re.DOTALL|re.I)
    content = re.sub(r'<[^>]+>', ' ', content)
    content = re.sub(r'\s+', ' ', content).strip()
    content = unescape(content)
    return content


def extract_list_items(html, domain):
    """从列表页提取文章项 — 智能筛选，排除导航菜单"""
    items = []
    
    # 如果用 ljobx 区域规则，先缩小范围
    list_rule = LIST_RULES.get(domain)
    url_must = ""
    url_forbid = ""
    if list_rule:
        area_start, area_end = list_rule[0], list_rule[1]
        url_must = list_rule[2] or ""
        url_forbid = list_rule[3] if len(list_rule) > 3 else ""
        if area_start:
            sub = get_between(html, area_start, area_end)
            if sub:
                html = sub
    
    # 过滤导航关键词
    NAV_KEYWORDS = ['首页', '中国政府网', '手机版', '无障碍', '繁体', 'English',
                    '登录', '注册', '搜索', '返回', '关于', '联系', '邮箱']
    
    # 找带日期的 <li> 中的 <a>
    li_pattern = re.compile(r'<li[^>]*>(.*?)</li>', re.DOTALL)
    for m in li_pattern.finditer(html):
        li_html = m.group(1)
        a_match = re.search(r'<a\s+[^>]*href="([^"]+)"[^>]*>(.*?)</a>', li_html)
        if not a_match:
            a_match = re.search(r"<a\s+[^>]*href='([^']+)'[^>]*>(.*?)</a>", li_html)
        if not a_match:
            continue
        url = a_match.group(1)
        title = re.sub(r'<[^>]+>', '', a_match.group(2)).strip()
        
        # 排除空/短标题
        if not title or len(title) < 5:
            continue
        # 排除导航
        if any(kw in title for kw in NAV_KEYWORDS):
            continue
        
        # URL 过滤
        if url_must and url_must not in url:
            continue
        if url_forbid and url_forbid in url:
            continue
        if url.startswith('javascript') or url.startswith('#'):
            continue
        
        # 日期
        date = ""
        date_m = re.search(r'(\d{4}[-/]\d{1,2}[-/]\d{1,2})', li_html)
        if date_m:
            date = date_m.group(1)
        
        # URL 用原始值，爬取时再补全
        items.append({"url": url, "title": title, "date": date})
    
    # 如果没有找到，尝试直接从页面找所有 <a>（不限 <li>）
    if len(items) < 3:
        a_pattern = re.compile(r'<a\s+[^>]*href="([^"]+)"[^>]*>(.*?)</a>', re.DOTALL)
        for m in a_pattern.finditer(html):
            url = m.group(1)
            title = re.sub(r'<[^>]+>', '', m.group(2)).strip()
            if not title or len(title) < 8:
                continue
            if any(kw in title for kw in NAV_KEYWORDS):
                continue
            if url.startswith('javascript') or url.startswith('#'):
                continue
            if url_must and url_must not in url:
                continue
            if url_forbid and url_forbid in url:
                continue
            # 去重
            if any(t['url'] == url for t in items):
                continue
            date = ""
            context = html[max(0, html.find(title)-300):html.find(title)+300]
            date_m = re.search(r'(\d{4}[-/]\d{1,2}[-/]\d{1,2})', context)
            if date_m:
                date = date_m.group(1)
            items.append({"url": url, "title": title, "date": date})
            if len(items) >= 10:
                break
    
    return items[:15]


def clean_text(text):
    """清洗正文"""
    if not text: return ""
    text = re.sub(r'<script[^>]*>.*?</script>', '', text, flags=re.DOTALL|re.I)
    text = re.sub(r'<style[^>]*>.*?</style>', '', text, flags=re.DOTALL|re.I)
    text = re.sub(r'</?p[^>]*>', '\n\n', text, flags=re.I)
    text = re.sub(r'<br\s*/?>', '\n', text, flags=re.I)
    text = re.sub(r'<[^>]+>', '', text)
    text = unescape(text)
    text = re.sub(r'&nbsp;', ' ', text)
    text = re.sub(r'\n{3,}', '\n\n', text)
    text = re.sub(r'[ \t]+', ' ', text)
    lines = [l.strip() for l in text.split('\n')]
    return '\n'.join(l for l in lines if l).strip()


# ── 通用正文提取（无规则兜底） ──

COMMON_CONTENT_SELECTORS = [
    'div[class*="content"]', 'div[class*="article"]', 'div[class*="text"]',
    'div[class*="detail"]', 'div[class*="TRS"]', 'div[class*="zoom"]',
    'div[id*="content"]', 'div[id*="article"]', 'div[id*="zoom"]',
    'article', 'div[class*="body"]', 'div[class*="main"]',
    'div[class*="show"]', 'div[class*="info"]',
    # fallback: body
    'body',
]

def extract_content_generic(html):
    """通用正文提取"""
    from bs4 import BeautifulSoup
    soup = BeautifulSoup(html, 'html.parser')
    
    # 去除 script/style
    for tag in soup(['script', 'style', 'nav', 'footer', 'header']):
        tag.decompose()
    
    # 尝试常见容器
    best_text = ""
    for sel in COMMON_CONTENT_SELECTORS:
        containers = soup.select(sel)
        for c in containers:
            text = c.get_text(separator='\n', strip=True)
            # 排除导航
            cls = ' '.join(c.get('class', []) or []) + ' ' + (c.get('id', '') or '')
            if any(kw in cls.lower() for kw in ['nav', 'menu', 'header', 'sidebar', 'footer']):
                continue
            if len(text) > len(best_text):
                best_text = text
    
    if best_text:
        return best_text[:5000]
    
    # fallback: body text
    body = soup.find('body')
    if body:
        return body.get_text(separator='\n', strip=True)[:5000]
    return ""


# ── DB ──

def init_db():
    conn = sqlite3.connect(QUALITY_DB)
    conn.executescript("""
        CREATE TABLE IF NOT EXISTS quality_results (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            site_name TEXT NOT NULL,
            domain TEXT,
            title TEXT,
            url TEXT UNIQUE,
            content TEXT,
            publish_date TEXT,
            summary TEXT,
            crawled_at TEXT DEFAULT (datetime('now','localtime'))
        );
        CREATE TABLE IF NOT EXISTS crawl_notes (
            site_name TEXT PRIMARY KEY,
            total INTEGER DEFAULT 0,
            status TEXT,
            crawled_at TEXT
        );
    """)
    conn.commit()
    conn.close()


def store(name, domain, title, url, content, date):
    conn = sqlite3.connect(QUALITY_DB)
    try:
        summary = content[:300] if content else (title[:200] if title else "")
        conn.execute("DELETE FROM quality_results WHERE url=?", (url,))
        conn.execute(
            "INSERT INTO quality_results (site_name, domain, title, url, content, publish_date, summary) VALUES (?,?,?,?,?,?,?)",
            (name, domain, title, url, content or "", date or "", summary),
        )
        conn.commit()
        return True
    except Exception as e:
        print(f"    ❌ 存储: {e}")
        return False
    finally:
        conn.close()


# ── 爬取单个站点 ──

def crawl_one(name, list_url, domain):
    print(f"\n{'─'*50}")
    print(f"📍 {name}")
    print(f"  URL: {list_url}")
    print(f"  详情规则: {'✅' if domain in DETAIL_RULES else '❌'}")
    print(f"  列表规则: {'✅' if domain in LIST_RULES else '❌'}")
    
    # 1. 列表页
    html, _ = fetch(list_url)
    if not html:
        print(f"  ❌ 列表页失败")
        return 0
    
    items = extract_list_items(html, domain)
    if not items:
        print(f"  ❌ 列表解析失败")
        # 尝试直接从页面抓所有链接
        a_pattern = re.compile(r'<a[^>]*href="([^"]+)"[^>]*>(.*?)</a>', re.DOTALL)
        for m in a_pattern.finditer(html):
            url = m.group(1)
            title = re.sub(r'<[^>]+>', '', m.group(2)).strip()
            if title and len(title) > 10 and url not in ['#', 'javascript:void(0)']:
                if not url.startswith('http'):
                    base_url = list_url.rsplit('/', 1)[0] if '/' in list_url else list_url
                    url = base_url + '/' + url.lstrip('/')
                date = ""
                date_m = re.search(r'(\d{4}[-/]\d{1,2}[-/]\d{1,2})', html[html.find(title)-200:html.find(title)+200])
                if date_m:
                    date = date_m.group(1)
                items.append({"url": url, "title": title, "date": date})
                if len(items) >= 10:
                    break
    
    print(f"  📋 列表: {len(items)} 条，取前5")
    
    # 获取基准 URL 用于补全相对地址
    base_url = list_url.rsplit('/', 1)[0] if '/' in list_url else list_url
    
    count = 0
    for i, item in enumerate(items[:5], 1):
        detail_url = item["url"]
        # 补全相对URL
        if not detail_url.startswith('http'):
            from urllib.parse import urljoin
            detail_url = urljoin(list_url, detail_url)
        title = item["title"]
        date = item["date"]
        
        print(f"  [{i}/5] {title[:35]}...", end=" ")
        sys.stdout.flush()
        
        det_html, final_url = fetch(detail_url)
        if not det_html:
            print("❌ 无法获取详情页")
            continue
        
        # 提取正文
        content = ""
        if domain in DETAIL_RULES:
            content = extract_content_by_rules(det_html, domain)
        if not content:
            content = extract_content_generic(det_html)
        
        # 用规则提取标题
        det_title = ""
        if domain in DETAIL_RULES:
            det_title = extract_title_by_rules(det_html, domain)
        
        title_final = det_title or title
        cleaned = clean_text(content) if content else ""
        
        if store(name, domain, title_final, final_url, cleaned, date):
            count += 1
            print(f"✅ {len(cleaned)}B")
        else:
            print("❌ 入库失败")
        
        time.sleep(1)
    
    return count


# ── 同步到服务器 ──

def sync():
    print(f"\n📤 同步到服务器...")
    conn = sqlite3.connect(QUALITY_DB)
    rows = conn.execute("SELECT * FROM quality_results").fetchall()
    conn.close()
    if not rows:
        print("  ⏭️ 无数据")
        return
    
    batch_size = 30
    for i in range(0, len(rows), batch_size):
        batch = rows[i:i+batch_size]
        vals = []
        for r in batch:
            def e(v):
                if v is None: return "NULL"
                return "'" + str(v).replace("'", "''") + "'"
            vals.append(f"({e(r['site_name'])},{e(r['domain'])},{e(r['title'])},{e(r['url'])},{e(r['content'])},{e(r['publish_date'])},{e(r['summary'])})")
        
        sql = f"INSERT OR IGNORE INTO quality_results (site_name,domain,title,url,content,publish_date,summary) VALUES {','.join(vals)};"
        b64 = base64.b64encode(sql.encode()).decode()
        subprocess.run(
            ["ssh", "-o", "StrictHostKeyChecking=no", SERVER_SSH,
             f"echo {b64} | base64 -d | sqlite3 {SERVER_QUALITY_DB}"],
            capture_output=True, timeout=60,
        )
    print(f"  ✅ {len(rows)} 条已同步")


# ── 主程序 ──

if __name__ == "__main__":
    print("="*50)
    print("9个站点快速测试爬虫")
    print("="*50)
    
    init_db()
    
    total = 0
    for name, list_url, domain in SITES:
        n = crawl_one(name, list_url, domain)
        total += n
        print(f"  → {name}: {n}/5")
    
    print(f"\n{'='*50}")
    print(f"✅ 共爬取 {total}/{45} 条")
    
    if total > 0:
        sync()