#!/usr/bin/env python3
import os
"""
隆昌市人民政府 — 环境保护 爬虫
===========================
标准政府CMS列表页模式：
  python3 crawl_longchang.py              # 全量爬取
  python3 crawl_longchang.py --test       # 测试模式（只爬前1页）

数据流：本地JSONL → SSH管道 → 服务器 search.db
"""
import sys, os, re, json, subprocess
from datetime import datetime
from urllib.parse import urljoin

# 添加父目录到路径
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import fetch_page, clean_html as extract_html_content
import sys as _SYS
_MAX_PG = int(_SYS.argv[1]) if len(_SYS.argv) > 1 and _SYS.argv[1].isdigit() else None
if _MAX_PG is not None:
    print('[AutoPg] max_pages=' + str(_MAX_PG))
# END AUTO PAGES

SITE_NAME = "隆昌市人民政府"
DOMAIN = "www.longchang.gov.cn"
BASE_URL = "https://www.longchang.gov.cn"
LIST_URL = "https://www.longchang.gov.cn/lcs/hjbhjc/zfxxgk_gknrz.shtml"
TOTAL_PAGES = 10  # 最多爬10页

def parse_list(html, base_url):
    """解析列表页，返回 [(title, url, date_str), ...]"""
    items = []
    # 查找 <dd><a href="..." title="...">title</a></dd>
    pattern = re.compile(
        r'<dd[^>]*>.*?<a\s+href="([^"]+)"[^>]*title=\'([^\']*)\'[^>]*>',
        re.IGNORECASE | re.DOTALL
    )
    for m in pattern.finditer(html):
        href = m.group(1).strip()
        title = m.group(2).strip()
        full_url = urljoin(base_url, href)
        if title and full_url:
            items.append((title, full_url))
    
    return items

def fetch_detail(url):
    """获取详情页：标题、正文HTML、发布日期"""
    html = fetch_page(url)
    if not html:
        return None, None, None
    
    # 标题
    title = None
    m = re.search(r'<h2[^>]*>(.*?)</h2>', html, re.DOTALL)
    if m:
        title = re.sub(r'<[^>]+>', '', m.group(1)).strip()
    
    # 发布日期
    publish_date = None
    m = re.search(r'发布时间[：:]\s*(\d{4}[-/]\d{1,2}[-/]\d{1,2})', html)
    if m:
        publish_date = m.group(1).replace('/', '-')
    
    # 正文 — 找 content 区域
    content = None
    # 尝试多个容器选择器
    for pattern in [
        r'<!--maincontent.*?-->(.*?)<!--.*?maincontent.*?-->',
        r'class="(?:content|article|main|text|TRS_Editor|news_content|con_content)[^"]*"[^>]*>(.*?)</(?:div|section)>',
        r'<div[^>]*id="[^"]*(?:content|article|main|text|UCAP-CONTENT)[^"]*"[^>]*>(.*?)</div>',
    ]:
        m = re.search(pattern, html, re.DOTALL | re.IGNORECASE)
        if m:
            content = m.group(1).strip()
            break
    
    if not content:
        # 回退：取第一个 h2 之后到 footer/script 之前
        m = re.search(r'<h2[^>]*>.*?</h2>\s*(.*?)(?:<div\s+class="[^"]*foot[^"]*"|<script|<footer)', html, re.DOTALL)
        if m:
            content = m.group(1).strip()
    
    if content:
        content = extract_html_content(content)
    
    return title, content, publish_date


def run(max_pages=None):
    """主爬取函数"""
    if max_pages is None:
        max_pages = TOTAL_PAGES
    
    print(f"🔍 {SITE_NAME} — 爬取 {max_pages} 页")
    
    all_items = []
    
    for page in range(1, min(max_pages, _MAX_PG or max_pages) + 1):
        if page == 1:
            url = LIST_URL
        else:
            url = LIST_URL.replace('.shtml', f'_{page}.shtml')
        
        print(f"  📃 第{page}页: {url}")
        html = fetch_page(url)
        if not html:
            print(f"    ✗ 获取失败")
            break
        
        items = parse_list(html, url)
        if not items:
            print(f"    ➖ 无条目")
            # 如果不是第1页且无数据，停止
            if page > 1:
                break
            continue
        
        print(f"    📄 {len(items)} 条")
        
        for title, page_url in items:
            # 获取详情
            detail_title, content, publish_date = fetch_detail(page_url)
            final_title = detail_title or title
            
            if not content:
                print(f"    ⚠️ [{final_title[:30]}] 没有正文")
                continue
            
            all_items.append({
                'title': final_title,
                'content': content,
                'publish_date': publish_date or '',
                'page_url': page_url,
                'site_name': SITE_NAME
            })
            
            if len(all_items) % 5 == 0:
                print(f"    📊 已收集 {len(all_items)} 条")
    
    print(f"\n📊 共爬取 {len(all_items)} 条有效记录")
    
    if all_items:
        # 直接通过SSH管道推送到服务器
        global PASSWORD, SERVER
        PASSWORD = ""   # 2026-09-10 改用 SSH 密钥认证(~/.ssh/config: hw-backup), 明文已移除
        SERVER = "root@1.94.217.116"
        
        jsonl = "\n".join(json.dumps(it, ensure_ascii=False) for it in all_items)
        tag = f"lc_{int(datetime.now().timestamp())}"
        
        # 传文件到服务器
        proc = subprocess.run(
            ["ssh", "-o", "StrictHostKeyChecking=no", SERVER, f"cat > /tmp/{tag}.jsonl"],
            input=jsonl, capture_output=True, text=True, timeout=30
        )
        if proc.returncode == 0:
            # 导入
            result = subprocess.run(
                f"ssh -o StrictHostKeyChecking=no {SERVER} 'python3 /tmp/import_jsonl.py /tmp/{tag}.jsonl'",
                shell=True, capture_output=True, text=True, timeout=60
            )
            print(f"  {result.stdout.strip()}")
    
    return all_items


if __name__ == '__main__':
    test_mode = '--test' in sys.argv
    if test_mode:
        run(max_pages=1)
    else:
        run()