#!/usr/bin/env python3
"""平阴县环境保护爬虫 (API版 - 政府信息公开平台)"""
import sys, re, os, json
sys.path.insert(0, '/root')
sys.path.insert(0, '/root/gov_crawler')
from urllib.parse import urljoin
import urllib.request, ssl
from bs4 import BeautifulSoup

SITE_NAME = "平阴县环境保护"
API_URL = "http://www.pingyin.gov.cn/gongkai/api/mdocument"
CHANNEL_ID = "6389a48a3759918282639b4d"
PAGES = 5
PER_PAGE = 50

ssl_ctx = ssl.create_default_context()
ssl_ctx.check_hostname = False
ssl_ctx.verify_mode = ssl.CERT_NONE


def html_table_to_html(table, base_url=""):
    """保留 HTML 表格结构，仅将相对链接/图片转绝对 URL"""
    import urllib.parse
    tbl = BeautifulSoup(str(table), 'html.parser')
    for a in tbl.find_all('a'):
        href = a.get('href', '')
        if href and not href.startswith(('http', 'javascript', '#')):
            a['href'] = urllib.parse.urljoin(base_url, href) if base_url else href
    for img in tbl.find_all('img'):
        src = img.get('src', '')
        if src and not src.startswith(('http', '//', 'data:')):
            img['src'] = urllib.parse.urljoin(base_url, src) if base_url else src
    return str(tbl)
def fetch_page(page_num):
    """调用API获取列表页数据"""
    data = json.dumps({
        "method": "mList",
        "params": {
            "channel_name": CHANNEL_ID,
            "and": [],
            "page": page_num,
            "per_page": PER_PAGE
        }
    }).encode()
    
    req = urllib.request.Request(API_URL, data=data, headers={
        "User-Agent": "Mozilla/5.0",
        "Content-Type": "application/json",
        "Authorization": "Bearer",
        "X-Requested-With": "XMLHttpRequest",
    })
    resp = urllib.request.urlopen(req, timeout=30, context=ssl_ctx)
    return json.loads(resp.read().decode("utf-8"))


def clean_content(html_content):
    """将API返回的HTML正文转为Markdown格式"""
    if not html_content:
        return ""
    soup = BeautifulSoup(html_content, 'html.parser')
    parts = []
    
    for el in soup.find_all(['p', 'table', 'img']):
        if el.name == 'p':
            if el.find_parent('table'):
                continue
            txt = el.get_text(' ', strip=True)
            txt = re.sub(r'\s+', ' ', txt)
            txt = re.sub(r'(?<=[\u4e00-\u9fff])[ \t\u3000]+(?=[\u4e00-\u9fff])', '', txt)
            if txt:
                parts.append(txt)
        elif el.name == 'table':
            tbl_html = html_table_to_html(el, "")
            if tbl_html:
                parts.append(tbl_html)
        elif el.name == 'img':
            src = el.get('src', '')
            alt = el.get('alt', '图片')
            if src and not src.startswith('data:'):
                full_src = urljoin("http://www.pingyin.gov.cn", src)
                parts.append(f'![{alt}]({full_src})')
    
    return '\n\n'.join(parts)


def extract_attachments(item):
    """从relevance字段提取附件"""
    attachments = []
    rel = item.get('relevance', {})
    for key in ['files', 'pdfs', 'words', 'documents']:
        for att in rel.get(key, []):
            if isinstance(att, dict):
                name = att.get('name', '')
                url = att.get('url', '')
                if url and not url.startswith('http'):
                    url = urljoin("http://www.pingyin.gov.cn", url)
                if name and url:
                    attachments.append({"name": name, "url": url})
    return attachments


def main():
    all_items = []
    print(f"=== {SITE_NAME} ===")
    
    for page in range(1, PAGES + 1):
        print(f"\n[Page {page}/{PAGES}]")
        try:
            result = fetch_page(page)
            items = result.get('items', [])
            print(f"  items: {len(items)}, total: {result.get('total', '?')}")
            all_items.extend(items)
        except Exception as e:
            print(f"  [ERROR] {e}")
    
    print(f"\n=== API返回: {len(all_items)} 条 ===")
    
    # 去重
    seen_ids = set()
    unique_items = []
    for item in all_items:
        iid = item.get('_id', '')
        if iid not in seen_ids:
            seen_ids.add(iid)
            unique_items.append(item)
    print(f"去重后: {len(unique_items)} 条")
    
    # 转换格式
    db_items = []
    for item in unique_items:
        title = re.sub(r'\s+', ' ', item.get('title', '')).strip()
        # 日期: 优先 release, 然后 _createtime
        date_str = item.get('release', '') or item.get('_createtime', '')
        if date_str:
            date_str = date_str[:10]
        
        # 来源
        source = ''
        froms = item.get('froms', {})
        if isinstance(froms, dict):
            source = froms.get('name', '')
        if not source:
            source = item.get('organization', '') or item.get('author', '')
        
        # 正文
        html_content = item.get('content', '')
        content = clean_content(html_content)
        summary = re.sub(r'\s+', '', content)[:200] if content else ''
        
        # 附件
        attachments = extract_attachments(item)
        attachment_json = json.dumps(attachments, ensure_ascii=False) if attachments else ''
        
        # 内嵌附件链接到正文
        if attachments:
            if content:
                content += '\n\n'
            content += '\n'.join(f'[{a["name"]}]({a["url"]})' for a in attachments)
        
        # URL
        url = item.get('router', '')
        if not url:
            url = f"http://www.pingyin.gov.cn/gongkai/site/doc_{item.get('_id', '')}.html"
        
        db_items.append({
            'site_name': SITE_NAME,
            'source_url': url,
            'url': url,
            'title': title,
            'pub_date': date_str,
            'summary': summary,
            'content': content,
            'attachments': attachment_json,
        })
    
    # 入库
    try:
        from crawler_lib import push_to_searchdb
        push_to_searchdb(db_items, SITE_NAME)
        print(f"\n✅ 入库完成: {len(db_items)} 条")
    except Exception as e:
        print(f"\n❌ 入库异常: {e}")
        import traceback
        traceback.print_exc()
        with open(f'/root/pingyin_hjbh.json', 'w', encoding='utf-8') as f:
            json.dump(db_items, f, ensure_ascii=False, indent=2)
        print(f"  数据已保存到 /root/pingyin_hjbh.json")


if __name__ == '__main__':
    main()