#!/usr/bin/env python3
"""
晋江市人民政府 - 建设项目环境影响评价 爬虫
URL: https://www.jinjiang.gov.cn/xxgk/zdxxgk/hjbh/jsxmyxpj/
API: POST https://www.jinjiang.gov.cn/ssp/search/api/v2/external
"""

import sys, json, time, requests, sqlite3, re
from bs4 import BeautifulSoup
from urllib.parse import urljoin

BASE_URL = "https://www.jinjiang.gov.cn/xxgk/zdxxgk/hjbh/jsxmyxpj/"
API_URL = "https://www.jinjiang.gov.cn/ssp/search/api/v2/external"
SITE_NAME = "jinjiang"
SEARCH_DB = "/root/temp_search.db"

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
    "Content-Type": "application/json",
    "Referer": "https://www.jinjiang.gov.cn/",
}

def get_detail_url(rel_url):
    return urljoin(BASE_URL, rel_url)

def fetch_list_page(session, page):
    """调用SSP API获取一页列表"""
    payload = {
        "siteId": "000000008fd72c27018fdd69fb8b0002",
        "apiName": "WCMDOCAPI",
        "pageSize": 100,
        "relevanceField": "",
        "relevanceType": "all",
        "sortField": "docorderpri desc,docreltime desc",
        "matchMode": "smart",
        "aggregateField": "publishyear",
        "isCollapse": "Y",
        "page": page,
        "filter": {
            "docstatus": [{"eq": 10}],
            "chnlid": [{"in": ["51723"]}],
            "publishyear": []
        }
    }
    resp = session.post(API_URL, json=payload, timeout=30)
    if resp.status_code != 200:
        print(f"  [ERROR] API page {page}: HTTP {resp.status_code}")
        return [], 0
    
    data = resp.json()
    if not data.get("data"):
        print(f"  [ERROR] API page {page}: no data in response")
        return [], 0
    
    result = data["data"].get("result", [])
    total = int(data["data"].get("total", 0))
    return result, total

def fetch_detail_content(url, session):
    """获取详情页HTML正文"""
    try:
        resp = session.get(url, timeout=30)
        if resp.status_code != 200:
            return "", None, None
        soup = BeautifulSoup(resp.text, 'html.parser')
        content_div = soup.select_one('div.TRS_Editor')
        if not content_div:
            content_div = soup.select_one('div.article_content')
        content = str(content_div) if content_div else ""
        
        meta_title = soup.find('meta', attrs={"name": "ArticleTitle"})
        title = meta_title['content'].strip() if meta_title and meta_title.get('content') else None
        meta_date = soup.find('meta', attrs={"name": "PubDate"})
        date = meta_date['content'].strip() if meta_date and meta_date.get('content') else None
        return content, title, date
    except Exception as e:
        return "", None, None

def main():
    print(f"===== 晋江市-建设项目环境影响评价 爬虫 =====")
    print(f"API: {API_URL}")
    
    session = requests.Session()
    session.headers.update(HEADERS)
    
    # Step 1: 获取所有列表页
    print("\n--- 第1步: 获取列表(API) ---")
    
    # 先获取第一页，知道总数
    page1_results, total = fetch_list_page(session, 1)
    if not page1_results:
        print("API无数据，退出")
        return
    
    print(f"总记录: {total}, 每页100条, 共{(total + 99) // 100}页")
    
    all_items = []
    all_results = page1_results
    print(f"  [OK] 第1页: {len(page1_results)} 条 [{page1_results[-1]['doctitle'][:30]} ~ {page1_results[0]['doctitle'][:30]}]")
    
    total_pages = (total + 99) // 100
    for page in range(2, total_pages + 1):
        results, _ = fetch_list_page(session, page)
        if not results:
            print(f"  [END] 第{page}页: 无数据")
            break
        all_results.extend(results)
        print(f"  [OK] 第{page}页: {len(results)} 条 [{results[-1]['doctitle'][:30]} ~ {results[0]['doctitle'][:30]}]")
        time.sleep(0.5)
    
    print(f"\n共获取: {len(all_results)} 条")
    
    # Step 2: 获取详情页
    print(f"\n--- 第2步: 获取详情页 ---")
    
    conn = sqlite3.connect(SEARCH_DB, timeout=60)
    c = conn.cursor()
    c.execute('''CREATE TABLE IF NOT EXISTS gov_raw (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        title TEXT, content TEXT, source_url TEXT UNIQUE,
        site_name TEXT, publish_date TEXT,
        crawl_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP
    )''')
    
    inserted = 0
    short_count = 0
    skip_count = 0
    
    for idx, item in enumerate(all_results):
        # 从API结果提取信息
        title = item.get("doctitle", "")
        # docreltime是Unix毫秒时间戳
        ts = int(item.get("docreltime", 0)) / 1000
        import datetime
        date = datetime.datetime.fromtimestamp(ts).strftime("%Y-%m-%d") if ts > 0 else ""
        
        # 详情页URL
        detail_url = item.get("docpuburl", "")
        if not detail_url:
            skip_count += 1
            continue
        
        # 确保URL完整
        if detail_url.startswith("./"):
            detail_url = urljoin(BASE_URL, detail_url[2:])
        elif not detail_url.startswith("http"):
            detail_url = urljoin(BASE_URL, detail_url)
        
        print(f"  [{idx+1}/{len(all_results)}] {date} {title[:40]}...", end=" ")
        sys.stdout.flush()
        
        content, det_title, det_date = fetch_detail_content(detail_url, session)
        if not content or len(content.strip()) < 200:
            content = content or ""
            print(f"SHORT({len(content.strip())})")
            short_count += 1
        else:
            print(f"OK({len(content.strip())})")
        
        final_title = det_title or title
        final_date = det_date or date
        
        try:
            c.execute("INSERT OR IGNORE INTO gov_raw (title, content, source_url, site_name, publish_date) VALUES (?, ?, ?, ?, ?)",
                      (final_title, content, detail_url, SITE_NAME, final_date))
            if c.rowcount > 0:
                inserted += 1
        except Exception as e:
            print(f"    [DB ERROR] {e}")
    
    conn.commit()
    conn.close()
    
    print(f"\n{'='*50}")
    print(f"完成！总计: {inserted} 条入库 (跳过{skip_count}条无URL)")
    print(f"有内容: {len(all_results)-short_count-skip_count} 条, SHORT: {short_count} 条")
    print(f"{'='*50}")

if __name__ == "__main__":
    main()