#!/usr/bin/env python3
"""海南省生态环境厅 - 企业自主环评公示"""
import sys, re, json, sqlite3, requests, warnings
from datetime import datetime, timedelta
from bs4 import BeautifulSoup
import os
warnings.filterwarnings("ignore")

BASE_URL = "http://218.77.183.197:8007"
HOST = "hnsthb.hainan.gov.cn:8007"
API_PATH = "/hbcenter/qyzzHpYsgsController.do"
HEADERS = {
    "Host": HOST,
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
    "Accept": "application/json, text/javascript, */*; q=0.01",
}
SITE_NAME = "海南省生态环境厅-企业自主环评公示"
DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
CUT_DATE = (datetime.now() - timedelta(days=365*3)).strftime("%Y-%m-%d")
PAGE_SIZE = 30

def fetch_json(url, label=""):
    try:
        r = requests.get(url, headers=HEADERS, timeout=20)
        return r.json()
    except Exception as e:
        print(f"[ERROR] {label}: {e}")
        return None

def fetch_html(url, label=""):
    try:
        h = dict(HEADERS)
        h["Accept"] = "text/html,*/*"
        r = requests.get(url, headers=h, timeout=20)
        r.encoding = "utf-8"
        return r.text
    except Exception as e:
        print(f"[ERROR] {label}: {e}")
        return ""

def parse_detail(html, item):
    """从详情页提取标题、日期、正文HTML"""
    soup = BeautifulSoup(html, "lxml")
    title = item.get("title", "") or item.get("xmName", "")
    pub_date = ""
    
    # Date from detail page
    m = re.search(r"发布时间[：:]\s*(\d{4}-\d{2}-\d{2})", html)
    if m:
        pub_date = m.group(1)
    else:
        fb = item.get("fbTime", "")
        if fb:
            pub_date = fb[:10]
    
    # Content: find div after hr (style="margin:0 20px")
    content_div = soup.find("div", style=re.compile(r"margin:0\s*20px"))
    if not content_div:
        # Fallback: find div after hr
        hr = soup.find("hr")
        if hr:
            content_div = hr.find_next_sibling("div")
    
    if content_div:
        # Remove the metadata div (font12right)
        meta = content_div.find("div", class_="font12right")
        if meta:
            meta.extract()
        # Remove <br/> at start
        br = content_div.find("br")
        if br:
            br.extract()
        content_html = str(content_div)
    else:
        content_html = ""
    
    return title, pub_date, content_html

def main(mode="full"):
    start = datetime.now()
    print(f"[海南环评] mode={mode}, cut_date={CUT_DATE}")
    
    api_url = f"{BASE_URL}{API_PATH}?datagrid&field=id,xmName,title,enterpriseName,typeOfPublicity,fbTime&rows=1&page=1&sort=fbTime&order=desc"
    data = fetch_json(api_url, "首请求")
    if not data:
        print("[ERROR] API不可用")
        return
    total = data.get("total", 0)
    print(f"[海南环评] 总量: {total}")
    
    all_items = []
    page = 1
    while True:
        api_url = f"{BASE_URL}{API_PATH}?datagrid&field=id,xmName,title,enterpriseName,typeOfPublicity,fbTime&rows={PAGE_SIZE}&page={page}&sort=fbTime&order=desc"
        data = fetch_json(api_url, f"第{page}页")
        if not data:
            break
        rows = data.get("rows", [])
        if not rows:
            break
        all_items.extend(rows)
        fb_first = rows[0].get("fbTime","?")[:10]
        fb_last = rows[-1].get("fbTime","?")[:10]
        print(f"[海南环评] 第{page}页: {len(rows)}条 ({fb_first} ~ {fb_last})")
        oldest = rows[-1].get("fbTime", "")
        if oldest and oldest[:10] < CUT_DATE:
            print(f"[海南环评] {oldest[:10]} 超过3年线，停止")
            break
        page += 1
    
    print(f"[海南环评] 列表共 {len(all_items)} 条")
    
    conn = sqlite3.connect(DB_PATH)
    c = conn.cursor()
    new, exists, skipped = 0, 0, 0
    
    for idx, item in enumerate(all_items):
        fb_time = item.get("fbTime", "")
        if fb_time and fb_time[:10] < CUT_DATE:
            skipped += 1
            continue
        
        item_id = item["id"]
        detail_url = f"{BASE_URL}{API_PATH}?goGsDetail&id={item_id}"
        page_url = f"http://{HOST}{API_PATH}?goGsDetail&id={item_id}"
        
        html = fetch_html(detail_url, f"详情{idx+1}")
        if not html:
            continue
        
        title, pub_date, content_html = parse_detail(html, item)
        summary = BeautifulSoup(content_html, "lxml").get_text(strip=True)[:200] if content_html else title
        pd = pub_date or fb_time[:10]
        
        # Use REPLACE to update existing records with corrected content
        c.execute("SELECT id FROM gov_raw WHERE page_url=?", (page_url,))
        row = c.fetchone()
        if row:
            c.execute("UPDATE gov_raw SET content=?, summary=?, title=?, publish_date=? WHERE id=?",
                      (content_html, summary, title, pd, row[0]))
            exists += 1
        else:
            c.execute("INSERT INTO gov_raw (title, page_url, site_name, summary, content, publish_date) VALUES (?,?,?,?,?,?)",
                      (title, page_url, SITE_NAME, summary, content_html, pd))
            new += 1
        
        if (idx + 1) % 20 == 0:
            conn.commit()
            el = (datetime.now() - start).total_seconds()
            print(f"[海南环评] {idx+1}/{len(all_items)}, 新增{new}, 已更新{exists}, {el:.0f}s")
    
    conn.commit()
    conn.close()
    el = (datetime.now() - start).total_seconds()
    print(f"[海南环评] 完成! 新增{new}, 已更新{exists}, 跳过旧{skipped}, 耗时{el:.0f}s")

if __name__ == "__main__":
    main(sys.argv[1] if len(sys.argv) > 1 else "full")