#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
吉林省生态环境厅 - 环境影响评价受理公示
TRS CMS, createPageHTML(42, 0, "index", "html")
"""
import re, sys, os, time, sqlite3, requests

SITE_NAME = "吉林省生态环境厅-受理公示"
BASE_URL = "https://sthjt.jl.gov.cn"
LIST_DIR = "/ywdt/gkgs/gszx/slgs"
LIST_BASE = BASE_URL + LIST_DIR
SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
TOTAL_PAGES = 42

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/125.0.0.0 Safari/537.36",
}

def log(msg):
    print(f"[{time.strftime('%H:%M:%S')}] {msg}")

def retry_get(url, max_retries=3):
    for attempt in range(1, max_retries + 1):
        try:
            r = requests.get(url, headers=HEADERS, timeout=30)
            r.encoding = "utf-8"
            return r.text
        except Exception as e:
            if attempt < max_retries:
                time.sleep(attempt * 2)
            else:
                log(f"获取失败: {str(e)[:50]}")
    return None

def fetch_list_page(page):
    """page从1开始"""
    if page == 1:
        url = LIST_BASE + "/"
    else:
        url = LIST_BASE + f"/index_{page-1}.html"
    return retry_get(url)

def parse_articles(html):
    arts = []
    for item in re.findall(r'<li[^>]*>(.*?)</li>', html, re.DOTALL):
        m = re.search(r'<a[^>]*href="([^"]*)"[^>]*title="([^"]*)"', item)
        dm = re.search(r'<span>(\d{4}-\d{2}-\d{2})</span>', item)
        if m and dm:
            url = m.group(1).strip()
            if url.startswith("./"):
                url = LIST_BASE + url[1:]
            elif not url.startswith("http"):
                url = BASE_URL + url if url.startswith("/") else LIST_BASE + "/" + url
            title = m.group(2).strip()
            pub_date = dm.group(1).strip()
            arts.append({"title": title, "url": url, "pub_date": pub_date})
    return arts

def fetch_detail(url):
    html = retry_get(url)
    if not html:
        return "", []
    
    # Content from <div class="mt50" id="content">
    cm = re.search(r'<div class="mt50"[^>]*id="content">(.*?)</div>\s*(?:<style|</div>|<div)', html, re.DOTALL)
    ch = cm.group(1) if cm else ""
    
    # Get plain text from <p> tags
    lines = []
    for p in re.findall(r'<p[^>]*>(.*?)</p>', ch, re.DOTALL):
        t = re.sub(r'<[^>]+>', '', p).strip().replace('&nbsp;', ' ').replace('\u3000', ' ').strip()
        if t and len(t) > 5:
            lines.append(t)
    content = "\n".join(lines)
    
    # PDF attachments (in the table below content)
    atts = []
    for m in re.finditer(r'<a[^>]*href="([^"]*\.pdf)"[^>]*>([^<]*)</a>', html, re.I):
        link = m.group(1)
        if link.startswith("./"):
            link = LIST_BASE + link[1:]
        elif not link.startswith("http"):
            link = LIST_BASE + "/" + link
        atts.append({"url": link, "name": m.group(2).strip() or link.rsplit("/", 1)[-1]})
    
    return content, atts

def main():
    log("开始爬取吉林省生态环境厅受理公示...")
    
    all_arts = []
    for p in range(1, TOTAL_PAGES + 1):
        html = fetch_list_page(p)
        if not html:
            log(f"第{p}页失败，跳过")
            continue
        arts = parse_articles(html)
        all_arts.extend(arts)
        if p == 1 or p % 10 == 0:
            log(f"第{p}页: {len(arts)} 条 (累计{len(all_arts)})")
        time.sleep(0.5)
    
    log(f"列表共 {len(all_arts)} 条")
    
    db = sqlite3.connect(SEARCH_DB, timeout=30)
    db.execute("PRAGMA journal_mode=WAL")
    existing = set()
    try:
        cur = db.execute("SELECT title FROM gov_raw WHERE site_name=?", (SITE_NAME,))
        for row in cur.fetchall():
            existing.add(row[0])
    except:
        pass
    log(f"已有记录: {len(existing)} 条")
    
    saved, skipped = 0, 0
    for art in all_arts:
        if art["title"] in existing:
            skipped += 1
            continue
        try:
            content, atts = fetch_detail(art["url"])
        except:
            content, atts = "", []
        body = ""
        if content:
            body = "<p>" + content.replace("\n", "</p><p>") + "</p>"
        for a in atts:
            body += f'<p><a href="{a["url"]}">{a["name"]}</a></p>'
        try:
            db.execute(
                "INSERT INTO gov_raw (title, content, source_url, publish_date, site_name) VALUES (?,?,?,?,?)",
                (art["title"], body, art["url"], art["pub_date"], SITE_NAME)
            )
            db.commit()
            saved += 1
        except Exception as e:
            log(f"DB写入失败: {art['title'][:30]} {e}")
        if saved % 50 == 0 and saved > 0:
            log(f"进度: {saved} 存, {skipped} 跳")
        time.sleep(0.3)
    
    db.close()
    log(f"完成! 新增 {saved} 条, 跳过 {skipped} 条")

if __name__ == "__main__":
    main()