#!/usr/bin/env python3
"""北海经济开发区 — 建设项目环境影响评价
http://www.sdbh.gov.cn/col/col197145/index.html?vc_xxgkarea=11371643004387601P-1&number=BH0230208
大汉版通CMS，POST分页，20条/页
"""
import os, sys, re, time
import requests
from bs4 import BeautifulSoup
from datetime import datetime, timedelta
from urllib.parse import urljoin

DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
BASE_URL = "http://www.sdbh.gov.cn"
SITE_NAME = "北海经济开发区-建设项目环境影响评价"
THRESHOLD = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
SEARCH_URL = "http://www.sdbh.gov.cn/module/xxgk/search.jsp"

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}
session = requests.Session()
session.headers.update(HEADERS)
session.verify = False
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
import sqlite3

def store_item(title, date_str, content, page_url):
    try:
        conn = sqlite3.connect(DB_PATH, timeout=10)
        c = conn.cursor()
        summary = (content.strip()[:200] if content.strip() else '')
        date_rank = 0
        if date_str:
            try:
                date_rank = int(datetime.strptime(date_str[:10], "%Y-%m-%d").timestamp())
            except:
                pass
        c.execute("""INSERT OR IGNORE INTO gov_raw (title, publish_date, content, page_url, source_url, site_name, summary, date_rank)
                     VALUES (?,?,?,?,?,?,?,?)""",
                  (title.strip(), date_str, content.strip(), page_url.strip(), page_url.strip(), SITE_NAME, summary, date_rank))
        a = c.rowcount; conn.commit(); conn.close(); return a
    except Exception as e:
        print(f"[ERROR] 写入失败: {e}"); return 0

def fetch(url, max_retries=3, post_data=None):
    for i in range(max_retries):
        try:
            if post_data:
                r = session.post(url, data=post_data, timeout=15)
            else:
                r = session.get(url, timeout=15)
            r.encoding = "utf-8"
            if r.status_code == 200:
                return r.text
        except Exception as e:
            print(f"[WARN] {url} 请求失败: {e}")
        if i < max_retries - 1:
            time.sleep(2)
    return None

def fetch_list_page(page_num):
    """Fetch a list page via POST"""
    post_data = {
        "divid": "div195978",
        "infotypeId": "BH0230208",
        "jdid": "457",
        "area": "",
        "sortfield": "createdatetime:0",
        "standardXxgk": "1",
    }
    if page_num > 1:
        post_data["currpage"] = str(page_num)
    html = fetch(SEARCH_URL, post_data=post_data)
    return html

def parse_list_page(html):
    """解析列表页，返回 [(title, date_str, detail_url), ...]"""
    soup = BeautifulSoup(html, "html.parser")
    items = []
    zd = soup.find("div", class_=lambda c: c and "zfxxgk_zdgkc" in str(c))
    if not zd:
        return items
    for li in zd.find_all("li"):
        a = li.find("a", href=True)
        b = li.find("b")
        if not a:
            continue
        title = a.get_text(strip=True)
        href = a["href"]
        if not title or not href:
            continue
        full_url = urljoin(BASE_URL, href)
        date_str = b.get_text(strip=True) if b else ""
        items.append((title, date_str, full_url))
    return items

def parse_detail_page(url):
    html = fetch(url)
    if not html:
        return None, None, None
    soup = BeautifulSoup(html, "html.parser")
    
    # 标题
    title = ""
    h1 = soup.find("h1")
    if h1:
        title = h1.get_text(strip=True)
    if not title:
        t = soup.find("title")
        if t:
            title = t.get_text(strip=True).split("_")[0].strip()
    
    # 日期
    date_str = ""
    m = re.search(r'发布日期[：:]?\s*(\d{4}-\d{1,2}-\d{1,2})', soup.get_text())
    if m:
        date_str = m.group(1)
    
    # 正文 - 从 div.content 提取 <p> 标签
    content_parts = []
    body = soup.find("div", class_=lambda c: c and "content" in str(c))
    if body:
        for p in body.find_all("p"):
            txt = p.get_text(strip=True)
            if txt and len(txt) > 3:
                skip_words = ["扫一扫", "分享到", "上一篇", "下一篇", "责任编辑", "编辑:"]
                if any(w in txt for w in skip_words):
                    continue
                content_parts.append(txt)
    
    content = "\n\n".join(content_parts)
    
    # 附件
    attachments = []
    if body:
        for a in body.find_all("a", href=True):
            h = a["href"].lower()
            if re.search(r'\.(pdf|doc|docx|xls|xlsx|zip|rar|ppt|pptx)$', h):
                fname = a.get_text(strip=True) or os.path.basename(a["href"])
                full_url = urljoin(url, a["href"])
                attachments.append(f"[{fname}]({full_url})")
    if attachments:
        if content:
            content += "\n\n---\n**附件：**\n" + "\n".join(attachments)
        else:
            content = "\n".join(attachments)
    
    return title, date_str, content

def main():
    print(f"[START] {SITE_NAME} 阈值: {THRESHOLD}")
    total_new = 0
    total_skip = 0
    total_err = 0
    
    for pg in range(1, 9):  # pages 1-8
        print(f"\n[PAGE] 第{pg}页")
        html = fetch_list_page(pg)
        if not html:
            print(f"  [ERR] 获取失败")
            break
        
        items = parse_list_page(html)
        if not items:
            print(f"  [END] 无条目")
            break
        
        print(f"  [LIST] {len(items)}条")
        
        has_recent = False
        for title, date_str, detail_url in items:
            if date_str and date_str < THRESHOLD:
                print(f"  [SKIP] {date_str} {title[:40]}")
                total_skip += 1
                continue
            has_recent = True
            
            # Handle direct PDF links
            if detail_url.lower().endswith('.pdf'):
                st = store_item(title, date_str or "", f"[查看PDF]({detail_url})", detail_url)
                if st:
                    total_new += 1
                    print(f"  [PDF] {date_str} {title[:40]}")
                else:
                    total_err += 1
                continue
            
            t, d, c = parse_detail_page(detail_url)
            if t and c:
                final_date = d or date_str or ""
                st = store_item(t, final_date, c, detail_url)
                if st:
                    total_new += 1
                    print(f"  [OK] {final_date} {t[:40]}")
                else:
                    total_err += 1
                    print(f"  [DUP] {final_date} {t[:40]}")
            else:
                total_err += 1
                print(f"  [ERR] {d or date_str or ''} {title[:40]} 解析失败 {detail_url}")
            time.sleep(0.3)
        
        if not has_recent:
            print(f"  [STOP] 本页无阈值内条目")
            break
        time.sleep(0.5)
    
    print(f"\n[DONE] 新增: {total_new}, 跳过: {total_skip}, 失败: {total_err}")

if __name__ == "__main__":
    main()