#!/usr/bin/env python3
"""绥化绿色环境网 — 环保项目公示
http://www.shlshj.com/col.jsp?id=121
凡科建站(FKW) CMS, m561pageno=N 分页
"""
import os, sys, re, time
import requests
from bs4 import BeautifulSoup
from datetime import datetime, timedelta
from urllib.parse import urljoin

DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
BASE_URL = "http://www.shlshj.com/col.jsp?id=121"
SITE_NAME = "绥化绿色环境网-环保项目公示"
THRESHOLD = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
MAX_PAGES = 14

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}
session = requests.Session()
session.headers.update(HEADERS)
import sqlite3

def store_item(title, date_str, content, page_url):
    try:
        conn = sqlite3.connect(DB_PATH, timeout=10)
        c = conn.cursor()
        summary = (content.strip()[:200] if content.strip() else '')
        date_rank = 0
        if date_str:
            try:
                date_rank = int(datetime.strptime(date_str[:10], "%Y-%m-%d").timestamp())
            except:
                pass
        c.execute("""INSERT OR IGNORE INTO gov_raw (title, publish_date, content, page_url, source_url, site_name, summary, date_rank)
                     VALUES (?,?,?,?,?,?,?,?)""",
                  (title.strip(), date_str, content.strip(), page_url.strip(), page_url.strip(), SITE_NAME, summary, date_rank))
        a = c.rowcount; conn.commit(); conn.close(); return a
    except Exception as e:
        print(f"[ERROR] 写入失败: {e}"); return 0

def parse_list_page(html):
    soup = BeautifulSoup(html, "html.parser")
    items = []
    for a in soup.select("a.fk-newsListTitle"):
        href = a.get("href", "")
        title = a.get_text(strip=True)
        if not href or not title or len(title) < 5:
            continue
        # Get full URL
        full_url = href if href.startswith("http") else urljoin("http://www.shlshj.com", href)
        # Find the paired date (next sibling a.fk-newsListDate or parent then find date)
        date_str = ""
        parent_li = a.find_parent("li") or a.find_parent("div", class_=re.compile(r"J_module"))
        if parent_li:
            d_a = parent_li.find("a", class_="fk-newsListDate")
            if d_a:
                date_str = d_a.get_text(strip=True)
        # Try finding by looking for date anchor near this link
        if not date_str:
            for sibling in a.find_all_next("a", class_="fk-newsListDate", limit=1):
                date_str = sibling.get_text(strip=True)
        items.append((title, date_str, full_url))
    return items

def parse_detail_page(html, page_url):
    soup = BeautifulSoup(html, "html.parser")

    # Title from <title>
    title = ""
    tt = soup.find("title")
    if tt:
        title = tt.get_text(strip=True).replace(" - 绥化绿色环境网", "")

    # Date: <span class="newsInfo">发表时间：YYYY-MM-DD
    date_str = ""
    ni = soup.find("span", class_="newsInfo")
    if ni:
        m = re.search(r"(\d{4}-\d{1,2}-\d{1,2})", ni.get_text())
        if m: date_str = m.group(1)
    if not date_str:
        for m in re.finditer(r"(\d{4}-\d{1,2}-\d{1,2})", html[:5000]):
            date_str = m.group(1)

    # Content from richContent0
    body_parts = []
    rc = soup.find("div", class_="richContent0")
    if rc:
        for p in rc.find_all("p"):
            t = p.get_text(strip=True)
            if t and len(t) > 3:
                body_parts.append(t)
        for table in rc.find_all("table"):
            rows = []
            for tr in table.find_all("tr"):
                cells = [td.get_text(strip=True) for td in tr.find_all(["td", "th"])]
                if cells: rows.append(" | ".join(cells))
            if rows: body_parts.append("表格:\n" + "\n".join(rows))
    else:
        # Fallback: any p tags with substantial text
        for p in soup.find_all("p"):
            t = p.get_text(strip=True)
            if t and len(t) > 5:
                body_parts.append(t)

    # Dedup
    seen = set()
    unique = []
    for part in body_parts:
        if part not in seen: seen.add(part); unique.append(part)
    content = "\n\n".join(unique)

    # Attachments
    atts = []
    for a in soup.find_all("a", href=True):
        h = a["href"]
        if re.search(r"\.(pdf|doc|docx|xls|xlsx|zip|rar|caj)$", h, re.I):
            t = a.get_text(strip=True) or "附件"
            fu = h if h.startswith("http") else urljoin("http://www.shlshj.com", h)
            link = f"[{t}]({fu})"
            if link not in atts: atts.append(link)

    if atts:
        if content:
            content += "\n\n---\n附件：\n" + "\n".join(atts)
        else:
            content = "附件：\n" + "\n".join(atts)

    return title, date_str, content

def main():
    print(f"[INFO] 开始爬取: {SITE_NAME}")
    print(f"[INFO] 阈值: {THRESHOLD}, 最大页数: {MAX_PAGES}")
    total_new, stop = 0, False
    for page_num in range(1, MAX_PAGES + 1):
        if stop: break
        if page_num == 1:
            page_url = BASE_URL
        else:
            page_url = f"http://www.shlshj.com/col.jsp?m561pageno={page_num}&id=121"
        if page_num > 1:
            time.sleep(1.0)
        try:
            r = session.get(page_url, timeout=30)
            r.encoding = "utf-8"
        except Exception as e:
            print(f"[ERROR] 请求失败: {e}"); time.sleep(2); continue
        if r.status_code != 200:
            print(f"[PAGE] {page_num}: HTTP {r.status_code}, 停止"); break
        items = parse_list_page(r.text)
        if not items:
            print(f"[PAGE] {page_num}: 无数据, 停止"); break
        print(f"[PAGE] 第{page_num}页: {len(items)}条 [{items[0][1]}] {items[0][0][:35]}")
        for title, date_str, detail_url in items:
            if date_str and date_str < THRESHOLD:
                print(f"  [SKIP] {date_str} 超阈值: {title[:30]}"); continue
            time.sleep(0.5)
            try:
                rd = session.get(detail_url, timeout=30)
                rd.encoding = "utf-8"
            except:
                print(f"  [!] 请求失败: {detail_url[:60]}"); continue
            if rd.status_code != 200: continue
            dt, dd, dc = parse_detail_page(rd.text, detail_url)
            ft = dt if dt else title
            fd = dd or date_str
            affected = store_item(ft, fd, dc, detail_url)
            if affected > 0:
                total_new += 1
                print(f"  [+] {ft[:35]}...({fd})")
            else:
                print(f"  [-] 已存在")
    print(f"\n=== 完成, 新增: {total_new} ===")

if __name__ == "__main__":
    main()