#!/usr/bin/env python3
"""新浦化学 - 项目公示 爬虫
奕云企服 CMS，静态HTML列表页，PDF附件直链
列表页 /project.html ~ /project/17.html，10条/页，共162条
正文格式: 标题 + 内嵌PDF链接
"""

import os
import sys
import re
import requests
from datetime import datetime, timedelta

DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "spchemicals.com.cn-项目公示"
BASE_URL = "https://www.spchemicals.com.cn"
LIST_URL = f"{BASE_URL}/project.html"
TOTAL_PAGES = 17

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
}

CUTOFF_DATE = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
print(f"[info] 日期过滤: >= {CUTOFF_DATE}")


def fetch_page(page):
    """获取单页列表HTML"""
    if page == 1:
        url = LIST_URL
    else:
        url = f"{BASE_URL}/project/{page}.html"

    try:
        r = requests.get(url, headers=HEADERS, timeout=15)
        r.encoding = "utf-8"
        return r.text
    except Exception as e:
        print(f"[error] 第{page}页请求失败: {e}")
        return ""


def parse_page_items(html):
    """从HTML中解析条目列表"""
    items = []
    # 匹配 <li class="wow..."><a href="...pdf"><p>title</p><span>date</span></a></li>
    # 使用更宽松的匹配
    pattern = r'<li[^>]*>\s*<a href="([^"]+)"[^>]*>\s*<p>([\s\S]*?)</p>\s*<span>([^<]+)</span>\s*</a>\s*</li>'
    for m in re.finditer(pattern, html):
        pdf_url = m.group(1).strip()
        title = m.group(2).strip()
        date = m.group(3).strip()
        if pdf_url.startswith("/"):
            pdf_url = BASE_URL + pdf_url
        items.append({"title": title, "date": date, "pdf_url": pdf_url})
    return items


def make_content(title, pdf_url):
    """正文格式: 标题 + 内嵌PDF链接"""
    return f"""<h3>{title}</h3>
<p>PDF附件: <a href="{pdf_url}" target="_blank">{pdf_url}</a></p>"""


def main():
    import sqlite3

    conn = sqlite3.connect(DB_PATH)
    c = conn.cursor()
    c.execute("PRAGMA journal_mode=WAL")

    new_count = 0
    skip_count = 0
    err_count = 0

    for page in range(1, TOTAL_PAGES + 1):
        html = fetch_page(page)
        if not html:
            continue

        items = parse_page_items(html)
        if not items:
            print(f"[warn] 第{page}页未解析到条目")
            continue

        page_new = 0
        page_skip = 0
        for item in items:
            date = item["date"]
            if date < CUTOFF_DATE:
                page_skip += 1
                continue

            title = item["title"]
            pdf_url = item["pdf_url"]

            # 检查是否已存在（用page_url去重）
            c.execute("SELECT COUNT(*) FROM gov_raw WHERE page_url=?", (pdf_url,))
            if c.fetchone()[0] > 0:
                page_skip += 1
                continue

            # 生成内容: 标题 + 内嵌PDF链接
            content = make_content(title, pdf_url)

            try:
                c.execute("""
                    INSERT OR IGNORE INTO gov_raw (page_url, title, content, site_name, publish_date, summary, category)
                    VALUES (?, ?, ?, ?, ?, ?, ?)
                """, (pdf_url, title, content, SITE_NAME, date, title, "项目公示"))
                if c.rowcount > 0:
                    page_new += 1
            except Exception as e:
                print(f"  [error] 入库失败: {e}")
                err_count += 1

        conn.commit()
        new_count += page_new
        skip_count += page_skip
        print(f"[page {page}/{TOTAL_PAGES}] +{page_new} new, {page_skip} skipped, cumulative: {new_count} new")

    conn.close()
    print(f"\n[完成] 新浦化学: 新增 {new_count} 条, 跳过 {skip_count} 条, 错误 {err_count}")


if __name__ == "__main__":
    main()