#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
山东华鲁恒升 - 公示公告 (GSGG)
站点: www.hl-hengsheng.com/GSGG/index.html
CMS: 静态 HTML (附件上传列表)
列表: a[href=../GSGG/2026-08-13/uuid.pdf] > div.news-item (标题 div.text-[#333333], 日期 12/2026-08)
详情: 无详情页 — 纯 PDF 外链公告 (正文 = 标题 + URL 内嵌段)
分页: /GSGG/index.html?page=N (跳页 JS)
用法: python3 crawl_hl-hengsheng_gsgg.py [--pages=N | N]
"""
import re, sys, os, json, html as html_mod
from urllib.parse import urljoin
import requests, sqlite3

SITE_NAME = "华鲁恒升-公示公告"
BASE_URL = "https://www.hl-hengsheng.com"
LIST_URL = "https://www.hl-hengsheng.com/GSGG/index.html"
DOMAIN = "www.hl-hengsheng.com"
GROUP = "企业"
CUTOFF = "2023-01-01"
DB_PATH = os.environ.get("DB_PATH", "/root/search.db")

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
    "Referer": LIST_URL,
}

def clean_title(t):
    t = re.sub(r"·| | |\u200b|\ufeff", "", t or "")
    t = re.sub(r"^[•·]\s*", "", t)
    return re.sub(r"\s+", " ", t).strip()

def fetch(url, retries=3):
    for i in range(retries):
        try:
            resp = requests.get(url, timeout=30, headers=HEADERS, verify=False)
            resp.encoding = "utf-8"
            if resp.status_code == 200:
                return resp.text
        except Exception:
            if i < retries - 1:
                import time; time.sleep(2)
    return None

def extract_list_items(html_text):
    """附件条目: <a href="../GSGG/2026-08-13/uuid.pdf"> <div class="news-item"> <div>12</div><div>2026-08</div> <div class="text-[#333333]">标题</div>"""
    items = []
    if not html_text:
        return items
    seen = set()
    for m in re.finditer(
        r'<a[^>]+href="([^"]*(?:GSGG|gsgg)/20\d{2}-\d{2}-\d{2}/[^"]+\.(?:pdf|doc|docx|xls|xlsx|zip|rar|wps))"[^>]*>(.*?)</a>',
        html_text, re.DOTALL | re.I):
        href, block = m.group(1).strip(), m.group(2)
        if href in seen:
            continue
        seen.add(href)
        # 标题: div.text-[#333333] 或类似 class 含 text-[#333333]
        tm = re.search(r'class="[^"]*text-\[#333333\][^"]*"[^>]*>(.*?)</div>', block, re.DOTALL)
        title = ""
        if tm:
            title = clean_title(re.sub(r"<[^>]+>", "", tm.group(1)))
        if not title:
            title = clean_title(re.sub(r"<[^>]+>", "", block))[:80]
        # 日期: <div>12</div><div>2026-08</div>
        dm = re.search(r'>(\d{1,2})</div>\s*<div[^>]*>(\d{4}-\d{2})</div>', block)
        pub_date = ""
        if dm:
            day, ym = dm.group(1).zfill(2), dm.group(2)
            pub_date = f"{ym}-{day}"
        if not pub_date:
            dm2 = re.search(r'20\d{2}-\d{2}-\d{2}', href)
            if dm2:
                pub_date = dm2.group(0)
        abs_url = href if href.startswith("http") else urljoin(LIST_URL, href)
        items.append({"title": title or os.path.basename(href), "url": abs_url, "pub_date": pub_date, "href_raw": href})
    return items

def _store(it):
    """纯 PDF 外链公告: 正文 = 标题 + URL 内嵌段"""
    fname = os.path.basename(it["url"])
    body = f'<p>{html_mod.escape(it["title"])}</p>\n<p><a href="{it["url"]}">{html_mod.escape(fname)}</a></p>'
    row = {
        "site_name": SITE_NAME,
        "source_url": it["url"],
        "page_url": it["url"],
        "title": it["title"],
        "publish_date": it["pub_date"],
        "content": body,
        "summary": body[:500],
        "category": "环境保护",
        "script_name": os.path.basename(__file__),
    }
    conn = sqlite3.connect(DB_PATH, timeout=60)
    conn.execute("PRAGMA busy_timeout=60000")
    c = conn.cursor()
    try:
        c.execute(
            "INSERT OR IGNORE INTO gov_raw (site_name, source_url, page_url, title, publish_date, content, summary, category, script_name) VALUES (?,?,?,?,?,?,?,?,?)",
            (row["site_name"], row["source_url"], row["page_url"], row["title"], row["publish_date"], row["content"], row["summary"], row["category"], row["script_name"]))
        conn.commit()
    except Exception:
        pass
    conn.close()

def crawl(pages=1):
    stored = skipped = 0
    seen_urls = set()
    for page in range(1, pages + 1):
        if page == 1:
            html_text = fetch(LIST_URL)
        else:
            html_text = fetch(f"{LIST_URL}?page={page}")
        if not html_text:
            break
        items = extract_list_items(html_text)
        if not items:
            break
        for it in items:
            if it["url"] in seen_urls:
                continue
            seen_urls.add(it["url"])
            if it["pub_date"] and it["pub_date"] < CUTOFF:
                continue
            _store(it)
            stored += 1
        print(f"[{SITE_NAME}] P{page}: {len(items)} items", flush=True)
        time.sleep(0.3)
    print(f"[{SITE_NAME}] Result: {stored} stored, {skipped} skipped")

if __name__ == "__main__":
    import time
    _MAX_PG = 1
    for a in sys.argv[1:]:
        if a.startswith("--pages="):
            try:
                _MAX_PG = int(a.split("=", 1)[1])
            except ValueError:
                pass
        elif a.isdigit():
            _MAX_PG = int(a)
    crawl(pages=_MAX_PG)