#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
唐山三友化工股份有限公司 - 公司公告 (channels/1738.html) 爬虫
列表: http://www.sanyou-chem.com.cn/sygfx/channels/1738.html
  GB2312 编码, 每页9条
  列表结构: <div class="newstitle"><a href="PDF_URL" target="_blank">标题</a></div><div class="newsdate">日期</div>
  公告为纯 PDF 外链 (static.sse.com.cn 上交所披露平台), 无详情页
  分页: 1738.html / 1738_2.html ~ 1738_148.html
正文格式(用户要求): 标题 + URL 内嵌
  <p>公告标题</p><p><a href="PDF完整URL">PDF名称</a></p>
用法: python3 crawl_sanyou_gsgg.py [--pages=N] [--start=N]
"""
import os, sys, re, time, json
import requests
from urllib.parse import urljoin

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

BASE_URL = "http://www.sanyou-chem.com.cn"
SITE_NAME = "唐山三友化工-公司公告"
CATEGORY = "公司公告"
LIST_URL = BASE_URL + "/sygfx/channels/1738.html"
PAGE_URL = BASE_URL + "/sygfx/channels/1738_{}.html"
MAX_PAGES = 148

UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"
HEADERS = {"User-Agent": UA, "Accept-Language": "zh-CN,zh;q=0.9"}

LIST_RE = re.compile(
    r'<div class="newstitle">\s*<a href="([^"]+)"[^>]*>(.*?)</a>\s*</div>\s*<div class="newsdate">\s*([^<]+?)\s*</div>',
    re.S
)


def parse_args():
    pages = 0
    start = 1
    for a in sys.argv[1:]:
        if a.startswith("--pages="):
            pages = int(a.split("=")[1])
        elif a.startswith("--start="):
            start = int(a.split("=")[1])
        elif a.isdigit():
            pages = int(a)
    return pages, start


def clean_title(t):
    t = re.sub(r"\s+", " ", t or "")
    t = re.sub(r"^[•·\-\s]+", "", t)
    return t.strip()


def fetch(session, url):
    try:
        r = session.get(url, timeout=25)
        if r.status_code != 200:
            return ""
        return r.content.decode("gbk", errors="ignore")
    except Exception:
        return ""


def parse_list(html):
    items = []
    for m in LIST_RE.finditer(html):
        href = m.group(1).strip()
        title = clean_title(re.sub(r"<[^>]+>", "", m.group(2)))
        date = m.group(3).strip()
        if not title or not href:
            continue
        if not href.startswith("http"):
            href = urljoin(BASE_URL, href)
        # 只保留 PDF 外链公告
        if not href.lower().endswith(".pdf"):
            continue
        date = date.strip()
        # 日期格式: 2026-04-25
        dm = re.search(r"(\d{4})-(\d{2})-(\d{2})", date)
        if dm:
            date = "%s-%s-%s" % (dm.group(1), dm.group(2), dm.group(3))
        items.append({"title": title, "url": href, "date": date})
    return items


def main():
    max_pages, start = parse_args()
    if max_pages == 0:
        max_pages = MAX_PAGES
    print(f"[SANYOU-GSGG] SITE={SITE_NAME} start={start} max_pages={max_pages}", flush=True)
    session = requests.Session()
    session.headers.update(HEADERS)

    all_items = []
    seen = set()
    for pg in range(start, max_pages + 1):
        if pg == 1:
            url = LIST_URL
        else:
            url = PAGE_URL.format(pg)
        html = fetch(session, url)
        if not html or len(html) < 500:
            print(f"  Page {pg}: 空/失败, 停止", flush=True)
            break
        items = parse_list(html)
        if not items:
            print(f"  Page {pg}: 无公告, 停止", flush=True)
            break
        new_count = 0
        for it in items:
            if it["url"] in seen:
                continue
            seen.add(it["url"])
            all_items.append(it)
            new_count += 1
        print(f"  Page {pg}: {len(items)}条(新{new_count}) 累计{len(all_items)}", flush=True)
        time.sleep(0.3)

    print(f"\n共 {len(all_items)} 条, 生成正文(标题+URL内嵌)...", flush=True)
    results = []
    for it in all_items:
        # 正文: 标题 + URL 内嵌 (用户要求格式)
        pdf_name = it["url"].split("/")[-1]
        content = (
            f'<p>{it["title"]}</p>\n'
            f'<p><a href="{it["url"]}">{pdf_name}</a></p>'
        )
        results.append({
            "site_name": SITE_NAME,
            "source_url": it["url"],
            "url": it["url"],
            "title": it["title"],
            "pub_date": it["date"],
            "content": content,
            "summary": it["title"],
            "category": CATEGORY,
            "attachments": json.dumps([{"name": pdf_name, "url": it["url"]}], ensure_ascii=False),
        })

    print(f"  pushing {len(results)} items...", flush=True)
    push_to_searchdb(results, batch_label=SITE_NAME)
    empty = sum(1 for r in results if not (r.get("content") or "").strip())
    dates = [r["pub_date"] for r in results if r["pub_date"]]
    print(f"  DONE. pushed={len(results)} 空正文={empty}", flush=True)
    print(f"  日期范围: {min(dates) if dates else '-'} ~ {max(dates) if dates else '-'}", flush=True)


if __name__ == "__main__":
    main()