#!/usr/bin/env python3
"""金能化学（青岛）有限公司 - 新闻 爬虫
URL: http://jinnengchem.com/news.asp?Sortid=1
分页: news.asp?Sortid=1&Page=N (5页)
详情: new.asp?Sortid=1&id=NNN → div.detail_new_text
"""

import sys
import re
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin
from datetime import datetime, timedelta

BASE_URL = "http://jinnengchem.com"
LIST_URL = "http://jinnengchem.com/news.asp?Sortid=1"
COLUMN = "新闻"
SITE = "金能化学"
PROVINCE = "企业"

HEADERS = {"User-Agent": "Mozilla/5.0"}
session = requests.Session()
session.headers.update(HEADERS)


def log(msg):
    sys.stderr.write(msg + "\n")
    sys.stderr.flush()


def extract_list(page_url):
    """Extract (title, url, date) from a list page."""
    try:
        resp = session.get(page_url, timeout=15)
        resp.encoding = "utf-8"
    except Exception as e:
        log(f"  [ERROR] {page_url} - {e}")
        return []

    soup = BeautifulSoup(resp.text, "html.parser")
    items = []
    for li in soup.select("ul > li"):
        link = li.select_one("a[href^='new.asp']")
        if not link:
            continue
        href = link.get("href", "").strip()
        url = urljoin(BASE_URL, href)

        # Title from <p> inside <a>
        title = ""
        p_el = link.select_one("p")
        if p_el:
            title = p_el.get_text(strip=True)
        if not title:
            title = link.get("title", "") or link.get_text(strip=True)
        if not title:
            continue

        # Date from <div class="time"><h4>DD</h4><h5>YYYY-M</h5></div>
        date_str = ""
        time_div = li.select_one("div.time")
        if time_div:
            day_el = time_div.select_one("h4")
            ym_el = time_div.select_one("h5")
            if day_el and ym_el:
                day = day_el.get_text(strip=True).zfill(2)
                ym = ym_el.get_text(strip=True).strip()
                try:
                    parts = ym.split("-")
                    if len(parts) == 2:
                        date_str = f"{parts[0]}-{parts[1].zfill(2)}-{day}"
                except:
                    pass

        items.append({"title": title, "url": url, "date": date_str})
    return items


def extract_detail(detail_url):
    """Extract content from detail page."""
    try:
        resp = session.get(detail_url, timeout=15)
        resp.encoding = "utf-8"
    except Exception as e:
        log(f"  [ERROR] {detail_url} - {e}")
        return "", ""

    soup = BeautifulSoup(resp.text, "html.parser")
    # Get full title: prefer detail_new_text first strong text,
    # fallback to div.subtitle h4
    title = ""
    content_div = soup.select_one("div.detail_new_text")

    # Primary: div.subtitle h4
    subtitle_h4 = soup.select_one("div.subtitle h4")
    if subtitle_h4:
        title = subtitle_h4.get_text(strip=True)

    # Override: if content has a first strong that's longer than h4 (h4 was truncated)
    if content_div and title:
        first_strong = content_div.select_one("strong")
        if first_strong:
            st = first_strong.get_text(strip=True)
            if len(st) > len(title):
                title = st

    # Fallback: no h4 found, use strong text
    if not title and content_div:
        strong_els = content_div.find_all("strong")
        strong_texts = [s.get_text(strip=True) for s in strong_els if s.get_text(strip=True) and len(s.get_text(strip=True)) > 5]
        # Only take first 2-3 strong texts to avoid content leakage
        title = " ".join(strong_texts[:3])

    content = ""
    if content_div:
        # Insert paragraph markers at block-level boundaries
        for p_tag in content_div.find_all(["p", "h1", "h2", "h3", "h4", "h5", "h6"]):
            p_tag.append("\u00b6P\u00b6")
        for div_tag in content_div.find_all("div", recursive=False):
            if div_tag.get_text(strip=True):
                div_tag.append("\u00b6P\u00b6")
        for br in content_div.find_all("br"):
            br.replace_with("\u00b6P\u00b6")
        for tag in content_div.find_all(["span", "b", "strong", "font", "em", "i"]):
            tag.unwrap()
        content = content_div.get_text(separator="", strip=True)
        content = content.replace("\u00b6P\u00b6", "\n\n")
        content = re.sub(r"\n{3,}", "\n\n", content)
        content = content.strip()

    return title, content


def crawl_all(months_back=36):
    """Full crawl."""
    cutoff = datetime.now() - timedelta(days=months_back * 30) if months_back else None
    all_items, seen_urls = [], set()

    for page in range(1, 6):
        url = f"{BASE_URL}/news.asp?Sortid=1&Page={page}" if page > 1 else LIST_URL
        items = extract_list(url)
        if not items:
            continue
        new_count = 0
        for item in items:
            if item["url"] not in seen_urls:
                seen_urls.add(item["url"])
                all_items.append(item)
                new_count += 1
        log(f"  [LIST] page {page}/5 → +{new_count}")

    log(f"\n共 {len(all_items)} 条待爬")

    for idx, item in enumerate(all_items, 1):
        title, content = extract_detail(item["url"])
        if title:
            item["title"] = title
        item["content"] = content or f"[{item['title']}]({item['url']})"
        output_item(item, idx)

    log(f"\n[DONE] 共爬取 {len(all_items)} 条")


def crawl_incremental():
    items = extract_list(LIST_URL)
    log(f"[LIST] → {len(items)} 条")
    for idx, item in enumerate(items, 1):
        title, content = extract_detail(item["url"])
        if title:
            item["title"] = title
        item["content"] = content or f"[{item['title']}]({item['url']})"
        output_item(item, idx)
    log(f"\n[DONE] 增量 {len(items)} 条")


def output_item(item, idx):
    import json
    print(json.dumps({
        "title": item.get("title", ""),
        "page_url": item.get("url", ""),
        "publish_date": item.get("date", ""),
        "content": item.get("content", ""),
        "site_name": f"{SITE}-{COLUMN}",
        "column": COLUMN,
        "province": PROVINCE,
    }, ensure_ascii=False))


if __name__ == "__main__":
    mode = sys.argv[1] if len(sys.argv) > 1 else "incremental"
    if mode == "incremental":
        crawl_incremental()
    elif mode == "full":
        crawl_all(int(sys.argv[2]) if len(sys.argv) > 2 else 36)
    elif mode == "list":
        items = extract_list(LIST_URL)
        log(f"共 {len(items)} 条")
        for it in items[:5]:
            log(f"  {it['date']} | {it['title'][:40]} | {it['url']}")
    else:
        log(f"Usage: {sys.argv[0]} [incremental|full [months]]")