#!/usr/bin/env python3
"""伊春新闻网-经济新闻 (yichun.dbw.cn/jjxw)
单页: index.shtml -> div.box10 (GB2312编码)
详情: /system/YYYY/MM/DD/ID.shtml -> div.box10 (第二个)
"""
import sys, os, re, json, time
from datetime import datetime, timedelta
import requests
from bs4 import BeautifulSoup

BASE = "https://yichun.dbw.cn"
SITE = "伊春新闻网-经济新闻"
COLUMN = "经济新闻"
PROVINCE = "企业"

HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
session = requests.Session()
session.headers.update(HEADERS)
MARKER = "\x00P\x00"

def log(msg):
    sys.stderr.write(msg + "\n")
    sys.stderr.flush()

def fetch_list():
    url = f"{BASE}/jjxw/index.shtml"
    try:
        r = session.get(url, timeout=20)
        r.encoding = 'gb2312'
        return r.text
    except Exception as e:
        log(f"  [ERROR] list: {e}")
        return None

def fix_url(href):
    if href.startswith('//'):
        return 'https:' + href
    if href.startswith('http'):
        return href
    return BASE + '/jjxw/' + href.lstrip('./')

def parse_list(html):
    soup = BeautifulSoup(html, "html.parser")
    items = []
    seen = set()
    for div in soup.find_all("div", class_="box10"):
        for a in div.find_all("a", href=True):
            href = a['href']
            if 'system' not in href:
                continue
            title = a.get_text(strip=True)
            if not title or len(title) < 5:
                continue
            full_url = fix_url(href)
            if full_url in seen:
                continue
            seen.add(full_url)
            # Extract date from parent text YY-MM-DD
            parent_text = a.parent.get_text(strip=True) if a.parent else ""
            dm = re.search(r'(\d{2})-(\d{2})-(\d{2})', parent_text)
            date_str = ""
            if dm:
                y, m, d = dm.group(1), dm.group(2), dm.group(3)
                date_str = f"20{y}-{m}-{d}"
            items.append({"title": title, "url": full_url, "date": date_str})
    return items

def extract_content(detail_url, title):
    try:
        r = session.get(detail_url, timeout=20)
        r.encoding = 'gb2312'
    except Exception as e:
        return f"[{title}]({detail_url})", title, ""

    soup = BeautifulSoup(r.text, "html.parser")

    # Get title from page
    h1 = soup.select_one("div.left_title, h1, .article_title")
    if h1:
        t = h1.get_text(strip=True)
        if t:
            title = t

    # Content: second div.box10 (first is navigation)
    box10s = soup.find_all("div", class_="box10")
    content_el = box10s[1] if len(box10s) > 1 else box10s[0] if box10s else None
    if not content_el:
        return f"[{title}]({detail_url})", title, ""

    # Save tables
    tables_html = []
    for table in content_el.find_all("table"):
        tables_html.append(str(table))
        table.decompose()

    # Paragraph markers
    for tag in content_el.find_all(["p", "h1", "h2", "h3", "h4", "h5", "h6"]):
        tag.insert(0, MARKER)
        tag.append(MARKER)
    for br in content_el.find_all("br"):
        br.replace_with(MARKER)

    # Unwrap inline
    for tag in content_el.find_all(["span", "b", "strong", "font", "em", "i", "u", "s"]):
        tag.unwrap()

    text = content_el.get_text(separator="", strip=True)
    text = re.sub(r"\x00P\x00(\s*\x00P\x00)+", "\x00P\x00", text)
    text = text.replace("\x00P\x00", "\n\n")
    text = re.sub(r"\n{3,}", "\n\n", text)
    text = text.strip()

    for tbl_html in tables_html:
        text += f"\n\n{tbl_html}"

    if not text.strip() or len(text.strip()) < 20:
        text = f"[{title}]({detail_url})"

    return text, title, ""

def crawl_all(months_back=0):
    cutoff = datetime.now() - timedelta(days=months_back * 30) if months_back else None
    html = fetch_list()
    if not html:
        return
    items = parse_list(html)
    # Filter by date
    filtered = []
    for item in items:
        if cutoff and item["date"]:
            try:
                d = datetime.strptime(item["date"], "%Y-%m-%d")
                if d < cutoff:
                    continue
            except ValueError:
                pass
        filtered.append(item)
    log(f"[LIST] {len(filtered)} 条 (过滤后)")
    for idx, item in enumerate(filtered, 1):
        content, title, _ = extract_content(item["url"], item["title"])
        if title:
            item["title"] = title
        item["content"] = content
        output_item(item, idx)
        if idx % 10 == 0:
            log(f"  [PROGRESS] {idx}/{len(filtered)}")
    log(f"[DONE] 共爬取 {len(filtered)} 条")

def crawl_incremental():
    crawl_all()

def output_item(item, idx):
    record = {
        "title": item.get("title", ""),
        "page_url": item.get("url", ""),
        "publish_date": item.get("date", ""),
        "content": item.get("content", ""),
        "site_name": SITE,
        "column": COLUMN,
        "province": PROVINCE,
    }
    print(json.dumps(record, ensure_ascii=False))

if __name__ == "__main__":
    mode = sys.argv[1] if len(sys.argv) > 1 else "incremental"
    if mode == "incremental" or mode == "full":
        crawl_all()
    elif mode == "list":
        html = fetch_list()
        items = parse_list(html)
        log(f"共 {len(items)} 条")
        for it in items[:5]:
            log(f"  {it['date']} | {it['title'][:50]} | {it['url']}")
    else:
        log(f"Usage: {sys.argv[0]} [incremental|full|list]")