#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Crawler for 石大胜华新材料集团 - 信息公开 (Enterprise)
CMS: Custom PHP (Yunyou Network)
List: div.newslist > h3 > a[href], p.newslisttime (YYYY.MM.DD)
Pagination: list.php?catid=65&page=N, 6/page, ~29 pages
Detail: div.newsshowc content, title from div.newsshowt h3

Usage:
    python3 crawl_sinodmc_xxgk.py             # Full crawl
    python3 crawl_sinodmc_xxgk.py --max-pages 1  # Incremental
"""

import requests, re, json, sqlite3, time, os, sys
from datetime import datetime
from bs4 import BeautifulSoup
from urllib.parse import urljoin

DB_PATH = "/root/search.db"
SITE_NAME = "石大胜华新材料集团-信息公开"
CATEGORY = "信息公开"
GROUP = "企业-山东"
BASE_URL = "https://www.sinodmc.com"
LIST_URL = "https://www.sinodmc.com/list.php?catid=65"
MAX_PAGES = 29

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}


# ---------- Content extraction ----------

SKIP_TEXTS = [
    "【打印本页】", "【关闭窗口】", "打印本页", "关闭窗口",
    "转载分享：", "浏览量：", "相关附件：", "相关稿件：",
    "扫一扫在手机打开", "附件：", "累计次数：",
]


def render_paragraph(p_tag, detail_url):
    parts = []
    for elem in p_tag.children:
        if isinstance(elem, str):
            t = elem.strip()
            if t:
                parts.append(t)
        elif elem.name == "a":
            href = elem.get("href", "").strip()
            text = elem.get_text(strip=True)
            if href and text:
                full = urljoin(detail_url, href)
                parts.append(f'<p><a href="{full}">{text}</a></p>')
            elif text:
                parts.append(text)
        elif elem.name == "img":
            src = elem.get("src", "")
            if src:
                alt = elem.get("alt", "")
                full_src = urljoin(detail_url, src)
                parts.append(f'<p><a href="{full_src}">查看图片</a></p>' if alt else f'<p><a href="{full_src}">查看图片</a></p>')
        elif elem.name == "br":
            parts.append("\n")
        elif hasattr(elem, "get_text"):
            t = elem.get_text(" ", strip=True)
            if t:
                parts.append(t)
    return " ".join(parts).strip()


def extract_tables_as_html(content_div, detail_url):
    """Extract <table> elements as raw HTML. Remove from content_div."""
    tables = []
    for table in content_div.find_all("table"):
        table_html = str(table)
        if len(table.get_text(strip=True)) >= 15:
            tables.append(table_html)
        table.decompose()
    return tables


def extract_images(content_div, detail_url):
    imgs = []
    for img in content_div.find_all("img"):
        src = img.get("src", "")
        if src:
            full_src = urljoin(detail_url, src)
            alt = img.get("alt", "")
            imgs.append(f'<p><a href="{full_src}">查看图片</a></p>' if alt else f'<p><a href="{full_src}">查看图片</a></p>')
            img.decompose()
    return imgs


def extract_content(content_div, detail_url):
    parts = []

    for s in content_div.find_all(["script", "style"]):
        s.decompose()

    table_htmls = extract_tables_as_html(content_div, detail_url)
    img_lines = extract_images(content_div, detail_url)

    for child in list(content_div.children):
        if child.name is None:
            continue
        tn = child.name.lower()

        if tn == "p":
            # Skip <p> inside <table> (already extracted)
            if child.find_parent("table"):
                continue
            txt = ""
            # Handle spans with mixed content (text + sub-elements)
            for sub in child.children:
                if isinstance(sub, str):
                    t = sub.strip()
                    if t:
                        txt += t + " "
                elif sub.name == "span":
                    # Skip span that contains a table
                    if sub.find("table"):
                        continue
                    t = sub.get_text(" ", strip=True)
                    if t:
                        txt += t + " "
                elif sub.name == "br":
                    txt += "\n"
                elif sub.name == "a":
                    href = sub.get("href", "")
                    text = sub.get_text(strip=True)
                    if href and text:
                        full = urljoin(detail_url, href)
                        txt += f'<p><a href="{full}">{text}</a></p> '
                    elif text:
                        txt += text + " "
                elif sub.name == "img":
                    src = sub.get("src", "")
                    if src:
                        alt = sub.get("alt", "")
                        full_src = urljoin(detail_url, src)
                        txt += f'<p><a href="{full_src}">查看图片</a></p> ' if alt else f'<p><a href="{full_src}">查看图片</a></p> '
                else:
                    t = sub.get_text(" ", strip=True)
                    if t:
                        txt += t + " "

            txt = re.sub(r"\s+", " ", txt).strip()
            if txt and not any(sk in txt for sk in SKIP_TEXTS):
                if not parts or txt != parts[-1]:
                    parts.append(txt)

        elif tn == "div":
            for sub in child.find_all("p", recursive=False):
                txt = sub.get_text(" ", strip=True)
                if txt and not any(sk in txt for sk in SKIP_TEXTS):
                    if not parts or txt != parts[-1]:
                        parts.append(txt)

    result = "\n\n".join(parts)
    if table_htmls:
        result += "\n\n" + "\n\n".join(table_htmls)
    if img_lines:
        result += "\n\n" + "\n\n".join(img_lines)

    return result.strip()


# ---------- Attachments ----------

def extract_attachments(soup, detail_url):
    atts = []
    seen = set()
    for a in soup.find_all("a", href=True):
        href = a["href"]
        if re.search(r"\.(pdf|doc|docx|xls|xlsx|rar|zip|txt)$", href, re.I):
            full_url = urljoin(detail_url, href)
            name = a.get_text(strip=True) or href.split("/")[-1].split("?")[0]
            if full_url not in seen:
                seen.add(full_url)
                atts.append({"name": name, "url": full_url})
    return atts


# ---------- Detail ----------

def fetch_detail(url):
    try:
        r = requests.get(url, headers=HEADERS, timeout=20)
        r.encoding = "utf-8"
    except Exception as e:
        print(f"  fetch_detail error {url}: {e}", file=sys.stderr)
        return None, None, None, []

    soup = BeautifulSoup(r.text, "html.parser")

    # --- Title ---
    title = ""
    h3 = soup.select_one("div.newsshowt h3")
    if h3:
        title = h3.get_text(strip=True)

    # --- Date ---
    date_str = ""
    p_time = soup.select_one("div.newsshowt p")
    if p_time:
        m = re.search(r"时间[：:]\s*(\d{4}-\d{2}-\d{2})", p_time.get_text())
        if m:
            date_str = m.group(1)

    # --- Content ---
    content_div = soup.find("div", class_="newsshowc")

    attachments = extract_attachments(soup, url)

    if content_div:
        content = extract_content(content_div, url)
    else:
        content = ""

    # Image-only fallback
    if len(content.strip()) < 20:
        content = f'<p><a href="{url}">{title}</a></p>'
        imgs = soup.find_all("img")
        img_lines = []
        for img in imgs:
            src = img.get("src", "")
            if src:
                full_src = urljoin(url, src)
                alt = img.get("alt", "")
                img_lines.append(f'<p><a href="{full_src}">查看图片</a></p>' if alt else f'<p><a href="{full_src}">查看图片</a></p>')
        if img_lines:
            content += "\n\n" + "\n\n".join(img_lines)

    # Append attachments
    if attachments:
        for a in attachments:
            inline_marker = f"[{a['name']}]({a['url']})"
            if inline_marker not in content:
                content += f"\n\n附件：[{a['name']}]({a['url']})"

    return title, date_str, content, attachments


# ---------- List ----------

def get_list_url(page_num):
    return f"{LIST_URL}&page={page_num}"


def extract_list_page(page_num):
    url = get_list_url(page_num)
    try:
        r = requests.get(url, headers=HEADERS, timeout=20)
        r.encoding = "utf-8"
    except Exception as e:
        print(f"  Error fetching {url}: {e}", file=sys.stderr)
        return []

    if r.status_code != 200:
        print(f"  Page {page_num}: HTTP {r.status_code}", file=sys.stderr)
        return []

    soup = BeautifulSoup(r.text, "html.parser")
    items = []

    for newsdiv in soup.find_all("div", class_="newslist"):
        a = newsdiv.find("a")
        if not a or not a.get("href"):
            continue

        href = a["href"].strip()
        title = a.get("title", "").strip() or a.get_text(strip=True)
        page_url = urljoin(url, href)

        # Date
        time_p = newsdiv.find("p", class_="newslisttime")
        date_str = time_p.get_text(strip=True) if time_p else ""
        # Normalize date YYYY.MM.DD -> YYYY-MM-DD
        date_str = date_str.replace(".", "-")

        if title and page_url:
            items.append((title, page_url, date_str))

    return items


# ---------- DB ----------

def init_db():
    conn = sqlite3.connect(DB_PATH, timeout=60)
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute("PRAGMA busy_timeout=5000")
    return conn


def save_article(conn, title, page_url, publish_date, content, attachments):
    summary = content if content else title
    summary = re.sub(r"\s+", " ", summary).strip()
    att_json = json.dumps(attachments, ensure_ascii=False) if attachments else "[]"
    try:
        cur = conn.execute(
            """INSERT OR IGNORE INTO gov_raw
               (site_name, source_url, page_url, title, publish_date, summary, content, category, attachments, group_name)
               VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
            (SITE_NAME, page_url, page_url, title.strip(), publish_date,
             summary, content, CATEGORY, att_json, GROUP),
        )
        return cur.rowcount > 0
    except Exception as e:
        print(f"  DB error: {e}", file=sys.stderr)
        return False


# ---------- Main ----------

def main():
    max_pages = MAX_PAGES
    if len(sys.argv) > 1:
        try:
            max_pages = int(sys.argv[1])
        except ValueError:
            if sys.argv[1] in ("--max-pages",) and len(sys.argv) > 2:
                try:
                    max_pages = int(sys.argv[2])
                except ValueError:
                    pass

    print(f"Scraping {SITE_NAME}, max pages: {max_pages}", file=sys.stderr)
    conn = init_db()
    total_new = 0
    total_found = 0

    # Clean old data for this site
    conn.execute("DELETE FROM gov_raw WHERE site_name = ?", (SITE_NAME,))
    conn.commit()
    print(f"  Cleared old data for {SITE_NAME}", file=sys.stderr)

    for pn in range(1, max_pages + 1):
        articles = extract_list_page(pn)
        if not articles:
            print(f"  Page {pn}: 0 articles (end)", file=sys.stderr)
            break

        total_found += len(articles)

        for title, page_url, date in articles:
            if date and date < "2020-01-01":
                print(f"  Skip old: {date} {title[:40]}", file=sys.stderr)
                continue

            # Check if already exists
            cur = conn.execute("SELECT id FROM gov_raw WHERE page_url = ?", (page_url,))
            if cur.fetchone():
                continue

            det_title, det_date, det_content, det_atts = fetch_detail(page_url)
            final_title = det_title or title
            final_date = det_date or date
            final_content = det_content or f'<p><a href="{page_url}">{final_title}</a></p>'
            final_atts = det_atts or []

            if save_article(conn, final_title, page_url, final_date, final_content, final_atts):
                total_new += 1
                if total_new <= 5:
                    print(f"  + {final_date} {final_title[:50]}", file=sys.stderr)

            time.sleep(0.3)

        print(f"  Page {pn}: {len(articles)} found, {total_new} total", file=sys.stderr)

    conn.commit()
    conn.close()
    print(f"\nDone! Found: {total_found}, New: {total_new}", file=sys.stderr)


if __name__ == "__main__":
    main()