#!/usr/bin/env python3
"""
池州市生态环境局 - 环境影响评价公众参与
https://sthjj.chizhou.gov.cn/News/showList/5578/page_1.html
"""
import re
import sys
import time
import json
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin

DB_PATH = "/root/search.db"
SITE_NAME = "池州市生态环境局-环评公众参与"
CATEGORY = "安徽"
BASE_URL = "https://sthjj.chizhou.gov.cn"
MAX_PAGES = 5

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
    "Referer": "https://sthjj.chizhou.gov.cn/",
}

INSERT_SQL = """
INSERT OR REPLACE INTO gov_raw (site_name, page_url, title, publish_date, summary, content, category, attachments, script_name) VALUES (?, ?, ?, ?, '', ?, ?, ?, 'crawl_chizhou.py')
"""


def init_db():
    import sqlite3
    conn = sqlite3.connect(DB_PATH, timeout=60)
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute("PRAGMA busy_timeout=5000")
    return conn


def extract_list_page(page_num):
    url = f"https://sthjj.chizhou.gov.cn/News/showList/5578/page_{page_num}.html"
    try:
        r = requests.get(url, headers=HEADERS, timeout=30)
        r.encoding = "utf-8"
    except Exception as e:
        print(f"  [ERROR] Failed to fetch page {page_num}: {e}", flush=True)
        return []

    if r.status_code != 200:
        print(f"  [ERROR] HTTP {r.status_code} for page {page_num}", flush=True)
        return []

    soup = BeautifulSoup(r.text, "html.parser")
    items = []

    list_div = soup.select_one("div.m-cglist")
    if not list_div:
        print(f"  [ERROR] No list div on page {page_num}", flush=True)
        return []

    for li in list_div.find_all("li"):
        a_tag = li.find("a", href=True)
        if not a_tag:
            continue

        title = a_tag.get("title", "").strip()
        if not title:
            title = a_tag.get_text(strip=True)
        title = re.sub(r'\s+', ' ', title).strip()

        href = a_tag["href"].strip()
        if not href or href.startswith("javascript"):
            continue
        full_url = urljoin(BASE_URL, href)

        date_span = li.find("span")
        date_str = date_span.get_text(strip=True) if date_span else ""

        items.append({"title": title, "url": full_url, "date": date_str})

    print(f"  Page {page_num}: {len(items)} items", flush=True)
    return items


def extract_detail(url):
    try:
        r = requests.get(url, headers=HEADERS, timeout=30)
        r.encoding = "utf-8"
    except Exception as e:
        print(f"  [ERROR] Failed to fetch {url}: {e}", flush=True)
        return None

    if r.status_code != 200:
        print(f"  [ERROR] HTTP {r.status_code} for {url}", flush=True)
        return None

    soup = BeautifulSoup(r.text, "html.parser")

    title = ""
    meta_title = soup.find("meta", attrs={"name": "ArticleTitle"})
    if meta_title and meta_title.get("content"):
        title = meta_title["content"].strip()

    pub_date = ""
    meta_date = soup.find("meta", attrs={"name": "PubDate"})
    if meta_date and meta_date.get("content"):
        pub_date = meta_date["content"].strip()
        pub_date = re.sub(r'\s.*', '', pub_date)

    content_div = soup.select_one("div.j-fontContent")
    if not content_div:
        content_div = soup.select_one("div.m-dttexts")
    if not content_div:
        content_div = soup.select_one("div.m-detailbox")

    content_text = ""
    attachments_list = []

    if content_div:
        # Build content preserving paragraphs and tables
        content_html_str = str(content_div)
        all_tables = content_div.find_all("table")
        for ti, table in enumerate(all_tables):
            marker = f"\n__TABLE_{ti}__\n"
            content_html_str = content_html_str.replace(str(table), marker, 1)

        # Get text preserving <p> as paragraph breaks, but joining <span> fragments with space
        clean_soup = BeautifulSoup(content_html_str, "html.parser")
        
        parts = []
        for p_tag in clean_soup.find_all("p"):
            t = p_tag.get_text(" ", strip=True)
            if t:
                parts.append(t)

        # Build final text: paragraphs separated by double newlines
        content_text = "\n\n".join(parts)
        
        # Clean up span artifacts
        content_text = re.sub(r' +', ' ', content_text)
        content_text = re.sub(r' ([.,;:!?，。；：！？、\)）】》"])', r'\1', content_text)
        content_text = re.sub(r'(["\'\(（【《]) ', r'\1', content_text)
        
        # If there are tables, insert them at marker positions
        if all_tables:
            table_texts = []
            for ti, table in enumerate(all_tables):
                rows = table.find_all("tr")
                if not rows:
                    table_texts.append("")
                    continue
                num_cols = 0
                for row in rows:
                    cells = row.find_all(["td", "th"])
                    num_cols = max(num_cols, len(cells))

                if num_cols == 2:
                    t_lines = ['| 项 目 | 内 容 |', '|------|------|']
                    for row in rows:
                        cells = row.find_all(["td", "th"])
                        if len(cells) >= 2:
                            label = cells[0].get_text(" ", strip=True)
                            value = cells[1].get_text(" ", strip=True)
                            t_lines.append(f"| {label} | {value} |" if label else f"| | {value} |")
                    table_texts.append("\n".join(t_lines))
                else:
                    col_widths = [0] * num_cols
                    for row in rows:
                        cells = row.find_all(["td", "th"])
                        for ic, cell in enumerate(cells):
                            cell_text = cell.get_text(" ", strip=True)
                            col_widths[ic] = max(col_widths[ic], len(cell_text))
                    col_widths = [max(6, min(w + 2, 40)) for w in col_widths]
                    t_lines = []
                    for ri, row in enumerate(rows):
                        cells = row.find_all(["td", "th"])
                        cell_texts = []
                        for ci in range(num_cols):
                            txt = cells[ci].get_text(" ", strip=True) if ci < len(cells) else ""
                            cell_texts.append(txt.ljust(col_widths[ci]))
                        sep = "| "
                        t_lines.append(sep + sep.join(cell_texts) + " |")
                        if ri == 0:
                            sep_line = []
                            for w in col_widths:
                                sep_line.append("-" * w)
                            t_lines.append("|-" + "-|-".join(sep_line) + "-|")
                    table_texts.append("\n".join(t_lines))
            
            # Replace table markers in content_text
            for ti in range(len(all_tables)):
                marker = f"\n__TABLE_{ti}__\n"
                if table_texts[ti]:
                    content_text = content_text.replace(marker, f"\n\n{table_texts[ti]}\n\n")

        for a_tag in content_div.find_all("a", href=True):
            href = a_tag["href"]
            if re.search(r'\.(pdf|doc|docx|xls|xlsx|rar|zip)$', href, re.IGNORECASE):
                att_title = a_tag.get_text(strip=True) or href.split("/")[-1]
                full_url = urljoin(url, href)
                attachments_list.append({"name": att_title, "url": full_url})

    if len(content_text.strip()) < 20:
        content_text = f"[{title or url.split('/')[-1]}]({url})\n（本文为PDF附件）"

    return {
        "title": title,
        "content": content_text,
        "pub_date": pub_date,
        "attachments": json.dumps(attachments_list, ensure_ascii=False) if attachments_list else "",
    }


def crawl(test_mode=False, max_pages=MAX_PAGES):
    print(f"[{SITE_NAME}] Starting crawl, max_pages={max_pages}", flush=True)
    conn = init_db()
    cur = conn.cursor()

    total_inserted = 0
    total_skipped = 0
    pages_crawled = 0

    for page_num in range(1, max_pages + 1):
        items = extract_list_page(page_num)
        if not items:
            print(f"  Page {page_num} empty, stopping", flush=True)
            break

        pages_crawled += 1
        for item in items:
            title = re.sub(r'\s+', ' ', item["title"]).strip()
            url = item["url"]
            date = item["date"]
            # Check if already exists for THIS site
            cur.execute("SELECT id FROM gov_raw WHERE page_url = ? AND site_name = ?", (url, SITE_NAME))
            if cur.fetchone():
                total_skipped += 1
                continue

            detail = extract_detail(url)
            if detail is None:
                total_skipped += 1
                continue

            if detail["title"] and len(detail["title"]) > len(title):
                title = detail["title"]

            content = detail["content"]
            pub_date = date or detail["pub_date"]
            attachments = detail["attachments"]

            cur.execute(INSERT_SQL, (
                SITE_NAME, url, title, pub_date,
                content, CATEGORY, attachments,
            ))
            total_inserted += 1

            if test_mode and total_inserted >= 10:
                break

        conn.commit()

        if test_mode and total_inserted >= 10:
            print(f"  Test mode: stopping after {total_inserted} inserts", flush=True)
            break

        time.sleep(0.5)

    conn.close()
    print(f"[{SITE_NAME}] Done. Inserted={total_inserted}, Skipped={total_skipped}, Pages={pages_crawled}", flush=True)
    return total_inserted


if __name__ == "__main__":
    test_mode = "--test" in sys.argv
    mp = 3 if test_mode else MAX_PAGES
    if "--max-pages" in sys.argv:
        idx = sys.argv.index("--max-pages")
        if idx + 1 < len(sys.argv):
            mp = int(sys.argv[idx + 1])
    crawl(test_mode=test_mode, max_pages=mp)