#!/usr/bin/env python3
"""
武汉智汇元环保科技 (zhy-ep.com)
CMS: 织梦 (DedeCMS), 静态分页
多栏目通用脚本 - 通过 --column 指定栏目
  dikuai: 土壤地块调查公示 (list_11)
  jungong: 竣工验收公示 (list_12)
  huanping: 环评公示 (list_??)
  gongsi: 公司新闻 (list_??)
"""
import sys
import re
import os
import sqlite3
import urllib.request
import ssl

# === COLUMNS ===
COLUMNS = {
    "dikuai": {
        "name": "武汉智汇元-土壤地块调查公示",
        "path": "dikuai",
        "list_id": 11,
    },
    "jungong": {
        "name": "武汉智汇元-竣工验收公示",
        "path": "jungong",
        "list_id": 12,
    },
}

BASE_URL = "https://www.zhy-ep.com"
DB_PATH = "/root/search.db"
CUTOFF_DATE = "2023-01-01"
PAGES_DEFAULT = 3  # default pages for incremental

ssl_ctx = ssl.create_default_context()
ssl_ctx.check_hostname = False
ssl_ctx.verify_mode = ssl.CERT_NONE


def fetch(url):
    req = urllib.request.Request(url, headers={
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
    })
    try:
        resp = urllib.request.urlopen(req, timeout=20, context=ssl_ctx)
        return resp.read().decode("utf-8", errors="replace")
    except Exception as e:
        print(f"  [FETCH ERROR] {url}: {e}")
        return None


def parse_list(html):
    """Parse list page - extract items from li.flex.wow.fadeInUp"""
    items = []
    pattern = r'<li class="flex wow fadeInUp">.*?<a href="([^"]*)"[^>]*>.*?<b>([^<]+)</b>.*?<span>(\d{4}-\d{2}-\d{2})</span>'
    for match in re.finditer(pattern, html, re.DOTALL):
        href = match.group(1).strip()
        title = match.group(2).strip()
        date = match.group(3).strip()
        if not href.startswith("http"):
            href = BASE_URL + href
        items.append({"url": href, "title": title, "date": date})
    return items


def parse_detail(html):
    """Extract title, date, content from detail page"""
    title = ""
    date = ""
    content = ""

    # Title from <title> (strip site suffix)
    title_match = re.search(r'<title>(.*?)</title>', html)
    if title_match:
        raw = title_match.group(1)
        # Remove " - 武汉智汇元环保科技有限公司" suffix
        title = re.sub(r'_[^_]+$| - [^-]+$', '', raw).strip()
        if not title:
            title = raw

    # Date from page content - look for date span
    date_match = re.search(r'<span>(\d{4}-\d{2}-\d{2})</span>', html)
    if date_match:
        date = date_match.group(1)

    # Content from div.content2
    content_idx = html.find('class="content2 wow fadeInUp"')
    if content_idx < 0:
        content_idx = html.find('class="content2')
    if content_idx < 0:
        content_idx = html.find('id="content"')
    if content_idx < 0:
        content_idx = html.find('class="content"')

    if content_idx >= 0:
        # Find the opening tag
        start = html.rfind("<div", 0, content_idx)
        if start < 0:
            start = content_idx

        # Find end - look for next section or common closing patterns
        end_markers = [
            '<div class="footer',
            '<div class="copyright',
            '<div class="prenext',
            '<div class="share',
            '<div class="related',
            '<div class="page"',
        ]
        end = len(html)
        for marker in end_markers:
            pos = html.find(marker, content_idx)
            if 0 < pos < end:
                end = pos

        content_html = html[start:end]

        # Remove scripts and styles
        text = re.sub(r'<script[^>]*>.*?</script>', '', content_html, flags=re.DOTALL)
        text = re.sub(r'<style[^>]*>.*?</style>', '', text, flags=re.DOTALL)

        # Preserve paragraph breaks BEFORE stripping tags
        # <br> → newline, </p> → double newline, </div> → newline
        text = re.sub(r'<br\s*/?>', '\n', text)
        text = re.sub(r'</p>', '\n\n', text)
        text = re.sub(r'</div>\s*', '\n', text)
        # Remove carriage returns
        text = text.replace('\r', '')
        # Clean HTML entities first for better text
        text = text.replace('&nbsp;', ' ')
        text = text.replace('&ldquo;', '\u201c').replace('&rdquo;', '\u201d')
        text = text.replace('&mdash;', '\u2014').replace('&middot;', '\u00b7')
        text = text.replace('&amp;', '&').replace('&lt;', '<').replace('&gt;', '>')
        # Convert <sup> tags inline (e.g. m<sup>2</sup> → m²)
        text = re.sub(r'<sup>(\d+)</sup>', lambda m: {'2': '²', '3': '³'}.get(m.group(1), m.group(1)), text)

        # Extract tables as markdown
        tables = []
        table_pattern = r'<table[^>]*>.*?</table>'
        for table_match in re.finditer(table_pattern, text, re.DOTALL):
            table_html = table_match.group(0)
            # Simple table to text conversion
            rows = re.findall(r'<tr[^>]*>(.*?)</tr>', table_html, re.DOTALL)
            table_lines = []
            for row in rows:
                cells = re.findall(r'<t[dh][^>]*>(.*?)</t[dh]>', row, re.DOTALL)
                cell_texts = []
                for cell in cells:
                    cell_text = re.sub(r'<[^>]+>', '', cell).strip()
                    cell_texts.append(cell_text)
                if cell_texts and any(cell_texts):
                    table_lines.append("| " + " | ".join(cell_texts) + " |")
            if table_lines:
                # Add header separator if there are rows
                if len(table_lines) >= 2:
                    header_cells = table_lines[0].count("|") - 1
                    table_lines.insert(1, "|" + "---|" * header_cells)
                tables.append("\n".join(table_lines))

        # Extract images as markdown
        imgs = re.findall(r'<img[^>]*src="([^"]+)"[^>]*>', text)
        img_lines = []
        for img_src in imgs:
            if img_src.startswith("http"):
                full_src = img_src
            elif img_src.startswith("/"):
                full_src = BASE_URL + img_src
            else:
                full_src = img_src  # relative, unlikely
            img_lines.append(f"![image]({full_src})")

        # Get clean text
        clean = re.sub(r'<[^>]+>', '', text)
        # Normalize whitespace: collapse spaces, keep max 2 newlines
        clean = re.sub(r'[ \t]+', ' ', clean)
        clean = re.sub(r' *\n *', '\n', clean)
        clean = re.sub(r'\n{3,}', '\n\n', clean)
        clean = clean.strip()

        # Build final content: text + tables + images
        parts = []
        if clean:
            parts.append(clean)
        if tables:
            parts.append("\n" + "\n\n".join(tables))
        if img_lines:
            parts.append("\n" + "\n".join(img_lines))

        content = "\n\n".join(parts)

    return {"title": title, "date": date, "content": content}


def insert_to_db(items, site_name):
    conn = sqlite3.connect(DB_PATH, timeout=30)
    c = conn.cursor()
    inserted = 0
    skipped = 0

    for item in items:
        title = item.get("title", "")
        url = item.get("url", "")
        date = item.get("date", "")
        content = item.get("content", "")

        if not content and not title:
            skipped += 1
            continue

        try:
            c.execute("""
                INSERT OR IGNORE INTO gov_raw
                (title, page_url, site_name, publish_date, content, summary, attachments, source_url, date_rank)
                VALUES (?, ?, ?, ?, ?, '', '', ?, CAST(strftime('%s', ?) AS INTEGER))
            """, (title, url, site_name, date, content, url, date))
            if c.rowcount > 0:
                inserted += 1
            else:
                skipped += 1
        except Exception as e:
            print(f"  [DB ERROR] {title[:30]}: {e}")
            skipped += 1

    conn.commit()
    conn.close()
    return inserted, skipped


def main():
    column = "dikuai"  # default
    max_pages = PAGES_DEFAULT

    i = 1
    while i < len(sys.argv):
        if sys.argv[i] in ("--column", "-c") and i + 1 < len(sys.argv):
            column = sys.argv[i + 1]
            i += 2
        elif sys.argv[i].isdigit():
            max_pages = int(sys.argv[i])
            i += 1
        else:
            i += 1

    if column not in COLUMNS:
        print(f"[ERROR] Unknown column '{column}'. Available: {list(COLUMNS.keys())}")
        return

    col_conf = COLUMNS[column]
    site_name = col_conf["name"]
    list_id = col_conf["list_id"]
    list_url = f"{BASE_URL}/html/xinwen/{col_conf['path']}/"
    list_tpl = f"{BASE_URL}/html/xinwen/{col_conf['path']}/list_{list_id}_{{}}.html"

    print(f"[INFO] {site_name} - 爬虫, pages={max_pages}")

    all_items = []
    total_new = 0
    total_old = 0

    for page_no in range(1, max_pages + 1):
        if page_no == 1:
            url = list_url
        else:
            url = list_tpl.format(page_no)

        print(f"\n  [Page {page_no}] {url}")
        html = fetch(url)
        if not html:
            print(f"  [SKIP] Cannot fetch page {page_no}")
            continue

        items = parse_list(html)
        print(f"  Found {len(items)} items")

        if not items:
            print(f"  [STOP] No items")
            break

        for item in items:
            if item["date"] < CUTOFF_DATE:
                print(f"  [STOP] Date {item['date']} < {CUTOFF_DATE}")
                break

            print(f"    {item['date']} {item['title'][:50]}...")
            detail_html = fetch(item["url"])
            if not detail_html:
                print(f"    [SKIP] Cannot fetch detail")
                total_old += 1
                continue

            detail = parse_detail(detail_html)
            if detail["title"]:
                item["title"] = detail["title"]
            if detail["date"]:
                item["date"] = detail["date"]
            item["content"] = detail["content"]
            all_items.append(item)

        if all_items:
            new, old = insert_to_db(all_items, site_name)
            total_new += new
            total_old += old
            print(f"  [DB] +{new} new, {old} existing")
            all_items = []

    if all_items:
        new, old = insert_to_db(all_items, site_name)
        total_new += new
        total_old += old

    print(f"\n[DONE] 新增: {total_new}, 跳过: {total_old}")

    if total_new > 0:
        conn = sqlite3.connect(DB_PATH, timeout=30)
        conn.execute("SELECT 1 /* noop: gov_search 由触发器维护, 无需 rebuild */")
        conn.commit()
        conn.close()
        print("FTS rebuilt")


if __name__ == "__main__":
    main()