#!/usr/bin/env python3
"""Crawl chq.mas.gov.cn - 马鞍山慈湖高新区通知公告 (Lonsun CMS)"""
import sys, re, json, sqlite3, urllib.request, urllib.error, traceback, ssl
from datetime import datetime
from urllib.parse import urljoin

ssl._create_default_https_context = ssl._create_unverified_context

BASE = "https://chq.mas.gov.cn"
COLUMN_URL = "https://chq.mas.gov.cn/content/column/4716840"
SITE_NAME = "chq_mas_tzgg"
DB_PATH = "/root/search.db"
PAGE_SIZE = 20
MAX_PAGES = 5

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
}


def fetch_list_page(page):
    url = f"{COLUMN_URL}?pageIndex={page}&pageSize={PAGE_SIZE}"
    req = urllib.request.Request(url, headers=headers)
    resp = urllib.request.urlopen(req, timeout=15)
    html = resp.read().decode("utf-8", errors="replace")
    return html


def parse_list(html):
    """Extract article URLs and dates from list page HTML."""
    items = re.findall(
        r'<li[^>]*>.*?<a href="([^"]+)"[^>]*title="([^"]*)"[^>]*>.*?</a>.*?<span[^>]*class="right date"[^>]*>([^<]+)</span>',
        html, re.DOTALL)
    result = []
    for href, title, date_str in items:
        date = date_str.strip()
        # Only include chq.mas.gov.cn URLs
        if 'chq.mas.gov.cn' not in href:
            continue
        full_url = urljoin(BASE, href) if not href.startswith('http') else href
        result.append({"url": full_url, "title": title.strip(), "date": date})
    return result


def fetch_detail(url):
    """Fetch article detail page and extract content + attachments."""
    try:
        req = urllib.request.Request(url, headers=headers)
        resp = urllib.request.urlopen(req, timeout=15)
        html = resp.read().decode("utf-8", errors="replace")

        # Find content div
        m = re.search(r'<div[^>]*class="wzcon j-fontContent"[^>]*>(.*?)</div>\s*<(?:script|style|div)', html, re.DOTALL)
        if not m:
            m = re.search(r'class="wzcon j-fontContent"[^>]*>(.*?)</div>', html, re.DOTALL)
        if not m:
            return ""

        content_html = m.group(1).strip()

        # Extract attachment links
        attachments = []
        for href in re.findall(r'href="([^"]*\.(?:pdf|doc|docx|xls|xlsx|zip|rar))"', content_html, re.I):
            full = urljoin(url, href)
            # Get text from <a> tag
            a_match = re.search(r'<a[^>]*href="' + re.escape(href) + r'"[^>]*>(.*?)</a>', content_html, re.I | re.DOTALL)
            text = ""
            if a_match:
                text = re.sub(r'<[^>]+>', '', a_match.group(1)).strip()
            if not text:
                text = href.split('/')[-1].rsplit('.', 1)[0]
            attachments.append(f"[{text}]({full})")
            print(f"    +ATT: {text[:40]}")

        # Build final content: HTML table preserved, attachment links appended
        content = content_html
        if attachments:
            content += '\n\n**附件：**\n' + '\n'.join(attachments)

        return content
    except Exception as e:
        print(f"  Detail error: {e}")
        return ""


def save_to_db(records):
    if not records:
        return 0
    conn = sqlite3.connect(DB_PATH)
    conn.execute("PRAGMA busy_timeout=30000")
    c = conn.cursor()
    count = 0
    for r in records:
        existing = c.execute("SELECT id FROM gov_raw WHERE source_url=?", (r["url"],)).fetchone()
        if existing:
            print(f"  SKIP: {r['title'][:40]}")
            continue
        c.execute("""INSERT INTO gov_raw (site_name, source_url, page_url, title, publish_date, date_rank, summary, content)
            VALUES (?,?,?,?,?,?,?,?)""",
                  (SITE_NAME, r["url"], r["url"], r["title"], r["date"],
                   int(datetime.strptime(r["date"], "%Y-%m-%d").timestamp()) if r["date"] else 0,
                   r["content"][:200] if r["content"] else "", r["content"]))
        conn.commit()
        count += 1
        print(f"  OK: {r['title'][:50]} | {r['date']}")
    conn.close()
    return count


def main():
    print(f"=== Crawling {SITE_NAME} ===")

    # Fetch all list pages
    all_items = []
    for page in range(1, MAX_PAGES + 1):
        try:
            html = fetch_list_page(page)
            items = parse_list(html)
            if not items:
                print(f"Page {page}: no items, stopping")
                break
            print(f"Page {page}: {len(items)} items")
            all_items.extend(items)
            if len(all_items) >= 2009:
                break
        except Exception as e:
            print(f"Page {page}: error - {e}")
            break

    print(f"\nTotal list items: {len(all_items)}")

    # Fetch detail pages
    saved = 0
    for i, item in enumerate(all_items):
        print(f"[{i+1}/{len(all_items)}] {item['title'][:50]}...")
        try:
            content = fetch_detail(item["url"])
            item["content"] = content
            # Save immediately
            conn = sqlite3.connect(DB_PATH)
            conn.execute("PRAGMA busy_timeout=30000")
            c = conn.cursor()
            existing = c.execute("SELECT id FROM gov_raw WHERE source_url=?", (item["url"],)).fetchone()
            if existing:
                print(f"  SKIP: {item['title'][:40]}")
                conn.close()
                continue
            c.execute("""INSERT INTO gov_raw (site_name, source_url, page_url, title, publish_date, date_rank, summary, content)
                VALUES (?,?,?,?,?,?,?,?)""",
                      (SITE_NAME, item["url"], item["url"], item["title"], item["date"],
                       int(datetime.strptime(item["date"], "%Y-%m-%d").timestamp()) if item["date"] else 0,
                       content[:200] if content else "", content))
            conn.commit()
            conn.close()
            saved += 1
            print(f"  OK: {item['title'][:50]} | {item['date']}")
        except Exception as e:
            print(f"  Error importing {item['title'][:40]}: {e}")

    print(f"\n=== Complete: {saved}/{len(all_items)} new records ===")


if __name__ == "__main__":
    main()