#!/usr/bin/env python3
"""Crawl jkq.mas.gov.cn - 马鞍山经开区通知公告 (Lonsun CMS)"""
import sys, re, json, sqlite3, urllib.request, urllib.error, traceback, ssl
from datetime import datetime
from urllib.parse import urljoin



import sys as _SYS
_MAX_PG = int(_SYS.argv[1]) if len(_SYS.argv) > 1 and _SYS.argv[1].isdigit() else None
if _MAX_PG is not None:
    print('[AutoPg] max_pages=' + str(_MAX_PG))
# END AUTO PAGES
ssl._create_default_https_context = ssl._create_unverified_context

BASE = "https://jkq.mas.gov.cn"
COLUMN_URL = "https://jkq.mas.gov.cn/content/column/4716728"
SITE_NAME = "jkq_mas_tzgg"
DB_PATH = "/root/search.db"
PAGE_SIZE = 20
MAX_PAGES = 5

headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}


def extract_div(html, class_pat):
    m = re.search(r'(<div[^>]*class="[^"]*' + class_pat + r'[^"]*"[^>]*>)', html)
    if not m:
        return ""
    start, tag = m.start(), m.group(1)
    content = html[start:]
    depth = 0
    for i in range(len(content)):
        if content[i:i+4] == '<div': depth += 1
        elif content[i:i+6] == '</div>':
            depth -= 1
            if depth == 0:
                return content[len(tag):i].strip()
    return ""


def fetch_list_page(page):
    url = f"{COLUMN_URL}?pageIndex={page}&pageSize={PAGE_SIZE}"
    req = urllib.request.Request(url, headers=headers)
    resp = urllib.request.urlopen(req, timeout=15)
    return resp.read().decode("utf-8", errors="replace")


def parse_list(html):
    items = re.findall(r'<a href="([^"]+)"[^>]*title="([^"]*)"[^>]*class="left">', html)
    dates = re.findall(r'<span[^>]*class="right date"[^>]*>([^<]+)</span>', html)
    result = []
    for i, (href, title) in enumerate(items):
        u = href if href.startswith('http') else urljoin(BASE, href)
        result.append({"url": u, "title": title.strip(), "date": dates[i].strip() if i < len(dates) else ""})
    return result


def fetch_detail(url):
    try:
        req = urllib.request.Request(url, headers=headers)
        resp = urllib.request.urlopen(req, timeout=15)
        html = resp.read().decode("utf-8", errors="replace")
        return extract_div(html, r'wzcon')
    except:
        return ""


def main():
    print(f"=== Crawling {SITE_NAME} ===")
    all_items = []
    for pg in range(1, min(MAX_PAGES, _MAX_PG or MAX_PAGES)+1):
        try:
            items = parse_list(fetch_list_page(pg))
            if not items:
                break
            print(f"Page {pg}: {len(items)} items")
            all_items.extend(items)
        except Exception as e:
            print(f"Page {pg}: {e}")
            break

    print(f"Total: {len(all_items)} items")

    saved = 0
    for i, item in enumerate(all_items):
        print(f"[{i+1}/{len(all_items)}] {item['title'][:50]}...")
        try:
            content = fetch_detail(item["url"])
            conn = sqlite3.connect(DB_PATH, timeout=60)
            conn.execute("PRAGMA busy_timeout=30000")
            c = conn.cursor()
            if c.execute("SELECT id FROM gov_raw WHERE source_url=?", (item["url"],)).fetchone():
                conn.close()
                continue
            date = item["date"][:10]
            ts = int(datetime.strptime(date, "%Y-%m-%d").timestamp()) if date else 0
            c.execute("""INSERT INTO gov_raw (site_name, source_url, page_url, title, publish_date, date_rank, summary, content)
                VALUES (?,?,?,?,?,?,?,?)""",
                      (SITE_NAME, item["url"], item["url"], item["title"], date,
                       ts, content[:200] if content else "", content))
            conn.commit()
            conn.close()
            saved += 1
            print(f"  OK")
        except Exception as e:
            print(f"  Error: {e}")

    print(f"\n=== Complete: {saved}/{len(all_items)} new records ===")


if __name__ == "__main__":
    main()