#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Crawler for 吉木萨尔县人民政府 - 环评审批
# API: /content/page?channelIds=6378&page=N&size=15&orderBy=38
# Content: div.article-content div.article-view

import requests, sqlite3, re, sys, json, urllib.parse
from datetime import datetime, timedelta
import os

API_URL = "https://www.jmser.gov.cn/content/page?channelIds=6378&page={}&size=15&orderBy=38"
DETAIL_BASE = "https://www.jmser.gov.cn"
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
    "Referer": "https://www.jmser.gov.cn/p192/hpsp.html"
}
DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
CUTOFF_DATE = (datetime.now() - timedelta(days=365*3)).strftime("%Y-%m-%d")
SITE_NAME = "吉木萨尔县环评审批"
FULL = "--full" in sys.argv

TITLE_PAT = re.compile(r'<meta\s+name="ArticleTitle"\s+content="([^"]+)"')
DATE_PAT = re.compile(r'<meta\s+name="PubDate"\s+content="([^"]+)"')
CONTENT_PAT = re.compile(r'<div class="article-content">\s*<div class="article-view">([\s\S]*?)</div>\s*</div>')

def fetch_page(page):
    try:
        r = requests.get(API_URL.format(page), headers=HEADERS, timeout=15)
        return r.json() if r.status_code == 200 else None
    except Exception as e:
        print(f"  API error: {e}")
        return None

def extract_detail(detail_url):
    try:
        r = requests.get(detail_url, headers=HEADERS, timeout=20)
        r.encoding = "utf-8"
        html = r.text
    except Exception as e:
        print(f"  Detail error {detail_url}: {e}")
        return None, None, None

    tm = TITLE_PAT.search(html)
    dm = DATE_PAT.search(html)
    cm = CONTENT_PAT.search(html)

    title = tm.group(1).strip() if tm else None
    pub_date = dm.group(1)[:10] if dm else None
    content = cm.group(1).strip() if cm else None

    return title, pub_date, content

def run():
    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()
    total_new = total_skip = total_before = 0

    first_data = fetch_page(1)
    if not first_data:
        print("ERROR: Cannot fetch page 1")
        return

    total_elements = first_data.get("data", {}).get("totalElements", 0)
    total_pages = (total_elements + 14) // 15
    print(f"Total: {total_elements} articles, {total_pages} pages")

    pages_to_crawl = min(total_pages, 5) if not FULL else total_pages
    print(f"Crawling {pages_to_crawl} pages")

    for pg in range(1, pages_to_crawl + 1):
        data = fetch_page(pg)
        if not data:
            print(f"Page {pg}: no data")
            continue

        articles = data.get("data", {}).get("content", [])
        if not articles:
            print(f"Page {pg}: empty")
            break

        page_new = page_skip = page_before = 0
        for art in articles:
            detail_url = art.get("urlWhole", "")
            # Skip external URLs (not on jmser.gov.cn)
            parsed = urllib.parse.urlparse(detail_url)
            if parsed.netloc and "jmser.gov.cn" not in parsed.netloc:
                print(f"  Skip (external): {art.get('title','')[:50]}...")
                page_skip += 1
                continue
            title = art.get("title", "").strip()
            release_time = art.get("releaseTime", "")
            pub_date = release_time[:10] if release_time else ""

            if pub_date and pub_date < CUTOFF_DATE:
                page_before += 1
                continue

            c.execute("SELECT 1 FROM gov_raw WHERE page_url = ?", (detail_url,))
            if c.fetchone():
                page_skip += 1
                continue

            detail_title, detail_date, content = extract_detail(detail_url)
            ft = detail_title or title
            fd = detail_date or pub_date

            if content is None:
                print(f"  Skip (no content): {ft[:40]}...")
                page_skip += 1
                continue

            c.execute(
                "INSERT OR IGNORE INTO gov_raw "
                "(site_name, source_url, page_url, title, publish_date, content, summary) "
                "VALUES (?,?,?,?,?,?,?)",
                (SITE_NAME, detail_url, detail_url, ft, fd, content, ft)
            )
            page_new += 1
            total_new += 1

        conn.commit()
        print(f"Page {pg}: +{page_new} new, {page_skip} skip, {page_before} pre-cutoff")

        if page_before == len(articles) and pg < pages_to_crawl:
            print("All remaining before cutoff, stopping")
            break

    conn.close()
    print(f"\nDone: {total_new} new, {total_skip} skip, {total_before} before cutoff")
    return total_new, total_skip

if __name__ == "__main__":
    run()