#!/usr/bin/env python3
"""宜春市生态环境局 - 拟受理项目公示"""
import sys, re, json, requests, sqlite3
from datetime import datetime, timedelta
import os

DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
API_URL = "http://sthjj.yichun.gov.cn/queryList"
SITE_NAME = "宜春市生态环境局-拟受理项目公示"
HEADERS = {
    "User-Agent": "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)",
    "Content-Type": "application/json",
}

def get_conn():
    conn = sqlite3.connect(DB_PATH)
    conn.execute("PRAGMA journal_mode=WAL")
    return conn

def extract_date(text):
    if not text: return None
    m = re.search(r"(\d{4}-\d{2}-\d{2})", str(text))
    return m.group(1) if m else None

def is_recent(date_str, years=3):
    if not date_str: return False
    try:
        dt = datetime.strptime(date_str, "%Y-%m-%d")
        return dt >= (datetime.now() - timedelta(days=365*years))
    except: return False

def fetch_page(page_num, page_size=20):
    payload = {
        "current": page_num,
        "pageSize": page_size,
        "channelCode": "nslxmgs",
        "websiteCode": "ycssthjj",
    }
    try:
        resp = requests.post(API_URL, json=payload, headers=HEADERS, timeout=15)
        if resp.status_code != 200:
            print("  HTTP %d" % resp.status_code)
            return None
        data = resp.json()
        return data.get("data")
    except Exception as e:
        print("  Error:", e)
        return None

def crawl(incremental=False):
    conn = get_conn()
    cursor = conn.cursor()
    max_pages = 1 if incremental else 5
    total_new = total_skip = total_old = 0

    for page in range(1, max_pages + 1):
        print("Page %d/%d..." % (page, max_pages))
        result = fetch_page(page)
        if not result:
            print("  No data, stopping")
            break
        items = result.get("results", [])
        total = result.get("total", 0)
        print("  Got %d items (total: %d)" % (len(items), total))
        if not items: break

        for item in items:
            src = item.get("source", {})
            title = (src.get("showTitle") or src.get("title", "")).strip()
            pub_date = src.get("pubDate", "")
            date_str = extract_date(pub_date)
            content_obj = src.get("content", {}) or {}
            content = content_obj.get("content", "") if isinstance(content_obj, dict) else ""
            urls_str = src.get("urls", "{}")
            try:
                urls = json.loads(urls_str) if isinstance(urls_str, str) else urls_str
                page_url = "http://sthjj.yichun.gov.cn" + urls.get("pc", "")
            except:
                page_url = ""

            if not title or not page_url:
                total_skip += 1
                continue
            if date_str and not is_recent(date_str):
                total_old += 1
                continue

            summary = re.sub(r"<[^>]+>", "", content)[:200] if content else ""
            cursor.execute("""
                INSERT OR IGNORE INTO gov_raw
                (source_url, page_url, title, summary, content, site_name, publish_date)
                VALUES (?, ?, ?, ?, ?, ?, ?)
            """, (page_url, page_url, title, summary, content, SITE_NAME, date_str or ""))
            if cursor.rowcount > 0:
                total_new += 1
        conn.commit()

    conn.close()
    print("")
    print("Done. New: %d, Skipped: %d, Old: %d" % (total_new, total_skip, total_old))

if __name__ == "__main__":
    crawl(incremental="incremental" in sys.argv)