#!/usr/bin/env python3
import os
"""Crawl 吉林省生态环境厅 - 环境影响评价受理公示"""
import sys, re, os, requests, time
from datetime import datetime, timedelta
import sqlite3

SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "吉林省生态环境厅-受理公示"
BASE_URL = "http://sthjt.jl.gov.cn"
LIST_PREFIX = BASE_URL + "/ywdt/gkgs/gszx/slgs/"
CUTOFF = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

# Playwright session (shared across requests)
_pw_page = None
_pw_context = None
_pw_browser = None
import sys as _SYS
_MAX_PG = int(_SYS.argv[1]) if len(_SYS.argv) > 1 and _SYS.argv[1].isdigit() else None
if _MAX_PG is not None:
    print('[AutoPg] max_pages=' + str(_MAX_PG))
# END AUTO PAGES


def _get_pw_session():
    global _pw_page, _pw_context, _pw_browser
    if _pw_page is not None:
        return _pw_page
    from playwright.sync_api import sync_playwright
    _pw_play = sync_playwright().start()
    _pw_browser = _pw_play.chromium.launch(headless=True, args=['--disable-dev-shm-usage'])
    _pw_context = _pw_browser.new_context(
        user_agent=HEADERS["User-Agent"],
        viewport={"width": 1920, "height": 1080}
    )
    _pw_page = _pw_context.new_page()
    return _pw_page

def _close_pw():
    global _pw_page, _pw_context, _pw_browser
    if _pw_page:
        try: _pw_page.context.close()
        except: pass
        _pw_page = None
    if _pw_browser:
        try: _pw_browser.close()
        except: pass
        _pw_browser = None

def fetch_page(url, is_list=False):
    """Try requests first, fall back to Playwright if WAF challenge detected"""
    try:
        r = requests.get(url, headers=HEADERS, timeout=20)
        r.encoding = 'utf-8'
        html = r.text
        if '浏览器环境检查' not in html[:500]:
            return html
    except Exception:
        pass
    
    # Fallback to Playwright
    for attempt in range(3):
        try:
            page = _get_pw_session()
            page.goto(url, wait_until='networkidle', timeout=30000)
            time.sleep(1)
            html = page.content()
            if 'currency_ul_list' in html or 'TRS_UEDITOR' in html or 'ArticleTitle' in html:
                return html
            if '浏览器环境检查' in html:
                print("  WAF still active, retrying...", flush=True)
                time.sleep(2)
                continue
            return html
        except Exception as e:
            print("  PW attempt %d error: %s" % (attempt+1, e), flush=True)
            time.sleep(2)
    return ""

def get_list_page(page):
    if page == 0:
        url = LIST_PREFIX + "index.html"
    else:
        url = LIST_PREFIX + "index_%d.html" % page
    html = fetch_page(url)
    if not html or 'currency_ul_list' not in html:
        print("  List page %d empty (WAF?)" % page, flush=True)
        return [], False
    items = []
    for m in re.finditer(r'<li[^>]*>\s*<a\s+href="([^"]+)"[^>]*title="([^"]*)"[^>]*>.*?</a>\s*<span>([^<]+)</span>', html, re.DOTALL):
        url = m.group(1).strip()
        title = m.group(2).strip()
        date = m.group(3).strip()
        if not url.startswith('http'):
            if url.startswith('./'):
                url = LIST_PREFIX + url[2:]
            else:
                url = BASE_URL + url if url.startswith('/') else LIST_PREFIX + url
        items.append({'title': title[:500], 'url': url, 'date': date})
    total_pages = None
    pm = re.search(r'var countPage\s*=\s*(\d+)', html)
    if pm:
        total_pages = int(pm.group(1))
    has_more = page + 1 < total_pages if total_pages else True
    return items, has_more

def extract_content(html):
    m = re.search(r'<div\s+class="trs_editor_view\s+TRS_UEDITOR[^"]*"[^>]*>(.*?)</div>\s*</div>', html, re.DOTALL)
    if m:
        return m.group(1).strip()
    m2 = re.search(r'id="content"[^>]*>(.*?)</div>\s*</div>', html, re.DOTALL)
    if m2:
        return m2.group(1).strip()
    return ""

def crawl():
    print("=== %s ===" % SITE_NAME, flush=True)
    print("Cutoff: %s" % CUTOFF, flush=True)
    all_items = []
    page = 0
    while True:
        items, has_more = get_list_page(page)
        if not items:
            if page == 0:
                print("  First page empty, aborting!", flush=True)
                return
            break
        for item in items:
            if item['date'] >= CUTOFF:
                all_items.append(item)
        last_date = items[-1]['date']
        if last_date < CUTOFF or not has_more:
            break
        page += 1
        if _MAX_PG and page >= _MAX_PG: break
        if page % 5 == 0:
            print("  Scanned page %d (%d items so far)..." % (page, len(all_items)), flush=True)
    print("Items in 3yr range: %d" % len(all_items), flush=True)
    conn = sqlite3.connect(SEARCH_DB, timeout=60)
    c = conn.cursor()
    inserted = 0
    empty = 0
    for idx, item in enumerate(all_items):
        html = fetch_page(item['url'])
        if not html or '浏览器环境检查' in html[:500]:
            empty += 1
            print("  FAIL %s | %s" % (item['title'][:40], item['date']), flush=True)
            continue
        title = item['title']
        mt = re.search(r'<meta\s+name="ArticleTitle"\s+content="([^"]*)"', html)
        if mt and mt.group(1).strip():
            title = mt.group(1).strip()[:500]
        date = item['date']
        md = re.search(r'<meta\s+name="PubDate"\s+content="([^"]*)"', html)
        if md:
            d = md.group(1).strip()[:10]
            if d:
                date = d
        content = extract_content(html)
        if not content:
            empty += 1
            print("  EMPTY %s | %s" % (title[:40], date), flush=True)
            continue
        for attempt in range(3):
            try:
                c.execute("""INSERT OR REPLACE INTO gov_raw (source_url, page_url, site_name, title, content, summary, publish_date, category, script_name) VALUES (?,?,?,?,?,?,?,?, 'crawl_jilin.py')""", (
                    item['url'], item['url'], SITE_NAME, title, content,
                    item['title'] if item['title'] != title else "",
                    date, 'eia'
                ))
                break
            except sqlite3.OperationalError as e:
                if 'locked' in str(e) and attempt < 2:
                    time.sleep(1)
                    continue
                raise
        inserted += 1
        if (idx + 1) % 10 == 0:
            for _ in range(3):
                try:
                    conn.commit()
                    break
                except sqlite3.OperationalError as e:
                    if 'locked' in str(e):
                        time.sleep(1)
                        continue
                    raise
            print("  Progress: %d/%d" % (idx + 1, len(all_items)), flush=True)
    conn.commit()
    conn.close()
    _close_pw()
    print("Done: %d inserted, %d empty" % (inserted, empty), flush=True)

if __name__ == "__main__":
    crawl()