#!/usr/bin/env python3
"""crawl_gscxhj.py - 甘肃创新环境科技 (公司公示 article-23)"""
import sys, os, re, time
from datetime import datetime, timezone, timedelta
import requests
from bs4 import BeautifulSoup
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

SITE_NAME = "甘肃创新环境-公司公示"
BASE_URL = "http://www.gscxhj.com"
MAX_PAGES = 5
THREE_YEARS_AGO = (datetime.now(timezone.utc) - timedelta(days=365*3)).strftime("%Y-%m-%d")

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
}

def create_session():
    s = requests.Session()
    s.headers.update(HEADERS)
    s.verify = False
    try:
        r = s.get(BASE_URL + "/", timeout=15)
        for c in re.findall(r"document\.cookie=\x27([^\x27]+)\x27", r.text):
            kv = c.split("=", 1)
            if len(kv) == 2:
                s.cookies.set(kv[0], kv[1].rstrip(";"))
        if re.findall(r"document\.cookie", r.text):
            s.get(BASE_URL + "/", timeout=15)
    except:
        pass
    return s

def fetch_page(session, page_num):
    if page_num == 1:
        url = BASE_URL + "/article-23.html"
    else:
        url = f"{BASE_URL}/article23_{page_num-1}.html"
    try:
        r = session.get(url, timeout=20)
        r.encoding = "utf-8"
        return r.text
    except Exception as e:
        print(f"  [ERROR] page {page_num}: {e}")
        return None

def parse_items(html):
    soup = BeautifulSoup(html, "html.parser")
    items = []
    for td in soup.find_all("div", class_=lambda c: c and "newsCenterTop" in c
                            and "col-lg-12" in c and "Topl" not in c and "Topr" not in c):
        ld = td.find("div", class_=lambda c: c and "newsCenterTopl" in c)
        if not ld:
            continue
        a = ld.find("a")
        if not a or not a.get("href"):
            continue
        url = a["href"].strip()
        if not url.startswith("http"):
            url = BASE_URL + url
        rd = td.find("div", class_=lambda c: c and "newsCenterTopr" in c)
        if not rd:
            continue
        h4 = rd.find("h4")
        title = h4.get_text(strip=True) if h4 else ""
        span = rd.find("span")
        date_text = span.get_text(strip=True) if span else ""
        items.append((title, url, date_text))
    return items

def fetch_detail(session, url):
    import re as _re
    try:
        r = session.get(url, timeout=20)
        r.encoding = "utf-8"
        html = r.text
        # Handle cookie challenge (403 + JS cookie)
        if r.status_code == 403 and "document.cookie" in html:
            m = _re.search(r'document\.cookie=.wtime=([^;\'"\"]+)', html)
            wtime = m.group(1) if m else None
            m = _re.search(r'document\.cookie=.wtoken=([^;\'"\"]+)', html)
            wtoken = m.group(1) if m else None
            if wtime: session.cookies.set("wtime", wtime)
            if wtoken: session.cookies.set("wtoken", wtoken)
            r = session.get(url, timeout=20)
            r.encoding = "utf-8"
            html = r.text
    except:
        return None, None, None
    soup = BeautifulSoup(html, "html.parser")
    title_tag = soup.find("title")
    title = title_tag.get_text(strip=True) if title_tag else ""
    title = re.sub(r"\s*[-–—|]\s*甘肃创新环境.*$", "", title).strip()
    dm = re.search(r"(\d{4}-\d{2}-\d{2})", html)
    pub_date = dm.group(1) if dm else ""
    cd = soup.find("div", class_=lambda c: c and "newsdetails" in c)
    if cd:
        content = str(cd)
        content = re.sub(r"<script[^>]*>.*?</script>", "", content, flags=re.DOTALL|re.I)
        content = re.sub(r"<style[^>]*>.*?</style>", "", content, flags=re.DOTALL|re.I)
        return title or url.split("/")[-1], content, pub_date
    return None, None, pub_date

def has_next_page(html):
    soup = BeautifulSoup(html, "html.parser")
    pd = soup.find("div", class_=lambda c: c and "pro_page" in c)
    if pd:
        for a in pd.find_all("a"):
            if "下一页" in a.get_text(strip=True):
                return True
    return False

def main(incremental=False, limit=None):
    print(f"\n{'='*50}\n{SITE_NAME}\n{'='*50}")
    session = create_session()
    all_items = []
    seen = set()

    for page in range(1, MAX_PAGES + 1):
        print(f"\n--- Page {page} ---", end=" ", flush=True)
        html = fetch_page(session, page)
        if not html:
            break
        items = parse_items(html)
        if not items:
            print("empty, stop")
            break
        print(f"{len(items)} items", end="")

        page_exhausted = True
        new_count = 0
        for title, url, date_text in items:
            if url in seen:
                continue
            seen.add(url)
            if date_text and date_text < THREE_YEARS_AGO:
                continue
            page_exhausted = False
            new_count += 1
            if incremental:
                import sqlite3 as s3
                conn = s3.connect("/root/search.db")
                exists = conn.execute("SELECT 1 FROM gov_raw WHERE page_url=?", (url,)).fetchone()
                conn.close()
                if exists:
                    continue
            all_items.append((title, url, date_text))

        print(f" | keep: {new_count}")
        if page_exhausted:
            print("  All items past 3yr cutoff, stop")
            break
        if not has_next_page(html):
            break
        time.sleep(0.5)
        if limit and len(all_items) >= limit:
            all_items = all_items[:limit]
            break

    print(f"\nTotal: {len(all_items)} items")

    if not all_items:
        print("  No new items")
        return

    results = []
    for i, (title, url, list_date) in enumerate(all_items):
        print(f"  [{i+1}/{len(all_items)}] {title[:50]}...", end=" ", flush=True)
        dt, content, pub_date = fetch_detail(session, url)
        ftitle = dt or title
        fdate = pub_date or list_date
        summary = re.sub(r"<[^>]+>", " ", content or "").strip()[:300]
        summary = re.sub(r"\s+", " ", summary)
        results.append({
            "site_name": SITE_NAME, "title": ftitle, "url": url,
            "source_url": url, "content": content or "",
            "pub_date": fdate, "summary": summary, "tags": SITE_NAME,
        })
        print("OK")
        time.sleep(0.5)

    if results:
        push_to_searchdb(results, "gscxhj")
    print(f"\nDone! {len(results)} records")

if __name__ == "__main__":
    t0 = time.time()
    inc = len(sys.argv) > 1 and sys.argv[1] in ("1", "--incremental")
    limit = None
    if "--limit" in sys.argv:
        limit = int(sys.argv[sys.argv.index("--limit") + 1])
    main(incremental=inc, limit=limit)
    print(f"Time: {time.time()-t0:.1f}s")