#!/usr/bin/env python3
import os
"""汝阳县公示公告爬虫 - ry.gov.cn"""
import re, sys, time, json, urllib.request, urllib.error, sqlite3, os, ssl
from datetime import datetime, timedelta
from bs4 import BeautifulSoup

# ── config ──


import sys as _SYS
_MAX_PG = int(_SYS.argv[1]) if len(_SYS.argv) > 1 and _SYS.argv[1].isdigit() else None
if _MAX_PG is not None:
    print('[AutoPg] max_pages=' + str(_MAX_PG))
# END AUTO PAGES
BASE_URL = "https://www.ry.gov.cn/ryzx/gsgg"
SITE_NAME = "汝阳县政府-公示公告"
CUTOFF_DATE = (datetime.now() - timedelta(days=365*3)).strftime("%Y-%m-%d")  # 3 years
MAX_PAGES = 55  # safety limit
DELAY = 1.0

DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE

def fetch(url, retries=3):
    for i in range(retries):
        try:
            req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"})
            with urllib.request.urlopen(req, context=ctx, timeout=20) as r:
                return r.read().decode("utf-8", errors="replace")
        except Exception as e:
            if i < retries - 1:
                time.sleep(DELAY * (i+1))
                continue
            print(f"  [WARN] fetch failed: {url} - {e}", flush=True)
            return None

def parse_list(html):
    """Extract (url, title, date) from list page"""
    soup = BeautifulSoup(html, "html.parser")
    items = []
    for tr in soup.select("table tr"):
        tds = tr.find_all("td")
        if len(tds) >= 2:
            a = tds[0].find("a")
            date_span = tds[1].find("span")
            if a and a.get("href") and date_span:
                url = a["href"].strip()
                if url.startswith("/"):
                    url = "https://www.ry.gov.cn" + url
                elif url.startswith("//"):
                    url = "https:" + url
                title = a.get("title", a.get_text(strip=True))
                date = date_span.get_text(strip=True)
                if re.match(r"\d{4}-\d{2}-\d{2}", date):
                    items.append((url, title, date))
    return items

def parse_detail(html, url):
    """Extract content from detail page"""
    soup = BeautifulSoup(html, "html.parser")
    
    # Title
    title_el = soup.select_one(".content_top h3")
    title = title_el.get_text(strip=True) if title_el else ""
    
    # Date
    date_el = soup.find("span", string=re.compile(r"日期"))
    date = ""
    if date_el:
        m = re.search(r"(\d{4}-\d{2}-\d{2})", date_el.get_text())
        if m:
            date = m.group(1)
    
    # Source
    source_el = soup.find("span", id="test")
    source = source_el.get_text(strip=True).replace("来源：", "").strip() if source_el else ""
    
    # Content
    content_div = soup.select_one(".mailbox_content_wznrs")
    content = ""
    if content_div:
        content = str(content_div)
    
    # Attachments
    attachments = []
    attach_div = soup.select_one(".attachmentBox")
    if attach_div:
        for a in attach_div.find_all("a"):
            href = a.get("href", "")
            name = a.get_text(strip=True)
            if href and name:
                if href.startswith("/"):
                    href = "https://www.ry.gov.cn" + href
                elif href.startswith("//"):
                    href = "https:" + href
                elif not href.startswith("http"):
                    href = urllib.parse.urljoin(url, href)
                attachments.append(f'<a href="{href}">{name}</a>')
    
    full_content = content
    if attachments:
        full_content += '<div class="attachments">' + "<br/>".join(attachments) + "</div>"
    
    return title, date, source, full_content

def save_to_db(items):
    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()
    inserted = 0
    for url, title, date, source, content in items:
        try:
            # Use summary for source info, page_url for the URL
            summary = f"来源：{source}" if source else ""
            c.execute("""
                INSERT OR IGNORE INTO gov_raw 
                (page_url, title, publish_date, site_name, source_url, summary, content)
                VALUES (?, ?, ?, ?, ?, ?, ?)
            """, (url, title, date, SITE_NAME, url, summary, content))
            if c.rowcount > 0:
                inserted += 1
            else:
                # Existing record - update content if empty
                c.execute("UPDATE gov_raw SET content=? WHERE page_url=? AND (content IS NULL OR content='')",
                         (content, url))
                if c.rowcount > 0:
                    inserted += 1
        except Exception as e:
            print(f"  [ERROR] DB insert: {url} - {e}", flush=True)
    conn.commit()
    conn.close()
    return inserted

def rebuild_fts():
    """Rebuild FTS index for this site's records"""
    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()
    try:
        c.execute("""
            INSERT INTO gov_search(gov_search, rowid, title, site_name, summary)
            VALUES('rebuild', 0, '', '', '')
        """)
    except:
        pass
    conn.commit()
    conn.close()

def main():
    print(f"[ry] 汝阳县公示公告爬虫 - 截止日期: {CUTOFF_DATE}", flush=True)
    
    all_items = []
    # Collect list pages 0-45 (3 year cutoff found at page 45)
    for page in range(min(46, _MAX_PG or 46)):
        if page == 0:
            url = f"{BASE_URL}/index.html"
        else:
            url = f"{BASE_URL}/index_{page}.html"
        
        print(f"[ry] 列表页 {page+1}/46: {url}", flush=True)
        html = fetch(url)
        if not html:
            print(f"  [SKIP] 无法获取列表页", flush=True)
            continue
        
        items = parse_list(html)
        if not items:
            print(f"  [SKIP] 空列表", flush=True)
            break
        
        # Check cutoff
        dates = sorted(set(d for _,_,d in items), reverse=True)
        oldest = min(d for _,_,d in items) if items else "9999-99-99"
        newest = max(d for _,_,d in items) if items else "0000-00-00"
        print(f"  → {len(items)}条, 日期范围 {oldest} ~ {newest}", flush=True)
        
        for url, title, date in items:
            if date >= CUTOFF_DATE:
                all_items.append((url, title, date))
        
        if oldest < CUTOFF_DATE:
            print(f"  [END] 遇到 {oldest} < 截止 {CUTOFF_DATE}，停止列表翻页", flush=True)
            break
        
        time.sleep(DELAY)
    
    print(f"\n[ry] 共收集 {len(all_items)} 条待抓取详情", flush=True)
    
    # Fetch details
    detailed = []
    for i, (url, title, date) in enumerate(all_items, 1):
        print(f"[ry] 详情 {i}/{len(all_items)}: {title[:30]}...", flush=True)
        html = fetch(url)
        if not html:
            print(f"  [SKIP] 详情页获取失败", flush=True)
            detailed.append((url, title, date, "", ""))
            continue
        
        detail_title, detail_date, source, content = parse_detail(html, url)
        final_title = detail_title or title
        final_date = detail_date or date
        
        content_len = len(content) if content else 0
        print(f"  → 正文 {content_len}字, 来源: {source or '(无)'}", flush=True)
        detailed.append((url, final_title, final_date, source, content))
        time.sleep(DELAY)
    
    # Save to DB
    inserted = save_to_db(detailed)
    print(f"\n[ry] ✅ 入库 {inserted} 条", flush=True)
    
    # Rebuild FTS
    rebuild_fts()
    print(f"[ry] ✅ FTS索引已重建", flush=True)
    
    print(f"[ry] ✅ 完成", flush=True)

if __name__ == "__main__":
    main()