#!/usr/bin/env python3
"""北海经济开发区 — 通知公告 (col118450)
http://www.sdbh.gov.cn/col/col118450/index.html?uid=699737&pageNum=0
TRS jPage AJAX: /module/web/jpage/dataproxy.jsp
Total: 511 records × 15/page = 35 pages
Page 1 data embedded in HTML <script type="text/xml"><datastore>
Pages 2+ via POST to dataproxy.jsp with session cookies
Detail: div.bg-article.bfr_article_content or div.content
"""

import requests, re, sys, os, time, sqlite3
from bs4 import BeautifulSoup
from datetime import datetime, timedelta

BASE_URL = "http://www.sdbh.gov.cn"
LIST_URL = BASE_URL + "/col/col118450/index.html?uid=699737&pageNum=0"
SITE_NAME = "北海经济开发区-通知公告"
GROUP = "企业"
INDUSTRY = "政府公告"
SCRIPT_NAME = "crawl_sdbh_notice.py"
MAX_PAGES = 5
PER_PAGE = 15

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}
SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
CUTOFF = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")

session = requests.Session()
session.headers.update(HEADERS)
session.verify = False

import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)


def fetch(url, post_data=None):
    for i in range(3):
        try:
            if post_data:
                r = session.post(url, data=post_data, timeout=15)
            else:
                r = session.get(url, timeout=15)
            r.encoding = "utf-8"
            if r.status_code == 200:
                return r.text
        except Exception as e:
            print(f"[WARN] {url[:60]} failed: {e}")
        time.sleep(2)
    return None


def parse_list_xml(html):
    """Extract items from embedded XML dataStore in the first page"""
    items = []
    m = re.search(r'<div id="699737">.*?<script[^>]*type="text/xml">(.*?)</script>', html, re.DOTALL)
    if not m:
        return items
    
    xml_data = m.group(1)
    # Extract records
    records = re.findall(
        r'<a href="([^"]*)"[^>]*title="([^"]*)"[^>]*>.*?</a><span>([^<]*)</span>',
        xml_data, re.DOTALL
    )
    for href, title, date in records:
        if not href.startswith("http"):
            href = BASE_URL + href
        items.append((title.strip(), href.strip(), date.strip()))
    return items


def parse_list_ajax(html):
    """Parse AJAX response from dataproxy.jsp"""
    items = []
    records = re.findall(
        r'<a href="([^"]*)"[^>]*title="([^"]*)"[^>]*>.*?</a><span>([^<]*)</span>',
        html, re.DOTALL
    )
    for href, title, date in records:
        if not href.startswith("http"):
            href = BASE_URL + href
        items.append((title.strip(), href.strip(), date.strip()))
    return items


def parse_detail(url):
    """Parse detail page"""
    html = fetch(url)
    if not html:
        return "", "", ""
    
    soup = BeautifulSoup(html, "html.parser")
    
    # Title
    title = ""
    h1 = soup.find("h1")
    if h1:
        title = h1.get_text(strip=True)
    if not title:
        meta = soup.find("meta", attrs={"name": "ArticleTitle"})
        if meta and meta.get("content"):
            title = meta["content"].strip()
    if not title:
        m = soup.find("title")
        if m:
            title = m.get_text(strip=True).split(" ")[0]
    
    # Date
    date_str = ""
    m = re.search(r'发布日期[：:]\s*(\d{4}[-/]\d{1,2}[-/]\d{1,2})', html)
    if m:
        date_str = m.group(1).replace("/", "-")
    if not date_str:
        meta = soup.find("meta", attrs={"name": "PubDate"})
        if meta and meta.get("content"):
            m = re.search(r"(\d{4}-\d{2}-\d{2})", meta["content"])
            if m:
                date_str = m.group(1)
    
    # Content
    content = ""
    body = soup.find("div", class_=lambda c: c and "content" in str(c) if c else False)
    if not body:
        body = soup.find("div", class_="bg-article")
    if body:
        content = str(body)
        # Clean
        content = re.sub(r'<script[^>]*>.*?</script>', '', content, flags=re.DOTALL)
        content = re.sub(r'<style[^>]*>.*?</style>', '', content, flags=re.DOTALL)
        # Remove share/footer elements
        content = re.sub(r'<div[^>]*class="[^"]*bshare[^"]*"[^>]*>.*?</div>', '', content, flags=re.DOTALL)
        content = content.strip()
    
    return title, date_str, content


def crawl(pages=MAX_PAGES):
    conn = sqlite3.connect(SEARCH_DB, timeout=60)
    cursor = conn.cursor()
    
    # First, visit the list page to establish session
    print(f"[{SCRIPT_NAME}] Loading first page...")
    html = fetch(LIST_URL)
    if not html:
        print("[ERROR] Cannot load list page")
        return
    
    all_count = 0
    skip_count = 0
    
    for page in range(1, pages + 1):
        if page == 1:
            # Page 1: extract from embedded XML
            items = parse_list_xml(html)
        else:
            # Page 2+: call dataproxy.jsp with startrecord/endrecord
            start_record = (page - 1) * PER_PAGE + 1
            end_record = page * PER_PAGE
            post_data = {
                "startrecord": str(start_record),
                "endrecord": str(end_record),
                "perpage": str(PER_PAGE),
                "webid": "457",
                "path": f"{BASE_URL}/",
                "columnid": "118450",
                "unitid": "699737",
                "permissiontype": "0",
                "col": "1",
                "sourceContentType": "1"
            }
            ajax_html = fetch(f"{BASE_URL}/module/web/jpage/dataproxy.jsp", post_data=post_data)
            if not ajax_html or 'csrf' in ajax_html:
                print(f"[{SCRIPT_NAME}] No AJAX data for page {page}, stopping")
                break
            items = parse_list_ajax(ajax_html)
        
        if not items:
            print(f"[{SCRIPT_NAME}] No items on page {page}")
            continue
        
        print(f"[{SCRIPT_NAME}] Page {page}: {len(items)} items")
        
        for title, item_url, date_str in items:
            existing = cursor.execute(
                "SELECT id FROM gov_raw WHERE page_url = ?", (item_url,)
            ).fetchone()
            if existing:
                skip_count += 1
                continue
            
            detail_title, detail_date, content = parse_detail(item_url)
            if not detail_title:
                detail_title = title
            if not detail_date:
                detail_date = date_str
            if not content or len(content.strip()) < 50:
                content = "正文为空"
            
            try:
                cursor.execute("""
                    INSERT INTO gov_raw (source_url, page_url, title, publish_date, site_name, content, group_name, industry)
                    VALUES (?, ?, ?, ?, ?, ?, ?, ?)
                """, (item_url, item_url, detail_title, detail_date, SITE_NAME, content, GROUP, INDUSTRY))
                conn.commit()
                all_count += 1
                print(f"[{SCRIPT_NAME}] +{all_count}: {detail_title[:40]} ({detail_date})")
            except Exception as e:
                conn.rollback()
                skip_count += 1
            
            time.sleep(0.5)
        
        time.sleep(0.5)
    
    conn.close()
    print(f"\n[{SCRIPT_NAME}] Done. New: {all_count}, Skipped: {skip_count}")
    return all_count


if __name__ == "__main__":
    pages = MAX_PAGES
    if len(sys.argv) > 1 and sys.argv[1].startswith("--pages="):
        pages = int(sys.argv[1].split("=")[1])
    elif len(sys.argv) > 2 and sys.argv[1] == "--pages":
        pages = int(sys.argv[2])
    crawl(pages=pages)