import requests, re, sys, time, json, os, sqlite3
import urllib3
urllib3.disable_warnings()

"""峡江县-公告公示 爬虫
域名: www.xiajiang.gov.cn
CMS: ZCMS AJAX (api-ajax_list-{page}.html)
参数: ajax_type=7_news, catid=21, siteid=7
列表API: 每页20条, 每次注入40条
详情页: div.show_content
"""

BASE_URL = "http://www.xiajiang.gov.cn"
SITE_NAME = "峡江县-公告公示"
GROUP = "江西"
INDUSTRY = "政府公告"
SCRIPT_NAME = "crawl_xiajiang_gggs.py"
MAX_PAGES = 5

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9",
}

SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")

session = requests.Session()
session.headers.update(HEADERS)
session.verify = False

# Get homepage for session cookie
try:
    session.get(BASE_URL, timeout=10)
except:
    pass

def fetch_list(page):
    """Fetch one page of items via ZCMS AJAX API"""
    url = f"{BASE_URL}/api-ajax_list-{page}.html"
    headers = {**HEADERS, "X-Requested-With": "XMLHttpRequest", "Referer": f"{BASE_URL}/news-list-gonggaogongshi.html"}
    data = {
        "ajax_type[]": ["7_news", "21", "7", "news", "Y-m-d", "40", "20",
                        ["is_top DESC", "displayorder DESC", "inputtime DESC"], ""],
        "is_ds": "1"
    }
    for retry in range(3):
        try:
            r = session.post(url, data=data, headers=headers, timeout=15)
            if r.status_code == 200:
                d = r.json()
                items = d.get("data", [])
                total = d.get("total", 0)
                return items, total
        except Exception as e:
            if retry < 2:
                time.sleep(2)
    return [], 0

def fetch_detail(url):
    """Fetch detail page HTML"""
    for retry in range(3):
        try:
            r = session.get(url, timeout=15)
            r.encoding = "utf-8"
            if r.status_code == 200:
                return r.text
        except Exception as e:
            if retry < 2:
                time.sleep(2)
    return None

def parse_detail(html):
    """Extract title, date, content from detail page"""
    # Title - try meta first
    title = ""
    m = re.search(r'<meta[^>]*name="ArticleTitle"[^>]*content="([^"]*)"', html)
    if m:
        title = m.group(1).strip()
    if not title:
        m = re.search(r'<title>(.*?)</title>', html)
        if m:
            title = m.group(1).replace("峡江县人民政府-公告公示 ", "").replace("峡江县人民政府-", "").strip()
    
    # Date
    date_str = ""
    m = re.search(r'PubDate[^>]*content="([^"]*)"', html)
    if m:
        dm = re.search(r'(\d{4}-\d{2}-\d{2})', m.group(1))
        if dm:
            date_str = dm.group(1)
    if not date_str:
        m = re.search(r'(\d{4}-\d{2}-\d{2})', html)
        if m:
            date_str = m.group(1)
    
    # Content - div.show_content or div.content
    content = ""
    for pat in ['class="show_content"', 'class="content"', 'id="zoom"']:
        idx = html.find(pat)
        if idx < 0:
            continue
        # Find opening tag
        start = html.rfind("<div", 0, idx)
        if start < 0:
            continue
        # Count div depth to find matching close
        depth = 0
        for i in range(start, len(html)):
            if html[i:i+4] == "<div" and (i+4 >= len(html) or html[i+4] in " >\n\r\t"):
                depth += 1
            elif html[i:i+6] == "</div>":
                depth -= 1
                if depth == 0:
                    raw = html[start:i+6]
                    # Clean scripts and styles
                    raw = re.sub(r'<(script|style)[^>]*>.*?</\1>', '', raw, flags=re.DOTALL|re.I)
                    raw = raw.strip()
                    if len(raw) > 50:
                        content = raw
                    break
        if content:
            break
    
    return title, date_str, content

def crawl():
    conn = sqlite3.connect(SEARCH_DB, timeout=60)
    cursor = conn.cursor()
    
    all_items = []
    for pg in range(1, MAX_PAGES + 1):
        items, total = fetch_list(pg)
        if not items:
            if pg == 1:
                print(f"[{SCRIPT_NAME}] ERROR: API returned empty!")
                return
            else:
                print(f"[{SCRIPT_NAME}] Page {pg}: empty, done")
                break
        print(f"[{SCRIPT_NAME}] Page {pg}: {len(items)} items (total={total})")
        for item in items:
            all_items.append({
                "url": f"{BASE_URL}/news-show-{item.get('id', '')}.html",
                "title": item.get("title", ""),
                "date": str(item.get("inputtime", ""))[:10],
            })
        time.sleep(0.3)
    
    print(f"[{SCRIPT_NAME}] Total items from API: {len(all_items)}")
    
    new_count = 0
    skip_count = 0
    
    for item in all_items:
        item_url = item["url"]
        item_title = item["title"]
        item_date = item["date"]
        
        # Check if exists
        existing = cursor.execute(
            "SELECT id FROM gov_raw WHERE page_url = ?", (item_url,)
        ).fetchone()
        if existing:
            skip_count += 1
            continue
        
        # Fetch detail
        detail_html = fetch_detail(item_url)
        if not detail_html:
            skip_count += 1
            continue
        
        real_title, real_date, content = parse_detail(detail_html)
        if not real_title:
            real_title = item_title
        if not real_date:
            real_date = item_date
        if not content:
            content = "正文为空"
        
        try:
            cursor.execute("""
                INSERT INTO gov_raw (source_url, page_url, title, publish_date, site_name, content, group_name, industry)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?)
            """, (item_url, item_url, real_title, real_date, SITE_NAME, content, GROUP, INDUSTRY))
            conn.commit()
            new_count += 1
            print(f"[{SCRIPT_NAME}] +{new_count}: {real_title[:40]} ({real_date})")
        except Exception as e:
            conn.rollback()
            skip_count += 1
        
        time.sleep(0.5)
    
    conn.close()
    print(f"\n[{SCRIPT_NAME}] Done. New: {new_count}, Skipped: {skip_count}")

if __name__ == "__main__":
    crawl()