#!/usr/bin/env python3
"""
Crawler for 湘潭县云湖桥镇 - 信息公开
https://www.xtx.gov.cn/9932/9940/
"""
import sys, os, re, time
from datetime import datetime, date
from urllib.parse import urljoin
import requests

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

SITE_NAME = "xtx.gov.cn-云湖桥镇-信息公开"
BASE_URL = "https://www.xtx.gov.cn/9932/9940/"
CATEGORY = "环评公示"
CUTOFF = date(2023, 6, 16)
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}


def fetch_html(url):
    try:
        r = requests.get(url, headers=HEADERS, timeout=30, verify=False)
        r.encoding = 'gb2312'
        return r.text
    except Exception as e:
        print(f"  [ERROR] fetch failed: {e}")
        return None


def extract_meta(html, name):
    m = re.search(r'<meta[^>]*name="' + name + r'"[^>]*content=[\'"](.*?)[\'"]', html)
    return m.group(1).strip() if m else ""


def extract_content(html):
    """Extract article body from xl-xqnr div."""
    idx = html.find("xl-xqnr")
    if idx < 0:
        return ""
    
    start = html.rfind("<div", 0, idx)
    if start < 0:
        start = idx
    
    content = html[start:]
    depth = 0
    for i, c in enumerate(content):
        if c == '<':
            if content[i:i+4] == '<!--':
                end = content.find('-->', i+4)
                if end > i:
                    i = end + 3
                    continue
            if content[i:i+4] == '<div' and (len(content) <= i+4 or content[i+4] in (' ', '>', 'c', 'i')):
                depth += 1
            elif content[i:i+6] == '</div>' and i > 0 and content[i-1] != '!':
                depth -= 1
                if depth == 0:
                    content = content[:i+6]
                    break
    
    content = re.sub(r'<script[^>]*>.*?</script>', '', content, flags=re.DOTALL|re.I)
    content = re.sub(r'<style[^>]*>.*?</style>', '', content, flags=re.DOTALL|re.I)
    return content.strip()


def extract_items_from_page(page_url):
    items = []
    html = fetch_html(page_url)
    if not html:
        return items
    
    tbody_m = re.search(r'<tbody>(.*?)</tbody>', html, re.DOTALL)
    if not tbody_m:
        return items
    
    tbody = tbody_m.group(1)
    for tr in re.finditer(r'<tr[^>]*>(.*?)</tr>', tbody, re.DOTALL):
        tds = re.findall(r'<td[^>]*>(.*?)</td>', tr.group(1), re.DOTALL)
        if len(tds) < 3:
            continue
        
        a_m = re.search(r'<a[^>]*href=[\'"](.*?)[\'"][^>]*>(.*?)</a>', tds[1], re.DOTALL)
        if not a_m:
            continue
        
        href = a_m.group(1).strip()
        title = re.sub(r'<[^>]+>', '', a_m.group(2)).strip()
        if not title:
            continue
        
        date_str = re.sub(r'<[^>]+>', '', tds[2]).strip()
        items.append((urljoin(page_url, href), title, date_str))
    
    return items


def main():
    print(f"站点: {SITE_NAME}  截止: {CUTOFF}\n")
    
    # Collect all pages
    print("[收集列表页...]")
    all_items = []
    
    # Pages 1-5: static HTML
    for p in range(5):
        url = BASE_URL if p == 0 else urljoin(BASE_URL, f"index_{p}.htm")
        items = extract_items_from_page(url)
        print(f"  page {p} (htm): {len(items)} items")
        all_items.extend(items)
    
    # Pages 6-8: JSP dynamic
    for offset in [75, 90, 105]:
        jsp_url = urljoin(BASE_URL, f"index.jsp?pager.offset={offset}&pager.desc=false")
        items = extract_items_from_page(jsp_url)
        print(f"  offset={offset} (jsp): {len(items)} items")
        all_items.extend(items)
        if len(items) < 15:
            break  # Last page
    
    print(f"\n总计: {len(all_items)} 条")
    
    # Process details
    print("\n[处理详情页...]")
    batch = []
    skip = 0
    
    for i, (href, title, date_str) in enumerate(all_items, 1):
        print(f"\n[{i}/{len(all_items)}]", end=" ")
        
        # Parse date
        dt = None
        try:
            dt = datetime.strptime(date_str.strip(), "%Y-%m-%d").date()
        except:
            pass
        
        if dt and dt < CUTOFF:
            print(f"[SKIP] {dt}")
            skip += 1
            continue
        
        # Fetch detail
        html = fetch_html(href)
        if not html:
            print("[ERROR] fetch fail")
            skip += 1
            continue
        
        meta_title = extract_meta(html, "ArticleTitle") or title
        pub_date = extract_meta(html, "PubDate")[:10]
        
        if pub_date:
            try:
                dt = datetime.strptime(pub_date, "%Y-%m-%d").date()
            except:
                pass
        
        if dt and dt < CUTOFF:
            print(f"[SKIP] {dt}")
            skip += 1
            continue
        
        content = extract_content(html)
        
        # Fix relative attachment URLs
        content = re.sub(
            r'href="(\.\./\.\./uploadfiles/|uploadfiles/)',
            f'href="https://www.xtx.gov.cn/',
            content
        )
        
        summary = re.sub(r'<[^>]+>', ' ', content)
        summary = re.sub(r'\s+', ' ', summary).strip()[:300]
        
        entry = {
            "site_name": SITE_NAME,
            "title": meta_title,
            "url": href,
            "source_url": href,
            "pub_date": str(dt) if dt else date_str,
            "content": content,
            "summary": summary,
            "category": CATEGORY,
            "tags": "",
        }
        batch.append(entry)
        print(f"[OK] {meta_title[:40]}...")
        time.sleep(0.3)
    
    print(f"\n{'='*50}")
    print(f"入库: {len(batch)}, 跳过: {skip}")
    if batch:
        push_to_searchdb(batch)
    print(f"{'='*50}")


if __name__ == "__main__":
    main()