#!/usr/bin/env python3
"""
Crawl 青岛市生态环境局 (mbee.qingdao.gov.cn) - 建设项目环境影响评价公示 + 政务服务
ASP.NET WebForms + UpdatePanel pagination.
All data written to /root/search.db gov_raw table.
"""

import sys, re, json, time, os, html as html_mod
import requests
from bs4 import BeautifulSoup
from datetime import datetime, timedelta
import os

BASE_URL = "http://mbee.qingdao.gov.cn:8082"
DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "青岛市生态环境局"

# All sub-sections to crawl
SECTIONS = [
    {
        "name": "审批受理情况公示",
        "list_path": "/m2/ZWGKNew/webgs/list1.aspx",
        "detail_prefix": "/m2/webgs/list1detaile.aspx",
        "m": 357, "max_pages": 0,
        "date_field": "受理时间"
    },
    {
        "name": "拟审批准情况公示",
        "list_path": "/m2/ZWGKNew/webgs/list2.aspx",
        "detail_prefix": "/m2/webgs/list2detaile.aspx",
        "m": 358, "max_pages": 0,
        "date_field": "拟审批时间"
    },
    {
        "name": "审批决定公示",
        "list_path": "/m2/ZWGKNew/webgs/list3.aspx",
        "detail_prefix": "/m2/webgs/list3detaile.aspx",
        "m": 359, "max_pages": 0,
        "date_field": "发文时间"
    },
    {
        "name": "告知承诺审批决定公示",
        "list_path": "/m2/ZWGKNew/webgs/gzcnsgslist.aspx",
        "detail_prefix": "/m2/webgs/gzcnsgsdetaile.aspx",
        "m": 473, "max_pages": 0,
        "date_field": "发文时间"
    },
    {
        "name": "验收决定公示",
        "list_path": "/m2/ZWGKNew/webys/list3.aspx",
        "detail_prefix": "/m2/webys/list3detaile.aspx",
        "m": 362, "max_pages": 0,
        "date_field": "发文时间"
    },
]

# Filter: only keep records from last 3 years
THREE_YEARS_AGO = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")

session = requests.Session()
session.headers.update({
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
})

def extract_viewstate(html_text):
    """Extract ASP.NET __VIEWSTATE, __VIEWSTATEGENERATOR, __EVENTVALIDATION"""
    result = {}
    for name in ['__VIEWSTATE', '__VIEWSTATEGENERATOR', '__EVENTVALIDATION']:
        m = re.search(r'id="' + name + r'"[^>]*value="([^"]*)"', html_text)
        if m:
            result[name] = html_mod.unescape(m.group(1))
    return result

def fetch_page(url, page_num=1, viewstate=None):
    """Fetch a page, returning (html, new_viewstate). 
    For page 1, does GET. For page 2+, POSTs with __doPostBack."""
    if page_num == 1 or viewstate is None:
        resp = session.get(url, timeout=30)
        html = resp.text
        new_vs = extract_viewstate(html)
        return html, new_vs
    
    # POST for subsequent pages
    post_data = {
        **viewstate,
        '__EVENTTARGET': 'ctl00$right_content$Pagination$pg',
        '__EVENTARGUMENT': str(page_num),
        '__LASTFOCUS': '',
    }
    resp = session.post(url, data=post_data, timeout=30)
    html = resp.text
    new_vs = extract_viewstate(html)
    return html, new_vs

def parse_list_page(html_text, section):
    """Parse list page and return list of dicts with title, url, date, summary"""
    records = []
    soup = BeautifulSoup(html_text, 'html.parser')
    
    # Find table - could be in UpdatePanel or directly
    updatepanel = soup.find('div', id='right_content_UpdatePanel1')
    if updatepanel:
        table = updatepanel.find('table')
    else:
        table = soup.find('table', class_='table-list')
    
    if not table:
        return records
    
    rows = table.find_all('tr')
    for row in rows:
        cells = row.find_all('td')
        if len(cells) < 4:
            continue
        
        # Try to find link in first cell
        link = cells[0].find('a') if cells[0] else None
        if not link:
            continue
        
        title = link.get_text(strip=True)
        href = link.get('href', '')
        if not href or not title:
            continue
        
        # Make href absolute
        if href.startswith('/'):
            href = BASE_URL + href
        elif not href.startswith('http'):
            href = BASE_URL + '/' + href.lstrip('/')
        
        # Date from table (last column or second-to-last)
        date_str = ""
        for cell in reversed(cells):
            txt = cell.get_text(strip=True)
            dm = re.search(r'(\d{4}[-/]\d{1,2}[-/]\d{1,2})', txt)
            if dm:
                date_str = dm.group(1).replace('/', '-')
                break
        
        # Summary from table
        summary_parts = []
        for cell in cells[1:-1]:
            txt = cell.get_text(strip=True)
            if txt:
                summary_parts.append(txt)
        summary = ' | '.join(summary_parts)
        
        records.append({
            "title": title,
            "url": href,
            "date": date_str,
            "summary": summary,
            "section": section["name"],
        })
    
    return records

def fetch_detail(url):
    """Fetch detail page, return (content_html, date_str, full_title)"""
    try:
        resp = session.get(url, timeout=30)
        resp.encoding = 'utf-8'
    except Exception as e:
        return "", "", ""

    soup = BeautifulSoup(resp.text, 'html.parser')

    # Find content container
    detail = soup.find('div', class_='detail-box')
    if not detail:
        detail = soup.find('div', class_='table-detail')

    content_html = ""
    date_str = ""
    full_title = ""

    if detail:
        # Extract content preserving HTML structure
        content_html = str(detail)

        # Extract full title from "项目名称：" field
        name_label = detail.find("td", string=lambda t: t and "项目名称" in t)
        if name_label:
            name_cell = name_label.find_next_sibling("td")
            if name_cell:
                full_title = name_cell.get_text(strip=True)

        # Try to find date from the detail page
        date_match = re.search(r'(?:受理时间|发文时间|公示时间|拟审批时间|审批时间)[：:]\s*(\d{4}年\d{1,2}月\d{1,2}日)', detail.get_text())
        if date_match:
            dt = date_match.group(1)
            dt = dt.replace('年', '-').replace('月', '-').replace('日', '')
            date_str = dt

    return content_html, date_str, full_title


def insert_to_db(records, conn):
    """Insert records into gov_raw table"""
    cursor = conn.cursor()
    inserted = 0
    skipped = 0
    
    for rec in records:
        # Skip if date is too old
        if rec["date"] and rec["date"] < THREE_YEARS_AGO:
            skipped += 1
            continue
        
        try:
            cursor.execute("""
                INSERT OR IGNORE INTO gov_raw 
                (page_url, source_url, title, summary, content, publish_date, site_name)
                VALUES (?, ?, ?, ?, ?, ?, ?)
            """, (
                rec["url"],
                "http://mbee.qingdao.gov.cn",
                rec["title"],
                rec.get("summary", ""),
                rec.get("content", ""),
                rec.get("date", ""),
                f"{SITE_NAME} - {rec.get('section', '')}",
            ))
            if cursor.rowcount > 0:
                inserted += 1
        except Exception as e:
            print(f"  DB error: {e}")
    
    conn.commit()
    return inserted, skipped

def crawl_section(section, max_pages=0, incremental=False):
    """Crawl a single section and return record count"""
    section_name = section["name"]
    list_url = f"{BASE_URL}{section['list_path']}?m={section['m']}"
    
    print(f"\n=== {section_name} ===")
    print(f"URL: {list_url}")
    
    all_records = []
    page = 1
    viewstate = None
    has_more = True
    
    # Determine max pages
    if incremental:
        pages_to_crawl = 1
    elif max_pages > 0:
        pages_to_crawl = max_pages
    elif section["max_pages"] > 0:
        pages_to_crawl = section["max_pages"]
    else:
        pages_to_crawl = 5  # default cap
    
    while has_more and page <= pages_to_crawl:
        print(f"  Page {page}...", end=" ", flush=True)
        
        try:
            html_text, viewstate = fetch_page(list_url, page, viewstate)
            
            # Check for redirect / error
            if 'Object moved' in html_text:
                print("ERROR: page redirected (invalid viewstate)")
                break
            
            records = parse_list_page(html_text, section)
            
            if not records:
                print("no records found")
                break
            
            print(f"{len(records)} records", end="", flush=True)
            all_records.extend(records)
            
            # Check pagination for continuation
            soup = BeautifulSoup(html_text, 'html.parser')
            pg = soup.find('div', class_='pagination')
            if pg:
                pg_text = pg.get_text()
                total_pages_match = re.search(r'共\s*(\d+)\s*页', pg_text)
                current_page_match = re.search(r'第\s*(\d+)\s*页', pg_text)
                
                total_pages = int(total_pages_match.group(1)) if total_pages_match else page
                
                # Log total info
                records_count = re.search(r'共\s*(\d+)\s*条', pg_text)
                if records_count:
                    print(f" / {records_count.group(1)}条 total, {total_pages}页", end="")
                
                # Update max_pages from actual total
                if section["max_pages"] == 0 and max_pages == 0:
                    # Auto-detect: if total pages < 5, use all; else cap at 5
                    if total_pages < 5:
                        pages_to_crawl = total_pages
                    # Already capped at 5
                
                # Check if there are more pages
                if current_page_match:
                    current = int(current_page_match.group(1))
                    has_more = current < total_pages and current < pages_to_crawl
                else:
                    has_more = page < total_pages and page < pages_to_crawl
            else:
                has_more = False
            
            print()
            page += 1
            
            # Small delay
            time.sleep(0.3)
            
        except Exception as e:
            print(f"ERROR: {e}")
            break
    
    print(f"  Total: {len(all_records)} records collected")
    return all_records

def main():
    incremental = 'incremental' in sys.argv or sys.argv[-1] == '1'
    
    import sqlite3
    conn = sqlite3.connect(DB_PATH, timeout=60)
    
    total_inserted = 0
    total_records = 0
    
    for section in SECTIONS:
        records = crawl_section(section, incremental=incremental)
        total_records += len(records)
        
        if not records:
            continue
        
        # Fetch detail content for each record
        print(f"  Fetching detail pages...")
        for i, rec in enumerate(records):
            if i > 0 and i % 10 == 0:
                print(f"    {i}/{len(records)}", flush=True)
            
            content, date_from_detail, full_title = fetch_detail(rec["url"])
            if content:
                rec["content"] = content
            if full_title and len(full_title) > len(rec.get("title", "")):
                rec["title"] = full_title
            if date_from_detail and not rec["date"]:
                rec["date"] = date_from_detail
            elif date_from_detail:
                # Prefer detail page date over list page date (more precise)
                rec["date"] = date_from_detail
        
        # Insert to DB
        inserted, skipped = insert_to_db(records, conn)
        print(f"  Inserted: {inserted}, Skipped (old): {skipped}")
        total_inserted += inserted
    
    conn.close()
    
    print(f"\n=== SUMMARY ===")
    print(f"Total records collected: {total_records}")
    print(f"Total inserted: {total_inserted}")

if __name__ == "__main__":
    main()