#!/usr/bin/env python3
"""
Crawl 朝阳县 - 建设项目环境影响评价审批
https://www.cyx.gov.cn/cyxzf/zwgk/zwgkbzhgfhzl/gkly/sthj/xzxk/jsxm/glist.html
"""
import requests, re, sqlite3, os, time, socket
from datetime import datetime, timedelta
import os

# cyx.gov.cn DNS 间歇性失败 (2026-08-24 发现: 系统DNS超时, IP直连42.4.4.248通)
# 强制 IP 直连绕过不稳定 DNS, 否则日跑会静默失败(+0条)
_CYX_IP = "42.4.4.248"
_CYX_HOST = "www.cyx.gov.cn"
_orig_getaddrinfo = socket.getaddrinfo
def _patched_getaddrinfo(host, *args, **kwargs):
    if host == _CYX_HOST:
        host = _CYX_IP
    return _orig_getaddrinfo(host, *args, **kwargs)
socket.getaddrinfo = _patched_getaddrinfo

SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
NAME = "朝阳县-建设项目环评审批"
DOMAIN = "www.cyx.gov.cn"
BASE = "https://www.cyx.gov.cn"
LIST_URL = "/cyxzf/zwgk/zwgkbzhgfhzl/gkly/sthj/xzxk/jsxm/glist"

PAGES = 4
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/125.0.0.0"}
NOW = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
CUTOFF_DATE = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")

def extract_content(html):
    """Extract from main-info-info div"""
    idx = html.find('class="main-info-info"')
    if idx < 0:
        return ""
    
    div_start = html.rfind("<div", 0, idx)
    if div_start < 0:
        return ""
    
    section = html[div_start:]
    depth = 0
    content = ""
    for i in range(len(section)):
        if section[i:i+4] == "<div" and (i+4 >= len(section) or section[i+4] in " >\n\r\t"):
            depth += 1
        elif section[i:i+6] == "</div>":
            depth -= 1
            if depth == 0:
                gt_pos = section.find(">", 0, i)
                if gt_pos > 0:
                    content = section[gt_pos+1:i]
                else:
                    content = section[7:i]
                break
    
    content = re.sub(r'<script[^>]*>.*?</script>', '', content, flags=re.DOTALL|re.I)
    content = re.sub(r'<style[^>]*>.*?</style>', '', content, flags=re.DOTALL|re.I)
    content = re.sub(r'<!--.*?-->', '', content, flags=re.DOTALL)
    return content.strip()

def fetch_detail(session, url):
    """Fetch detail page"""
    try:
        r = session.get(url, timeout=30)
        if r.status_code != 200:
            return None, None, None
        r.encoding = "utf-8"
        html = r.text
    except:
        return None, None, None
    
    content = extract_content(html)
    
    # Title
    title = ""
    tm = re.search(r"<title>(.*?)<", html)
    if tm:
        title = tm.group(1).strip()
        # Remove suffix
        title = re.sub(r'-.*$', '', title).strip()
    
    # Date from meta or display
    date_str = ""
    for pat in [
        r'<meta name="PubDate"[^>]*content="([^"]+)"',
        r'发布时间[：:]\s*(\d{4}-\d{2}-\d{2})',
    ]:
        dm = re.search(pat, html)
        if dm:
            raw = dm.group(1).strip().split(" ")[0]
            try:
                dt = datetime.strptime(raw, "%Y-%m-%d")
                date_str = dt.strftime("%Y-%m-%d")
            except:
                date_str = raw
            break
    
    summary = re.sub(r'<[^>]+>', '', content)[:200] if content else ""
    summary = re.sub(r'\s+', ' ', summary).strip()
    
    return title, date_str, content

def parse_list(html):
    """Extract items from list page"""
    items = []
    for m in re.finditer(
        r'<li><a[^>]*href="([^"]+)"[^>]*>(.*?)</a><span>(\d{4}-\d{2}-\d{2})</span>',
        html, re.DOTALL
    ):
        url = m.group(1).strip()
        title = m.group(2).strip()
        date = m.group(3).strip()
        if not url.startswith("http"):
            url = BASE + url
        items.append({"url": url, "title": title, "date": date})
    return items

def insert_item(conn, item):
    """Insert or replace one record"""
    title = item["title"].replace("'", "''")
    page_url = item["url"].replace("'", "''")
    content = item.get("content", "").replace("'", "''")
    summary = item.get("summary", "").replace("'", "''")
    date = item.get("date", "").replace("'", "''")
    
    date_rank = 0
    try:
        date_rank = int(datetime.strptime(date, "%Y-%m-%d").timestamp())
    except:
        date_rank = int(time.time())
    
    sql = f"""INSERT OR REPLACE INTO gov_raw
        (site_name, page_url, title, publish_date, content, summary, date_rank, category)
        VALUES ('{NAME}', '{page_url}', '{title}', '{date}',
                '{content}', '{summary}', {date_rank}, '环评审批')"""
    try:
        conn.execute(sql)
        conn.commit()
        return True
    except Exception as e:
        print(f"    ❌ DB: {e}")
        return False

def main():
    print(f"\n{'='*60}")
    print(f"🚀 {NAME}")
    print(f"   页面: 1~{PAGES}")
    print(f"   日期截止: {CUTOFF_DATE}")
    print(f"{'='*60}\n")
    
    session = requests.Session()
    session.headers.update(HEADERS)
    conn = sqlite3.connect(SEARCH_DB, timeout=60)
    conn.execute("PRAGMA journal_mode=WAL")
    
    total_new = 0
    start_time = time.time()
    
    for page in range(0, PAGES):
        if page == 0:
            list_url = f"{BASE}{LIST_URL}.html"
        else:
            list_url = f"{BASE}{LIST_URL}{page+1}.html"
        
        print(f"📄 第{page+1}/{PAGES}页")
        
        try:
            r = session.get(list_url, timeout=20)
            if r.status_code != 200:
                print(f"  ❌ HTTP {r.status_code}")
                continue
            r.encoding = "utf-8"
            items = parse_list(r.text)
        except Exception as e:
            print(f"  ❌ 错误: {e}")
            continue
        
        if not items:
            print(f"  ⚠️ 无数据")
            continue
        
        print(f"  📋 {len(items)} 条")
        
        for idx, item in enumerate(items):
            if item["date"] < CUTOFF_DATE:
                continue
            
            has = conn.execute(
                "SELECT 1 FROM gov_raw WHERE page_url=? AND content IS NOT NULL AND content != ''",
                (item["url"],)
            ).fetchone()
            
            if has:
                print(f"  [{idx+1}/{len(items)}] ⏭️ {item['title'][:35]}...")
                continue
            
            print(f"  [{idx+1}/{len(items)}] {item['title'][:35]}...", end=" ", flush=True)
            
            title, date, content = fetch_detail(session, item["url"])
            
            if content:
                item["content"] = content
                item["summary"] = re.sub(r'<[^>]+>', '', content)[:200] if content else item["title"]
                item["summary"] = re.sub(r'\s+', ' ', item["summary"]).strip()
                item["title"] = title or item["title"]
                item["date"] = date or item["date"]
                if insert_item(conn, item):
                    total_new += 1
                    print(f"✅ {len(content):,}字")
                else:
                    print(f"❌ DB失败")
            else:
                item["content"] = ""
                item["summary"] = item["title"]
                if insert_item(conn, item):
                    print(f"⚠️ 无正文")
                else:
                    print(f"❌ DB失败")
            
            time.sleep(0.5)
        
        print()
    
    elapsed = time.time() - start_time
    print(f"{'='*60}")
    print(f"📊 完成！新增 {total_new} 条")
    print(f"⏱ 耗时: {elapsed:.0f}s ({elapsed/60:.1f}min)")
    print(f"{'='*60}")
    conn.close()
    return total_new

if __name__ == "__main__":
    main()