#!/usr/bin/env python3
"""
第七师胡杨河市(南谯) - 生态环境 (nqs.gov.cn)
列表: /ztzl/xzzfgs/bm5/sthjj7/zhgs16
详情: /ztzl/xzzfgs/bm5/sthjj7/zhgs16/content_XXXXX
正文: div.body
"""
import sys, os, re, time
from datetime import datetime, timezone, timedelta
import requests, urllib3
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

SITE_NAME = "第七师胡杨河市-生态环境"
BASE_URL = "http://www.nqs.gov.cn"
LIST_URL = "http://www.nqs.gov.cn/ztzl/xzzfgs/bm5/sthjj7/zhgs16"
MAX_PAGES = 5
CUTOFF = (datetime.now(timezone.utc) - timedelta(days=365*3)).strftime("%Y-%m-%d")
HEADERS = {"User-Agent": "Mozilla/5.0"}

def fetch(url):
    try:
        r = requests.get(url, headers=HEADERS, timeout=20, verify=False)
        r.encoding = 'utf-8'
        return r.text
    except:
        return None

def parse_list(html):
    """<ul class="newsList"><li><span class="date">date</span><a href="..." title="...">title</a></li>"""
    items = []
    m = re.search(r'<ul class="newsList">(.*?)</ul>', html, re.DOTALL)
    if not m:
        return items
    ul = m.group(1)
    for m2 in re.finditer(r'<li[^>]*>.*?<span[^>]*class="date"[^>]*>(\d{4}-\d{2}-\d{2}).*?<a[^>]*href="([^"]+)"[^>]*>(.*?)</a>', ul, re.DOTALL):
        date, href, title = m2.group(1), m2.group(2), re.sub(r'<[^>]+>', '', m2.group(3)).strip()
        if not href.startswith('http'):
            href = BASE_URL + href
        items.append((title, href, date))
    return items

def get_total_pages(html):
    return 1  # 单页，全部展示

def fetch_detail(url):
    html = fetch(url)
    if not html:
        return None, None, None
    title = ""
    m = re.search(r'ArticleTitle["\']\s*content=["\']([^"\']+)', html, re.I)
    if m: title = m.group(1).strip()
    if not title:
        m = re.search(r'<h2[^>]*class="title"[^>]*>(.*?)</h2>', html, re.DOTALL)
        if m: title = re.sub(r'<[^>]+>', '', m.group(1)).strip()
    if not title:
        m = re.search(r'<title>([^<]+)', html)
        if m: title = re.sub(r'[-_—|].*', '', m.group(1)).strip()
    content = ""
    m = re.search(r'<article[^>]*class="articleCon"[^>]*>(.*?)</article>', html, re.DOTALL)
    if m:
        raw = m.group(1)
        raw = re.sub(r'<script[^>]*>.*?</script>', '', raw, flags=re.DOTALL|re.I)
        raw = re.sub(r'<style[^>]*>.*?</style>', '', raw, flags=re.DOTALL|re.I)
        raw = re.sub(r' style="[^"]*"', '', raw)
        content = raw.strip()
    date = ""
    m = re.search(r'PubDate["\'][^>]*content=["\']([^"\']+)', html, re.I)
    if m: date = m.group(1)[:10]
    return title, content, date

def run(args=None):
    incremental = False
    if args and '1' in args:
        incremental = True
    print(f"爬取: {SITE_NAME}")
    html = fetch(LIST_URL)
    if not html:
        print("  ❌ 无法获取列表页")
        return
    items = parse_list(html)
    print(f"  列表: {len(items)} 条")
    
    results = []
    for i, (title, url, date) in enumerate(items):
        print(f"  [{i+1}/{len(items)}] {title[:40]}...", end=" ", flush=True)
        dt, content, d2 = fetch_detail(url)
        if not content or len(content) < 30:
            print("内容过短", content[:50])
            continue
        results.append({
            "site_name": SITE_NAME, "title": dt or title,
            "url": url, "content": content,
            "pub_date": d2 or date,
            "category": "环评公示", "tags": "第七师",
        })
        print(f"✅ ({len(content)}字)")
        time.sleep(0.3)
    
    if results:
        print(f"\n  入库 {len(results)} 条")
        push_to_searchdb(results, "nqs_sthj")
    print("完成")

if __name__ == "__main__":
    run(sys.argv[1] if len(sys.argv)>1 else None)