#!/usr/bin/env python3
"""
信阳/常熟化工园 - 政策公告 (58.210.182.61:10520)
列表: /gongshixinxi
详情: /gongshixinxi/{id}
正文: div.content
"""
import sys, os, re, time, sqlite3
from datetime import datetime, timezone, timedelta
import requests, urllib3
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

SITE_NAME = "常熟化工产业园-政策公告"
BASE_URL = "http://58.210.182.61:10520"
LIST_URL = "http://58.210.182.61:10520/gongshixinxi"
DB_PATH = "/root/search.db"
HEADERS = {"User-Agent": "Mozilla/5.0"}
TIMEOUT = 15


def exists_by_title(title):
    """站点会重编号文章(1471→1472 同标题重发)，page_url 唯一约束拦不住 → 按标题去重"""
    try:
        conn = sqlite3.connect(DB_PATH, timeout=60)
        conn.execute('PRAGMA busy_timeout=5000')
        r = conn.execute(
            "SELECT 1 FROM gov_raw WHERE site_name=? AND title=? LIMIT 1",
            (SITE_NAME, title)).fetchone()
        conn.close()
        return r is not None
    except Exception:
        return False

def fetch(url):
    try:
        r = requests.get(url, headers=HEADERS, timeout=TIMEOUT, verify=False)
        r.encoding = 'utf-8'
        return r.text
    except:
        return None

def parse_list(html):
    """<div class="list">...<a href="/gongshixinxi/N">title</a> date"""
    items = []
    m = re.search(r'<div[^>]*class="[^"]*list[^"]*"[^>]*>(.*?)</div>\s*</div>', html, re.DOTALL)
    if m:
        ul_html = m.group(1)
        for m2 in re.finditer(r'<a[^>]*href="([^"]+)"[^>]*>(.*?)</a>.*?(\d{4}-\d{2}-\d{2})', ul_html, re.DOTALL):
            href = m2.group(1)
            title = re.sub(r'<[^>]+>', '', m2.group(2)).strip()
            date = m2.group(3).strip()
            if title and len(title) > 5:
                if not href.startswith('http'):
                    href = BASE_URL + href
                items.append((title, href, date))
    return items

def fetch_detail(url):
    html = fetch(url)
    if not html:
        return None, None, None
    title = ""
    m = re.search(r'<h1>([^<]+)', html)
    if m: title = m.group(1).strip()
    if not title:
        m = re.search(r'<title>([^<]+)', html)
        if m: title = m.group(1).strip()
    content = ""
    # 正文在 div.article (cleaner，不含h1和metadata)
    m = re.search(r'<div[^>]*class="[^"]*article[^"]*"[^>]*>(.*?)</div>\s*</div>', html, re.DOTALL)
    if m:
        raw = m.group(1)
        raw = re.sub(r'<script[^>]*>.*?</script>', '', raw, flags=re.DOTALL|re.I)
        raw = re.sub(r' style="[^"]*"', '', raw)
        raw = re.sub(r'<br\s*/?>', '\n', raw)
        content = raw.strip()
    date = ""
    # ⚠️ 先剥注释/脚本再提取日期：CMS 模板注释里带 createDate/updateDate 内部日期
    # (如 2021-08-26 10:40:04)，裸正则会误抓成发布日期(曾抓出未来日期 2026-09-19)
    html_clean = re.sub(r'<!--.*?-->', '', html, flags=re.DOTALL)
    html_clean = re.sub(r'<script[^>]*>.*?</script>', '', html_clean, flags=re.DOTALL | re.I)
    m = re.search(r'发布时间[：:]\s*(\d{4}-\d{2}-\d{2})', html_clean)
    if m:
        date = m.group(1)
    return title, content, date

def run(args=None):
    print(f"爬取: {SITE_NAME}")
    html = fetch(LIST_URL)
    if not html:
        print("  ❌ 无法获取列表页")
        return
    items = parse_list(html)
    print(f"  列表: {len(items)} 条")
    
    results = []
    for i, (title, url, date) in enumerate(items):
        print(f"  [{i+1}/{len(items)}] {title[:40]}...", end=" ", flush=True)
        dt, content, d2 = fetch_detail(url)
        if not content or len(content) < 30:
            print("内容过短")
            continue
        if exists_by_title(dt or title):
            print("⏭️ 标题已存在，跳过")
            continue
        # 发布日期兜底：详情页日期缺失或为未来日期 → 用列表页日期
        today = datetime.now().strftime('%Y-%m-%d')
        final_date = d2 if d2 and d2 <= today else date
        results.append({
            "site_name": SITE_NAME, "title": dt or title,
            "url": url, "content": content,
            "pub_date": final_date,
            "category": "环评公示", "tags": "常熟化工园",
        })
        print(f"✅ ({len(content)}字)")
        time.sleep(0.3)
    
    if results:
        print(f"\n  入库 {len(results)} 条")
        push_to_searchdb(results, "xyhpgk")
    print("完成")

if __name__ == "__main__":
    run(sys.argv[1] if len(sys.argv)>1 else None)