#!/usr/bin/env python3
"""crawl_jingmen_jtysj.py — 荆门市交通运输局 公示公告 (Hanweb)"""
import os, re, sys, json
from datetime import datetime, timedelta
try:
    from urllib.request import Request, urlopen
except ImportError:
    from urllib2 import Request, urlopen

DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "jtysj.jingmen.gov.cn-公示公告"
BASE_URL = "http://jtysj.jingmen.gov.cn"
CUTOFF_DATE = (datetime.now() - timedelta(days=int(sys.argv[1]) if len(sys.argv) > 1 else 365*3)).strftime("%Y-%m-%d")
CATEGORY = "公示公告"
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

TOTAL_RECORDS = 225
PER_PAGE = 25
PROXY_URL = "/module/web/jpage/dataproxy.jsp"
WEBID = "8"
COLUMNID = "3078"
UNITID = "55055"

def fetch(url, encoding="utf-8"):
    req = Request(url, headers=HEADERS)
    resp = urlopen(req, timeout=30)
    data = resp.read()
    return data.decode(encoding, errors="replace")

def post_fetch(url, data_bytes):
    """POST方式获取数据"""
    req = Request(url, data=data_bytes, headers=HEADERS)
    resp = urlopen(req, timeout=30)
    return resp.read().decode("utf-8", errors="replace")

def parse_records_from_xml(xml_text):
    """从XML recordset中解析记录"""
    records = []
    for m in re.finditer(r'<record><!\[CDATA\[(.*?)\]\]></record>', xml_text, re.DOTALL):
        cdata = m.group(1)
        a = re.search(r'<a[^>]*href="([^"]+)"[^>]*title="([^"]*)"', cdata)
        if a:
            href = a.group(1)
            title = a.group(2)
            date_m = re.search(r'<span>(\d{4}-\d{2}-\d{2})</span>', cdata)
            date = date_m.group(1) if date_m else ""
            if href.startswith("/"):
                href = BASE_URL + href
            records.append((href, title.strip(), date))
    return records

def parse_list_page(url):
    """解析列表页，从XML record中提取 (url, title, date)"""
    html = fetch(url)
    return parse_records_from_xml(html)

def parse_ajax_page(page):
    """通过AJAX接口获取分页数据（POST请求）"""
    params = "page=%d&col=1&webid=%s&path=/&columnid=%s&unitid=%s&sourceContentType=1&webname=%s&permissiontype=0" % (
        page, WEBID, COLUMNID, UNITID, "荆门市交通运输局")
    url = BASE_URL + PROXY_URL
    html = post_fetch(url, params.encode("utf-8"))
    return parse_records_from_xml(html)

def parse_detail(url):
    """解析详情页，返回 (title, pub_date, source, content_html)"""
    html = fetch(url)
    
    # 标题
    title = ""
    t = re.search(r'<p class="con-title">\s*([^<]+)</p>', html)
    if t:
        title = t.group(1).strip()
    if not title:
        t = re.search(r'<meta name="ArticleTitle"[^>]*content="([^"]+)"', html)
        if t:
            title = t.group(1).strip()
    
    # 日期
    pub_date = ""
    p = re.search(r'<meta name="PubDate"[^>]*content="(\d{4}-\d{2}-\d{2})', html)
    if p:
        pub_date = p.group(1)
    if not pub_date:
        p = re.search(r'发布日期[：:](\d{4}-\d{2}-\d{2})', html)
        if p:
            pub_date = p.group(1)
    
    # 来源
    source = ""
    s = re.search(r'<meta name="contentSource"[^>]*content="([^"]+)"', html)
    if s:
        source = s.group(1).strip()
    if not source:
        s = re.search(r'信息来源[：:]([^<]+)', html)
        if s:
            source = s.group(1).strip()
    
    # 正文 - class="main-txt" id="content"
    content_html = ""
    content_start = html.find('<div class="main-txt" id="content">')
    if content_start < 0:
        content_start = html.find('class="main-txt"')
    if content_start > 0:
        inner = html[content_start:]
        # 结束：下一个 </div> 匹配到 con-main 层级外
        depth = 0
        end_pos = -1
        for i, ch in enumerate(inner):
            if ch == '<':
                if inner[i:i+6] == '</div>':
                    depth -= 1
                    if depth < 0:
                        # 找到了 main-txt 的闭合
                        # 但 main-txt 在 con-main 内，所以深度为零时是对的
                        end_pos = i + 6
                        break
                elif inner[i:i+2] in ['<d', '<D'] and inner[i:i+4] in ['<div', '<Div', '<DIV']:
                    depth += 1
                elif inner[i:i+3] in ['<p>', '<P>'] or inner[i:i+4].startswith('<p '):
                    pass  # p 标签不需要计数深度
        if end_pos > 0:
            content_html = inner[:end_pos]
            # 去掉ZJEG_RSS标签和ContentStart/ContentEnd
            content_html = re.sub(r'<!--ZJEG_RSS\.content\.(begin|end)-->', '', content_html)
            content_html = re.sub(r'<meta name="ContentStart">', '', content_html)
            content_html = re.sub(r'<meta name="ContentEnd">', '', content_html)
    
    return title, pub_date, source, content_html

def save_to_db(records_data):
    """批量保存到SQLite"""
    import sqlite3
    conn = sqlite3.connect(DB_PATH, timeout=60)
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute("PRAGMA busy_timeout=5000")
    c = conn.cursor()
    
    # 确保表存在
    c.execute("""CREATE TABLE IF NOT EXISTS gov_raw (
        id INTEGER PRIMARY KEY, site_name TEXT, source_url TEXT,
        page_url TEXT, title TEXT, publish_date TEXT,
        date_rank INTEGER DEFAULT 0, summary TEXT, status TEXT,
        category TEXT DEFAULT '', visits INTEGER DEFAULT 0,
        content TEXT DEFAULT '', tags TEXT DEFAULT ''
    )""")
    c.execute("CREATE UNIQUE INDEX IF NOT EXISTS idx_temp_page_url ON gov_raw(page_url)")
    
    total = len(records_data)
    added = 0
    for i, (url, title, pub_date, source, content) in enumerate(records_data):
        date_rank = 0
        if pub_date:
            try:
                dt = datetime.strptime(pub_date, "%Y-%m-%d")
                date_rank = int(dt.strftime("%Y%m%d"))
            except:
                pass
        c.execute("""INSERT OR IGNORE INTO gov_raw
            (page_url, site_name, title, publish_date, source_url, content, category, date_rank, summary, status, visits, tags)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?, '', 'published', 0, '')""",
            (url, SITE_NAME, title, pub_date, source, content, CATEGORY, date_rank))
        if c.rowcount > 0:
            added += 1
        if (i + 1) % 25 == 0:
            conn.commit()
            print(f"  [{i+1}/{total}] 已入库 {added} 条新数据")
    conn.commit()
    conn.close()
    return added

def main():
    print(f"站点: {SITE_NAME}")
    print(f"截止日期: {CUTOFF_DATE}")
    print(f"总记录数: {TOTAL_RECORDS} 条, 每页 {PER_PAGE} 条")
    
    # 第1步：获取列表
    all_records = []
    
    # 首页（从HTML中提取XML record，含前3组约75条）
    print("\n[1/2] 解析列表页...")
    records = parse_list_page(BASE_URL + "/col/col3078/index.html")
    print(f"  首页提取: {len(records)} 条")
    all_records.extend(records)
    
    # AJAX分页补充（返回全量，但去重只取遗漏的旧数据）
    try:
        records2 = parse_ajax_page(1)
        print(f"  AJAX补充: {len(records2)} 条")
        existing = {(u, t, d) for u, t, d in all_records}
        new_recs = [(u, t, d) for u, t, d in records2 if (u, t, d) not in existing]
        print(f"  其中新增: {len(new_recs)} 条")
        all_records.extend(new_recs)
    except Exception as e:
        print(f"  AJAX补充失败: {e}")
    
    print(f"\n列表共获取: {len(all_records)} 条")
    
    # 筛选日期
    filtered = [(u, t, d) for u, t, d in all_records if d >= CUTOFF_DATE]
    print(f"  日期筛选后 ({CUTOFF_DATE} 后): {len(filtered)} 条")
    
    if not filtered:
        print("  无新数据，跳过详情解析")
        return
    
    # 第2步：获取详情
    print(f"\n[2/2] 解析详情页 ({len(filtered)} 条)...")
    detail_data = []
    for i, (url, title, date) in enumerate(filtered):
        try:
            t, pub_date, src, content = parse_detail(url)
            final_title = t or title
            final_date = pub_date or date
            final_src = src or SITE_NAME
            if content and len(content.strip()) > 50:
                detail_data.append((url, final_title, final_date, final_src, content))
                print(f"  [{i+1}/{len(filtered)}] ✓ {final_title[:40]}...")
            else:
                # 正文太短可能是附件类，存标题+URL
                detail_data.append((url, final_title, final_date, final_src, ""))
                print(f"  [{i+1}/{len(filtered)}] ⚠ 正文过短: {final_title[:40]}...")
        except Exception as e:
            print(f"  [{i+1}/{len(filtered)}] ✗ {url}: {e}")
        # 每页提交
        if (i + 1) % 10 == 0:
            pass  # 后面统一入库
    
    # 第3步：入库
    print(f"\n入库 {len(detail_data)} 条...")
    added = save_to_db(detail_data)
    print(f"\n✅ 完成！新增 {added} 条，累计 {len(detail_data)} 条")

if __name__ == "__main__":
    main()