#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
葫芦岛经济开发区-通知公告 爬虫 (TRS CMS)
站点: jjkfq.hld.gov.cn
栏目: /xwdt/tzgg/index.html
"""

import sys, os, re, json, sqlite3, math, urllib.request, urllib.error, warnings
warnings.filterwarnings('ignore')

_HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, _HERE)
sys.path.insert(0, os.path.join(_HERE, '..', 'crawler'))

from crawler_lib import fetch_page

SITE_NAME = "葫芦岛经济开发区-通知公告"
DOMAIN = "jjkfq.hld.gov.cn"
BASE_URL = "https://jjkfq.hld.gov.cn"
SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")

MAX_PAGES = 4  # index.html ~ index_3.html

def insert_db(items):
    if not items:
        return
    db = sqlite3.connect(SEARCH_DB, timeout=60)
    db.execute("PRAGMA journal_mode=WAL")
    db.execute("PRAGMA busy_timeout=8000")
    db.execute("PRAGMA synchronous=NORMAL")
    db.execute("""CREATE TABLE IF NOT EXISTS gov_raw (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        site_name TEXT, source_url TEXT UNIQUE, page_url TEXT,
        title TEXT, publish_date TEXT, summary TEXT,
        content TEXT, status TEXT, category TEXT, tags TEXT,
        attachments TEXT
    )""")
    db.execute("""CREATE VIRTUAL TABLE IF NOT EXISTS gov_search USING fts5(
        title, site_name, summary, content='gov_raw', content_rowid='id'
    )""")
    db.commit()
    ok = skip = 0
    for item in items:
        try:
            title = (item.get("title") or "")[:500]
            summary = (item.get("summary") or title)[:500]
            content = item.get("content") or ""
            pub_date = (item.get("publish_date") or "")[:10]
            source_url = item.get("source_url") or ""
            page_url = item.get("url") or source_url
            attachments = item.get("attachments") or ""
            db.execute("""INSERT OR IGNORE INTO gov_raw
                (site_name, source_url, page_url, title, publish_date,
                 summary, content, status, attachments)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
                (SITE_NAME, source_url, page_url, title, pub_date,
                 summary, content, "active", attachments))
            if db.total_changes > 0:
                ok += 1
            else:
                skip += 1
        except:
            skip += 1
    db.commit()
    try:
        db.execute("""INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary)
            SELECT r.id, r.title, r.site_name, r.summary
            FROM gov_raw r
            WHERE r.site_name=? AND r.id NOT IN (SELECT rowid FROM gov_search)""",
            (SITE_NAME,))
        db.commit()
    except Exception as e:
        print(f"  [FTS ERROR] {e}")
    db.close()
    print(f"  [DB] 新增: {ok}, 跳过: {skip}")
    return ok

def clean_content(html):
    if not html:
        return ""
    html = re.sub(r'<style[^>]*>.*?</style>', '', html, flags=re.DOTALL | re.IGNORECASE)
    html = re.sub(r'<script[^>]*>.*?</script>', '', html, flags=re.DOTALL | re.IGNORECASE)
    html = re.sub(r'<!--.*?-->', '', html, flags=re.DOTALL)
    return html.strip()

def extract_text(html):
    if not html:
        return ""
    text = html
    text = re.sub(r'<br\s*/?>', '\n', text, flags=re.IGNORECASE)
    text = re.sub(r'</p>', '\n\n', text, flags=re.IGNORECASE)
    text = re.sub(r'</div>', '\n', text, flags=re.IGNORECASE)
    text = re.sub(r'</tr>', '\n', text, flags=re.IGNORECASE)
    text = re.sub(r'</th>', '\n', text, flags=re.IGNORECASE)
    text = re.sub(r'</td>', '\n', text, flags=re.IGNORECASE)
    text = re.sub(r'<(?!/?(?:table|tr|td|th|thead|tbody|tfoot|img))[^>]*>', '', text)
    text = re.sub(r'&nbsp;', ' ', text)
    text = re.sub(r'&lt;', '<', text)
    text = re.sub(r'&gt;', '>', text)
    text = re.sub(r'&amp;', '&', text)
    text = re.sub(r'\n{3,}', '\n\n', text)
    text = re.sub(r'[ \t]+', ' ', text)
    text = re.sub(r'(?<=[\u4e00-\u9fff])[ \t]+(?=[\u4e00-\u9fff])', '', text)
    return text.strip()

def extract_attachments(html, base_url=BASE_URL):
    if not html:
        return ""
    ext_pattern = r'\.(doc|docx|pdf|xls|xlsx|rar|zip)(\?[^\s"\'<>]*)?'
    links = re.findall(
        r'<a[^>]*href=["\']([^"\']*?(?:' + ext_pattern + r'))["\'][^>]*>([^<]*)</a>',
        html, re.IGNORECASE)
    if not links:
        return ""
    parts = []
    for href, _, _, text in links:
        full_url = href if href.startswith('http') else (base_url.rstrip('/') + '/' + href.lstrip('/'))
        parts.append(f"[附件: {text.strip()}]({full_url})")
    return "\n".join(parts)

def fetch_list(page=0):
    """获取列表页，page=0 = index.html, page=1 = index_1.html, ..."""
    if page == 0:
        url = f"{BASE_URL}/xwdt/tzgg/index.html"
    else:
        url = f"{BASE_URL}/xwdt/tzgg/index_{page}.html"
    html = fetch_page(url, timeout=30)
    if not html:
        return []
    # 提取 li 列表
    items = []
    pattern = re.compile(
        r'<li><a\s+href=[\'"]([^\'"]+)[\'"]\s+title=[\'"]([^\'"]*)[\'"][^>]*>'
        r'(.*?)</a><span\s+class=[\'"]date[\'"]>\[([^\]]+)\]</span></li>',
        re.DOTALL | re.IGNORECASE
    )
    for m in pattern.finditer(html):
        href = m.group(1).strip()
        title = m.group(2).strip() or m.group(3).strip()
        date_str = m.group(4).strip()
        
        # 构建完整 URL
        if href.startswith('http'):
            full_url = href
        elif href.startswith('./'):
            # ./202607/t20260702_1241012.html
            # Current page path: /xwdt/tzgg/index.html
            full_url = f"{BASE_URL}/xwdt/tzgg/{href[2:]}"
        elif href.startswith('/'):
            full_url = BASE_URL + href
        else:
            full_url = f"{BASE_URL}/xwdt/tzgg/{href}"
        
        items.append({
            "title": title,
            "url": full_url,
            "date": date_str,
        })
    return items

def fetch_detail(item):
    """获取详情页内容"""
    url = item["url"]
    html = fetch_page(url, timeout=30)
    if not html:
        return None

    # 从 TRS_Editor 提取内容 (content_con 容器内)
    idx = html.find('TRS_Editor')
    if idx < 0:
        return None
    
    tag_end = html.find('>', idx)
    if tag_end < 0:
        return None
    rest = html[tag_end+1:]
    
    depth = 1
    pos = 0
    while depth > 0 and pos < len(rest):
        if rest.startswith('<div', pos):
            depth += 1
            pos += 4
        elif rest[pos:pos+6] == '</div>':
            depth -= 1
            pos += 6
        else:
            pos += 1
    raw = rest[:pos-6]
    
    cleaned = clean_content(raw)
    attachments = extract_attachments(raw)
    text_content = extract_text(cleaned)
    
    if attachments:
        text_content = text_content.rstrip() + "\n\n--- 附件 ---\n" + attachments
    
    # 提取图片
    imgs = re.findall(r'<img[^>]*src=["\']([^"\']+)["\']', raw, re.IGNORECASE)
    img_descs = []
    for img_src in imgs:
        full_src = img_src if img_src.startswith('http') else (url.rsplit('/', 1)[0] + '/' + img_src.lstrip('./'))
        img_descs.append(f"![图片]({full_src})")
    if img_descs:
        if text_content:
            text_content += "\n\n"
        text_content += "\n".join(img_descs)
    
    return {
        "title": item["title"],
        "content": text_content,
        "attachments": attachments,
    }

def run(max_pages=None):
    print(f"\n{'='*60}")
    print(f"🚀 {SITE_NAME}")
    print(f"{'='*60}")
    
    if max_pages and max_pages > 0:
        total_pages = min(max_pages, MAX_PAGES)
    else:
        total_pages = MAX_PAGES
    
    # 获取所有页
    print("\n📥 获取列表...")
    all_items = []
    for p in range(total_pages):
        items = fetch_list(p)
        if items:
            all_items.extend(items)
            print(f"   第 {p+1}/{total_pages} 页: {len(items)} 条")
        else:
            print(f"   第 {p+1}/{total_pages} 页: 空")
    
    print(f"   共获取: {len(all_items)} 条")
    
    # 逐条详情
    success = fail = 0
    db_items = []
    
    for i, item in enumerate(all_items, 1):
        print(f"\n[{i}/{len(all_items)}] {item['title'][:60]}...")
        
        detail = fetch_detail(item)
        if detail is None:
            fail += 1
            continue
        
        content_text = detail["content"]
        
        db_items.append({
            "title": item["title"],
            "source_url": item["url"],
            "url": item["url"],
            "publish_date": item["date"][:10],
            "summary": item["title"][:200],
            "content": content_text,
            "attachments": detail.get("attachments", ""),
        })
        success += 1
        print(f"   ✅ 正文: {len(content_text)} 字, 日期: {item['date']}")
    
    print(f"\n{'='*60}")
    print(f"📦 写入数据库... 成功: {success}, 失败: {fail}, 跳过: {len(all_items)-success-fail}")
    if db_items:
        insert_db(db_items)
    print(f"{'='*60}")
    print(f"✅ 完成! 共处理 {len(all_items)} 条，成功 {success}，失败 {fail}")

if __name__ == "__main__":
    max_p = None
    args = sys.argv[1:]
    for i, arg in enumerate(args):
        if arg == "--pages" and i + 1 < len(args):
            try:
                max_p = int(args[i + 1])
            except:
                pass
    if max_p is None:
        for arg in args:
            try:
                max_p = int(arg.lstrip('-'))
                break
            except:
                pass
    run(max_pages=max_p)