#!/usr/bin/env python3
"""
博兴县人民政府 - 通知公告 爬虫
Hanweb CMS: JSONP分页 + requests详情页
"""
import json, os, re, sqlite3, subprocess, sys, time, urllib.parse, urllib.request

SITE_NAME = "博兴县人民政府"
LIST_URL = "http://www.boxing.gov.cn/col/col117915/index.html"
BASE_URL = "http://www.boxing.gov.cn"
DB_PATH = os.path.expanduser("~/gov_crawler/boxing_results.db")
SERVER_SSH = "root@1.94.217.116"
SERVER_SEARCH_DB = "/root/search.db"
TOTAL_RECORDS = 4401  # from page metadata
PER_PAGE = 45

# Dataproxy URL template for pagination
PROXY_URL = "http://www.boxing.gov.cn/module/web/jpage/dataproxy.jsp?page={page}&webid=453&path=http://www.boxing.gov.cn/&columnid=117915&unitid=692184&webname=%25E5%258D%259A%25E5%2585%25B4%25E5%258E%25BF%25E4%25BA%25BA%25E6%25B0%2591%25E6%2594%25BF%25E5%25BA%259C&permissiontype=0"

def init_db():
    conn = sqlite3.connect(DB_PATH, timeout=60)
    conn.execute('''CREATE TABLE IF NOT EXISTS crawl_results (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        title TEXT, url TEXT UNIQUE, content TEXT,
        publish_date TEXT, summary TEXT
    )''')
    conn.commit()
    return conn

def esc(v):
    s = str(v or "")
    return s.replace("'", "''")

def fetch(url, timeout=20):
    req = urllib.request.Request(url, headers={
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
    })
    resp = urllib.request.urlopen(req, timeout=timeout)
    return resp.read().decode('utf-8', 'ignore')

def extract_links_from_page(html):
    """Extract links from the embedded CDATA records on the first page."""
    links = []
    records = re.findall(r'<record>.*?CDATA\[(.*?)\]\]>.*?</record>', html, re.DOTALL)
    for rec in records:
        a = re.search(r'<a[^>]*href="([^"]+)"[^>]*title="([^"]*)"', rec)
        date = re.search(r'<span>([^<]+)</span>', rec)
        if a:
            url = a.group(1)
            if url.startswith('/'):
                url = BASE_URL + url
            links.append((a.group(2), url, date.group(1) if date else ""))
    return links

    

BASE_URL_HTTP = "http://www.boxing.gov.cn"

def extract_content(html):
    """Extract article content from detail page as HTML (preserves links/paragraphs)."""
    # Content in <div class="content contpm"> — end before <div class="share">
    m = re.search(r'<div class="content contpm[^"]*"[^>]*>(.*?)(?:<div class="share|<div class="bot)', html, re.DOTALL)
    if not m:
        # Fallback: try the old pattern
        m = re.search(r'<div class="content contpm[^"]*"[^>]*>(.*?)</div>\s*</div>', html, re.DOTALL)
    if not m:
        return "", ""
    
    content_html = m.group(1)
    
    # Strip HTML comments and meta tags
    content_html = re.sub(r'<!--.*?-->', '', content_html, flags=re.DOTALL)
    content_html = re.sub(r'<meta[^>]*>', '', content_html)
    # Strip iframe (PDF viewer - not needed, we keep the <a> link instead)
    content_html = re.sub(r'<iframe[^>]*>.*?</iframe>', '', content_html, flags=re.DOTALL)
    # Replace <img> with alt/title text (preserves link text inside <a>)
    def replace_img(m):
        t = m.group(0)
        # Icon/decorative images → strip silently
        if '/icons/' in t or '/ico/' in t:
            return ''
        mt = re.search(r'title="([^"]*)"', t)
        if mt: title = mt.group(1)
        else: title = ''
        if not title:
            ma = re.search(r'alt="([^"]*)"', t)
            if ma: title = ma.group(1)
        return title or '[图片]'
    content_html = re.sub(r'<img[^>]*>', replace_img, content_html)
    
    # Convert <p> to <div> for html2text rendering (each paragraph = separate line)
    content_html = re.sub(r'<p[^>]*>', '<div>', content_html)
    content_html = content_html.replace('</p>', '</div>')
    
    # Fix relative URLs in <a> tags - make them absolute
    content_html = re.sub(r'href="(/(?:module|attach)[^"]*)"',
                          lambda m: 'href="' + BASE_URL_HTTP + m.group(1) + '"', content_html)
    content_html = re.sub(r"href='(/(?:module|attach)[^']*)'",
                          lambda m: "href='" + BASE_URL_HTTP + m.group(1) + "'", content_html)
    # Also fix any other relative URLs (e.g. /art/...)
    content_html = re.sub(r'href="(/(?!http)[^"]*)"',
                          lambda m: 'href="' + BASE_URL_HTTP + m.group(1) + '"', content_html)
    content_html = re.sub(r"href='(/(?!http)[^']*)'",
                          lambda m: "href='" + BASE_URL_HTTP + m.group(1) + "'", content_html)
    
    # Entities
    content_html = content_html.replace('&nbsp;', ' ')
    content_html = content_html.replace('&amp;', '&')
    content_html = content_html.replace('&lt;', '<')
    content_html = content_html.replace('&gt;', '>')
    
    # Normalize whitespace: collapse excess spaces but keep block structure
    content_html = re.sub(r'>\s+<', '>\\n<', content_html)
    lines = []
    for line in content_html.split('\\n'):
        stripped = line.strip()
        if stripped:
            lines.append(stripped)
    content_html = '\\n'.join(lines)
    
    # Clean empty <a> tags
    content_html = re.sub(r'<a[^>]*>\s*</a>', '', content_html)
    # Clean empty <div> tags
    content_html = re.sub(r'<div>\s*</div>', '', content_html)
    
    content_html = content_html.strip()
    # Remove trailing </div> from content div's own closing tag
    content_html = re.sub(r'</div>\s*$', '', content_html)
    content_html = content_html.strip()
    
    # Summary - plain text version
    summary = re.sub(r'<[^>]+>', '', content_html)
    summary = re.sub(r'\\s+', ' ', summary)[:500].strip()
    
    return content_html, summary

def extract_pub_date(html):
    """Extract publish date from detail page."""
    d = re.search(r'发布日期：(\d{4}-\d{2}-\d{2})', html)
    if d:
        return d.group(1)
    return ""

def sync_to_server():
    """将本地数据直接写入 search.db（服务器本地模式）"""
    print("\n📤 同步到 search.db...")

    conn = sqlite3.connect(DB_PATH, timeout=60)
    rows = conn.execute("SELECT title, url, content, publish_date, summary FROM crawl_results ORDER BY id").fetchall()
    conn.close()

    if not rows:
        print("  本地没有数据")
        return

    dst = sqlite3.connect("/root/search.db", timeout=60)
    dst.execute("PRAGMA journal_mode=WAL")

    site_name = "博兴县人民政府"
    new_count = 0
    for r in rows:
        title, url, content, pub_date, summary = r
        try:
            dst.execute(
                "INSERT OR IGNORE INTO gov_raw "
                "(title, page_url, content, publish_date, summary, site_name, tags) "
                "VALUES (?,?,?,?,?,?,?)",
                (title, url, (content or "")[:500000], pub_date or "",
                 (summary or "")[:300], site_name, "")
            )
            if dst.total_changes > 0:
                new_count += 1
        except Exception as e:
            print(f"  Error: {e}")

    if new_count > 0:
        dst.commit()
        # Update FTS
        dst.execute(
            "INSERT OR REPLACE INTO gov_search(rowid,title,site_name,summary) "
            "SELECT r.id,r.title,r.site_name,r.summary FROM gov_raw r "
            "WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?",
            (site_name,))
        dst.commit()

    total = dst.execute(
        "SELECT COUNT(*) FROM gov_raw WHERE site_name=?", (site_name,)).fetchone()[0]
    dst.close()

    print(f"  OK {new_count}/{len(rows)} 条同步到 search.db (DB共{total}条)")


def crawl(test_limit=0):
    conn = init_db()
    stats = {"new": 0, "skip": 0, "errors": 0}
    all_links = []
    
    # Build an opener with cookie support
    import http.cookiejar
    cj = http.cookiejar.CookieJar()
    opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cj))
    
    def fetch_with_cookies(url, post_data=None, timeout=20):
        req = urllib.request.Request(url, data=post_data, headers={
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
            'Referer': LIST_URL,
        })
        if post_data:
            req.add_header('Content-Type', 'application/x-www-form-urlencoded')
        resp = opener.open(req, timeout=timeout)
        return resp.read().decode('utf-8', 'ignore')
    
    # ---- Page 1 (embedded in HTML) ----
    print("📃 第1页")
    html = fetch_with_cookies(LIST_URL)
    links = extract_links_from_page(html)
    all_links.extend(links)
    print(f"  📄 提取 {len(links)} 条")
    
    # ---- Pages 2+ (via dataproxy AJAX POST) ----
    # The jpage plugin uses POST with startrecord/endrecord/perpage as URL params
    # and ajaxParam as POST body. Group size=3 pages per AJAX call.
    BATCH_SIZE = 45  # 3 pages × 15 per page
    webname_enc = urllib.parse.quote('博兴县人民政府')
    proxy_url = "http://www.boxing.gov.cn/module/web/jpage/dataproxy.jsp"
    ajax_param = {
        'page': '2',
        'col': '1',
        'webid': '453',
        'path': 'http://www.boxing.gov.cn/',
        'columnid': '117915',
        'sourceContentType': '1',
        'unitid': '692184',
        'webname': '博兴县人民政府',
        'permissiontype': '0',
    }
    
    start_rec = BATCH_SIZE + 1
    existing_urls = {u for _, u, _ in all_links}
    
    # 本地已有 url(增量依据, 不再每轮清空)
    local_exist = {u for (u,) in conn.execute("SELECT url FROM crawl_results").fetchall()}
    print(f"  📚 本地已有 {len(local_exist)} 条")

    for sr in [46, 91, 136, 181, 226, 271, 316, 361, 500, 1000, 1500, 2000, 2500, 3000, 3500, 4000, 4300]:
        er = sr + BATCH_SIZE - 1
        if sr > TOTAL_RECORDS:
            break
        
        print(f"📃 提取 (rec {sr}-{er})")
        
        url = f"{proxy_url}?startrecord={sr}&endrecord={er}&perpage=15&unitid=692184&webid=453&path=http%3A%2F%2Fwww.boxing.gov.cn%2F&webname={webname_enc}&col=1&columnid=117915&sourceContentType=1&permissiontype=0"
        post_body = urllib.parse.urlencode(ajax_param).encode()
        
        try:
            data = fetch_with_cookies(url, post_body)
            new_links = extract_links_from_page(data)
            new_unique = [(t, u, d) for t, u, d in new_links if u not in existing_urls]
            
            if new_unique:
                all_links.extend(new_unique)
                existing_urls.update(u for _, u, _ in new_unique)
                print(f"  📄 +{len(new_unique)} 条")

            if len(new_links) < 10:
                print(f"  ⏹️ 数据耗尽")
                break
        except Exception as e:
            print(f"  ⚠️ 请求失败: {str(e)[:50]}")
        
        time.sleep(0.3)
    
    print(f"\n📄 共提取 {len(all_links)} 条链接，开始抓取详情页...")

    # 增量: 只抓尚未入库的(按最新在前已收集 ~360 条深窗, 覆盖历史缺口),
    # 不再"遇首个已入库即截断"(会再次漏掉 test 时期的历史缺口)
    all_links = [x for x in all_links if x[1] not in local_exist]
    print(f"  ⏩ 过滤后待抓详情 {len(all_links)} 条 (本地已有 {len(local_exist)})")
    
    for idx, (title, url, list_date) in enumerate(all_links):
        if test_limit > 0 and idx >= test_limit:
            break
        try:
            html = fetch(url)
            
            content, summary = extract_content(html)
            pub_date = extract_pub_date(html) or list_date
            
            conn.execute(
                "INSERT OR IGNORE INTO crawl_results (title, url, content, publish_date, summary) VALUES (?,?,?,?,?)",
                (title[:500], url, content, pub_date, summary)
            )
            conn.commit()
            
            size = len(content or "")
            print(f"  ✅ {size}B | {title[:50]}")
            stats['new'] += 1
            
        except Exception as e:
            print(f"  ❌ {str(e)[:60]} | {title[:40]}")
            stats['errors'] += 1
        
        time.sleep(0.3)
    
    conn.close()
    
    print(f"\n{'='*50}")
    print(f"🏁 新增:{stats['new']} 跳过:{stats['skip']} 错误:{stats['errors']}")
    
    if stats['new'] > 0:
        sync_to_server()
    else:
        print("⚠️ 无新数据，跳过同步")

if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('--full', action='store_true')
    parser.add_argument('--test', type=int, default=0)
    parser.add_argument('--sync', action='store_true')
    args = parser.parse_args()
    
    if args.sync:
        sync_to_server()
    else:
        crawl(test_limit=args.test)