#!/usr/bin/env python3
"""
荆州生态环境局 — 环评公示 爬虫
========================
API模式（POST JSON → 列表页）：
  python3 crawl_jz.py                  # 增量（默认：只爬第1页）
  python3 crawl_jz.py --full           # 全量：223页
  python3 crawl_jz.py --pages=5        # 指定页数
  python3 crawl_jz.py --sync           # 同步本地数据到服务器

数据流：本地 quality_results.db → SCP sql文件 → 服务器 search.db
"""

import sys, os, re, json, sqlite3, hashlib, time, base64, subprocess, html as html_mod
from datetime import datetime
from urllib.parse import urljoin

# ─── 配置 ───
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
QUALITY_DB = os.path.join(BASE_DIR, "quality_results.db")
SERVER_SSH = "root@1.94.217.116"
SERVER_DB = "/root/search.db"

SITE_NAME = "荆州市生态环境局"
DOMAIN = "jzssthjj.zwgk.jingzhou.gov.cn"
API_URL = "http://jzssthjj.zwgk.jingzhou.gov.cn/api/content_center/document/list-data-new"
BASE_URL = "http://jzssthjj.zwgk.jingzhou.gov.cn"
TOTAL_PAGES = 223

# ─── HTTP 工具 ───
import ssl, urllib.request
CTX = ssl._create_unverified_context()
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

def http_get(url, timeout=20):
    req = urllib.request.Request(url, headers=HEADERS)
    try: return urllib.request.urlopen(req, timeout=timeout, context=CTX).read().decode("utf-8", errors="replace")
    except: return None

def http_post_json(url, data, timeout=20):
    req = urllib.request.Request(url, data=json.dumps(data).encode('utf-8'),
                                 headers={**HEADERS, 'Content-Type': 'application/json;charset=UTF-8'})
    try: return json.loads(urllib.request.urlopen(req, timeout=timeout, context=CTX).read())
    except: return None

# ─── 列表页解析（API模式）───
def fetch_list_page(page):
    """通过API获取一页列表"""
    data = http_post_json(API_URL, {
        "dept_id": 90, "column_id": 35663, "has_related": False,
        "is_contain_sub": True, "page_index": page, "page_size": 20,
        "return_data_type": "", "randNumber": time.time()
    })
    if not data or data.get('code') != 200:
        return None, 0

    items = data['data'].get('list', [])
    total = data['data'].get('total', 0)
    return items, total

# ─── 详情页解析 ───
def get_detail(url):
    """获取详情页：正文HTML（<!--正文内容-->↔<!--结束-->）"""
    html = http_get(url)
    if not html:
        return None

    # 标题
    title = ''
    tm = re.search(r'<title>(.*?)</title>', html, re.DOTALL)
    if tm:
        title = re.sub(r'\s*[-_].*$', '', tm.group(1)).strip()

    # 正文
    content = ''
    cm = re.search(r'<!--正文内容-->(.*?)<!--结束-->', html, re.DOTALL)
    if cm:
        content = cm.group(1).strip()
        content = re.sub(r'<script[^>]*>.*?</script>', '', content, flags=re.DOTALL|re.I)
        content = re.sub(r'<style[^>]*>.*?</style>', '', content, flags=re.DOTALL|re.I)
        # 附件链接保留，清理多余空段落
        content = re.sub(r'<p>\s*<br\s*/?>\s*</p>', '', content)
        if len(content) > 10000:
            content = content[:10000] + '...'

    return {'title': title, 'content': content}

def extract_plain_text(html_content):
    """从HTML提取纯文本，保留段落"""
    if not html_content: return ''
    text = re.sub(r'</(p|div|li|h[1-6]|tr|td|th)>', '\n\n', html_content)
    text = re.sub(r'<br\s*/?>', '\n', text)
    text = re.sub(r'<[^>]+>', '', text)
    text = html_mod.unescape(text)
    text = re.sub(r'[ \t]+', ' ', text)
    text = re.sub(r'\n{3,}', '\n\n', text)
    text = text.strip()[:300]
    # 过滤常见噪声文本行
    noise_lines = ['扫一扫', '打印|', '关闭', '相关链接', '相关文档', '相关图片', '相关音频', '相关视频', '相关附件']
    clean_lines = []
    for line in text.split('\n'):
        ls = line.strip()
        if not ls:
            clean_lines.append('')
        elif not any(n in ls for n in noise_lines):
            clean_lines.append(ls)
    text = '\n'.join(clean_lines)
    text = re.sub(r'\n{3,}', '\n\n', text)
    return text.strip()

# ─── DB 操作 ───
def ensure_table():
    conn = sqlite3.connect(QUALITY_DB)
    conn.execute("""
        CREATE TABLE IF NOT EXISTS crawl_results (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            site_id INTEGER DEFAULT 0,
            title TEXT,
            url TEXT UNIQUE,
            content TEXT,
            publish_date TEXT,
            summary TEXT,
            domain TEXT,
            category TEXT DEFAULT '',
            content_hash TEXT,
            crawl_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP
        )
    """)
    conn.commit()
    conn.close()

def is_url_crawled(url):
    conn = sqlite3.connect(QUALITY_DB)
    cur = conn.execute("SELECT id FROM crawl_results WHERE url=?", (url,))
    r = cur.fetchone()
    conn.close()
    return r is not None

def store_item(item):
    # 验证：跳过无效标题（HTML代码、空白、过短）
    title = item.get('title', '').strip()
    if not title or len(title) < 3 or '<' in title or '>' in title:
        print(f"  ⚠️ 跳过无效标题: '{item.get('title','')[:40]}'")
        return False
    # 验证：无效日期
    date = item.get('date', '').strip()[:10]
    if date and not re.match(r'^\d{4}-\d{2}-\d{2}$', date):
        print(f"  ⚠️ 跳过无效日期: '{date}'")
        return False

    conn = sqlite3.connect(QUALITY_DB)
    try:
        conn.execute(
            "INSERT OR IGNORE INTO crawl_results "
            "(title, url, content, publish_date, summary, domain, category, content_hash) "
            "VALUES (?,?,?,?,?,?,?,?)",
            (title, item['url'], item.get('content',''), date,
             item.get('summary',''), DOMAIN, '环评阶段',
             hashlib.md5((item.get('content','') or '').encode()).hexdigest())
        )
        conn.commit()
        return conn.total_changes > 0
    finally:
        conn.close()

# ─── 同步到服务器 ───
def sync_to_server():
    """将本地数据直接写入 search.db（服务器本地模式）"""
    print("\n📤 同步到 search.db...")

    conn = sqlite3.connect(QUALITY_DB)
    rows = conn.execute(
        "SELECT title, url, content, publish_date, summary FROM crawl_results WHERE domain=? ORDER BY id",
        (DOMAIN,)
    ).fetchall()
    conn.close()

    if not rows:
        print("  本地没有数据")
        return

    dst = sqlite3.connect("/root/search.db")
    dst.execute("PRAGMA journal_mode=WAL")

    site_name = "荆州市生态环境局"
    new_count = 0
    for r in rows:
        title, url, content, pub_date, summary = r
        # 同步时也过滤无效数据
        title = (title or '').strip()
        if not title or len(title) < 3 or '<' in title or '>' in title:
            continue
        pub_date = (pub_date or '').strip()[:10]
        if pub_date and not re.match(r'^\d{4}(-\d{2}){2}$', pub_date):
            pub_date = ''
        try:
            dst.execute(
                "INSERT OR IGNORE INTO gov_raw "
                "(title, page_url, content, publish_date, summary, site_name, tags) "
                "VALUES (?,?,?,?,?,?,?)",
                (title, url, (content or "")[:8000], pub_date,
                 (summary or "")[:300], site_name, "")
            )
            if dst.total_changes > 0:
                new_count += 1
        except Exception as e:
            print(f"  Error: {e}")

    if new_count > 0:
        dst.commit()
        # Update FTS
        dst.execute(
            "INSERT INTO gov_search(rowid,title,site_name,summary) "
            "SELECT r.id,r.title,r.site_name,r.summary FROM gov_raw r "
            "WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?",
            (site_name,))
        dst.commit()

    total = dst.execute(
        "SELECT COUNT(*) FROM gov_raw WHERE site_name=?", (site_name,)).fetchone()[0]
    dst.close()

    print(f"  OK {new_count}/{len(rows)} 条同步到 search.db (DB共{total}条)")


# ─── 主逻辑 ───
def main():
    args = sys.argv[1:]
    full_mode = '--full' in args
    sync_only = '--sync' in args

    if sync_only:
        sync_to_server(); return

    pages_to_crawl = TOTAL_PAGES if full_mode else 1
    pages_match = [a for a in args if a.startswith('--pages=')]
    if pages_match:
        pages_to_crawl = int(pages_match[0].split('=')[1])
    mode = "全量" if full_mode else f"增量（{pages_to_crawl}页）"

    print(f"🔍 {SITE_NAME}爬虫 — {mode}模式 ({TOTAL_PAGES}页)")
    ensure_table()

    total_site = 0
    new_total = skip_total = 0

    for page in range(1, pages_to_crawl + 1):
        items, total = fetch_list_page(page)
        if items is None:
            print(f"  [p{page:3d}/{pages_to_crawl}] ✗ API失败", end='')
            if page > 1: break   # 连续失败则退出
            print(); continue

        if total_site == 0: total_site = total

        page_new = page_skip = 0
        for item in items:
            page_url = item.get('pub_url', '')
            if not page_url: continue

            if is_url_crawled(page_url):
                page_skip += 1; skip_total += 1; continue

            # 获取详情
            detail = get_detail(page_url)
            entry = {
                'title': item.get('title', ''),
                'url': page_url,
                'date': (item.get('rel_time','') or '')[:10],
                'content': '',
                'summary': '',
            }
            if detail:
                entry['title'] = detail.get('title') or entry['title']
                entry['content'] = detail.get('content', '')
                entry['summary'] = extract_plain_text(detail.get('content',''))

            store_item(entry)
            page_new += 1; new_total += 1

        print(f"  [p{page:3d}/{pages_to_crawl}] ✓ {len(items)}条 (新增{page_new} 跳过{page_skip})" + ' '*10, end='\r')
        if page % 20 == 0:
            print()
        time.sleep(0.3)

    print(f"\n\n📊 完成！新增: {new_total} | 跳过: {skip_total} | 总数: {total_site}")

    if new_total > 0:
        sync_to_server()
    else:
        print("  无新数据，跳过同步")

if __name__ == '__main__':
    main()