#!/usr/bin/env python3 -u
"""
每日推送：2026年后含"项目"关键词的数据 → Notion数据库
"""

import os, re, sys, json, subprocess, ssl, time
from urllib.request import Request, urlopen
from urllib.error import HTTPError
from concurrent.futures import ThreadPoolExecutor, as_completed

DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
NOTION_API_KEY = "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"
NOTION_DB_ID = "3a6399b8-3c5b-803d-b92f-da0c63a31a6d"
PUSHED_URLS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "notion_pushed_urls.json")
# 查重策略: JSON文件为主(权威, 无10k上限), Notion API仅首次初始化和兜底
DATA_SOURCE_ID = "3a6399b8-3c5b-8025-8989-000b93a513e1"
BATCH_SIZE = 50
TARGET_INDUSTRIES = []  # 全行业推送（用户2026-08-22改回）; 行业写入Notion select属性, 可按行业筛选
KEYWORDS = ["项目", "工程"]  # 用户2026-08-22: 标题含 项目 或 工程 任一即推（全行业）
# 行业 id -> 中文标签 (与 search_app INDUSTRIES 对齐; other/空 -> 其他)
INDUSTRY_LABEL = {
    "power": "电力", "terminals": "终端/接收站", "port": "港口码头",
    "transmission": "管道输送", "production": "生产/开采", "altfuel": "替代燃料",
    "hpi": "石油炼制", "cpi": "化工加工", "metals": "金属与矿物",
    "pulp": "制浆/造纸/木材", "food": "食品与饮料", "logistics": "物流",
    "manufacturing": "工业制造", "pharma": "制药与生物技术",
}
MAX_WORKERS = 2
# 2026-08-24: 只查最近 N 天(避免全量6.2万行扫描~4min); 去重由 pushed set 兜底(含全部历史URL)
DAYS_WINDOW = 7
# 2026-08-24 用户规则: 永不推送 2026-06-01 之前的数据(清理存量后双保险, 防止未来改大窗口误推老数据)
MIN_PUBLISH_DATE = "2026-06-01"
_SSL_CTX = ssl.create_default_context()

# 加载爬虫配置映射表
CONFIG_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "daily_crawl_config.json")
SCRIPT_MAP = {}
if os.path.exists(CONFIG_PATH):
    with open(CONFIG_PATH, "r", encoding="utf-8") as f:
        for entry in json.load(f):
            name = entry.get("name", "")
            script = entry.get("script", "")
            if not name and script:
                name = entry.get("group", "")
            if name:
                if script:
                    SCRIPT_MAP[name] = script.replace('.py', '')
                else:
                    SCRIPT_MAP[name] = name


def notion_api(method, path, data=None, retries=5):
    url = "https://api.notion.com/v1" + path
    body = json.dumps(data).encode() if data else None

    for attempt in range(retries):
        try:
            req = Request(url, data=body, headers={
                "Authorization": "Bearer " + NOTION_API_KEY,
                "Notion-Version": "2022-06-28",
                "Content-Type": "application/json",
            }, method=method)
            resp = urlopen(req, timeout=30, context=_SSL_CTX)
            return json.loads(resp.read())
        except HTTPError as e:
            err = e.read().decode()
            # 429 rate limit: 指数退避重试
            if e.code == 429:
                if attempt < retries - 1:
                    wait = min(2 ** (attempt + 2), 30)
                    time.sleep(wait)
                    continue
                return {"_error": True, "status": e.code, "message": err[:600]}
            # 400类错误(非429)不重试
            if e.code >= 400 and e.code < 500:
                return {"_error": True, "status": e.code, "message": err[:600]}
            if attempt < retries - 1:
                time.sleep(2 ** attempt)
                continue
            return {"_error": True, "status": e.code, "message": err[:600]}
        except Exception as e:
            if attempt < retries - 1:
                time.sleep(2 ** attempt)
                continue
            return {"_error": True, "status": 0, "message": str(e)[:200]}


def get_script_name(site_name):
    """从daily_crawl_config.json匹配站点名称->爬虫标识名"""
    if not site_name:
        return ""
    # 精确匹配
    if site_name in SCRIPT_MAP:
        return SCRIPT_MAP[site_name]
    # 模糊匹配: config name应该包含在site_name中或反之
    best = ""
    best_len = 0
    for cname, label in SCRIPT_MAP.items():
        if cname in site_name or site_name in cname:
            match_len = min(len(cname), len(site_name))
            if match_len > best_len:
                best = label
                best_len = match_len
    # 如果还匹配不上，尝试用地名前缀匹配(如"沂源县"匹配"沂源县-环评审批信息")
    if not best:
        for prefix_len in range(4, 1, -1):
            if len(site_name) >= prefix_len:
                prefix = site_name[:prefix_len]
                for cname, label in SCRIPT_MAP.items():
                    if prefix in cname:
                        return label
    return best


def fetch_records(limit=None):
    """查询符合条件的记录（按行业+关键词）"""
    limit_clause = " LIMIT {}".format(limit) if limit else ""

    # 关键词条件: 用 gov_bigram 2字词索引替代 LIKE 全表扫描 (LIKE ~100s, bigram ~21s)
    # 2026-08-24 修复: 06:04 推送因 LIKE 超时120s崩溃
    gram_conds = " OR ".join(
        "EXISTS (SELECT 1 FROM gov_bigram WHERE gram='{}' AND rowid = gov_raw.id)".format(kw)
        for kw in KEYWORDS
    )

    # 日期窗口: 只查最近 DAYS_WINDOW 天, 避免全量6.2万行扫描(实测~4min超时风险)
    # 2026-08-24 修复: 全量查询即使bigram优化仍~4min, 日期窗口+索引压到秒级
    # 2026-08-24 用户规则: 永不早于 MIN_PUBLISH_DATE (清理6.1前存量后双保险)
    import datetime
    since = max((datetime.date.today() - datetime.timedelta(days=DAYS_WINDOW)).isoformat(), MIN_PUBLISH_DATE)

    ind_sql = ''
    if TARGET_INDUSTRIES:
        ind_sql = ' AND industry IN ({})'.format(','.join("'{}'".format(i) for i in TARGET_INDUSTRIES))
    sql = """SELECT COALESCE(NULLIF(page_url, ''), source_url) AS page_url,
       title, publish_date, site_name, group_name, industry
FROM gov_raw
WHERE publish_date >= '{since}'
  AND (COALESCE(NULLIF(page_url, ''), source_url) IS NOT NULL
       AND COALESCE(NULLIF(page_url, ''), source_url) != '')
  {ind_sql}
  AND ({gram_conds})
ORDER BY publish_date DESC""".format(since=since, ind_sql=ind_sql, gram_conds=gram_conds) + limit_clause

    r = subprocess.run(["sqlite3", "-json", DB_PATH],
                       input=sql, capture_output=True, text=True, timeout=300)
    if r.returncode != 0 or not r.stdout.strip():
        return []
    recs = json.loads(r.stdout)
    # 2026-08-24: gov_bigram 是正文分词, bigram EXISTS 会误收标题不含关键词的行
    # (如巡察整改/食品安全案例, 正文含"项目"但标题没有) → Python 层按标题再过滤
    recs = [rec for rec in recs if any(kw in (rec.get("title") or "") for kw in KEYWORDS)]
    return recs

def parse_date(raw):
    """将各种中文日期格式转为ISO 8601 (YYYY-MM-DD)"""
    import re
    s = (raw or "").strip()
    m = re.search(r"(\d{4})-(\d{1,2})-(\d{1,2})", s)
    if m:
        return "{}-{:02d}-{:02d}".format(int(m.group(1)), int(m.group(2)), int(m.group(3)))
    m = re.search(r"(\d{4})[年/.]?(\d{1,2})[月/.]?(\d{1,2})", s)
    if m:
        return "{}-{:02d}-{:02d}".format(int(m.group(1)), int(m.group(2)), int(m.group(3)))
    m = re.search(r"(\d{4})", s)
    if m:
        return m.group(1) + "-01-01"
    return None


def push_to_notion(record):
    """推送单条到Notion"""
    title = (record.get("title") or "").strip()
    page_url = (record.get("page_url") or "").strip()
    pub_date = parse_date(record.get("publish_date"))
    site_name = (record.get("site_name") or "").strip()
    script_name = get_script_name(site_name)

    if not title or not page_url:
        return {"error": "missing title or url"}

    props = {
        "标题": {"title": [{"text": {"content": title[:2000]}}]},
        "URL": {"url": page_url},
    }
    if site_name:
        props["站点名称"] = {"rich_text": [{"text": {"content": site_name[:100]}}]}
    if script_name:
        props["脚本名称"] = {"rich_text": [{"text": {"content": script_name[:100]}}]}
    # 行业 select 属性: 英文id -> 中文标签 (record 含 industry 字段)
    ind_id = (record.get("industry") or "").strip()
    ind_label = INDUSTRY_LABEL.get(ind_id, "其他")
    props["行业"] = {"select": {"name": ind_label}}
    if pub_date and len(pub_date) == 10:
        props["发布日期"] = {"date": {"start": pub_date}}

    data = {
        "parent": {"database_id": NOTION_DB_ID},
        "properties": props,
    }

    result = notion_api("POST", "/pages", data)
    if result.get("_error"):
        return {"url": page_url, "title": title[:50], "error": result.get("message")}
    return {"url": page_url, "title": title[:50], "id": result.get("id")}


def load_pushed_urls():
    """从本地JSON文件加载已推送URL (权威查重基准, 无10k API上限)"""
    urls = set()
    if os.path.exists(PUSHED_URLS_FILE):
        try:
            with open(PUSHED_URLS_FILE, "r", encoding="utf-8") as f:
                data = json.load(f)
            for rec in data.get("records", []):
                if rec.get("url"):
                    urls.add(rec["url"])
        except Exception as e:
            print("  !! 读取JSON查重文件失败: {}".format(e))
    return urls


def get_pushed_urls():
    """合并查重: 本地JSON文件 + Notion API兜底补充(API有10k上限, 仅用于初始化/补充)"""
    urls = load_pushed_urls()
    print("  本地JSON查重文件: {}条".format(len(urls)))
    if os.path.exists(PUSHED_URLS_FILE):
        return urls  # JSON已存在则以JSON为准, 不再全量拉Notion(慢且10k截断)
    # 首次运行/文件缺失: 拉Notion全量兜底(注意: linked DB视图query有10k硬上限, 仅用于初始)
    urls = set()
    cursor = None
    retries = 0
    while True:
        params = {"page_size": 100}
        if cursor:
            params["start_cursor"] = cursor
        result = notion_api("POST", "/databases/" + NOTION_DB_ID + "/query", params)
        if result.get("_error"):
            # 任何错误(含429/5xx/网络)都重试, 绝不break提前退出 — 否则漏掉分页导致去重不全重复推送
            if retries < 8:
                retries += 1
                wait = min(2 ** retries, 30)
                print("  !! Notion分页错误({}), {}s后重试 {}/8".format(
                    result.get("message", result.get("status", "?")), wait, retries))
                time.sleep(wait)
                continue
            print("  !! Notion分页重试8次仍失败, 提前结束(去重可能不全)")
            break
        retries = 0
        if not result.get("results"):
            break
        for item in result["results"]:
            props = item.get("properties", {})
            url_prop = props.get("URL", {})
            if url_prop.get("type") == "url":
                val = url_prop.get("url")
                if val:
                    urls.add(val)
        cursor = result.get("next_cursor")
        if not result.get("has_more"):
            break
        time.sleep(0.5)  # 限流间隔
    return urls


def main():
    import argparse
    parser = argparse.ArgumentParser(description="推送项目数据到Notion")
    parser.add_argument("--limit", type=int, default=0, help="限制推送条数(0=全部)")
    parser.add_argument("--skip-dedup", action="store_true", help="跳过去重(用于首次全量)")
    args = parser.parse_args()

    print("[项目推送] Starting...")

    # 去重
    if args.skip_dedup:
        pushed = set()
        print("  跳过去重")
    else:
        print("  查询Notion已有URL (去重)...")
        pushed = get_pushed_urls()
        print("  Notion中已有: {}条".format(len(pushed)))

    # 查询待推送记录
    all_records = fetch_records(limit=args.limit if args.limit else None)
    print("  数据库匹配: {}条".format(len(all_records)))

    # 过滤已推送的
    new_records = [r for r in all_records if r["page_url"] not in pushed]
    total = len(new_records)
    print("  待推送: {}条".format(total))

    if total == 0:
        print("[项目推送] Done: 无新记录")
        return

    # 推送
    print("  写入Notion ({} workers)...".format(MAX_WORKERS))
    ok = fail = 0
    failed_urls = set()
    t0 = time.time()

    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as ex:
        fut_map = {ex.submit(push_to_notion, r): r for r in new_records}
        for i, fut in enumerate(as_completed(fut_map), 1):
            r = fut.result()
            if r.get("id"):
                ok += 1
            else:
                fail += 1
                if r.get("url"):
                    failed_urls.add(r["url"])
                if r.get("error"):
                    print("    FAIL: {} - {}".format(r.get("title", "?"), r["error"][:100]))
            if i % 200 == 0:
                print("    {}/{} ({:.0f}s)".format(i, total, time.time() - t0))

    t = time.time() - t0
    print("\n[项目推送] Done: {}成功, {}失败, {:.1f}s, {:.0f}条/分钟".format(
        ok, fail, t, total / (t / 60) if t > 0 else 0))

    # 推送成功后: 合并新URL回JSON查重文件 (增量持久化)
    # 超时/网络类失败(urlopen error/read timed out) 服务端可能已写入 → 一并加入防重复;
    # 只有 missing title or url 这类确定失败的不加 (它没有有效 url)
    if ok > 0:
        pushed = load_pushed_urls()
        for r in new_records:
            pu = r.get("page_url")
            if not pu:
                continue
            err = (r.get("error") or "")
            is_timeout = ("timed out" in err) or ("handshake" in err) or ("timeout" in err.lower())
            if pu in failed_urls and not is_timeout:
                continue  # 确定失败且非超时 → 明天重试
            pushed.add(pu)
        try:
            existing = []
            if os.path.exists(PUSHED_URLS_FILE):
                with open(PUSHED_URLS_FILE, "r", encoding="utf-8") as f:
                    existing = json.load(f).get("records", [])
            seen = {rec.get("url") for rec in existing if rec.get("url")}
            for u in sorted(pushed):
                if u not in seen:
                    existing.append({"url": u})
                    seen.add(u)
            _seen2 = set()
            _dedup = []
            for _r in existing:
                _u = _r.get("url")
                if not _u or _u in _seen2:
                    continue
                _seen2.add(_u)
                _dedup.append(_r)
            existing = _dedup
            with open(PUSHED_URLS_FILE, "w", encoding="utf-8") as f:
                json.dump({"exported_at": time.strftime("%Y-%m-%d %H:%M:%S"),
                           "total": len(existing), "records": existing}, f, ensure_ascii=False)
            print("  已更新JSON查重文件: {}条".format(len(existing)))
        except Exception as e:
            print("  !! 写JSON查重文件失败: {}".format(e))


if __name__ == "__main__":
    main()