#!/bin/bash
set -e

WORKDIR=/root/gov_crawler
SNAPSHOT_DIR=/mnt/data/snapshots
DATE=$(date +%Y%m%d)
SNAPSHOT_FILE="gov_crawler_core_${DATE}.tar.gz"
RETENTION=7

mkdir -p "$SNAPSHOT_DIR"
cd "$WORKDIR"

echo "[snapshot] Creating ${SNAPSHOT_DIR}/${SNAPSHOT_FILE} ..."

# 逐项添加，跳过不存在的文件
INCLUDES=()

add() {
  for f in "$@"; do
    if [ -e "$f" ] || [ -L "$f" ]; then
      INCLUDES+=("$f")
    fi
  done
}

# 脚本
add crawl_*.py crawler_lib.py search_app.py crawler_admin.py
add *.sh

# 配置
add daily_crawl_config.json daily_crawl_config_temp.json sites_cache.json
add data/*.json data/*.csv data/*.txt

# scrapy 核心（只取源码）
add gov_crawler/__init__.py gov_crawler/items.py gov_crawler/pipelines.py gov_crawler/settings.py
add gov_crawler/middlewares.py gov_crawler/middlewares_retry.py gov_crawler/middlewares_tls.py
add gov_crawler/spiders/*.py

# 工具
add utils/*.py

# 其他脚本
add fix_*.py validate*.py

echo "[snapshot] Including ${#INCLUDES[@]} paths..."

tar -czf "${SNAPSHOT_DIR}/${SNAPSHOT_FILE}" \
  --exclude='*.db' --exclude='*.db-wal' --exclude='*.db-shm' \
  --exclude='output' --exclude='venv' --exclude='*.jsonl' \
  --exclude='*.log' --exclude='*.gz' --exclude='__pycache__' \
  --exclude='.git' \
  "${INCLUDES[@]}"

SIZE=$(du -sh "${SNAPSHOT_DIR}/${SNAPSHOT_FILE}" | cut -f1)
echo "[snapshot] Done: ${SNAPSHOT_FILE} (${SIZE})"

# 保留最近7份
cd "$SNAPSHOT_DIR"
COUNT=$(ls gov_crawler_core_*.tar.gz 2>/dev/null | wc -l)
if [ "$COUNT" -gt "$RETENTION" ]; then
  TO_DELETE=$((COUNT - RETENTION))
  echo "[snapshot] Removing $TO_DELETE old snapshot(s)..."
  ls -1 gov_crawler_core_*.tar.gz | sort | head -n $TO_DELETE | while read f; do
    rm -f "$f"
    echo "  removed: $f"
  done
fi