#!/bin/bash
# ZC 数据同步与解析：WebDAV → 下载 PDF → parse_zc.py → ZC.db
# 每日增量：只下载新PDF并解析

WEBDAV_BASE="https://dav.jianguoyun.com/dav/Downloads_nuc"
AUTH="chniir0000@outlook.com:REDACTED_SEE_DOTENV"
PDF_DIR="/mnt/data/zc_pdfs"
DB="/root/ZC.db"
TRACKING_FILE="/mnt/data/zc_pdfs/.processed.txt"
LOG="/var/log/zc_sync.log"

mkdir -p "$PDF_DIR"

log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "$LOG"
}

log "=== 开始 ZC 同步与解析 ==="

# 1. 列出 WebDAV 所有 PDF（只取文件名）
log "列出 WebDAV PDF 列表..."
curl -s -u "$AUTH" -X PROPFIND -H 'Depth: 1' "$WEBDAV_BASE/" 2>/dev/null | \
    grep -o 'href>[^<]*\.pdf<' | sed 's/href>//;s/<$//' | while read -r href; do
    filename=$(basename "$href" | python3 -c "import sys,urllib.parse; print(urllib.parse.unquote(sys.stdin.read().strip()))" 2>/dev/null)
    if [ -z "$filename" ]; then continue; fi
    
    # 跳过已处理的
    if grep -q "^$filename$" "$TRACKING_FILE" 2>/dev/null; then
        continue
    fi
    
    log "  下载: $filename"
    curl -sL -u "$AUTH" -o "$PDF_DIR/$filename" "$WEBDAV_BASE/$filename" 2>/dev/null
    
    if [ -s "$PDF_DIR/$filename" ]; then
        echo "$filename" >> "$TRACKING_FILE"
    else
        log "  ✗ 下载失败: $filename"
        rm -f "$PDF_DIR/$filename"
    fi
done

# 2. 检查是否有新 PDF
NEW_COUNT=$(find "$PDF_DIR" -name '*.pdf' -newer "$TRACKING_FILE" 2>/dev/null | wc -l)
if [ "$NEW_COUNT" -eq 0 ]; then
    log "无新 PDF，跳过解析"
    log "=== 完成 ==="
    exit 0
fi

log "发现 $NEW_COUNT 个新 PDF，开始解析..."

# 3. 运行解析生成 SQL 并导入 DB
cd "$PDF_DIR"
for pdf in *.pdf; do
    [ -f "$pdf" ] || continue
    log "  解析: $pdf"
    
    # 用 python3 运行解析，输出 SQL 并导入 DB
    python3 -c "
import sys
sys.path.insert(0, '/root/gov_crawler')
exec(open('/root/gov_crawler/parse_zc.py').read().replace('PDF_DIR = os.path.expanduser(\"~/Nutstore Files/Downloads_nuc\")', 'PDF_DIR = \"$PDF_DIR\"'))
" --batch 2>/dev/null | sqlite3 "$DB" 2>&1
    
    # 标记已处理
    echo "$pdf" >> "$TRACKING_FILE"
done

# 4. 验证结果
COUNT=$(sqlite3 "$DB" 'SELECT COUNT(*) FROM zc_projects')
log "ZC.db 现有 $COUNT 条项目"
log "=== ZC 同步完成 ==="