#!/bin/bash
# Daily incremental crawl runner - robust version with timeout per crawler
cd /root/gov_crawler

JSON_FILE="daily_crawl_config.json"
RESULTS_FILE="/tmp/daily_crawl_results2.json"

# Parse enabled crawlers and run them one by one
TOTAL=$(python3 -c "import json; d=json.load(open()); print(len([c for c in d[crawlers] if c.get(enabled,False)]))")
echo "=== Daily Incremental Crawl Start ==="
echo "Time: $(date +%Y-%m-%d %H:%M:%S)"
echo "Total enabled sites: $TOTAL"
echo ""

INDEX=0
TOTAL_NEW=0
FAIL_COUNT=0
declare -a FAIL_NAMES
declare -a RESULTS_JSON

while IFS=