#!/bin/bash
cd ~/gov_crawler
source venv/bin/activate

echo "=== Starting direct_server crawlers ==="
date

# Run all direct_server scripts
python3 -c "
import json, subprocess
cfg = json.load(open('daily_crawl_config.json', encoding='utf-8'))
crawlers = [c for c in cfg['crawlers'] if c.get('enabled') and c.get('incremental') and c.get('sync_mode') == 'direct_server']
for c in crawlers:
    print(f'▶ {c[\"name\"]}')
" && echo "---" && \
python3 crawl_xx_sthj.py 1 2>&1 | grep -E '(新增|完成|Error|ERROR|入库|上传|推送|爬取|列表|✅|❌|完成|失败|🔍|同步|💾|📤)' && echo "---xx_sthj---" && \
python3 crawl_nqs_sthj.py 1 2>&1 | grep -E '(新增|完成|Error|ERROR|入库|上传|推送|爬取|列表|✅|❌|完成|失败|🔍|同步|💾|📤)' && echo "---nqs_sthj---" && \
python3 crawl_hg_hjbh.py 1 2>&1 | grep -E '(新增|完成|Error|ERROR|入库|上传|推送|爬取|列表|✅|❌|完成|失败|🔍|同步|💾|📤)' && echo "---hg_hjbh---" && \
python3 crawl_xyhpgk.py 1 2>&1 | grep -E '(新增|完成|Error|ERROR|入库|上传|推送|爬取|列表|✅|❌|完成|失败|🔍|同步|💾|📤)' && echo "---xyhpgk---"