#!/bin/bash
cd ~/gov_crawler
export PATH=venv/bin:$PATH

# ===== PHASE 1: Remaining direct_server crawlers =====
echo "=== REMAINING DIRECT ==="

for script in \
  "crawl_yiyang.py" \
  "crawl_gdee_spqgs.py" \
  "crawl_zx_tzgg.py" \
  "crawl_zhangwu_tzgg.py 1" \
  "crawl_jiangyin_tzgg.py" \
  "crawl_zhanyi_hpgl.py" \
  "crawl_zhostar_hp.py" \
  "crawl_cngy_hjbh.py" \
  "crawl_hetang_tzgg.py 1" \
  "crawl_suyu_gsgg.py" \
  "crawl_longchang_gsgg.py" \
  "crawl_teda_hpgs.py 1" \
  "crawl_ahjs_hpgs.py 1" \
  "crawl_ahys_hpgs.py 1" \
  "crawl_daying_gk.py 1" \
  "crawl_dongyue_news.py" \
  "crawl_gzcsx_tzgg.py 1"
do
  set -- $script
  f=$1; shift
  echo "[$f] starting..."
  python3 -u "$f" "$@" 2>&1 | grep -E '(新增|入库|上传|同步|💾|📤|✅|❌|完成|Error|ERROR)' | tail -3
  echo "---"
done

echo ""
echo "=== SCRAPY TO SEARCHDB ==="

# ===== PHASE 2: scrapy_to_searchdb crawlers =====
for script in \
  "crawl_fogang.py" \
  "crawl_xjhbcy.py" \
  "crawl_xjzdkfq.py" \
  "crawl_jz.py" \
  "crawl_eiafans.py --pages=10" \
  "crawl_cnhu.py" \
  "crawl_gzpbjjkfq_gsgg.py 1" \
  "crawl_linquan_zxjd.py 1" \
  "crawl_shccig_tzgg.py 1" \
  "crawl_taihe_zxjd.py 1" \
  "crawl_xifeng_yqxx.py 1" \
  "crawl_xjht_sthj.py 1" \
  "crawl_yd_spjg.py 1" \
  "crawl_yd_yjzj.py 1" \
  "crawl_ybty_xxgs.py 1" \
  "crawl_zhangshu_hjbh.py 1" \
  "crawl_jiangmen_jsxm.py 1" \
  "crawl_ahgc_hpgs.py 1" \
  "crawl_xjcbcr_yjzj.py 1" \
  "crawl_xjyiwu_ghxx.py 1" \
  "crawl_fengxin_qyfw.py 1"
do
  set -- $script
  f=$1; shift
  echo "[$f] starting..."
  python3 -u "$f" "$@" 2>&1 | grep -E '(新增|入库|上传|同步|💾|📤|✅|❌|完成|Error|ERROR)' | tail -5
  echo "---"
done

echo ""
echo "=== ALL DONE ==="
date