#!/usr/bin/env bash
# Batch-analyse the four 上片 clips end to end, unattended.
#
# Picks its own detection backend: if a locally-built TensorRT engine exists AND
# actually returns detections, it uses the GPU service; otherwise it falls back
# to several parallel CPU services. The GPU check is not optional -- the engine
# shipped in the repo loaded fine, ran 8x faster, and returned ZERO detections
# (it had been built for a different device), so "the service is healthy" proves
# nothing. Only a non-zero detection count on real frames does.
set -uo pipefail
cd "$(dirname "$0")/.."

REPO=/home/brad/proj/sonice-ai-sentinel-edge
PY=$REPO/.cache/roven-real-model-eval/venv/bin/python
MODELS=$REPO/.cache/roven-real-model-eval/models/dfine_s_obj2coco_onnx
CLIPS=(load-0800 load-1300 load-1730 load-2200)
mkdir -p logs metrics runs

echo "[batch] waiting for frame extraction to finish"
while pgrep -f extract_clip.sh >/dev/null; do sleep 20; done
for c in "${CLIPS[@]}"; do echo "  $c: $(ls media/proxies/${c}_frames 2>/dev/null | wc -l) frames"; done

# Wait for the TensorRT build too, else we would probe a half-written engine,
# fall back to CPU, and throw away an 8x speedup that lands minutes later.
if pgrep -x trtexec >/dev/null; then
  echo "[batch] TensorRT engine still building; waiting up to 45 min"
  for _ in $(seq 90); do
    pgrep -x trtexec >/dev/null || break
    sleep 30
  done
  pgrep -x trtexec >/dev/null && echo "[batch] trtexec still going, proceeding without it" \
                              || echo "[batch] trtexec finished"
fi

# ---- choose detection backend -------------------------------------------
# Every port is probed FUNCTIONALLY before use: it must return detections on a
# frame known to contain three people. Two separate incidents forced this --
# a TensorRT engine that loaded, ran 8x faster and returned zero detections, and
# port 18089 which answered healthz with PASS because an unrelated SAM
# segmentation service from a previous session was sitting on it. A health
# check proves the socket is alive, nothing more.
PORTS="18085"
KNOWN_GOOD=$PWD/media/proxies/a1_frames/f_00001.jpg   # 09:30:00, 3 people

probe_port() {
  local p=$1
  local n
  n=$(curl -s -m40 -X POST "http://127.0.0.1:$p/roven/detect" -H 'Content-Type: application/json' \
      -d "{\"schema_version\":\"roven.edge.detector_service_request.v0\",\"run_id\":\"probe\",\"model_profile\":\"ms1_live_detector\",\"frames\":[{\"image\":\"p\",\"path\":\"$KNOWN_GOOD\",\"camera_id\":\"c\",\"time\":\"2026-07-28T09:30:00\",\"frame_ref\":\"file://p\"}]}" \
      | $PY -c "import sys,json
try:
    d=json.load(sys.stdin); print(sum(len(r.get('detections',[])) for r in d.get('records',[])))
except Exception: print(0)" 2>/dev/null)
  [[ "${n:-0}" -ge 1 ]]
}

probe_gpu() {
  [[ -s $MODELS/model_local_b1_640.engine ]] || return 1
  pkill -f "port 18087" 2>/dev/null
  pkill -f "dfine-glass-gpu" 2>/dev/null
  sleep 3
  setsid $PY $REPO/examples/roven-dfine-onnx-detector-service.py \
    --model $MODELS/model_fp16.onnx --engine $MODELS/model_local_b1_640.engine \
    --execution-provider gpu --host 127.0.0.1 --port 18087 \
    --imgsz 640 --conf 0.50 --classes person \
    --work-dir $REPO/runtime/live-services/dfine-glass-gpu2 \
    </dev/null > logs/dfine-gpu2.log 2>&1 &
  for _ in $(seq 40); do
    curl -s -m3 http://127.0.0.1:18087/healthz | grep -q PASS && break
    sleep 3
  done
  rm -f /tmp/gpuprobe.jsonl
  $PY scripts/detect_clip.py --frames-dir media/proxies/load-0800_frames \
    --out /tmp/gpuprobe.jsonl --ports 18087 --limit 40 >/dev/null 2>&1
  local nd
  nd=$($PY - <<'EOF'
import json
try:
    print(sum(len(json.loads(l)["detections"]) for l in open("/tmp/gpuprobe.jsonl")))
except Exception:
    print(0)
EOF
)
  echo "[batch] GPU probe: $nd detections in 40 frames"
  [[ "$nd" -gt 10 ]]
}

if probe_gpu; then
  PORTS="18087"
  echo "[batch] using GPU (TensorRT) on :18087"
else
  echo "[batch] GPU unusable -> parallel CPU services"
  # Fresh high ports only. Never reuse a port just because something there is
  # answering healthz -- that is how a SAM service ended up serving a third of
  # the detection traffic and silently returning nothing.
  for p in 18091 18092; do
    ss -ltn 2>/dev/null | grep -q ":$p " && { echo "[batch] port $p busy, skipping"; continue; }
    setsid $PY $REPO/examples/roven-dfine-onnx-detector-service.py \
      --model $MODELS/model_quantized.onnx --execution-provider cpu \
      --host 127.0.0.1 --port $p --imgsz 640 --conf 0.50 --classes person \
      --work-dir $REPO/runtime/live-services/dfine-cpu-$p \
      </dev/null > logs/dfine-cpu-$p.log 2>&1 &
  done
  echo "[batch] waiting for CPU services to load their model"
  sleep 45
  GOOD=()
  for p in 18085 18091 18092; do
    if probe_port $p; then GOOD+=("$p"); echo "[batch]   :$p OK"; else echo "[batch]   :$p REJECTED (no detections on known-good frame)"; fi
  done
  [[ ${#GOOD[@]} -eq 0 ]] && { echo "[batch] FATAL: no working detector"; exit 1; }
  PORTS=$(IFS=,; echo "${GOOD[*]}")
  echo "[batch] using CPU services $PORTS"
fi

# ---- detect + analyse ----------------------------------------------------
pkill -f "detect_clip.py --frames-dir media/proxies/load-0800" 2>/dev/null
sleep 2
for c in "${CLIPS[@]}"; do
  n=$(ls media/proxies/${c}_frames 2>/dev/null | wc -l)
  [[ "$n" -lt 1000 ]] && { echo "[batch] $c: only $n frames, skipping"; continue; }
  mkdir -p runs/$c
  echo "[batch] === $c: detect ($n frames) ==="
  $PY scripts/detect_clip.py --frames-dir media/proxies/${c}_frames \
    --out runs/$c/detections.jsonl --ports "$PORTS" --video-t0 0 --run-id "$c" \
    2>&1 | tee logs/detect_$c.log | tail -3
  echo "[batch] === $c: analyse ==="
  ./scripts/run_clip.sh "$c" 2>&1 | tee logs/run_$c.log | tail -6
  $PY scripts/build_clips_index.py --out runs/clips.json
  echo "[batch] $c COMPLETE"
done

$PY scripts/build_clips_index.py --out runs/clips.json
echo "[batch] ALL DONE"