DataOps-platform/scripts/validation/kestra_queue_soak.sh

#!/bin/sh
set -eu

compose_file=${COMPOSE_FILE:-deploy/docker/docker-compose.yml}
duration=${KESTRA_SOAK_SECONDS:-120}
interval=${KESTRA_SOAK_INTERVAL_SECONDS:-5}
deadline=$(( $(date +%s) + duration ))
kestra_container=$(docker compose -f "$compose_file" ps -q kestra)
initial_restarts=$(docker inspect -f '{{.RestartCount}}' "$kestra_container")
checks=0

while [ "$(date +%s)" -lt "$deadline" ]; do
  docker compose -f "$compose_file" exec -T kestra \
    curl -fsS http://127.0.0.1:8081/health >/dev/null
  current_restarts=$(docker inspect -f '{{.RestartCount}}' "$kestra_container")
  if [ "$current_restarts" != "$initial_restarts" ]; then
    echo "Kestra restarted during soak: ${initial_restarts} -> ${current_restarts}" >&2
    exit 1
  fi
  checks=$((checks + 1))
  sleep "$interval"
done

if docker compose -f "$compose_file" logs --since "${duration}s" kestra |
  grep -Eiq 'broken pipe|connection is closed|connection reset|hikaripool.*failed|outofmemory|fatal'; then
  echo "Kestra emitted a fatal database/queue signature during soak" >&2
  exit 1
fi

echo "Kestra soak passed: ${checks} health checks, restart count ${initial_restarts}"