DataOps-platform/deployment/compose/docker-compose.eai-18188.yml

name: dataops-eai

services:
  postgres:
    image: docker.m.daocloud.io/pgvector/pgvector:pg16
    environment:
      POSTGRES_DB: dataops
      POSTGRES_USER: ${DATAOPS_POSTGRES_USER}
      POSTGRES_PASSWORD: ${DATAOPS_POSTGRES_PASSWORD}
    volumes:
      - dataops-eai-postgres:/var/lib/postgresql/data
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U ${DATAOPS_POSTGRES_USER} -d dataops"]
      interval: 5s
      timeout: 5s
      retries: 30
    networks: [dataops-eai]

  kestra-db-init:
    image: docker.m.daocloud.io/library/postgres:16-alpine
    environment:
      PGPASSWORD: ${DATAOPS_POSTGRES_PASSWORD}
    command:
      - /bin/sh
      - -ec
      - >-
        until pg_isready -h postgres -U ${DATAOPS_POSTGRES_USER} -d dataops; do sleep 2; done;
        psql -h postgres -U ${DATAOPS_POSTGRES_USER} -d postgres -tc "SELECT 1 FROM pg_database WHERE datname='kestra'" |
        grep -q 1 || psql -h postgres -U ${DATAOPS_POSTGRES_USER} -d postgres -c "CREATE DATABASE kestra";
    depends_on:
      postgres:
        condition: service_healthy
    restart: "no"
    profiles: ["orchestration"]
    networks: [dataops-eai]

  db-role-init:
    image: dataops-eai-migrator:local
    build:
      context: ../..
      dockerfile: deployment/compose/backend.Dockerfile.eai
      target: migrator
    environment:
      DB_ROLE_INIT_DATABASE_URL: postgresql://${DATAOPS_POSTGRES_USER}:${DATAOPS_POSTGRES_PASSWORD}@postgres:5432/dataops
      MIGRATION_DATABASE_URL: postgresql://${DATAOPS_POSTGRES_USER}:${DATAOPS_POSTGRES_PASSWORD}@postgres:5432/dataops
      DATABASE_URL: postgresql://dataops_app:${DATAOPS_RUNTIME_PASSWORD}@postgres:5432/dataops
      DATAOPS_MIGRATOR_USER: ${DATAOPS_POSTGRES_USER}
      DATAOPS_RUNTIME_USER: dataops_app
      DATAOPS_RUNTIME_PASSWORD: ${DATAOPS_RUNTIME_PASSWORD}
    command: ["python", "-m", "app.core.edge_gateway.runtime_roles"]
    depends_on:
      postgres:
        condition: service_healthy
    restart: "no"
    networks: [dataops-eai]

  db-migrate:
    image: dataops-eai-migrator:local
    build:
      context: ../..
      dockerfile: deployment/compose/backend.Dockerfile.eai
      target: migrator
    command: ["alembic", "-c", "alembic.ini", "upgrade", "head"]
    environment:
      MIGRATION_DATABASE_URL: postgresql://${DATAOPS_POSTGRES_USER}:${DATAOPS_POSTGRES_PASSWORD}@postgres:5432/dataops
    depends_on:
      db-role-init:
        condition: service_completed_successfully
    restart: "no"
    networks: [dataops-eai]

  neo4j:
    image: docker.m.daocloud.io/library/neo4j:5.26-community
    environment:
      NEO4J_AUTH: neo4j/${DATAOPS_NEO4J_PASSWORD}
      NEO4J_server_memory_heap_initial__size: 256m
      NEO4J_server_memory_heap_max__size: 512m
      NEO4J_server_memory_pagecache_size: 256m
    volumes:
      - dataops-eai-neo4j:/data
    healthcheck:
      test: ["CMD-SHELL", "cypher-shell -u neo4j -p '${DATAOPS_NEO4J_PASSWORD}' 'RETURN 1' >/dev/null 2>&1"]
      interval: 10s
      timeout: 10s
      retries: 30
      start_period: 20s
    networks: [dataops-eai]

  minio:
    image: docker.m.daocloud.io/minio/minio:RELEASE.2025-04-22T22-12-26Z
    command: server /data --console-address ":9001"
    environment:
      MINIO_ROOT_USER: ${DATAOPS_MINIO_USER}
      MINIO_ROOT_PASSWORD: ${DATAOPS_MINIO_PASSWORD}
    volumes:
      - dataops-eai-minio:/data
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
      interval: 5s
      timeout: 5s
      retries: 30
    networks: [dataops-eai]

  minio-init:
    image: docker.m.daocloud.io/minio/mc:RELEASE.2025-04-16T18-13-26Z
    environment:
      MINIO_ROOT_USER: ${DATAOPS_MINIO_USER}
      MINIO_ROOT_PASSWORD: ${DATAOPS_MINIO_PASSWORD}
    entrypoint:
      - /bin/sh
      - -ec
      - mc alias set local http://minio:9000 "$${MINIO_ROOT_USER}" "$${MINIO_ROOT_PASSWORD}" && mc mb --ignore-existing local/dataops-bucket
    depends_on:
      minio:
        condition: service_healthy
    restart: "no"
    networks: [dataops-eai]

  kestra:
    image: docker.m.daocloud.io/kestra/kestra:v1.3.20
    command: ["server", "standalone", "--worker-thread=8", "--config", "/etc/kestra/application.yml"]
    environment:
      KESTRA_DB_HOST: postgres
      KESTRA_DB_NAME: kestra
      KESTRA_DB_USERNAME: ${DATAOPS_POSTGRES_USER}
      KESTRA_DB_PASSWORD: ${DATAOPS_POSTGRES_PASSWORD}
      KESTRA_USERNAME: ${KESTRA_USERNAME}
      KESTRA_PASSWORD: ${KESTRA_PASSWORD}
    volumes:
      - dataops-eai-kestra:/app/storage
      - ../../deploy/docker/kestra/application.yml:/etc/kestra/application.yml:ro
    depends_on:
      kestra-db-init:
        condition: service_completed_successfully
    healthcheck:
      test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:8081/health"]
      interval: 10s
      timeout: 5s
      retries: 60
      start_period: 60s
    restart: unless-stopped
    profiles: ["orchestration"]
    networks: [dataops-eai]

  runner:
    image: dataops-eai-runner:local
    build:
      context: ../..
      dockerfile: deployment/compose/runner.Dockerfile.eai
    environment:
      RUNNER_LISTEN_HOST: 0.0.0.0
      RUNNER_LISTEN_PORT: 5600
      RUNNER_TASK_TOKEN_SECRET: ${RUNNER_TASK_TOKEN_SECRET}
      RUNNER_MINIO_HOST: minio:9000
      RUNNER_MINIO_USER: ${DATAOPS_MINIO_USER}
      RUNNER_MINIO_PASSWORD: ${DATAOPS_MINIO_PASSWORD}
      RUNNER_MINIO_BUCKET: dataops-bucket
      RUNNER_MINIO_SECURE: "false"
      DATABASE_URL: postgresql://dataops_app:${DATAOPS_RUNTIME_PASSWORD}@postgres:5432/dataops
      NEO4J_URI: bolt://neo4j:7687
      NEO4J_USER: neo4j
      NEO4J_PASSWORD: ${DATAOPS_NEO4J_PASSWORD}
      DATASOURCE_CREDENTIAL_MASTER_KEY: ${DATASOURCE_CREDENTIAL_MASTER_KEY}
      DATASOURCE_CREDENTIAL_KEY_VERSION: v1
    read_only: true
    tmpfs: [/tmp]
    depends_on:
      db-migrate:
        condition: service_completed_successfully
      neo4j:
        condition: service_healthy
      minio-init:
        condition: service_completed_successfully
    healthcheck:
      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:5600/health', timeout=5)"]
      interval: 10s
      timeout: 5s
      retries: 20
      start_period: 20s
    networks: [dataops-eai]

  backend:
    image: dataops-eai-backend:local
    build:
      context: ../..
      dockerfile: deployment/compose/backend.Dockerfile.eai
      target: runtime
    environment:
      FLASK_ENV: production
      LISTEN_HOST: 0.0.0.0
      LISTEN_PORT: 5500
      GUNICORN_WORKERS: 2
      DATABASE_URL: postgresql://dataops_app:${DATAOPS_RUNTIME_PASSWORD}@postgres:5432/dataops
      NEO4J_URI: bolt://neo4j:7687
      NEO4J_HTTP_URI: http://neo4j:7474
      NEO4J_USER: neo4j
      NEO4J_PASSWORD: ${DATAOPS_NEO4J_PASSWORD}
      NEO4J_ENCRYPTED: "false"
      MINIO_HOST: minio:9000
      MINIO_USER: ${DATAOPS_MINIO_USER}
      MINIO_PASSWORD: ${DATAOPS_MINIO_PASSWORD}
      MINIO_BUCKET: dataops-bucket
      MINIO_SECURE: "false"
      DATAFLOW_DEFAULT_DB_HOST: postgres
      DATAFLOW_DEFAULT_DB_PORT: 5432
      DATAFLOW_DEFAULT_DB_NAME: dataops
      DATAFLOW_SCHEMA: dags
      RUNNER_TASK_TOKEN_SECRET: ${RUNNER_TASK_TOKEN_SECRET}
      API_BASE_URL: http://backend:5500/api
      SECRET_KEY: ${DATAOPS_SECRET_KEY}
      AUDIT_EVIDENCE_SECRET: ${AUDIT_EVIDENCE_SECRET}
      AUDIT_EVIDENCE_KEY_VERSION: v1
      RULE_GENERATION_RECEIPT_SECRET: ${RULE_GENERATION_RECEIPT_SECRET}
      DATASOURCE_CREDENTIAL_MASTER_KEY: ${DATASOURCE_CREDENTIAL_MASTER_KEY}
      DATASOURCE_CREDENTIAL_KEY_VERSION: v1
      LOG_TO_CONSOLE: "true"
    ports: ["127.0.0.1:${BACKEND_PORT}:5500"]
    volumes:
      - dataops-eai-backend-logs:/app/logs
      - dataops-eai-upload:/data/upload
      - dataops-eai-archive:/data/archive
    depends_on:
      db-migrate:
        condition: service_completed_successfully
      neo4j:
        condition: service_healthy
      minio-init:
        condition: service_completed_successfully
      runner:
        condition: service_healthy
    healthcheck:
      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:5500/api/system/health', timeout=5)"]
      interval: 10s
      timeout: 10s
      retries: 20
      start_period: 20s
    networks: [dataops-eai]

  frontend:
    image: dataops-eai-frontend:local
    build:
      context: ../..
      dockerfile: deployment/compose/frontend.Dockerfile.eai
      args:
        VUE_APP_N8N_URL: ""
        VUE_APP_N8N_CHAT_URL: ""
        VUE_APP_MINIO_PUBLIC_URL: ""
        VUE_APP_PREVIEW_URL: ""
    ports: ["127.0.0.1:${FRONTEND_PORT}:80"]
    depends_on:
      backend:
        condition: service_healthy
    healthcheck:
      test: ["CMD-SHELL", "wget --spider -q http://127.0.0.1/"]
      interval: 10s
      timeout: 5s
      retries: 20
    networks: [dataops-eai]

networks:
  dataops-eai:
    name: dataops-eai-network

volumes:
  dataops-eai-postgres:
  dataops-eai-neo4j:
  dataops-eai-minio:
  dataops-eai-kestra:
  dataops-eai-backend-logs:
  dataops-eai-upload:
  dataops-eai-archive: