DataOps-platform/deploy/docker/docker-compose.yml

name: dataops-test

services:
  postgres:
    image: pgvector/pgvector:pg16
    environment:
      POSTGRES_DB: dataops
      POSTGRES_USER: dataops
      POSTGRES_PASSWORD: dataops-test-password
    ports:
      - "${DATAOPS_POSTGRES_PORT:-15432}:5432"
    volumes:
      - dataops-test-postgres:/var/lib/postgresql/data
      - ./postgres/init:/docker-entrypoint-initdb.d:ro
      - ../../database:/dataops-schema:ro
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U dataops -d dataops"]
      interval: 5s
      timeout: 5s
      retries: 20
    networks:
      - dataops-test-net

  kestra-db-init:
    image: postgres:16-alpine
    environment:
      PGPASSWORD: dataops-test-password
    volumes:
      - ./postgres/init/010-kestra-runtime.sql:/opt/dataops/010-kestra-runtime.sql:ro
    command:
      - psql
      - -v
      - ON_ERROR_STOP=1
      - -h
      - postgres
      - -U
      - dataops
      - -d
      - postgres
      - -f
      - /opt/dataops/010-kestra-runtime.sql
    depends_on:
      postgres:
        condition: service_healthy
    restart: "no"
    networks:
      - dataops-test-net

  source-postgres:
    image: postgres:16-alpine
    environment:
      POSTGRES_DB: acceptance
      POSTGRES_USER: source_reader
      POSTGRES_PASSWORD: source-test-password
    ports:
      - "${SOURCE_POSTGRES_PORT:-25432}:5432"
    volumes:
      - dataops-test-source-postgres:/var/lib/postgresql/data
      - ./datasources/postgres/init.sql:/docker-entrypoint-initdb.d/010-acceptance.sql:ro
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U source_reader -d acceptance"]
      interval: 5s
      timeout: 5s
      retries: 20
    networks:
      - dataops-test-net

  source-mysql:
    image: mysql:8.4
    environment:
      MYSQL_DATABASE: acceptance
      MYSQL_USER: source_reader
      MYSQL_PASSWORD: source-test-password
      MYSQL_ROOT_PASSWORD: source-root-test-password
    ports:
      - "${SOURCE_MYSQL_PORT:-23306}:3306"
    volumes:
      - dataops-test-source-mysql:/var/lib/mysql
      - ./datasources/mysql/init.sql:/docker-entrypoint-initdb.d/010-acceptance.sql:ro
    healthcheck:
      test:
        - CMD-SHELL
        - mysqladmin ping -h 127.0.0.1 -u source_reader -psource-test-password --silent
      interval: 5s
      timeout: 5s
      retries: 30
      start_period: 20s
    networks:
      - dataops-test-net

  neo4j:
    image: neo4j:5.26-community
    environment:
      NEO4J_AUTH: neo4j/Passw0rd
      NEO4J_server_memory_heap_initial__size: 256m
      NEO4J_server_memory_heap_max__size: 512m
      NEO4J_server_memory_pagecache_size: 256m
    ports:
      - "${NEO4J_HTTP_PORT:-17474}:7474"
      - "${NEO4J_BOLT_PORT:-17687}:7687"
    volumes:
      - dataops-test-neo4j:/data
    healthcheck:
      test: ["CMD-SHELL", "cypher-shell -u neo4j -p Passw0rd 'RETURN 1' >/dev/null 2>&1"]
      interval: 10s
      timeout: 10s
      retries: 30
      start_period: 20s
    networks:
      - dataops-test-net

  minio:
    image: minio/minio:RELEASE.2025-04-22T22-12-26Z
    command: server /data --console-address ":9001"
    environment:
      MINIO_ROOT_USER: dataops-test
      MINIO_ROOT_PASSWORD: dataops-test-password
    ports:
      - "${MINIO_API_PORT:-19000}:9000"
      - "${MINIO_CONSOLE_PORT:-19001}:9001"
    volumes:
      - dataops-test-minio:/data
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
      interval: 5s
      timeout: 5s
      retries: 20
    networks:
      - dataops-test-net

  minio-init:
    image: minio/mc:RELEASE.2025-04-16T18-13-26Z
    depends_on:
      minio:
        condition: service_healthy
    entrypoint:
      - /bin/sh
      - -c
      - >-
        mc alias set local http://minio:9000 dataops-test dataops-test-password &&
        mc mb --ignore-existing local/dataops-bucket
    restart: "no"
    networks:
      - dataops-test-net

  n8n:
    image: n8nio/n8n:1.100.1
    environment:
      DB_TYPE: postgresdb
      DB_POSTGRESDB_HOST: postgres
      DB_POSTGRESDB_PORT: 5432
      DB_POSTGRESDB_DATABASE: n8n
      DB_POSTGRESDB_USER: dataops
      DB_POSTGRESDB_PASSWORD: dataops-test-password
      N8N_HOST: localhost
      N8N_PORT: 5678
      N8N_PROTOCOL: http
      N8N_SECURE_COOKIE: "false"
      N8N_ENCRYPTION_KEY: dataops-local-test-encryption-key
      N8N_ENFORCE_SETTINGS_FILE_PERMISSIONS: "true"
      WEBHOOK_URL: http://localhost:15678/
      GENERIC_TIMEZONE: Asia/Shanghai
      TZ: Asia/Shanghai
    ports:
      - "${N8N_PORT:-15678}:5678"
    volumes:
      - dataops-test-n8n:/home/node/.n8n
    depends_on:
      postgres:
        condition: service_healthy
    healthcheck:
      test: ["CMD-SHELL", "wget --spider -q http://127.0.0.1:5678/healthz"]
      interval: 10s
      timeout: 5s
      retries: 30
      start_period: 30s
    networks:
      - dataops-test-net

  kestra:
    image: kestra/kestra:v1.3.20
    command:
      - server
      - standalone
      - --worker-thread=16
      - --config
      - /etc/kestra/application.yml
    stop_grace_period: 6m
    environment:
      KESTRA_DB_HOST: postgres
      KESTRA_DB_NAME: kestra
      KESTRA_DB_USERNAME: kestra_runtime
      KESTRA_DB_PASSWORD: kestra-local-test-password
      KESTRA_USERNAME: ${KESTRA_USERNAME:-admin@dataops.local}
      KESTRA_PASSWORD: ${KESTRA_PASSWORD:-DataOpsKestra1!}
    ports:
      - "127.0.0.1:${KESTRA_PORT:-18080}:8080"
    volumes:
      - dataops-test-kestra:/app/storage
      - ./kestra/application.yml:/etc/kestra/application.yml:ro
    depends_on:
      kestra-db-init:
        condition: service_completed_successfully
    healthcheck:
      test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:8081/health"]
      interval: 10s
      timeout: 5s
      retries: 60
      start_period: 60s
    restart: unless-stopped
    networks:
      - dataops-test-net

  kestra-mcp:
    image: ghcr.io/kestra-io/mcp-server-python@sha256:1fd62028f60914af31e244ddb0086ed5017420313e2c8547ea9c01b8d24dfc1d
    profiles:
      - kestra-mcp
    stdin_open: true
    environment:
      KESTRA_BASE_URL: http://kestra:8080/api/v1
      KESTRA_TENANT_ID: main
      KESTRA_USERNAME: ${KESTRA_USERNAME:-admin@dataops.local}
      KESTRA_PASSWORD: ${KESTRA_PASSWORD:-DataOpsKestra1!}
      KESTRA_MCP_DISABLED_TOOLS: ee,files,kv,namespace
      KESTRA_MCP_LOG_LEVEL: ERROR
    depends_on:
      kestra:
        condition: service_healthy
    networks:
      - dataops-test-net

  dataops-scheduling-mcp:
    image: dataops-platform-mcp:local
    build:
      context: ../..
      dockerfile: deploy/docker/mcp.Dockerfile
    profiles:
      - dataops-mcp
    stdin_open: true
    read_only: true
    tmpfs:
      - /tmp:size=32m,mode=1777
    pids_limit: 64
    environment:
      DATAOPS_MCP_SUBJECT: ${DATAOPS_MCP_SUBJECT:-}
      DATAOPS_MCP_ROLES: ${DATAOPS_MCP_ROLES:-}
      DATAOPS_MCP_BUSINESS_DOMAINS: ${DATAOPS_MCP_BUSINESS_DOMAINS:-}
      DATAOPS_MCP_ENVIRONMENTS: ${DATAOPS_MCP_ENVIRONMENTS:-}
      DATAOPS_MCP_CORRELATION_ID: ${DATAOPS_MCP_CORRELATION_ID:-}
      DATAOPS_MCP_DATABASE_URL: postgresql://dataops:dataops-test-password@postgres:5432/dataops
      KESTRA_BASE_URL: http://kestra:8080/api/v1
      KESTRA_TENANT_ID: main
      KESTRA_USERNAME: ${KESTRA_USERNAME:-admin@dataops.local}
      KESTRA_PASSWORD: ${KESTRA_PASSWORD:-DataOpsKestra1!}
      RUNNER_TASK_TOKEN_SECRET: ${RUNNER_TASK_TOKEN_SECRET:-dataops-local-runner-task-token-secret-change-me}
      RUNNER_TASK_TOKEN_TTL_SECONDS: 60
    command:
      - --kind
      - scheduling
      - --factory
      - app.core.mcp.bootstrap:build_scheduling_gateway
    depends_on:
      postgres:
        condition: service_healthy
      kestra:
        condition: service_healthy
      runner:
        condition: service_healthy
    networks:
      - dataops-test-net

  dataops-context-mcp:
    image: dataops-platform-mcp:local
    build:
      context: ../..
      dockerfile: deploy/docker/mcp.Dockerfile
    profiles:
      - dataops-mcp
    stdin_open: true
    read_only: true
    tmpfs:
      - /tmp:size=32m,mode=1777
    pids_limit: 64
    environment:
      DATAOPS_MCP_SUBJECT: ${DATAOPS_MCP_SUBJECT:-}
      DATAOPS_MCP_ROLES: ${DATAOPS_MCP_ROLES:-}
      DATAOPS_MCP_BUSINESS_DOMAINS: ${DATAOPS_MCP_BUSINESS_DOMAINS:-}
      DATAOPS_MCP_ENVIRONMENTS: ${DATAOPS_MCP_ENVIRONMENTS:-}
      DATAOPS_MCP_CORRELATION_ID: ${DATAOPS_MCP_CORRELATION_ID:-}
      DATAOPS_MCP_DATABASE_URL: postgresql://dataops:dataops-test-password@postgres:5432/dataops
      DATAOPS_MCP_CONTEXT_MAX_CONCURRENCY: 10
    command:
      - --kind
      - context
      - --factory
      - app.core.mcp.bootstrap:build_context_service
    depends_on:
      postgres:
        condition: service_healthy
    networks:
      - dataops-test-net

  runner:
    image: ${DATAOPS_RUNNER_IMAGE:-dataops-platform-runner:local}
    build:
      context: ../..
      dockerfile: deploy/docker/runner.Dockerfile
    environment:
      RUNNER_LISTEN_HOST: 0.0.0.0
      RUNNER_LISTEN_PORT: 5600
      RUNNER_WORKERS: 2
      RUNNER_THREADS: 2
      RUNNER_TASK_TOKEN_SECRET: ${RUNNER_TASK_TOKEN_SECRET:-dataops-local-runner-task-token-secret-change-me}
      RUNNER_TASK_TOKEN_TTL_SECONDS: 60
      RUNNER_DATASOURCE_POOL_SIZE: 1
      RUNNER_DATASOURCE_MAX_OVERFLOW: 1
      RUNNER_DATASOURCE_MAX_IDLE_POOLS: 4
      RUNNER_DATASOURCE_CONNECTION_BUDGET: 32
      RUNNER_HTTP_ALLOWED_HOSTS: ${RUNNER_HTTP_ALLOWED_HOSTS:-}
      RUNNER_MINIO_HOST: minio:9000
      RUNNER_MINIO_USER: dataops-test
      RUNNER_MINIO_PASSWORD: dataops-test-password
      RUNNER_MINIO_BUCKET: dataops-bucket
      RUNNER_MINIO_SECURE: "false"
      RUNNER_ARTIFACT_MAX_BYTES: 33554432
      RUNNER_ARTIFACT_MAX_ROWS: 100000
      RUNNER_ARTIFACT_MEMORY_LIMIT_BYTES: 268435456
      RUNNER_ARTIFACT_TTL_SECONDS: 3600
      DATABASE_URL: postgresql://dataops:dataops-test-password@postgres:5432/dataops
      NEO4J_URI: bolt://neo4j:7687
      NEO4J_USER: neo4j
      NEO4J_PASSWORD: Passw0rd
      DATASOURCE_CREDENTIAL_MASTER_KEY: ${DATASOURCE_CREDENTIAL_MASTER_KEY:-MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=}
      DATASOURCE_CREDENTIAL_KEY_VERSION: ${DATASOURCE_CREDENTIAL_KEY_VERSION:-v1}
    ports:
      - "127.0.0.1:${RUNNER_PORT:-15600}:5600"
    read_only: true
    tmpfs:
      - /tmp:size=64m,mode=1777
    pids_limit: 128
    deploy:
      resources:
        limits:
          cpus: "1.0"
          memory: 512M
          pids: 128
    depends_on:
      postgres:
        condition: service_healthy
      neo4j:
        condition: service_healthy
      minio-init:
        condition: service_completed_successfully
    healthcheck:
      test:
        - CMD
        - python
        - -c
        - import urllib.request; urllib.request.urlopen('http://127.0.0.1:5600/health', timeout=5)
      interval: 10s
      timeout: 5s
      retries: 20
      start_period: 20s
    networks:
      dataops-test-net:
        aliases:
          - dataops-runner

  backend:
    image: ${DATAOPS_BACKEND_IMAGE:-dataops-platform-backend:local}
    build:
      context: ../..
      dockerfile: deploy/docker/backend.Dockerfile
    environment:
      FLASK_ENV: production
      LISTEN_HOST: 0.0.0.0
      LISTEN_PORT: 5500
      GUNICORN_WORKERS: 2
      DATABASE_URL: postgresql://dataops:dataops-test-password@postgres:5432/dataops
      NEO4J_URI: bolt://neo4j:7687
      NEO4J_HTTP_URI: http://neo4j:7474
      NEO4J_USER: neo4j
      NEO4J_PASSWORD: Passw0rd
      NEO4J_ENCRYPTED: "false"
      MINIO_HOST: minio:9000
      MINIO_USER: dataops-test
      MINIO_PASSWORD: dataops-test-password
      MINIO_BUCKET: dataops-bucket
      MINIO_SECURE: "false"
      DATAFLOW_DEFAULT_DB_HOST: postgres
      DATAFLOW_DEFAULT_DB_PORT: 5432
      DATAFLOW_DEFAULT_DB_NAME: dataops
      DATAFLOW_SCHEMA: dags
      N8N_API_URL: http://n8n:5678
      N8N_API_KEY: ${N8N_API_KEY:-}
      KESTRA_BASE_URL: http://kestra:8080/api/v1
      KESTRA_TENANT_ID: main
      KESTRA_USERNAME: ${KESTRA_USERNAME:-admin@dataops.local}
      KESTRA_PASSWORD: ${KESTRA_PASSWORD:-DataOpsKestra1!}
      RUNNER_TASK_TOKEN_SECRET: ${RUNNER_TASK_TOKEN_SECRET:-dataops-local-runner-task-token-secret-change-me}
      RUNNER_TASK_TOKEN_TTL_SECONDS: 60
      DATA_FACTORY_ACTIVATION_ENABLED: ${DATA_FACTORY_ACTIVATION_ENABLED:-false}
      API_BASE_URL: http://backend:5500/api
      DEEPSEEK_API_KEY: ${DEEPSEEK_API_KEY:-}
      LLM_BASE_URL: https://api.deepseek.com
      LLM_MODEL_NAME: deepseek-chat
      KNOWLEDGE_ENABLED: "true"
      KNOWLEDGE_LIGHTRAG_ENABLED: ${KNOWLEDGE_LIGHTRAG_ENABLED:-false}
      KNOWLEDGE_LIGHTRAG_SHADOW_ONLY: "true"
      KNOWLEDGE_LIGHTRAG_BASE_URL: http://lightrag:9621
      KNOWLEDGE_LIGHTRAG_API_KEY: ${LIGHTRAG_API_KEY:-local-shadow-only-key}
      QWEN_EMBEDDING_BASE_URL: ${QWEN_EMBEDDING_BASE_URL:-}
      QWEN_EMBEDDING_API_KEY: ${QWEN_EMBEDDING_API_KEY:-}
      QWEN_EMBEDDING_MODEL: ${QWEN_EMBEDDING_MODEL:-text-embedding-v3}
      SECRET_KEY: dataops-local-test-secret-key
      AUDIT_EVIDENCE_SECRET: ${AUDIT_EVIDENCE_SECRET:-dataops-local-audit-evidence-secret-change-me}
      AUDIT_EVIDENCE_KEY_VERSION: ${AUDIT_EVIDENCE_KEY_VERSION:-local-v1}
      SECURITY_SIEM_HOST_ALLOWLIST: ${SECURITY_SIEM_HOST_ALLOWLIST:-}
      RULE_GENERATION_RECEIPT_SECRET: ${RULE_GENERATION_RECEIPT_SECRET:-}
      DATASOURCE_CREDENTIAL_MASTER_KEY: ${DATASOURCE_CREDENTIAL_MASTER_KEY:-MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=}
      DATASOURCE_CREDENTIAL_KEY_VERSION: ${DATASOURCE_CREDENTIAL_KEY_VERSION:-v1}
      LOG_DIR: /app/logs
      LOG_TO_CONSOLE: "true"
    ports:
      - "${BACKEND_PORT:-15500}:5500"
    volumes:
      - dataops-test-backend-logs:/app/logs
      - dataops-test-upload:/data/upload
      - dataops-test-archive:/data/archive
    depends_on:
      postgres:
        condition: service_healthy
      neo4j:
        condition: service_healthy
      minio-init:
        condition: service_completed_successfully
      kestra:
        condition: service_healthy
      runner:
        condition: service_healthy
    healthcheck:
      test:
        - CMD
        - python
        - -c
        - import urllib.request; urllib.request.urlopen('http://127.0.0.1:5500/api/system/health', timeout=5)
      interval: 10s
      timeout: 10s
      retries: 20
      start_period: 20s
    networks:
      - dataops-test-net

  lightrag-neo4j:
    image: neo4j:5.26-community
    profiles:
      - knowledge-shadow
    environment:
      NEO4J_AUTH: neo4j/lightrag-test-password
      NEO4J_server_memory_heap_initial__size: 256m
      NEO4J_server_memory_heap_max__size: 512m
      NEO4J_server_memory_pagecache_size: 256m
    volumes:
      - dataops-test-lightrag-neo4j:/data
    healthcheck:
      test: ["CMD-SHELL", "cypher-shell -u neo4j -p lightrag-test-password 'RETURN 1' >/dev/null 2>&1"]
      interval: 10s
      timeout: 10s
      retries: 30
      start_period: 20s
    networks:
      - dataops-test-net

  lightrag:
    image: ghcr.io/hkuds/lightrag@sha256:67ccf8d9f74eb29da872bf8b3e6513605f5ac601fb0509c5b0ca16d98d2d307d
    profiles:
      - knowledge-shadow
    environment:
      HOST: 0.0.0.0
      PORT: 9621
      WORKSPACE: ${LIGHTRAG_SHADOW_WORKSPACE:-dataops-global-shadow-g1}
      LIGHTRAG_API_KEY: ${LIGHTRAG_API_KEY:-local-shadow-only-key}
      LIGHTRAG_KV_STORAGE: PGKVStorage
      LIGHTRAG_DOC_STATUS_STORAGE: PGDocStatusStorage
      LIGHTRAG_VECTOR_STORAGE: PGVectorStorage
      LIGHTRAG_GRAPH_STORAGE: Neo4JStorage
      POSTGRES_HOST: postgres
      POSTGRES_PORT: 5432
      POSTGRES_USER: lightrag
      POSTGRES_PASSWORD: lightrag-test-password
      POSTGRES_DATABASE: dataops_lightrag
      POSTGRES_ENABLE_VECTOR: "true"
      NEO4J_URI: bolt://lightrag-neo4j:7687
      NEO4J_USERNAME: neo4j
      NEO4J_PASSWORD: lightrag-test-password
      NEO4J_DATABASE: neo4j
      LLM_BINDING: openai
      LLM_BINDING_HOST: ${LIGHTRAG_LLM_BASE_URL:-https://api.deepseek.com/v1}
      LLM_BINDING_API_KEY: ${LIGHTRAG_LLM_API_KEY:-}
      LLM_MODEL: ${LIGHTRAG_LLM_MODEL:-deepseek-chat}
      EMBEDDING_BINDING: openai
      EMBEDDING_BINDING_HOST: ${QWEN_EMBEDDING_BASE_URL:-}
      EMBEDDING_BINDING_API_KEY: ${QWEN_EMBEDDING_API_KEY:-}
      EMBEDDING_MODEL: ${QWEN_EMBEDDING_MODEL:-text-embedding-v3}
      EMBEDDING_DIM: 1024
      LIGHTRAG_PARSER: "*:legacy-F"
    depends_on:
      postgres:
        condition: service_healthy
      lightrag-neo4j:
        condition: service_healthy
    healthcheck:
      test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:9621/health >/dev/null"]
      interval: 10s
      timeout: 5s
      retries: 30
      start_period: 30s
    networks:
      - dataops-test-net

  lightrag-projector:
    image: ${DATAOPS_LIGHTRAG_PROJECTOR_IMAGE:-dataops-platform-lightrag-projector:local}
    build:
      context: ../..
      dockerfile: deploy/docker/backend.Dockerfile
    profiles:
      - knowledge-shadow
    command: ["python", "-m", "app.core.knowledge.lightrag.worker"]
    environment:
      DATABASE_URL: postgresql://dataops:dataops-test-password@postgres:5432/dataops
      KNOWLEDGE_LIGHTRAG_BASE_URL: http://lightrag:9621
      KNOWLEDGE_LIGHTRAG_API_KEY: ${LIGHTRAG_API_KEY:-local-shadow-only-key}
      KNOWLEDGE_LIGHTRAG_TIMEOUT_SECONDS: 15
      LIGHTRAG_SHADOW_WORKSPACE: ${LIGHTRAG_SHADOW_WORKSPACE:-dataops-global-shadow-g1}
    depends_on:
      postgres:
        condition: service_healthy
      lightrag:
        condition: service_healthy
    restart: unless-stopped
    networks:
      - dataops-test-net

  frontend:
    image: ${DATAOPS_FRONTEND_IMAGE:-dataops-platform-frontend:local}
    build:
      context: ../..
      dockerfile: deploy/docker/frontend.Dockerfile
      args:
        VUE_APP_N8N_URL: http://localhost:15678
        VUE_APP_N8N_CHAT_URL: ""
        VUE_APP_MINIO_PUBLIC_URL: http://localhost:19000/dataops-bucket
        VUE_APP_PREVIEW_URL: ""
    ports:
      - "${FRONTEND_PORT:-18183}:80"
    depends_on:
      backend:
        condition: service_healthy
    healthcheck:
      test: ["CMD-SHELL", "wget --spider -q http://127.0.0.1/"]
      interval: 10s
      timeout: 5s
      retries: 20
    networks:
      - dataops-test-net

networks:
  dataops-test-net:
    name: ${DATAOPS_NETWORK_NAME:-dataops-test-net}

volumes:
  dataops-test-postgres:
  dataops-test-source-postgres:
  dataops-test-source-mysql:
  dataops-test-neo4j:
  dataops-test-minio:
  dataops-test-n8n:
  dataops-test-kestra:
  dataops-test-backend-logs:
  dataops-test-upload:
  dataops-test-archive:
  dataops-test-lightrag-neo4j: