docker-compose-cloudlab.yml

version: "3"

services:
  redis-state:
    image: kubasz51/faasm-redis:${REDIS_VERSION}

  redis-queue:
    image: kubasz51/faasm-redis:${REDIS_VERSION}

  # minio:
  #   image: kubasz51/faasm-minio:${FAASM_VERSION}
  #   ports:
  #     - "9000:9000"
  #   environment:
  #     MINIO_ROOT_USER: minio
  #     MINIO_ROOT_PASSWORD: minio123
  #   volumes:
  #     - ./dev/minio/data:/data/minio
  #   healthcheck:
  #     test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ]
  #     interval: 5s
  #     timeout: 3s
  #     retries: 3

  ceph-mon1:
    image: kubasz51/faasm-ceph:${FAASM_VERSION}
    hostname: ceph-mon1
    entrypoint: ["/usr/bin/tini-static", "/bin/bash", "/usr/local/code/faasm/bin/run_ceph_mon.sh"]
    deploy:
      placement:
        constraints: [ node.labels.rank == leader ]
    volumes:
      - ./:/usr/local/code/faasm/
      - ./deploy/conf/ceph/ceph.conf:/etc/ceph/ceph.conf:ro
      - ${FAASM_BUILD_DIR}:${FAASM_BUILD_MOUNT}
      - ${FAASM_BUILD_DIR}/lib/libfaasm_ceph.so:/usr/lib/x86_64-linux-gnu/rados-classes/libcls_faasm.so:ro
      - ./dev/faasm-local/:/usr/local/faasm/
      - ./dev/dev-run/:/run/faasm
      - /sys/kernel/debug:/sys/kernel/debug
      - /sys/kernel/debug/tracing:/sys/kernel/debug/tracing
      - ./dev/container/shared_store:/data
    restart: on-failure

  ceph-osd1: &ceph-osd1
    image: kubasz51/faasm-ceph:${FAASM_VERSION}
    hostname: ceph-osd1
    entrypoint: ["/usr/bin/tini-static", "/bin/bash", "/usr/local/code/faasm/bin/run_ceph_osd.sh"]
    depends_on:
      - ceph-mon1
    deploy:
      placement:
        constraints: [ node.labels.name == storage0 ]
    volumes:
      - ./:/usr/local/code/faasm/
      - ./deploy/conf/ceph/ceph.conf:/etc/ceph/ceph.conf:ro
      - ${FAASM_BUILD_DIR}:${FAASM_BUILD_MOUNT}
      - ${FAASM_BUILD_DIR}/lib/libfaasm_ceph.so:/usr/lib/x86_64-linux-gnu/rados-classes/libcls_faasm.so:ro
      - ./dev/faasm-local/:/usr/local/faasm/
      - ./dev/dev-run/:/run/faasm
      - /sys/kernel/debug:/sys/kernel/debug
      - /sys/kernel/debug/tracing:/sys/kernel/debug/tracing
      - /dev:/dev
      - /mnt/ceph:/var/lib/ceph/osd/ceph-0
      - /mnt/ceph:/var/lib/ceph/osd/ceph-1
      - /mnt/ceph:/var/lib/ceph/osd/ceph-2
      - ./dev/container/shared_store:/osd
    restart: on-failure

  ceph-osd2:
    <<: *ceph-osd1
    hostname: ceph-osd2
    deploy:
      placement:
        constraints: [ node.labels.name == storage1 ]

  ceph-osd3:
    <<: *ceph-osd1
    hostname: ceph-osd3
    deploy:
      placement:
        constraints: [ node.labels.name == storage2 ]

  # ceph-osd4:
  #   <<: *ceph-osd1
  #   hostname: ceph-osd4
  #   deploy:
  #     placement:
  #       constraints: [ node.labels.name == storage3 ]

  # ceph-osd5:
  #   <<: *ceph-osd1
  #   hostname: ceph-osd5
  #   deploy:
  #     placement:
  #       constraints: [ node.labels.name == storage4 ]

  upload:
    image: kubasz51/faasm-upload:${FAASM_VERSION}
    ports:
      - "8002:8002"
    expose:
      - "5000"
    depends_on:
      - redis-queue
      - redis-state
      - ceph-mon1
      - ceph-osd1
      - ceph-osd2
      - ceph-osd3
    restart: on-failure
    deploy:
      placement: 
        constraints: [ node.labels.rank == leader ]
    volumes:
      - ./:/usr/local/code/faasm/
      - ${FAASM_BUILD_DIR}:${FAASM_BUILD_MOUNT}
      - ./dev/faasm-local/:${FAASM_LOCAL_MOUNT}
      - ./deploy/conf/ceph/ceph.conf:/etc/ceph/ceph.conf:ro
      - ./dev/faasm-local/wasm/:/usr/local/faasm/wasm
      - ./dev/faasm-local/object/:/usr/local/faasm/object
      - ./bin/entrypoint_upload.sh:/entrypoint.sh
      - /sys/kernel/debug:/sys/kernel/debug
      - /sys/kernel/debug/tracing:/sys/kernel/debug/tracing
    environment:
      - CODEGEN_TARGETS=x86_64:skylake;aarch64:thunderx2t99
      - DELTA_SNAPSHOT_ENCODING=pages=64
      - LOG_LEVEL=${LOG_LEVEL:-debug}
      - PYTHON_CODEGEN=${PYTHON_CODEGEN:-off}
      - REDIS_QUEUE_HOST=redis-queue
      - REDIS_STATE_HOST=redis-state
      - LD_LIBRARY_PATH=/build/faasm/third-party/lib:/usr/local/lib
      - WASM_VM=${WASM_VM:-wavm}
      - ASAN_OPTIONS=verbosity=1:halt_on_error=1
      - TSAN_OPTIONS=halt_on_error=1:suppressions=/usr/local/code/faasm/thread-sanitizer-ignorelist.txt:history_size=7:second_deadlock_stack=1
      - UBSAN_OPTIONS="print_stacktrace=1:halt_on_error=1
    healthcheck:
      test: [ "CMD", "curl", "-f", "http://localhost:8002/ping" ]
      interval: 5s
      timeout: 3s
      retries: 10

  # Distributed worker, used for executing as a local cluster
  # worker:
  #   image: kubasz51/faasm-worker:${FAASM_VERSION}
  #   security_opt:
  #     - seccomp:unconfined
  #   # Allows for tracy thread&ctx switch tracking to work better
  #   pid: "host"
  #   depends_on:
  #     - upload
  #   expose:
  #     - "8080"
  #     - "5000"
  #     - "8003-8010"
  #   ports:
  #     - "8080:8080"
  #     - "8086:8086"
  #   privileged: true
  #   deploy:
  #     replicas: 2
  #     placement:
  #       spread: [ node.labels.type == compute ]
  #   volumes:
  #     - ./:/usr/local/code/faasm/
  #     - /tmp/faasm-monitor:/var/faasm-monitor
  #     - ./deploy/conf/ceph/ceph.conf:/etc/ceph/ceph.conf:ro
  #     - ${FAASM_BUILD_DIR}:${FAASM_BUILD_MOUNT}
  #     - ./dev/faasm-local/:${FAASM_LOCAL_MOUNT}
  #     - aesmd-socket:/var/run/aesmd
  #     - ./dev/container/shared_store/:/usr/local/faasm/debug_shared_store/
  #     - ${SGX_DEVICE_MOUNT_DIR:-./dev/faasm-local/sgx}:/dev/sgx
  #     - ./bin/entrypoint_worker.sh:/entrypoint.sh
  #     - /sys/kernel/debug:/sys/kernel/debug
  #     - /sys/kernel/debug/tracing:/sys/kernel/debug/tracing
  #   environment:
  #     - TINI_SUBREAPER=1
  #     - CODEGEN_TARGETS=x86_64:skylake;aarch64:thunderx2t99
  #     - CAPTURE_STDOUT=on
  #     # 02/11/2022 - Temporarily disable CGroup mode, to mitigate a potential
  #     # mismatch between the host OS and the container image. Ubuntu 20.04
  #     # uses CGroup v1, whereas Ubuntu 22.04 uses CGroup v2. This change will
  #     # be reverted once we upgrade container images to 22.04
  #     - CGROUP_MODE=off
  #     - DELTA_SNAPSHOT_ENCODING=pages=64
  #     - GLOBAL_MESSAGE_TIMEOUT=30000
  #     - LOG_LEVEL=${LOG_LEVEL:-debug}
  #     - NETNS_MODE=off
  #     - REDIS_QUEUE_HOST=redis-queue
  #     - REDIS_STATE_HOST=redis-state
  #     - SCHEDULER_MONITOR_FILE=/var/faasm-monitor
  #     - STATE_MODE=redis
  #     - LD_LIBRARY_PATH=/build/faasm/third-party/lib:/usr/local/lib
  #     - ENDPOINT_HOST=worker
  #     - ENDPOINT_NUM_THREADS=4
  #     - OVERRIDE_CPU_COUNT=8
  #     - VM_ARENA_MODE=${VM_ARENA_MODE:-}
  #     - ASAN_OPTIONS=verbosity=1:halt_on_error=1
  #     - TSAN_OPTIONS=halt_on_error=1:suppressions=/usr/local/code/faasm/thread-sanitizer-ignorelist.txt:history_size=7:second_deadlock_stack=1
  #     - UBSAN_OPTIONS="print_stacktrace=1:halt_on_error=1

  # Distributed worker, used for executing as a local cluster
  worker-0:
    image: kubasz51/faasm-worker:${FAASM_VERSION}
    security_opt:
      - seccomp:unconfined
    # Allows for tracy thread&ctx switch tracking to work better
    pid: "host"
    depends_on:
      - upload
    expose:
      - "8080"
      - "5000"
      - "8003-8010"
    ports:
      - "8078:8080"
      - "8084:8086"
    privileged: true
    deploy:
      placement:
        constraints: [ node.labels.name == compute0 ]
    volumes:
      - ./:/usr/local/code/faasm/
      - /tmp/faasm-monitor:/var/faasm-monitor
      - ./deploy/conf/ceph/ceph.conf:/etc/ceph/ceph.conf:ro
      - ${FAASM_BUILD_DIR}:${FAASM_BUILD_MOUNT}
      - ./dev/faasm-local/:${FAASM_LOCAL_MOUNT}
      - aesmd-socket:/var/run/aesmd
      - ./dev/container/shared_store/:/usr/local/faasm/debug_shared_store/
      - ${SGX_DEVICE_MOUNT_DIR:-./dev/faasm-local/sgx}:/dev/sgx
      - ./bin/entrypoint_worker.sh:/entrypoint.sh
      - /sys/kernel/debug:/sys/kernel/debug
      - /sys/kernel/debug/tracing:/sys/kernel/debug/tracing
    environment:
      - TINI_SUBREAPER=1
      - CODEGEN_TARGETS=x86_64:skylake;aarch64:thunderx2t99
      - CAPTURE_STDOUT=on
      # 02/11/2022 - Temporarily disable CGroup mode, to mitigate a potential
      # mismatch between the host OS and the container image. Ubuntu 20.04
      # uses CGroup v1, whereas Ubuntu 22.04 uses CGroup v2. This change will
      # be reverted once we upgrade container images to 22.04
      - CGROUP_MODE=off
      - DELTA_SNAPSHOT_ENCODING=pages=64
      - GLOBAL_MESSAGE_TIMEOUT=30000
      - LOAD_BALANCE_POLICY=least_load
      - LOG_LEVEL=${LOG_LEVEL:-debug}
      - NETNS_MODE=off
      - REDIS_QUEUE_HOST=redis-queue
      - REDIS_STATE_HOST=redis-state
      - SCHEDULER_MONITOR_FILE=/var/faasm-monitor
      - STATE_MODE=redis
      - LD_LIBRARY_PATH=/build/faasm/third-party/lib:/usr/local/lib
      - ENDPOINT_HOST=worker-0
      - ENDPOINT_NUM_THREADS=4
      - OVERRIDE_CPU_COUNT=0
      - VM_ARENA_MODE=${VM_ARENA_MODE:-}
      - ASAN_OPTIONS=verbosity=1:halt_on_error=1
      - TSAN_OPTIONS=halt_on_error=1:suppressions=/usr/local/code/faasm/thread-sanitizer-ignorelist.txt:history_size=7:second_deadlock_stack=1
      - UBSAN_OPTIONS="print_stacktrace=1:halt_on_error=1

  # Distributed worker, used for executing as a local cluster
  worker-1:
    image: kubasz51/faasm-worker:${FAASM_VERSION}
    security_opt:
      - seccomp:unconfined
    # Allows for tracy thread&ctx switch tracking to work better
    pid: "host"
    depends_on:
      - upload
    expose:
      - "8080"
      - "5000"
      - "8003-8010"
    ports:
      - "8079:8080"
      - "8085:8086"
    privileged: true
    deploy:
      placement:
        constraints: [ node.labels.name == compute1 ]
    volumes:
      - ./:/usr/local/code/faasm/
      - /tmp/faasm-monitor:/var/faasm-monitor
      - ./deploy/conf/ceph/ceph.conf:/etc/ceph/ceph.conf:ro
      - ${FAASM_BUILD_DIR}:${FAASM_BUILD_MOUNT}
      - ./dev/faasm-local/:${FAASM_LOCAL_MOUNT}
      - aesmd-socket:/var/run/aesmd
      - ./dev/container/shared_store/:/usr/local/faasm/debug_shared_store/
      - ${SGX_DEVICE_MOUNT_DIR:-./dev/faasm-local/sgx}:/dev/sgx
      - ./bin/entrypoint_worker.sh:/entrypoint.sh
      - /sys/kernel/debug:/sys/kernel/debug
      - /sys/kernel/debug/tracing:/sys/kernel/debug/tracing
    environment:
      - TINI_SUBREAPER=1
      - CODEGEN_TARGETS=x86_64:skylake;aarch64:thunderx2t99
      - CAPTURE_STDOUT=on
      # 02/11/2022 - Temporarily disable CGroup mode, to mitigate a potential
      # mismatch between the host OS and the container image. Ubuntu 20.04
      # uses CGroup v1, whereas Ubuntu 22.04 uses CGroup v2. This change will
      # be reverted once we upgrade container images to 22.04
      - CGROUP_MODE=off
      - DELTA_SNAPSHOT_ENCODING=pages=64
      - LOAD_BALANCE_POLICY=least_load
      - GLOBAL_MESSAGE_TIMEOUT=30000
      - LOG_LEVEL=${LOG_LEVEL:-debug}
      - NETNS_MODE=off
      - REDIS_QUEUE_HOST=redis-queue
      - REDIS_STATE_HOST=redis-state
      - SCHEDULER_MONITOR_FILE=/var/faasm-monitor
      - STATE_MODE=redis
      - LD_LIBRARY_PATH=/build/faasm/third-party/lib:/usr/local/lib
      - ENDPOINT_HOST=worker-1
      - ENDPOINT_NUM_THREADS=4
      - OVERRIDE_CPU_COUNT=0
      - VM_ARENA_MODE=${VM_ARENA_MODE:-}
      - ASAN_OPTIONS=verbosity=1:halt_on_error=1
      - TSAN_OPTIONS=halt_on_error=1:suppressions=/usr/local/code/faasm/thread-sanitizer-ignorelist.txt:history_size=7:second_deadlock_stack=1
      - UBSAN_OPTIONS="print_stacktrace=1:halt_on_error=1

  # Distributed worker, used for executing as a local cluster
  worker-2:
    image: kubasz51/faasm-worker:${FAASM_VERSION}
    security_opt:
      - seccomp:unconfined
    # Allows for tracy thread&ctx switch tracking to work better
    pid: "host"
    depends_on:
      - upload
    expose:
      - "8080"
      - "5000"
      - "8003-8010"
    ports:
      - "8080:8080"
      - "8086:8086"
    privileged: true
    deploy:
      placement:
        constraints: [ node.labels.name == compute2 ]
    volumes:
      - ./:/usr/local/code/faasm/
      - /tmp/faasm-monitor:/var/faasm-monitor
      - ./deploy/conf/ceph/ceph.conf:/etc/ceph/ceph.conf:ro
      - ${FAASM_BUILD_DIR}:${FAASM_BUILD_MOUNT}
      - ./dev/faasm-local/:${FAASM_LOCAL_MOUNT}
      - aesmd-socket:/var/run/aesmd
      - ./dev/container/shared_store/:/usr/local/faasm/debug_shared_store/
      - ${SGX_DEVICE_MOUNT_DIR:-./dev/faasm-local/sgx}:/dev/sgx
      - ./bin/entrypoint_worker.sh:/entrypoint.sh
      - /sys/kernel/debug:/sys/kernel/debug
      - /sys/kernel/debug/tracing:/sys/kernel/debug/tracing
    environment:
      - TINI_SUBREAPER=1
      - CODEGEN_TARGETS=x86_64:skylake;aarch64:thunderx2t99
      - CAPTURE_STDOUT=on
      - LOAD_BALANCE_POLICY=least_load
      # 02/11/2022 - Temporarily disable CGroup mode, to mitigate a potential
      # mismatch between the host OS and the container image. Ubuntu 20.04
      # uses CGroup v1, whereas Ubuntu 22.04 uses CGroup v2. This change will
      # be reverted once we upgrade container images to 22.04
      - CGROUP_MODE=off
      - DELTA_SNAPSHOT_ENCODING=pages=64
      - GLOBAL_MESSAGE_TIMEOUT=30000
      - LOG_LEVEL=${LOG_LEVEL:-debug}
      - NETNS_MODE=off
      - REDIS_QUEUE_HOST=redis-queue
      - REDIS_STATE_HOST=redis-state
      - SCHEDULER_MONITOR_FILE=/var/faasm-monitor
      - STATE_MODE=redis
      - LD_LIBRARY_PATH=/build/faasm/third-party/lib:/usr/local/lib
      - ENDPOINT_HOST=worker-2
      - ENDPOINT_NUM_THREADS=4
      - OVERRIDE_CPU_COUNT=0
      - VM_ARENA_MODE=${VM_ARENA_MODE:-}
      - ASAN_OPTIONS=verbosity=1:halt_on_error=1
      - TSAN_OPTIONS=halt_on_error=1:suppressions=/usr/local/code/faasm/thread-sanitizer-ignorelist.txt:history_size=7:second_deadlock_stack=1
      - UBSAN_OPTIONS="print_stacktrace=1:halt_on_error=1

  worker-storage:
    image: kubasz51/faasm-worker:${FAASM_VERSION}
    security_opt:
      - seccomp:unconfined
    # Allows for tracy thread&ctx switch tracking to work better
    pid: "host"
    depends_on:
      - upload
    expose:
      - "8080"
      - "5000"
      - "8003-8010"
    ports:
      - "8080"
      - "8087:8086"
    privileged: true
    deploy:
      replicas: 3
      placement:
        constraints: [ node.labels.type == storage ]
    volumes:
      - ./:/usr/local/code/faasm/
      - /tmp/faasm-monitor-storage:/var/faasm-monitor
      - ./deploy/conf/ceph/ceph.conf:/etc/ceph/ceph.conf:ro
      - ${FAASM_BUILD_DIR}:${FAASM_BUILD_MOUNT}
      - ./dev/faasm-local/:${FAASM_LOCAL_MOUNT}
      - ./dev/dev-run/:/run/faasm
      - aesmd-socket:/var/run/aesmd
      - ./dev/container/shared_store/:/usr/local/faasm/debug_shared_store/
      - ${SGX_DEVICE_MOUNT_DIR:-./dev/faasm-local/sgx}:/dev/sgx
      - ./bin/entrypoint_worker.sh:/entrypoint.sh
      - /sys/kernel/debug:/sys/kernel/debug
      - /sys/kernel/debug/tracing:/sys/kernel/debug/tracing
    environment:
      - IS_STORAGE_NODE=1
      - TINI_SUBREAPER=1
      - OFFLOAD_CPU_THRESHOLD=0.7
      - OFFLOAD_RAM_THRESHOLD=0.8
      - OFFLOAD_LOAD_AVG_FACTOR_THRESHOLD=0.7
      - CODEGEN_TARGETS=x86_64:skylake;aarch64:thunderx2t99
      - CAPTURE_STDOUT=on
      - CGROUP_MODE=off
      - DELTA_SNAPSHOT_ENCODING=pages=64
      - GLOBAL_MESSAGE_TIMEOUT=30000
      - LOG_LEVEL=${LOG_LEVEL:-debug}
      - NETNS_MODE=off
      - REDIS_QUEUE_HOST=redis-queue
      - REDIS_STATE_HOST=redis-state
      - SCHEDULER_MONITOR_FILE=/var/faasm-monitor
      - STATE_MODE=redis
      - LD_LIBRARY_PATH=/build/faasm/third-party/lib:/usr/local/lib
      - ENDPOINT_HOST=worker-storage
      - ENDPOINT_NUM_THREADS=4
      - OVERRIDE_CPU_COUNT=0
      - VM_ARENA_MODE=${VM_ARENA_MODE:-}
      - ASAN_OPTIONS=verbosity=1:halt_on_error=1
      - TSAN_OPTIONS=halt_on_error=1:suppressions=/usr/local/code/faasm/thread-sanitizer-ignorelist.txt:history_size=7:second_deadlock_stack=1
      - UBSAN_OPTIONS="print_stacktrace=1:halt_on_error=1

  # Load balancer for local cluster
  nginx:
    image: nginx:latest
    volumes:
      - ./deploy/conf/nginx-local.conf:/etc/nginx/nginx.conf:ro

  # C/C++ functions
  cpp:
    image: ${CPP_CLI_IMAGE}
    working_dir: /code/cpp
    stdin_open: true
    tty: true
    deploy:
      replicas: 7
      # placement:
      #   constraints: [ node.labels.type == loadgen ]
    volumes:
      - ./clients/cpp:/code/cpp
      - ./dev/faasm-local/:${FAASM_LOCAL_MOUNT}
      - ./deploy/conf/ceph/ceph.conf:/etc/ceph/ceph.conf:ro

  # Python functions
  # python:
  #   image: ${PYTHON_CLI_IMAGE}
  #   working_dir: /code/python
  #   stdin_open: true
  #   tty: true
  #   volumes:
  #     - ./clients/python:/code/python
  #     - ./dev/faasm-local/:${FAASM_LOCAL_MOUNT}
  #     - ./deploy/conf/ceph/ceph.conf:/etc/ceph/ceph.conf:ro

  # Faasm dev
  faasm-cli:
    image: ${FAASM_CLI_IMAGE}
    working_dir: /usr/local/code/faasm
    stdin_open: true
    tty: true
    privileged: true
    deploy:
      replicas: 7
    depends_on:
      - redis-state
      - redis-queue
    environment:
      - UPLOAD_HOST=${UPLOAD_HOST:-upload}
      - INVOKE_HOST=${INVOKE_HOST:-worker-0}
      - LOG_LEVEL=${LOG_LEVEL:-debug}
      - REDIS_QUEUE_HOST=redis-queue
      - REDIS_STATE_HOST=redis-state
      - LD_LIBRARY_PATH=/build/faasm/third-party/lib:/usr/local/lib
      - ASAN_OPTIONS=halt_on_error=1:quarantine_size_mb=16
      - LSAN_OPTIONS=suppressions=./leak-sanitizer-ignorelist.txt
      - TSAN_OPTIONS="halt_on_error=1 suppressions=./thread-sanitizer-ignorelist.txt history_size=0 flush_memory_ms=5000"
      - UBSAN_OPTIONS=print_stacktrace=1:halt_on_error=1:suppressions=./ub-sanitizer-ignorelist.txt
      - WASM_VM=${WASM_VM:-wavm}
      - SGX_AESM_ADDR=1
      - AZ_ATTESTATION_PROVIDER_URL=https://faasmattprov.eus2.attest.azure.net
      - AZDCAP_DEBUG_LOG_LEVEL=info
    volumes:
      - ./:/usr/local/code/faasm/
      - ${FAASM_BUILD_DIR}:${FAASM_BUILD_MOUNT}
      - ./deploy/conf/ceph/ceph.conf:/etc/ceph/ceph.conf:ro
      - ./dev/faasm-local/:${FAASM_LOCAL_MOUNT}
      - ${CONAN_CACHE_MOUNT_SOURCE}:/root/.conan
      - aesmd-socket:/var/run/aesmd
      - ${SGX_DEVICE_MOUNT_DIR:-./dev/faasm-local/sgx}:/dev/sgx

  # SGX AESM service to mock the remote attestation environment in production
  # There is no need to re-create the image every time, as it does not depend
  # on Faasm's code
  aesmd:
    image: faasm/sgx-aesmd:0.8.9
    devices:
      - ${SGX_DEVICE_MOUNT_DIR:-./dev/faasm-local/sgx}/enclave
      - ${SGX_DEVICE_MOUNT_DIR:-./dev/faasm-local/sgx}/provision
    environment:
      - AZDCAP_DEBUG_LOG_LEVEL=info
    volumes:
      - aesmd-socket:/var/run/aesmd
    stdin_open: true
    tty: true

  # Distributed tests server
  dist-test-server:
    image: ${FAASM_CLI_IMAGE}
    working_dir: /build/faasm
    depends_on:
      - redis-queue
      - redis-state
      - ceph-mon1
      - ceph-osd1
      - ceph-osd2
      - ceph-osd3
      - upload
    environment:
      - LD_LIBRARY_PATH=/usr/local/lib
      - LOG_LEVEL=${LOG_LEVEL:-debug}
      - REDIS_QUEUE_HOST=redis-queue
      - REDIS_STATE_HOST=redis-state
      - OVERRIDE_CPU_COUNT=4
      - CGROUP_MODE=off
      - GLOBAL_MESSAGE_TIMEOUT=120000
      - BOUND_TIMEOUT=60000
    command: ./bin/dist_test_server
    volumes:
      - ./:/usr/local/code/faasm/
      - ${FAASM_BUILD_DIR}:${FAASM_BUILD_MOUNT}

  # Profiling/ perf testing
  vector:
    image: netflixoss/vector:latest
    depends_on:
      - faasm-cli
    ports:
      - "80:80"

# This volume is only needed when running SGX in hardware mode, in an
# SGX-enabled machine. It enables communication between the AESMD container
# and the SGX-enabled client or worker. It is needed for remote attestation.
volumes:
  aesmd-socket:
    external: ${AESMD_SOCKET_EXTERNAL_VOLUME:-false}