refactor: compose file vars

humanprotocol · Jan 22, 2025 · e7370e3 · e7370e3
1 parent 8f72de8
commit e7370e3
Show file tree

Hide file tree

Showing 2 changed files with 60 additions and 55 deletions.
diff --git a/scripts/cvat/docker-compose.local.yml b/scripts/cvat/docker-compose.local.yml
@@ -1,13 +1,13 @@
 name: human-protocol-local-setup
 
-x-default-config:
+x-service-default-config:
   restart: &default-restart unless-stopped
   logging: &default-logging
     options:
       max-size: 10m
       max-file: 3
 
-x-default-vars:
+x-hardcoded-vars:
   frontend_default_port: &frontend_default_port 3000
   all_interfaces_ip: &all_interfaces_ip '0.0.0.0'
   web3_env: &web3_env testnet
@@ -16,77 +16,81 @@ x-default-vars:
   redis_host: &redis_host redis
   redis_port: &redis_port 6379
   minio_host: &minio_host minio
+  node_env: &node_env local
+  cvat_lb_url: &cvat_lb_url 'http://cvat-lb:8080'
+  cvat_oracle_storage_provider: &cvat_oracle_storage_provider aws
 
-x-env-variables:
-  backend_apps_internal_port: &backend_apps_internal_port ${BACKEND_APPS_INTERNAL_PORT:?}
+x-general-env-variables:
+  # GENERAL VARS
+  subgraph_api_key: &subgraph_api_key ${SUBGRAPH_API_KEY}
+  sendgrid_api_key: &sendgrid_api_key ${SENDGRID_API_KEY:-sendgrid-disabled}
+  rpc_url_polygon_amoy: &rpc_url_polygon_amoy ${RPC_URL_POLYGON_AMOY:-}
+  # POSTGRES VARS
   postgres_user: &postgres_user ${POSTGRES_USER:-default}
   postgres_password: &postgres_password ${POSTGRES_PASSWORD:-qwerty}
+  # MINIO VARS
   minio_port: &minio_port ${MINIO_PORT:?}
   minio_console_port: &minio_console_port ${MINIO_CONSOLE_PORT:-9001}
-  # same default is in minio docs
   minio_root_user: &minio_root_user ${MINIO_ROOT_USER:-minioadmin}
-  # same default is in minio docs
   minio_root_password: &minio_root_password ${MINIO_ROOT_PASSWORD:-minioadmin}
   minio_services_access_key: &minio_services_access_key ${MINIO_SERVICES_ACCESS_KEY:-human-oracle}
   minio_services_secret_key: &minio_services_secret_key ${MINIO_SERVICES_SECRET_KEY:-human-oracle-s3-secret}
-  minio_bucket_manifests: &minio_bucket_manifests ${MINIO_BUCKET_MANIFESTS:-manifests}
-  minio_bucket_datasets: &minio_bucket_datasets ${MINIO_BUCKET_MANIFESTS:-datasets}
-  minio_bucket_rep_o: &minio_bucket_rep_o ${MINIO_BUCKET_REPUTATION_ORACLE:-reputation-oracle}
-  minio_bucket_exc_o: &minio_bucket_exc_o ${MINIO_BUCKET_EXCHANGE_ORACLE:-exchange-oracle}
-  minio_bucket_rec_o: &minio_bucket_rec_o ${MINIO_BUCKET_RECORDING_ORACLE:-recording-oracle}
-  node_env: &node_env local
-  human_app_email: &human_app_email ${HUMAN_APP_EMAIL:?}
-  rpc_url_polygon_amoy: &rpc_url_polygon_amoy ${RPC_URL_POLYGON_AMOY:-}
-  subgraph_api_key: &subgraph_api_key ${SUBGRAPH_API_KEY}
-  sendgrid_api_key: &sendgrid_api_key ${SENDGRID_API_KEY}
+  # BUCKET NAMES
+  bucket_name_manifests: &bucket_name_manifests ${BUCKET_NAME_MANIFESTS:-manifests}
+  bucket_name_datasets: &bucket_name_datasets ${BUCKET_NAME_DATASETS:-datasets}
+  bucket_name_rep_o: &bucket_name_rep_o ${BUCKET_NAME_REPUTATION_ORACLE:-reputation-oracle}
+  bucket_name_exc_o: &bucket_name_exc_o ${BUCKET_NAME_EXCHANGE_ORACLE:-exchange-oracle}
+  bucket_name_rec_o: &bucket_name_rec_o ${BUCKET_NAME_RECORDING_ORACLE:-recording-oracle}
+  # WEB3 ADDRESSES
   reputation_oracle_address: &reputation_oracle_address ${REPUTATION_ORACLE_ADDRESS:?}
   exchange_oracle_address: &exchange_oracle_address ${EXCHANGE_ORACLE_ADDRESS:?}
   recording_oracle_address: &recording_oracle_address ${RECORDING_ORACLE_ADDRESS:?}
+  # OTHER
+  backend_apps_internal_port: &backend_apps_internal_port ${BACKEND_APPS_INTERNAL_PORT:?}
+  human_app_email: &human_app_email ${HUMAN_APP_EMAIL:?}
   reputation_oracle_jwt_public_key: &reputation_oracle_jwt_public_key ${REPUTATION_ORACLE_JWT_PUBLIC_KEY:?}
+  cvat_oracle_storage_endpoint: &cvat_oracle_storage_endpoint minio:${MINIO_PORT:?}
 
-x-service-environment:
-  shared_vars: &shared_vars
+x-service-env-vars-groups:
+  postgres_auth_vars: &postgres_auth_vars
+    POSTGRES_USER: *postgres_user
+    POSTGRES_PASSWORD: *postgres_password
+  redis_app_vars: &redis_app_vars
+    REDIS_HOST: *redis_host
+    REDIS_PORT: *redis_port
+  nodejs_app_vars: &nodejs_app_vars
     NODE_ENV: *node_env
     WEB3_ENV: *web3_env
-    HUMAN_APP_EMAIL: *human_app_email
     RPC_URL_POLYGON_AMOY: *rpc_url_polygon_amoy
     SENDGRID_API_KEY: *sendgrid_api_key
     SUBGRAPH_API_KEY: *subgraph_api_key
     HCAPTCHA_SITE_KEY: ${HCAPTCHA_SITE_KEY:-10000000-ffff-ffff-ffff-000000000001}
     HCAPTCHA_SECRET: ${HCAPTCHA_SECRET:-0x0000000000000000000000000000000000000000}
     HCAPTCHA_API_KEY: ${HCAPTCHA_API_KEY:-test}
-  postgres_auth_vars: &postgres_auth_vars
-    POSTGRES_USER: *postgres_user
-    POSTGRES_PASSWORD: *postgres_password
-  postgres_connection_vars: &postgres_connection_vars
+  nodejs_app_postgres_vars: &nodejs_app_postgres_vars
     <<: *postgres_auth_vars
     POSTGRES_HOST: *postgres_host
     POSTGRES_PORT: *postgres_port
     POSTGRES_SSL: 'false'
-  postgres_cvat_oracles_vars: &postgres_cvat_oracles_vars
-    PG_HOST: *postgres_host
-    PG_PORT: *postgres_port
-    PG_USER: *postgres_user
-    PG_PASSWORD: *postgres_password
-  redis_connection_vars: &redis_connection_vars
-    REDIS_HOST: *redis_host
-    REDIS_PORT: *redis_port
-  s3_connection_vars: &s3_connection_vars
+  nodejs_app_s3_vars: &nodejs_app_s3_vars
     S3_ENDPOINT: *minio_host
     S3_PORT: *minio_port
     S3_ACCESS_KEY: *minio_services_access_key
     S3_SECRET_KEY: *minio_services_secret_key
     S3_USE_SSL: 'false'
-  cvat_oracle_storage_provider: &cvat_oracle_storage_provider aws
-  cvat_oracle_storage_endpoint: &cvat_oracle_storage_endpoint minio:${MINIO_PORT:?}
+  cvat_oracle_postgres_vars: &cvat_oracle_postgres_vars:
+    PG_HOST: *postgres_host
+    PG_PORT: *postgres_port
+    PG_USER: *postgres_user
+    PG_PASSWORD: *postgres_password
   cvat_oracle_storage_connection_vars: &cvat_oracle_storage_connection_vars
     STORAGE_PROVIDER: *cvat_oracle_storage_provider
     STORAGE_ENDPOINT_URL: *cvat_oracle_storage_endpoint
     STORAGE_ACCESS_KEY: *minio_services_access_key
     STORAGE_SECRET_KEY: *minio_services_secret_key
     STORAGE_USE_SSL: 'false'
-  cvat_vars: &cvat_vars
-    CVAT_URL: "http://cvat-lb:8080"
+  cvat_connection_vars: &cvat_connection_vars
+    CVAT_URL: *cvat_lb_url
     CVAT_ADMIN: ${CVAT_ADMIN:-human-protocol}
     CVAT_ADMIN_PASS: ${CVAT_ADMIN_PASS:-qwe123qwe123Q!}
     CVAT_ORG_SLUG: HumanAppLocal
@@ -181,11 +185,11 @@ services:
       MINIO_ROOT_PASSWORD: *minio_root_password
       SERVICES_ACCESS_KEY: *minio_services_access_key
       SERVICES_SECRET_KEY: *minio_services_secret_key
-      BUCKET_MANIFESTS: *minio_bucket_manifests
-      BUCKET_DATASETS: *minio_bucket_datasets
-      BUCKET_REPUTATION_ORACLE: *minio_bucket_rep_o
-      BUCKET_EXCHANGE_ORACLE: *minio_bucket_exc_o
-      BUCKET_RECORDING_ORACLE: *minio_bucket_rec_o
+      BUCKET_MANIFESTS: *bucket_name_manifests
+      BUCKET_DATASETS: *bucket_name_datasets
+      BUCKET_REPUTATION_ORACLE: *bucket_name_rep_o
+      BUCKET_EXCHANGE_ORACLE: *bucket_name_exc_o
+      BUCKET_RECORDING_ORACLE: *bucket_name_rec_o
     entrypoint: >
       /bin/sh -c "
         mc alias set myminio http://minio:9000 $$MINIO_ROOT_USER $$MINIO_ROOT_PASSWORD
@@ -236,12 +240,13 @@ services:
         published: ${REPUTATION_ORACLE_PORT:-5001}
     env_file: ./.env.reputation-oracle.local
     environment:
-      <<: [*shared_vars, *postgres_connection_vars, *s3_connection_vars]
+      <<: [*nodejs_app_vars, *nodejs_app_postgres_vars, *nodejs_app_s3_vars]
       HOST: *all_interfaces_ip
       PORT: *backend_apps_internal_port
       POSTGRES_DATABASE: reputation-oracle
-      S3_BUCKET: *minio_bucket_rep_o
+      S3_BUCKET: *bucket_name_rep_o
       KYC_API_PRIVATE_KEY: ${KYC_API_PRIVATE_KEY:-none}
+      HUMAN_APP_EMAIL: *human_app_email
       # It is accessed by user, not from container
       # so put here exposed port, not internal
       FE_URL: http://localhost:${HUMAN_APP_CLIENT_PORT:?}
@@ -270,11 +275,12 @@ services:
         published: ${HUMAN_APP_SERVER_PORT:-5002}
     env_file: ./.env.human-app-server.local
     environment:
-      <<: [*shared_vars, *redis_connection_vars]
+      <<: [*nodejs_app_vars, *redis_app_vars]
       HOST: *all_interfaces_ip
       PORT: *backend_apps_internal_port
       REDIS_DB: 1
       RPC_URL: *rpc_url_polygon_amoy
+      HUMAN_APP_EMAIL: *human_app_email
       REPUTATION_ORACLE_URL: "http://reputation-oracle:${BACKEND_APPS_INTERNAL_PORT:?}"
       REPUTATION_ORACLE_ADDRESS: *reputation_oracle_address
 
@@ -337,16 +343,16 @@ services:
       HOST: *all_interfaces_ip
       PORT: *backend_apps_internal_port
       <<: [
-        *redis_connection_vars,
-        *postgres_cvat_oracles_vars,
+        *cvat_oracle_postgres_vars,
+        *redis_app_vars,
         *cvat_oracle_storage_connection_vars,
-        *cvat_vars
+        *cvat_connection_vars
       ]
       REDIS_DB: 2
       PG_DB: exchange-oracle
       POLYGON_AMOY_RPC_API_URL: *rpc_url_polygon_amoy
       POLYGON_AMOY_ADDR: *exchange_oracle_address
-      STORAGE_BUCKET_NAME: *minio_bucket_exc_o
+      STORAGE_BUCKET_NAME: *bucket_name_exc_o
       HUMAN_APP_JWT_KEY: *reputation_oracle_jwt_public_key
       CVAT_INCOMING_WEBHOOKS_URL: "http://exchange-oracle.app:${BACKEND_APPS_INTERNAL_PORT:?}/cvat-webhook"
       LOCALHOST_RECORDING_ORACLE_URL: "http://recording-oracle:${BACKEND_APPS_INTERNAL_PORT:?}/webhook"
@@ -384,20 +390,20 @@ services:
       HOST: *all_interfaces_ip
       PORT: *backend_apps_internal_port
       <<: [
-        *postgres_cvat_oracles_vars,
+        *cvat_oracle_postgres_vars,
         *cvat_oracle_storage_connection_vars,
-        *cvat_vars
+        *cvat_connection_vars
       ]
       PG_DB: recording-oracle
       POLYGON_AMOY_RPC_API_URL: *rpc_url_polygon_amoy
       POLYGON_AMOY_ADDR: *recording_oracle_address
-      STORAGE_RESULTS_BUCKET_NAME: *minio_bucket_rec_o
+      STORAGE_RESULTS_BUCKET_NAME: *bucket_name_rec_o
       STORAGE_USE_PATH_STYLE: "true"
       EXCHANGE_ORACLE_STORAGE_PROVIDER: *cvat_oracle_storage_provider
       EXCHANGE_ORACLE_STORAGE_ENDPOINT_URL: *cvat_oracle_storage_endpoint
       EXCHANGE_ORACLE_STORAGE_ACCESS_KEY: *minio_services_access_key
       EXCHANGE_ORACLE_STORAGE_SECRET_KEY: *minio_services_secret_key
-      EXCHANGE_ORACLE_STORAGE_RESULTS_BUCKET_NAME: *minio_bucket_exc_o
+      EXCHANGE_ORACLE_STORAGE_RESULTS_BUCKET_NAME: *bucket_name_exc_o
       EXCHANGE_ORACLE_STORAGE_USE_SSL: 'false'
       LOCALHOST_EXCHANGE_ORACLE_URL: "http://exchange-oracle:${BACKEND_APPS_INTERNAL_PORT:?}/webhook"
       LOCALHOST_REPUTATION_ORACLE_URL: "http://reputation-oracle:${BACKEND_APPS_INTERNAL_PORT:?}/webhook"
@@ -430,11 +436,11 @@ services:
         published: ${JOB_LAUNCHER_PORT:-5005}
     env_file: ./.env.job-launcher.local
     environment:
-      <<: [*shared_vars, *postgres_connection_vars, *s3_connection_vars]
+      <<: [*nodejs_app_vars, *nodejs_app_postgres_vars, *nodejs_app_s3_vars]
       HOST: *all_interfaces_ip
       PORT: *backend_apps_internal_port
       POSTGRES_DATABASE: job-launcher
-      S3_BUCKET: *minio_bucket_manifests
+      S3_BUCKET: *bucket_name_manifests
       REPUTATION_ORACLES: *exchange_oracle_address
       CVAT_EXCHANGE_ORACLE_ADDRESS: *exchange_oracle_address
       CVAT_RECORDING_ORACLE_ADDRESS: *recording_oracle_address

diff --git a/scripts/cvat/env-files/.env.compose b/scripts/cvat/env-files/.env.compose
@@ -19,4 +19,3 @@ RECORDING_ORACLE_ADDRESS=replace_me
 
 # Might be empty
 SUBGRAPH_API_KEY=
-SENDGRID_API_KEY=sendgrid-disabled
Original file line number	Diff line number	Diff line change
Expand Up		@@ -19,4 +19,3 @@ RECORDING_ORACLE_ADDRESS=replace_me

		# Might be empty
		SUBGRAPH_API_KEY=
		SENDGRID_API_KEY=sendgrid-disabled