Merge pull request #38 from arenaxr/demo_setup

Restructure stack config

.github/workflows/demo-branch.yml

@@ -0,0 +1,19 @@
+name: Update demo branch
+on:
+  workflow_dispatch:
+  push:
+
+jobs:
+  copy-to-branches:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Copy To Branches Action
+        uses: planetoftheweb/[email protected]
+        env:
+          key: master
+          branches: demo master
+          files: docker-compose.yaml docker-compose.demo.yaml conf-templates/*
+          exclude: conf-templates/localdev conf-templates/prod conf-templates/staging

VERSION

@@ -1,10 +1,14 @@
 # this file is automatically updated by update-versions.sh  
-ARENA_SERVICES=v1.0.3
-ARENA_CORE=v1.26.3
-ARENA_PERSIST=v1.0.4
-ARENA_ACCOUNT=v1.2.0
-ARENA_BROKER=v1.6.10
-ARENA_FILESTORE=v2.20.1
-ORCHESTRATOR=v1.1.0
-ARENA_RUNTIME_NATIVE=v1.1.0
-ARENA_INIT_UTILS=v1.0.1
+ARENA_SERVICES_VERSION=v1.0.3
+ARENA_WEB_CORE_VERSION=v1.26.3
+ARENA_PERSIST_VERSION=v1.0.2a
+ARENA_ACCOUNT_VERSION=v1.2.0
+ARENA_BROKER_VERSION=v1.6.10
+ARENA_FILESTORE_VERSION=v2.28.0
+ARENA_INIT_UTILS_VERSION=v1.0.4
+CERTBOT_VERSION=latest
+MONGO_VERSION=latest
+CADVISOR_VERSION=latest
+PROM_EXPORTER_VERSION=latest
+PROMETHEUS_VERSION=latest
+GRAPHANA_VERSION=latest

VERSION.preprod

@@ -0,0 +1,13 @@
+ARENA_SERVICES_VERSION=latest
+ARENA_WEB_CORE_VERSION=latest
+ARENA_PERSIST_VERSION=latest
+ARENA_ACCOUNT_VERSION=latest
+ARENA_BROKER_VERSION=latest
+ARENA_FILESTORE_VERSION=latest
+ARENA_INIT_UTILS_VERSION=latest
+CERTBOT_VERSION=latest
+MONGO_VERSION=latest
+CADVISOR_VERSION=latest
+PROM_EXPORTER_VERSION=latest
+PROMETHEUS_VERSION=latest
+GRAPHANA_VERSION=latest

backup.sh

@@ -1,15 +1,24 @@
 #!/bin/bash
 #!/bin/bash
-DIR="$(dirname "${BASH_SOURCE[0]}")"
-ARENA_DOCKER_REPO_FOLDER="$(realpath "${DIR}")"
+
+ARENA_DOCKER_REPO_FOLDER=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 source $ARENA_DOCKER_REPO_FOLDER/.env
+
+if [ -z ${BACKUP_USER+x} ]; then 
+  logger -s "BACKUP_USER not set; please set BACKUP_USER in .env" 
+  exit 1
+fi
+
 BACKUP_PATH=$(realpath -s $ARENA_DOCKER_REPO_FOLDER)/data/backup/$HOSTNAME 
 DATA_PATH=$(realpath -s $ARENA_DOCKER_REPO_FOLDER)/data 
 [ ! -d "$BACKUP_PATH" ] && mkdir $BACKUP_PATH
+
 BFOLDER=$BACKUP_PATH/mongodb
 [ ! -d "$BFOLDER" ] && mkdir $BFOLDER
+
 # backup mongodb using mongodump
 sudo /usr/bin/docker exec -it -e HOSTNAME=$HOSTNAME arena-services-docker_mongodb_1 sh -c 'exec mongodump --db arena_persist --collection arenaobjects --out /backup/$HOSTNAME/mongodb'
+
 # backup other services by copying files
 DATA_FOLDERS=( "arena-store" "grafana" "account" )
 for d in "${DATA_FOLDERS[@]}"
@@ -19,4 +28,5 @@ do
   echo "cp -R $DATA_PATH/$d/ $BFOLDER/"
   cp -R $DATA_PATH/$d/ $BFOLDER
 done
+
 chown -R $BACKUP_USER $DATA_PATH/backup/*

build-arena-core.sh

@@ -1,17 +1,28 @@
 #!/bin/bash
+# build arena-web-core javascript
+# ALWAYS_YES="true" indicates that we answer yes to all questions: create new secrets, tokens, regenerate config files
 
-echo -e "\n\e[1m### Building ARENA core js\e[0m\n"
+# load utils
+source init-utils/bash-common-utils.sh 
 
-echo -e "Skipping. For now, you will have to setup a node environment on the host to build ARENA core js.\n"
+echocolor ${HIGHLIGHT} "### Building ARENA core js."
+
+printf "Skipping. For now, you will have to setup a node environment on the host to build ARENA core js.\n"
 exit 0
 
-read -p "Build js (production instances - started with ./prod.sh - can skip this step) ? (y/N) " -r
 
-# load ARENA_DOCKER_REPO_FOLDER var
+# try to load ARENA_DOCKER_REPO_FOLDER var
 export $(grep "^ARENA_DOCKER_REPO_FOLDER" .env | xargs)
 
+if [ -z ${ARENA_DOCKER_REPO_FOLDER+x} ]; then 
+  ARENA_DOCKER_REPO_FOLDER=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )    
+  printf "ARENA_DOCKER_REPO_FOLDER not set. Using inferred folder from script path: $ARENA_DOCKER_REPO_FOLDER\n".
+  printf "Add ARENA_DOCKER_REPO_FOLDER to .env if this is incorrect\n".
+fi
+
+readprompt "Build js (production instances - started with ./prod.sh - can skip this step) ? (y/N) "
 if [[ $REPLY =~ ^[Yy]$ ]]; then
     [ -x "$(command -v git)" ] && cd ${ARENA_DOCKER_REPO_FOLDER}/arena-web-core && git checkout master && git pull
     mkdir -p ${ARENA_DOCKER_REPO_FOLDER}/arena-web-core/dist
-    docker run -it --rm -v ${ARENA_DOCKER_REPO_FOLDER}:/arena -w /arena/arena-web-core conixcenter/arena-services-docker-init-utils sh -c "npm install --also=dev && npm run build"
+    docker run -it --rm -v ${ARENA_DOCKER_REPO_FOLDER}:/arena -w /arena/arena-web-core arenaxrorg/arena-services-docker-init-utils:$ARENA_INIT_UTILS_VERSION sh -c "npm install --also=dev && npm run build"
 fi

cleanup.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+# cleanup previous config created with init.sh; run with sudo
+
+[ -d conf ] && ( rm -fr conf_bak; mv conf conf_bak && echo "Backup conf in conf_bak" ) || echo "No conf folder found"
+[ -d data ] && ( rm -fr data_bak; mv data data_bak && echo "Backup data in data_bak") || echo "No conf folder found"
+[ -f .env ] && ( rm -fr .env_bak; mv .env .env_bak && echo "Backup .env in .env_bak") || echo "No .env found"
+
+if [ -d "./data/certbot/conf/live" ]; then
+  echo "Existing certificate/letsencrypt data found (deleting and retrying to create certificates might bump into letsencrypt retry limits)."
+  read -p "Continue and remove certificate/letsencrypt files ? (y/N) " decision
+  if [ "$decision" != "Y" ] && [ "$decision" != "y" ]; then
+    exit
+  fi
+  rm -fr ./data/certbot/conf/*
+else
+  echo "No certificate/letsencrypt data found."
+fi

conf-templates/README.md

@@ -0,0 +1,43 @@
+# Config Templates
+
+This folder has templates for the config files used (nginx, mosquitto, account, persist, web client, ...).
+These templates have variables replaced from values in env variables using `envsubst`.
+
+When init.sh is executed, this folder's structure will be copied into conf/ and files will be created from templates using `envsubst`.
+
+## Structure
+
+The root of this folder has the config files that are common to all compose configurations (prod, devlocal, staging, ...). A folder for each compose configuration has the files that are specific to it. 
+
+When init.sh is executed, configuration files in conf/arena-web-conf will be copied to each of the compose configuration folders (e.g. conf/prod/arena-web-conf/, conf/devlocal/arena-web-conf/, ...)
+
+```
+.
+├── arena-web-conf 						# arena web client config, common to all compose configurations
+│             ├── ...
+│             └── gauth.json
+├── demo								# demo compose configuration config files
+│             ├── arena-web-conf 		# arena web client config for demo; init.sh copies files from ../../arena-web-conf
+│             │             ├── defaults.js
+│             │             ├── defaults.json
+│             │             └── ...
+│             └── arena-web.conf
+│
+├── localdev							# devlocal compose configuration config files
+│             ├── arena-web-conf        # arena web client config for localdev; init.sh copies files from ../../arena-web-conf
+│             │             ├── defaults.js
+│             │             └── defaults.json
+│             │             └── ...
+│             └── arena-web.conf
+├── prod								# prod compose configuration config files
+│             ├── arena-web-conf		# arena web client config for prod; init.sh copies files from ../../arena-web-conf
+│             │             ├── defaults.js
+│             │             └── defaults.json
+│             │             └── ...
+│             └── ...
+├── ...
+│
+│   # these are config files common to all setups
+├── mosquitto.conf
+└── ...
+```

conf-templates/arena-account-settings.py.tmpl → conf-templates/account-settings.py.tmpl

@@ -272,4 +272,4 @@ PUBSUB = {
     },
     "mqtt_realm": "${ARENA_REALM}",
     "mqtt_username": "arena_account",
-}
+}

conf-templates/arena-web-conf/gauth.json.tmpl

@@ -11,4 +11,4 @@
       "http://localhost"
     ]
   }
-}
+}