Adaptive Postgres HA Support

.github/Makefile

@@ -2,7 +2,8 @@
 
 ROOT = $(dir $(realpath $(firstword $(MAKEFILE_LIST))))
 
-all: workflows/nightly.yml workflows/release.yml workflows/tests.yml workflows/tests-managed-pg.yml
+all: workflows/nightly.yml workflows/release.yml workflows/tests.yml \
+    workflows/tests-managed-pg.yml workflows/tests-ha.yml
 
 workflows/%.yml: workflows.src/%.tpl.yml workflows.src/%.targets.yml
 	$(ROOT)/workflows.src/render.py $* $*.targets.yml
@@ -12,3 +13,6 @@ workflows.src/tests.tpl.yml: workflows.src/tests.inc.yml
 
 workflows.src/tests-managed-pg.tpl.yml: workflows.src/tests.inc.yml
 	touch $(ROOT)/workflows.src/tests-managed-pg.tpl.yml
+
+workflows.src/tests-ha.tpl.yml: workflows.src/tests.inc.yml
+	touch $(ROOT)/workflows.src/tests-ha.tpl.yml

.github/workflows.src/tests-ha.targets.yml

@@ -0,0 +1 @@
+data:

.github/workflows.src/tests-ha.tpl.yml

@@ -0,0 +1,49 @@
+<% from "tests.inc.yml" import build, calc_cache_key, restore_cache -%>
+name: High Availability Tests
+
+on:
+  workflow_run:
+    workflows: ["Tests"]
+    types:
+      - completed
+
+jobs:
+  build:
+    runs-on: ubuntu-18.04
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+
+    steps:
+    <%- call build() -%>
+    - name: Compute cache keys and download the running times log
+      env:
+        GIST_TOKEN: ${{ secrets.CI_BOT_GIST_TOKEN }}
+      run: |
+        << calc_cache_key()|indent >>
+
+        curl \
+          -H "Accept: application/vnd.github.v3+json" \
+          -u edgedb-ci:$GIST_TOKEN \
+          https://api.github.com/gists/8b722a65397f7c4c0df72f5394efa04c \
+        | jq '.files."time_stats.csv".raw_url' \
+        | xargs curl > .tmp/time_stats.csv
+    <%- endcall %>
+
+  ha-test:
+    needs: build
+    runs-on: ubuntu-18.04
+
+    steps:
+    <<- restore_cache() >>
+
+    # Run the test
+
+    - name: Test
+      env:
+        SHARD: ${{ matrix.shard }}
+        EDGEDB_TEST_HA: 1
+        EDGEDB_TEST_CONSUL_PATH: build/stolon/bin/consul
+        EDGEDB_TEST_STOLON_CTL: build/stolon/bin/stolonctl
+        EDGEDB_TEST_STOLON_SENTINEL: build/stolon/bin/stolon-sentinel
+        EDGEDB_TEST_STOLON_KEEPER: build/stolon/bin/stolon-keeper
+      run: |
+        edb test -j1 -v test_stolon

.github/workflows.src/tests-managed-pg.tpl.yml

@@ -82,10 +82,10 @@ jobs:
 
     - name: Test
       env:
-        EDGEDB_TEST_POSTGRES_DSN: postgres://edbtest:${{ secrets.AWS_RDS_PASSWORD }}@${{ needs.setup-aws-rds.outputs.pghost }}/postgres
+        EDGEDB_TEST_BACKEND_DSN: postgres://edbtest:${{ secrets.AWS_RDS_PASSWORD }}@${{ needs.setup-aws-rds.outputs.pghost }}/postgres
       run: |
-        edb server --bootstrap-only --postgres-dsn=$EDGEDB_TEST_POSTGRES_DSN
-        edb test -j2 -v --postgres-dsn=$EDGEDB_TEST_POSTGRES_DSN
+        edb server --bootstrap-only --backend-dsn=$EDGEDB_TEST_BACKEND_DSN
+        edb test -j2 -v --backend-dsn=$EDGEDB_TEST_BACKEND_DSN
 
   teardown-aws-rds:
     runs-on: ubuntu-latest
@@ -194,10 +194,10 @@ jobs:
 
     - name: Test
       env:
-        EDGEDB_TEST_POSTGRES_DSN: postgres://${{ steps.pguser.outputs.stdout }}:${{ steps.pgpass.outputs.stdout }}@${{ steps.pghost.outputs.stdout }}:${{ steps.pgport.outputs.stdout }}/${{ steps.pgdatabase.outputs.stdout }}
+        EDGEDB_TEST_BACKEND_DSN: postgres://${{ steps.pguser.outputs.stdout }}:${{ steps.pgpass.outputs.stdout }}@${{ steps.pghost.outputs.stdout }}:${{ steps.pgport.outputs.stdout }}/${{ steps.pgdatabase.outputs.stdout }}
       run: |
-        edb server --bootstrap-only --postgres-dsn=$EDGEDB_TEST_POSTGRES_DSN
-        edb test -j2 -v --postgres-dsn=$EDGEDB_TEST_POSTGRES_DSN
+        edb server --bootstrap-only --backend-dsn=$EDGEDB_TEST_BACKEND_DSN
+        edb test -j2 -v --backend-dsn=$EDGEDB_TEST_BACKEND_DSN
 
   teardown-do-database:
     runs-on: ubuntu-latest
@@ -269,10 +269,10 @@ jobs:
 
     - name: Test
       env:
-        EDGEDB_TEST_POSTGRES_DSN: postgres://postgres:${{ secrets.AWS_RDS_PASSWORD }}@${{ needs.setup-gcp-cloud-sql.outputs.pghost }}/postgres
+        EDGEDB_TEST_BACKEND_DSN: postgres://postgres:${{ secrets.AWS_RDS_PASSWORD }}@${{ needs.setup-gcp-cloud-sql.outputs.pghost }}/postgres
       run: |
-        edb server --bootstrap-only --postgres-dsn=$EDGEDB_TEST_POSTGRES_DSN
-        edb test -j2 -v --postgres-dsn=$EDGEDB_TEST_POSTGRES_DSN
+        edb server --bootstrap-only --backend-dsn=$EDGEDB_TEST_BACKEND_DSN
+        edb test -j2 -v --backend-dsn=$EDGEDB_TEST_BACKEND_DSN
 
   teardown-gcp-cloud-sql:
     runs-on: ubuntu-latest

.github/workflows.src/tests.inc.yml

@@ -70,9 +70,9 @@
       id: ext-cache
       with:
         path: build/extensions
-        key: edb-ext-v1-${{ hashFiles('.tmp/ext_cache_key.txt') }}
+        key: edb-ext-v2-${{ hashFiles('.tmp/ext_cache_key.txt') }}
         restore-keys: |
-          edb-ext-v1-
+          edb-ext-v2-
 
     - name: Handle cached PostgreSQL build
       uses: actions/cache@v2
@@ -81,13 +81,21 @@
         path: build/postgres/install
         key: edb-postgres-v1-${{ env.POSTGRES_GIT_REV }}
 
+    - name: Handle cached Stolon build
+      uses: actions/cache@v2
+      id: stolon-cache
+      with:
+        path: build/stolon/bin
+        key: edb-stolon-v2-${{ env.STOLON_GIT_REV }}
+
     # Install system dependencies for building
 
     - name: Install system deps
       if: |
         steps.cli-cache.outputs.cache-hit != 'true' ||
         steps.rust-cache.outputs.cache-hit != 'true' ||
         steps.ext-cache.outputs.cache-hit != 'true' ||
+        steps.stolon-cache.outputs.cache-hit != 'true' ||
         steps.postgres-cache.outputs.cache-hit != 'true'
       run: |
         sudo apt-get update
@@ -157,9 +165,9 @@
       if: steps.ext-cache.outputs.cache-hit != 'true'
       with:
         path: ${{ env.BUILD_TEMP }}/edb
-        key: edb-ext-build-v1-${{ hashFiles('.tmp/ext_cache_key.txt') }}
+        key: edb-ext-build-v2-${{ hashFiles('.tmp/ext_cache_key.txt') }}
         restore-keys: |
-          edb-ext-build-v1-
+          edb-ext-build-v2-
 
     - name: Build Cython extensions
       env:
@@ -208,6 +216,31 @@
           cp build/postgres/stamp build/postgres/install/
         fi
 
+    # Build Stolon
+
+    - name: Set up Go
+      if: steps.stolon-cache.outputs.cache-hit != 'true'
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.16
+
+    - uses: actions/checkout@v2
+      if: steps.stolon-cache.outputs.cache-hit != 'true'
+      with:
+        repository: sorintlab/stolon
+        path: build/stolon
+        ref: ${{ env.STOLON_GIT_REV }}
+        fetch-depth: 0
+        submodules: false
+
+    - name: Build Stolon
+      if: steps.stolon-cache.outputs.cache-hit != 'true'
+      run: |
+        mkdir -p build/stolon/bin/
+        curl -fsSL https://releases.hashicorp.com/consul/1.10.1/consul_1.10.1_linux_amd64.zip | zcat > build/stolon/bin/consul
+        chmod +x build/stolon/bin/consul
+        cd build/stolon && make
+
     # Install edgedb-server and populate egg-info
 
     - name: Install edgedb-server and populate egg-info
@@ -248,9 +281,11 @@
     python setup.py -q ci_helper --type ext >.tmp/ext_cache_key.txt
     python setup.py -q ci_helper --type parsers >.tmp/parsers_cache_key.txt
     python setup.py -q ci_helper --type postgres >.tmp/postgres_git_rev.txt
+    echo 'v0.16.0' >.tmp/stolon_git_rev.txt
     python setup.py -q ci_helper --type bootstrap >.tmp/bootstrap_cache_key.txt
     echo EDGEDBCLI_GIT_REV=$(cat .tmp/edgedbcli_git_rev.txt) >> $GITHUB_ENV
     echo POSTGRES_GIT_REV=$(cat .tmp/postgres_git_rev.txt) >> $GITHUB_ENV
+    echo STOLON_GIT_REV=$(cat .tmp/stolon_git_rev.txt) >> $GITHUB_ENV
     echo BUILD_LIB=$(python setup.py -q ci_helper --type build_lib) >> $GITHUB_ENV
     echo BUILD_TEMP=$(python setup.py -q ci_helper --type build_temp) >> $GITHUB_ENV
 <%- endmacro %>
@@ -272,6 +307,7 @@
       run: |
         echo EDGEDBCLI_GIT_REV=$(cat .tmp/edgedbcli_git_rev.txt) >> $GITHUB_ENV
         echo POSTGRES_GIT_REV=$(cat .tmp/postgres_git_rev.txt) >> $GITHUB_ENV
+        echo STOLON_GIT_REV=$(cat .tmp/stolon_git_rev.txt) >> $GITHUB_ENV
         echo BUILD_LIB=$(python setup.py -q ci_helper --type build_lib) >> $GITHUB_ENV
         echo BUILD_TEMP=$(python setup.py -q ci_helper --type build_temp) >> $GITHUB_ENV
 
@@ -296,7 +332,7 @@
       id: ext-cache
       with:
         path: build/extensions
-        key: edb-ext-v1-${{ hashFiles('.tmp/ext_cache_key.txt') }}
+        key: edb-ext-v2-${{ hashFiles('.tmp/ext_cache_key.txt') }}
 
     - name: Restore compiled parsers cache
       uses: actions/cache@v2
@@ -312,6 +348,13 @@
         path: build/postgres/install
         key: edb-postgres-v1-${{ env.POSTGRES_GIT_REV }}
 
+    - name: Restore cached Stolon build
+      uses: actions/cache@v2
+      id: stolon-cache
+      with:
+        path: build/stolon/bin
+        key: edb-stolon-v2-${{ env.STOLON_GIT_REV }}
+
     - name: Restore bootstrap cache
       uses: actions/cache@v2
       id: bootstrap-cache
@@ -327,6 +370,7 @@
         steps.ext-cache.outputs.cache-hit != 'true' ||
         steps.parsers-cache.outputs.cache-hit != 'true' ||
         steps.postgres-cache.outputs.cache-hit != 'true' ||
+        steps.stolon-cache.outputs.cache-hit != 'true' ||
         steps.bootstrap-cache.outputs.cache-hit != 'true'
       run: |
         echo ::error::Cannot retrieve build cache.