name: "PG 13: Migration Tests"
on:
push:
branches: ['main', '*.*-dev', '*.*.*-dev']
pull_request:
branches: ['main']
jobs:
run-pg-13-migration-tests:
strategy:
matrix:
version: [2024.2.1.0-b185, 2.20.8.0-b53, 2024.1.3.1-b8, 2.23.1.0-b220]
BETA_FAST_DATA_EXPORT: [0, 1]
test_group:
- offline
- live_basic
- live_advanced
env:
BETA_FAST_DATA_EXPORT: ${{ matrix.BETA_FAST_DATA_EXPORT }}
runs-on: ubuntu-22.04
services:
postgres:
image: postgres:13
env:
POSTGRES_PASSWORD: postgres
# Set health checks to wait until postgres has started
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
# Maps tcp port 5432 on service container to the host
- 5432:5432
steps:
- uses: actions/checkout@v3
- uses: actions/setup-java@v3
# https://github.com/actions/setup-java
with:
distribution: "temurin"
java-version: "17"
check-latest: true
- name: Cache local Maven repository
uses: actions/cache@v3
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- name: "Enable postgres with wal_level as logical and install postgis"
run: |
docker exec ${{ job.services.postgres.id }} sh -c "echo 'wal_level=logical' >> /var/lib/postgresql/data/postgresql.conf"
docker exec ${{ job.services.postgres.id }} sh -c "apt-get update && apt-get install -y postgresql-13-postgis postgresql-13-postgis-3"
docker restart ${{ job.services.postgres.id }}
sleep 10
- name: Install python3 and psycopg2
run: |
sudo apt install -y python3
sudo apt install -y libpq-dev
sudo apt install python3-psycopg2
- name: Run installer script to setup voyager
run: |
cd installer_scripts
yes | ./install-yb-voyager --install-from-local-source --only-pg-support
sudo rm /usr/bin/pg_dump
sudo ln -s /usr/lib/postgresql/17/bin/pg_dump /usr/bin/pg_dump
sudo rm /usr/bin/pg_restore
sudo ln -s /usr/lib/postgresql/17/bin/pg_restore /usr/bin/pg_restore
pg_dump --version
pg_restore --version
psql --version
env:
ON_INSTALLER_ERROR_OUTPUT_LOG: Y
- name: Test PostgreSQL Connection
run: |
psql "postgresql://postgres:postgres@127.0.0.1:5432/postgres" -c "SELECT version();"
- name: Create PostgreSQL user
run: |
./migtests/scripts/postgresql/create_pg_user
- name: Start YugabyteDB cluster
run: |
docker run -d --name yugabytedb \
-p7000:7000 -p9000:9000 -p15433:15433 -p5433:5433 -p9042:9042 \
yugabytedb/yugabyte:${{ matrix.version }} \
bin/yugabyted start --tserver_flags="ysql_hba_conf_csv={host all yugabyte all trust,host all all all md5}" --background=false --ui=false
sleep 20
- name: Test YugabyteDB connection
run: |
psql "postgresql://yugabyte:@127.0.0.1:5433/yugabyte" -c "SELECT version();"
- name: Create YugabyteDB user
run: |
./migtests/scripts/yugabytedb/create_yb_user
- name: Enable yb-tserver-n1 and yb-master-n1 name resolution
run: |
echo "127.0.0.1 yb-tserver-n1" | sudo tee -a /etc/hosts
echo "127.0.0.1 yb-master-n1" | sudo tee -a /etc/hosts
psql "postgresql://yugabyte@yb-tserver-n1:5433/yugabyte" -c "SELECT version();"
- name: Log yb_servers()
run: |
psql "postgresql://yugabyte@yb-tserver-n1:5433/yugabyte" -c "SELECT * from yb_servers();"
- name: "TEST: pg-table-list-flags-test (table-list and exclude-table-list)"
if: ${{ !cancelled() && matrix.test_group == 'offline' }}
run: migtests/scripts/run-test.sh pg/table-list-flags-tests
- name: "TEST: pg-table-list-file-path-test (table-list-file-path and exclude-table-list-file-path)"
if: ${{ !cancelled() && matrix.test_group == 'offline' }}
run: migtests/scripts/run-test.sh pg/table-list-flags-tests env-file-path-flags.sh
- name: "TEST: pg-case-sensitivity-single-table"
if: ${{ !cancelled() && matrix.test_group == 'offline' }}
run: migtests/scripts/run-test-export-data.sh pg/case-sensitivity-single-table
- name: "TEST: pg-datatypes"
if: ${{ !cancelled() && matrix.test_group == 'offline' }}
run: migtests/scripts/run-test.sh pg/datatypes
- name: "TEST: pg-constraints"
if: ${{ !cancelled() && matrix.test_group == 'offline' }}
run: migtests/scripts/run-test.sh pg/constraints
- name: "TEST: pg-sequences"
if: ${{ !cancelled() && matrix.test_group == 'offline' }}
run: migtests/scripts/run-test.sh pg/sequences
- name: "TEST: pg-indexes"
if: ${{ !cancelled() && matrix.test_group == 'offline' }}
run: migtests/scripts/run-test.sh pg/indexes
- name: "TEST: pg-partitions"
if: ${{ !cancelled() && matrix.test_group == 'offline' }}
run: migtests/scripts/run-test.sh pg/partitions
- name: "TEST: pg-partitions with (table-list)"
if: ${{ !cancelled() && matrix.test_group == 'offline' }}
run: EXPORT_TABLE_LIST='customers,sales,emp,p2.boston,p2.london,p2.sydney,range_columns_partition_test,sales_region,test_partitions_sequences' migtests/scripts/run-test.sh pg/partitions
# Broken for v2.15 and v2.16: https://github.com/yugabyte/yugabyte-db/issues/14529
# Fixed in 2.17.1.0-b368
- name: "TEST: pg-partitions-with-indexes"
if: ${{ !cancelled() && matrix.test_group == 'offline' }}
run: migtests/scripts/run-test.sh pg/partitions-with-indexes
- name: "TEST: pg-views-and-rules"
if: ${{ !cancelled() && matrix.test_group == 'offline' }}
run: migtests/scripts/run-test.sh pg/views-and-rules
- name: "TEST: pg-misc-objects-1 (Types, case-sensitive-table-name, Domain)"
if: ${{ !cancelled() && matrix.test_group == 'offline' }}
run: migtests/scripts/run-test.sh pg/misc-objects-1
- name: "TEST: pg-misc-objects-2 (Aggregates, Procedures, triggers, functions, extensions, inline comments)"
if: ${{ !cancelled() && matrix.test_group == 'offline' }}
run: migtests/scripts/run-test.sh pg/misc-objects-2
- name: "TEST: pg-dependent-ddls"
if: ${{ !cancelled() && matrix.test_group == 'offline' }}
run: migtests/scripts/run-test.sh pg/dependent-ddls
- name: "TEST: pg-multiple-schemas"
if: ${{ !cancelled() && matrix.test_group == 'offline' }}
run: migtests/scripts/run-test.sh pg/multiple-schemas
- name: "TEST: pg-codependent-schemas"
if: ${{ !cancelled() && matrix.test_group == 'offline' }}
run: migtests/scripts/run-test.sh pg/codependent-schemas
- name: "TEST: pg-sample-schema-emp"
if: ${{ !cancelled() && matrix.test_group == 'offline' }}
run: migtests/scripts/run-test.sh pg/sample-employee
- name: "TEST: pg-hasura-ecommerce"
if: ${{ !cancelled() && matrix.test_group == 'offline' }}
run: migtests/scripts/run-test.sh pg/hasura-ecommerce
- name: "TEST: pg-case-sensitivity-reserved-words-offline"
if: ${{ !cancelled() && matrix.test_group == 'offline' }}
run: migtests/scripts/run-test.sh pg/case-sensitivity-reserved-words
- name: "TEST: pg-basic-non-public-live-migration-test"
if: ${{ !cancelled() && matrix.test_group == 'live_basic' }}
run: migtests/scripts/live-migration-run-test.sh pg/basic-non-public-live-test
- name: "TEST: pg-basic-public-fall-forward-test"
if: ${{ !cancelled() && matrix.test_group == 'live_basic' }}
run: migtests/scripts/live-migration-fallf-run-test.sh pg/basic-public-live-test
# - name: "TEST: pg-basic-non-public-fall-back-test"
# run: migtests/scripts/live-migration-fallb-run-test.sh pg/basic-non-public-live-test
- name: "TEST: pg-datatypes-fall-back-test"
if: ${{ !cancelled() && matrix.test_group == 'live_basic' }}
run: migtests/scripts/live-migration-fallb-run-test.sh pg/datatypes
# case sensitive table names are not yet supported in live migration, to restricting test only to a few tables.
- name: "TEST: pg-live-migration-multiple-schemas"
if: ${{ !cancelled() && matrix.test_group == 'live_advanced' }}
run: EXPORT_TABLE_LIST="ext_test,tt,audit,recipients,session_log,schema2.ext_test,schema2.tt,schema2.audit,schema2.recipients,schema2.session_log" migtests/scripts/live-migration-run-test.sh pg/multiple-schemas
- name: "TEST: pg-unique-key-conflicts-test"
if: ${{ !cancelled() && matrix.test_group == 'live_advanced' }}
run: migtests/scripts/live-migration-fallf-run-test.sh pg/unique-key-conflicts-test
- name: "TEST: pg-live-migration-partitions-fall-forward"
if: ${{ !cancelled() && matrix.test_group == 'live_advanced' }}
run: migtests/scripts/live-migration-fallf-run-test.sh pg/partitions