Upgrade to PG17 in Github actions (#2114)

Fixed Validation files for the VIEW/MVIEW's select statements Skipping the SET transaction_timeout = 0; in the import schema to be not executed on YB. Fixes #2121
yugabyte · Dec 27, 2024 · 47a8e50 · 47a8e50
1 parent d148a04
commit 47a8e50
Show file tree

Hide file tree

Showing 15 changed files with 320 additions and 70 deletions.
diff --git a/.github/workflows/misc-migtests.yml b/.github/workflows/misc-migtests.yml
@@ -12,9 +12,9 @@ jobs:
 
     services:
         postgres:
-          image: postgres:15
+          image: postgres:17
           env:
-            POSTGRES_PASSWORD: secret
+            POSTGRES_PASSWORD: postgres
           # Set health checks to wait until postgres has started
           options: >-
             --health-cmd pg_isready
@@ -55,17 +55,19 @@ jobs:
           sudo apt install -y libpq-dev
           sudo apt install python3-psycopg2
 
+      #TODO Remove the install PG 17 command once we do that in installer script 
       - name: Run installer script to setup voyager
         run: |
           cd installer_scripts
           yes | ./install-yb-voyager --install-from-local-source --only-pg-support
-          echo "/usr/lib/postgresql/16/bin" >> "$GITHUB_PATH"
+          sudo apt-get -y install postgresql-17
+          echo "/usr/lib/postgresql/17/bin" >> "$GITHUB_PATH"
         env:
           ON_INSTALLER_ERROR_OUTPUT_LOG: Y
 
       - name: Test PostgreSQL Connection
         run: |
-          psql "postgresql://postgres:secret@127.0.0.1:5432/postgres" -c "SELECT version();"
+          psql "postgresql://postgres:postgres@127.0.0.1:5432/postgres" -c "SELECT version();"
 
       - name: Create PostgreSQL user
         run: |

diff --git a/.github/workflows/pg-13-migtests.yml b/.github/workflows/pg-13-migtests.yml
@@ -0,0 +1,219 @@
+name: "PG 13: Migration Tests"
+
+on:
+  push:
+    branches: ['main', '*.*-dev', '*.*.*-dev']
+  pull_request:
+    branches: ['main']
+
+jobs:
+  run-pg-13-migration-tests:
+    strategy:
+      matrix:
+        version: [2024.2.0.0-b145, 2.20.8.0-b53, 2024.1.3.1-b8, 2.23.1.0-b220]
+        BETA_FAST_DATA_EXPORT: [0, 1]
+        test_group:
+          - offline
+          - live_basic
+          - live_advanced
+
+    env:
+      BETA_FAST_DATA_EXPORT: ${{ matrix.BETA_FAST_DATA_EXPORT }}
+    runs-on: ubuntu-22.04
+    services:
+      postgres:
+        image: postgres:13
+        env:
+          POSTGRES_PASSWORD: postgres
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          # Maps tcp port 5432 on service container to the host
+          - 5432:5432
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-java@v3
+        # https://github.com/actions/setup-java
+        with:
+          distribution: "temurin"
+          java-version: "17"
+          check-latest: true
+
+      - name: Cache local Maven repository
+        uses: actions/cache@v3
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-
+
+      - name: "Enable postgres with wal_level as logical and install postgis"
+        run: |
+          docker exec ${{ job.services.postgres.id }} sh -c "echo 'wal_level=logical' >> /var/lib/postgresql/data/postgresql.conf"
+          docker exec ${{ job.services.postgres.id }} sh -c "apt-get update && apt-get install -y postgresql-13-postgis postgresql-13-postgis-3"
+          docker restart ${{ job.services.postgres.id }}
+          sleep 10
+
+      - name: Install python3 and psycopg2
+        run: |
+          sudo apt install -y python3
+          sudo apt install -y libpq-dev
+          sudo apt install python3-psycopg2
+
+      - name: Run installer script to setup voyager
+        run: |
+          cd installer_scripts
+          yes | ./install-yb-voyager --install-from-local-source --only-pg-support
+          sudo rm /usr/bin/pg_dump
+          sudo ln -s /usr/lib/postgresql/16/bin/pg_dump /usr/bin/pg_dump
+          sudo rm /usr/bin/pg_restore
+          sudo ln -s /usr/lib/postgresql/16/bin/pg_restore /usr/bin/pg_restore
+          pg_dump --version
+        env:
+          ON_INSTALLER_ERROR_OUTPUT_LOG: Y
+
+      - name: Test PostgreSQL Connection
+        run: |
+          psql "postgresql://postgres:[email protected]:5432/postgres" -c "SELECT version();"
+
+      - name: Create PostgreSQL user
+        run: |
+          ./migtests/scripts/postgresql/create_pg_user
+
+      - name: Start YugabyteDB cluster
+        run: |
+          docker run -d --name yugabytedb \
+            -p7000:7000 -p9000:9000 -p15433:15433 -p5433:5433 -p9042:9042 \
+            yugabytedb/yugabyte:${{ matrix.version }} \
+            bin/yugabyted start --background=false --ui=false
+          sleep 20
+
+      - name: Test YugabyteDB connection
+        run: |
+          psql "postgresql://yugabyte:@127.0.0.1:5433/yugabyte" -c "SELECT version();"
+
+      - name: Create YugabyteDB user
+        run: |
+          ./migtests/scripts/yugabytedb/create_yb_user
+
+      - name: Enable yb-tserver-n1 and yb-master-n1 name resolution
+        run: |
+          echo "127.0.0.1	yb-tserver-n1" | sudo tee -a /etc/hosts
+          echo "127.0.0.1 yb-master-n1" | sudo tee -a /etc/hosts
+          psql "postgresql://yugabyte@yb-tserver-n1:5433/yugabyte" -c "SELECT version();"
+
+      - name: "TEST: pg-table-list-flags-test (table-list and exclude-table-list)"
+        if: ${{ !cancelled() && matrix.test_group == 'offline' }}
+        run: migtests/scripts/run-test.sh pg/table-list-flags-tests
+
+      - name: "TEST: pg-table-list-file-path-test (table-list-file-path and exclude-table-list-file-path)"
+        if: ${{ !cancelled() && matrix.test_group == 'offline' }}
+        run: migtests/scripts/run-test.sh pg/table-list-flags-tests env-file-path-flags.sh
+
+      - name: "TEST: pg-case-sensitivity-single-table"
+        if: ${{ !cancelled() && matrix.test_group == 'offline' }}
+        run: migtests/scripts/run-test-export-data.sh pg/case-sensitivity-single-table
+
+      - name: "TEST: pg-dvdrental"
+        if: ${{ !cancelled() && matrix.test_group == 'offline' }}
+        run: migtests/scripts/run-test.sh pg/dvdrental
+
+      - name: "TEST: pg-datatypes"
+        if: ${{ !cancelled() && matrix.test_group == 'offline' }}
+        run: migtests/scripts/run-test.sh pg/datatypes
+
+      - name: "TEST: pg-constraints"
+        if: ${{ !cancelled() && matrix.test_group == 'offline' }}
+        run: migtests/scripts/run-test.sh pg/constraints
+
+      - name: "TEST: pg-sequences"
+        if: ${{ !cancelled() && matrix.test_group == 'offline' }}
+        run: migtests/scripts/run-test.sh pg/sequences
+
+      - name: "TEST: pg-indexes"
+        if: ${{ !cancelled() && matrix.test_group == 'offline' }}
+        run: migtests/scripts/run-test.sh pg/indexes
+
+      - name: "TEST: pg-partitions"
+        if: ${{ !cancelled() && matrix.test_group == 'offline' }}
+        run: migtests/scripts/run-test.sh pg/partitions 
+
+      - name: "TEST: pg-partitions with (table-list)"
+        if: ${{ !cancelled() && matrix.test_group == 'offline' }}
+        run: EXPORT_TABLE_LIST='customers,sales,emp,p2.boston,p2.london,p2.sydney,range_columns_partition_test,sales_region,test_partitions_sequences' migtests/scripts/run-test.sh pg/partitions 
+
+      # Broken for v2.15 and v2.16: https://github.com/yugabyte/yugabyte-db/issues/14529
+      # Fixed in 2.17.1.0-b368
+      - name: "TEST: pg-partitions-with-indexes"
+        if: ${{ !cancelled() && matrix.test_group == 'offline' }}
+        run: migtests/scripts/run-test.sh pg/partitions-with-indexes
+
+      - name: "TEST: pg-views-and-rules"
+        if: ${{ !cancelled() && matrix.test_group == 'offline' }}
+        run: migtests/scripts/run-test.sh pg/views-and-rules
+
+      - name: "TEST: pg-misc-objects-1 (Types, case-sensitive-table-name, Domain)"
+        if: ${{ !cancelled() && matrix.test_group == 'offline' }}
+        run: migtests/scripts/run-test.sh pg/misc-objects-1
+
+      - name: "TEST: pg-misc-objects-2 (Aggregates, Procedures, triggers, functions, extensions, inline comments)"
+        if: ${{ !cancelled() && matrix.test_group == 'offline' }}
+        run: migtests/scripts/run-test.sh pg/misc-objects-2
+
+      - name: "TEST: pg-dependent-ddls"
+        if: ${{ !cancelled() && matrix.test_group == 'offline' }}
+        run: migtests/scripts/run-test.sh pg/dependent-ddls
+
+      - name: "TEST: pg-multiple-schemas"
+        if: ${{ !cancelled() && matrix.test_group == 'offline' }}
+        run: migtests/scripts/run-test.sh pg/multiple-schemas
+
+      - name: "TEST: pg-codependent-schemas"
+        if: ${{ !cancelled() && matrix.test_group == 'offline' }}
+        run: migtests/scripts/run-test.sh pg/codependent-schemas
+
+      - name: "TEST: pg-sample-schema-emp"
+        if: ${{ !cancelled() && matrix.test_group == 'offline' }}
+        run: migtests/scripts/run-test.sh pg/sample-employee
+
+      - name: "TEST: pg-hasura-ecommerce"
+        if: ${{ !cancelled() && matrix.test_group == 'offline' }}
+        run: migtests/scripts/run-test.sh pg/hasura-ecommerce
+
+      - name: "TEST: pg-case-sensitivity-reserved-words-offline"
+        if: ${{ !cancelled() && matrix.test_group == 'offline' }}
+        run: migtests/scripts/run-test.sh pg/case-sensitivity-reserved-words
+
+      - name: "TEST: pg-basic-non-public-live-migration-test"
+        if: ${{ !cancelled() && matrix.test_group == 'live_basic' }}
+        run: migtests/scripts/live-migration-run-test.sh pg/basic-non-public-live-test
+
+      - name: "TEST: pg-basic-public-fall-forward-test"
+        if: ${{ !cancelled() && matrix.test_group == 'live_basic' }}
+        run: migtests/scripts/live-migration-fallf-run-test.sh pg/basic-public-live-test
+
+      # - name: "TEST: pg-basic-non-public-fall-back-test"
+      #   run: migtests/scripts/live-migration-fallb-run-test.sh pg/basic-non-public-live-test
+
+      - name: "TEST: pg-datatypes-fall-back-test"
+        if: ${{ !cancelled() && matrix.test_group == 'live_basic' }}
+        run: migtests/scripts/live-migration-fallb-run-test.sh pg/datatypes
+
+      # case sensitive table names are not yet supported in live migration, to restricting test only to a few tables.
+      - name: "TEST: pg-live-migration-multiple-schemas"
+        if: ${{ !cancelled() && matrix.test_group == 'live_advanced' }}
+        run: EXPORT_TABLE_LIST="ext_test,tt,audit,recipients,session_log,schema2.ext_test,schema2.tt,schema2.audit,schema2.recipients,schema2.session_log" migtests/scripts/live-migration-run-test.sh  pg/multiple-schemas
+
+      - name: "TEST: pg-unique-key-conflicts-test"
+        if: ${{ !cancelled() && matrix.test_group == 'live_advanced' }}
+        run: migtests/scripts/live-migration-fallf-run-test.sh pg/unique-key-conflicts-test
+
+      - name: "TEST: pg-live-migration-partitions-fall-forward"
+        if: ${{ !cancelled() && matrix.test_group == 'live_advanced' }}
+        run: migtests/scripts/live-migration-fallf-run-test.sh pg/partitions
+
diff --git a/.github/workflows/pg-migtests.yml → .github/workflows/pg-17-migtests.yml b/.github/workflows/pg-migtests.yml → .github/workflows/pg-17-migtests.yml
@@ -1,4 +1,4 @@
-name: "PG 13: Migration Tests"
+name: "PG 17: Migration Tests"
 
 on:
   push:
@@ -7,7 +7,7 @@ on:
     branches: ['main']
 
 jobs:
-  run-pg-migration-tests:
+  run-pg-17-migration-tests:
     strategy:
       matrix:
         version: [2024.2.0.0-b145, 2.20.8.0-b53, 2024.1.3.1-b8, 2.23.1.0-b220]
@@ -22,9 +22,9 @@ jobs:
     runs-on: ubuntu-22.04
     services:
       postgres:
-        image: postgres:13
+        image: postgres:17
         env:
-          POSTGRES_PASSWORD: secret
+          POSTGRES_PASSWORD: postgres
         # Set health checks to wait until postgres has started
         options: >-
           --health-cmd pg_isready
@@ -55,7 +55,7 @@ jobs:
       - name: "Enable postgres with wal_level as logical and install postgis"
         run: |
           docker exec ${{ job.services.postgres.id }} sh -c "echo 'wal_level=logical' >> /var/lib/postgresql/data/postgresql.conf"
-          docker exec ${{ job.services.postgres.id }} sh -c "apt-get update && apt-get install -y postgresql-13-postgis postgresql-13-postgis-3"
+          docker exec ${{ job.services.postgres.id }} sh -c "apt-get update && apt-get install -y postgresql-17-postgis postgresql-17-postgis-3"
           docker restart ${{ job.services.postgres.id }}
           sleep 10
 
@@ -65,21 +65,23 @@ jobs:
           sudo apt install -y libpq-dev
           sudo apt install python3-psycopg2
 
+      #TODO Remove the install PG 17 command once we do that in installer script 
       - name: Run installer script to setup voyager
         run: |
           cd installer_scripts
           yes | ./install-yb-voyager --install-from-local-source --only-pg-support
+          sudo apt-get -y install postgresql-17 
           sudo rm /usr/bin/pg_dump
-          sudo ln -s /usr/lib/postgresql/16/bin/pg_dump /usr/bin/pg_dump
+          sudo ln -s /usr/lib/postgresql/17/bin/pg_dump /usr/bin/pg_dump
           sudo rm /usr/bin/pg_restore
-          sudo ln -s /usr/lib/postgresql/16/bin/pg_restore /usr/bin/pg_restore
+          sudo ln -s /usr/lib/postgresql/17/bin/pg_restore /usr/bin/pg_restore
           pg_dump --version
         env:
           ON_INSTALLER_ERROR_OUTPUT_LOG: Y
 
       - name: Test PostgreSQL Connection
         run: |
-          psql "postgresql://postgres:secret@127.0.0.1:5432/postgres" -c "SELECT version();"
+          psql "postgresql://postgres:postgres@127.0.0.1:5432/postgres" -c "SELECT version();"
 
       - name: Create PostgreSQL user
         run: |
@@ -125,7 +127,7 @@ jobs:
 
       - name: "TEST: PG sample schemas (pgtbrus)"
         if: ${{ !cancelled() && matrix.test_group == 'offline' }}
-        run: migtests/scripts/run-schema-migration.sh pg/pgtbrus\
+        run: migtests/scripts/run-schema-migration.sh pg/pgtbrus
 
       - name: "TEST: PG sample schemas (stackexchange)"
         if: ${{ !cancelled() && matrix.test_group == 'offline' }}

diff --git a/.github/workflows/pg-9-migtests.yml b/.github/workflows/pg-9-migtests.yml
@@ -7,7 +7,7 @@ on:
     branches: ['main']
 
 jobs:
-  run-pg-migration-tests:
+  run-pg-9-migration-tests:
     strategy:
       matrix:
         version: [2024.2.0.0-b145]
@@ -19,7 +19,7 @@ jobs:
       postgres:
         image: postgres:9
         env:
-          POSTGRES_PASSWORD: secret
+          POSTGRES_PASSWORD: postgres
         # Set health checks to wait until postgres has started
         options: >-
           --health-cmd pg_isready
@@ -72,7 +72,7 @@ jobs:
 
       - name: Test PostgreSQL Connection
         run: |
-          psql "postgresql://postgres:secret@127.0.0.1:5432/postgres" -c "SELECT version();"
+          psql "postgresql://postgres:postgres@127.0.0.1:5432/postgres" -c "SELECT version();"
 
       - name: Create PostgreSQL user
         run: |

diff --git a/migtests/scripts/postgresql/env.sh b/migtests/scripts/postgresql/env.sh
@@ -3,4 +3,4 @@ export SOURCE_DB_PORT=${SOURCE_DB_PORT:-5432}
 export SOURCE_DB_USER=${SOURCE_DB_USER:-"ybvoyager"}
 export SOURCE_DB_PASSWORD=${SOURCE_DB_PASSWORD:-'Test@123#$%^&*()!'}
 export SOURCE_DB_ADMIN_USER=${SOURCE_DB_ADMIN_USER:-"postgres"}
-export SOURCE_DB_ADMIN_PASSWORD=${SOURCE_DB_ADMIN_PASSWORD:-"secret"}
+export SOURCE_DB_ADMIN_PASSWORD=${SOURCE_DB_ADMIN_PASSWORD:-"postgres"}
diff --git a/migtests/scripts/postgresql/ff_env.sh b/migtests/scripts/postgresql/ff_env.sh
@@ -2,4 +2,4 @@ export SOURCE_REPLICA_DB_NAME=${SOURCE_REPLICA_DB_NAME:-"ff_db"}
 export SOURCE_REPLICA_DB_HOST=${SOURCE_REPLICA_DB_HOST:-"127.0.0.1"}
 export SOURCE_REPLICA_DB_PORT=${SOURCE_REPLICA_DB_PORT:-"5432"}
 export SOURCE_REPLICA_DB_USER=${SOURCE_REPLICA_DB_USER:-"postgres"}
-export SOURCE_REPLICA_DB_PASSWORD=${SOURCE_REPLICA_DB_PASSWORD:-"secret"}
+export SOURCE_REPLICA_DB_PASSWORD=${SOURCE_REPLICA_DB_PASSWORD:-"postgres"}