Improve CI scripts (#103)

* improve CI scripts supporting tests with PostGIS on CI Actions --------- Co-authored-by: MinhLA <[email protected]>
pgspider · Sep 17, 2024 · 0db18ef · 0db18ef
1 parent f2475fb
commit 0db18ef
Show file tree

Hide file tree

Showing 7 changed files with 138 additions and 25 deletions.
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -26,47 +26,60 @@ jobs:
     env:
       SQLITE_VERSION : "3420000"
       SQLITE_YEAR: "2023"
+      POSTGIS_VERSION : "3.4.2"
       HTTP_PROXY: ""
       HTTPS_PROXY: ""
     strategy:
       fail-fast: false
       matrix:
         pg: ${{ fromJSON(needs.detect-pgversion.outputs.pgversion) }}
+        config: ["default", "postgis"]
 
-    name: Test on PostgreSQL ${{ matrix.pg }}
+    name: Test on PostgreSQL ${{ matrix.pg }}, ${{ matrix.config }} mode
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
 
       - name: tar
         run: tar zcvf sqlite_fdw.tar.gz ./*
-      
+
       - name: set_proxy
         run: bash GitHubActions/env.sh
 
       - name: install locales
         run: bash GitHubActions/install_locales.sh
-            
-      - name: build PostgreSQL ${{ matrix.pg }}
+
+      - name: build PostgreSQL ${{ matrix.pg }} with ${{ matrix.config }}
         run: bash GitHubActions/build_postgres.sh ${{ matrix.pg }}
 
-      - name: install SQLite
-        run: bash GitHubActions/install_sqlite.sh ${{ env.SQLITE_VERSION }} ${{ env.SQLITE_YEAR }}
-
-      - name: build sqlite_fdw
-        run: bash GitHubActions/build_sqlite_fdw.sh ${{ matrix.pg }}
-
+      - name: build PostGIS ${{ env.POSTGIS_VERSION }} for PostgreSQL ${{ matrix.pg }}, ${{ matrix.config }} mode
+        run: |
+          if [[ "${{ matrix.config }}" == "postgis" ]]; then
+            bash GitHubActions/build_postgis.sh ${{ matrix.pg }} ${{ env.POSTGIS_VERSION }}
+          fi
+
+      - name: install SQLite, ${{ matrix.config }} mode
+        run: |
+          if [[ "${{ matrix.config }}" == "default" ]]; then
+            bash GitHubActions/install_sqlite.sh ${{ env.SQLITE_VERSION }} ${{ env.SQLITE_YEAR }}
+          elif [[ "${{ matrix.config }}" == "postgis" ]]; then
+            bash GitHubActions/install_sqlite.sh ${{ env.SQLITE_VERSION }} ${{ env.SQLITE_YEAR }} --enable-rtree
+          fi
+
+      - name: build sqlite_fdw, ${{ matrix.config }} mode
+        run: |
+          bash GitHubActions/build_sqlite_fdw.sh ${{ matrix.pg }} ${{ matrix.config }}
+
       - name: execute sqlite_fdw test
-        run: bash GitHubActions/execute_test.sh ${{ matrix.pg }}
-      
+        run: bash GitHubActions/execute_test.sh ${{ matrix.pg }} ${{ matrix.config }}
+
       - name: download output files (regression.diffs)
         if: failure()
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ matrix.pg }}-test-results
+          name: ${{ matrix.pg }}-${{ matrix.config }}-test-results
           path: |
             workdir/postgresql-${{ matrix.pg }}/contrib/sqlite_fdw/regression.diffs
             workdir/postgresql-${{ matrix.pg }}/contrib/sqlite_fdw/regression.out
             workdir/postgresql-${{ matrix.pg }}/contrib/sqlite_fdw/results
           retention-days: 7
-
diff --git a/GitHubActions/build_postgis.sh b/GitHubActions/build_postgis.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+################################################################################
+#
+# This script downloads PostgreSQL from the official web site into ./workdir
+# then builds it.
+#
+# Usage: ./build_postgres.sh pg_version postgis_version
+#     pg_version is a PostgreSQL version to be installed like 16.0.
+#     postgis_version is a PostGIS version to be installed.
+#
+# Requirements
+# - be able to connect to the PostgreSQL official web site by curl.
+#
+################################################################################
+
+POSTGRESQL_VERSION=$1
+POSTGIS_VERSION=$2
+
+cd ./workdir
+cd postgresql-${POSTGRESQL_VERSION}
+
+# Install necessary dependencies
+sudo apt update
+sudo apt install -y build-essential libxml2-dev libgeos-dev libproj-dev libgdal-dev libjson-c-dev libprotobuf-c-dev protobuf-c-compiler
+
+GEOS_CONFIG_PATH=$(which geos-config)
+
+# Download and compile PostGIS
+cd contrib
+wget http://download.osgeo.org/postgis/source/postgis-${POSTGIS_VERSION}.tar.gz
+tar -xzf postgis-${POSTGIS_VERSION}.tar.gz
+mv postgis-${POSTGIS_VERSION} postgis -v
+cd postgis
+echo " - PostGIS directory"
+export LD_LIBRARY_PATH=/usr/local/lib/:$LD_LIBRARY_PATH
+./configure --with-pgconfig=/usr/local/pgsql/bin/pg_config --with-geosconfig=$GEOS_CONFIG_PATH
+make
+sudo make install
diff --git a/GitHubActions/build_postgres.sh b/GitHubActions/build_postgres.sh
@@ -5,19 +5,35 @@
 # This script downloads PostgreSQL from the official web site into ./workdir
 # then builds it.
 #
-# Usage: ./build_postgres.sh pg_version
+# Usage: ./build_postgres.sh pg_version [configure_options]
 #     pg_version is a PostgreSQL version to be installed like 16.0.
+#     configure_options are a list of option for postgres server.
 #
 # Requirements
 # - be able to connect to the PostgreSQL official web site by curl.
 #
 ################################################################################
 
 VERSION=$1
+CONFIGURE_OPTIONS=""
+
+while (( "$#" )); do
+  CONFIGURE_OPTIONS="$CONFIGURE_OPTIONS $2"
+  shift
+done
+
 mkdir -p ./workdir
 cd ./workdir
 curl -O https://ftp.postgresql.org/pub/source/v${VERSION}/postgresql-${VERSION}.tar.bz2
 tar xjf postgresql-${VERSION}.tar.bz2
 cd postgresql-${VERSION}
-./configure
+
+if [ -z "$CONFIGURE_OPTIONS" ]; then
+  ./configure
+else
+  ./configure $CONFIGURE_OPTIONS
+fi
+
 make
+sudo make install
+sudo chown -R $USER /usr/local/pgsql
diff --git a/GitHubActions/build_sqlite_fdw.sh b/GitHubActions/build_sqlite_fdw.sh
@@ -4,8 +4,9 @@
 #
 # This script builds sqlite_fdw in PostgreSQL source tree.
 #
-# Usage: ./build_sqlite_fdw.sh pg_version
+# Usage: ./build_sqlite_fdw.sh pg_version mode
 #     pg_version is a PostgreSQL version like 16.0 to be built in.
+#     mode is flag for sqlite_fdw compiler.
 #
 # Requirements
 # - the source code of sqlite_fdw is available by git clone.
@@ -14,8 +15,17 @@
 ################################################################################
 
 VERSION=$1
+MODE="$2"
+
 mkdir -p ./workdir/postgresql-${VERSION}/contrib/sqlite_fdw
 tar zxvf ./sqlite_fdw.tar.gz -C ./workdir/postgresql-${VERSION}/contrib/sqlite_fdw/
 cd ./workdir/postgresql-${VERSION}/contrib/sqlite_fdw
 export LD_LIBRARY_PATH=/usr/local/lib/:$LD_LIBRARY_PATH
-make
+
+if [ "$MODE" == "postgis" ]; then
+  make ENABLE_GIS=1
+else
+  make
+fi
+
+sudo make install
diff --git a/GitHubActions/execute_test.sh b/GitHubActions/execute_test.sh
@@ -6,8 +6,9 @@
 # sqlite_fdw. If all tests are passed, this script will exit successfully.
 # Otherwise, it will exit with failure.
 
-# Usage: ./execute_test.sh pg_version
+# Usage: ./execute_test.sh pg_version mode
 #     pg_version is a PostgreSQL version to be tested like 16.0.
+#	  mode is flag for sqlite_fdw compiler.
 #
 # Requiremets
 # - the source code of PostgreSQL is located in ./workdir/postgresql-{pg_version}.
@@ -20,9 +21,26 @@
 ################################################################################
 
 VERSION=$1
+MODE="$2"
+
 cd ./workdir/postgresql-${VERSION}/contrib/sqlite_fdw
+
+if [ "$MODE" == "postgis" ]; then
+	MAKEFILE_OPT="ENABLE_GIS=1"
+	echo "$MODE mode, makefile option = $MAKEFILE_OPT"
+
+	# Start postgres server
+	POSTGRES_HOME=/usr/local/pgsql
+	${POSTGRES_HOME}/bin/initdb ${POSTGRES_HOME}/databases
+	${POSTGRES_HOME}/bin/pg_ctl -D ${POSTGRES_HOME}/databases -l logfile start
+
+	# Change the testing method
+	sed -i 's/make check/make installcheck/' test.sh
+fi
+
+# Execute test script
 chmod +x ./test.sh
-./test.sh
+./test.sh $MAKEFILE_OPT
 
 last_line=$(tail -n 1 make_check.out)
 third_line_from_the_last=$(tail -n 3 make_check.out | head -n 1)

diff --git a/GitHubActions/install_sqlite.sh b/GitHubActions/install_sqlite.sh
@@ -5,11 +5,12 @@
 # This sript downloads SQLite source code from the official web site into
 # ./workdir then builds and installs it.
 #
-# Usage: ./install_sqlite.sh version year
+# Usage: ./install_sqlite.sh version year [configure_options]
 #     version: SQLite version to be installed
 #     year: A year of SQLite released. It is used for determining a download URL.
+#     configure_options are a list of option for sqlite server.
 #
-#     Ex) ./install_sqlite.sh 3420000 2023
+#     Ex) ./install_sqlite.sh 3420000 2023 --enable-rtree
 #
 # Requirements
 # - be able to connect to the SQLite official web site by curl.
@@ -19,11 +20,25 @@
 
 VERSION=$1
 YEAR=$2
+
+CONFIGURE_OPTIONS=""
+
+while (( "$#" )); do
+  CONFIGURE_OPTIONS="$CONFIGURE_OPTIONS $3"
+  shift
+done
+
 mkdir -p ./workdir
 cd ./workdir
 curl -O https://www.sqlite.org/${YEAR}/sqlite-src-${VERSION}.zip
-unzip sqlite-src-${VERSION}.zip
+unzip sqlite-src-${VERSION}.zip > /dev/null
 cd sqlite-src-${VERSION}
-./configure --enable-fts5
+
+if [ -z "$CONFIGURE_OPTIONS" ]; then
+  ./configure --enable-fts5
+else
+  ./configure --enable-fts5 $CONFIGURE_OPTIONS
+fi
+
 make
 sudo make install
diff --git a/test.sh b/test.sh
@@ -1,3 +1,5 @@
+#!/bin/bash
+
 testdir='/tmp/sqlite_fdw_test';
 rm -rf "$testdir";
 mkdir "$testdir";
@@ -10,6 +12,6 @@ sqlite3 "$testdir/selectfunc.db" < sql/init_data/init_selectfunc.sql;
 
 sed -i 's/REGRESS =.*/REGRESS = extra\/sqlite_fdw_post extra\/bitstring extra\/bool extra\/float4 extra\/float8 extra\/int4 extra\/int8 extra\/numeric extra\/out_of_range extra\/timestamp extra\/uuid extra\/join extra\/limit extra\/aggregates extra\/prepare extra\/select_having extra\/select extra\/insert extra\/update extra\/encodings sqlite_fdw type aggregate selectfunc /' Makefile
 
-make clean;
+make clean $1;
 make $1;
 make check $1 | tee make_check.out;