internetofwater · EthanGrahn · Mar 15, 2022 · Mar 14, 2022 · Mar 14, 2022 · Mar 14, 2022
diff --git a/database/ci/Dockerfile b/database/ci/Dockerfile
@@ -1,11 +1,14 @@
-ARG LIQUIBASE_DOCKER_VERSION=3.10
-ARG POSTGIS_DOCKER_VERSION=12-3.0
+ARG DOCKER_MIRROR
+ARG LIQUIBASE_DOCKER_VERSION
+ARG POSTGIS_DOCKER_VERSION
 
-FROM liquibase/liquibase:${LIQUIBASE_DOCKER_VERSION} as Liquibase
+FROM ${DOCKER_MIRROR:-""}liquibase/liquibase:${LIQUIBASE_DOCKER_VERSION:-"3.10"} as Liquibase
 
-FROM postgis/postgis:${POSTGIS_DOCKER_VERSION} AS Postgres
+FROM ${DOCKER_MIRROR:-""}postgis/postgis:${POSTGIS_DOCKER_VERSION:-"12-3.0"} AS Postgres
 
-LABEL [email protected]
+LABEL org.opencontainers.image.authors="[email protected]"
+LABEL org.opencontainers.image.title="NLDI CI Database"
+LABEL org.opencontainers.image.description="A sample NLDI database used for continuous integration testing."
 
 ENV LIQUIBASE_HOME /liquibase
 ENV LIQUIBASE_WORKSPACE /liquibase

diff --git a/database/demo/Dockerfile b/database/demo/Dockerfile
@@ -1,11 +1,14 @@
-ARG LIQUIBASE_DOCKER_VERSION=3.10
-ARG POSTGIS_DOCKER_VERSION=12-3.0
+ARG DOCKER_MIRROR
+ARG LIQUIBASE_DOCKER_VERSION
+ARG POSTGIS_DOCKER_VERSION
 
-FROM liquibase/liquibase:${LIQUIBASE_DOCKER_VERSION} as Liquibase
+FROM ${DOCKER_MIRROR:-""}liquibase/liquibase:${LIQUIBASE_DOCKER_VERSION:-"3.10"} as Liquibase
 
-FROM postgis/postgis:${POSTGIS_DOCKER_VERSION} AS Postgres
+FROM ${DOCKER_MIRROR:-""}postgis/postgis:${POSTGIS_DOCKER_VERSION:-"12-3.0"} AS Postgres
 
-LABEL [email protected]
+LABEL org.opencontainers.image.authors="[email protected]"
+LABEL org.opencontainers.image.title="NLDI Demo Database"
+LABEL org.opencontainers.image.description="A demo of the NLDI database containing sample data."
 
 ENV LIQUIBASE_HOME /liquibase
 ENV LIQUIBASE_WORKSPACE /liquibase
@@ -20,10 +23,8 @@ RUN apt-get update && apt-get install -y curl
 
 COPY ./liquibase/changeLogs $LIQUIBASE_WORKSPACE
 COPY ./liquibase/scripts/*.sh /docker-entrypoint-initdb.d/
-COPY ./liquibase/scripts/dbInit /docker-entrypoint-initdb.d/
-COPY ./liquibase/scripts/dbCi /docker-entrypoint-initdb.d/
-COPY ./liquibase/scripts/dbDemo /docker-entrypoint-initdb.d/
-
+COPY ./liquibase/scripts/dbInit/*.sh /docker-entrypoint-initdb.d/
+COPY ./liquibase/scripts/dbDemo/*.sh /docker-entrypoint-initdb.d/
 
 RUN curl -L --verbose "https://github.com/ACWI-SSWD/nldi-db/releases/download/artifacts-1.0.0/nhdplus.yahara.pgdump.gz" -o $LIQUIBASE_HOME/nhdplus.yahara.pgdump.gz
 RUN curl -L --verbose "https://github.com/ACWI-SSWD/nldi-db/releases/download/artifacts-1.0.0/characteristic_data.yahara.pgdump.gz" -o $LIQUIBASE_HOME/characteristic_data.yahara.pgdump.gz

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -6,9 +6,10 @@ services:
     build:
       context: .
       dockerfile: ./database/ci/Dockerfile
-    networks:
-      nldi:
-        ipv4_address: ${DB_CI_IPV4}
+      args:
+        DOCKER_MIRROR: ${DOCKER_MIRROR}
+        LIQUIBASE_DOCKER_VERSION: ${LIQUIBASE_DOCKER_VERSION}
+        POSTGIS_DOCKER_VERSION: ${POSTGIS_DOCKER_VERSION}
     environment:
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
       - POSTGRES_DB=${NLDI_DATABASE_NAME}
@@ -20,7 +21,7 @@ services:
       - NHDPLUS_SCHEMA_OWNER_USERNAME=${NHDPLUS_SCHEMA_OWNER_USERNAME}
       - NLDI_READ_ONLY_USERNAME=${NLDI_READ_ONLY_USERNAME}
       - NLDI_READ_ONLY_PASSWORD=${NLDI_READ_ONLY_PASSWORD}
-      - NLDI_DATABASE_ADDRESS=127.0.0.1
+      - NLDI_DATABASE_ADDRESS=${NLDI_DATABASE_ADDRESS}
     ports:
       - ${DB_CI_PORT}:5432
     container_name: nldi-db-ci
@@ -30,9 +31,10 @@ services:
     build:
       context: .
       dockerfile: ./database/demo/Dockerfile
-    networks:
-      nldi:
-        ipv4_address: ${DB_DEMO_IPV4}
+      args:
+        DOCKER_MIRROR: ${DOCKER_MIRROR}
+        LIQUIBASE_DOCKER_VERSION: ${LIQUIBASE_DOCKER_VERSION}
+        POSTGIS_DOCKER_VERSION: ${POSTGIS_DOCKER_VERSION}
     environment:
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
       - POSTGRES_DB=${NLDI_DATABASE_NAME}
@@ -44,17 +46,13 @@ services:
       - NHDPLUS_SCHEMA_OWNER_USERNAME=${NHDPLUS_SCHEMA_OWNER_USERNAME}
       - NLDI_READ_ONLY_USERNAME=${NLDI_READ_ONLY_USERNAME}
       - NLDI_READ_ONLY_PASSWORD=${NLDI_READ_ONLY_PASSWORD}
-      - NLDI_DATABASE_ADDRESS=127.0.0.1
+      - NLDI_DATABASE_ADDRESS=${NLDI_DATABASE_ADDRESS}
     ports:
       - ${DB_DEMO_PORT}:5432
     container_name: nldi-db-demo
 
   db:
     image: postgis/postgis:12-3.0
-    networks:
-      nldi:
-        aliases:
-          - nldi-db
     environment:
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
       - POSTGRES_DB=${NLDI_DATABASE_NAME}
@@ -68,10 +66,10 @@ services:
       - db
     build:
       context: ./liquibase
-    networks:
-      nldi:
-        aliases:
-          - nldi-liquibase
+      args:
+        DOCKER_MIRROR: ${DOCKER_MIRROR}
+        LIQUIBASE_DOCKER_VERSION: ${LIQUIBASE_DOCKER_VERSION}
+    network_mode: "host"
     environment:
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
       - POSTGRES_DB=${NLDI_DATABASE_NAME}
@@ -83,13 +81,8 @@ services:
       - NHDPLUS_SCHEMA_OWNER_USERNAME=${NHDPLUS_SCHEMA_OWNER_USERNAME}
       - NLDI_READ_ONLY_USERNAME=${NLDI_READ_ONLY_USERNAME}
       - NLDI_READ_ONLY_PASSWORD=${NLDI_READ_ONLY_PASSWORD}
-      - NLDI_DATABASE_ADDRESS=nldi-db # using the network alias from the db service
+      - NLDI_DATABASE_ADDRESS=${NLDI_DATABASE_ADDRESS}
     volumes:
       - ./liquibase/changeLogs:/liquibase/workspace/
       - ./liquibase/scripts/dbInit:/docker-entrypoint-initdb.d
     container_name: nldi-liquibase
-
-networks:
-  nldi:
-    external:
-      name: ${LOCAL_NETWORK_NAME}
diff --git a/liquibase/Dockerfile b/liquibase/Dockerfile
@@ -1,16 +1,32 @@
-ARG LIQUIBASE_DOCKER_VERSION=3.10
+ARG DOCKER_MIRROR
+ARG LIQUIBASE_DOCKER_VERSION
 
-FROM liquibase/liquibase:${LIQUIBASE_DOCKER_VERSION}
+FROM ${DOCKER_MIRROR:-""}liquibase/liquibase:${LIQUIBASE_DOCKER_VERSION:-"3.10"}
 
-LABEL maintainer="[[email protected]](mailto:[email protected])"
+LABEL org.opencontainers.image.authors="[email protected]"
+LABEL org.opencontainers.image.title="NLDI Liquibase"
+LABEL org.opencontainers.image.description="Liquibase container for configuring the NLDI database."
 
 ENV LIQUIBASE_HOME /liquibase
 ENV LIQUIBASE_WORKSPACE $LIQUIBASE_HOME/workspace
-ENV LOCALONLY "-c listen_addresses='127.0.0.1, ::1'"
+
+ENV NLDI_DATABASE_ADDRESS=missing_db_address
+ENV NLDI_DATABASE_NAME=nldi
+ENV NLDI_DB_OWNER_USERNAME=missing_owner_username
+ENV NLDI_DB_OWNER_PASSWORD=missing_owner_password
+ENV NLDI_SCHEMA_OWNER_USERNAME=missing_nldi_schema_owner_username
+ENV NLDI_SCHEMA_OWNER_PASSWORD=missing_nldi_schema_owner_password
+ENV NHDPLUS_SCHEMA_OWNER_USERNAME=missing_nhdplus_schema_owner_username
+ENV NLDI_READ_ONLY_USERNAME=missing_read_only_username
+ENV NLDI_READ_ONLY_PASSWORD=missing_read_only_password
+ENV POSTGRES_PASSWORD=missing_postgres_password
 
 USER root
 COPY ./docker-entrypoint.sh /usr/local/bin/
 RUN chmod +x /usr/local/bin/docker-entrypoint.sh
 USER liquibase
 
+COPY ./changeLogs /liquibase/workspace/.
+COPY ./scripts/dbInit/* /docker-entrypoint-initdb.d/
+
 ENTRYPOINT ["docker-entrypoint.sh"]
diff --git a/liquibase/changeLogs/nldi/nldi_data/tables.sql b/liquibase/changeLogs/nldi/nldi_data/tables.sql
@@ -44,60 +44,27 @@ create table nldi_data.feature_wqp ( ) inherits (nldi_data.feature);
 alter table nldi_data.feature_wqp owner to ${NLDI_SCHEMA_OWNER_USERNAME};
 --rollback drop table nldi_data.feature_wqp;
 
---changeset drsteini:create.nldi_data.feature_wqp_temp
---preconditions onFail:MARK_RAN onError:HALT
---precondition-sql-check expectedResult:0 select count(*) from information_schema.tables where table_schema = 'nldi_data' and table_name = 'feature_wqp_temp'
-create table nldi_data.feature_wqp_temp (like nldi_data.feature);
-alter table nldi_data.feature_wqp_temp owner to ${NLDI_SCHEMA_OWNER_USERNAME};
---rollback drop table nldi_data.feature_wqp_temp;
-
-
 --changeset drsteini:create.nldi_data.feature_np21_nwis
 --preconditions onFail:MARK_RAN onError:HALT
 --precondition-sql-check expectedResult:0 select count(*) from information_schema.tables where table_schema = 'nldi_data' and table_name = 'feature_np21_nwis'
 create table nldi_data.feature_np21_nwis ( ) inherits (nldi_data.feature);
 alter table nldi_data.feature_np21_nwis owner to ${NLDI_SCHEMA_OWNER_USERNAME};
 --rollback drop table nldi_data.feature_np21_nwis;
 
---changeset drsteini:create.nldi_data.feature_np21_nwis_temp
---preconditions onFail:MARK_RAN onError:HALT
---precondition-sql-check expectedResult:0 select count(*) from information_schema.tables where table_schema = 'nldi_data' and table_name = 'feature_np21_nwis_temp'
-create table nldi_data.feature_np21_nwis_temp (like nldi_data.feature);
-alter table nldi_data.feature_np21_nwis_temp owner to ${NLDI_SCHEMA_OWNER_USERNAME};
---rollback drop table nldi_data.feature_np21_nwis_temp;
-
 --changeset drsteini:create.nldi_data.sqlinjection_test context:ci
 --preconditions onFail:MARK_RAN onError:HALT
 --precondition-sql-check expectedResult:0 select count(*) from information_schema.tables where table_schema = 'nldi_data' and table_name = 'feature; select * from pg_class;'
 create table nldi_data."feature; select * from pg_class;" ( ) inherits (nldi_data.feature);
 alter table nldi_data."feature; select * from pg_class;"  owner to ${NLDI_SCHEMA_OWNER_USERNAME};
 --rollback drop table nldi_data."feature; select * from pg_class;";
 
-
---changeset drsteini:create.nldi_data.sqlinjection_test_temp context:ci
---preconditions onFail:MARK_RAN onError:HALT
---precondition-sql-check expectedResult:0 select count(*) from information_schema.tables where table_schema = 'nldi_data' and table_name = 'feature; select * from pg_class;_temp'
-create table nldi_data."feature; select * from pg_class;_temp" (like nldi_data.feature);
-alter table nldi_data."feature; select * from pg_class;_temp" owner to ${NLDI_SCHEMA_OWNER_USERNAME};
---rollback drop table nldi_data."feature; select * from pg_class;_temp";
-
-
 --changeset drsteini:create.nldi_data.feature_huc12pp
 --preconditions onFail:MARK_RAN onError:HALT
 --precondition-sql-check expectedResult:0 select count(*) from information_schema.tables where table_schema = 'nldi_data' and table_name = 'feature_huc12pp'
 create table nldi_data.feature_huc12pp ( ) inherits (nldi_data.feature);
 alter table nldi_data.feature_huc12pp owner to ${NLDI_SCHEMA_OWNER_USERNAME};
 --rollback drop table nldi_data.feature_huc12pp;
 
-
---changeset drsteini:create.nldi_data.feature_huc12pp_temp
---preconditions onFail:MARK_RAN onError:HALT
---precondition-sql-check expectedResult:0 select count(*) from information_schema.tables where table_schema = 'nldi_data' and table_name = 'feature_huc12pp_temp'
-create table nldi_data.feature_huc12pp_temp (like nldi_data.feature);
-alter table nldi_data.feature_huc12pp_temp owner to ${NLDI_SCHEMA_OWNER_USERNAME};
---rollback drop table nldi_data.feature_huc12pp_temp;
-
-
 --changeset drsteini:create.nldi_data.web_service_log
 --preconditions onFail:MARK_RAN onError:HALT
 --precondition-sql-check expectedResult:0 select count(*) from information_schema.tables where table_schema = 'nldi_data' and table_name = 'web_service_log'
@@ -114,3 +81,22 @@ create table nldi_data.web_service_log
 alter table nldi_data.web_service_log owner to ${NLDI_SCHEMA_OWNER_USERNAME};
 --rollback drop table nldi_data.web_service_log;
 
+--changeset egrahn:drop.nldi_data.feature_wqp_temp
+--preconditions onFail:MARK_RAN onError:HALT
+--precondition-sql-check expectedResult:t select to_regclass('nldi_data.feature_wqp_temp') is not null
+drop table nldi_data.feature_wqp_temp;
+
+--changeset egrahn:drop.nldi_data.feature_np21_nwis_temp
+--preconditions onFail:MARK_RAN onError:HALT
+--precondition-sql-check expectedResult:t select to_regclass('nldi_data.feature_np21_nwis_temp') is not null
+drop table nldi_data.feature_np21_nwis_temp;
+
+--changeset egrahn:drop.nldi_data.sqlinjection_test_temp
+--preconditions onFail:MARK_RAN onError:HALT
+--precondition-sql-check expectedResult:t select to_regclass('nldi_data."feature; select * from pg_class;_temp"') is not null
+drop table nldi_data."feature; select * from pg_class;_temp";
+
+--changeset egrahn:drop.nldi_data.feature_huc12pp_temp
+--preconditions onFail:MARK_RAN onError:HALT
+--precondition-sql-check expectedResult:t select to_regclass('nldi_data.feature_huc12pp_temp') is not null
+drop table nldi_data.feature_huc12pp_temp;
diff --git a/liquibase/scripts/dbDemo/z4_load_demo.sh b/liquibase/scripts/dbDemo/z4_load_demo.sh
@@ -1,8 +1,7 @@
 #!/bin/bash 
 
-# Restart postgres to make sure we can connect
-pg_ctl -D "$PGDATA" -m fast -o "$LOCALONLY" -w restart
-
+gunzip -c ${LIQUIBASE_HOME}/nhdplus.yahara.pgdump.gz | pg_restore -h 127.0.0.1 -p 5432 -U postgres -w -a -d ${NLDI_DATABASE_NAME}
+gunzip -c ${LIQUIBASE_HOME}/characteristic_data.yahara.pgdump.gz | pg_restore -h 127.0.0.1 -p 5432 -U postgres -w -a -d ${NLDI_DATABASE_NAME} -n characteristic_data
 gunzip -c ${LIQUIBASE_HOME}/nldi_data.crawler_source.pgdump.gz | pg_restore -h 127.0.0.1 -p 5432 -U nldi -w -a -d ${NLDI_DATABASE_NAME}
 gunzip -c ${LIQUIBASE_HOME}/feature_wqp_yahara.backup.gz | pg_restore -h 127.0.0.1 -p 5432 -U nldi -w -a -d ${NLDI_DATABASE_NAME}
 gunzip -c ${LIQUIBASE_HOME}/feature_huc12pp_yahara.backup.gz | pg_restore -h 127.0.0.1 -p 5432 -U nldi -w -a -d ${NLDI_DATABASE_NAME}

diff --git a/liquibase/scripts/dbInit/z1_postgres_liquibase.sh b/liquibase/scripts/dbInit/z1_postgres_liquibase.sh
@@ -15,7 +15,7 @@ ${LIQUIBASE_HOME}/liquibase \
 	--username=postgres \
 	--password=${POSTGRES_PASSWORD} \
 	--contexts=${CONTEXTS} \
-	--logLevel=debug \
+	--logLevel=${LOG_LEVEL:-info} \
 	--liquibaseCatalogName=public \
 	--liquibaseSchemaName=public \
 	update \
@@ -37,7 +37,7 @@ ${LIQUIBASE_HOME}/liquibase \
 	--username=postgres \
 	--password=${POSTGRES_PASSWORD} \
 	--contexts=demo \
-	--logLevel=debug \
+	--logLevel=${LOG_LEVEL:-info} \
 	--liquibaseCatalogName=public \
 	--liquibaseSchemaName=public \
 	update \

diff --git a/liquibase/scripts/dbInit/z2_nldi_liquibase.sh b/liquibase/scripts/dbInit/z2_nldi_liquibase.sh
@@ -8,7 +8,7 @@ ${LIQUIBASE_HOME}/liquibase \
 	--username=${NLDI_DB_OWNER_USERNAME} \
 	--password=${NLDI_DB_OWNER_PASSWORD} \
 	--contexts=${CONTEXTS} \
-	--logLevel=debug\
+	--logLevel=${LOG_LEVEL:-info} \
 	--liquibaseCatalogName=nldi_data \
 	--liquibaseSchemaName=nldi_data \
 	update \