From 8c27a311697774ce8c8252be8203f8c64c7096c3 Mon Sep 17 00:00:00 2001 From: Max Grossman Date: Mon, 27 Feb 2023 15:04:11 -0500 Subject: [PATCH 01/26] add docker development environment --- .gitignore | 6 + Makefile.docker | 48 +++++ docker-compose.yml | 64 ++++++ docker/Dockerfile.core-services | 196 ++++++++++++++++++ docker/Dockerfile.frontend | 40 ++++ docker/Dockerfile.postgres | 35 ++++ ...DatabaseConfigServicesTranslationsLocal.sh | 3 + docker/scripts/core-services-configure.sh | 19 ++ docker/scripts/core-services-entrypoint.sh | 69 ++++++ docker/scripts/core-services-healthcheck.sh | 28 +++ docker/scripts/frontend-entrypoint.sh | 8 + docker/scripts/frontend-healthcheck.sh | 9 + docker/scripts/postgres-entrypoint.sh | 32 +++ docker/scripts/postgres-healthcheck.sh | 3 + docker/scripts/postgres-install.sh | 65 ++++++ docker/scripts/tomcat_configure.sh | 18 ++ hoot.env.example | 8 + 17 files changed, 651 insertions(+) create mode 100644 Makefile.docker create mode 100644 docker-compose.yml create mode 100644 docker/Dockerfile.core-services create mode 100644 docker/Dockerfile.frontend create mode 100644 docker/Dockerfile.postgres create mode 100755 docker/scripts/DatabaseConfigServicesTranslationsLocal.sh create mode 100755 docker/scripts/core-services-configure.sh create mode 100755 docker/scripts/core-services-entrypoint.sh create mode 100755 docker/scripts/core-services-healthcheck.sh create mode 100755 docker/scripts/frontend-entrypoint.sh create mode 100755 docker/scripts/frontend-healthcheck.sh create mode 100755 docker/scripts/postgres-entrypoint.sh create mode 100755 docker/scripts/postgres-healthcheck.sh create mode 100755 docker/scripts/postgres-install.sh create mode 100755 docker/scripts/tomcat_configure.sh create mode 100644 hoot.env.example diff --git a/.gitignore b/.gitignore index ac972a15f4..85ae0393ce 100644 --- a/.gitignore +++ b/.gitignore @@ -185,4 +185,10 @@ config.rpath 0 tbs/0 .DS_Store +.ccache .ccache/**/* +.m2 +.env +hoot.env +.bash_history +.config diff --git a/Makefile.docker b/Makefile.docker new file mode 100644 index 0000000000..462c4fae2c --- /dev/null +++ b/Makefile.docker @@ -0,0 +1,48 @@ +UID=$(shell id -u) +GID=$(shell id -g) + +hoot.env: + cp hoot.env.example hoot.env + +.env: + touch .env + mkdir bin lib .ccache + echo "HOOT_UID=$(UID)" >> .env + echo "HOOT_GID=$(GID)" >> .env + echo "HOOT_USER=hoot" >> .env + echo "HOOT_GROUP=hoot" >> .env + echo "HOOT_HOME=/var/lib/hootenanny" >> .env + echo "POSTGRESQL_VERSION=14" >> .env + +conf/database/DatabaseConfigLocal.sh: + touch conf/database/DatabaseConfigLocal.sh + chmod +x conf/database/DatabaseConfigLocal.sh + echo "#!/bin/bash;" >> conf/database/DatabaseConfigLocal.sh + echo "set -euo pipefail;" >> conf/database/DatabaseConfigLocal.sh + echo "export DB_HOST=postgres\n" >> conf/database/DatabaseConfigLocal.sh + +hoot-ui-2x/README.md: + git submodule update --init + +build: .env hoot.env conf/database/DatabaseConfigLocal.sh hoot-ui-2x/README.md + DOCKER_BUILDKIT=1 docker compose build --no-cache + +up: .env hoot.env conf/database/DatabaseConfigLocal.sh build hoot-ui-2x/README.md + DOCKER_BUILDKIT=1 docker compose up -d + +down: .env hoot.env + DOCKER_BUILDKIT=1 docker compose down + +distclean: .env hoot.env + DOCKER_BUILDKIT=1 docker compose down --volumes --rmi all + rm -fr .ccache lib bin \ + .env hoot.env hoot-ui/node_modules hoot-ui-2x/node_modules \ + conf/database/DatabaseConfigLocal.sh + +PHONY: \ + build \ + distclean \ + down \ + up + + diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000000..0297edff28 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,64 @@ +version: "3.7" +x-args: &default-args + hoot_uid: "${HOOT_UID}" + hoot_gid: "${HOOT_GID}" + hoot_user: "${HOOT_USER}" + hoot_group: "${HOOT_GROUP}" + hoot_home: "${HOOT_HOME}" + postgresql_version: "${POSTGRESQL_VERSION}" +services: + frontend: + build: + context: . + dockerfile: docker/Dockerfile.frontend + args: *default-args + hostname: frontend + ports: + - 8080:8080 + env_file: + - hoot.env + volumes: + - ./hoot-ui-2x/package.json:${HOOT_HOME}/hoot-ui-2x/package.json + - ./hoot-ui-2x/data:${HOOT_HOME}/hoot-ui-2x/data + - ./hoot-ui-2x/css:${HOOT_HOME}/hoot-ui-2x/css + - ./hoot-ui-2x/dist:${HOOT_HOME}/hoot-ui-2x/dist:rw + - ./hoot-ui-2x/img:${HOOT_HOME}/hoot-ui-2x/img + - ./hoot-ui-2x/modules:${HOOT_HOME}/hoot-ui-2x/modules:rw + - ./hoot-ui-2x/node_modules:${HOOT_HOME}/hoot-ui-2x/node_modules + - ./hoot-ui-2x/scripts:${HOOT_HOME}/hoot-ui-2x/scripts + - ./hoot-ui-2x/svg:${HOOT_HOME}/hoot-ui-2x/svg + - ./hoot-ui-2x/webpack-config:${HOOT_HOME}/hoot-ui-2x/webpack-config:rw + - ./hoot-ui-2x/webpack.config.babel.js:${HOOT_HOME}/hoot-ui-2x/webpack.config.babel.js + - ./hoot-ui-2x/build_data.js:${HOOT_HOME}/hoot-ui-2x/build_data.js + postgres: + build: + context: . + dockerfile: docker/Dockerfile.postgres + args: *default-args + hostname: postgres + env_file: + - hoot.env + volumes: + - postgres-data:/var/lib/${POSTGRESQL_VERSION}/data:rw + core-services: + depends_on: + postgres: + condition: service_healthy + frontend: + condition: service_healthy + build: + context: . + dockerfile: docker/Dockerfile.core-services + args: *default-args + hostname: core-services + env_file: + - hoot.env + ports: + - 8888:8080 + - 8094:8094 + - 8096:8096 + - 8101:8101 + volumes: + - ./:${HOOT_HOME}:rw +volumes: + postgres-data: diff --git a/docker/Dockerfile.core-services b/docker/Dockerfile.core-services new file mode 100644 index 0000000000..2db1615c85 --- /dev/null +++ b/docker/Dockerfile.core-services @@ -0,0 +1,196 @@ +FROM centos:7 + +ARG hoot_user +ARG hoot_uid +ARG hoot_gid +ARG hoot_group +ARG hoot_shell=/bin/bash +ARG hoot_home + +ARG glpk_version=4.64 +ARG liboauthcpp_version=0.1.0 +ARG libphonenumber_version=8.12.39 +ARG libpostal_version=1.1 +ARG node_version=14.16.1 +ARG npm_version=6.14.12 +ARG v8_version=8.4.371.19 +ARG stxxl_version=1.3.1 +ARG armadillo_version=10.8.2 +ARG gdal_version=3.2.3 +ARG geos_version=3.9.3 +ARG libgeotiff_version=1.6.0 +ARG proj_version=7.2.1 +ARG devtoolset_version=8 +ARG v8_version=8.4.371.19 +ARG stxxl_version=1.3.1 +ARG postgresql_version=14 + + +ENV HOOT_HOME=$hoot_home + +COPY scripts/yum/geoint-repo.sh scripts/yum/hoot-repo.sh scripts/yum/pgdg-repo.sh /tmp + +RUN --mount=type=cache,target=/var/cache/yum \ + /tmp/geoint-repo.sh && \ + /tmp/hoot-repo.sh && \ + /tmp/pgdg-repo.sh $postgresql_version && \ + yum-config-manager --save \ + --setopt=base.repo_gpgcheck=1 \ + --setopt=extras.repo_gpgcheck=1 \ + --setopt=updates.repo_gpgcheck=1 &> /dev/null && \ + yum -y install epel-release centos-release-scl + +RUN yum install -y \ + armadillo-$armadillo_version \ + devtoolset-$devtoolset_version \ + devtoolset-$devtoolset_version-libasan-devel \ + geos-$geos_version \ + geos-devel-$geos_version \ + glpk-$glpk_version \ + glpk-devel-$glpk_version \ + gdal-$gdal_version \ + gdal-devel-$gdal_version \ + gdal-python-tools-$gdal_version \ + google-chrome-stable-$google_chrome_version \ + libgeotiff-$libgeotiff_version \ + libgeotiff-devel-$libgeotiff_version \ + liboauthcpp-$liboauthcpp_version \ + liboauthcpp-devel-$liboauthcpp_version \ + libphonenumber-$libphonenumber_version \ + libphonenumber-devel-$libphonenumber_version \ + libpostal-$libpostal_version \ + libpostal-data-$libpostal_version \ + libpostal-devel-$libpostal_version \ + npm-$npm_version \ + nodejs-$node_version \ + nodejs-devel-$node_version \ + nodejs-docs-$node_version \ + nodejs-libs-$node_version \ + postgresql$postgresql_version \ + postgresql$postgresql_version-contrib \ + postgresql$postgresql_version-server \ + proj-$proj_version \ + proj-devel-$proj_version \ + stxxl-$stxxl_version \ + stxxl-devel-$stxxl_version \ + v8-devel-$v8_version + +RUN yum -y install \ + asciidoc \ + autoconf \ + autoconf-archive \ + automake \ + bison \ + boost-devel \ + bzip2 \ + ccache \ + cmake \ + cppunit-devel \ + dblatex \ + doxygen \ + gcc-c++ \ + git \ + git-core \ + gnuplot \ + lcov \ + libffi-devel \ + libicu-devel \ + libpng-devel \ + libtool \ + liquibase \ + maven \ + m4 \ + mlocate \ + opencv \ + opencv-core \ + opencv-devel \ + opencv-python \ + osmosis \ + java-1.8.0-openjdk \ + perl-XML-LibXML \ + parallel \ + make \ + protobuf \ + protobuf-compiler \ + protobuf-devel \ + python \ + python-devel \ + python3 \ + python3-devel \ + python3-matplotlib \ + python3-pip \ + python3-setuptools \ + qt5-qtbase \ + qt5-qtbase-devel \ + qt5-qtbase-postgresql \ + qt5-qtwebkit \ + qt5-qtwebkit-devel \ + readline-devel \ + redhat-lsb-core \ + sqlite-devel \ + swig \ + tex-fonts-hebrew \ + texlive \ + texlive-collection-fontsrecommended \ + texlive-collection-langcyrillic \ + tomcat8 \ + unzip \ + vim \ + wamerican-insane \ + w3m \ + wget \ + words \ + xorg-x11-server-Xvfb \ + zip + +RUN yum install -y postgresql$postgresql_version-devel && \ + alternatives --install /usr/bin/pg_config pgsql-pg_config /usr/pgsql-$postgresql_version/bin/pg_config 500 + +RUN groupadd --non-unique -g ${hoot_gid} ${hoot_group} && \ + useradd -d ${hoot_home} -m -s ${hoot_shell} -u ${hoot_uid} -g ${hoot_gid} ${hoot_user} && \ + chmod 0755 ${hoot_home} + +ARG tomcat_config=/etc/tomcat8 +ARG tomcat_logs=/var/log/tomcat8 + +# vars needed for core entrypoint +ENV DEVTOOLSET_VERSION=$devtoolset_version +ENV HOOT_USER=$hoot_user +ENV PG_MAJOR_VERSION=$postgresql_version +ENV PATH=$PATH:/usr/pgsql-$postgresql_version/bin +ENV MANPATH=/opt/rh/devtoolset-$DEVTOOLSET_VERSION/root/usr/share/ +ENV TOMCAT_CONFIG=$tomcat_config +ENV TOMCAT8_HOME=/var/lib/tomcat8 +ENV TOMCAT_SERVER=$tomcat_config/server.xml + +# services configuration files. +RUN echo -e '\ +export GDAL_DATA=$GDAL_DATA\n\ +export HOOT_HOME=$HOOT_HOME\n\ +export HOOT_WORKING_NAME=hootenanny\n\ +export JAVA_HOME=$JRE_HOME\n\ +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:$JAVA_HOME/lib/amd64/server:$HOOT_HOME/lib\n\ +export PATH=$HOOT_HOME/bin:$PATH\n'\ +>> $tomcat_config/conf.d/hoot.conf + +RUN install -d -m 0775 $tomcat_logs +RUN echo '\n\ +Please login to the host to view the logs:\n\ + sudo journalctl -u tomcat8\n\ +EOF\n'\ >> $tomcat_logs/catalina.out + +RUN chown $hoot_user:$hoot_user $tomcat_logs/catalina.out + +COPY docker/scripts/tomcat_configure.sh /tmp +RUN /tmp/tomcat_configure.sh && rm -f /tmp/tomcat_configure.sh + +# let hoot user execute tomcat8 commands +RUN usermod -a -G tomcat $hoot_user + +COPY docker/scripts/core-services-entrypoint.sh /docker-entrypoint.sh +COPY docker/scripts/core-services-healthcheck.sh /docker-healthcheck.sh + +WORKDIR ${hoot_home} +USER ${hoot_user} +ENTRYPOINT /docker-entrypoint.sh +HEALTHCHECK --interval=15s --start-period=1m CMD ["/docker-healthcheck.sh"] \ No newline at end of file diff --git a/docker/Dockerfile.frontend b/docker/Dockerfile.frontend new file mode 100644 index 0000000000..e17e67f68e --- /dev/null +++ b/docker/Dockerfile.frontend @@ -0,0 +1,40 @@ +FROM centos:7 + +ARG hoot_user +ARG hoot_uid +ARG hoot_gid +ARG hoot_group +ARG hoot_shell=/bin/bash +ARG hoot_home +ARG node_version=16 + +ENV HOOT_HOME=$hoot_home + +COPY scripts/node/nodesource-repo.sh scripts/yum/hoot-repo.sh /tmp/ + +RUN --mount=type=cache,target=/var/cache/yum \ + /tmp/nodesource-repo.sh $node_version && \ + /tmp/hoot-repo.sh && \ + yum-config-manager --save \ + --setopt=base.repo_gpgcheck=1 \ + --setopt=extras.repo_gpgcheck=1 \ + --setopt=updates.repo_gpgcheck=1 &> /dev/null && \ + yum makecache -y && \ + yum -q -y install epel-release && \ + yum -y install \ + bzip2 \ + nodejs-devel yarn \ + google-chrome-stable-91.0.4472.114 && \ + yum -q -y clean all && rm -f /tmp*.sh + +RUN groupadd --non-unique -g ${hoot_gid} ${hoot_group} && \ + useradd -d ${hoot_home} -m -s ${hoot_shell} -u ${hoot_uid} -g ${hoot_gid} ${hoot_user} && \ + chmod 0755 ${hoot_home} + +COPY docker/scripts/frontend-entrypoint.sh /docker-entrypoint.sh +COPY docker/scripts/frontend-healthcheck.sh /docker-healthcheck.sh + +USER ${hoot_user} +WORKDIR ${hoot_home}/hoot-ui-2x +ENTRYPOINT /docker-entrypoint.sh +HEALTHCHECK --interval=15s --start-period=1m CMD ["/docker-healthcheck.sh"] \ No newline at end of file diff --git a/docker/Dockerfile.postgres b/docker/Dockerfile.postgres new file mode 100644 index 0000000000..7e882d969a --- /dev/null +++ b/docker/Dockerfile.postgres @@ -0,0 +1,35 @@ +FROM centos:7 + +ARG postgresql_version +ARG pgdata=/var/lib/pgsql/$postgresql_version/data + +ENV LANG=en_US.UTF-8 \ + LC_ALL=en_US.UTF-8 \ + PGDATA=${pgdata} + +COPY scripts/yum/geoint-repo.sh \ + scripts/yum/pgdg-repo.sh \ + docker/scripts/postgres-install.sh \ + /tmp/ + +RUN --mount=type=cache,target=/var/cache/yum \ + /tmp/geoint-repo.sh && \ + /tmp/pgdg-repo.sh ${postgresql_version} && \ + yum-config-manager --save \ + --setopt=base.repo_gpgcheck=1 \ + --setopt=extras.repo_gpgcheck=1 \ + --setopt=updates.repo_gpgcheck=1 &> /dev/null && \ + yum -q -y install epel-release + +RUN /tmp/postgres-install.sh $postgresql_version && rm -f /tmp/*.sh + +COPY docker/scripts/postgres-entrypoint.sh /docker-entrypoint.sh +COPY docker/scripts/postgres-healthcheck.sh /docker-healthcheck.sh + +# Set up data volume, entrypoint, etc. +USER postgres +ENTRYPOINT ["/docker-entrypoint.sh"] +EXPOSE 5432/tcp +HEALTHCHECK --interval=15s CMD ["/docker-healthcheck.sh"] +CMD ["postgres"] +VOLUME ${pgdata} diff --git a/docker/scripts/DatabaseConfigServicesTranslationsLocal.sh b/docker/scripts/DatabaseConfigServicesTranslationsLocal.sh new file mode 100755 index 0000000000..f839c7dda9 --- /dev/null +++ b/docker/scripts/DatabaseConfigServicesTranslationsLocal.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +export DB_HOST=postgres diff --git a/docker/scripts/core-services-configure.sh b/docker/scripts/core-services-configure.sh new file mode 100755 index 0000000000..e9eaae9877 --- /dev/null +++ b/docker/scripts/core-services-configure.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -euo pipefail + +cp LocalConfig.pri.orig LocalConfig.pri + +set +u +source SetupEnv.sh +set -u + +source conf/database/DatabaseConfig.sh + +echo "JAVA_HOME=$JAVA_HOME" +echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" + +aclocal +autoconf +autoheader +automake --add-missing --copy +./configure --with-services diff --git a/docker/scripts/core-services-entrypoint.sh b/docker/scripts/core-services-entrypoint.sh new file mode 100755 index 0000000000..4f56a5c2bb --- /dev/null +++ b/docker/scripts/core-services-entrypoint.sh @@ -0,0 +1,69 @@ +#!/bin/bash +set -euo pipefail + +. /opt/rh/devtoolset-$DEVTOOLSET_VERSION/enable + + +source ./SetupEnv.sh +source conf/database/DatabaseConfig.sh + +start_tomcat() +{ + . /etc/tomcat8/tomcat8.conf + . /etc/sysconfig/tomcat8 + NAME= /usr/libexec/tomcat8/server start +} + +stop_tomcat() +{ + /usr/libexec/tomcat8/server stop +} + +copy_war_and_frontend_to_tomcat() +{ + rm -rf $TOMCAT8_HOME/webapps/hootenanny-id/ + cp -R hoot-ui-2x/dist $TOMCAT8_HOME/webapps/hootenanny-id/ + chown -R $HOOT_USER:tomcat $TOMCAT8_HOME/webapps/hootenanny-id/ + + rm -f $TOMCAT8_HOME/webapps/hoot-services.war + rm -rf $TOMCAT8_HOME/webapps/hoot-services/ + cp hoot-services/target/hoot-services-$HOOT_USER.war $TOMCAT8_HOME/webapps/hoot-services.war +} + +if [ ! -f configure ]; then + ./docker/scripts/core-services-configure.sh +fi + +if [ "${HOOT_BUILD_CORE:-0}" = "1" ] || [ ! -f ./bin/hoot.bin ]; then + # everything the core target does except build servces + make -f Makefile.hoot -j$(nproc) \ + makedirs js-make qt-make model-build \ + conf/dictionary/words.sqlite \ + conf/dictionary/WordsAbridged.sqlite \ + hoot-services/src/main/resources/language-translation/langdetect-183.bin +fi + +if [ "${HOOT_BUILD_HOOT_SERVICES_UI:-0}" = "1" ] || [ ! -f hoot-services/target/hoot-services-$HOOT_USER.war ]; then + make services-build +fi; + +# translation server is tied to services so need to build schema files and its dependencies +if [ "${HOOT_BUILD_JS_SCHEMA:-0}" = "1" ] || [ ! -f translations/tds71_schema.js ]; then + make -f Makefile.hoot js-make +fi + +if [ "${HOOT_BUILD_TRANSLATION_SERVER:-0}" = "1" ] || [ ! -d translations/node_modules ]; then + make -f Makefile.hoot translations-test +fi; + +if [ "${HOOT_BUILD_NODE_EXPORT_SERVER:-0}" = "1" ] || [ ! -d node-export-server/node_modules ]; then + pushd node-export-server; + npm install; + popd +fi; + +export GDAL_DATA=$(gdal-config --datadir) + +copy_war_and_frontend_to_tomcat +npm start --prefix "${HOOT_HOME}/node-export-server" & +start_tomcat \ No newline at end of file diff --git a/docker/scripts/core-services-healthcheck.sh b/docker/scripts/core-services-healthcheck.sh new file mode 100755 index 0000000000..672d887ce6 --- /dev/null +++ b/docker/scripts/core-services-healthcheck.sh @@ -0,0 +1,28 @@ +#!/bin/sh +set -eu +. ./SetupEnv.sh && hoot version && \ +/usr/bin/curl \ + --fail \ + --output /dev/null \ + --silent \ + --user-agent docker-healthcheck \ + "http://0.0.0.0:8080/hootenanny-id/" && \ +/usr/bin/curl \ + --request OPTIONS \ + --fail \ + --output /dev/null \ + --silent \ + --user-agent docker-healthcheck \ + "http://0.0.0.0:8096/" && \ +/usr/bin/curl \ + --fail \ + --output /dev/null \ + --silent \ + --user-agent docker-healthcheck \ + "http://0.0.0.0:8094/translations" && \ +/usr/bin/curl \ + --fail \ + --output /dev/null \ + --silent \ + --user-agent docker-healthcheck \ + "http://0.0.0.0:8101/options" \ No newline at end of file diff --git a/docker/scripts/frontend-entrypoint.sh b/docker/scripts/frontend-entrypoint.sh new file mode 100755 index 0000000000..af1570f7e7 --- /dev/null +++ b/docker/scripts/frontend-entrypoint.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -euo pipefail + +if [ "${HOOT_NPM_INSTALL:-0}" == "1" ] || [ ! -d node_modules ]; then + npm install && npm run all +fi + +npm start diff --git a/docker/scripts/frontend-healthcheck.sh b/docker/scripts/frontend-healthcheck.sh new file mode 100755 index 0000000000..3e18baead7 --- /dev/null +++ b/docker/scripts/frontend-healthcheck.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu +/usr/bin/curl \ + --insecure \ + --fail \ + --output /dev/null \ + --silent \ + --user-agent docker-healthcheck \ + "https://0.0.0.0:8080/globals.js" # use /globals.js to curcumvent the 304 behavior of non-authorized requests to / diff --git a/docker/scripts/postgres-entrypoint.sh b/docker/scripts/postgres-entrypoint.sh new file mode 100755 index 0000000000..d38dfec3c1 --- /dev/null +++ b/docker/scripts/postgres-entrypoint.sh @@ -0,0 +1,32 @@ +#!/bin/bash +set -eu + +update_postgres_setting () { + local pg_setting="$1" + local pg_value="$2" + if grep -q -e "^${pg_setting}\s\+\?=" "${PGDATA}/postgresql.conf"; then + # Modify existing setting. + sed -i -e "s#^${pg_setting}\s\+\?=\s\+\?.\+#${pg_setting} = ${pg_value}#g" \ + "${PGDATA}/postgresql.conf" + else + # New setting. + echo "${pg_setting} = ${pg_value}" >> "${PGDATA}/postgresql.conf" + fi +} + +# Customize any PostgreSQL settings from POSTGRES_SETTINGS environment. +if [ -n "${POSTGRES_SETTINGS:-}" ]; then + read -d '#' -a POSTGRES_SETTINGS_ARR <<< "$POSTGRES_SETTINGS#" + for pg_setting_pair in "${POSTGRES_SETTINGS_ARR[@]}"; do + pg_setting="$(echo "${pg_setting_pair}" | awk -F= '{ print $1 }')" + pg_value="$(echo "${pg_setting_pair}" | awk -F= '{ print $2 }')" + update_postgres_setting "${pg_setting}" "${pg_value}" + done +fi + +# ensure first argument is `postgres`. +if [ "${1#-}" != "$1" ]; then + set -- postgres "$@" +fi + +exec "$@" diff --git a/docker/scripts/postgres-healthcheck.sh b/docker/scripts/postgres-healthcheck.sh new file mode 100755 index 0000000000..9a5fbcb18c --- /dev/null +++ b/docker/scripts/postgres-healthcheck.sh @@ -0,0 +1,3 @@ +#!/bin/sh +set -eu +/usr/bin/pg_isready -h "${PGHOST:-0.0.0.0}" \ No newline at end of file diff --git a/docker/scripts/postgres-install.sh b/docker/scripts/postgres-install.sh new file mode 100755 index 0000000000..da15886177 --- /dev/null +++ b/docker/scripts/postgres-install.sh @@ -0,0 +1,65 @@ +#!/bin/bash +set -eu + +POSTGRES_VERSION="${1}" + +yum -q -y install \ + postgresql${POSTGRES_VERSION} \ + postgresql${POSTGRES_VERSION}-contrib \ + postgresql${POSTGRES_VERSION}-server + +alternatives --install /usr/bin/createdb pgsql-createdb "/usr/pgsql-${POSTGRES_VERSION}/bin/createdb" 500 +alternatives --install /usr/bin/createuser pgsql-createuser "/usr/pgsql-${POSTGRES_VERSION}/bin/createuser" 500 +alternatives --install /usr/bin/initdb pgsql-initdb "/usr/pgsql-${POSTGRES_VERSION}/bin/initdb" 500 +alternatives --install /usr/bin/pg_config pgsql-pg_config "/usr/pgsql-${POSTGRES_VERSION}/bin/pg_config" 500 +alternatives --install /usr/bin/pg_ctl pgsql-pg_ctl "/usr/pgsql-${POSTGRES_VERSION}/bin/pg_ctl" 500 +alternatives --install /usr/bin/pg_isready pgsql-pg_isready "/usr/pgsql-${POSTGRES_VERSION}/bin/pg_isready" 500 +alternatives --install /usr/bin/postgres pgsql-postgres "/usr/pgsql-${POSTGRES_VERSION}/bin/postgres" 500 + +# init and start postgres +if [ ! -e /var/lib/pgsql/$POSTGRES_VERSION/data/PG_VERSION ]; then + PGSETUP_INITDB_OPTIONS="-E UTF-8 --lc-collate=en_US.UTF-8 --lc-ctype=en_US.UTF-8" \ + su postgres -c "initdb -D $PGDATA --locale $LANG" +fi + +su postgres -c "pg_ctl -D $PGDATA -s -w start" + + +while ! /usr/pgsql-$POSTGRES_VERSION/bin/pg_isready; do + echo "Waiting for postgres to start" + sleep 1 +done + +# create Hoot services db +if ! psql -lqt | cut -d \| -f 1 | grep -qw hoot; then + su postgres -c "createuser --superuser hoot || true"; + su postgres -c "psql -c \"ALTER USER hoot WITH PASSWORD 'hoottest';\"" + su postgres -c "createdb hoot --owner=hoot" + su postgres -c "psql -d hoot -c \"CREATE EXTENSION hstore;\"" +fi + +su postgres -c "pg_ctl -D $PGDATA -s -m fast -w stop" + +# configure Postgres settings +PG_HB_CONF=/var/lib/pgsql/$POSTGRES_VERSION/data/pg_hba.conf +if ! grep -i --quiet hoot $PG_HB_CONF; then + sed -i '1ihost all all 0.0.0.0/0 md5' $PG_HB_CONF +fi +POSTGRES_CONF=/var/lib/pgsql/$POSTGRES_VERSION/data/postgresql.conf +if ! grep -i --quiet HOOT $POSTGRES_CONF; then + sed -i "s/^max_connections/\#max_connections/" $POSTGRES_CONF + sed -i "s/^shared_buffers/\#shared_buffers/" $POSTGRES_CONF + cat >> $POSTGRES_CONF <@ \n &@' $TOMCAT_SERVER +fi + +# OSM OAuth2 requires redirect URLs to use https +# Generate a keystore with a self-signed cert for localhost +if [ ! -f $TOMCAT_CONFIG/localhost-rsa.jks ]; then + keytool -genkey -keyalg RSA -noprompt -alias tomcat -dname "CN=localhost, OU=NA, O=NA, L=NA, S=NA, C=NA" -keystore $TOMCAT_CONFIG/localhost-rsa.jks -validity 9999 -storepass changeme -keypass changeme + chown root:tomcat $TOMCAT_CONFIG/localhost-rsa.jks +fi + +# Enable SSL using self-signed cert +if ! grep -i --quiet 'certificateKeyAlias="tomcat"' ${TOMCAT_CONFIG}/server.xml; then + sed -i.bak 's@@\n \n \n \n \n\n &@' ${TOMCAT_CONFIG}/server.xml +fi diff --git a/hoot.env.example b/hoot.env.example new file mode 100644 index 0000000000..bdb63a3e3e --- /dev/null +++ b/hoot.env.example @@ -0,0 +1,8 @@ +HOOT_BUILD_CORE=0 +HOOT_BUILD_HOOT_SERVICES_UI=0 +HOOT_BUILD_JS_SCHEMA=0 +HOOT_BUILD_TRANSLATION_SERVER=0 +HOOT_BUILD_NODE_EXPORT_SERVER=0 +HOOT_SERVICES_HOST=core-services +HOOT_SERVICES_PORT=8080 +HOOT_NPM_INSTALL=0 \ No newline at end of file From ff9d8dde562c289a1b48e0c5138faec0cf5dd2b8 Mon Sep 17 00:00:00 2001 From: Max Grossman Date: Tue, 28 Feb 2023 12:44:05 -0500 Subject: [PATCH 02/26] newlines --- Makefile.docker | 1 - docker/Dockerfile.core-services | 3 ++- docker/Dockerfile.frontend | 3 ++- docker/scripts/core-services-entrypoint.sh | 3 ++- docker/scripts/core-services-healthcheck.sh | 3 ++- docker/scripts/postgres-healthcheck.sh | 3 ++- 6 files changed, 10 insertions(+), 6 deletions(-) diff --git a/Makefile.docker b/Makefile.docker index 462c4fae2c..9fcc37977d 100644 --- a/Makefile.docker +++ b/Makefile.docker @@ -45,4 +45,3 @@ PHONY: \ down \ up - diff --git a/docker/Dockerfile.core-services b/docker/Dockerfile.core-services index 2db1615c85..380c64f810 100644 --- a/docker/Dockerfile.core-services +++ b/docker/Dockerfile.core-services @@ -193,4 +193,5 @@ COPY docker/scripts/core-services-healthcheck.sh /docker-healthcheck.sh WORKDIR ${hoot_home} USER ${hoot_user} ENTRYPOINT /docker-entrypoint.sh -HEALTHCHECK --interval=15s --start-period=1m CMD ["/docker-healthcheck.sh"] \ No newline at end of file +HEALTHCHECK --interval=15s --start-period=1m CMD ["/docker-healthcheck.sh"] + diff --git a/docker/Dockerfile.frontend b/docker/Dockerfile.frontend index e17e67f68e..59246bc3bf 100644 --- a/docker/Dockerfile.frontend +++ b/docker/Dockerfile.frontend @@ -37,4 +37,5 @@ COPY docker/scripts/frontend-healthcheck.sh /docker-healthcheck.sh USER ${hoot_user} WORKDIR ${hoot_home}/hoot-ui-2x ENTRYPOINT /docker-entrypoint.sh -HEALTHCHECK --interval=15s --start-period=1m CMD ["/docker-healthcheck.sh"] \ No newline at end of file +HEALTHCHECK --interval=15s --start-period=1m CMD ["/docker-healthcheck.sh"] + diff --git a/docker/scripts/core-services-entrypoint.sh b/docker/scripts/core-services-entrypoint.sh index 4f56a5c2bb..4c0cc4d8aa 100755 --- a/docker/scripts/core-services-entrypoint.sh +++ b/docker/scripts/core-services-entrypoint.sh @@ -66,4 +66,5 @@ export GDAL_DATA=$(gdal-config --datadir) copy_war_and_frontend_to_tomcat npm start --prefix "${HOOT_HOME}/node-export-server" & -start_tomcat \ No newline at end of file +start_tomcat + diff --git a/docker/scripts/core-services-healthcheck.sh b/docker/scripts/core-services-healthcheck.sh index 672d887ce6..714fa882d4 100755 --- a/docker/scripts/core-services-healthcheck.sh +++ b/docker/scripts/core-services-healthcheck.sh @@ -25,4 +25,5 @@ set -eu --output /dev/null \ --silent \ --user-agent docker-healthcheck \ - "http://0.0.0.0:8101/options" \ No newline at end of file + "http://0.0.0.0:8101/options" + diff --git a/docker/scripts/postgres-healthcheck.sh b/docker/scripts/postgres-healthcheck.sh index 9a5fbcb18c..178014f13a 100755 --- a/docker/scripts/postgres-healthcheck.sh +++ b/docker/scripts/postgres-healthcheck.sh @@ -1,3 +1,4 @@ #!/bin/sh set -eu -/usr/bin/pg_isready -h "${PGHOST:-0.0.0.0}" \ No newline at end of file +/usr/bin/pg_isready -h "${PGHOST:-0.0.0.0}" + From 4a63997edba049027ae8878318b936e4b23b081f Mon Sep 17 00:00:00 2001 From: Max Grossman Date: Tue, 28 Feb 2023 12:49:03 -0500 Subject: [PATCH 03/26] do not make build dep of up. this way build can be separate way to hard rebuild images --- Makefile.docker | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.docker b/Makefile.docker index 9fcc37977d..d91d5d46a5 100644 --- a/Makefile.docker +++ b/Makefile.docker @@ -27,7 +27,7 @@ hoot-ui-2x/README.md: build: .env hoot.env conf/database/DatabaseConfigLocal.sh hoot-ui-2x/README.md DOCKER_BUILDKIT=1 docker compose build --no-cache -up: .env hoot.env conf/database/DatabaseConfigLocal.sh build hoot-ui-2x/README.md +up: .env hoot.env conf/database/DatabaseConfigLocal.sh hoot-ui-2x/README.md DOCKER_BUILDKIT=1 docker compose up -d down: .env hoot.env From 35f9578369ef9cdfeae3271fbb19268c82cf6d13 Mon Sep 17 00:00:00 2001 From: Max Grossman Date: Tue, 28 Feb 2023 16:43:01 -0500 Subject: [PATCH 04/26] just mount hoot-ui-2x dir --- docker-compose.yml | 13 +------------ hoot.env.example | 3 ++- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 0297edff28..037f2807cc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,18 +18,7 @@ services: env_file: - hoot.env volumes: - - ./hoot-ui-2x/package.json:${HOOT_HOME}/hoot-ui-2x/package.json - - ./hoot-ui-2x/data:${HOOT_HOME}/hoot-ui-2x/data - - ./hoot-ui-2x/css:${HOOT_HOME}/hoot-ui-2x/css - - ./hoot-ui-2x/dist:${HOOT_HOME}/hoot-ui-2x/dist:rw - - ./hoot-ui-2x/img:${HOOT_HOME}/hoot-ui-2x/img - - ./hoot-ui-2x/modules:${HOOT_HOME}/hoot-ui-2x/modules:rw - - ./hoot-ui-2x/node_modules:${HOOT_HOME}/hoot-ui-2x/node_modules - - ./hoot-ui-2x/scripts:${HOOT_HOME}/hoot-ui-2x/scripts - - ./hoot-ui-2x/svg:${HOOT_HOME}/hoot-ui-2x/svg - - ./hoot-ui-2x/webpack-config:${HOOT_HOME}/hoot-ui-2x/webpack-config:rw - - ./hoot-ui-2x/webpack.config.babel.js:${HOOT_HOME}/hoot-ui-2x/webpack.config.babel.js - - ./hoot-ui-2x/build_data.js:${HOOT_HOME}/hoot-ui-2x/build_data.js + - ./hoot-ui-2x:${HOOT_HOME}/hoot-ui-2x:rw postgres: build: context: . diff --git a/hoot.env.example b/hoot.env.example index bdb63a3e3e..a1fb173c42 100644 --- a/hoot.env.example +++ b/hoot.env.example @@ -5,4 +5,5 @@ HOOT_BUILD_TRANSLATION_SERVER=0 HOOT_BUILD_NODE_EXPORT_SERVER=0 HOOT_SERVICES_HOST=core-services HOOT_SERVICES_PORT=8080 -HOOT_NPM_INSTALL=0 \ No newline at end of file +HOOT_NPM_INSTALL=0 + From ffcd681a7e870df3eb9b357b6e5ff15b442f3740 Mon Sep 17 00:00:00 2001 From: Max Grossman Date: Tue, 28 Feb 2023 17:31:32 -0500 Subject: [PATCH 05/26] removing extra lines --- Makefile.docker | 7 +++---- docker/Dockerfile.core-services | 1 - docker/Dockerfile.frontend | 1 - docker/scripts/core-services-entrypoint.sh | 1 - docker/scripts/core-services-healthcheck.sh | 1 - docker/scripts/postgres-healthcheck.sh | 1 - hoot.env.example | 1 - 7 files changed, 3 insertions(+), 10 deletions(-) diff --git a/Makefile.docker b/Makefile.docker index d91d5d46a5..0d76be2267 100644 --- a/Makefile.docker +++ b/Makefile.docker @@ -4,9 +4,9 @@ GID=$(shell id -g) hoot.env: cp hoot.env.example hoot.env -.env: +.env: hoot-ui-2x/README.md touch .env - mkdir bin lib .ccache + mkdir -p bin lib .ccache echo "HOOT_UID=$(UID)" >> .env echo "HOOT_GID=$(GID)" >> .env echo "HOOT_USER=hoot" >> .env @@ -27,7 +27,7 @@ hoot-ui-2x/README.md: build: .env hoot.env conf/database/DatabaseConfigLocal.sh hoot-ui-2x/README.md DOCKER_BUILDKIT=1 docker compose build --no-cache -up: .env hoot.env conf/database/DatabaseConfigLocal.sh hoot-ui-2x/README.md +up: .env hoot.env conf/database/DatabaseConfigLocal.sh build hoot-ui-2x/README.md DOCKER_BUILDKIT=1 docker compose up -d down: .env hoot.env @@ -44,4 +44,3 @@ PHONY: \ distclean \ down \ up - diff --git a/docker/Dockerfile.core-services b/docker/Dockerfile.core-services index 380c64f810..bf3287f681 100644 --- a/docker/Dockerfile.core-services +++ b/docker/Dockerfile.core-services @@ -194,4 +194,3 @@ WORKDIR ${hoot_home} USER ${hoot_user} ENTRYPOINT /docker-entrypoint.sh HEALTHCHECK --interval=15s --start-period=1m CMD ["/docker-healthcheck.sh"] - diff --git a/docker/Dockerfile.frontend b/docker/Dockerfile.frontend index 59246bc3bf..df2e0060be 100644 --- a/docker/Dockerfile.frontend +++ b/docker/Dockerfile.frontend @@ -38,4 +38,3 @@ USER ${hoot_user} WORKDIR ${hoot_home}/hoot-ui-2x ENTRYPOINT /docker-entrypoint.sh HEALTHCHECK --interval=15s --start-period=1m CMD ["/docker-healthcheck.sh"] - diff --git a/docker/scripts/core-services-entrypoint.sh b/docker/scripts/core-services-entrypoint.sh index 4c0cc4d8aa..e0e661c9b0 100755 --- a/docker/scripts/core-services-entrypoint.sh +++ b/docker/scripts/core-services-entrypoint.sh @@ -67,4 +67,3 @@ export GDAL_DATA=$(gdal-config --datadir) copy_war_and_frontend_to_tomcat npm start --prefix "${HOOT_HOME}/node-export-server" & start_tomcat - diff --git a/docker/scripts/core-services-healthcheck.sh b/docker/scripts/core-services-healthcheck.sh index 714fa882d4..5f3d6cd721 100755 --- a/docker/scripts/core-services-healthcheck.sh +++ b/docker/scripts/core-services-healthcheck.sh @@ -26,4 +26,3 @@ set -eu --silent \ --user-agent docker-healthcheck \ "http://0.0.0.0:8101/options" - diff --git a/docker/scripts/postgres-healthcheck.sh b/docker/scripts/postgres-healthcheck.sh index 178014f13a..c7527ea47b 100755 --- a/docker/scripts/postgres-healthcheck.sh +++ b/docker/scripts/postgres-healthcheck.sh @@ -1,4 +1,3 @@ #!/bin/sh set -eu /usr/bin/pg_isready -h "${PGHOST:-0.0.0.0}" - diff --git a/hoot.env.example b/hoot.env.example index a1fb173c42..5d51012541 100644 --- a/hoot.env.example +++ b/hoot.env.example @@ -6,4 +6,3 @@ HOOT_BUILD_NODE_EXPORT_SERVER=0 HOOT_SERVICES_HOST=core-services HOOT_SERVICES_PORT=8080 HOOT_NPM_INSTALL=0 - From 92f4a4a743b310028ec23cca4148f1984975c19c Mon Sep 17 00:00:00 2001 From: Max Grossman Date: Mon, 6 Mar 2023 09:16:38 -0500 Subject: [PATCH 06/26] adding missing and fixing incorrect volumes --- Makefile.docker | 2 +- docker-compose.yml | 7 ++- docker/scripts/core-services-entrypoint.sh | 13 +++-- docker/scripts/core-services-healthcheck.sh | 57 +++++++++++---------- docker/scripts/tomcat_configure.sh | 4 ++ 5 files changed, 51 insertions(+), 32 deletions(-) diff --git a/Makefile.docker b/Makefile.docker index 0d76be2267..219f121c93 100644 --- a/Makefile.docker +++ b/Makefile.docker @@ -27,7 +27,7 @@ hoot-ui-2x/README.md: build: .env hoot.env conf/database/DatabaseConfigLocal.sh hoot-ui-2x/README.md DOCKER_BUILDKIT=1 docker compose build --no-cache -up: .env hoot.env conf/database/DatabaseConfigLocal.sh build hoot-ui-2x/README.md +up: .env hoot.env conf/database/DatabaseConfigLocal.sh hoot-ui-2x/README.md DOCKER_BUILDKIT=1 docker compose up -d down: .env hoot.env diff --git a/docker-compose.yml b/docker-compose.yml index 037f2807cc..566f72e64f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,7 +28,7 @@ services: env_file: - hoot.env volumes: - - postgres-data:/var/lib/${POSTGRESQL_VERSION}/data:rw + - postgres-data:/var/lib/pgsql/${POSTGRESQL_VERSION}/data:rw core-services: depends_on: postgres: @@ -49,5 +49,10 @@ services: - 8101:8101 volumes: - ./:${HOOT_HOME}:rw + - tomcat8-hoot-services:/var/lib/tomcat8/webapps/hoot-services:rw + - tomcat8-hootenanny-id:/var/lib/tomcat8/webapps/hootenanny-id:rw + volumes: postgres-data: + tomcat8-hoot-services: + tomcat8-hootenanny-id: \ No newline at end of file diff --git a/docker/scripts/core-services-entrypoint.sh b/docker/scripts/core-services-entrypoint.sh index e0e661c9b0..1cbb33ce6b 100755 --- a/docker/scripts/core-services-entrypoint.sh +++ b/docker/scripts/core-services-entrypoint.sh @@ -21,15 +21,17 @@ stop_tomcat() copy_war_and_frontend_to_tomcat() { - rm -rf $TOMCAT8_HOME/webapps/hootenanny-id/ - cp -R hoot-ui-2x/dist $TOMCAT8_HOME/webapps/hootenanny-id/ - chown -R $HOOT_USER:tomcat $TOMCAT8_HOME/webapps/hootenanny-id/ + rm -rf $TOMCAT8_HOME/webapps/hootenanny-id/* + cp -R hoot-ui-2x/dist/ $TOMCAT8_HOME/webapps/hootenanny-id/ rm -f $TOMCAT8_HOME/webapps/hoot-services.war - rm -rf $TOMCAT8_HOME/webapps/hoot-services/ + rm -rf $TOMCAT8_HOME/webapps/hoot-services/* cp hoot-services/target/hoot-services-$HOOT_USER.war $TOMCAT8_HOME/webapps/hoot-services.war + cp -R hoot-services/target/hoot-services-$HOOT_USER/* $TOMCAT8_HOME/webapps/hoot-services } +touch core-services-building.txt + if [ ! -f configure ]; then ./docker/scripts/core-services-configure.sh fi @@ -66,4 +68,7 @@ export GDAL_DATA=$(gdal-config --datadir) copy_war_and_frontend_to_tomcat npm start --prefix "${HOOT_HOME}/node-export-server" & + +rm -f core-services-building.txt + start_tomcat diff --git a/docker/scripts/core-services-healthcheck.sh b/docker/scripts/core-services-healthcheck.sh index 5f3d6cd721..7b250c6410 100755 --- a/docker/scripts/core-services-healthcheck.sh +++ b/docker/scripts/core-services-healthcheck.sh @@ -1,28 +1,33 @@ #!/bin/sh set -eu -. ./SetupEnv.sh && hoot version && \ -/usr/bin/curl \ - --fail \ - --output /dev/null \ - --silent \ - --user-agent docker-healthcheck \ - "http://0.0.0.0:8080/hootenanny-id/" && \ -/usr/bin/curl \ - --request OPTIONS \ - --fail \ - --output /dev/null \ - --silent \ - --user-agent docker-healthcheck \ - "http://0.0.0.0:8096/" && \ -/usr/bin/curl \ - --fail \ - --output /dev/null \ - --silent \ - --user-agent docker-healthcheck \ - "http://0.0.0.0:8094/translations" && \ -/usr/bin/curl \ - --fail \ - --output /dev/null \ - --silent \ - --user-agent docker-healthcheck \ - "http://0.0.0.0:8101/options" + +if [ -f core-services-building.txt ]; then + sleep 5 +else + . ./SetupEnv.sh && hoot version && \ + /usr/bin/curl \ + --fail \ + --output /dev/null \ + --silent \ + --user-agent docker-healthcheck \ + "http://0.0.0.0:8080/hootenanny-id/" && \ + /usr/bin/curl \ + --request OPTIONS \ + --fail \ + --output /dev/null \ + --silent \ + --user-agent docker-healthcheck \ + "http://0.0.0.0:8096/" && \ + /usr/bin/curl \ + --fail \ + --output /dev/null \ + --silent \ + --user-agent docker-healthcheck \ + "http://0.0.0.0:8094/translations" && \ + /usr/bin/curl \ + --fail \ + --output /dev/null \ + --silent \ + --user-agent docker-healthcheck \ + "http://0.0.0.0:8101/options" +fi \ No newline at end of file diff --git a/docker/scripts/tomcat_configure.sh b/docker/scripts/tomcat_configure.sh index c3daae9b78..6fbda7f200 100755 --- a/docker/scripts/tomcat_configure.sh +++ b/docker/scripts/tomcat_configure.sh @@ -16,3 +16,7 @@ fi if ! grep -i --quiet 'certificateKeyAlias="tomcat"' ${TOMCAT_CONFIG}/server.xml; then sed -i.bak 's@@\n \n \n \n \n\n &@' ${TOMCAT_CONFIG}/server.xml fi + +mkdir -p $TOMCAT8_HOME/webapps/{hootenanny-id,hoot-services} +chown -R $HOOT_USER:tomcat $TOMCAT8_HOME/webapps/hootenanny-id/ +chown -R $HOOT_USER:tomcat $TOMCAT8_HOME/webapps/hoot-services/ \ No newline at end of file From 00aadc2c9db6b5e6b7e59505ef415c5e7957424c Mon Sep 17 00:00:00 2001 From: Max Grossman Date: Mon, 6 Mar 2023 10:50:02 -0500 Subject: [PATCH 07/26] let us attach a remote debugger for services --- docker-compose.yml | 1 + docker/Dockerfile.core-services | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 566f72e64f..8f13244b27 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -47,6 +47,7 @@ services: - 8094:8094 - 8096:8096 - 8101:8101 + - 8000:8585 volumes: - ./:${HOOT_HOME}:rw - tomcat8-hoot-services:/var/lib/tomcat8/webapps/hoot-services:rw diff --git a/docker/Dockerfile.core-services b/docker/Dockerfile.core-services index bf3287f681..53fb93e0e2 100644 --- a/docker/Dockerfile.core-services +++ b/docker/Dockerfile.core-services @@ -170,7 +170,8 @@ export HOOT_HOME=$HOOT_HOME\n\ export HOOT_WORKING_NAME=hootenanny\n\ export JAVA_HOME=$JRE_HOME\n\ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:$JAVA_HOME/lib/amd64/server:$HOOT_HOME/lib\n\ -export PATH=$HOOT_HOME/bin:$PATH\n'\ +export PATH=$HOOT_HOME/bin:$PATH\n\ +export JAVA_OPTS="${JAVA_OPTS} -Xdebug -Xrunjdwp:transport=dt_socket,address=8585,server=y,suspend=n"\n'\ >> $tomcat_config/conf.d/hoot.conf RUN install -d -m 0775 $tomcat_logs From d34649a20de6e634205c0bd63484be00b92bc521 Mon Sep 17 00:00:00 2001 From: Max Grossman Date: Mon, 6 Mar 2023 18:23:15 -0500 Subject: [PATCH 08/26] only copy files/war when we build them --- docker/scripts/core-services-entrypoint.sh | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/docker/scripts/core-services-entrypoint.sh b/docker/scripts/core-services-entrypoint.sh index 1cbb33ce6b..69bf64215c 100755 --- a/docker/scripts/core-services-entrypoint.sh +++ b/docker/scripts/core-services-entrypoint.sh @@ -19,17 +19,20 @@ stop_tomcat() /usr/libexec/tomcat8/server stop } -copy_war_and_frontend_to_tomcat() +copy_war_to_tomcat() { - rm -rf $TOMCAT8_HOME/webapps/hootenanny-id/* - cp -R hoot-ui-2x/dist/ $TOMCAT8_HOME/webapps/hootenanny-id/ - rm -f $TOMCAT8_HOME/webapps/hoot-services.war - rm -rf $TOMCAT8_HOME/webapps/hoot-services/* cp hoot-services/target/hoot-services-$HOOT_USER.war $TOMCAT8_HOME/webapps/hoot-services.war + rm -rf $TOMCAT8_HOME/webapps/hoot-services/* cp -R hoot-services/target/hoot-services-$HOOT_USER/* $TOMCAT8_HOME/webapps/hoot-services } +copy_frontend_to_tomcat() +{ + rm -rf $TOMCAT8_HOME/webapps/hootenanny-id/* + cp -R hoot-ui-2x/dist/ $TOMCAT8_HOME/webapps/hootenanny-id/ +} + touch core-services-building.txt if [ ! -f configure ]; then @@ -47,11 +50,13 @@ fi if [ "${HOOT_BUILD_HOOT_SERVICES_UI:-0}" = "1" ] || [ ! -f hoot-services/target/hoot-services-$HOOT_USER.war ]; then make services-build + copy_war_to_tomcat fi; # translation server is tied to services so need to build schema files and its dependencies if [ "${HOOT_BUILD_JS_SCHEMA:-0}" = "1" ] || [ ! -f translations/tds71_schema.js ]; then make -f Makefile.hoot js-make + copy_frontend_to_tomcat fi if [ "${HOOT_BUILD_TRANSLATION_SERVER:-0}" = "1" ] || [ ! -d translations/node_modules ]; then @@ -66,9 +71,6 @@ fi; export GDAL_DATA=$(gdal-config --datadir) -copy_war_and_frontend_to_tomcat -npm start --prefix "${HOOT_HOME}/node-export-server" & - rm -f core-services-building.txt - +npm start --prefix "${HOOT_HOME}/node-export-server" & start_tomcat From 1597befa990d4bc25212684e42c8f482a66ee8fc Mon Sep 17 00:00:00 2001 From: Max Grossman Date: Thu, 9 Mar 2023 10:23:44 -0500 Subject: [PATCH 09/26] add db_host_osmapi to DatabaseConfigLocal target --- Makefile.docker | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile.docker b/Makefile.docker index 219f121c93..efa3d09787 100644 --- a/Makefile.docker +++ b/Makefile.docker @@ -19,7 +19,8 @@ conf/database/DatabaseConfigLocal.sh: chmod +x conf/database/DatabaseConfigLocal.sh echo "#!/bin/bash;" >> conf/database/DatabaseConfigLocal.sh echo "set -euo pipefail;" >> conf/database/DatabaseConfigLocal.sh - echo "export DB_HOST=postgres\n" >> conf/database/DatabaseConfigLocal.sh + echo "export DB_HOST=postgres" >> conf/database/DatabaseConfigLocal.sh + echo "export DB_OSMAPI_HOST=postgres" >> conf/database/DatabaseConfigLocal.sh hoot-ui-2x/README.md: git submodule update --init From a146a0a27dfe66e39636620d29293a293f426ae7 Mon Sep 17 00:00:00 2001 From: Max Grossman Date: Thu, 9 Mar 2023 11:10:04 -0500 Subject: [PATCH 10/26] correct osmapi host name --- Makefile.docker | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.docker b/Makefile.docker index efa3d09787..e7d5a9fca5 100644 --- a/Makefile.docker +++ b/Makefile.docker @@ -20,7 +20,7 @@ conf/database/DatabaseConfigLocal.sh: echo "#!/bin/bash;" >> conf/database/DatabaseConfigLocal.sh echo "set -euo pipefail;" >> conf/database/DatabaseConfigLocal.sh echo "export DB_HOST=postgres" >> conf/database/DatabaseConfigLocal.sh - echo "export DB_OSMAPI_HOST=postgres" >> conf/database/DatabaseConfigLocal.sh + echo "export DB_HOST_OSMAPI=postgres" >> conf/database/DatabaseConfigLocal.sh hoot-ui-2x/README.md: git submodule update --init From 68b5d30b0266063a5ffcc2d77c3d7882bf184e81 Mon Sep 17 00:00:00 2001 From: Brian Hatchl Date: Thu, 9 Mar 2023 11:56:22 -0500 Subject: [PATCH 11/26] handle older versions of docker/docker-compose --- Makefile.docker | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/Makefile.docker b/Makefile.docker index e7d5a9fca5..fd5ecd09ca 100644 --- a/Makefile.docker +++ b/Makefile.docker @@ -1,6 +1,26 @@ UID=$(shell id -u) GID=$(shell id -g) +DOCKER_VERSION := $(shell docker --version 2>/dev/null) + +ifndef DOCKER_VERSION +$(error "command docker is not available, please install Docker") +endif + +# The "new" version integrates compose in the docker command. +# It also fixes a bug with stopping log output when the "--follow" option is set +COMPOSE_COMMAND=docker compose + +DOCKER_COMPOSE_NEW := $(shell docker compose version 2>/dev/null) +ifndef DOCKER_COMPOSE_NEW +DOCKER_COMPOSE_OLD := $(shell docker-compose --version 2>/dev/null) +ifdef DOCKER_COMPOSE_OLD +COMPOSE_COMMAND = docker-compose +else +$(error "docker compose is not available, please install it") +endif +endif + hoot.env: cp hoot.env.example hoot.env @@ -26,16 +46,16 @@ hoot-ui-2x/README.md: git submodule update --init build: .env hoot.env conf/database/DatabaseConfigLocal.sh hoot-ui-2x/README.md - DOCKER_BUILDKIT=1 docker compose build --no-cache + DOCKER_BUILDKIT=1 $(COMPOSE_COMMAND) build --no-cache up: .env hoot.env conf/database/DatabaseConfigLocal.sh hoot-ui-2x/README.md - DOCKER_BUILDKIT=1 docker compose up -d + DOCKER_BUILDKIT=1 $(COMPOSE_COMMAND) up -d down: .env hoot.env - DOCKER_BUILDKIT=1 docker compose down + DOCKER_BUILDKIT=1 $(COMPOSE_COMMAND) down distclean: .env hoot.env - DOCKER_BUILDKIT=1 docker compose down --volumes --rmi all + DOCKER_BUILDKIT=1 $(COMPOSE_COMMAND) down --volumes --rmi all rm -fr .ccache lib bin \ .env hoot.env hoot-ui/node_modules hoot-ui-2x/node_modules \ conf/database/DatabaseConfigLocal.sh From 6a3454a79953208272400dafcbd8631106e72753 Mon Sep 17 00:00:00 2001 From: Brian Hatchl Date: Thu, 9 Mar 2023 11:58:19 -0500 Subject: [PATCH 12/26] git ignore .npm --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 85ae0393ce..faebaca57e 100644 --- a/.gitignore +++ b/.gitignore @@ -192,3 +192,4 @@ tbs/0 hoot.env .bash_history .config +.npm From f011038f67bd5127feabbcc0bec1c0f52b51f797 Mon Sep 17 00:00:00 2001 From: Max Grossman Date: Thu, 9 Mar 2023 12:17:59 -0500 Subject: [PATCH 13/26] update hoot-ui-2x submodule --- hoot-ui-2x | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hoot-ui-2x b/hoot-ui-2x index 0a91f2fe31..ff93449d1d 160000 --- a/hoot-ui-2x +++ b/hoot-ui-2x @@ -1 +1 @@ -Subproject commit 0a91f2fe31e9cea74cd710c8829655b77d1fc026 +Subproject commit ff93449d1d4c2668a42f52cbd88c93d358a5feac From eba2ea6d7fdafe8d2e6854ea9c6e992cf945e686 Mon Sep 17 00:00:00 2001 From: Max Grossman Date: Thu, 9 Mar 2023 16:03:53 -0500 Subject: [PATCH 14/26] add missing new line --- docker/scripts/tomcat_configure.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/scripts/tomcat_configure.sh b/docker/scripts/tomcat_configure.sh index 6fbda7f200..ee6c1ce4c7 100755 --- a/docker/scripts/tomcat_configure.sh +++ b/docker/scripts/tomcat_configure.sh @@ -19,4 +19,5 @@ fi mkdir -p $TOMCAT8_HOME/webapps/{hootenanny-id,hoot-services} chown -R $HOOT_USER:tomcat $TOMCAT8_HOME/webapps/hootenanny-id/ -chown -R $HOOT_USER:tomcat $TOMCAT8_HOME/webapps/hoot-services/ \ No newline at end of file +chown -R $HOOT_USER:tomcat $TOMCAT8_HOME/webapps/hoot-services/ + From 19abb8682a92c753e2aa367d68b421b20d48786a Mon Sep 17 00:00:00 2001 From: Brian Hatchl Date: Fri, 31 Mar 2023 17:10:52 -0400 Subject: [PATCH 15/26] Updates to docker PR to work in fresh git repo clone (#5614) * legacy hoot submodule hoot-ui/ has been removed * don't need hootenanny-id in core-services container add parent submodule git volume to frontend to allow npm deps from github to work * use node 14 to match core-services, install git for use in npm * don't need deploy frontend to tomcat add a make clean target just deploy exploded webapp, not war file * use unprotected auth endpoint as health check * don't need hootenanny-id dir in tomcat * add clean property, remove UI mention from build services property * Submodule hoot-ui-2x 9df7cba39..ba88f7ad9: > dist/build updates > use https to fetch github dependency, refresh lock versions --- Makefile.docker | 2 +- docker-compose.yml | 3 +-- docker/Dockerfile.frontend | 4 +++- docker/scripts/core-services-entrypoint.sh | 15 +++++---------- docker/scripts/core-services-healthcheck.sh | 2 +- docker/scripts/tomcat_configure.sh | 3 +-- hoot-ui-2x | 2 +- hoot.env.example | 3 ++- 8 files changed, 15 insertions(+), 19 deletions(-) diff --git a/Makefile.docker b/Makefile.docker index fd5ecd09ca..dda88c17ba 100644 --- a/Makefile.docker +++ b/Makefile.docker @@ -57,7 +57,7 @@ down: .env hoot.env distclean: .env hoot.env DOCKER_BUILDKIT=1 $(COMPOSE_COMMAND) down --volumes --rmi all rm -fr .ccache lib bin \ - .env hoot.env hoot-ui/node_modules hoot-ui-2x/node_modules \ + .env hoot.env hoot-ui-2x/node_modules \ conf/database/DatabaseConfigLocal.sh PHONY: \ diff --git a/docker-compose.yml b/docker-compose.yml index 8f13244b27..7a28adacdb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,6 +19,7 @@ services: - hoot.env volumes: - ./hoot-ui-2x:${HOOT_HOME}/hoot-ui-2x:rw + - ./.git/modules/hoot-ui-2x:${HOOT_HOME}/.git/modules/hoot-ui-2x:rw postgres: build: context: . @@ -51,9 +52,7 @@ services: volumes: - ./:${HOOT_HOME}:rw - tomcat8-hoot-services:/var/lib/tomcat8/webapps/hoot-services:rw - - tomcat8-hootenanny-id:/var/lib/tomcat8/webapps/hootenanny-id:rw volumes: postgres-data: tomcat8-hoot-services: - tomcat8-hootenanny-id: \ No newline at end of file diff --git a/docker/Dockerfile.frontend b/docker/Dockerfile.frontend index df2e0060be..9b4b62c4f8 100644 --- a/docker/Dockerfile.frontend +++ b/docker/Dockerfile.frontend @@ -6,7 +6,7 @@ ARG hoot_gid ARG hoot_group ARG hoot_shell=/bin/bash ARG hoot_home -ARG node_version=16 +ARG node_version=14 ENV HOOT_HOME=$hoot_home @@ -23,6 +23,8 @@ RUN --mount=type=cache,target=/var/cache/yum \ yum -q -y install epel-release && \ yum -y install \ bzip2 \ + git \ + git-core \ nodejs-devel yarn \ google-chrome-stable-91.0.4472.114 && \ yum -q -y clean all && rm -f /tmp*.sh diff --git a/docker/scripts/core-services-entrypoint.sh b/docker/scripts/core-services-entrypoint.sh index 69bf64215c..c2c5413ed1 100755 --- a/docker/scripts/core-services-entrypoint.sh +++ b/docker/scripts/core-services-entrypoint.sh @@ -21,20 +21,16 @@ stop_tomcat() copy_war_to_tomcat() { - rm -f $TOMCAT8_HOME/webapps/hoot-services.war - cp hoot-services/target/hoot-services-$HOOT_USER.war $TOMCAT8_HOME/webapps/hoot-services.war rm -rf $TOMCAT8_HOME/webapps/hoot-services/* cp -R hoot-services/target/hoot-services-$HOOT_USER/* $TOMCAT8_HOME/webapps/hoot-services } -copy_frontend_to_tomcat() -{ - rm -rf $TOMCAT8_HOME/webapps/hootenanny-id/* - cp -R hoot-ui-2x/dist/ $TOMCAT8_HOME/webapps/hootenanny-id/ -} - touch core-services-building.txt +if [ "${HOOT_CLEAN:-0}" = "1" ]; then + make -f Makefile.hoot clean +fi; + if [ ! -f configure ]; then ./docker/scripts/core-services-configure.sh fi @@ -48,7 +44,7 @@ if [ "${HOOT_BUILD_CORE:-0}" = "1" ] || [ ! -f ./bin/hoot.bin ]; then hoot-services/src/main/resources/language-translation/langdetect-183.bin fi -if [ "${HOOT_BUILD_HOOT_SERVICES_UI:-0}" = "1" ] || [ ! -f hoot-services/target/hoot-services-$HOOT_USER.war ]; then +if [ "${HOOT_BUILD_HOOT_SERVICES:-0}" = "1" ] || [ ! -f hoot-services/target/hoot-services-$HOOT_USER.war ]; then make services-build copy_war_to_tomcat fi; @@ -56,7 +52,6 @@ fi; # translation server is tied to services so need to build schema files and its dependencies if [ "${HOOT_BUILD_JS_SCHEMA:-0}" = "1" ] || [ ! -f translations/tds71_schema.js ]; then make -f Makefile.hoot js-make - copy_frontend_to_tomcat fi if [ "${HOOT_BUILD_TRANSLATION_SERVER:-0}" = "1" ] || [ ! -d translations/node_modules ]; then diff --git a/docker/scripts/core-services-healthcheck.sh b/docker/scripts/core-services-healthcheck.sh index 7b250c6410..348d3000b9 100755 --- a/docker/scripts/core-services-healthcheck.sh +++ b/docker/scripts/core-services-healthcheck.sh @@ -10,7 +10,7 @@ else --output /dev/null \ --silent \ --user-agent docker-healthcheck \ - "http://0.0.0.0:8080/hootenanny-id/" && \ + "http://0.0.0.0:8080/hoot-services/auth/oauth2/authorize" && \ /usr/bin/curl \ --request OPTIONS \ --fail \ diff --git a/docker/scripts/tomcat_configure.sh b/docker/scripts/tomcat_configure.sh index ee6c1ce4c7..909d499b0f 100755 --- a/docker/scripts/tomcat_configure.sh +++ b/docker/scripts/tomcat_configure.sh @@ -17,7 +17,6 @@ if ! grep -i --quiet 'certificateKeyAlias="tomcat"' ${TOMCAT_CONFIG}/server.xml; sed -i.bak 's@@\n \n \n \n \n\n &@' ${TOMCAT_CONFIG}/server.xml fi -mkdir -p $TOMCAT8_HOME/webapps/{hootenanny-id,hoot-services} -chown -R $HOOT_USER:tomcat $TOMCAT8_HOME/webapps/hootenanny-id/ +mkdir -p $TOMCAT8_HOME/webapps/hoot-services chown -R $HOOT_USER:tomcat $TOMCAT8_HOME/webapps/hoot-services/ diff --git a/hoot-ui-2x b/hoot-ui-2x index 9df7cba396..ba88f7ad96 160000 --- a/hoot-ui-2x +++ b/hoot-ui-2x @@ -1 +1 @@ -Subproject commit 9df7cba39633c2b9f69d881b5ee2b47e3aa963f2 +Subproject commit ba88f7ad96a975a9b7266a55ee31f86006ef08e0 diff --git a/hoot.env.example b/hoot.env.example index 5d51012541..d407560d41 100644 --- a/hoot.env.example +++ b/hoot.env.example @@ -1,5 +1,6 @@ +HOOT_CLEAN=0 HOOT_BUILD_CORE=0 -HOOT_BUILD_HOOT_SERVICES_UI=0 +HOOT_BUILD_HOOT_SERVICES=0 HOOT_BUILD_JS_SCHEMA=0 HOOT_BUILD_TRANSLATION_SERVER=0 HOOT_BUILD_NODE_EXPORT_SERVER=0 From 388e63c172549c66e33ac167c494b0099b4d8cab Mon Sep 17 00:00:00 2001 From: Brian Hatchl Date: Thu, 13 Apr 2023 11:34:43 -0400 Subject: [PATCH 16/26] restore FORCE to always attempt VersionDefines.h rebuild --- Makefile.hoot | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Makefile.hoot b/Makefile.hoot index 3b3f9021c1..b46f151a31 100644 --- a/Makefile.hoot +++ b/Makefile.hoot @@ -399,8 +399,11 @@ else echo "UI tests must specify both --with-services and --with-uitests." endif -HOOT_VERSION_FILE: hoot-core/src/main/cpp/hoot/core/info/VersionDefines.h - -include HOOT_VERSION_FILE +FORCE: + +HOOT_VERSION_FILE: FORCE + $(SHELL_PATH) ./HOOT_VERSION_GEN; +-include HOOT_VERSION_FILE hoot-core/src/main/cpp/hoot/core/info/VersionDefines.h: $(SHELL_PATH) ./HOOT_VERSION_GEN; From 335e4a966a202f767b844b11517faa62ee8cdc45 Mon Sep 17 00:00:00 2001 From: Brian Hatchl Date: Thu, 13 Apr 2023 14:05:21 -0400 Subject: [PATCH 17/26] this seems to be causing ./HOOT_VERSION_GEN to loop --- Makefile.hoot | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile.hoot b/Makefile.hoot index b46f151a31..b1eda74b1e 100644 --- a/Makefile.hoot +++ b/Makefile.hoot @@ -403,7 +403,6 @@ FORCE: HOOT_VERSION_FILE: FORCE $(SHELL_PATH) ./HOOT_VERSION_GEN; --include HOOT_VERSION_FILE hoot-core/src/main/cpp/hoot/core/info/VersionDefines.h: $(SHELL_PATH) ./HOOT_VERSION_GEN; From f062ae9332491c58a1e4cb07fb53660ae7fe8ced Mon Sep 17 00:00:00 2001 From: Brian Hatchl Date: Thu, 13 Apr 2023 15:08:48 -0400 Subject: [PATCH 18/26] don't need this --- Makefile.hoot | 3 --- 1 file changed, 3 deletions(-) diff --git a/Makefile.hoot b/Makefile.hoot index b1eda74b1e..ca8eaf66a3 100644 --- a/Makefile.hoot +++ b/Makefile.hoot @@ -404,9 +404,6 @@ FORCE: HOOT_VERSION_FILE: FORCE $(SHELL_PATH) ./HOOT_VERSION_GEN; -hoot-core/src/main/cpp/hoot/core/info/VersionDefines.h: - $(SHELL_PATH) ./HOOT_VERSION_GEN; - rebuild-db: scripts/database/DeleteAllTables.sh cd hoot-services && $(MAKE) -f Makefile build From 2204e2d11c2149ea7c2f8e51cc7f322a4b43815c Mon Sep 17 00:00:00 2001 From: Brian Hatchl Date: Thu, 13 Apr 2023 17:03:08 -0400 Subject: [PATCH 19/26] revert changes to see if it affects 'make archive' --- Makefile.hoot | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile.hoot b/Makefile.hoot index ca8eaf66a3..3b3f9021c1 100644 --- a/Makefile.hoot +++ b/Makefile.hoot @@ -399,9 +399,10 @@ else echo "UI tests must specify both --with-services and --with-uitests." endif -FORCE: +HOOT_VERSION_FILE: hoot-core/src/main/cpp/hoot/core/info/VersionDefines.h + -include HOOT_VERSION_FILE -HOOT_VERSION_FILE: FORCE +hoot-core/src/main/cpp/hoot/core/info/VersionDefines.h: $(SHELL_PATH) ./HOOT_VERSION_GEN; rebuild-db: From 5a6d33275486a800173d1c4f071a255bb6cc4749 Mon Sep 17 00:00:00 2001 From: Brian Hatchl Date: Fri, 14 Apr 2023 09:16:51 -0400 Subject: [PATCH 20/26] try .PHONE instead of FORCE --- Makefile.hoot | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Makefile.hoot b/Makefile.hoot index 3b3f9021c1..8a91030773 100644 --- a/Makefile.hoot +++ b/Makefile.hoot @@ -9,7 +9,7 @@ endif SHELL=/bin/bash -.PHONY: clean-ccache +.PHONY: clean-ccache HOOT_VERSION_FILE # This list is for manually rebuilding the schema files. The files have been commited to the main GitHub repo SCHEMA_FILES= translations/tds71_schema.js translations/tds70_schema.js translations/tds61_schema.js translations/tds40_schema.js translations/mgcp_schema.js translations/ggdm30_schema.js translations/tds71_thematic_schema.js translations/tds71_full_schema.js translations/tds61_full_schema.js translations/tds70_full_schema.js translations/tds40_full_schema.js translations/ggdm30_full_schema.js translations/etds71_rules.js translations/etds70_rules.js translations/etds61_rules.js translations/etds40_rules.js translations/emgcp_rules.js translations/eggdm30_rules.js translations/etds71_osm_rules.js translations/etds61_osm_rules.js translations/etds40_osm_rules.js translations/emgcp_osm_rules.js translations/eggdm30_osm_rules.js translations/etds70_osm_rules.js @@ -399,11 +399,9 @@ else echo "UI tests must specify both --with-services and --with-uitests." endif -HOOT_VERSION_FILE: hoot-core/src/main/cpp/hoot/core/info/VersionDefines.h - -include HOOT_VERSION_FILE - -hoot-core/src/main/cpp/hoot/core/info/VersionDefines.h: +HOOT_VERSION_FILE: $(SHELL_PATH) ./HOOT_VERSION_GEN; +-include HOOT_VERSION_FILE rebuild-db: scripts/database/DeleteAllTables.sh From 89553be8d5d6a02d8f6a813b25b34b1d71e3dc5a Mon Sep 17 00:00:00 2001 From: Brian Hatchl Date: Fri, 16 Jun 2023 15:57:19 -0400 Subject: [PATCH 21/26] Submodule hoot-ui-2x 521e79585..d2bbe9541: > Merge remote-tracking branch 'origin/hoot2x' into docker_updates --- hoot-ui-2x | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hoot-ui-2x b/hoot-ui-2x index 521e795857..d2bbe9541f 160000 --- a/hoot-ui-2x +++ b/hoot-ui-2x @@ -1 +1 @@ -Subproject commit 521e7958578ccd4a88d3c7ce0e97825f18e7d376 +Subproject commit d2bbe9541f50e0aa491c08638c3e2cc9396c5893 From 5de4d6c1b24b1edfdc5e3b27d81c4a3592861a42 Mon Sep 17 00:00:00 2001 From: Brian Hatchl Date: Mon, 19 Jun 2023 15:10:17 -0400 Subject: [PATCH 22/26] wip: add some docker command refs --- Docker.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 Docker.md diff --git a/Docker.md b/Docker.md new file mode 100644 index 0000000000..7148adeba8 --- /dev/null +++ b/Docker.md @@ -0,0 +1,32 @@ +## Docker development environment cheat sheet + + +To start the containers +>make -f Makefile.docker up + + +To stop the containers +>make -f Makefile.docker down + + +To tail tomcat logs +>docker-compose logs --follow core-services + + +To bash to the core-services container +>docker-compose exec core-services bash + + +To force clean/build on container start, modify hoot.env _e.g._ +``` +HOOT_CLEAN=0 +HOOT_BUILD_CORE=1 +HOOT_BUILD_HOOT_SERVICES=1 +HOOT_BUILD_JS_SCHEMA=0 +HOOT_BUILD_TRANSLATION_SERVER=0 +HOOT_BUILD_NODE_EXPORT_SERVER=0 +HOOT_SERVICES_HOST=core-services +HOOT_SERVICES_PORT=8080 +HOOT_NPM_INSTALL=0 +``` +to set any properties from 0 to 1 and then stop/start the containers. \ No newline at end of file From 405490cf5780223b67178f01a078a258f869cbeb Mon Sep 17 00:00:00 2001 From: Brian Hatchl Date: Mon, 19 Jun 2023 16:20:28 -0400 Subject: [PATCH 23/26] need to run configure each time because generated Makefile can get stale --- docker/scripts/core-services-entrypoint.sh | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/docker/scripts/core-services-entrypoint.sh b/docker/scripts/core-services-entrypoint.sh index c2c5413ed1..fefc302b2e 100755 --- a/docker/scripts/core-services-entrypoint.sh +++ b/docker/scripts/core-services-entrypoint.sh @@ -28,20 +28,13 @@ copy_war_to_tomcat() touch core-services-building.txt if [ "${HOOT_CLEAN:-0}" = "1" ]; then - make -f Makefile.hoot clean -fi; - -if [ ! -f configure ]; then ./docker/scripts/core-services-configure.sh -fi + make clean +fi; if [ "${HOOT_BUILD_CORE:-0}" = "1" ] || [ ! -f ./bin/hoot.bin ]; then - # everything the core target does except build servces - make -f Makefile.hoot -j$(nproc) \ - makedirs js-make qt-make model-build \ - conf/dictionary/words.sqlite \ - conf/dictionary/WordsAbridged.sqlite \ - hoot-services/src/main/resources/language-translation/langdetect-183.bin + ./docker/scripts/core-services-configure.sh + make core -j$(nproc) fi if [ "${HOOT_BUILD_HOOT_SERVICES:-0}" = "1" ] || [ ! -f hoot-services/target/hoot-services-$HOOT_USER.war ]; then From c2e8e000d980b8bea9e10c05b074367d31150167 Mon Sep 17 00:00:00 2001 From: Brian Hatchl Date: Wed, 21 Jun 2023 10:47:55 -0400 Subject: [PATCH 24/26] need to call this too when core builds services --- docker/scripts/core-services-entrypoint.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/scripts/core-services-entrypoint.sh b/docker/scripts/core-services-entrypoint.sh index fefc302b2e..1f7ddcc671 100755 --- a/docker/scripts/core-services-entrypoint.sh +++ b/docker/scripts/core-services-entrypoint.sh @@ -35,6 +35,7 @@ fi; if [ "${HOOT_BUILD_CORE:-0}" = "1" ] || [ ! -f ./bin/hoot.bin ]; then ./docker/scripts/core-services-configure.sh make core -j$(nproc) + copy_war_to_tomcat fi if [ "${HOOT_BUILD_HOOT_SERVICES:-0}" = "1" ] || [ ! -f hoot-services/target/hoot-services-$HOOT_USER.war ]; then From f06d3bb58782670a0276f6e631e79a83ddb0023e Mon Sep 17 00:00:00 2001 From: Brian Hatchl Date: Tue, 26 Sep 2023 14:42:17 -0400 Subject: [PATCH 25/26] Move versions out to .env file and expose in docker-compose.yml (#5675) * consolidate lib version configs in VagrantProvisionVars.sh for now * add other ARG values * move versions to .env template file * missed adding this template file * downgrade glpk * use PG 14 * don't need bash * these are written in by Makefile.docker * PG 13 is now working for me --- .env_versions | 16 ++++++++++++++ Makefile.docker | 3 +-- VagrantProvisionCentOS7.sh | 4 ++-- VagrantProvisionCentOS7Deps.sh | 2 +- VagrantProvisionVars.sh | 10 +-------- docker-compose.yml | 16 ++++++++++++++ docker/Dockerfile.core-services | 37 ++++++++++++++++----------------- docker/Dockerfile.frontend | 2 +- 8 files changed, 56 insertions(+), 34 deletions(-) create mode 100644 .env_versions diff --git a/.env_versions b/.env_versions new file mode 100644 index 0000000000..d820ae17d8 --- /dev/null +++ b/.env_versions @@ -0,0 +1,16 @@ +POSTGRESQL_VERSION=13 +GLPK_VERSION=4.64 +LIBOAUTHCPP_VERSION=0.1.0 +LIBPHONENUMBER_VERSION=8.12.39 +LIBPOSTAL_VERSION=1.1 +NODE_VERSION=14.16.1 +NPM_VERSION=6.14.12 +V8_VERSION=8.4.371.19 +STXXL_VERSION=1.3.1 +ARMADILLO_VERSION=10.8.2 +GDAL_VERSION=3.2.3 +GEOS_VERSION=3.9.3 +LIBGEOTIFF_VERSION=1.6.0 +PROJ_VERSION=7.2.1 +DEVTOOLSET_VERSION=8 +JDK_VERSION=1.8.0 diff --git a/Makefile.docker b/Makefile.docker index dda88c17ba..cdda68d94a 100644 --- a/Makefile.docker +++ b/Makefile.docker @@ -25,14 +25,13 @@ hoot.env: cp hoot.env.example hoot.env .env: hoot-ui-2x/README.md - touch .env + cp .env_versions .env mkdir -p bin lib .ccache echo "HOOT_UID=$(UID)" >> .env echo "HOOT_GID=$(GID)" >> .env echo "HOOT_USER=hoot" >> .env echo "HOOT_GROUP=hoot" >> .env echo "HOOT_HOME=/var/lib/hootenanny" >> .env - echo "POSTGRESQL_VERSION=14" >> .env conf/database/DatabaseConfigLocal.sh: touch conf/database/DatabaseConfigLocal.sh diff --git a/VagrantProvisionCentOS7.sh b/VagrantProvisionCentOS7.sh index 392510088c..0b40666e9a 100755 --- a/VagrantProvisionCentOS7.sh +++ b/VagrantProvisionCentOS7.sh @@ -96,11 +96,11 @@ fi if ! grep --quiet "export JAVA_HOME" ~/.bash_profile; then echo "Adding Java home to profile..." - echo "export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk" >> ~/.bash_profile + echo "export JAVA_HOME=/usr/lib/jvm/java-$JDK_VERSION-openjdk" >> ~/.bash_profile echo "export PATH=\$PATH:\$JAVA_HOME/bin" >> ~/.bash_profile source ~/.bash_profile else - sed -i '/^export JAVA_HOME=.*/c\export JAVA_HOME=\/usr\/lib\/jvm\/java-1.8.0-openjdk' ~/.bash_profile + sed -i '/^export JAVA_HOME=.*/c\export JAVA_HOME=\/usr\/lib\/jvm\/java-\$JDK_VERSION-openjdk' ~/.bash_profile fi # Update the GDAL_DATA folder in ~/.bash_profile diff --git a/VagrantProvisionCentOS7Deps.sh b/VagrantProvisionCentOS7Deps.sh index 8b8d1d3476..b279149707 100755 --- a/VagrantProvisionCentOS7Deps.sh +++ b/VagrantProvisionCentOS7Deps.sh @@ -148,7 +148,7 @@ sudo yum -y install \ opencv-devel \ opencv-python \ osmosis \ - java-1.8.0-openjdk \ + java-${JDK_VERSION}-openjdk \ perl-XML-LibXML \ parallel \ postgresql${POSTGRESQL_VERSION_DOTLESS} \ diff --git a/VagrantProvisionVars.sh b/VagrantProvisionVars.sh index 43f950ab8d..03d18a9048 100644 --- a/VagrantProvisionVars.sh +++ b/VagrantProvisionVars.sh @@ -1,9 +1,6 @@ #!/usr/bin/env bash -export JDK_VERSION=1.8.0_161 -export JDK_URL=http://download.oracle.com/otn-pub/java/jdk/8u161-b12/2f38c3b165be4555a1fa6e98c45e0808/jdk-8u161-linux-x64.tar.gz -export JDK_TAR=jdk-8u161-linux-x64.tar.gz -export JDK_MD5=99051574a0d90871ed24a91a5d321ed2 +export JDK_VERSION=1.8.0 # Hoot deps library versions export GLPK_VERSION=4.64 @@ -27,10 +24,5 @@ export PROJ_VERSION=7.2.1 export POSTGRESQL_VERSION=13 export POSTGRESQL_VERSION_DOTLESS="$(echo "$POSTGRESQL_VERSION" | awk '{ gsub(/\./, ""); print substr($0, 1, 2) }')" -# FGDB 1.5 is required to compile using g++ >= 5.1 -# https://trac.osgeo.org/gdal/wiki/FileGDB#HowtodealwithGCC5.1C11ABIonLinux -export FGDB_VERSION=1.5.1 -export FGDB_URL=https://github.com/Esri/file-geodatabase-api/raw/master/FileGDB_API_${FGDB_VERSION}/ - # Devtoolset export DEVTOOLSET_VERSION=8 diff --git a/docker-compose.yml b/docker-compose.yml index 7a28adacdb..161d10a903 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,6 +6,22 @@ x-args: &default-args hoot_group: "${HOOT_GROUP}" hoot_home: "${HOOT_HOME}" postgresql_version: "${POSTGRESQL_VERSION}" + glpk_version: "${GLPK_VERSION}" + liboauthcpp_version: "${LIBOAUTHCPP_VERSION}" + libphonenumber_version: "${LIBPHONENUMBER_VERSION}" + libpostal_version: "${LIBPOSTAL_VERSION}" + node_version: "${NODE_VERSION}" + npm_version: "${NPM_VERSION}" + v8_version: "${V8_VERSION}" + stxxl_version: "${STXXL_VERSION}" + armadillo_version: "${ARMADILLO_VERSION}" + gdal_version: "${GDAL_VERSION}" + geos_version: "${GEOS_VERSION}" + libgeotiff_version: "${LIBGEOTIFF_VERSION}" + proj_version: "${PROJ_VERSION}" + devtoolset_version: "${DEVTOOLSET_VERSION}" + stxxl_version: "${STXXL_VERSION}" + jdk_version: "${JDK_VERSION}" services: frontend: build: diff --git a/docker/Dockerfile.core-services b/docker/Dockerfile.core-services index 53fb93e0e2..d12e046854 100644 --- a/docker/Dockerfile.core-services +++ b/docker/Dockerfile.core-services @@ -6,24 +6,23 @@ ARG hoot_gid ARG hoot_group ARG hoot_shell=/bin/bash ARG hoot_home - -ARG glpk_version=4.64 -ARG liboauthcpp_version=0.1.0 -ARG libphonenumber_version=8.12.39 -ARG libpostal_version=1.1 -ARG node_version=14.16.1 -ARG npm_version=6.14.12 -ARG v8_version=8.4.371.19 -ARG stxxl_version=1.3.1 -ARG armadillo_version=10.8.2 -ARG gdal_version=3.2.3 -ARG geos_version=3.9.3 -ARG libgeotiff_version=1.6.0 -ARG proj_version=7.2.1 -ARG devtoolset_version=8 -ARG v8_version=8.4.371.19 -ARG stxxl_version=1.3.1 -ARG postgresql_version=14 +ARG glpk_version +ARG liboauthcpp_version +ARG libphonenumber_version +ARG libpostal_version +ARG node_version +ARG npm_version +ARG v8_version +ARG stxxl_version +ARG armadillo_version +ARG gdal_version +ARG geos_version +ARG libgeotiff_version +ARG proj_version +ARG devtoolset_version +ARG stxxl_version +ARG postgresql_version +ARG jdk_version ENV HOOT_HOME=$hoot_home @@ -106,7 +105,7 @@ RUN yum -y install \ opencv-devel \ opencv-python \ osmosis \ - java-1.8.0-openjdk \ + java-$jdk_version-openjdk \ perl-XML-LibXML \ parallel \ make \ diff --git a/docker/Dockerfile.frontend b/docker/Dockerfile.frontend index 9b4b62c4f8..3554dccc4b 100644 --- a/docker/Dockerfile.frontend +++ b/docker/Dockerfile.frontend @@ -6,7 +6,7 @@ ARG hoot_gid ARG hoot_group ARG hoot_shell=/bin/bash ARG hoot_home -ARG node_version=14 +ARG node_version ENV HOOT_HOME=$hoot_home From 974780c12f0d4f42c637a3f64a28f3913e9929cb Mon Sep 17 00:00:00 2001 From: Brian Hatchl Date: Tue, 26 Sep 2023 16:28:47 -0400 Subject: [PATCH 26/26] Submodule hoot-ui-2x 7e0099d18...804b1502f: > update build info > Docker updates (#2050) > update build info > Bump semver from 5.7.1 to 5.7.2 (#2063) > Bump word-wrap from 1.2.3 to 1.2.4 (#2064) > Treat folders with null parent id as being under root (id=0) (#2065) > update build info > Add option to clip grail pull data to extent (#2062) < replace with hoot2x (reverting lock update) < Merge remote-tracking branch 'origin/hoot2x' into docker_updates < Merge remote-tracking branch 'origin/hoot2x' into docker_updates < dist/build updates < use https to fetch github dependency, refresh lock versions --- hoot-ui-2x | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hoot-ui-2x b/hoot-ui-2x index 7e0099d184..804b1502f5 160000 --- a/hoot-ui-2x +++ b/hoot-ui-2x @@ -1 +1 @@ -Subproject commit 7e0099d184bae903d744b8a3126d8293933c87a5 +Subproject commit 804b1502f5478ca467c7fb71d97eee64e30eaea3