From 06e30f5c20ffd49d1f60075c595282db15804e94 Mon Sep 17 00:00:00 2001 From: Max Grossman Date: Wed, 27 Sep 2023 10:01:09 -0400 Subject: [PATCH] Docker dev env (#5583) * add docker development environment * newlines * do not make build dep of up. this way build can be separate way to hard rebuild images * just mount hoot-ui-2x dir * removing extra lines * adding missing and fixing incorrect volumes * let us attach a remote debugger for services * only copy files/war when we build them * add db_host_osmapi to DatabaseConfigLocal target * correct osmapi host name * handle older versions of docker/docker-compose * git ignore .npm * update hoot-ui-2x submodule * add missing new line * Updates to docker PR to work in fresh git repo clone (#5614) * legacy hoot submodule hoot-ui/ has been removed * don't need hootenanny-id in core-services container add parent submodule git volume to frontend to allow npm deps from github to work * use node 14 to match core-services, install git for use in npm * don't need deploy frontend to tomcat add a make clean target just deploy exploded webapp, not war file * use unprotected auth endpoint as health check * don't need hootenanny-id dir in tomcat * add clean property, remove UI mention from build services property * Submodule hoot-ui-2x 9df7cba39..ba88f7ad9: > dist/build updates > use https to fetch github dependency, refresh lock versions * restore FORCE to always attempt VersionDefines.h rebuild * this seems to be causing ./HOOT_VERSION_GEN to loop * don't need this * revert changes to see if it affects 'make archive' * try .PHONE instead of FORCE * Submodule hoot-ui-2x 521e79585..d2bbe9541: > Merge remote-tracking branch 'origin/hoot2x' into docker_updates * wip: add some docker command refs * need to run configure each time because generated Makefile can get stale * need to call this too when core builds services * Move versions out to .env file and expose in docker-compose.yml (#5675) * consolidate lib version configs in VagrantProvisionVars.sh for now * add other ARG values * move versions to .env template file * missed adding this template file * downgrade glpk * use PG 14 * don't need bash * these are written in by Makefile.docker * PG 13 is now working for me * Submodule hoot-ui-2x 7e0099d18...804b1502f: > update build info > Docker updates (#2050) > update build info > Bump semver from 5.7.1 to 5.7.2 (#2063) > Bump word-wrap from 1.2.3 to 1.2.4 (#2064) > Treat folders with null parent id as being under root (id=0) (#2065) > update build info > Add option to clip grail pull data to extent (#2062) < replace with hoot2x (reverting lock update) < Merge remote-tracking branch 'origin/hoot2x' into docker_updates < Merge remote-tracking branch 'origin/hoot2x' into docker_updates < dist/build updates < use https to fetch github dependency, refresh lock versions --------- Co-authored-by: Brian Hatchl --- .env_versions | 16 ++ .gitignore | 7 + Docker.md | 32 +++ Makefile.docker | 66 ++++++ Makefile.hoot | 8 +- VagrantProvisionCentOS7.sh | 4 +- VagrantProvisionCentOS7Deps.sh | 2 +- VagrantProvisionVars.sh | 10 +- docker-compose.yml | 74 +++++++ docker/Dockerfile.core-services | 196 ++++++++++++++++++ docker/Dockerfile.frontend | 42 ++++ docker/Dockerfile.postgres | 35 ++++ ...DatabaseConfigServicesTranslationsLocal.sh | 3 + docker/scripts/core-services-configure.sh | 19 ++ docker/scripts/core-services-entrypoint.sh | 65 ++++++ docker/scripts/core-services-healthcheck.sh | 33 +++ docker/scripts/frontend-entrypoint.sh | 8 + docker/scripts/frontend-healthcheck.sh | 9 + docker/scripts/postgres-entrypoint.sh | 32 +++ docker/scripts/postgres-healthcheck.sh | 3 + docker/scripts/postgres-install.sh | 65 ++++++ docker/scripts/tomcat_configure.sh | 22 ++ hoot-ui-2x | 2 +- hoot.env.example | 9 + 24 files changed, 744 insertions(+), 18 deletions(-) create mode 100644 .env_versions create mode 100644 Docker.md create mode 100644 Makefile.docker create mode 100644 docker-compose.yml create mode 100644 docker/Dockerfile.core-services create mode 100644 docker/Dockerfile.frontend create mode 100644 docker/Dockerfile.postgres create mode 100755 docker/scripts/DatabaseConfigServicesTranslationsLocal.sh create mode 100755 docker/scripts/core-services-configure.sh create mode 100755 docker/scripts/core-services-entrypoint.sh create mode 100755 docker/scripts/core-services-healthcheck.sh create mode 100755 docker/scripts/frontend-entrypoint.sh create mode 100755 docker/scripts/frontend-healthcheck.sh create mode 100755 docker/scripts/postgres-entrypoint.sh create mode 100755 docker/scripts/postgres-healthcheck.sh create mode 100755 docker/scripts/postgres-install.sh create mode 100755 docker/scripts/tomcat_configure.sh create mode 100644 hoot.env.example diff --git a/.env_versions b/.env_versions new file mode 100644 index 0000000000..d820ae17d8 --- /dev/null +++ b/.env_versions @@ -0,0 +1,16 @@ +POSTGRESQL_VERSION=13 +GLPK_VERSION=4.64 +LIBOAUTHCPP_VERSION=0.1.0 +LIBPHONENUMBER_VERSION=8.12.39 +LIBPOSTAL_VERSION=1.1 +NODE_VERSION=14.16.1 +NPM_VERSION=6.14.12 +V8_VERSION=8.4.371.19 +STXXL_VERSION=1.3.1 +ARMADILLO_VERSION=10.8.2 +GDAL_VERSION=3.2.3 +GEOS_VERSION=3.9.3 +LIBGEOTIFF_VERSION=1.6.0 +PROJ_VERSION=7.2.1 +DEVTOOLSET_VERSION=8 +JDK_VERSION=1.8.0 diff --git a/.gitignore b/.gitignore index ac972a15f4..faebaca57e 100644 --- a/.gitignore +++ b/.gitignore @@ -185,4 +185,11 @@ config.rpath 0 tbs/0 .DS_Store +.ccache .ccache/**/* +.m2 +.env +hoot.env +.bash_history +.config +.npm diff --git a/Docker.md b/Docker.md new file mode 100644 index 0000000000..7148adeba8 --- /dev/null +++ b/Docker.md @@ -0,0 +1,32 @@ +## Docker development environment cheat sheet + + +To start the containers +>make -f Makefile.docker up + + +To stop the containers +>make -f Makefile.docker down + + +To tail tomcat logs +>docker-compose logs --follow core-services + + +To bash to the core-services container +>docker-compose exec core-services bash + + +To force clean/build on container start, modify hoot.env _e.g._ +``` +HOOT_CLEAN=0 +HOOT_BUILD_CORE=1 +HOOT_BUILD_HOOT_SERVICES=1 +HOOT_BUILD_JS_SCHEMA=0 +HOOT_BUILD_TRANSLATION_SERVER=0 +HOOT_BUILD_NODE_EXPORT_SERVER=0 +HOOT_SERVICES_HOST=core-services +HOOT_SERVICES_PORT=8080 +HOOT_NPM_INSTALL=0 +``` +to set any properties from 0 to 1 and then stop/start the containers. \ No newline at end of file diff --git a/Makefile.docker b/Makefile.docker new file mode 100644 index 0000000000..cdda68d94a --- /dev/null +++ b/Makefile.docker @@ -0,0 +1,66 @@ +UID=$(shell id -u) +GID=$(shell id -g) + +DOCKER_VERSION := $(shell docker --version 2>/dev/null) + +ifndef DOCKER_VERSION +$(error "command docker is not available, please install Docker") +endif + +# The "new" version integrates compose in the docker command. +# It also fixes a bug with stopping log output when the "--follow" option is set +COMPOSE_COMMAND=docker compose + +DOCKER_COMPOSE_NEW := $(shell docker compose version 2>/dev/null) +ifndef DOCKER_COMPOSE_NEW +DOCKER_COMPOSE_OLD := $(shell docker-compose --version 2>/dev/null) +ifdef DOCKER_COMPOSE_OLD +COMPOSE_COMMAND = docker-compose +else +$(error "docker compose is not available, please install it") +endif +endif + +hoot.env: + cp hoot.env.example hoot.env + +.env: hoot-ui-2x/README.md + cp .env_versions .env + mkdir -p bin lib .ccache + echo "HOOT_UID=$(UID)" >> .env + echo "HOOT_GID=$(GID)" >> .env + echo "HOOT_USER=hoot" >> .env + echo "HOOT_GROUP=hoot" >> .env + echo "HOOT_HOME=/var/lib/hootenanny" >> .env + +conf/database/DatabaseConfigLocal.sh: + touch conf/database/DatabaseConfigLocal.sh + chmod +x conf/database/DatabaseConfigLocal.sh + echo "#!/bin/bash;" >> conf/database/DatabaseConfigLocal.sh + echo "set -euo pipefail;" >> conf/database/DatabaseConfigLocal.sh + echo "export DB_HOST=postgres" >> conf/database/DatabaseConfigLocal.sh + echo "export DB_HOST_OSMAPI=postgres" >> conf/database/DatabaseConfigLocal.sh + +hoot-ui-2x/README.md: + git submodule update --init + +build: .env hoot.env conf/database/DatabaseConfigLocal.sh hoot-ui-2x/README.md + DOCKER_BUILDKIT=1 $(COMPOSE_COMMAND) build --no-cache + +up: .env hoot.env conf/database/DatabaseConfigLocal.sh hoot-ui-2x/README.md + DOCKER_BUILDKIT=1 $(COMPOSE_COMMAND) up -d + +down: .env hoot.env + DOCKER_BUILDKIT=1 $(COMPOSE_COMMAND) down + +distclean: .env hoot.env + DOCKER_BUILDKIT=1 $(COMPOSE_COMMAND) down --volumes --rmi all + rm -fr .ccache lib bin \ + .env hoot.env hoot-ui-2x/node_modules \ + conf/database/DatabaseConfigLocal.sh + +PHONY: \ + build \ + distclean \ + down \ + up diff --git a/Makefile.hoot b/Makefile.hoot index 5976c42e8f..5b275fda42 100644 --- a/Makefile.hoot +++ b/Makefile.hoot @@ -9,7 +9,7 @@ endif SHELL=/bin/bash -.PHONY: clean-ccache +.PHONY: clean-ccache HOOT_VERSION_FILE # This list is for manually rebuilding the schema files. The files have been commited to the main GitHub repo SCHEMA_FILES= translations/tds71_schema.js translations/tds70_schema.js translations/tds61_schema.js translations/tds40_schema.js translations/mgcp_schema.js translations/mgcp_thematic_schema.js translations/ggdm30_schema.js translations/tds71_thematic_schema.js translations/tds71_full_schema.js translations/tds61_full_schema.js translations/tds70_full_schema.js translations/tds40_full_schema.js translations/ggdm30_full_schema.js translations/etds71_rules.js translations/etds70_rules.js translations/etds61_rules.js translations/etds40_rules.js translations/emgcp_rules.js translations/eggdm30_rules.js translations/etds71_osm_rules.js translations/etds61_osm_rules.js translations/etds40_osm_rules.js translations/emgcp_osm_rules.js translations/eggdm30_osm_rules.js translations/etds70_osm_rules.js @@ -404,11 +404,9 @@ else echo "UI tests must specify both --with-services and --with-uitests." endif -HOOT_VERSION_FILE: hoot-core/src/main/cpp/hoot/core/info/VersionDefines.h - -include HOOT_VERSION_FILE - -hoot-core/src/main/cpp/hoot/core/info/VersionDefines.h: +HOOT_VERSION_FILE: $(SHELL_PATH) ./HOOT_VERSION_GEN; +-include HOOT_VERSION_FILE rebuild-db: scripts/database/DeleteAllTables.sh diff --git a/VagrantProvisionCentOS7.sh b/VagrantProvisionCentOS7.sh index 392510088c..0b40666e9a 100755 --- a/VagrantProvisionCentOS7.sh +++ b/VagrantProvisionCentOS7.sh @@ -96,11 +96,11 @@ fi if ! grep --quiet "export JAVA_HOME" ~/.bash_profile; then echo "Adding Java home to profile..." - echo "export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk" >> ~/.bash_profile + echo "export JAVA_HOME=/usr/lib/jvm/java-$JDK_VERSION-openjdk" >> ~/.bash_profile echo "export PATH=\$PATH:\$JAVA_HOME/bin" >> ~/.bash_profile source ~/.bash_profile else - sed -i '/^export JAVA_HOME=.*/c\export JAVA_HOME=\/usr\/lib\/jvm\/java-1.8.0-openjdk' ~/.bash_profile + sed -i '/^export JAVA_HOME=.*/c\export JAVA_HOME=\/usr\/lib\/jvm\/java-\$JDK_VERSION-openjdk' ~/.bash_profile fi # Update the GDAL_DATA folder in ~/.bash_profile diff --git a/VagrantProvisionCentOS7Deps.sh b/VagrantProvisionCentOS7Deps.sh index 8b8d1d3476..b279149707 100755 --- a/VagrantProvisionCentOS7Deps.sh +++ b/VagrantProvisionCentOS7Deps.sh @@ -148,7 +148,7 @@ sudo yum -y install \ opencv-devel \ opencv-python \ osmosis \ - java-1.8.0-openjdk \ + java-${JDK_VERSION}-openjdk \ perl-XML-LibXML \ parallel \ postgresql${POSTGRESQL_VERSION_DOTLESS} \ diff --git a/VagrantProvisionVars.sh b/VagrantProvisionVars.sh index 43f950ab8d..03d18a9048 100644 --- a/VagrantProvisionVars.sh +++ b/VagrantProvisionVars.sh @@ -1,9 +1,6 @@ #!/usr/bin/env bash -export JDK_VERSION=1.8.0_161 -export JDK_URL=http://download.oracle.com/otn-pub/java/jdk/8u161-b12/2f38c3b165be4555a1fa6e98c45e0808/jdk-8u161-linux-x64.tar.gz -export JDK_TAR=jdk-8u161-linux-x64.tar.gz -export JDK_MD5=99051574a0d90871ed24a91a5d321ed2 +export JDK_VERSION=1.8.0 # Hoot deps library versions export GLPK_VERSION=4.64 @@ -27,10 +24,5 @@ export PROJ_VERSION=7.2.1 export POSTGRESQL_VERSION=13 export POSTGRESQL_VERSION_DOTLESS="$(echo "$POSTGRESQL_VERSION" | awk '{ gsub(/\./, ""); print substr($0, 1, 2) }')" -# FGDB 1.5 is required to compile using g++ >= 5.1 -# https://trac.osgeo.org/gdal/wiki/FileGDB#HowtodealwithGCC5.1C11ABIonLinux -export FGDB_VERSION=1.5.1 -export FGDB_URL=https://github.com/Esri/file-geodatabase-api/raw/master/FileGDB_API_${FGDB_VERSION}/ - # Devtoolset export DEVTOOLSET_VERSION=8 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000000..161d10a903 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,74 @@ +version: "3.7" +x-args: &default-args + hoot_uid: "${HOOT_UID}" + hoot_gid: "${HOOT_GID}" + hoot_user: "${HOOT_USER}" + hoot_group: "${HOOT_GROUP}" + hoot_home: "${HOOT_HOME}" + postgresql_version: "${POSTGRESQL_VERSION}" + glpk_version: "${GLPK_VERSION}" + liboauthcpp_version: "${LIBOAUTHCPP_VERSION}" + libphonenumber_version: "${LIBPHONENUMBER_VERSION}" + libpostal_version: "${LIBPOSTAL_VERSION}" + node_version: "${NODE_VERSION}" + npm_version: "${NPM_VERSION}" + v8_version: "${V8_VERSION}" + stxxl_version: "${STXXL_VERSION}" + armadillo_version: "${ARMADILLO_VERSION}" + gdal_version: "${GDAL_VERSION}" + geos_version: "${GEOS_VERSION}" + libgeotiff_version: "${LIBGEOTIFF_VERSION}" + proj_version: "${PROJ_VERSION}" + devtoolset_version: "${DEVTOOLSET_VERSION}" + stxxl_version: "${STXXL_VERSION}" + jdk_version: "${JDK_VERSION}" +services: + frontend: + build: + context: . + dockerfile: docker/Dockerfile.frontend + args: *default-args + hostname: frontend + ports: + - 8080:8080 + env_file: + - hoot.env + volumes: + - ./hoot-ui-2x:${HOOT_HOME}/hoot-ui-2x:rw + - ./.git/modules/hoot-ui-2x:${HOOT_HOME}/.git/modules/hoot-ui-2x:rw + postgres: + build: + context: . + dockerfile: docker/Dockerfile.postgres + args: *default-args + hostname: postgres + env_file: + - hoot.env + volumes: + - postgres-data:/var/lib/pgsql/${POSTGRESQL_VERSION}/data:rw + core-services: + depends_on: + postgres: + condition: service_healthy + frontend: + condition: service_healthy + build: + context: . + dockerfile: docker/Dockerfile.core-services + args: *default-args + hostname: core-services + env_file: + - hoot.env + ports: + - 8888:8080 + - 8094:8094 + - 8096:8096 + - 8101:8101 + - 8000:8585 + volumes: + - ./:${HOOT_HOME}:rw + - tomcat8-hoot-services:/var/lib/tomcat8/webapps/hoot-services:rw + +volumes: + postgres-data: + tomcat8-hoot-services: diff --git a/docker/Dockerfile.core-services b/docker/Dockerfile.core-services new file mode 100644 index 0000000000..d12e046854 --- /dev/null +++ b/docker/Dockerfile.core-services @@ -0,0 +1,196 @@ +FROM centos:7 + +ARG hoot_user +ARG hoot_uid +ARG hoot_gid +ARG hoot_group +ARG hoot_shell=/bin/bash +ARG hoot_home +ARG glpk_version +ARG liboauthcpp_version +ARG libphonenumber_version +ARG libpostal_version +ARG node_version +ARG npm_version +ARG v8_version +ARG stxxl_version +ARG armadillo_version +ARG gdal_version +ARG geos_version +ARG libgeotiff_version +ARG proj_version +ARG devtoolset_version +ARG stxxl_version +ARG postgresql_version +ARG jdk_version + + +ENV HOOT_HOME=$hoot_home + +COPY scripts/yum/geoint-repo.sh scripts/yum/hoot-repo.sh scripts/yum/pgdg-repo.sh /tmp + +RUN --mount=type=cache,target=/var/cache/yum \ + /tmp/geoint-repo.sh && \ + /tmp/hoot-repo.sh && \ + /tmp/pgdg-repo.sh $postgresql_version && \ + yum-config-manager --save \ + --setopt=base.repo_gpgcheck=1 \ + --setopt=extras.repo_gpgcheck=1 \ + --setopt=updates.repo_gpgcheck=1 &> /dev/null && \ + yum -y install epel-release centos-release-scl + +RUN yum install -y \ + armadillo-$armadillo_version \ + devtoolset-$devtoolset_version \ + devtoolset-$devtoolset_version-libasan-devel \ + geos-$geos_version \ + geos-devel-$geos_version \ + glpk-$glpk_version \ + glpk-devel-$glpk_version \ + gdal-$gdal_version \ + gdal-devel-$gdal_version \ + gdal-python-tools-$gdal_version \ + google-chrome-stable-$google_chrome_version \ + libgeotiff-$libgeotiff_version \ + libgeotiff-devel-$libgeotiff_version \ + liboauthcpp-$liboauthcpp_version \ + liboauthcpp-devel-$liboauthcpp_version \ + libphonenumber-$libphonenumber_version \ + libphonenumber-devel-$libphonenumber_version \ + libpostal-$libpostal_version \ + libpostal-data-$libpostal_version \ + libpostal-devel-$libpostal_version \ + npm-$npm_version \ + nodejs-$node_version \ + nodejs-devel-$node_version \ + nodejs-docs-$node_version \ + nodejs-libs-$node_version \ + postgresql$postgresql_version \ + postgresql$postgresql_version-contrib \ + postgresql$postgresql_version-server \ + proj-$proj_version \ + proj-devel-$proj_version \ + stxxl-$stxxl_version \ + stxxl-devel-$stxxl_version \ + v8-devel-$v8_version + +RUN yum -y install \ + asciidoc \ + autoconf \ + autoconf-archive \ + automake \ + bison \ + boost-devel \ + bzip2 \ + ccache \ + cmake \ + cppunit-devel \ + dblatex \ + doxygen \ + gcc-c++ \ + git \ + git-core \ + gnuplot \ + lcov \ + libffi-devel \ + libicu-devel \ + libpng-devel \ + libtool \ + liquibase \ + maven \ + m4 \ + mlocate \ + opencv \ + opencv-core \ + opencv-devel \ + opencv-python \ + osmosis \ + java-$jdk_version-openjdk \ + perl-XML-LibXML \ + parallel \ + make \ + protobuf \ + protobuf-compiler \ + protobuf-devel \ + python \ + python-devel \ + python3 \ + python3-devel \ + python3-matplotlib \ + python3-pip \ + python3-setuptools \ + qt5-qtbase \ + qt5-qtbase-devel \ + qt5-qtbase-postgresql \ + qt5-qtwebkit \ + qt5-qtwebkit-devel \ + readline-devel \ + redhat-lsb-core \ + sqlite-devel \ + swig \ + tex-fonts-hebrew \ + texlive \ + texlive-collection-fontsrecommended \ + texlive-collection-langcyrillic \ + tomcat8 \ + unzip \ + vim \ + wamerican-insane \ + w3m \ + wget \ + words \ + xorg-x11-server-Xvfb \ + zip + +RUN yum install -y postgresql$postgresql_version-devel && \ + alternatives --install /usr/bin/pg_config pgsql-pg_config /usr/pgsql-$postgresql_version/bin/pg_config 500 + +RUN groupadd --non-unique -g ${hoot_gid} ${hoot_group} && \ + useradd -d ${hoot_home} -m -s ${hoot_shell} -u ${hoot_uid} -g ${hoot_gid} ${hoot_user} && \ + chmod 0755 ${hoot_home} + +ARG tomcat_config=/etc/tomcat8 +ARG tomcat_logs=/var/log/tomcat8 + +# vars needed for core entrypoint +ENV DEVTOOLSET_VERSION=$devtoolset_version +ENV HOOT_USER=$hoot_user +ENV PG_MAJOR_VERSION=$postgresql_version +ENV PATH=$PATH:/usr/pgsql-$postgresql_version/bin +ENV MANPATH=/opt/rh/devtoolset-$DEVTOOLSET_VERSION/root/usr/share/ +ENV TOMCAT_CONFIG=$tomcat_config +ENV TOMCAT8_HOME=/var/lib/tomcat8 +ENV TOMCAT_SERVER=$tomcat_config/server.xml + +# services configuration files. +RUN echo -e '\ +export GDAL_DATA=$GDAL_DATA\n\ +export HOOT_HOME=$HOOT_HOME\n\ +export HOOT_WORKING_NAME=hootenanny\n\ +export JAVA_HOME=$JRE_HOME\n\ +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:$JAVA_HOME/lib/amd64/server:$HOOT_HOME/lib\n\ +export PATH=$HOOT_HOME/bin:$PATH\n\ +export JAVA_OPTS="${JAVA_OPTS} -Xdebug -Xrunjdwp:transport=dt_socket,address=8585,server=y,suspend=n"\n'\ +>> $tomcat_config/conf.d/hoot.conf + +RUN install -d -m 0775 $tomcat_logs +RUN echo '\n\ +Please login to the host to view the logs:\n\ + sudo journalctl -u tomcat8\n\ +EOF\n'\ >> $tomcat_logs/catalina.out + +RUN chown $hoot_user:$hoot_user $tomcat_logs/catalina.out + +COPY docker/scripts/tomcat_configure.sh /tmp +RUN /tmp/tomcat_configure.sh && rm -f /tmp/tomcat_configure.sh + +# let hoot user execute tomcat8 commands +RUN usermod -a -G tomcat $hoot_user + +COPY docker/scripts/core-services-entrypoint.sh /docker-entrypoint.sh +COPY docker/scripts/core-services-healthcheck.sh /docker-healthcheck.sh + +WORKDIR ${hoot_home} +USER ${hoot_user} +ENTRYPOINT /docker-entrypoint.sh +HEALTHCHECK --interval=15s --start-period=1m CMD ["/docker-healthcheck.sh"] diff --git a/docker/Dockerfile.frontend b/docker/Dockerfile.frontend new file mode 100644 index 0000000000..3554dccc4b --- /dev/null +++ b/docker/Dockerfile.frontend @@ -0,0 +1,42 @@ +FROM centos:7 + +ARG hoot_user +ARG hoot_uid +ARG hoot_gid +ARG hoot_group +ARG hoot_shell=/bin/bash +ARG hoot_home +ARG node_version + +ENV HOOT_HOME=$hoot_home + +COPY scripts/node/nodesource-repo.sh scripts/yum/hoot-repo.sh /tmp/ + +RUN --mount=type=cache,target=/var/cache/yum \ + /tmp/nodesource-repo.sh $node_version && \ + /tmp/hoot-repo.sh && \ + yum-config-manager --save \ + --setopt=base.repo_gpgcheck=1 \ + --setopt=extras.repo_gpgcheck=1 \ + --setopt=updates.repo_gpgcheck=1 &> /dev/null && \ + yum makecache -y && \ + yum -q -y install epel-release && \ + yum -y install \ + bzip2 \ + git \ + git-core \ + nodejs-devel yarn \ + google-chrome-stable-91.0.4472.114 && \ + yum -q -y clean all && rm -f /tmp*.sh + +RUN groupadd --non-unique -g ${hoot_gid} ${hoot_group} && \ + useradd -d ${hoot_home} -m -s ${hoot_shell} -u ${hoot_uid} -g ${hoot_gid} ${hoot_user} && \ + chmod 0755 ${hoot_home} + +COPY docker/scripts/frontend-entrypoint.sh /docker-entrypoint.sh +COPY docker/scripts/frontend-healthcheck.sh /docker-healthcheck.sh + +USER ${hoot_user} +WORKDIR ${hoot_home}/hoot-ui-2x +ENTRYPOINT /docker-entrypoint.sh +HEALTHCHECK --interval=15s --start-period=1m CMD ["/docker-healthcheck.sh"] diff --git a/docker/Dockerfile.postgres b/docker/Dockerfile.postgres new file mode 100644 index 0000000000..7e882d969a --- /dev/null +++ b/docker/Dockerfile.postgres @@ -0,0 +1,35 @@ +FROM centos:7 + +ARG postgresql_version +ARG pgdata=/var/lib/pgsql/$postgresql_version/data + +ENV LANG=en_US.UTF-8 \ + LC_ALL=en_US.UTF-8 \ + PGDATA=${pgdata} + +COPY scripts/yum/geoint-repo.sh \ + scripts/yum/pgdg-repo.sh \ + docker/scripts/postgres-install.sh \ + /tmp/ + +RUN --mount=type=cache,target=/var/cache/yum \ + /tmp/geoint-repo.sh && \ + /tmp/pgdg-repo.sh ${postgresql_version} && \ + yum-config-manager --save \ + --setopt=base.repo_gpgcheck=1 \ + --setopt=extras.repo_gpgcheck=1 \ + --setopt=updates.repo_gpgcheck=1 &> /dev/null && \ + yum -q -y install epel-release + +RUN /tmp/postgres-install.sh $postgresql_version && rm -f /tmp/*.sh + +COPY docker/scripts/postgres-entrypoint.sh /docker-entrypoint.sh +COPY docker/scripts/postgres-healthcheck.sh /docker-healthcheck.sh + +# Set up data volume, entrypoint, etc. +USER postgres +ENTRYPOINT ["/docker-entrypoint.sh"] +EXPOSE 5432/tcp +HEALTHCHECK --interval=15s CMD ["/docker-healthcheck.sh"] +CMD ["postgres"] +VOLUME ${pgdata} diff --git a/docker/scripts/DatabaseConfigServicesTranslationsLocal.sh b/docker/scripts/DatabaseConfigServicesTranslationsLocal.sh new file mode 100755 index 0000000000..f839c7dda9 --- /dev/null +++ b/docker/scripts/DatabaseConfigServicesTranslationsLocal.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +export DB_HOST=postgres diff --git a/docker/scripts/core-services-configure.sh b/docker/scripts/core-services-configure.sh new file mode 100755 index 0000000000..e9eaae9877 --- /dev/null +++ b/docker/scripts/core-services-configure.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -euo pipefail + +cp LocalConfig.pri.orig LocalConfig.pri + +set +u +source SetupEnv.sh +set -u + +source conf/database/DatabaseConfig.sh + +echo "JAVA_HOME=$JAVA_HOME" +echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" + +aclocal +autoconf +autoheader +automake --add-missing --copy +./configure --with-services diff --git a/docker/scripts/core-services-entrypoint.sh b/docker/scripts/core-services-entrypoint.sh new file mode 100755 index 0000000000..1f7ddcc671 --- /dev/null +++ b/docker/scripts/core-services-entrypoint.sh @@ -0,0 +1,65 @@ +#!/bin/bash +set -euo pipefail + +. /opt/rh/devtoolset-$DEVTOOLSET_VERSION/enable + + +source ./SetupEnv.sh +source conf/database/DatabaseConfig.sh + +start_tomcat() +{ + . /etc/tomcat8/tomcat8.conf + . /etc/sysconfig/tomcat8 + NAME= /usr/libexec/tomcat8/server start +} + +stop_tomcat() +{ + /usr/libexec/tomcat8/server stop +} + +copy_war_to_tomcat() +{ + rm -rf $TOMCAT8_HOME/webapps/hoot-services/* + cp -R hoot-services/target/hoot-services-$HOOT_USER/* $TOMCAT8_HOME/webapps/hoot-services +} + +touch core-services-building.txt + +if [ "${HOOT_CLEAN:-0}" = "1" ]; then + ./docker/scripts/core-services-configure.sh + make clean +fi; + +if [ "${HOOT_BUILD_CORE:-0}" = "1" ] || [ ! -f ./bin/hoot.bin ]; then + ./docker/scripts/core-services-configure.sh + make core -j$(nproc) + copy_war_to_tomcat +fi + +if [ "${HOOT_BUILD_HOOT_SERVICES:-0}" = "1" ] || [ ! -f hoot-services/target/hoot-services-$HOOT_USER.war ]; then + make services-build + copy_war_to_tomcat +fi; + +# translation server is tied to services so need to build schema files and its dependencies +if [ "${HOOT_BUILD_JS_SCHEMA:-0}" = "1" ] || [ ! -f translations/tds71_schema.js ]; then + make -f Makefile.hoot js-make +fi + +if [ "${HOOT_BUILD_TRANSLATION_SERVER:-0}" = "1" ] || [ ! -d translations/node_modules ]; then + make -f Makefile.hoot translations-test +fi; + +if [ "${HOOT_BUILD_NODE_EXPORT_SERVER:-0}" = "1" ] || [ ! -d node-export-server/node_modules ]; then + pushd node-export-server; + npm install; + popd +fi; + +export GDAL_DATA=$(gdal-config --datadir) + +rm -f core-services-building.txt +npm start --prefix "${HOOT_HOME}/node-export-server" & +start_tomcat diff --git a/docker/scripts/core-services-healthcheck.sh b/docker/scripts/core-services-healthcheck.sh new file mode 100755 index 0000000000..348d3000b9 --- /dev/null +++ b/docker/scripts/core-services-healthcheck.sh @@ -0,0 +1,33 @@ +#!/bin/sh +set -eu + +if [ -f core-services-building.txt ]; then + sleep 5 +else + . ./SetupEnv.sh && hoot version && \ + /usr/bin/curl \ + --fail \ + --output /dev/null \ + --silent \ + --user-agent docker-healthcheck \ + "http://0.0.0.0:8080/hoot-services/auth/oauth2/authorize" && \ + /usr/bin/curl \ + --request OPTIONS \ + --fail \ + --output /dev/null \ + --silent \ + --user-agent docker-healthcheck \ + "http://0.0.0.0:8096/" && \ + /usr/bin/curl \ + --fail \ + --output /dev/null \ + --silent \ + --user-agent docker-healthcheck \ + "http://0.0.0.0:8094/translations" && \ + /usr/bin/curl \ + --fail \ + --output /dev/null \ + --silent \ + --user-agent docker-healthcheck \ + "http://0.0.0.0:8101/options" +fi \ No newline at end of file diff --git a/docker/scripts/frontend-entrypoint.sh b/docker/scripts/frontend-entrypoint.sh new file mode 100755 index 0000000000..af1570f7e7 --- /dev/null +++ b/docker/scripts/frontend-entrypoint.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -euo pipefail + +if [ "${HOOT_NPM_INSTALL:-0}" == "1" ] || [ ! -d node_modules ]; then + npm install && npm run all +fi + +npm start diff --git a/docker/scripts/frontend-healthcheck.sh b/docker/scripts/frontend-healthcheck.sh new file mode 100755 index 0000000000..3e18baead7 --- /dev/null +++ b/docker/scripts/frontend-healthcheck.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu +/usr/bin/curl \ + --insecure \ + --fail \ + --output /dev/null \ + --silent \ + --user-agent docker-healthcheck \ + "https://0.0.0.0:8080/globals.js" # use /globals.js to curcumvent the 304 behavior of non-authorized requests to / diff --git a/docker/scripts/postgres-entrypoint.sh b/docker/scripts/postgres-entrypoint.sh new file mode 100755 index 0000000000..d38dfec3c1 --- /dev/null +++ b/docker/scripts/postgres-entrypoint.sh @@ -0,0 +1,32 @@ +#!/bin/bash +set -eu + +update_postgres_setting () { + local pg_setting="$1" + local pg_value="$2" + if grep -q -e "^${pg_setting}\s\+\?=" "${PGDATA}/postgresql.conf"; then + # Modify existing setting. + sed -i -e "s#^${pg_setting}\s\+\?=\s\+\?.\+#${pg_setting} = ${pg_value}#g" \ + "${PGDATA}/postgresql.conf" + else + # New setting. + echo "${pg_setting} = ${pg_value}" >> "${PGDATA}/postgresql.conf" + fi +} + +# Customize any PostgreSQL settings from POSTGRES_SETTINGS environment. +if [ -n "${POSTGRES_SETTINGS:-}" ]; then + read -d '#' -a POSTGRES_SETTINGS_ARR <<< "$POSTGRES_SETTINGS#" + for pg_setting_pair in "${POSTGRES_SETTINGS_ARR[@]}"; do + pg_setting="$(echo "${pg_setting_pair}" | awk -F= '{ print $1 }')" + pg_value="$(echo "${pg_setting_pair}" | awk -F= '{ print $2 }')" + update_postgres_setting "${pg_setting}" "${pg_value}" + done +fi + +# ensure first argument is `postgres`. +if [ "${1#-}" != "$1" ]; then + set -- postgres "$@" +fi + +exec "$@" diff --git a/docker/scripts/postgres-healthcheck.sh b/docker/scripts/postgres-healthcheck.sh new file mode 100755 index 0000000000..c7527ea47b --- /dev/null +++ b/docker/scripts/postgres-healthcheck.sh @@ -0,0 +1,3 @@ +#!/bin/sh +set -eu +/usr/bin/pg_isready -h "${PGHOST:-0.0.0.0}" diff --git a/docker/scripts/postgres-install.sh b/docker/scripts/postgres-install.sh new file mode 100755 index 0000000000..da15886177 --- /dev/null +++ b/docker/scripts/postgres-install.sh @@ -0,0 +1,65 @@ +#!/bin/bash +set -eu + +POSTGRES_VERSION="${1}" + +yum -q -y install \ + postgresql${POSTGRES_VERSION} \ + postgresql${POSTGRES_VERSION}-contrib \ + postgresql${POSTGRES_VERSION}-server + +alternatives --install /usr/bin/createdb pgsql-createdb "/usr/pgsql-${POSTGRES_VERSION}/bin/createdb" 500 +alternatives --install /usr/bin/createuser pgsql-createuser "/usr/pgsql-${POSTGRES_VERSION}/bin/createuser" 500 +alternatives --install /usr/bin/initdb pgsql-initdb "/usr/pgsql-${POSTGRES_VERSION}/bin/initdb" 500 +alternatives --install /usr/bin/pg_config pgsql-pg_config "/usr/pgsql-${POSTGRES_VERSION}/bin/pg_config" 500 +alternatives --install /usr/bin/pg_ctl pgsql-pg_ctl "/usr/pgsql-${POSTGRES_VERSION}/bin/pg_ctl" 500 +alternatives --install /usr/bin/pg_isready pgsql-pg_isready "/usr/pgsql-${POSTGRES_VERSION}/bin/pg_isready" 500 +alternatives --install /usr/bin/postgres pgsql-postgres "/usr/pgsql-${POSTGRES_VERSION}/bin/postgres" 500 + +# init and start postgres +if [ ! -e /var/lib/pgsql/$POSTGRES_VERSION/data/PG_VERSION ]; then + PGSETUP_INITDB_OPTIONS="-E UTF-8 --lc-collate=en_US.UTF-8 --lc-ctype=en_US.UTF-8" \ + su postgres -c "initdb -D $PGDATA --locale $LANG" +fi + +su postgres -c "pg_ctl -D $PGDATA -s -w start" + + +while ! /usr/pgsql-$POSTGRES_VERSION/bin/pg_isready; do + echo "Waiting for postgres to start" + sleep 1 +done + +# create Hoot services db +if ! psql -lqt | cut -d \| -f 1 | grep -qw hoot; then + su postgres -c "createuser --superuser hoot || true"; + su postgres -c "psql -c \"ALTER USER hoot WITH PASSWORD 'hoottest';\"" + su postgres -c "createdb hoot --owner=hoot" + su postgres -c "psql -d hoot -c \"CREATE EXTENSION hstore;\"" +fi + +su postgres -c "pg_ctl -D $PGDATA -s -m fast -w stop" + +# configure Postgres settings +PG_HB_CONF=/var/lib/pgsql/$POSTGRES_VERSION/data/pg_hba.conf +if ! grep -i --quiet hoot $PG_HB_CONF; then + sed -i '1ihost all all 0.0.0.0/0 md5' $PG_HB_CONF +fi +POSTGRES_CONF=/var/lib/pgsql/$POSTGRES_VERSION/data/postgresql.conf +if ! grep -i --quiet HOOT $POSTGRES_CONF; then + sed -i "s/^max_connections/\#max_connections/" $POSTGRES_CONF + sed -i "s/^shared_buffers/\#shared_buffers/" $POSTGRES_CONF + cat >> $POSTGRES_CONF <@ \n &@' $TOMCAT_SERVER +fi + +# OSM OAuth2 requires redirect URLs to use https +# Generate a keystore with a self-signed cert for localhost +if [ ! -f $TOMCAT_CONFIG/localhost-rsa.jks ]; then + keytool -genkey -keyalg RSA -noprompt -alias tomcat -dname "CN=localhost, OU=NA, O=NA, L=NA, S=NA, C=NA" -keystore $TOMCAT_CONFIG/localhost-rsa.jks -validity 9999 -storepass changeme -keypass changeme + chown root:tomcat $TOMCAT_CONFIG/localhost-rsa.jks +fi + +# Enable SSL using self-signed cert +if ! grep -i --quiet 'certificateKeyAlias="tomcat"' ${TOMCAT_CONFIG}/server.xml; then + sed -i.bak 's@@\n \n \n \n \n\n &@' ${TOMCAT_CONFIG}/server.xml +fi + +mkdir -p $TOMCAT8_HOME/webapps/hoot-services +chown -R $HOOT_USER:tomcat $TOMCAT8_HOME/webapps/hoot-services/ + diff --git a/hoot-ui-2x b/hoot-ui-2x index eb2e97aaf1..804b1502f5 160000 --- a/hoot-ui-2x +++ b/hoot-ui-2x @@ -1 +1 @@ -Subproject commit eb2e97aaf1d16766399a111ae7b9f54065901f39 +Subproject commit 804b1502f5478ca467c7fb71d97eee64e30eaea3 diff --git a/hoot.env.example b/hoot.env.example new file mode 100644 index 0000000000..d407560d41 --- /dev/null +++ b/hoot.env.example @@ -0,0 +1,9 @@ +HOOT_CLEAN=0 +HOOT_BUILD_CORE=0 +HOOT_BUILD_HOOT_SERVICES=0 +HOOT_BUILD_JS_SCHEMA=0 +HOOT_BUILD_TRANSLATION_SERVER=0 +HOOT_BUILD_NODE_EXPORT_SERVER=0 +HOOT_SERVICES_HOST=core-services +HOOT_SERVICES_PORT=8080 +HOOT_NPM_INSTALL=0