From 2e9788642b10a573102285d81ce27b93cd1380b3 Mon Sep 17 00:00:00 2001 From: Vidar Date: Wed, 8 Aug 2018 10:54:44 +0200 Subject: [PATCH] EZEE-1903: Added bin/generate-solr-config.sh (#120) * Added bin/generate-solr-config.sh * Adjust generator script to be usable for travis and safe for BSD use * [Travis] Bump Solr 6.6 usage to 6.6.5 * Update generate-solr-config.sh * Update init_solr.sh * Update generate-solr-config.sh * Fix usage of backticks --- .travis.yml | 2 +- README.md | 12 +++- bin/.travis/init_solr.sh | 31 ++------- bin/generate-solr-config.sh | 134 ++++++++++++++++++++++++++++++++++++ 4 files changed, 151 insertions(+), 28 deletions(-) create mode 100755 bin/generate-solr-config.sh diff --git a/.travis.yml b/.travis.yml index 2ba59bb03..cd7ffd226 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,7 @@ matrix: - php: 7.2 env: TEST_CONFIG="phpunit-integration-legacy-solr.xml" SOLR_VERSION="6.5.1" CORES_SETUP="shared" - php: 5.6 - env: TEST_CONFIG="phpunit-integration-legacy-solr.xml" SOLR_VERSION="6.6.0" CORES_SETUP="single" SOLR_CORES="collection1" + env: TEST_CONFIG="phpunit-integration-legacy-solr.xml" SOLR_VERSION="6.6.5" CORES_SETUP="single" SOLR_CORES="collection1" # test only master and stable branches (+ Pull requests against those) branches: diff --git a/README.md b/README.md index df19fa54c..e2c58ad78 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ For Contributing to this Bundle, you should make sure to run both unit and integ ## Modify solrconfig.xml to remove section that doesn't agree with our schema sed -i.bak '//,/<\/updateRequestProcessorChain>/d' server/ez/template/solrconfig.xml ``` - + ###### For use in production/dev Note that Solr Bundle does not commit changes directly on repository updates, which can lead to issues of content not showing up in the index. You can control this by adjusting `autoSoftCommit` *(visibility @@ -91,6 +91,16 @@ For Contributing to this Bundle, you should make sure to run both unit and integ ${solr.autoSoftCommit.maxTime:20} + ###### TIP for automating generating of config + You may also use the command line tool `bin/generate-solr-config.sh` to generate the Solr 6 configuration instead of these instructions. + This is particular convenient if deploying to eZ Platform Cloud (Platform.sh), but can also be used for on-premise installs. + + The script should be executed from the eZ Platform root directory, run the following for more info: + + ```bash + ./vendor/ezsystems/ezplatform-solr-search-engine/bin/generate-solr-config.sh --help + ``` + 4. Start Solr diff --git a/bin/.travis/init_solr.sh b/bin/.travis/init_solr.sh index 94ea53f9c..4dccf08d2 100755 --- a/bin/.travis/init_solr.sh +++ b/bin/.travis/init_solr.sh @@ -10,7 +10,7 @@ default_cores[2]='core2' default_cores[3]='core3' SOLR_PORT=${SOLR_PORT:-8983} -SOLR_VERSION=${SOLR_VERSION:-6.6.0} +SOLR_VERSION=${SOLR_VERSION:-6.6.5} SOLR_DEBUG=${SOLR_DEBUG:-false} SOLR_HOME=${SOLR_HOME:-ez} SOLR_CONFIG=${SOLR_CONFIG:-${default_config_files[*]}} @@ -21,7 +21,7 @@ SOLR_INSTALL_DIR="${SOLR_DIR}/${SOLR_VERSION}" download() { case ${SOLR_VERSION} in # PS!!: Append versions and don't remove old once, kernel uses this script! - 4.10.4|6.3.0|6.4.1|6.4.2|6.5.1|6.6.0 ) + 4.10.4|6.3.0|6.4.1|6.4.2|6.5.1|6.6.0|6.6.5 ) url="http://archive.apache.org/dist/lucene/solr/${SOLR_VERSION}/solr-${SOLR_VERSION}.tgz" ;; *) @@ -162,29 +162,6 @@ solr4_run() { echo 'Started' } -# Configure for Solr 6, see solr4_configure() for 4.10 -configure() { - home_dir="${SOLR_INSTALL_DIR}/server/${SOLR_HOME}" - template_dir="${home_dir}/template" - config_dir="${SOLR_INSTALL_DIR}/server/solr/configsets/basic_configs/conf" - - create_dir ${home_dir} - create_dir ${template_dir} - - files=${SOLR_CONFIG} - files+=("${config_dir}/currency.xml") - files+=("${config_dir}/solrconfig.xml") - files+=("${config_dir}/stopwords.txt") - files+=("${config_dir}/synonyms.txt") - files+=("${config_dir}/elevate.xml") - - copy_files ${template_dir} "${files[*]}" - copy_file "${SOLR_INSTALL_DIR}/server/solr/solr.xml" ${home_dir} - - # modify solrconfig.xml to remove section that doesn't agree with our schema - sed -i.bak '//,/<\/updateRequestProcessorChain>/d' "${template_dir}/solrconfig.xml" -} - # Run for Solr 6, see solr4_run() for 4.10 run() { echo "Running with version ${SOLR_VERSION} in standalone mode" @@ -221,7 +198,9 @@ create_core() { download if [[ ${SOLR_VERSION} == 6* ]] ; then - configure + ./bin/generate-solr-config.sh \ + --solr-install-dir="${SOLR_INSTALL_DIR}" \ + --destination-dir="${SOLR_INSTALL_DIR}/server/${SOLR_HOME}/template" run else solr4_configure diff --git a/bin/generate-solr-config.sh b/bin/generate-solr-config.sh new file mode 100755 index 000000000..edbf7c8b2 --- /dev/null +++ b/bin/generate-solr-config.sh @@ -0,0 +1,134 @@ +#!/usr/bin/env bash + +set -e + +# Default paramters, if not overloaded by user arguments +DESTINATION_DIR=.platform/configsets/solr6/conf +SOLR_VERSION=6.6.5 +FORCE=false +SOLR_INSTALL_DIR="" + +show_help() { + cat << EOF +Script for generting solr config +This config can be used to configure solr on eZ Platform Cloud (Platform.sh) or elsewhere. +The script should be executed from the eZ Platform root directory. + +Help (this text): +./vendor/ezsystems/ezplatform-solr-search-engine/bin/generate-solr-config.sh --help + +Usage with eZ Platform Cloud (arguments here can be skipped as they have default values): +./vendor/ezsystems/ezplatform-solr-search-engine/bin/generate-solr-config.sh \\ + --destination-dir=.platform/configsets/solr6/conf \\ + --solr-version=6.6.5 + +Usage with on-premise version of Solr: +./vendor/ezsystems/ezplatform-solr-search-engine/bin/generate-solr-config.sh \\ + --destination-dir=/opt/solr/server/ez/template \\ + --solr-install-dir=/opt/solr + +Warning: + This script only supports Solr 6 and higher !! + + +Arguments: + [--destination-dir=] : Location where solr config should be stored + Default value is .platform/configsets/solr6/conf + [-f|--force] : Overwrite destination-dir if it already exists + [--solr-install-dir] : Existing downloaded Solr install to copy base config from. + [--solr-version] : Solr version to download & copy base config from, used only if --solr-install-dir is unset + [-h|--help] : Help text (this text) +EOF +} + +realpath() { + [[ $1 = /* ]] && echo "$1" || echo "$PWD/${1#./}" +} + + +EZ_SCRIPT=`realpath $0` +EZ_BUNDLE_PATH="`dirname $EZ_SCRIPT`/.." + +## Parse arguments +for i in "$@"; do + case $i in + --destination-dir=*) + DESTINATION_DIR="${i#*=}" + ;; + -f|--force) + FORCE=true + ;; + --solr-version=*) + SOLR_VERSION="${i#*=}" + ;; + --solr-install-dir=*) + SOLR_INSTALL_DIR="${i#*=}" + SOLR_INSTALL_DIR="${SOLR_INSTALL_DIR/#\~/$HOME}" + ;; + -h|--help) + show_help + exit 0 + ;; + *) + show_help "${i}" + exit 1 + ;; + esac +done + + +if [ `whoami` == "root" ]; then + echo "Error : Do not run this script as root" + exit 1 +fi + +if [ -e $DESTINATION_DIR ]; then + if [ "$FORCE" == "true" ]; then + echo -e "\033[0;31mDestination directory ($DESTINATION_DIR) already exists, removing in 5 seconds.... \033[0m" + sleep 5 + rm -Rf $DESTINATION_DIR + else + echo -e "\033[1;31mError: Destination dir already exists ($DESTINATION_DIR). Use -f parameter to force \033[0m" + exit 1 + fi +fi + +if [ "$SOLR_INSTALL_DIR" == "" ]; then + # If we where not provided existing install directory we'll temporary download version of solr 6 to generate config. + GENERATE_SOLR_TMPDIR=`mktemp -d` + echo "Downloading solr bundle:" + curl http://archive.apache.org/dist/lucene/solr/${SOLR_VERSION}/solr-${SOLR_VERSION}.tgz > $GENERATE_SOLR_TMPDIR/solr-${SOLR_VERSION}.tgz + + echo "Untaring" + cd $GENERATE_SOLR_TMPDIR + tar -xzf solr-${SOLR_VERSION}.tgz + cd - > /dev/null 2>&1 + echo "done extracting Solr" + SOLR_INSTALL_DIR="${GENERATE_SOLR_TMPDIR}/solr-${SOLR_VERSION}" +fi + +mkdir -p $DESTINATION_DIR +cp -a ${EZ_BUNDLE_PATH}/lib/Resources/config/solr/* $DESTINATION_DIR +cp ${SOLR_INSTALL_DIR}/server/solr/configsets/basic_configs/conf/{currency.xml,solrconfig.xml,stopwords.txt,synonyms.txt,elevate.xml} $DESTINATION_DIR + +if [[ ! $DESTINATION_DIR =~ ^\.platform ]]; then + # If we are not targeting .platform(.sh) config, we also output default solr.xml + cp -f ${SOLR_INSTALL_DIR}/server/solr/solr.xml $DESTINATION_DIR/.. +else + echo "NOTE: Skipped copying ${SOLR_INSTALL_DIR}/server/solr/solr.xml given destination dir is a '.platform/' config folder" +fi + +# Adapt autoSoftCommit to have a recommended value, and remove add-unknown-fields-to-the-schema +sed -i.bak '//,/<\/updateRequestProcessorChain>/d' $DESTINATION_DIR/solrconfig.xml +sed -i.bak2 's/${solr.autoSoftCommit.maxTime:-1}/${solr.autoSoftCommit.maxTime:20}/' $DESTINATION_DIR/solrconfig.xml + +if [ "$GENERATE_SOLR_TMPDIR" != "" ]; then + echo Removing temp dir: $GENERATE_SOLR_TMPDIR + rm -Rf ${GENERATE_SOLR_TMPDIR} +fi + +echo -e "\033[0;32mDone generating config to $DESTINATION_DIR ! \033[0m" + +if [[ $DESTINATION_DIR =~ ^\.platform ]]; then + echo "NOTE: You also need to enable solr service in '.platform.app.yaml' and '.platform/services.yaml'." +fi