Version for ES 8.0.1

.github/workflows/maven.yml

@@ -9,9 +9,9 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
-    - name: Set up JDK 1.8
+    - name: Set up JDK 17
       uses: actions/setup-java@v1
       with:
-        java-version: 1.8
+        java-version: 17
     - name: Build with Maven
       run: mvn -B package --file pom.xml

pom.xml

@@ -3,7 +3,7 @@
 
     <groupId>org.querqy</groupId>
     <artifactId>querqy-elasticsearch</artifactId>
-    <version>1.6.es7172.0-SNAPSHOT</version>
+    <version>1.6.es800.0-SNAPSHOT</version>
     <packaging>jar</packaging>
 
     <prerequisites>
@@ -48,20 +48,21 @@
         <maven-surefire-plugin.version>2.22.2</maven-surefire-plugin.version>
 
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-        <java.version>1.8</java.version>
+        <java.version>16</java.version>
         <junit.version>4.12</junit.version>
         <hamcrest.version>2.1</hamcrest.version>
         <mockito.version>2.27.0</mockito.version>
         <spatial4j.version>0.7</spatial4j.version>
         <jts.version>1.15.0</jts.version>
 
-        <log4j.version>2.17.0</log4j.version>
-        <querqy.lucene.version>5.2.lucene810.0</querqy.lucene.version>
-        <es.major.version>7</es.major.version>
-        <es.minor.version>17</es.minor.version>
-        <es.bugfix.version>2</es.bugfix.version>
+        <querqy.lucene.version>5.4.lucene900.0</querqy.lucene.version>
+        <es.major.version>8</es.major.version>
+        <es.minor.version>0</es.minor.version>
+        <es.bugfix.version>1</es.bugfix.version>
         <elasticsearch.version>${es.major.version}.${es.minor.version}.${es.bugfix.version}</elasticsearch.version>
-        <jacoco.version>0.8.5</jacoco.version>
+        <jacoco.version>0.8.8</jacoco.version>
+        <maven.compiler.source>16</maven.compiler.source>
+        <maven.compiler.target>16</maven.compiler.target>
 
     </properties>
 
@@ -85,13 +86,6 @@
             <scope>provided</scope>
         </dependency>
 
-        <dependency>
-            <groupId>org.apache.logging.log4j</groupId>
-            <artifactId>log4j-core</artifactId>
-            <version>${log4j.version}</version>
-            <scope>provided</scope>
-        </dependency>
-
         <dependency>
             <groupId>org.elasticsearch.test</groupId>
             <artifactId>framework</artifactId>
@@ -192,8 +186,7 @@
                     <artifactId>maven-compiler-plugin</artifactId>
                     <version>${maven-compiler-plugin.version}</version>
                     <configuration>
-                        <source>${java.version}</source>
-                        <target>${java.version}</target>
+                        <release>${java.version}</release>
                         <showDeprecation>true</showDeprecation>
                     </configuration>
                 </plugin>

src/main/java/querqy/elasticsearch/DismaxSearchEngineRequestAdapter.java

@@ -204,6 +204,12 @@ public Optional<Float> getTiebreaker() {
         return queryBuilder.getTieBreaker();
     }
 
+    @Override
+    public Optional<Float> getMultiMatchTiebreaker() {
+        // TODO
+        return Optional.empty();
+    }
+
     /**
      * <p>Apply the 'minimum should match' setting of the request.</p>
      * <p>It will be the responsibility of the LuceneSearchEngineRequestAdapter implementation to derive the
@@ -293,7 +299,7 @@ public List<Query> getAdditiveBoosts(final QuerqyQuery<?> userQuery) throws Synt
 
             }
         }
-        return null;
+        return Collections.emptyList();
     }
 
     /**
@@ -308,7 +314,7 @@ public List<Query> getAdditiveBoosts(final QuerqyQuery<?> userQuery) throws Synt
      */
     @Override
     public List<Query> getMultiplicativeBoosts(final QuerqyQuery<?> userQuery) throws SyntaxException {
-        return null;
+        return Collections.emptyList();
     }
 
     @Override

src/main/java/querqy/elasticsearch/RewriterShardContext.java

@@ -100,7 +100,7 @@ public synchronized RewriterFactoryAndLogging loadFactory(final String rewriterI
             final GetResponse response;
 
             try {
-                response = client.prepareGet(".querqy", null, rewriterId).execute().get();
+                response = client.prepareGet(".querqy", rewriterId).execute().get();
             } catch (InterruptedException | ExecutionException e) {
                 throw new ElasticsearchException("Could not load rewriter " + rewriterId, e);
             }

src/main/java/querqy/elasticsearch/rewriter/SimpleCommonRulesRewriterFactory.java

@@ -7,6 +7,7 @@
 import querqy.rewrite.RewriterFactory;
 import querqy.rewrite.commonrules.QuerqyParserFactory;
 import querqy.rewrite.commonrules.WhiteSpaceQuerqyParserFactory;
+import querqy.rewrite.commonrules.model.BoostInstruction;
 import querqy.rewrite.commonrules.select.ExpressionCriteriaSelectionStrategyFactory;
 import querqy.rewrite.commonrules.select.SelectionStrategyFactory;
 
@@ -45,8 +46,8 @@ public void configure(final Map<String, Object> config) {
 
         try {
             delegate = new querqy.rewrite.commonrules.SimpleCommonRulesRewriterFactory(rewriterId,
-                    new StringReader(rules), allowBooleanInput, querqyParser, ignoreCase, selectionStrategyFactories,
-                    DEFAULT_SELECTION_STRATEGY_FACTORY, false);
+                    new StringReader(rules), allowBooleanInput, BoostInstruction.BoostMethod.ADDITIVE,
+                    querqyParser, ignoreCase, selectionStrategyFactories, DEFAULT_SELECTION_STRATEGY_FACTORY, false);
         } catch (final IOException e) {
             throw new ElasticsearchException(e);
         }
@@ -72,8 +73,8 @@ public List<String> validateConfiguration(final Map<String, Object> config) {
         final boolean allowBooleanInput = ConfigUtils.getArg(config, "allowBooleanInput", false);
         try {
             new querqy.rewrite.commonrules.SimpleCommonRulesRewriterFactory(rewriterId,
-                    new StringReader(rules), allowBooleanInput, querqyParser, ignoreCase, Collections.emptyMap(),
-                    DEFAULT_SELECTION_STRATEGY_FACTORY, false);
+                    new StringReader(rules), allowBooleanInput, BoostInstruction.BoostMethod.ADDITIVE,
+                    querqyParser, ignoreCase, Collections.emptyMap(), DEFAULT_SELECTION_STRATEGY_FACTORY, false);
         } catch (final IOException e) {
             return Collections.singletonList("Cannot create rewriter: " + e.getMessage());
         }

src/main/java/querqy/elasticsearch/rewriter/WordBreakCompoundRewriterFactory.java

@@ -6,8 +6,11 @@
 import querqy.elasticsearch.ConfigUtils;
 import querqy.elasticsearch.DismaxSearchEngineRequestAdapter;
 import querqy.elasticsearch.ESRewriterFactory;
+import querqy.lucene.contrib.rewrite.wordbreak.LuceneCompounder;
+import querqy.lucene.contrib.rewrite.wordbreak.MorphologicalCompounder;
 import querqy.lucene.contrib.rewrite.wordbreak.MorphologicalWordBreaker;
 import querqy.lucene.contrib.rewrite.wordbreak.Morphology;
+import querqy.lucene.contrib.rewrite.wordbreak.MorphologyProvider;
 import querqy.lucene.contrib.rewrite.wordbreak.SpellCheckerCompounder;
 import querqy.lucene.contrib.rewrite.wordbreak.WordBreakCompoundRewriter;
 import querqy.model.ExpandedQuery;
@@ -17,11 +20,7 @@
 import querqy.rewrite.SearchEngineRequestAdapter;
 import querqy.trie.TrieMap;
 
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
 import java.util.Collections;
-import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -50,21 +49,23 @@ public class WordBreakCompoundRewriterFactory extends ESRewriterFactory {
     static final boolean DEFAULT_ALWAYS_ADD_REVERSE_COMPOUNDS = false;
     static final boolean DEFAULT_VERIFY_DECOMPOUND_COLLATION = false;
 
+    static final String DEFAULT_MORPHOLOGY_NAME = "DEFAULT";
+
+    private static final MorphologyProvider MORPHOLOGY_PROVIDER = new MorphologyProvider();
 
     private String dictionaryField;
 
     private boolean lowerCaseInput = DEFAULT_LOWER_CASE_INPUT;
     private boolean alwaysAddReverseCompounds = DEFAULT_ALWAYS_ADD_REVERSE_COMPOUNDS;
 
     private WordBreakSpellChecker spellChecker;
-    private SpellCheckerCompounder compounder;
+    private LuceneCompounder compounder;
     private MorphologicalWordBreaker wordBreaker;
     private TrieMap<Boolean> reverseCompoundTriggerWords;
     private TrieMap<Boolean> protectedWords;
     private int maxDecompoundExpansions = DEFAULT_MAX_DECOMPOUND_EXPANSIONS;
     private boolean verifyDecompoundCollation = DEFAULT_VERIFY_DECOMPOUND_COLLATION;
 
-
     public WordBreakCompoundRewriterFactory(final String rewriterId) {
         super(rewriterId);
     }
@@ -89,21 +90,49 @@ public void configure(final Map<String, Object> config) {
         spellChecker.setMaxCombineWordLength(maxCombineLength);
         spellChecker.setMinBreakWordLength(minBreakLength);
         spellChecker.setMaxEvaluations(100);
-        compounder = new SpellCheckerCompounder(spellChecker, dictionaryField, lowerCaseInput);
 
-        final Morphology morphology = ConfigUtils.getEnumArg(config, "morphology", Morphology.class)
-                .orElse(Morphology.DEFAULT);
+        final String defaultMorphologyName = ConfigUtils.getStringArg(config, "morphology", DEFAULT_MORPHOLOGY_NAME);
 
-        wordBreaker = new MorphologicalWordBreaker(morphology, dictionaryField, lowerCaseInput, minSuggestionFreq,
-                minBreakLength, MAX_EVALUATIONS);
+        Map<String, Object> compoundConf = (Map<String, Object>) config.get("compound");
+        if (compoundConf == null) {
+            compoundConf = Collections.emptyMap();
+        }
+
+        final String compoundMorphologyName = ConfigUtils.getStringArg(compoundConf, "morphology",
+                defaultMorphologyName);
+
+        final Optional<Morphology> compoundMorphology = MORPHOLOGY_PROVIDER.get(compoundMorphologyName);
+
+        if (compoundMorphology.isEmpty() || compoundMorphology.get() == MorphologyProvider.DEFAULT) {
+            // use WordBreakSpellChecker when DEFAULT for backwards compatibility
+            final WordBreakSpellChecker spellChecker = new WordBreakSpellChecker();
+            spellChecker.setMaxChanges(MAX_CHANGES);
+            spellChecker.setMinSuggestionFrequency(minSuggestionFreq);
+            spellChecker.setMaxCombineWordLength(maxCombineLength);
+            spellChecker.setMinBreakWordLength(minBreakLength);
+            spellChecker.setMaxEvaluations(100);
+            compounder = new SpellCheckerCompounder(spellChecker, dictionaryField, lowerCaseInput);
+        } else {
+            compounder = new MorphologicalCompounder(compoundMorphology.get(), dictionaryField, lowerCaseInput,
+                    minSuggestionFreq);
+        }
 
-        reverseCompoundTriggerWords = ConfigUtils.getTrieSetArg(config, "reverseCompoundTriggerWords");
-        protectedWords = ConfigUtils.getTrieSetArg(config, "protectedWords");
 
         Map<String, Object> decompoundConf = (Map<String, Object>) config.get("decompound");
         if (decompoundConf == null) {
             decompoundConf = Collections.emptyMap();
         }
+        final String decompoundMorphologyName = ConfigUtils.getStringArg(decompoundConf, "morphology",
+                defaultMorphologyName);
+        final Optional<Morphology> decompoundMorphology = MORPHOLOGY_PROVIDER.get(decompoundMorphologyName);
+
+
+        wordBreaker = new MorphologicalWordBreaker(decompoundMorphology.get(), dictionaryField, lowerCaseInput,
+                minSuggestionFreq, minBreakLength, MAX_EVALUATIONS);
+
+        reverseCompoundTriggerWords = ConfigUtils.getTrieSetArg(config, "reverseCompoundTriggerWords");
+        protectedWords = ConfigUtils.getTrieSetArg(config, "protectedWords");
+
         maxDecompoundExpansions = ConfigUtils.getArg(decompoundConf, "maxExpansions",
                 DEFAULT_MAX_DECOMPOUND_EXPANSIONS);
         verifyDecompoundCollation =  ConfigUtils.getArg(decompoundConf, "verifyCollation",
@@ -122,11 +151,30 @@ public List<String> validateConfiguration(final Map<String, Object> config) {
         }
 
         ConfigUtils.getStringArg(config, "morphology").ifPresent(morphologyName -> {
-            if (Arrays.stream(Morphology.values()).map(Enum::name).noneMatch(name -> name.equals(morphologyName))) {
+            if (!MORPHOLOGY_PROVIDER.exists(morphologyName)) {
                 errors.add("Unknown morphology: " + morphologyName);
             }
         });
 
+        final Map<String, Object> decompoundConf = (Map<String, Object>) config.get("decompound");
+        if (decompoundConf != null) {
+            ConfigUtils.getStringArg(decompoundConf, "morphology").ifPresent(morphologyName -> {
+                if (!MORPHOLOGY_PROVIDER.exists(morphologyName)) {
+                    errors.add("Unknown decompound morphology: " + morphologyName);
+                }
+            });
+        }
+
+        final Map<String, Object> compoundConf = (Map<String, Object>) config.get("compound");
+        if (compoundConf != null) {
+            ConfigUtils.getStringArg(compoundConf, "morphology").ifPresent(morphologyName -> {
+                if (!MORPHOLOGY_PROVIDER.exists(morphologyName)) {
+                    errors.add("Unknown compound morphology: " + morphologyName);
+                }
+            });
+        }
+
+
         return errors;
 
     }
@@ -192,7 +240,7 @@ private IndexReader getShardIndexReader(final DismaxSearchEngineRequestAdapter s
         return searchEngineRequestAdapter.getSearchExecutionContext().searcher().getTopReaderContext().reader();
     }
 
-    public SpellCheckerCompounder getCompounder() {
+    public LuceneCompounder getCompounder() {
         return compounder;
     }
 

src/main/java/querqy/elasticsearch/rewriterstore/NodesClearRewriterCacheRequest.java

@@ -1,9 +1,9 @@
 package querqy.elasticsearch.rewriterstore;
 
-import org.elasticsearch.action.support.nodes.BaseNodeRequest;
 import org.elasticsearch.action.support.nodes.BaseNodesRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.transport.TransportRequest;
 
 import java.io.IOException;
 import java.util.Optional;
@@ -41,7 +41,7 @@ public NodeRequest newNodeRequest() {
         return new NodeRequest(rewriterId);
     }
 
-    public static class NodeRequest extends BaseNodeRequest {
+    public static class NodeRequest extends TransportRequest {
 
         final String rewriterId;
 

src/main/java/querqy/elasticsearch/rewriterstore/NodesReloadRewriterRequest.java

@@ -1,9 +1,9 @@
 package querqy.elasticsearch.rewriterstore;
 
-import org.elasticsearch.action.support.nodes.BaseNodeRequest;
 import org.elasticsearch.action.support.nodes.BaseNodesRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.transport.TransportRequest;
 
 import java.io.IOException;
 
@@ -36,7 +36,7 @@ public String getRewriterId() {
     }
 
 
-    public static class NodeRequest extends BaseNodeRequest {
+    public static class NodeRequest extends TransportRequest {
 
         String rewriterId;
 

src/main/java/querqy/elasticsearch/rewriterstore/TransportDeleteRewriterAction.java

@@ -14,8 +14,6 @@
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 
-import java.util.function.Supplier;
-
 public class TransportDeleteRewriterAction  extends HandledTransportAction<DeleteRewriterRequest, DeleteRewriterResponse> {
 
     private final Client client;
@@ -32,8 +30,7 @@ public TransportDeleteRewriterAction(final TransportService transportService, fi
     protected void doExecute(final Task task, final DeleteRewriterRequest request,
                              final ActionListener<DeleteRewriterResponse> listener) {
 
-        final DeleteRequestBuilder deleteRequest = client.prepareDelete(QUERQY_INDEX_NAME, null,
-                request.getRewriterId());
+        final DeleteRequestBuilder deleteRequest = client.prepareDelete(QUERQY_INDEX_NAME, request.getRewriterId());
 
         deleteRequest.execute(new ActionListener<DeleteResponse>() {
 

src/main/java/querqy/elasticsearch/rewriterstore/TransportNodesClearRewriterCacheAction.java

@@ -9,6 +9,7 @@
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import querqy.elasticsearch.RewriterShardContexts;
@@ -60,7 +61,7 @@ protected NodesClearRewriterCacheResponse.NodeResponse newNodeResponse(final Str
 
     @Override
     protected NodesClearRewriterCacheResponse.NodeResponse nodeOperation(
-            final NodesClearRewriterCacheRequest.NodeRequest request) {
+            final NodesClearRewriterCacheRequest.NodeRequest request, final Task task) {
 
         final Optional<String> rewriterId = request.getRewriterId();
         if (rewriterId.isPresent()) {

src/main/java/querqy/elasticsearch/rewriterstore/TransportNodesReloadRewriterAction.java

@@ -9,6 +9,7 @@
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.indices.IndicesService;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import querqy.elasticsearch.RewriterShardContexts;
@@ -59,7 +60,7 @@ protected NodesReloadRewriterResponse.NodeResponse newNodeResponse(final StreamI
 
     @Override
     protected NodesReloadRewriterResponse.NodeResponse nodeOperation(
-            final NodesReloadRewriterRequest.NodeRequest request) {
+            final NodesReloadRewriterRequest.NodeRequest request, final Task task) {
         try {
             rewriterShardContexts.reloadRewriter(request.getRewriterId());
             return new NodesReloadRewriterResponse.NodeResponse(clusterService.localNode(), null);