Update/solr8.6 (#11)

* Update to Solr 8.6 * Adding a test workflow * Fix workflow name * Working on fixing the random number error * Bringing over the highlighter helpers * Release workflow
o19s · Sep 10, 2020 · bed05ea · bed05ea
1 parent 7479b28
commit bed05ea
Show file tree

Hide file tree

Showing 9 changed files with 285 additions and 11 deletions.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -0,0 +1,43 @@
+name: Java CI
+
+on:
+  push:
+    tags:
+      - 'v*.*.*'
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set release version Name
+        run: echo ::set-env name=RELEASE_VERSION::${GITHUB_REF#refs/*/}
+      - name: Set up JDK 1.8
+        uses: actions/setup-java@v1
+        with:
+          java-version: 1.8
+      - name: Build with Maven
+        run: mvn -B package --file pom.xml
+        env:
+          MAVEN_OPTS: -Dtest.solr.allowed.securerandom=NativePRNG
+      - name: Stage jar
+        run: cp target/*.jar payload-component-${{ env.RELEASE_VERSION }}.jar
+      - name: Create Release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ env.RELEASE_VERSION }}
+          release_name: ${{ env.RELEASE_VERSION }}
+          draft: false
+          prerelease: false
+      - name: Upload Release
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./payload-component-${{ env.RELEASE_VERSION }}.jar
+          asset_name: payload-component-${{ env.RELEASE_VERSION }}.jar
+          asset_content_type: application/java-archive
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -0,0 +1,18 @@
+name: Java CI
+
+on: [push]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up JDK 1.8
+        uses: actions/setup-java@v1
+        with:
+          java-version: 1.8
+      - name: Build with Maven
+        run: mvn -B package --file pom.xml
+        env:
+          MAVEN_OPTS: -Dtest.solr.allowed.securerandom=NativePRNG
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,4 @@
 .DS_Store
 .idea/*
 target/*
+solr-payloads.iml
diff --git a/pom.xml b/pom.xml
@@ -6,7 +6,7 @@
 
   <groupId>com.o19s.payloads</groupId>
   <artifactId>solr-payloads</artifactId>
-  <version>1.0.3-solr7.1.0-SNAPSHOT</version>
+  <version>1.1.4-solr8.6.0</version>
 
   <name>solr-payloads</name>
   <url>https://github.com/o19s/payload-component</url>
@@ -16,9 +16,8 @@
     <maven.compiler.source>1.8</maven.compiler.source>
     <maven.compiler.target>1.8</maven.compiler.target>
 
-    <junit.version>4.11</junit.version>
-    <slf4j.version>1.7.7</slf4j.version>
-    <solr.version>7.1.0</solr.version>
+    <junit.version>4.12</junit.version>
+    <solr.version>8.6.0</solr.version>
   </properties>
 
   <dependencies>
@@ -28,12 +27,6 @@
       <version>${solr.version}</version>
       <scope>test</scope>
     </dependency>
-    <dependency>
-      <groupId>org.slf4j</groupId>
-      <artifactId>slf4j-log4j12</artifactId>
-      <version>${slf4j.version}</version>
-      <scope>test</scope>
-    </dependency>
     <dependency>
       <groupId>commons-logging</groupId>
       <artifactId>commons-logging</artifactId>

diff --git a/src/main/java/com/o19s/hl/OffsetFormatter.java b/src/main/java/com/o19s/hl/OffsetFormatter.java
@@ -0,0 +1,22 @@
+package com.o19s.hl;
+
+import org.apache.lucene.search.highlight.Formatter;
+import org.apache.solr.common.params.HighlightParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.highlight.HighlightingPluginBase;
+import org.apache.solr.highlight.SolrFormatter;
+
+public class OffsetFormatter extends HighlightingPluginBase implements SolrFormatter {
+    @Override
+    public Formatter getFormatter(String fieldName, SolrParams params) {
+        return new SimpleOffsetFormatter(
+                params.getFieldParam(fieldName, HighlightParams.SIMPLE_PRE, "<em data-num-tokens=\"$numTokens\" data-score=\"$score\" data-end-offset=\"$endOffset\" data-start-offset=\"$startOffset\">"),
+                params.getFieldParam(fieldName, HighlightParams.SIMPLE_POST, "</em>")
+        );
+    }
+
+    @Override
+    public String getDescription() {
+        return "OffsetFormatter";
+    }
+}
diff --git a/src/main/java/com/o19s/hl/SimpleOffsetFormatter.java b/src/main/java/com/o19s/hl/SimpleOffsetFormatter.java
@@ -0,0 +1,48 @@
+package com.o19s.hl;
+
+import org.apache.lucene.search.highlight.Formatter;
+import org.apache.lucene.search.highlight.TokenGroup;
+
+public class SimpleOffsetFormatter implements Formatter {
+    private static final String DEFAULT_PRE_TAG = "<b>";
+    private static final String DEFAULT_POST_TAG = "</b>";
+
+    private String preTag;
+    private String postTag;
+
+    public SimpleOffsetFormatter() {
+        this(DEFAULT_PRE_TAG, DEFAULT_POST_TAG);
+    }
+
+    public SimpleOffsetFormatter(String preTag, String postTag) {
+        this.preTag = preTag;
+        this.postTag = postTag;
+    }
+
+    @Override
+    public String highlightTerm(String matchedText, TokenGroup tokenGroup) {
+        if (tokenGroup.getTotalScore() <= 0) {
+            return matchedText;
+        }
+
+        // Allocate StringBuilder with the right number of characters from the
+        // beginning, to avoid char[] allocations in the middle of appends.
+        StringBuilder returnBuffer = new StringBuilder(preTag.length() + matchedText.length() + postTag.length());
+
+        String replacedPre = preTag
+                // Total score for the group
+                .replace("$score", Float.toString(tokenGroup.getTotalScore()))
+
+                // Offsets
+                .replace("$endOffset", Integer.toString(tokenGroup.getEndOffset()))
+                .replace("$startOffset", Integer.toString(tokenGroup.getStartOffset()))
+
+                // Number of tokens in the group
+                .replace("$numTokens", Integer.toString(tokenGroup.getNumTokens()));
+
+        returnBuffer.append(replacedPre);
+        returnBuffer.append(matchedText);
+        returnBuffer.append(postTag);
+        return returnBuffer.toString();
+    }
+}
diff --git a/src/test/java/com/o19s/hl/OffsetTests.java b/src/test/java/com/o19s/hl/OffsetTests.java
@@ -0,0 +1,40 @@
+package com.o19s.hl;
+
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.util.TestHarness;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.util.HashMap;
+
+public class OffsetTests extends SolrTestCaseJ4 {
+    @BeforeClass
+    public static void beforeClass() throws Exception {
+        initCore("solrconfig.xml","schema.xml");
+    }
+
+    @Test
+    public void testOffsets() {
+        HashMap<String, String> args = new HashMap<>();
+        args.put("defType", "edismax");
+        args.put("hl", "true");
+        args.put("hl.fl", "content");
+        args.put("qf", "content");
+        args.put("q.alt", "*:*");
+        TestHarness.LocalRequestFactory sumLRF = h.getRequestFactory(
+                "standard", 0, 200, args);
+
+        assertU(adoc("content", "A long day's night.",
+                "id", "1"));
+        assertU(commit());
+        assertU(optimize());
+        assertQ("Offset test",
+                sumLRF.makeRequest("night"),
+                "//lst[@name='highlighting']/lst[@name='1']",
+                "//lst[@name='1']/arr[@name='content']/str[contains(text(),'data-num-tokens=\"1\"')]",
+                "//lst[@name='1']/arr[@name='content']/str[contains(text(),'data-score=\"1.0\"')]",
+                "//lst[@name='1']/arr[@name='content']/str[contains(text(),'data-start-offset=\"13\"')]",
+                "//lst[@name='1']/arr[@name='content']/str[contains(text(),'data-end-offset=\"18\"')]"
+        );
+    }
+}
diff --git a/src/test/resources/solr/collection1/conf/schema.xml b/src/test/resources/solr/collection1/conf/schema.xml
@@ -7,6 +7,7 @@
     -->
 
     <field name="id" type="string" indexed="true"  stored="true"  multiValued="false" />
+    <field name="content" type="text" indexed="true"  stored="true"  multiValued="false" termOffsets="true" termPositions="true" termVectors="true" />
     <field name="content_payload" type="payload" indexed="true"  stored="true"  multiValued="false" termPayloads="true" termOffsets="true" termPositions="true" termVectors="true" />
     <field name="content_payload_b64" type="payload_b64" indexed="true" stored="true" multiValued="false" termPayloads="true" termOffsets="true" termPositions="true" termVectors="true" />
     <field name="content_payload_buffered" type="payload_buffered" indexed="true" stored="true" multiValued="true" termPayloads="true" termOffsets="true" termPositions="true" termVectors="true" />
@@ -61,6 +62,18 @@
         </analyzer>
     </fieldType>
 
+    <!-- Simple text analysis -->
+    <fieldType name="text" class="solr.TextField" positionIncrementGap="100">
+        <analyzer type="index">
+            <tokenizer class="solr.StandardTokenizerFactory"/>
+            <filter class="solr.LowerCaseFilterFactory"/>
+        </analyzer>
+        <analyzer type="query">
+            <tokenizer class="solr.StandardTokenizerFactory"/>
+            <filter class="solr.LowerCaseFilterFactory"/>
+        </analyzer>
+    </fieldType>
+
     <!-- Simple types -->
     <fieldtype name="string" class="solr.StrField"      sortMissingLast="true" omitNorms="true"/>
     <fieldType name="long"   class="solr.TrieLongField" precisionStep="0" positionIncrementGap="0"/>

diff --git a/src/test/resources/solr/collection1/conf/solrconfig.xml b/src/test/resources/solr/collection1/conf/solrconfig.xml
@@ -85,7 +85,103 @@
   <requestHandler name="/analysis/field" startup="lazy" class="solr.FieldAnalysisRequestHandler" />
   <requestHandler name="/update" class="solr.UpdateRequestHandler"  />
 
-
+  <searchComponent class="solr.HighlightComponent" name="highlight">
+    <highlighting>
+      <!-- Configure the standard fragmenter -->
+      <!-- This could most likely be commented out in the "default" case -->
+      <fragmenter name="gap"
+                  default="true"
+                  class="solr.highlight.GapFragmenter">
+        <lst name="defaults">
+          <int name="hl.fragsize">100</int>
+        </lst>
+      </fragmenter>
+
+      <!-- A regular-expression-based fragmenter
+           (for sentence extraction)
+        -->
+      <fragmenter name="regex"
+                  class="solr.highlight.RegexFragmenter">
+        <lst name="defaults">
+          <!-- slightly smaller fragsizes work better because of slop -->
+          <int name="hl.fragsize">70</int>
+          <!-- allow 50% slop on fragment sizes -->
+          <float name="hl.regex.slop">0.5</float>
+          <!-- a basic sentence pattern -->
+          <str name="hl.regex.pattern">[-\w ,/\n\&quot;&apos;]{20,200}</str>
+        </lst>
+      </fragmenter>
+
+      <!-- Configure the standard formatter -->
+      <formatter name="html"
+                 default="true"
+                 class="com.o19s.hl.OffsetFormatter">
+      </formatter>
+
+      <!-- Configure the standard encoder -->
+      <encoder name="html"
+               class="solr.highlight.HtmlEncoder" />
+
+      <!-- Configure the standard fragListBuilder -->
+      <fragListBuilder name="simple"
+                       class="solr.highlight.SimpleFragListBuilder"/>
+
+      <!-- Configure the single fragListBuilder -->
+      <fragListBuilder name="single"
+                       class="solr.highlight.SingleFragListBuilder"/>
+
+      <!-- Configure the weighted fragListBuilder -->
+      <fragListBuilder name="weighted"
+                       default="true"
+                       class="solr.highlight.WeightedFragListBuilder"/>
+
+      <!-- default tag FragmentsBuilder -->
+      <fragmentsBuilder name="default"
+                        default="true"
+                        class="solr.highlight.ScoreOrderFragmentsBuilder">
+        <!--
+        <lst name="defaults">
+          <str name="hl.multiValuedSeparatorChar">/</str>
+        </lst>
+        -->
+      </fragmentsBuilder>
+
+      <!-- multi-colored tag FragmentsBuilder -->
+      <fragmentsBuilder name="colored"
+                        class="solr.highlight.ScoreOrderFragmentsBuilder">
+        <lst name="defaults">
+          <str name="hl.tag.pre"><![CDATA[
+               <b style="background:yellow">,<b style="background:lawgreen">,
+               <b style="background:aquamarine">,<b style="background:magenta">,
+               <b style="background:palegreen">,<b style="background:coral">,
+               <b style="background:wheat">,<b style="background:khaki">,
+               <b style="background:lime">,<b style="background:deepskyblue">]]></str>
+          <str name="hl.tag.post"><![CDATA[</b>]]></str>
+        </lst>
+      </fragmentsBuilder>
+
+      <boundaryScanner name="default"
+                       default="true"
+                       class="solr.highlight.SimpleBoundaryScanner">
+        <lst name="defaults">
+          <str name="hl.bs.maxScan">10</str>
+          <str name="hl.bs.chars">.,!? &#9;&#10;&#13;</str>
+        </lst>
+      </boundaryScanner>
+
+      <boundaryScanner name="breakIterator"
+                       class="solr.highlight.BreakIteratorBoundaryScanner">
+        <lst name="defaults">
+          <!-- type should be one of CHARACTER, WORD(default), LINE and SENTENCE -->
+          <str name="hl.bs.type">WORD</str>
+          <!-- language and country are used when constructing Locale object.  -->
+          <!-- And the Locale object will be used when getting instance of BreakIterator -->
+          <str name="hl.bs.language">en</str>
+          <str name="hl.bs.country">US</str>
+        </lst>
+      </boundaryScanner>
+    </highlighting>
+  </searchComponent>
 
 
   <!-- config for the admin interface -->