OPENNLP-1567 - OpenNLP Models: Provide a Finder / Loader Implementation

README.md

@@ -64,6 +64,7 @@ You can also follow our [mailing lists](http://opennlp.apache.org/mailing-lists.
 Currently, the library has different packages:
 
 * `opennlp-tools` : The core toolkit.
+* `opennlp-tools-models` : A set of classes to load [OpenNLP models](https://github.com/apache/opennlp-models) from the classpath.
 * `opennlp-uima` : A set of [Apache UIMA](https://uima.apache.org) annotators.
 * `opennlp-brat-annotator` : A set of annotators for [BRAT](http://brat.nlplab.org/)
 * `opennlp-morfologik-addon` : An addon for Morfologik

opennlp-tools-models/pom.xml

@@ -0,0 +1,105 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one
+   or more contributor license agreements.  See the NOTICE file
+   distributed with this work for additional information
+   regarding copyright ownership.  The ASF licenses this file
+   to you under the Apache License, Version 2.0 (the
+   "License"); you may not use this file except in compliance
+   with the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing,
+   software distributed under the License is distributed on an
+   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+   KIND, either express or implied.  See the License for the
+   specific language governing permissions and limitations
+   under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.apache.opennlp</groupId>
+        <artifactId>opennlp</artifactId>
+        <version>2.3.4-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>opennlp-tools-models</artifactId>
+    <packaging>jar</packaging>
+    <name>Apache OpenNLP Tools Models</name>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.opennlp</groupId>
+            <artifactId>opennlp-tools</artifactId>
+            <version>${project.version}</version>
+            <scope>provided</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>io.github.classgraph</groupId>
+            <artifactId>classgraph</artifactId>
+            <version>${classgraph.version}</version>
+            <optional>true</optional>
+        </dependency>
+
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-api</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-engine</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-simple</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <!-- models -->
+        <dependency>
+            <groupId>org.apache.opennlp</groupId>
+            <artifactId>opennlp-models-sentdetect-en</artifactId>
+            <version>${opennlp.models.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.opennlp</groupId>
+            <artifactId>opennlp-models-langdetect</artifactId>
+            <version>${opennlp.models.version}</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <version>${maven.surefire.plugin}</version>
+                <configuration>
+                    <argLine>-Xmx2048m -Dorg.slf4j.simpleLogger.defaultLogLevel=off --add-opens java.base/jdk.internal.loader=ALL-UNNAMED</argLine>
+                    <forkCount>${opennlp.forkCount}</forkCount>
+                    <useSystemClassLoader>false</useSystemClassLoader>
+                    <failIfNoSpecifiedTests>false</failIfNoSpecifiedTests>
+                </configuration>
+            </plugin>
+
+        </plugins>
+    </build>
+</project>

opennlp-tools-models/src/main/java/opennlp/tools/models/AbstractClassPathModelFinder.java

@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.models;
+
+import java.net.URI;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * Enables the detection of OpenNLP models in the classpath. By default, this class will search
+ * for JAR files starting with "opennlp-models-*". This wildcard pattern can be adjusted by
+ * using the alternative constructor of this class.
+ */
+public abstract class AbstractClassPathModelFinder implements ClassPathModelFinder {
+
+  protected static final String JAR = "jar";
+
+  private final String jarModelPrefix;
+  private Set<ClassPathModelEntry> models;
+
+  /**
+   * By default, it scans for "opennlp-models-*.jar".
+   */
+  public AbstractClassPathModelFinder() {
+    this(OPENNLP_MODEL_JAR_PREFIX);
+  }
+
+  /**
+   * @param jarModelPrefix The leafnames of the jars that should be canned (e.g. "opennlp.jar").
+   *                       May contain a wildcard glob ("opennlp-*.jar"). It must not be {@code null}.
+   */
+  public AbstractClassPathModelFinder(String jarModelPrefix) {
+    Objects.requireNonNull(jarModelPrefix, "modelJarPrefix must not be null");
+    this.jarModelPrefix = jarModelPrefix;
+  }
+
+  @Override
+  public Set<ClassPathModelEntry> findModels(boolean reloadCache) {
+
+    if (this.models == null || reloadCache) {
+      final List<URI> classpathModels = getMatchingURIs("*.bin", getContext());
+      final List<URI> classPathProperties = getMatchingURIs("model.properties", getContext());
+
+      this.models = new HashSet<>();
+
+      for (URI model : classpathModels) {
+        URI m = null;
+        for (URI prop : classPathProperties) {
+          if (jarPathsMatch(model, prop)) {
+            m = prop;
+            break;
+          }
+        }
+        this.models.add(new ClassPathModelEntry(model, Optional.ofNullable(m)));
+
+      }
+    }
+    return this.models;
+  }
+
+  /**
+   * Subclasses can implement this method to provide additional context to
+   * {@link AbstractClassPathModelFinder#getMatchingURIs(String, Object)}.
+   *
+   * @return a context information. May be {@code null}.
+   */
+  protected abstract Object getContext();
+
+  /**
+   * Return matching classpath URIs for the given pattern.
+   *
+   * @param wildcardPattern the pattern. Must not be {@code null}.
+   * @param context         an object holding context information. It might be {@code null}.
+   * @return a list of matching classpath URIs.
+   */
+  protected abstract List<URI> getMatchingURIs(String wildcardPattern, Object context);
+
+  protected boolean jarPathsMatch(URI uri1, URI uri2) {
+    final String[] parts1 = parseJarURI(uri1);
+    final String[] parts2 = parseJarURI(uri2);
+
+    if (parts1 == null || parts2 == null) {
+      return false;
+    }
+
+    return parts1[0].equals(parts2[0]);
+  }
+
+  protected String[] parseJarURI(URI uri) {
+    if (JAR.equals(uri.getScheme())) {
+      final String ssp = uri.getSchemeSpecificPart();
+      final int separatorIndex = ssp.indexOf("!/");
+      if (separatorIndex > 0) {
+        final String jarFileUri = ssp.substring(0, separatorIndex);
+        final String entryPath = ssp.substring(separatorIndex + 2);
+        return new String[] {jarFileUri, entryPath};
+      }
+    }
+    return null;
+  }
+
+  protected String getJarModelPrefix() {
+    return jarModelPrefix;
+  }
+
+}

opennlp-tools-models/src/main/java/opennlp/tools/models/ClassPathModel.java

@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.models;
+
+import java.util.Properties;
+
+public record ClassPathModel(Properties properties, byte[] model) {
+
+  public String getModelVersion() {
+    return properties != null ? properties.getProperty("model.version", "unknown") : "unknown";
+  }
+
+  public String getModelName() {
+    return properties != null ? properties.getProperty("model.name", "unknown") : "unknown";
+  }
+
+  public String getModelSHA256() {
+    return properties != null ? properties.getProperty("model.sha256", "unknown") : "unknown";
+  }
+
+}

opennlp-tools-models/src/main/java/opennlp/tools/models/ClassPathModelEntry.java

@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.models;
+
+import java.net.URI;
+import java.util.Optional;
+
+public record ClassPathModelEntry(URI model, Optional<URI> properties) {
+
+}

opennlp-tools-models/src/main/java/opennlp/tools/models/ClassPathModelFinder.java

@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.models;
+
+import java.util.Set;
+
+public interface ClassPathModelFinder {
+
+  String OPENNLP_MODEL_JAR_PREFIX = "opennlp-models-*.jar";
+
+  /**
+   * Finds OpenNLP models within the classpath.
+   *
+   * @param reloadCache {@code true}, if the internal cache should explicitly be reloaded
+   * @return A Set of {@link ClassPathModelEntry ClassPathModelEntries}. It might be empty.
+   */
+  Set<ClassPathModelEntry> findModels(boolean reloadCache);
+}

opennlp-tools-models/src/main/java/opennlp/tools/models/ClassPathModelLoader.java

@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.models;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Objects;
+import java.util.Properties;
+
+/**
+ * Responsible for loading OpenNLP models from the classpath.
+ */
+public class ClassPathModelLoader {
+
+  /**
+   * Loads a {@link ClassPathModel} from a {@link ClassPathModelEntry}
+   *
+   * @param entry must not be {@code null}.
+   * @return a {@link ClassPathModel} containing the model resources.
+   * @throws IOException thrown if something went wrong during reading resources from the classpath.
+   */
+  public ClassPathModel load(ClassPathModelEntry entry) throws IOException {
+    Objects.requireNonNull(entry, "entry must not be null");
+    Objects.requireNonNull(entry.properties(), "entry.properties() must not be null");
+    Objects.requireNonNull(entry.model(), "entry.model() must not be null");
+
+    final Properties properties = new Properties();
+
+    if (entry.properties().isPresent()) {
+      try (InputStream inputStream = entry.properties().get().toURL().openStream()) {
+        properties.load(inputStream);
+      }
+    }
+
+    final byte[] model;
+    try (InputStream inputStream = entry.model().toURL().openStream()) {
+      model = inputStream.readAllBytes();
+    }
+
+    return new ClassPathModel(properties, model);
+  }
+}