Skip to content
This repository has been archived by the owner on Mar 3, 2020. It is now read-only.

Commit

Permalink
Initial commit
Browse files Browse the repository at this point in the history
  • Loading branch information
@ficolo
ficolo committed Aug 23, 2017
1 parent 58c52aa commit ac02dc2
Show file tree
Hide file tree
Showing 6 changed files with 393 additions and 0 deletions.
54 changes: 54 additions & 0 deletions index-bioschemas/build-ivy.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
<?xml version="1.0"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project name="index-bioschemas" default="deps-jar" xmlns:ivy="antlib:org.apache.ivy.ant">

<property name="ivy.install.version" value="2.1.0" />
<condition property="ivy.home" value="${env.IVY_HOME}">
<isset property="env.IVY_HOME" />
</condition>
<property name="ivy.home" value="${user.home}/.ant" />
<property name="ivy.checksums" value="" />
<property name="ivy.jar.dir" value="${ivy.home}/lib" />
<property name="ivy.jar.file" value="${ivy.jar.dir}/ivy.jar" />

<target name="download-ivy" unless="offline">

<mkdir dir="${ivy.jar.dir}"/>
<!-- download Ivy from web site so that it can be used even without any special installation -->
<get src="http://repo2.maven.org/maven2/org/apache/ivy/ivy/${ivy.install.version}/ivy-${ivy.install.version}.jar"
dest="${ivy.jar.file}" usetimestamp="true"/>
</target>

<target name="init-ivy" depends="download-ivy">
<!-- try to load ivy here from ivy home, in case the user has not already dropped
it into ant's lib dir (note that the latter copy will always take precedence).
We will not fail as long as local lib dir exists (it may be empty) and
ivy is in at least one of ant's lib dir or the local lib dir. -->
<path id="ivy.lib.path">
<fileset dir="${ivy.jar.dir}" includes="*.jar"/>

</path>
<taskdef resource="org/apache/ivy/ant/antlib.xml"
uri="antlib:org.apache.ivy.ant" classpathref="ivy.lib.path"/>
</target>

<target name="deps-jar" depends="init-ivy">
<ivy:retrieve pattern="lib/[artifact]-[revision].[ext]"/>
</target>

</project>
7 changes: 7 additions & 0 deletions index-bioschemas/build.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
<?xml version="1.0"?>

<project name="index-bioschemas" default="jar-core">

<import file="../build-plugin.xml"/>

</project>
26 changes: 26 additions & 0 deletions index-bioschemas/ivy.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
<ivy-module version="1.0">
<info organisation="org.apache.nutch" module="${ant.project.name}">
<license name="Apache 2.0"/>
<ivyauthor name="Federico Lopez" url="http://github.com/ficolo"/>
<description>
Bioschemas plugin for Apache Nutch
</description>
</info>

<configurations>
<include file="${nutch.root}/ivy/ivy-configurations.xml"/>
</configurations>

<publications>
<!--get the artifact from our module name-->
<artifact conf="master"/>
</publications>

<dependencies>
<!-- https://mvnrepository.com/artifact/org.apache.any23/apache-any23-api -->
<dependency org="org.apache.any23" name="apache-any23-api" rev="2.0" conf="*->default"/>
<!-- https://mvnrepository.com/artifact/org.apache.any23/apache-any23-core -->
<dependency org="org.apache.any23" name="apache-any23-core" rev="2.0" conf="*->default"/>
</dependencies>

</ivy-module>
172 changes: 172 additions & 0 deletions index-bioschemas/plugin.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,172 @@
<?xml version="1.0" encoding="UTF-8"?>
<plugin
id="index-bioschemas"
name="Bioschemas Indexing Filter"
version="1.0.0"
provider-name="bioschemas.org">


<runtime>
<library name="index-bioschemas.jar">
<export name="*"/>
</library>
<library name="apache-any23-api-2.0.jar"/>
<library name="jcip-annotations-1.0.jar"/>
<library name="rdf4j-repository-sail-2.1.3.jar"/>
<library name="apache-any23-core-2.0.jar"/>
<library name="jcommander-1.30.jar"/>
<library name="rdf4j-repository-sparql-2.1.3.jar"/>
<library name="apache-any23-csvutils-2.0.jar"/>
<library name="jdom2-2.0.4.jar"/>
<library name="rdf4j-rio-api-2.1.3.jar"/>
<library name="apache-any23-encoding-2.0.jar"/>
<library name="jempbox-1.8.12.jar"/>
<library name="rdf4j-rio-datatypes-2.1.3.jar"/>
<library name="apache-any23-mime-2.0.jar"/>
<library name="jhighlight-1.0.2.jar"/>
<library name="rdf4j-rio-jsonld-2.1.3.jar"/>
<library name="apache-mime4j-core-0.7.2.jar"/>
<library name="jj2000-5.2.jar"/>
<library name="rdf4j-rio-languages-2.1.3.jar"/>
<library name="apache-mime4j-dom-0.7.2.jar"/>
<library name="jmatio-1.2.jar"/>
<library name="rdf4j-rio-n3-2.1.3.jar"/>
<library name="asm-5.0.4.jar"/>
<library name="jna-4.1.0.jar"/>
<library name="rdf4j-rio-nquads-2.1.3.jar"/>
<library name="bcmail-jdk15on-1.54.jar"/>
<library name="joda-time-2.2.jar"/>
<library name="rdf4j-rio-ntriples-2.1.3.jar"/>
<library name="bcpkix-jdk15on-1.54.jar"/>
<library name="json-20140107.jar"/>
<library name="rdf4j-rio-rdfxml-2.1.3.jar"/>
<library name="bcprov-jdk15on-1.54.jar"/>
<library name="json-simple-1.1.1.jar"/>
<library name="rdf4j-rio-trig-2.1.3.jar"/>
<library name="boilerpipe-1.1.0.jar"/>
<library name="jsoup-1.7.2.jar"/>
<library name="rdf4j-rio-trix-2.1.3.jar"/>
<library name="bzip2-0.9.1.jar"/>
<library name="jsr-275-0.9.3.jar"/>
<library name="rdf4j-rio-turtle-2.1.3.jar"/>
<library name="c3p0-0.9.1.1.jar"/>
<library name="juniversalchardet-1.0.3.jar"/>
<library name="rdf4j-sail-api-2.1.3.jar"/>
<library name="cdm-4.5.5.jar"/>
<library name="junrar-0.7.jar"/>
<library name="rdf4j-sail-base-2.1.3.jar"/>
<library name="commons-codec-1.10.jar"/>
<library name="jwnl-1.3.3.jar"/>
<library name="rdf4j-sail-inferencer-2.1.3.jar"/>
<library name="commons-compress-1.13.jar"/>
<library name="mapdb-1.0.8.jar"/>
<library name="rdf4j-sail-memory-2.1.3.jar"/>
<library name="commons-csv-1.0-SNAPSHOT-rev1148315.jar"/>
<library name="maven-scm-api-1.4.jar"/>
<library name="rdf4j-sail-model-2.1.3.jar"/>
<library name="commons-exec-1.3.jar"/>
<library name="maven-scm-provider-svn-commons-1.4.jar"/>
<library name="rdf4j-util-2.1.3.jar"/>
<library name="commons-httpclient-3.1.jar"/>
<library name="maven-scm-provider-svnexe-1.4.jar"/>
<library name="regexp-1.3.jar"/>
<library name="commons-io-2.4.jar"/>
<library name="metadata-extractor-2.9.1.jar"/>
<library name="rome-1.5.1.jar"/>
<library name="commons-lang-2.6.jar"/>
<library name="nekohtml-1.9.20.jar"/>
<library name="rome-utils-1.5.1.jar"/>
<library name="commons-logging-1.2.jar"/>
<library name="netcdf4-4.5.5.jar"/>
<library name="semargl-core-0.7.jar"/>
<library name="commons-logging-api-1.1.jar"/>
<library name="opennlp-maxent-3.0.3.jar"/>
<library name="semargl-rdf-0.7.jar"/>
<library name="commons-vfs2-2.0.jar"/>
<library name="opennlp-tools-1.5.3.jar"/>
<library name="semargl-rdf4j-0.7.jar"/>
<library name="cxf-core-3.0.3.jar"/>
<library name="pdfbox-2.0.3.jar"/>
<library name="semargl-rdfa-0.7.jar"/>
<library name="cxf-rt-frontend-jaxrs-3.0.3.jar"/>
<library name="pdfbox-debugger-2.0.3.jar"/>
<library name="sis-metadata-0.6.jar"/>
<library name="cxf-rt-rs-client-3.0.3.jar"/>
<library name="pdfbox-tools-2.0.3.jar"/>
<library name="sis-netcdf-0.6.jar"/>
<library name="cxf-rt-transports-http-3.0.3.jar"/>
<library name="plexus-utils-1.5.6.jar"/>
<library name="sis-referencing-0.6.jar"/>
<library name="dom4j-1.6.1.jar"/>
<library name="sis-storage-0.6.jar"/>
<library name="ehcache-core-2.6.2.jar"/>
<library name="poi-3.8.jar"/>
<library name="sis-utility-0.6.jar"/>
<library name="fontbox-2.0.3.jar"/>
<library name="poi-ooxml-3.8.jar"/>
<library name="slf4j-api-1.7.21.jar"/>
<library name="geoapi-3.0.0.jar"/>
<library name="poi-ooxml-schemas-3.8.jar"/>
<library name="snakeyaml-1.17.jar"/>
<library name="grib-4.5.5.jar"/>
<library name="poi-scratchpad-3.8.jar"/>
<library name="stax2-api-3.1.4.jar"/>
<library name="gson-2.2.4.jar"/>
<library name="protobuf-java-2.5.0.jar"/>
<library name="tagsoup-1.2.1.jar"/>
<library name="guava-18.0.jar"/>
<library name="quartz-2.2.0.jar"/>
<library name="tika-core-1.14.jar"/>
<library name="httpclient-4.5.2.jar"/>
<library name="rdf4j-http-client-2.1.3.jar"/>
<library name="tika-parsers-1.14.jar"/>
<library name="httpcore-4.4.5.jar"/>
<library name="rdf4j-http-protocol-2.1.3.jar"/>
<library name="udunits-4.5.5.jar"/>
<library name="httpmime-4.5.2.jar"/>
<library name="rdf4j-model-2.1.3.jar"/>
<library name="vorbis-java-core-0.8.jar"/>
<library name="httpservices-4.5.5.jar"/>
<library name="rdf4j-query-2.1.3.jar"/>
<library name="vorbis-java-tika-0.8.jar"/>
<library name="rdf4j-queryalgebra-evaluation-2.1.3.jar"/>
<library name="woodstox-core-asl-4.4.1.jar"/>
<library name="isoparser-1.1.18.jar"/>
<library name="rdf4j-queryalgebra-model-2.1.3.jar"/>
<library name="xercesImpl-2.10.0.jar"/>
<library name="jackcess-2.1.4.jar"/>
<library name="rdf4j-queryparser-api-2.1.3.jar"/>
<library name="xml-apis-1.4.01.jar"/>
<library name="jackcess-encrypt-2.1.1.jar"/>
<library name="rdf4j-queryparser-serql-2.1.3.jar"/>
<library name="xmlbeans-2.3.0.jar"/>
<library name="jackson-core-2.8.2.jar"/>
<library name="rdf4j-queryparser-sparql-2.1.3.jar"/>
<library name="xmlschema-core-2.1.0.jar"/>
<library name="java-libpst-0.8.1.jar"/>
<library name="rdf4j-queryresultio-api-2.1.3.jar"/>
<library name="xmpcore-5.1.2.jar"/>
<library name="javax.annotation-api-1.2.jar"/>
<library name="rdf4j-queryresultio-sparqlxml-2.1.3.jar"/>
<library name="xz-1.5.jar"/>
<library name="javax.ws.rs-api-2.0.1.jar"/>
<library name="rdf4j-repository-api-2.1.3.jar"/>
</runtime>

<requires>
<import plugin="nutch-extensionpoints"/>
</requires>

<extension id="org.apache.nutch.indexer.bioschemas.BioschemasIndexingFilter"
name="Bioschemas Indexing Filter"
point="org.apache.nutch.indexer.IndexingFilter">
<implementation id="org.apache.nutch.indexer.bioschemas.BioschemasIndexingFilter"
class="org.apache.nutch.indexer.bioschemas.BioschemasIndexingFilter"/>
</extension>
<extension id="org.apache.nutch.parse.bioschemas.BioschemasParserFilter"
name="Bioschemas Parser Filter"
point="org.apache.nutch.parse.HtmlParseFilter">
<implementation id="org.apache.nutch.parse.bioschemas.BioschemasParserFilter"
class="org.apache.nutch.parse.bioschemas.BioschemasParserFilter" />
</extension>
</plugin>
33 changes: 33 additions & 0 deletions index-bioschemas/src/java/org/apache/nutch/indexer/bioschemas/BioschemasIndexingFilter.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
package org.apache.nutch.indexer.bioschemas;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.nutch.crawl.CrawlDatum;
import org.apache.nutch.crawl.Inlinks;
import org.apache.nutch.indexer.IndexingException;
import org.apache.nutch.indexer.IndexingFilter;
import org.apache.nutch.indexer.NutchDocument;
import org.apache.nutch.parse.Parse;

public class BioschemasIndexingFilter implements IndexingFilter{

Configuration conf = null;

@Override
public Configuration getConf() {
return this.conf;
}

@Override
public void setConf(Configuration conf) {
this.conf = conf;
}

@Override
public NutchDocument filter(NutchDocument doc, Parse parse, Text url, CrawlDatum datum, Inlinks inlinks)
throws IndexingException {
doc.add("bioschemas", parse.getData().getParseMeta().get("bioschemas"));
return doc;
}

}
101 changes: 101 additions & 0 deletions index-bioschemas/src/java/org/apache/nutch/parse/bioschemas/BioschemasParserFilter.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
package org.apache.nutch.parse.bioschemas;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringWriter;
import java.lang.invoke.MethodHandles;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Map;

import javax.xml.transform.OutputKeys;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;

import org.apache.any23.extractor.html.TagSoupParser;
import org.apache.any23.extractor.microdata.ItemScope;
import org.apache.any23.extractor.microdata.MicrodataParserReport;
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.metadata.Metadata;
import org.apache.nutch.parse.HTMLMetaTags;
import org.apache.nutch.parse.HtmlParseFilter;
import org.apache.nutch.parse.ParseResult;
import org.apache.nutch.protocol.Content;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.DocumentFragment;
import org.w3c.dom.Node;

public class BioschemasParserFilter implements HtmlParseFilter {

private Configuration conf = null;
private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());

@Override
public Configuration getConf() {
return this.conf;
}

@Override
public void setConf(Configuration conf) {
this.conf = conf;

}

@Override
public ParseResult filter(Content content, ParseResult parseResult, HTMLMetaTags metaTags, DocumentFragment doc) {
try {
Map<String, StringBuilder> jsonItems = parse(nodeToString(doc), content.getUrl());
Metadata metadata = parseResult.get(content.getUrl()).getData().getParseMeta();
StringBuilder jsonOutput = new StringBuilder("{");
jsonItems.forEach((key, value)-> jsonOutput.append(key + ": [" + value + "]"));
jsonOutput.append("}");
metadata.add("bioschemas", jsonOutput.toString());
} catch (IOException e1) {
LOG.error("Failed to parse microdata");
e1.printStackTrace();
}
return parseResult;
}


private String nodeToString(Node node) {
StringWriter sw = new StringWriter();
try {
Transformer t = TransformerFactory.newInstance().newTransformer();
t.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes");
t.transform(new DOMSource(node), new StreamResult(sw));
} catch (TransformerException te) {
System.out.println("nodeToString Transformer Exception");
}
return sw.toString();
}


private Map<String, StringBuilder> parse(String html, String uri) throws IOException {
InputStream stream = new ByteArrayInputStream(html.getBytes(StandardCharsets.UTF_8));
final TagSoupParser tagSoupParser = new TagSoupParser(stream, uri);

final MicrodataParserReport report = org.apache.any23.extractor.microdata.MicrodataParser.getMicrodata(tagSoupParser.getDOM());
final ItemScope[] itemScopes = report.getDetectedItemScopes();
Map<String, StringBuilder> jsonItems = new HashMap<String, StringBuilder>();
for(ItemScope item : itemScopes) {
String type = item.getType().toString();
type = type.substring(type.lastIndexOf('/') + 1);
if(jsonItems.containsKey(type)) {
jsonItems.get(type).append(", " + item.toJSON());
} else {
StringBuilder firstItem = new StringBuilder(item.toJSON());
jsonItems.put(type, firstItem);
}
}

return jsonItems;
}


}

0 comments on commit ac02dc2

Please sign in to comment.