diff --git a/.github/workflows/qc.yml b/.github/workflows/qc.yml
index 07909b2..bab048d 100644
--- a/.github/workflows/qc.yml
+++ b/.github/workflows/qc.yml
@@ -19,15 +19,15 @@ jobs:
ontology_qc:
# The type of runner that the job will run on
runs-on: ubuntu-latest
- container: obolibrary/odkfull:v1.4
+ container: obolibrary/odkfull:v1.5.3
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v3
- name: Run ontology QC checks
env:
DEFAULT_BRANCH: master
- run: cd src/ontology && make ROBOT_ENV='ROBOT_JAVA_ARGS=-Xmx6G' test IMP=false PAT=false
+ run: cd src/ontology && make ROBOT_ENV='ROBOT_JAVA_ARGS=-Xmx6G' test IMP=false PAT=false MIR=false
diff --git a/docs/odk-workflows/RepoManagement.md b/docs/odk-workflows/RepoManagement.md
index 92531ca..1b4a6f0 100644
--- a/docs/odk-workflows/RepoManagement.md
+++ b/docs/odk-workflows/RepoManagement.md
@@ -2,7 +2,7 @@
## Updating your ODK repository
-Your ODK repositories configuration is managed in `src/ontology/omo-odk.yaml`. Once you have made your changes, you can run the following to apply your changes to the repository:
+Your ODK repositories configuration is managed in `src/ontology/omo-odk.yaml`. The [ODK Project Configuration Schema](https://github.com/INCATools/ontology-development-kit/blob/master/docs/project-schema.md) defines all possible parameters that can be used in this config YAML. Once you have made your changes, you can run the following to apply your changes to the repository:
```
diff --git a/src/ontology/Makefile b/src/ontology/Makefile
index 25617eb..119f916 100644
--- a/src/ontology/Makefile
+++ b/src/ontology/Makefile
@@ -1,7 +1,7 @@
# ----------------------------------------
# Makefile for omo
# Generated using ontology-development-kit
-# ODK Version: v1.4.3
+# ODK Version: v1.5.3
# ----------------------------------------
# IMPORTANT: DO NOT EDIT THIS FILE. To override default make goals, use omo.Makefile instead
@@ -9,6 +9,9 @@
# ----------------------------------------
# More information: https://github.com/INCATools/ontology-development-kit/
+# Fingerprint of the configuration file when this Makefile was last generated
+CONFIG_HASH= d1df776cd3bd82b36491a506bef1b4f4df2263a95d7b40914d0e7ea1e185a0f8
+
# ----------------------------------------
# Standard Constants
@@ -44,7 +47,7 @@ REPORT_PROFILE_OPTS = --profile $(ROBOT_PROFILE)
OBO_FORMAT_OPTIONS =
SPARQL_VALIDATION_CHECKS = owldef-self-reference iri-range label-with-iri multiple-replaced_by
SPARQL_EXPORTS = basic-report
-ODK_VERSION_MAKEFILE = v1.4.3
+ODK_VERSION_MAKEFILE = v1.5.3
TODAY ?= $(shell date +%Y-%m-%d)
OBODATE ?= $(shell date +'%d:%m:%Y %H:%M')
@@ -59,6 +62,11 @@ FORMATS = $(sort owl obo json owl)
FORMATS_INCL_TSV = $(sort $(FORMATS) tsv)
RELEASE_ARTEFACTS = $(sort $(ONT)-full )
+ifeq ($(ODK_DEBUG),yes)
+ODK_DEBUG_FILE = debug.log
+SHELL = $(SCRIPTSDIR)/run-command.sh
+endif
+
# ----------------------------------------
# Top-level targets
# ----------------------------------------
@@ -69,12 +77,16 @@ RELEASE_ARTEFACTS = $(sort $(ONT)-full )
all: all_odk
.PHONY: all_odk
-all_odk: odkversion test all_assets
+all_odk: odkversion config_check test custom_reports all_assets
.PHONY: test
test: odkversion reason_test sparql_test robot_reports $(REPORTDIR)/validate_profile_owl2dl_$(ONT).owl.txt
echo "Finished running all tests successfully."
+.PHONY: test
+test_fast:
+ $(MAKE_FAST) test
+
.PHONY: release_diff
release_diff: $(REPORTDIR)/release-diff.md
@@ -85,13 +97,52 @@ reason_test: $(EDIT_PREPROCESSED)
.PHONY: odkversion
odkversion:
- echo "ODK Makefile version: $(ODK_VERSION_MAKEFILE) (this is the version of the ODK with which this Makefile was generated, \
- not the version of the ODK you are running)" &&\
- echo "ROBOT version (ODK): " && $(ROBOT) --version
+ @echo "ODK Makefile $(ODK_VERSION_MAKEFILE)"
+ @odk-info --tools
+.PHONY: config_check
+config_check:
+ @if [ "$$(sha256sum $(ONT)-odk.yaml | cut -c1-64)" = "$(CONFIG_HASH)" ]; then \
+ echo "Repository is up-to-date." ; else \
+ echo "Your ODK configuration has changed since this Makefile was generated. You may need to run 'make update_repo'." ; fi
+
$(TMPDIR) $(REPORTDIR) $(MIRRORDIR) $(IMPORTDIR) $(COMPONENTSDIR) $(SUBSETDIR):
mkdir -p $@
+# ----------------------------------------
+# ODK-managed ROBOT plugins
+# ----------------------------------------
+
+# Make sure ROBOT knows where to find plugins
+export ROBOT_PLUGINS_DIRECTORY=$(TMPDIR)/plugins
+
+# Override this rule in omo.Makefile to install custom plugins
+.PHONY: custom_robot_plugins
+custom_robot_plugins:
+
+
+.PHONY: extra_robot_plugins
+extra_robot_plugins:
+
+
+# Install all ROBOT plugins to the runtime plugins directory
+.PHONY: all_robot_plugins
+all_robot_plugins: $(foreach plugin,$(notdir $(wildcard /tools/robot-plugins/*.jar)),$(ROBOT_PLUGINS_DIRECTORY)/$(plugin)) \
+ $(foreach plugin,$(notdir $(wildcard ../../plugins/*.jar)),$(ROBOT_PLUGINS_DIRECTORY)/$(plugin)) \
+ custom_robot_plugins extra_robot_plugins \
+
+# Default rule to install plugins
+$(ROBOT_PLUGINS_DIRECTORY)/%.jar:
+ @mkdir -p $(ROBOT_PLUGINS_DIRECTORY)
+ @if [ -f ../../plugins/$*.jar ]; then \
+ ln ../../plugins/$*.jar $@ ; \
+ elif [ -f /tools/robot-plugins/$*.jar ]; then \
+ cp /tools/robot-plugins/$*.jar $@ ; \
+ fi
+
+# Specific rules for supplementary plugins defined in configuration
+
+
# ----------------------------------------
# Release assets
# ----------------------------------------
@@ -181,10 +232,10 @@ validate_profile_%: $(REPORTDIR)/validate_profile_owl2dl_%.txt
SPARQL_VALIDATION_QUERIES = $(foreach V,$(SPARQL_VALIDATION_CHECKS),$(SPARQLDIR)/$(V)-violation.sparql)
-sparql_test: $(EDIT_PREPROCESSED) catalog-v001.xml | $(REPORTDIR)
+sparql_test: $(EDIT_PREPROCESSED) | $(REPORTDIR)
ifneq ($(SPARQL_VALIDATION_QUERIES),)
- $(ROBOT) verify --catalog catalog-v001.xml -i $(EDIT_PREPROCESSED) --queries $(SPARQL_VALIDATION_QUERIES) -O $(REPORTDIR)
+ $(ROBOT) verify -i $(EDIT_PREPROCESSED) --queries $(SPARQL_VALIDATION_QUERIES) -O $(REPORTDIR)
endif
# ----------------------------------------
@@ -197,6 +248,14 @@ $(REPORTDIR)/$(SRC)-obo-report.tsv: $(SRCMERGED) | $(REPORTDIR)
$(REPORTDIR)/%-obo-report.tsv: % | $(REPORTDIR)
$(ROBOT) report -i $< $(REPORT_LABEL) $(REPORT_PROFILE_OPTS) --fail-on $(REPORT_FAIL_ON) --print 5 -o $@
+check_for_robot_updates:
+ @cut -f2 "/tools/robot_report_profile.txt" | sort > $(TMPDIR)/sorted_tsv2.txt
+ @cut -f2 "$(ROBOT_PROFILE)" | sort > $(TMPDIR)/sorted_tsv1.txt
+ @comm -23 $(TMPDIR)/sorted_tsv2.txt $(TMPDIR)/sorted_tsv1.txt > $(TMPDIR)/missing.txt
+ @echo "Missing tests:"
+ @cat $(TMPDIR)/missing.txt
+ @rm $(TMPDIR)/sorted_tsv1.txt $(TMPDIR)/sorted_tsv2.txt $(TMPDIR)/missing.txt $(TMPDIR)/report_profile_robot.txt
+
# ----------------------------------------
# Release assets
# ----------------------------------------
@@ -213,7 +272,7 @@ RELEASE_ASSETS = \
$(SUBSET_FILES)
.PHONY: all_assets
-all_assets: $(ASSETS)
+all_assets: $(ASSETS) check_rdfxml_assets
.PHONY: show_assets
show_assets:
@@ -221,7 +280,7 @@ show_assets:
du -sh $(ASSETS)
check_rdfxml_%: %
- @check-rdfxml $<
+ @check-rdfxml $<
.PHONY: check_rdfxml_assets
check_rdfxml_assets: $(foreach product,$(MAIN_PRODUCTS),check_rdfxml_$(product).owl)
@@ -333,12 +392,17 @@ IMP=true # Global parameter to bypass import generation
MIR=true # Global parameter to bypass mirror generation
IMP_LARGE=true # Global parameter to bypass handling of large imports
+ifeq ($(strip $(MIR)),true)
$(MIRRORDIR)/%.owl: mirror-% | $(MIRRORDIR)
- if [ $(IMP) = true ] && [ $(MIR) = true ] && [ -f $(TMPDIR)/mirror-$*.owl ]; then if cmp -s $(TMPDIR)/mirror-$*.owl $@ ; then echo "Mirror identical, ignoring."; else echo "Mirrors different, updating." &&\
+ if [ -f $(TMPDIR)/mirror-$*.owl ]; then if cmp -s $(TMPDIR)/mirror-$*.owl $@ ; then echo "Mirror identical, ignoring."; else echo "Mirrors different, updating." &&\
cp $(TMPDIR)/mirror-$*.owl $@; fi; fi
+else # MIR=false
+$(MIRRORDIR)/%.owl:
+ @echo "Not refreshing $@ because the mirrorring pipeline is disabled (MIR=$(MIR))."
+endif
@@ -477,13 +541,16 @@ update_repo:
update_docs:
mkdocs gh-deploy --config-file ../../mkdocs.yaml
-# Note to future generations: prepending ./ is a safety measure to ensure that
-# the environment does not malicously set `CLEANFILES` to `\`.
+# Note to future generations: computing the real path relative to the
+# current directory is a way to ensure we only clean up directories that
+# are located below the current directory, regardless of the contents of
+# the *DIR variables.
.PHONY: clean
clean:
- [ -n "$(MIRRORDIR)" ] && [ $(MIRRORDIR) != "." ] && [ $(MIRRORDIR) != "/" ] && [ $(MIRRORDIR) != ".." ] && [ -d ./$(MIRRORDIR) ] && rm -rf ./$(MIRRORDIR)/*
- [ -n "$(TMPDIR)" ] && [ $(TMPDIR) != "." ] && [ $(TMPDIR) != "/" ] && [ $(TMPDIR) != ".." ] && [ -d ./$(TMPDIR) ] && rm -rf ./$(TMPDIR)/*
- [ -n "$(UPDATEREPODIR)" ] && [ $(UPDATEREPODIR) != "." ] && [ $(UPDATEREPODIR) != "/" ] && [ $(UPDATEREPODIR) != ".." ] && [ -d ./$(UPDATEREPODIR) ] && rm -rf ./$(UPDATEREPODIR)/*
+ for dir in $(MIRRORDIR) $(TMPDIR) $(UPDATEREPODIR) ; do \
+ reldir=$$(realpath --relative-to=$$(pwd) $$dir) ; \
+ case $$reldir in .*|"") ;; *) rm -rf $$reldir/* ;; esac \
+ done
rm -f $(CLEANFILES)
.PHONY: help
@@ -502,6 +569,7 @@ Core commands:
* prepare_release_fast: Run the entire release pipeline without refreshing imports, recreating components or recompiling patterns.
* update_repo: Update the ODK repository setup using the config file omo-odk.yaml
* test: Running all validation tests
+* test_fast: Runs the test suite, but without updating imports or components
* odkversion: Show the current version of the ODK Makefile and ROBOT.
* clean: Delete all temporary files
* help: Print ODK Usage information
@@ -519,7 +587,7 @@ Imports management:
Editor utilities:
* validate_idranges: Make sure your ID ranges file is formatted correctly
-* normalize_src: Load and safe your omo-edit file after you to make sure its serialised correctly
+* normalize_src: Load and save your omo-edit file after you to make sure its serialised correctly
* explain_unsat: If you have unsatisfiable classes, this command will create a markdown file (tmp/explain_unsat.md) which will explain all your unsatisfiable classes
* validate-all-tsv: Check all your tsv files for possible problems in syntax. Use ALL_TSV_FILES variable to list files
* validate-tsv: Check a tsv file for syntactic problems with tsvalid. Use TSV variable to pass filepath, e.g. make TSV=../my.tsv validate-tsv.
diff --git a/src/ontology/run.sh b/src/ontology/run.sh
index cebc839..833f7e6 100755
--- a/src/ontology/run.sh
+++ b/src/ontology/run.sh
@@ -14,6 +14,8 @@
#
# See README-editors.md for more details.
+set -e
+
if [ -f run.sh.conf ]; then
. ./run.sh.conf
fi
@@ -29,6 +31,22 @@ elif [ -f "$HOME/Library/Application Support/ontology-development-kit/github/tok
GH_TOKEN=$(cat "$HOME/Library/Application Support/ontology-development-kit/github/token")
fi
+# SSH agent socket
+# On macOS, we cannot use $SSH_AUTH_SOCK directly,
+# we need to use a "magic" socket instead.
+case "$(uname)" in
+Darwin)
+ ODK_SSH_AUTH_SOCKET=/run/host-services/ssh-auth.sock
+ ;;
+*)
+ ODK_SSH_AUTH_SOCKET=$SSH_AUTH_SOCK
+ ;;
+esac
+ODK_SSH_BIND=
+if [ -n "$ODK_SSH_AUTH_SOCKET" ]; then
+ ODK_SSH_BIND=",$ODK_SSH_AUTH_SOCKET:/run/host-services/ssh-auth.sock"
+fi
+
ODK_IMAGE=${ODK_IMAGE:-odkfull}
TAG_IN_IMAGE=$(echo $ODK_IMAGE | awk -F':' '{ print $2 }')
if [ -n "$TAG_IN_IMAGE" ]; then
@@ -40,16 +58,28 @@ ODK_TAG=${ODK_TAG:-latest}
ODK_JAVA_OPTS=${ODK_JAVA_OPTS:--Xmx12G}
ODK_DEBUG=${ODK_DEBUG:-no}
+ODK_USER_ID=${ODK_USER_ID:-$(id -u)}
+ODK_GROUP_ID=${ODK_GROUP_ID:-$(id -g)}
+
+# Convert OWLAPI_* environment variables to the OWLAPI as Java options
+# See http://owlcs.github.io/owlapi/apidocs_4/org/semanticweb/owlapi/model/parameters/ConfigurationOptions.html
+# for a list of allowed options
+OWLAPI_OPTIONS_NAMESPACE=org.semanticweb.owlapi.model.parameters.ConfigurationOptions
+for owlapi_var in $(env | sed -n s/^OWLAPI_//p) ; do
+ ODK_JAVA_OPTS="$ODK_JAVA_OPTS -D$OWLAPI_OPTIONS_NAMESPACE.${owlapi_var%=*}=${owlapi_var#*=}"
+done
+
TIMECMD=
if [ x$ODK_DEBUG = xyes ]; then
# If you wish to change the format string, take care of using
# non-breaking spaces (U+00A0) instead of normal spaces, to
# prevent the shell from tokenizing the format string.
- echo "Running ${IMAGE} with ${ODK_JAVA_OPTS} of memory for ROBOT and Java-based pipeline steps."
+ echo "Running obolibrary/${ODK_IMAGE}:${ODK_TAG} with '${ODK_JAVA_OPTS}' as options for ROBOT and other Java-based pipeline steps."
TIMECMD="/usr/bin/time -f ### DEBUG STATS ###\nElapsed time: %E\nPeak memory: %M kb"
fi
+rm -f tmp/debug.log
-VOLUME_BIND=$PWD/../../:/work
+VOLUME_BIND=$PWD/../../:/work$ODK_SSH_BIND
WORK_DIR=/work/src/ontology
if [ -n "$ODK_BINDS" ]; then
@@ -59,14 +89,14 @@ fi
if [ -n "$USE_SINGULARITY" ]; then
singularity exec --cleanenv $ODK_SINGULARITY_OPTIONS \
- --env "ROBOT_JAVA_ARGS=$ODK_JAVA_OPTS,JAVA_OPTS=$ODK_JAVA_OPTS" \
+ --env "ROBOT_JAVA_ARGS=$ODK_JAVA_OPTS,JAVA_OPTS=$ODK_JAVA_OPTS,SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock,ODK_USER_ID=$ODK_USER_ID,ODK_GROUP_ID=$ODK_GROUP_ID,ODK_DEBUG=$ODK_DEBUG" \
--bind $VOLUME_BIND \
-W $WORK_DIR \
docker://obolibrary/$ODK_IMAGE:$ODK_TAG $TIMECMD "$@"
else
BIND_OPTIONS="-v $(echo $VOLUME_BIND | sed 's/,/ -v /')"
docker run $ODK_DOCKER_OPTIONS $BIND_OPTIONS -w $WORK_DIR \
- -e ROBOT_JAVA_ARGS="$ODK_JAVA_OPTS" -e JAVA_OPTS="$ODK_JAVA_OPTS" \
+ -e ROBOT_JAVA_ARGS="$ODK_JAVA_OPTS" -e JAVA_OPTS="$ODK_JAVA_OPTS" -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock -e ODK_USER_ID=$ODK_USER_ID -e ODK_GROUP_ID=$ODK_GROUP_ID -e ODK_DEBUG=$ODK_DEBUG \
--rm -ti obolibrary/$ODK_IMAGE:$ODK_TAG $TIMECMD "$@"
fi
diff --git a/src/scripts/run-command.sh b/src/scripts/run-command.sh
new file mode 100755
index 0000000..45d431d
--- /dev/null
+++ b/src/scripts/run-command.sh
@@ -0,0 +1,4 @@
+#!/bin/sh
+ODK_DEBUG_FILE=${ODK_DEBUG_FILE:-debug.log}
+echo "Command: sh $@" >> $ODK_DEBUG_FILE
+/usr/bin/time -a -o $ODK_DEBUG_FILE -f "Elapsed time: %E\nPeak memory: %M kb" /bin/sh "$@"
diff --git a/src/scripts/update_repo.sh b/src/scripts/update_repo.sh
index 3ff4660..40f8ea7 100644
--- a/src/scripts/update_repo.sh
+++ b/src/scripts/update_repo.sh
@@ -26,7 +26,8 @@ cp target/$OID/src/ontology/run.sh $SRCDIR/ontology/
cp -r target/$OID/src/sparql/* $SRCDIR/sparql/
mkdir -p $ROOTDIR/.github
mkdir -p $ROOTDIR/.github/workflows
-cp -n target/$OID/.github/workflows/qc.yml $ROOTDIR/.github/workflows/qc.yml
+cp target/$OID/.github/workflows/qc.yml $ROOTDIR/.github/workflows/qc.yml
+
cp target/$OID/.github/workflows/docs.yml $ROOTDIR/.github/workflows/docs.yml
@@ -35,5 +36,5 @@ cp target/$OID/.github/workflows/docs.yml $ROOTDIR/.github/workflows/docs.yml
cp -n target/$OID/mkdocs.yaml $ROOTDIR/
echo "WARNING: These files should be manually migrated: mkdocs.yaml, .gitignore, src/ontology/catalog.xml (if you added a new import or component)"
-echo "WARNING: Your QC workflows have not been updated automatically. Please update the ODK version number(s) in .github/workflows/qc.yml."
+
echo "Ontology repository update successfully completed."
\ No newline at end of file
diff --git a/src/sparql/class-count-by-prefix.sparql b/src/sparql/class-count-by-prefix.sparql
index 0c4d284..a6a4851 100644
--- a/src/sparql/class-count-by-prefix.sparql
+++ b/src/sparql/class-count-by-prefix.sparql
@@ -7,4 +7,4 @@ SELECT ?prefix (COUNT(DISTINCT ?cls) AS ?numberOfClasses) WHERE
FILTER (!isBlank(?cls))
BIND( STRBEFORE(STRAFTER(str(?cls),"http://purl.obolibrary.org/obo/"), "_") AS ?prefix)
}
-GROUP BY ?prefix
+GROUP BY ?prefix
\ No newline at end of file
diff --git a/src/sparql/dc-properties-violation.sparql b/src/sparql/dc-properties-violation.sparql
new file mode 100644
index 0000000..106ee1a
--- /dev/null
+++ b/src/sparql/dc-properties-violation.sparql
@@ -0,0 +1,11 @@
+# The purpose of this violation is to make sure people update
+# from using the deprecated DC Elements 1.1 namespace (http://purl.org/dc/elements/1.1/)
+# to using the recommended DC Terms namespace (http://purl.org/dc/terms/)
+# See also discussion on https://github.com/oborel/obo-relations/pull/692
+
+SELECT ?term ?predicate WHERE {
+ ?term ?predicate ?value .
+ FILTER(STRSTARTS(STR(?predicate), "http://purl.org/dc/elements/1.1/"))
+ FILTER(isIRI(?term) && (STRSTARTS(str(?term), "http://purl.obolibrary.org/obo/OMO_") || STRSTARTS(str(?term), "http://purl.obolibrary.org/obo/IAO_") || STRSTARTS(str(?term), "http://www.geneontology.org/formats/oboInOwl#")))
+}
+
diff --git a/src/sparql/illegal-date-violation.sparql b/src/sparql/illegal-date-violation.sparql
index 25ec660..9e05358 100644
--- a/src/sparql/illegal-date-violation.sparql
+++ b/src/sparql/illegal-date-violation.sparql
@@ -1,10 +1,12 @@
PREFIX dct:
PREFIX xsd:
+PREFIX oboInOwl:
SELECT DISTINCT ?term ?property ?value WHERE
{
- VALUES ?property {dct:date dct:issued dct:created}
+ VALUES ?property {dct:date dct:issued dct:created oboInOwl:creation_date}
?term ?property ?value .
FILTER (datatype(?value) != xsd:date || !regex(str(?value), '^\\d{4}-\\d\\d-\\d\\d$'))
+ FILTER (datatype(?value) != xsd:dateTime || !regex(str(?value), '^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}Z'))
}