Merge branch 'master' into pr/refactor/resources-validators

ncbo · Feb 28, 2023 · 29f3527 · 29f3527
2 parents 46d1b9f + 919c20d
commit 29f3527
Show file tree

Hide file tree

Showing 31 changed files with 196 additions and 133 deletions.
diff --git a/.github/workflows/ruby-unit-test.yml b/.github/workflows/ruby-unit-test.yml
@@ -12,7 +12,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby-version: ['2.7']
+        ruby-version: ['2.7', '3.0']
 
     steps:
     - uses: actions/checkout@v3

diff --git a/.gitignore b/.gitignore
@@ -25,3 +25,5 @@ doc/
 
 .idea/*
 projectFilesBackup/*
+
+config/config.rb
diff --git a/Gemfile.lock b/Gemfile.lock
@@ -34,15 +34,15 @@ GEM
       public_suffix (>= 2.0.2, < 6.0)
     builder (3.2.4)
     coderay (1.1.3)
-    concurrent-ruby (1.1.10)
+    concurrent-ruby (1.2.0)
     connection_pool (2.3.0)
     cube-ruby (0.0.3)
     daemons (1.4.1)
     docile (1.4.0)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
     eventmachine (1.2.7)
-    faraday (1.10.2)
+    faraday (1.10.3)
       faraday-em_http (~> 1.0)
       faraday-em_synchrony (~> 1.0)
       faraday-excon (~> 1.1)
@@ -70,7 +70,7 @@ GEM
       domain_name (~> 0.5)
     i18n (0.9.5)
       concurrent-ruby (~> 1.0)
-    json_pure (2.6.2)
+    json_pure (2.6.3)
     macaddr (1.7.2)
       systemu (~> 2.6.5)
     method_source (1.0.0)
@@ -79,28 +79,28 @@ GEM
     mime-types-data (3.2022.0105)
     minitest (4.7.5)
     multi_json (1.15.0)
-    multipart-post (2.2.3)
+    multipart-post (2.3.0)
     mustermann (3.0.0)
       ruby2_keywords (~> 0.0.1)
     net-http-persistent (2.9.4)
     netrc (0.11.0)
-    pry (0.14.1)
+    pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    public_suffix (5.0.0)
-    rack (2.2.4)
+    public_suffix (5.0.1)
+    rack (2.2.6.2)
     rack-accept (0.4.5)
       rack (>= 0.4)
     rack-post-body-to-params (0.1.8)
       activesupport (>= 2.3)
-    rack-protection (3.0.3)
+    rack-protection (3.0.5)
       rack
     rake (13.0.6)
     rdf (1.0.8)
       addressable (>= 2.2)
-    redis (5.0.5)
+    redis (5.0.6)
       redis-client (>= 0.9.0)
-    redis-client (0.11.1)
+    redis-client (0.12.1)
       connection_pool
     rest-client (2.1.0)
       http-accept (>= 1.7.0, < 2.0)
@@ -112,7 +112,7 @@ GEM
       builder (>= 2.1.2)
       faraday (>= 0.9, < 3, != 2.0.0)
     ruby2_keywords (0.0.5)
-    simplecov (0.21.2)
+    simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
@@ -121,10 +121,10 @@ GEM
       simplecov (~> 0.19)
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
-    sinatra (3.0.3)
+    sinatra (3.0.5)
       mustermann (~> 3.0)
       rack (~> 2.2, >= 2.2.4)
-      rack-protection (= 3.0.3)
+      rack-protection (= 3.0.5)
       tilt (~> 2.0)
     systemu (2.6.5)
     thin (1.8.1)
@@ -163,4 +163,4 @@ DEPENDENCIES
   uuid
 
 BUNDLED WITH
-   2.3.22
+   2.3.15
diff --git a/Rakefile b/Rakefile
@@ -107,7 +107,6 @@ end
 desc "Console for working with data"
 task :console do
   require_relative "test/test_case"
-  GooTest.configure_goo
   binding.pry
 end
 

diff --git a/config/config.rb.sample b/config/config.rb.sample
@@ -0,0 +1,23 @@
+Goo.config do |config|
+  # 4store
+  config.goo_backend_name    = '4store'
+  config.goo_port            = 8080
+  config.goo_host            = 'localhost'
+  config.goo_path_query      = '/sparql/'
+  config.goo_path_data       = '/data/'
+  config.goo_path_update     = '/update/'
+
+  # AllegroGraph
+  # config.goo_backend_name  = 'AG'
+  # config.goo_port          = 10035
+  # config.goo_host          = 'localhost'
+  # config.goo_path_query    = "/repositories/ontoportal"
+  # config.goo_path_data     = "/repositories/ontoportal/statements/"
+  # config.goo_path_update   = "/repositories/ontoportal/statements/"
+
+  config.search_server_url   = 'http://localhost:8983/solr/term_search_core1'
+  config.redis_host          = 'localhost'
+  config.redis_port          = 6379
+  config.bioportal_namespace = 'http://data.bioontology.org/'
+  config.queries_debug       = false
+end
diff --git a/lib/goo.rb b/lib/goo.rb
@@ -12,6 +12,7 @@
 require 'uuid'
 require "cube"
 
+require_relative "goo/config/config"
 require_relative "goo/sparql/sparql"
 require_relative "goo/search/search"
 require_relative "goo/base/base"
@@ -82,16 +83,6 @@ def self.add_sparql_backend(name, *opts)
     @@sparql_backends.freeze
   end
 
-  def self.test_reset
-    if @@sparql_backends[:main][:query].url.to_s["localhost"].nil?
-      raise Exception, "only for testing"
-    end
-    @@sparql_backends[:main][:query]=Goo::SPARQL::Client.new("http://localhost:9000/sparql/",
-                 {protocol: "1.1", "Content-Type" => "application/x-www-form-urlencoded",
-                   read_timeout: 300,
-                  redis_cache: @@redis_client })
-  end
-
   def self.use_cache=(value)
     @@use_cache = value
     set_sparql_cache

diff --git a/lib/goo/config/config.rb b/lib/goo/config/config.rb
@@ -0,0 +1,78 @@
+require 'ostruct'
+
+module Goo
+  extend self
+  attr_reader :settings
+
+  @settings = OpenStruct.new
+  @settings_run = false
+
+  def config(&block)
+    return if @settings_run
+    @settings_run = true
+
+    yield @settings if block_given?
+
+    # Set defaults
+    @settings.goo_backend_name    ||= ENV['GOO_BACKEND_NAME'] || '4store'
+    @settings.goo_port            ||= ENV['GOO_PORT'] || 9000
+    @settings.goo_host            ||= ENV['GOO_HOST'] || 'localhost"'
+    @settings.goo_path_query      ||= ENV['GOO_PATH_QUERY'] || '/sparql/'
+    @settings.goo_path_data       ||= ENV['GOO_PATH_DATA'] || '/data/'
+    @settings.goo_path_update     ||= ENV['GOO_PATH_UPDATE'] || '/update/'
+    @settings.search_server_url   ||= ENV['SEARCH_SERVER_URL'] || 'http://localhost:8983/solr/term_search_core1'
+    @settings.redis_host          ||= ENV['REDIS_HOST'] || 'localhost'
+    @settings.redis_port          ||= ENV['REDIS_PORT'] || 6379
+    @settings.bioportal_namespace ||= ENV['BIOPORTAL_NAMESPACE'] || 'http://data.bioontology.org/'
+    @settings.queries_debug       ||= ENV['QUERIES_DEBUG'] || false
+
+    puts "(GOO) >> Using RDF store #{@settings.goo_host}:#{@settings.goo_port}#{@settings.goo_path_query}"
+    puts "(GOO) >> Using term search server at #{@settings.search_server_url}"
+    puts "(GOO) >> Using Redis instance at #{@settings.redis_host}:#{@settings.redis_port}"
+
+    connect_goo
+  end
+
+  def connect_goo
+    begin
+      Goo.configure do |conf|
+        conf.queries_debug(@settings.queries_debug)
+        conf.add_sparql_backend(:main,
+                                backend_name: @settings.goo_backend_name,
+                                query: "http://#{@settings.goo_host}:#{@settings.goo_port}#{@settings.goo_path_query}",
+                                data: "http://#{@settings.goo_host}:#{@settings.goo_port}#{@settings.goo_path_data}",
+                                update: "http://#{@settings.goo_host}:#{@settings.goo_port}#{@settings.goo_path_update}",
+                                options: { rules: :NONE })
+        conf.add_search_backend(:main, service: @settings.search_server_url)
+        conf.add_redis_backend(host: @settings.goo_redis_host, port: @settings.goo_redis_port)
+
+        conf.add_namespace(:omv, RDF::Vocabulary.new("http://omv.org/ontology/"))
+        conf.add_namespace(:skos, RDF::Vocabulary.new("http://www.w3.org/2004/02/skos/core#"))
+        conf.add_namespace(:owl, RDF::Vocabulary.new("http://www.w3.org/2002/07/owl#"))
+        conf.add_namespace(:rdfs, RDF::Vocabulary.new("http://www.w3.org/2000/01/rdf-schema#"))
+        conf.add_namespace(:goo, RDF::Vocabulary.new("http://goo.org/default/"), default = true)
+        conf.add_namespace(:metadata, RDF::Vocabulary.new("http://goo.org/metadata/"))
+        conf.add_namespace(:foaf, RDF::Vocabulary.new("http://xmlns.com/foaf/0.1/"))
+        conf.add_namespace(:rdf, RDF::Vocabulary.new("http://www.w3.org/1999/02/22-rdf-syntax-ns#"))
+        conf.add_namespace(:tiger, RDF::Vocabulary.new("http://www.census.gov/tiger/2002/vocab#"))
+        conf.add_namespace(:nemo, RDF::Vocabulary.new("http://purl.bioontology.org/NEMO/ontology/NEMO_annotation_properties.owl#"))
+        conf.add_namespace(:bioportal, RDF::Vocabulary.new(@settings.bioportal_namespace))
+        conf.use_cache = false
+      end
+    rescue Exception => e
+      abort("EXITING: Goo cannot connect to triplestore and/or search server:\n  #{e}\n#{e.backtrace.join("\n")}")
+    end
+  end
+
+  def self.test_reset
+    if @@sparql_backends[:main][:query].url.to_s["localhost"].nil?
+      raise Exception, "only for testing"
+    end
+    @@sparql_backends[:main][:query] = Goo::SPARQL::Client.new("http://#{@settings.goo_host}:#{@settings.goo_port}#{@settings.goo_path_query}",
+                                                             {protocol: "1.1", "Content-Type" => "application/x-www-form-urlencoded",
+                                                              read_timeout: 300,
+                                                              redis_cache: @@redis_client })
+  end
+
+
+end
diff --git a/lib/goo/search/search.rb b/lib/goo/search/search.rb
@@ -65,6 +65,14 @@ def search(q, params={}, connection_name=:main)
       def indexBatch(collection, connection_name=:main)
         docs = Array.new
         collection.each do |c|
+
+
+
+          # c.bring(:prefLabel)
+          # binding.pry if c.prefLabel == "biodiversity"
+
+
+
           docs << c.indexable_object
         end
         Goo.search_connection(connection_name).add(docs)

diff --git a/lib/goo/sparql/queries.rb b/lib/goo/sparql/queries.rb
@@ -568,6 +568,10 @@ def self.model_load_sliced(*options)
         if page
           offset = (page[:page_i]-1) * page[:page_size]
           select.slice(offset,page[:page_size])
+          # mdorf, 1/12/2023, AllegroGraph returns duplicate results across
+          # different pages unless the order_by clause is explicitly specified
+          # see https://github.com/ncbo/bioportal-project/issues/264
+          select.order(:id)
         end
         select.distinct(true)
         if query_options && !binding_as
@@ -766,7 +770,7 @@ def self.model_load_sliced(*options)
           if collection.is_a?Array and collection.length == 1
             collection_value = collection.first
           end
-          if collection.respond_to?:id
+          if collection.respond_to?(:id)
             collection_value = collection
           end
         end

diff --git a/lib/goo/validators/enforce.rb b/lib/goo/validators/enforce.rb
@@ -70,10 +70,11 @@ def check(validator_class, inst, attr, value, opt, *options)
           validator = validator_class.new(inst, attr, value, *options)
           add_error(opt, validator.error) unless validator.valid?
         end
+
         def enforce_by_attribute(model, attr)
            model.model_settings[:attributes][attr][:enforce]
         end
-
+        
         def call_proc(opt,inst, attr)
           # This should return an array like [:name_of_error1, "Error message 1", :name_of_error2, "Error message 2"]
           errors = opt.call(inst, attr)

diff --git a/test/app/bioportal.rb b/test/app/bioportal.rb
@@ -4,8 +4,6 @@
 require_relative '../test_case'
 require_relative './query_profiler'
 
-GooTest.configure_goo
-
 module Test
   module BioPortal 
     class Ontology < Goo::Base::Resource

diff --git a/test/app/models.rb b/test/app/models.rb
@@ -1,7 +1,5 @@
 require_relative '../test_case'
 
-GooTest.configure_goo
-
 module Test
   module Models
 

diff --git a/test/app/test_app.rb b/test/app/test_app.rb
@@ -1,6 +1,4 @@
 require_relative '../test_case'
 require_relative 'bioportal'
 
-GooTest.configure_goo
-
 binding.pry
diff --git a/test/console.rb b/test/console.rb
@@ -1,5 +1,4 @@
 require_relative "../lib/goo.rb"
 require_relative "./test_case.rb"
 
-GooTest.configure_goo
 binding.pry
diff --git a/test/test_basic_persistence.rb b/test/test_basic_persistence.rb
@@ -1,7 +1,5 @@
 require_relative 'test_case'
 
-GooTest.configure_goo
-
 module Dep
   class Ontology < Goo::Base::Resource
     model :ontology, name_with: :name

diff --git a/test/test_cache.rb b/test/test_cache.rb
@@ -1,7 +1,4 @@
 require_relative 'test_case'
-
-GooTest.configure_goo
-
 require_relative 'models'
 
 class TestCache < MiniTest::Unit::TestCase

diff --git a/test/test_case.rb b/test/test_case.rb
@@ -20,6 +20,7 @@
 MiniTest::Unit.autorun
 
 require_relative "../lib/goo.rb"
+require_relative '../config/config'
 
 class GooTest
 
@@ -59,35 +60,6 @@ def _run_suite(suite, type)
 
   MiniTest::Unit.runner = GooTest::Unit.new
 
-  def self.configure_goo
-    if not Goo.configure?
-      Goo.configure do |conf|
-        conf.add_redis_backend(host: "localhost")
-        conf.add_namespace(:omv, RDF::Vocabulary.new("http://omv.org/ontology/"))
-        conf.add_namespace(:skos, RDF::Vocabulary.new("http://www.w3.org/2004/02/skos/core#"))
-        conf.add_namespace(:owl, RDF::Vocabulary.new("http://www.w3.org/2002/07/owl#"))
-        conf.add_namespace(:rdfs, RDF::Vocabulary.new("http://www.w3.org/2000/01/rdf-schema#"))
-        conf.add_namespace(:goo, RDF::Vocabulary.new("http://goo.org/default/"),default=true)
-        conf.add_namespace(:metadata, RDF::Vocabulary.new("http://goo.org/metadata/"))
-        conf.add_namespace(:foaf, RDF::Vocabulary.new("http://xmlns.com/foaf/0.1/"))
-        conf.add_namespace(:rdf, RDF::Vocabulary.new("http://www.w3.org/1999/02/22-rdf-syntax-ns#"))
-        conf.add_namespace(:tiger, RDF::Vocabulary.new("http://www.census.gov/tiger/2002/vocab#"))
-        conf.add_namespace(:bioportal, RDF::Vocabulary.new("http://data.bioontology.org/"))
-        conf.add_namespace(:nemo, RDF::Vocabulary.new("http://purl.bioontology.org/NEMO/ontology/NEMO_annotation_properties.owl#"))
-        conf.add_sparql_backend(
-          :main, 
-          backend_name: "4store",
-          query: "http://localhost:9000/sparql/",
-          data: "http://localhost:9000/data/",
-          update: "http://localhost:9000/update/",
-          options: { rules: :NONE }
-        )
-        conf.add_search_backend(:main, service: "http://localhost:8983/solr/term_search_core1")
-        conf.use_cache = false
-      end
-    end
-  end
-
   def self.triples_for_subject(resource_id)
     rs = Goo.sparql_query_client.query("SELECT * WHERE { #{resource_id.to_ntriples} ?p ?o . }")
     count = 0
Original file line number	Diff line number	Diff line change
Expand Up		@@ -25,3 +25,5 @@ doc/

		.idea/*
		projectFilesBackup/*

		config/config.rb