From 1b6344b9cf78f8c681c24b0dba899ff2ede49f8c Mon Sep 17 00:00:00 2001 From: Timothy Redmond Date: Thu, 1 Dec 2022 14:38:29 -0800 Subject: [PATCH 1/6] Undid the broken fix to tests involving enforcement of the :parents attribute values. --- lib/goo/validators/enforce.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/goo/validators/enforce.rb b/lib/goo/validators/enforce.rb index 2e1b9e56..d326839b 100644 --- a/lib/goo/validators/enforce.rb +++ b/lib/goo/validators/enforce.rb @@ -56,8 +56,6 @@ def self.enforce(inst,attr,value) errors_by_opt = {} enforce_opts.each do |opt| case opt - when :class - nil when :unique unless value.nil? dup = Goo::SPARQL::Queries.duplicate_attribute_value?(inst,attr) From c1c1c95cc0a983dc350c2dcf8515183f22b07f16 Mon Sep 17 00:00:00 2001 From: Timothy Redmond Date: Fri, 2 Dec 2022 14:14:14 -0800 Subject: [PATCH 2/6] Added Goo.add_model statements to make up for the fact that there is more than one attribute range model for the :class attribute. Not an ideal solution. --- test/test_model_complex.rb | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/test/test_model_complex.rb b/test/test_model_complex.rb index e2237c1c..9a79da0a 100644 --- a/test/test_model_complex.rb +++ b/test/test_model_complex.rb @@ -185,6 +185,11 @@ def test_multiple_collection() def test_collection() + # This call is not usually necessary as it is usually covered by + # the model declaration above. See the explanation in + # https://github.com/ncbo/goo/commit/0e09816b121750b3bb875a5c24cb79865287fcf4#commitcomment-90304626 + Goo.add_model(:class, Term) + submission = Submission.new(name: "submission1") unless submission.exist? submission.save @@ -313,6 +318,11 @@ def test_two_resources_same_id def test_parents_inverse_children + # This call is not usually necessary as it is usually covered by + # the model declaration above. See the explanation in + # https://github.com/ncbo/goo/commit/0e09816b121750b3bb875a5c24cb79865287fcf4#commitcomment-90304626 + Goo.add_model(:class, Term) + submission = Submission.new(name: "submission1") unless submission.exist? submission.save @@ -320,6 +330,7 @@ def test_parents_inverse_children submission = Submission.find("submission1").first end + terms = Term.in(submission) terms.each do |t| t.delete @@ -653,6 +664,11 @@ def test_empty_pages assert_equal 0, GooTest.count_pattern("GRAPH #{submission.id.to_ntriples} { #{t.id.to_ntriples} ?p ?o . }") end + # This call is not usually necessary as it is usually covered by + # the model declaration above. See the explanation in + # https://github.com/ncbo/goo/commit/0e09816b121750b3bb875a5c24cb79865287fcf4#commitcomment-90304626 + Goo.add_model(:class, Term) + terms = [] 10.times do |i| term = Term.new @@ -679,6 +695,12 @@ def test_empty_pages end def test_readonly_pages_with_include + + # This call is not usually necessary as it is usually covered by + # the model declaration above. See the explanation in + # https://github.com/ncbo/goo/commit/0e09816b121750b3bb875a5c24cb79865287fcf4#commitcomment-90304626 + Goo.add_model(:class, Term) + submission = Submission.new(name: "submission1") unless submission.exist? submission.save @@ -691,6 +713,7 @@ def test_readonly_pages_with_include assert_equal(0, GooTest.count_pattern("GRAPH #{submission.id.to_ntriples} { #{t.id.to_ntriples} ?p ?o . }")) end + terms = [] 10.times do |i| term = Term.new From 9a5d60d6937658dbb62690dcb33039ac03fb7556 Mon Sep 17 00:00:00 2001 From: Alex Skrenchuk Date: Thu, 15 Dec 2022 11:45:28 -0800 Subject: [PATCH 3/6] Add ruby v3.0 to the matrix for unit tests --- .github/workflows/ruby-unit-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ruby-unit-test.yml b/.github/workflows/ruby-unit-test.yml index ccb161f3..c2540c22 100644 --- a/.github/workflows/ruby-unit-test.yml +++ b/.github/workflows/ruby-unit-test.yml @@ -12,7 +12,7 @@ jobs: strategy: fail-fast: false matrix: - ruby-version: ['2.7'] + ruby-version: ['2.7', '3.0'] steps: - uses: actions/checkout@v3 From 70dd9a4e5c792be261ca5b17b64498b37f63ba77 Mon Sep 17 00:00:00 2001 From: mdorf Date: Tue, 24 Jan 2023 16:39:00 -0800 Subject: [PATCH 4/6] implemented #128 (configurable goo) and fixed ncbo/bioportal-project#264 (empty nodes in AG) --- .gitignore | 2 + Rakefile | 1 - config/config.rb.sample | 23 ++++++++++ lib/goo.rb | 11 +---- lib/goo/config/config.rb | 78 ++++++++++++++++++++++++++++++++++ lib/goo/search/search.rb | 8 ++++ lib/goo/sparql/queries.rb | 6 ++- test/app/bioportal.rb | 2 - test/app/models.rb | 2 - test/app/test_app.rb | 2 - test/console.rb | 1 - test/test_basic_persistence.rb | 2 - test/test_cache.rb | 3 -- test/test_case.rb | 30 +------------ test/test_chunks_write.rb | 15 +++++-- test/test_collections.rb | 2 - test/test_dsl_settings.rb | 2 - test/test_enum.rb | 2 - test/test_index.rb | 2 - test/test_inmutable.rb | 2 - test/test_inverse.rb | 2 - test/test_model_complex.rb | 2 - test/test_name_with.rb | 2 - test/test_namespaces.rb | 2 - test/test_read_only.rb | 2 - test/test_schemaless.rb | 2 - test/test_search.rb | 2 - test/test_where.rb | 3 -- 28 files changed, 130 insertions(+), 83 deletions(-) create mode 100644 config/config.rb.sample create mode 100644 lib/goo/config/config.rb diff --git a/.gitignore b/.gitignore index f887556d..5dcefa7c 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,5 @@ doc/ .idea/* projectFilesBackup/* + +config/config.rb \ No newline at end of file diff --git a/Rakefile b/Rakefile index e593ddce..42ddf39d 100644 --- a/Rakefile +++ b/Rakefile @@ -107,7 +107,6 @@ end desc "Console for working with data" task :console do require_relative "test/test_case" - GooTest.configure_goo binding.pry end diff --git a/config/config.rb.sample b/config/config.rb.sample new file mode 100644 index 00000000..12abdccb --- /dev/null +++ b/config/config.rb.sample @@ -0,0 +1,23 @@ +Goo.config do |config| + # 4store + config.goo_backend_name = '4store' + config.goo_port = 8080 + config.goo_host = 'localhost' + config.goo_path_query = '/sparql/' + config.goo_path_data = '/data/' + config.goo_path_update = '/update/' + + # AllegroGraph + # config.goo_backend_name = 'AG' + # config.goo_port = 10035 + # config.goo_host = 'localhost' + # config.goo_path_query = "/repositories/ontoportal" + # config.goo_path_data = "/repositories/ontoportal/statements/" + # config.goo_path_update = "/repositories/ontoportal/statements/" + + config.search_server_url = 'http://localhost:8983/solr/term_search_core1' + config.redis_host = 'localhost' + config.redis_port = 6379 + config.bioportal_namespace = 'http://data.bioontology.org/' + config.queries_debug = false +end \ No newline at end of file diff --git a/lib/goo.rb b/lib/goo.rb index 71df0691..9930d71d 100644 --- a/lib/goo.rb +++ b/lib/goo.rb @@ -12,6 +12,7 @@ require 'uuid' require "cube" +require_relative "goo/config/config" require_relative "goo/sparql/sparql" require_relative "goo/search/search" require_relative "goo/base/base" @@ -78,16 +79,6 @@ def self.add_sparql_backend(name, *opts) @@sparql_backends.freeze end - def self.test_reset - if @@sparql_backends[:main][:query].url.to_s["localhost"].nil? - raise Exception, "only for testing" - end - @@sparql_backends[:main][:query]=Goo::SPARQL::Client.new("http://localhost:9000/sparql/", - {protocol: "1.1", "Content-Type" => "application/x-www-form-urlencoded", - read_timeout: 300, - redis_cache: @@redis_client }) - end - def self.use_cache=(value) @@use_cache = value set_sparql_cache diff --git a/lib/goo/config/config.rb b/lib/goo/config/config.rb new file mode 100644 index 00000000..bd356d31 --- /dev/null +++ b/lib/goo/config/config.rb @@ -0,0 +1,78 @@ +require 'ostruct' + +module Goo + extend self + attr_reader :settings + + @settings = OpenStruct.new + @settings_run = false + + def config(&block) + return if @settings_run + @settings_run = true + + yield @settings if block_given? + + # Set defaults + @settings.goo_backend_name ||= ENV['GOO_BACKEND_NAME'] || '4store' + @settings.goo_port ||= ENV['GOO_PORT'] || 9000 + @settings.goo_host ||= ENV['GOO_HOST'] || 'localhost"' + @settings.goo_path_query ||= ENV['GOO_PATH_QUERY'] || '/sparql/' + @settings.goo_path_data ||= ENV['GOO_PATH_DATA'] || '/data/' + @settings.goo_path_update ||= ENV['GOO_PATH_UPDATE'] || '/update/' + @settings.search_server_url ||= ENV['SEARCH_SERVER_URL'] || 'http://localhost:8983/solr/term_search_core1' + @settings.redis_host ||= ENV['REDIS_HOST'] || 'localhost' + @settings.redis_port ||= ENV['REDIS_PORT'] || 6379 + @settings.bioportal_namespace ||= ENV['BIOPORTAL_NAMESPACE'] || 'http://data.bioontology.org/' + @settings.queries_debug ||= ENV['QUERIES_DEBUG'] || false + + puts "(GOO) >> Using RDF store #{@settings.goo_host}:#{@settings.goo_port}#{@settings.goo_path_query}" + puts "(GOO) >> Using term search server at #{@settings.search_server_url}" + puts "(GOO) >> Using Redis instance at #{@settings.redis_host}:#{@settings.redis_port}" + + connect_goo + end + + def connect_goo + begin + Goo.configure do |conf| + conf.queries_debug(@settings.queries_debug) + conf.add_sparql_backend(:main, + backend_name: @settings.goo_backend_name, + query: "http://#{@settings.goo_host}:#{@settings.goo_port}#{@settings.goo_path_query}", + data: "http://#{@settings.goo_host}:#{@settings.goo_port}#{@settings.goo_path_data}", + update: "http://#{@settings.goo_host}:#{@settings.goo_port}#{@settings.goo_path_update}", + options: { rules: :NONE }) + conf.add_search_backend(:main, service: @settings.search_server_url) + conf.add_redis_backend(host: @settings.goo_redis_host, port: @settings.goo_redis_port) + + conf.add_namespace(:omv, RDF::Vocabulary.new("http://omv.org/ontology/")) + conf.add_namespace(:skos, RDF::Vocabulary.new("http://www.w3.org/2004/02/skos/core#")) + conf.add_namespace(:owl, RDF::Vocabulary.new("http://www.w3.org/2002/07/owl#")) + conf.add_namespace(:rdfs, RDF::Vocabulary.new("http://www.w3.org/2000/01/rdf-schema#")) + conf.add_namespace(:goo, RDF::Vocabulary.new("http://goo.org/default/"), default = true) + conf.add_namespace(:metadata, RDF::Vocabulary.new("http://goo.org/metadata/")) + conf.add_namespace(:foaf, RDF::Vocabulary.new("http://xmlns.com/foaf/0.1/")) + conf.add_namespace(:rdf, RDF::Vocabulary.new("http://www.w3.org/1999/02/22-rdf-syntax-ns#")) + conf.add_namespace(:tiger, RDF::Vocabulary.new("http://www.census.gov/tiger/2002/vocab#")) + conf.add_namespace(:nemo, RDF::Vocabulary.new("http://purl.bioontology.org/NEMO/ontology/NEMO_annotation_properties.owl#")) + conf.add_namespace(:bioportal, RDF::Vocabulary.new(@settings.bioportal_namespace)) + conf.use_cache = false + end + rescue Exception => e + abort("EXITING: Goo cannot connect to triplestore and/or search server:\n #{e}\n#{e.backtrace.join("\n")}") + end + end + + def self.test_reset + if @@sparql_backends[:main][:query].url.to_s["localhost"].nil? + raise Exception, "only for testing" + end + @@sparql_backends[:main][:query] = Goo::SPARQL::Client.new("http://#{@settings.goo_host}:#{@settings.goo_port}#{@settings.goo_path_query}", + {protocol: "1.1", "Content-Type" => "application/x-www-form-urlencoded", + read_timeout: 300, + redis_cache: @@redis_client }) + end + + +end diff --git a/lib/goo/search/search.rb b/lib/goo/search/search.rb index 730f21de..2987b4e2 100644 --- a/lib/goo/search/search.rb +++ b/lib/goo/search/search.rb @@ -65,6 +65,14 @@ def search(q, params={}, connection_name=:main) def indexBatch(collection, connection_name=:main) docs = Array.new collection.each do |c| + + + + # c.bring(:prefLabel) + # binding.pry if c.prefLabel == "biodiversity" + + + docs << c.indexable_object end Goo.search_connection(connection_name).add(docs) diff --git a/lib/goo/sparql/queries.rb b/lib/goo/sparql/queries.rb index 60996ac0..2a49e438 100644 --- a/lib/goo/sparql/queries.rb +++ b/lib/goo/sparql/queries.rb @@ -568,6 +568,10 @@ def self.model_load_sliced(*options) if page offset = (page[:page_i]-1) * page[:page_size] select.slice(offset,page[:page_size]) + # mdorf, 1/12/2023, AllegroGraph returns duplicate results across + # different pages unless the order_by clause is explicitly specified + # see https://github.com/ncbo/bioportal-project/issues/264 + select.order(:id) end select.distinct(true) if query_options && !binding_as @@ -766,7 +770,7 @@ def self.model_load_sliced(*options) if collection.is_a?Array and collection.length == 1 collection_value = collection.first end - if collection.respond_to?:id + if collection.respond_to?(:id) collection_value = collection end end diff --git a/test/app/bioportal.rb b/test/app/bioportal.rb index 2cef2c2b..2b3c57b3 100644 --- a/test/app/bioportal.rb +++ b/test/app/bioportal.rb @@ -4,8 +4,6 @@ require_relative '../test_case' require_relative './query_profiler' -GooTest.configure_goo - module Test module BioPortal class Ontology < Goo::Base::Resource diff --git a/test/app/models.rb b/test/app/models.rb index 5aeb2a2e..876b70df 100644 --- a/test/app/models.rb +++ b/test/app/models.rb @@ -1,7 +1,5 @@ require_relative '../test_case' -GooTest.configure_goo - module Test module Models diff --git a/test/app/test_app.rb b/test/app/test_app.rb index 26a88d60..4d444ad4 100644 --- a/test/app/test_app.rb +++ b/test/app/test_app.rb @@ -1,6 +1,4 @@ require_relative '../test_case' require_relative 'bioportal' -GooTest.configure_goo - binding.pry diff --git a/test/console.rb b/test/console.rb index e64d4adf..39d19aa2 100644 --- a/test/console.rb +++ b/test/console.rb @@ -1,5 +1,4 @@ require_relative "../lib/goo.rb" require_relative "./test_case.rb" -GooTest.configure_goo binding.pry diff --git a/test/test_basic_persistence.rb b/test/test_basic_persistence.rb index 0cafdfbd..665a5d60 100644 --- a/test/test_basic_persistence.rb +++ b/test/test_basic_persistence.rb @@ -1,7 +1,5 @@ require_relative 'test_case' -GooTest.configure_goo - module Dep class Ontology < Goo::Base::Resource model :ontology, name_with: :name diff --git a/test/test_cache.rb b/test/test_cache.rb index aecbdef3..be01f518 100644 --- a/test/test_cache.rb +++ b/test/test_cache.rb @@ -1,7 +1,4 @@ require_relative 'test_case' - -GooTest.configure_goo - require_relative 'models' class TestCache < MiniTest::Unit::TestCase diff --git a/test/test_case.rb b/test/test_case.rb index fa2b3706..1f8ec5be 100644 --- a/test/test_case.rb +++ b/test/test_case.rb @@ -13,6 +13,7 @@ MiniTest::Unit.autorun require_relative "../lib/goo.rb" +require_relative '../config/config' class GooTest @@ -52,35 +53,6 @@ def _run_suite(suite, type) MiniTest::Unit.runner = GooTest::Unit.new - def self.configure_goo - if not Goo.configure? - Goo.configure do |conf| - conf.add_redis_backend(host: "localhost") - conf.add_namespace(:omv, RDF::Vocabulary.new("http://omv.org/ontology/")) - conf.add_namespace(:skos, RDF::Vocabulary.new("http://www.w3.org/2004/02/skos/core#")) - conf.add_namespace(:owl, RDF::Vocabulary.new("http://www.w3.org/2002/07/owl#")) - conf.add_namespace(:rdfs, RDF::Vocabulary.new("http://www.w3.org/2000/01/rdf-schema#")) - conf.add_namespace(:goo, RDF::Vocabulary.new("http://goo.org/default/"),default=true) - conf.add_namespace(:metadata, RDF::Vocabulary.new("http://goo.org/metadata/")) - conf.add_namespace(:foaf, RDF::Vocabulary.new("http://xmlns.com/foaf/0.1/")) - conf.add_namespace(:rdf, RDF::Vocabulary.new("http://www.w3.org/1999/02/22-rdf-syntax-ns#")) - conf.add_namespace(:tiger, RDF::Vocabulary.new("http://www.census.gov/tiger/2002/vocab#")) - conf.add_namespace(:bioportal, RDF::Vocabulary.new("http://data.bioontology.org/")) - conf.add_namespace(:nemo, RDF::Vocabulary.new("http://purl.bioontology.org/NEMO/ontology/NEMO_annotation_properties.owl#")) - conf.add_sparql_backend( - :main, - backend_name: "4store", - query: "http://localhost:9000/sparql/", - data: "http://localhost:9000/data/", - update: "http://localhost:9000/update/", - options: { rules: :NONE } - ) - conf.add_search_backend(:main, service: "http://localhost:8983/solr/term_search_core1") - conf.use_cache = false - end - end - end - def self.triples_for_subject(resource_id) rs = Goo.sparql_query_client.query("SELECT * WHERE { #{resource_id.to_ntriples} ?p ?o . }") count = 0 diff --git a/test/test_chunks_write.rb b/test/test_chunks_write.rb index a7988d40..66d9500f 100644 --- a/test/test_chunks_write.rb +++ b/test/test_chunks_write.rb @@ -1,7 +1,5 @@ require_relative 'test_case' -GooTest.configure_goo - module TestChunkWrite ONT_ID = "http:://example.org/data/nemo" ONT_ID_EXTRA = "http:://example.org/data/nemo/extra" @@ -23,9 +21,20 @@ def self.after_suite def self._delete graphs = [ONT_ID,ONT_ID_EXTRA] url = Goo.sparql_data_client.url + + + # url = File.join(url, "") + + graphs.each do |graph| # This bypasses the chunks stuff - params = { method: :delete, url: "#{url.to_s}#{graph.to_s}", timeout: nil } + delete_url = url.to_s + (Goo.sparql_backend_name.downcase === 'ag' ? "?context=#{CGI.escape(graph.to_s)}" : graph.to_s) + params = { method: :delete, url: delete_url, timeout: nil } + + + binding.pry + + RestClient::Request.execute(params) end end diff --git a/test/test_collections.rb b/test/test_collections.rb index 390ad349..02bc9c3a 100644 --- a/test/test_collections.rb +++ b/test/test_collections.rb @@ -1,7 +1,5 @@ require_relative 'test_case' -GooTest.configure_goo - #collection on attribute class Issue < Goo::Base::Resource model :issue, collection: :owner, name_with: :description diff --git a/test/test_dsl_settings.rb b/test/test_dsl_settings.rb index c444e829..69ab9f6e 100644 --- a/test/test_dsl_settings.rb +++ b/test/test_dsl_settings.rb @@ -1,7 +1,5 @@ require_relative 'test_case' -GooTest.configure_goo - class StatusModel < Goo::Base::Resource model :status_model, name_with: :name attribute :description, enforce: [ :existence, :unique] diff --git a/test/test_enum.rb b/test/test_enum.rb index db41c343..eaf13af2 100644 --- a/test/test_enum.rb +++ b/test/test_enum.rb @@ -1,7 +1,5 @@ require_relative 'test_case' -GooTest.configure_goo - module TestEnum VALUES = ["uploaded","removed","archived"] diff --git a/test/test_index.rb b/test/test_index.rb index 4d781973..bf4b8937 100644 --- a/test/test_index.rb +++ b/test/test_index.rb @@ -1,8 +1,6 @@ require_relative 'test_case' require_relative './app/models' -GooTest.configure_goo - module TestIndex class TestSchemaless < MiniTest::Unit::TestCase diff --git a/test/test_inmutable.rb b/test/test_inmutable.rb index 9d6037c0..0b1a8c2a 100644 --- a/test/test_inmutable.rb +++ b/test/test_inmutable.rb @@ -1,7 +1,5 @@ require_relative 'test_case' -GooTest.configure_goo - module TestInmutable class Status < Goo::Base::Resource model :status, :inmutable, name_with: :code diff --git a/test/test_inverse.rb b/test/test_inverse.rb index e926a572..2fbb4479 100644 --- a/test/test_inverse.rb +++ b/test/test_inverse.rb @@ -1,7 +1,5 @@ require_relative 'test_case' -GooTest.configure_goo - class Task < Goo::Base::Resource model :task, name_with: :description attribute :description, enforce: [ :existence, :unique] diff --git a/test/test_model_complex.rb b/test/test_model_complex.rb index e2237c1c..1b8f604a 100644 --- a/test/test_model_complex.rb +++ b/test/test_model_complex.rb @@ -1,7 +1,5 @@ require_relative 'test_case' -GooTest.configure_goo - module TestComplex class Submission < Goo::Base::Resource diff --git a/test/test_name_with.rb b/test/test_name_with.rb index 7ba4df42..c2f226a4 100644 --- a/test/test_name_with.rb +++ b/test/test_name_with.rb @@ -1,7 +1,5 @@ require_relative 'test_case' -GooTest.configure_goo - class NameWith < Goo::Base::Resource model :name_with, name_with: lambda { |s| id_generator(s) } attribute :name, enforce: [ :existence, :string, :unique ] diff --git a/test/test_namespaces.rb b/test/test_namespaces.rb index 78ba9a93..6c4bddc0 100644 --- a/test/test_namespaces.rb +++ b/test/test_namespaces.rb @@ -1,7 +1,5 @@ require_relative 'test_case' -GooTest.configure_goo - class NamespacesModel < Goo::Base::Resource model :namespaces, namespace: :rdfs, name_with: :name attribute :name, enforce: [ :existence, :string, :unique ], namespace: :skos diff --git a/test/test_read_only.rb b/test/test_read_only.rb index 4496c463..0b139e18 100644 --- a/test/test_read_only.rb +++ b/test/test_read_only.rb @@ -1,8 +1,6 @@ require_relative 'test_case' require_relative 'test_where' -GooTest.configure_goo - module TestReadOnly class TestReadOnlyWithStruct < TestWhere diff --git a/test/test_schemaless.rb b/test/test_schemaless.rb index f95a17d5..5d0b4bdd 100644 --- a/test/test_schemaless.rb +++ b/test/test_schemaless.rb @@ -1,7 +1,5 @@ require_relative 'test_case' -GooTest.configure_goo - module TestSChemaless ONT_ID = "http:://example.org/data/nemo" diff --git a/test/test_search.rb b/test/test_search.rb index 433dee86..180062d1 100644 --- a/test/test_search.rb +++ b/test/test_search.rb @@ -1,7 +1,5 @@ require_relative 'test_case' -GooTest.configure_goo - module TestSearch class TermSearch < Goo::Base::Resource diff --git a/test/test_where.rb b/test/test_where.rb index 1f5f9634..41201f7c 100644 --- a/test/test_where.rb +++ b/test/test_where.rb @@ -1,7 +1,4 @@ require_relative 'test_case' - -GooTest.configure_goo - require_relative 'models' class TestWhere < MiniTest::Unit::TestCase From 21eaa18f64f9834cc875a4becdc30bf5efb56a97 Mon Sep 17 00:00:00 2001 From: mdorf Date: Tue, 24 Jan 2023 16:40:03 -0800 Subject: [PATCH 5/6] Gemfile.lock update --- Gemfile.lock | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index a1dff571..2af590e4 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -34,7 +34,7 @@ GEM public_suffix (>= 2.0.2, < 6.0) builder (3.2.4) coderay (1.1.3) - concurrent-ruby (1.1.10) + concurrent-ruby (1.2.0) connection_pool (2.3.0) cube-ruby (0.0.3) daemons (1.4.1) @@ -42,7 +42,7 @@ GEM domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) eventmachine (1.2.7) - faraday (1.10.2) + faraday (1.10.3) faraday-em_http (~> 1.0) faraday-em_synchrony (~> 1.0) faraday-excon (~> 1.1) @@ -70,7 +70,7 @@ GEM domain_name (~> 0.5) i18n (0.9.5) concurrent-ruby (~> 1.0) - json_pure (2.6.2) + json_pure (2.6.3) macaddr (1.7.2) systemu (~> 2.6.5) method_source (1.0.0) @@ -84,23 +84,23 @@ GEM ruby2_keywords (~> 0.0.1) net-http-persistent (2.9.4) netrc (0.11.0) - pry (0.14.1) + pry (0.14.2) coderay (~> 1.1) method_source (~> 1.0) - public_suffix (5.0.0) - rack (2.2.4) + public_suffix (5.0.1) + rack (2.2.6.2) rack-accept (0.4.5) rack (>= 0.4) rack-post-body-to-params (0.1.8) activesupport (>= 2.3) - rack-protection (3.0.2) + rack-protection (3.0.5) rack rake (13.0.6) rdf (1.0.8) addressable (>= 2.2) - redis (5.0.5) + redis (5.0.6) redis-client (>= 0.9.0) - redis-client (0.11.0) + redis-client (0.12.1) connection_pool rest-client (2.1.0) http-accept (>= 1.7.0, < 2.0) @@ -111,16 +111,16 @@ GEM builder (>= 2.1.2) faraday (>= 0.9, < 3, != 2.0.0) ruby2_keywords (0.0.5) - simplecov (0.21.2) + simplecov (0.22.0) docile (~> 1.1) simplecov-html (~> 0.11) simplecov_json_formatter (~> 0.1) simplecov-html (0.12.3) simplecov_json_formatter (0.1.4) - sinatra (3.0.2) + sinatra (3.0.5) mustermann (~> 3.0) rack (~> 2.2, >= 2.2.4) - rack-protection (= 3.0.2) + rack-protection (= 3.0.5) tilt (~> 2.0) systemu (2.6.5) thin (1.8.1) From 15023141f6051d4fa6cba6081d082c720327b0c9 Mon Sep 17 00:00:00 2001 From: mdorf Date: Mon, 30 Jan 2023 16:04:44 -0800 Subject: [PATCH 6/6] Gemfile.lock update --- Gemfile.lock | 2 +- test/test_chunks_write.rb | 84 ++++++++++++++++++--------------------- 2 files changed, 39 insertions(+), 47 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 96a767fc..5b282424 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -79,7 +79,7 @@ GEM mime-types-data (3.2022.0105) minitest (4.7.5) multi_json (1.15.0) - multipart-post (2.2.3) + multipart-post (2.3.0) mustermann (3.0.0) ruby2_keywords (~> 0.0.1) net-http-persistent (2.9.4) diff --git a/test/test_chunks_write.rb b/test/test_chunks_write.rb index 66d9500f..64281772 100644 --- a/test/test_chunks_write.rb +++ b/test/test_chunks_write.rb @@ -1,11 +1,14 @@ require_relative 'test_case' module TestChunkWrite - ONT_ID = "http:://example.org/data/nemo" - ONT_ID_EXTRA = "http:://example.org/data/nemo/extra" + ONT_ID = "http://example.org/data/nemo" + ONT_ID_EXTRA = "http://example.org/data/nemo/extra" class TestChunkWrite < MiniTest::Unit::TestCase + BACKEND_4STORE = '4store' + BACKEND_AG = 'ag' + def initialize(*args) super(*args) end @@ -19,24 +22,13 @@ def self.after_suite end def self._delete - graphs = [ONT_ID,ONT_ID_EXTRA] + graphs = [ONT_ID, ONT_ID_EXTRA] url = Goo.sparql_data_client.url - - - # url = File.join(url, "") - - - graphs.each do |graph| + graphs.each { |graph| # This bypasses the chunks stuff - delete_url = url.to_s + (Goo.sparql_backend_name.downcase === 'ag' ? "?context=#{CGI.escape(graph.to_s)}" : graph.to_s) - params = { method: :delete, url: delete_url, timeout: nil } - - - binding.pry - - + params = self.params_for_backend(:delete, graph.to_s) RestClient::Request.execute(params) - end + } end def test_put_data @@ -84,14 +76,7 @@ def test_reentrant_queries ntriples_file_path = "./test/data/nemo_ontology.ntriples" # Bypass in chunks - url = Goo.sparql_data_client.url - params = { - method: :put, - url: "#{url.to_s}#{ONT_ID}", - payload: File.read(ntriples_file_path), - headers: {content_type: "application/x-turtle"}, - timeout: nil - } + params = self.class.params_for_backend(:put, ONT_ID, ntriples_file_path) RestClient::Request.execute(params) tput = Thread.new { @@ -146,16 +131,7 @@ def test_reentrant_queries def test_query_flood ntriples_file_path = "./test/data/nemo_ontology.ntriples" - - # Bypass in chunks - url = Goo.sparql_data_client.url - params = { - method: :put, - url: "#{url.to_s}#{ONT_ID}", - payload: File.read(ntriples_file_path), - headers: {content_type: "application/x-turtle"}, - timeout: nil - } + params = self.class.params_for_backend(:put, ONT_ID, ntriples_file_path) RestClient::Request.execute(params) tput = Thread.new { @@ -174,21 +150,37 @@ def test_query_flood } end - log_status = [] - Thread.new { - 10.times do |i| - log_status << Goo.sparql_query_client.status - sleep(1.2) + if Goo.sparql_backend_name.downcase === BACKEND_4STORE + log_status = [] + Thread.new { + 10.times do |i| + log_status << Goo.sparql_query_client.status + sleep(1.2) + end + } + + threads.each do |t| + t.join end - } + tput.join - threads.each do |t| - t.join + assert log_status.map { |x| x[:outstanding] }.max > 0 + assert_equal 16, log_status.map { |x| x[:running] }.max end - tput.join + end - assert log_status.map { |x| x[:outstanding] }.max > 0 - assert_equal 16, log_status.map { |x| x[:running] }.max + def self.params_for_backend(method, graph_name, ntriples_file_path = nil) + url = Goo.sparql_data_client.url + params = {method: method, headers: {content_type: "application/x-turtle"}, timeout: nil} + + if Goo.sparql_backend_name.downcase === BACKEND_AG + params[:url] = "#{url.to_s}?context=%22#{CGI.escape(graph_name)}%22" + params[:payload] = File.read(ntriples_file_path) if ntriples_file_path + else + params[:url] = "#{url.to_s}#{graph_name}" + params[:payload] = File.read(ntriples_file_path) if ntriples_file_path + end + params end end