diff --git a/.github/workflows/ruby-unit-test.yml b/.github/workflows/ruby-unit-test.yml index 988859c2..906e1ad5 100644 --- a/.github/workflows/ruby-unit-test.yml +++ b/.github/workflows/ruby-unit-test.yml @@ -16,7 +16,7 @@ jobs: triplestore: ['fs', 'ag'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install Dependencies run: sudo apt-get -y install raptor2-utils - name: Set up Ruby @@ -30,5 +30,8 @@ jobs: - name: Run tests run: bundle exec rake test:docker:${{ matrix.triplestore }} TESTOPTS="-v" - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - + uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} + flags: unittests + verbose: true diff --git a/Gemfile b/Gemfile index b6e65383..bfbe2049 100644 --- a/Gemfile +++ b/Gemfile @@ -21,4 +21,4 @@ group :profiling do gem "thin" end -gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'master' +gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'develop' diff --git a/Gemfile.lock b/Gemfile.lock index d3c6d348..12860fa5 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ GIT remote: https://github.com/ncbo/sparql-client.git - revision: d418d56a6c9ff5692f925b45739a2a1c66bca851 - branch: master + revision: 55e7dbf858eb571c767bc67868f9af61663859cb + branch: develop specs: sparql-client (1.0.1) json_pure (>= 1.4) @@ -35,7 +35,7 @@ GEM base64 (0.2.0) builder (3.2.4) coderay (1.1.3) - concurrent-ruby (1.2.2) + concurrent-ruby (1.2.3) connection_pool (2.4.1) cube-ruby (0.0.3) daemons (1.4.1) @@ -58,7 +58,7 @@ GEM method_source (1.0.0) mime-types (3.5.2) mime-types-data (~> 3.2015) - mime-types-data (3.2023.1205) + mime-types-data (3.2024.0305) minitest (4.7.5) multi_json (1.15.0) mustermann (3.0.0) @@ -69,7 +69,7 @@ GEM coderay (~> 1.1) method_source (~> 1.0) public_suffix (5.0.4) - rack (2.2.8) + rack (2.2.8.1) rack-accept (0.4.5) rack (>= 0.4) rack-post-body-to-params (0.1.8) @@ -80,9 +80,9 @@ GEM rake (13.1.0) rdf (1.0.8) addressable (>= 2.2) - redis (5.0.8) + redis (5.1.0) redis-client (>= 0.17.0) - redis-client (0.19.1) + redis-client (0.21.0) connection_pool rest-client (2.1.0) http-accept (>= 1.7.0, < 2.0) @@ -140,4 +140,4 @@ DEPENDENCIES uuid BUNDLED WITH - 2.3.22 + 2.4.22 diff --git a/docker-compose.yml b/docker-compose.yml index a582512f..71b2fe70 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,7 @@ services: retries: 30 solr-ut: - image: ontoportal/solr-ut:0.0.2 + image: ontoportal/solr-ut:0.1.0 ports: - 8983:8983 healthcheck: @@ -21,7 +21,7 @@ services: retries: 5 agraph-ut: - image: franzinc/agraph:v8.0.0 + image: franzinc/agraph:v8.1.0 platform: linux/amd64 environment: - AGRAPH_SUPER_USER=test @@ -38,12 +38,12 @@ services: ; agtool users add anonymous ; agtool users grant anonymous root:ontoportal_test:rw ; tail -f /agraph/data/agraph.log" - # healthcheck: - # test: ["CMD-SHELL", "curl -m 1 -sf http://127.0.0.1:10035/repositories/ontoportal_test/status | grep -iqE '(^running|^lingering)' || exit 1"] - # start_period: 60s - # interval: 10s - # timeout: 5s - # retries: 5 + healthcheck: + test: ["CMD-SHELL", "curl -m 1 -sf http://127.0.0.1:10035/repositories/ontoportal_test/status | grep -iqE '(^running|^lingering)' || exit 1"] + start_period: 60s + interval: 10s + timeout: 5s + retries: 5 profiles: - ag diff --git a/lib/goo/sparql/client.rb b/lib/goo/sparql/client.rb index 35b60b5f..8f7ad9e1 100644 --- a/lib/goo/sparql/client.rb +++ b/lib/goo/sparql/client.rb @@ -83,41 +83,28 @@ def delete_data_graph(graph) def append_triples_no_bnodes(graph,file_path,mime_type_in) bnodes_filter = nil dir = nil - - if file_path.end_with?("ttl") + response = nil + if file_path.end_with?('ttl') bnodes_filter = file_path else - bnodes_filter,dir = bnodes_filter_file(file_path,mime_type_in) - end - mime_type = "text/turtle" - - if mime_type_in == "text/x-nquads" - mime_type = "text/x-nquads" - graph = "http://data.bogus.graph/uri" + bnodes_filter, dir = bnodes_filter_file(file_path, mime_type_in) + end + chunk_lines = 500_000 # number of line + file = File.foreach(bnodes_filter) + lines = [] + file.each_entry do |line| + lines << line + if lines.size == chunk_lines + response = execute_append_request graph, lines.join, mime_type_in + lines.clear + end end - data_file = File.read(bnodes_filter) - params = {method: :post, url: "#{url.to_s}", headers: {"content-type" => mime_type, "mime-type" => mime_type}, timeout: nil} - backend_name = Goo.sparql_backend_name - - if backend_name == BACKEND_4STORE - params[:payload] = { - graph: graph.to_s, - data: data_file, - "mime-type" => mime_type - } - #for some reason \\\\ breaks parsing - params[:payload][:data] = params[:payload][:data].split("\n").map { |x| x.sub("\\\\","") }.join("\n") - else - params[:url] << "?context=#{CGI.escape("<#{graph.to_s}>")}" - params[:payload] = data_file - end + response = execute_append_request graph, lines.join, mime_type_in unless lines.empty? - response = RestClient::Request.execute(params) unless dir.nil? File.delete(bnodes_filter) - begin FileUtils.rm_rf(dir) rescue => e @@ -196,6 +183,35 @@ def status resp[:outstanding] = outstanding resp end + + private + + def execute_append_request(graph, data_file, mime_type_in) + mime_type = "text/turtle" + + if mime_type_in == "text/x-nquads" + mime_type = "text/x-nquads" + graph = "http://data.bogus.graph/uri" + end + + params = {method: :post, url: "#{url.to_s}", headers: {"content-type" => mime_type, "mime-type" => mime_type}, timeout: nil} + backend_name = Goo.sparql_backend_name + + if backend_name == BACKEND_4STORE + params[:payload] = { + graph: graph.to_s, + data: data_file, + 'mime-type' => mime_type + } + #for some reason \\\\ breaks parsing + params[:payload][:data] = params[:payload][:data].split("\n").map { |x| x.sub("\\\\","") }.join("\n") + else + params[:url] << "?context=#{CGI.escape("<#{graph.to_s}>")}" + params[:payload] = data_file + end + + RestClient::Request.execute(params) + end end end end diff --git a/lib/goo/sparql/solutions_mapper.rb b/lib/goo/sparql/solutions_mapper.rb index 011904bc..4fbc7dba 100644 --- a/lib/goo/sparql/solutions_mapper.rb +++ b/lib/goo/sparql/solutions_mapper.rb @@ -35,10 +35,14 @@ def map_each_solutions(select) list_attributes = Set.new(klass.attributes(:list)) all_attributes = Set.new(klass.attributes(:all)) - # for using prefixes before queries - # mdorf, 7/27/2023, AllegroGraph supplied a patch (rfe17161-7.3.1.fasl.patch) - # that enables implicit internal ordering. The patch requires the prefix below - select.prefix('franzOption_imposeImplicitBasicOrdering: ') if @options[:page] + if @options[:page] + # for using prefixes before queries + # mdorf, 7/27/2023, AllegroGraph supplied a patch (rfe17161-7.3.1.fasl.patch) + # that enables implicit internal ordering. The patch requires the prefix below + select.prefix('franzOption_imposeImplicitBasicOrdering: ') + # mdorf, 1/24/2024, AllegroGraph 8 introduced a new feature that allows caching OFFSET/LIMIT queries + select.prefix('franzOption_allowCachingResults: ') + end # for troubleshooting specific queries (write 1 of 3) # ont_to_parse = 'OAE'