From 4df8aae9af75acb0f6ff21fc46a70b9d923fed65 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Sun, 18 Dec 2022 01:25:27 +0100 Subject: [PATCH 01/53] add generalized helpers for request parameters The request parameters are: page, page_size, display, sortby, order --- helpers/request_params_helper.rb | 56 ++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 helpers/request_params_helper.rb diff --git a/helpers/request_params_helper.rb b/helpers/request_params_helper.rb new file mode 100644 index 00000000..3b0163b3 --- /dev/null +++ b/helpers/request_params_helper.rb @@ -0,0 +1,56 @@ +require 'sinatra/base' + +module Sinatra + module Helpers + module RequestParamsHelper + + def settings_params(klass) + page, size = page_params + attributes = get_attributes_to_include(includes_param, klass) + order_by = get_order_by_from(@params) + bring_unmapped = bring_unmapped?(includes_param, klass) + + [attributes, page, size, order_by, bring_unmapped] + end + + def is_set?(param) + !param.nil? && param != "" + end + + + + + + def get_order_by_from(params, default_order = :asc) + if is_set?(params['sortby']) + orders = (params["order"] || default_order.to_s).split(',') + out = params['sortby'].split(',').map.with_index do |param, index| + sort_order_item(param, orders[index] || default_order) + end + out.to_h + end + end + + def get_attributes_to_include(includes_param, klass) + ld = klass.goo_attrs_to_load(includes_param) + ld.delete(:properties) + ld + end + + def bring_unmapped?(includes_param) + (includes_param && includes_param.include?(:all)) + end + + def bring_unmapped_to(page_data, sub, klass) + klass.in(sub).models(page_data).include(:unmapped).all + end + + private + def sort_order_item(param , order) + [param.to_sym, order.to_sym] + end + end + end +end + +helpers Sinatra::Helpers::RequestParamsHelper \ No newline at end of file From 05cb9fffac178c3ed27f0458aa2a2eedbea79196 Mon Sep 17 00:00:00 2001 From: OntoPortal Bot Date: Wed, 21 Dec 2022 01:46:26 +0100 Subject: [PATCH 02/53] [ontoportal-bot] Gemfile.lock update --- Gemfile.lock | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 37e90e9b..1510a87d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -128,7 +128,7 @@ GEM concurrent-ruby (1.1.10) cube-ruby (0.0.3) dante (0.2.0) - date (3.3.2) + date (3.3.3) declarative (0.0.20) docile (1.4.0) domain_name (0.5.20190701) @@ -338,7 +338,6 @@ GEM macaddr (~> 1.0) PLATFORMS - x86_64-darwin-21 x86_64-linux DEPENDENCIES @@ -392,4 +391,4 @@ DEPENDENCIES unicorn-worker-killer BUNDLED WITH - 2.3.23 + 2.3.15 From 59c64a91acfdddd76da0a31c623b0074fa57851f Mon Sep 17 00:00:00 2001 From: OntoPortal Bot Date: Wed, 21 Dec 2022 01:51:34 +0100 Subject: [PATCH 03/53] [ontoportal-bot] Gemfile.lock update --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 1085b56f..5bece252 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -128,7 +128,7 @@ GEM concurrent-ruby (1.1.10) cube-ruby (0.0.3) dante (0.2.0) - date (3.3.2) + date (3.3.3) declarative (0.0.20) docile (1.4.0) domain_name (0.5.20190701) From b9bbc5a40038e6b28b97ff2050a817d2c62ec04a Mon Sep 17 00:00:00 2001 From: OntoPortal Bot Date: Sat, 24 Dec 2022 06:20:34 +0100 Subject: [PATCH 04/53] [ontoportal-bot] Gemfile.lock update --- Gemfile.lock | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 1510a87d..cf89600d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -37,13 +37,13 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ncbo_cron.git - revision: d09999177235029ba30a00cd59d10f7ca1a1fa93 + revision: 3296b76c5287a4b2644f123e736c156f3a6d219d branch: master specs: ncbo_cron (0.0.1) dante goo - google-api-client + google-apis-analytics_v3 mlanett-redis-lock multi_json ncbo_annotator @@ -53,7 +53,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: 9c5c56cf7b4546ccc45431a82bbce9a33184e4cc + revision: c98145605413e459d3c09a41d3c28a5acd0c49e4 branch: master specs: ontologies_linked_data (0.0.1) @@ -160,22 +160,24 @@ GEM ffi (1.15.5) get_process_mem (0.2.7) ffi (~> 1.0) - google-api-client (0.52.0) + google-apis-analytics_v3 (0.12.0) + google-apis-core (>= 0.9.1, < 2.a) + google-apis-core (0.9.2) addressable (~> 2.5, >= 2.5.1) - googleauth (~> 0.9) - httpclient (>= 2.8.1, < 3.0) + googleauth (>= 0.16.2, < 2.a) + httpclient (>= 2.8.1, < 3.a) mini_mime (~> 1.0) representable (~> 3.0) - retriable (>= 2.0, < 4.0) + retriable (>= 2.0, < 4.a) rexml - signet (~> 0.12) - googleauth (0.17.1) - faraday (>= 0.17.3, < 2.0) + webrick + googleauth (1.3.0) + faraday (>= 0.17.3, < 3.a) jwt (>= 1.4, < 3.0) memoist (~> 0.16) multi_json (~> 1.11) os (>= 0.9, < 2.0) - signet (~> 0.15) + signet (>= 0.16, < 2.a) haml (5.2.2) temple (>= 0.8.0) tilt @@ -189,7 +191,7 @@ GEM json-schema (2.8.1) addressable (>= 2.4) json_pure (2.6.3) - jwt (2.5.0) + jwt (2.6.0) kgio (2.11.4) libxml-ruby (3.2.4) logger (1.5.3) @@ -213,7 +215,7 @@ GEM multi_json (1.15.0) multipart-post (2.2.3) net-http-persistent (2.9.4) - net-imap (0.3.2) + net-imap (0.3.4) date net-protocol net-pop (0.1.2) @@ -292,7 +294,7 @@ GEM faraday (>= 0.17.5, < 3.a) jwt (>= 1.5, < 3.0) multi_json (~> 1.10) - simplecov (0.21.2) + simplecov (0.22.0) docile (~> 1.1) simplecov-html (~> 0.11) simplecov_json_formatter (~> 0.1) @@ -336,6 +338,7 @@ GEM unicorn (>= 4, < 7) uuid (2.3.9) macaddr (~> 1.0) + webrick (1.7.0) PLATFORMS x86_64-linux From b31f96f7f0638c95ed8e29e96bf871823a92d621 Mon Sep 17 00:00:00 2001 From: OntoPortal Bot Date: Sun, 1 Jan 2023 18:05:17 +0100 Subject: [PATCH 05/53] [ontoportal-bot] Gemfile.lock update --- Gemfile.lock | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 5bece252..7ecf42c4 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -37,13 +37,13 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ncbo_cron.git - revision: d09999177235029ba30a00cd59d10f7ca1a1fa93 + revision: 3296b76c5287a4b2644f123e736c156f3a6d219d branch: master specs: ncbo_cron (0.0.1) dante goo - google-api-client + google-apis-analytics_v3 mlanett-redis-lock multi_json ncbo_annotator @@ -53,7 +53,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: 9c5c56cf7b4546ccc45431a82bbce9a33184e4cc + revision: 2adf80367f9bf2bfe8ddd870205910350606aa83 branch: master specs: ontologies_linked_data (0.0.1) @@ -160,22 +160,24 @@ GEM ffi (1.15.5) get_process_mem (0.2.7) ffi (~> 1.0) - google-api-client (0.52.0) + google-apis-analytics_v3 (0.12.0) + google-apis-core (>= 0.9.1, < 2.a) + google-apis-core (0.9.2) addressable (~> 2.5, >= 2.5.1) - googleauth (~> 0.9) - httpclient (>= 2.8.1, < 3.0) + googleauth (>= 0.16.2, < 2.a) + httpclient (>= 2.8.1, < 3.a) mini_mime (~> 1.0) representable (~> 3.0) - retriable (>= 2.0, < 4.0) + retriable (>= 2.0, < 4.a) rexml - signet (~> 0.12) - googleauth (0.17.1) - faraday (>= 0.17.3, < 2.0) + webrick + googleauth (1.3.0) + faraday (>= 0.17.3, < 3.a) jwt (>= 1.4, < 3.0) memoist (~> 0.16) multi_json (~> 1.11) os (>= 0.9, < 2.0) - signet (~> 0.15) + signet (>= 0.16, < 2.a) haml (5.2.2) temple (>= 0.8.0) tilt @@ -189,9 +191,9 @@ GEM json-schema (2.8.1) addressable (>= 2.4) json_pure (2.6.3) - jwt (2.5.0) + jwt (2.6.0) kgio (2.11.4) - libxml-ruby (3.2.4) + libxml-ruby (4.0.0) logger (1.5.3) macaddr (1.7.2) systemu (~> 2.6.5) @@ -213,7 +215,7 @@ GEM multi_json (1.15.0) multipart-post (2.2.3) net-http-persistent (2.9.4) - net-imap (0.3.2) + net-imap (0.3.4) date net-protocol net-pop (0.1.2) @@ -292,7 +294,7 @@ GEM faraday (>= 0.17.5, < 3.a) jwt (>= 1.5, < 3.0) multi_json (~> 1.10) - simplecov (0.21.2) + simplecov (0.22.0) docile (~> 1.1) simplecov-html (~> 0.11) simplecov_json_formatter (~> 0.1) @@ -336,6 +338,7 @@ GEM unicorn (>= 4, < 7) uuid (2.3.9) macaddr (~> 1.0) + webrick (1.7.0) PLATFORMS x86_64-linux From bcb05fc24be4856475e93e952fb6e30b74de45ae Mon Sep 17 00:00:00 2001 From: OntoPortal Bot Date: Mon, 2 Jan 2023 09:36:01 +0100 Subject: [PATCH 06/53] [ontoportal-bot] Gemfile.lock update --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 7ecf42c4..8e7424d6 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -53,7 +53,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: 2adf80367f9bf2bfe8ddd870205910350606aa83 + revision: b1d8504f434d3821b967fa07cbe17f7f3d286b41 branch: master specs: ontologies_linked_data (0.0.1) From 312a3357c5dcff92afe297b4e5b593d72f34c03e Mon Sep 17 00:00:00 2001 From: OntoPortal Bot Date: Mon, 2 Jan 2023 09:47:46 +0100 Subject: [PATCH 07/53] [ontoportal-bot] Gemfile.lock update --- Gemfile.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index cf89600d..110b4103 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -53,7 +53,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: c98145605413e459d3c09a41d3c28a5acd0c49e4 + revision: b1d8504f434d3821b967fa07cbe17f7f3d286b41 branch: master specs: ontologies_linked_data (0.0.1) @@ -193,7 +193,7 @@ GEM json_pure (2.6.3) jwt (2.6.0) kgio (2.11.4) - libxml-ruby (3.2.4) + libxml-ruby (4.0.0) logger (1.5.3) macaddr (1.7.2) systemu (~> 2.6.5) From 2cd094c42991025bfbbcaf7c07a62a54e0deb405 Mon Sep 17 00:00:00 2001 From: OntoPortal Bot Date: Mon, 2 Jan 2023 08:52:23 +0000 Subject: [PATCH 08/53] [ontoportal-bot] Gemfile.lock update --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 110b4103..5d2170a7 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -394,4 +394,4 @@ DEPENDENCIES unicorn-worker-killer BUNDLED WITH - 2.3.15 + 2.3.14 From 24d8ac54b9ee0cc15c0e158c4e14be54b98a763c Mon Sep 17 00:00:00 2001 From: OntoPortal Bot Date: Tue, 3 Jan 2023 17:43:40 +0100 Subject: [PATCH 09/53] [ontoportal-bot] Gemfile.lock update --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 5d2170a7..110b4103 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -394,4 +394,4 @@ DEPENDENCIES unicorn-worker-killer BUNDLED WITH - 2.3.14 + 2.3.15 From 80c690923a778ffee2f3aac791f9fc35f8eed3eb Mon Sep 17 00:00:00 2001 From: OntoPortal Bot Date: Tue, 3 Jan 2023 16:59:54 +0000 Subject: [PATCH 10/53] [ontoportal-bot] Gemfile.lock update --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 110b4103..5d2170a7 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -394,4 +394,4 @@ DEPENDENCIES unicorn-worker-killer BUNDLED WITH - 2.3.15 + 2.3.14 From b83d1fa2cada7516ee0ddec4203e2329e034f7e7 Mon Sep 17 00:00:00 2001 From: OntoPortal Bot Date: Mon, 9 Jan 2023 15:11:52 +0100 Subject: [PATCH 11/53] [ontoportal-bot] Gemfile.lock update --- Gemfile.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 5d2170a7..a423b2e1 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -162,7 +162,7 @@ GEM ffi (~> 1.0) google-apis-analytics_v3 (0.12.0) google-apis-core (>= 0.9.1, < 2.a) - google-apis-core (0.9.2) + google-apis-core (0.9.4) addressable (~> 2.5, >= 2.5.1) googleauth (>= 0.16.2, < 2.a) httpclient (>= 2.8.1, < 3.a) @@ -236,7 +236,7 @@ GEM parseconfig (1.1.2) pony (1.13.1) mail (>= 2.0) - pry (0.14.1) + pry (0.14.2) coderay (~> 1.1) method_source (~> 1.0) public_suffix (5.0.1) @@ -394,4 +394,4 @@ DEPENDENCIES unicorn-worker-killer BUNDLED WITH - 2.3.14 + 2.3.15 From fdf1845f5c3926e24961a73475c640ca1ec6fa7d Mon Sep 17 00:00:00 2001 From: OntoPortal Bot Date: Tue, 10 Jan 2023 17:50:21 +0100 Subject: [PATCH 12/53] [ontoportal-bot] Gemfile.lock update --- Gemfile.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 8e7424d6..8e325333 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -162,7 +162,7 @@ GEM ffi (~> 1.0) google-apis-analytics_v3 (0.12.0) google-apis-core (>= 0.9.1, < 2.a) - google-apis-core (0.9.2) + google-apis-core (0.9.4) addressable (~> 2.5, >= 2.5.1) googleauth (>= 0.16.2, < 2.a) httpclient (>= 2.8.1, < 3.a) @@ -228,7 +228,7 @@ GEM net-protocol net-ssh (7.0.1) netrc (0.11.0) - newrelic_rpm (8.14.0) + newrelic_rpm (8.15.0) oj (2.18.5) omni_logger (0.1.4) logger @@ -236,7 +236,7 @@ GEM parseconfig (1.1.2) pony (1.13.1) mail (>= 2.0) - pry (0.14.1) + pry (0.14.2) coderay (~> 1.1) method_source (~> 1.0) public_suffix (5.0.1) From 598636e3fae492edbc61c9e054d482760657cd08 Mon Sep 17 00:00:00 2001 From: OntoPortal Bot Date: Tue, 10 Jan 2023 17:15:58 +0000 Subject: [PATCH 13/53] [ontoportal-bot] Gemfile.lock update --- Gemfile.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index a423b2e1..b50de4b3 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -228,7 +228,7 @@ GEM net-protocol net-ssh (7.0.1) netrc (0.11.0) - newrelic_rpm (8.14.0) + newrelic_rpm (8.15.0) oj (2.18.5) omni_logger (0.1.4) logger @@ -394,4 +394,4 @@ DEPENDENCIES unicorn-worker-killer BUNDLED WITH - 2.3.15 + 2.3.14 From 8967fa40bd472a1f02a27e8b36ce4aabc0a2bbaa Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Thu, 19 Jan 2023 16:00:03 +0100 Subject: [PATCH 14/53] add requests filters --- helpers/request_params_helper.rb | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/helpers/request_params_helper.rb b/helpers/request_params_helper.rb index 3b0163b3..f9780875 100644 --- a/helpers/request_params_helper.rb +++ b/helpers/request_params_helper.rb @@ -17,14 +17,18 @@ def is_set?(param) !param.nil? && param != "" end + def filter? + is_set?(@params["filter_by"]) + end - - + def filter + build_filter + end def get_order_by_from(params, default_order = :asc) if is_set?(params['sortby']) orders = (params["order"] || default_order.to_s).split(',') - out = params['sortby'].split(',').map.with_index do |param, index| + out = params['sortby'].split(',').map.with_index do |param, index| sort_order_item(param, orders[index] || default_order) end out.to_h @@ -46,9 +50,14 @@ def bring_unmapped_to(page_data, sub, klass) end private - def sort_order_item(param , order) + + def sort_order_item(param, order) [param.to_sym, order.to_sym] end + + def build_filter(value = @params["filter_value"]) + Goo::Filter.new(@params["filter_by"].to_sym).regex(value) + end end end end From f9a0586c61b6be49ffd3c7da70552c93c03fabab Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Thu, 19 Jan 2023 16:10:55 +0100 Subject: [PATCH 15/53] remove generalized helpers from instances_helper --- helpers/instances_helper.rb | 48 +------------------------------------ 1 file changed, 1 insertion(+), 47 deletions(-) diff --git a/helpers/instances_helper.rb b/helpers/instances_helper.rb index c902c3b5..c9796544 100644 --- a/helpers/instances_helper.rb +++ b/helpers/instances_helper.rb @@ -3,58 +3,12 @@ module Sinatra module Helpers module InstancesHelper - - # TODO: generalize this to all routes (maybe in application_helper) - def settings_params(klass) - page, size = page_params - attributes = get_attributes_to_include(includes_param, klass) - order_by = get_order_by_from(@params) - bring_unmapped = bring_unmapped?(includes_param) - filter_by_label = label_regex_filter - - [attributes, page, size, filter_by_label, order_by, bring_unmapped] - end - - def is_set?(param) - !param.nil? && param != "" - end - def label_regex_filter (Goo::Filter.new(:label).regex(@params["search"])) if is_set?(@params["search"]) end def filter_classes_by(class_uri) - class_uri.nil? ? nil :{types: RDF::URI.new(class_uri.to_s)} - end - - def get_order_by_from(params, default_order = :asc) - if is_set?(params['sortby']) - orders = (params["order"] || default_order.to_s).split(',') - out = params['sortby'].split(',').map.with_index do |param, index| - sort_order_item(param, orders[index] || default_order) - end - out.to_h - end - - end - - def get_attributes_to_include(includes_param, klass) - ld = klass.goo_attrs_to_load(includes_param) - ld.delete(:properties) - ld - end - - def bring_unmapped?(includes_param) - (includes_param && includes_param.include?(:all)) - end - - def bring_unmapped_to(page_data, sub, klass = LinkedData::Models::Instance) - klass.in(sub).models(page_data).include(:unmapped).all - end - - private - def sort_order_item(param , order) - [param.to_sym, order.to_sym] + class_uri.nil? ? nil : { types: RDF::URI.new(class_uri.to_s) } end end end From 8655f2608d9b79d0cf96aed4de8f72023e455211 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Thu, 19 Jan 2023 16:17:00 +0100 Subject: [PATCH 16/53] add request filter to retrieve_latest_submissions --- helpers/application_helper.rb | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/helpers/application_helper.rb b/helpers/application_helper.rb index 3ba7411c..6c44f25f 100644 --- a/helpers/application_helper.rb +++ b/helpers/application_helper.rb @@ -370,15 +370,14 @@ def retrieve_latest_submissions(options = {}) end submissions_query = submissions_query.filter(Goo::Filter.new(ontology: [:viewOf]).unbound) unless include_views + submissions_query = submissions_query.filter(filter) if filter? # When asking to display all metadata, we are using bring_remaining on each submission. Slower but best way to retrieve all attrs if includes_param.first == :all - including = [:submissionId, {:contact=>[:name, :email], :ontology=>[:administeredBy, :acronym, :name, :summaryOnly, :ontologyType, :viewingRestriction, :acl, + includes = [:submissionId, {:contact=>[:name, :email], :ontology=>[:administeredBy, :acronym, :name, :summaryOnly, :ontologyType, :viewingRestriction, :acl, :group, :hasDomain, :views, :viewOf, :flat], :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym]}, :submissionStatus] - submissions = submissions_query.include(including).to_a - else - submissions = submissions_query.include(includes).to_a end - + submissions = submissions_query.include(includes).to_a + # Figure out latest parsed submissions using all submissions latest_submissions = {} submissions.each do |sub| @@ -391,7 +390,7 @@ def retrieve_latest_submissions(options = {}) latest_submissions[sub.ontology.acronym] ||= sub latest_submissions[sub.ontology.acronym] = sub if sub.submissionId.to_i > latest_submissions[sub.ontology.acronym].submissionId.to_i end - return latest_submissions + latest_submissions end def get_ontology_and_submission From 9508f1de3d833371c4946b8073288f5b6d712ef8 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Thu, 19 Jan 2023 17:24:35 +0100 Subject: [PATCH 17/53] remove no more used parameter bring_unmapped? --- helpers/request_params_helper.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/request_params_helper.rb b/helpers/request_params_helper.rb index f9780875..e7ec091a 100644 --- a/helpers/request_params_helper.rb +++ b/helpers/request_params_helper.rb @@ -8,7 +8,7 @@ def settings_params(klass) page, size = page_params attributes = get_attributes_to_include(includes_param, klass) order_by = get_order_by_from(@params) - bring_unmapped = bring_unmapped?(includes_param, klass) + bring_unmapped = bring_unmapped?(includes_param) [attributes, page, size, order_by, bring_unmapped] end From 2ca440df6f2ac0aec2ad2c74a72ce7dc7029af0f Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Thu, 19 Jan 2023 17:23:40 +0100 Subject: [PATCH 18/53] update the usage of the olds settings_params --- controllers/classes_controller.rb | 2 +- controllers/collection_controller.rb | 2 +- controllers/instances_controller.rb | 16 ++++++++-------- controllers/skos_xl_label_controller.rb | 4 ++-- helpers/collections_helper.rb | 2 +- helpers/schemes_helper.rb | 2 +- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/controllers/classes_controller.rb b/controllers/classes_controller.rb index ed8662c2..c8e55bf8 100644 --- a/controllers/classes_controller.rb +++ b/controllers/classes_controller.rb @@ -9,7 +9,7 @@ class ClassesController < ApplicationController cls_count = submission.class_count(LOGGER) error 403, "Unable to display classes due to missing metrics for #{submission.id.to_s}. Please contact the administrator." if cls_count < 0 - attributes, page, size, filter_by_label, order_by_hash, bring_unmapped_needed = settings_params(LinkedData::Models::Class) + attributes, page, size, order_by_hash, bring_unmapped_needed = settings_params(LinkedData::Models::Class) check_last_modified_segment(LinkedData::Models::Class, [ont.acronym]) index = LinkedData::Models::Class.in(submission) diff --git a/controllers/collection_controller.rb b/controllers/collection_controller.rb index 20883e77..46cd6b1a 100644 --- a/controllers/collection_controller.rb +++ b/controllers/collection_controller.rb @@ -30,7 +30,7 @@ class CollectionsController < ApplicationController get '/members' do ont, submission = get_ontology_and_submission - attributes, page, size, filter_by_label, order_by, bring_unmapped_needed = settings_params(LinkedData::Models::Class) + attributes, page, size, order_by, bring_unmapped_needed = settings_params(LinkedData::Models::Class) collection_uri = get_collection_uri(params) data = LinkedData::Models::Class.where(memberOf: collection_uri).in(submission).include(attributes).page(page,size).all reply data diff --git a/controllers/instances_controller.rb b/controllers/instances_controller.rb index 8e1a8d3e..954dd6e6 100644 --- a/controllers/instances_controller.rb +++ b/controllers/instances_controller.rb @@ -7,8 +7,8 @@ class InstancesController < ApplicationController check_last_modified_segment(LinkedData::Models::Instance, [ont.acronym]) cls = get_class(sub) error 404 if cls.nil? - - attributes, page, size, filter_by_label, order_by, bring_unmapped_needed = settings_params(LinkedData::Models::Instance) + filter_by_label = label_regex_filter + attributes, page, size, order_by, bring_unmapped_needed = settings_params(LinkedData::Models::Instance) @@ -20,7 +20,7 @@ class InstancesController < ApplicationController page_data.order_by(order_by) unless order_by.nil? page_data = page_data.page(page,size).all - bring_unmapped_to page_data , sub if bring_unmapped_needed + bring_unmapped_to page_data , sub, LinkedData::Models::Instance if bring_unmapped_needed reply page_data end @@ -30,8 +30,8 @@ class InstancesController < ApplicationController get do ont, sub = get_ontology_and_submission check_last_modified_segment(LinkedData::Models::Instance, [ont.acronym]) - - attributes, page, size, filter_by_label, order_by, bring_unmapped_needed = settings_params(LinkedData::Models::Instance) + filter_by_label = label_regex_filter + attributes, page, size, order_by, bring_unmapped_needed = settings_params(LinkedData::Models::Instance) page_data = LinkedData::Models::Instance.where @@ -42,7 +42,7 @@ class InstancesController < ApplicationController page_data.order_by(order_by) unless order_by.nil? page_data = page_data.page(page,size).all - bring_unmapped_to page_data , sub if bring_unmapped_needed + bring_unmapped_to page_data , sub, LinkedData::Models::Instance if bring_unmapped_needed reply page_data end @@ -51,11 +51,11 @@ class InstancesController < ApplicationController ont, sub = get_ontology_and_submission check_last_modified_segment(LinkedData::Models::Instance, [ont.acronym]) - attributes, page, size, filter_by_label, order_by, bring_unmapped_needed = settings_params(LinkedData::Models::Instance) + attributes, page, size, order_by, bring_unmapped_needed = settings_params(LinkedData::Models::Instance) page_data = LinkedData::Models::Instance.find(@params["inst"]).include(attributes).in(sub).first - bring_unmapped_to [page_data] , sub if bring_unmapped_needed + bring_unmapped_to [page_data] , sub, LinkedData::Models::Instance if bring_unmapped_needed reply page_data end diff --git a/controllers/skos_xl_label_controller.rb b/controllers/skos_xl_label_controller.rb index ae4977e5..72a00a19 100644 --- a/controllers/skos_xl_label_controller.rb +++ b/controllers/skos_xl_label_controller.rb @@ -3,7 +3,7 @@ class SkosXlLabelController < ApplicationController namespace "/ontologies/:ontology/skos_xl_labels" do get do ont, submission = get_ontology_and_submission - attributes, page, size, filter_by_label, order_by, bring_unmapped_needed = settings_params(LinkedData::Models::SKOS::Label) + attributes, page, size, order_by, bring_unmapped_needed = settings_params(LinkedData::Models::SKOS::Label) labels = LinkedData::Models::SKOS::Label.where.in(submission).include(attributes).page(page, size).all if labels && bring_unmapped_needed LinkedData::Models::SKOS::Label.in(submission).models(labels).include(:unmapped).all @@ -13,7 +13,7 @@ class SkosXlLabelController < ApplicationController get '/:id' do ont, submission = get_ontology_and_submission - attributes, page, size, filter_by_label, order_by, bring_unmapped_needed = settings_params(LinkedData::Models::SKOS::Label) + attributes, page, size, order_by, bring_unmapped_needed = settings_params(LinkedData::Models::SKOS::Label) label = LinkedData::Models::SKOS::Label.find(params[:id]).in(submission).include(attributes).first if label && bring_unmapped_needed LinkedData::Models::SKOS::Label.in(submission).models([label]).include(:unmapped).all diff --git a/helpers/collections_helper.rb b/helpers/collections_helper.rb index ada83fd4..06fa4212 100644 --- a/helpers/collections_helper.rb +++ b/helpers/collections_helper.rb @@ -5,7 +5,7 @@ module Helpers module CollectionsHelper def collections_setting_params ont, submission = get_ontology_and_submission - attributes, page, size, filter_by_label, order_by, bring_unmapped_needed = settings_params(LinkedData::Models::SKOS::Collection) + attributes, page, size, order_by, bring_unmapped_needed = settings_params(LinkedData::Models::SKOS::Collection) [submission, attributes, bring_unmapped_needed] end diff --git a/helpers/schemes_helper.rb b/helpers/schemes_helper.rb index 85f5f765..18f08e14 100644 --- a/helpers/schemes_helper.rb +++ b/helpers/schemes_helper.rb @@ -5,7 +5,7 @@ module Helpers module SchemesHelper def schemes_setting_params ont, submission = get_ontology_and_submission - attributes, page, size, filter_by_label, order_by, bring_unmapped_needed = settings_params(LinkedData::Models::SKOS::Scheme) + attributes, page, size, order_by, bring_unmapped_needed = settings_params(LinkedData::Models::SKOS::Scheme) [submission, attributes, bring_unmapped_needed] end From eac6f35a32200f822c99dd81deb034679536a513 Mon Sep 17 00:00:00 2001 From: OntoPortal Bot Date: Fri, 20 Jan 2023 08:38:38 +0100 Subject: [PATCH 19/53] [ontoportal-bot] Gemfile.lock update --- Gemfile | 4 ++-- Gemfile.lock | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Gemfile b/Gemfile index cc21c703..5ff855ea 100644 --- a/Gemfile +++ b/Gemfile @@ -42,12 +42,12 @@ gem 'haml', '~> 5.2.2' # pin see https://github.com/ncbo/ontologies_api/pull/107 gem 'redcarpet' # NCBO gems (can be from a local dev path or from rubygems/git) -gem 'goo', git: 'https://github.com/ontoportal-lirmm/goo.git', branch: 'master' +gem 'goo', git: 'https://github.com/ontoportal-lirmm/goo.git', branch: 'development' gem 'ncbo_annotator', git: 'https://github.com/ontoportal-lirmm/ncbo_annotator.git', branch: 'master' gem 'ncbo_cron', git: 'https://github.com/ontoportal-lirmm/ncbo_cron.git', branch: 'master' gem 'ncbo_ontology_recommender', git: 'https://github.com/ncbo/ncbo_ontology_recommender.git', branch: 'master' gem 'sparql-client', github: 'ontoportal-lirmm/sparql-client', branch: 'master' -gem 'ontologies_linked_data', git: 'https://github.com/ontoportal-lirmm/ontologies_linked_data.git', branch: 'master' +gem 'ontologies_linked_data', git: 'https://github.com/ontoportal-lirmm/ontologies_linked_data.git', branch: 'development' group :development do # bcrypt_pbkdf and ed35519 is required for capistrano deployments when using ed25519 keys; see https://github.com/miloserdow/capistrano-deploy/issues/42 diff --git a/Gemfile.lock b/Gemfile.lock index b50de4b3..f520628d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -11,8 +11,8 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/goo.git - revision: 92968978e5d2436c648bf7956e25aff05440b59d - branch: master + revision: cda6aff2338e2a2831e4e7bf716abdf8fa8483d2 + branch: development specs: goo (0.0.2) addressable (~> 2.8) @@ -53,8 +53,8 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: b1d8504f434d3821b967fa07cbe17f7f3d286b41 - branch: master + revision: 679c6468b87f85b3993583c111108e183e37c3e5 + branch: development specs: ontologies_linked_data (0.0.1) activesupport @@ -134,7 +134,7 @@ GEM domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) ed25519 (1.3.0) - faraday (1.10.2) + faraday (1.10.3) faraday-em_http (~> 1.0) faraday-em_synchrony (~> 1.0) faraday-excon (~> 1.1) @@ -162,7 +162,7 @@ GEM ffi (~> 1.0) google-apis-analytics_v3 (0.12.0) google-apis-core (>= 0.9.1, < 2.a) - google-apis-core (0.9.4) + google-apis-core (0.9.5) addressable (~> 2.5, >= 2.5.1) googleauth (>= 0.16.2, < 2.a) httpclient (>= 2.8.1, < 3.a) @@ -197,7 +197,7 @@ GEM logger (1.5.3) macaddr (1.7.2) systemu (~> 2.6.5) - mail (2.8.0) + mail (2.8.0.1) mini_mime (>= 0.1.1) net-imap net-pop @@ -394,4 +394,4 @@ DEPENDENCIES unicorn-worker-killer BUNDLED WITH - 2.3.14 + 2.3.15 From 3a12f000f0c7c8d6cd53c1a6f7a57abdb24119b3 Mon Sep 17 00:00:00 2001 From: OntoPortal Bot Date: Mon, 23 Jan 2023 10:42:52 +0100 Subject: [PATCH 20/53] [ontoportal-bot] Gemfile.lock update --- Gemfile.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index f520628d..5dc7a928 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -37,7 +37,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ncbo_cron.git - revision: 3296b76c5287a4b2644f123e736c156f3a6d219d + revision: 8db3481116c57d2a21dc8f32bcd1695d95442280 branch: master specs: ncbo_cron (0.0.1) @@ -53,7 +53,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: 679c6468b87f85b3993583c111108e183e37c3e5 + revision: f4395ea6b32df8cee25b3ccf8515c9c75d18b19a branch: development specs: ontologies_linked_data (0.0.1) From f568e52c1fff3ed18bb8b93362d84cd4622763c0 Mon Sep 17 00:00:00 2001 From: OntoPortal Bot Date: Mon, 23 Jan 2023 10:02:18 +0000 Subject: [PATCH 21/53] [ontoportal-bot] Gemfile.lock update --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 5dc7a928..c3a8dacc 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -394,4 +394,4 @@ DEPENDENCIES unicorn-worker-killer BUNDLED WITH - 2.3.15 + 2.3.14 From a3121099089120c40a938e2a97f6bf3f78c50f2c Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Mon, 6 Mar 2023 16:58:37 +0100 Subject: [PATCH 22/53] add request_lang middleware --- Gemfile | 3 ++- Gemfile.lock | 37 ++++++++++++++++++++++--------------- app.rb | 6 ++++++ lib/rack/request_lang.rb | 16 ++++++++++++++++ 4 files changed, 46 insertions(+), 16 deletions(-) create mode 100644 lib/rack/request_lang.rb diff --git a/Gemfile b/Gemfile index 5ff855ea..c80ea2e9 100644 --- a/Gemfile +++ b/Gemfile @@ -13,6 +13,7 @@ gem 'rake', '~> 10.0' gem 'sinatra', '~> 1.0' gem 'sinatra-advanced-routes' gem 'sinatra-contrib', '~> 1.0' +gem 'request_store' # Rack middleware gem 'ffi' @@ -47,7 +48,7 @@ gem 'ncbo_annotator', git: 'https://github.com/ontoportal-lirmm/ncbo_annotator.g gem 'ncbo_cron', git: 'https://github.com/ontoportal-lirmm/ncbo_cron.git', branch: 'master' gem 'ncbo_ontology_recommender', git: 'https://github.com/ncbo/ncbo_ontology_recommender.git', branch: 'master' gem 'sparql-client', github: 'ontoportal-lirmm/sparql-client', branch: 'master' -gem 'ontologies_linked_data', git: 'https://github.com/ontoportal-lirmm/ontologies_linked_data.git', branch: 'development' +gem 'ontologies_linked_data', git: 'https://github.com/ontoportal-lirmm/ontologies_linked_data.git', branch: 'feature/support-multilingual-read-one-language-from-request-parameter' group :development do # bcrypt_pbkdf and ed35519 is required for capistrano deployments when using ed25519 keys; see https://github.com/miloserdow/capistrano-deploy/issues/42 diff --git a/Gemfile.lock b/Gemfile.lock index d02647ba..ba50a063 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -11,7 +11,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/goo.git - revision: cda6aff2338e2a2831e4e7bf716abdf8fa8483d2 + revision: b769c165906163e30a026dba511ae1069c4eed3d branch: development specs: goo (0.0.2) @@ -53,8 +53,8 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: f4395ea6b32df8cee25b3ccf8515c9c75d18b19a - branch: development + revision: f8c8f569cc282ec31ad181cdbe355f7899983b3f + branch: feature/support-multilingual-read-one-language-from-request-parameter specs: ontologies_linked_data (0.0.1) activesupport @@ -112,7 +112,7 @@ GEM bcrypt_pbkdf (1.1.0) bigdecimal (1.4.2) builder (3.2.4) - capistrano (3.17.1) + capistrano (3.17.2) airbrussh (>= 1.0.0) i18n rake (>= 10.0.0) @@ -125,7 +125,8 @@ GEM capistrano (~> 3.1) sshkit (~> 1.3) coderay (1.1.3) - concurrent-ruby (1.2.0) + concurrent-ruby (1.2.2) + connection_pool (2.3.0) cube-ruby (0.0.3) dante (0.2.0) date (3.3.3) @@ -160,8 +161,8 @@ GEM ffi (1.15.5) get_process_mem (0.2.7) ffi (~> 1.0) - google-apis-analytics_v3 (0.12.0) - google-apis-core (>= 0.9.1, < 2.a) + google-apis-analytics_v3 (0.13.0) + google-apis-core (>= 0.11.0, < 2.a) google-apis-core (0.11.0) addressable (~> 2.5, >= 2.5.1) googleauth (>= 0.16.2, < 2.a) @@ -206,7 +207,7 @@ GEM method_source (1.0.0) mime-types (3.4.1) mime-types-data (~> 3.2015) - mime-types-data (3.2022.0105) + mime-types-data (3.2023.0218.1) mini_mime (1.1.2) minitest (4.7.5) minitest-stub_any_instance (1.0.3) @@ -228,7 +229,7 @@ GEM net-protocol net-ssh (7.0.1) netrc (0.11.0) - newrelic_rpm (8.16.0) + newrelic_rpm (9.0.0) oj (2.18.5) omni_logger (0.1.4) logger @@ -256,24 +257,29 @@ GEM rack-test (2.0.2) rack (>= 1.3) rack-timeout (0.6.3) - raindrops (0.20.0) + raindrops (0.20.1) rake (10.5.0) rdf (1.0.8) addressable (>= 2.2) redcarpet (3.6.0) - redis (4.8.1) + redis (5.0.6) + redis-client (>= 0.9.0) redis-activesupport (5.3.0) activesupport (>= 3, < 8) redis-store (>= 1.3, < 2) + redis-client (0.13.0) + connection_pool redis-rack-cache (2.2.1) rack-cache (>= 1.10, < 2) redis-store (>= 1.6, < 2) - redis-store (1.9.1) - redis (>= 4, < 5) + redis-store (1.9.2) + redis (>= 4, < 6) representable (3.2.0) declarative (< 0.1.0) trailblazer-option (>= 0.1.1, < 0.2.0) uber (< 0.2.0) + request_store (1.5.1) + rack (>= 1.4) rest-client (2.1.0) http-accept (>= 1.7.0, < 2.0) http-cookie (>= 1.0.2, < 2.0) @@ -316,12 +322,12 @@ GEM rack-test sinatra (~> 1.4.0) tilt (>= 1.3, < 3) - sshkit (1.21.3) + sshkit (1.21.4) net-scp (>= 1.1.2) net-ssh (>= 2.8.0) systemu (2.6.5) temple (0.10.0) - tilt (2.0.11) + tilt (2.1.0) timeout (0.3.2) trailblazer-option (0.1.2) tzinfo (2.0.6) @@ -384,6 +390,7 @@ DEPENDENCIES redis redis-activesupport redis-rack-cache (~> 2.0) + request_store shotgun! simplecov simplecov-cobertura diff --git a/app.rb b/app.rb index 5360ae4b..46457c86 100644 --- a/app.rb +++ b/app.rb @@ -29,6 +29,7 @@ require_relative 'lib/rack/cube_reporter' require_relative 'lib/rack/param_translator' require_relative 'lib/rack/slice_detection' +require_relative 'lib/rack/request_lang' # Logging setup require_relative "config/logging" @@ -36,6 +37,8 @@ # Inflector setup require_relative "config/inflections" +require 'request_store' + # Protection settings set :protection, :except => :path_traversal @@ -143,6 +146,9 @@ use Rack::PostBodyToParams use Rack::ParamTranslator +use RequestStore::Middleware +use Rack::RequestLang + use LinkedData::Security::Authorization use LinkedData::Security::AccessDenied diff --git a/lib/rack/request_lang.rb b/lib/rack/request_lang.rb new file mode 100644 index 00000000..b2221041 --- /dev/null +++ b/lib/rack/request_lang.rb @@ -0,0 +1,16 @@ +module Rack + class RequestLang + + def initialize(app = nil, options = {}) + @app = app + end + + def call(env) + r = Rack::Request.new(env) + lang = r.params["lang"] || r.params["language"] + lang = lang.upcase.to_sym if lang + RequestStore.store[:requested_lang] = lang + @app.call(env) + end + end +end \ No newline at end of file From 207884bede33c54e764117843843220d26244674 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Mon, 6 Mar 2023 19:17:40 +0100 Subject: [PATCH 23/53] pin redis gem version to 4.8.1 --- Gemfile | 2 +- Gemfile.lock | 82 ++++++++++++++++++++++++++++++---------------------- 2 files changed, 49 insertions(+), 35 deletions(-) diff --git a/Gemfile b/Gemfile index 82282bee..1737403e 100644 --- a/Gemfile +++ b/Gemfile @@ -26,7 +26,7 @@ gem 'rack-timeout' gem 'redis-rack-cache', '~> 2.0' # Data access (caching) -gem 'redis' +gem 'redis', '~> 4.8.1' gem 'redis-activesupport' # Monitoring diff --git a/Gemfile.lock b/Gemfile.lock index 27deaffe..e423a14a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,6 +1,6 @@ GIT remote: https://github.com/ncbo/goo.git - revision: 077c674a6e277a51dca4ca681e49e3e3a55b918a + revision: 919c20dec58375eb8a4dd1aed47864e2ad7bacfa branch: master specs: goo (0.0.2) @@ -15,7 +15,7 @@ GIT GIT remote: https://github.com/ncbo/ncbo_annotator.git - revision: 71d41e3afb35dafe29abfb6d9becaadc725bad36 + revision: 964f0680799421ab24eddc974d9f2995c6c88734 branch: master specs: ncbo_annotator (0.0.1) @@ -26,7 +26,7 @@ GIT GIT remote: https://github.com/ncbo/ncbo_cron.git - revision: a1a4babc0b5e36325b5af667a6a50b7af1f5f891 + revision: c09d76eeb131caf422c18ba49580db1b47df86dc branch: master specs: ncbo_cron (0.0.1) @@ -42,7 +42,7 @@ GIT GIT remote: https://github.com/ncbo/ncbo_ontology_recommender.git - revision: 6010ff60b99dc1282822b8a1fb59bd59f453755f + revision: d0ac992c88bd417f2f2137ba62934c3c41b6db7c branch: master specs: ncbo_ontology_recommender (0.0.1) @@ -53,7 +53,7 @@ GIT GIT remote: https://github.com/ncbo/ontologies_linked_data.git - revision: 4f9139d870c3b1771af1127afa17b679bd0f60dc + revision: 546b3a45a6f359c09f0cc95d954f03e979b4847a branch: master specs: ontologies_linked_data (0.0.1) @@ -112,7 +112,7 @@ GEM bcrypt_pbkdf (1.1.0) bigdecimal (1.4.2) builder (3.2.4) - capistrano (3.17.1) + capistrano (3.17.2) airbrussh (>= 1.0.0) i18n rake (>= 10.0.0) @@ -125,15 +125,16 @@ GEM capistrano (~> 3.1) sshkit (~> 1.3) coderay (1.1.3) - concurrent-ruby (1.1.10) + concurrent-ruby (1.2.2) cube-ruby (0.0.3) dante (0.2.0) + date (3.3.3) declarative (0.0.20) docile (1.4.0) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) ed25519 (1.3.0) - faraday (1.10.2) + faraday (1.10.3) faraday-em_http (~> 1.0) faraday-em_synchrony (~> 1.0) faraday-excon (~> 1.1) @@ -159,9 +160,9 @@ GEM ffi (1.15.5) get_process_mem (0.2.7) ffi (~> 1.0) - google-apis-analytics_v3 (0.12.0) - google-apis-core (>= 0.9.1, < 2.a) - google-apis-core (0.9.1) + google-apis-analytics_v3 (0.13.0) + google-apis-core (>= 0.11.0, < 2.a) + google-apis-core (0.11.0) addressable (~> 2.5, >= 2.5.1) googleauth (>= 0.16.2, < 2.a) httpclient (>= 2.8.1, < 3.a) @@ -186,36 +187,48 @@ GEM httpclient (2.8.3) i18n (0.9.5) concurrent-ruby (~> 1.0) - json (2.6.2) + json (2.6.3) json-schema (2.8.1) addressable (>= 2.4) - json_pure (2.6.2) - jwt (2.5.0) + json_pure (2.6.3) + jwt (2.7.0) kgio (2.11.4) - libxml-ruby (3.2.4) - logger (1.5.1) + libxml-ruby (4.0.0) + logger (1.5.3) macaddr (1.7.2) systemu (~> 2.6.5) - mail (2.7.1) + mail (2.8.1) mini_mime (>= 0.1.1) + net-imap + net-pop + net-smtp memoist (0.16.2) method_source (1.0.0) mime-types (3.4.1) mime-types-data (~> 3.2015) - mime-types-data (3.2022.0105) + mime-types-data (3.2023.0218.1) mini_mime (1.1.2) minitest (4.7.5) minitest-stub_any_instance (1.0.3) mlanett-redis-lock (0.2.7) redis multi_json (1.15.0) - multipart-post (2.2.3) + multipart-post (2.3.0) net-http-persistent (2.9.4) + net-imap (0.3.4) + date + net-protocol + net-pop (0.1.2) + net-protocol + net-protocol (0.2.1) + timeout net-scp (4.0.0) net-ssh (>= 2.6.5, < 8.0.0) + net-smtp (0.3.3) + net-protocol net-ssh (7.0.1) netrc (0.11.0) - newrelic_rpm (8.13.1) + newrelic_rpm (9.0.0) oj (2.18.5) omni_logger (0.1.4) logger @@ -223,10 +236,10 @@ GEM parseconfig (1.1.2) pony (1.13.1) mail (>= 2.0) - pry (0.14.1) + pry (0.14.2) coderay (~> 1.1) method_source (~> 1.0) - public_suffix (5.0.0) + public_suffix (5.0.1) rack (1.6.13) rack-accept (0.4.5) rack (>= 0.4) @@ -243,20 +256,20 @@ GEM rack-test (2.0.2) rack (>= 1.3) rack-timeout (0.6.3) - raindrops (0.20.0) + raindrops (0.20.1) rake (10.5.0) rdf (1.0.8) addressable (>= 2.2) - redcarpet (3.5.1) - redis (4.8.0) + redcarpet (3.6.0) + redis (4.8.1) redis-activesupport (5.3.0) activesupport (>= 3, < 8) redis-store (>= 1.3, < 2) redis-rack-cache (2.2.1) rack-cache (>= 1.10, < 2) redis-store (>= 1.6, < 2) - redis-store (1.9.1) - redis (>= 4, < 5) + redis-store (1.9.2) + redis (>= 4, < 6) representable (3.2.0) declarative (< 0.1.0) trailblazer-option (>= 0.1.1, < 0.2.0) @@ -281,7 +294,7 @@ GEM faraday (>= 0.17.5, < 3.a) jwt (>= 1.5, < 3.0) multi_json (~> 1.10) - simplecov (0.21.2) + simplecov (0.22.0) docile (~> 1.1) simplecov-html (~> 0.11) simplecov_json_formatter (~> 0.1) @@ -303,14 +316,15 @@ GEM rack-test sinatra (~> 1.4.0) tilt (>= 1.3, < 3) - sshkit (1.21.3) + sshkit (1.21.4) net-scp (>= 1.1.2) net-ssh (>= 2.8.0) systemu (2.6.5) - temple (0.9.1) - tilt (2.0.11) + temple (0.10.0) + tilt (2.1.0) + timeout (0.3.2) trailblazer-option (0.1.2) - tzinfo (2.0.5) + tzinfo (2.0.6) concurrent-ruby (~> 1.0) uber (0.1.0) unf (0.1.4) @@ -324,7 +338,7 @@ GEM unicorn (>= 4, < 7) uuid (2.3.9) macaddr (~> 1.0) - webrick (1.7.0) + webrick (1.8.1) PLATFORMS x86_64-darwin-21 @@ -367,7 +381,7 @@ DEPENDENCIES rack-timeout rake (~> 10.0) redcarpet - redis + redis (~> 4.8.1) redis-activesupport redis-rack-cache (~> 2.0) shotgun! From ae23ce082cab23158d0f038e9a7d956776d14376 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Mon, 6 Mar 2023 19:31:15 +0100 Subject: [PATCH 24/53] update Gemfile.lock --- Gemfile | 2 +- Gemfile.lock | 10 +++------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/Gemfile b/Gemfile index b8371e74..71cdf420 100644 --- a/Gemfile +++ b/Gemfile @@ -48,7 +48,7 @@ gem 'ncbo_annotator', git: 'https://github.com/ontoportal-lirmm/ncbo_annotator.g gem 'ncbo_cron', git: 'https://github.com/ontoportal-lirmm/ncbo_cron.git', branch: 'master' gem 'ncbo_ontology_recommender', git: 'https://github.com/ncbo/ncbo_ontology_recommender.git', branch: 'master' gem 'sparql-client', github: 'ontoportal-lirmm/sparql-client', branch: 'master' -gem 'ontologies_linked_data', git: 'https://github.com/ontoportal-lirmm/ontologies_linked_data.git', branch: 'feature/support-multilingual-read-one-language-from-request-parameter' +gem 'ontologies_linked_data', git: 'https://github.com/ontoportal-lirmm/ontologies_linked_data.git', branch: 'development' group :development do # bcrypt_pbkdf and ed35519 is required for capistrano deployments when using ed25519 keys; see https://github.com/miloserdow/capistrano-deploy/issues/42 diff --git a/Gemfile.lock b/Gemfile.lock index 449708f4..38c87943 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -53,8 +53,8 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: f8c8f569cc282ec31ad181cdbe355f7899983b3f - branch: feature/support-multilingual-read-one-language-from-request-parameter + revision: 69756f8a7cff39d283065217559c68820d6d95e3 + branch: development specs: ontologies_linked_data (0.0.1) activesupport @@ -126,7 +126,6 @@ GEM sshkit (~> 1.3) coderay (1.1.3) concurrent-ruby (1.2.2) - connection_pool (2.3.0) cube-ruby (0.0.3) dante (0.2.0) date (3.3.3) @@ -262,13 +261,10 @@ GEM rdf (1.0.8) addressable (>= 2.2) redcarpet (3.6.0) - redis (5.0.6) - redis-client (>= 0.9.0) + redis (4.8.1) redis-activesupport (5.3.0) activesupport (>= 3, < 8) redis-store (>= 1.3, < 2) - redis-client (0.13.0) - connection_pool redis-rack-cache (2.2.1) rack-cache (>= 1.10, < 2) redis-store (>= 1.6, < 2) From be31e4f6dd9b3a3d1a29d99eb7087367e731e45f Mon Sep 17 00:00:00 2001 From: OntoPortal Bot Date: Tue, 7 Mar 2023 18:18:16 +0000 Subject: [PATCH 25/53] [ontoportal-bot] Gemfile.lock update --- Gemfile.lock | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 3d5d0c97..e29b281b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -107,7 +107,7 @@ GEM public_suffix (>= 2.0.2, < 6.0) airbrussh (1.4.1) sshkit (>= 1.6.1, != 1.7.0) - backports (3.23.0) + backports (3.24.0) bcrypt (3.1.18) bcrypt_pbkdf (1.1.0) bigdecimal (1.4.2) @@ -342,9 +342,7 @@ GEM macaddr (~> 1.0) webrick (1.8.1) - PLATFORMS - x86_64-darwin-21 x86_64-linux DEPENDENCIES From 77bde061c596e22cf68aab477e6d1eb1270c1fd7 Mon Sep 17 00:00:00 2001 From: OntoPortal Bot Date: Tue, 7 Mar 2023 18:34:47 +0000 Subject: [PATCH 26/53] [ontoportal-bot] Gemfile.lock update --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index e29b281b..f176a828 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -11,7 +11,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/goo.git - revision: b769c165906163e30a026dba511ae1069c4eed3d + revision: 597436e366417be0ec523c65d97e18f94e906b73 branch: development specs: goo (0.0.2) From 0d8e4fdcf64a9e2475104b5783987b2dac2b5623 Mon Sep 17 00:00:00 2001 From: OntoPortal Bot Date: Tue, 7 Mar 2023 18:42:30 +0000 Subject: [PATCH 27/53] [ontoportal-bot] Gemfile.lock update --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index f176a828..4b3f0d6d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -53,7 +53,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: 69756f8a7cff39d283065217559c68820d6d95e3 + revision: 4609ce3101bad8b9507488e50da92eb23c8a92ab branch: development specs: ontologies_linked_data (0.0.1) From aed3bf580e56b45beaf921a8dcbd6520f685b1f1 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Mon, 20 Mar 2023 06:07:11 +0100 Subject: [PATCH 28/53] update Gemfile to use ontologies_linked_data new metadata branch --- Gemfile | 2 +- Gemfile.lock | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Gemfile b/Gemfile index 71cdf420..da2c84f9 100644 --- a/Gemfile +++ b/Gemfile @@ -48,7 +48,7 @@ gem 'ncbo_annotator', git: 'https://github.com/ontoportal-lirmm/ncbo_annotator.g gem 'ncbo_cron', git: 'https://github.com/ontoportal-lirmm/ncbo_cron.git', branch: 'master' gem 'ncbo_ontology_recommender', git: 'https://github.com/ncbo/ncbo_ontology_recommender.git', branch: 'master' gem 'sparql-client', github: 'ontoportal-lirmm/sparql-client', branch: 'master' -gem 'ontologies_linked_data', git: 'https://github.com/ontoportal-lirmm/ontologies_linked_data.git', branch: 'development' +gem 'ontologies_linked_data', git: 'https://github.com/ontoportal-lirmm/ontologies_linked_data.git', branch: 'feature/migrate-attrbiutes-metadata-to-scheme-file' group :development do # bcrypt_pbkdf and ed35519 is required for capistrano deployments when using ed25519 keys; see https://github.com/miloserdow/capistrano-deploy/issues/42 diff --git a/Gemfile.lock b/Gemfile.lock index 4b3f0d6d..9613a47f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -11,7 +11,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/goo.git - revision: 597436e366417be0ec523c65d97e18f94e906b73 + revision: f9b73ce6b6fac92bffd0a2a1b9dc9ba9266acf79 branch: development specs: goo (0.0.2) @@ -53,8 +53,8 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: 4609ce3101bad8b9507488e50da92eb23c8a92ab - branch: development + revision: 18c1daaf0fdcdfced75a062a70fe9796034d40d9 + branch: feature/migrate-attrbiutes-metadata-to-scheme-file specs: ontologies_linked_data (0.0.1) activesupport @@ -226,7 +226,7 @@ GEM net-ssh (>= 2.6.5, < 8.0.0) net-smtp (0.3.3) net-protocol - net-ssh (7.0.1) + net-ssh (7.1.0) netrc (0.11.0) newrelic_rpm (9.0.0) oj (2.18.5) @@ -253,7 +253,7 @@ GEM rack (>= 1.2.0) rack-protection (1.5.5) rack - rack-test (2.0.2) + rack-test (2.1.0) rack (>= 1.3) rack-timeout (0.6.3) raindrops (0.20.1) @@ -343,6 +343,7 @@ GEM webrick (1.8.1) PLATFORMS + x86_64-darwin-21 x86_64-linux DEPENDENCIES From f0f56a68317f2ad29df406a88c62cb55726f3625 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Mon, 20 Mar 2023 06:42:49 +0100 Subject: [PATCH 29/53] update TestOntologySubmissionsController to be adapted to the new model --- test/controllers/test_ontology_submissions_controller.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/controllers/test_ontology_submissions_controller.rb b/test/controllers/test_ontology_submissions_controller.rb index 7500dce4..253fef54 100644 --- a/test/controllers/test_ontology_submissions_controller.rb +++ b/test/controllers/test_ontology_submissions_controller.rb @@ -18,7 +18,10 @@ def self._set_vars administeredBy: "tim", "file" => Rack::Test::UploadedFile.new(@@test_file, ""), released: DateTime.now.to_s, - contact: [{name: "test_name", email: "test@example.org"}] + contact: [{name: "test_name", email: "test@example.org"}], + URI: 'https://test.com/test', + status: 'production', + description: 'ontology description' } @@status_uploaded = "UPLOADED" @@status_rdf = "RDF" From 535e96e1aa133af6cd6527531e8e875f8454e725 Mon Sep 17 00:00:00 2001 From: OntoPortal Bot Date: Mon, 20 Mar 2023 10:18:23 +0000 Subject: [PATCH 30/53] [ontoportal-bot] Gemfile.lock update --- Gemfile.lock | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 9613a47f..566e61d5 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -53,7 +53,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: 18c1daaf0fdcdfced75a062a70fe9796034d40d9 + revision: 8da57befa2e7f4953ab74e74ebf87da1352ab06a branch: feature/migrate-attrbiutes-metadata-to-scheme-file specs: ontologies_linked_data (0.0.1) @@ -343,7 +343,6 @@ GEM webrick (1.8.1) PLATFORMS - x86_64-darwin-21 x86_64-linux DEPENDENCIES From 7320f50aa9558dc5fbc26220c6f9d5b9c8c4014a Mon Sep 17 00:00:00 2001 From: OntoPortal Bot Date: Mon, 20 Mar 2023 10:27:06 +0000 Subject: [PATCH 31/53] [ontoportal-bot] Gemfile.lock update --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 566e61d5..d79d63fd 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -53,7 +53,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: 8da57befa2e7f4953ab74e74ebf87da1352ab06a + revision: f2271662e6429d7878c66296ed855042251b0f01 branch: feature/migrate-attrbiutes-metadata-to-scheme-file specs: ontologies_linked_data (0.0.1) From 62f5831cbbf369bed5479ed06d3c804551335a44 Mon Sep 17 00:00:00 2001 From: OntoPortal Bot Date: Tue, 21 Mar 2023 09:01:03 +0000 Subject: [PATCH 32/53] [ontoportal-bot] Gemfile.lock update --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index d79d63fd..7b61090b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -53,7 +53,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: f2271662e6429d7878c66296ed855042251b0f01 + revision: 1e6886bec9c0c07bf5bfaad51e095a04c5bd0197 branch: feature/migrate-attrbiutes-metadata-to-scheme-file specs: ontologies_linked_data (0.0.1) From e259c1c6509c8329c6311355fddc6526c7050aee Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Tue, 21 Mar 2023 13:33:32 +0100 Subject: [PATCH 33/53] in submission_metadata rename display with category --- helpers/metadata_helper.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helpers/metadata_helper.rb b/helpers/metadata_helper.rb index db61c414..7699e6a0 100644 --- a/helpers/metadata_helper.rb +++ b/helpers/metadata_helper.rb @@ -64,9 +64,9 @@ def klass_metadata(klass, type) # Get display from the metadata if klass.attribute_settings(attr)[:display].nil? - attr_settings[:display] = "no" + attr_settings[:category] = "no" else - attr_settings[:display] = klass.attribute_settings(attr)[:display] + attr_settings[:category] = klass.attribute_settings(attr)[:display] end if !klass.attribute_settings(attr)[:helpText].nil? From 2bf3ac8e43df39299e958a3b6a8e9a23ab8bbab8 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Tue, 21 Mar 2023 13:34:02 +0100 Subject: [PATCH 34/53] add to submission_metadata description and example fields --- helpers/metadata_helper.rb | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/helpers/metadata_helper.rb b/helpers/metadata_helper.rb index 7699e6a0..2c5d7182 100644 --- a/helpers/metadata_helper.rb +++ b/helpers/metadata_helper.rb @@ -69,10 +69,18 @@ def klass_metadata(klass, type) attr_settings[:category] = klass.attribute_settings(attr)[:display] end - if !klass.attribute_settings(attr)[:helpText].nil? + unless klass.attribute_settings(attr)[:helpText].nil? attr_settings[:helpText] = klass.attribute_settings(attr)[:helpText] end + unless klass.attribute_settings(attr)[:description].nil? + attr_settings[:description] = klass.attribute_settings(attr)[:description] + end + + unless klass.attribute_settings(attr)[:example].nil? + attr_settings[:example] = klass.attribute_settings(attr)[:example] + end + attr_settings[:@context] = { "@vocab" => "#{id_url_prefix}metadata/" } From ab8c876b036f8db95848e79981561f759efa6e93 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Wed, 26 Apr 2023 09:21:47 +0200 Subject: [PATCH 35/53] add the option to do pagination for the submission endpoint --- controllers/ontology_submissions_controller.rb | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/controllers/ontology_submissions_controller.rb b/controllers/ontology_submissions_controller.rb index cf55659d..8749786d 100644 --- a/controllers/ontology_submissions_controller.rb +++ b/controllers/ontology_submissions_controller.rb @@ -1,9 +1,13 @@ class OntologySubmissionsController < ApplicationController get "/submissions" do check_last_modified_collection(LinkedData::Models::OntologySubmission) - #using appplication_helper method - options = {also_include_views: params["also_include_views"], status: (params["include_status"] || "ANY")} - reply retrieve_latest_submissions(options).values + options = { + also_include_views: params["also_include_views"], + status: (params["include_status"] || "ANY") + } + subs = retrieve_latest_submissions(options) + subs = subs.values unless page? + reply subs end ## From c6454e48f1854fb15aabedf2a2911179c118e91d Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Wed, 26 Apr 2023 09:22:25 +0200 Subject: [PATCH 36/53] extract retrieve_latest_submissions method to submission helper --- helpers/application_helper.rb | 38 ++++++-------------------- helpers/submission_helper.rb | 51 +++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 30 deletions(-) create mode 100644 helpers/submission_helper.rb diff --git a/helpers/application_helper.rb b/helpers/application_helper.rb index 6c44f25f..84c26497 100644 --- a/helpers/application_helper.rb +++ b/helpers/application_helper.rb @@ -355,40 +355,18 @@ def replace_url_prefix(id) end def retrieve_latest_submissions(options = {}) - status = (options[:status] || "RDF").to_s.upcase - include_ready = status.eql?("READY") ? true : false - status = "RDF" if status.eql?("READY") - any = true if status.eql?("ANY") - include_views = options[:also_include_views] || false - includes = OntologySubmission.goo_attrs_to_load(includes_param) - - includes << :submissionStatus unless includes.include?(:submissionStatus) - if any - submissions_query = OntologySubmission.where - else - submissions_query = OntologySubmission.where(submissionStatus: [ code: status]) - end + submissions = retrieve_submissions(options) - submissions_query = submissions_query.filter(Goo::Filter.new(ontology: [:viewOf]).unbound) unless include_views - submissions_query = submissions_query.filter(filter) if filter? - # When asking to display all metadata, we are using bring_remaining on each submission. Slower but best way to retrieve all attrs - if includes_param.first == :all - includes = [:submissionId, {:contact=>[:name, :email], :ontology=>[:administeredBy, :acronym, :name, :summaryOnly, :ontologyType, :viewingRestriction, :acl, - :group, :hasDomain, :views, :viewOf, :flat], :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym]}, :submissionStatus] - end - submissions = submissions_query.include(includes).to_a - - # Figure out latest parsed submissions using all submissions - latest_submissions = {} + latest_submissions = page? ? submissions : {} # latest_submission doest not work with pagination submissions.each do |sub| # To retrieve all metadata, but slow when a lot of ontologies - if includes_param.first == :all - sub.bring_remaining + sub.bring_remaining if includes_param.first == :all + unless page? + next if include_ready?(options) && !sub.ready? + next if sub.ontology.nil? + latest_submissions[sub.ontology.acronym] ||= sub + latest_submissions[sub.ontology.acronym] = sub if sub.submissionId.to_i > latest_submissions[sub.ontology.acronym].submissionId.to_i end - next if include_ready && !sub.ready? - next if sub.ontology.nil? - latest_submissions[sub.ontology.acronym] ||= sub - latest_submissions[sub.ontology.acronym] = sub if sub.submissionId.to_i > latest_submissions[sub.ontology.acronym].submissionId.to_i end latest_submissions end diff --git a/helpers/submission_helper.rb b/helpers/submission_helper.rb new file mode 100644 index 00000000..38108050 --- /dev/null +++ b/helpers/submission_helper.rb @@ -0,0 +1,51 @@ +require 'sinatra/base' + +module Sinatra + module Helpers + module SubmissionHelper + + def retrieve_submissions(options) + status = (options[:status] || "RDF").to_s.upcase + status = "RDF" if status.eql?("READY") + any = status.eql?("ANY") + include_views = options[:also_include_views] || false + includes, page, size, order_by, _ = settings_params(LinkedData::Models::OntologySubmission) + includes << :submissionStatus unless includes.include?(:submissionStatus) + + + submissions_query = LinkedData::Models::OntologySubmission + if any + submissions_query = submissions_query.where + else + submissions_query = submissions_query.where({submissionStatus: [ code: status]}) + end + + submissions_query = apply_filters(submissions_query) + submissions_query = submissions_query.filter(Goo::Filter.new(ontology: [:viewOf]).unbound) unless include_views + submissions_query = submissions_query.filter(filter) if filter? + + # When asking to display all metadata, we are using bring_remaining on each submission. Slower but best way to retrieve all attrs + if includes_param.first == :all + includes = [:submissionId, {:contact=>[:name, :email], :ontology=>[:administeredBy, :acronym, :name, :summaryOnly, :ontologyType, :viewingRestriction, :acl, + :group, :hasDomain, :views, :viewOf, :flat], :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym]}, :submissionStatus] + elsif includes.find{|v| v.is_a?(Hash) && v.keys.first.eql?(:ontology)} + includes << {:ontology=>[:administeredBy, :acronym, :name, :viewingRestriction, :group, :hasDomain]} + end + + submissions = submissions_query.include(includes) + if page? + submissions.page(page, size).all + else + submissions.to_a + end + end + + def include_ready?(options) + options[:status] && options[:status].to_s.upcase.eql?("READY") + end + + end + end +end + +helpers Sinatra::Helpers::SubmissionHelper \ No newline at end of file From f1b1234644f394fa902730df82180114f6e1996c Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Wed, 26 Apr 2023 09:23:13 +0200 Subject: [PATCH 37/53] implement apply_filters to submissions endpoint using SPARQL FILTERs --- helpers/request_params_helper.rb | 51 ++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/helpers/request_params_helper.rb b/helpers/request_params_helper.rb index e7ec091a..251af299 100644 --- a/helpers/request_params_helper.rb +++ b/helpers/request_params_helper.rb @@ -13,6 +13,10 @@ def settings_params(klass) [attributes, page, size, order_by, bring_unmapped] end + def page? + !params[:page].nil? + end + def is_set?(param) !param.nil? && param != "" end @@ -25,6 +29,53 @@ def filter build_filter end + def apply_filters(query) + + filters = { + naturalLanguage: params[:naturalLanguage]&.split(',') , #%w[http://lexvo.org/id/iso639-3/fra http://lexvo.org/id/iso639-3/eng], + hasOntologyLanguage_acronym: params[:hasOntologyLanguage]&.split(',') , #%w[OWL SKOS], + ontology_hasDomain_acronym: params[:hasDomain]&.split(',') , #%w[Crop Vue_francais], + ontology_group_acronym: params[:group]&.split(','), #%w[RICE CROP], + ontology_name: Array(params[:name]) + Array(params[:name]&.capitalize), + isOfType: params[:isOfType]&.split(','), #["http://omv.ontoware.org/2005/05/ontology#Vocabulary"], + viewingRestriction: params[:viewingRestriction]&.split(','), #["private"] + } + inverse_filters = { + status: params[:status], #"retired", + submissionStatus: params[:submissionStatus] #"RDF", + } + + filters.each do |key , values| + attr = extract_attr(key) + next if Array(values).empty? + + filter = Goo::Filter.new(attr).regex(values.first) + values.drop(1).each do |v| + filter = filter.or(Goo::Filter.new(attr).regex(v)) + end + query = query.filter(filter) + end + + inverse_filters.each do |key ,value| + attr = extract_attr(key) + next unless value + + filter = Goo::Filter.new(attr).regex("^(?:(?!#{value}).)*$") + query = query.filter(filter) + end + query + end + + def extract_attr(key) + attr, sub_attr, sub_sub_attr = key.to_s.split('_') + + return attr.to_sym unless sub_attr + + return {attr.to_sym => [sub_attr.to_sym]} unless sub_sub_attr + + {attr.to_sym => [sub_attr.to_sym => sub_sub_attr.to_sym]} + end + def get_order_by_from(params, default_order = :asc) if is_set?(params['sortby']) orders = (params["order"] || default_order.to_s).split(',') From 64e7286516e0d5b783f9c8746b42216608c493a6 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Fri, 28 Apr 2023 10:49:41 +0200 Subject: [PATCH 38/53] add test for submissions endpoint pagination --- Gemfile | 2 +- Gemfile.lock | 17 +++++++++-------- .../test_ontology_submissions_controller.rb | 15 +++++++++++++++ 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/Gemfile b/Gemfile index da2c84f9..71cdf420 100644 --- a/Gemfile +++ b/Gemfile @@ -48,7 +48,7 @@ gem 'ncbo_annotator', git: 'https://github.com/ontoportal-lirmm/ncbo_annotator.g gem 'ncbo_cron', git: 'https://github.com/ontoportal-lirmm/ncbo_cron.git', branch: 'master' gem 'ncbo_ontology_recommender', git: 'https://github.com/ncbo/ncbo_ontology_recommender.git', branch: 'master' gem 'sparql-client', github: 'ontoportal-lirmm/sparql-client', branch: 'master' -gem 'ontologies_linked_data', git: 'https://github.com/ontoportal-lirmm/ontologies_linked_data.git', branch: 'feature/migrate-attrbiutes-metadata-to-scheme-file' +gem 'ontologies_linked_data', git: 'https://github.com/ontoportal-lirmm/ontologies_linked_data.git', branch: 'development' group :development do # bcrypt_pbkdf and ed35519 is required for capistrano deployments when using ed25519 keys; see https://github.com/miloserdow/capistrano-deploy/issues/42 diff --git a/Gemfile.lock b/Gemfile.lock index 7b61090b..d76badc2 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -11,7 +11,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/goo.git - revision: f9b73ce6b6fac92bffd0a2a1b9dc9ba9266acf79 + revision: 1484ad56e39810208ee08a14d1319b6bc112f647 branch: development specs: goo (0.0.2) @@ -53,8 +53,8 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: 1e6886bec9c0c07bf5bfaad51e095a04c5bd0197 - branch: feature/migrate-attrbiutes-metadata-to-scheme-file + revision: f2168c8ad9ec447338da914d10a352df558494b5 + branch: development specs: ontologies_linked_data (0.0.1) activesupport @@ -103,11 +103,11 @@ GEM activesupport (3.2.22.5) i18n (~> 0.6, >= 0.6.4) multi_json (~> 1.0) - addressable (2.8.1) + addressable (2.8.4) public_suffix (>= 2.0.2, < 6.0) airbrussh (1.4.1) sshkit (>= 1.6.1, != 1.7.0) - backports (3.24.0) + backports (3.24.1) bcrypt (3.1.18) bcrypt_pbkdf (1.1.0) bigdecimal (1.4.2) @@ -171,7 +171,7 @@ GEM retriable (>= 2.0, < 4.a) rexml webrick - googleauth (1.3.0) + googleauth (1.5.2) faraday (>= 0.17.3, < 3.a) jwt (>= 1.4, < 3.0) memoist (~> 0.16) @@ -228,7 +228,7 @@ GEM net-protocol net-ssh (7.1.0) netrc (0.11.0) - newrelic_rpm (9.0.0) + newrelic_rpm (9.2.0) oj (2.18.5) omni_logger (0.1.4) logger @@ -249,7 +249,7 @@ GEM rack (>= 0.4) rack-cors (1.0.6) rack (>= 1.6.0) - rack-mini-profiler (3.0.0) + rack-mini-profiler (3.1.0) rack (>= 1.2.0) rack-protection (1.5.5) rack @@ -343,6 +343,7 @@ GEM webrick (1.8.1) PLATFORMS + x86_64-darwin-21 x86_64-linux DEPENDENCIES diff --git a/test/controllers/test_ontology_submissions_controller.rb b/test/controllers/test_ontology_submissions_controller.rb index 253fef54..ee8576bd 100644 --- a/test/controllers/test_ontology_submissions_controller.rb +++ b/test/controllers/test_ontology_submissions_controller.rb @@ -195,4 +195,19 @@ def test_download_acl_only end end + def test_submissions_pagination + num_onts_created, created_ont_acronyms = create_ontologies_and_submissions(ont_count: 2, submission_count: 2) + + get "/submissions" + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + + assert_equal 2, submissions.length + + + get "/submissions?page=1&pagesize=1" + assert last_response.ok? + submissions = MultiJson.load(last_response.body) + assert_equal 1, submissions["collection"].length + end end From 7e86f58e4ac1a2de75dc2cd269375cbe3f38c9d1 Mon Sep 17 00:00:00 2001 From: OntoPortal Bot Date: Fri, 28 Apr 2023 09:20:55 +0000 Subject: [PATCH 39/53] [ontoportal-bot] Gemfile.lock update --- Gemfile.lock | 1 - 1 file changed, 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index d76badc2..152465f7 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -343,7 +343,6 @@ GEM webrick (1.8.1) PLATFORMS - x86_64-darwin-21 x86_64-linux DEPENDENCIES From 1b7d514d949ce50bce74440082b83048b3a3aeec Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Mon, 1 May 2023 00:43:39 +0200 Subject: [PATCH 40/53] fix private only submission filter --- helpers/request_params_helper.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/request_params_helper.rb b/helpers/request_params_helper.rb index 251af299..30c4bf43 100644 --- a/helpers/request_params_helper.rb +++ b/helpers/request_params_helper.rb @@ -38,7 +38,7 @@ def apply_filters(query) ontology_group_acronym: params[:group]&.split(','), #%w[RICE CROP], ontology_name: Array(params[:name]) + Array(params[:name]&.capitalize), isOfType: params[:isOfType]&.split(','), #["http://omv.ontoware.org/2005/05/ontology#Vocabulary"], - viewingRestriction: params[:viewingRestriction]&.split(','), #["private"] + ontology_viewingRestriction: params[:viewingRestriction]&.split(','), #["private"] } inverse_filters = { status: params[:status], #"retired", From 4cb42d0a333d5a57da6780e7a90fa8448f732cfc Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Mon, 1 May 2023 17:56:54 +0200 Subject: [PATCH 41/53] add hasFormalityLevel filter for submissions endpoint --- helpers/request_params_helper.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/helpers/request_params_helper.rb b/helpers/request_params_helper.rb index 30c4bf43..3f973dec 100644 --- a/helpers/request_params_helper.rb +++ b/helpers/request_params_helper.rb @@ -38,6 +38,7 @@ def apply_filters(query) ontology_group_acronym: params[:group]&.split(','), #%w[RICE CROP], ontology_name: Array(params[:name]) + Array(params[:name]&.capitalize), isOfType: params[:isOfType]&.split(','), #["http://omv.ontoware.org/2005/05/ontology#Vocabulary"], + hasFormalityLevel: params[:hasFormalityLevel]&.split(','), #["http://w3id.org/nkos/nkostype#thesaurus"], ontology_viewingRestriction: params[:viewingRestriction]&.split(','), #["private"] } inverse_filters = { From 864e42fd8b8deffc6bc8c9e74765a1643026ee0a Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Mon, 1 May 2023 17:31:54 +0200 Subject: [PATCH 42/53] add for ontology: reviews, notes, projects on the submissions endpoints --- controllers/ontology_submissions_controller.rb | 2 ++ helpers/submission_helper.rb | 8 +++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/controllers/ontology_submissions_controller.rb b/controllers/ontology_submissions_controller.rb index 8749786d..9dd0d241 100644 --- a/controllers/ontology_submissions_controller.rb +++ b/controllers/ontology_submissions_controller.rb @@ -7,6 +7,8 @@ class OntologySubmissionsController < ApplicationController } subs = retrieve_latest_submissions(options) subs = subs.values unless page? + # Force to show ontology reviews, notes and projects by default only for this request + LinkedData::Models::Ontology.serialize_default(*(LinkedData::Models::Ontology.hypermedia_settings[:serialize_default] + [:reviews, :notes, :projects])) reply subs end diff --git a/helpers/submission_helper.rb b/helpers/submission_helper.rb index 38108050..73550369 100644 --- a/helpers/submission_helper.rb +++ b/helpers/submission_helper.rb @@ -26,10 +26,12 @@ def retrieve_submissions(options) # When asking to display all metadata, we are using bring_remaining on each submission. Slower but best way to retrieve all attrs if includes_param.first == :all - includes = [:submissionId, {:contact=>[:name, :email], :ontology=>[:administeredBy, :acronym, :name, :summaryOnly, :ontologyType, :viewingRestriction, :acl, - :group, :hasDomain, :views, :viewOf, :flat], :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym]}, :submissionStatus] + includes = [:submissionId, {:contact=>[:name, :email], + :ontology=>[:administeredBy, :acronym, :name, :summaryOnly, :ontologyType, :viewingRestriction, :acl, + :group, :hasDomain, :views, :viewOf, :flat, :notes, :reviews, :projects], + :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym]}, :submissionStatus] elsif includes.find{|v| v.is_a?(Hash) && v.keys.first.eql?(:ontology)} - includes << {:ontology=>[:administeredBy, :acronym, :name, :viewingRestriction, :group, :hasDomain]} + includes << {:ontology=>[:administeredBy, :acronym, :name, :viewingRestriction, :group, :hasDomain,:notes, :reviews, :projects]} end submissions = submissions_query.include(includes) From 04971936b6468897e41e42046aa8174a4a214a13 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Mon, 1 May 2023 18:01:37 +0200 Subject: [PATCH 43/53] bring submission metrics for submissions endpoints --- controllers/ontologies_controller.rb | 2 +- controllers/ontology_submissions_controller.rb | 2 +- helpers/submission_helper.rb | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/controllers/ontologies_controller.rb b/controllers/ontologies_controller.rb index da1b748c..51b77cbd 100644 --- a/controllers/ontologies_controller.rb +++ b/controllers/ontologies_controller.rb @@ -47,7 +47,7 @@ class OntologiesController < ApplicationController latest.bring({:contact=>[:name, :email], :ontology=>[:acronym, :name, :administeredBy, :group, :viewingRestriction, :doNotUpdate, :flat, :hasDomain, :summaryOnly, :acl, :viewOf, :ontologyType], - :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym]}) + :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym], :metrics =>[:classes, :individuals, :properties]}) else latest.bring(*OntologySubmission.goo_attrs_to_load(includes_param)) end diff --git a/controllers/ontology_submissions_controller.rb b/controllers/ontology_submissions_controller.rb index 9dd0d241..69c2a4cd 100644 --- a/controllers/ontology_submissions_controller.rb +++ b/controllers/ontology_submissions_controller.rb @@ -32,7 +32,7 @@ class OntologySubmissionsController < ApplicationController # When asking to display all metadata, we are using bring_remaining which is more performant than including all metadata (remove this when the query to get metadata will be fixed) ont.bring(submissions: [:released, :creationDate, :status, :submissionId, {:contact=>[:name, :email], :ontology=>[:administeredBy, :acronym, :name, :summaryOnly, :ontologyType, :viewingRestriction, :acl, :group, :hasDomain, :views, :viewOf, :flat], - :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym]}, :submissionStatus]) + :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym]}, :submissionStatus], :metrics =>[:classes, :individuals, :properties]) ont.submissions.each do |sub| sub.bring_remaining diff --git a/helpers/submission_helper.rb b/helpers/submission_helper.rb index 73550369..df141802 100644 --- a/helpers/submission_helper.rb +++ b/helpers/submission_helper.rb @@ -29,7 +29,8 @@ def retrieve_submissions(options) includes = [:submissionId, {:contact=>[:name, :email], :ontology=>[:administeredBy, :acronym, :name, :summaryOnly, :ontologyType, :viewingRestriction, :acl, :group, :hasDomain, :views, :viewOf, :flat, :notes, :reviews, :projects], - :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym]}, :submissionStatus] + :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym], :metrics =>[:classes, :individuals, :properties]}, + :submissionStatus] elsif includes.find{|v| v.is_a?(Hash) && v.keys.first.eql?(:ontology)} includes << {:ontology=>[:administeredBy, :acronym, :name, :viewingRestriction, :group, :hasDomain,:notes, :reviews, :projects]} end From 3abc4306a0ae0a26f46e00239552f884e3dc60f0 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Mon, 1 May 2023 18:04:53 +0200 Subject: [PATCH 44/53] bring all contact attributes if asked in the submissions endpoints --- controllers/ontologies_controller.rb | 6 +++++- helpers/submission_helper.rb | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/controllers/ontologies_controller.rb b/controllers/ontologies_controller.rb index 51b77cbd..1a8f2ebf 100644 --- a/controllers/ontologies_controller.rb +++ b/controllers/ontologies_controller.rb @@ -49,7 +49,11 @@ class OntologiesController < ApplicationController :hasDomain, :summaryOnly, :acl, :viewOf, :ontologyType], :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym], :metrics =>[:classes, :individuals, :properties]}) else - latest.bring(*OntologySubmission.goo_attrs_to_load(includes_param)) + includes = OntologySubmission.goo_attrs_to_load(includes_param) + + includes << {:contact=>[:name, :email]} if includes.find{|v| v.is_a?(Hash) && v.keys.first.eql?(:contact)} + + latest.bring(*includes) end end #remove the whole previous if block and replace by it: latest.bring(*OntologySubmission.goo_attrs_to_load(includes_param)) if latest diff --git a/helpers/submission_helper.rb b/helpers/submission_helper.rb index df141802..e19061c0 100644 --- a/helpers/submission_helper.rb +++ b/helpers/submission_helper.rb @@ -33,6 +33,8 @@ def retrieve_submissions(options) :submissionStatus] elsif includes.find{|v| v.is_a?(Hash) && v.keys.first.eql?(:ontology)} includes << {:ontology=>[:administeredBy, :acronym, :name, :viewingRestriction, :group, :hasDomain,:notes, :reviews, :projects]} + elsif includes.find{|v| v.is_a?(Hash) && v.keys.first.eql?(:contact)} + includes << {:contact=>[:name, :email]} end submissions = submissions_query.include(includes) From 69ef8bd9fb2fa268361d9f911a375e96db3c5671 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Mon, 1 May 2023 18:08:56 +0200 Subject: [PATCH 45/53] refactor submissions endpoint filters by extracting some methods --- helpers/request_params_helper.rb | 77 +++++++++++++++++++++----------- 1 file changed, 50 insertions(+), 27 deletions(-) diff --git a/helpers/request_params_helper.rb b/helpers/request_params_helper.rb index 3f973dec..d3ced82d 100644 --- a/helpers/request_params_helper.rb +++ b/helpers/request_params_helper.rb @@ -46,36 +46,11 @@ def apply_filters(query) submissionStatus: params[:submissionStatus] #"RDF", } - filters.each do |key , values| - attr = extract_attr(key) - next if Array(values).empty? - - filter = Goo::Filter.new(attr).regex(values.first) - values.drop(1).each do |v| - filter = filter.or(Goo::Filter.new(attr).regex(v)) - end - query = query.filter(filter) - end - - inverse_filters.each do |key ,value| - attr = extract_attr(key) - next unless value + query = add_direct_filters(filters, query) - filter = Goo::Filter.new(attr).regex("^(?:(?!#{value}).)*$") - query = query.filter(filter) - end - query + query = add_inverse_filters(inverse_filters, query) end - def extract_attr(key) - attr, sub_attr, sub_sub_attr = key.to_s.split('_') - - return attr.to_sym unless sub_attr - - return {attr.to_sym => [sub_attr.to_sym]} unless sub_sub_attr - - {attr.to_sym => [sub_attr.to_sym => sub_sub_attr.to_sym]} - end def get_order_by_from(params, default_order = :asc) if is_set?(params['sortby']) @@ -102,6 +77,54 @@ def bring_unmapped_to(page_data, sub, klass) end private + def extract_attr(key) + attr, sub_attr, sub_sub_attr = key.to_s.split('_') + + return attr.to_sym unless sub_attr + + return {attr.to_sym => [sub_attr.to_sym]} unless sub_sub_attr + + {attr.to_sym => [sub_attr.to_sym => sub_sub_attr.to_sym]} + end + + def add_direct_filters(filters, query) + filters.each do |key, values| + attr = extract_attr(key) + next if Array(values).empty? + + filter = Goo::Filter.new(attr).regex(values.first) + values.drop(1).each do |v| + filter = filter.or(Goo::Filter.new(attr).regex(v)) + end + query = query.filter(filter) + end + query + end + + def add_inverse_filters(inverse_filters, query) + inverse_filters.each do |key, value| + attr = extract_attr(key) + next unless value + + filter = Goo::Filter.new(attr).regex("^(?:(?!#{value}).)*$") + query = query.filter(filter) + end + query + end + + def add_acronym_name_filters(query) + if params[:acronym] + filter = Goo::Filter.new(extract_attr(:ontology_acronym)).regex(params[:acronym]) + if params[:name] + filter.or(Goo::Filter.new(extract_attr(:ontology_name)).regex(params[:name])) + end + query = query.filter(filter) + elsif params[:name] + filter = Goo::Filter.new(extract_attr(:ontology_name)).regex(params[:name]) + query = query.filter(filter) + end + query + end def sort_order_item(param, order) [param.to_sym, order.to_sym] From 1655473ab592af03ee34e64dd8c1af92fbb18a95 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Mon, 1 May 2023 18:09:48 +0200 Subject: [PATCH 46/53] add ontology acronym or name filters for submissions endpoints --- helpers/request_params_helper.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/helpers/request_params_helper.rb b/helpers/request_params_helper.rb index d3ced82d..a9b52cc0 100644 --- a/helpers/request_params_helper.rb +++ b/helpers/request_params_helper.rb @@ -49,6 +49,8 @@ def apply_filters(query) query = add_direct_filters(filters, query) query = add_inverse_filters(inverse_filters, query) + + query = add_acronym_name_filters(query) end From 060ed269e78027b5d01d22ea2d2829e11d3c048f Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Mon, 1 May 2023 18:10:34 +0200 Subject: [PATCH 47/53] add submissions endpoint order_by option --- Gemfile.lock | 11 +++++------ helpers/request_params_helper.rb | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index d76badc2..5dda09cb 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -11,7 +11,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/goo.git - revision: 1484ad56e39810208ee08a14d1319b6bc112f647 + revision: c3456c45c12ed92d4a3ae43cac7c1d4cdbf290b6 branch: development specs: goo (0.0.2) @@ -53,7 +53,7 @@ GIT GIT remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git - revision: f2168c8ad9ec447338da914d10a352df558494b5 + revision: 4ac9873f3d016259196a17cfc9c6ab8149468ec9 branch: development specs: ontologies_linked_data (0.0.1) @@ -193,7 +193,7 @@ GEM json_pure (2.6.3) jwt (2.7.0) kgio (2.11.4) - libxml-ruby (4.0.0) + libxml-ruby (4.1.0) logger (1.5.3) macaddr (1.7.2) systemu (~> 2.6.5) @@ -228,7 +228,7 @@ GEM net-protocol net-ssh (7.1.0) netrc (0.11.0) - newrelic_rpm (9.2.0) + newrelic_rpm (9.2.1) oj (2.18.5) omni_logger (0.1.4) logger @@ -344,7 +344,6 @@ GEM PLATFORMS x86_64-darwin-21 - x86_64-linux DEPENDENCIES activesupport (~> 3.0) @@ -398,4 +397,4 @@ DEPENDENCIES unicorn-worker-killer BUNDLED WITH - 2.3.14 + 2.3.23 diff --git a/helpers/request_params_helper.rb b/helpers/request_params_helper.rb index a9b52cc0..cf8ba1f8 100644 --- a/helpers/request_params_helper.rb +++ b/helpers/request_params_helper.rb @@ -51,6 +51,8 @@ def apply_filters(query) query = add_inverse_filters(inverse_filters, query) query = add_acronym_name_filters(query) + + add_order_by_patterns(query) end @@ -128,6 +130,19 @@ def add_acronym_name_filters(query) query end + def add_order_by_patterns(query) + if params[:order_by] + attr, sub_attr = params[:order_by].to_s.split('_') + if sub_attr + order_pattern = { attr.to_sym => { sub_attr.to_sym => (sub_attr.eql?("name") ? :asc : :desc) } } + else + order_pattern = { attr.to_sym => :desc } + end + query = query.order_by(order_pattern) + end + query + end + def sort_order_item(param, order) [param.to_sym, order.to_sym] end From 059d7e1bd1737dfff76479d86ade4e0dc81e26e1 Mon Sep 17 00:00:00 2001 From: OntoPortal Bot Date: Mon, 1 May 2023 19:14:56 +0000 Subject: [PATCH 48/53] [ontoportal-bot] Gemfile.lock update --- Gemfile.lock | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 89a9d393..5e8cfa16 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -343,10 +343,8 @@ GEM webrick (1.8.1) PLATFORMS - x86_64-darwin-21 x86_64-linux - DEPENDENCIES activesupport (~> 3.0) bcrypt_pbkdf (>= 1.0, < 2.0) @@ -399,4 +397,4 @@ DEPENDENCIES unicorn-worker-killer BUNDLED WITH - 2.3.23 + 2.3.14 From 3f29887a8236c17eae5cbf5f0b8c8bcf2615aea8 Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Mon, 1 May 2023 21:45:28 +0200 Subject: [PATCH 49/53] fix including ontology and contacts in the submissions endpoints --- helpers/submission_helper.rb | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/helpers/submission_helper.rb b/helpers/submission_helper.rb index e19061c0..6621ed43 100644 --- a/helpers/submission_helper.rb +++ b/helpers/submission_helper.rb @@ -31,12 +31,16 @@ def retrieve_submissions(options) :group, :hasDomain, :views, :viewOf, :flat, :notes, :reviews, :projects], :submissionStatus=>[:code], :hasOntologyLanguage=>[:acronym], :metrics =>[:classes, :individuals, :properties]}, :submissionStatus] - elsif includes.find{|v| v.is_a?(Hash) && v.keys.first.eql?(:ontology)} - includes << {:ontology=>[:administeredBy, :acronym, :name, :viewingRestriction, :group, :hasDomain,:notes, :reviews, :projects]} - elsif includes.find{|v| v.is_a?(Hash) && v.keys.first.eql?(:contact)} - includes << {:contact=>[:name, :email]} - end + else + if includes.find{|v| v.is_a?(Hash) && v.keys.include?(:ontology)} + includes << {:ontology=>[:administeredBy, :acronym, :name, :viewingRestriction, :group, :hasDomain,:notes, :reviews, :projects]} + end + if includes.find{|v| v.is_a?(Hash) && v.keys.include?(:contact)} + includes << {:contact=>[:name, :email]} + end + end + submissions = submissions_query.include(includes) if page? submissions.page(page, size).all From b45618a53a09be206f9cff64fb35c8e1a0b5ff7f Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Mon, 1 May 2023 23:34:19 +0200 Subject: [PATCH 50/53] fix list admin filter_access control (e.g for submissions endpoints) --- helpers/access_control_helper.rb | 4 ---- helpers/submission_helper.rb | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/helpers/access_control_helper.rb b/helpers/access_control_helper.rb index 1de3bee5..74416866 100644 --- a/helpers/access_control_helper.rb +++ b/helpers/access_control_helper.rb @@ -10,11 +10,7 @@ module AccessControlHelper def check_access(obj) return obj unless LinkedData.settings.enable_security if obj.is_a?(Enumerable) - if obj.first.is_a?(LinkedData::Models::Base) && obj.first.access_based_on? - check_access(obj.first) - else filter_access(obj) - end else if obj.respond_to?(:read_restricted?) && obj.read_restricted? readable = obj.readable?(env["REMOTE_USER"]) diff --git a/helpers/submission_helper.rb b/helpers/submission_helper.rb index 6621ed43..0a082823 100644 --- a/helpers/submission_helper.rb +++ b/helpers/submission_helper.rb @@ -33,14 +33,14 @@ def retrieve_submissions(options) :submissionStatus] else if includes.find{|v| v.is_a?(Hash) && v.keys.include?(:ontology)} - includes << {:ontology=>[:administeredBy, :acronym, :name, :viewingRestriction, :group, :hasDomain,:notes, :reviews, :projects]} + includes << {:ontology=>[:administeredBy, :acronym, :name, :viewingRestriction, :group, :hasDomain,:notes, :reviews, :projects,:acl]} end if includes.find{|v| v.is_a?(Hash) && v.keys.include?(:contact)} includes << {:contact=>[:name, :email]} end end - + submissions = submissions_query.include(includes) if page? submissions.page(page, size).all From b1e9ddf605bc7e232fe346b25f6f291fe094448e Mon Sep 17 00:00:00 2001 From: OntoPortal Bot Date: Mon, 1 May 2023 21:36:20 +0000 Subject: [PATCH 51/53] [ontoportal-bot] Gemfile.lock update --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 5e8cfa16..1d512537 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -228,7 +228,7 @@ GEM net-protocol net-ssh (7.1.0) netrc (0.11.0) - newrelic_rpm (9.2.1) + newrelic_rpm (9.2.2) oj (2.18.5) omni_logger (0.1.4) logger From 34f19a3fd9f865f7e547d36275bba1ae6dd0229d Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Tue, 2 May 2023 00:35:56 +0200 Subject: [PATCH 52/53] check access of ontologies in /ontologies/:acronym/submissions endpoint --- controllers/ontology_submissions_controller.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/controllers/ontology_submissions_controller.rb b/controllers/ontology_submissions_controller.rb index 69c2a4cd..96411a8d 100644 --- a/controllers/ontology_submissions_controller.rb +++ b/controllers/ontology_submissions_controller.rb @@ -25,9 +25,10 @@ class OntologySubmissionsController < ApplicationController ## # Display all submissions of an ontology get do - ont = Ontology.find(params["acronym"]).include(:acronym).first + ont = Ontology.find(params["acronym"]).include(:acronym, :administeredBy, :acl, :viewingRestriction).first error 422, "Ontology #{params["acronym"]} does not exist" unless ont check_last_modified_segment(LinkedData::Models::OntologySubmission, [ont.acronym]) + check_access(ont) if includes_param.first == :all # When asking to display all metadata, we are using bring_remaining which is more performant than including all metadata (remove this when the query to get metadata will be fixed) ont.bring(submissions: [:released, :creationDate, :status, :submissionId, From d5dcb9edcb972c29fe97d777575e2de4f79398ba Mon Sep 17 00:00:00 2001 From: Syphax Bouazzouni Date: Tue, 2 May 2023 03:58:27 +0200 Subject: [PATCH 53/53] include ontology viewOf attribute in the submission endpoints --- helpers/submission_helper.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/submission_helper.rb b/helpers/submission_helper.rb index 0a082823..07e4f27a 100644 --- a/helpers/submission_helper.rb +++ b/helpers/submission_helper.rb @@ -33,7 +33,7 @@ def retrieve_submissions(options) :submissionStatus] else if includes.find{|v| v.is_a?(Hash) && v.keys.include?(:ontology)} - includes << {:ontology=>[:administeredBy, :acronym, :name, :viewingRestriction, :group, :hasDomain,:notes, :reviews, :projects,:acl]} + includes << {:ontology=>[:administeredBy, :acronym, :name, :viewingRestriction, :group, :hasDomain,:notes, :reviews, :projects,:acl, :viewOf]} end if includes.find{|v| v.is_a?(Hash) && v.keys.include?(:contact)}