diff --git a/.ruby-version b/.ruby-version index 818bd47a..fd2a0186 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -3.0.6 +3.1.0 diff --git a/Dockerfile b/Dockerfile index b6130644..896b90cb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -ARG RUBY_VERSION=3.0 +ARG RUBY_VERSION=3.1 ARG DISTRO_NAME=bullseye FROM ruby:$RUBY_VERSION-$DISTRO_NAME @@ -14,8 +14,6 @@ COPY Gemfile* /srv/ontoportal/ontologies_api/ WORKDIR /srv/ontoportal/ontologies_api -RUN gem update --system -RUN gem install bundler ENV BUNDLE_PATH=/srv/ontoportal/bundle RUN bundle install diff --git a/Gemfile b/Gemfile index 26aa283e..19bc5eda 100644 --- a/Gemfile +++ b/Gemfile @@ -46,7 +46,7 @@ gem 'goo', github: 'ncbo/goo', branch: 'master' gem 'ncbo_annotator', github: 'ncbo/ncbo_annotator', branch: 'master' gem 'ncbo_cron', github: 'ncbo/ncbo_cron', branch: 'master' gem 'ncbo_ontology_recommender', github: 'ncbo/ncbo_ontology_recommender', branch: 'master' -gem 'ontologies_linked_data', github: 'ncbo/ontologies_linked_data', branch: 'master' +gem 'ontologies_linked_data', github: 'ontoportal-lirmm/ontologies_linked_data', branch: 'feature/extract-agroportal-model-to-ontoportal' gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'master' group :development do @@ -80,3 +80,5 @@ group :test do gem 'webmock', '~> 3.19.1' gem 'webrick' end + +gem 'net-ftp' \ No newline at end of file diff --git a/Gemfile.lock b/Gemfile.lock index e7e28b9c..95c35bb9 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -27,7 +27,7 @@ GIT GIT remote: https://github.com/ncbo/ncbo_cron.git - revision: 1e0cb7f5e230e1d19c02f11f3fd76ed9831a3807 + revision: c703f1387bf633999a285938067a76a1bff3ec87 branch: master specs: ncbo_cron (0.0.1) @@ -55,9 +55,19 @@ GIT redis GIT - remote: https://github.com/ncbo/ontologies_linked_data.git - revision: 536cbe5cef9e23abc299ed44942c0b3966f9c18d + remote: https://github.com/ncbo/sparql-client.git + revision: e89c26aa96f184dbe9b52d51e04fb3d9ba998dbc branch: master + specs: + sparql-client (1.0.1) + json_pure (>= 1.4) + net-http-persistent (= 2.9.4) + rdf (>= 1.0) + +GIT + remote: https://github.com/ontoportal-lirmm/ontologies_linked_data.git + revision: 0622986d71175da6eda30da3c222d7dea2fb3b46 + branch: feature/extract-agroportal-model-to-ontoportal specs: ontologies_linked_data (0.0.1) activesupport @@ -74,16 +84,6 @@ GIT rsolr rubyzip -GIT - remote: https://github.com/ncbo/sparql-client.git - revision: e89c26aa96f184dbe9b52d51e04fb3d9ba998dbc - branch: master - specs: - sparql-client (1.0.1) - json_pure (>= 1.4) - net-http-persistent (= 2.9.4) - rdf (>= 1.0) - GIT remote: https://github.com/palexander/rack-post-body-to-params.git revision: 0fd30e710386d0cb8a3a6833d9549d7b655d5398 @@ -118,7 +118,7 @@ GEM bcrypt (3.1.20) bcrypt_pbkdf (1.1.1) bcrypt_pbkdf (1.1.1-arm64-darwin) - bigdecimal (3.1.8) + bigdecimal (3.1.9) builder (3.3.0) capistrano (3.19.2) airbrussh (>= 1.0.0) @@ -133,17 +133,17 @@ GEM capistrano (~> 3.1) sshkit (~> 1.3) coderay (1.1.3) - concurrent-ruby (1.3.4) - connection_pool (2.4.1) + concurrent-ruby (1.3.5) + connection_pool (2.5.0) crack (0.4.5) rexml cube-ruby (0.0.3) dante (0.2.0) - date (3.4.0) + date (3.4.1) docile (1.4.1) domain_name (0.6.20240107) ed25519 (1.3.0) - faraday (2.12.1) + faraday (2.12.2) faraday-net_http (>= 2.0, < 3.5) json logger @@ -153,24 +153,25 @@ GEM net-http (>= 0.5.0) faraday-retry (2.2.1) faraday (~> 2.0) - ffi (1.17.0-aarch64-linux-gnu) - ffi (1.17.0-arm64-darwin) - ffi (1.17.0-x86_64-linux-gnu) - gapic-common (0.23.0) + ffi (1.17.1) + ffi (1.17.1-arm64-darwin) + gapic-common (0.25.0) faraday (>= 1.9, < 3.a) faraday-retry (>= 1.0, < 3.a) + google-cloud-env (~> 2.2) + google-logging-utils (~> 0.1) google-protobuf (>= 3.25, < 5.a) googleapis-common-protos (~> 1.6) googleapis-common-protos-types (~> 1.15) - googleauth (~> 1.11) - grpc (~> 1.65) + googleauth (~> 1.12) + grpc (~> 1.66) get_process_mem (0.2.7) ffi (~> 1.0) - google-analytics-data (0.6.1) + google-analytics-data (0.7.0) google-analytics-data-v1beta (>= 0.11, < 2.a) google-cloud-core (~> 1.6) - google-analytics-data-v1beta (0.13.1) - gapic-common (>= 0.21.1, < 2.a) + google-analytics-data-v1beta (0.16.0) + gapic-common (>= 0.25.0, < 2.a) google-cloud-errors (~> 1.0) google-cloud-core (1.7.1) google-cloud-env (>= 1.0, < 3.a) @@ -178,29 +179,31 @@ GEM google-cloud-env (2.2.1) faraday (>= 1.0, < 3.a) google-cloud-errors (1.4.0) - google-protobuf (3.25.5-aarch64-linux) - google-protobuf (3.25.5-arm64-darwin) - google-protobuf (3.25.5-x86_64-linux) + google-logging-utils (0.1.0) + google-protobuf (3.25.6-aarch64-linux) + google-protobuf (3.25.6-arm64-darwin) + google-protobuf (3.25.6-x86_64-linux) googleapis-common-protos (1.6.0) google-protobuf (>= 3.18, < 5.a) googleapis-common-protos-types (~> 1.7) grpc (~> 1.41) - googleapis-common-protos-types (1.16.0) + googleapis-common-protos-types (1.18.0) google-protobuf (>= 3.18, < 5.a) - googleauth (1.11.2) + googleauth (1.13.1) faraday (>= 1.0, < 3.a) - google-cloud-env (~> 2.1) + google-cloud-env (~> 2.2) + google-logging-utils (~> 0.1) jwt (>= 1.4, < 3.0) multi_json (~> 1.11) os (>= 0.9, < 2.0) signet (>= 0.16, < 2.a) - grpc (1.67.0-aarch64-linux) + grpc (1.70.1-aarch64-linux) google-protobuf (>= 3.25, < 5.0) googleapis-common-protos-types (~> 1.0) - grpc (1.67.0-arm64-darwin) + grpc (1.70.1-arm64-darwin) google-protobuf (>= 3.25, < 5.0) googleapis-common-protos-types (~> 1.0) - grpc (1.67.0-x86_64-linux) + grpc (1.70.1-x86_64-linux) google-protobuf (>= 3.25, < 5.0) googleapis-common-protos-types (~> 1.0) haml (5.2.2) @@ -208,20 +211,21 @@ GEM tilt hashdiff (1.1.2) http-accept (1.7.0) - http-cookie (1.0.7) + http-cookie (1.0.8) domain_name (~> 0.5) - i18n (1.14.6) + i18n (1.14.7) concurrent-ruby (~> 1.0) - json (2.8.2) + json (2.10.1) json-schema (2.8.1) addressable (>= 2.4) json_pure (2.8.1) - jwt (2.9.3) + jwt (2.10.1) base64 kgio (2.11.4) - language_server-protocol (3.17.0.3) + language_server-protocol (3.17.0.4) libxml-ruby (5.0.3) - logger (1.6.1) + lint_roller (1.1.0) + logger (1.6.6) macaddr (1.7.2) systemu (~> 2.6.5) mail (2.8.1) @@ -233,35 +237,38 @@ GEM mime-types (3.6.0) logger mime-types-data (~> 3.2015) - mime-types-data (3.2024.1105) + mime-types-data (3.2025.0204) mini_mime (1.1.5) - minitest (5.25.2) + minitest (5.25.4) minitest-hooks (1.5.2) minitest (> 5.3) minitest-stub_any_instance (1.0.3) mlanett-redis-lock (0.2.7) redis multi_json (1.15.0) - net-http (0.5.0) + net-ftp (0.3.8) + net-protocol + time + net-http (0.6.0) uri net-http-persistent (2.9.4) - net-imap (0.4.18) + net-imap (0.5.6) date net-protocol net-pop (0.1.2) net-protocol net-protocol (0.2.2) timeout - net-scp (4.0.0) + net-scp (4.1.0) net-ssh (>= 2.6.5, < 8.0.0) net-sftp (4.0.0) net-ssh (>= 5.0.0, < 8.0.0) - net-smtp (0.5.0) + net-smtp (0.5.1) net-protocol net-ssh (7.3.0) netrc (0.11.0) - newrelic_rpm (9.16.0) - oj (3.16.7) + newrelic_rpm (9.17.0) + oj (3.16.9) bigdecimal (>= 3.0) ostruct (>= 0.2) omni_logger (0.1.4) @@ -270,12 +277,12 @@ GEM ostruct (0.6.1) parallel (1.26.3) parseconfig (1.1.2) - parser (3.3.6.0) + parser (3.3.7.1) ast (~> 2.4.1) racc pony (1.13.1) mail (>= 2.0) - pry (0.15.0) + pry (0.15.2) coderay (~> 1.1) method_source (~> 1.0) public_suffix (6.0.1) @@ -293,7 +300,7 @@ GEM rack (>= 1.2.0) rack-protection (1.5.5) rack - rack-test (2.1.0) + rack-test (2.2.0) rack (>= 1.3) rack-timeout (0.7.0) rainbow (3.1.1) @@ -304,14 +311,14 @@ GEM redcarpet (3.6.0) redis (5.3.0) redis-client (>= 0.22.0) - redis-client (0.22.2) + redis-client (0.23.2) connection_pool redis-rack-cache (2.2.1) rack-cache (>= 1.10, < 2) redis-store (>= 1.6, < 2) redis-store (1.11.0) redis (>= 4, < 6) - regexp_parser (2.9.2) + regexp_parser (2.10.0) request_store (1.7.0) rack (>= 1.4) rest-client (2.1.0) @@ -319,25 +326,26 @@ GEM http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) - rexml (3.3.9) + rexml (3.4.1) rsolr (2.6.0) builder (>= 2.1.2) faraday (>= 0.9, < 3, != 2.0.0) - rubocop (1.69.0) + rubocop (1.72.2) json (~> 2.3) - language_server-protocol (>= 3.17.0) + language_server-protocol (~> 3.17.0.2) + lint_roller (~> 1.1.0) parallel (~> 1.10) parser (>= 3.3.0.2) rainbow (>= 2.2.2, < 4.0) - regexp_parser (>= 2.4, < 3.0) - rubocop-ast (>= 1.36.1, < 2.0) + regexp_parser (>= 2.9.3, < 3.0) + rubocop-ast (>= 1.38.0, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 2.4.0, < 4.0) - rubocop-ast (1.36.1) + rubocop-ast (1.38.0) parser (>= 3.3.1.0) ruby-progressbar (1.13.0) ruby-xxHash (0.4.0.2) - rubyzip (2.3.2) + rubyzip (2.4.1) rufus-scheduler (2.0.24) tzinfo (>= 0.3.22) signet (0.19.0) @@ -367,8 +375,9 @@ GEM rack-test sinatra (~> 1.4.0) tilt (>= 1.3, < 3) - sshkit (1.23.2) + sshkit (1.24.0) base64 + logger net-scp (>= 1.1.2) net-sftp (>= 2.1.2) net-ssh (>= 2.8.0) @@ -376,11 +385,13 @@ GEM systemu (2.6.5) temple (0.10.3) thread_safe (0.3.6) - tilt (2.4.0) - timeout (0.4.2) + tilt (2.6.0) + time (0.4.1) + date + timeout (0.4.3) tzinfo (1.2.11) thread_safe (~> 0.1) - unicode-display_width (3.1.2) + unicode-display_width (3.1.4) unicode-emoji (~> 4.0, >= 4.0.4) unicode-emoji (4.0.4) unicorn (6.1.0) @@ -396,11 +407,12 @@ GEM addressable (>= 2.8.0) crack (>= 0.3.2) hashdiff (>= 0.4.0, < 2.0.0) - webrick (1.9.0) + webrick (1.9.1) PLATFORMS aarch64-linux arm64-darwin-22 + arm64-darwin-24 x86_64-linux DEPENDENCIES @@ -425,6 +437,7 @@ DEPENDENCIES ncbo_annotator! ncbo_cron! ncbo_ontology_recommender! + net-ftp newrelic_rpm oj ontologies_linked_data! @@ -458,4 +471,4 @@ DEPENDENCIES webrick BUNDLED WITH - 2.5.7 + 2.6.3 diff --git a/controllers/submission_metadata_controller.rb b/controllers/submission_metadata_controller.rb new file mode 100644 index 00000000..e9a256d2 --- /dev/null +++ b/controllers/submission_metadata_controller.rb @@ -0,0 +1,14 @@ +class SubmissionMetadataController < ApplicationController + + ## + # Display all metadata for submissions + get '/submission_metadata' do + reply klass_metadata(LinkedData::Models::OntologySubmission, 'submission_metadata') + end + + ## + # Display all metadata for ontologies + get '/ontology_metadata' do + reply klass_metadata(LinkedData::Models::Ontology, 'ontology_metadata') + end +end \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 11811fe7..7aa0569d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ x-app: &app build: context: . args: - RUBY_VERSION: '3.0' + RUBY_VERSION: '3.1' # Increase the version number in the image tag every time Dockerfile or its arguments is changed image: ontologies_api:0.0.5 environment: &env diff --git a/helpers/application_helper.rb b/helpers/application_helper.rb index ad813e31..ebd37e7f 100644 --- a/helpers/application_helper.rb +++ b/helpers/application_helper.rb @@ -11,10 +11,10 @@ def h(text) Rack::Utils.escape_html(text) end - ## + ## # Populate +obj+ using values from +params+ # Will also try to find related objects using a Goo lookup. - # TODO: Currerntly, this allows for mass-assignment of everything, which will permit + # TODO: Currently, this allows for mass-assignment of everything, which will permit # users to overwrite any attribute, including things like passwords. def populate_from_params(obj, params) return if obj.nil? @@ -23,14 +23,15 @@ def populate_from_params(obj, params) if obj.is_a?(LinkedData::Models::Base) obj.bring_remaining if obj.exist? no_writable_attributes = obj.class.attributes(:all) - obj.class.attributes - params = params.reject {|k,v| no_writable_attributes.include? k.to_sym} + params = params.reject { |k, v| no_writable_attributes.include? k.to_sym } end params.each do |attribute, value| next if value.nil? - # Deal with empty strings + # Deal with empty strings for String and URI empty_string = value.is_a?(String) && value.empty? - old_string_value_exists = obj.respond_to?(attribute) && obj.send(attribute).is_a?(String) + old_string_value_exists = obj.respond_to?(attribute) && (obj.send(attribute).is_a?(String) || obj.send(attribute).is_a?(RDF::URI)) + old_string_value_exists ||= obj.respond_to?(attribute) && obj.send(attribute).is_a?(LinkedData::Models::Base) if old_string_value_exists && empty_string value = nil elsif empty_string @@ -51,6 +52,10 @@ def populate_from_params(obj, params) value = is_arr ? value : [value] new_value = [] value.each do |cls| + if uri_as_needed(cls["ontology"]).nil? + new_value << cls + next + end sub = LinkedData::Models::Ontology.find(uri_as_needed(cls["ontology"])).first.latest_submission new_value << LinkedData::Models::Class.find(cls["class"]).in(sub).first end @@ -58,8 +63,8 @@ def populate_from_params(obj, params) elsif attr_cls && not_hash_or_array || (attr_cls && not_array_of_hashes) # Replace the initial value with the object, handling Arrays as appropriate if value.is_a?(Array) - value = value.map {|e| attr_cls.find(uri_as_needed(e)).include(attr_cls.attributes).first} - else + value = value.map { |e| attr_cls.find(uri_as_needed(e)).include(attr_cls.attributes).first } + elsif !value.nil? value = attr_cls.find(uri_as_needed(value)).include(attr_cls.attributes).first end elsif attr_cls @@ -67,28 +72,31 @@ def populate_from_params(obj, params) if value.is_a?(Array) retrieved_values = [] value.each do |e| + e = e.to_h retrieved_value = attr_cls.where(e.symbolize_keys).first - if retrieved_value - retrieved_values << retrieved_value - else - retrieved_values << populate_from_params(attr_cls.new, e.symbolize_keys).save - end + retrieved_values << (retrieved_value || populate_from_params(attr_cls.new, e.symbolize_keys).save) end else - retrieved_values = attr_cls.where(value.symbolize_keys).to_a - unless retrieved_values - retrieved_values = populate_from_params(attr_cls.new, e.symbolize_keys).save - end + retrieved_values = attr_cls.where(value.to_h.symbolize_keys).to_a + retrieved_values ||= populate_from_params(attr_cls.new, e.symbolize_keys).save end value = retrieved_values elsif attribute_settings && attribute_settings[:enforce] && attribute_settings[:enforce].include?(:date_time) # TODO: Remove this awful hack when obj.class.model_settings[:range][attribute] contains DateTime class - value = DateTime.parse(value) + is_array = value.is_a?(Array) + value = Array(value).map { |v| DateTime.parse(v) } + value = value.first unless is_array + value + elsif attribute_settings && attribute_settings[:enforce] && attribute_settings[:enforce].include?(:uri) && attribute_settings[:enforce].include?(:list) + # in case its a list of URI, convert all value to IRI + value = value.map { |v| RDF::IRI.new(v) } elsif attribute_settings && attribute_settings[:enforce] && attribute_settings[:enforce].include?(:uri) # TODO: Remove this awful hack when obj.class.model_settings[:range][attribute] contains RDF::IRI class - value = RDF::IRI.new(value) + unless value.nil? + # If pass an empty string for an URI : set it as nil. + value = RDF::IRI.new(value) + end end - # Don't populate naming attributes if they exist if obj.class.model_settings[:name_with] != attribute || obj.send(attribute).nil? obj.send("#{attribute}=", value) if obj.respond_to?("#{attribute}=") @@ -203,11 +211,11 @@ def restricted_ontologies(params=nil) found_onts = onts.length > 0 Ontology.where.models(onts).include(*Ontology.access_control_load_attrs).all else - if params["also_include_views"] == "true" - onts = Ontology.where.include(Ontology.goo_attrs_to_load()).to_a + onts = if params["also_include_views"] == "true" + Ontology.where.include(Ontology.goo_attrs_to_load()).to_a else - onts = Ontology.where.filter(Goo::Filter.new(:viewOf).unbound).include(Ontology.goo_attrs_to_load()).to_a - end + Ontology.where.filter(Goo::Filter.new(:viewOf).unbound).include(Ontology.goo_attrs_to_load()).to_a + end found_onts = onts.length > 0 @@ -358,11 +366,11 @@ def retrieve_latest_submissions(options = {}) include_views = options[:also_include_views] || false includes = OntologySubmission.goo_attrs_to_load(includes_param) includes << :submissionStatus unless includes.include?(:submissionStatus) - if any - submissions_query = OntologySubmission.where + submissions_query = if any + OntologySubmission.where else - submissions_query = OntologySubmission.where(submissionStatus: [ code: status]) - end + OntologySubmission.where(submissionStatus: [ code: status]) + end submissions_query = submissions_query.filter(Goo::Filter.new(ontology: [:viewOf]).unbound) unless include_views submissions = submissions_query.include(includes).to_a diff --git a/helpers/metadata_helper.rb b/helpers/metadata_helper.rb new file mode 100644 index 00000000..090fd14c --- /dev/null +++ b/helpers/metadata_helper.rb @@ -0,0 +1,89 @@ +require 'sinatra/base' +require 'date' + +module Sinatra + module Helpers + module MetadataHelper + + def klass_metadata(klass, type) + all_attr = [] + klass.attributes(:all).each do |attr| + + id_url_prefix = if LinkedData.settings.id_url_prefix.nil? || LinkedData.settings.id_url_prefix.empty? + 'http://data.bioontology.org/' + else + LinkedData.settings.id_url_prefix + end + + attr_settings = {} + attr_settings[:@id] = "#{id_url_prefix}#{type}/#{attr.to_s}" + attr_settings[:@type] = "#{id_url_prefix}metadata/#{type.camelize}" + attr_settings[:attribute] = attr.to_s + + # Get metadata namespace + attr_settings[:namespace] = if klass.attribute_settings(attr)[:namespace].nil? + nil + else + klass.attribute_settings(attr)[:namespace].to_s + end + + # Get metadata label if one + attr_settings[:label] = if klass.attribute_settings(attr)[:label].nil? + nil + else + klass.attribute_settings(attr)[:label] + end + + # Get if it is an extracted metadata + attr_settings[:extracted] = klass.attribute_settings(attr)[:extractedMetadata].eql?('true') ? 'true' : 'false' + + # Get mappings of the metadata + attr_settings[:metadataMappings] = if klass.attribute_settings(attr)[:metadataMappings].nil? + nil + else + klass.attribute_settings(attr)[:metadataMappings] + end + + # Get enforced from the metadata + attr_settings[:enforce] = [] + klass.attribute_settings(attr)[:enforce]&.each do |enforced| + next if enforced.is_a?(Proc) + + attr_settings[:enforce] << enforced.to_s + end + + # Get enforcedValues from the metadata + attr_settings[:enforcedValues] = klass.attribute_settings(attr)[:enforcedValues] + + # Get display from the metadata + attr_settings[:category] = if klass.attribute_settings(attr)[:display].nil? + 'no' + else + klass.attribute_settings(attr)[:display] + end + + unless klass.attribute_settings(attr)[:helpText].nil? + attr_settings[:helpText] = klass.attribute_settings(attr)[:helpText] + end + + unless klass.attribute_settings(attr)[:description].nil? + attr_settings[:description] = klass.attribute_settings(attr)[:description] + end + + unless klass.attribute_settings(attr)[:example].nil? + attr_settings[:example] = klass.attribute_settings(attr)[:example] + end + + attr_settings[:@context] = { + '@vocab' => "#{id_url_prefix}metadata/" + } + + all_attr << attr_settings + end + all_attr + end + end + end +end + +helpers Sinatra::Helpers::MetadataHelper diff --git a/test/controllers/test_ontology_submissions_controller.rb b/test/controllers/test_ontology_submissions_controller.rb index d0f58582..b388126f 100644 --- a/test/controllers/test_ontology_submissions_controller.rb +++ b/test/controllers/test_ontology_submissions_controller.rb @@ -20,7 +20,7 @@ def self._set_vars "file" => Rack::Test::UploadedFile.new(@@test_file, ""), released: DateTime.now.to_s, contact: [{name: "test_name", email: "test3@example.org"}], - URI: 'https://test.com/test', + uri: 'https://test.com/test', status: 'production', description: 'ontology description' }