From 0d042a1eb4bba24c873dd87afa0a672df653c9af Mon Sep 17 00:00:00 2001
From: Vincent Emonet <vincent.emonet@lirmm.fr>
Date: Tue, 25 Oct 2016 15:41:18 +0200
Subject: [PATCH 001/168] add comment

---
 lib/goo/sparql/queries.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/goo/sparql/queries.rb b/lib/goo/sparql/queries.rb
index 7e6adb35..b8d282ab 100644
--- a/lib/goo/sparql/queries.rb
+++ b/lib/goo/sparql/queries.rb
@@ -724,6 +724,7 @@ def self.model_load_sliced(*options)
               unless models_by_id[id].class.handler?(v)
                 unless object.nil? && !models_by_id[id].instance_variable_get("@#{v.to_s}").nil?
                   if v != :id
+                    # TODO: handle multilingual values here
                     # if multiple language values are included for a given property, set the
                     # corresponding model attribute to the English language value - NCBO-1662
                     if sol[v].kind_of?(RDF::Literal)

From ae67841e4f60e56c1bd867ce0a2738e0739b0ceb Mon Sep 17 00:00:00 2001
From: Vincent Emonet <vincent.emonet@gmail.com>
Date: Fri, 9 Jun 2017 12:29:31 +0200
Subject: [PATCH 002/168] add Goo.main_languages, an array to define the
 main_languages of the Portal

---
 lib/goo.rb | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/lib/goo.rb b/lib/goo.rb
index 43a1e408..3fe28dc7 100644
--- a/lib/goo.rb
+++ b/lib/goo.rb
@@ -23,6 +23,8 @@ module Goo
 
   @@resource_options = Set.new([:persistent]).freeze
 
+  @@main_languages = ["en","eng"]
+
   @@configure_flag = false
   @@sparql_backends = {}
   @@model_by_name = {}
@@ -87,6 +89,14 @@ def self.test_reset
                   redis_cache: @@redis_client })
   end
 
+  def self.main_languages
+    @@main_languages
+  end
+
+  def self.main_languages=(value)
+    @@main_languages = value
+  end
+
   def self.use_cache=(value)
     @@use_cache = value
     set_sparql_cache

From 18d5ecad231e0fa609d86da5ae0de26c372b7c93 Mon Sep 17 00:00:00 2001
From: Vincent Emonet <vincent.emonet@gmail.com>
Date: Fri, 9 Jun 2017 12:54:44 +0200
Subject: [PATCH 003/168] Use 'Goo.main_languages' to get a property value. now
 when retrieving a value only the object with a language included in
 main_languages will be taken (value with no lang defined are taken too). Be
 careful if an ontology contains several prefLabel with lang defined here

---
 lib/goo.rb                |  1 +
 lib/goo/sparql/queries.rb | 14 ++++++++++----
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/lib/goo.rb b/lib/goo.rb
index 3fe28dc7..5929ab70 100644
--- a/lib/goo.rb
+++ b/lib/goo.rb
@@ -23,6 +23,7 @@ module Goo
 
   @@resource_options = Set.new([:persistent]).freeze
 
+  # Define the languages from which the properties values will be taken (be careful if prefLabel with different lang, only one will be taken)
   @@main_languages = ["en","eng"]
 
   @@configure_flag = false
diff --git a/lib/goo/sparql/queries.rb b/lib/goo/sparql/queries.rb
index c775b015..78488886 100644
--- a/lib/goo/sparql/queries.rb
+++ b/lib/goo/sparql/queries.rb
@@ -582,7 +582,6 @@ def self.model_load_sliced(*options)
         end
 
         expand_equivalent_predicates(select,equivalent_predicates)
-        var_set_hash = {}
 
         select.each_solution do |sol|
           next if sol[:some_type] && klass.type_uri(collection) != sol[:some_type]
@@ -725,14 +724,21 @@ def self.model_load_sliced(*options)
               unless models_by_id[id].class.handler?(v)
                 unless object.nil? && !models_by_id[id].instance_variable_get("@#{v.to_s}").nil?
                   if v != :id
-                    # TODO: handle multilingual values here
                     # if multiple language values are included for a given property, set the
                     # corresponding model attribute to the English language value - NCBO-1662
                     if sol[v].kind_of?(RDF::Literal)
                       key = "#{v}#__#{id.to_s}"
-                      models_by_id[id].send("#{v}=", object, on_load: true) unless var_set_hash[key]
                       lang = sol[v].language
-                      var_set_hash[key] = true if lang == :EN || lang == :en
+
+                      #models_by_id[id].send("#{v}=", object, on_load: true) unless var_set_hash[key]
+                      #var_set_hash[key] = true if lang == :EN || lang == :en
+
+                      # We add the value only if it's language is in the main languages or if lang is nil
+                      if (Goo.main_languages.nil?)
+                        models_by_id[id].send("#{v}=", object, on_load: true)
+                      elsif (lang.nil? || Goo.main_languages.include?(lang.to_s.downcase) || Goo.main_languages.include?(lang.to_s.upcase))
+                        models_by_id[id].send("#{v}=", object, on_load: true)
+                      end
                     else
                       models_by_id[id].send("#{v}=", object, on_load: true)
                     end

From fa40b50851eee2b118d769a0590ed26f6ace89e6 Mon Sep 17 00:00:00 2001
From: Vincent Emonet <vincent.emonet@gmail.com>
Date: Fri, 9 Jun 2017 13:45:37 +0200
Subject: [PATCH 004/168] add main_lang and accepted_lang instead of just
 main_languages. The main_lang is the main lang used to extract prefLabels.
 Accepted lang is used to extract all others properties

---
 lib/goo.rb                | 19 ++++++++++++++-----
 lib/goo/sparql/queries.rb | 23 +++++++++++++++++++++--
 2 files changed, 35 insertions(+), 7 deletions(-)

diff --git a/lib/goo.rb b/lib/goo.rb
index 5929ab70..ec0e9148 100644
--- a/lib/goo.rb
+++ b/lib/goo.rb
@@ -24,7 +24,8 @@ module Goo
   @@resource_options = Set.new([:persistent]).freeze
 
   # Define the languages from which the properties values will be taken (be careful if prefLabel with different lang, only one will be taken)
-  @@main_languages = ["en","eng"]
+  @@main_lang = "en"
+  @@accepted_lang = ["en","eng"]
 
   @@configure_flag = false
   @@sparql_backends = {}
@@ -90,12 +91,20 @@ def self.test_reset
                   redis_cache: @@redis_client })
   end
 
-  def self.main_languages
-    @@main_languages
+  def self.main_lang
+    @@main_lang
   end
 
-  def self.main_languages=(value)
-    @@main_languages = value
+  def self.main_lang=(value)
+    @@main_lang = value
+  end
+
+  def self.accepted_lang
+    @@accepted_lang
+  end
+
+  def self.accepted_lang=(value)
+    @@accepted_lang = value
   end
 
   def self.use_cache=(value)
diff --git a/lib/goo/sparql/queries.rb b/lib/goo/sparql/queries.rb
index 78488886..f9257f28 100644
--- a/lib/goo/sparql/queries.rb
+++ b/lib/goo/sparql/queries.rb
@@ -582,6 +582,8 @@ def self.model_load_sliced(*options)
         end
 
         expand_equivalent_predicates(select,equivalent_predicates)
+        main_lang_hash = {}
+        accepted_lang_hash = {}
 
         select.each_solution do |sol|
           next if sol[:some_type] && klass.type_uri(collection) != sol[:some_type]
@@ -734,9 +736,26 @@ def self.model_load_sliced(*options)
                       #var_set_hash[key] = true if lang == :EN || lang == :en
 
                       # We add the value only if it's language is in the main languages or if lang is nil
-                      if (Goo.main_languages.nil?)
+                      if (Goo.accepted_lang.nil? || Goo.main_lang.nil?)
                         models_by_id[id].send("#{v}=", object, on_load: true)
-                      elsif (lang.nil? || Goo.main_languages.include?(lang.to_s.downcase) || Goo.main_languages.include?(lang.to_s.upcase))
+
+                      elsif (v.to_s.eql?("prefLabel"))
+                        # Special treatment for prefLabel where we want to extract the main_lang first, then accepted lang if no main.
+                        # Then anything if no main or accepted found
+                        if lang.to_s.downcase.eql?(Goo.main_lang)
+                          models_by_id[id].send("#{v}=", object, on_load: true)
+                          main_lang_hash[key] = true
+                        end
+                        if !main_lang_hash[key]
+                          if Goo.accepted_lang.include?(lang.to_s.downcase)
+                            models_by_id[id].send("#{v}=", object, on_load: true)
+                            accepted_lang_hash[key] = true
+                          elsif !accepted_lang_hash[key]
+                            models_by_id[id].send("#{v}=", object, on_load: true)
+                          end
+                        end
+
+                      elsif (lang.nil? || Goo.accepted_lang.include?(lang.to_s.downcase))
                         models_by_id[id].send("#{v}=", object, on_load: true)
                       end
                     else

From 172f7867aa9775866eda2ef43f9015f8611971f9 Mon Sep 17 00:00:00 2001
From: Vincent Emonet <vincent.emonet@gmail.com>
Date: Fri, 9 Jun 2017 14:02:40 +0200
Subject: [PATCH 005/168] remove accepted_lang, now only used main_lang

---
 lib/goo.rb                | 11 +----------
 lib/goo/sparql/queries.rb | 22 ++++++++--------------
 2 files changed, 9 insertions(+), 24 deletions(-)

diff --git a/lib/goo.rb b/lib/goo.rb
index ec0e9148..16bf3352 100644
--- a/lib/goo.rb
+++ b/lib/goo.rb
@@ -24,8 +24,7 @@ module Goo
   @@resource_options = Set.new([:persistent]).freeze
 
   # Define the languages from which the properties values will be taken (be careful if prefLabel with different lang, only one will be taken)
-  @@main_lang = "en"
-  @@accepted_lang = ["en","eng"]
+  @@main_lang = ["en","eng"]
 
   @@configure_flag = false
   @@sparql_backends = {}
@@ -99,14 +98,6 @@ def self.main_lang=(value)
     @@main_lang = value
   end
 
-  def self.accepted_lang
-    @@accepted_lang
-  end
-
-  def self.accepted_lang=(value)
-    @@accepted_lang = value
-  end
-
   def self.use_cache=(value)
     @@use_cache = value
     set_sparql_cache
diff --git a/lib/goo/sparql/queries.rb b/lib/goo/sparql/queries.rb
index f9257f28..2da5583d 100644
--- a/lib/goo/sparql/queries.rb
+++ b/lib/goo/sparql/queries.rb
@@ -583,7 +583,6 @@ def self.model_load_sliced(*options)
 
         expand_equivalent_predicates(select,equivalent_predicates)
         main_lang_hash = {}
-        accepted_lang_hash = {}
 
         select.each_solution do |sol|
           next if sol[:some_type] && klass.type_uri(collection) != sol[:some_type]
@@ -736,26 +735,21 @@ def self.model_load_sliced(*options)
                       #var_set_hash[key] = true if lang == :EN || lang == :en
 
                       # We add the value only if it's language is in the main languages or if lang is nil
-                      if (Goo.accepted_lang.nil? || Goo.main_lang.nil?)
+                      if Goo.main_lang.nil?
                         models_by_id[id].send("#{v}=", object, on_load: true)
 
                       elsif (v.to_s.eql?("prefLabel"))
-                        # Special treatment for prefLabel where we want to extract the main_lang first, then accepted lang if no main.
-                        # Then anything if no main or accepted found
-                        if lang.to_s.downcase.eql?(Goo.main_lang)
-                          models_by_id[id].send("#{v}=", object, on_load: true)
-                          main_lang_hash[key] = true
-                        end
+                        # Special treatment for prefLabel where we want to extract the main_lang first, or anything else
                         if !main_lang_hash[key]
-                          if Goo.accepted_lang.include?(lang.to_s.downcase)
-                            models_by_id[id].send("#{v}=", object, on_load: true)
-                            accepted_lang_hash[key] = true
-                          elsif !accepted_lang_hash[key]
-                            models_by_id[id].send("#{v}=", object, on_load: true)
+
+                          models_by_id[id].send("#{v}=", object, on_load: true)
+                          if Goo.main_lang.include?(lang.to_s.downcase)
+                            # If prefLabel from the main_lang found we stop looking for prefLabel
+                            main_lang_hash[key] = true
                           end
                         end
 
-                      elsif (lang.nil? || Goo.accepted_lang.include?(lang.to_s.downcase))
+                      elsif (lang.nil? || Goo.main_lang.include?(lang.to_s.downcase))
                         models_by_id[id].send("#{v}=", object, on_load: true)
                       end
                     else

From 157082420fab1fb9b575771325b6a22af9df3a13 Mon Sep 17 00:00:00 2001
From: Vincent Emonet <vincent.emonet@gmail.com>
Date: Mon, 12 Jun 2017 18:18:38 +0200
Subject: [PATCH 006/168] add comment on how solr docs are generated

---
 lib/goo/search/search.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/goo/search/search.rb b/lib/goo/search/search.rb
index 742a5995..56a13cc8 100644
--- a/lib/goo/search/search.rb
+++ b/lib/goo/search/search.rb
@@ -29,6 +29,7 @@ def get_indexable_object()
         #in solr
         doc[:resource_id] = doc[:id].to_s
         doc[:id] = get_index_id.to_s
+        # id: clsUri_ONTO-ACRO_submissionNumber. i.e.: http://lod.nal.usda.gov/nalt/5260_NALT_4
         doc
       end
 

From 2e547094cb226df96f3e7d95c64c55fcd78e192c Mon Sep 17 00:00:00 2001
From: Vincent Emonet <vincent.emonet@gmail.com>
Date: Tue, 13 Jun 2017 03:02:03 +0200
Subject: [PATCH 007/168] now only index labels with the language included in
 main_lang

---
 lib/goo/base/resource.rb | 32 ++++++++++++++++++++++++--------
 1 file changed, 24 insertions(+), 8 deletions(-)

diff --git a/lib/goo/base/resource.rb b/lib/goo/base/resource.rb
index c12f6203..d102323c 100644
--- a/lib/goo/base/resource.rb
+++ b/lib/goo/base/resource.rb
@@ -228,6 +228,7 @@ def graph
         return col ? col.id : nil
       end
 
+      # Retrieve unmapped attribute from an instance (i.e.: a Class)
       def self.map_attributes(inst,equivalent_predicates=nil)
         if (inst.kind_of?(Goo::Base::Resource) && inst.unmapped.nil?) ||
             (!inst.respond_to?(:unmapped) && inst[:unmapped].nil?)
@@ -267,7 +268,27 @@ def self.map_attributes(inst,equivalent_predicates=nil)
             else
               object = unmapped_string_keys[attr_uri]
             end
-            object = object.map { |o| o.is_a?(RDF::URI) ? o : o.object }
+
+            #binding.pry if inst.id.to_s.eql?("http://lirmm.fr/2015/resource/AGROOE_c_03")
+            # Now include only literal that have language in the main_langs or nil
+            # Old way: object = object.map { |o| o.is_a?(RDF::URI) ? o : o.object }
+
+            object = object.map { |o| if o.is_a?(RDF::URI)
+                                        o
+                                      else
+                                        if o.respond_to?("language")
+                                          # Include only literal that have language in the main_langs or nil
+                                          if o.language.nil?
+                                            o.object
+                                          elsif Goo.main_lang.include?(o.language.to_s.downcase)
+                                            o.object
+                                          end
+                                        else
+                                          o.object
+                                        end
+                                      end }
+            object = object.compact
+
             if klass.range(attr)
               object = object.map { |o|
                 o.is_a?(RDF::URI) ? klass.range_object(attr,o) : o }
@@ -281,13 +302,8 @@ def self.map_attributes(inst,equivalent_predicates=nil)
               inst.send("#{attr}=",object, on_load: true)
             end
           else
-            inst.send("#{attr}=",
-                      list_attrs.include?(attr) ? [] : nil, on_load: true)
-            if inst.id.to_s == "http://purl.obolibrary.org/obo/IAO_0000415"
-              if attr == :definition
-               # binding.pry
-              end
-            end
+            inst.send("#{attr}=", list_attrs.include?(attr) ? [] : nil, on_load: true)
+            #binding.pry if inst.id.to_s.eql?("http://lirmm.fr/2015/resource/AGROOE_c_03")
           end
 
         end

From f31367938720c5dbf7b45165ce579ff9e87bafc0 Mon Sep 17 00:00:00 2001
From: Vincent Emonet <vincent.emonet@gmail.com>
Date: Tue, 13 Jun 2017 11:34:22 +0200
Subject: [PATCH 008/168] now take prefLabel with a lang that is in main_lang
 in priority, then nil if not available

---
 lib/goo/base/resource.rb | 25 ++++++++++++++++++++-----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/lib/goo/base/resource.rb b/lib/goo/base/resource.rb
index d102323c..6b52315a 100644
--- a/lib/goo/base/resource.rb
+++ b/lib/goo/base/resource.rb
@@ -269,17 +269,24 @@ def self.map_attributes(inst,equivalent_predicates=nil)
               object = unmapped_string_keys[attr_uri]
             end
 
-            #binding.pry if inst.id.to_s.eql?("http://lirmm.fr/2015/resource/AGROOE_c_03")
+            #binding.pry if inst.id.to_s.eql?("http://lirmm.fr/2015/resource/AGROOE_c_03") && attr.to_s.eql?("prefLabel")
             # Now include only literal that have language in the main_langs or nil
             # Old way: object = object.map { |o| o.is_a?(RDF::URI) ? o : o.object }
-
+            prefLabelNilLang = []
             object = object.map { |o| if o.is_a?(RDF::URI)
                                         o
                                       else
                                         if o.respond_to?("language")
                                           # Include only literal that have language in the main_langs or nil
                                           if o.language.nil?
-                                            o.object
+                                            if attr.to_s.eql?("prefLabel")
+                                              # For prefLabel we want to take a value with a defined lang in priority
+                                              # And one with nil lang if not available
+                                              prefLabelNilLang << o.object
+                                              nil
+                                            else
+                                              o.object
+                                            end
                                           elsif Goo.main_lang.include?(o.language.to_s.downcase)
                                             o.object
                                           end
@@ -294,7 +301,16 @@ def self.map_attributes(inst,equivalent_predicates=nil)
                 o.is_a?(RDF::URI) ? klass.range_object(attr,o) : o }
             end
             unless list_attrs.include?(attr)
-              object = object.first
+              if attr.to_s.eql?("prefLabel")
+                if object.empty?
+                  # If no value with a lang within main_lang for prefLabel, we take the nil lang
+                  object = prefLabelNilLang.first
+                else
+                  object = object.first
+                end
+              else
+                object = object.first
+              end
             end
             if inst.respond_to?(:klass)
               inst[attr] = object
@@ -303,7 +319,6 @@ def self.map_attributes(inst,equivalent_predicates=nil)
             end
           else
             inst.send("#{attr}=", list_attrs.include?(attr) ? [] : nil, on_load: true)
-            #binding.pry if inst.id.to_s.eql?("http://lirmm.fr/2015/resource/AGROOE_c_03")
           end
 
         end

From 4d32d7ed5d825829f58faef31cd41960f056c443 Mon Sep 17 00:00:00 2001
From: Vincent Emonet <vincent.emonet@gmail.com>
Date: Tue, 13 Jun 2017 16:01:36 +0200
Subject: [PATCH 009/168] retrieve label from bad langs

---
 lib/goo/base/resource.rb | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/lib/goo/base/resource.rb b/lib/goo/base/resource.rb
index 6b52315a..24a05e71 100644
--- a/lib/goo/base/resource.rb
+++ b/lib/goo/base/resource.rb
@@ -271,8 +271,9 @@ def self.map_attributes(inst,equivalent_predicates=nil)
 
             #binding.pry if inst.id.to_s.eql?("http://lirmm.fr/2015/resource/AGROOE_c_03") && attr.to_s.eql?("prefLabel")
             # Now include only literal that have language in the main_langs or nil
-            # Old way: object = object.map { |o| o.is_a?(RDF::URI) ? o : o.object }
+            # Olw way: object = object.map { |o| o.is_a?(RDF::URI) ? o : o.object }
             prefLabelNilLang = []
+            attrBadLang = []
             object = object.map { |o| if o.is_a?(RDF::URI)
                                         o
                                       else
@@ -289,12 +290,18 @@ def self.map_attributes(inst,equivalent_predicates=nil)
                                             end
                                           elsif Goo.main_lang.include?(o.language.to_s.downcase)
                                             o.object
+                                          else
+                                            attrBadLang << o.object
                                           end
                                         else
                                           o.object
                                         end
                                       end }
             object = object.compact
+            if object.nil? || object.empty?
+              # If no label have been found in the main_langs, then we take from the not accepted lang
+              object = attrBadLang.compact
+            end
 
             if klass.range(attr)
               object = object.map { |o|

From 07340fc906ca7612fe03660b0a75be01c3177ebe Mon Sep 17 00:00:00 2001
From: Vincent Emonet <vincent.emonet@gmail.com>
Date: Tue, 13 Jun 2017 17:37:37 +0200
Subject: [PATCH 010/168] fix how we retrieve labels when no labels frfrom one
 of the main_lang

---
 lib/goo/base/resource.rb          | 12 +++++++-----
 lib/goo/base/settings/settings.rb |  8 ++++----
 lib/goo/search/search.rb          |  4 +++-
 3 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/lib/goo/base/resource.rb b/lib/goo/base/resource.rb
index 24a05e71..a8281d00 100644
--- a/lib/goo/base/resource.rb
+++ b/lib/goo/base/resource.rb
@@ -292,16 +292,13 @@ def self.map_attributes(inst,equivalent_predicates=nil)
                                             o.object
                                           else
                                             attrBadLang << o.object
+                                            nil
                                           end
                                         else
                                           o.object
                                         end
                                       end }
             object = object.compact
-            if object.nil? || object.empty?
-              # If no label have been found in the main_langs, then we take from the not accepted lang
-              object = attrBadLang.compact
-            end
 
             if klass.range(attr)
               object = object.map { |o|
@@ -311,7 +308,11 @@ def self.map_attributes(inst,equivalent_predicates=nil)
               if attr.to_s.eql?("prefLabel")
                 if object.empty?
                   # If no value with a lang within main_lang for prefLabel, we take the nil lang
-                  object = prefLabelNilLang.first
+                  if prefLabelNilLang.length > 0
+                    object = prefLabelNilLang.first
+                  else
+                    object = attrBadLang.compact.first
+                  end
                 else
                   object = object.first
                 end
@@ -319,6 +320,7 @@ def self.map_attributes(inst,equivalent_predicates=nil)
                 object = object.first
               end
             end
+
             if inst.respond_to?(:klass)
               inst[attr] = object
             else
diff --git a/lib/goo/base/settings/settings.rb b/lib/goo/base/settings/settings.rb
index 2a274454..ce3e9a21 100644
--- a/lib/goo/base/settings/settings.rb
+++ b/lib/goo/base/settings/settings.rb
@@ -232,15 +232,13 @@ def shape_attribute(attr)
               raise ArgumentError, "Method based attributes cannot be set"
             end
             if self.class.inverse?(attr) && !(args && args.last.instance_of?(Hash) && args.last[:on_load])
-              raise ArgumentError,
-                "`#{attr}` is an inverse attribute. Values cannot be assigned."
+              raise ArgumentError, "`#{attr}` is an inverse attribute. Values cannot be assigned."
             end
             @loaded_attributes.add(attr)
             value = args[0]
             unless args.last.instance_of?(Hash) and args.last[:on_load]
               if self.persistent? and self.class.name_with == attr
-                raise ArgumentError, 
-                    "`#{attr}` attribute is used to name this resource and cannot be modified."
+                raise ArgumentError, "`#{attr}` attribute is used to name this resource and cannot be modified."
               end
               prev = self.instance_variable_get("@#{attr}")
               if !prev.nil? and !@modified_attributes.include?(attr)
@@ -266,9 +264,11 @@ def shape_attribute(attr)
               @loaded_attributes << attr
               return value
             end
+
             if (not @persistent) or @loaded_attributes.include?(attr)
               return self.instance_variable_get("@#{attr}")
             else
+              # TODO: bug here when no labels from one of the main_lang available... (when it is called by ontologies_linked_data ontologies_submission)
               raise Goo::Base::AttributeNotLoaded, "Attribute `#{attr}` is not loaded for #{self.id}. Loaded attributes: #{@loaded_attributes.inspect}."
             end
           end
diff --git a/lib/goo/search/search.rb b/lib/goo/search/search.rb
index 56a13cc8..06f81c0d 100644
--- a/lib/goo/search/search.rb
+++ b/lib/goo/search/search.rb
@@ -24,9 +24,11 @@ def get_index_id()
         self.class.model_settings[:search_options][:index_id].call(self)
       end
 
+      # Get the doc that will be indexed in solr
       def get_indexable_object()
+        # To make the code less readable the guys that wrote it managed to hide the real function called by this line
+        # It is "get_index_doc" in ontologies_linked_data Class.rb
         doc = self.class.model_settings[:search_options][:document].call(self)
-        #in solr
         doc[:resource_id] = doc[:id].to_s
         doc[:id] = get_index_id.to_s
         # id: clsUri_ONTO-ACRO_submissionNumber. i.e.: http://lod.nal.usda.gov/nalt/5260_NALT_4

From 773a0f3408d21d350463e92258655c6406e21c43 Mon Sep 17 00:00:00 2001
From: Vincent Emonet <vincent.emonet@gmail.com>
Date: Tue, 13 Jun 2017 20:42:27 +0200
Subject: [PATCH 011/168] avoid attributeNotLoaded error for empty labels now

---
 lib/goo/sparql/queries.rb | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/lib/goo/sparql/queries.rb b/lib/goo/sparql/queries.rb
index 2da5583d..72962273 100644
--- a/lib/goo/sparql/queries.rb
+++ b/lib/goo/sparql/queries.rb
@@ -584,7 +584,9 @@ def self.model_load_sliced(*options)
         expand_equivalent_predicates(select,equivalent_predicates)
         main_lang_hash = {}
 
+
         select.each_solution do |sol|
+          attr_to_load_if_empty = []
           next if sol[:some_type] && klass.type_uri(collection) != sol[:some_type]
           if count
             return sol[:count_var].object
@@ -748,9 +750,10 @@ def self.model_load_sliced(*options)
                             main_lang_hash[key] = true
                           end
                         end
-
                       elsif (lang.nil? || Goo.main_lang.include?(lang.to_s.downcase))
                         models_by_id[id].send("#{v}=", object, on_load: true)
+                      else
+                        attr_to_load_if_empty << v
                       end
                     else
                       models_by_id[id].send("#{v}=", object, on_load: true)
@@ -760,6 +763,12 @@ def self.model_load_sliced(*options)
               end
             end
           end
+          attr_to_load_if_empty.each do |empty_attr|
+            # To avoid bug where the attr is not loaded (because the data model is really bad)
+            if !models_by_id[id].loaded_attributes.include?(empty_attr.to_sym)
+              models_by_id[id].send("#{empty_attr}=", nil, on_load: true)
+            end
+          end
         end
         return models_by_id if bnode_extraction
 

From a17bc26053b2cd15f05bbeb5a01648403b78f94f Mon Sep 17 00:00:00 2001
From: Vincent Emonet <vincent.emonet@gmail.com>
Date: Tue, 13 Jun 2017 20:45:00 +0200
Subject: [PATCH 012/168] stop trying to get bad lang values for indexing

---
 lib/goo/base/resource.rb | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/lib/goo/base/resource.rb b/lib/goo/base/resource.rb
index a8281d00..34d2feb1 100644
--- a/lib/goo/base/resource.rb
+++ b/lib/goo/base/resource.rb
@@ -273,7 +273,6 @@ def self.map_attributes(inst,equivalent_predicates=nil)
             # Now include only literal that have language in the main_langs or nil
             # Olw way: object = object.map { |o| o.is_a?(RDF::URI) ? o : o.object }
             prefLabelNilLang = []
-            attrBadLang = []
             object = object.map { |o| if o.is_a?(RDF::URI)
                                         o
                                       else
@@ -291,7 +290,6 @@ def self.map_attributes(inst,equivalent_predicates=nil)
                                           elsif Goo.main_lang.include?(o.language.to_s.downcase)
                                             o.object
                                           else
-                                            attrBadLang << o.object
                                             nil
                                           end
                                         else
@@ -310,8 +308,6 @@ def self.map_attributes(inst,equivalent_predicates=nil)
                   # If no value with a lang within main_lang for prefLabel, we take the nil lang
                   if prefLabelNilLang.length > 0
                     object = prefLabelNilLang.first
-                  else
-                    object = attrBadLang.compact.first
                   end
                 else
                   object = object.first

From 9a2fe3146f660e5a9eae7e8cddc46fe059fc4253 Mon Sep 17 00:00:00 2001
From: Vincent Emonet <vincent.emonet@gmail.com>
Date: Tue, 13 Jun 2017 20:49:53 +0200
Subject: [PATCH 013/168] return array instead of nil when no value for attr

---
 lib/goo/sparql/queries.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/goo/sparql/queries.rb b/lib/goo/sparql/queries.rb
index 72962273..4d94d090 100644
--- a/lib/goo/sparql/queries.rb
+++ b/lib/goo/sparql/queries.rb
@@ -764,9 +764,9 @@ def self.model_load_sliced(*options)
             end
           end
           attr_to_load_if_empty.each do |empty_attr|
-            # To avoid bug where the attr is not loaded (because the data model is really bad)
+            # To avoid bug where the attr is not loaded, we return an empty array (because the data model is really bad)
             if !models_by_id[id].loaded_attributes.include?(empty_attr.to_sym)
-              models_by_id[id].send("#{empty_attr}=", nil, on_load: true)
+              models_by_id[id].send("#{empty_attr}=", [], on_load: true)
             end
           end
         end

From 711b8d90bdc4351dcaf8ebde97cae87b828a5079 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A9r=C3=B4me=20LAMARQUE?= <jerome.lamarque@gmail.com>
Date: Tue, 16 Jun 2020 22:10:16 +0000
Subject: [PATCH 014/168] Gemfile.lock update

---
 Gemfile.lock | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index b508c486..4281e29a 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -33,7 +33,7 @@ GEM
       tzinfo (~> 0.3.37)
     addressable (2.3.5)
     builder (3.2.4)
-    coderay (1.1.2)
+    coderay (1.1.3)
     concurrent-ruby (1.1.6)
     cube-ruby (0.0.3)
     daemons (1.3.1)
@@ -54,7 +54,7 @@ GEM
     method_source (1.0.0)
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2019.1009)
+    mime-types-data (3.2020.0512)
     minitest (4.7.5)
     multi_json (1.14.1)
     multipart-post (2.1.1)
@@ -62,10 +62,10 @@ GEM
       ruby2_keywords (~> 0.0.1)
     net-http-persistent (2.9.4)
     netrc (0.11.0)
-    pry (0.13.0)
+    pry (0.13.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    rack (2.2.2)
+    rack (2.2.3)
     rack-accept (0.4.5)
       rack (>= 0.4)
     rack-post-body-to-params (0.1.8)
@@ -75,7 +75,7 @@ GEM
     rake (13.0.1)
     rdf (1.0.8)
       addressable (>= 2.2)
-    redis (4.1.3)
+    redis (4.2.1)
     rest-client (2.1.0)
       http-accept (>= 1.7.0, < 2.0)
       http-cookie (>= 1.0.2, < 2.0)
@@ -101,7 +101,7 @@ GEM
       rack (>= 1, < 3)
     thread_safe (0.3.6)
     tilt (2.0.10)
-    tzinfo (0.3.56)
+    tzinfo (0.3.57)
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.7.7)
@@ -110,7 +110,6 @@ GEM
 
 PLATFORMS
   ruby
-  x86_64-darwin-16
 
 DEPENDENCIES
   activesupport

From 894aba0b10a96a20c07202a634035bf2ff0cd519 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Lamarque?= <jerome.lamarque@lirmm.fr>
Date: Fri, 26 Jun 2020 09:31:25 +0200
Subject: [PATCH 015/168] Fixing  merge issue with NCBO master
 (goo/search/search.rb)

---
 lib/goo/search/search.rb | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/lib/goo/search/search.rb b/lib/goo/search/search.rb
index 730f21de..1dc72ea9 100644
--- a/lib/goo/search/search.rb
+++ b/lib/goo/search/search.rb
@@ -82,6 +82,17 @@ def unindexByQuery(query, connection_name=:main)
         Goo.search_connection(connection_name).delete_by_query(query)
       end
 
+      # Get the doc that will be indexed in solr
+      def get_indexable_object()
+        # To make the code less readable the guys that wrote it managed to hide the real function called by this line
+        # It is "get_index_doc" in ontologies_linked_data Class.rb
+        doc = self.class.model_settings[:search_options][:document].call(self)
+        doc[:resource_id] = doc[:id].to_s
+        doc[:id] = get_index_id.to_s
+        # id: clsUri_ONTO-ACRO_submissionNumber. i.e.: http://lod.nal.usda.gov/nalt/5260_NALT_4
+        doc
+      end
+
       def indexCommit(attrs=nil, connection_name=:main)
         Goo.search_connection(connection_name).commit(:commit_attributes => attrs || {})
       end

From 1f5c0e3fd2233b939d79a66198d23f0a3592cf57 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A9r=C3=B4me=20LAMARQUE?= <jerome.lamarque@gmail.com>
Date: Fri, 26 Jun 2020 07:36:03 +0000
Subject: [PATCH 016/168] Gemfile.lock update

---
 Gemfile.lock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index 4281e29a..b73b61a1 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -1,6 +1,6 @@
 GIT
   remote: https://github.com/ncbo/sparql-client.git
-  revision: 0e7f983ec590bf3cac4369366a0bffb80bd9adb2
+  revision: 298f9c9a8d2637650a62f8102679aa30084bf878
   branch: master
   specs:
     sparql-client (1.0.1)

From bf42c5c632cfc6475b38fc4e2694d2b8e65d32e4 Mon Sep 17 00:00:00 2001
From: OntoPortal Bot <ontoportal.bot.lirmm@gmail.com>
Date: Fri, 3 Jul 2020 08:43:53 +0000
Subject: [PATCH 017/168]  Gemfile.lock update

---
 Gemfile.lock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index b73b61a1..26f4ee99 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -48,7 +48,7 @@ GEM
       domain_name (~> 0.5)
     i18n (0.9.5)
       concurrent-ruby (~> 1.0)
-    json_pure (2.3.0)
+    json_pure (2.3.1)
     macaddr (1.7.2)
       systemu (~> 2.6.5)
     method_source (1.0.0)

From 09e8627a5a894f43eb4c9fd336a999da4a3015c3 Mon Sep 17 00:00:00 2001
From: OntoPortal Bot <ontoportal.bot.lirmm@gmail.com>
Date: Fri, 10 Jul 2020 09:32:18 +0200
Subject: [PATCH 018/168] [ontoportal-bot] Gemfile.lock update

---
 Gemfile.lock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index 26f4ee99..b57e8564 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -56,7 +56,7 @@ GEM
       mime-types-data (~> 3.2015)
     mime-types-data (3.2020.0512)
     minitest (4.7.5)
-    multi_json (1.14.1)
+    multi_json (1.15.0)
     multipart-post (2.1.1)
     mustermann (1.1.1)
       ruby2_keywords (~> 0.0.1)

From 34aa8878748aca41f94b65fd3ce28629d54f2d92 Mon Sep 17 00:00:00 2001
From: OntoPortal Bot <ontoportal.bot.lirmm@gmail.com>
Date: Thu, 16 Jul 2020 11:44:55 +0200
Subject: [PATCH 019/168] [ontoportal-bot] Gemfile.lock update

---
 Gemfile.lock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index b57e8564..b393b128 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -1,6 +1,6 @@
 GIT
   remote: https://github.com/ncbo/sparql-client.git
-  revision: 298f9c9a8d2637650a62f8102679aa30084bf878
+  revision: 87bc0e8976519cdcdef06169556677467a10fcfc
   branch: master
   specs:
     sparql-client (1.0.1)

From 819d8122422fe16c3934468ab1658863883ed629 Mon Sep 17 00:00:00 2001
From: OntoPortal Bot <ontoportal.bot.lirmm@gmail.com>
Date: Tue, 25 Aug 2020 07:30:27 +0000
Subject: [PATCH 020/168] [ontoportal-bot] Gemfile.lock update

---
 Gemfile.lock | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index b393b128..7401eb5b 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -34,7 +34,7 @@ GEM
     addressable (2.3.5)
     builder (3.2.4)
     coderay (1.1.3)
-    concurrent-ruby (1.1.6)
+    concurrent-ruby (1.1.7)
     cube-ruby (0.0.3)
     daemons (1.3.1)
     docile (1.3.2)
@@ -85,7 +85,7 @@ GEM
       builder (>= 2.1.2)
       faraday (>= 0.9.0)
     ruby2_keywords (0.0.2)
-    simplecov (0.18.5)
+    simplecov (0.19.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
     simplecov-html (0.12.2)

From 5d4b78df14266a1c2ee86887fbd342e72e8eb1a5 Mon Sep 17 00:00:00 2001
From: OntoPortal Bot <ontoportal.bot.lirmm@gmail.com>
Date: Thu, 10 Sep 2020 13:34:10 +0000
Subject: [PATCH 021/168] [ontoportal-bot] Gemfile.lock update

---
 Gemfile.lock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index 400f4099..e69c63fa 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -75,7 +75,7 @@ GEM
     rake (13.0.1)
     rdf (1.0.8)
       addressable (>= 2.2)
-    redis (4.2.1)
+    redis (4.2.2)
     rest-client (2.1.0)
       http-accept (>= 1.7.0, < 2.0)
       http-cookie (>= 1.0.2, < 2.0)

From 7dee2100dc26273158ddc6f4c4bbecd9d64d903d Mon Sep 17 00:00:00 2001
From: OntoPortal Bot <ontoportal.bot.lirmm@gmail.com>
Date: Thu, 1 Oct 2020 10:58:23 +0200
Subject: [PATCH 022/168] [ontoportal-bot] Gemfile.lock update

---
 Gemfile.lock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index e69c63fa..0dd2c26c 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -88,7 +88,7 @@ GEM
     simplecov (0.19.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
-    simplecov-html (0.12.2)
+    simplecov-html (0.12.3)
     sinatra (2.1.0)
       mustermann (~> 1.0)
       rack (~> 2.2)

From ba3edcdd1e2f6ee630b3cca29e08cc209229186e Mon Sep 17 00:00:00 2001
From: OntoPortal Bot <ontoportal.bot.lirmm@gmail.com>
Date: Fri, 30 Oct 2020 18:39:31 +0100
Subject: [PATCH 023/168] [ontoportal-bot] Gemfile.lock update

---
 Gemfile.lock | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index 0dd2c26c..fdaa5132 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -41,8 +41,9 @@ GEM
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
     eventmachine (1.2.7)
-    faraday (1.0.1)
+    faraday (1.1.0)
       multipart-post (>= 1.2, < 3)
+      ruby2_keywords
     http-accept (1.7.0)
     http-cookie (1.0.3)
       domain_name (~> 0.5)
@@ -85,7 +86,7 @@ GEM
       builder (>= 2.1.2)
       faraday (>= 0.9.0)
     ruby2_keywords (0.0.2)
-    simplecov (0.19.0)
+    simplecov (0.19.1)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
     simplecov-html (0.12.3)

From 89a8f665f70644989adf70cd132bf0f38ef3414d Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Tue, 28 Dec 2021 16:54:30 +0100
Subject: [PATCH 024/168] fix #4 ; add regex filter

---
 lib/goo/base/filter.rb    | 22 +++++++++++++---------
 lib/goo/sparql/queries.rb | 25 ++++++++++++++++---------
 2 files changed, 29 insertions(+), 18 deletions(-)

diff --git a/lib/goo/base/filter.rb b/lib/goo/base/filter.rb
index 66f2095d..50fa58ec 100644
--- a/lib/goo/base/filter.rb
+++ b/lib/goo/base/filter.rb
@@ -11,50 +11,54 @@ def initialize(pattern)
       end
 
       def >(value)
-        @filter_tree << FILTER_TUPLE.new(:>,value)
+        @filter_tree << FILTER_TUPLE.new(:>, value)
         self
       end
 
       def <(value)
-        @filter_tree << FILTER_TUPLE.new(:<,value)
+        @filter_tree << FILTER_TUPLE.new(:<, value)
         self
       end
 
       def <=(value)
-        @filter_tree << FILTER_TUPLE.new(:<=,value)
+        @filter_tree << FILTER_TUPLE.new(:<=, value)
         self
       end
 
       def >=(value)
-        @filter_tree << FILTER_TUPLE.new(:>=,value)
+        @filter_tree << FILTER_TUPLE.new(:>=, value)
         self
       end
 
       def or(value)
-        @filter_tree << FILTER_TUPLE.new(:or,value)
+        @filter_tree << FILTER_TUPLE.new(:or, value)
         self
       end
 
       def ==(value)
-        @filter_tree << FILTER_TUPLE.new(:==,value)
+        @filter_tree << FILTER_TUPLE.new(:==, value)
         self
       end
 
       def and(value)
-        @filter_tree << FILTER_TUPLE.new(:and,value)
+        @filter_tree << FILTER_TUPLE.new(:and, value)
         self
       end
 
       def unbound
-        @filter_tree << FILTER_TUPLE.new(:unbound,nil)
+        @filter_tree << FILTER_TUPLE.new(:unbound, nil)
         self
       end
 
       def bound
-        @filter_tree << FILTER_TUPLE.new(:bound,nil)
+        @filter_tree << FILTER_TUPLE.new(:bound, nil)
         self
       end
 
+      def regex(value)
+        @filter_tree << FILTER_TUPLE.new(:regex, value)
+        self 
+      end
     end
   end
 end
diff --git a/lib/goo/sparql/queries.rb b/lib/goo/sparql/queries.rb
index 8ed102dd..7cc04642 100644
--- a/lib/goo/sparql/queries.rb
+++ b/lib/goo/sparql/queries.rb
@@ -131,22 +131,29 @@ def self.query_filter_sparql(klass,filter,filter_patterns,filter_graphs,
           end
           filter_var = inspected_patterns[filter_pattern_match]
           if !filter_operation.value.instance_of?(Goo::Filter)
-            unless filter_operation.operator == :unbound || filter_operation.operator == :bound
+            case filter_operation.operator
+            when  :unbound
+              filter_operations << "!BOUND(?#{filter_var.to_s})"
+              return :optional
+
+            when :bound
+              filter_operations << "BOUND(?#{filter_var.to_s})"
+              return :optional
+            when :regex
+              if  filter_operation.value.is_a?(String)
+                filter_operations << "REGEX(?#{filter_var.to_s} , \"#{filter_operation.value.to_s}\")"
+              end
+
+            else
               value = RDF::Literal.new(filter_operation.value)
               if filter_operation.value.is_a? String
                 value = RDF::Literal.new(filter_operation.value, :datatype => RDF::XSD.string)
               end
               filter_operations << (
                 "?#{filter_var.to_s} #{sparql_op_string(filter_operation.operator)} " +
-                " #{value.to_ntriples}")
-            else
-              if filter_operation.operator == :unbound
-                filter_operations << "!BOUND(?#{filter_var.to_s})"
-              else
-                filter_operations << "BOUND(?#{filter_var.to_s})"
-              end
-              return :optional
+                  " #{value.to_ntriples}")
             end
+
           else
             filter_operations << "#{sparql_op_string(filter_operation.operator)}"
             query_filter_sparql(klass,filter_operation.value,filter_patterns,

From 694aa19dc46941d01535f07996320307f74eb84d Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Tue, 15 Mar 2022 18:13:23 +0100
Subject: [PATCH 025/168] add main_languages configuration variable

---
 lib/goo.rb | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/lib/goo.rb b/lib/goo.rb
index 1867c7d6..fedae62e 100644
--- a/lib/goo.rb
+++ b/lib/goo.rb
@@ -25,6 +25,8 @@ module Goo
 
   @@resource_options = Set.new([:persistent]).freeze
 
+  # Define the languages from which the properties values will be taken
+  @@main_languages = %w[en]
   @@configure_flag = false
   @@sparql_backends = {}
   @@model_by_name = {}
@@ -42,6 +44,19 @@ module Goo
 
   @@slice_loading_size = 500
 
+
+  def self.main_languages
+    @@main_languages
+  end
+  def self.main_languages=(lang)
+    @@main_languages = lang
+  end
+
+  def self.language_includes(lang)
+    lang_str = lang.to_s
+    main_languages.index { |l| lang_str.downcase.eql?(l) || lang_str.upcase.eql?(l)}
+  end
+
   def self.add_namespace(shortcut, namespace,default=false)
     if !(namespace.instance_of? RDF::Vocabulary)
       raise ArgumentError, "Namespace must be a RDF::Vocabulary object"

From eec4d6066655ba4657dab173cb6d0765e8ce6e45 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Tue, 15 Mar 2022 18:15:19 +0100
Subject: [PATCH 026/168] add LanguageFilter class to filter an attribute
 values by the languages

---
 lib/goo/sparql/mixins/solution_lang_filter.rb | 83 +++++++++++++++++++
 lib/goo/sparql/sparql.rb                      |  1 +
 2 files changed, 84 insertions(+)
 create mode 100644 lib/goo/sparql/mixins/solution_lang_filter.rb

diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
new file mode 100644
index 00000000..af02b44a
--- /dev/null
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -0,0 +1,83 @@
+module Goo
+  module SPARQL
+    module Solution
+      class  LanguageFilter
+
+        def initialize
+          @other_languages_values = {}
+        end
+
+        def other_languages_values
+          @other_languages_values
+        end
+
+        def main_lang_filter(id, attr, old_values, new_value)
+          index, value = lang_index  old_values, new_value
+          save_other_lang_val(id, attr, index, new_value) unless index.eql? :no_lang
+          [index, value]
+        end
+
+        def fill_models_with_other_languages(models_by_id, list_attributes)
+          @other_languages_values.each  do |id, languages_values|
+            languages_values.each do |attr, index_values|
+              model_attribute_val = models_by_id[id].instance_variable_get("@#{attr.to_s}")
+              values = languages_values_to_set(index_values, model_attribute_val)
+
+              if !values.nil? && list_attributes.include?(attr)
+                models_by_id[id].send("#{attr.to_s}=", values || [], on_load: true)
+              elsif !values.nil?
+                models_by_id[id].send("#{attr.to_s}=", values.first || nil, on_load: true)
+              end
+            end
+          end
+        end
+
+        def languages_values_to_set(language_values, no_lang_values)
+
+          values = nil
+          matched_lang, not_matched_lang = matched_languages(language_values, no_lang_values)
+          if !matched_lang.empty?
+            main_lang = Array(matched_lang[:'0']) + Array(matched_lang[:no_lang])
+            secondary_languages = matched_lang.select { |key| key != :'0' && key != :no_lang }.sort.map { |x| x[1] }.flatten
+            values = main_lang + secondary_languages
+          elsif !not_matched_lang.empty?
+            values = not_matched_lang
+          end
+          values
+        end
+
+        private
+
+        def lang_index(object, new_value)
+          lang = new_value.language
+          if lang.nil?
+            [:no_lang, object]
+          else
+            index = Goo.language_includes(lang)
+            index = index ? index.to_s.to_sym : :not_matched
+            [index, new_value]
+          end
+        end
+
+        def save_other_lang_val(id, attr, index, value)
+          @other_languages_values[id] ||= {}
+          @other_languages_values[id][attr] ||= {}
+          @other_languages_values[id][attr][index] ||= []
+          
+          unless @other_languages_values[id][attr][index].include?(value.to_s)
+            @other_languages_values[id][attr][index] += Array(value.to_s)
+          end
+        end
+        
+
+
+        def matched_languages(index_values, model_attribute_val)
+          not_matched_lang = index_values[:not_matched]
+          matched_lang = index_values.reject { |key| key == :not_matched }
+          matched_lang[:no_lang] = Array(model_attribute_val) unless model_attribute_val.nil?
+          [matched_lang, not_matched_lang]
+        end
+      end
+    end
+  end
+end
diff --git a/lib/goo/sparql/sparql.rb b/lib/goo/sparql/sparql.rb
index dfd3d0a6..6fa1d582 100644
--- a/lib/goo/sparql/sparql.rb
+++ b/lib/goo/sparql/sparql.rb
@@ -1,6 +1,7 @@
 require "sparql/client"
 
 require_relative "mixins/query_pattern"
+require_relative "mixins/solution_lang_filter"
 require_relative "query_builder"
 require_relative "solutions_mapper"
 require_relative "client"

From 31b0bb051c1bf52e8fea19bd77a3dd073d0f436f Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Tue, 15 Mar 2022 18:15:44 +0100
Subject: [PATCH 027/168] use LanguageFilter in the solution mapper

---
 lib/goo/sparql/solutions_mapper.rb | 47 +++++++++++++++---------------
 1 file changed, 23 insertions(+), 24 deletions(-)

diff --git a/lib/goo/sparql/solutions_mapper.rb b/lib/goo/sparql/solutions_mapper.rb
index 386b3f18..209bf316 100644
--- a/lib/goo/sparql/solutions_mapper.rb
+++ b/lib/goo/sparql/solutions_mapper.rb
@@ -19,6 +19,7 @@ def initialize(aggregate_projections, bnode_extraction, embed_struct,
         @variables = variables
         @options = options
 
+
       end
 
       def map_each_solutions(select)
@@ -31,9 +32,9 @@ def map_each_solutions(select)
 
         found = Set.new
         objects_new = {}
-        var_set_hash = {}
         list_attributes = Set.new(klass.attributes(:list))
         all_attributes = Set.new(klass.attributes(:all))
+        @lang_filter = Goo::SPARQL::Solution::LanguageFilter.new
 
         select.each_solution do |sol|
           next if sol[:some_type] && klass.type_uri(collection) != sol[:some_type]
@@ -83,11 +84,13 @@ def map_each_solutions(select)
 
             object = object_to_array(id, @klass_struct, @models_by_id, object, v) if list_attributes.include?(v)
 
-            model_map_attributes_values(id, var_set_hash, @models_by_id, object, sol, v) unless object.nil?
+            model_map_attributes_values(id, object, sol, v)
           end
         end
+        @lang_filter.fill_models_with_other_languages(@models_by_id, list_attributes)
 
         return @models_by_id if @bnode_extraction
+
         model_set_collection_attributes(collection, klass, @models_by_id, objects_new)
 
         #remove from models_by_id elements that were not touched
@@ -119,7 +122,6 @@ def model_set_unmapped(models_by_id, sol)
         end
       end
 
-
       def create_struct(bnode_extraction, klass, models_by_id, sol, variables)
         list_attributes = Set.new(klass.attributes(:list))
         struct = klass.range(bnode_extraction).new
@@ -144,6 +146,7 @@ def create_class_model(id, klass, klass_struct)
         klass_model.klass = klass if klass_struct
         klass_model
       end
+
       def models_unmapped_to_array(models_by_id)
         models_by_id.each do |idm, m|
           m.unmmaped_to_array
@@ -215,36 +218,32 @@ def model_set_collection_attributes(collection, klass, models_by_id, objects_new
       def get_collection_value(collection, klass)
         collection_value = nil
         if klass.collection_opts.instance_of?(Symbol)
-          if collection.is_a?(Array) && (collection.length == 1)
-            collection_value = collection.first
-          end
-          if collection.respond_to? :id
-            collection_value = collection
-          end
+          collection_value = collection.first if collection.is_a?(Array) && (collection.length == 1)
+          collection_value = collection if collection.respond_to? :id
         end
         collection_value
       end
 
-      def model_map_attributes_values(id, var_set_hash, models_by_id, object, sol, v)
-        if models_by_id[id].respond_to?(:klass)
-          models_by_id[id][v] = object if models_by_id[id][v].nil?
+      def model_map_attributes_values(id, object, sol, v)
+
+        if @models_by_id[id].respond_to?(:klass)
+          @models_by_id[id][v] = object if @models_by_id[id][v].nil?
         else
-          model_attribute_val = models_by_id[id].instance_variable_get("@#{v.to_s}")
-          if (!models_by_id[id].class.handler?(v) || model_attribute_val.nil?) && v != :id
+          model_attribute_val = @models_by_id[id].instance_variable_get("@#{v.to_s}")
+          if (!@models_by_id[id].class.handler?(v) || model_attribute_val.nil?) && v != :id
             # if multiple language values are included for a given property, set the
             # corresponding model attribute to the English language value - NCBO-1662
             if sol[v].kind_of?(RDF::Literal)
-              key = "#{v}#__#{id.to_s}"
-              models_by_id[id].send("#{v}=", object, on_load: true) unless var_set_hash[key]
-              lang = sol[v].language
-              var_set_hash[key] = true if lang == :EN || lang == :en
-            else
-              models_by_id[id].send("#{v}=", object, on_load: true)
+              index, value = @lang_filter.main_lang_filter id, v, object, sol[v]
+              @models_by_id[id].send("#{v}=", value, on_load: true) if index.eql? :no_lang
+            elsif model_attribute_val.nil?
+              @models_by_id[id].send("#{v}=", object, on_load: true)
             end
           end
         end
       end
 
+
       def object_to_array(id, klass_struct, models_by_id, object, v)
         pre = klass_struct ? models_by_id[id][v] :
                 models_by_id[id].instance_variable_get("@#{v}")
@@ -301,11 +300,11 @@ def get_pre_val(id, models_by_id, object, v, read_only)
         if models_by_id[id] &&
           ((models_by_id[id].respond_to?(:klass) && models_by_id[id]) ||
             models_by_id[id].loaded_attributes.include?(v))
-          if !read_only
-            pre_val = models_by_id[id].instance_variable_get("@#{v}")
+          pre_val = if !read_only
+            models_by_id[id].instance_variable_get("@#{v}")
           else
-            pre_val = models_by_id[id][v]
-          end
+            models_by_id[id][v]
+                    end
 
           pre_val = pre_val.select { |x| x.id == object }.first if pre_val.is_a?(Array)
         end

From 55a931367c2ec1dd6c8c07e51db8d01bc8e0a298 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Tue, 15 Mar 2022 18:18:30 +0100
Subject: [PATCH 028/168] use the LanguageFilter in map_attributes

---
 lib/goo/base/resource.rb | 206 ++++++++++++++++++---------------------
 1 file changed, 96 insertions(+), 110 deletions(-)

diff --git a/lib/goo/base/resource.rb b/lib/goo/base/resource.rb
index c12f6203..34504069 100644
--- a/lib/goo/base/resource.rb
+++ b/lib/goo/base/resource.rb
@@ -42,9 +42,7 @@ def valid?
         self.class.attributes.each do |attr|
           inst_value = self.instance_variable_get("@#{attr}")
           attr_errors = Goo::Validators::Enforce.enforce(self,attr,inst_value)
-          unless attr_errors.nil?
-            validation_errors[attr] = attr_errors
-          end
+          validation_errors[attr] = attr_errors unless attr_errors.nil?
         end
 
         if !@persistent && validation_errors.length == 0
@@ -70,18 +68,14 @@ def valid?
       end
 
       def id=(new_id)
-        if !@id.nil? and @persistent
-          raise ArgumentError, "The id of a persistent object cannot be changed."
-        end
+        raise ArgumentError, "The id of a persistent object cannot be changed." if !@id.nil? and @persistent
         raise ArgumentError, "ID must be an RDF::URI" unless new_id.kind_of?(RDF::URI)
         @id = new_id
       end
 
       def id
         if @id.nil?
-          if self.class.name_with == :id
-            raise IDGenerationError, ":id must be set if configured in name_with"
-          end
+          raise IDGenerationError, ":id must be set if configured in name_with" if self.class.name_with == :id
           custom_name = self.class.name_with
           if custom_name.instance_of?(Symbol)
             @id = id_from_attribute()
@@ -153,9 +147,7 @@ def unmmaped_to_array
 
       def delete(*args)
         if self.kind_of?(Goo::Base::Enum)
-          unless args[0] && args[0][:init_enum]
-            raise ArgumentError, "Enums cannot be deleted"
-          end
+          raise ArgumentError, "Enums cannot be deleted" unless args[0] && args[0][:init_enum]
         end
 
         raise ArgumentError, "This object is not persistent and cannot be deleted" if !@persistent
@@ -163,9 +155,7 @@ def delete(*args)
         if !fully_loaded?
           missing = missing_load_attributes
           options_load = { models: [ self ], klass: self.class, :include => missing }
-          if self.class.collection_opts
-            options_load[:collection] = self.collection
-          end
+          options_load[:collection] = self.collection if self.class.collection_opts
           Goo::SPARQL::Queries.model_load(options_load)
         end
 
@@ -181,9 +171,7 @@ def delete(*args)
         end
         @persistent = false
         @modified = true
-        if self.class.inmutable? && self.class.inm_instances
-          self.class.load_inmutable_instances
-        end
+        self.class.load_inmutable_instances if self.class.inmutable? && self.class.inm_instances
         return nil
       end
 
@@ -191,15 +179,11 @@ def bring(*opts)
         opts.each do |k|
           if k.kind_of?(Hash)
             k.each do |k2,v|
-              if self.class.handler?(k2)
-                raise ArgumentError, "Unable to bring a method based attr #{k2}"
-              end
+              raise ArgumentError, "Unable to bring a method based attr #{k2}" if self.class.handler?(k2)
               self.instance_variable_set("@#{k2}",nil)
             end
           else
-            if self.class.handler?(k)
-              raise ArgumentError, "Unable to bring a method based attr #{k}"
-            end
+            raise ArgumentError, "Unable to bring a method based attr #{k}" if self.class.handler?(k)
             self.instance_variable_set("@#{k}",nil)
           end
         end
@@ -214,9 +198,7 @@ def bring(*opts)
 
       def graph
         opts = self.class.collection_opts
-        if opts.nil?
-          return self.class.uri_type
-        end
+        return self.class.uri_type if opts.nil?
         col = collection
         if col.is_a?Array
           if col.length == 1
@@ -228,79 +210,14 @@ def graph
         return col ? col.id : nil
       end
 
-      def self.map_attributes(inst,equivalent_predicates=nil)
-        if (inst.kind_of?(Goo::Base::Resource) && inst.unmapped.nil?) ||
-            (!inst.respond_to?(:unmapped) && inst[:unmapped].nil?)
-          raise ArgumentError, "Resource.map_attributes only works for :unmapped instances"
-        end
-        klass = inst.respond_to?(:klass) ? inst[:klass] : inst.class
-        unmapped = inst.respond_to?(:klass) ? inst[:unmapped] : inst.unmapped
-        list_attrs = klass.attributes(:list)
-        unmapped_string_keys = Hash.new
-        unmapped.each do |k,v|
-          unmapped_string_keys[k.to_s] = v
-        end
-        klass.attributes.each do |attr|
-          next if inst.class.collection?(attr) #collection is already there
-          next unless inst.respond_to?(attr)
-          attr_uri = klass.attribute_uri(attr,inst.collection).to_s
-          if unmapped_string_keys.include?(attr_uri.to_s) ||
-              (equivalent_predicates && equivalent_predicates.include?(attr_uri))
-            object = nil
-            if !unmapped_string_keys.include?(attr_uri)
-              equivalent_predicates[attr_uri].each do |eq_attr|
-                if object.nil? and !unmapped_string_keys[eq_attr].nil?
-                  object = unmapped_string_keys[eq_attr].dup
-                else
-                  if object.is_a?Array
-                    if !unmapped_string_keys[eq_attr].nil?
-                      object.concat(unmapped_string_keys[eq_attr])
-                    end
-                  end
-                end
-              end
-              if object.nil?
-                inst.send("#{attr}=",
-                           list_attrs.include?(attr) ? [] : nil, on_load: true)
-                next
-              end
-            else
-              object = unmapped_string_keys[attr_uri]
-            end
-            object = object.map { |o| o.is_a?(RDF::URI) ? o : o.object }
-            if klass.range(attr)
-              object = object.map { |o|
-                o.is_a?(RDF::URI) ? klass.range_object(attr,o) : o }
-            end
-            unless list_attrs.include?(attr)
-              object = object.first
-            end
-            if inst.respond_to?(:klass)
-              inst[attr] = object
-            else
-              inst.send("#{attr}=",object, on_load: true)
-            end
-          else
-            inst.send("#{attr}=",
-                      list_attrs.include?(attr) ? [] : nil, on_load: true)
-            if inst.id.to_s == "http://purl.obolibrary.org/obo/IAO_0000415"
-              if attr == :definition
-               # binding.pry
-              end
-            end
-          end
 
-        end
-      end
 
       def collection
         opts = self.class.collection_opts
         if opts.instance_of?(Symbol)
           if self.class.attributes.include?(opts)
             value = self.send("#{opts}")
-            if value.nil?
-              raise ArgumentError, "Collection `#{opts}` is nil"
-            end
+            raise ArgumentError, "Collection `#{opts}` is nil" if value.nil?
             return value
           else
             raise ArgumentError, "Collection `#{opts}` is not an attribute"
@@ -315,9 +232,7 @@ def add_aggregate(attribute,aggregate,value)
       def save(*opts)
 
         if self.kind_of?(Goo::Base::Enum)
-          unless opts[0] && opts[0][:init_enum]
-            raise ArgumentError, "Enums can only be created on initialization"
-          end
+          raise ArgumentError, "Enums can only be created on initialization" unless opts[0] && opts[0][:init_enum]
         end
         batch_file = nil
         if opts && opts.length > 0
@@ -327,9 +242,7 @@ def save(*opts)
         end
 
         if !batch_file
-          if not modified?
-            return self
-          end
+          return self if not modified?
           raise Goo::Base::NotValidException, "Object is not valid. Check errors." unless valid?
         end
 
@@ -368,9 +281,7 @@ def save(*opts)
 
         @modified_attributes = Set.new
         @persistent = true
-        if self.class.inmutable? && self.class.inm_instances
-          self.class.load_inmutable_instances
-        end
+        self.class.load_inmutable_instances if self.class.inmutable? && self.class.inm_instances
         return self
       end
 
@@ -408,9 +319,7 @@ def to_hash
             end
           end
           @unmapped.each do |attr,values|
-            unless all_attr_uris.include?(attr)
-              attr_hash[attr] = values.map { |v| v.to_s }
-            end
+            attr_hash[attr] = values.map { |v| v.to_s } unless all_attr_uris.include?(attr)
           end
         end
         attr_hash[:id] = @id
@@ -430,13 +339,90 @@ def self.range_object(attr,id)
         return range_object
       end
 
-      def self.find(id, *options)
-        if !id.instance_of?(RDF::URI) && self.name_with == :id
-          id = RDF::URI.new(id)
+
+
+      def self.map_attributes(inst,equivalent_predicates=nil)
+        if (inst.kind_of?(Goo::Base::Resource) && inst.unmapped.nil?) ||
+          (!inst.respond_to?(:unmapped) && inst[:unmapped].nil?)
+          raise ArgumentError, "Resource.map_attributes only works for :unmapped instances"
         end
-        unless id.instance_of?(RDF::URI)
-          id = id_from_unique_attribute(name_with(),id)
+        klass = inst.respond_to?(:klass) ? inst[:klass] : inst.class
+        unmapped = inst.respond_to?(:klass) ? inst[:unmapped] : inst.unmapped
+        list_attrs = klass.attributes(:list)
+        unmapped_string_keys = Hash.new
+        unmapped.each do |k,v|
+          unmapped_string_keys[k.to_s] = v
+        end
+        klass.attributes.each do |attr|
+          next if inst.class.collection?(attr) #collection is already there
+          next unless inst.respond_to?(attr)
+          attr_uri = klass.attribute_uri(attr,inst.collection).to_s
+          if unmapped_string_keys.include?(attr_uri.to_s) ||
+            (equivalent_predicates && equivalent_predicates.include?(attr_uri))
+            object = nil
+            if !unmapped_string_keys.include?(attr_uri)
+              equivalent_predicates[attr_uri].each do |eq_attr|
+                if object.nil? and !unmapped_string_keys[eq_attr].nil?
+                  object = unmapped_string_keys[eq_attr].dup
+                else
+                  if object.is_a?Array
+                    object.concat(unmapped_string_keys[eq_attr]) if !unmapped_string_keys[eq_attr].nil?
+                  end
+                end
+              end
+              if object.nil?
+                inst.send("#{attr}=", list_attrs.include?(attr) ? [] : nil, on_load: true)
+                next
+              end
+            else
+              object = unmapped_string_keys[attr_uri]
+            end
+
+            lang_filter = Goo::SPARQL::Solution::LanguageFilter.new
+
+            object = object.map do |o|
+              if o.is_a?(RDF::URI)
+                o
+              else
+                literal = o
+                index, lang_val  = lang_filter.main_lang_filter inst.id.to_s, attr, literal, literal
+                lang_val.to_s if index.eql? :no_lang
+              end
+            end
+
+            object = object.compact
+
+            other_languages_values = lang_filter.other_languages_values
+            other_languages_values = other_languages_values[inst.id.to_s][attr] unless other_languages_values.empty?
+            unless other_languages_values.nil?
+              object = lang_filter.languages_values_to_set(other_languages_values, object)
+            end
+
+            if klass.range(attr)
+              object = object.map { |o|
+                o.is_a?(RDF::URI) ? klass.range_object(attr,o) : o }
+            end
+            object = object.first unless list_attrs.include?(attr)
+            if inst.respond_to?(:klass)
+              inst[attr] = object
+            else
+              inst.send("#{attr}=",object, on_load: true)
+            end
+          else
+            inst.send("#{attr}=",
+                      list_attrs.include?(attr) ? [] : nil, on_load: true)
+            if inst.id.to_s == "http://purl.obolibrary.org/obo/IAO_0000415"
+              if attr == :definition
+                # binding.pry
+              end
+            end
+          end
+
         end
+      end
+      def self.find(id, *options)
+        id = RDF::URI.new(id) if !id.instance_of?(RDF::URI) && self.name_with == :id
+        id = id_from_unique_attribute(name_with(),id) unless id.instance_of?(RDF::URI)
         if self.inmutable? && self.inm_instances && self.inm_instances[id]
           w = Goo::Base::Where.new(self)
           w.instance_variable_set("@result", [self.inm_instances[id]])

From 1108250e41ff37f65ad7659d8aa81ab251b93c8b Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Wed, 16 Mar 2022 08:45:39 +0100
Subject: [PATCH 029/168] add include variable to the solution mapper arguments

---
 lib/goo/sparql/loader.rb           |  7 ++++---
 lib/goo/sparql/solutions_mapper.rb | 14 ++++++++------
 2 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/lib/goo/sparql/loader.rb b/lib/goo/sparql/loader.rb
index d877d47d..a542c820 100644
--- a/lib/goo/sparql/loader.rb
+++ b/lib/goo/sparql/loader.rb
@@ -93,7 +93,8 @@ def self.model_load_sliced(*options)
         expand_equivalent_predicates(select, equivalent_predicates)
         solution_mapper = Goo::SPARQL::SolutionMapper.new aggregate_projections, bnode_extraction,
                                                           embed_struct, incl_embed, klass_struct, models_by_id,
-                                                          predicates_map, unmapped, variables, options
+                                                          predicates_map, unmapped, variables, incl, options
+
         solution_mapper.map_each_solutions(select)
       end
 
@@ -147,10 +148,10 @@ def self.get_predicate_map(predicates)
           predicates_map = {}
           uniq_p.each do |p|
             i = 0
-            key = ("var_" + p.last_part + i.to_s).to_sym
+            key = ("var_#{p.last_part}#{i.to_s}").to_sym
             while predicates_map.include?(key)
               i += 1
-              key = ("var_" + p.last_part + i.to_s).to_sym
+              key = ("var_#{p.last_part}#{i.to_s}").to_sym
               break if i > 10
             end
             predicates_map[key] = p
diff --git a/lib/goo/sparql/solutions_mapper.rb b/lib/goo/sparql/solutions_mapper.rb
index 209bf316..57303f4d 100644
--- a/lib/goo/sparql/solutions_mapper.rb
+++ b/lib/goo/sparql/solutions_mapper.rb
@@ -6,7 +6,7 @@ class SolutionMapper
 
       def initialize(aggregate_projections, bnode_extraction, embed_struct,
                      incl_embed, klass_struct, models_by_id,
-                     predicates_map, unmapped, variables, options)
+                     predicates_map, unmapped, variables, incl, options)
 
         @aggregate_projections = aggregate_projections
         @bnode_extraction = bnode_extraction
@@ -17,6 +17,7 @@ def initialize(aggregate_projections, bnode_extraction, embed_struct,
         @predicates_map = predicates_map
         @unmapped = unmapped
         @variables = variables
+        @incl = incl
         @options = options
 
 
@@ -28,7 +29,7 @@ def map_each_solutions(select)
         klass = @options[:klass]
         read_only = @options[:read_only]
         collection = @options[:collection]
-        incl = @options[:include]
+
 
         found = Set.new
         objects_new = {}
@@ -72,7 +73,7 @@ def map_each_solutions(select)
             object = sol[v] || nil
 
             #bnodes
-            if object.kind_of?(RDF::Node) && object.anonymous? && incl.include?(v)
+            if object.kind_of?(RDF::Node) && object.anonymous? && @incl.include?(v)
               initialize_object(id, klass, object, objects_new, v)
               next
             end
@@ -101,6 +102,7 @@ def map_each_solutions(select)
         #next level of embed attributes
         include_embed_attributes(collection, @incl_embed, klass, objects_new) if @incl_embed && !@incl_embed.empty?
 
+
         #bnodes
         bnodes = objects_new.select { |id, obj| id.is_a?(RDF::Node) && id.anonymous? }
         include_bnodes(bnodes, collection, klass, @models_by_id) unless bnodes.empty?
@@ -182,7 +184,7 @@ def include_embed_attributes(collection, incl_embed, klass, objects_new)
           }.values
           unless range_objs.empty?
             range_objs.uniq!
-            attr_range.where().models(range_objs).in(collection).include(*next_attrs).all
+            attr_range.where.models(range_objs).in(collection).include(*next_attrs).all
           end
         end
       end
@@ -225,7 +227,7 @@ def get_collection_value(collection, klass)
       end
 
       def model_map_attributes_values(id, object, sol, v)
-
+        #binding.pry if v.eql? :programs
         if @models_by_id[id].respond_to?(:klass)
           @models_by_id[id][v] = object if @models_by_id[id][v].nil?
         else
@@ -236,7 +238,7 @@ def model_map_attributes_values(id, object, sol, v)
             if sol[v].kind_of?(RDF::Literal)
               index, value = @lang_filter.main_lang_filter id, v, object, sol[v]
               @models_by_id[id].send("#{v}=", value, on_load: true) if index.eql? :no_lang
-            elsif model_attribute_val.nil?
+            elsif model_attribute_val.nil? || !object.nil?
               @models_by_id[id].send("#{v}=", object, on_load: true)
             end
           end

From 7c9aa7260d828e74d780d967d8947eeb49d7015b Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Wed, 16 Mar 2022 14:26:37 +0100
Subject: [PATCH 030/168] change the language filter to take only one language

---
 lib/goo.rb                                    | 2 ++
 lib/goo/sparql/mixins/solution_lang_filter.rb | 8 ++++++--
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/lib/goo.rb b/lib/goo.rb
index fedae62e..cf7267ce 100644
--- a/lib/goo.rb
+++ b/lib/goo.rb
@@ -26,7 +26,9 @@ module Goo
   @@resource_options = Set.new([:persistent]).freeze
 
   # Define the languages from which the properties values will be taken
+  # It choose the first language that match otherwise return all the values
   @@main_languages = %w[en]
+
   @@configure_flag = false
   @@sparql_backends = {}
   @@model_by_name = {}
diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index af02b44a..f2bcc4ae 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -38,8 +38,12 @@ def languages_values_to_set(language_values, no_lang_values)
           matched_lang, not_matched_lang = matched_languages(language_values, no_lang_values)
           if !matched_lang.empty?
             main_lang = Array(matched_lang[:'0']) + Array(matched_lang[:no_lang])
-            secondary_languages = matched_lang.select { |key| key != :'0' && key != :no_lang }.sort.map { |x| x[1] }.flatten
-            values = main_lang + secondary_languages
+            if main_lang.empty?
+              secondary_languages = matched_lang.select { |key| key != :'0' && key != :no_lang }.sort.map { |x| x[1] }
+              values = secondary_languages.first
+            else
+              values = main_lang
+            end
           elsif !not_matched_lang.empty?
             values = not_matched_lang
           end

From ba27011fd2b093ff04d522477010d146602d0b62 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Fri, 18 Mar 2022 18:45:17 +0100
Subject: [PATCH 031/168] add the condition of nil? or empty? before adding the
 no_lang

---
 lib/goo/sparql/mixins/solution_lang_filter.rb | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index f2bcc4ae..9f0a5568 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -78,7 +78,9 @@ def save_other_lang_val(id, attr, index, value)
         def matched_languages(index_values, model_attribute_val)
           not_matched_lang = index_values[:not_matched]
           matched_lang = index_values.reject { |key| key == :not_matched }
-          matched_lang[:no_lang] = Array(model_attribute_val) unless model_attribute_val.nil?
+          unless model_attribute_val.nil? || model_attribute_val.empty?
+            matched_lang[:no_lang] = Array(model_attribute_val)
+          end
           [matched_lang, not_matched_lang]
         end
       end

From c7106bd6bb6d602b1aff30447ee34691f5f9984d Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Mon, 16 May 2022 10:54:16 +0200
Subject: [PATCH 032/168] uniquify the language values

---
 lib/goo/sparql/mixins/solution_lang_filter.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index 9f0a5568..fb4d116b 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -47,7 +47,7 @@ def languages_values_to_set(language_values, no_lang_values)
           elsif !not_matched_lang.empty?
             values = not_matched_lang
           end
-          values
+          values&.uniq
         end
 
         private

From 835aefa458186485eef60d3845315c478a95ff23 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Wed, 20 Jul 2022 15:36:05 +0200
Subject: [PATCH 033/168] re-implement the lang filter for the query builder

---
 Gemfile                                       |  2 +-
 Gemfile.lock                                  |  4 +-
 lib/goo/base/resource.rb                      |  2 +-
 lib/goo/sparql/mixins/solution_lang_filter.rb | 38 ++++++++++++-------
 lib/goo/sparql/solutions_mapper.rb            | 34 ++++++++---------
 5 files changed, 45 insertions(+), 35 deletions(-)

diff --git a/Gemfile b/Gemfile
index 2ca6a3b7..30167e35 100644
--- a/Gemfile
+++ b/Gemfile
@@ -18,4 +18,4 @@ group :profiling do
   gem 'thin'
 end
 
-gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'master'
+gem 'sparql-client', github: 'ontoportal-lirmm/sparql-client', branch: 'master'
diff --git a/Gemfile.lock b/Gemfile.lock
index 93e4d6e1..8480f287 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -1,6 +1,6 @@
 GIT
-  remote: https://github.com/ncbo/sparql-client.git
-  revision: fb4a89b420f8eb6dda5190a126b6c62e32c4c0c9
+  remote: https://github.com/ontoportal-lirmm/sparql-client.git
+  revision: aed51baf4106fd0f3d0e3f9238f0aad9406aa3f0
   branch: master
   specs:
     sparql-client (1.0.1)
diff --git a/lib/goo/base/resource.rb b/lib/goo/base/resource.rb
index 34504069..13c4b61a 100644
--- a/lib/goo/base/resource.rb
+++ b/lib/goo/base/resource.rb
@@ -385,7 +385,7 @@ def self.map_attributes(inst,equivalent_predicates=nil)
                 o
               else
                 literal = o
-                index, lang_val  = lang_filter.main_lang_filter inst.id.to_s, attr, literal, literal
+                index, lang_val  = lang_filter.main_lang_filter inst.id.to_s, attr, literal
                 lang_val.to_s if index.eql? :no_lang
               end
             end
diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index fb4d116b..efb33943 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -7,13 +7,11 @@ def initialize
           @other_languages_values = {}
         end
 
-        def other_languages_values
-          @other_languages_values
-        end
+        attr_reader :other_languages_values
 
-        def main_lang_filter(id, attr, old_values, new_value)
-          index, value = lang_index  old_values, new_value
-          save_other_lang_val(id, attr, index, new_value) unless index.eql? :no_lang
+        def main_lang_filter(id, attr, value)
+          index, value = lang_index value
+          save_other_lang_val(id, attr, index, value) unless index.nil? ||index.eql?(:no_lang)
           [index, value]
         end
 
@@ -22,11 +20,22 @@ def fill_models_with_other_languages(models_by_id, list_attributes)
             languages_values.each do |attr, index_values|
               model_attribute_val = models_by_id[id].instance_variable_get("@#{attr.to_s}")
               values = languages_values_to_set(index_values, model_attribute_val)
-
+              m = models_by_id[id]
+              value = nil
+              is_struct = m.respond_to?(:klass)
               if !values.nil? && list_attributes.include?(attr)
-                models_by_id[id].send("#{attr.to_s}=", values || [], on_load: true)
+                value = values || []
+
               elsif !values.nil?
-                models_by_id[id].send("#{attr.to_s}=", values.first || nil, on_load: true)
+                value = values.first || nil
+              end
+
+              if value
+                if is_struct
+                  m[attr] = value
+                else
+                  m.send("#{attr}=", value, on_load: true)
+                end
               end
             end
           end
@@ -52,14 +61,17 @@ def languages_values_to_set(language_values, no_lang_values)
 
         private
 
-        def lang_index(object, new_value)
-          lang = new_value.language
+        def lang_index(object)
+          return [nil, object] unless  object.is_a?(RDF::Literal)
+
+          lang = object.language
+
           if lang.nil?
             [:no_lang, object]
           else
             index = Goo.language_includes(lang)
             index = index ? index.to_s.to_sym : :not_matched
-            [index, new_value]
+            [index, object]
           end
         end
 
@@ -72,8 +84,6 @@ def save_other_lang_val(id, attr, index, value)
             @other_languages_values[id][attr][index] += Array(value.to_s)
           end
         end
-        
-
 
         def matched_languages(index_values, model_attribute_val)
           not_matched_lang = index_values[:not_matched]
diff --git a/lib/goo/sparql/solutions_mapper.rb b/lib/goo/sparql/solutions_mapper.rb
index e990beef..585f3ca7 100644
--- a/lib/goo/sparql/solutions_mapper.rb
+++ b/lib/goo/sparql/solutions_mapper.rb
@@ -6,7 +6,7 @@ class SolutionMapper
 
       def initialize(aggregate_projections, bnode_extraction, embed_struct,
                      incl_embed, klass_struct, models_by_id,
-                     properties_to_include, unmapped, variables,ids, options)
+                     properties_to_include, unmapped, variables, ids, options)
 
         @aggregate_projections = aggregate_projections
         @bnode_extraction = bnode_extraction
@@ -72,11 +72,13 @@ def map_each_solutions(select)
             next
           end
 
-          object, objects_new = get_value_object(id, objects_new, object, list_attributes, v)
-          add_object_to_model(id, object, v, var_set_hash)
+          # if multiple language values are included for a given property, set the
+          # corresponding model attribute to the English language value - NCBO-1662
+          language, object = get_object_language(id, object, predicate)
+          object, objects_new = get_value_object(id, objects_new, object, list_attributes, predicate)
+          add_object_to_model(id, object, predicate, language)
         end
         @lang_filter.fill_models_with_other_languages(@models_by_id, list_attributes)
-
         init_unloaded_attributes(found, list_attributes)
 
         return @models_by_id if @bnode_extraction
@@ -91,7 +93,6 @@ def map_each_solutions(select)
         #next level of embed attributes
         include_embed_attributes(@incl_embed, objects_new) if @incl_embed && !@incl_embed.empty?
 
-
         #bnodes
         blank_nodes = objects_new.select { |id, obj| id.is_a?(RDF::Node) && id.anonymous? }
         include_bnodes(blank_nodes, @models_by_id) unless blank_nodes.empty?
@@ -103,6 +104,10 @@ def map_each_solutions(select)
 
       private
 
+      def get_object_language(id, object, predicate)
+        @lang_filter.main_lang_filter id, predicate, object
+      end
+
       def init_unloaded_attributes(found, list_attributes)
         return if @incl.nil?
 
@@ -167,25 +172,20 @@ def get_value_object(id, objects_new, object, list_attributes, predicate)
             object.uniq!
           end
         end
-        [object,objects_new]
+        [object, objects_new]
       end
 
-      def add_object_to_model(id, object, predicate, var_set_hash)
+      def add_object_to_model(id, object, predicate, lang)
         if @models_by_id[id].respond_to?(:klass)
           @models_by_id[id][predicate] = object unless object.nil? && !@models_by_id[id][predicate].nil?
         elsif !@models_by_id[id].class.handler?(predicate) &&
-          !(object.nil? && !@models_by_id[id].instance_variable_get("@#{predicate}").nil?) &&
-          predicate != :id
-          # if multiple language values are included for a given property, set the
-          # corresponding model attribute to the English language value - NCBO-1662
-          if object.is_a?(RDF::Literal)
-            key = "#{predicate}#__#{id}"
-            @models_by_id[id].send("#{predicate}=", object, on_load: true) unless var_set_hash[key]
-            lang = object.language
-            var_set_hash[key] = true if %i[EN en].include?(lang)
-          else
+              !(object.nil? && !@models_by_id[id].instance_variable_get("@#{predicate}").nil?) &&
+              predicate != :id
+
+          if (lang&.eql?(:no_lang)) || !lang
             @models_by_id[id].send("#{predicate}=", object, on_load: true)
           end
+
         end
       end
 

From d1d34dcf455491f3e6413a60e8c4edd977830cf8 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Wed, 20 Jul 2022 15:36:05 +0200
Subject: [PATCH 034/168] re-implement the lang filter for the query builder

---
 Gemfile                                       |  2 +-
 Gemfile.lock                                  |  4 +-
 lib/goo/base/resource.rb                      |  2 +-
 lib/goo/sparql/mixins/solution_lang_filter.rb | 38 ++++++++++------
 lib/goo/sparql/solutions_mapper.rb            | 44 +++++++++----------
 5 files changed, 50 insertions(+), 40 deletions(-)

diff --git a/Gemfile b/Gemfile
index 2ca6a3b7..30167e35 100644
--- a/Gemfile
+++ b/Gemfile
@@ -18,4 +18,4 @@ group :profiling do
   gem 'thin'
 end
 
-gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'master'
+gem 'sparql-client', github: 'ontoportal-lirmm/sparql-client', branch: 'master'
diff --git a/Gemfile.lock b/Gemfile.lock
index 93e4d6e1..8480f287 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -1,6 +1,6 @@
 GIT
-  remote: https://github.com/ncbo/sparql-client.git
-  revision: fb4a89b420f8eb6dda5190a126b6c62e32c4c0c9
+  remote: https://github.com/ontoportal-lirmm/sparql-client.git
+  revision: aed51baf4106fd0f3d0e3f9238f0aad9406aa3f0
   branch: master
   specs:
     sparql-client (1.0.1)
diff --git a/lib/goo/base/resource.rb b/lib/goo/base/resource.rb
index 34504069..13c4b61a 100644
--- a/lib/goo/base/resource.rb
+++ b/lib/goo/base/resource.rb
@@ -385,7 +385,7 @@ def self.map_attributes(inst,equivalent_predicates=nil)
                 o
               else
                 literal = o
-                index, lang_val  = lang_filter.main_lang_filter inst.id.to_s, attr, literal, literal
+                index, lang_val  = lang_filter.main_lang_filter inst.id.to_s, attr, literal
                 lang_val.to_s if index.eql? :no_lang
               end
             end
diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index fb4d116b..efb33943 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -7,13 +7,11 @@ def initialize
           @other_languages_values = {}
         end
 
-        def other_languages_values
-          @other_languages_values
-        end
+        attr_reader :other_languages_values
 
-        def main_lang_filter(id, attr, old_values, new_value)
-          index, value = lang_index  old_values, new_value
-          save_other_lang_val(id, attr, index, new_value) unless index.eql? :no_lang
+        def main_lang_filter(id, attr, value)
+          index, value = lang_index value
+          save_other_lang_val(id, attr, index, value) unless index.nil? ||index.eql?(:no_lang)
           [index, value]
         end
 
@@ -22,11 +20,22 @@ def fill_models_with_other_languages(models_by_id, list_attributes)
             languages_values.each do |attr, index_values|
               model_attribute_val = models_by_id[id].instance_variable_get("@#{attr.to_s}")
               values = languages_values_to_set(index_values, model_attribute_val)
-
+              m = models_by_id[id]
+              value = nil
+              is_struct = m.respond_to?(:klass)
               if !values.nil? && list_attributes.include?(attr)
-                models_by_id[id].send("#{attr.to_s}=", values || [], on_load: true)
+                value = values || []
+
               elsif !values.nil?
-                models_by_id[id].send("#{attr.to_s}=", values.first || nil, on_load: true)
+                value = values.first || nil
+              end
+
+              if value
+                if is_struct
+                  m[attr] = value
+                else
+                  m.send("#{attr}=", value, on_load: true)
+                end
               end
             end
           end
@@ -52,14 +61,17 @@ def languages_values_to_set(language_values, no_lang_values)
 
         private
 
-        def lang_index(object, new_value)
-          lang = new_value.language
+        def lang_index(object)
+          return [nil, object] unless  object.is_a?(RDF::Literal)
+
+          lang = object.language
+
           if lang.nil?
             [:no_lang, object]
           else
             index = Goo.language_includes(lang)
             index = index ? index.to_s.to_sym : :not_matched
-            [index, new_value]
+            [index, object]
           end
         end
 
@@ -72,8 +84,6 @@ def save_other_lang_val(id, attr, index, value)
             @other_languages_values[id][attr][index] += Array(value.to_s)
           end
         end
-        
-
 
         def matched_languages(index_values, model_attribute_val)
           not_matched_lang = index_values[:not_matched]
diff --git a/lib/goo/sparql/solutions_mapper.rb b/lib/goo/sparql/solutions_mapper.rb
index 9dfc6ba4..86a23b58 100644
--- a/lib/goo/sparql/solutions_mapper.rb
+++ b/lib/goo/sparql/solutions_mapper.rb
@@ -6,7 +6,7 @@ class SolutionMapper
 
       def initialize(aggregate_projections, bnode_extraction, embed_struct,
                      incl_embed, klass_struct, models_by_id,
-                     properties_to_include, unmapped, variables,ids, options)
+                     properties_to_include, unmapped, variables, ids, options)
 
         @aggregate_projections = aggregate_projections
         @bnode_extraction = bnode_extraction
@@ -35,7 +35,7 @@ def map_each_solutions(select)
         var_set_hash = {}
         list_attributes = Set.new(@klass.attributes(:list))
         all_attributes = Set.new(@klass.attributes(:all))
-
+        @lang_filter = Goo::SPARQL::Solution::LanguageFilter.new
         
         select.each_solution do |sol|
           next if sol[:some_type] && @klass.type_uri(@collection) != sol[:some_type]
@@ -61,23 +61,25 @@ def map_each_solutions(select)
             next
           end
 
-          v = sol[:attributeProperty].to_s.to_sym
+          predicate = sol[:attributeProperty].to_s.to_sym
 
-          next if v.nil? || !all_attributes.include?(v)
+          next if predicate.nil? || !all_attributes.include?(predicate)
 
           object = sol[:attributeObject]
 
           #bnodes
-          if bnode_id?(object, v)
-            objects_new = bnode_id_tuple(id, object, objects_new, v)
+          if bnode_id?(object, predicate)
+            objects_new = bnode_id_tuple(id, object, objects_new, predicate)
             next
           end
 
-          object, objects_new = get_value_object(id, objects_new, object, list_attributes, v)
-          add_object_to_model(id, object, v, var_set_hash)
+          # if multiple language values are included for a given property, set the
+          # corresponding model attribute to the English language value - NCBO-1662
+          language, object = get_object_language(id, object, predicate)
+          object, objects_new = get_value_object(id, objects_new, object, list_attributes, predicate)
+          add_object_to_model(id, object, predicate, language)
         end
         @lang_filter.fill_models_with_other_languages(@models_by_id, list_attributes)
-
         init_unloaded_attributes(found, list_attributes)
 
         return @models_by_id if @bnode_extraction
@@ -92,7 +94,6 @@ def map_each_solutions(select)
         #next level of embed attributes
         include_embed_attributes(@incl_embed, objects_new) if @incl_embed && !@incl_embed.empty?
 
-
         #bnodes
         blank_nodes = objects_new.select { |id, obj| id.is_a?(RDF::Node) && id.anonymous? }
         include_bnodes(blank_nodes, @models_by_id) unless blank_nodes.empty?
@@ -104,6 +105,10 @@ def map_each_solutions(select)
 
       private
 
+      def get_object_language(id, object, predicate)
+        @lang_filter.main_lang_filter id, predicate, object
+      end
+
       def init_unloaded_attributes(found, list_attributes)
         return if @incl.nil?
 
@@ -168,25 +173,20 @@ def get_value_object(id, objects_new, object, list_attributes, predicate)
             object.uniq!
           end
         end
-        [object,objects_new]
+        [object, objects_new]
       end
 
-      def add_object_to_model(id, object, predicate, var_set_hash)
+      def add_object_to_model(id, object, predicate, lang)
         if @models_by_id[id].respond_to?(:klass)
           @models_by_id[id][predicate] = object unless object.nil? && !@models_by_id[id][predicate].nil?
         elsif !@models_by_id[id].class.handler?(predicate) &&
-          !(object.nil? && !@models_by_id[id].instance_variable_get("@#{predicate}").nil?) &&
-          predicate != :id
-          # if multiple language values are included for a given property, set the
-          # corresponding model attribute to the English language value - NCBO-1662
-          if object.is_a?(RDF::Literal)
-            key = "#{predicate}#__#{id}"
-            @models_by_id[id].send("#{predicate}=", object, on_load: true) unless var_set_hash[key]
-            lang = object.language
-            var_set_hash[key] = true if %i[EN en].include?(lang)
-          else
+              !(object.nil? && !@models_by_id[id].instance_variable_get("@#{predicate}").nil?) &&
+              predicate != :id
+
+          if (lang&.eql?(:no_lang)) || !lang
             @models_by_id[id].send("#{predicate}=", object, on_load: true)
           end
+
         end
       end
 

From 8c84c9ed0a050c592093853d0739e7c4f14f25df Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Wed, 23 Nov 2022 09:34:58 +0100
Subject: [PATCH 035/168] force attribute values with lang to be string

---
 lib/goo/sparql/mixins/solution_lang_filter.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index fb4d116b..6c0205f0 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -78,8 +78,8 @@ def save_other_lang_val(id, attr, index, value)
         def matched_languages(index_values, model_attribute_val)
           not_matched_lang = index_values[:not_matched]
           matched_lang = index_values.reject { |key| key == :not_matched }
-          unless model_attribute_val.nil? || model_attribute_val.empty?
-            matched_lang[:no_lang] = Array(model_attribute_val)
+          unless model_attribute_val.nil? || model_attribute_val.to_s.empty?
+            matched_lang[:no_lang] = Array(model_attribute_val.to_s)
           end
           [matched_lang, not_matched_lang]
         end

From 34a9fdfed3b79e5b796af154ff7407bf63d3411f Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Wed, 23 Nov 2022 09:34:58 +0100
Subject: [PATCH 036/168] force attribute values with lang to be Array

---
 lib/goo/sparql/mixins/solution_lang_filter.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index fb4d116b..4fb12292 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -78,7 +78,7 @@ def save_other_lang_val(id, attr, index, value)
         def matched_languages(index_values, model_attribute_val)
           not_matched_lang = index_values[:not_matched]
           matched_lang = index_values.reject { |key| key == :not_matched }
-          unless model_attribute_val.nil? || model_attribute_val.empty?
+          unless model_attribute_val.nil? || Array(model_attribute_val).empty?
             matched_lang[:no_lang] = Array(model_attribute_val)
           end
           [matched_lang, not_matched_lang]

From ba4f9101985d92fc0b316121744801fcbad40b15 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Sat, 17 Dec 2022 14:57:57 +0100
Subject: [PATCH 037/168] merge partial fix to ncbo/bioportal-project#251

---
 lib/goo/sparql/query_builder.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/goo/sparql/query_builder.rb b/lib/goo/sparql/query_builder.rb
index a3c35a43..274cd647 100644
--- a/lib/goo/sparql/query_builder.rb
+++ b/lib/goo/sparql/query_builder.rb
@@ -147,7 +147,7 @@ def ids_filter(ids)
         filter_id = []
 
         ids.each do |id|
-          filter_id << "?id = #{id.to_ntriples.to_s}"
+          filter_id << "?id = #{id.to_ntriples.to_s.gsub(' ', '%20')}"
         end
         filter_id_str = filter_id.join ' || '
         @query.filter filter_id_str

From 3ddcf11fa852011ae779bb7220d1eb8762287c45 Mon Sep 17 00:00:00 2001
From: Timothy Redmond <tredmond@stanford.edu>
Date: Fri, 4 Nov 2022 14:28:20 -0700
Subject: [PATCH 038/168] Revert "A small (2 line) fix for 3 or 12 (depending
 on how it is counted) tests + removal at least one error."

This reverts commit 0e09816b121750b3bb875a5c24cb79865287fcf4.
---
 lib/goo/validators/enforce.rb | 2 --
 1 file changed, 2 deletions(-)

diff --git a/lib/goo/validators/enforce.rb b/lib/goo/validators/enforce.rb
index 2e1b9e56..d326839b 100644
--- a/lib/goo/validators/enforce.rb
+++ b/lib/goo/validators/enforce.rb
@@ -56,8 +56,6 @@ def self.enforce(inst,attr,value)
         errors_by_opt = {}
         enforce_opts.each do |opt|
           case opt
-          when :class
-            nil
           when :unique
             unless value.nil?
               dup = Goo::SPARQL::Queries.duplicate_attribute_value?(inst,attr)

From 86553d235c89114a48c5a58832f67664e14c8158 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Sat, 17 Dec 2022 16:01:14 +0100
Subject: [PATCH 039/168] fix test_model_complex.rb tests with a uniq and not
 reused model name

---
 test/test_model_complex.rb | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/test/test_model_complex.rb b/test/test_model_complex.rb
index e2237c1c..8f904d8b 100644
--- a/test/test_model_complex.rb
+++ b/test/test_model_complex.rb
@@ -10,7 +10,7 @@ class Submission < Goo::Base::Resource
 end
 
 class Term < Goo::Base::Resource
-  model :class,
+  model :term,
         namespace: :owl,
         collection: :submission,
         name_with: :id,
@@ -25,22 +25,22 @@ class Term < Goo::Base::Resource
   attribute :parents, 
             namespace: :rdfs, 
             property: lambda { |x| tree_property(x) },
-            enforce: [:list, :class]
+            enforce: [:list, :term]
 
   attribute :ancestors, 
             namespace: :rdfs, 
             property: lambda { |x| tree_property(x) },
-            enforce: [:list, :class], transitive: true
+            enforce: [:list, :term], transitive: true
 
   attribute :children, 
             namespace: :rdfs, 
             property: lambda { |x| tree_property(x) },
-            inverse: { on: :class , attribute: :parents }
+            inverse: { on: :term , attribute: :parents }
 
   attribute :descendants, 
             namespace: :rdfs, 
             property: lambda { |x| tree_property(x) },
-            inverse: { on: :class , attribute: :parents }, 
+            inverse: { on: :term , attribute: :parents },
             transitive: true
 
   def self.tree_property(*args)

From 8e0e46d7785a0c58f92edc9bb81e06c592d8b745 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Sat, 17 Dec 2022 17:13:33 +0100
Subject: [PATCH 040/168] fix test_embed_struct in test_read_only.rb

---
 test/test_read_only.rb | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/test/test_read_only.rb b/test/test_read_only.rb
index 4496c463..268f7c86 100644
--- a/test/test_read_only.rb
+++ b/test/test_read_only.rb
@@ -39,11 +39,17 @@ def test_struct_find
     end
 
     def test_embed_struct
-      skip "not yet"
+
       students = Student.where(enrolled: [university: [name: "Stanford"]])
                 .include(:name)
-                .include(enrolled: [:name, university: [ :address ]])
+                .include(enrolled: [:name, university: [ :address, :name ]])
                 .read_only.all
+
+      assert_equal 3, students.size
+      students.each do |st|
+        assert st.enrolled.any? {|e| e.is_a?(Struct) && e.university.name.eql?('Stanford')}
+      end
+
     end
   end
 end

From 361940a81fb4ca1bc9f85c2ee12c6d375dfc53b6 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Sat, 17 Dec 2022 17:15:55 +0100
Subject: [PATCH 041/168] fix test_reentrant_queries by ensuring  the write
 thread is still alive

---
 test/test_chunks_write.rb | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/test/test_chunks_write.rb b/test/test_chunks_write.rb
index a7988d40..4f0a8676 100644
--- a/test/test_chunks_write.rb
+++ b/test/test_chunks_write.rb
@@ -71,7 +71,6 @@ def test_put_delete_data
     end
 
     def test_reentrant_queries
-      skip "TODO: why does this test fail?"
       ntriples_file_path = "./test/data/nemo_ontology.ntriples"
 
       # Bypass in chunks
@@ -87,8 +86,8 @@ def test_reentrant_queries
 
       tput = Thread.new {
         Goo.sparql_data_client.put_triples(ONT_ID_EXTRA, ntriples_file_path, mime_type="application/x-turtle")
+        sleep(1.5)
       }
-      sleep(1.5)
       count_queries = 0
       tq = Thread.new {
        5.times do
@@ -112,8 +111,8 @@ def test_reentrant_queries
 
       tdelete = Thread.new {
         Goo.sparql_data_client.delete_graph(ONT_ID_EXTRA)
+        sleep(1.5)
       }
-      sleep(1.5)
       count_queries = 0
       tq = Thread.new {
        5.times do

From a30af42241aa4d9f1ac237ea88942db3cad0765f Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Sat, 17 Dec 2022 17:16:29 +0100
Subject: [PATCH 042/168] update test_inverse_on_collection test

---
 test/test_collections.rb | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/test/test_collections.rb b/test/test_collections.rb
index 390ad349..2177c669 100644
--- a/test/test_collections.rb
+++ b/test/test_collections.rb
@@ -91,18 +91,18 @@ def test_unique_per_collection
   def test_inverse_on_collection
     skip "Not supported inverse on collection"
 
-    john = User.find("John").include(:name).first || 
-      User.new(name: "John").save()
+    john = User.find("John").include(:name).first || User.new(name: "John").save
+
     5.times do |i|
-      Issue.new(description: "issue_#{i}", owner: john).save
+      Issue.find("issue_#{i}").in(john) || Issue.new(description: "issue_#{i}", owner: john).save
     end
-    
-    binding.pry
-    User.find("John",include: [:issues]).first.issues
-    User.find("John",include: [issues: [:desciption]]).first.issues
 
-    5.times do |i|
-      Issue.find("issue_#{i}", collection: john).delete
+    issues = User.find("John").include(:issues).first.issues
+    assert_equal 5, issues.size
+
+    issues.each do |issue|
+      assert_equal "issue_#{i}", issue.description
+      assert_equal john, issue.collection
     end
   end
 

From 4c87b64dfa6ad51825da522b0a07fc1abcac76e0 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Thu, 19 Jan 2023 10:06:47 +0100
Subject: [PATCH 043/168] add REGEX filter unit test

---
 test/test_where.rb | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/test/test_where.rb b/test/test_where.rb
index 1f5f9634..bca4b2ea 100644
--- a/test/test_where.rb
+++ b/test/test_where.rb
@@ -499,6 +499,12 @@ def test_filter
     f = Goo::Filter.new(enrolled: [ :xxx ]).unbound
     st = Student.where.filter(f).all
     assert st.length == 7
+
+    f = Goo::Filter.new(:name).regex("n") # will find all students that contains "n" in there name
+    st = Student.where.filter(f).include(:name).all # return "John" , "Daniel"  and  "Susan"
+
+    assert_equal 3, st.length
+    assert_equal ["John","Daniel","Susan"].sort, st.map { |x| x.name }.sort
   end
 
   def test_aggregated

From 03d8cccb5a86cc46328715a7ca2e58b44646d2c1 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Thu, 19 Jan 2023 10:07:38 +0100
Subject: [PATCH 044/168] update query_filter_sparql to handle REGEX

---
 lib/goo/sparql/queries.rb       | 56 +--------------------------------
 lib/goo/sparql/query_builder.rb | 36 ++++++++++-----------
 2 files changed, 17 insertions(+), 75 deletions(-)

diff --git a/lib/goo/sparql/queries.rb b/lib/goo/sparql/queries.rb
index fb8b1ec5..54189b08 100644
--- a/lib/goo/sparql/queries.rb
+++ b/lib/goo/sparql/queries.rb
@@ -50,61 +50,7 @@ def self.model_exist(model,id=nil,store=:main)
         return so.true?
       end
 
-      def self.query_filter_sparql(klass,filter,filter_patterns,filter_graphs,
-                                   filter_operations,
-                                   internal_variables,
-                                   inspected_patterns,
-                                   collection)
-        #create a object variable to project the value in the filter
-        filter.filter_tree.each do |filter_operation|
-          filter_pattern_match = {}
-          if filter.pattern.instance_of?(Symbol)
-            filter_pattern_match[filter.pattern] = []
-          else
-            filter_pattern_match = filter.pattern
-          end
-          unless inspected_patterns.include?(filter_pattern_match)
-            attr = filter_pattern_match.keys.first
-            patterns_for_match(klass, attr, filter_pattern_match[attr],
-                               filter_graphs, filter_patterns,
-                                   [],internal_variables,
-                                   subject=:id,in_union=false,in_aggregate=false,
-                                   collection=collection)
-            inspected_patterns[filter_pattern_match] = internal_variables.last
-          end
-          filter_var = inspected_patterns[filter_pattern_match]
-          if !filter_operation.value.instance_of?(Goo::Filter)
-            case filter_operation.operator
-            when  :unbound
-              filter_operations << "!BOUND(?#{filter_var.to_s})"
-              return :optional
-
-            when :bound
-              filter_operations << "BOUND(?#{filter_var.to_s})"
-              return :optional
-            when :regex
-              if  filter_operation.value.is_a?(String)
-                filter_operations << "REGEX(?#{filter_var.to_s} , \"#{filter_operation.value.to_s}\")"
-              end
-
-            else
-              value = RDF::Literal.new(filter_operation.value)
-              if filter_operation.value.is_a? String
-                value = RDF::Literal.new(filter_operation.value, :datatype => RDF::XSD.string)
-              end
-              filter_operations << (
-                "?#{filter_var.to_s} #{sparql_op_string(filter_operation.operator)} " +
-                  " #{value.to_ntriples}")
-            end
-
-          else
-            filter_operations << "#{sparql_op_string(filter_operation.operator)}"
-            query_filter_sparql(klass,filter_operation.value,filter_patterns,
-                                filter_graphs,filter_operations,
-                                internal_variables,inspected_patterns,collection)
-          end
-        end
-      end
+   
 
       def self.model_load(*options)
         Goo::SPARQL::Loader.model_load(*options)
diff --git a/lib/goo/sparql/query_builder.rb b/lib/goo/sparql/query_builder.rb
index 274cd647..965277fc 100644
--- a/lib/goo/sparql/query_builder.rb
+++ b/lib/goo/sparql/query_builder.rb
@@ -310,28 +310,24 @@ def query_filter_sparql(klass, filter, filter_patterns, filter_graphs,
           end
           filter_var = inspected_patterns[filter_pattern_match]
 
-          if !filter_operation.value.instance_of?(Goo::Filter)
-            if filter_operation.operator == :unbound || filter_operation.operator == :bound
-              if filter_operation.operator == :unbound
-                filter_operations << "!BOUND(?#{filter_var.to_s})"
-              else
-                filter_operations << "BOUND(?#{filter_var.to_s})"
-              end
+          unless filter_operation.value.instance_of?(Goo::Filter)
+            case filter_operation.operator
+            when  :unbound
+              filter_operations << "!BOUND(?#{filter_var.to_s})"
               return :optional
-            else
-              value = RDF::Literal.new(filter_operation.value)
-              if filter_operation.value.is_a? String
-                value = RDF::Literal.new(filter_operation.value, :datatype => RDF::XSD.string)
+
+            when :bound
+              filter_operations << "BOUND(?#{filter_var.to_s})"
+              return :optional
+            when :regex
+              if  filter_operation.value.is_a?(String)
+                filter_operations << "REGEX(STR(?#{filter_var.to_s}) , \"#{filter_operation.value.to_s}\")"
               end
-              filter_operations << (
-                "?#{filter_var.to_s} #{sparql_op_string(filter_operation.operator)} " +
-                  " #{value.to_ntriples}")
-            end
-          else
-            filter_operations << "#{sparql_op_string(filter_operation.operator)}"
-            query_filter_sparql(klass, filter_operation.value, filter_patterns,
-                                filter_graphs, filter_operations,
-                                internal_variables, inspected_patterns, collection)
+            else
+              filter_operations << "#{sparql_op_string(filter_operation.operator)}"
+              query_filter_sparql(klass, filter_operation.value, filter_patterns,
+                                  filter_graphs, filter_operations,
+                                  internal_variables, inspected_patterns, collection)
           end
         end
       end

From f20479fc49c694f06ca2ca91d388d9ba9875bc2d Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Mon, 27 Feb 2023 02:46:13 +0100
Subject: [PATCH 045/168] add tests for the new dsl to write property data
 types

---
 test/test_dsl_settings.rb | 54 +++++++++++++++++++++++++++++----------
 1 file changed, 40 insertions(+), 14 deletions(-)

diff --git a/test/test_dsl_settings.rb b/test/test_dsl_settings.rb
index c444e829..a4de17e2 100644
--- a/test/test_dsl_settings.rb
+++ b/test/test_dsl_settings.rb
@@ -2,6 +2,22 @@
 
 GooTest.configure_goo
 
+class NewPersonModel < Goo::Base::Resource
+  model :person_model_new, name_with: :name
+  attribute :name, type: :string, enforce: [ :existence, :unique]
+  attribute :multiple_values, type: [:list, :integer], enforce: [ :existence, :min_3, :max_5 ]
+  attribute :one_number, type: :integer,enforce: [ :existence ] #by default not a list
+  attribute :birth_date, type: :date_time, enforce: [ :existence ]
+
+  attribute :created, type: DateTime ,
+            default: lambda { |record| DateTime.now },
+            namespace: :omv
+
+  attribute :friends, type: NewPersonModel , enforce: [ :existence]
+  attribute :status, type: :status, enforce: [ :existence],
+            default: lambda { |record| StatusModel.find("single") }
+end
+
 class StatusModel < Goo::Base::Resource
   model :status_model, name_with: :name
   attribute :description, enforce: [ :existence, :unique]
@@ -37,8 +53,25 @@ def initialize(*args)
     super(*args)
   end
 
+  def test_data_type_dsl
+    _test_attributes_enforce NewPersonModel
+  end
+
   def test_attributes_set_get
+    _test_attributes_enforce PersonModel
+  end
+
+  def test_default_value
+    #default is on save ... returns`
     person = PersonModel.new
+    assert_equal nil, person.created
+  end
+
+
+  private
+  def _test_attributes_enforce(model)
+    person = model.new
+    model_key_name = model.model_name
     assert(person.respond_to? :id)
     assert(person.kind_of? Goo::Base::Resource)
     assert !person.valid?
@@ -67,7 +100,7 @@ def test_attributes_set_get
     assert !person.valid?
     assert !person.errors[:birth_date]
 
-    person.birth_date = "X" 
+    person.birth_date = "X"
     assert !person.valid?
     assert person.errors[:birth_date][:date_time]
 
@@ -103,17 +136,17 @@ def test_attributes_set_get
       person.multiple_values << 99
     end
 
-    friends = [PersonModel.new , PersonModel.new]
+    friends = [model.new , model.new]
     person.friends = friends
     assert !person.valid?
     assert person.errors[:friends][:no_list]
-    person.friends = PersonModel.new
+    person.friends = model.new
     assert !person.valid?
-    assert person.errors[:friends][:person_model]
+    assert person.errors[:friends][model_key_name]
     person.friends = "some one"
     assert !person.valid?
-    assert person.errors[:friends][:person_model]
-    person.friends = PersonModel.new
+    assert person.errors[:friends][model_key_name]
+    person.friends = model.new
 
     person.one_number = 99
     assert !person.valid?
@@ -127,7 +160,7 @@ def test_attributes_set_get
     assert !person.valid?
     assert person.errors[:one_number][:no_list]
 
-    person.one_number = 99 
+    person.one_number = 99
     assert_equal(99, person.one_number)
     assert !person.valid?
     assert !person.errors[:one_number]
@@ -138,11 +171,4 @@ def test_attributes_set_get
     #there are assigned objects that are not saved
     assert !person.valid?
   end
-
-  def test_default_value
-    #default is on save ... returns`
-    person = PersonModel.new
-    assert_equal nil, person.created
-  end
-
 end

From b6cbb1ab185a96ec961bdb08ee1f36665e2d0518 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Mon, 27 Feb 2023 02:46:51 +0100
Subject: [PATCH 046/168] append the property :type values to the :enforce
 array

---
 lib/goo/base/settings/settings.rb | 23 ++++++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)

diff --git a/lib/goo/base/settings/settings.rb b/lib/goo/base/settings/settings.rb
index ce3e9a21..e1937396 100644
--- a/lib/goo/base/settings/settings.rb
+++ b/lib/goo/base/settings/settings.rb
@@ -185,9 +185,12 @@ def attribute(*args)
           attr_name = attr_name.to_sym
           options = options.pop
           options = {} if options.nil?
-          if options[:enforce].nil? or !options[:enforce].include?(:list)
-            options[:enforce] = options[:enforce] ? (options[:enforce] << :no_list) : [:no_list]
-          end
+
+          options[:enforce] ||= []
+
+          set_data_type(options)
+          set_no_list_by_default(options)
+
           @model_settings[:attributes][attr_name] = options
           shape_attribute(attr_name)
           namespace = attribute_namespace(attr_name)
@@ -372,6 +375,20 @@ def read_only(attributes)
           instance
         end
 
+        private
+
+        def set_no_list_by_default(options)
+          if options[:enforce].nil? or !options[:enforce].include?(:list)
+            options[:enforce] = options[:enforce] ? (options[:enforce] << :no_list) : [:no_list]
+          end
+        end
+        def set_data_type(options)
+          if options[:type]
+            options[:enforce] += Array(options[:type])
+            options[:enforce].uniq!
+            options.delete :type
+          end
+        end
       end
     end
   end

From 209ea16d3d64e6183b77e93b0d09958300d22c45 Mon Sep 17 00:00:00 2001
From: HADDAD Zineddine <hz_haddad@esi.dz>
Date: Mon, 27 Feb 2023 10:50:14 +0100
Subject: [PATCH 047/168] update solution mapper to support multilingual

---
 lib/goo/sparql/solutions_mapper.rb | 33 ++++++++++++++++++------------
 1 file changed, 20 insertions(+), 13 deletions(-)

diff --git a/lib/goo/sparql/solutions_mapper.rb b/lib/goo/sparql/solutions_mapper.rb
index 77b20ae0..0544c2dd 100644
--- a/lib/goo/sparql/solutions_mapper.rb
+++ b/lib/goo/sparql/solutions_mapper.rb
@@ -72,13 +72,12 @@ def map_each_solutions(select)
             next
           end
 
-          # if multiple language values are included for a given property, set the
-          # corresponding model attribute to the English language value - NCBO-1662
-          language, object = get_object_language(id, object, predicate)
-          object, objects_new = get_value_object(id, objects_new, object, list_attributes, predicate)
-          add_object_to_model(id, object, predicate, language)
+          lang = object_language(object)
+
+          objects, objects_new = get_value_object(id, objects_new, object, list_attributes, predicate)
+          add_object_to_model(id, objects, predicate, lang, :EN)
         end
-        @lang_filter.fill_models_with_other_languages(@models_by_id, list_attributes)
+        # @lang_filter.fill_models_with_other_languages(@models_by_id, list_attributes)
         init_unloaded_attributes(found, list_attributes)
 
         return @models_by_id if @bnode_extraction
@@ -175,17 +174,25 @@ def get_value_object(id, objects_new, object, list_attributes, predicate)
         [object, objects_new]
       end
 
-      def add_object_to_model(id, object, predicate, lang)
+      def object_language(new_value)
+        new_value.language || :no_lang if new_value.is_a?(RDF::Literal)
+      end
+
+      def add_object_to_model(id, object, predicate, language, requested_lang = nil)
         if @models_by_id[id].respond_to?(:klass)
           @models_by_id[id][predicate] = object unless object.nil? && !@models_by_id[id][predicate].nil?
         elsif !@models_by_id[id].class.handler?(predicate) &&
               !(object.nil? && !@models_by_id[id].instance_variable_get("@#{predicate}").nil?) &&
               predicate != :id
 
-          if (lang&.eql?(:no_lang)) || !lang
-            @models_by_id[id].send("#{predicate}=", object, on_load: true)
-          end
-
+            if language.nil? 
+              @models_by_id[id].send("#{predicate}=", object, on_load: true)
+            else 
+              if language.eql?(requested_lang) || language.eql?(:no_lang) || requested_lang.nil?
+                @models_by_id[id].send("#{predicate}=", object, on_load: true)
+              end
+            end
+          
         end
       end
 
@@ -215,6 +222,7 @@ def preloaded_or_new_struct(object, objects_new, pre_val, predicate)
       def preloaded_value(id, predicate)
         if !@read_only
           @models_by_id[id].instance_variable_get("@#{predicate}")
+          
         else
           @models_by_id[id][predicate]
         end
@@ -457,5 +465,4 @@ def add_aggregations_to_model(sol)
       end
     end
   end
-end
-
+end
\ No newline at end of file

From 207a4835ed8ee6d8746415deef6bc423891bb924 Mon Sep 17 00:00:00 2001
From: HADDAD Zineddine <hz_haddad@esi.dz>
Date: Mon, 27 Feb 2023 11:12:25 +0100
Subject: [PATCH 048/168] update solution mapper to support multilingual

---
 lib/goo/sparql/solutions_mapper.rb | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/goo/sparql/solutions_mapper.rb b/lib/goo/sparql/solutions_mapper.rb
index 0544c2dd..09b0983a 100644
--- a/lib/goo/sparql/solutions_mapper.rb
+++ b/lib/goo/sparql/solutions_mapper.rb
@@ -465,4 +465,5 @@ def add_aggregations_to_model(sol)
       end
     end
   end
-end
\ No newline at end of file
+end
+

From 1a7a6bf14021a2a78b3c2416dc89ed30baec7b35 Mon Sep 17 00:00:00 2001
From: HADDAD Zineddine <hz_haddad@esi.dz>
Date: Mon, 27 Feb 2023 14:40:46 +0100
Subject: [PATCH 049/168] fix typo ( name )

---
 lib/goo/sparql/solutions_mapper.rb | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/lib/goo/sparql/solutions_mapper.rb b/lib/goo/sparql/solutions_mapper.rb
index 09b0983a..7416a78c 100644
--- a/lib/goo/sparql/solutions_mapper.rb
+++ b/lib/goo/sparql/solutions_mapper.rb
@@ -178,18 +178,18 @@ def object_language(new_value)
         new_value.language || :no_lang if new_value.is_a?(RDF::Literal)
       end
 
-      def add_object_to_model(id, object, predicate, language, requested_lang = nil)
+      def add_object_to_model(id, objects, predicate, language, requested_lang = nil)
         if @models_by_id[id].respond_to?(:klass)
-          @models_by_id[id][predicate] = object unless object.nil? && !@models_by_id[id][predicate].nil?
+          @models_by_id[id][predicate] = objects unless objects.nil? && !@models_by_id[id][predicate].nil?
         elsif !@models_by_id[id].class.handler?(predicate) &&
-              !(object.nil? && !@models_by_id[id].instance_variable_get("@#{predicate}").nil?) &&
+              !(objects.nil? && !@models_by_id[id].instance_variable_get("@#{predicate}").nil?) &&
               predicate != :id
 
             if language.nil? 
-              @models_by_id[id].send("#{predicate}=", object, on_load: true)
+              @models_by_id[id].send("#{predicate}=", objects, on_load: true)
             else 
               if language.eql?(requested_lang) || language.eql?(:no_lang) || requested_lang.nil?
-                @models_by_id[id].send("#{predicate}=", object, on_load: true)
+                @models_by_id[id].send("#{predicate}=", objects, on_load: true)
               end
             end
           

From 5168edf1004031b107bce369fab51f279b4a1956 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Tue, 28 Feb 2023 06:23:24 +0100
Subject: [PATCH 050/168] add symmetric validator tests for no_list and list
 cases

---
 test/models.rb          |  5 ++-
 test/test_validators.rb | 82 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 86 insertions(+), 1 deletion(-)

diff --git a/test/models.rb b/test/models.rb
index cd606eed..7d490a4a 100644
--- a/test/models.rb
+++ b/test/models.rb
@@ -101,7 +101,10 @@ def self.create_test_case_data
   end
 
   def self.delete_test_case_data
-    objects = [Student, University, Program, Category, Address]
+    delete_all [Student, University, Program, Category, Address]
+  end
+
+  def self.delete_all(objects)
     objects.each do |obj|
       obj.where.include(obj.attributes).each do |i|
         i.delete
diff --git a/test/test_validators.rb b/test/test_validators.rb
index d81410a4..9be94798 100644
--- a/test/test_validators.rb
+++ b/test/test_validators.rb
@@ -25,6 +25,13 @@ class RangeTestModel < Goo::Base::Resource
   attribute :weight, enforce: [:float, :min_3, :max_5]
 end
 
+class SymmetricTestModel < Goo::Base::Resource
+  model :symmetric_test_model, name_with: :name
+  attribute :name, enforce: [:unique, :existence]
+  attribute :friend, enforce: [SymmetricTestModel, :symmetric]
+  attribute :friends, enforce: [SymmetricTestModel, :symmetric, :list]
+end
+
 
 
 class TestValidators < MiniTest::Unit::TestCase
@@ -39,6 +46,7 @@ def self.before_suite
 
   def self.after_suite
     GooTestData.delete_test_case_data
+    GooTestData.delete_all [SymmetricTestModel]
   end
 
 
@@ -181,4 +189,78 @@ def test_value_range_validator
 
   end
 
+  def test_symmetric_validator_no_list
+    p1 = SymmetricTestModel.new
+    p2 = SymmetricTestModel.new
+    p3 = SymmetricTestModel.new
+    p1.name = "p1"
+    p2.name = "p2"
+    p3.name = "p3"
+
+    p2.save
+    p3.save
+
+    p1.friend = p2
+
+    refute p1.valid?
+    assert p1.errors[:friend][:symmetric]
+
+    p3.friend = p1
+
+    refute p1.valid?
+
+    p2.friend = p1
+    p1.friend = p2
+
+    assert p1.valid?
+
+    p1.save
+
+    assert p2.valid?
+
+  end
+
+  def test_symmetric_validator_list
+    p1 = SymmetricTestModel.new
+    p2 = SymmetricTestModel.new
+    p3 = SymmetricTestModel.new
+    p4 = SymmetricTestModel.new
+    p1.name = "p1"
+    p2.name = "p2"
+    p3.name = "p3"
+    p4.name = "p4"
+
+    p2.save
+    p3.save
+    p4.save
+
+    p1.friends = [p2, p3]
+
+    refute p1.valid?
+    assert p1.errors[:friends][:symmetric]
+
+    p2.friends = [p1, p3, p4]
+    p3.friends = [p2]
+    p4.friends = [p2]
+
+    refute p1.valid?
+    refute p2.valid?
+
+
+    p3.friends = [p2, p1]
+
+    assert p1.valid?
+    p1.save
+
+    assert p3.valid?
+    p3.save
+
+
+    assert p2.valid?
+
+    p2.save
+
+    assert p4.valid?
+
+  end
 end

From 810771a1c550ba765baef6ee0b7065a67f32e749 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Tue, 28 Feb 2023 06:24:26 +0100
Subject: [PATCH 051/168] implement symmetric validator

---
 lib/goo/validators/enforce.rb                 |  2 +
 .../validators/implementations/existence.rb   |  5 ++-
 .../validators/implementations/symmetric.rb   | 44 +++++++++++++++++++
 3 files changed, 50 insertions(+), 1 deletion(-)
 create mode 100644 lib/goo/validators/implementations/symmetric.rb

diff --git a/lib/goo/validators/enforce.rb b/lib/goo/validators/enforce.rb
index 9283c070..70953645 100644
--- a/lib/goo/validators/enforce.rb
+++ b/lib/goo/validators/enforce.rb
@@ -39,6 +39,8 @@ def enforce(inst,attr,value)
               check Goo::Validators::DataType, inst, attr, value, opt, DateTime
             when :float, Float
               check Goo::Validators::DataType, inst, attr, value, opt, Float
+            when :symmetric
+              check Goo::Validators::Symmetric, inst, attr, value, opt
             when Proc
               call_proc(opt, inst, attr)
             when /^max_/, /^min_/
diff --git a/lib/goo/validators/implementations/existence.rb b/lib/goo/validators/implementations/existence.rb
index a551f35d..6e759154 100644
--- a/lib/goo/validators/implementations/existence.rb
+++ b/lib/goo/validators/implementations/existence.rb
@@ -8,9 +8,12 @@ class Existence < ValidatorBase
       error_message ->(obj) { "`#{@value}` value cannot be nil"}
 
       validity_check -> (obj) do
-        not (@value.nil? || self.class.empty?(@value) || self.class.empty_array?(@value))
+        not self.class.empty_value?(@value)
       end
 
+      def self.empty_value?(value)
+        value.nil? || self.empty?(value) || self.empty_array?(value)
+      end
       def self.empty?(value)
         empty_string?(value) || empty_to_s?(value)
       end
diff --git a/lib/goo/validators/implementations/symmetric.rb b/lib/goo/validators/implementations/symmetric.rb
new file mode 100644
index 00000000..c1c9055c
--- /dev/null
+++ b/lib/goo/validators/implementations/symmetric.rb
@@ -0,0 +1,44 @@
+module Goo
+  module Validators
+    class Symmetric < ValidatorBase
+      include Validator
+
+      key :symmetric
+
+      error_message ->(obj) {
+        "symmetric error"
+      }
+
+      validity_check -> (obj) do
+        return true if Existence.empty_value?(@value)
+
+        return Array(@value).select{|x| not self.class.symmetric?(@attr,x, @inst)}.empty?
+      end
+
+      def self.symmetric?(attr, value, source_object)
+        if self.respond_to?(attr, value)
+          target_values = self.attr_value(attr, value)
+          return target_values.any?{ |target_object| self.equivalent?(target_object, source_object)}
+        end
+
+        return false
+      end
+
+      def self.respond_to?(attr, object)
+        object && object.respond_to?(attr)
+      end
+
+      def self.attr_value(attr, object)
+        Array(object.send(attr))
+      end
+
+      def self.equivalent?(object1, object2)
+        if object1.respond_to?(:id) && object2.respond_to?(:id)
+          object1.id.eql?(object2.id)
+        else
+          object2 == object1
+        end
+      end
+    end
+  end
+end
\ No newline at end of file

From 0f20d7a25f7b8e7964120a1ebdc764f00d4c815b Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Tue, 28 Feb 2023 08:31:42 +0100
Subject: [PATCH 052/168] move re used methods to the parent class

---
 .../validators/implementations/data_type.rb   | 10 +++---
 .../validators/implementations/existence.rb   | 16 ---------
 .../validators/implementations/object_type.rb |  8 ++---
 .../validators/implementations/value_range.rb | 12 +++----
 lib/goo/validators/validator.rb               | 36 +++++++++++++++++++
 5 files changed, 51 insertions(+), 31 deletions(-)

diff --git a/lib/goo/validators/implementations/data_type.rb b/lib/goo/validators/implementations/data_type.rb
index 419ab37a..0ea65ab3 100644
--- a/lib/goo/validators/implementations/data_type.rb
+++ b/lib/goo/validators/implementations/data_type.rb
@@ -15,7 +15,7 @@ class DataType < ValidatorBase
       }
 
       validity_check -> (obj) do
-        self.class.enforce_type(@type, @value)
+        self.enforce_type(@type, @value)
       end
 
       def initialize(inst, attr, value, type)
@@ -25,7 +25,7 @@ def initialize(inst, attr, value, type)
 
 
 
-      def self.enforce_type(type, value)
+      def enforce_type(type, value)
         return true if value.nil?
 
         if type == :boolean
@@ -44,13 +44,13 @@ def self.enforce_type(type, value)
 
       end
 
-      def self.enforce_type_uri(value)
+      def enforce_type_uri(value)
         return true  if value.nil?
 
         value.is_a?(RDF::URI) && value.valid?
       end
 
-      def self.enforce_type_boolean(value)
+      def enforce_type_boolean(value)
         if value.kind_of? Array
           return value.select { |x| !is_a_boolean?(x) }.empty?
         else
@@ -58,7 +58,7 @@ def self.enforce_type_boolean(value)
         end
       end
 
-      def self.is_a_boolean?(value)
+      def is_a_boolean?(value)
         return (value.class == TrueClass) || (value.class == FalseClass)
       end
     end
diff --git a/lib/goo/validators/implementations/existence.rb b/lib/goo/validators/implementations/existence.rb
index 6e759154..fcf04d61 100644
--- a/lib/goo/validators/implementations/existence.rb
+++ b/lib/goo/validators/implementations/existence.rb
@@ -11,23 +11,7 @@ class Existence < ValidatorBase
         not self.class.empty_value?(@value)
       end
 
-      def self.empty_value?(value)
-        value.nil? || self.empty?(value) || self.empty_array?(value)
-      end
-      def self.empty?(value)
-        empty_string?(value) || empty_to_s?(value)
-      end
-      def self.empty_string?(string)
-        string.is_a?(String) && string.strip.empty?
-      end
 
-      def self.empty_to_s?(object)
-        object && object.to_s&.strip.empty?
-      end
-
-      def self.empty_array?(array)
-        array.is_a?(Array) && array && array.reject{|x|  x.nil? || empty?(x)}.empty?
-      end
     end
   end
 end
\ No newline at end of file
diff --git a/lib/goo/validators/implementations/object_type.rb b/lib/goo/validators/implementations/object_type.rb
index 41f0349c..3af97b41 100644
--- a/lib/goo/validators/implementations/object_type.rb
+++ b/lib/goo/validators/implementations/object_type.rb
@@ -16,12 +16,12 @@ class ObjectType < ValidatorBase
       validity_check -> (obj) do
         values = Array(@value)
 
-        unless values.select { |v| !self.class.is_a_model?(v, @model_range) }.empty?
+        unless values.select { |v| !self.is_a_model?(v, @model_range) }.empty?
           @error = :no_range
           return false
         end
 
-        unless values.select { |v| !self.class.persistent?(v) }.empty?
+        unless values.select { |v| !self.persistent?(v) }.empty?
           @error = :persistence
           return false
         end
@@ -34,11 +34,11 @@ def initialize(inst, attr, value, model_range)
         @model_range = model_range
       end
 
-      def self.is_a_model?(value, model_range)
+      def is_a_model?(value, model_range)
         value.is_a?(model_range) || (value.respond_to?(:klass) && value[:klass] == model_range)
       end
 
-      def self.persistent?(value)
+      def persistent?(value)
         value.respond_to?(:klass) || value.persistent?
       end
     end
diff --git a/lib/goo/validators/implementations/value_range.rb b/lib/goo/validators/implementations/value_range.rb
index 64e85341..71440bcf 100644
--- a/lib/goo/validators/implementations/value_range.rb
+++ b/lib/goo/validators/implementations/value_range.rb
@@ -6,7 +6,7 @@ class ValueRange < ValidatorBase
       keys [:min_, :max_]
 
       error_message ->(obj) {
-        value = self.class.value_length(@value)
+        value = self.value_length(@value)
         if @type == :min
           "#{@attr} value has length `#{value}` and the min length is `#{@range}`"
         else
@@ -15,27 +15,27 @@ class ValueRange < ValidatorBase
       }
 
       validity_check -> (obj) do
-        self.class.enforce_range_length(@type, @range, @value)
+        self.enforce_range_length(@type, @range, @value)
       end
 
       def initialize(inst, attr, value, type)
         super(inst, attr, value)
         @type = type.index("max_") ? :max : :min
-        @range = self.class.range(type)
+        @range = self.range(type)
       end
 
-      def self.enforce_range_length(type_range, range, value)
+      def enforce_range_length(type_range, range, value)
         return false if value.nil?
         value_length = self.value_length(value)
 
         (type_range.eql?(:min) && (value_length >= range)) || (type_range.eql?(:max) && (value_length <= range))
       end
 
-      def self.range(opt)
+      def range(opt)
         opt[4..opt.length].to_i
       end
 
-      def self.value_length(value)
+      def value_length(value)
         return 0 if value.nil?
 
         if value.is_a?(String) || value.is_a?(Array)
diff --git a/lib/goo/validators/validator.rb b/lib/goo/validators/validator.rb
index 3eae58c5..0655c0df 100644
--- a/lib/goo/validators/validator.rb
+++ b/lib/goo/validators/validator.rb
@@ -52,10 +52,46 @@ def error_message(message)
         def validator_settings
           @validator_settings ||= {}
         end
+
+        def ids
+          validator_settings[:id]
+        end
+
+        def equivalent_value?(object1, object2)
+          if object1.respond_to?(:id) && object2.respond_to?(:id)
+            object1.id.eql?(object2.id)
+          else
+            object2 == object1
+          end
+        end
+
+        def attr_value(attr, object)
+          Array(object.send(attr))
+        end
+
+        def empty_value?(value)
+          value.nil? || empty?(value) || empty_array?(value)
+        end
+        def empty?(value)
+          empty_string?(value) || empty_to_s?(value)
+        end
+        def empty_string?(string)
+          string.is_a?(String) && string.strip.empty?
+        end
+
+        def empty_to_s?(object)
+          object && object.to_s&.strip.empty?
+        end
+
+        def empty_array?(array)
+          array.is_a?(Array) && array && array.reject{|x|  x.nil? || empty?(x)}.empty?
+        end
       end
 
 
 
+
+
     end
   end
 end

From 9e4739c8cd177367df4d7e02d373a956db780e00 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Tue, 28 Feb 2023 08:33:19 +0100
Subject: [PATCH 053/168] update symmetric code and error message

---
 .../validators/implementations/symmetric.rb   | 27 ++++++-------------
 1 file changed, 8 insertions(+), 19 deletions(-)

diff --git a/lib/goo/validators/implementations/symmetric.rb b/lib/goo/validators/implementations/symmetric.rb
index c1c9055c..e9ceb3f4 100644
--- a/lib/goo/validators/implementations/symmetric.rb
+++ b/lib/goo/validators/implementations/symmetric.rb
@@ -6,39 +6,28 @@ class Symmetric < ValidatorBase
       key :symmetric
 
       error_message ->(obj) {
-        "symmetric error"
+        "`#{@attr}` must be symmetric"
       }
 
       validity_check -> (obj) do
-        return true if Existence.empty_value?(@value)
+        return true if self.class.empty_value?(@value)
 
-        return Array(@value).select{|x| not self.class.symmetric?(@attr,x, @inst)}.empty?
+        return Array(@value).select{|x| not symmetric?(@attr,x, @inst)}.empty?
       end
 
-      def self.symmetric?(attr, value, source_object)
-        if self.respond_to?(attr, value)
-          target_values = self.attr_value(attr, value)
-          return target_values.any?{ |target_object| self.equivalent?(target_object, source_object)}
+      def symmetric?(attr, value, source_object)
+        if respond_to?(attr, value)
+          target_values = self.class.attr_value(attr, value)
+          return target_values.any?{ |target_object| self.class.equivalent_value?(target_object, source_object)}
         end
 
         return false
       end
 
-      def self.respond_to?(attr, object)
+      def respond_to?(attr, object)
         object && object.respond_to?(attr)
       end
 
-      def self.attr_value(attr, object)
-        Array(object.send(attr))
-      end
-
-      def self.equivalent?(object1, object2)
-        if object1.respond_to?(:id) && object2.respond_to?(:id)
-          object1.id.eql?(object2.id)
-        else
-          object2 == object1
-        end
-      end
     end
   end
 end
\ No newline at end of file

From 05bb6cbafb727ea6fc1990c4440f5927f9ef9f76 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Tue, 28 Feb 2023 08:34:02 +0100
Subject: [PATCH 054/168] add distinct of validator tests

---
 test/test_validators.rb | 35 ++++++++++++++++++++++++++++++++++-
 1 file changed, 34 insertions(+), 1 deletion(-)

diff --git a/test/test_validators.rb b/test/test_validators.rb
index 9be94798..02b5ed8a 100644
--- a/test/test_validators.rb
+++ b/test/test_validators.rb
@@ -32,6 +32,13 @@ class SymmetricTestModel < Goo::Base::Resource
   attribute :friends, enforce: [SymmetricTestModel, :symmetric, :list]
 end
 
+class DistinctOfTestModel < Goo::Base::Resource
+  model :symmetric_test_model, name_with: :name
+  attribute :name, enforce: [:unique, :existence, :string]
+  attribute :last_name, enforce: [:distinct_of_name, :string]
+  attribute :names, enforce: [:list, :string]
+  attribute :last_names, enforce: [:list, :distinct_of_names, :string]
+end
 
 
 class TestValidators < MiniTest::Unit::TestCase
@@ -217,7 +224,7 @@ def test_symmetric_validator_no_list
     p1.save
 
     assert p2.valid?
-
+    GooTestData.delete_all [SymmetricTestModel]
   end
 
   def test_symmetric_validator_list
@@ -261,6 +268,32 @@ def test_symmetric_validator_list
     p2.save
 
     assert p4.valid?
+    GooTestData.delete_all [SymmetricTestModel]
+  end
+
+  def test_distinct_of_validator
+    p = DistinctOfTestModel.new
+    p.name = "p1"
+    p.last_name = "p1"
+    p.names = ["p1", "p2"]
+    p.last_names = ["p1", "p2"]
+
+
+    refute p.valid?
+
+    p.last_name = "last name"
+    p.last_names = ["last name 1", "last name 2"]
+
+    assert p.valid?
+
+    p.last_name = "last name"
+    p.last_names = ["last name 1", "p2"]
 
+    refute p.valid?
+
+    p.last_name = ""
+    p.last_names = []
+
+    assert p.valid?
   end
 end

From fb62419689644a3058c245a7511d61b31528106b Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Tue, 28 Feb 2023 08:34:23 +0100
Subject: [PATCH 055/168] implement distinct_of validator

---
 lib/goo/validators/enforce.rb                 |  2 ++
 .../validators/implementations/distinct_of.rb | 36 +++++++++++++++++++
 2 files changed, 38 insertions(+)
 create mode 100644 lib/goo/validators/implementations/distinct_of.rb

diff --git a/lib/goo/validators/enforce.rb b/lib/goo/validators/enforce.rb
index 70953645..bc0fd4c1 100644
--- a/lib/goo/validators/enforce.rb
+++ b/lib/goo/validators/enforce.rb
@@ -41,6 +41,8 @@ def enforce(inst,attr,value)
               check Goo::Validators::DataType, inst, attr, value, opt, Float
             when :symmetric
               check Goo::Validators::Symmetric, inst, attr, value, opt
+            when /^distinct_of_/
+              check Goo::Validators::DistinctOf, inst, attr, value, opt, opt
             when Proc
               call_proc(opt, inst, attr)
             when /^max_/, /^min_/
diff --git a/lib/goo/validators/implementations/distinct_of.rb b/lib/goo/validators/implementations/distinct_of.rb
new file mode 100644
index 00000000..820f5125
--- /dev/null
+++ b/lib/goo/validators/implementations/distinct_of.rb
@@ -0,0 +1,36 @@
+module Goo
+  module Validators
+    class DistinctOf < ValidatorBase
+      include Validator
+
+      key :distinct_of_
+
+      error_message ->(obj) { "`#{@attr}` must be distinct of `#{@property}`"}
+
+      validity_check -> (obj) do
+        return true if self.class.empty_value?(@value)
+
+        self.distinct?(@inst, @property, @value)
+      end
+
+      def initialize(inst, attr, value, key)
+        super(inst, attr, value)
+        @property = property(key)
+      end
+
+      def property(opt)
+        opt[self.class.ids.size..opt.length].to_sym
+      end
+
+      def distinct?(inst, property, value)
+        target_values = self.class.attr_value(property, inst)
+        current_values = Array(value)
+
+        !current_values.any?{ |x| self.find_any?(target_values, x)}
+      end
+      def find_any?(array, value)
+        array.any?{ |x| self.class.equivalent_value?(value, x)}
+      end
+    end
+  end
+end
\ No newline at end of file

From 5cd980d6a7014975a95d893fa32cdb5209f17859 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Tue, 28 Feb 2023 09:05:12 +0100
Subject: [PATCH 056/168] add superior_equal_to validator tests

---
 test/test_validators.rb | 27 ++++++++++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/test/test_validators.rb b/test/test_validators.rb
index 02b5ed8a..c0989180 100644
--- a/test/test_validators.rb
+++ b/test/test_validators.rb
@@ -33,13 +33,20 @@ class SymmetricTestModel < Goo::Base::Resource
 end
 
 class DistinctOfTestModel < Goo::Base::Resource
-  model :symmetric_test_model, name_with: :name
+  model :distinct_of_test_model, name_with: :name
   attribute :name, enforce: [:unique, :existence, :string]
   attribute :last_name, enforce: [:distinct_of_name, :string]
   attribute :names, enforce: [:list, :string]
   attribute :last_names, enforce: [:list, :distinct_of_names, :string]
 end
 
+class SuperiorToTestModel < Goo::Base::Resource
+  model :superior_to_test_model, name_with: :name
+  attribute :name, enforce: [:unique, :existence, :string]
+  attribute :birth_date, enforce: [:date_time]
+  attribute :death_date, enforce: [:superior_equal_to_birth_date, :date_time]
+end
+
 
 class TestValidators < MiniTest::Unit::TestCase
 
@@ -296,4 +303,22 @@ def test_distinct_of_validator
 
     assert p.valid?
   end
+
+  def test_superior_equal_to_validator
+    p = SuperiorToTestModel.new
+    p.name = "p"
+    p.birth_date = DateTime.parse('1998-12-02')
+    p.death_date = DateTime.parse('1995-12-02')
+
+    refute p.valid?
+    assert p.errors[:death_date][:superior_equal_to_birth_date]
+
+    p.death_date = DateTime.parse('2023-12-02')
+
+    assert p.valid?
+
+    p.birth_date = nil
+
+    assert p.valid?
+  end
 end

From 0d17a84da13c304ab20993d953186f93fa2b3423 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Tue, 28 Feb 2023 09:06:02 +0100
Subject: [PATCH 057/168] extract property method to ValidatorBase class

---
 lib/goo/validators/implementations/distinct_of.rb | 4 +---
 lib/goo/validators/validator.rb                   | 6 +++++-
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/lib/goo/validators/implementations/distinct_of.rb b/lib/goo/validators/implementations/distinct_of.rb
index 820f5125..289a9de2 100644
--- a/lib/goo/validators/implementations/distinct_of.rb
+++ b/lib/goo/validators/implementations/distinct_of.rb
@@ -18,9 +18,7 @@ def initialize(inst, attr, value, key)
         @property = property(key)
       end
 
-      def property(opt)
-        opt[self.class.ids.size..opt.length].to_sym
-      end
+
 
       def distinct?(inst, property, value)
         target_values = self.class.attr_value(property, inst)
diff --git a/lib/goo/validators/validator.rb b/lib/goo/validators/validator.rb
index 0655c0df..fcb056ea 100644
--- a/lib/goo/validators/validator.rb
+++ b/lib/goo/validators/validator.rb
@@ -54,7 +54,11 @@ def validator_settings
         end
 
         def ids
-          validator_settings[:id]
+          Array(validator_settings[:id])
+        end
+
+        def property(key)
+          key[ids.first.size..key.size].to_sym
         end
 
         def equivalent_value?(object1, object2)

From 5740a6b1c5da58decef197261b5532bc4b23fd72 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Tue, 28 Feb 2023 09:06:29 +0100
Subject: [PATCH 058/168] implement superior_equal_to validator

---
 lib/goo/validators/enforce.rb                 |  2 ++
 .../implementations/superior_equal_to.rb      | 26 +++++++++++++++++++
 2 files changed, 28 insertions(+)
 create mode 100644 lib/goo/validators/implementations/superior_equal_to.rb

diff --git a/lib/goo/validators/enforce.rb b/lib/goo/validators/enforce.rb
index bc0fd4c1..7c4b8345 100644
--- a/lib/goo/validators/enforce.rb
+++ b/lib/goo/validators/enforce.rb
@@ -43,6 +43,8 @@ def enforce(inst,attr,value)
               check Goo::Validators::Symmetric, inst, attr, value, opt
             when /^distinct_of_/
               check Goo::Validators::DistinctOf, inst, attr, value, opt, opt
+            when /^superior_equal_to_/
+              check Goo::Validators::SuperiorEqualTo, inst, attr, value, opt, opt
             when Proc
               call_proc(opt, inst, attr)
             when /^max_/, /^min_/
diff --git a/lib/goo/validators/implementations/superior_equal_to.rb b/lib/goo/validators/implementations/superior_equal_to.rb
new file mode 100644
index 00000000..91508f30
--- /dev/null
+++ b/lib/goo/validators/implementations/superior_equal_to.rb
@@ -0,0 +1,26 @@
+module Goo
+  module Validators
+    class SuperiorEqualTo < ValidatorBase
+      include Validator
+
+      key :superior_equal_to_
+
+      error_message ->(obj) {
+        "`#{@attr}` must be superior or equal to `#{@property}`"
+      }
+
+      validity_check -> (obj) do
+        target_values = self.class.attr_value(@property, @inst)
+
+        return true if target_values.empty?
+
+        return @value >= target_values.first
+      end
+
+      def initialize(inst, attr, value, key)
+        super(inst, attr, value)
+        @property = self.class.property(key)
+      end
+    end
+  end
+end

From 9707004cb2a55d60449dde6221fef4e44514f765 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Tue, 28 Feb 2023 10:08:22 +0100
Subject: [PATCH 059/168] add inverse of validator tests

---
 test/test_validators.rb | 66 ++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 65 insertions(+), 1 deletion(-)

diff --git a/test/test_validators.rb b/test/test_validators.rb
index c0989180..c01a543b 100644
--- a/test/test_validators.rb
+++ b/test/test_validators.rb
@@ -47,6 +47,15 @@ class SuperiorToTestModel < Goo::Base::Resource
   attribute :death_date, enforce: [:superior_equal_to_birth_date, :date_time]
 end
 
+class InverseOfTestModel < Goo::Base::Resource
+  model :inverse_test_model_one, name_with: :name
+  attribute :name, enforce: [:unique, :existence, :string]
+  attribute :state, enforce: [InverseOfTestModel]
+  attribute :city, enforce: [:inverse_of_state, InverseOfTestModel]
+  attribute :states, enforce: [InverseOfTestModel, :list]
+  attribute :cities, enforce: [:inverse_of_states, InverseOfTestModel, :list]
+end
+
 
 class TestValidators < MiniTest::Unit::TestCase
 
@@ -60,7 +69,7 @@ def self.before_suite
 
   def self.after_suite
     GooTestData.delete_test_case_data
-    GooTestData.delete_all [SymmetricTestModel]
+    GooTestData.delete_all [SymmetricTestModel, InverseOfTestModel]
   end
 
 
@@ -321,4 +330,59 @@ def test_superior_equal_to_validator
 
     assert p.valid?
   end
+
+  def test_inverse_of_validator_no_list
+    GooTestData.delete_all [InverseOfTestModel]
+    p1 = InverseOfTestModel.new
+    p2 = InverseOfTestModel.new
+
+    p1.name = 'p1'
+    p2.name = 'p2'
+
+
+    p2.save
+
+    p1.city = p2
+
+    refute p1.valid?
+    assert p1.errors[:city][:inverse_of_state]
+
+
+    p2.state = p1
+
+    assert p1.valid?
+
+  end
+
+  def test_inverse_of_validator_list
+    GooTestData.delete_all [InverseOfTestModel]
+    p1 = InverseOfTestModel.new
+    p2 = InverseOfTestModel.new
+    p3 = InverseOfTestModel.new
+    p4 = InverseOfTestModel.new
+
+    p1.name = 'p1'
+    p2.name = 'p2'
+    p3.name = 'p3'
+    p4.name = 'p4'
+
+    p2.save
+    p3.save
+
+    p1.cities = [p2,p3]
+
+    refute p1.valid?
+    assert p1.errors[:cities][:inverse_of_states]
+
+    p2.states = [p1, p4]
+    p3.states = [p2, p4]
+
+    refute p1.valid?
+    assert p1.errors[:cities][:inverse_of_states]
+
+    p3.states = [p2, p4, p1]
+
+    assert p1.valid?
+
+  end
 end

From 9ea0be87010b734d363d0c891ed73eedae518d97 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Tue, 28 Feb 2023 10:09:19 +0100
Subject: [PATCH 060/168] implement inverse_of validator

---
 lib/goo/validators/enforce.rb                 |  2 ++
 .../validators/implementations/inverse_of.rb  | 35 +++++++++++++++++++
 lib/goo/validators/validator.rb               |  5 +++
 3 files changed, 42 insertions(+)
 create mode 100644 lib/goo/validators/implementations/inverse_of.rb

diff --git a/lib/goo/validators/enforce.rb b/lib/goo/validators/enforce.rb
index 7c4b8345..e9efe249 100644
--- a/lib/goo/validators/enforce.rb
+++ b/lib/goo/validators/enforce.rb
@@ -45,6 +45,8 @@ def enforce(inst,attr,value)
               check Goo::Validators::DistinctOf, inst, attr, value, opt, opt
             when /^superior_equal_to_/
               check Goo::Validators::SuperiorEqualTo, inst, attr, value, opt, opt
+            when /^inverse_of_/
+              check Goo::Validators::InverseOf, inst, attr, value, opt, opt
             when Proc
               call_proc(opt, inst, attr)
             when /^max_/, /^min_/
diff --git a/lib/goo/validators/implementations/inverse_of.rb b/lib/goo/validators/implementations/inverse_of.rb
new file mode 100644
index 00000000..60518af3
--- /dev/null
+++ b/lib/goo/validators/implementations/inverse_of.rb
@@ -0,0 +1,35 @@
+module Goo
+  module Validators
+    class InverseOf < ValidatorBase
+      include Validator
+
+      key :inverse_of_
+
+      error_message ->(obj) {
+        "`#{@attr}` must be the inverse of ``#{@property}``"
+      }
+
+      validity_check -> (obj) do
+        return true if self.class.empty_value?(@value)
+
+        return Array(@value).select{|x| not inverse?(@property,x, @inst)}.empty?
+      end
+
+      def initialize(inst, attr, value, key)
+        super(inst, attr, value)
+        @property = self.class.property(key)
+      end
+
+      def inverse?(attr, value, source_object)
+        if self.class.respond_to?(attr, value)
+          target_values = self.class.attr_value(attr, value)
+          return target_values.any?{ |target_object| self.class.equivalent_value?(target_object, source_object)}
+        end
+
+        false
+      end
+
+
+    end
+  end
+end
\ No newline at end of file
diff --git a/lib/goo/validators/validator.rb b/lib/goo/validators/validator.rb
index fcb056ea..c133c33e 100644
--- a/lib/goo/validators/validator.rb
+++ b/lib/goo/validators/validator.rb
@@ -61,6 +61,11 @@ def property(key)
           key[ids.first.size..key.size].to_sym
         end
 
+        def respond_to?(attr, object)
+          object && object.respond_to?(attr)
+        end
+
+
         def equivalent_value?(object1, object2)
           if object1.respond_to?(:id) && object2.respond_to?(:id)
             object1.id.eql?(object2.id)

From ba8bf8af538c464f3f65c5ec702fb8d830ac4b4c Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Tue, 28 Feb 2023 10:24:53 +0100
Subject: [PATCH 061/168] use the class method property in distinct of

---
 lib/goo/validators/implementations/distinct_of.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/goo/validators/implementations/distinct_of.rb b/lib/goo/validators/implementations/distinct_of.rb
index 289a9de2..2e93313b 100644
--- a/lib/goo/validators/implementations/distinct_of.rb
+++ b/lib/goo/validators/implementations/distinct_of.rb
@@ -15,7 +15,7 @@ class DistinctOf < ValidatorBase
 
       def initialize(inst, attr, value, key)
         super(inst, attr, value)
-        @property = property(key)
+        @property = self.class.property(key)
       end
 
 

From f061e51612f07555680907a7809faa03f3566c1e Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Tue, 28 Feb 2023 20:48:16 +0100
Subject: [PATCH 062/168] add proc validator tests

---
 test/test_validators.rb | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/test/test_validators.rb b/test/test_validators.rb
index c01a543b..8795fccf 100644
--- a/test/test_validators.rb
+++ b/test/test_validators.rb
@@ -57,6 +57,29 @@ class InverseOfTestModel < Goo::Base::Resource
 end
 
 
+class ProcValidatorsTestModel < Goo::Base::Resource
+  model :proc_validator_test_model, name_with: :name
+  attribute :name, enforce: [:unique, :equal_to_test]
+  attribute :last_name, enforce: [:unique, ->(inst, attr) {  equal_to_test_2(inst, attr)}]
+
+
+  def self.equal_to_test_2(inst, attr)
+    value = inst.send(attr)
+
+    return nil if value && value.eql?('test 2')
+
+    [:equal_to_test_2, "#{attr} need to be equal to `test 2`"]
+  end
+
+  def equal_to_test(inst, attr)
+    value = inst.send(attr)
+
+    return nil if  value && value.eql?('test')
+
+    [:equal_to_test, "#{attr} need to be equal to `test`"]
+  end
+end
+
 class TestValidators < MiniTest::Unit::TestCase
 
   def self.before_suite
@@ -385,4 +408,20 @@ def test_inverse_of_validator_list
     assert p1.valid?
 
   end
+
+
+  def test_proc_validators
+    p = ProcValidatorsTestModel.new
+    p.name = "hi"
+    p.last_name = "hi"
+
+    refute p.valid?
+    assert p.errors[:name][:equal_to_test]
+    assert p.errors[:last_name][:equal_to_test_2]
+
+    p.name = "test"
+    p.last_name = "test 2"
+
+    assert p.valid?
+  end
 end

From b7cccab30510c3b4e428be66996dbbf84b52c892 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Tue, 28 Feb 2023 20:50:01 +0100
Subject: [PATCH 063/168] add instance proc validators

---
 lib/goo/validators/enforce.rb | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/lib/goo/validators/enforce.rb b/lib/goo/validators/enforce.rb
index e9efe249..8860effc 100644
--- a/lib/goo/validators/enforce.rb
+++ b/lib/goo/validators/enforce.rb
@@ -53,8 +53,11 @@ def enforce(inst,attr,value)
               type = opt.to_s.index("max_") ? :max : :min
               check Goo::Validators::ValueRange, inst, attr, value, type, opt.to_s
             else
-              model_range = object_type(opt)
-              check_object_type inst, attr, value, model_range
+              if object_type?(opt)
+                check_object_type inst, attr, value, opt
+              elsif instance_proc?(inst, opt)
+                call_proc(inst.method(opt), inst, attr)
+              end
             end
           end
 
@@ -67,8 +70,16 @@ def object_type(opt)
           opt.respond_to?(:shape_attribute) ? opt : Goo.model_by_name(opt)
         end
 
-        def check_object_type(inst, attr, value, model_range)
+        def object_type?(opt)
+          opt.respond_to?(:shape_attribute) ? opt : Goo.model_by_name(opt)
+        end
+
+        def instance_proc?(inst, opt)
+          inst.respond_to? opt
+        end
 
+        def check_object_type(inst, attr, value, opt)
+          model_range = object_type(opt)
           if model_range && !value.nil?
             check Goo::Validators::ObjectType, inst, attr, value, model_range.model_name, model_range
           end
@@ -82,9 +93,9 @@ def enforce_by_attribute(model, attr)
            model.model_settings[:attributes][attr][:enforce]
         end
 
-        def call_proc(opt,inst, attr)
+        def call_proc(proc,inst, attr)
           # This should return an array like [:name_of_error1, "Error message 1", :name_of_error2, "Error message 2"]
-          errors = opt.call(inst, attr)
+          errors = proc.call(inst, attr) || []
           errors.each_slice(2) do |e|
             next if e.nil? || e.compact.empty?
             add_error(e[0].to_sym, e[1])

From 73def7491f36ac49e945fb3480a9d20893b56f52 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Tue, 28 Feb 2023 21:11:23 +0100
Subject: [PATCH 064/168] fix call_proc validator to test if the returned
 values are correct

---
 lib/goo/validators/enforce.rb | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/lib/goo/validators/enforce.rb b/lib/goo/validators/enforce.rb
index 8860effc..1a53eaf0 100644
--- a/lib/goo/validators/enforce.rb
+++ b/lib/goo/validators/enforce.rb
@@ -75,7 +75,7 @@ def object_type?(opt)
         end
 
         def instance_proc?(inst, opt)
-          inst.respond_to? opt
+          opt && (opt.is_a?(Symbol) || opt.is_a?(String)) && inst.respond_to?(opt)
         end
 
         def check_object_type(inst, attr, value, opt)
@@ -95,7 +95,10 @@ def enforce_by_attribute(model, attr)
 
         def call_proc(proc,inst, attr)
           # This should return an array like [:name_of_error1, "Error message 1", :name_of_error2, "Error message 2"]
-          errors = proc.call(inst, attr) || []
+          errors = proc.call(inst, attr)
+
+          return unless !errors.nil? && errors.is_a?(Array)
+
           errors.each_slice(2) do |e|
             next if e.nil? || e.compact.empty?
             add_error(e[0].to_sym, e[1])

From fe85a2bbc38af344a5088eb913288c5b93a513d1 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Wed, 1 Mar 2023 00:22:11 +0100
Subject: [PATCH 065/168] add model_with_yaml_scheme test

---
 test/data/yaml_scheme_model_test.yml | 11 ++++++++
 test/test_dsl_settings.rb            | 38 ++++++++++++++++++++++++++++
 2 files changed, 49 insertions(+)
 create mode 100644 test/data/yaml_scheme_model_test.yml

diff --git a/test/data/yaml_scheme_model_test.yml b/test/data/yaml_scheme_model_test.yml
new file mode 100644
index 00000000..fd8c4921
--- /dev/null
+++ b/test/data/yaml_scheme_model_test.yml
@@ -0,0 +1,11 @@
+name:
+  label: 'Name'
+  description: 'Person name'
+  equivalents: ['test:name' , 'test2:name', 'test3:person_name']
+  help: 'Put the person name as string'
+  example: 'John'
+nationality:
+  label: 'Person nationality'
+  enforcedValues: {'fr': 'france', 'us': 'USA'}
+
+
diff --git a/test/test_dsl_settings.rb b/test/test_dsl_settings.rb
index a4de17e2..9a8f03df 100644
--- a/test/test_dsl_settings.rb
+++ b/test/test_dsl_settings.rb
@@ -48,6 +48,22 @@ def initialize(attributes = {})
   end
 end
 
+
+class YamlSchemeModelTest < Goo::Base::Resource
+  model :yaml_scheme_model_test, name_with: :name, scheme: 'test/data/yaml_scheme_model_test.yml'
+  attribute :name, enforce: [ :existence, :string, :unique]
+  attribute :last_name, enforce: [ :existence, :string, :unique]
+  attribute :birth_date, enforce: [ :existence, :date_time ]
+  attribute :nationality, enforce: [ :existence, :string ]
+  attribute :created, enforce: [ DateTime ],
+            default: lambda { |record| DateTime.now },
+            namespace: :omv
+  attribute :friends, enforce: [ :existence , PersonModel]
+  attribute :status, enforce: [ :existence, :status ],
+            default: lambda { |record| StatusModel.find("single") }
+end
+
+
 class TestDSLSeeting < MiniTest::Unit::TestCase
   def initialize(*args)
     super(*args)
@@ -171,4 +187,26 @@ def _test_attributes_enforce(model)
     #there are assigned objects that are not saved
     assert !person.valid?
   end
+
+  def test_model_with_yaml_scheme
+
+    settings = YamlSchemeModelTest.model_settings
+    attributes_settings = settings[:attributes]
+
+
+    assert_equal "test/data/yaml_scheme_model_test.yml", settings[:scheme]
+
+    assert_equal 'Name', attributes_settings[:name][:label]
+    assert_equal 'Person name', attributes_settings[:name][:description]
+    assert_equal %w[test:name test2:name test3:person_name], attributes_settings[:name][:equivalents]
+    assert_equal 'Put the person name as string', attributes_settings[:name][:help]
+    assert_equal 'John', attributes_settings[:name][:example]
+
+
+    assert_equal 'Person nationality', attributes_settings[:nationality][:label]
+    hash = {fr: 'france', us: 'USA'}
+    assert_equal hash, attributes_settings[:nationality][:enforcedValues]
+
+  end
+
 end

From 4255a098890dd8f391cb520ff96c533da37022ae Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Wed, 1 Mar 2023 00:23:05 +0100
Subject: [PATCH 066/168] implement YAMLScheme module

---
 lib/goo/base/settings/yaml_settings.rb | 45 ++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)
 create mode 100644 lib/goo/base/settings/yaml_settings.rb

diff --git a/lib/goo/base/settings/yaml_settings.rb b/lib/goo/base/settings/yaml_settings.rb
new file mode 100644
index 00000000..8a931b3a
--- /dev/null
+++ b/lib/goo/base/settings/yaml_settings.rb
@@ -0,0 +1,45 @@
+require 'yaml'
+
+module Goo
+  module Base
+    module Settings
+      module YAMLScheme
+        attr_reader :yaml_settings
+
+        def init_yaml_scheme_settings
+          scheme_file_path = @model_settings[:scheme]
+          @yaml_settings = read_yaml_settings_file(scheme_file_path)
+        end
+
+        def attribute_yaml_settings(attr)
+
+          return {} if yaml_settings.nil?
+
+          yaml_settings[attr.to_sym]
+        end
+
+
+
+        private
+
+        def load_yaml_scheme_options(attr)
+          settings = attribute_settings(attr)
+          yaml_settings = attribute_yaml_settings(attr)
+          settings.merge! yaml_settings unless yaml_settings.nil? || yaml_settings.empty?
+        end
+
+        def read_yaml_settings_file(scheme_file_path)
+          return  if scheme_file_path.nil?
+
+          yaml_contents = File.read(scheme_file_path) rescue return
+
+          YAML.safe_load(yaml_contents, symbolize_names: true)
+        end
+      end
+    end
+  end
+end
+
+
+
+

From 0d467ad4ed923e3b06a74b728413b76db4a80ffc Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Wed, 1 Mar 2023 00:23:32 +0100
Subject: [PATCH 067/168] use YAMLScheme module in Settings module

---
 lib/goo/base/settings/settings.rb | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/lib/goo/base/settings/settings.rb b/lib/goo/base/settings/settings.rb
index e1937396..3d343d5d 100644
--- a/lib/goo/base/settings/settings.rb
+++ b/lib/goo/base/settings/settings.rb
@@ -1,4 +1,5 @@
 require 'active_support/core_ext/string'
+require_relative 'yaml_settings'
 
 module Goo
   module Base
@@ -12,8 +13,10 @@ module ClassMethods
         attr_reader :model_name
         attr_reader :attribute_uris
 
+        include YAMLScheme
+
         def default_model_options
-          return {}
+           {}
         end
 
         def model(*args)
@@ -34,7 +37,9 @@ def model(*args)
 
           @model_settings = default_model_options.merge(options || {})
 
-          unless options.include?:name_with
+          init_yaml_scheme_settings
+
+          unless options.include? :name_with
             raise ArgumentError, "The model `#{model_name}` definition should include the :name_with option"
           end
           Goo.add_model(@model_name,self)
@@ -192,6 +197,7 @@ def attribute(*args)
           set_no_list_by_default(options)
 
           @model_settings[:attributes][attr_name] = options
+          load_yaml_scheme_options(attr_name)
           shape_attribute(attr_name)
           namespace = attribute_namespace(attr_name)
           namespace = namespace || @model_settings[:namespace]

From a903826a9d6893a5ff5c84972b941966c35b9258 Mon Sep 17 00:00:00 2001
From: HADDAD Zineddine <hz_haddad@esi.dz>
Date: Wed, 1 Mar 2023 14:02:28 +0100
Subject: [PATCH 068/168] use platform lang and code refacto

---
 lib/goo/base/resource.rb                      |   3 +-
 lib/goo/sparql/mixins/solution_lang_filter.rb |  48 ++--
 lib/goo/sparql/solutions_mapper.rb            | 208 ++++++++++--------
 3 files changed, 143 insertions(+), 116 deletions(-)

diff --git a/lib/goo/base/resource.rb b/lib/goo/base/resource.rb
index 13c4b61a..e6981c0b 100644
--- a/lib/goo/base/resource.rb
+++ b/lib/goo/base/resource.rb
@@ -134,7 +134,8 @@ def missing_load_attributes
 
       def unmapped_set(attribute,value)
         @unmapped ||= {}
-        (@unmapped[attribute] ||= Set.new) << value
+        @unmapped[attribute] ||= Set.new 
+        @unmapped[attribute] << value unless value.nil?
       end
 
       def unmmaped_to_array
diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index b5254786..eb9e754c 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -15,26 +15,36 @@ def main_lang_filter(id, attr, value)
           [index, value]
         end
 
-        def fill_models_with_other_languages(models_by_id, list_attributes)
-          @other_languages_values.each  do |id, languages_values|
-            languages_values.each do |attr, index_values|
-              model_attribute_val = models_by_id[id].instance_variable_get("@#{attr.to_s}")
-              values = languages_values_to_set(index_values, model_attribute_val)
-              m = models_by_id[id]
-              value = nil
-              is_struct = m.respond_to?(:klass)
-              if !values.nil? && list_attributes.include?(attr)
-                value = values || []
-
-              elsif !values.nil?
-                value = values.first || nil
-              end
+        def find_model_objects_by_lang(objects_by_lang, lang, model_id, predicate)
+          objects_by_lang[lang]&.find { |obj| obj[:id].eql?(model_id) && obj[:predicate].eql?(predicate) }
+        end
+
+        def fill_models_with_other_languages(models_by_id, objects_by_lang, list_attributes, attributes,klass)
+      
+          other_platform_languages = [:EN, :FR]
+
+          unless other_platform_languages.empty?
+            
+            models_by_id&.each do |id, model|
+              
+              attributes&.each do |attr|
+                
+                other_platform_languages.each do |lang|
+                  
+                  model_attribute_val = model.instance_variable_get("@#{attr}")
+                  model_objects_by_lang = find_model_objects_by_lang(objects_by_lang, lang, id, attr) || []
+
+                  next if model_objects_by_lang.empty?
 
-              if value
-                if is_struct
-                  m[attr] = value
-                else
-                  m.send("#{attr}=", value, on_load: true)
+                  if list_attributes.include?(attr)
+                    model_attribute_val ||= []
+                    if model_attribute_val.empty?
+                      model.send("#{attr}=", model_attribute_val + model_objects_by_lang[:objects], on_load: true)
+                    end
+                  elsif !model_attribute_val
+                    model.send("#{attr}=", model_objects_by_lang[:objects][0] , on_load: true)
+                  end
+                  
                 end
               end
             end
diff --git a/lib/goo/sparql/solutions_mapper.rb b/lib/goo/sparql/solutions_mapper.rb
index 7416a78c..e6157fcf 100644
--- a/lib/goo/sparql/solutions_mapper.rb
+++ b/lib/goo/sparql/solutions_mapper.rb
@@ -1,7 +1,6 @@
 module Goo
   module SPARQL
     class SolutionMapper
-
       BNODES_TUPLES = Struct.new(:id, :attribute)
 
       def initialize(aggregate_projections, bnode_extraction, embed_struct,
@@ -19,18 +18,16 @@ def initialize(aggregate_projections, bnode_extraction, embed_struct,
         @variables = variables
         @ids = ids
         @klass = options[:klass]
-        @klass = options[:klass]
         @read_only = options[:read_only]
         @incl = options[:include]
         @count = options[:count]
         @collection = options[:collection]
+        @objects_by_lang = {}
       end
 
-
-
       def map_each_solutions(select)
-
         found = Set.new
+        fill_models_with_platform_languages = false
         objects_new = {}
         list_attributes = Set.new(@klass.attributes(:list))
         all_attributes = Set.new(@klass.attributes(:all))
@@ -66,34 +63,45 @@ def map_each_solutions(select)
 
           object = sol[:attributeObject]
 
-          #bnodes
+          # bnodes
           if bnode_id?(object, predicate)
             objects_new = bnode_id_tuple(id, object, objects_new, predicate)
             next
           end
 
-          lang = object_language(object)
+          lang = object_language(object) # if lang is nil, it means that the object is not a literal
+
+          requested_lang = nil
+
+          if requested_lang.nil?
+            requested_lang = :ES # Goo.main_languages[0] || :EN
+            fill_models_with_platform_languages = true
+          end
 
           objects, objects_new = get_value_object(id, objects_new, object, list_attributes, predicate)
-          add_object_to_model(id, objects, predicate, lang, :EN)
+          add_object_to_model(id, objects, object, predicate, lang, requested_lang)
+        end
+     
+        if fill_models_with_platform_languages
+          @lang_filter.fill_models_with_other_languages(@models_by_id, @objects_by_lang, list_attributes, @incl, @klass)
         end
-        # @lang_filter.fill_models_with_other_languages(@models_by_id, list_attributes)
+
         init_unloaded_attributes(found, list_attributes)
 
         return @models_by_id if @bnode_extraction
 
         model_set_collection_attributes(@models_by_id, objects_new)
 
-        #remove from models_by_id elements that were not touched
-        @models_by_id.select! { |k, m| found.include?(k) }
+        # remove from models_by_id elements that were not touched
+        @models_by_id.select! { |k, _m| found.include?(k) }
 
         models_set_all_persistent(@models_by_id) unless @read_only
 
-        #next level of embed attributes
+        # next level of embed attributes
         include_embed_attributes(@incl_embed, objects_new) if @incl_embed && !@incl_embed.empty?
 
-        #bnodes
-        blank_nodes = objects_new.select { |id, obj| id.is_a?(RDF::Node) && id.anonymous? }
+        # bnodes
+        blank_nodes = objects_new.select { |id, _obj| id.is_a?(RDF::Node) && id.anonymous? }
         include_bnodes(blank_nodes, @models_by_id) unless blank_nodes.empty?
 
         models_unmapped_to_array(@models_by_id) if @unmapped
@@ -103,8 +111,8 @@ def map_each_solutions(select)
 
       private
 
-      def get_object_language(id, object, predicate)
-        @lang_filter.main_lang_filter id, predicate, object
+      def get_object_language(id, new_value, predicate)
+        @lang_filter.main_lang_filter id, predicate, new_value
       end
 
       def init_unloaded_attributes(found, list_attributes)
@@ -134,10 +142,10 @@ def init_unloaded_attributes(found, list_attributes)
       end
 
       def get_value_object(id, objects_new, object, list_attributes, predicate)
-        object = object.object if object && !(object.is_a? RDF::URI)
+        object = object.object if object.is_a?(RDF::Literal)
         range_for_v = @klass.range(predicate)
-        #binding.pry if v.eql?(:enrolled)
-        #dependent model creation
+        # binding.pry if v.eql?(:enrolled)
+        # dependent model creation
 
         if object.is_a?(RDF::URI) && (predicate != :id) && !range_for_v.nil?
           if objects_new.include?(object)
@@ -147,7 +155,7 @@ def get_value_object(id, objects_new, object, list_attributes, predicate)
             object, objects_new = if !@read_only
                                     preloaded_or_new_object(object, objects_new, pre_val, predicate)
                                   else
-                                    #depedent read only
+                                    # depedent read only
                                     preloaded_or_new_struct(object, objects_new, pre_val, predicate)
                                   end
           else
@@ -156,20 +164,15 @@ def get_value_object(id, objects_new, object, list_attributes, predicate)
         end
 
         if list_attributes.include?(predicate)
-          # To handle attr that are lists
-          pre = if @klass_struct
-                  @models_by_id[id][predicate]
-                else
-                  @models_by_id[id].instance_variable_get("@#{predicate}")
-                end
-          if object.nil? && pre.nil?
-            object = []
-          elsif object.nil? && !pre.nil?
-            object = pre
-          elsif object
-            object = !pre ? [object] : (pre.dup << object)
-            object.uniq!
-          end
+          pre = @klass_struct ? @models_by_id[id][predicate] : @models_by_id[id].instance_variable_get("@#{predicate}")
+
+          object = []  if object.nil? && pre.nil?
+
+          object = pre if object.nil? && !pre.nil?
+
+          object = pre.nil? ? [object] : (pre.dup << object)
+          object.uniq
+
         end
         [object, objects_new]
       end
@@ -178,21 +181,39 @@ def object_language(new_value)
         new_value.language || :no_lang if new_value.is_a?(RDF::Literal)
       end
 
-      def add_object_to_model(id, objects, predicate, language, requested_lang = nil)
+      def language_match?(language, requested_lang = nil)
+        !language.nil? && (language.eql?(requested_lang) || language.eql?(:no_lang) || requested_lang.nil?)
+      end
+
+      def add_object_to_model(id, objects, current_obj, predicate, language, requested_lang = nil)
         if @models_by_id[id].respond_to?(:klass)
           @models_by_id[id][predicate] = objects unless objects.nil? && !@models_by_id[id][predicate].nil?
         elsif !@models_by_id[id].class.handler?(predicate) &&
               !(objects.nil? && !@models_by_id[id].instance_variable_get("@#{predicate}").nil?) &&
               predicate != :id
 
-            if language.nil? 
-              @models_by_id[id].send("#{predicate}=", objects, on_load: true)
-            else 
-              if language.eql?(requested_lang) || language.eql?(:no_lang) || requested_lang.nil?
-                @models_by_id[id].send("#{predicate}=", objects, on_load: true)
-              end
-            end
-          
+          if language.nil? # the object is a non-literal
+            return @models_by_id[id].send("#{predicate}=", objects, on_load: true)
+          end
+
+          if language_match?(language, requested_lang) # the object is a literal and the language matches
+            return @models_by_id[id].send("#{predicate}=", objects, on_load: true)
+          end
+
+
+          # the object is a literal and the language does not match , so we store it in a hash
+          @objects_by_lang[language] ||= []
+          item = @objects_by_lang[language].find { |obj| obj[:id] == id && obj[:predicate] == predicate }
+         
+          if item
+            # If an item with the matching id exists, update its attributes
+            item[:objects] << current_obj.object
+            item[:predicate] = predicate
+          else
+            # If an item with the matching id does not exist, add the new item to the array
+            @objects_by_lang[language] << { id: id, objects: [current_obj.object], predicate: predicate }
+          end
+        
         end
       end
 
@@ -222,7 +243,7 @@ def preloaded_or_new_struct(object, objects_new, pre_val, predicate)
       def preloaded_value(id, predicate)
         if !@read_only
           @models_by_id[id].instance_variable_get("@#{predicate}")
-          
+
         else
           @models_by_id[id][predicate]
         end
@@ -239,9 +260,7 @@ def bnode_id?(object, predicate)
 
       def bnode_id_tuple(id, object, objects_new, predicate)
         range = @klass.range(predicate)
-        if range.respond_to?(:new)
-          objects_new[object] = BNODES_TUPLES.new(id, predicate)
-        end
+        objects_new[object] = BNODES_TUPLES.new(id, predicate) if range.respond_to?(:new)
         objects_new
       end
 
@@ -255,11 +274,13 @@ def create_model(id)
         @models_by_id[id] = create_class_model(id, @klass, @klass_struct) unless @models_by_id.include?(id)
       end
 
-      def model_set_unmapped(id, predicate, value)
+      def model_set_unmapped(id, predicate, value, _requested_lang = nil)
+        value = nil if value.is_a?(RDF::Literal) && !language_match?(value.language, nil)
 
-        if @models_by_id[id].respond_to? :klass #struct
+        if @models_by_id[id].respond_to? :klass # struct
           @models_by_id[id][:unmapped] ||= {}
-          (@models_by_id[id][:unmapped][predicate] ||= []) << value
+          @models_by_id[id][:unmapped][predicate] ||= []
+          @models_by_id[id][:unmapped][predicate]  << value unless value.nil?
         else
           @models_by_id[id].unmapped_set(predicate, value)
         end
@@ -270,6 +291,7 @@ def create_struct(bnode_extraction, models_by_id, sol, variables)
         struct = @klass.range(bnode_extraction).new
         variables.each do |v|
           next if v == :id
+
           svalue = sol[v]
           struct[v] = svalue.is_a?(RDF::Node) ? svalue : svalue.object
         end
@@ -290,25 +312,25 @@ def create_class_model(id, klass, klass_struct)
       end
 
       def models_unmapped_to_array(models_by_id)
-        models_by_id.each do |idm, m|
+        models_by_id.each do |_idm, m|
           m.unmmaped_to_array
         end
       end
 
       def include_bnodes(bnodes, models_by_id)
-        #group by attribute
-        attrs = bnodes.map { |x, y| y.attribute }.uniq
+        # group by attribute
+        attrs = bnodes.map { |_x, y| y.attribute }.uniq
         attrs.each do |attr|
           struct = @klass.range(attr)
 
-          #bnodes that are in a range of goo ground models
-          #for example parents and children in LD class models
-          #we skip this cases for the moment
+          # bnodes that are in a range of goo ground models
+          # for example parents and children in LD class models
+          # we skip this cases for the moment
           next if struct.respond_to?(:model_name)
 
           bnode_attrs = struct.new.to_h.keys
-          ids = bnodes.select { |x, y| y.attribute == attr }.map { |x, y| y.id }
-          @klass.where.models(models_by_id.select { |x, y| ids.include?(x) }.values)
+          ids = bnodes.select { |_x, y| y.attribute == attr }.map { |_x, y| y.id }
+          @klass.where.models(models_by_id.select { |x, _y| ids.include?(x) }.values)
                 .in(@collection)
                 .include(bnode: { attr => bnode_attrs }).all
         end
@@ -316,44 +338,46 @@ def include_bnodes(bnodes, models_by_id)
 
       def include_embed_attributes(incl_embed, objects_new)
         incl_embed.each do |attr, next_attrs|
-          #anything to join ?
+          # anything to join ?
           attr_range = @klass.range(attr)
           next if attr_range.nil?
-          range_objs = objects_new.select { |id, obj|
+
+          range_objs = objects_new.select do |_id, obj|
             obj.instance_of?(attr_range) || (obj.respond_to?(:klass) && obj[:klass] == attr_range)
-          }.values
-          unless range_objs.empty?
-            range_objs.uniq!
-            query = attr_range.where().models(range_objs).in(@collection).include(*next_attrs)
-            query = query.read_only if @read_only
-            query.all
-          end
+          end.values
+          next if range_objs.empty?
+
+          range_objs.uniq!
+          query = attr_range.where.models(range_objs).in(@collection).include(*next_attrs)
+          query = query.read_only if @read_only
+          query.all
         end
       end
 
       def models_set_all_persistent(models_by_id)
         return unless @ids
-        models_by_id.each do |k, m|
+
+        models_by_id.each do |_k, m|
           m.persistent = true
         end
       end
 
       def model_set_collection_attributes(models_by_id, objects_new)
         collection_value = get_collection_value
-        if collection_value
-          collection_attribute = @klass.collection_opts
-          models_by_id.each do |id, m|
-            m.send("#{collection_attribute}=", collection_value)
-          end
-          objects_new.each do |id, obj_new|
-            if obj_new.respond_to?(:klass)
-              collection_attribute = obj_new[:klass].collection_opts
-              obj_new[collection_attribute] = collection_value
-            elsif obj_new.class.respond_to?(:collection_opts) &&
-              obj_new.class.collection_opts.instance_of?(Symbol)
-              collection_attribute = obj_new.class.collection_opts
-              obj_new.send("#{collection_attribute}=", collection_value)
-            end
+        return unless collection_value
+
+        collection_attribute = @klass.collection_opts
+        models_by_id.each do |_id, m|
+          m.send("#{collection_attribute}=", collection_value)
+        end
+        objects_new.each do |_id, obj_new|
+          if obj_new.respond_to?(:klass)
+            collection_attribute = obj_new[:klass].collection_opts
+            obj_new[collection_attribute] = collection_value
+          elsif obj_new.class.respond_to?(:collection_opts) &&
+                obj_new.class.collection_opts.instance_of?(Symbol)
+            collection_attribute = obj_new.class.collection_opts
+            obj_new.send("#{collection_attribute}=", collection_value)
           end
         end
       end
@@ -361,17 +385,12 @@ def model_set_collection_attributes(models_by_id, objects_new)
       def get_collection_value
         collection_value = nil
         if @klass.collection_opts.instance_of?(Symbol)
-          if @collection.is_a?(Array) && (@collection.length == 1)
-            collection_value = @collection.first
-          end
-          if @collection.respond_to? :id
-            collection_value = @collection
-          end
+          collection_value = @collection.first if @collection.is_a?(Array) && (@collection.length == 1)
+          collection_value = @collection if @collection.respond_to? :id
         end
         collection_value
       end
 
-
       def object_to_array(id, klass_struct, models_by_id, object, predicate)
         pre = if klass_struct
                 models_by_id[id][predicate]
@@ -390,7 +409,6 @@ def object_to_array(id, klass_struct, models_by_id, object, predicate)
       end
 
       def dependent_model_creation(embed_struct, id, models_by_id, object, objects_new, v, options)
-
         read_only = options[:read_only]
         if object.is_a?(RDF::URI) && v != :id
           range_for_v = @klass.range(v)
@@ -409,13 +427,12 @@ def dependent_model_creation(embed_struct, id, models_by_id, object, objects_new
       end
 
       def get_object_from_range(pre_val, embed_struct, object, objects_new, predicate)
-
         range_for_v = @klass.range(predicate)
         if !@read_only
           object = pre_val || @klass.range_object(predicate, object)
           objects_new[object.id] = object
         else
-          #depedent read only
+          # depedent read only
           struct = pre_val || embed_struct[predicate].new
           struct.id = object
           struct.klass = range_for_v
@@ -428,8 +445,8 @@ def get_object_from_range(pre_val, embed_struct, object, objects_new, predicate)
       def get_pre_val(id, models_by_id, object, predicate)
         pre_val = nil
         if models_by_id[id] &&
-          ((models_by_id[id].respond_to?(:klass) && models_by_id[id]) ||
-            models_by_id[id].loaded_attributes.include?(predicate))
+           ((models_by_id[id].respond_to?(:klass) && models_by_id[id]) ||
+             models_by_id[id].loaded_attributes.include?(predicate))
           pre_val = if !@read_only
                       models_by_id[id].instance_variable_get("@#{predicate}")
                     else
@@ -448,7 +465,7 @@ def add_unmapped_to_model(sol)
         id = sol[:id]
         value = sol[:attributeObject]
 
-        model_set_unmapped(id, @properties_to_include[predicate][:uri], value)
+        model_set_unmapped(id, @properties_to_include[predicate][:uri], value, :ES)
       end
 
       def add_aggregations_to_model(sol)
@@ -466,4 +483,3 @@ def add_aggregations_to_model(sol)
     end
   end
 end
-

From ef7f5cf438a872ed456b7d86b5d0936b0a0e8af6 Mon Sep 17 00:00:00 2001
From: HADDAD Zineddine <hz_haddad@esi.dz>
Date: Wed, 1 Mar 2023 14:49:07 +0100
Subject: [PATCH 069/168] filter by lang in properties

---
 lib/goo/sparql/solutions_mapper.rb | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/lib/goo/sparql/solutions_mapper.rb b/lib/goo/sparql/solutions_mapper.rb
index e6157fcf..8acbdeb7 100644
--- a/lib/goo/sparql/solutions_mapper.rb
+++ b/lib/goo/sparql/solutions_mapper.rb
@@ -274,8 +274,8 @@ def create_model(id)
         @models_by_id[id] = create_class_model(id, @klass, @klass_struct) unless @models_by_id.include?(id)
       end
 
-      def model_set_unmapped(id, predicate, value, _requested_lang = nil)
-        value = nil if value.is_a?(RDF::Literal) && !language_match?(value.language, nil)
+      def model_set_unmapped(id, predicate, value, requested_lang = nil)
+        value = nil if value.is_a?(RDF::Literal) && !language_match?(value.language, requested_lang)
 
         if @models_by_id[id].respond_to? :klass # struct
           @models_by_id[id][:unmapped] ||= {}
@@ -465,7 +465,9 @@ def add_unmapped_to_model(sol)
         id = sol[:id]
         value = sol[:attributeObject]
 
-        model_set_unmapped(id, @properties_to_include[predicate][:uri], value, :ES)
+        requested_lang = :FR
+
+        model_set_unmapped(id, @properties_to_include[predicate][:uri], value, requested_lang)
       end
 
       def add_aggregations_to_model(sol)

From 562065bff482f980a76a5f0e40e2b6091fe4c089 Mon Sep 17 00:00:00 2001
From: HADDAD Zineddine <hz_haddad@esi.dz>
Date: Wed, 1 Mar 2023 16:25:17 +0100
Subject: [PATCH 070/168] do some refactoring

---
 lib/goo/sparql/mixins/solution_lang_filter.rb |  2 +-
 lib/goo/sparql/solutions_mapper.rb            | 41 ++++++++-----------
 2 files changed, 19 insertions(+), 24 deletions(-)

diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index eb9e754c..2a20a5bb 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -21,7 +21,7 @@ def find_model_objects_by_lang(objects_by_lang, lang, model_id, predicate)
 
         def fill_models_with_other_languages(models_by_id, objects_by_lang, list_attributes, attributes,klass)
       
-          other_platform_languages = [:EN, :FR]
+          other_platform_languages = Goo.main_languages[1..-1]
 
           unless other_platform_languages.empty?
             
diff --git a/lib/goo/sparql/solutions_mapper.rb b/lib/goo/sparql/solutions_mapper.rb
index 8acbdeb7..f8ef20bc 100644
--- a/lib/goo/sparql/solutions_mapper.rb
+++ b/lib/goo/sparql/solutions_mapper.rb
@@ -48,7 +48,7 @@ def map_each_solutions(select)
           end
 
           if @unmapped
-            add_unmapped_to_model(sol)
+            add_unmapped_to_model(sol, requested_lang)
             next
           end
 
@@ -74,7 +74,7 @@ def map_each_solutions(select)
           requested_lang = nil
 
           if requested_lang.nil?
-            requested_lang = :ES # Goo.main_languages[0] || :EN
+            requested_lang = Goo.main_languages[0] || :EN
             fill_models_with_platform_languages = true
           end
 
@@ -111,10 +111,6 @@ def map_each_solutions(select)
 
       private
 
-      def get_object_language(id, new_value, predicate)
-        @lang_filter.main_lang_filter id, predicate, new_value
-      end
-
       def init_unloaded_attributes(found, list_attributes)
         return if @incl.nil?
 
@@ -142,7 +138,7 @@ def init_unloaded_attributes(found, list_attributes)
       end
 
       def get_value_object(id, objects_new, object, list_attributes, predicate)
-        object = object.object if object.is_a?(RDF::Literal)
+        object = object.object if object && !(object.is_a? RDF::URI)
         range_for_v = @klass.range(predicate)
         # binding.pry if v.eql?(:enrolled)
         # dependent model creation
@@ -200,20 +196,21 @@ def add_object_to_model(id, objects, current_obj, predicate, language, requested
             return @models_by_id[id].send("#{predicate}=", objects, on_load: true)
           end
 
+          store_objects_by_lang(id, predicate, current_obj, language)      
+        end
+      end
 
-          # the object is a literal and the language does not match , so we store it in a hash
-          @objects_by_lang[language] ||= []
-          item = @objects_by_lang[language].find { |obj| obj[:id] == id && obj[:predicate] == predicate }
-         
-          if item
-            # If an item with the matching id exists, update its attributes
-            item[:objects] << current_obj.object
-            item[:predicate] = predicate
-          else
-            # If an item with the matching id does not exist, add the new item to the array
-            @objects_by_lang[language] << { id: id, objects: [current_obj.object], predicate: predicate }
-          end
-        
+      def store_objects_by_lang(id, predicate, object, language)
+        @objects_by_lang[language] ||= []
+        item = @objects_by_lang[language].find { |obj| obj[:id] == id && obj[:predicate] == predicate }
+       
+        if item
+          # If an item with the matching id exists, update its attributes
+          item[:objects] << object.object
+          item[:predicate] = predicate
+        else
+          # If an item with the matching id does not exist, add the new item to the array
+          @objects_by_lang[language] << { id: id, objects: [object.object], predicate: predicate }
         end
       end
 
@@ -458,15 +455,13 @@ def get_pre_val(id, models_by_id, object, predicate)
         pre_val
       end
 
-      def add_unmapped_to_model(sol)
+      def add_unmapped_to_model(sol, requested_lang)
         predicate = sol[:attributeProperty].to_s.to_sym
         return unless @properties_to_include[predicate]
 
         id = sol[:id]
         value = sol[:attributeObject]
 
-        requested_lang = :FR
-
         model_set_unmapped(id, @properties_to_include[predicate][:uri], value, requested_lang)
       end
 

From 6f9e73ddda35e0555c0fda1d4393e7b1c38d1bdd Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Fri, 3 Mar 2023 09:01:39 +0100
Subject: [PATCH 071/168] add unmapped_get to goo resources

---
 lib/goo/base/resource.rb | 926 ++++++++++++++++++++-------------------
 1 file changed, 465 insertions(+), 461 deletions(-)

diff --git a/lib/goo/base/resource.rb b/lib/goo/base/resource.rb
index e6981c0b..8f7da133 100644
--- a/lib/goo/base/resource.rb
+++ b/lib/goo/base/resource.rb
@@ -1,461 +1,465 @@
-require 'cgi'
-require_relative "settings/settings"
-
-module Goo
-  module Base
-    AGGREGATE_VALUE = Struct.new(:attribute,:aggregate,:value)
-
-    class IDGenerationError < StandardError; end;
-
-    class Resource
-      include Goo::Base::Settings
-      include Goo::Search
-
-      attr_reader :loaded_attributes
-      attr_reader :modified_attributes
-      attr_reader :errors
-      attr_reader :aggregates
-      attr_reader :unmapped
-
-      attr_reader :id
-
-      def initialize(*args)
-        @loaded_attributes = Set.new
-        @modified_attributes = Set.new
-        @previous_values = nil
-        @persistent = false
-        @aggregates = nil
-        @unmapped = nil
-
-        attributes = args[0] || {}
-        opt_symbols = Goo.resource_options
-        attributes.each do |attr,value|
-          next if opt_symbols.include?(attr)
-          self.send("#{attr}=",value)
-        end
-
-        @id = attributes.delete :id
-      end
-
-      def valid?
-        validation_errors = {}
-        self.class.attributes.each do |attr|
-          inst_value = self.instance_variable_get("@#{attr}")
-          attr_errors = Goo::Validators::Enforce.enforce(self,attr,inst_value)
-          validation_errors[attr] = attr_errors unless attr_errors.nil?
-        end
-
-        if !@persistent && validation_errors.length == 0
-          uattr = self.class.name_with.instance_of?(Symbol) ? self.class.name_with : :proc_naming
-          if validation_errors[uattr].nil?
-            begin
-              if self.exist?(from_valid=true)
-                validation_errors[uattr] = validation_errors[uattr] || {}
-                validation_errors[uattr][:duplicate] =
-                  "There is already a persistent resource with id `#{@id.to_s}`"
-              end
-            rescue ArgumentError => e
-              (validation_errors[uattr] ||= {})[:existence] = e.message
-            end
-            if self.class.name_with == :id && @id.nil?
-              (validation_errors[:id] ||= {})[:existence] = ":id must be set if configured in name_with"
-            end
-          end
-        end
-
-        @errors = validation_errors
-        return @errors.length == 0
-      end
-
-      def id=(new_id)
-        raise ArgumentError, "The id of a persistent object cannot be changed." if !@id.nil? and @persistent
-        raise ArgumentError, "ID must be an RDF::URI" unless new_id.kind_of?(RDF::URI)
-        @id = new_id
-      end
-
-      def id
-        if @id.nil?
-          raise IDGenerationError, ":id must be set if configured in name_with" if self.class.name_with == :id
-          custom_name = self.class.name_with
-          if custom_name.instance_of?(Symbol)
-            @id = id_from_attribute()
-          elsif custom_name
-            begin
-              @id = custom_name.call(self)
-            rescue => e
-              raise IDGenerationError, "Problem with custom id generation: #{e.message}"
-            end
-          else
-            raise IDGenerationError, "custom_name is nil. settings for this model are incorrect."
-          end
-        end
-        return @id
-      end
-
-      def persistent?
-        return @persistent
-      end
-
-      def persistent=(val)
-        @persistent=val
-      end
-
-      def modified?
-        return modified_attributes.length > 0
-      end
-
-      def exist?(from_valid=false)
-        #generate id with proc
-        begin
-          id() unless self.class.name_with.kind_of?(Symbol)
-        rescue IDGenerationError
-        end
-
-        _id = @id
-        if _id.nil? && !from_valid && self.class.name_with.is_a?(Symbol)
-          begin
-            _id = id_from_attribute()
-          rescue IDGenerationError
-          end
-        end
-        return false unless _id
-        return Goo::SPARQL::Queries.model_exist(self,id=_id)
-      end
-
-      def fully_loaded?
-        #every declared attributed has been loaded
-        return @loaded_attributes == Set.new(self.class.attributes)
-      end
-
-      def missing_load_attributes
-        #every declared attributed has been loaded
-        return Set.new(self.class.attributes) - @loaded_attributes
-      end
-
-      def unmapped_set(attribute,value)
-        @unmapped ||= {}
-        @unmapped[attribute] ||= Set.new 
-        @unmapped[attribute] << value unless value.nil?
-      end
-
-      def unmmaped_to_array
-        cpy = {}
-        @unmapped.each do |attr,v|
-          cpy[attr] = v.to_a
-        end
-        @unmapped = cpy
-      end
-
-      def delete(*args)
-        if self.kind_of?(Goo::Base::Enum)
-          raise ArgumentError, "Enums cannot be deleted" unless args[0] && args[0][:init_enum]
-        end
-
-        raise ArgumentError, "This object is not persistent and cannot be deleted" if !@persistent
-
-        if !fully_loaded?
-          missing = missing_load_attributes
-          options_load = { models: [ self ], klass: self.class, :include => missing }
-          options_load[:collection] = self.collection if self.class.collection_opts
-          Goo::SPARQL::Queries.model_load(options_load)
-        end
-
-        graph_delete,bnode_delete = Goo::SPARQL::Triples.model_delete_triples(self)
-
-        begin
-          bnode_delete.each do |attr,delete_query|
-            Goo.sparql_update_client.update(delete_query)
-          end
-          Goo.sparql_update_client.delete_data(graph_delete, graph: self.graph)
-        rescue Exception => e
-          raise e
-        end
-        @persistent = false
-        @modified = true
-        self.class.load_inmutable_instances if self.class.inmutable? && self.class.inm_instances
-        return nil
-      end
-
-      def bring(*opts)
-        opts.each do |k|
-          if k.kind_of?(Hash)
-            k.each do |k2,v|
-              raise ArgumentError, "Unable to bring a method based attr #{k2}" if self.class.handler?(k2)
-              self.instance_variable_set("@#{k2}",nil)
-            end
-          else
-            raise ArgumentError, "Unable to bring a method based attr #{k}" if self.class.handler?(k)
-            self.instance_variable_set("@#{k}",nil)
-          end
-        end
-        query = self.class.where.models([self]).include(*opts)
-        if self.class.collection_opts.instance_of?(Symbol)
-          collection_attribute = self.class.collection_opts
-          query.in(self.send("#{collection_attribute}"))
-        end
-        query.all
-        self
-      end
-
-      def graph
-        opts = self.class.collection_opts
-        return self.class.uri_type if opts.nil?
-        col = collection
-        if col.is_a?Array
-          if col.length == 1
-            col = col.first
-          else
-            raise Exception, "collection in save only can be len=1"
-          end
-        end
-        return col ? col.id : nil
-      end
-
-
-
-      def collection
-        opts = self.class.collection_opts
-        if opts.instance_of?(Symbol)
-          if self.class.attributes.include?(opts)
-            value = self.send("#{opts}")
-            raise ArgumentError, "Collection `#{opts}` is nil" if value.nil?
-            return value
-          else
-            raise ArgumentError, "Collection `#{opts}` is not an attribute"
-          end
-        end
-      end
-
-      def add_aggregate(attribute,aggregate,value)
-        (@aggregates ||= []) << AGGREGATE_VALUE.new(attribute,aggregate,value)
-      end
-
-      def save(*opts)
-
-        if self.kind_of?(Goo::Base::Enum)
-          raise ArgumentError, "Enums can only be created on initialization" unless opts[0] && opts[0][:init_enum]
-        end
-        batch_file = nil
-        if opts && opts.length > 0
-          if opts.first.is_a?(Hash) && opts.first[:batch] && opts.first[:batch].is_a?(File)
-            batch_file = opts.first[:batch]
-          end
-        end
-
-        if !batch_file
-          return self if not modified?
-          raise Goo::Base::NotValidException, "Object is not valid. Check errors." unless valid?
-        end
-
-        graph_insert, graph_delete = Goo::SPARQL::Triples.model_update_triples(self)
-        graph = self.graph()
-        if graph_delete and graph_delete.size > 0
-          begin
-            Goo.sparql_update_client.delete_data(graph_delete, graph: graph)
-          rescue Exception => e
-            raise e
-          end
-        end
-        if graph_insert and graph_insert.size > 0
-          begin
-            if batch_file
-              lines = []
-              graph_insert.each do |t|
-                lines << [t.subject.to_ntriples,
-                          t.predicate.to_ntriples,
-                          t.object.to_ntriples,
-                          graph.to_ntriples,
-                          ".\n" ].join(' ')
-              end
-              batch_file.write(lines.join(""))
-              batch_file.flush()
-            else
-              Goo.sparql_update_client.insert_data(graph_insert, graph: graph)
-            end
-          rescue Exception => e
-            raise e
-          end
-        end
-
-        #after save all attributes where loaded
-        @loaded_attributes = Set.new(self.class.attributes).union(@loaded_attributes)
-
-        @modified_attributes = Set.new
-        @persistent = true
-        self.class.load_inmutable_instances if self.class.inmutable? && self.class.inm_instances
-        return self
-      end
-
-      def bring?(attr)
-        return @persistent &&
-                 !@loaded_attributes.include?(attr) &&
-                 !@modified_attributes.include?(attr)
-      end
-
-      def bring_remaining
-        to_bring = []
-        self.class.attributes.each do |attr|
-          to_bring << attr if self.bring?(attr)
-        end
-        self.bring(*to_bring)
-      end
-
-      def previous_values
-        return @previous_values
-      end
-
-      def to_hash
-        attr_hash = {}
-        self.class.attributes.each do |attr|
-          v = self.instance_variable_get("@#{attr}")
-          attr_hash[attr]=v unless v.nil?
-        end
-        if @unmapped
-          all_attr_uris = Set.new
-          self.class.attributes.each do |attr|
-            if self.class.collection_opts
-              all_attr_uris << self.class.attribute_uri(attr,self.collection)
-            else
-              all_attr_uris << self.class.attribute_uri(attr)
-            end
-          end
-          @unmapped.each do |attr,values|
-            attr_hash[attr] = values.map { |v| v.to_s } unless all_attr_uris.include?(attr)
-          end
-        end
-        attr_hash[:id] = @id
-        return attr_hash
-      end
-
-      ###
-      # Class level methods
-      # ##
-
-      def self.range_object(attr,id)
-        klass_range = self.range(attr)
-        return nil if klass_range.nil?
-        range_object = klass_range.new
-        range_object.id = id
-        range_object.persistent = true
-        return range_object
-      end
-
-
-
-      def self.map_attributes(inst,equivalent_predicates=nil)
-        if (inst.kind_of?(Goo::Base::Resource) && inst.unmapped.nil?) ||
-          (!inst.respond_to?(:unmapped) && inst[:unmapped].nil?)
-          raise ArgumentError, "Resource.map_attributes only works for :unmapped instances"
-        end
-        klass = inst.respond_to?(:klass) ? inst[:klass] : inst.class
-        unmapped = inst.respond_to?(:klass) ? inst[:unmapped] : inst.unmapped
-        list_attrs = klass.attributes(:list)
-        unmapped_string_keys = Hash.new
-        unmapped.each do |k,v|
-          unmapped_string_keys[k.to_s] = v
-        end
-        klass.attributes.each do |attr|
-          next if inst.class.collection?(attr) #collection is already there
-          next unless inst.respond_to?(attr)
-          attr_uri = klass.attribute_uri(attr,inst.collection).to_s
-          if unmapped_string_keys.include?(attr_uri.to_s) ||
-            (equivalent_predicates && equivalent_predicates.include?(attr_uri))
-            object = nil
-            if !unmapped_string_keys.include?(attr_uri)
-              equivalent_predicates[attr_uri].each do |eq_attr|
-                if object.nil? and !unmapped_string_keys[eq_attr].nil?
-                  object = unmapped_string_keys[eq_attr].dup
-                else
-                  if object.is_a?Array
-                    object.concat(unmapped_string_keys[eq_attr]) if !unmapped_string_keys[eq_attr].nil?
-                  end
-                end
-              end
-              if object.nil?
-                inst.send("#{attr}=", list_attrs.include?(attr) ? [] : nil, on_load: true)
-                next
-              end
-            else
-              object = unmapped_string_keys[attr_uri]
-            end
-
-            lang_filter = Goo::SPARQL::Solution::LanguageFilter.new
-
-            object = object.map do |o|
-              if o.is_a?(RDF::URI)
-                o
-              else
-                literal = o
-                index, lang_val  = lang_filter.main_lang_filter inst.id.to_s, attr, literal
-                lang_val.to_s if index.eql? :no_lang
-              end
-            end
-
-            object = object.compact
-
-            other_languages_values = lang_filter.other_languages_values
-            other_languages_values = other_languages_values[inst.id.to_s][attr] unless other_languages_values.empty?
-            unless other_languages_values.nil?
-              object = lang_filter.languages_values_to_set(other_languages_values, object)
-            end
-
-            if klass.range(attr)
-              object = object.map { |o|
-                o.is_a?(RDF::URI) ? klass.range_object(attr,o) : o }
-            end
-            object = object.first unless list_attrs.include?(attr)
-            if inst.respond_to?(:klass)
-              inst[attr] = object
-            else
-              inst.send("#{attr}=",object, on_load: true)
-            end
-          else
-            inst.send("#{attr}=",
-                      list_attrs.include?(attr) ? [] : nil, on_load: true)
-            if inst.id.to_s == "http://purl.obolibrary.org/obo/IAO_0000415"
-              if attr == :definition
-                # binding.pry
-              end
-            end
-          end
-
-        end
-      end
-      def self.find(id, *options)
-        id = RDF::URI.new(id) if !id.instance_of?(RDF::URI) && self.name_with == :id
-        id = id_from_unique_attribute(name_with(),id) unless id.instance_of?(RDF::URI)
-        if self.inmutable? && self.inm_instances && self.inm_instances[id]
-          w = Goo::Base::Where.new(self)
-          w.instance_variable_set("@result", [self.inm_instances[id]])
-          return w
-        end
-        options_load = { ids: [id], klass: self }.merge(options[-1] || {})
-        options_load[:find] = true
-        where = Goo::Base::Where.new(self)
-        where.where_options_load = options_load
-        return where
-      end
-
-      def self.in(collection)
-        return where.in(collection)
-      end
-
-      def self.where(*match)
-        return Goo::Base::Where.new(self,*match)
-      end
-
-      def self.all
-        return self.where.all
-      end
-
-      protected
-      def id_from_attribute()
-        uattr = self.class.name_with
-        uvalue = self.send("#{uattr}")
-        return self.class.id_from_unique_attribute(uattr,uvalue)
-      end
-
-    end
-
-  end
-end
+require 'cgi'
+require_relative "settings/settings"
+
+module Goo
+  module Base
+    AGGREGATE_VALUE = Struct.new(:attribute,:aggregate,:value)
+
+    class IDGenerationError < StandardError; end;
+
+    class Resource
+      include Goo::Base::Settings
+      include Goo::Search
+
+      attr_reader :loaded_attributes
+      attr_reader :modified_attributes
+      attr_reader :errors
+      attr_reader :aggregates
+      attr_reader :unmapped
+
+      attr_reader :id
+
+      def initialize(*args)
+        @loaded_attributes = Set.new
+        @modified_attributes = Set.new
+        @previous_values = nil
+        @persistent = false
+        @aggregates = nil
+        @unmapped = nil
+
+        attributes = args[0] || {}
+        opt_symbols = Goo.resource_options
+        attributes.each do |attr,value|
+          next if opt_symbols.include?(attr)
+          self.send("#{attr}=",value)
+        end
+
+        @id = attributes.delete :id
+      end
+
+      def valid?
+        validation_errors = {}
+        self.class.attributes.each do |attr|
+          inst_value = self.instance_variable_get("@#{attr}")
+          attr_errors = Goo::Validators::Enforce.enforce(self,attr,inst_value)
+          validation_errors[attr] = attr_errors unless attr_errors.nil?
+        end
+
+        if !@persistent && validation_errors.length == 0
+          uattr = self.class.name_with.instance_of?(Symbol) ? self.class.name_with : :proc_naming
+          if validation_errors[uattr].nil?
+            begin
+              if self.exist?(from_valid=true)
+                validation_errors[uattr] = validation_errors[uattr] || {}
+                validation_errors[uattr][:duplicate] =
+                  "There is already a persistent resource with id `#{@id.to_s}`"
+              end
+            rescue ArgumentError => e
+              (validation_errors[uattr] ||= {})[:existence] = e.message
+            end
+            if self.class.name_with == :id && @id.nil?
+              (validation_errors[:id] ||= {})[:existence] = ":id must be set if configured in name_with"
+            end
+          end
+        end
+
+        @errors = validation_errors
+        return @errors.length == 0
+      end
+
+      def id=(new_id)
+        raise ArgumentError, "The id of a persistent object cannot be changed." if !@id.nil? and @persistent
+        raise ArgumentError, "ID must be an RDF::URI" unless new_id.kind_of?(RDF::URI)
+        @id = new_id
+      end
+
+      def id
+        if @id.nil?
+          raise IDGenerationError, ":id must be set if configured in name_with" if self.class.name_with == :id
+          custom_name = self.class.name_with
+          if custom_name.instance_of?(Symbol)
+            @id = id_from_attribute()
+          elsif custom_name
+            begin
+              @id = custom_name.call(self)
+            rescue => e
+              raise IDGenerationError, "Problem with custom id generation: #{e.message}"
+            end
+          else
+            raise IDGenerationError, "custom_name is nil. settings for this model are incorrect."
+          end
+        end
+        return @id
+      end
+
+      def persistent?
+        return @persistent
+      end
+
+      def persistent=(val)
+        @persistent=val
+      end
+
+      def modified?
+        return modified_attributes.length > 0
+      end
+
+      def exist?(from_valid=false)
+        #generate id with proc
+        begin
+          id() unless self.class.name_with.kind_of?(Symbol)
+        rescue IDGenerationError
+        end
+
+        _id = @id
+        if _id.nil? && !from_valid && self.class.name_with.is_a?(Symbol)
+          begin
+            _id = id_from_attribute()
+          rescue IDGenerationError
+          end
+        end
+        return false unless _id
+        return Goo::SPARQL::Queries.model_exist(self,id=_id)
+      end
+
+      def fully_loaded?
+        #every declared attributed has been loaded
+        return @loaded_attributes == Set.new(self.class.attributes)
+      end
+
+      def missing_load_attributes
+        #every declared attributed has been loaded
+        return Set.new(self.class.attributes) - @loaded_attributes
+      end
+
+      def unmapped_set(attribute,value)
+        @unmapped ||= {}
+        @unmapped[attribute] ||= Set.new 
+        @unmapped[attribute] << value unless value.nil?
+      end
+ 
+      def unmapped_get(attribute)
+        @unmapped[attribute]
+      end
+
+      def unmmaped_to_array
+        cpy = {}
+        @unmapped.each do |attr,v|
+          cpy[attr] = v.to_a
+        end
+        @unmapped = cpy
+      end
+
+      def delete(*args)
+        if self.kind_of?(Goo::Base::Enum)
+          raise ArgumentError, "Enums cannot be deleted" unless args[0] && args[0][:init_enum]
+        end
+
+        raise ArgumentError, "This object is not persistent and cannot be deleted" if !@persistent
+
+        if !fully_loaded?
+          missing = missing_load_attributes
+          options_load = { models: [ self ], klass: self.class, :include => missing }
+          options_load[:collection] = self.collection if self.class.collection_opts
+          Goo::SPARQL::Queries.model_load(options_load)
+        end
+
+        graph_delete,bnode_delete = Goo::SPARQL::Triples.model_delete_triples(self)
+
+        begin
+          bnode_delete.each do |attr,delete_query|
+            Goo.sparql_update_client.update(delete_query)
+          end
+          Goo.sparql_update_client.delete_data(graph_delete, graph: self.graph)
+        rescue Exception => e
+          raise e
+        end
+        @persistent = false
+        @modified = true
+        self.class.load_inmutable_instances if self.class.inmutable? && self.class.inm_instances
+        return nil
+      end
+
+      def bring(*opts)
+        opts.each do |k|
+          if k.kind_of?(Hash)
+            k.each do |k2,v|
+              raise ArgumentError, "Unable to bring a method based attr #{k2}" if self.class.handler?(k2)
+              self.instance_variable_set("@#{k2}",nil)
+            end
+          else
+            raise ArgumentError, "Unable to bring a method based attr #{k}" if self.class.handler?(k)
+            self.instance_variable_set("@#{k}",nil)
+          end
+        end
+        query = self.class.where.models([self]).include(*opts)
+        if self.class.collection_opts.instance_of?(Symbol)
+          collection_attribute = self.class.collection_opts
+          query.in(self.send("#{collection_attribute}"))
+        end
+        query.all
+        self
+      end
+
+      def graph
+        opts = self.class.collection_opts
+        return self.class.uri_type if opts.nil?
+        col = collection
+        if col.is_a?Array
+          if col.length == 1
+            col = col.first
+          else
+            raise Exception, "collection in save only can be len=1"
+          end
+        end
+        return col ? col.id : nil
+      end
+
+
+
+      def collection
+        opts = self.class.collection_opts
+        if opts.instance_of?(Symbol)
+          if self.class.attributes.include?(opts)
+            value = self.send("#{opts}")
+            raise ArgumentError, "Collection `#{opts}` is nil" if value.nil?
+            return value
+          else
+            raise ArgumentError, "Collection `#{opts}` is not an attribute"
+          end
+        end
+      end
+
+      def add_aggregate(attribute,aggregate,value)
+        (@aggregates ||= []) << AGGREGATE_VALUE.new(attribute,aggregate,value)
+      end
+
+      def save(*opts)
+
+        if self.kind_of?(Goo::Base::Enum)
+          raise ArgumentError, "Enums can only be created on initialization" unless opts[0] && opts[0][:init_enum]
+        end
+        batch_file = nil
+        if opts && opts.length > 0
+          if opts.first.is_a?(Hash) && opts.first[:batch] && opts.first[:batch].is_a?(File)
+            batch_file = opts.first[:batch]
+          end
+        end
+
+        if !batch_file
+          return self if not modified?
+          raise Goo::Base::NotValidException, "Object is not valid. Check errors." unless valid?
+        end
+
+        graph_insert, graph_delete = Goo::SPARQL::Triples.model_update_triples(self)
+        graph = self.graph()
+        if graph_delete and graph_delete.size > 0
+          begin
+            Goo.sparql_update_client.delete_data(graph_delete, graph: graph)
+          rescue Exception => e
+            raise e
+          end
+        end
+        if graph_insert and graph_insert.size > 0
+          begin
+            if batch_file
+              lines = []
+              graph_insert.each do |t|
+                lines << [t.subject.to_ntriples,
+                          t.predicate.to_ntriples,
+                          t.object.to_ntriples,
+                          graph.to_ntriples,
+                          ".\n" ].join(' ')
+              end
+              batch_file.write(lines.join(""))
+              batch_file.flush()
+            else
+              Goo.sparql_update_client.insert_data(graph_insert, graph: graph)
+            end
+          rescue Exception => e
+            raise e
+          end
+        end
+
+        #after save all attributes where loaded
+        @loaded_attributes = Set.new(self.class.attributes).union(@loaded_attributes)
+
+        @modified_attributes = Set.new
+        @persistent = true
+        self.class.load_inmutable_instances if self.class.inmutable? && self.class.inm_instances
+        return self
+      end
+
+      def bring?(attr)
+        return @persistent &&
+                 !@loaded_attributes.include?(attr) &&
+                 !@modified_attributes.include?(attr)
+      end
+
+      def bring_remaining
+        to_bring = []
+        self.class.attributes.each do |attr|
+          to_bring << attr if self.bring?(attr)
+        end
+        self.bring(*to_bring)
+      end
+
+      def previous_values
+        return @previous_values
+      end
+
+      def to_hash
+        attr_hash = {}
+        self.class.attributes.each do |attr|
+          v = self.instance_variable_get("@#{attr}")
+          attr_hash[attr]=v unless v.nil?
+        end
+        if @unmapped
+          all_attr_uris = Set.new
+          self.class.attributes.each do |attr|
+            if self.class.collection_opts
+              all_attr_uris << self.class.attribute_uri(attr,self.collection)
+            else
+              all_attr_uris << self.class.attribute_uri(attr)
+            end
+          end
+          @unmapped.each do |attr,values|
+            attr_hash[attr] = values.map { |v| v.to_s } unless all_attr_uris.include?(attr)
+          end
+        end
+        attr_hash[:id] = @id
+        return attr_hash
+      end
+
+      ###
+      # Class level methods
+      # ##
+
+      def self.range_object(attr,id)
+        klass_range = self.range(attr)
+        return nil if klass_range.nil?
+        range_object = klass_range.new
+        range_object.id = id
+        range_object.persistent = true
+        return range_object
+      end
+
+
+
+      def self.map_attributes(inst,equivalent_predicates=nil)
+        if (inst.kind_of?(Goo::Base::Resource) && inst.unmapped.nil?) ||
+          (!inst.respond_to?(:unmapped) && inst[:unmapped].nil?)
+          raise ArgumentError, "Resource.map_attributes only works for :unmapped instances"
+        end
+        klass = inst.respond_to?(:klass) ? inst[:klass] : inst.class
+        unmapped = inst.respond_to?(:klass) ? inst[:unmapped] : inst.unmapped
+        list_attrs = klass.attributes(:list)
+        unmapped_string_keys = Hash.new
+        unmapped.each do |k,v|
+          unmapped_string_keys[k.to_s] = v
+        end
+        klass.attributes.each do |attr|
+          next if inst.class.collection?(attr) #collection is already there
+          next unless inst.respond_to?(attr)
+          attr_uri = klass.attribute_uri(attr,inst.collection).to_s
+          if unmapped_string_keys.include?(attr_uri.to_s) ||
+            (equivalent_predicates && equivalent_predicates.include?(attr_uri))
+            object = nil
+            if !unmapped_string_keys.include?(attr_uri)
+              equivalent_predicates[attr_uri].each do |eq_attr|
+                if object.nil? and !unmapped_string_keys[eq_attr].nil?
+                  object = unmapped_string_keys[eq_attr].dup
+                else
+                  if object.is_a?Array
+                    object.concat(unmapped_string_keys[eq_attr]) if !unmapped_string_keys[eq_attr].nil?
+                  end
+                end
+              end
+              if object.nil?
+                inst.send("#{attr}=", list_attrs.include?(attr) ? [] : nil, on_load: true)
+                next
+              end
+            else
+              object = unmapped_string_keys[attr_uri]
+            end
+
+            lang_filter = Goo::SPARQL::Solution::LanguageFilter.new
+
+            object = object.map do |o|
+              if o.is_a?(RDF::URI)
+                o
+              else
+                literal = o
+                index, lang_val  = lang_filter.main_lang_filter inst.id.to_s, attr, literal
+                lang_val.to_s if index.eql? :no_lang
+              end
+            end
+
+            object = object.compact
+
+            other_languages_values = lang_filter.other_languages_values
+            other_languages_values = other_languages_values[inst.id.to_s][attr] unless other_languages_values.empty?
+            unless other_languages_values.nil?
+              object = lang_filter.languages_values_to_set(other_languages_values, object)
+            end
+
+            if klass.range(attr)
+              object = object.map { |o|
+                o.is_a?(RDF::URI) ? klass.range_object(attr,o) : o }
+            end
+            object = object.first unless list_attrs.include?(attr)
+            if inst.respond_to?(:klass)
+              inst[attr] = object
+            else
+              inst.send("#{attr}=",object, on_load: true)
+            end
+          else
+            inst.send("#{attr}=",
+                      list_attrs.include?(attr) ? [] : nil, on_load: true)
+            if inst.id.to_s == "http://purl.obolibrary.org/obo/IAO_0000415"
+              if attr == :definition
+                # binding.pry
+              end
+            end
+          end
+
+        end
+      end
+      def self.find(id, *options)
+        id = RDF::URI.new(id) if !id.instance_of?(RDF::URI) && self.name_with == :id
+        id = id_from_unique_attribute(name_with(),id) unless id.instance_of?(RDF::URI)
+        if self.inmutable? && self.inm_instances && self.inm_instances[id]
+          w = Goo::Base::Where.new(self)
+          w.instance_variable_set("@result", [self.inm_instances[id]])
+          return w
+        end
+        options_load = { ids: [id], klass: self }.merge(options[-1] || {})
+        options_load[:find] = true
+        where = Goo::Base::Where.new(self)
+        where.where_options_load = options_load
+        return where
+      end
+
+      def self.in(collection)
+        return where.in(collection)
+      end
+
+      def self.where(*match)
+        return Goo::Base::Where.new(self,*match)
+      end
+
+      def self.all
+        return self.where.all
+      end
+
+      protected
+      def id_from_attribute()
+        uattr = self.class.name_with
+        uvalue = self.send("#{uattr}")
+        return self.class.id_from_unique_attribute(uattr,uvalue)
+      end
+
+    end
+
+  end
+end

From 9f91124afaabb426c62e5b1a5a7c4375c78e61e9 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Fri, 3 Mar 2023 09:05:18 +0100
Subject: [PATCH 072/168] update lang filter module to support requested_lang
 and portal_lang

---
 lib/goo/sparql/mixins/solution_lang_filter.rb | 226 +++++++++---------
 lib/goo/sparql/solutions_mapper.rb            |  36 +--
 2 files changed, 119 insertions(+), 143 deletions(-)

diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index 2a20a5bb..b6333593 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -1,109 +1,117 @@
-module Goo
-  module SPARQL
-    module Solution
-      class  LanguageFilter
-
-        def initialize
-          @other_languages_values = {}
-        end
-
-        attr_reader :other_languages_values
-
-        def main_lang_filter(id, attr, value)
-          index, value = lang_index value
-          save_other_lang_val(id, attr, index, value) unless index.nil? ||index.eql?(:no_lang)
-          [index, value]
-        end
-
-        def find_model_objects_by_lang(objects_by_lang, lang, model_id, predicate)
-          objects_by_lang[lang]&.find { |obj| obj[:id].eql?(model_id) && obj[:predicate].eql?(predicate) }
-        end
-
-        def fill_models_with_other_languages(models_by_id, objects_by_lang, list_attributes, attributes,klass)
-      
-          other_platform_languages = Goo.main_languages[1..-1]
-
-          unless other_platform_languages.empty?
-            
-            models_by_id&.each do |id, model|
-              
-              attributes&.each do |attr|
-                
-                other_platform_languages.each do |lang|
-                  
-                  model_attribute_val = model.instance_variable_get("@#{attr}")
-                  model_objects_by_lang = find_model_objects_by_lang(objects_by_lang, lang, id, attr) || []
-
-                  next if model_objects_by_lang.empty?
-
-                  if list_attributes.include?(attr)
-                    model_attribute_val ||= []
-                    if model_attribute_val.empty?
-                      model.send("#{attr}=", model_attribute_val + model_objects_by_lang[:objects], on_load: true)
-                    end
-                  elsif !model_attribute_val
-                    model.send("#{attr}=", model_objects_by_lang[:objects][0] , on_load: true)
-                  end
-                  
-                end
-              end
-            end
-          end
-        end
-
-        def languages_values_to_set(language_values, no_lang_values)
-
-          values = nil
-          matched_lang, not_matched_lang = matched_languages(language_values, no_lang_values)
-          if !matched_lang.empty?
-            main_lang = Array(matched_lang[:'0']) + Array(matched_lang[:no_lang])
-            if main_lang.empty?
-              secondary_languages = matched_lang.select { |key| key != :'0' && key != :no_lang }.sort.map { |x| x[1] }
-              values = secondary_languages.first
-            else
-              values = main_lang
-            end
-          elsif !not_matched_lang.empty?
-            values = not_matched_lang
-          end
-          values&.uniq
-        end
-
-        private
-
-        def lang_index(object)
-          return [nil, object] unless  object.is_a?(RDF::Literal)
-
-          lang = object.language
-
-          if lang.nil?
-            [:no_lang, object]
-          else
-            index = Goo.language_includes(lang)
-            index = index ? index.to_s.to_sym : :not_matched
-            [index, object]
-          end
-        end
-
-        def save_other_lang_val(id, attr, index, value)
-          @other_languages_values[id] ||= {}
-          @other_languages_values[id][attr] ||= {}
-          @other_languages_values[id][attr][index] ||= []
-          
-          unless @other_languages_values[id][attr][index].include?(value.to_s)
-            @other_languages_values[id][attr][index] += Array(value.to_s)
-          end
-        end
-
-        def matched_languages(index_values, model_attribute_val)
-          not_matched_lang = index_values[:not_matched]
-          matched_lang = index_values.reject { |key| key == :not_matched }
-          unless model_attribute_val.nil? || Array(model_attribute_val).empty?
-            matched_lang[:no_lang] = Array(model_attribute_val)
-          end
-          [matched_lang, not_matched_lang]
-        end
-      end
-    end
-  end
-end
+module Goo
+  module SPARQL
+    module Solution
+      class  LanguageFilter
+        
+        attr_reader :requested_lang, :unmapped, :objects_by_lang
+
+        def initialize(requested_lang: nil, unmapped: false, list_attributes: [])
+          @list_attributes = list_attributes
+          @objects_by_lang = {}
+          @unmapped = unmapped
+          @requested_lang = requested_lang
+
+        end
+
+        def object_language(new_value)
+          new_value.language || :no_lang if new_value.is_a?(RDF::Literal)
+        end
+  
+        def language_match?(language)
+          !language.nil? && (language.eql?(requested_lang) || language.eql?(:no_lang) || requested_lang.nil?)
+        end
+
+        def store_objects_by_lang(id, predicate, object, language)
+          # store objects in this format: [id][predicate][language] = [objects]
+
+          objects_by_lang[id] ||= {}
+          objects_by_lang[id][predicate] ||= {}
+          objects_by_lang[id][predicate][language] ||= []
+
+          objects_by_lang[id][predicate][language] << object.object
+
+        end
+
+        def fill_models_with_other_languages(models_by_id)
+
+          other_platform_languages = Goo.main_languages[1..] || []
+
+          objects_by_lang.each do |id, predicates|
+            model = models_by_id[id]
+            predicates.each do |predicate, languages|
+              model_attribute_val = get_model_attribute_value(model, predicate)
+              next unless model_attribute_val.nil? || model_attribute_val.empty?
+
+              other_platform_languages.each do |platform_language|
+                if languages[platform_language]
+                  save_value_to_model(model, languages[platform_language], predicate, unmapped)
+                  break
+                end
+              end
+              model_attribute_val = get_model_attribute_value(model, predicate)
+              if model_attribute_val.nil? || model_attribute_val.empty?
+                save_value_to_model(model, languages.values.flatten.uniq, predicate, unmapped)
+              end
+            end
+          end
+        
+        end
+        
+
+
+        def model_set_unmapped(model, predicate, value, language)
+          if language.nil? || language_match?(language)
+            return add_unmapped_to_model(model, predicate, value)
+          end
+          
+          store_objects_by_lang(model.id, predicate, value, language)
+        end
+
+
+        private
+
+        def get_model_attribute_value(model, predicate)
+          if unmapped
+            unmapped_get(model, predicate)
+          else
+            model.instance_variable_get("@#{predicate}")
+          end
+        end
+
+
+        def add_unmapped_to_model(model, predicate, value)
+          if model.respond_to? :klass # struct
+            model[:unmapped] ||= {}
+            model[:unmapped][predicate] ||= []
+            model[:unmapped][predicate]  << value unless value.nil?
+          else
+            model.unmapped_set(predicate, value)
+          end
+        end
+
+        def save_value_to_model(model, value, predicate, unmapped)
+          if unmapped
+            add_unmapped_to_model(model, predicate, value)
+          else
+            value = Array(value).min unless list_attributes?(predicate)
+            model.send("#{predicate}=", value, on_load: true)
+          end
+        end
+
+        def unmapped_get(model, predicate)
+          if model && model.respond_to?(:klass) # struct
+            model[:unmapped]&.dig(predicate)
+          else
+            model.unmapped_get(predicate)
+          end
+
+        end
+
+        def list_attributes?(predicate)
+          @list_attributes.include?(predicate)
+        end
+
+      end
+    end
+  end
+end
diff --git a/lib/goo/sparql/solutions_mapper.rb b/lib/goo/sparql/solutions_mapper.rb
index f8ef20bc..01ade699 100644
--- a/lib/goo/sparql/solutions_mapper.rb
+++ b/lib/goo/sparql/solutions_mapper.rb
@@ -173,14 +173,7 @@ def get_value_object(id, objects_new, object, list_attributes, predicate)
         [object, objects_new]
       end
 
-      def object_language(new_value)
-        new_value.language || :no_lang if new_value.is_a?(RDF::Literal)
-      end
-
-      def language_match?(language, requested_lang = nil)
-        !language.nil? && (language.eql?(requested_lang) || language.eql?(:no_lang) || requested_lang.nil?)
-      end
-
+      def add_object_to_model(id, objects, current_obj, predicate, language) 
       def add_object_to_model(id, objects, current_obj, predicate, language, requested_lang = nil)
         if @models_by_id[id].respond_to?(:klass)
           @models_by_id[id][predicate] = objects unless objects.nil? && !@models_by_id[id][predicate].nil?
@@ -197,21 +190,7 @@ def add_object_to_model(id, objects, current_obj, predicate, language, requested
           end
 
           store_objects_by_lang(id, predicate, current_obj, language)      
-        end
-      end
-
-      def store_objects_by_lang(id, predicate, object, language)
-        @objects_by_lang[language] ||= []
-        item = @objects_by_lang[language].find { |obj| obj[:id] == id && obj[:predicate] == predicate }
-       
-        if item
-          # If an item with the matching id exists, update its attributes
-          item[:objects] << object.object
-          item[:predicate] = predicate
-        else
-          # If an item with the matching id does not exist, add the new item to the array
-          @objects_by_lang[language] << { id: id, objects: [object.object], predicate: predicate }
-        end
+        end
       end
 
       def get_preload_value(id, object, predicate)
@@ -271,17 +250,6 @@ def create_model(id)
         @models_by_id[id] = create_class_model(id, @klass, @klass_struct) unless @models_by_id.include?(id)
       end
 
-      def model_set_unmapped(id, predicate, value, requested_lang = nil)
-        value = nil if value.is_a?(RDF::Literal) && !language_match?(value.language, requested_lang)
-
-        if @models_by_id[id].respond_to? :klass # struct
-          @models_by_id[id][:unmapped] ||= {}
-          @models_by_id[id][:unmapped][predicate] ||= []
-          @models_by_id[id][:unmapped][predicate]  << value unless value.nil?
-        else
-          @models_by_id[id].unmapped_set(predicate, value)
-        end
-      end
 
       def create_struct(bnode_extraction, models_by_id, sol, variables)
         list_attributes = Set.new(@klass.attributes(:list))

From 536e22fea70ec871ad33a5d9984d1c2c3ef1cff0 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Fri, 3 Mar 2023 09:36:37 +0100
Subject: [PATCH 073/168] use the new lang filter module in the solution_mapper

---
 lib/goo/sparql/solutions_mapper.rb | 900 +++++++++++++++--------------
 1 file changed, 452 insertions(+), 448 deletions(-)

diff --git a/lib/goo/sparql/solutions_mapper.rb b/lib/goo/sparql/solutions_mapper.rb
index 01ade699..ca830719 100644
--- a/lib/goo/sparql/solutions_mapper.rb
+++ b/lib/goo/sparql/solutions_mapper.rb
@@ -1,450 +1,454 @@
-module Goo
-  module SPARQL
-    class SolutionMapper
-      BNODES_TUPLES = Struct.new(:id, :attribute)
-
-      def initialize(aggregate_projections, bnode_extraction, embed_struct,
-                     incl_embed, klass_struct, models_by_id,
-                     properties_to_include, unmapped, variables, ids, options)
-
-        @aggregate_projections = aggregate_projections
-        @bnode_extraction = bnode_extraction
-        @embed_struct = embed_struct
-        @incl_embed = incl_embed
-        @klass_struct = klass_struct
-        @models_by_id = models_by_id
-        @properties_to_include = properties_to_include
-        @unmapped = unmapped
-        @variables = variables
-        @ids = ids
-        @klass = options[:klass]
-        @read_only = options[:read_only]
-        @incl = options[:include]
-        @count = options[:count]
-        @collection = options[:collection]
-        @objects_by_lang = {}
-      end
-
-      def map_each_solutions(select)
-        found = Set.new
-        fill_models_with_platform_languages = false
-        objects_new = {}
-        list_attributes = Set.new(@klass.attributes(:list))
-        all_attributes = Set.new(@klass.attributes(:all))
-        @lang_filter = Goo::SPARQL::Solution::LanguageFilter.new
-
-        select.each_solution do |sol|
-          next if sol[:some_type] && @klass.type_uri(@collection) != sol[:some_type]
-          return sol[:count_var].object if @count
-
-          found.add(sol[:id])
-          id = sol[:id]
-
-          create_model(id)
-
-          if @bnode_extraction
-            add_bnode_to_model(sol)
-            next
-          end
-
-          if @unmapped
-            add_unmapped_to_model(sol, requested_lang)
-            next
-          end
-
-          if @aggregate_projections
-            add_aggregations_to_model(sol)
-            next
-          end
-
-          predicate = sol[:attributeProperty].to_s.to_sym
-
-          next if predicate.nil? || !all_attributes.include?(predicate)
-
-          object = sol[:attributeObject]
-
-          # bnodes
-          if bnode_id?(object, predicate)
-            objects_new = bnode_id_tuple(id, object, objects_new, predicate)
-            next
-          end
-
-          lang = object_language(object) # if lang is nil, it means that the object is not a literal
-
-          requested_lang = nil
-
-          if requested_lang.nil?
-            requested_lang = Goo.main_languages[0] || :EN
-            fill_models_with_platform_languages = true
-          end
-
-          objects, objects_new = get_value_object(id, objects_new, object, list_attributes, predicate)
-          add_object_to_model(id, objects, object, predicate, lang, requested_lang)
-        end
-     
-        if fill_models_with_platform_languages
-          @lang_filter.fill_models_with_other_languages(@models_by_id, @objects_by_lang, list_attributes, @incl, @klass)
-        end
-
-        init_unloaded_attributes(found, list_attributes)
-
-        return @models_by_id if @bnode_extraction
-
-        model_set_collection_attributes(@models_by_id, objects_new)
-
-        # remove from models_by_id elements that were not touched
-        @models_by_id.select! { |k, _m| found.include?(k) }
-
-        models_set_all_persistent(@models_by_id) unless @read_only
-
-        # next level of embed attributes
-        include_embed_attributes(@incl_embed, objects_new) if @incl_embed && !@incl_embed.empty?
-
-        # bnodes
-        blank_nodes = objects_new.select { |id, _obj| id.is_a?(RDF::Node) && id.anonymous? }
-        include_bnodes(blank_nodes, @models_by_id) unless blank_nodes.empty?
-
-        models_unmapped_to_array(@models_by_id) if @unmapped
-
-        @models_by_id
-      end
-
-      private
-
-      def init_unloaded_attributes(found, list_attributes)
-        return if @incl.nil?
-
-        # Here we are setting to nil all attributes that have been included but not found in the triplestore
-        found.uniq.each do |model_id|
-          m = @models_by_id[model_id]
-          @incl.each do |attr_to_incl|
-            is_handler = m.respond_to?(:handler?) && m.class.handler?(attr_to_incl)
-            next if attr_to_incl.to_s.eql?('unmapped') || is_handler
-
-            loaded = m.respond_to?('loaded_attributes') && m.loaded_attributes.include?(attr_to_incl)
-            is_list = list_attributes.include?(attr_to_incl)
-            is_struct = m.respond_to?(:klass)
-
-            # Go through all models queried
-            if is_struct
-              m[attr_to_incl] = [] if is_list && m[attr_to_incl].nil?
-            elsif is_list && (!loaded || m.send(attr_to_incl.to_s).nil?)
-              m.send("#{attr_to_incl}=", [], on_load: true)
-            elsif !loaded && !is_list && m.respond_to?("#{attr_to_incl}=")
-              m.send("#{attr_to_incl}=", nil, on_load: true)
-            end
-          end
-        end
-      end
-
-      def get_value_object(id, objects_new, object, list_attributes, predicate)
-        object = object.object if object && !(object.is_a? RDF::URI)
-        range_for_v = @klass.range(predicate)
-        # binding.pry if v.eql?(:enrolled)
-        # dependent model creation
-
-        if object.is_a?(RDF::URI) && (predicate != :id) && !range_for_v.nil?
-          if objects_new.include?(object)
-            object = objects_new[object]
-          elsif !range_for_v.inmutable?
-            pre_val = get_preload_value(id, object, predicate)
-            object, objects_new = if !@read_only
-                                    preloaded_or_new_object(object, objects_new, pre_val, predicate)
-                                  else
-                                    # depedent read only
-                                    preloaded_or_new_struct(object, objects_new, pre_val, predicate)
-                                  end
-          else
-            object = range_for_v.find(object).first
-          end
-        end
-
-        if list_attributes.include?(predicate)
-          pre = @klass_struct ? @models_by_id[id][predicate] : @models_by_id[id].instance_variable_get("@#{predicate}")
-
-          object = []  if object.nil? && pre.nil?
-
-          object = pre if object.nil? && !pre.nil?
-
-          object = pre.nil? ? [object] : (pre.dup << object)
-          object.uniq
-
-        end
-        [object, objects_new]
-      end
-
+module Goo
+  module SPARQL
+    class SolutionMapper
+      BNODES_TUPLES = Struct.new(:id, :attribute)
+
+      def initialize(aggregate_projections, bnode_extraction, embed_struct,
+                     incl_embed, klass_struct, models_by_id,
+                     properties_to_include, unmapped, variables, ids, options)
+
+        @aggregate_projections = aggregate_projections
+        @bnode_extraction = bnode_extraction
+        @embed_struct = embed_struct
+        @incl_embed = incl_embed
+        @klass_struct = klass_struct
+        @models_by_id = models_by_id
+        @properties_to_include = properties_to_include
+        @unmapped = unmapped
+        @variables = variables
+        @ids = ids
+        @klass = options[:klass]
+        @read_only = options[:read_only]
+        @incl = options[:include]
+        @count = options[:count]
+        @collection = options[:collection]
+        @objects_by_lang = {}
+        @requested_lang = :EN
+        Goo.main_languages = [:ES, :FR, :EN]
+      end
+      
+      def map_each_solutions(select)
+        found = Set.new
+        fill_models_with_platform_languages = init_requested_lang
+        objects_new = {}
+        list_attributes = Set.new(@klass.attributes(:list))
+        all_attributes = Set.new(@klass.attributes(:all))
+        @lang_filter = Goo::SPARQL::Solution::LanguageFilter.new(requested_lang: @requested_lang, unmapped: @unmapped,
+           list_attributes: list_attributes)
+
+        
+        select.each_solution do |sol|
+          next if sol[:some_type] && @klass.type_uri(@collection) != sol[:some_type]
+          return sol[:count_var].object if @count
+
+          found.add(sol[:id])
+          id = sol[:id]
+
+          create_model(id)
+
+          if @bnode_extraction
+            add_bnode_to_model(sol)
+            next
+          end
+
+          if @unmapped
+            add_unmapped_to_model(sol)
+            next
+          end
+
+          if @aggregate_projections
+            add_aggregations_to_model(sol)
+            next
+          end
+
+          predicate = sol[:attributeProperty].to_s.to_sym
+
+          next if predicate.nil? || !all_attributes.include?(predicate)
+
+          object = sol[:attributeObject]
+
+          # bnodes
+          if bnode_id?(object, predicate)
+            objects_new = bnode_id_tuple(id, object, objects_new, predicate)
+            next
+          end
+
+          lang = @lang_filter.object_language(object) # if lang is nil, it means that the object is not a literal
+
+
+          objects, objects_new = get_value_object(id, objects_new, object, list_attributes, predicate)
+          add_object_to_model(id, objects, object, predicate, lang)
+        end
+      
+  
+        if fill_models_with_platform_languages
+          @lang_filter.fill_models_with_other_languages(@models_by_id)
+        end
+
+        init_unloaded_attributes(found, list_attributes)
+
+        return @models_by_id if @bnode_extraction
+
+        model_set_collection_attributes(@models_by_id, objects_new)
+
+        # remove from models_by_id elements that were not touched
+        @models_by_id.select! { |k, _m| found.include?(k) }
+
+        models_set_all_persistent(@models_by_id) unless @read_only
+
+        # next level of embed attributes
+        include_embed_attributes(@incl_embed, objects_new) if @incl_embed && !@incl_embed.empty?
+
+        # bnodes
+        blank_nodes = objects_new.select { |id, _obj| id.is_a?(RDF::Node) && id.anonymous? }
+        include_bnodes(blank_nodes, @models_by_id) unless blank_nodes.empty?
+
+        models_unmapped_to_array(@models_by_id) if @unmapped
+
+        @models_by_id
+      end
+
+      private
+
+      def init_requested_lang 
+        if @requested_lang.nil?
+          @requested_lang = Goo.main_languages[0] || :EN
+          return true
+        end
+        
+        false
+      end
+
+      def init_unloaded_attributes(found, list_attributes)
+        return if @incl.nil?
+
+        # Here we are setting to nil all attributes that have been included but not found in the triplestore
+        found.uniq.each do |model_id|
+          m = @models_by_id[model_id]
+          @incl.each do |attr_to_incl|
+            is_handler = m.respond_to?(:handler?) && m.class.handler?(attr_to_incl)
+            next if attr_to_incl.to_s.eql?('unmapped') || is_handler
+
+            loaded = m.respond_to?('loaded_attributes') && m.loaded_attributes.include?(attr_to_incl)
+            is_list = list_attributes.include?(attr_to_incl)
+            is_struct = m.respond_to?(:klass)
+
+            # Go through all models queried
+            if is_struct
+              m[attr_to_incl] = [] if is_list && m[attr_to_incl].nil?
+            elsif is_list && (!loaded || m.send(attr_to_incl.to_s).nil?)
+              m.send("#{attr_to_incl}=", [], on_load: true)
+            elsif !loaded && !is_list && m.respond_to?("#{attr_to_incl}=")
+              m.send("#{attr_to_incl}=", nil, on_load: true)
+            end
+          end
+        end
+      end
+
+      def get_value_object(id, objects_new, object, list_attributes, predicate)
+        object = object.object if object && !(object.is_a? RDF::URI)
+        range_for_v = @klass.range(predicate)
+       
+
+        if object.is_a?(RDF::URI) && (predicate != :id) && !range_for_v.nil?
+          if objects_new.include?(object)
+            object = objects_new[object]
+          elsif !range_for_v.inmutable?
+            pre_val = get_preload_value(id, object, predicate)
+            object, objects_new = if !@read_only
+                                    preloaded_or_new_object(object, objects_new, pre_val, predicate)
+                                  else
+                                    # depedent read only
+                                    preloaded_or_new_struct(object, objects_new, pre_val, predicate)
+                                  end
+          else
+            object = range_for_v.find(object).first
+          end
+        end
+
+        if list_attributes.include?(predicate)
+          pre = @klass_struct ? @models_by_id[id][predicate] : @models_by_id[id].instance_variable_get("@#{predicate}")
+
+          if object.nil?
+            object = pre.nil? ? [] : pre
+          else
+            object = pre.nil? ? [object] : (pre.dup << object)
+            object.uniq!
+          end
+
+        end
+        [object, objects_new]
+      end
+
       def add_object_to_model(id, objects, current_obj, predicate, language) 
-      def add_object_to_model(id, objects, current_obj, predicate, language, requested_lang = nil)
-        if @models_by_id[id].respond_to?(:klass)
-          @models_by_id[id][predicate] = objects unless objects.nil? && !@models_by_id[id][predicate].nil?
-        elsif !@models_by_id[id].class.handler?(predicate) &&
-              !(objects.nil? && !@models_by_id[id].instance_variable_get("@#{predicate}").nil?) &&
-              predicate != :id
-
-          if language.nil? # the object is a non-literal
-            return @models_by_id[id].send("#{predicate}=", objects, on_load: true)
-          end
-
-          if language_match?(language, requested_lang) # the object is a literal and the language matches
-            return @models_by_id[id].send("#{predicate}=", objects, on_load: true)
-          end
-
-          store_objects_by_lang(id, predicate, current_obj, language)      
+        if @models_by_id[id].respond_to?(:klass)
+          @models_by_id[id][predicate] = objects unless objects.nil? && !@models_by_id[id][predicate].nil?
+        elsif !@models_by_id[id].class.handler?(predicate) &&
+              !(objects.nil? && !@models_by_id[id].instance_variable_get("@#{predicate}").nil?) &&
+              predicate != :id
+
+          if language.nil? || @lang_filter.language_match?(language)
+            return @models_by_id[id].send("#{predicate}=", objects, on_load: true)
+          end
+
+          @lang_filter.store_objects_by_lang(id, predicate, current_obj, language)
+        end
+      end
+
+      def get_preload_value(id, object, predicate)
+        pre_val = nil
+        if predicate_preloaded?(id, predicate)
+          pre_val = preloaded_value(id, predicate)
+          pre_val = pre_val.select { |x| x.id == object }.first if pre_val.is_a?(Array)
+        end
+        pre_val
+      end
+
+      def preloaded_or_new_object(object, objects_new, pre_val, predicate)
+        object = pre_val || @klass.range_object(predicate, object)
+        objects_new[object.id] = object
+        [object, objects_new]
+      end
+
+      def preloaded_or_new_struct(object, objects_new, pre_val, predicate)
+        struct = pre_val || @embed_struct[predicate].new
+        struct.id = object
+        struct.klass = @klass.range(predicate)
+        objects_new[struct.id] = struct
+        [struct, objects_new]
+      end
+
+      def preloaded_value(id, predicate)
+        if !@read_only
+          @models_by_id[id].instance_variable_get("@#{predicate}")
+
+        else
+          @models_by_id[id][predicate]
+        end
+      end
+
+      def predicate_preloaded?(id, predicate)
+        @models_by_id[id] &&
+          (@models_by_id[id].respond_to?(:klass) || @models_by_id[id].loaded_attributes.include?(predicate))
+      end
+
+      def bnode_id?(object, predicate)
+        object.is_a?(RDF::Node) && object.anonymous? && @incl.include?(predicate)
+      end
+
+      def bnode_id_tuple(id, object, objects_new, predicate)
+        range = @klass.range(predicate)
+        objects_new[object] = BNODES_TUPLES.new(id, predicate) if range.respond_to?(:new)
+        objects_new
+      end
+
+      def add_bnode_to_model(sol)
+        id = sol[:id]
+        struct = create_struct(@bnode_extraction, @models_by_id, sol, @variables)
+        @models_by_id[id].send("#{@bnode_extraction}=", struct)
+      end
+
+      def create_model(id)
+        @models_by_id[id] = create_class_model(id, @klass, @klass_struct) unless @models_by_id.include?(id)
+      end
+
+
+      def create_struct(bnode_extraction, models_by_id, sol, variables)
+        list_attributes = Set.new(@klass.attributes(:list))
+        struct = @klass.range(bnode_extraction).new
+        variables.each do |v|
+          next if v == :id
+
+          svalue = sol[v]
+          struct[v] = svalue.is_a?(RDF::Node) ? svalue : svalue.object
+        end
+        if list_attributes.include?(bnode_extraction)
+          pre = models_by_id[sol[:id]].instance_variable_get("@#{bnode_extraction}")
+          pre = pre ? (pre.dup << struct) : [struct]
+          struct = pre
+        end
+        struct
+      end
+
+      def create_class_model(id, klass, klass_struct)
+        klass_model = klass_struct ? klass_struct.new : klass.new
+        klass_model.id = id
+        klass_model.persistent = true unless klass_struct
+        klass_model.klass = klass if klass_struct
+        klass_model
+      end
+
+      def models_unmapped_to_array(models_by_id)
+        models_by_id.each do |_idm, m|
+          m.unmmaped_to_array
+        end
+      end
+
+      def include_bnodes(bnodes, models_by_id)
+        # group by attribute
+        attrs = bnodes.map { |_x, y| y.attribute }.uniq
+        attrs.each do |attr|
+          struct = @klass.range(attr)
+
+          # bnodes that are in a range of goo ground models
+          # for example parents and children in LD class models
+          # we skip this cases for the moment
+          next if struct.respond_to?(:model_name)
+
+          bnode_attrs = struct.new.to_h.keys
+          ids = bnodes.select { |_x, y| y.attribute == attr }.map { |_x, y| y.id }
+          @klass.where.models(models_by_id.select { |x, _y| ids.include?(x) }.values)
+                .in(@collection)
+                .include(bnode: { attr => bnode_attrs }).all
+        end
+      end
+
+      def include_embed_attributes(incl_embed, objects_new)
+        incl_embed.each do |attr, next_attrs|
+          # anything to join ?
+          attr_range = @klass.range(attr)
+          next if attr_range.nil?
+
+          range_objs = objects_new.select do |_id, obj|
+            obj.instance_of?(attr_range) || (obj.respond_to?(:klass) && obj[:klass] == attr_range)
+          end.values
+          next if range_objs.empty?
+
+          range_objs.uniq!
+          query = attr_range.where.models(range_objs).in(@collection).include(*next_attrs)
+          query = query.read_only if @read_only
+          query.all
+        end
+      end
+
+      def models_set_all_persistent(models_by_id)
+        return unless @ids
+
+        models_by_id.each do |_k, m|
+          m.persistent = true
+        end
+      end
+
+      def model_set_collection_attributes(models_by_id, objects_new)
+        collection_value = get_collection_value
+        return unless collection_value
+
+        collection_attribute = @klass.collection_opts
+        models_by_id.each do |_id, m|
+          m.send("#{collection_attribute}=", collection_value)
+        end
+        objects_new.each do |_id, obj_new|
+          if obj_new.respond_to?(:klass)
+            collection_attribute = obj_new[:klass].collection_opts
+            obj_new[collection_attribute] = collection_value
+          elsif obj_new.class.respond_to?(:collection_opts) &&
+                obj_new.class.collection_opts.instance_of?(Symbol)
+            collection_attribute = obj_new.class.collection_opts
+            obj_new.send("#{collection_attribute}=", collection_value)
+          end
+        end
+      end
+
+      def get_collection_value
+        collection_value = nil
+        if @klass.collection_opts.instance_of?(Symbol)
+          collection_value = @collection.first if @collection.is_a?(Array) && (@collection.length == 1)
+          collection_value = @collection if @collection.respond_to? :id
+        end
+        collection_value
+      end
+
+      def object_to_array(id, klass_struct, models_by_id, object, predicate)
+        pre = if klass_struct
+                models_by_id[id][predicate]
+              else
+                models_by_id[id].instance_variable_get("@#{predicate}")
+              end
+        if object.nil? && pre.nil?
+          object = []
+        elsif object.nil? && !pre.nil?
+          object = pre
+        elsif object
+          object = !pre ? [object] : (pre.dup << object)
+          object.uniq!
+        end
+        object
+      end
+
+      def dependent_model_creation(embed_struct, id, models_by_id, object, objects_new, v, options)
+        read_only = options[:read_only]
+        if object.is_a?(RDF::URI) && v != :id
+          range_for_v = @klass.range(v)
+          if range_for_v
+            if objects_new.include?(object)
+              object = objects_new[object]
+            elsif !range_for_v.inmutable?
+              pre_val = get_pre_val(id, models_by_id, object, v, read_only)
+              object = get_object_from_range(pre_val, embed_struct, object, objects_new, v, options)
+            else
+              object = range_for_v.find(object).first
+            end
+          end
+        end
+        object
+      end
+
+      def get_object_from_range(pre_val, embed_struct, object, objects_new, predicate)
+        range_for_v = @klass.range(predicate)
+        if !@read_only
+          object = pre_val || @klass.range_object(predicate, object)
+          objects_new[object.id] = object
+        else
+          # depedent read only
+          struct = pre_val || embed_struct[predicate].new
+          struct.id = object
+          struct.klass = range_for_v
+          objects_new[struct.id] = struct
+          object = struct
+        end
+        object
+      end
+
+      def get_pre_val(id, models_by_id, object, predicate)
+        pre_val = nil
+        if models_by_id[id] &&
+           ((models_by_id[id].respond_to?(:klass) && models_by_id[id]) ||
+             models_by_id[id].loaded_attributes.include?(predicate))
+          pre_val = if !@read_only
+                      models_by_id[id].instance_variable_get("@#{predicate}")
+                    else
+                      models_by_id[id][predicate]
+                    end
+
+          pre_val = pre_val.select { |x| x.id == object }.first if pre_val.is_a?(Array)
+        end
+        pre_val
+      end
+
+      def add_unmapped_to_model(sol)
+        predicate = sol[:attributeProperty].to_s.to_sym
+        return unless @properties_to_include[predicate]
+
+        id = sol[:id]
+        value = sol[:attributeObject]
+
+        language = @lang_filter.object_language(value)
+
+        @lang_filter.model_set_unmapped(@models_by_id[id], @properties_to_include[predicate][:uri], value, language)
+      end
+
+      def add_aggregations_to_model(sol)
+        id = sol[:id]
+        @aggregate_projections&.each do |aggregate_key, aggregate_val|
+          if @models_by_id[id].respond_to?(:add_aggregate)
+            @models_by_id[id].add_aggregate(aggregate_val[1], aggregate_val[0], sol[aggregate_key].object)
+          else
+            (@models_by_id[id].aggregates ||= []) << Goo::Base::AGGREGATE_VALUE.new(aggregate_val[1],
+                                                                                    aggregate_val[0],
+                                                                                    sol[aggregate_key].object)
+          end
         end
-      end
-
-      def get_preload_value(id, object, predicate)
-        pre_val = nil
-        if predicate_preloaded?(id, predicate)
-          pre_val = preloaded_value(id, predicate)
-          pre_val = pre_val.select { |x| x.id == object }.first if pre_val.is_a?(Array)
-        end
-        pre_val
-      end
-
-      def preloaded_or_new_object(object, objects_new, pre_val, predicate)
-        object = pre_val || @klass.range_object(predicate, object)
-        objects_new[object.id] = object
-        [object, objects_new]
-      end
-
-      def preloaded_or_new_struct(object, objects_new, pre_val, predicate)
-        struct = pre_val || @embed_struct[predicate].new
-        struct.id = object
-        struct.klass = @klass.range(predicate)
-        objects_new[struct.id] = struct
-        [struct, objects_new]
-      end
-
-      def preloaded_value(id, predicate)
-        if !@read_only
-          @models_by_id[id].instance_variable_get("@#{predicate}")
-
-        else
-          @models_by_id[id][predicate]
-        end
-      end
-
-      def predicate_preloaded?(id, predicate)
-        @models_by_id[id] &&
-          (@models_by_id[id].respond_to?(:klass) || @models_by_id[id].loaded_attributes.include?(predicate))
-      end
-
-      def bnode_id?(object, predicate)
-        object.is_a?(RDF::Node) && object.anonymous? && @incl.include?(predicate)
-      end
-
-      def bnode_id_tuple(id, object, objects_new, predicate)
-        range = @klass.range(predicate)
-        objects_new[object] = BNODES_TUPLES.new(id, predicate) if range.respond_to?(:new)
-        objects_new
-      end
-
-      def add_bnode_to_model(sol)
-        id = sol[:id]
-        struct = create_struct(@bnode_extraction, @models_by_id, sol, @variables)
-        @models_by_id[id].send("#{@bnode_extraction}=", struct)
-      end
-
-      def create_model(id)
-        @models_by_id[id] = create_class_model(id, @klass, @klass_struct) unless @models_by_id.include?(id)
-      end
-
-
-      def create_struct(bnode_extraction, models_by_id, sol, variables)
-        list_attributes = Set.new(@klass.attributes(:list))
-        struct = @klass.range(bnode_extraction).new
-        variables.each do |v|
-          next if v == :id
-
-          svalue = sol[v]
-          struct[v] = svalue.is_a?(RDF::Node) ? svalue : svalue.object
-        end
-        if list_attributes.include?(bnode_extraction)
-          pre = models_by_id[sol[:id]].instance_variable_get("@#{bnode_extraction}")
-          pre = pre ? (pre.dup << struct) : [struct]
-          struct = pre
-        end
-        struct
-      end
-
-      def create_class_model(id, klass, klass_struct)
-        klass_model = klass_struct ? klass_struct.new : klass.new
-        klass_model.id = id
-        klass_model.persistent = true unless klass_struct
-        klass_model.klass = klass if klass_struct
-        klass_model
-      end
-
-      def models_unmapped_to_array(models_by_id)
-        models_by_id.each do |_idm, m|
-          m.unmmaped_to_array
-        end
-      end
-
-      def include_bnodes(bnodes, models_by_id)
-        # group by attribute
-        attrs = bnodes.map { |_x, y| y.attribute }.uniq
-        attrs.each do |attr|
-          struct = @klass.range(attr)
-
-          # bnodes that are in a range of goo ground models
-          # for example parents and children in LD class models
-          # we skip this cases for the moment
-          next if struct.respond_to?(:model_name)
-
-          bnode_attrs = struct.new.to_h.keys
-          ids = bnodes.select { |_x, y| y.attribute == attr }.map { |_x, y| y.id }
-          @klass.where.models(models_by_id.select { |x, _y| ids.include?(x) }.values)
-                .in(@collection)
-                .include(bnode: { attr => bnode_attrs }).all
-        end
-      end
-
-      def include_embed_attributes(incl_embed, objects_new)
-        incl_embed.each do |attr, next_attrs|
-          # anything to join ?
-          attr_range = @klass.range(attr)
-          next if attr_range.nil?
-
-          range_objs = objects_new.select do |_id, obj|
-            obj.instance_of?(attr_range) || (obj.respond_to?(:klass) && obj[:klass] == attr_range)
-          end.values
-          next if range_objs.empty?
-
-          range_objs.uniq!
-          query = attr_range.where.models(range_objs).in(@collection).include(*next_attrs)
-          query = query.read_only if @read_only
-          query.all
-        end
-      end
-
-      def models_set_all_persistent(models_by_id)
-        return unless @ids
-
-        models_by_id.each do |_k, m|
-          m.persistent = true
-        end
-      end
-
-      def model_set_collection_attributes(models_by_id, objects_new)
-        collection_value = get_collection_value
-        return unless collection_value
-
-        collection_attribute = @klass.collection_opts
-        models_by_id.each do |_id, m|
-          m.send("#{collection_attribute}=", collection_value)
-        end
-        objects_new.each do |_id, obj_new|
-          if obj_new.respond_to?(:klass)
-            collection_attribute = obj_new[:klass].collection_opts
-            obj_new[collection_attribute] = collection_value
-          elsif obj_new.class.respond_to?(:collection_opts) &&
-                obj_new.class.collection_opts.instance_of?(Symbol)
-            collection_attribute = obj_new.class.collection_opts
-            obj_new.send("#{collection_attribute}=", collection_value)
-          end
-        end
-      end
-
-      def get_collection_value
-        collection_value = nil
-        if @klass.collection_opts.instance_of?(Symbol)
-          collection_value = @collection.first if @collection.is_a?(Array) && (@collection.length == 1)
-          collection_value = @collection if @collection.respond_to? :id
-        end
-        collection_value
-      end
-
-      def object_to_array(id, klass_struct, models_by_id, object, predicate)
-        pre = if klass_struct
-                models_by_id[id][predicate]
-              else
-                models_by_id[id].instance_variable_get("@#{predicate}")
-              end
-        if object.nil? && pre.nil?
-          object = []
-        elsif object.nil? && !pre.nil?
-          object = pre
-        elsif object
-          object = !pre ? [object] : (pre.dup << object)
-          object.uniq!
-        end
-        object
-      end
-
-      def dependent_model_creation(embed_struct, id, models_by_id, object, objects_new, v, options)
-        read_only = options[:read_only]
-        if object.is_a?(RDF::URI) && v != :id
-          range_for_v = @klass.range(v)
-          if range_for_v
-            if objects_new.include?(object)
-              object = objects_new[object]
-            elsif !range_for_v.inmutable?
-              pre_val = get_pre_val(id, models_by_id, object, v, read_only)
-              object = get_object_from_range(pre_val, embed_struct, object, objects_new, v, options)
-            else
-              object = range_for_v.find(object).first
-            end
-          end
-        end
-        object
-      end
-
-      def get_object_from_range(pre_val, embed_struct, object, objects_new, predicate)
-        range_for_v = @klass.range(predicate)
-        if !@read_only
-          object = pre_val || @klass.range_object(predicate, object)
-          objects_new[object.id] = object
-        else
-          # depedent read only
-          struct = pre_val || embed_struct[predicate].new
-          struct.id = object
-          struct.klass = range_for_v
-          objects_new[struct.id] = struct
-          object = struct
-        end
-        object
-      end
-
-      def get_pre_val(id, models_by_id, object, predicate)
-        pre_val = nil
-        if models_by_id[id] &&
-           ((models_by_id[id].respond_to?(:klass) && models_by_id[id]) ||
-             models_by_id[id].loaded_attributes.include?(predicate))
-          pre_val = if !@read_only
-                      models_by_id[id].instance_variable_get("@#{predicate}")
-                    else
-                      models_by_id[id][predicate]
-                    end
-
-          pre_val = pre_val.select { |x| x.id == object }.first if pre_val.is_a?(Array)
-        end
-        pre_val
-      end
-
-      def add_unmapped_to_model(sol, requested_lang)
-        predicate = sol[:attributeProperty].to_s.to_sym
-        return unless @properties_to_include[predicate]
-
-        id = sol[:id]
-        value = sol[:attributeObject]
-
-        model_set_unmapped(id, @properties_to_include[predicate][:uri], value, requested_lang)
-      end
-
-      def add_aggregations_to_model(sol)
-        id = sol[:id]
-        @aggregate_projections&.each do |aggregate_key, aggregate_val|
-          if @models_by_id[id].respond_to?(:add_aggregate)
-            @models_by_id[id].add_aggregate(aggregate_val[1], aggregate_val[0], sol[aggregate_key].object)
-          else
-            (@models_by_id[id].aggregates ||= []) << Goo::Base::AGGREGATE_VALUE.new(aggregate_val[1],
-                                                                                    aggregate_val[0],
-                                                                                    sol[aggregate_key].object)
-          end
-        end
-      end
-    end
-  end
-end
+      end
+    end
+  end
+end

From 085eb41c101c3e924219ad1080bacb218ca52c47 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Fri, 3 Mar 2023 09:56:47 +0100
Subject: [PATCH 074/168] remove the usage of the old lang filter module in
 map_attributes

---
 lib/goo/base/resource.rb | 20 +-------------------
 1 file changed, 1 insertion(+), 19 deletions(-)

diff --git a/lib/goo/base/resource.rb b/lib/goo/base/resource.rb
index 8f7da133..c17191a8 100644
--- a/lib/goo/base/resource.rb
+++ b/lib/goo/base/resource.rb
@@ -383,25 +383,7 @@ def self.map_attributes(inst,equivalent_predicates=nil)
               object = unmapped_string_keys[attr_uri]
             end
 
-            lang_filter = Goo::SPARQL::Solution::LanguageFilter.new
-
-            object = object.map do |o|
-              if o.is_a?(RDF::URI)
-                o
-              else
-                literal = o
-                index, lang_val  = lang_filter.main_lang_filter inst.id.to_s, attr, literal
-                lang_val.to_s if index.eql? :no_lang
-              end
-            end
-
-            object = object.compact
-
-            other_languages_values = lang_filter.other_languages_values
-            other_languages_values = other_languages_values[inst.id.to_s][attr] unless other_languages_values.empty?
-            unless other_languages_values.nil?
-              object = lang_filter.languages_values_to_set(other_languages_values, object)
-            end
+            object = object.map {|o| o.is_a?(RDF::URI) ? o : o.object}
 
             if klass.range(attr)
               object = object.map { |o|

From 48389a03cfd7d203c2f49c5797eee0fbb90d6779 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Sat, 4 Mar 2023 15:52:29 +0100
Subject: [PATCH 075/168] add request language global variable

---
 lib/goo.rb                         | 9 +++++++++
 lib/goo/sparql/solutions_mapper.rb | 3 +--
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/lib/goo.rb b/lib/goo.rb
index eb941d9d..ff0e6279 100644
--- a/lib/goo.rb
+++ b/lib/goo.rb
@@ -32,6 +32,7 @@ module Goo
   # Define the languages from which the properties values will be taken
   # It choose the first language that match otherwise return all the values
   @@main_languages = %w[en]
+  @@requested_language = nil
 
   @@configure_flag = false
   @@sparql_backends = {}
@@ -58,6 +59,14 @@ def self.main_languages=(lang)
     @@main_languages = lang
   end
 
+  def self.requested_language
+    @@requested_language
+  end
+
+  def self.requested_language=(lang)
+    @@requested_language = lang
+  end
+
   def self.language_includes(lang)
     lang_str = lang.to_s
     main_languages.index { |l| lang_str.downcase.eql?(l) || lang_str.upcase.eql?(l)}
diff --git a/lib/goo/sparql/solutions_mapper.rb b/lib/goo/sparql/solutions_mapper.rb
index ca830719..d80aae67 100644
--- a/lib/goo/sparql/solutions_mapper.rb
+++ b/lib/goo/sparql/solutions_mapper.rb
@@ -23,8 +23,7 @@ def initialize(aggregate_projections, bnode_extraction, embed_struct,
         @count = options[:count]
         @collection = options[:collection]
         @objects_by_lang = {}
-        @requested_lang = :EN
-        Goo.main_languages = [:ES, :FR, :EN]
+        @requested_lang =  Goo.requested_language
       end
       
       def map_each_solutions(select)

From 08bdaa16735e60c04d9e2179c02c3775d985a01e Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Sat, 4 Mar 2023 15:59:00 +0100
Subject: [PATCH 076/168] fix datatype check for list values

---
 .../validators/implementations/data_type.rb   | 27 ++++++++++++-------
 test/test_validators.rb                       |  5 +++-
 2 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/lib/goo/validators/implementations/data_type.rb b/lib/goo/validators/implementations/data_type.rb
index 0ea65ab3..04f46d0c 100644
--- a/lib/goo/validators/implementations/data_type.rb
+++ b/lib/goo/validators/implementations/data_type.rb
@@ -29,16 +29,16 @@ def enforce_type(type, value)
         return true if value.nil?
 
         if type == :boolean
-          return self.enforce_type_boolean(value)
+          self.enforce_type_boolean(value)
         elsif type.eql?(:uri) || type.eql?(RDF::URI)
-          return  self.enforce_type_uri(value)
+          self.enforce_type_uri(value)
         elsif type.eql?(:uri) || type.eql?(Array)
-          return value.is_a? Array
+          value.is_a? Array
         else
           if value.is_a? Array
-            return value.select{|x| !x.is_a?(type)}.empty?
+             value.select{|x| !x.is_a?(type)}.empty?
           else
-            return value.is_a? type
+             value.is_a? type
           end
         end
 
@@ -47,19 +47,28 @@ def enforce_type(type, value)
       def enforce_type_uri(value)
         return true  if value.nil?
 
-        value.is_a?(RDF::URI) && value.valid?
+        if value.kind_of? Array
+          value.select { |x| !is_a_uri?(x) }.empty?
+        else
+          is_a_uri?(value)
+        end
+
       end
 
       def enforce_type_boolean(value)
         if value.kind_of? Array
-          return value.select { |x| !is_a_boolean?(x) }.empty?
+          value.select { |x| !is_a_boolean?(x) }.empty?
         else
-          return  is_a_boolean?(value)
+          is_a_boolean?(value)
         end
       end
 
       def is_a_boolean?(value)
-        return (value.class == TrueClass) || (value.class == FalseClass)
+         (value.class == TrueClass) || (value.class == FalseClass)
+      end
+
+      def is_a_uri?(value)
+        value.is_a?(RDF::URI) && value.valid?
       end
     end
   end
diff --git a/test/test_validators.rb b/test/test_validators.rb
index 8795fccf..a8e69dbe 100644
--- a/test/test_validators.rb
+++ b/test/test_validators.rb
@@ -12,6 +12,7 @@ class Person < Goo::Base::Resource
   attribute :birth_date, enforce: [ :date_time ]
   attribute :male, enforce: [:boolean]
   attribute :social, enforce: [:uri]
+  attribute :socials, enforce: [:uri, :list]
   attribute :weight, enforce: [:float]
   attribute :friends, enforce: [Person, :list]
 end
@@ -143,6 +144,7 @@ def test_datatype_validators
     p.birth_date = 100
     p.male =  "ok"
     p.social =  100
+    p.socials =  [100]
     p.weight =  100
 
 
@@ -161,7 +163,8 @@ def test_datatype_validators
     p.one_number = 12
     p.birth_date = DateTime.parse('1978-01-01')
     p.male =  true
-    p.social =  RDF::URI.new('https://test.com/')
+    p.social = RDF::URI.new('https://test.com/')
+    p.socials = [RDF::URI.new('https://test.com/'), RDF::URI.new('https://test.com/')]
     p.weight =  100.0
     #good types are  valid
     assert p.valid?

From 5e5c77252c8652ef06c190e51ee34eef35a9424d Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Sat, 4 Mar 2023 16:03:10 +0100
Subject: [PATCH 077/168] remove old unused test if clause

---
 lib/goo/base/resource.rb | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/lib/goo/base/resource.rb b/lib/goo/base/resource.rb
index fe11f89c..d4459bf4 100644
--- a/lib/goo/base/resource.rb
+++ b/lib/goo/base/resource.rb
@@ -383,11 +383,6 @@ def self.map_attributes(inst,equivalent_predicates=nil)
           else
             inst.send("#{attr}=",
                       list_attrs.include?(attr) ? [] : nil, on_load: true)
-            if inst.id.to_s == "http://purl.obolibrary.org/obo/IAO_0000415"
-              if attr == :definition
-                # binding.pry
-              end
-            end
           end
 
         end

From 4cc7a7fb9f8591957cde3c592ef4c3a59090d3a5 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Sat, 4 Mar 2023 17:11:24 +0100
Subject: [PATCH 078/168] for no unmapped values cast them to object before
 sending

---
 lib/goo/sparql/mixins/solution_lang_filter.rb | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index b6333593..6f1a7aff 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -28,8 +28,7 @@ def store_objects_by_lang(id, predicate, object, language)
           objects_by_lang[id][predicate] ||= {}
           objects_by_lang[id][predicate][language] ||= []
 
-          objects_by_lang[id][predicate][language] << object.object
-
+          objects_by_lang[id][predicate][language] << object
         end
 
         def fill_models_with_other_languages(models_by_id)
@@ -89,12 +88,13 @@ def add_unmapped_to_model(model, predicate, value)
           end
         end
 
-        def save_value_to_model(model, value, predicate, unmapped)
+        def save_value_to_model(model, values, predicate, unmapped)
           if unmapped
-            add_unmapped_to_model(model, predicate, value)
+            add_unmapped_to_model(model, predicate, values)
           else
-            value = Array(value).min unless list_attributes?(predicate)
-            model.send("#{predicate}=", value, on_load: true)
+            values = values.map(&:object)
+            values = Array(values).min unless list_attributes?(predicate)
+            model.send("#{predicate}=", values, on_load: true)
           end
         end
 

From 660e8c525a3c7c992379876efc9dcb4649cfe59c Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Sat, 4 Mar 2023 17:11:49 +0100
Subject: [PATCH 079/168] for resource unmapped_set merge new value if an array

---
 lib/goo/base/resource.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/goo/base/resource.rb b/lib/goo/base/resource.rb
index d4459bf4..44cacdda 100644
--- a/lib/goo/base/resource.rb
+++ b/lib/goo/base/resource.rb
@@ -119,8 +119,8 @@ def missing_load_attributes
 
       def unmapped_set(attribute,value)
         @unmapped ||= {}
-        @unmapped[attribute] ||= Set.new 
-        @unmapped[attribute] << value unless value.nil?
+        @unmapped[attribute] ||= Set.new
+        @unmapped[attribute].merge(Array(value)) unless value.nil?
       end
  
       def unmapped_get(attribute)

From d30c5748f27c05d88d5db423f7c4b2355412483b Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Sat, 4 Mar 2023 17:12:02 +0100
Subject: [PATCH 080/168] prevent add_object_to_model if no_lang and previous
 value exist

---
 lib/goo/sparql/solutions_mapper.rb | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/lib/goo/sparql/solutions_mapper.rb b/lib/goo/sparql/solutions_mapper.rb
index d80aae67..d54e0418 100644
--- a/lib/goo/sparql/solutions_mapper.rb
+++ b/lib/goo/sparql/solutions_mapper.rb
@@ -186,7 +186,11 @@ def add_object_to_model(id, objects, current_obj, predicate, language)
               !(objects.nil? && !@models_by_id[id].instance_variable_get("@#{predicate}").nil?) &&
               predicate != :id
 
-          if language.nil? || @lang_filter.language_match?(language)
+          if language.nil?
+            return @models_by_id[id].send("#{predicate}=", objects, on_load: true)
+          elsif @lang_filter.language_match?(language)
+            return if language.eql?(:no_lang) && !@models_by_id[id].instance_variable_get("@#{predicate}").nil? && !objects.is_a?(Array)
+
             return @models_by_id[id].send("#{predicate}=", objects, on_load: true)
           end
 

From 96e57326ebdf454d92929cb4a27e6bfbb3e424e4 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Sat, 4 Mar 2023 22:04:21 +0100
Subject: [PATCH 081/168] move from the mapper  lang_filter related code to 
 lang_filter module

---
 lib/goo/sparql/mixins/solution_lang_filter.rb | 33 ++++++++++++++-
 lib/goo/sparql/solutions_mapper.rb            | 40 ++++---------------
 2 files changed, 38 insertions(+), 35 deletions(-)

diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index 6f1a7aff..f624a8d2 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -10,7 +10,7 @@ def initialize(requested_lang: nil, unmapped: false, list_attributes: [])
           @objects_by_lang = {}
           @unmapped = unmapped
           @requested_lang = requested_lang
-
+          @fill_other_languages = init_requested_lang
         end
 
         def object_language(new_value)
@@ -33,6 +33,8 @@ def store_objects_by_lang(id, predicate, object, language)
 
         def fill_models_with_other_languages(models_by_id)
 
+          return unless fill_other_languages?
+
           other_platform_languages = Goo.main_languages[1..] || []
 
           objects_by_lang.each do |id, predicates|
@@ -57,8 +59,21 @@ def fill_models_with_other_languages(models_by_id)
         end
         
 
+        def model_set_value(model, predicate, objects, object)
+          language = object_language(object) # if lang is nil, it means that the object is not a literal
+          if language.nil?
+            return model.send("#{predicate}=", objects, on_load: true)
+          elsif language_match?(language)
+            return if language.eql?(:no_lang) && !model.instance_variable_get("@#{predicate}").nil? && !objects.is_a?(Array)
+
+            return model.send("#{predicate}=", objects, on_load: true)
+          end
 
-        def model_set_unmapped(model, predicate, value, language)
+          store_objects_by_lang(model.id, predicate, object, language)
+        end
+
+        def model_set_unmapped(model, predicate, value)
+          language = object_language(value)
           if language.nil? || language_match?(language)
             return add_unmapped_to_model(model, predicate, value)
           end
@@ -69,6 +84,15 @@ def model_set_unmapped(model, predicate, value, language)
 
         private
 
+        def init_requested_lang
+          if @requested_lang.nil?
+            @requested_lang = Goo.main_languages[0] || :EN
+            return true
+          end
+
+          false
+        end
+
         def get_model_attribute_value(model, predicate)
           if unmapped
             unmapped_get(model, predicate)
@@ -111,6 +135,11 @@ def list_attributes?(predicate)
           @list_attributes.include?(predicate)
         end
 
+
+        def fill_other_languages?
+          @fill_other_languages
+        end
+
       end
     end
   end
diff --git a/lib/goo/sparql/solutions_mapper.rb b/lib/goo/sparql/solutions_mapper.rb
index d54e0418..b29814b2 100644
--- a/lib/goo/sparql/solutions_mapper.rb
+++ b/lib/goo/sparql/solutions_mapper.rb
@@ -22,13 +22,11 @@ def initialize(aggregate_projections, bnode_extraction, embed_struct,
         @incl = options[:include]
         @count = options[:count]
         @collection = options[:collection]
-        @objects_by_lang = {}
         @requested_lang =  Goo.requested_language
       end
       
       def map_each_solutions(select)
         found = Set.new
-        fill_models_with_platform_languages = init_requested_lang
         objects_new = {}
         list_attributes = Set.new(@klass.attributes(:list))
         all_attributes = Set.new(@klass.attributes(:all))
@@ -72,17 +70,12 @@ def map_each_solutions(select)
             next
           end
 
-          lang = @lang_filter.object_language(object) # if lang is nil, it means that the object is not a literal
-
-
           objects, objects_new = get_value_object(id, objects_new, object, list_attributes, predicate)
-          add_object_to_model(id, objects, object, predicate, lang)
+          add_object_to_model(id, objects, object, predicate)
         end
       
-  
-        if fill_models_with_platform_languages
-          @lang_filter.fill_models_with_other_languages(@models_by_id)
-        end
+
+        @lang_filter.fill_models_with_other_languages(@models_by_id)
 
         init_unloaded_attributes(found, list_attributes)
 
@@ -109,15 +102,6 @@ def map_each_solutions(select)
 
       private
 
-      def init_requested_lang 
-        if @requested_lang.nil?
-          @requested_lang = Goo.main_languages[0] || :EN
-          return true
-        end
-        
-        false
-      end
-
       def init_unloaded_attributes(found, list_attributes)
         return if @incl.nil?
 
@@ -179,22 +163,14 @@ def get_value_object(id, objects_new, object, list_attributes, predicate)
         [object, objects_new]
       end
 
-      def add_object_to_model(id, objects, current_obj, predicate, language) 
+      def add_object_to_model(id, objects, current_obj, predicate)
+
         if @models_by_id[id].respond_to?(:klass)
           @models_by_id[id][predicate] = objects unless objects.nil? && !@models_by_id[id][predicate].nil?
         elsif !@models_by_id[id].class.handler?(predicate) &&
               !(objects.nil? && !@models_by_id[id].instance_variable_get("@#{predicate}").nil?) &&
               predicate != :id
-
-          if language.nil?
-            return @models_by_id[id].send("#{predicate}=", objects, on_load: true)
-          elsif @lang_filter.language_match?(language)
-            return if language.eql?(:no_lang) && !@models_by_id[id].instance_variable_get("@#{predicate}").nil? && !objects.is_a?(Array)
-
-            return @models_by_id[id].send("#{predicate}=", objects, on_load: true)
-          end
-
-          @lang_filter.store_objects_by_lang(id, predicate, current_obj, language)
+          @lang_filter.model_set_value(@models_by_id[id], predicate, objects, current_obj)
         end
       end
 
@@ -435,9 +411,7 @@ def add_unmapped_to_model(sol)
         id = sol[:id]
         value = sol[:attributeObject]
 
-        language = @lang_filter.object_language(value)
-
-        @lang_filter.model_set_unmapped(@models_by_id[id], @properties_to_include[predicate][:uri], value, language)
+        @lang_filter.model_set_unmapped(@models_by_id[id], @properties_to_include[predicate][:uri], value)
       end
 
       def add_aggregations_to_model(sol)

From 92bed925de041eb5a40adbe8788a74c6c7cb1020 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Sat, 4 Mar 2023 22:18:26 +0100
Subject: [PATCH 082/168] move internal lang filter module methods to private
 section

---
 lib/goo/sparql/mixins/solution_lang_filter.rb | 35 ++++++++++---------
 1 file changed, 18 insertions(+), 17 deletions(-)

diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index f624a8d2..58bde592 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -13,23 +13,6 @@ def initialize(requested_lang: nil, unmapped: false, list_attributes: [])
           @fill_other_languages = init_requested_lang
         end
 
-        def object_language(new_value)
-          new_value.language || :no_lang if new_value.is_a?(RDF::Literal)
-        end
-  
-        def language_match?(language)
-          !language.nil? && (language.eql?(requested_lang) || language.eql?(:no_lang) || requested_lang.nil?)
-        end
-
-        def store_objects_by_lang(id, predicate, object, language)
-          # store objects in this format: [id][predicate][language] = [objects]
-
-          objects_by_lang[id] ||= {}
-          objects_by_lang[id][predicate] ||= {}
-          objects_by_lang[id][predicate][language] ||= []
-
-          objects_by_lang[id][predicate][language] << object
-        end
 
         def fill_models_with_other_languages(models_by_id)
 
@@ -84,6 +67,24 @@ def model_set_unmapped(model, predicate, value)
 
         private
 
+        def object_language(new_value)
+          new_value.language || :no_lang if new_value.is_a?(RDF::Literal)
+        end
+        
+        def language_match?(language)
+          !language.nil? && (language.eql?(requested_lang) || language.eql?(:no_lang) || requested_lang.nil?)
+        end
+
+        def store_objects_by_lang(id, predicate, object, language)
+          # store objects in this format: [id][predicate][language] = [objects]
+
+          objects_by_lang[id] ||= {}
+          objects_by_lang[id][predicate] ||= {}
+          objects_by_lang[id][predicate][language] ||= []
+
+          objects_by_lang[id][predicate][language] << object
+        end
+
         def init_requested_lang
           if @requested_lang.nil?
             @requested_lang = Goo.main_languages[0] || :EN

From bcbf686d4373a41a214f2094beeedef920c33f6f Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Sat, 4 Mar 2023 23:12:23 +0100
Subject: [PATCH 083/168] add request_store gem to save request language
 globally

---
 Gemfile      |  1 +
 Gemfile.lock | 15 +++++++++------
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/Gemfile b/Gemfile
index edb00975..3564fe3b 100644
--- a/Gemfile
+++ b/Gemfile
@@ -7,6 +7,7 @@ gem "cube-ruby", require: "cube"
 gem "faraday", '~> 1.9'
 gem "rake"
 gem "uuid"
+gem "request_store"
 
 group :test do
   gem "minitest", '< 5.0'
diff --git a/Gemfile.lock b/Gemfile.lock
index 34a6c39c..9fe7bd02 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -34,7 +34,7 @@ GEM
       public_suffix (>= 2.0.2, < 6.0)
     builder (3.2.4)
     coderay (1.1.3)
-    concurrent-ruby (1.1.10)
+    concurrent-ruby (1.2.2)
     connection_pool (2.3.0)
     cube-ruby (0.0.3)
     daemons (1.4.1)
@@ -76,10 +76,10 @@ GEM
     method_source (1.0.0)
     mime-types (3.4.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2022.0105)
+    mime-types-data (3.2023.0218.1)
     minitest (4.7.5)
     multi_json (1.15.0)
-    multipart-post (2.2.3)
+    multipart-post (2.3.0)
     mustermann (3.0.0)
       ruby2_keywords (~> 0.0.1)
     net-http-persistent (2.9.4)
@@ -88,7 +88,7 @@ GEM
       coderay (~> 1.1)
       method_source (~> 1.0)
     public_suffix (5.0.1)
-    rack (2.2.6.2)
+    rack (2.2.6.3)
     rack-accept (0.4.5)
       rack (>= 0.4)
     rack-post-body-to-params (0.1.8)
@@ -100,8 +100,10 @@ GEM
       addressable (>= 2.2)
     redis (5.0.6)
       redis-client (>= 0.9.0)
-    redis-client (0.12.1)
+    redis-client (0.13.0)
       connection_pool
+    request_store (1.5.1)
+      rack (>= 1.4)
     rest-client (2.1.0)
       http-accept (>= 1.7.0, < 2.0)
       http-cookie (>= 1.0.2, < 2.0)
@@ -132,7 +134,7 @@ GEM
       eventmachine (~> 1.0, >= 1.0.4)
       rack (>= 1, < 3)
     thread_safe (0.3.6)
-    tilt (2.0.11)
+    tilt (2.1.0)
     tzinfo (0.3.61)
     unf (0.1.4)
       unf_ext
@@ -155,6 +157,7 @@ DEPENDENCIES
   rack-accept
   rack-post-body-to-params
   rake
+  request_store
   simplecov
   simplecov-cobertura
   sinatra

From f1a5845df36d70deaed54d146f38ee93b54976a3 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Sat, 4 Mar 2023 23:13:00 +0100
Subject: [PATCH 084/168] save requested  language in model_load options

---
 lib/goo/sparql/loader.rb           | 6 ++++++
 lib/goo/sparql/solutions_mapper.rb | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/lib/goo/sparql/loader.rb b/lib/goo/sparql/loader.rb
index 821aba26..094fbba2 100644
--- a/lib/goo/sparql/loader.rb
+++ b/lib/goo/sparql/loader.rb
@@ -1,3 +1,4 @@
+require 'request_store'
 module Goo
   module SPARQL
     module Loader
@@ -6,8 +7,10 @@ class << self
 
         def model_load(*options)
           options = options.last
+          set_request_lang(options)
           if options[:models] && options[:models].is_a?(Array) && \
            (options[:models].length > Goo.slice_loading_size)
+
             options = options.dup
             models = options[:models]
             include_options = options[:include]
@@ -96,6 +99,9 @@ def model_load_sliced(*options)
 
         private
 
+        def set_request_lang(options)
+          options[:requested_lang] = RequestStore.store[:requested_lang]
+        end
         def expand_equivalent_predicates(properties_to_include, eq_p)
 
           return unless eq_p && !eq_p.empty?
diff --git a/lib/goo/sparql/solutions_mapper.rb b/lib/goo/sparql/solutions_mapper.rb
index b29814b2..b9bf19d3 100644
--- a/lib/goo/sparql/solutions_mapper.rb
+++ b/lib/goo/sparql/solutions_mapper.rb
@@ -22,7 +22,7 @@ def initialize(aggregate_projections, bnode_extraction, embed_struct,
         @incl = options[:include]
         @count = options[:count]
         @collection = options[:collection]
-        @requested_lang =  Goo.requested_language
+        @requested_lang =  options[:requested_lang]
       end
       
       def map_each_solutions(select)

From ac8709a86a4c4d2178d34de0e18367bcd131ebed Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Sun, 5 Mar 2023 23:22:05 +0100
Subject: [PATCH 085/168] force requested_lang and portal_langs to be upcase
 and symbol

---
 lib/goo/sparql/mixins/solution_lang_filter.rb | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index 58bde592..192372d2 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -11,6 +11,7 @@ def initialize(requested_lang: nil, unmapped: false, list_attributes: [])
           @unmapped = unmapped
           @requested_lang = requested_lang
           @fill_other_languages = init_requested_lang
+          @requested_lang = @requested_lang.to_s.upcase.to_sym
         end
 
 
@@ -27,7 +28,7 @@ def fill_models_with_other_languages(models_by_id)
               next unless model_attribute_val.nil? || model_attribute_val.empty?
 
               other_platform_languages.each do |platform_language|
-                if languages[platform_language]
+                if languages[platform_language.to_s.upcase.to_sym]
                   save_value_to_model(model, languages[platform_language], predicate, unmapped)
                   break
                 end
@@ -70,7 +71,7 @@ def model_set_unmapped(model, predicate, value)
         def object_language(new_value)
           new_value.language || :no_lang if new_value.is_a?(RDF::Literal)
         end
-        
+
         def language_match?(language)
           !language.nil? && (language.eql?(requested_lang) || language.eql?(:no_lang) || requested_lang.nil?)
         end

From 8dbb74b57ab2286c865a001c0f61b4e5398cd979 Mon Sep 17 00:00:00 2001
From: HADDAD Zineddine <hz_haddad@esi.dz>
Date: Mon, 6 Mar 2023 13:35:20 +0100
Subject: [PATCH 086/168] change methodes/vars names

---
 lib/goo/sparql/mixins/solution_lang_filter.rb | 11 +++++------
 lib/goo/sparql/solutions_mapper.rb            |  4 ++--
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index 58bde592..56e9a963 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -13,8 +13,7 @@ def initialize(requested_lang: nil, unmapped: false, list_attributes: [])
           @fill_other_languages = init_requested_lang
         end
 
-
-        def fill_models_with_other_languages(models_by_id)
+        def enrich_models(models_by_id)
 
           return unless fill_other_languages?
 
@@ -28,13 +27,13 @@ def fill_models_with_other_languages(models_by_id)
 
               other_platform_languages.each do |platform_language|
                 if languages[platform_language]
-                  save_value_to_model(model, languages[platform_language], predicate, unmapped)
+                  save_model_values(model, languages[platform_language], predicate, unmapped)
                   break
                 end
               end
               model_attribute_val = get_model_attribute_value(model, predicate)
               if model_attribute_val.nil? || model_attribute_val.empty?
-                save_value_to_model(model, languages.values.flatten.uniq, predicate, unmapped)
+                save_model_values(model, languages.values.flatten.uniq, predicate, unmapped)
               end
             end
           end
@@ -42,7 +41,7 @@ def fill_models_with_other_languages(models_by_id)
         end
         
 
-        def model_set_value(model, predicate, objects, object)
+        def set_model_value(model, predicate, objects, object)
           language = object_language(object) # if lang is nil, it means that the object is not a literal
           if language.nil?
             return model.send("#{predicate}=", objects, on_load: true)
@@ -113,7 +112,7 @@ def add_unmapped_to_model(model, predicate, value)
           end
         end
 
-        def save_value_to_model(model, values, predicate, unmapped)
+        def save_model_values(model, values, predicate, unmapped)
           if unmapped
             add_unmapped_to_model(model, predicate, values)
           else
diff --git a/lib/goo/sparql/solutions_mapper.rb b/lib/goo/sparql/solutions_mapper.rb
index b9bf19d3..3117b39c 100644
--- a/lib/goo/sparql/solutions_mapper.rb
+++ b/lib/goo/sparql/solutions_mapper.rb
@@ -75,7 +75,7 @@ def map_each_solutions(select)
         end
       
 
-        @lang_filter.fill_models_with_other_languages(@models_by_id)
+        @lang_filter.enrich_models(@models_by_id)
 
         init_unloaded_attributes(found, list_attributes)
 
@@ -170,7 +170,7 @@ def add_object_to_model(id, objects, current_obj, predicate)
         elsif !@models_by_id[id].class.handler?(predicate) &&
               !(objects.nil? && !@models_by_id[id].instance_variable_get("@#{predicate}").nil?) &&
               predicate != :id
-          @lang_filter.model_set_value(@models_by_id[id], predicate, objects, current_obj)
+          @lang_filter.set_model_value(@models_by_id[id], predicate, objects, current_obj)
         end
       end
 

From 996922a9dfae06da9a7214e1322e1d403d3f1b39 Mon Sep 17 00:00:00 2001
From: HADDAD Zineddine <hz_haddad@esi.dz>
Date: Mon, 6 Mar 2023 13:50:19 +0100
Subject: [PATCH 087/168] get the last item in objects instead of passing the
 current object

---
 lib/goo/sparql/mixins/solution_lang_filter.rb | 8 +++++---
 lib/goo/sparql/solutions_mapper.rb            | 6 +++---
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index 1aec2b99..0ca882a8 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -42,8 +42,9 @@ def enrich_models(models_by_id)
         end
         
 
-        def set_model_value(model, predicate, objects, object)
-          language = object_language(object) # if lang is nil, it means that the object is not a literal
+        def set_model_value(model, predicate, objects)
+          new_value = Array(objects).last
+          language = object_language(new_value) # if lang is nil, it means that the object is not a literal
           if language.nil?
             return model.send("#{predicate}=", objects, on_load: true)
           elsif language_match?(language)
@@ -52,7 +53,8 @@ def set_model_value(model, predicate, objects, object)
             return model.send("#{predicate}=", objects, on_load: true)
           end
 
-          store_objects_by_lang(model.id, predicate, object, language)
+
+          store_objects_by_lang(model.id, predicate, new_value, language)
         end
 
         def model_set_unmapped(model, predicate, value)
diff --git a/lib/goo/sparql/solutions_mapper.rb b/lib/goo/sparql/solutions_mapper.rb
index 3117b39c..8a102aac 100644
--- a/lib/goo/sparql/solutions_mapper.rb
+++ b/lib/goo/sparql/solutions_mapper.rb
@@ -71,7 +71,7 @@ def map_each_solutions(select)
           end
 
           objects, objects_new = get_value_object(id, objects_new, object, list_attributes, predicate)
-          add_object_to_model(id, objects, object, predicate)
+          add_object_to_model(id, objects, predicate)
         end
       
 
@@ -163,14 +163,14 @@ def get_value_object(id, objects_new, object, list_attributes, predicate)
         [object, objects_new]
       end
 
-      def add_object_to_model(id, objects, current_obj, predicate)
+      def add_object_to_model(id, objects, predicate)
 
         if @models_by_id[id].respond_to?(:klass)
           @models_by_id[id][predicate] = objects unless objects.nil? && !@models_by_id[id][predicate].nil?
         elsif !@models_by_id[id].class.handler?(predicate) &&
               !(objects.nil? && !@models_by_id[id].instance_variable_get("@#{predicate}").nil?) &&
               predicate != :id
-          @lang_filter.set_model_value(@models_by_id[id], predicate, objects, current_obj)
+          @lang_filter.set_model_value(@models_by_id[id], predicate, objects)
         end
       end
 

From b769c165906163e30a026dba511ae1069c4eed3d Mon Sep 17 00:00:00 2001
From: HADDAD Zineddine <hz_haddad@esi.dz>
Date: Mon, 6 Mar 2023 13:50:19 +0100
Subject: [PATCH 088/168] Revert "get the last item in objects instead of
 passing the current object"

This reverts commit 996922a9dfae06da9a7214e1322e1d403d3f1b39.
---
 lib/goo/sparql/mixins/solution_lang_filter.rb | 8 +++-----
 lib/goo/sparql/solutions_mapper.rb            | 6 +++---
 2 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index 0ca882a8..1aec2b99 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -42,9 +42,8 @@ def enrich_models(models_by_id)
         end
         
 
-        def set_model_value(model, predicate, objects)
-          new_value = Array(objects).last
-          language = object_language(new_value) # if lang is nil, it means that the object is not a literal
+        def set_model_value(model, predicate, objects, object)
+          language = object_language(object) # if lang is nil, it means that the object is not a literal
           if language.nil?
             return model.send("#{predicate}=", objects, on_load: true)
           elsif language_match?(language)
@@ -53,8 +52,7 @@ def set_model_value(model, predicate, objects)
             return model.send("#{predicate}=", objects, on_load: true)
           end
 
-
-          store_objects_by_lang(model.id, predicate, new_value, language)
+          store_objects_by_lang(model.id, predicate, object, language)
         end
 
         def model_set_unmapped(model, predicate, value)
diff --git a/lib/goo/sparql/solutions_mapper.rb b/lib/goo/sparql/solutions_mapper.rb
index 8a102aac..3117b39c 100644
--- a/lib/goo/sparql/solutions_mapper.rb
+++ b/lib/goo/sparql/solutions_mapper.rb
@@ -71,7 +71,7 @@ def map_each_solutions(select)
           end
 
           objects, objects_new = get_value_object(id, objects_new, object, list_attributes, predicate)
-          add_object_to_model(id, objects, predicate)
+          add_object_to_model(id, objects, object, predicate)
         end
       
 
@@ -163,14 +163,14 @@ def get_value_object(id, objects_new, object, list_attributes, predicate)
         [object, objects_new]
       end
 
-      def add_object_to_model(id, objects, predicate)
+      def add_object_to_model(id, objects, current_obj, predicate)
 
         if @models_by_id[id].respond_to?(:klass)
           @models_by_id[id][predicate] = objects unless objects.nil? && !@models_by_id[id][predicate].nil?
         elsif !@models_by_id[id].class.handler?(predicate) &&
               !(objects.nil? && !@models_by_id[id].instance_variable_get("@#{predicate}").nil?) &&
               predicate != :id
-          @lang_filter.set_model_value(@models_by_id[id], predicate, objects)
+          @lang_filter.set_model_value(@models_by_id[id], predicate, objects, current_obj)
         end
       end
 

From 30ed9f9be4529e873d6cb728effe730a8f3c1670 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Tue, 7 Mar 2023 19:32:47 +0100
Subject: [PATCH 089/168] handle this case where values is nil in
 save_model_values

---
 lib/goo/sparql/mixins/solution_lang_filter.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index 1aec2b99..8102c82e 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -117,8 +117,8 @@ def save_model_values(model, values, predicate, unmapped)
           if unmapped
             add_unmapped_to_model(model, predicate, values)
           else
-            values = values.map(&:object)
-            values = Array(values).min unless list_attributes?(predicate)
+            values = Array(values).map(&:object)
+            values = values.min unless list_attributes?(predicate)
             model.send("#{predicate}=", values, on_load: true)
           end
         end

From 07b06c31937dacb3bd0e7bcaf2e8a024d145bd31 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Tue, 14 Mar 2023 22:59:57 +0100
Subject: [PATCH 090/168] handle the casf of nil values for  the
 SuperiorEqualTo validator

---
 lib/goo/validators/implementations/superior_equal_to.rb | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/goo/validators/implementations/superior_equal_to.rb b/lib/goo/validators/implementations/superior_equal_to.rb
index 91508f30..38012010 100644
--- a/lib/goo/validators/implementations/superior_equal_to.rb
+++ b/lib/goo/validators/implementations/superior_equal_to.rb
@@ -11,10 +11,10 @@ class SuperiorEqualTo < ValidatorBase
 
       validity_check -> (obj) do
         target_values = self.class.attr_value(@property, @inst)
+        target_value = target_values.first
+        return true if target_value.nil? || @value.nil?
 
-        return true if target_values.empty?
-
-        return @value >= target_values.first
+        return @value >= target_value
       end
 
       def initialize(inst, attr, value, key)

From c83687e9a65c25313df6d74be96e83ace9d939c7 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Thu, 16 Mar 2023 02:21:57 +0100
Subject: [PATCH 091/168] add onUpdate callback tests

---
 test/test_update_callbacks.rb | 53 +++++++++++++++++++++++++++++++++++
 1 file changed, 53 insertions(+)
 create mode 100644 test/test_update_callbacks.rb

diff --git a/test/test_update_callbacks.rb b/test/test_update_callbacks.rb
new file mode 100644
index 00000000..bef38a68
--- /dev/null
+++ b/test/test_update_callbacks.rb
@@ -0,0 +1,53 @@
+require_relative 'test_case'
+
+
+require_relative 'models'
+
+class TestUpdateCallBack < Goo::Base::Resource
+  model :update_callback_model, name_with: :code
+  attribute :code, enforce: [:string, :existence]
+  attribute :name, enforce: [:string, :existence]
+  attribute :first_name, onUpdate: :update_name
+  attribute :last_name, onUpdate: :update_name
+
+
+  def update_name(inst, attr)
+    self.name = self.first_name + self.last_name
+  end
+end
+
+class TestUpdateCallBacks < MiniTest::Unit::TestCase
+
+  def self.before_suite
+    GooTestData.delete_all [TestUpdateCallBack]
+  end
+
+  def self.after_suite
+    GooTestData.delete_all [TestUpdateCallBack]
+  end
+
+
+  def test_update_callback
+    p = TestUpdateCallBack.new
+    p.code = "1"
+    p.name = "name"
+    p.first_name = "first_name"
+    p.last_name = "last_name"
+
+    assert p.valid?
+    p.save
+
+    p.bring_remaining
+
+    assert_equal p.first_name + p.last_name, p.name
+
+    p.last_name = "last_name2"
+    p.save
+
+    p.bring_remaining
+    assert_equal  "last_name2",  p.last_name
+    assert_equal p.first_name + p.last_name, p.name
+  end
+
+end
+

From e4979ffbb8e1d203d0db9368be1871ba172a5b69 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Thu, 16 Mar 2023 02:22:46 +0100
Subject: [PATCH 092/168] implement enforce_callback to run an attribute
 callback

---
 lib/goo/validators/enforce.rb | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/lib/goo/validators/enforce.rb b/lib/goo/validators/enforce.rb
index 1a53eaf0..3c90e204 100644
--- a/lib/goo/validators/enforce.rb
+++ b/lib/goo/validators/enforce.rb
@@ -64,6 +64,17 @@ def enforce(inst,attr,value)
           errors_by_opt.length > 0 ? errors_by_opt : nil
         end
 
+        def enforce_callback(inst, attr)
+          callbacks = Array(inst.class.update_callbacks(attr))
+          callbacks.each do |proc|
+            if instance_proc?(inst, proc)
+              call_proc(inst.method(proc), inst, attr)
+            elsif proc.is_a?(Proc)
+              call_proc(proc, inst, attr)
+            end
+          end
+        end
+
         private
 
         def object_type(opt)
@@ -115,6 +126,10 @@ def add_error(opt, err)
       def self.enforce(inst,attr,value)
         EnforceInstance.new.enforce(inst,attr,value)
       end
+
+      def self.enforce_callbacks(inst, attr)
+        EnforceInstance.new.enforce_callback(inst, attr)
+      end
     end
   end
 end

From c7f7092a228b563d4637533131446015d0f36957 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Thu, 16 Mar 2023 02:24:19 +0100
Subject: [PATCH 093/168] move the attribute default callback to the save
 method

---
 lib/goo/base/resource.rb  | 11 +++++++++++
 lib/goo/sparql/triples.rb | 10 ----------
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/lib/goo/base/resource.rb b/lib/goo/base/resource.rb
index 44cacdda..cdd2d755 100644
--- a/lib/goo/base/resource.rb
+++ b/lib/goo/base/resource.rb
@@ -236,6 +236,17 @@ def save(*opts)
           raise Goo::Base::NotValidException, "Object is not valid. Check errors." unless valid?
         end
 
+        #set default values before saving
+        unless self.persistent?
+          self.class.attributes_with_defaults.each do |attr|
+            value = self.send("#{attr}")
+            if value.nil?
+              value = self.class.default(attr).call(self)
+              self.send("#{attr}=", value)
+            end
+          end
+        end
+
         graph_insert, graph_delete = Goo::SPARQL::Triples.model_update_triples(self)
         graph = self.graph()
         if graph_delete and graph_delete.size > 0
diff --git a/lib/goo/sparql/triples.rb b/lib/goo/sparql/triples.rb
index cb840df9..df3f9f1d 100644
--- a/lib/goo/sparql/triples.rb
+++ b/lib/goo/sparql/triples.rb
@@ -67,16 +67,6 @@ def self.model_update_triples(model)
         unless model.persistent?
           graph_insert << [subject, RDF.type, model.class.uri_type(model.collection)]
         end
-        #set default values before saving
-        if not model.persistent?
-          model.class.attributes_with_defaults.each do |attr|
-            value = model.send("#{attr}")
-            if value.nil?
-              value = model.class.default(attr).call(model)
-              model.send("#{attr}=",value)
-            end
-          end
-        end
 
         model.modified_attributes.each do |attr|
           next if model.class.collection?(attr)

From f81b15fd04c10e563d40bf0ab724fa9d810a18ce Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Thu, 16 Mar 2023 02:25:06 +0100
Subject: [PATCH 094/168] implement onUpdate DSL in the ressource settings

---
 lib/goo/base/settings/settings.rb | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/lib/goo/base/settings/settings.rb b/lib/goo/base/settings/settings.rb
index 3d343d5d..5f669d08 100644
--- a/lib/goo/base/settings/settings.rb
+++ b/lib/goo/base/settings/settings.rb
@@ -96,6 +96,16 @@ def attributes_with_defaults
                   select{ |attr,opts| opts[:default] }).keys()
         end
 
+        def attributes_with_update_callbacks
+           (@model_settings[:attributes].
+            select{ |attr,opts| opts[:onUpdate] }).keys
+        end
+
+
+        def update_callbacks(attr)
+          @model_settings[:attributes][attr][:onUpdate]
+        end
+
         def default(attr)
           return @model_settings[:attributes][attr][:default]
         end

From 714f085983a2695f4ea8caf7dea00b8883139cb3 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Thu, 16 Mar 2023 02:25:49 +0100
Subject: [PATCH 095/168] call to the attributes onUpdate callback in the save
 method

---
 lib/goo/base/resource.rb | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/lib/goo/base/resource.rb b/lib/goo/base/resource.rb
index cdd2d755..eddbf273 100644
--- a/lib/goo/base/resource.rb
+++ b/lib/goo/base/resource.rb
@@ -225,10 +225,13 @@ def save(*opts)
           raise ArgumentError, "Enums can only be created on initialization" unless opts[0] && opts[0][:init_enum]
         end
         batch_file = nil
-        if opts && opts.length > 0
-          if opts.first.is_a?(Hash) && opts.first[:batch] && opts.first[:batch].is_a?(File)
+        callbacks = true
+        if opts && opts.length > 0 && opts.first.is_a?(Hash)
+          if opts.first[:batch] && opts.first[:batch].is_a?(File)
             batch_file = opts.first[:batch]
           end
+
+          callbacks = opts.first[:callbacks]
         end
 
         if !batch_file
@@ -247,8 +250,17 @@ def save(*opts)
           end
         end
 
+        #call update callback before saving
+        if callbacks
+          self.class.attributes_with_update_callbacks.each do |attr|
+            Goo::Validators::Enforce.enforce_callbacks(self, attr)
+          end
+        end
+
         graph_insert, graph_delete = Goo::SPARQL::Triples.model_update_triples(self)
-        graph = self.graph()
+        graph = self.graph
+
+
         if graph_delete and graph_delete.size > 0
           begin
             Goo.sparql_update_client.delete_data(graph_delete, graph: graph)

From f8caff72119c592ce77adc42520c3ee1924d0ebd Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Fri, 17 Mar 2023 03:30:39 +0100
Subject: [PATCH 096/168] in validators bring attribute if needed

---
 lib/goo/validators/validator.rb | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/goo/validators/validator.rb b/lib/goo/validators/validator.rb
index c133c33e..22d453ac 100644
--- a/lib/goo/validators/validator.rb
+++ b/lib/goo/validators/validator.rb
@@ -75,6 +75,8 @@ def equivalent_value?(object1, object2)
         end
 
         def attr_value(attr, object)
+          object.bring attr if object.respond_to?(:bring?) && object.bring?(attr)
+
           Array(object.send(attr))
         end
 

From 7de77a0b000b61735857f2cc60e2ebda6e5c7878 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Sun, 19 Mar 2023 09:16:58 +0100
Subject: [PATCH 097/168] make superior_equal_to works for list attributes

---
 lib/goo/validators/implementations/superior_equal_to.rb | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/goo/validators/implementations/superior_equal_to.rb b/lib/goo/validators/implementations/superior_equal_to.rb
index 38012010..46676794 100644
--- a/lib/goo/validators/implementations/superior_equal_to.rb
+++ b/lib/goo/validators/implementations/superior_equal_to.rb
@@ -11,10 +11,10 @@ class SuperiorEqualTo < ValidatorBase
 
       validity_check -> (obj) do
         target_values = self.class.attr_value(@property, @inst)
-        target_value = target_values.first
-        return true if target_value.nil? || @value.nil?
 
-        return @value >= target_value
+        return true if target_values.nil? || target_values.empty?
+
+        return  Array(@value).all? {|v| v.nil? || target_values.all?{|t_v| v >= t_v}}
       end
 
       def initialize(inst, attr, value, key)

From 5ca8b16ab99d4d714d22359885f0b0755b6dbb84 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Sun, 19 Mar 2023 14:44:31 +0100
Subject: [PATCH 098/168] add email validator test

---
 test/test_validators.rb | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/test/test_validators.rb b/test/test_validators.rb
index a8e69dbe..5110da80 100644
--- a/test/test_validators.rb
+++ b/test/test_validators.rb
@@ -12,6 +12,7 @@ class Person < Goo::Base::Resource
   attribute :birth_date, enforce: [ :date_time ]
   attribute :male, enforce: [:boolean]
   attribute :social, enforce: [:uri]
+  attribute :email, enforce: [:email]
   attribute :socials, enforce: [:uri, :list]
   attribute :weight, enforce: [:float]
   attribute :friends, enforce: [Person, :list]
@@ -146,8 +147,7 @@ def test_datatype_validators
     p.social =  100
     p.socials =  [100]
     p.weight =  100
-
-
+    p.email = "test@test"
     #wrong types are not valid
     refute p.valid?
     assert p.errors[:last_name][:string]
@@ -157,6 +157,7 @@ def test_datatype_validators
     assert p.errors[:birth_date][:date_time]
     assert p.errors[:male][:boolean]
     assert p.errors[:social][:uri]
+    assert p.errors[:email][:email]
 
     p.last_name =  "hello"
     p.multiple_values = [22,11]
@@ -166,6 +167,7 @@ def test_datatype_validators
     p.social = RDF::URI.new('https://test.com/')
     p.socials = [RDF::URI.new('https://test.com/'), RDF::URI.new('https://test.com/')]
     p.weight =  100.0
+    p.email = "test@test.hi.com"
     #good types are  valid
     assert p.valid?
   end

From 6f4f50f82a14f6e9d1ddbd0d7b80b32632d205ea Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Sun, 19 Mar 2023 14:44:46 +0100
Subject: [PATCH 099/168] implement email validator

---
 lib/goo/validators/enforce.rb               |  2 ++
 lib/goo/validators/implementations/email.rb | 22 +++++++++++++++++++++
 2 files changed, 24 insertions(+)
 create mode 100644 lib/goo/validators/implementations/email.rb

diff --git a/lib/goo/validators/enforce.rb b/lib/goo/validators/enforce.rb
index 1a53eaf0..4d0c09f4 100644
--- a/lib/goo/validators/enforce.rb
+++ b/lib/goo/validators/enforce.rb
@@ -41,6 +41,8 @@ def enforce(inst,attr,value)
               check Goo::Validators::DataType, inst, attr, value, opt, Float
             when :symmetric
               check Goo::Validators::Symmetric, inst, attr, value, opt
+            when :email
+              check Goo::Validators::Email, inst, attr, value, opt
             when /^distinct_of_/
               check Goo::Validators::DistinctOf, inst, attr, value, opt, opt
             when /^superior_equal_to_/
diff --git a/lib/goo/validators/implementations/email.rb b/lib/goo/validators/implementations/email.rb
new file mode 100644
index 00000000..f8405714
--- /dev/null
+++ b/lib/goo/validators/implementations/email.rb
@@ -0,0 +1,22 @@
+module Goo
+  module Validators
+    class Email < ValidatorBase
+      include Validator
+      EMAIL_REGEXP = /\A[\w+\-.]+@[a-z\d\-]+(\.[a-z\d\-]+)*\.[a-z]+\z/i
+      key :email
+
+      error_message ->(obj) {
+        if @value.kind_of? Array
+          return "All values in attribute `#{@attr}` must be a valid emails"
+        else
+          return "Attribute `#{@attr}` with the value `#{@value}` must be a valid email"
+
+        end
+      }
+
+      validity_check -> (obj) do
+        @value.nil? || @value.match?(EMAIL_REGEXP)
+      end
+    end
+  end
+end
\ No newline at end of file

From f78687914f1ceae3de6139ac131754a4ed1d9873 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Sat, 22 Apr 2023 22:03:37 +0100
Subject: [PATCH 100/168] add filters patterns to select variables

---
 lib/goo/sparql/query_builder.rb | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/lib/goo/sparql/query_builder.rb b/lib/goo/sparql/query_builder.rb
index a6e4f634..d44944dd 100644
--- a/lib/goo/sparql/query_builder.rb
+++ b/lib/goo/sparql/query_builder.rb
@@ -33,13 +33,12 @@ def build_select_query(ids, variables, graphs, patterns,
         @order_by, variables, optional_patterns = init_order_by(@count, @klass, @order_by, optional_patterns, variables)
         variables, patterns = add_some_type_to_id(patterns, query_options, variables)
 
-        query_filter_str, patterns, optional_patterns =
+        query_filter_str, patterns, optional_patterns, filter_variables =
           filter_query_strings(@collection, graphs, internal_variables, @klass, optional_patterns, patterns, @query_filters)
-
         variables = [] if @count
         variables.delete :some_type
 
-        select_distinct(variables, aggregate_projections)
+        select_distinct(variables, aggregate_projections, filter_variables)
           .from(graphs)
           .where(patterns)
           .union_bind_in_where(properties_to_include)
@@ -135,10 +134,10 @@ def from(graphs)
         self
       end
 
-      def select_distinct(variables, aggregate_projections)
-
+      def select_distinct(variables, aggregate_projections, filter_variables)
         select_vars = variables.dup
         reject_aggregations_from_vars(select_vars, aggregate_projections) if aggregate_projections
+        select_vars = (select_vars + filter_variables).uniq  if @page # Fix for 4store pagination with a filter
         @query = @query.select(*select_vars).distinct(true)
         self
       end
@@ -347,8 +346,8 @@ def filter_query_strings(collection, graphs, internal_variables, klass,
                                optional_patterns, patterns,
                                query_filters)
         query_filter_str = []
-
         filter_graphs = []
+        filter_variables = []
         inspected_patterns = {}
         query_filters&.each do |query_filter|
           filter_operations = []
@@ -365,9 +364,9 @@ def filter_query_strings(collection, graphs, internal_variables, klass,
               patterns.concat(filter_patterns)
             end
           end
+          filter_variables << inspected_patterns.values.last
         end
-
-        [query_filter_str, patterns, optional_patterns, internal_variables]
+        [query_filter_str, patterns, optional_patterns, filter_variables]
       end
 
       def reject_aggregations_from_vars(variables, aggregate_projections)

From a613c80e48e60f4f3389e8ee1ad6e45551916434 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Sat, 22 Apr 2023 22:05:55 +0100
Subject: [PATCH 101/168] make make regex filter no-case sensitive

---
 lib/goo/sparql/query_builder.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/goo/sparql/query_builder.rb b/lib/goo/sparql/query_builder.rb
index a6e4f634..3cca37c4 100644
--- a/lib/goo/sparql/query_builder.rb
+++ b/lib/goo/sparql/query_builder.rb
@@ -321,7 +321,7 @@ def query_filter_sparql(klass, filter, filter_patterns, filter_graphs,
               return :optional
             when :regex
               if  filter_operation.value.is_a?(String)
-                filter_operations << "REGEX(STR(?#{filter_var.to_s}) , \"#{filter_operation.value.to_s}\")"
+                filter_operations << "REGEX(STR(?#{filter_var.to_s}) , \"#{filter_operation.value.to_s}\", \"i\")"
               end
 
             else

From 88676a15dae6afd64059226141983b8e50d5d8d4 Mon Sep 17 00:00:00 2001
From: HADDAD Zineddine <hz_haddad@esi.dz>
Date: Sun, 23 Apr 2023 17:30:31 +0200
Subject: [PATCH 102/168] if requested_lang = 'all' return all

---
 lib/goo/sparql/mixins/solution_lang_filter.rb | 30 ++++++++++++++-----
 lib/goo/sparql/solutions_mapper.rb            |  8 ++---
 2 files changed, 27 insertions(+), 11 deletions(-)

diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index 8102c82e..21c0af09 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -9,13 +9,12 @@ def initialize(requested_lang: nil, unmapped: false, list_attributes: [])
           @list_attributes = list_attributes
           @objects_by_lang = {}
           @unmapped = unmapped
-          @requested_lang = requested_lang
           @fill_other_languages = init_requested_lang
-          @requested_lang = @requested_lang.to_s.upcase.to_sym
+          @requested_lang = requested_lang
         end
 
         def enrich_models(models_by_id)
-
+          
           return unless fill_other_languages?
 
           other_platform_languages = Goo.main_languages[1..] || []
@@ -43,15 +42,27 @@ def enrich_models(models_by_id)
         
 
         def set_model_value(model, predicate, objects, object)
-          language = object_language(object) # if lang is nil, it means that the object is not a literal
-          if language.nil?
+                    
+          if requested_lang.eql?(:ALL)
+            return model.send("#{predicate}=", objects, on_load: true)
+          end
+          
+          unless is_literal(object)
             return model.send("#{predicate}=", objects, on_load: true)
-          elsif language_match?(language)
+          end
+          
+          # here the object is a literal (but it could be with no language) 
+
+          language = object_language(object)
+
+          if language_match?(language)
             return if language.eql?(:no_lang) && !model.instance_variable_get("@#{predicate}").nil? && !objects.is_a?(Array)
 
             return model.send("#{predicate}=", objects, on_load: true)
           end
 
+          # here the object is a literal with a different language , so we store it for to enrich the model later with the other platform languages
+
           store_objects_by_lang(model.id, predicate, object, language)
         end
 
@@ -72,7 +83,8 @@ def object_language(new_value)
         end
 
         def language_match?(language)
-          !language.nil? && (language.eql?(requested_lang) || language.eql?(:no_lang) || requested_lang.nil?)
+          # no_lang means that the object is not a literal
+          language.eql?(requested_lang) || language.eql?(:no_lang)
         end
 
         def store_objects_by_lang(id, predicate, object, language)
@@ -141,6 +153,10 @@ def fill_other_languages?
           @fill_other_languages
         end
 
+        def is_literal(object)
+          return object_language(object).nil? ? false : true
+        end
+
       end
     end
   end
diff --git a/lib/goo/sparql/solutions_mapper.rb b/lib/goo/sparql/solutions_mapper.rb
index 3117b39c..119f4113 100644
--- a/lib/goo/sparql/solutions_mapper.rb
+++ b/lib/goo/sparql/solutions_mapper.rb
@@ -22,7 +22,7 @@ def initialize(aggregate_projections, bnode_extraction, embed_struct,
         @incl = options[:include]
         @count = options[:count]
         @collection = options[:collection]
-        @requested_lang =  options[:requested_lang]
+        @requested_lang =  options[:requested_lang] || :ALL
       end
       
       def map_each_solutions(select)
@@ -30,9 +30,9 @@ def map_each_solutions(select)
         objects_new = {}
         list_attributes = Set.new(@klass.attributes(:list))
         all_attributes = Set.new(@klass.attributes(:all))
+
         @lang_filter = Goo::SPARQL::Solution::LanguageFilter.new(requested_lang: @requested_lang, unmapped: @unmapped,
            list_attributes: list_attributes)
-
         
         select.each_solution do |sol|
           next if sol[:some_type] && @klass.type_uri(@collection) != sol[:some_type]
@@ -74,8 +74,8 @@ def map_each_solutions(select)
           add_object_to_model(id, objects, object, predicate)
         end
       
-
-        @lang_filter.enrich_models(@models_by_id)
+        # for this moment we are not going to enrich models , maybe we will use it if the results are empty  
+        # @lang_filter.enrich_models(@models_by_id)
 
         init_unloaded_attributes(found, list_attributes)
 

From 688ec5e8bdcba6b1f3bfaca7e4c2fd15f299d924 Mon Sep 17 00:00:00 2001
From: HADDAD Zineddine <hz_haddad@esi.dz>
Date: Mon, 24 Apr 2023 17:25:40 +0200
Subject: [PATCH 103/168] support select multilanguage

---
 lib/goo/sparql/mixins/solution_lang_filter.rb | 11 ++++++++++-
 lib/goo/sparql/solutions_mapper.rb            | 11 +++++++++--
 2 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index 21c0af09..c3f78889 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -84,7 +84,16 @@ def object_language(new_value)
 
         def language_match?(language)
           # no_lang means that the object is not a literal
-          language.eql?(requested_lang) || language.eql?(:no_lang)
+          if language.eql?(:no_lang)
+            return true 
+          end
+
+          if requested_lang.is_a?(Array)
+            return requested_lang.include?(language)
+          end
+
+          return language.eql?(requested_lang)
+
         end
 
         def store_objects_by_lang(id, predicate, object, language)
diff --git a/lib/goo/sparql/solutions_mapper.rb b/lib/goo/sparql/solutions_mapper.rb
index 119f4113..9c97032f 100644
--- a/lib/goo/sparql/solutions_mapper.rb
+++ b/lib/goo/sparql/solutions_mapper.rb
@@ -4,7 +4,7 @@ class SolutionMapper
       BNODES_TUPLES = Struct.new(:id, :attribute)
 
       def initialize(aggregate_projections, bnode_extraction, embed_struct,
-                     incl_embed, klass_struct, models_by_id,
+                      incl_embed, klass_struct, models_by_id,
                      properties_to_include, unmapped, variables, ids, options)
 
         @aggregate_projections = aggregate_projections
@@ -22,7 +22,7 @@ def initialize(aggregate_projections, bnode_extraction, embed_struct,
         @incl = options[:include]
         @count = options[:count]
         @collection = options[:collection]
-        @requested_lang =  options[:requested_lang] || :ALL
+        @requested_lang =  get_language(options[:requested_lang].to_s) 
       end
       
       def map_each_solutions(select)
@@ -102,6 +102,13 @@ def map_each_solutions(select)
 
       private
 
+      def get_language(languages)
+        languages = 'ALL' if languages.nil? || languages.empty? 
+        lang = languages.split(',').map {|l| l.upcase.to_sym}
+        return lang.length == 1 ? lang.first : lang
+      end
+
+
       def init_unloaded_attributes(found, list_attributes)
         return if @incl.nil?
 

From 7871af7ef5fab2b41158edb1ba989a9f22c4aee9 Mon Sep 17 00:00:00 2001
From: HADDAD Zineddine <hz_haddad@esi.dz>
Date: Wed, 26 Apr 2023 15:33:06 +0200
Subject: [PATCH 104/168] show the values with their corresponding language

---
 lib/goo/sparql/mixins/solution_lang_filter.rb | 55 ++++++-------------
 lib/goo/sparql/solutions_mapper.rb            |  2 +-
 2 files changed, 19 insertions(+), 38 deletions(-)

diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index c3f78889..60eae002 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -15,55 +15,31 @@ def initialize(requested_lang: nil, unmapped: false, list_attributes: [])
 
         def enrich_models(models_by_id)
           
-          return unless fill_other_languages?
-
-          other_platform_languages = Goo.main_languages[1..] || []
-
+          ## if the requested language is ALL, we can enrich the models with the objects by language
           objects_by_lang.each do |id, predicates|
             model = models_by_id[id]
-            predicates.each do |predicate, languages|
-              model_attribute_val = get_model_attribute_value(model, predicate)
-              next unless model_attribute_val.nil? || model_attribute_val.empty?
-
-              other_platform_languages.each do |platform_language|
-                if languages[platform_language.to_s.upcase.to_sym]
-                    save_model_values(model, languages[platform_language], predicate, unmapped)
-                  break
-                end
-              end
-              model_attribute_val = get_model_attribute_value(model, predicate)
-              if model_attribute_val.nil? || model_attribute_val.empty?
-                save_model_values(model, languages.values.flatten.uniq, predicate, unmapped)
+            predicates.each do |predicate, values|
+              if predicate.eql?(:synonym) || predicate.eql?(:prefLabel)
+                save_model_values(model, values, predicate, unmapped)
               end
             end
-          end
-        
+          end  
         end
         
 
         def set_model_value(model, predicate, objects, object)
                     
-          if requested_lang.eql?(:ALL)
-            return model.send("#{predicate}=", objects, on_load: true)
-          end
-          
-          unless is_literal(object)
-            return model.send("#{predicate}=", objects, on_load: true)
-          end
-          
-          # here the object is a literal (but it could be with no language) 
-
           language = object_language(object)
 
-          if language_match?(language)
-            return if language.eql?(:no_lang) && !model.instance_variable_get("@#{predicate}").nil? && !objects.is_a?(Array)
+          if requested_lang.eql?(:ALL) || !is_literal(object) || language_match?(language)
+            model.send("#{predicate}=", objects, on_load: true)
+          end 
 
-            return model.send("#{predicate}=", objects, on_load: true)
+          if requested_lang.eql?(:ALL) || requested_lang.is_a?(Array)
+            language = "@none" if language.nil? || language.eql?(:no_lang)
+            store_objects_by_lang(model.id, predicate, object, language)
           end
 
-          # here the object is a literal with a different language , so we store it for to enrich the model later with the other platform languages
-
-          store_objects_by_lang(model.id, predicate, object, language)
         end
 
         def model_set_unmapped(model, predicate, value)
@@ -99,6 +75,8 @@ def language_match?(language)
         def store_objects_by_lang(id, predicate, object, language)
           # store objects in this format: [id][predicate][language] = [objects]
 
+          return if requested_lang.is_a?(Array) && !requested_lang.include?(language)
+
           objects_by_lang[id] ||= {}
           objects_by_lang[id][predicate] ||= {}
           objects_by_lang[id][predicate][language] ||= []
@@ -138,8 +116,11 @@ def save_model_values(model, values, predicate, unmapped)
           if unmapped
             add_unmapped_to_model(model, predicate, values)
           else
-            values = Array(values).map(&:object)
-            values = values.min unless list_attributes?(predicate)
+            
+            if !list_attributes?(predicate)
+              values = values.map { |k, v| [k, v.first] }.to_h
+            end
+
             model.send("#{predicate}=", values, on_load: true)
           end
         end
diff --git a/lib/goo/sparql/solutions_mapper.rb b/lib/goo/sparql/solutions_mapper.rb
index 9c97032f..17c00a24 100644
--- a/lib/goo/sparql/solutions_mapper.rb
+++ b/lib/goo/sparql/solutions_mapper.rb
@@ -75,7 +75,7 @@ def map_each_solutions(select)
         end
       
         # for this moment we are not going to enrich models , maybe we will use it if the results are empty  
-        # @lang_filter.enrich_models(@models_by_id)
+        @lang_filter.enrich_models(@models_by_id)
 
         init_unloaded_attributes(found, list_attributes)
 

From dd5f3ef1d05bf6b29d04ec0bea21f2f8fdf53b51 Mon Sep 17 00:00:00 2001
From: HADDAD Zineddine <hz_haddad@esi.dz>
Date: Wed, 26 Apr 2023 15:39:14 +0200
Subject: [PATCH 105/168] use @attributes_to_translate

---
 lib/goo/sparql/mixins/solution_lang_filter.rb | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index 60eae002..272f2cdc 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -6,6 +6,7 @@ class  LanguageFilter
         attr_reader :requested_lang, :unmapped, :objects_by_lang
 
         def initialize(requested_lang: nil, unmapped: false, list_attributes: [])
+          @attributes_to_translate = [:synonym, :prefLabel]
           @list_attributes = list_attributes
           @objects_by_lang = {}
           @unmapped = unmapped
@@ -19,7 +20,7 @@ def enrich_models(models_by_id)
           objects_by_lang.each do |id, predicates|
             model = models_by_id[id]
             predicates.each do |predicate, values|
-              if predicate.eql?(:synonym) || predicate.eql?(:prefLabel)
+              if @attributes_to_translate.any? { |attr| predicate.eql?(attr) }
                 save_model_values(model, values, predicate, unmapped)
               end
             end

From 5f81e7b624af2b1bbb09f9434ff86111862fb4bc Mon Sep 17 00:00:00 2001
From: HADDAD Zineddine <hz_haddad@esi.dz>
Date: Fri, 28 Apr 2023 14:17:21 +0200
Subject: [PATCH 106/168] change methode name

---
 lib/goo/sparql/mixins/solution_lang_filter.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index 272f2cdc..f705db78 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -32,7 +32,7 @@ def set_model_value(model, predicate, objects, object)
                     
           language = object_language(object)
 
-          if requested_lang.eql?(:ALL) || !is_literal(object) || language_match?(language)
+          if requested_lang.eql?(:ALL) || !literal?(object) || language_match?(language)
             model.send("#{predicate}=", objects, on_load: true)
           end 
 
@@ -144,7 +144,7 @@ def fill_other_languages?
           @fill_other_languages
         end
 
-        def is_literal(object)
+        def literal?(object)
           return object_language(object).nil? ? false : true
         end
 

From bb685a207e5b552fddd5c272076bb10984e1b6ae Mon Sep 17 00:00:00 2001
From: HADDAD Zineddine <hz_haddad@esi.dz>
Date: Fri, 28 Apr 2023 14:20:34 +0200
Subject: [PATCH 107/168] remove platform languages

---
 lib/goo/sparql/mixins/solution_lang_filter.rb | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index f705db78..d801fb09 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -10,7 +10,6 @@ def initialize(requested_lang: nil, unmapped: false, list_attributes: [])
           @list_attributes = list_attributes
           @objects_by_lang = {}
           @unmapped = unmapped
-          @fill_other_languages = init_requested_lang
           @requested_lang = requested_lang
         end
 
@@ -85,14 +84,6 @@ def store_objects_by_lang(id, predicate, object, language)
           objects_by_lang[id][predicate][language] << object
         end
 
-        def init_requested_lang
-          if @requested_lang.nil?
-            @requested_lang = Goo.main_languages[0] || :EN
-            return true
-          end
-
-          false
-        end
 
         def get_model_attribute_value(model, predicate)
           if unmapped
@@ -140,10 +131,6 @@ def list_attributes?(predicate)
         end
 
 
-        def fill_other_languages?
-          @fill_other_languages
-        end
-
         def literal?(object)
           return object_language(object).nil? ? false : true
         end

From 778b696c3dc193443b1d2ff08853bcbcd4709d0a Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Mon, 1 May 2023 18:33:51 +0200
Subject: [PATCH 108/168] add complex_order_by unit test

---
 test/test_where.rb | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/test/test_where.rb b/test/test_where.rb
index bca4b2ea..30d933e3 100644
--- a/test/test_where.rb
+++ b/test/test_where.rb
@@ -600,4 +600,15 @@ def test_include_inverse_with_find
     end
   end
 
+  def test_complex_order_by
+    u = University.where.include(address: [:country]).order_by(address: {country: :asc}).all
+    countries = u.map {|x| x.address.map{|a| a.country}}.flatten
+    assert_equal countries.sort, countries
+
+
+    u = University.where.include(address: [:country]).order_by(address: {country: :desc}).all
+    countries = u.map {|x| x.address.map{|a| a.country}}.flatten
+    assert_equal countries.sort{|a,b| b<=>a }, countries
+  end
+
 end

From b7a4c56d17587cca1433c65bd74b17451a6475d3 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Mon, 1 May 2023 19:37:44 +0200
Subject: [PATCH 109/168] refactor query_builder to extract internal_variables
 as instance variable

---
 lib/goo/sparql/query_builder.rb | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/lib/goo/sparql/query_builder.rb b/lib/goo/sparql/query_builder.rb
index 5227dac2..9cbb5636 100644
--- a/lib/goo/sparql/query_builder.rb
+++ b/lib/goo/sparql/query_builder.rb
@@ -17,24 +17,25 @@ def initialize(options)
         @model_query_options = options[:query_options]
         @enable_rules = options[:rules]
         @order_by = options[:order_by]
-
+        @internal_variables_map = {}
         @query = get_client
       end
 
       def build_select_query(ids, variables, graphs, patterns,
                              query_options, properties_to_include)
 
-        internal_variables = graph_match(@collection, @graph_match, graphs, @klass, patterns, query_options, @unions)
+        patterns = graph_match(@collection, @graph_match, graphs, @klass, patterns, query_options, @unions)
 
         aggregate_projections, aggregate_vars,
           variables, optional_patterns = get_aggregate_vars(@aggregate, @collection, graphs,
-                                                            @klass, @unions, variables, internal_variables)
+                                                            @klass, @unions, variables)
 
         @order_by, variables, optional_patterns = init_order_by(@count, @klass, @order_by, optional_patterns, variables)
         variables, patterns = add_some_type_to_id(patterns, query_options, variables)
 
         query_filter_str, patterns, optional_patterns, filter_variables =
-          filter_query_strings(@collection, graphs, internal_variables, @klass, optional_patterns, patterns, @query_filters)
+          filter_query_strings(@collection, graphs, @klass, optional_patterns, patterns, @query_filters)
+
         variables = [] if @count
         variables.delete :some_type
 
@@ -169,6 +170,7 @@ def patterns_for_match(klass, attr, value, graphs, patterns, unions,
             value = "#{attr}_agg_#{in_aggregate}".to_sym
           end
           internal_variables << value
+          @internal_variables_map[attr] = value
         end
 
         add_rules(attr, klass, query_options)
@@ -209,7 +211,7 @@ def walk_pattern(klass, match_patterns, graphs, patterns, unions,
         end
       end
 
-      def get_aggregate_vars(aggregate, collection, graphs, klass, unions, variables, internal_variables)
+      def get_aggregate_vars(aggregate, collection, graphs, klass, unions, variables)
         # mdorf, 6/03/20 If aggregate projections (sub-SELECT within main SELECT) use an alias, that alias cannot appear in the main SELECT
         # https://github.com/ncbo/goo/issues/106
         # See last sentence in https://www.w3.org/TR/sparql11-query/#aggregateExample
@@ -240,8 +242,6 @@ def get_aggregate_vars(aggregate, collection, graphs, klass, unions, variables,
       end
 
       def graph_match(collection, graph_match, graphs, klass, patterns, query_options, unions)
-        internal_variables = []
-
         if graph_match
           #make it deterministic - for caching
           graph_match_iteration = Goo::Base::PatternIteration.new(graph_match)
@@ -249,7 +249,7 @@ def graph_match(collection, graph_match, graphs, klass, patterns, query_options,
                        internal_variables, in_aggregate = false, query_options, collection)
           graphs.uniq!
         end
-        internal_variables
+        patterns
       end
 
       def get_client
@@ -342,7 +342,7 @@ def query_filter_sparql(klass, filter, filter_patterns, filter_graphs,
         end
       end
 
-      def filter_query_strings(collection, graphs, internal_variables, klass,
+      def filter_query_strings(collection, graphs, klass,
                                optional_patterns, patterns,
                                query_filters)
         query_filter_str = []
@@ -382,6 +382,9 @@ def add_some_type_to_id(patterns, query_options, variables)
         [variables, patterns]
       end
 
+      def internal_variables
+        @internal_variables_map.values
+      end
     end
   end
 end

From 4df6681401ca332c4a8f91042782787dc0c36b8f Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Mon, 1 May 2023 19:38:37 +0200
Subject: [PATCH 110/168] update order_by to work for joined patterns (object
 attributes)

---
 lib/goo/sparql/query_builder.rb | 32 ++++++++++++++++++++++++--------
 1 file changed, 24 insertions(+), 8 deletions(-)

diff --git a/lib/goo/sparql/query_builder.rb b/lib/goo/sparql/query_builder.rb
index 9cbb5636..40e888d0 100644
--- a/lib/goo/sparql/query_builder.rb
+++ b/lib/goo/sparql/query_builder.rb
@@ -30,7 +30,7 @@ def build_select_query(ids, variables, graphs, patterns,
           variables, optional_patterns = get_aggregate_vars(@aggregate, @collection, graphs,
                                                             @klass, @unions, variables)
 
-        @order_by, variables, optional_patterns = init_order_by(@count, @klass, @order_by, optional_patterns, variables)
+        @order_by, variables, optional_patterns = init_order_by(@count, @klass, @order_by, optional_patterns, variables,patterns, query_options, graphs)
         variables, patterns = add_some_type_to_id(patterns, query_options, variables)
 
         query_filter_str, patterns, optional_patterns, filter_variables =
@@ -55,7 +55,7 @@ def build_select_query(ids, variables, graphs, patterns,
         @query.union(*@unions) unless @unions.empty?
 
         ids_filter(ids) if ids
-        order_by if @order_by # TODO  test if work
+        order_by if @order_by
 
         put_query_aggregate_vars(aggregate_vars) if aggregate_vars
         count if @count
@@ -118,7 +118,13 @@ def put_query_aggregate_vars(aggregate_vars)
       end
 
       def order_by
-        order_by_str = @order_by.map { |attr, order| "#{order.to_s.upcase}(?#{attr})" }
+        order_by_str = @order_by.map do |attr, order|
+          if order.is_a?(Hash)
+            sub_attr, order = order.first
+            attr = @internal_variables_map[sub_attr]
+          end
+          "#{order.to_s.upcase}(?#{attr})"
+        end
         @query.order_by(*order_by_str)
         self
       end
@@ -256,21 +262,31 @@ def get_client
         Goo.sparql_query_client(@store)
       end
 
-      def init_order_by(count, klass, order_by, optional_patterns, variables)
+      def init_order_by(count, klass, order_by, optional_patterns, variables, patterns, query_options, graphs)
         order_by = nil if count
         if order_by
           order_by = order_by.first
           #simple ordering ... needs to use pattern inspection
           order_by.each do |attr, direction|
-            quad = query_pattern(klass, attr)
-            optional_patterns << quad[1]
+
+            if direction.is_a?(Hash)
+              sub_attr, direction = direction.first
+              graph_match_iteration = Goo::Base::PatternIteration.new(Goo::Base::Pattern.new({attr => [sub_attr]}))
+              old_internal = internal_variables.dup
+              walk_pattern(klass, graph_match_iteration, graphs, optional_patterns, @unions, internal_variables, in_aggregate = false, query_options, @collection)
+              variables << (internal_variables - old_internal).last
+            else
+              quad = query_pattern(klass, attr)
+              optional_patterns << quad[1]
+              variables << attr
+            end
+
             #patterns << quad[1]
             #mdorf, 9/22/16 If an ORDER BY clause exists, the columns used in the ORDER BY should be present in the SPARQL select
             #variables << attr unless variables.include?(attr)
           end
-          variables = %i[id attributeProperty attributeObject]
         end
-        [order_by, variables, optional_patterns]
+        [order_by, variables, optional_patterns, patterns]
       end
 
       def sparql_op_string(op)

From 7ffeccdf6b6c79611dc1a36639ad525fab2dbc95 Mon Sep 17 00:00:00 2001
From: HADDAD Zineddine <hz_haddad@esi.dz>
Date: Tue, 9 May 2023 14:01:49 +0200
Subject: [PATCH 111/168] downcase lang key

---
 lib/goo/sparql/mixins/solution_lang_filter.rb | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index d801fb09..4d5f2b24 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -6,7 +6,7 @@ class  LanguageFilter
         attr_reader :requested_lang, :unmapped, :objects_by_lang
 
         def initialize(requested_lang: nil, unmapped: false, list_attributes: [])
-          @attributes_to_translate = [:synonym, :prefLabel]
+          @attributes_to_translate = [:synonym, :prefLabel, :definition, :cui, :semanticType, :obsolete, :inScheme, :memberOf, :created, :modified, :links]
           @list_attributes = list_attributes
           @objects_by_lang = {}
           @unmapped = unmapped
@@ -19,8 +19,8 @@ def enrich_models(models_by_id)
           objects_by_lang.each do |id, predicates|
             model = models_by_id[id]
             predicates.each do |predicate, values|
-              if @attributes_to_translate.any? { |attr| predicate.eql?(attr) }
-                save_model_values(model, values, predicate, unmapped)
+              if !@attributes_to_translate.any? { |attr| predicate.eql?(attr) }
+                save_model_values(model, values, predicate, unmapped) 
               end
             end
           end  
@@ -77,11 +77,13 @@ def store_objects_by_lang(id, predicate, object, language)
 
           return if requested_lang.is_a?(Array) && !requested_lang.include?(language)
 
+          language_key = language.downcase  
+            
           objects_by_lang[id] ||= {}
           objects_by_lang[id][predicate] ||= {}
-          objects_by_lang[id][predicate][language] ||= []
+          objects_by_lang[id][predicate][language_key] ||= []
 
-          objects_by_lang[id][predicate][language] << object
+          objects_by_lang[id][predicate][language_key] << object
         end
 
 

From 3967febae703a659a09725a5c5ccd59c4c1ff9f7 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Wed, 10 May 2023 06:27:45 +0200
Subject: [PATCH 112/168] Fix the issue of undefined 'id'  of the language
 filter module

---
 lib/goo/sparql/mixins/solution_lang_filter.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index 4d5f2b24..0f2edec6 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -6,7 +6,7 @@ class  LanguageFilter
         attr_reader :requested_lang, :unmapped, :objects_by_lang
 
         def initialize(requested_lang: nil, unmapped: false, list_attributes: [])
-          @attributes_to_translate = [:synonym, :prefLabel, :definition, :cui, :semanticType, :obsolete, :inScheme, :memberOf, :created, :modified, :links]
+          @attributes_to_translate = [:synonym, :prefLabel, :definition]
           @list_attributes = list_attributes
           @objects_by_lang = {}
           @unmapped = unmapped
@@ -19,7 +19,7 @@ def enrich_models(models_by_id)
           objects_by_lang.each do |id, predicates|
             model = models_by_id[id]
             predicates.each do |predicate, values|
-              if !@attributes_to_translate.any? { |attr| predicate.eql?(attr) }
+              if @attributes_to_translate.any? { |attr| predicate.eql?(attr) }
                 save_model_values(model, values, predicate, unmapped) 
               end
             end

From 78e6420388f671aff68b226410d56e8887e39425 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Wed, 10 May 2023 06:28:17 +0200
Subject: [PATCH 113/168] Show literal attribute if we requested all the
 languages

---
 lib/goo/sparql/mixins/solution_lang_filter.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index 0f2edec6..59a2c8bf 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -44,7 +44,7 @@ def set_model_value(model, predicate, objects, object)
 
         def model_set_unmapped(model, predicate, value)
           language = object_language(value)
-          if language.nil? || language_match?(language)
+          if requested_lang.eql?(:ALL) || language.nil? || language_match?(language)
             return add_unmapped_to_model(model, predicate, value)
           end
           

From 7e1410c2bc349bc3eacca6882e9386d6db0b9e53 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Wed, 10 May 2023 06:36:43 +0200
Subject: [PATCH 114/168] Use portal language by default in the language filter
 module

---
 lib/goo/sparql/solutions_mapper.rb | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/lib/goo/sparql/solutions_mapper.rb b/lib/goo/sparql/solutions_mapper.rb
index 17c00a24..620e1182 100644
--- a/lib/goo/sparql/solutions_mapper.rb
+++ b/lib/goo/sparql/solutions_mapper.rb
@@ -103,11 +103,14 @@ def map_each_solutions(select)
       private
 
       def get_language(languages)
-        languages = 'ALL' if languages.nil? || languages.empty? 
+        languages = portal_language if languages.nil? || languages.empty?
         lang = languages.split(',').map {|l| l.upcase.to_sym}
-        return lang.length == 1 ? lang.first : lang
+        lang.length == 1 ? lang.first : lang
       end
 
+      def portal_language
+        Goo.main_languages.first
+      end
 
       def init_unloaded_attributes(found, list_attributes)
         return if @incl.nil?

From 40df9b009cd7a7a2124f59940887c0e346758ca9 Mon Sep 17 00:00:00 2001
From: HADDAD Zineddine <hz_haddad@esi.dz>
Date: Thu, 1 Jun 2023 16:37:23 +0200
Subject: [PATCH 115/168] group unmapped properties by lang

---
 lib/goo/base/resource.rb                      |  3 ++-
 lib/goo/sparql/mixins/solution_lang_filter.rb | 15 +++++++++++++++
 lib/goo/sparql/solutions_mapper.rb            | 16 ++++++++++++++--
 3 files changed, 31 insertions(+), 3 deletions(-)

diff --git a/lib/goo/base/resource.rb b/lib/goo/base/resource.rb
index eddbf273..083ef27e 100644
--- a/lib/goo/base/resource.rb
+++ b/lib/goo/base/resource.rb
@@ -15,7 +15,7 @@ class Resource
       attr_reader :modified_attributes
       attr_reader :errors
       attr_reader :aggregates
-      attr_reader :unmapped
+      attr_accessor :unmapped
 
       attr_reader :id
 
@@ -129,6 +129,7 @@ def unmapped_get(attribute)
 
       def unmmaped_to_array
         cpy = {}
+        
         @unmapped.each do |attr,v|
           cpy[attr] = v.to_a
         end
diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index 59a2c8bf..3d4a66b8 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -51,9 +51,24 @@ def model_set_unmapped(model, predicate, value)
           store_objects_by_lang(model.id, predicate, value, language)
         end
 
+        def model_group_by_lang(model, requested_lang)
+          unmapped = model.unmapped 
+          cpy = {}
+  
+          unmapped.each do |attr, v|          
+             cpy[attr] = is_a_uri?(v.first) ? v.to_a : v.group_by { |x| x.language.to_s }
+          end
+  
+          model.unmapped = cpy
+        end
+
 
         private
 
+        def is_a_uri?(value)
+          value.is_a?(RDF::URI) && value.valid?
+        end
+
         def object_language(new_value)
           new_value.language || :no_lang if new_value.is_a?(RDF::Literal)
         end
diff --git a/lib/goo/sparql/solutions_mapper.rb b/lib/goo/sparql/solutions_mapper.rb
index 620e1182..6361eb9c 100644
--- a/lib/goo/sparql/solutions_mapper.rb
+++ b/lib/goo/sparql/solutions_mapper.rb
@@ -35,6 +35,7 @@ def map_each_solutions(select)
            list_attributes: list_attributes)
         
         select.each_solution do |sol|
+          
           next if sol[:some_type] && @klass.type_uri(@collection) != sol[:some_type]
           return sol[:count_var].object if @count
 
@@ -96,7 +97,8 @@ def map_each_solutions(select)
         include_bnodes(blank_nodes, @models_by_id) unless blank_nodes.empty?
 
         models_unmapped_to_array(@models_by_id) if @unmapped
-
+        
+        
         @models_by_id
       end
 
@@ -269,10 +271,20 @@ def create_class_model(id, klass, klass_struct)
 
       def models_unmapped_to_array(models_by_id)
         models_by_id.each do |_idm, m|
-          m.unmmaped_to_array
+          if is_multiple_langs?
+            @lang_filter.model_group_by_lang(m, @requested_lang)
+          else
+            m.unmmaped_to_array
+          end
         end
       end
 
+
+      def is_multiple_langs?
+        return true if @requested_lang.is_a?(Array) || @requested_lang.eql?(:ALL)
+        false
+      end
+
       def include_bnodes(bnodes, models_by_id)
         # group by attribute
         attrs = bnodes.map { |_x, y| y.attribute }.uniq

From cbd0186490813e968a62196982481a3afbf7f277 Mon Sep 17 00:00:00 2001
From: HADDAD Zineddine <hz_haddad@esi.dz>
Date: Tue, 6 Jun 2023 16:52:58 +0200
Subject: [PATCH 116/168] Feature: group unmapped properties by language  (#38)

* group unmapped properties by lang

* downcase language keys of unmapped properties

---------

Co-authored-by: Syphax bouazzouni <gs_bouazzouni@esi.dz>
---
 lib/goo/sparql/mixins/solution_lang_filter.rb | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index 3d4a66b8..de41c7a8 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -56,7 +56,7 @@ def model_group_by_lang(model, requested_lang)
           cpy = {}
   
           unmapped.each do |attr, v|          
-             cpy[attr] = is_a_uri?(v.first) ? v.to_a : v.group_by { |x| x.language.to_s }
+            cpy[attr] = group_by_lang(v)
           end
   
           model.unmapped = cpy
@@ -65,6 +65,18 @@ def model_group_by_lang(model, requested_lang)
 
         private
 
+        def group_by_lang(values)
+          
+          return values.to_a if is_a_uri?(values.first)
+          
+          values = values.group_by { |x| x.language ? x.language.to_s.downcase : :none }
+                              
+          no_lang = values[:none] || []
+          return no_lang if !no_lang.empty? && no_lang.all? { |x| !x.plain? }
+
+          values 
+        end
+
         def is_a_uri?(value)
           value.is_a?(RDF::URI) && value.valid?
         end

From e0bf04f4231250cea9aa6c25177b7b6f2ff9dabd Mon Sep 17 00:00:00 2001
From: HADDAD Zineddine <hz_haddad@esi.dz>
Date: Tue, 13 Jun 2023 16:17:49 +0200
Subject: [PATCH 117/168] assert that pre in an array in get_value_object

---
 lib/goo/sparql/solutions_mapper.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/goo/sparql/solutions_mapper.rb b/lib/goo/sparql/solutions_mapper.rb
index 6361eb9c..1e6da4b4 100644
--- a/lib/goo/sparql/solutions_mapper.rb
+++ b/lib/goo/sparql/solutions_mapper.rb
@@ -166,8 +166,8 @@ def get_value_object(id, objects_new, object, list_attributes, predicate)
 
           if object.nil?
             object = pre.nil? ? [] : pre
-          else
-            object = pre.nil? ? [object] : (pre.dup << object)
+          else            
+            object = pre.nil? ? [object] : (Array(pre).dup << object)
             object.uniq!
           end
 

From 39b7f2f2e9c398b981fb03feaa1eebc270877341 Mon Sep 17 00:00:00 2001
From: HADDAD Zineddine <hz_haddad@esi.dz>
Date: Tue, 13 Jun 2023 16:18:05 +0200
Subject: [PATCH 118/168] add label to attributes_to_translate

---
 lib/goo/sparql/mixins/solution_lang_filter.rb | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index de41c7a8..22a109ce 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -6,7 +6,7 @@ class  LanguageFilter
         attr_reader :requested_lang, :unmapped, :objects_by_lang
 
         def initialize(requested_lang: nil, unmapped: false, list_attributes: [])
-          @attributes_to_translate = [:synonym, :prefLabel, :definition]
+          @attributes_to_translate = [:synonym, :prefLabel, :definition, :label]
           @list_attributes = list_attributes
           @objects_by_lang = {}
           @unmapped = unmapped
@@ -15,7 +15,6 @@ def initialize(requested_lang: nil, unmapped: false, list_attributes: [])
 
         def enrich_models(models_by_id)
           
-          ## if the requested language is ALL, we can enrich the models with the objects by language
           objects_by_lang.each do |id, predicates|
             model = models_by_id[id]
             predicates.each do |predicate, values|

From 8af1d47029e203cf006025ca54180cfd6fa53956 Mon Sep 17 00:00:00 2001
From: HADDAD Zineddine <hz_haddad@esi.dz>
Date: Tue, 20 Jun 2023 19:07:28 +0200
Subject: [PATCH 119/168] update define_method

---
 lib/goo/base/settings/settings.rb | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/lib/goo/base/settings/settings.rb b/lib/goo/base/settings/settings.rb
index 5f669d08..3f5780a8 100644
--- a/lib/goo/base/settings/settings.rb
+++ b/lib/goo/base/settings/settings.rb
@@ -274,9 +274,13 @@ def shape_attribute(attr)
             self.instance_variable_set("@#{attr}",value)
           end
           define_method("#{attr}") do |*args|
+            attr_value = self.instance_variable_get("@#{attr}")
+            attr_value = attr_value.values.first if  attr_value.is_a?(Hash) && !args.include?(:show_with_language)
+
+
             if self.class.handler?(attr)
               if @loaded_attributes.include?(attr)
-                return self.instance_variable_get("@#{attr}")
+                return attr_value
               end
               value = self.send("#{self.class.handler(attr)}")
               self.instance_variable_set("@#{attr}",value)
@@ -285,7 +289,7 @@ def shape_attribute(attr)
             end
 
             if (not @persistent) or @loaded_attributes.include?(attr)
-              return self.instance_variable_get("@#{attr}")
+                return attr_value
             else
               # TODO: bug here when no labels from one of the main_lang available... (when it is called by ontologies_linked_data ontologies_submission)
               raise Goo::Base::AttributeNotLoaded, "Attribute `#{attr}` is not loaded for #{self.id}. Loaded attributes: #{@loaded_attributes.inspect}."

From 5b7c78f67449512c511e8a4b752ca2ee724d6786 Mon Sep 17 00:00:00 2001
From: HADDAD Zineddine <hz_haddad@esi.dz>
Date: Tue, 20 Jun 2023 19:07:46 +0200
Subject: [PATCH 120/168] update solution mapper

---
 lib/goo/sparql/mixins/solution_lang_filter.rb | 24 ++++++++-----------
 1 file changed, 10 insertions(+), 14 deletions(-)

diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index 22a109ce..fde181b9 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -17,10 +17,12 @@ def enrich_models(models_by_id)
           
           objects_by_lang.each do |id, predicates|
             model = models_by_id[id]
-            predicates.each do |predicate, values|
-              if @attributes_to_translate.any? { |attr| predicate.eql?(attr) }
-                save_model_values(model, values, predicate, unmapped) 
-              end
+            predicates.each do |predicate, values|   
+            
+              if values.values.all? { |v| v.all? { |x| literal?(x) && x.plain?} }
+                save_model_values(model, values, predicate, unmapped)
+                
+              end 
             end
           end  
         end
@@ -100,7 +102,6 @@ def language_match?(language)
 
         def store_objects_by_lang(id, predicate, object, language)
           # store objects in this format: [id][predicate][language] = [objects]
-
           return if requested_lang.is_a?(Array) && !requested_lang.include?(language)
 
           language_key = language.downcase  
@@ -123,6 +124,7 @@ def get_model_attribute_value(model, predicate)
 
 
         def add_unmapped_to_model(model, predicate, value)
+          
           if model.respond_to? :klass # struct
             model[:unmapped] ||= {}
             model[:unmapped][predicate] ||= []
@@ -133,16 +135,10 @@ def add_unmapped_to_model(model, predicate, value)
         end
 
         def save_model_values(model, values, predicate, unmapped)
-          if unmapped
-            add_unmapped_to_model(model, predicate, values)
-          else
-            
-            if !list_attributes?(predicate)
-              values = values.map { |k, v| [k, v.first] }.to_h
-            end
+          add_unmapped_to_model(model, predicate, values) if unmapped
+          values = values.map { |k, v| [k, v.first] }.to_h unless list_attributes?(predicate)
 
-            model.send("#{predicate}=", values, on_load: true)
-          end
+          model.send("#{predicate}=", values, on_load: true)
         end
 
         def unmapped_get(model, predicate)

From 3161fbb11f6f1721204b38f0e5f76e53f8aa109f Mon Sep 17 00:00:00 2001
From: HADDAD Zineddine <hz_haddad@esi.dz>
Date: Tue, 20 Jun 2023 19:08:00 +0200
Subject: [PATCH 121/168] update get_preload_value

---
 lib/goo/sparql/solutions_mapper.rb | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/goo/sparql/solutions_mapper.rb b/lib/goo/sparql/solutions_mapper.rb
index 1e6da4b4..1f6e234c 100644
--- a/lib/goo/sparql/solutions_mapper.rb
+++ b/lib/goo/sparql/solutions_mapper.rb
@@ -98,7 +98,7 @@ def map_each_solutions(select)
 
         models_unmapped_to_array(@models_by_id) if @unmapped
         
-        
+       
         @models_by_id
       end
 
@@ -189,8 +189,8 @@ def add_object_to_model(id, objects, current_obj, predicate)
       def get_preload_value(id, object, predicate)
         pre_val = nil
         if predicate_preloaded?(id, predicate)
-          pre_val = preloaded_value(id, predicate)
-          pre_val = pre_val.select { |x| x.id == object }.first if pre_val.is_a?(Array)
+          pre_val = preloaded_value(id, predicate)          
+          pre_val = pre_val.select { |x|  x.id  == object  }.first if pre_val.is_a?(Array)
         end
         pre_val
       end

From 6c1790433c6897dd398dce89ff724646e1c7e6af Mon Sep 17 00:00:00 2001
From: HADDAD Zineddine <hz_haddad@esi.dz>
Date: Wed, 28 Jun 2023 02:29:45 +0200
Subject: [PATCH 122/168] Feature: Support multi lingual -  add show_language
 argument to the attributes getters (#39)

* update define_method

* update solution mapper

* update get_preload_value
---
 lib/goo/base/settings/settings.rb             |  8 +++++--
 lib/goo/sparql/mixins/solution_lang_filter.rb | 24 ++++++++-----------
 lib/goo/sparql/solutions_mapper.rb            |  6 ++---
 3 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/lib/goo/base/settings/settings.rb b/lib/goo/base/settings/settings.rb
index 5f669d08..3f5780a8 100644
--- a/lib/goo/base/settings/settings.rb
+++ b/lib/goo/base/settings/settings.rb
@@ -274,9 +274,13 @@ def shape_attribute(attr)
             self.instance_variable_set("@#{attr}",value)
           end
           define_method("#{attr}") do |*args|
+            attr_value = self.instance_variable_get("@#{attr}")
+            attr_value = attr_value.values.first if  attr_value.is_a?(Hash) && !args.include?(:show_with_language)
+
+
             if self.class.handler?(attr)
               if @loaded_attributes.include?(attr)
-                return self.instance_variable_get("@#{attr}")
+                return attr_value
               end
               value = self.send("#{self.class.handler(attr)}")
               self.instance_variable_set("@#{attr}",value)
@@ -285,7 +289,7 @@ def shape_attribute(attr)
             end
 
             if (not @persistent) or @loaded_attributes.include?(attr)
-              return self.instance_variable_get("@#{attr}")
+                return attr_value
             else
               # TODO: bug here when no labels from one of the main_lang available... (when it is called by ontologies_linked_data ontologies_submission)
               raise Goo::Base::AttributeNotLoaded, "Attribute `#{attr}` is not loaded for #{self.id}. Loaded attributes: #{@loaded_attributes.inspect}."
diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index 22a109ce..fde181b9 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -17,10 +17,12 @@ def enrich_models(models_by_id)
           
           objects_by_lang.each do |id, predicates|
             model = models_by_id[id]
-            predicates.each do |predicate, values|
-              if @attributes_to_translate.any? { |attr| predicate.eql?(attr) }
-                save_model_values(model, values, predicate, unmapped) 
-              end
+            predicates.each do |predicate, values|   
+            
+              if values.values.all? { |v| v.all? { |x| literal?(x) && x.plain?} }
+                save_model_values(model, values, predicate, unmapped)
+                
+              end 
             end
           end  
         end
@@ -100,7 +102,6 @@ def language_match?(language)
 
         def store_objects_by_lang(id, predicate, object, language)
           # store objects in this format: [id][predicate][language] = [objects]
-
           return if requested_lang.is_a?(Array) && !requested_lang.include?(language)
 
           language_key = language.downcase  
@@ -123,6 +124,7 @@ def get_model_attribute_value(model, predicate)
 
 
         def add_unmapped_to_model(model, predicate, value)
+          
           if model.respond_to? :klass # struct
             model[:unmapped] ||= {}
             model[:unmapped][predicate] ||= []
@@ -133,16 +135,10 @@ def add_unmapped_to_model(model, predicate, value)
         end
 
         def save_model_values(model, values, predicate, unmapped)
-          if unmapped
-            add_unmapped_to_model(model, predicate, values)
-          else
-            
-            if !list_attributes?(predicate)
-              values = values.map { |k, v| [k, v.first] }.to_h
-            end
+          add_unmapped_to_model(model, predicate, values) if unmapped
+          values = values.map { |k, v| [k, v.first] }.to_h unless list_attributes?(predicate)
 
-            model.send("#{predicate}=", values, on_load: true)
-          end
+          model.send("#{predicate}=", values, on_load: true)
         end
 
         def unmapped_get(model, predicate)
diff --git a/lib/goo/sparql/solutions_mapper.rb b/lib/goo/sparql/solutions_mapper.rb
index 1e6da4b4..1f6e234c 100644
--- a/lib/goo/sparql/solutions_mapper.rb
+++ b/lib/goo/sparql/solutions_mapper.rb
@@ -98,7 +98,7 @@ def map_each_solutions(select)
 
         models_unmapped_to_array(@models_by_id) if @unmapped
         
-        
+       
         @models_by_id
       end
 
@@ -189,8 +189,8 @@ def add_object_to_model(id, objects, current_obj, predicate)
       def get_preload_value(id, object, predicate)
         pre_val = nil
         if predicate_preloaded?(id, predicate)
-          pre_val = preloaded_value(id, predicate)
-          pre_val = pre_val.select { |x| x.id == object }.first if pre_val.is_a?(Array)
+          pre_val = preloaded_value(id, predicate)          
+          pre_val = pre_val.select { |x|  x.id  == object  }.first if pre_val.is_a?(Array)
         end
         pre_val
       end

From cca637e29efeb40867ee47806e0e08fc32609f25 Mon Sep 17 00:00:00 2001
From: HADDAD Zineddine <hz_haddad@esi.dz>
Date: Thu, 13 Jul 2023 18:45:41 +0200
Subject: [PATCH 123/168] fix save_model_values if unmmaped condition

---
 lib/goo/sparql/mixins/solution_lang_filter.rb | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index fde181b9..746aece4 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -135,10 +135,15 @@ def add_unmapped_to_model(model, predicate, value)
         end
 
         def save_model_values(model, values, predicate, unmapped)
-          add_unmapped_to_model(model, predicate, values) if unmapped
-          values = values.map { |k, v| [k, v.first] }.to_h unless list_attributes?(predicate)
+          if unmapped
+            add_unmapped_to_model(model, predicate, values) 
+
+          else 
+            values = values.map { |k, v| [k, v.first] }.to_h unless list_attributes?(predicate)
+
+            model.send("#{predicate}=", values, on_load: true)
+          end
 
-          model.send("#{predicate}=", values, on_load: true)
         end
 
         def unmapped_get(model, predicate)

From 94959cd3c34c2d7d6fa27b9220aa908d67114cd3 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Fri, 14 Jul 2023 18:02:36 +0200
Subject: [PATCH 124/168] fix getters for list attributes  to not take only the
 first value

---
 lib/goo/base/settings/settings.rb | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/lib/goo/base/settings/settings.rb b/lib/goo/base/settings/settings.rb
index 3f5780a8..6c416298 100644
--- a/lib/goo/base/settings/settings.rb
+++ b/lib/goo/base/settings/settings.rb
@@ -275,7 +275,12 @@ def shape_attribute(attr)
           end
           define_method("#{attr}") do |*args|
             attr_value = self.instance_variable_get("@#{attr}")
-            attr_value = attr_value.values.first if  attr_value.is_a?(Hash) && !args.include?(:show_with_language)
+
+            if self.class.not_show_all_languages?(attr_value, args)
+              is_array = attr_value.values.first.is_a?(Array)
+              attr_value = attr_value.values.flatten
+              attr_value = attr_value.first unless is_array
+            end
 
 
             if self.class.handler?(attr)
@@ -395,6 +400,14 @@ def read_only(attributes)
           instance
         end
 
+        def show_all_languages?(args)
+          args.include?(:show_with_language)
+        end
+
+        def not_show_all_languages?(values, args)
+          values.is_a?(Hash) && !show_all_languages?(args)
+        end
+
         private
 
         def set_no_list_by_default(options)

From 37da0d36cd59de4cf477da07dbdbda41003b4c43 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Fri, 14 Jul 2023 18:03:25 +0200
Subject: [PATCH 125/168] remove the languages hash for the unmapped if not a
 mutli langual asked

---
 lib/goo/base/resource.rb | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/lib/goo/base/resource.rb b/lib/goo/base/resource.rb
index 083ef27e..633c9594 100644
--- a/lib/goo/base/resource.rb
+++ b/lib/goo/base/resource.rb
@@ -15,7 +15,7 @@ class Resource
       attr_reader :modified_attributes
       attr_reader :errors
       attr_reader :aggregates
-      attr_accessor :unmapped
+      attr_writer :unmapped
 
       attr_reader :id
 
@@ -136,6 +136,12 @@ def unmmaped_to_array
         @unmapped = cpy
       end
 
+      def unmapped(*args)
+        @unmapped.transform_values do  |language_values|
+          self.class.not_show_all_languages?(language_values, args) ?  language_values.values.flatten: language_values
+        end
+      end
+
       def delete(*args)
         if self.kind_of?(Goo::Base::Enum)
           raise ArgumentError, "Enums cannot be deleted" unless args[0] && args[0][:init_enum]

From a65048089f1ddae6fed7e1fffecce64bb96df450 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Fri, 14 Jul 2023 18:05:28 +0200
Subject: [PATCH 126/168] move some language helper from the mapper to the
 lang_filter module

---
 lib/goo/sparql/mixins/solution_lang_filter.rb | 21 +++++++++++++++++--
 lib/goo/sparql/solutions_mapper.rb            | 16 +-------------
 2 files changed, 20 insertions(+), 17 deletions(-)

diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index fde181b9..127da1a4 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -63,6 +63,13 @@ def model_group_by_lang(model, requested_lang)
           model.unmapped = cpy
         end
 
+        def models_unmapped_to_array(m)
+          if show_all_languages?
+            model_group_by_lang(m)
+          else
+            m.unmmaped_to_array
+          end
+        end
 
         private
 
@@ -155,8 +162,18 @@ def list_attributes?(predicate)
         end
 
 
-        def literal?(object)
-          return object_language(object).nil? ? false : true
+        def show_all_languages?
+          @requested_lang.is_a?(Array) || @requested_lang.eql?(:ALL)
+        end
+
+        def get_language(languages)
+          languages = portal_language if languages.nil? || languages.empty?
+          lang = languages.to_s.split(',').map { |l| l.upcase.to_sym }
+          lang.length == 1 ? lang.first : lang
+        end
+
+        def portal_language
+          Goo.main_languages.first
         end
 
       end
diff --git a/lib/goo/sparql/solutions_mapper.rb b/lib/goo/sparql/solutions_mapper.rb
index 1f6e234c..4cfd9104 100644
--- a/lib/goo/sparql/solutions_mapper.rb
+++ b/lib/goo/sparql/solutions_mapper.rb
@@ -104,16 +104,6 @@ def map_each_solutions(select)
 
       private
 
-      def get_language(languages)
-        languages = portal_language if languages.nil? || languages.empty?
-        lang = languages.split(',').map {|l| l.upcase.to_sym}
-        lang.length == 1 ? lang.first : lang
-      end
-
-      def portal_language
-        Goo.main_languages.first
-      end
-
       def init_unloaded_attributes(found, list_attributes)
         return if @incl.nil?
 
@@ -271,11 +261,7 @@ def create_class_model(id, klass, klass_struct)
 
       def models_unmapped_to_array(models_by_id)
         models_by_id.each do |_idm, m|
-          if is_multiple_langs?
-            @lang_filter.model_group_by_lang(m, @requested_lang)
-          else
-            m.unmmaped_to_array
-          end
+          @lang_filter.models_unmapped_to_array(m)
         end
       end
 

From 6785c4bbf5814ce8fedff0f8f87798f969bf36de Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Fri, 14 Jul 2023 18:09:49 +0200
Subject: [PATCH 127/168] move @requested_lang variable from the mapper to the
 lang_filter module

---
 lib/goo/sparql/mixins/solution_lang_filter.rb | 4 ++--
 lib/goo/sparql/solutions_mapper.rb            | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index 127da1a4..f29ac635 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -5,12 +5,12 @@ class  LanguageFilter
         
         attr_reader :requested_lang, :unmapped, :objects_by_lang
 
-        def initialize(requested_lang: nil, unmapped: false, list_attributes: [])
+        def initialize(requested_lang: RequestStore.store[:requested_lang], unmapped: false, list_attributes: [])
           @attributes_to_translate = [:synonym, :prefLabel, :definition, :label]
           @list_attributes = list_attributes
           @objects_by_lang = {}
           @unmapped = unmapped
-          @requested_lang = requested_lang
+          @requested_lang = get_language(requested_lang)
         end
 
         def enrich_models(models_by_id)
diff --git a/lib/goo/sparql/solutions_mapper.rb b/lib/goo/sparql/solutions_mapper.rb
index 4cfd9104..13cdbbd6 100644
--- a/lib/goo/sparql/solutions_mapper.rb
+++ b/lib/goo/sparql/solutions_mapper.rb
@@ -22,7 +22,7 @@ def initialize(aggregate_projections, bnode_extraction, embed_struct,
         @incl = options[:include]
         @count = options[:count]
         @collection = options[:collection]
-        @requested_lang =  get_language(options[:requested_lang].to_s) 
+        @options = options
       end
       
       def map_each_solutions(select)
@@ -31,7 +31,7 @@ def map_each_solutions(select)
         list_attributes = Set.new(@klass.attributes(:list))
         all_attributes = Set.new(@klass.attributes(:all))
 
-        @lang_filter = Goo::SPARQL::Solution::LanguageFilter.new(requested_lang: @requested_lang, unmapped: @unmapped,
+        @lang_filter = Goo::SPARQL::Solution::LanguageFilter.new(requested_lang: @options[:requested_lang].to_s, unmapped: @unmapped,
            list_attributes: list_attributes)
         
         select.each_solution do |sol|

From 966ab12be7e46f3c278952762adcaa1eafefb710 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Fri, 14 Jul 2023 18:10:30 +0200
Subject: [PATCH 128/168] remove no more used @attributes_to_translate variable
 in lang_filter

---
 lib/goo/sparql/mixins/solution_lang_filter.rb | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index f29ac635..6d93c8e3 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -6,7 +6,6 @@ class  LanguageFilter
         attr_reader :requested_lang, :unmapped, :objects_by_lang
 
         def initialize(requested_lang: RequestStore.store[:requested_lang], unmapped: false, list_attributes: [])
-          @attributes_to_translate = [:synonym, :prefLabel, :definition, :label]
           @list_attributes = list_attributes
           @objects_by_lang = {}
           @unmapped = unmapped

From a990260bdea958ce293c1de75234ee0c25fb9213 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Fri, 14 Jul 2023 18:49:30 +0200
Subject: [PATCH 129/168] fix save_model_values method to not save RDF:Literal
 object but a string

---
 lib/goo/sparql/mixins/solution_lang_filter.rb | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index bc7ed15d..2df05692 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -145,7 +145,11 @@ def save_model_values(model, values, predicate, unmapped)
             add_unmapped_to_model(model, predicate, values) 
 
           else 
-            values = values.map { |k, v| [k, v.first] }.to_h unless list_attributes?(predicate)
+            values = values.map do  |language, values_literals|
+              values_string = values_literals.map{|x| x.object}
+              values_string = values_string.first unless list_attributes?(predicate)
+              [language, values_string]
+            end.to_h
 
             model.send("#{predicate}=", values, on_load: true)
           end

From 5d0a84b10b7adad31bbadd260d86c6d6d63b014b Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Fri, 14 Jul 2023 18:55:51 +0200
Subject: [PATCH 130/168] remove  not used method in lang filter module

---
 lib/goo/sparql/mixins/solution_lang_filter.rb | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index 2df05692..b6d5efd9 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -120,15 +120,6 @@ def store_objects_by_lang(id, predicate, object, language)
         end
 
 
-        def get_model_attribute_value(model, predicate)
-          if unmapped
-            unmapped_get(model, predicate)
-          else
-            model.instance_variable_get("@#{predicate}")
-          end
-        end
-
-
         def add_unmapped_to_model(model, predicate, value)
           
           if model.respond_to? :klass # struct

From 58b07f3f743e628f260519d81e3301f222e45095 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Fri, 14 Jul 2023 19:06:46 +0200
Subject: [PATCH 131/168] refecator and rename some methods of the lang_filter
 module

---
 lib/goo/sparql/mixins/solution_lang_filter.rb | 108 +++++++++---------
 1 file changed, 53 insertions(+), 55 deletions(-)

diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index b6d5efd9..7e2bd788 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -2,7 +2,7 @@ module Goo
   module SPARQL
     module Solution
       class  LanguageFilter
-        
+
         attr_reader :requested_lang, :unmapped, :objects_by_lang
 
         def initialize(requested_lang: RequestStore.store[:requested_lang], unmapped: false, list_attributes: [])
@@ -12,54 +12,29 @@ def initialize(requested_lang: RequestStore.store[:requested_lang], unmapped: fa
           @requested_lang = get_language(requested_lang)
         end
 
-        def enrich_models(models_by_id)
-          
+        def fill_models_with_all_languages(models_by_id)
           objects_by_lang.each do |id, predicates|
             model = models_by_id[id]
-            predicates.each do |predicate, values|   
-            
+            predicates.each do |predicate, values|
+
               if values.values.all? { |v| v.all? { |x| literal?(x) && x.plain?} }
-                save_model_values(model, values, predicate, unmapped)
-                
-              end 
+                pull_stored_values(model, values, predicate, @unmapped)
+              end
             end
-          end  
+          end
         end
-        
-
-        def set_model_value(model, predicate, objects, object)
-                    
-          language = object_language(object)
 
-          if requested_lang.eql?(:ALL) || !literal?(object) || language_match?(language)
-            model.send("#{predicate}=", objects, on_load: true)
-          end 
 
-          if requested_lang.eql?(:ALL) || requested_lang.is_a?(Array)
-            language = "@none" if language.nil? || language.eql?(:no_lang)
-            store_objects_by_lang(model.id, predicate, object, language)
+        def set_model_value(model, predicate, values, value)
+          set_value(model, predicate, value) do
+            model.send("#{predicate}=", values, on_load: true)
           end
-
         end
 
-        def model_set_unmapped(model, predicate, value)
-          language = object_language(value)
-          if requested_lang.eql?(:ALL) || language.nil? || language_match?(language)
+        def set_unmapped_value(model, predicate, value)
+          set_value(model, predicate, value) do
             return add_unmapped_to_model(model, predicate, value)
           end
-          
-          store_objects_by_lang(model.id, predicate, value, language)
-        end
-
-        def model_group_by_lang(model, requested_lang)
-          unmapped = model.unmapped 
-          cpy = {}
-  
-          unmapped.each do |attr, v|          
-            cpy[attr] = group_by_lang(v)
-          end
-  
-          model.unmapped = cpy
         end
 
         def models_unmapped_to_array(m)
@@ -72,16 +47,41 @@ def models_unmapped_to_array(m)
 
         private
 
+
+        def set_value(model, predicate, value, &block)
+          language = object_language(value)
+
+          if requested_lang.eql?(:ALL) || !literal?(value) || language_match?(language)
+            block.call
+          end
+
+          if requested_lang.eql?(:ALL) || requested_lang.is_a?(Array)
+            language = "@none" if language.nil? || language.eql?(:no_lang)
+            store_objects_by_lang(model.id, predicate, value, language)
+          end
+        end
+        
+        def model_group_by_lang(model)
+          unmapped = model.unmapped
+          cpy = {}
+
+          unmapped.each do |attr, v|
+            cpy[attr] = group_by_lang(v)
+          end
+
+          model.unmapped = cpy
+        end
+
         def group_by_lang(values)
-          
+
           return values.to_a if is_a_uri?(values.first)
-          
+
           values = values.group_by { |x| x.language ? x.language.to_s.downcase : :none }
-                              
+
           no_lang = values[:none] || []
           return no_lang if !no_lang.empty? && no_lang.all? { |x| !x.plain? }
 
-          values 
+          values
         end
 
         def is_a_uri?(value)
@@ -94,24 +94,23 @@ def object_language(new_value)
 
         def language_match?(language)
           # no_lang means that the object is not a literal
-          if language.eql?(:no_lang)
-            return true 
-          end
+          return true if language.eql?(:no_lang)
 
-          if requested_lang.is_a?(Array)
-            return requested_lang.include?(language)
-          end
+          return requested_lang.include?(language)  if requested_lang.is_a?(Array)
 
-          return language.eql?(requested_lang)
+          language.eql?(requested_lang)
+        end
 
+        def literal?(object)
+          !object_language(object).nil?
         end
 
         def store_objects_by_lang(id, predicate, object, language)
           # store objects in this format: [id][predicate][language] = [objects]
           return if requested_lang.is_a?(Array) && !requested_lang.include?(language)
 
-          language_key = language.downcase  
-            
+          language_key = language.downcase
+
           objects_by_lang[id] ||= {}
           objects_by_lang[id][predicate] ||= {}
           objects_by_lang[id][predicate][language_key] ||= []
@@ -121,7 +120,7 @@ def store_objects_by_lang(id, predicate, object, language)
 
 
         def add_unmapped_to_model(model, predicate, value)
-          
+
           if model.respond_to? :klass # struct
             model[:unmapped] ||= {}
             model[:unmapped][predicate] ||= []
@@ -131,11 +130,10 @@ def add_unmapped_to_model(model, predicate, value)
           end
         end
 
-        def save_model_values(model, values, predicate, unmapped)
+        def pull_stored_values(model, values, predicate, unmapped)
           if unmapped
-            add_unmapped_to_model(model, predicate, values) 
-
-          else 
+            add_unmapped_to_model(model, predicate, values)
+          else
             values = values.map do  |language, values_literals|
               values_string = values_literals.map{|x| x.object}
               values_string = values_string.first unless list_attributes?(predicate)

From efd0436fa52e75ad0cc7eb32a4a166f2c3b8efba Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Fri, 14 Jul 2023 19:07:27 +0200
Subject: [PATCH 132/168] use the new name of the lang filter methods in the
 solution mapper

---
 lib/goo/sparql/solutions_mapper.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/goo/sparql/solutions_mapper.rb b/lib/goo/sparql/solutions_mapper.rb
index 13cdbbd6..e537cae1 100644
--- a/lib/goo/sparql/solutions_mapper.rb
+++ b/lib/goo/sparql/solutions_mapper.rb
@@ -76,7 +76,7 @@ def map_each_solutions(select)
         end
       
         # for this moment we are not going to enrich models , maybe we will use it if the results are empty  
-        @lang_filter.enrich_models(@models_by_id)
+        @lang_filter.fill_models_with_all_languages(@models_by_id)
 
         init_unloaded_attributes(found, list_attributes)
 
@@ -419,7 +419,7 @@ def add_unmapped_to_model(sol)
         id = sol[:id]
         value = sol[:attributeObject]
 
-        @lang_filter.model_set_unmapped(@models_by_id[id], @properties_to_include[predicate][:uri], value)
+        @lang_filter.set_unmapped_value(@models_by_id[id], @properties_to_include[predicate][:uri], value)
       end
 
       def add_aggregations_to_model(sol)

From e26009af8a11ffbc6a7ab016c41645b76bcac936 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Fri, 14 Jul 2023 19:20:57 +0200
Subject: [PATCH 133/168] replace the getters argument to show languages from
 :show_all_languages to :show_languages: true

---
 lib/goo/base/settings/settings.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/goo/base/settings/settings.rb b/lib/goo/base/settings/settings.rb
index 6c416298..1308bef0 100644
--- a/lib/goo/base/settings/settings.rb
+++ b/lib/goo/base/settings/settings.rb
@@ -401,7 +401,7 @@ def read_only(attributes)
         end
 
         def show_all_languages?(args)
-          args.include?(:show_with_language)
+          args.first.is_a?(Hash) && args.first.keys.include?(:show_languages) && args.first[:show_languages]
         end
 
         def not_show_all_languages?(values, args)

From e5d4d1a87d77c2018e14476b02ec391465c2f97a Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Fri, 14 Jul 2023 20:56:55 +0200
Subject: [PATCH 134/168] catch transform_values of unmapped if it is nil

---
 lib/goo/base/resource.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/goo/base/resource.rb b/lib/goo/base/resource.rb
index 633c9594..5166a967 100644
--- a/lib/goo/base/resource.rb
+++ b/lib/goo/base/resource.rb
@@ -137,7 +137,7 @@ def unmmaped_to_array
       end
 
       def unmapped(*args)
-        @unmapped.transform_values do  |language_values|
+        @unmapped&.transform_values do  |language_values|
           self.class.not_show_all_languages?(language_values, args) ?  language_values.values.flatten: language_values
         end
       end

From 5952f8708f64f0a28dcf2ae44c4d8e4610682448 Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Sat, 15 Jul 2023 00:45:46 +0200
Subject: [PATCH 135/168] change the getters show_all_languages argument from
 to include_languages

---
 lib/goo/base/settings/settings.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/goo/base/settings/settings.rb b/lib/goo/base/settings/settings.rb
index 1308bef0..1263a44a 100644
--- a/lib/goo/base/settings/settings.rb
+++ b/lib/goo/base/settings/settings.rb
@@ -401,7 +401,7 @@ def read_only(attributes)
         end
 
         def show_all_languages?(args)
-          args.first.is_a?(Hash) && args.first.keys.include?(:show_languages) && args.first[:show_languages]
+          args.first.is_a?(Hash) && args.first.keys.include?(:include_languages) && args.first[:include_languages]
         end
 
         def not_show_all_languages?(values, args)

From a81f12ca1d5d6cf7c4b7c30718ff6bd8e4a56aff Mon Sep 17 00:00:00 2001
From: Syphax Bouazzouni <gs_bouazzouni@esi.dz>
Date: Sat, 15 Jul 2023 00:46:21 +0200
Subject: [PATCH 136/168] make the map_attributes handle the option showing all
 the languages

---
 lib/goo/base/resource.rb | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/lib/goo/base/resource.rb b/lib/goo/base/resource.rb
index 5166a967..02709f5e 100644
--- a/lib/goo/base/resource.rb
+++ b/lib/goo/base/resource.rb
@@ -361,13 +361,13 @@ def self.range_object(attr,id)
 
 
 
-      def self.map_attributes(inst,equivalent_predicates=nil)
+      def self.map_attributes(inst,equivalent_predicates=nil, include_languages: false)
         if (inst.kind_of?(Goo::Base::Resource) && inst.unmapped.nil?) ||
           (!inst.respond_to?(:unmapped) && inst[:unmapped].nil?)
           raise ArgumentError, "Resource.map_attributes only works for :unmapped instances"
         end
         klass = inst.respond_to?(:klass) ? inst[:klass] : inst.class
-        unmapped = inst.respond_to?(:klass) ? inst[:unmapped] : inst.unmapped
+        unmapped = inst.respond_to?(:klass) ? inst[:unmapped] : inst.unmapped(include_languages: include_languages)
         list_attrs = klass.attributes(:list)
         unmapped_string_keys = Hash.new
         unmapped.each do |k,v|
@@ -398,13 +398,18 @@ def self.map_attributes(inst,equivalent_predicates=nil)
               object = unmapped_string_keys[attr_uri]
             end
 
-            object = object.map {|o| o.is_a?(RDF::URI) ? o : o.object}
+            if object.is_a?(Hash)
+              object = object.transform_values{|values| Array(values).map{|o|o.is_a?(RDF::URI) ? o : o.object}}
+            else
+              object = object.map {|o| o.is_a?(RDF::URI) ? o : o.object}
+            end
 
             if klass.range(attr)
               object = object.map { |o|
                 o.is_a?(RDF::URI) ? klass.range_object(attr,o) : o }
             end
-            object = object.first unless list_attrs.include?(attr)
+
+            object = object.first unless list_attrs.include?(attr) || include_languages
             if inst.respond_to?(:klass)
               inst[attr] = object
             else

From cc0c27eb6428a6fcd5b2b159735a45d43ecac59a Mon Sep 17 00:00:00 2001
From: Syphax bouazzouni <gs_bouazzouni@esi.dz>
Date: Fri, 28 Apr 2023 09:41:47 +0200
Subject: [PATCH 137/168] Merge pull request #33 from
 ontoportal-lirmm/feature/make-regex-filter-no-case-sensitive

Feature: Make  the regex filter no-case sensitive
---
 lib/goo/sparql/query_builder.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/goo/sparql/query_builder.rb b/lib/goo/sparql/query_builder.rb
index a6e4f634..3cca37c4 100644
--- a/lib/goo/sparql/query_builder.rb
+++ b/lib/goo/sparql/query_builder.rb
@@ -321,7 +321,7 @@ def query_filter_sparql(klass, filter, filter_patterns, filter_graphs,
               return :optional
             when :regex
               if  filter_operation.value.is_a?(String)
-                filter_operations << "REGEX(STR(?#{filter_var.to_s}) , \"#{filter_operation.value.to_s}\")"
+                filter_operations << "REGEX(STR(?#{filter_var.to_s}) , \"#{filter_operation.value.to_s}\", \"i\")"
               end
 
             else

From 9753f042d69191b2e44eb8761318839dccd4c952 Mon Sep 17 00:00:00 2001
From: Syphax bouazzouni <gs_bouazzouni@esi.dz>
Date: Fri, 28 Apr 2023 09:38:35 +0200
Subject: [PATCH 138/168] Merge pull request #34 from
 ontoportal-lirmm/fix/filters-with-pagination-empty

Fix: Add filters patterns to select variables to resolve empty pagination
---
 lib/goo/sparql/query_builder.rb | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/lib/goo/sparql/query_builder.rb b/lib/goo/sparql/query_builder.rb
index 3cca37c4..5227dac2 100644
--- a/lib/goo/sparql/query_builder.rb
+++ b/lib/goo/sparql/query_builder.rb
@@ -33,13 +33,12 @@ def build_select_query(ids, variables, graphs, patterns,
         @order_by, variables, optional_patterns = init_order_by(@count, @klass, @order_by, optional_patterns, variables)
         variables, patterns = add_some_type_to_id(patterns, query_options, variables)
 
-        query_filter_str, patterns, optional_patterns =
+        query_filter_str, patterns, optional_patterns, filter_variables =
           filter_query_strings(@collection, graphs, internal_variables, @klass, optional_patterns, patterns, @query_filters)
-
         variables = [] if @count
         variables.delete :some_type
 
-        select_distinct(variables, aggregate_projections)
+        select_distinct(variables, aggregate_projections, filter_variables)
           .from(graphs)
           .where(patterns)
           .union_bind_in_where(properties_to_include)
@@ -135,10 +134,10 @@ def from(graphs)
         self
       end
 
-      def select_distinct(variables, aggregate_projections)
-
+      def select_distinct(variables, aggregate_projections, filter_variables)
         select_vars = variables.dup
         reject_aggregations_from_vars(select_vars, aggregate_projections) if aggregate_projections
+        select_vars = (select_vars + filter_variables).uniq  if @page # Fix for 4store pagination with a filter
         @query = @query.select(*select_vars).distinct(true)
         self
       end
@@ -347,8 +346,8 @@ def filter_query_strings(collection, graphs, internal_variables, klass,
                                optional_patterns, patterns,
                                query_filters)
         query_filter_str = []
-
         filter_graphs = []
+        filter_variables = []
         inspected_patterns = {}
         query_filters&.each do |query_filter|
           filter_operations = []
@@ -365,9 +364,9 @@ def filter_query_strings(collection, graphs, internal_variables, klass,
               patterns.concat(filter_patterns)
             end
           end
+          filter_variables << inspected_patterns.values.last
         end
-
-        [query_filter_str, patterns, optional_patterns, internal_variables]
+        [query_filter_str, patterns, optional_patterns, filter_variables]
       end
 
       def reject_aggregations_from_vars(variables, aggregate_projections)

From a82e6f54014765426b22db2a6d5f5ea8ba04ddbe Mon Sep 17 00:00:00 2001
From: Syphax bouazzouni <gs_bouazzouni@esi.dz>
Date: Mon, 1 May 2023 19:44:20 +0200
Subject: [PATCH 139/168] Merge pull request #36 from
 ontoportal-lirmm/feature/add-complex-oder-by

Feature: Add complex oder_by for joined attributes
---
 lib/goo/sparql/query_builder.rb | 53 ++++++++++++++++++++++-----------
 test/test_where.rb              | 11 +++++++
 2 files changed, 47 insertions(+), 17 deletions(-)

diff --git a/lib/goo/sparql/query_builder.rb b/lib/goo/sparql/query_builder.rb
index 5227dac2..40e888d0 100644
--- a/lib/goo/sparql/query_builder.rb
+++ b/lib/goo/sparql/query_builder.rb
@@ -17,24 +17,25 @@ def initialize(options)
         @model_query_options = options[:query_options]
         @enable_rules = options[:rules]
         @order_by = options[:order_by]
-
+        @internal_variables_map = {}
         @query = get_client
       end
 
       def build_select_query(ids, variables, graphs, patterns,
                              query_options, properties_to_include)
 
-        internal_variables = graph_match(@collection, @graph_match, graphs, @klass, patterns, query_options, @unions)
+        patterns = graph_match(@collection, @graph_match, graphs, @klass, patterns, query_options, @unions)
 
         aggregate_projections, aggregate_vars,
           variables, optional_patterns = get_aggregate_vars(@aggregate, @collection, graphs,
-                                                            @klass, @unions, variables, internal_variables)
+                                                            @klass, @unions, variables)
 
-        @order_by, variables, optional_patterns = init_order_by(@count, @klass, @order_by, optional_patterns, variables)
+        @order_by, variables, optional_patterns = init_order_by(@count, @klass, @order_by, optional_patterns, variables,patterns, query_options, graphs)
         variables, patterns = add_some_type_to_id(patterns, query_options, variables)
 
         query_filter_str, patterns, optional_patterns, filter_variables =
-          filter_query_strings(@collection, graphs, internal_variables, @klass, optional_patterns, patterns, @query_filters)
+          filter_query_strings(@collection, graphs, @klass, optional_patterns, patterns, @query_filters)
+
         variables = [] if @count
         variables.delete :some_type
 
@@ -54,7 +55,7 @@ def build_select_query(ids, variables, graphs, patterns,
         @query.union(*@unions) unless @unions.empty?
 
         ids_filter(ids) if ids
-        order_by if @order_by # TODO  test if work
+        order_by if @order_by
 
         put_query_aggregate_vars(aggregate_vars) if aggregate_vars
         count if @count
@@ -117,7 +118,13 @@ def put_query_aggregate_vars(aggregate_vars)
       end
 
       def order_by
-        order_by_str = @order_by.map { |attr, order| "#{order.to_s.upcase}(?#{attr})" }
+        order_by_str = @order_by.map do |attr, order|
+          if order.is_a?(Hash)
+            sub_attr, order = order.first
+            attr = @internal_variables_map[sub_attr]
+          end
+          "#{order.to_s.upcase}(?#{attr})"
+        end
         @query.order_by(*order_by_str)
         self
       end
@@ -169,6 +176,7 @@ def patterns_for_match(klass, attr, value, graphs, patterns, unions,
             value = "#{attr}_agg_#{in_aggregate}".to_sym
           end
           internal_variables << value
+          @internal_variables_map[attr] = value
         end
 
         add_rules(attr, klass, query_options)
@@ -209,7 +217,7 @@ def walk_pattern(klass, match_patterns, graphs, patterns, unions,
         end
       end
 
-      def get_aggregate_vars(aggregate, collection, graphs, klass, unions, variables, internal_variables)
+      def get_aggregate_vars(aggregate, collection, graphs, klass, unions, variables)
         # mdorf, 6/03/20 If aggregate projections (sub-SELECT within main SELECT) use an alias, that alias cannot appear in the main SELECT
         # https://github.com/ncbo/goo/issues/106
         # See last sentence in https://www.w3.org/TR/sparql11-query/#aggregateExample
@@ -240,8 +248,6 @@ def get_aggregate_vars(aggregate, collection, graphs, klass, unions, variables,
       end
 
       def graph_match(collection, graph_match, graphs, klass, patterns, query_options, unions)
-        internal_variables = []
-
         if graph_match
           #make it deterministic - for caching
           graph_match_iteration = Goo::Base::PatternIteration.new(graph_match)
@@ -249,28 +255,38 @@ def graph_match(collection, graph_match, graphs, klass, patterns, query_options,
                        internal_variables, in_aggregate = false, query_options, collection)
           graphs.uniq!
         end
-        internal_variables
+        patterns
       end
 
       def get_client
         Goo.sparql_query_client(@store)
       end
 
-      def init_order_by(count, klass, order_by, optional_patterns, variables)
+      def init_order_by(count, klass, order_by, optional_patterns, variables, patterns, query_options, graphs)
         order_by = nil if count
         if order_by
           order_by = order_by.first
           #simple ordering ... needs to use pattern inspection
           order_by.each do |attr, direction|
-            quad = query_pattern(klass, attr)
-            optional_patterns << quad[1]
+
+            if direction.is_a?(Hash)
+              sub_attr, direction = direction.first
+              graph_match_iteration = Goo::Base::PatternIteration.new(Goo::Base::Pattern.new({attr => [sub_attr]}))
+              old_internal = internal_variables.dup
+              walk_pattern(klass, graph_match_iteration, graphs, optional_patterns, @unions, internal_variables, in_aggregate = false, query_options, @collection)
+              variables << (internal_variables - old_internal).last
+            else
+              quad = query_pattern(klass, attr)
+              optional_patterns << quad[1]
+              variables << attr
+            end
+
             #patterns << quad[1]
             #mdorf, 9/22/16 If an ORDER BY clause exists, the columns used in the ORDER BY should be present in the SPARQL select
             #variables << attr unless variables.include?(attr)
           end
-          variables = %i[id attributeProperty attributeObject]
         end
-        [order_by, variables, optional_patterns]
+        [order_by, variables, optional_patterns, patterns]
       end
 
       def sparql_op_string(op)
@@ -342,7 +358,7 @@ def query_filter_sparql(klass, filter, filter_patterns, filter_graphs,
         end
       end
 
-      def filter_query_strings(collection, graphs, internal_variables, klass,
+      def filter_query_strings(collection, graphs, klass,
                                optional_patterns, patterns,
                                query_filters)
         query_filter_str = []
@@ -382,6 +398,9 @@ def add_some_type_to_id(patterns, query_options, variables)
         [variables, patterns]
       end
 
+      def internal_variables
+        @internal_variables_map.values
+      end
     end
   end
 end
diff --git a/test/test_where.rb b/test/test_where.rb
index bca4b2ea..30d933e3 100644
--- a/test/test_where.rb
+++ b/test/test_where.rb
@@ -600,4 +600,15 @@ def test_include_inverse_with_find
     end
   end
 
+  def test_complex_order_by
+    u = University.where.include(address: [:country]).order_by(address: {country: :asc}).all
+    countries = u.map {|x| x.address.map{|a| a.country}}.flatten
+    assert_equal countries.sort, countries
+
+
+    u = University.where.include(address: [:country]).order_by(address: {country: :desc}).all
+    countries = u.map {|x| x.address.map{|a| a.country}}.flatten
+    assert_equal countries.sort{|a,b| b<=>a }, countries
+  end
+
 end

From cd7d26d2fcbeb0992e175a736e21a4c146aab139 Mon Sep 17 00:00:00 2001
From: Syphax bouazzouni <gs_bouazzouni@esi.dz>
Date: Thu, 21 Sep 2023 19:20:07 +0200
Subject: [PATCH 140/168] fix order by an attribute that is already filtered

---
 lib/goo/sparql/query_builder.rb | 60 +++++++++++++++++++++------------
 1 file changed, 39 insertions(+), 21 deletions(-)

diff --git a/lib/goo/sparql/query_builder.rb b/lib/goo/sparql/query_builder.rb
index 40e888d0..d0814457 100644
--- a/lib/goo/sparql/query_builder.rb
+++ b/lib/goo/sparql/query_builder.rb
@@ -25,16 +25,14 @@ def build_select_query(ids, variables, graphs, patterns,
                              query_options, properties_to_include)
 
         patterns = graph_match(@collection, @graph_match, graphs, @klass, patterns, query_options, @unions)
+        variables, patterns = add_some_type_to_id(patterns, query_options, variables)
+        aggregate_projections, aggregate_vars, variables, optional_patterns = get_aggregate_vars(@aggregate, @collection, graphs, @klass, @unions, variables)
+        query_filter_str, patterns, optional_patterns, filter_variables =
+          filter_query_strings(@collection, graphs, @klass, optional_patterns, patterns, @query_filters)
 
-        aggregate_projections, aggregate_vars,
-          variables, optional_patterns = get_aggregate_vars(@aggregate, @collection, graphs,
-                                                            @klass, @unions, variables)
 
         @order_by, variables, optional_patterns = init_order_by(@count, @klass, @order_by, optional_patterns, variables,patterns, query_options, graphs)
-        variables, patterns = add_some_type_to_id(patterns, query_options, variables)
 
-        query_filter_str, patterns, optional_patterns, filter_variables =
-          filter_query_strings(@collection, graphs, @klass, optional_patterns, patterns, @query_filters)
 
         variables = [] if @count
         variables.delete :some_type
@@ -121,7 +119,7 @@ def order_by
         order_by_str = @order_by.map do |attr, order|
           if order.is_a?(Hash)
             sub_attr, order = order.first
-            attr = @internal_variables_map[sub_attr]
+            attr =  @internal_variables_map.select{ |internal_var, attr_var| attr_var.eql?({attr => sub_attr}) || attr_var.eql?(sub_attr)}.keys.last
           end
           "#{order.to_s.upcase}(?#{attr})"
         end
@@ -165,23 +163,24 @@ def ids_filter(ids)
       def patterns_for_match(klass, attr, value, graphs, patterns, unions,
                              internal_variables, subject = :id, in_union = false,
                              in_aggregate = false, query_options = {}, collection = nil)
+        new_internal_var = value
         if value.respond_to?(:each) || value.instance_of?(Symbol)
           next_pattern = value.instance_of?(Array) ? value.first : value
 
           #for filters
           next_pattern = { next_pattern => [] } if next_pattern.instance_of?(Symbol)
 
-          value = "internal_join_var_#{internal_variables.length}".to_sym
+          new_internal_var = "internal_join_var_#{internal_variables.length}".to_sym
           if in_aggregate
-            value = "#{attr}_agg_#{in_aggregate}".to_sym
+            new_internal_var = "#{attr}_agg_#{in_aggregate}".to_sym
           end
-          internal_variables << value
-          @internal_variables_map[attr] = value
+          internal_variables << new_internal_var
+          @internal_variables_map[new_internal_var] = value.empty? ? attr : {attr => value}
         end
 
         add_rules(attr, klass, query_options)
         graph, pattern =
-          query_pattern(klass, attr, value: value, subject: subject, collection: collection)
+          query_pattern(klass, attr, value: new_internal_var, subject: subject, collection: collection)
         if pattern
           if !in_union
             patterns << pattern
@@ -194,7 +193,7 @@ def patterns_for_match(klass, attr, value, graphs, patterns, unions,
           range = klass.range(attr)
           next_pattern.each do |next_attr, next_value|
             patterns_for_match(range, next_attr, next_value, graphs,
-                               patterns, unions, internal_variables, subject = value,
+                               patterns, unions, internal_variables, subject = new_internal_var,
                                in_union, in_aggregate, collection = collection)
           end
         end
@@ -270,11 +269,30 @@ def init_order_by(count, klass, order_by, optional_patterns, variables, patterns
           order_by.each do |attr, direction|
 
             if direction.is_a?(Hash)
+              # TODO this part can be improved/refactored, the complexity was added because order by don't work
+              # if the pattern is in the mandatory ones (variable `patterns`)
+              # and optional (variable `optional_patterns`) at the same time
               sub_attr, direction = direction.first
               graph_match_iteration = Goo::Base::PatternIteration.new(Goo::Base::Pattern.new({attr => [sub_attr]}))
               old_internal = internal_variables.dup
+              old_patterns = optional_patterns.dup
+
               walk_pattern(klass, graph_match_iteration, graphs, optional_patterns, @unions, internal_variables, in_aggregate = false, query_options, @collection)
-              variables << (internal_variables - old_internal).last
+              new_variables = (internal_variables - old_internal)
+              internal_variables.delete(new_variables)
+              new_patterns = optional_patterns - old_patterns
+              already_existent_pattern = patterns.select{|x| x[1].eql?(new_patterns.last[1])}.first
+
+              if already_existent_pattern
+                already_existent_variable = already_existent_pattern[2]
+                optional_patterns = old_patterns
+                key = @internal_variables_map.select{|key, value|  key.eql?(new_variables.last)}.keys.first
+                @internal_variables_map[key] = (already_existent_variable || new_variables.last) if key
+                variables << already_existent_variable
+              else
+                variables <<  new_variables.last
+              end
+
             else
               quad = query_pattern(klass, attr)
               optional_patterns << quad[1]
@@ -325,7 +343,12 @@ def query_filter_sparql(klass, filter, filter_patterns, filter_graphs,
           end
           filter_var = inspected_patterns[filter_pattern_match]
 
-          unless filter_operation.value.instance_of?(Goo::Filter)
+          if filter_operation.value.instance_of?(Goo::Filter)
+            filter_operations << "#{sparql_op_string(filter_operation.operator)}"
+            query_filter_sparql(klass, filter_operation.value, filter_patterns,
+                                filter_graphs, filter_operations,
+                                internal_variables, inspected_patterns, collection)
+          else
             case filter_operation.operator
             when  :unbound
               filter_operations << "!BOUND(?#{filter_var.to_s})"
@@ -349,11 +372,6 @@ def query_filter_sparql(klass, filter, filter_patterns, filter_graphs,
                   " #{value.to_ntriples}")
             end
 
-          else
-            filter_operations << "#{sparql_op_string(filter_operation.operator)}"
-            query_filter_sparql(klass, filter_operation.value, filter_patterns,
-                                filter_graphs, filter_operations,
-                                internal_variables, inspected_patterns, collection)
           end
         end
       end
@@ -399,7 +417,7 @@ def add_some_type_to_id(patterns, query_options, variables)
       end
 
       def internal_variables
-        @internal_variables_map.values
+        @internal_variables_map.keys
       end
     end
   end

From 891fa997fc35b50432fd645c4b0a11fd25eecf4d Mon Sep 17 00:00:00 2001
From: Syphax bouazzouni <gs_bouazzouni@esi.dz>
Date: Fri, 22 Sep 2023 03:30:17 +0200
Subject: [PATCH 141/168] don't add the filtered variables to the select clause
 of the query

---
 lib/goo/sparql/query_builder.rb | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/lib/goo/sparql/query_builder.rb b/lib/goo/sparql/query_builder.rb
index d0814457..a65f5ec9 100644
--- a/lib/goo/sparql/query_builder.rb
+++ b/lib/goo/sparql/query_builder.rb
@@ -288,15 +288,16 @@ def init_order_by(count, klass, order_by, optional_patterns, variables, patterns
                 optional_patterns = old_patterns
                 key = @internal_variables_map.select{|key, value|  key.eql?(new_variables.last)}.keys.first
                 @internal_variables_map[key] = (already_existent_variable || new_variables.last) if key
-                variables << already_existent_variable
+
+                #variables << already_existent_variable
               else
-                variables <<  new_variables.last
+                #variables <<  new_variables.last
               end
 
             else
               quad = query_pattern(klass, attr)
               optional_patterns << quad[1]
-              variables << attr
+              #variables << attr
             end
 
             #patterns << quad[1]
@@ -398,7 +399,7 @@ def filter_query_strings(collection, graphs, klass,
               patterns.concat(filter_patterns)
             end
           end
-          filter_variables << inspected_patterns.values.last
+          #filter_variables << inspected_patterns.values.last
         end
         [query_filter_str, patterns, optional_patterns, filter_variables]
       end

From e87c21e06164bf0bb08c7035a408fb24f0b2327b Mon Sep 17 00:00:00 2001
From: Syphax bouazzouni <gs_bouazzouni@esi.dz>
Date: Thu, 19 Oct 2023 17:19:14 +0200
Subject: [PATCH 142/168] add filters patterns to select variables

---
 lib/goo/sparql/query_builder.rb | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/goo/sparql/query_builder.rb b/lib/goo/sparql/query_builder.rb
index a65f5ec9..d1843dda 100644
--- a/lib/goo/sparql/query_builder.rb
+++ b/lib/goo/sparql/query_builder.rb
@@ -33,7 +33,8 @@ def build_select_query(ids, variables, graphs, patterns,
 
         @order_by, variables, optional_patterns = init_order_by(@count, @klass, @order_by, optional_patterns, variables,patterns, query_options, graphs)
 
-
+        query_filter_str, patterns, optional_patterns, filter_variables =
+          filter_query_strings(@collection, graphs, internal_variables, @klass, optional_patterns, patterns, @query_filters)
         variables = [] if @count
         variables.delete :some_type
 

From 19c9ce19291f3ba74f60191482216a5530c71dc1 Mon Sep 17 00:00:00 2001
From: Syphax bouazzouni <gs_bouazzouni@esi.dz>
Date: Thu, 19 Oct 2023 17:19:40 +0200
Subject: [PATCH 143/168] fix pagination with order_by with filter that returns
 empty pages for 4store

---
 lib/goo/sparql/query_builder.rb | 27 +++++++++++++++++----------
 test/test_where.rb              | 11 +++++++++++
 2 files changed, 28 insertions(+), 10 deletions(-)

diff --git a/lib/goo/sparql/query_builder.rb b/lib/goo/sparql/query_builder.rb
index d1843dda..6077bd60 100644
--- a/lib/goo/sparql/query_builder.rb
+++ b/lib/goo/sparql/query_builder.rb
@@ -32,13 +32,15 @@ def build_select_query(ids, variables, graphs, patterns,
 
 
         @order_by, variables, optional_patterns = init_order_by(@count, @klass, @order_by, optional_patterns, variables,patterns, query_options, graphs)
+        order_by_str, order_variables = order_by_string
+        variables, patterns = add_some_type_to_id(patterns, query_options, variables)
 
         query_filter_str, patterns, optional_patterns, filter_variables =
-          filter_query_strings(@collection, graphs, internal_variables, @klass, optional_patterns, patterns, @query_filters)
+          filter_query_strings(@collection, graphs, @klass, optional_patterns, patterns, @query_filters)
         variables = [] if @count
         variables.delete :some_type
 
-        select_distinct(variables, aggregate_projections, filter_variables)
+        select_distinct(variables, aggregate_projections, filter_variables, order_variables)
           .from(graphs)
           .where(patterns)
           .union_bind_in_where(properties_to_include)
@@ -54,7 +56,10 @@ def build_select_query(ids, variables, graphs, patterns,
         @query.union(*@unions) unless @unions.empty?
 
         ids_filter(ids) if ids
-        order_by if @order_by
+
+
+        @query.order_by(*order_by_str) if @order_by
+
 
         put_query_aggregate_vars(aggregate_vars) if aggregate_vars
         count if @count
@@ -116,16 +121,17 @@ def put_query_aggregate_vars(aggregate_vars)
         self
       end
 
-      def order_by
-        order_by_str = @order_by.map do |attr, order|
+      def order_by_string
+        order_variables = []
+        order_str = @order_by&.map do |attr, order|
           if order.is_a?(Hash)
             sub_attr, order = order.first
             attr =  @internal_variables_map.select{ |internal_var, attr_var| attr_var.eql?({attr => sub_attr}) || attr_var.eql?(sub_attr)}.keys.last
           end
+          order_variables << attr
           "#{order.to_s.upcase}(?#{attr})"
         end
-        @query.order_by(*order_by_str)
-        self
+        [order_str,order_variables]
       end
 
       def from(graphs)
@@ -140,10 +146,11 @@ def from(graphs)
         self
       end
 
-      def select_distinct(variables, aggregate_projections, filter_variables)
+      def select_distinct(variables, aggregate_variables, filter_variables, order_variables)
         select_vars = variables.dup
-        reject_aggregations_from_vars(select_vars, aggregate_projections) if aggregate_projections
-        select_vars = (select_vars + filter_variables).uniq  if @page # Fix for 4store pagination with a filter
+        reject_aggregations_from_vars(select_vars, aggregate_variables) if aggregate_variables
+        # Fix for 4store pagination with a filter https://github.com/ontoportal-lirmm/ontologies_api/issues/25
+        select_vars = (select_vars + filter_variables + order_variables).uniq  if @page
         @query = @query.select(*select_vars).distinct(true)
         self
       end
diff --git a/test/test_where.rb b/test/test_where.rb
index 30d933e3..748dca88 100644
--- a/test/test_where.rb
+++ b/test/test_where.rb
@@ -262,6 +262,17 @@ def test_embed_two_levels
     end
   end
 
+  def test_paging_with_filter_order
+    total_count = Student.where.count
+    page_1 = Student.where.page(1, total_count - 1).order_by(name: :asc).to_a
+    refute_empty page_1
+    assert page_1.next?
+    page_2 = Student.where.page(page_1.next_page, total_count - 1).order_by(name: :asc).to_a
+
+
+    refute_empty page_2
+    assert_equal total_count, page_1.size + page_2.size
+  end
 
   def test_unique_object_references
 

From c4dd04d90244edf9e3f8e651d68eb128d5944bf2 Mon Sep 17 00:00:00 2001
From: Syphax bouazzouni <gs_bouazzouni@esi.dz>
Date: Thu, 19 Oct 2023 19:09:16 +0200
Subject: [PATCH 144/168] include the in the select variables filtered
 variables

---
 lib/goo/sparql/query_builder.rb | 2 +-
 test/test_where.rb              | 8 +++++---
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/lib/goo/sparql/query_builder.rb b/lib/goo/sparql/query_builder.rb
index 6077bd60..3b0f3589 100644
--- a/lib/goo/sparql/query_builder.rb
+++ b/lib/goo/sparql/query_builder.rb
@@ -407,7 +407,7 @@ def filter_query_strings(collection, graphs, klass,
               patterns.concat(filter_patterns)
             end
           end
-          #filter_variables << inspected_patterns.values.last
+          filter_variables << inspected_patterns.values.last
         end
         [query_filter_str, patterns, optional_patterns, filter_variables]
       end
diff --git a/test/test_where.rb b/test/test_where.rb
index 748dca88..c80fed33 100644
--- a/test/test_where.rb
+++ b/test/test_where.rb
@@ -263,11 +263,13 @@ def test_embed_two_levels
   end
 
   def test_paging_with_filter_order
-    total_count = Student.where.count
-    page_1 = Student.where.page(1, total_count - 1).order_by(name: :asc).to_a
+
+    f = Goo::Filter.new(:birth_date) > DateTime.parse('1978-01-03')
+    total_count = Student.where.filter(f).count
+    page_1 = Student.where.include(:name, :birth_date).page(1, total_count - 1).filter(f).order_by(name: :asc).to_a
     refute_empty page_1
     assert page_1.next?
-    page_2 = Student.where.page(page_1.next_page, total_count - 1).order_by(name: :asc).to_a
+    page_2 = Student.where.include(:name, :birth_date).page(page_1.next_page, total_count - 1).filter(f).order_by(name: :asc).to_a
 
 
     refute_empty page_2

From 5247e8d55ebf3e2b6ccb8dc68a9b9df080e44924 Mon Sep 17 00:00:00 2001
From: Syphax bouazzouni <gs_bouazzouni@esi.dz>
Date: Thu, 19 Oct 2023 19:10:23 +0200
Subject: [PATCH 145/168] optimize pagination query by not re-doing the filters
 and order in the include query

---
 lib/goo/base/where.rb | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/lib/goo/base/where.rb b/lib/goo/base/where.rb
index 5bc0fa8c..81cd26ce 100644
--- a/lib/goo/base/where.rb
+++ b/lib/goo/base/where.rb
@@ -209,6 +209,11 @@ def process_query_intl(count=false)
 
         options_load[:ids] = ids if ids
         models_by_id = {}
+        if @page_i && (options_load[:models].length > 0)
+          options_load.delete(:filters)
+          options_load.delete(:order_by)
+        end
+
         if (@page_i && options_load[:models].length > 0) ||
             (!@page_i && (@count.nil? || @count > 0))
           models_by_id = Goo::SPARQL::Queries.model_load(options_load)

From 04419a8ac57a79ea5a6dd4918f5b0c677252ee7e Mon Sep 17 00:00:00 2001
From: Syphax bouazzouni <gs_bouazzouni@esi.dz>
Date: Mon, 4 Dec 2023 11:25:39 +0100
Subject: [PATCH 146/168] Merge to master: Release 2.3.3 - Multilingual (#44)

* Merge pull request #24 from ontoportal-lirmm/feature/support-multilingual-read-one-language-from-request-parameter

Feature : Support multilingual - Phase 1 - Read one language

* Merge branch pull request #32 from feature/language-return-all

* group unmapped properties by lang

* Feature: group unmapped properties by language  (#38)

* group unmapped properties by lang

* downcase language keys of unmapped properties

---------

Co-authored-by: Syphax bouazzouni <gs_bouazzouni@esi.dz>

* Feature: Support multi lingual -  add show_language argument to the attributes getters (#39)

* update define_method

* update solution mapper

* update get_preload_value

* Merge pull request #40 from ontoportal-lirmm/support-muli-lang-part-02

Fix: save_model_values if unmmaped condition

* Merge pull request #41 from ontoportal-lirmm/support-muli-lang-part-02

Feature: Refactor and fix the Language filter method

* Merge pull request #42 from ontoportal-lirmm/feature/support-multi-langual-search

Feature: Make map_attribute support showing all languages - Support multi language search

---------

Co-authored-by: HADDAD Zineddine <hz_haddad@esi.dz>
---
 Gemfile                                       |   1 +
 Gemfile.lock                                  |  15 +-
 lib/goo.rb                                    |   9 +
 lib/goo/base/resource.rb                      |  50 +-
 lib/goo/base/settings/settings.rb             |  35 +-
 lib/goo/sparql/loader.rb                      |   6 +
 lib/goo/sparql/mixins/solution_lang_filter.rb | 275 ++++--
 lib/goo/sparql/solutions_mapper.rb            | 900 +++++++++---------
 8 files changed, 695 insertions(+), 596 deletions(-)

diff --git a/Gemfile b/Gemfile
index edb00975..3564fe3b 100644
--- a/Gemfile
+++ b/Gemfile
@@ -7,6 +7,7 @@ gem "cube-ruby", require: "cube"
 gem "faraday", '~> 1.9'
 gem "rake"
 gem "uuid"
+gem "request_store"
 
 group :test do
   gem "minitest", '< 5.0'
diff --git a/Gemfile.lock b/Gemfile.lock
index 34a6c39c..9fe7bd02 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -34,7 +34,7 @@ GEM
       public_suffix (>= 2.0.2, < 6.0)
     builder (3.2.4)
     coderay (1.1.3)
-    concurrent-ruby (1.1.10)
+    concurrent-ruby (1.2.2)
     connection_pool (2.3.0)
     cube-ruby (0.0.3)
     daemons (1.4.1)
@@ -76,10 +76,10 @@ GEM
     method_source (1.0.0)
     mime-types (3.4.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2022.0105)
+    mime-types-data (3.2023.0218.1)
     minitest (4.7.5)
     multi_json (1.15.0)
-    multipart-post (2.2.3)
+    multipart-post (2.3.0)
     mustermann (3.0.0)
       ruby2_keywords (~> 0.0.1)
     net-http-persistent (2.9.4)
@@ -88,7 +88,7 @@ GEM
       coderay (~> 1.1)
       method_source (~> 1.0)
     public_suffix (5.0.1)
-    rack (2.2.6.2)
+    rack (2.2.6.3)
     rack-accept (0.4.5)
       rack (>= 0.4)
     rack-post-body-to-params (0.1.8)
@@ -100,8 +100,10 @@ GEM
       addressable (>= 2.2)
     redis (5.0.6)
       redis-client (>= 0.9.0)
-    redis-client (0.12.1)
+    redis-client (0.13.0)
       connection_pool
+    request_store (1.5.1)
+      rack (>= 1.4)
     rest-client (2.1.0)
       http-accept (>= 1.7.0, < 2.0)
       http-cookie (>= 1.0.2, < 2.0)
@@ -132,7 +134,7 @@ GEM
       eventmachine (~> 1.0, >= 1.0.4)
       rack (>= 1, < 3)
     thread_safe (0.3.6)
-    tilt (2.0.11)
+    tilt (2.1.0)
     tzinfo (0.3.61)
     unf (0.1.4)
       unf_ext
@@ -155,6 +157,7 @@ DEPENDENCIES
   rack-accept
   rack-post-body-to-params
   rake
+  request_store
   simplecov
   simplecov-cobertura
   sinatra
diff --git a/lib/goo.rb b/lib/goo.rb
index bb81541d..db863d2a 100644
--- a/lib/goo.rb
+++ b/lib/goo.rb
@@ -28,6 +28,7 @@ module Goo
   # Define the languages from which the properties values will be taken
   # It choose the first language that match otherwise return all the values
   @@main_languages = %w[en]
+  @@requested_language = nil
 
   @@configure_flag = false
   @@sparql_backends = {}
@@ -54,6 +55,14 @@ def self.main_languages=(lang)
     @@main_languages = lang
   end
 
+  def self.requested_language
+    @@requested_language
+  end
+
+  def self.requested_language=(lang)
+    @@requested_language = lang
+  end
+
   def self.language_includes(lang)
     lang_str = lang.to_s
     main_languages.index { |l| lang_str.downcase.eql?(l) || lang_str.upcase.eql?(l)}
diff --git a/lib/goo/base/resource.rb b/lib/goo/base/resource.rb
index 13c4b61a..88bbc8ce 100644
--- a/lib/goo/base/resource.rb
+++ b/lib/goo/base/resource.rb
@@ -15,7 +15,7 @@ class Resource
       attr_reader :modified_attributes
       attr_reader :errors
       attr_reader :aggregates
-      attr_reader :unmapped
+      attr_writer :unmapped
 
       attr_reader :id
 
@@ -134,17 +134,29 @@ def missing_load_attributes
 
       def unmapped_set(attribute,value)
         @unmapped ||= {}
-        (@unmapped[attribute] ||= Set.new) << value
+        @unmapped[attribute] ||= Set.new
+        @unmapped[attribute].merge(Array(value)) unless value.nil?
+      end
+ 
+      def unmapped_get(attribute)
+        @unmapped[attribute]
       end
 
       def unmmaped_to_array
         cpy = {}
+        
         @unmapped.each do |attr,v|
           cpy[attr] = v.to_a
         end
         @unmapped = cpy
       end
 
+      def unmapped(*args)
+        @unmapped.transform_values do  |language_values|
+          self.class.not_show_all_languages?(language_values, args) ?  language_values.values.flatten: language_values
+        end
+      end
+
       def delete(*args)
         if self.kind_of?(Goo::Base::Enum)
           raise ArgumentError, "Enums cannot be deleted" unless args[0] && args[0][:init_enum]
@@ -341,13 +353,13 @@ def self.range_object(attr,id)
 
 
 
-      def self.map_attributes(inst,equivalent_predicates=nil)
+      def self.map_attributes(inst,equivalent_predicates=nil, include_languages: false)
         if (inst.kind_of?(Goo::Base::Resource) && inst.unmapped.nil?) ||
           (!inst.respond_to?(:unmapped) && inst[:unmapped].nil?)
           raise ArgumentError, "Resource.map_attributes only works for :unmapped instances"
         end
         klass = inst.respond_to?(:klass) ? inst[:klass] : inst.class
-        unmapped = inst.respond_to?(:klass) ? inst[:unmapped] : inst.unmapped
+        unmapped = inst.respond_to?(:klass) ? inst[:unmapped] : inst.unmapped(include_languages: include_languages)
         list_attrs = klass.attributes(:list)
         unmapped_string_keys = Hash.new
         unmapped.each do |k,v|
@@ -378,31 +390,18 @@ def self.map_attributes(inst,equivalent_predicates=nil)
               object = unmapped_string_keys[attr_uri]
             end
 
-            lang_filter = Goo::SPARQL::Solution::LanguageFilter.new
-
-            object = object.map do |o|
-              if o.is_a?(RDF::URI)
-                o
-              else
-                literal = o
-                index, lang_val  = lang_filter.main_lang_filter inst.id.to_s, attr, literal
-                lang_val.to_s if index.eql? :no_lang
-              end
-            end
-
-            object = object.compact
-
-            other_languages_values = lang_filter.other_languages_values
-            other_languages_values = other_languages_values[inst.id.to_s][attr] unless other_languages_values.empty?
-            unless other_languages_values.nil?
-              object = lang_filter.languages_values_to_set(other_languages_values, object)
+            if object.is_a?(Hash)
+              object = object.transform_values{|values| Array(values).map{|o|o.is_a?(RDF::URI) ? o : o.object}}
+            else
+              object = object.map {|o| o.is_a?(RDF::URI) ? o : o.object}
             end
 
             if klass.range(attr)
               object = object.map { |o|
                 o.is_a?(RDF::URI) ? klass.range_object(attr,o) : o }
             end
-            object = object.first unless list_attrs.include?(attr)
+
+            object = object.first unless list_attrs.include?(attr) || include_languages
             if inst.respond_to?(:klass)
               inst[attr] = object
             else
@@ -411,11 +410,6 @@ def self.map_attributes(inst,equivalent_predicates=nil)
           else
             inst.send("#{attr}=",
                       list_attrs.include?(attr) ? [] : nil, on_load: true)
-            if inst.id.to_s == "http://purl.obolibrary.org/obo/IAO_0000415"
-              if attr == :definition
-                # binding.pry
-              end
-            end
           end
 
         end
diff --git a/lib/goo/base/settings/settings.rb b/lib/goo/base/settings/settings.rb
index ce3e9a21..a58daae0 100644
--- a/lib/goo/base/settings/settings.rb
+++ b/lib/goo/base/settings/settings.rb
@@ -255,9 +255,18 @@ def shape_attribute(attr)
             self.instance_variable_set("@#{attr}",value)
           end
           define_method("#{attr}") do |*args|
+            attr_value = self.instance_variable_get("@#{attr}")
+
+            if self.class.not_show_all_languages?(attr_value, args)
+              is_array = attr_value.values.first.is_a?(Array)
+              attr_value = attr_value.values.flatten
+              attr_value = attr_value.first unless is_array
+            end
+
+
             if self.class.handler?(attr)
               if @loaded_attributes.include?(attr)
-                return self.instance_variable_get("@#{attr}")
+                return attr_value
               end
               value = self.send("#{self.class.handler(attr)}")
               self.instance_variable_set("@#{attr}",value)
@@ -266,7 +275,7 @@ def shape_attribute(attr)
             end
 
             if (not @persistent) or @loaded_attributes.include?(attr)
-              return self.instance_variable_get("@#{attr}")
+                return attr_value
             else
               # TODO: bug here when no labels from one of the main_lang available... (when it is called by ontologies_linked_data ontologies_submission)
               raise Goo::Base::AttributeNotLoaded, "Attribute `#{attr}` is not loaded for #{self.id}. Loaded attributes: #{@loaded_attributes.inspect}."
@@ -372,6 +381,28 @@ def read_only(attributes)
           instance
         end
 
+        def show_all_languages?(args)
+          args.first.is_a?(Hash) && args.first.keys.include?(:include_languages) && args.first[:include_languages]
+        end
+
+        def not_show_all_languages?(values, args)
+          values.is_a?(Hash) && !show_all_languages?(args)
+        end
+
+        private
+
+        def set_no_list_by_default(options)
+          if options[:enforce].nil? or !options[:enforce].include?(:list)
+            options[:enforce] = options[:enforce] ? (options[:enforce] << :no_list) : [:no_list]
+          end
+        end
+        def set_data_type(options)
+          if options[:type]
+            options[:enforce] += Array(options[:type])
+            options[:enforce].uniq!
+            options.delete :type
+          end
+        end
       end
     end
   end
diff --git a/lib/goo/sparql/loader.rb b/lib/goo/sparql/loader.rb
index 821aba26..094fbba2 100644
--- a/lib/goo/sparql/loader.rb
+++ b/lib/goo/sparql/loader.rb
@@ -1,3 +1,4 @@
+require 'request_store'
 module Goo
   module SPARQL
     module Loader
@@ -6,8 +7,10 @@ class << self
 
         def model_load(*options)
           options = options.last
+          set_request_lang(options)
           if options[:models] && options[:models].is_a?(Array) && \
            (options[:models].length > Goo.slice_loading_size)
+
             options = options.dup
             models = options[:models]
             include_options = options[:include]
@@ -96,6 +99,9 @@ def model_load_sliced(*options)
 
         private
 
+        def set_request_lang(options)
+          options[:requested_lang] = RequestStore.store[:requested_lang]
+        end
         def expand_equivalent_predicates(properties_to_include, eq_p)
 
           return unless eq_p && !eq_p.empty?
diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index b5254786..8980dcdc 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -1,99 +1,176 @@
-module Goo
-  module SPARQL
-    module Solution
-      class  LanguageFilter
-
-        def initialize
-          @other_languages_values = {}
-        end
-
-        attr_reader :other_languages_values
-
-        def main_lang_filter(id, attr, value)
-          index, value = lang_index value
-          save_other_lang_val(id, attr, index, value) unless index.nil? ||index.eql?(:no_lang)
-          [index, value]
-        end
-
-        def fill_models_with_other_languages(models_by_id, list_attributes)
-          @other_languages_values.each  do |id, languages_values|
-            languages_values.each do |attr, index_values|
-              model_attribute_val = models_by_id[id].instance_variable_get("@#{attr.to_s}")
-              values = languages_values_to_set(index_values, model_attribute_val)
-              m = models_by_id[id]
-              value = nil
-              is_struct = m.respond_to?(:klass)
-              if !values.nil? && list_attributes.include?(attr)
-                value = values || []
-
-              elsif !values.nil?
-                value = values.first || nil
-              end
-
-              if value
-                if is_struct
-                  m[attr] = value
-                else
-                  m.send("#{attr}=", value, on_load: true)
-                end
-              end
-            end
-          end
-        end
-
-        def languages_values_to_set(language_values, no_lang_values)
-
-          values = nil
-          matched_lang, not_matched_lang = matched_languages(language_values, no_lang_values)
-          if !matched_lang.empty?
-            main_lang = Array(matched_lang[:'0']) + Array(matched_lang[:no_lang])
-            if main_lang.empty?
-              secondary_languages = matched_lang.select { |key| key != :'0' && key != :no_lang }.sort.map { |x| x[1] }
-              values = secondary_languages.first
-            else
-              values = main_lang
-            end
-          elsif !not_matched_lang.empty?
-            values = not_matched_lang
-          end
-          values&.uniq
-        end
-
-        private
-
-        def lang_index(object)
-          return [nil, object] unless  object.is_a?(RDF::Literal)
-
-          lang = object.language
-
-          if lang.nil?
-            [:no_lang, object]
-          else
-            index = Goo.language_includes(lang)
-            index = index ? index.to_s.to_sym : :not_matched
-            [index, object]
-          end
-        end
-
-        def save_other_lang_val(id, attr, index, value)
-          @other_languages_values[id] ||= {}
-          @other_languages_values[id][attr] ||= {}
-          @other_languages_values[id][attr][index] ||= []
-          
-          unless @other_languages_values[id][attr][index].include?(value.to_s)
-            @other_languages_values[id][attr][index] += Array(value.to_s)
-          end
-        end
-
-        def matched_languages(index_values, model_attribute_val)
-          not_matched_lang = index_values[:not_matched]
-          matched_lang = index_values.reject { |key| key == :not_matched }
-          unless model_attribute_val.nil? || Array(model_attribute_val).empty?
-            matched_lang[:no_lang] = Array(model_attribute_val)
-          end
-          [matched_lang, not_matched_lang]
-        end
-      end
-    end
-  end
-end
+module Goo
+  module SPARQL
+    module Solution
+      class  LanguageFilter
+
+        attr_reader :requested_lang, :unmapped, :objects_by_lang
+
+        def initialize(requested_lang: RequestStore.store[:requested_lang], unmapped: false, list_attributes: [])
+          @list_attributes = list_attributes
+          @objects_by_lang = {}
+          @unmapped = unmapped
+          @requested_lang = get_language(requested_lang)
+        end
+
+        def fill_models_with_all_languages(models_by_id)
+          objects_by_lang.each do |id, predicates|
+            model = models_by_id[id]
+            predicates.each do |predicate, values|
+
+              if values.values.all? { |v| v.all? { |x| literal?(x) && x.plain?} }
+                pull_stored_values(model, values, predicate, @unmapped)
+              end
+            end
+          end
+        end
+
+
+        def set_model_value(model, predicate, values)
+          set_value(model, predicate, values) do
+            model.send("#{predicate}=", values, on_load: true)
+          end
+        end
+
+        def set_unmapped_value(model, predicate, value)
+          set_value(model, predicate, value) do
+            return add_unmapped_to_model(model, predicate, value)
+          end
+        end
+
+        def models_unmapped_to_array(m)
+          if show_all_languages?
+            model_group_by_lang(m)
+          else
+            m.unmmaped_to_array
+          end
+        end
+
+        private
+
+
+        def set_value(model, predicate, value, &block)
+          language = object_language(value)
+
+          if requested_lang.eql?(:ALL) || !literal?(value) || language_match?(language)
+            block.call
+          end
+
+          if requested_lang.eql?(:ALL) || requested_lang.is_a?(Array)
+            language = "@none" if language.nil? || language.eql?(:no_lang)
+            store_objects_by_lang(model.id, predicate, value, language)
+          end
+        end
+        
+        def model_group_by_lang(model)
+          unmapped = model.unmapped
+          cpy = {}
+
+          unmapped.each do |attr, v|
+            cpy[attr] = group_by_lang(v)
+          end
+
+          model.unmapped = cpy
+        end
+
+        def group_by_lang(values)
+
+          return values.to_a if values.all?{|x| x.is_a?(RDF::URI) || !x.respond_to?(:language) }
+
+          values = values.group_by { |x| x.respond_to?(:language) && x.language ? x.language.to_s.downcase : :none }
+
+          no_lang = values[:none] || []
+          return no_lang if !no_lang.empty? && no_lang.all? { |x| x.respond_to?(:plain?) && !x.plain? }
+
+          values
+        end
+
+
+        def object_language(new_value)
+          new_value.language || :no_lang if new_value.is_a?(RDF::Literal)
+        end
+
+        def language_match?(language)
+          # no_lang means that the object is not a literal
+          return true if language.eql?(:no_lang)
+
+          return requested_lang.include?(language)  if requested_lang.is_a?(Array)
+
+          language.eql?(requested_lang)
+        end
+
+        def literal?(object)
+          !object_language(object).nil?
+        end
+
+        def store_objects_by_lang(id, predicate, object, language)
+          # store objects in this format: [id][predicate][language] = [objects]
+          return if requested_lang.is_a?(Array) && !requested_lang.include?(language)
+
+          language_key = language.downcase
+
+          objects_by_lang[id] ||= {}
+          objects_by_lang[id][predicate] ||= {}
+          objects_by_lang[id][predicate][language_key] ||= []
+
+          objects_by_lang[id][predicate][language_key] << object
+        end
+
+
+        def add_unmapped_to_model(model, predicate, value)
+
+          if model.respond_to? :klass # struct
+            model[:unmapped] ||= {}
+            model[:unmapped][predicate] ||= []
+            model[:unmapped][predicate]  << value unless value.nil?
+          else
+            model.unmapped_set(predicate, value)
+          end
+        end
+
+        def pull_stored_values(model, values, predicate, unmapped)
+          if unmapped
+            add_unmapped_to_model(model, predicate, values)
+          else
+            values = values.map do  |language, values_literals|
+              values_string = values_literals.map{|x| x.object}
+              values_string = values_string.first unless list_attributes?(predicate)
+              [language, values_string]
+            end.to_h
+
+            model.send("#{predicate}=", values, on_load: true)
+          end
+
+        end
+
+        def unmapped_get(model, predicate)
+          if model && model.respond_to?(:klass) # struct
+            model[:unmapped]&.dig(predicate)
+          else
+            model.unmapped_get(predicate)
+          end
+
+        end
+
+        def list_attributes?(predicate)
+          @list_attributes.include?(predicate)
+        end
+
+
+        def show_all_languages?
+          @requested_lang.is_a?(Array) || @requested_lang.eql?(:ALL)
+        end
+
+        def get_language(languages)
+          languages = portal_language if languages.nil? || languages.empty?
+          lang = languages.to_s.split(',').map { |l| l.upcase.to_sym }
+          lang.length == 1 ? lang.first : lang
+        end
+
+        def portal_language
+          Goo.main_languages.first
+        end
+
+      end
+    end
+  end
+end
diff --git a/lib/goo/sparql/solutions_mapper.rb b/lib/goo/sparql/solutions_mapper.rb
index 77b20ae0..879c1ff7 100644
--- a/lib/goo/sparql/solutions_mapper.rb
+++ b/lib/goo/sparql/solutions_mapper.rb
@@ -1,461 +1,439 @@
-module Goo
-  module SPARQL
-    class SolutionMapper
-
-      BNODES_TUPLES = Struct.new(:id, :attribute)
-
-      def initialize(aggregate_projections, bnode_extraction, embed_struct,
-                     incl_embed, klass_struct, models_by_id,
-                     properties_to_include, unmapped, variables, ids, options)
-
-        @aggregate_projections = aggregate_projections
-        @bnode_extraction = bnode_extraction
-        @embed_struct = embed_struct
-        @incl_embed = incl_embed
-        @klass_struct = klass_struct
-        @models_by_id = models_by_id
-        @properties_to_include = properties_to_include
-        @unmapped = unmapped
-        @variables = variables
-        @ids = ids
-        @klass = options[:klass]
-        @klass = options[:klass]
-        @read_only = options[:read_only]
-        @incl = options[:include]
-        @count = options[:count]
-        @collection = options[:collection]
-      end
-
-
-
-      def map_each_solutions(select)
-
-        found = Set.new
-        objects_new = {}
-        list_attributes = Set.new(@klass.attributes(:list))
-        all_attributes = Set.new(@klass.attributes(:all))
-        @lang_filter = Goo::SPARQL::Solution::LanguageFilter.new
-
-        select.each_solution do |sol|
-          next if sol[:some_type] && @klass.type_uri(@collection) != sol[:some_type]
-          return sol[:count_var].object if @count
-
-          found.add(sol[:id])
-          id = sol[:id]
-
-          create_model(id)
-
-          if @bnode_extraction
-            add_bnode_to_model(sol)
-            next
-          end
-
-          if @unmapped
-            add_unmapped_to_model(sol)
-            next
-          end
-
-          if @aggregate_projections
-            add_aggregations_to_model(sol)
-            next
-          end
-
-          predicate = sol[:attributeProperty].to_s.to_sym
-
-          next if predicate.nil? || !all_attributes.include?(predicate)
-
-          object = sol[:attributeObject]
-
-          #bnodes
-          if bnode_id?(object, predicate)
-            objects_new = bnode_id_tuple(id, object, objects_new, predicate)
-            next
-          end
-
-          # if multiple language values are included for a given property, set the
-          # corresponding model attribute to the English language value - NCBO-1662
-          language, object = get_object_language(id, object, predicate)
-          object, objects_new = get_value_object(id, objects_new, object, list_attributes, predicate)
-          add_object_to_model(id, object, predicate, language)
-        end
-        @lang_filter.fill_models_with_other_languages(@models_by_id, list_attributes)
-        init_unloaded_attributes(found, list_attributes)
-
-        return @models_by_id if @bnode_extraction
-
-        model_set_collection_attributes(@models_by_id, objects_new)
-
-        #remove from models_by_id elements that were not touched
-        @models_by_id.select! { |k, m| found.include?(k) }
-
-        models_set_all_persistent(@models_by_id) unless @read_only
-
-        #next level of embed attributes
-        include_embed_attributes(@incl_embed, objects_new) if @incl_embed && !@incl_embed.empty?
-
-        #bnodes
-        blank_nodes = objects_new.select { |id, obj| id.is_a?(RDF::Node) && id.anonymous? }
-        include_bnodes(blank_nodes, @models_by_id) unless blank_nodes.empty?
-
-        models_unmapped_to_array(@models_by_id) if @unmapped
-
-        @models_by_id
-      end
-
-      private
-
-      def get_object_language(id, object, predicate)
-        @lang_filter.main_lang_filter id, predicate, object
-      end
-
-      def init_unloaded_attributes(found, list_attributes)
-        return if @incl.nil?
-
-        # Here we are setting to nil all attributes that have been included but not found in the triplestore
-        found.uniq.each do |model_id|
-          m = @models_by_id[model_id]
-          @incl.each do |attr_to_incl|
-            is_handler = m.respond_to?(:handler?) && m.class.handler?(attr_to_incl)
-            next if attr_to_incl.to_s.eql?('unmapped') || is_handler
-
-            loaded = m.respond_to?('loaded_attributes') && m.loaded_attributes.include?(attr_to_incl)
-            is_list = list_attributes.include?(attr_to_incl)
-            is_struct = m.respond_to?(:klass)
-
-            # Go through all models queried
-            if is_struct
-              m[attr_to_incl] = [] if is_list && m[attr_to_incl].nil?
-            elsif is_list && (!loaded || m.send(attr_to_incl.to_s).nil?)
-              m.send("#{attr_to_incl}=", [], on_load: true)
-            elsif !loaded && !is_list && m.respond_to?("#{attr_to_incl}=")
-              m.send("#{attr_to_incl}=", nil, on_load: true)
-            end
-          end
-        end
-      end
-
-      def get_value_object(id, objects_new, object, list_attributes, predicate)
-        object = object.object if object && !(object.is_a? RDF::URI)
-        range_for_v = @klass.range(predicate)
-        #binding.pry if v.eql?(:enrolled)
-        #dependent model creation
-
-        if object.is_a?(RDF::URI) && (predicate != :id) && !range_for_v.nil?
-          if objects_new.include?(object)
-            object = objects_new[object]
-          elsif !range_for_v.inmutable?
-            pre_val = get_preload_value(id, object, predicate)
-            object, objects_new = if !@read_only
-                                    preloaded_or_new_object(object, objects_new, pre_val, predicate)
-                                  else
-                                    #depedent read only
-                                    preloaded_or_new_struct(object, objects_new, pre_val, predicate)
-                                  end
-          else
-            object = range_for_v.find(object).first
-          end
-        end
-
-        if list_attributes.include?(predicate)
-          # To handle attr that are lists
-          pre = if @klass_struct
-                  @models_by_id[id][predicate]
-                else
-                  @models_by_id[id].instance_variable_get("@#{predicate}")
-                end
-          if object.nil? && pre.nil?
-            object = []
-          elsif object.nil? && !pre.nil?
-            object = pre
-          elsif object
-            object = !pre ? [object] : (pre.dup << object)
-            object.uniq!
-          end
-        end
-        [object, objects_new]
-      end
-
-      def add_object_to_model(id, object, predicate, lang)
-        if @models_by_id[id].respond_to?(:klass)
-          @models_by_id[id][predicate] = object unless object.nil? && !@models_by_id[id][predicate].nil?
-        elsif !@models_by_id[id].class.handler?(predicate) &&
-              !(object.nil? && !@models_by_id[id].instance_variable_get("@#{predicate}").nil?) &&
-              predicate != :id
-
-          if (lang&.eql?(:no_lang)) || !lang
-            @models_by_id[id].send("#{predicate}=", object, on_load: true)
-          end
-
-        end
-      end
-
-      def get_preload_value(id, object, predicate)
-        pre_val = nil
-        if predicate_preloaded?(id, predicate)
-          pre_val = preloaded_value(id, predicate)
-          pre_val = pre_val.select { |x| x.id == object }.first if pre_val.is_a?(Array)
-        end
-        pre_val
-      end
-
-      def preloaded_or_new_object(object, objects_new, pre_val, predicate)
-        object = pre_val || @klass.range_object(predicate, object)
-        objects_new[object.id] = object
-        [object, objects_new]
-      end
-
-      def preloaded_or_new_struct(object, objects_new, pre_val, predicate)
-        struct = pre_val || @embed_struct[predicate].new
-        struct.id = object
-        struct.klass = @klass.range(predicate)
-        objects_new[struct.id] = struct
-        [struct, objects_new]
-      end
-
-      def preloaded_value(id, predicate)
-        if !@read_only
-          @models_by_id[id].instance_variable_get("@#{predicate}")
-        else
-          @models_by_id[id][predicate]
-        end
-      end
-
-      def predicate_preloaded?(id, predicate)
-        @models_by_id[id] &&
-          (@models_by_id[id].respond_to?(:klass) || @models_by_id[id].loaded_attributes.include?(predicate))
-      end
-
-      def bnode_id?(object, predicate)
-        object.is_a?(RDF::Node) && object.anonymous? && @incl.include?(predicate)
-      end
-
-      def bnode_id_tuple(id, object, objects_new, predicate)
-        range = @klass.range(predicate)
-        if range.respond_to?(:new)
-          objects_new[object] = BNODES_TUPLES.new(id, predicate)
-        end
-        objects_new
-      end
-
-      def add_bnode_to_model(sol)
-        id = sol[:id]
-        struct = create_struct(@bnode_extraction, @models_by_id, sol, @variables)
-        @models_by_id[id].send("#{@bnode_extraction}=", struct)
-      end
-
-      def create_model(id)
-        @models_by_id[id] = create_class_model(id, @klass, @klass_struct) unless @models_by_id.include?(id)
-      end
-
-      def model_set_unmapped(id, predicate, value)
-
-        if @models_by_id[id].respond_to? :klass #struct
-          @models_by_id[id][:unmapped] ||= {}
-          (@models_by_id[id][:unmapped][predicate] ||= []) << value
-        else
-          @models_by_id[id].unmapped_set(predicate, value)
-        end
-      end
-
-      def create_struct(bnode_extraction, models_by_id, sol, variables)
-        list_attributes = Set.new(@klass.attributes(:list))
-        struct = @klass.range(bnode_extraction).new
-        variables.each do |v|
-          next if v == :id
-          svalue = sol[v]
-          struct[v] = svalue.is_a?(RDF::Node) ? svalue : svalue.object
-        end
-        if list_attributes.include?(bnode_extraction)
-          pre = models_by_id[sol[:id]].instance_variable_get("@#{bnode_extraction}")
-          pre = pre ? (pre.dup << struct) : [struct]
-          struct = pre
-        end
-        struct
-      end
-
-      def create_class_model(id, klass, klass_struct)
-        klass_model = klass_struct ? klass_struct.new : klass.new
-        klass_model.id = id
-        klass_model.persistent = true unless klass_struct
-        klass_model.klass = klass if klass_struct
-        klass_model
-      end
-
-      def models_unmapped_to_array(models_by_id)
-        models_by_id.each do |idm, m|
-          m.unmmaped_to_array
-        end
-      end
-
-      def include_bnodes(bnodes, models_by_id)
-        #group by attribute
-        attrs = bnodes.map { |x, y| y.attribute }.uniq
-        attrs.each do |attr|
-          struct = @klass.range(attr)
-
-          #bnodes that are in a range of goo ground models
-          #for example parents and children in LD class models
-          #we skip this cases for the moment
-          next if struct.respond_to?(:model_name)
-
-          bnode_attrs = struct.new.to_h.keys
-          ids = bnodes.select { |x, y| y.attribute == attr }.map { |x, y| y.id }
-          @klass.where.models(models_by_id.select { |x, y| ids.include?(x) }.values)
-                .in(@collection)
-                .include(bnode: { attr => bnode_attrs }).all
-        end
-      end
-
-      def include_embed_attributes(incl_embed, objects_new)
-        incl_embed.each do |attr, next_attrs|
-          #anything to join ?
-          attr_range = @klass.range(attr)
-          next if attr_range.nil?
-          range_objs = objects_new.select { |id, obj|
-            obj.instance_of?(attr_range) || (obj.respond_to?(:klass) && obj[:klass] == attr_range)
-          }.values
-          unless range_objs.empty?
-            range_objs.uniq!
-            query = attr_range.where().models(range_objs).in(@collection).include(*next_attrs)
-            query = query.read_only if @read_only
-            query.all
-          end
-        end
-      end
-
-      def models_set_all_persistent(models_by_id)
-        return unless @ids
-        models_by_id.each do |k, m|
-          m.persistent = true
-        end
-      end
-
-      def model_set_collection_attributes(models_by_id, objects_new)
-        collection_value = get_collection_value
-        if collection_value
-          collection_attribute = @klass.collection_opts
-          models_by_id.each do |id, m|
-            m.send("#{collection_attribute}=", collection_value)
-          end
-          objects_new.each do |id, obj_new|
-            if obj_new.respond_to?(:klass)
-              collection_attribute = obj_new[:klass].collection_opts
-              obj_new[collection_attribute] = collection_value
-            elsif obj_new.class.respond_to?(:collection_opts) &&
-              obj_new.class.collection_opts.instance_of?(Symbol)
-              collection_attribute = obj_new.class.collection_opts
-              obj_new.send("#{collection_attribute}=", collection_value)
-            end
-          end
-        end
-      end
-
-      def get_collection_value
-        collection_value = nil
-        if @klass.collection_opts.instance_of?(Symbol)
-          if @collection.is_a?(Array) && (@collection.length == 1)
-            collection_value = @collection.first
-          end
-          if @collection.respond_to? :id
-            collection_value = @collection
-          end
-        end
-        collection_value
-      end
-
-
-      def object_to_array(id, klass_struct, models_by_id, object, predicate)
-        pre = if klass_struct
-                models_by_id[id][predicate]
-              else
-                models_by_id[id].instance_variable_get("@#{predicate}")
-              end
-        if object.nil? && pre.nil?
-          object = []
-        elsif object.nil? && !pre.nil?
-          object = pre
-        elsif object
-          object = !pre ? [object] : (pre.dup << object)
-          object.uniq!
-        end
-        object
-      end
-
-      def dependent_model_creation(embed_struct, id, models_by_id, object, objects_new, v, options)
-
-        read_only = options[:read_only]
-        if object.is_a?(RDF::URI) && v != :id
-          range_for_v = @klass.range(v)
-          if range_for_v
-            if objects_new.include?(object)
-              object = objects_new[object]
-            elsif !range_for_v.inmutable?
-              pre_val = get_pre_val(id, models_by_id, object, v, read_only)
-              object = get_object_from_range(pre_val, embed_struct, object, objects_new, v, options)
-            else
-              object = range_for_v.find(object).first
-            end
-          end
-        end
-        object
-      end
-
-      def get_object_from_range(pre_val, embed_struct, object, objects_new, predicate)
-
-        range_for_v = @klass.range(predicate)
-        if !@read_only
-          object = pre_val || @klass.range_object(predicate, object)
-          objects_new[object.id] = object
-        else
-          #depedent read only
-          struct = pre_val || embed_struct[predicate].new
-          struct.id = object
-          struct.klass = range_for_v
-          objects_new[struct.id] = struct
-          object = struct
-        end
-        object
-      end
-
-      def get_pre_val(id, models_by_id, object, predicate)
-        pre_val = nil
-        if models_by_id[id] &&
-          ((models_by_id[id].respond_to?(:klass) && models_by_id[id]) ||
-            models_by_id[id].loaded_attributes.include?(predicate))
-          pre_val = if !@read_only
-                      models_by_id[id].instance_variable_get("@#{predicate}")
-                    else
-                      models_by_id[id][predicate]
-                    end
-
-          pre_val = pre_val.select { |x| x.id == object }.first if pre_val.is_a?(Array)
-        end
-        pre_val
-      end
-
-      def add_unmapped_to_model(sol)
-        predicate = sol[:attributeProperty].to_s.to_sym
-        return unless @properties_to_include[predicate]
-
-        id = sol[:id]
-        value = sol[:attributeObject]
-
-        model_set_unmapped(id, @properties_to_include[predicate][:uri], value)
-      end
-
-      def add_aggregations_to_model(sol)
-        id = sol[:id]
-        @aggregate_projections&.each do |aggregate_key, aggregate_val|
-          if @models_by_id[id].respond_to?(:add_aggregate)
-            @models_by_id[id].add_aggregate(aggregate_val[1], aggregate_val[0], sol[aggregate_key].object)
-          else
-            (@models_by_id[id].aggregates ||= []) << Goo::Base::AGGREGATE_VALUE.new(aggregate_val[1],
-                                                                                    aggregate_val[0],
-                                                                                    sol[aggregate_key].object)
-          end
-        end
-      end
-    end
-  end
-end
-
+module Goo
+  module SPARQL
+    class SolutionMapper
+      BNODES_TUPLES = Struct.new(:id, :attribute)
+
+      def initialize(aggregate_projections, bnode_extraction, embed_struct,
+                      incl_embed, klass_struct, models_by_id,
+                     properties_to_include, unmapped, variables, ids, options)
+
+        @aggregate_projections = aggregate_projections
+        @bnode_extraction = bnode_extraction
+        @embed_struct = embed_struct
+        @incl_embed = incl_embed
+        @klass_struct = klass_struct
+        @models_by_id = models_by_id
+        @properties_to_include = properties_to_include
+        @unmapped = unmapped
+        @variables = variables
+        @ids = ids
+        @klass = options[:klass]
+        @read_only = options[:read_only]
+        @incl = options[:include]
+        @count = options[:count]
+        @collection = options[:collection]
+        @options = options
+      end
+      
+      def map_each_solutions(select)
+        found = Set.new
+        objects_new = {}
+        list_attributes = Set.new(@klass.attributes(:list))
+        all_attributes = Set.new(@klass.attributes(:all))
+
+        @lang_filter = Goo::SPARQL::Solution::LanguageFilter.new(requested_lang: @options[:requested_lang].to_s, unmapped: @unmapped,
+           list_attributes: list_attributes)
+        
+        select.each_solution do |sol|
+          
+          next if sol[:some_type] && @klass.type_uri(@collection) != sol[:some_type]
+          return sol[:count_var].object if @count
+
+          found.add(sol[:id])
+          id = sol[:id]
+
+          create_model(id)
+
+          if @bnode_extraction
+            add_bnode_to_model(sol)
+            next
+          end
+
+          if @unmapped
+            add_unmapped_to_model(sol)
+            next
+          end
+
+          if @aggregate_projections
+            add_aggregations_to_model(sol)
+            next
+          end
+
+          predicate = sol[:attributeProperty].to_s.to_sym
+
+          next if predicate.nil? || !all_attributes.include?(predicate)
+
+          object = sol[:attributeObject]
+
+          # bnodes
+          if bnode_id?(object, predicate)
+            objects_new = bnode_id_tuple(id, object, objects_new, predicate)
+            next
+          end
+
+          objects, objects_new = get_value_object(id, objects_new, object, list_attributes, predicate)
+          add_object_to_model(id, objects, predicate)
+        end
+      
+        # for this moment we are not going to enrich models , maybe we will use it if the results are empty  
+        @lang_filter.fill_models_with_all_languages(@models_by_id)
+
+        init_unloaded_attributes(found, list_attributes)
+
+        return @models_by_id if @bnode_extraction
+
+        model_set_collection_attributes(@models_by_id, objects_new)
+
+        # remove from models_by_id elements that were not touched
+        @models_by_id.select! { |k, _m| found.include?(k) }
+
+        models_set_all_persistent(@models_by_id) unless @read_only
+
+        # next level of embed attributes
+        include_embed_attributes(@incl_embed, objects_new) if @incl_embed && !@incl_embed.empty?
+
+        # bnodes
+        blank_nodes = objects_new.select { |id, _obj| id.is_a?(RDF::Node) && id.anonymous? }
+        include_bnodes(blank_nodes, @models_by_id) unless blank_nodes.empty?
+
+        models_unmapped_to_array(@models_by_id) if @unmapped
+        
+       
+        @models_by_id
+      end
+
+      private
+
+      def init_unloaded_attributes(found, list_attributes)
+        return if @incl.nil?
+
+        # Here we are setting to nil all attributes that have been included but not found in the triplestore
+        found.uniq.each do |model_id|
+          m = @models_by_id[model_id]
+          @incl.each do |attr_to_incl|
+            is_handler = m.respond_to?(:handler?) && m.class.handler?(attr_to_incl)
+            next if attr_to_incl.to_s.eql?('unmapped') || is_handler
+
+            loaded = m.respond_to?('loaded_attributes') && m.loaded_attributes.include?(attr_to_incl)
+            is_list = list_attributes.include?(attr_to_incl)
+            is_struct = m.respond_to?(:klass)
+
+            # Go through all models queried
+            if is_struct
+              m[attr_to_incl] = [] if is_list && m[attr_to_incl].nil?
+            elsif is_list && (!loaded || m.send(attr_to_incl.to_s).nil?)
+              m.send("#{attr_to_incl}=", [], on_load: true)
+            elsif !loaded && !is_list && m.respond_to?("#{attr_to_incl}=")
+              m.send("#{attr_to_incl}=", nil, on_load: true)
+            end
+          end
+        end
+      end
+
+      def get_value_object(id, objects_new, object, list_attributes, predicate)
+        object = object.object if object && !(object.is_a? RDF::URI)
+        range_for_v = @klass.range(predicate)
+       
+
+        if object.is_a?(RDF::URI) && (predicate != :id) && !range_for_v.nil?
+          if objects_new.include?(object)
+            object = objects_new[object]
+          elsif !range_for_v.inmutable?
+            pre_val = get_preload_value(id, object, predicate)
+            object, objects_new = if !@read_only
+                                    preloaded_or_new_object(object, objects_new, pre_val, predicate)
+                                  else
+                                    # depedent read only
+                                    preloaded_or_new_struct(object, objects_new, pre_val, predicate)
+                                  end
+          else
+            object = range_for_v.find(object).first
+          end
+        end
+
+        if list_attributes.include?(predicate)
+          pre = @klass_struct ? @models_by_id[id][predicate] : @models_by_id[id].instance_variable_get("@#{predicate}")
+
+          if object.nil?
+            object = pre.nil? ? [] : pre
+          else
+            object = pre.nil? ? [object] : (pre.dup << object)
+            object.uniq!
+          end
+
+        end
+        [object, objects_new]
+      end
+
+      def add_object_to_model(id, objects, predicate)
+
+        if @models_by_id[id].respond_to?(:klass)
+          @models_by_id[id][predicate] = objects unless objects.nil? && !@models_by_id[id][predicate].nil?
+        elsif !@models_by_id[id].class.handler?(predicate) &&
+              !(objects.nil? && !@models_by_id[id].instance_variable_get("@#{predicate}").nil?) &&
+              predicate != :id
+          @lang_filter.set_model_value(@models_by_id[id], predicate, objects)
+        end
+      end
+
+      def get_preload_value(id, object, predicate)
+        pre_val = nil
+        if predicate_preloaded?(id, predicate)
+          pre_val = preloaded_value(id, predicate)          
+          pre_val = pre_val.select { |x|  x.id  == object  }.first if pre_val.is_a?(Array)
+        end
+        pre_val
+      end
+
+      def preloaded_or_new_object(object, objects_new, pre_val, predicate)
+        object = pre_val || @klass.range_object(predicate, object)
+        objects_new[object.id] = object
+        [object, objects_new]
+      end
+
+      def preloaded_or_new_struct(object, objects_new, pre_val, predicate)
+        struct = pre_val || @embed_struct[predicate].new
+        struct.id = object
+        struct.klass = @klass.range(predicate)
+        objects_new[struct.id] = struct
+        [struct, objects_new]
+      end
+
+      def preloaded_value(id, predicate)
+        if !@read_only
+          @models_by_id[id].instance_variable_get("@#{predicate}")
+
+        else
+          @models_by_id[id][predicate]
+        end
+      end
+
+      def predicate_preloaded?(id, predicate)
+        @models_by_id[id] &&
+          (@models_by_id[id].respond_to?(:klass) || @models_by_id[id].loaded_attributes.include?(predicate))
+      end
+
+      def bnode_id?(object, predicate)
+        object.is_a?(RDF::Node) && object.anonymous? && @incl.include?(predicate)
+      end
+
+      def bnode_id_tuple(id, object, objects_new, predicate)
+        range = @klass.range(predicate)
+        objects_new[object] = BNODES_TUPLES.new(id, predicate) if range.respond_to?(:new)
+        objects_new
+      end
+
+      def add_bnode_to_model(sol)
+        id = sol[:id]
+        struct = create_struct(@bnode_extraction, @models_by_id, sol, @variables)
+        @models_by_id[id].send("#{@bnode_extraction}=", struct)
+      end
+
+      def create_model(id)
+        @models_by_id[id] = create_class_model(id, @klass, @klass_struct) unless @models_by_id.include?(id)
+      end
+
+
+      def create_struct(bnode_extraction, models_by_id, sol, variables)
+        list_attributes = Set.new(@klass.attributes(:list))
+        struct = @klass.range(bnode_extraction).new
+        variables.each do |v|
+          next if v == :id
+
+          svalue = sol[v]
+          struct[v] = svalue.is_a?(RDF::Node) ? svalue : svalue.object
+        end
+        if list_attributes.include?(bnode_extraction)
+          pre = models_by_id[sol[:id]].instance_variable_get("@#{bnode_extraction}")
+          pre = pre ? (pre.dup << struct) : [struct]
+          struct = pre
+        end
+        struct
+      end
+
+      def create_class_model(id, klass, klass_struct)
+        klass_model = klass_struct ? klass_struct.new : klass.new
+        klass_model.id = id
+        klass_model.persistent = true unless klass_struct
+        klass_model.klass = klass if klass_struct
+        klass_model
+      end
+
+      def models_unmapped_to_array(models_by_id)
+        models_by_id.each do |_idm, m|
+          @lang_filter.models_unmapped_to_array(m)
+        end
+      end
+
+
+      def is_multiple_langs?
+        return true if @requested_lang.is_a?(Array) || @requested_lang.eql?(:ALL)
+        false
+      end
+
+      def include_bnodes(bnodes, models_by_id)
+        # group by attribute
+        attrs = bnodes.map { |_x, y| y.attribute }.uniq
+        attrs.each do |attr|
+          struct = @klass.range(attr)
+
+          # bnodes that are in a range of goo ground models
+          # for example parents and children in LD class models
+          # we skip this cases for the moment
+          next if struct.respond_to?(:model_name)
+
+          bnode_attrs = struct.new.to_h.keys
+          ids = bnodes.select { |_x, y| y.attribute == attr }.map { |_x, y| y.id }
+          @klass.where.models(models_by_id.select { |x, _y| ids.include?(x) }.values)
+                .in(@collection)
+                .include(bnode: { attr => bnode_attrs }).all
+        end
+      end
+
+      def include_embed_attributes(incl_embed, objects_new)
+        incl_embed.each do |attr, next_attrs|
+          # anything to join ?
+          attr_range = @klass.range(attr)
+          next if attr_range.nil?
+
+          range_objs = objects_new.select do |_id, obj|
+            obj.instance_of?(attr_range) || (obj.respond_to?(:klass) && obj[:klass] == attr_range)
+          end.values
+          next if range_objs.empty?
+
+          range_objs.uniq!
+          query = attr_range.where.models(range_objs).in(@collection).include(*next_attrs)
+          query = query.read_only if @read_only
+          query.all
+        end
+      end
+
+      def models_set_all_persistent(models_by_id)
+        return unless @ids
+
+        models_by_id.each do |_k, m|
+          m.persistent = true
+        end
+      end
+
+      def model_set_collection_attributes(models_by_id, objects_new)
+        collection_value = get_collection_value
+        return unless collection_value
+
+        collection_attribute = @klass.collection_opts
+        models_by_id.each do |_id, m|
+          m.send("#{collection_attribute}=", collection_value)
+        end
+        objects_new.each do |_id, obj_new|
+          if obj_new.respond_to?(:klass)
+            collection_attribute = obj_new[:klass].collection_opts
+            obj_new[collection_attribute] = collection_value
+          elsif obj_new.class.respond_to?(:collection_opts) &&
+                obj_new.class.collection_opts.instance_of?(Symbol)
+            collection_attribute = obj_new.class.collection_opts
+            obj_new.send("#{collection_attribute}=", collection_value)
+          end
+        end
+      end
+
+      def get_collection_value
+        collection_value = nil
+        if @klass.collection_opts.instance_of?(Symbol)
+          collection_value = @collection.first if @collection.is_a?(Array) && (@collection.length == 1)
+          collection_value = @collection if @collection.respond_to? :id
+        end
+        collection_value
+      end
+
+      def object_to_array(id, klass_struct, models_by_id, object, predicate)
+        pre = if klass_struct
+                models_by_id[id][predicate]
+              else
+                models_by_id[id].instance_variable_get("@#{predicate}")
+              end
+        if object.nil? && pre.nil?
+          object = []
+        elsif object.nil? && !pre.nil?
+          object = pre
+        elsif object
+          object = !pre ? [object] : (pre.dup << object)
+          object.uniq!
+        end
+        object
+      end
+
+      def dependent_model_creation(embed_struct, id, models_by_id, object, objects_new, v, options)
+        read_only = options[:read_only]
+        if object.is_a?(RDF::URI) && v != :id
+          range_for_v = @klass.range(v)
+          if range_for_v
+            if objects_new.include?(object)
+              object = objects_new[object]
+            elsif !range_for_v.inmutable?
+              pre_val = get_pre_val(id, models_by_id, object, v, read_only)
+              object = get_object_from_range(pre_val, embed_struct, object, objects_new, v, options)
+            else
+              object = range_for_v.find(object).first
+            end
+          end
+        end
+        object
+      end
+
+      def get_object_from_range(pre_val, embed_struct, object, objects_new, predicate)
+        range_for_v = @klass.range(predicate)
+        if !@read_only
+          object = pre_val || @klass.range_object(predicate, object)
+          objects_new[object.id] = object
+        else
+          # depedent read only
+          struct = pre_val || embed_struct[predicate].new
+          struct.id = object
+          struct.klass = range_for_v
+          objects_new[struct.id] = struct
+          object = struct
+        end
+        object
+      end
+
+      def get_pre_val(id, models_by_id, object, predicate)
+        pre_val = nil
+        if models_by_id[id] &&
+           ((models_by_id[id].respond_to?(:klass) && models_by_id[id]) ||
+             models_by_id[id].loaded_attributes.include?(predicate))
+          pre_val = if !@read_only
+                      models_by_id[id].instance_variable_get("@#{predicate}")
+                    else
+                      models_by_id[id][predicate]
+                    end
+
+          pre_val = pre_val.select { |x| x.id == object }.first if pre_val.is_a?(Array)
+        end
+        pre_val
+      end
+
+      def add_unmapped_to_model(sol)
+        predicate = sol[:attributeProperty].to_s.to_sym
+        return unless @properties_to_include[predicate]
+
+        id = sol[:id]
+        value = sol[:attributeObject]
+
+        @lang_filter.set_unmapped_value(@models_by_id[id], @properties_to_include[predicate][:uri], value)
+      end
+
+      def add_aggregations_to_model(sol)
+        id = sol[:id]
+        @aggregate_projections&.each do |aggregate_key, aggregate_val|
+          if @models_by_id[id].respond_to?(:add_aggregate)
+            @models_by_id[id].add_aggregate(aggregate_val[1], aggregate_val[0], sol[aggregate_key].object)
+          else
+            (@models_by_id[id].aggregates ||= []) << Goo::Base::AGGREGATE_VALUE.new(aggregate_val[1],
+                                                                                    aggregate_val[0],
+                                                                                    sol[aggregate_key].object)
+          end
+        end
+      end
+    end
+  end
+end

From d4d9417c29e5dea346b482f69d710bba76ff39c2 Mon Sep 17 00:00:00 2001
From: Syphax bouazzouni <gs_bouazzouni@esi.dz>
Date: Mon, 4 Dec 2023 11:40:32 +0100
Subject: [PATCH 147/168] Merge to master: Release 2.3.4 - New model
 capabilities (validators, scheme file  and callbacks)  (#45)

* Merge pull request #23 from ontoportal-lirmm/feature/add-property-datatype-dsl

Feature: add property data type dsl

* Merge pull request #26 from ontoportal-lirmm/refactor/resources-validators

Refactor: resources validators

* Merge pull request #27 from ontoportal-lirmm/feature/implement-new-validators

Feature: Implement new validators

* Merge pull request #28 from ontoportal-lirmm/feature/implement-new-validators

Feature: Implement instance proc validators

* Merge pull request #30 from ontoportal-lirmm/feature/add-model-scheme-yml-file

Feature/add model scheme yml file

* Merge pull request #31 from ontoportal-lirmm/feature/add-models-on-update-callbacks

Feature: Add models on update callbacks
---
 lib/goo.rb                                    |   4 +
 lib/goo/base/resource.rb                      |  90 ++--
 lib/goo/base/settings/settings.rb             |  30 +-
 lib/goo/base/settings/yaml_settings.rb        |  45 ++
 lib/goo/sparql/triples.rb                     |  10 -
 lib/goo/validators/enforce.rb                 | 213 ++++-----
 .../validators/implementations/data_type.rb   |  66 +++
 .../validators/implementations/distinct_of.rb |  34 ++
 .../validators/implementations/existence.rb   |  17 +
 .../validators/implementations/inverse_of.rb  |  35 ++
 .../validators/implementations/object_type.rb |  46 ++
 .../implementations/superior_equal_to.rb      |  26 ++
 .../validators/implementations/symmetric.rb   |  33 ++
 lib/goo/validators/implementations/unique.rb  |  20 +
 .../validators/implementations/value_range.rb |  49 ++
 lib/goo/validators/validator.rb               | 111 +++++
 test/data/yaml_scheme_model_test.yml          |  11 +
 test/models.rb                                |   5 +-
 test/test_dsl_settings.rb                     |  86 +++-
 test/test_update_callbacks.rb                 |  53 +++
 test/test_validators.rb                       | 427 ++++++++++++++++++
 21 files changed, 1249 insertions(+), 162 deletions(-)
 create mode 100644 lib/goo/base/settings/yaml_settings.rb
 create mode 100644 lib/goo/validators/implementations/data_type.rb
 create mode 100644 lib/goo/validators/implementations/distinct_of.rb
 create mode 100644 lib/goo/validators/implementations/existence.rb
 create mode 100644 lib/goo/validators/implementations/inverse_of.rb
 create mode 100644 lib/goo/validators/implementations/object_type.rb
 create mode 100644 lib/goo/validators/implementations/superior_equal_to.rb
 create mode 100644 lib/goo/validators/implementations/symmetric.rb
 create mode 100644 lib/goo/validators/implementations/unique.rb
 create mode 100644 lib/goo/validators/implementations/value_range.rb
 create mode 100644 lib/goo/validators/validator.rb
 create mode 100644 test/data/yaml_scheme_model_test.yml
 create mode 100644 test/test_update_callbacks.rb
 create mode 100644 test/test_validators.rb

diff --git a/lib/goo.rb b/lib/goo.rb
index db863d2a..ff0e6279 100644
--- a/lib/goo.rb
+++ b/lib/goo.rb
@@ -16,6 +16,10 @@
 require_relative "goo/search/search"
 require_relative "goo/base/base"
 require_relative "goo/validators/enforce"
+require_relative "goo/validators/validator"
+project_root = File.dirname(File.absolute_path(__FILE__))
+Dir.glob("#{project_root}/goo/validators/implementations/*", &method(:require))
+
 require_relative "goo/utils/utils"
 require_relative "goo/mixins/sparql_client"
 
diff --git a/lib/goo/base/resource.rb b/lib/goo/base/resource.rb
index 88bbc8ce..26ac5859 100644
--- a/lib/goo/base/resource.rb
+++ b/lib/goo/base/resource.rb
@@ -74,23 +74,10 @@ def id=(new_id)
       end
 
       def id
-        if @id.nil?
-          raise IDGenerationError, ":id must be set if configured in name_with" if self.class.name_with == :id
-          custom_name = self.class.name_with
-          if custom_name.instance_of?(Symbol)
-            @id = id_from_attribute()
-          elsif custom_name
-            begin
-              @id = custom_name.call(self)
-            rescue => e
-              raise IDGenerationError, "Problem with custom id generation: #{e.message}"
-            end
-          else
-            raise IDGenerationError, "custom_name is nil. settings for this model are incorrect."
+        @id = generate_id if @id.nil?
+
+        @id
           end
-        end
-        return @id
-      end
 
       def persistent?
         return @persistent
@@ -104,22 +91,20 @@ def modified?
         return modified_attributes.length > 0
       end
 
-      def exist?(from_valid=false)
-        #generate id with proc
-        begin
-          id() unless self.class.name_with.kind_of?(Symbol)
-        rescue IDGenerationError
-        end
-
-        _id = @id
-        if _id.nil? && !from_valid && self.class.name_with.is_a?(Symbol)
+      def exist?(from_valid = false)
           begin
-            _id = id_from_attribute()
+          id unless self.class.name_with.kind_of?(Symbol)
           rescue IDGenerationError
+          # Ignored
           end
+
+        _id = @id
+        if from_valid || _id.nil?
+          _id = generate_id rescue _id = nil
         end
+
         return false unless _id
-        return Goo::SPARQL::Queries.model_exist(self,id=_id)
+        Goo::SPARQL::Queries.model_exist(self, id = _id)
       end
 
       def fully_loaded?
@@ -247,10 +232,13 @@ def save(*opts)
           raise ArgumentError, "Enums can only be created on initialization" unless opts[0] && opts[0][:init_enum]
         end
         batch_file = nil
-        if opts && opts.length > 0
-          if opts.first.is_a?(Hash) && opts.first[:batch] && opts.first[:batch].is_a?(File)
+        callbacks = true
+        if opts && opts.length > 0 && opts.first.is_a?(Hash)
+          if opts.first[:batch] && opts.first[:batch].is_a?(File)
             batch_file = opts.first[:batch]
           end
+
+          callbacks = opts.first[:callbacks]
         end
 
         if !batch_file
@@ -258,8 +246,28 @@ def save(*opts)
           raise Goo::Base::NotValidException, "Object is not valid. Check errors." unless valid?
         end
 
+        #set default values before saving
+        unless self.persistent?
+          self.class.attributes_with_defaults.each do |attr|
+            value = self.send("#{attr}")
+            if value.nil?
+              value = self.class.default(attr).call(self)
+              self.send("#{attr}=", value)
+            end
+          end
+        end
+
+        #call update callback before saving
+        if callbacks
+          self.class.attributes_with_update_callbacks.each do |attr|
+            Goo::Validators::Enforce.enforce_callbacks(self, attr)
+          end
+        end
+
         graph_insert, graph_delete = Goo::SPARQL::Triples.model_update_triples(self)
-        graph = self.graph()
+        graph = self.graph
+
+
         if graph_delete and graph_delete.size > 0
           begin
             Goo.sparql_update_client.delete_data(graph_delete, graph: graph)
@@ -442,10 +450,30 @@ def self.all
       end
 
       protected
+
       def id_from_attribute()
         uattr = self.class.name_with
         uvalue = self.send("#{uattr}")
-        return self.class.id_from_unique_attribute(uattr,uvalue)
+        return self.class.id_from_unique_attribute(uattr, uvalue)
+      end
+
+      def generate_id
+        return nil unless self.class.name_with
+
+        raise IDGenerationError, ":id must be set if configured in name_with" if self.class.name_with == :id
+        custom_name = self.class.name_with
+        if custom_name.instance_of?(Symbol)
+          id = id_from_attribute
+        elsif custom_name
+          begin
+            id = custom_name.call(self)
+          rescue => e
+            raise IDGenerationError, "Problem with custom id generation: #{e.message}"
+          end
+        else
+          raise IDGenerationError, "custom_name is nil. settings for this model are incorrect."
+        end
+        id
       end
 
     end
diff --git a/lib/goo/base/settings/settings.rb b/lib/goo/base/settings/settings.rb
index a58daae0..a7008087 100644
--- a/lib/goo/base/settings/settings.rb
+++ b/lib/goo/base/settings/settings.rb
@@ -1,4 +1,5 @@
 require 'active_support/core_ext/string'
+require_relative 'yaml_settings'
 
 module Goo
   module Base
@@ -12,8 +13,10 @@ module ClassMethods
         attr_reader :model_name
         attr_reader :attribute_uris
 
+        include YAMLScheme
+
         def default_model_options
-          return {}
+           {}
         end
 
         def model(*args)
@@ -34,7 +37,9 @@ def model(*args)
 
           @model_settings = default_model_options.merge(options || {})
 
-          unless options.include?:name_with
+          init_yaml_scheme_settings
+
+          unless options.include? :name_with
             raise ArgumentError, "The model `#{model_name}` definition should include the :name_with option"
           end
           Goo.add_model(@model_name,self)
@@ -91,6 +96,16 @@ def attributes_with_defaults
                   select{ |attr,opts| opts[:default] }).keys()
         end
 
+        def attributes_with_update_callbacks
+           (@model_settings[:attributes].
+            select{ |attr,opts| opts[:onUpdate] }).keys
+        end
+
+
+        def update_callbacks(attr)
+          @model_settings[:attributes][attr][:onUpdate]
+        end
+
         def default(attr)
           return @model_settings[:attributes][attr][:default]
         end
@@ -185,10 +200,14 @@ def attribute(*args)
           attr_name = attr_name.to_sym
           options = options.pop
           options = {} if options.nil?
-          if options[:enforce].nil? or !options[:enforce].include?(:list)
-            options[:enforce] = options[:enforce] ? (options[:enforce] << :no_list) : [:no_list]
-          end
+
+          options[:enforce] ||= []
+
+          set_data_type(options)
+          set_no_list_by_default(options)
+
           @model_settings[:attributes][attr_name] = options
+          load_yaml_scheme_options(attr_name)
           shape_attribute(attr_name)
           namespace = attribute_namespace(attr_name)
           namespace = namespace || @model_settings[:namespace]
@@ -381,6 +400,7 @@ def read_only(attributes)
           instance
         end
 
+
         def show_all_languages?(args)
           args.first.is_a?(Hash) && args.first.keys.include?(:include_languages) && args.first[:include_languages]
         end
diff --git a/lib/goo/base/settings/yaml_settings.rb b/lib/goo/base/settings/yaml_settings.rb
new file mode 100644
index 00000000..8a931b3a
--- /dev/null
+++ b/lib/goo/base/settings/yaml_settings.rb
@@ -0,0 +1,45 @@
+require 'yaml'
+
+module Goo
+  module Base
+    module Settings
+      module YAMLScheme
+        attr_reader :yaml_settings
+
+        def init_yaml_scheme_settings
+          scheme_file_path = @model_settings[:scheme]
+          @yaml_settings = read_yaml_settings_file(scheme_file_path)
+        end
+
+        def attribute_yaml_settings(attr)
+
+          return {} if yaml_settings.nil?
+
+          yaml_settings[attr.to_sym]
+        end
+
+
+
+        private
+
+        def load_yaml_scheme_options(attr)
+          settings = attribute_settings(attr)
+          yaml_settings = attribute_yaml_settings(attr)
+          settings.merge! yaml_settings unless yaml_settings.nil? || yaml_settings.empty?
+        end
+
+        def read_yaml_settings_file(scheme_file_path)
+          return  if scheme_file_path.nil?
+
+          yaml_contents = File.read(scheme_file_path) rescue return
+
+          YAML.safe_load(yaml_contents, symbolize_names: true)
+        end
+      end
+    end
+  end
+end
+
+
+
+
diff --git a/lib/goo/sparql/triples.rb b/lib/goo/sparql/triples.rb
index cb840df9..df3f9f1d 100644
--- a/lib/goo/sparql/triples.rb
+++ b/lib/goo/sparql/triples.rb
@@ -67,16 +67,6 @@ def self.model_update_triples(model)
         unless model.persistent?
           graph_insert << [subject, RDF.type, model.class.uri_type(model.collection)]
         end
-        #set default values before saving
-        if not model.persistent?
-          model.class.attributes_with_defaults.each do |attr|
-            value = model.send("#{attr}")
-            if value.nil?
-              value = model.class.default(attr).call(model)
-              model.send("#{attr}=",value)
-            end
-          end
-        end
 
         model.modified_attributes.each do |attr|
           next if model.class.collection?(attr)
diff --git a/lib/goo/validators/enforce.rb b/lib/goo/validators/enforce.rb
index d326839b..3c90e204 100644
--- a/lib/goo/validators/enforce.rb
+++ b/lib/goo/validators/enforce.rb
@@ -3,127 +3,132 @@ module Goo
   module Validators
     module Enforce
 
-      def self.enforce_by_attribute(model,attr)
-        return model.model_settings[:attributes][attr][:enforce]
-      end
+      class EnforceInstance
+        attr_reader :errors_by_opt
+        def initialize
+          @errors_by_opt = {}
+        end
+
+        def enforce(inst,attr,value)
+          enforce_opts = enforce_by_attribute(inst.class,attr)
+          return nil if enforce_opts.nil? or enforce_opts.length == 0
 
-      def self.enforce_type_boolean(attr,value)
-        if value.kind_of? Array
-          if (value.select {|x| !((x.class == TrueClass) || (x.class == FalseClass))} ).length > 0
-            return  "All values in attribute `#{attr}` must be `Boolean`"
+          enforce_opts.each do |opt|
+            case opt
+            when :unique
+              check Goo::Validators::Unique, inst, attr, value, opt
+            when :no_list
+              validator = Goo::Validators::DataType.new(inst, attr, value, Array)
+              if validator.valid? && !value.nil?
+                add_error(opt,
+                          "`#{attr}` is defined as non Array - it cannot hold multiple values")
+              end
+            when :existence
+              check Goo::Validators::Existence, inst, attr, value, opt
+            when :list, Array
+              check Goo::Validators::DataType, inst, attr, value,opt, Array
+            when :uri, RDF::URI
+              check Goo::Validators::DataType, inst, attr, value,opt, RDF::URI
+            when :string, String
+              check Goo::Validators::DataType, inst, attr, value,opt, String
+            when :integer, Integer
+              check Goo::Validators::DataType, inst, attr, value,opt, Integer
+            when :boolean
+              check Goo::Validators::DataType, inst, attr, value, opt,:boolean
+            when :date_time, DateTime
+              check Goo::Validators::DataType, inst, attr, value, opt, DateTime
+            when :float, Float
+              check Goo::Validators::DataType, inst, attr, value, opt, Float
+            when :symmetric
+              check Goo::Validators::Symmetric, inst, attr, value, opt
+            when /^distinct_of_/
+              check Goo::Validators::DistinctOf, inst, attr, value, opt, opt
+            when /^superior_equal_to_/
+              check Goo::Validators::SuperiorEqualTo, inst, attr, value, opt, opt
+            when /^inverse_of_/
+              check Goo::Validators::InverseOf, inst, attr, value, opt, opt
+            when Proc
+              call_proc(opt, inst, attr)
+            when /^max_/, /^min_/
+              type = opt.to_s.index("max_") ? :max : :min
+              check Goo::Validators::ValueRange, inst, attr, value, type, opt.to_s
+            else
+              if object_type?(opt)
+                check_object_type inst, attr, value, opt
+              elsif instance_proc?(inst, opt)
+                call_proc(inst.method(opt), inst, attr)
+              end
+            end
           end
-        else
-          if !((value.class == TrueClass) || (value.class == FalseClass))
-            return  "Attribute `#{attr}` value `#{value}` must be a `Boolean`"
+
+          errors_by_opt.length > 0 ? errors_by_opt : nil
+        end
+
+        def enforce_callback(inst, attr)
+          callbacks = Array(inst.class.update_callbacks(attr))
+          callbacks.each do |proc|
+            if instance_proc?(inst, proc)
+              call_proc(inst.method(proc), inst, attr)
+            elsif proc.is_a?(Proc)
+              call_proc(proc, inst, attr)
+            end
           end
         end
-      end
 
-      def self.enforce_type(attr,type,value)
-        if type == :boolean
-          return self.enforce_type_boolean(attr,value)
+        private
+
+        def object_type(opt)
+          opt.respond_to?(:shape_attribute) ? opt : Goo.model_by_name(opt)
         end
-        if value.kind_of? Array
-          if (value.select {|x| !(x.kind_of? type)} ).length > 0
-            return  "All values in attribute `#{attr}` must be `#{type.name}`"
-          end
-        else
-          if !(value.kind_of? type)
-            return  "Attribute `#{attr}` value `#{value}` must be a `#{type.name}`"
-          end
+
+        def object_type?(opt)
+          opt.respond_to?(:shape_attribute) ? opt : Goo.model_by_name(opt)
         end
-      end
 
-      def self.enforce_range_length(type_range,attr,opt_s,value)
-        if !value.nil? && !(value.kind_of?(Array) || value.kind_of?(String))
-          return "#{attr} value (#{value}) must be an Array or String - it has range length constraints"
+        def instance_proc?(inst, opt)
+          opt && (opt.is_a?(Symbol) || opt.is_a?(String)) && inst.respond_to?(opt)
         end
-        range = opt_s[4..opt_s.length].to_i
-        if type_range == :min
-          if !value.nil? && (value.length < range)
-            return "#{attr} value has length `#{value.length}` and the min length is `#{range}`"
+
+        def check_object_type(inst, attr, value, opt)
+          model_range = object_type(opt)
+          if model_range && !value.nil?
+            check Goo::Validators::ObjectType, inst, attr, value, model_range.model_name, model_range
           end
-        else
-          if !value.nil? && (value.length > range)
-            return "#{attr} value has length `#{value.length}` and the max length is `#{range}`"
+        end
+
+        def check(validator_class, inst, attr, value, opt, *options)
+          validator = validator_class.new(inst, attr, value, *options)
+          add_error(opt, validator.error) unless validator.valid?
+        end
+        def enforce_by_attribute(model, attr)
+           model.model_settings[:attributes][attr][:enforce]
+        end
+
+        def call_proc(proc,inst, attr)
+          # This should return an array like [:name_of_error1, "Error message 1", :name_of_error2, "Error message 2"]
+          errors = proc.call(inst, attr)
+
+          return unless !errors.nil? && errors.is_a?(Array)
+
+          errors.each_slice(2) do |e|
+            next if e.nil? || e.compact.empty?
+            add_error(e[0].to_sym, e[1])
           end
         end
+
+        def add_error(opt, err)
+          return if err.nil?
+          @errors_by_opt[opt] = err
+        end
       end
 
+
       def self.enforce(inst,attr,value)
-        enforce_opts = enforce_by_attribute(inst.class,attr)
-        return nil if enforce_opts.nil? or enforce_opts.length == 0
-        errors_by_opt = {}
-        enforce_opts.each do |opt|
-          case opt
-          when :unique
-            unless value.nil?
-              dup = Goo::SPARQL::Queries.duplicate_attribute_value?(inst,attr)
-              if dup
-                add_error(opt, errors_by_opt,
-                "`#{attr}` must be unique. " +
-                "There are other model instances with the same attribute value `#{value}`.")
-              end
-            end
-          when :no_list
-            if value.kind_of? Array
-              add_error(opt, errors_by_opt,
-                      "`#{attr}` is defined as non Array - it cannot hold multiple values")
-            end
-          when :existence
-            add_error(opt, errors_by_opt, "`#{attr}` value cannot be nil") if value.nil?
-          when :list, Array
-            if !value.nil? && !(value.kind_of? Array)
-              add_error(opt, errors_by_opt, "`#{attr}` value must be an Array")
-            end
-          when :uri, RDF::URI
-            add_error(opt, errors_by_opt, enforce_type(attr,RDF::URI,value)) unless value.nil?
-          when :string, String
-            add_error(opt, errors_by_opt, enforce_type(attr,String,value)) unless value.nil?
-          when :integer, Integer
-            add_error(opt, errors_by_opt, enforce_type(attr,Integer,value)) unless value.nil?
-          when :boolean
-            add_error(opt, errors_by_opt, enforce_type(attr,:boolean,value)) unless value.nil?
-          when :date_time, DateTime
-            add_error(opt, errors_by_opt, enforce_type(attr,DateTime,value)) unless value.nil?
-          when Proc
-            # This should return an array like [:name_of_error1, "Error message 1", :name_of_error2, "Error message 2"]
-            errors = opt.call(inst, attr)
-            errors.each_slice(2) do |e|
-              next if e.nil? || e.compact.empty?
-              add_error(e[0].to_sym, errors_by_opt, e[1]) rescue binding.pry
-            end
-          else
-            model_range = opt.respond_to?(:shape_attribute) ? opt : Goo.model_by_name(opt)
-            if model_range and !value.nil?
-              values = value.kind_of?(Array) ? value : [value]
-              values.each do |v|
-                if (!v.kind_of?(model_range)) && !(v.respond_to?(:klass) && v[:klass] == model_range)
-                  add_error(model_range.model_name, errors_by_opt,
-                            "`#{attr}` contains values that are not instance of `#{model_range.model_name}`")
-                else
-                  if !v.respond_to?(:klass) && !v.persistent?
-                    add_error(model_range.model_name, errors_by_opt,
-                              "`#{attr}` contains non persistent models. It will not save.")
-                  end
-                end
-              end
-            end
-            opt_s = opt.to_s
-            if opt_s.index("max_") == 0
-              add_error(:max, errors_by_opt, enforce_range_length(:max,attr,opt_s,value)) unless value.nil?
-            end
-            if opt_s.index("min_") == 0
-              add_error(:min, errors_by_opt, enforce_range_length(:min,attr,opt_s,value)) unless value.nil?
-            end
-          end
-        end
-        return errors_by_opt.length > 0 ? errors_by_opt : nil
+        EnforceInstance.new.enforce(inst,attr,value)
       end
 
-      def self.add_error(opt, h, err)
-        return if err.nil?
-        h[opt] = err
+      def self.enforce_callbacks(inst, attr)
+        EnforceInstance.new.enforce_callback(inst, attr)
       end
     end
   end
diff --git a/lib/goo/validators/implementations/data_type.rb b/lib/goo/validators/implementations/data_type.rb
new file mode 100644
index 00000000..0ea65ab3
--- /dev/null
+++ b/lib/goo/validators/implementations/data_type.rb
@@ -0,0 +1,66 @@
+module Goo
+  module Validators
+    class DataType < ValidatorBase
+      include Validator
+
+      keys [:list, :uri, :string, :integer, :boolean, :date_time, :float]
+
+      error_message ->(obj) {
+        if @value.kind_of? Array
+          return "All values in attribute `#{@attr}` must be `#{@type}`"
+        else
+          return "Attribute `#{@attr}` with the value `#{@value}` must be `#{@type}`"
+
+        end
+      }
+
+      validity_check -> (obj) do
+        self.enforce_type(@type, @value)
+      end
+
+      def initialize(inst, attr, value, type)
+        super(inst, attr, value)
+        @type = type
+      end
+
+
+
+      def enforce_type(type, value)
+        return true if value.nil?
+
+        if type == :boolean
+          return self.enforce_type_boolean(value)
+        elsif type.eql?(:uri) || type.eql?(RDF::URI)
+          return  self.enforce_type_uri(value)
+        elsif type.eql?(:uri) || type.eql?(Array)
+          return value.is_a? Array
+        else
+          if value.is_a? Array
+            return value.select{|x| !x.is_a?(type)}.empty?
+          else
+            return value.is_a? type
+          end
+        end
+
+      end
+
+      def enforce_type_uri(value)
+        return true  if value.nil?
+
+        value.is_a?(RDF::URI) && value.valid?
+      end
+
+      def enforce_type_boolean(value)
+        if value.kind_of? Array
+          return value.select { |x| !is_a_boolean?(x) }.empty?
+        else
+          return  is_a_boolean?(value)
+        end
+      end
+
+      def is_a_boolean?(value)
+        return (value.class == TrueClass) || (value.class == FalseClass)
+      end
+    end
+  end
+end
\ No newline at end of file
diff --git a/lib/goo/validators/implementations/distinct_of.rb b/lib/goo/validators/implementations/distinct_of.rb
new file mode 100644
index 00000000..2e93313b
--- /dev/null
+++ b/lib/goo/validators/implementations/distinct_of.rb
@@ -0,0 +1,34 @@
+module Goo
+  module Validators
+    class DistinctOf < ValidatorBase
+      include Validator
+
+      key :distinct_of_
+
+      error_message ->(obj) { "`#{@attr}` must be distinct of `#{@property}`"}
+
+      validity_check -> (obj) do
+        return true if self.class.empty_value?(@value)
+
+        self.distinct?(@inst, @property, @value)
+      end
+
+      def initialize(inst, attr, value, key)
+        super(inst, attr, value)
+        @property = self.class.property(key)
+      end
+
+
+
+      def distinct?(inst, property, value)
+        target_values = self.class.attr_value(property, inst)
+        current_values = Array(value)
+
+        !current_values.any?{ |x| self.find_any?(target_values, x)}
+      end
+      def find_any?(array, value)
+        array.any?{ |x| self.class.equivalent_value?(value, x)}
+      end
+    end
+  end
+end
\ No newline at end of file
diff --git a/lib/goo/validators/implementations/existence.rb b/lib/goo/validators/implementations/existence.rb
new file mode 100644
index 00000000..fcf04d61
--- /dev/null
+++ b/lib/goo/validators/implementations/existence.rb
@@ -0,0 +1,17 @@
+module Goo
+  module Validators
+    class Existence < ValidatorBase
+      include Validator
+
+      key :existence
+
+      error_message ->(obj) { "`#{@value}` value cannot be nil"}
+
+      validity_check -> (obj) do
+        not self.class.empty_value?(@value)
+      end
+
+
+    end
+  end
+end
\ No newline at end of file
diff --git a/lib/goo/validators/implementations/inverse_of.rb b/lib/goo/validators/implementations/inverse_of.rb
new file mode 100644
index 00000000..60518af3
--- /dev/null
+++ b/lib/goo/validators/implementations/inverse_of.rb
@@ -0,0 +1,35 @@
+module Goo
+  module Validators
+    class InverseOf < ValidatorBase
+      include Validator
+
+      key :inverse_of_
+
+      error_message ->(obj) {
+        "`#{@attr}` must be the inverse of ``#{@property}``"
+      }
+
+      validity_check -> (obj) do
+        return true if self.class.empty_value?(@value)
+
+        return Array(@value).select{|x| not inverse?(@property,x, @inst)}.empty?
+      end
+
+      def initialize(inst, attr, value, key)
+        super(inst, attr, value)
+        @property = self.class.property(key)
+      end
+
+      def inverse?(attr, value, source_object)
+        if self.class.respond_to?(attr, value)
+          target_values = self.class.attr_value(attr, value)
+          return target_values.any?{ |target_object| self.class.equivalent_value?(target_object, source_object)}
+        end
+
+        false
+      end
+
+
+    end
+  end
+end
\ No newline at end of file
diff --git a/lib/goo/validators/implementations/object_type.rb b/lib/goo/validators/implementations/object_type.rb
new file mode 100644
index 00000000..3af97b41
--- /dev/null
+++ b/lib/goo/validators/implementations/object_type.rb
@@ -0,0 +1,46 @@
+module Goo
+  module Validators
+    class ObjectType < ValidatorBase
+      include Validator
+
+      key :object_type
+
+      error_message ->(obj) {
+        if @error.eql?(:persistence)
+          "`#{@attr}` contains non persistent models. It will not save."
+        else
+          "`#{@attr}` contains values that are not instance of `#{@model_range.model_name}`"
+        end
+      }
+
+      validity_check -> (obj) do
+        values = Array(@value)
+
+        unless values.select { |v| !self.is_a_model?(v, @model_range) }.empty?
+          @error = :no_range
+          return false
+        end
+
+        unless values.select { |v| !self.persistent?(v) }.empty?
+          @error = :persistence
+          return false
+        end
+
+        return true
+      end
+
+      def initialize(inst, attr, value, model_range)
+        super(inst, attr, value)
+        @model_range = model_range
+      end
+
+      def is_a_model?(value, model_range)
+        value.is_a?(model_range) || (value.respond_to?(:klass) && value[:klass] == model_range)
+      end
+
+      def persistent?(value)
+        value.respond_to?(:klass) || value.persistent?
+      end
+    end
+  end
+end
diff --git a/lib/goo/validators/implementations/superior_equal_to.rb b/lib/goo/validators/implementations/superior_equal_to.rb
new file mode 100644
index 00000000..91508f30
--- /dev/null
+++ b/lib/goo/validators/implementations/superior_equal_to.rb
@@ -0,0 +1,26 @@
+module Goo
+  module Validators
+    class SuperiorEqualTo < ValidatorBase
+      include Validator
+
+      key :superior_equal_to_
+
+      error_message ->(obj) {
+        "`#{@attr}` must be superior or equal to `#{@property}`"
+      }
+
+      validity_check -> (obj) do
+        target_values = self.class.attr_value(@property, @inst)
+
+        return true if target_values.empty?
+
+        return @value >= target_values.first
+      end
+
+      def initialize(inst, attr, value, key)
+        super(inst, attr, value)
+        @property = self.class.property(key)
+      end
+    end
+  end
+end
diff --git a/lib/goo/validators/implementations/symmetric.rb b/lib/goo/validators/implementations/symmetric.rb
new file mode 100644
index 00000000..e9ceb3f4
--- /dev/null
+++ b/lib/goo/validators/implementations/symmetric.rb
@@ -0,0 +1,33 @@
+module Goo
+  module Validators
+    class Symmetric < ValidatorBase
+      include Validator
+
+      key :symmetric
+
+      error_message ->(obj) {
+        "`#{@attr}` must be symmetric"
+      }
+
+      validity_check -> (obj) do
+        return true if self.class.empty_value?(@value)
+
+        return Array(@value).select{|x| not symmetric?(@attr,x, @inst)}.empty?
+      end
+
+      def symmetric?(attr, value, source_object)
+        if respond_to?(attr, value)
+          target_values = self.class.attr_value(attr, value)
+          return target_values.any?{ |target_object| self.class.equivalent_value?(target_object, source_object)}
+        end
+
+        return false
+      end
+
+      def respond_to?(attr, object)
+        object && object.respond_to?(attr)
+      end
+
+    end
+  end
+end
\ No newline at end of file
diff --git a/lib/goo/validators/implementations/unique.rb b/lib/goo/validators/implementations/unique.rb
new file mode 100644
index 00000000..feb13a4b
--- /dev/null
+++ b/lib/goo/validators/implementations/unique.rb
@@ -0,0 +1,20 @@
+module Goo
+  module Validators
+    class Unique < ValidatorBase
+      include Validator
+
+      key :unique
+
+      error_message ->(obj) { "`#{@attr}` must be unique. " +
+        "There are other model instances with the same attribute value `#{@value}`."}
+
+      validity_check -> (obj) do
+        return true if @value.nil?
+
+        !Goo::SPARQL::Queries.duplicate_attribute_value?(@inst,@attr)
+      end
+
+
+    end
+  end
+end
\ No newline at end of file
diff --git a/lib/goo/validators/implementations/value_range.rb b/lib/goo/validators/implementations/value_range.rb
new file mode 100644
index 00000000..71440bcf
--- /dev/null
+++ b/lib/goo/validators/implementations/value_range.rb
@@ -0,0 +1,49 @@
+module Goo
+  module Validators
+    class ValueRange < ValidatorBase
+      include Validator
+
+      keys [:min_, :max_]
+
+      error_message ->(obj) {
+        value = self.value_length(@value)
+        if @type == :min
+          "#{@attr} value has length `#{value}` and the min length is `#{@range}`"
+        else
+          "#{@attr} value has length `#{value}` and the max length is `#{@range}`"
+        end
+      }
+
+      validity_check -> (obj) do
+        self.enforce_range_length(@type, @range, @value)
+      end
+
+      def initialize(inst, attr, value, type)
+        super(inst, attr, value)
+        @type = type.index("max_") ? :max : :min
+        @range = self.range(type)
+      end
+
+      def enforce_range_length(type_range, range, value)
+        return false if value.nil?
+        value_length = self.value_length(value)
+
+        (type_range.eql?(:min) && (value_length >= range)) || (type_range.eql?(:max) && (value_length <= range))
+      end
+
+      def range(opt)
+        opt[4..opt.length].to_i
+      end
+
+      def value_length(value)
+        return 0 if value.nil?
+
+        if value.is_a?(String) || value.is_a?(Array)
+          value.length
+        else
+          value
+        end
+      end
+    end
+  end
+end
diff --git a/lib/goo/validators/validator.rb b/lib/goo/validators/validator.rb
new file mode 100644
index 00000000..e7db80a4
--- /dev/null
+++ b/lib/goo/validators/validator.rb
@@ -0,0 +1,111 @@
+module Goo
+  module Validators
+
+    class ValidatorBase
+
+      def initialize(inst, attr, value)
+        @inst = inst
+        @attr = attr
+        @value = value
+      end
+
+      def valid?
+        self.instance_eval(&self.class.validator_settings[:check])
+      end
+
+      def error
+        message =  self.class.validator_settings[:message]
+        if message.is_a? Proc
+          self.instance_eval(&message)
+        else
+          message
+        end
+      end
+
+    end
+
+    module Validator
+
+      def self.included(base)
+        base.extend(ClassMethods)
+      end
+
+
+      module ClassMethods
+
+        def key(id)
+          validator_settings[:id] = id
+        end
+
+        def keys(ids)
+          key ids
+        end
+
+        def validity_check(block)
+          validator_settings[:check] = block
+        end
+
+        def error_message(message)
+          validator_settings[:message] = message
+        end
+
+        def validator_settings
+          @validator_settings ||= {}
+        end
+
+        def ids
+          Array(validator_settings[:id])
+        end
+
+        def property(key)
+          key[ids.first.size..key.size].to_sym
+        end
+
+        def respond_to?(attr, object)
+          object && object.respond_to?(attr)
+        end
+
+
+        def equivalent_value?(object1, object2)
+          if object1.respond_to?(:id) && object2.respond_to?(:id)
+            object1.id.eql?(object2.id)
+          else
+            object2 == object1
+          end
+        end
+
+        def attr_value(attr, object)
+          Array(object.send(attr))
+        end
+
+        def empty_value?(value)
+          value.nil? || empty?(value) || empty_array?(value)
+        end
+        def empty?(value)
+          empty_string?(value) || empty_to_s?(value)
+        end
+        def empty_string?(string)
+          string.is_a?(String) && string.strip.empty?
+        end
+
+        def empty_to_s?(object)
+          begin
+            object && object.to_s&.strip.empty?
+          rescue
+            return false
+          end
+        end
+
+        def empty_array?(array)
+          array.is_a?(Array) && array && array.reject{|x|  x.nil? || empty?(x)}.empty?
+        end
+      end
+
+
+
+
+
+    end
+  end
+end
+
diff --git a/test/data/yaml_scheme_model_test.yml b/test/data/yaml_scheme_model_test.yml
new file mode 100644
index 00000000..fd8c4921
--- /dev/null
+++ b/test/data/yaml_scheme_model_test.yml
@@ -0,0 +1,11 @@
+name:
+  label: 'Name'
+  description: 'Person name'
+  equivalents: ['test:name' , 'test2:name', 'test3:person_name']
+  help: 'Put the person name as string'
+  example: 'John'
+nationality:
+  label: 'Person nationality'
+  enforcedValues: {'fr': 'france', 'us': 'USA'}
+
+
diff --git a/test/models.rb b/test/models.rb
index cd606eed..7d490a4a 100644
--- a/test/models.rb
+++ b/test/models.rb
@@ -101,7 +101,10 @@ def self.create_test_case_data
   end
 
   def self.delete_test_case_data
-    objects = [Student, University, Program, Category, Address]
+    delete_all [Student, University, Program, Category, Address]
+  end
+
+  def self.delete_all(objects)
     objects.each do |obj|
       obj.where.include(obj.attributes).each do |i|
         i.delete
diff --git a/test/test_dsl_settings.rb b/test/test_dsl_settings.rb
index c444e829..9a8f03df 100644
--- a/test/test_dsl_settings.rb
+++ b/test/test_dsl_settings.rb
@@ -2,6 +2,22 @@
 
 GooTest.configure_goo
 
+class NewPersonModel < Goo::Base::Resource
+  model :person_model_new, name_with: :name
+  attribute :name, type: :string, enforce: [ :existence, :unique]
+  attribute :multiple_values, type: [:list, :integer], enforce: [ :existence, :min_3, :max_5 ]
+  attribute :one_number, type: :integer,enforce: [ :existence ] #by default not a list
+  attribute :birth_date, type: :date_time, enforce: [ :existence ]
+
+  attribute :created, type: DateTime ,
+            default: lambda { |record| DateTime.now },
+            namespace: :omv
+
+  attribute :friends, type: NewPersonModel , enforce: [ :existence]
+  attribute :status, type: :status, enforce: [ :existence],
+            default: lambda { |record| StatusModel.find("single") }
+end
+
 class StatusModel < Goo::Base::Resource
   model :status_model, name_with: :name
   attribute :description, enforce: [ :existence, :unique]
@@ -32,13 +48,46 @@ def initialize(attributes = {})
   end
 end
 
+
+class YamlSchemeModelTest < Goo::Base::Resource
+  model :yaml_scheme_model_test, name_with: :name, scheme: 'test/data/yaml_scheme_model_test.yml'
+  attribute :name, enforce: [ :existence, :string, :unique]
+  attribute :last_name, enforce: [ :existence, :string, :unique]
+  attribute :birth_date, enforce: [ :existence, :date_time ]
+  attribute :nationality, enforce: [ :existence, :string ]
+  attribute :created, enforce: [ DateTime ],
+            default: lambda { |record| DateTime.now },
+            namespace: :omv
+  attribute :friends, enforce: [ :existence , PersonModel]
+  attribute :status, enforce: [ :existence, :status ],
+            default: lambda { |record| StatusModel.find("single") }
+end
+
+
 class TestDSLSeeting < MiniTest::Unit::TestCase
   def initialize(*args)
     super(*args)
   end
 
+  def test_data_type_dsl
+    _test_attributes_enforce NewPersonModel
+  end
+
   def test_attributes_set_get
+    _test_attributes_enforce PersonModel
+  end
+
+  def test_default_value
+    #default is on save ... returns`
     person = PersonModel.new
+    assert_equal nil, person.created
+  end
+
+
+  private
+  def _test_attributes_enforce(model)
+    person = model.new
+    model_key_name = model.model_name
     assert(person.respond_to? :id)
     assert(person.kind_of? Goo::Base::Resource)
     assert !person.valid?
@@ -67,7 +116,7 @@ def test_attributes_set_get
     assert !person.valid?
     assert !person.errors[:birth_date]
 
-    person.birth_date = "X" 
+    person.birth_date = "X"
     assert !person.valid?
     assert person.errors[:birth_date][:date_time]
 
@@ -103,17 +152,17 @@ def test_attributes_set_get
       person.multiple_values << 99
     end
 
-    friends = [PersonModel.new , PersonModel.new]
+    friends = [model.new , model.new]
     person.friends = friends
     assert !person.valid?
     assert person.errors[:friends][:no_list]
-    person.friends = PersonModel.new
+    person.friends = model.new
     assert !person.valid?
-    assert person.errors[:friends][:person_model]
+    assert person.errors[:friends][model_key_name]
     person.friends = "some one"
     assert !person.valid?
-    assert person.errors[:friends][:person_model]
-    person.friends = PersonModel.new
+    assert person.errors[:friends][model_key_name]
+    person.friends = model.new
 
     person.one_number = 99
     assert !person.valid?
@@ -127,7 +176,7 @@ def test_attributes_set_get
     assert !person.valid?
     assert person.errors[:one_number][:no_list]
 
-    person.one_number = 99 
+    person.one_number = 99
     assert_equal(99, person.one_number)
     assert !person.valid?
     assert !person.errors[:one_number]
@@ -139,10 +188,25 @@ def test_attributes_set_get
     assert !person.valid?
   end
 
-  def test_default_value
-    #default is on save ... returns`
-    person = PersonModel.new
-    assert_equal nil, person.created
+  def test_model_with_yaml_scheme
+
+    settings = YamlSchemeModelTest.model_settings
+    attributes_settings = settings[:attributes]
+
+
+    assert_equal "test/data/yaml_scheme_model_test.yml", settings[:scheme]
+
+    assert_equal 'Name', attributes_settings[:name][:label]
+    assert_equal 'Person name', attributes_settings[:name][:description]
+    assert_equal %w[test:name test2:name test3:person_name], attributes_settings[:name][:equivalents]
+    assert_equal 'Put the person name as string', attributes_settings[:name][:help]
+    assert_equal 'John', attributes_settings[:name][:example]
+
+
+    assert_equal 'Person nationality', attributes_settings[:nationality][:label]
+    hash = {fr: 'france', us: 'USA'}
+    assert_equal hash, attributes_settings[:nationality][:enforcedValues]
+
   end
 
 end
diff --git a/test/test_update_callbacks.rb b/test/test_update_callbacks.rb
new file mode 100644
index 00000000..bef38a68
--- /dev/null
+++ b/test/test_update_callbacks.rb
@@ -0,0 +1,53 @@
+require_relative 'test_case'
+
+
+require_relative 'models'
+
+class TestUpdateCallBack < Goo::Base::Resource
+  model :update_callback_model, name_with: :code
+  attribute :code, enforce: [:string, :existence]
+  attribute :name, enforce: [:string, :existence]
+  attribute :first_name, onUpdate: :update_name
+  attribute :last_name, onUpdate: :update_name
+
+
+  def update_name(inst, attr)
+    self.name = self.first_name + self.last_name
+  end
+end
+
+class TestUpdateCallBacks < MiniTest::Unit::TestCase
+
+  def self.before_suite
+    GooTestData.delete_all [TestUpdateCallBack]
+  end
+
+  def self.after_suite
+    GooTestData.delete_all [TestUpdateCallBack]
+  end
+
+
+  def test_update_callback
+    p = TestUpdateCallBack.new
+    p.code = "1"
+    p.name = "name"
+    p.first_name = "first_name"
+    p.last_name = "last_name"
+
+    assert p.valid?
+    p.save
+
+    p.bring_remaining
+
+    assert_equal p.first_name + p.last_name, p.name
+
+    p.last_name = "last_name2"
+    p.save
+
+    p.bring_remaining
+    assert_equal  "last_name2",  p.last_name
+    assert_equal p.first_name + p.last_name, p.name
+  end
+
+end
+
diff --git a/test/test_validators.rb b/test/test_validators.rb
new file mode 100644
index 00000000..8795fccf
--- /dev/null
+++ b/test/test_validators.rb
@@ -0,0 +1,427 @@
+require_relative 'test_case'
+
+GooTest.configure_goo
+require_relative 'models'
+
+class Person < Goo::Base::Resource
+  model :person_model_validators, name_with: :name
+  attribute :name, enforce: [:string, :existence]
+  attribute :last_name, enforce: [:string]
+  attribute :multiple_values, enforce: [ :list, :integer]
+  attribute :one_number, enforce: [ :integer ]
+  attribute :birth_date, enforce: [ :date_time ]
+  attribute :male, enforce: [:boolean]
+  attribute :social, enforce: [:uri]
+  attribute :weight, enforce: [:float]
+  attribute :friends, enforce: [Person, :list]
+end
+
+
+class RangeTestModel < Goo::Base::Resource
+  model :range_test_model, name_with: :name
+  attribute :name, enforce: [:string, :existence, :min_3, :max_5]
+  attribute :multiple_values, enforce: [ :list, :integer, :min_3, :max_5 ]
+  attribute :one_number, enforce: [ :integer,  :min_3, :max_5]
+  attribute :weight, enforce: [:float, :min_3, :max_5]
+end
+
+class SymmetricTestModel < Goo::Base::Resource
+  model :symmetric_test_model, name_with: :name
+  attribute :name, enforce: [:unique, :existence]
+  attribute :friend, enforce: [SymmetricTestModel, :symmetric]
+  attribute :friends, enforce: [SymmetricTestModel, :symmetric, :list]
+end
+
+class DistinctOfTestModel < Goo::Base::Resource
+  model :distinct_of_test_model, name_with: :name
+  attribute :name, enforce: [:unique, :existence, :string]
+  attribute :last_name, enforce: [:distinct_of_name, :string]
+  attribute :names, enforce: [:list, :string]
+  attribute :last_names, enforce: [:list, :distinct_of_names, :string]
+end
+
+class SuperiorToTestModel < Goo::Base::Resource
+  model :superior_to_test_model, name_with: :name
+  attribute :name, enforce: [:unique, :existence, :string]
+  attribute :birth_date, enforce: [:date_time]
+  attribute :death_date, enforce: [:superior_equal_to_birth_date, :date_time]
+end
+
+class InverseOfTestModel < Goo::Base::Resource
+  model :inverse_test_model_one, name_with: :name
+  attribute :name, enforce: [:unique, :existence, :string]
+  attribute :state, enforce: [InverseOfTestModel]
+  attribute :city, enforce: [:inverse_of_state, InverseOfTestModel]
+  attribute :states, enforce: [InverseOfTestModel, :list]
+  attribute :cities, enforce: [:inverse_of_states, InverseOfTestModel, :list]
+end
+
+
+class ProcValidatorsTestModel < Goo::Base::Resource
+  model :proc_validator_test_model, name_with: :name
+  attribute :name, enforce: [:unique, :equal_to_test]
+  attribute :last_name, enforce: [:unique, ->(inst, attr) {  equal_to_test_2(inst, attr)}]
+
+
+  def self.equal_to_test_2(inst, attr)
+    value = inst.send(attr)
+
+    return nil if value && value.eql?('test 2')
+
+    [:equal_to_test_2, "#{attr} need to be equal to `test 2`"]
+  end
+
+  def equal_to_test(inst, attr)
+    value = inst.send(attr)
+
+    return nil if  value && value.eql?('test')
+
+    [:equal_to_test, "#{attr} need to be equal to `test`"]
+  end
+end
+
+class TestValidators < MiniTest::Unit::TestCase
+
+  def self.before_suite
+    begin
+      GooTestData.create_test_case_data
+    rescue Exception => e
+      puts e.message
+    end
+  end
+
+  def self.after_suite
+    GooTestData.delete_test_case_data
+    GooTestData.delete_all [SymmetricTestModel, InverseOfTestModel]
+  end
+
+
+  def test_unique_validator
+
+    s = Student.new
+    s.birth_date = DateTime.parse('1978-01-01')
+
+    s.name = "Susan"
+
+    refute s.valid?
+
+    s.name = "new"
+
+    assert s.valid?
+  end
+
+  def test_existence_validator
+    s = Student.new
+
+    refute s.valid?
+
+    assert s.errors[:name][:existence]
+    assert s.errors[:birth_date][:existence]
+
+
+    s.name = ''
+    s.birth_date = ''
+    assert s.errors[:name][:existence]
+    assert s.errors[:birth_date][:existence]
+
+
+    s.name = 'new'
+    s.birth_date = DateTime.parse('1978-01-01')
+
+    assert s.valid?
+  end
+
+  def test_datatype_validators
+    p = Person.new
+    p.name =  'test'
+    #nil  values are valid
+    assert p.valid?
+
+    p.last_name =  false
+    p.multiple_values = "hello"
+    p.one_number = "hello"
+    p.birth_date = 100
+    p.male =  "ok"
+    p.social =  100
+    p.weight =  100
+
+
+    #wrong types are not valid
+    refute p.valid?
+    assert p.errors[:last_name][:string]
+    assert p.errors[:multiple_values][:list]
+    assert p.errors[:multiple_values][:integer]
+    assert p.errors[:one_number][:integer]
+    assert p.errors[:birth_date][:date_time]
+    assert p.errors[:male][:boolean]
+    assert p.errors[:social][:uri]
+
+    p.last_name =  "hello"
+    p.multiple_values = [22,11]
+    p.one_number = 12
+    p.birth_date = DateTime.parse('1978-01-01')
+    p.male =  true
+    p.social =  RDF::URI.new('https://test.com/')
+    p.weight =  100.0
+    #good types are  valid
+    assert p.valid?
+  end
+
+  def test_uri_datatype_validator
+    p = Person.new
+    p.name =  'test'
+
+    assert p.valid?
+
+    p.social =  RDF::URI.new('') #empty uri
+    refute p.valid?
+
+    p.social =  RDF::URI.new('wrong/uri')
+    refute p.valid?
+
+    p.social =  RDF::URI.new('https://test.com/')
+    assert p.valid?
+  end
+
+  def test_object_type_validator
+    p = Person.new
+    p.name =  'test'
+    p.friends = [1]
+
+    refute p.valid?
+
+    new_person = Person.new
+    p.friends = [new_person]
+
+    refute p.valid?
+
+    new_person.persistent = true
+    p.friends = [new_person]
+
+    assert p.valid?
+  end
+
+  def test_value_range_validator
+    p = RangeTestModel.new
+
+    p.name =  "h"
+    p.multiple_values = [22,11]
+    p.one_number = 1
+    p.weight = 1.1
+
+    refute p.valid?
+    assert p.errors[:name][:min]
+    assert p.errors[:multiple_values][:min]
+    assert p.errors[:one_number][:min]
+    assert p.errors[:weight][:min]
+
+    p.name =  "hello hello"
+    p.multiple_values = [22,11,11,33,44, 55, 66]
+    p.one_number = 12
+    p.weight = 12.1
+
+    refute p.valid?
+    assert p.errors[:name][:max]
+    assert p.errors[:multiple_values][:max]
+    assert p.errors[:one_number][:max]
+    assert p.errors[:weight][:max]
+
+    p.name =  "hello"
+    p.multiple_values = [22,11,11,3]
+    p.one_number = 4
+    p.weight = 3.1
+
+    assert p.valid?
+
+  end
+
+  def test_symmetric_validator_no_list
+    p1 = SymmetricTestModel.new
+    p2 = SymmetricTestModel.new
+    p3 = SymmetricTestModel.new
+    p1.name = "p1"
+    p2.name = "p2"
+    p3.name = "p3"
+
+    p2.save
+    p3.save
+
+    p1.friend = p2
+
+    refute p1.valid?
+    assert p1.errors[:friend][:symmetric]
+
+    p3.friend = p1
+
+    refute p1.valid?
+
+    p2.friend = p1
+    p1.friend = p2
+
+    assert p1.valid?
+
+    p1.save
+
+    assert p2.valid?
+    GooTestData.delete_all [SymmetricTestModel]
+  end
+
+  def test_symmetric_validator_list
+    p1 = SymmetricTestModel.new
+    p2 = SymmetricTestModel.new
+    p3 = SymmetricTestModel.new
+    p4 = SymmetricTestModel.new
+    p1.name = "p1"
+    p2.name = "p2"
+    p3.name = "p3"
+    p4.name = "p4"
+
+    p2.save
+    p3.save
+    p4.save
+
+    p1.friends = [p2, p3]
+
+    refute p1.valid?
+    assert p1.errors[:friends][:symmetric]
+
+    p2.friends = [p1, p3, p4]
+    p3.friends = [p2]
+    p4.friends = [p2]
+
+    refute p1.valid?
+    refute p2.valid?
+
+
+    p3.friends = [p2, p1]
+
+    assert p1.valid?
+    p1.save
+
+    assert p3.valid?
+    p3.save
+
+
+    assert p2.valid?
+
+    p2.save
+
+    assert p4.valid?
+    GooTestData.delete_all [SymmetricTestModel]
+  end
+
+  def test_distinct_of_validator
+    p = DistinctOfTestModel.new
+    p.name = "p1"
+    p.last_name = "p1"
+    p.names = ["p1", "p2"]
+    p.last_names = ["p1", "p2"]
+
+
+    refute p.valid?
+
+    p.last_name = "last name"
+    p.last_names = ["last name 1", "last name 2"]
+
+    assert p.valid?
+
+    p.last_name = "last name"
+    p.last_names = ["last name 1", "p2"]
+
+    refute p.valid?
+
+    p.last_name = ""
+    p.last_names = []
+
+    assert p.valid?
+  end
+
+  def test_superior_equal_to_validator
+    p = SuperiorToTestModel.new
+    p.name = "p"
+    p.birth_date = DateTime.parse('1998-12-02')
+    p.death_date = DateTime.parse('1995-12-02')
+
+    refute p.valid?
+    assert p.errors[:death_date][:superior_equal_to_birth_date]
+
+    p.death_date = DateTime.parse('2023-12-02')
+
+    assert p.valid?
+
+    p.birth_date = nil
+
+    assert p.valid?
+  end
+
+  def test_inverse_of_validator_no_list
+    GooTestData.delete_all [InverseOfTestModel]
+    p1 = InverseOfTestModel.new
+    p2 = InverseOfTestModel.new
+
+    p1.name = 'p1'
+    p2.name = 'p2'
+
+
+    p2.save
+
+    p1.city = p2
+
+    refute p1.valid?
+    assert p1.errors[:city][:inverse_of_state]
+
+
+    p2.state = p1
+
+    assert p1.valid?
+
+  end
+
+  def test_inverse_of_validator_list
+    GooTestData.delete_all [InverseOfTestModel]
+    p1 = InverseOfTestModel.new
+    p2 = InverseOfTestModel.new
+    p3 = InverseOfTestModel.new
+    p4 = InverseOfTestModel.new
+
+    p1.name = 'p1'
+    p2.name = 'p2'
+    p3.name = 'p3'
+    p4.name = 'p4'
+
+    p2.save
+    p3.save
+
+    p1.cities = [p2,p3]
+
+    refute p1.valid?
+    assert p1.errors[:cities][:inverse_of_states]
+
+    p2.states = [p1, p4]
+    p3.states = [p2, p4]
+
+    refute p1.valid?
+    assert p1.errors[:cities][:inverse_of_states]
+
+    p3.states = [p2, p4, p1]
+
+    assert p1.valid?
+
+  end
+
+
+  def test_proc_validators
+    p = ProcValidatorsTestModel.new
+    p.name = "hi"
+    p.last_name = "hi"
+
+    refute p.valid?
+    assert p.errors[:name][:equal_to_test]
+    assert p.errors[:last_name][:equal_to_test_2]
+
+    p.name = "test"
+    p.last_name = "test 2"
+
+    assert p.valid?
+  end
+end

From 74ea47defc7f6260b045a6c6997bbe6a59c7bf62 Mon Sep 17 00:00:00 2001
From: Syphax bouazzouni <gs_bouazzouni@esi.dz>
Date: Tue, 5 Dec 2023 12:14:56 +0100
Subject: [PATCH 148/168] Merge to master - Release 2.3.5  - Optimize order by
 and filters queries  (#35)

* add tests for the new dsl to write property data types

* append the property :type values to the :enforce array

* update solution mapper to support multilingual

* update solution mapper to support multilingual

* fix typo ( name )

* add validators tests file

* add validator interface module

* implement data_type validator

* migrate existence validator to the new DSL

* migrate uniqueness validator to the new DSL

* implement object_type validator with the new DSL

* migrate range validator to the new DSL

* refactor the enforce module to use the new validators implementation

* force to regenerate the id when we update related attribute (named_with)

* require the validators implementation

* update existence validator to not accept empty to_s  objects

* update exist? test

* add symmetric validator tests for no_list and list cases

* implement symmetric validator

* move re used methods to the parent class

* update symmetric code and error message

* add distinct of validator tests

* implement distinct_of validator

* add superior_equal_to validator tests

* extract property method to ValidatorBase class

* implement superior_equal_to validator

* add inverse of validator tests

* implement inverse_of validator

* use the class method property in distinct of

* add proc validator tests

* add instance proc validators

* fix call_proc validator to test if the returned values are correct

* add model_with_yaml_scheme test

* implement YAMLScheme module

* use YAMLScheme module in Settings module

* use platform lang and code refacto

* filter by lang in properties

* do some refactoring

* add unmapped_get to goo resources

* update lang filter module to support requested_lang and portal_lang

* use the new lang filter module in the solution_mapper

* remove the usage of the old lang filter module in map_attributes

* add request language global variable

* fix datatype check for list values

* remove old unused test if clause

* for no unmapped values cast them to object before sending

* for resource unmapped_set merge new value if an array

* prevent add_object_to_model if no_lang and previous value exist

* move from the mapper  lang_filter related code to  lang_filter module

* move internal lang filter module methods to private section

* add request_store gem to save request language globally

* save requested  language in model_load options

* force requested_lang and portal_langs to be upcase and symbol

* change methodes/vars names

* get the last item in objects instead of passing the current object

* Revert "get the last item in objects instead of passing the current object"

This reverts commit 996922a9dfae06da9a7214e1322e1d403d3f1b39.

* handle this case where values is nil in save_model_values

* handle the casf of nil values for  the SuperiorEqualTo validator

* add onUpdate callback tests

* implement enforce_callback to run an attribute callback

* move the attribute default callback to the save method

* implement onUpdate DSL in the ressource settings

* call to the attributes onUpdate callback in the save method

* in validators bring attribute if needed

* make superior_equal_to works for list attributes

* add email validator test

* implement email validator

* add filters patterns to select variables

* make make regex filter no-case sensitive

* if requested_lang = 'all' return all

* support select multilanguage

* show the values with their corresponding language

* use @attributes_to_translate

* change methode name

* remove platform languages

* add complex_order_by unit test

* refactor query_builder to extract internal_variables as instance variable

* update order_by to work for joined patterns (object attributes)

* downcase lang key

* Fix the issue of undefined 'id'  of the language filter module

* Show literal attribute if we requested all the languages

* Use portal language by default in the language filter module

* group unmapped properties by lang

* Feature: group unmapped properties by language  (#38)

* group unmapped properties by lang

* downcase language keys of unmapped properties

---------

Co-authored-by: Syphax bouazzouni <gs_bouazzouni@esi.dz>

* assert that pre in an array in get_value_object

* add label to attributes_to_translate

* update define_method

* update solution mapper

* update get_preload_value

* Feature: Support multi lingual -  add show_language argument to the attributes getters (#39)

* update define_method

* update solution mapper

* update get_preload_value

* fix save_model_values if unmmaped condition

* fix getters for list attributes  to not take only the first value

* remove the languages hash for the unmapped if not a mutli langual asked

* move some language helper from the mapper to the lang_filter module

* move @requested_lang variable from the mapper to the lang_filter module

* remove no more used @attributes_to_translate variable in lang_filter

* fix save_model_values method to not save RDF:Literal object but a string

* remove  not used method in lang filter module

* refecator and rename some methods of the lang_filter module

* use the new name of the lang filter methods in the solution mapper

* replace the getters argument to show languages from :show_all_languages to :show_languages: true

* catch transform_values of unmapped if it is nil

* change the getters show_all_languages argument from to include_languages

* make the map_attributes handle the option showing all the languages

* fix order by an attribute that is already filtered

* don't add the filtered variables to the select clause of the query

* add filters patterns to select variables

* fix pagination with order_by with filter that returns empty pages for 4store

* include the in the select variables filtered variables

* optimize pagination query by not re-doing the filters and order in the include query

---------

Co-authored-by: HADDAD Zineddine <hz_haddad@esi.dz>
---
 lib/goo/base/resource.rb                      |   2 +-
 lib/goo/base/where.rb                         |   5 +
 lib/goo/sparql/mixins/solution_lang_filter.rb |   4 +-
 lib/goo/sparql/query_builder.rb               |  87 +-
 lib/goo/sparql/solutions_mapper.rb            | 878 +++++++++---------
 lib/goo/validators/enforce.rb                 |   2 +
 .../validators/implementations/data_type.rb   |  27 +-
 lib/goo/validators/implementations/email.rb   |  22 +
 .../implementations/superior_equal_to.rb      |   4 +-
 lib/goo/validators/validator.rb               |   2 +
 test/test_validators.rb                       |  11 +-
 test/test_where.rb                            |  13 +
 12 files changed, 569 insertions(+), 488 deletions(-)
 create mode 100644 lib/goo/validators/implementations/email.rb

diff --git a/lib/goo/base/resource.rb b/lib/goo/base/resource.rb
index 26ac5859..02709f5e 100644
--- a/lib/goo/base/resource.rb
+++ b/lib/goo/base/resource.rb
@@ -137,7 +137,7 @@ def unmmaped_to_array
       end
 
       def unmapped(*args)
-        @unmapped.transform_values do  |language_values|
+        @unmapped&.transform_values do  |language_values|
           self.class.not_show_all_languages?(language_values, args) ?  language_values.values.flatten: language_values
         end
       end
diff --git a/lib/goo/base/where.rb b/lib/goo/base/where.rb
index 5bc0fa8c..81cd26ce 100644
--- a/lib/goo/base/where.rb
+++ b/lib/goo/base/where.rb
@@ -209,6 +209,11 @@ def process_query_intl(count=false)
 
         options_load[:ids] = ids if ids
         models_by_id = {}
+        if @page_i && (options_load[:models].length > 0)
+          options_load.delete(:filters)
+          options_load.delete(:order_by)
+        end
+
         if (@page_i && options_load[:models].length > 0) ||
             (!@page_i && (@count.nil? || @count > 0))
           models_by_id = Goo::SPARQL::Queries.model_load(options_load)
diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index 8980dcdc..49d62c8e 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -25,8 +25,8 @@ def fill_models_with_all_languages(models_by_id)
         end
 
 
-        def set_model_value(model, predicate, values)
-          set_value(model, predicate, values) do
+        def set_model_value(model, predicate, values, value)
+          set_value(model, predicate, value) do
             model.send("#{predicate}=", values, on_load: true)
           end
         end
diff --git a/lib/goo/sparql/query_builder.rb b/lib/goo/sparql/query_builder.rb
index 40e888d0..31880859 100644
--- a/lib/goo/sparql/query_builder.rb
+++ b/lib/goo/sparql/query_builder.rb
@@ -25,21 +25,18 @@ def build_select_query(ids, variables, graphs, patterns,
                              query_options, properties_to_include)
 
         patterns = graph_match(@collection, @graph_match, graphs, @klass, patterns, query_options, @unions)
-
-        aggregate_projections, aggregate_vars,
-          variables, optional_patterns = get_aggregate_vars(@aggregate, @collection, graphs,
-                                                            @klass, @unions, variables)
-
-        @order_by, variables, optional_patterns = init_order_by(@count, @klass, @order_by, optional_patterns, variables,patterns, query_options, graphs)
         variables, patterns = add_some_type_to_id(patterns, query_options, variables)
-
+        aggregate_projections, aggregate_vars, variables, optional_patterns = get_aggregate_vars(@aggregate, @collection, graphs, @klass, @unions, variables)
         query_filter_str, patterns, optional_patterns, filter_variables =
           filter_query_strings(@collection, graphs, @klass, optional_patterns, patterns, @query_filters)
+        @order_by, variables, optional_patterns = init_order_by(@count, @klass, @order_by, optional_patterns, variables,patterns, query_options, graphs)
+        order_by_str, order_variables = order_by_string
+
 
         variables = [] if @count
         variables.delete :some_type
 
-        select_distinct(variables, aggregate_projections, filter_variables)
+        select_distinct(variables, aggregate_projections, filter_variables, order_variables)
           .from(graphs)
           .where(patterns)
           .union_bind_in_where(properties_to_include)
@@ -55,7 +52,10 @@ def build_select_query(ids, variables, graphs, patterns,
         @query.union(*@unions) unless @unions.empty?
 
         ids_filter(ids) if ids
-        order_by if @order_by
+
+
+        @query.order_by(*order_by_str) if @order_by
+
 
         put_query_aggregate_vars(aggregate_vars) if aggregate_vars
         count if @count
@@ -117,16 +117,17 @@ def put_query_aggregate_vars(aggregate_vars)
         self
       end
 
-      def order_by
-        order_by_str = @order_by.map do |attr, order|
+      def order_by_string
+        order_variables = []
+        order_str = @order_by&.map do |attr, order|
           if order.is_a?(Hash)
             sub_attr, order = order.first
-            attr = @internal_variables_map[sub_attr]
+            attr =  @internal_variables_map.select{ |internal_var, attr_var| attr_var.eql?({attr => sub_attr}) || attr_var.eql?(sub_attr)}.keys.last
           end
+          order_variables << attr
           "#{order.to_s.upcase}(?#{attr})"
         end
-        @query.order_by(*order_by_str)
-        self
+        [order_str,order_variables]
       end
 
       def from(graphs)
@@ -141,10 +142,11 @@ def from(graphs)
         self
       end
 
-      def select_distinct(variables, aggregate_projections, filter_variables)
+      def select_distinct(variables, aggregate_variables, filter_variables, order_variables)
         select_vars = variables.dup
-        reject_aggregations_from_vars(select_vars, aggregate_projections) if aggregate_projections
-        select_vars = (select_vars + filter_variables).uniq  if @page # Fix for 4store pagination with a filter
+        reject_aggregations_from_vars(select_vars, aggregate_variables) if aggregate_variables
+        # Fix for 4store pagination with a filter https://github.com/ontoportal-lirmm/ontologies_api/issues/25
+        select_vars = (select_vars + filter_variables + order_variables).uniq  if @page
         @query = @query.select(*select_vars).distinct(true)
         self
       end
@@ -165,23 +167,24 @@ def ids_filter(ids)
       def patterns_for_match(klass, attr, value, graphs, patterns, unions,
                              internal_variables, subject = :id, in_union = false,
                              in_aggregate = false, query_options = {}, collection = nil)
+        new_internal_var = value
         if value.respond_to?(:each) || value.instance_of?(Symbol)
           next_pattern = value.instance_of?(Array) ? value.first : value
 
           #for filters
           next_pattern = { next_pattern => [] } if next_pattern.instance_of?(Symbol)
 
-          value = "internal_join_var_#{internal_variables.length}".to_sym
+          new_internal_var = "internal_join_var_#{internal_variables.length}".to_sym
           if in_aggregate
-            value = "#{attr}_agg_#{in_aggregate}".to_sym
+            new_internal_var = "#{attr}_agg_#{in_aggregate}".to_sym
           end
-          internal_variables << value
-          @internal_variables_map[attr] = value
+          internal_variables << new_internal_var
+          @internal_variables_map[new_internal_var] = value.empty? ? attr : {attr => value}
         end
 
         add_rules(attr, klass, query_options)
         graph, pattern =
-          query_pattern(klass, attr, value: value, subject: subject, collection: collection)
+          query_pattern(klass, attr, value: new_internal_var, subject: subject, collection: collection)
         if pattern
           if !in_union
             patterns << pattern
@@ -194,7 +197,7 @@ def patterns_for_match(klass, attr, value, graphs, patterns, unions,
           range = klass.range(attr)
           next_pattern.each do |next_attr, next_value|
             patterns_for_match(range, next_attr, next_value, graphs,
-                               patterns, unions, internal_variables, subject = value,
+                               patterns, unions, internal_variables, subject = new_internal_var,
                                in_union, in_aggregate, collection = collection)
           end
         end
@@ -270,15 +273,35 @@ def init_order_by(count, klass, order_by, optional_patterns, variables, patterns
           order_by.each do |attr, direction|
 
             if direction.is_a?(Hash)
+              # TODO this part can be improved/refactored, the complexity was added because order by don't work
+              # if the pattern is in the mandatory ones (variable `patterns`)
+              # and optional (variable `optional_patterns`) at the same time
               sub_attr, direction = direction.first
               graph_match_iteration = Goo::Base::PatternIteration.new(Goo::Base::Pattern.new({attr => [sub_attr]}))
               old_internal = internal_variables.dup
+              old_patterns = optional_patterns.dup
+
               walk_pattern(klass, graph_match_iteration, graphs, optional_patterns, @unions, internal_variables, in_aggregate = false, query_options, @collection)
-              variables << (internal_variables - old_internal).last
+              new_variables = (internal_variables - old_internal)
+              internal_variables.delete(new_variables)
+              new_patterns = optional_patterns - old_patterns
+              already_existent_pattern = patterns.select{|x| x[1].eql?(new_patterns.last[1])}.first
+
+              if already_existent_pattern
+                already_existent_variable = already_existent_pattern[2]
+                optional_patterns = old_patterns
+                key = @internal_variables_map.select{|key, value|  key.eql?(new_variables.last)}.keys.first
+                @internal_variables_map[key] = (already_existent_variable || new_variables.last) if key
+
+                #variables << already_existent_variable
+              else
+                #variables <<  new_variables.last
+              end
+
             else
               quad = query_pattern(klass, attr)
               optional_patterns << quad[1]
-              variables << attr
+              #variables << attr
             end
 
             #patterns << quad[1]
@@ -325,7 +348,12 @@ def query_filter_sparql(klass, filter, filter_patterns, filter_graphs,
           end
           filter_var = inspected_patterns[filter_pattern_match]
 
-          unless filter_operation.value.instance_of?(Goo::Filter)
+          if filter_operation.value.instance_of?(Goo::Filter)
+            filter_operations << "#{sparql_op_string(filter_operation.operator)}"
+            query_filter_sparql(klass, filter_operation.value, filter_patterns,
+                                filter_graphs, filter_operations,
+                                internal_variables, inspected_patterns, collection)
+          else
             case filter_operation.operator
             when  :unbound
               filter_operations << "!BOUND(?#{filter_var.to_s})"
@@ -349,11 +377,6 @@ def query_filter_sparql(klass, filter, filter_patterns, filter_graphs,
                   " #{value.to_ntriples}")
             end
 
-          else
-            filter_operations << "#{sparql_op_string(filter_operation.operator)}"
-            query_filter_sparql(klass, filter_operation.value, filter_patterns,
-                                filter_graphs, filter_operations,
-                                internal_variables, inspected_patterns, collection)
           end
         end
       end
@@ -399,7 +422,7 @@ def add_some_type_to_id(patterns, query_options, variables)
       end
 
       def internal_variables
-        @internal_variables_map.values
+        @internal_variables_map.keys
       end
     end
   end
diff --git a/lib/goo/sparql/solutions_mapper.rb b/lib/goo/sparql/solutions_mapper.rb
index 879c1ff7..64d258d5 100644
--- a/lib/goo/sparql/solutions_mapper.rb
+++ b/lib/goo/sparql/solutions_mapper.rb
@@ -1,439 +1,439 @@
-module Goo
-  module SPARQL
-    class SolutionMapper
-      BNODES_TUPLES = Struct.new(:id, :attribute)
-
-      def initialize(aggregate_projections, bnode_extraction, embed_struct,
-                      incl_embed, klass_struct, models_by_id,
-                     properties_to_include, unmapped, variables, ids, options)
-
-        @aggregate_projections = aggregate_projections
-        @bnode_extraction = bnode_extraction
-        @embed_struct = embed_struct
-        @incl_embed = incl_embed
-        @klass_struct = klass_struct
-        @models_by_id = models_by_id
-        @properties_to_include = properties_to_include
-        @unmapped = unmapped
-        @variables = variables
-        @ids = ids
-        @klass = options[:klass]
-        @read_only = options[:read_only]
-        @incl = options[:include]
-        @count = options[:count]
-        @collection = options[:collection]
-        @options = options
-      end
-      
-      def map_each_solutions(select)
-        found = Set.new
-        objects_new = {}
-        list_attributes = Set.new(@klass.attributes(:list))
-        all_attributes = Set.new(@klass.attributes(:all))
-
-        @lang_filter = Goo::SPARQL::Solution::LanguageFilter.new(requested_lang: @options[:requested_lang].to_s, unmapped: @unmapped,
-           list_attributes: list_attributes)
-        
-        select.each_solution do |sol|
-          
-          next if sol[:some_type] && @klass.type_uri(@collection) != sol[:some_type]
-          return sol[:count_var].object if @count
-
-          found.add(sol[:id])
-          id = sol[:id]
-
-          create_model(id)
-
-          if @bnode_extraction
-            add_bnode_to_model(sol)
-            next
-          end
-
-          if @unmapped
-            add_unmapped_to_model(sol)
-            next
-          end
-
-          if @aggregate_projections
-            add_aggregations_to_model(sol)
-            next
-          end
-
-          predicate = sol[:attributeProperty].to_s.to_sym
-
-          next if predicate.nil? || !all_attributes.include?(predicate)
-
-          object = sol[:attributeObject]
-
-          # bnodes
-          if bnode_id?(object, predicate)
-            objects_new = bnode_id_tuple(id, object, objects_new, predicate)
-            next
-          end
-
-          objects, objects_new = get_value_object(id, objects_new, object, list_attributes, predicate)
-          add_object_to_model(id, objects, predicate)
-        end
-      
-        # for this moment we are not going to enrich models , maybe we will use it if the results are empty  
-        @lang_filter.fill_models_with_all_languages(@models_by_id)
-
-        init_unloaded_attributes(found, list_attributes)
-
-        return @models_by_id if @bnode_extraction
-
-        model_set_collection_attributes(@models_by_id, objects_new)
-
-        # remove from models_by_id elements that were not touched
-        @models_by_id.select! { |k, _m| found.include?(k) }
-
-        models_set_all_persistent(@models_by_id) unless @read_only
-
-        # next level of embed attributes
-        include_embed_attributes(@incl_embed, objects_new) if @incl_embed && !@incl_embed.empty?
-
-        # bnodes
-        blank_nodes = objects_new.select { |id, _obj| id.is_a?(RDF::Node) && id.anonymous? }
-        include_bnodes(blank_nodes, @models_by_id) unless blank_nodes.empty?
-
-        models_unmapped_to_array(@models_by_id) if @unmapped
-        
-       
-        @models_by_id
-      end
-
-      private
-
-      def init_unloaded_attributes(found, list_attributes)
-        return if @incl.nil?
-
-        # Here we are setting to nil all attributes that have been included but not found in the triplestore
-        found.uniq.each do |model_id|
-          m = @models_by_id[model_id]
-          @incl.each do |attr_to_incl|
-            is_handler = m.respond_to?(:handler?) && m.class.handler?(attr_to_incl)
-            next if attr_to_incl.to_s.eql?('unmapped') || is_handler
-
-            loaded = m.respond_to?('loaded_attributes') && m.loaded_attributes.include?(attr_to_incl)
-            is_list = list_attributes.include?(attr_to_incl)
-            is_struct = m.respond_to?(:klass)
-
-            # Go through all models queried
-            if is_struct
-              m[attr_to_incl] = [] if is_list && m[attr_to_incl].nil?
-            elsif is_list && (!loaded || m.send(attr_to_incl.to_s).nil?)
-              m.send("#{attr_to_incl}=", [], on_load: true)
-            elsif !loaded && !is_list && m.respond_to?("#{attr_to_incl}=")
-              m.send("#{attr_to_incl}=", nil, on_load: true)
-            end
-          end
-        end
-      end
-
-      def get_value_object(id, objects_new, object, list_attributes, predicate)
-        object = object.object if object && !(object.is_a? RDF::URI)
-        range_for_v = @klass.range(predicate)
-       
-
-        if object.is_a?(RDF::URI) && (predicate != :id) && !range_for_v.nil?
-          if objects_new.include?(object)
-            object = objects_new[object]
-          elsif !range_for_v.inmutable?
-            pre_val = get_preload_value(id, object, predicate)
-            object, objects_new = if !@read_only
-                                    preloaded_or_new_object(object, objects_new, pre_val, predicate)
-                                  else
-                                    # depedent read only
-                                    preloaded_or_new_struct(object, objects_new, pre_val, predicate)
-                                  end
-          else
-            object = range_for_v.find(object).first
-          end
-        end
-
-        if list_attributes.include?(predicate)
-          pre = @klass_struct ? @models_by_id[id][predicate] : @models_by_id[id].instance_variable_get("@#{predicate}")
-
-          if object.nil?
-            object = pre.nil? ? [] : pre
-          else
-            object = pre.nil? ? [object] : (pre.dup << object)
-            object.uniq!
-          end
-
-        end
-        [object, objects_new]
-      end
-
-      def add_object_to_model(id, objects, predicate)
-
-        if @models_by_id[id].respond_to?(:klass)
-          @models_by_id[id][predicate] = objects unless objects.nil? && !@models_by_id[id][predicate].nil?
-        elsif !@models_by_id[id].class.handler?(predicate) &&
-              !(objects.nil? && !@models_by_id[id].instance_variable_get("@#{predicate}").nil?) &&
-              predicate != :id
-          @lang_filter.set_model_value(@models_by_id[id], predicate, objects)
-        end
-      end
-
-      def get_preload_value(id, object, predicate)
-        pre_val = nil
-        if predicate_preloaded?(id, predicate)
-          pre_val = preloaded_value(id, predicate)          
-          pre_val = pre_val.select { |x|  x.id  == object  }.first if pre_val.is_a?(Array)
-        end
-        pre_val
-      end
-
-      def preloaded_or_new_object(object, objects_new, pre_val, predicate)
-        object = pre_val || @klass.range_object(predicate, object)
-        objects_new[object.id] = object
-        [object, objects_new]
-      end
-
-      def preloaded_or_new_struct(object, objects_new, pre_val, predicate)
-        struct = pre_val || @embed_struct[predicate].new
-        struct.id = object
-        struct.klass = @klass.range(predicate)
-        objects_new[struct.id] = struct
-        [struct, objects_new]
-      end
-
-      def preloaded_value(id, predicate)
-        if !@read_only
-          @models_by_id[id].instance_variable_get("@#{predicate}")
-
-        else
-          @models_by_id[id][predicate]
-        end
-      end
-
-      def predicate_preloaded?(id, predicate)
-        @models_by_id[id] &&
-          (@models_by_id[id].respond_to?(:klass) || @models_by_id[id].loaded_attributes.include?(predicate))
-      end
-
-      def bnode_id?(object, predicate)
-        object.is_a?(RDF::Node) && object.anonymous? && @incl.include?(predicate)
-      end
-
-      def bnode_id_tuple(id, object, objects_new, predicate)
-        range = @klass.range(predicate)
-        objects_new[object] = BNODES_TUPLES.new(id, predicate) if range.respond_to?(:new)
-        objects_new
-      end
-
-      def add_bnode_to_model(sol)
-        id = sol[:id]
-        struct = create_struct(@bnode_extraction, @models_by_id, sol, @variables)
-        @models_by_id[id].send("#{@bnode_extraction}=", struct)
-      end
-
-      def create_model(id)
-        @models_by_id[id] = create_class_model(id, @klass, @klass_struct) unless @models_by_id.include?(id)
-      end
-
-
-      def create_struct(bnode_extraction, models_by_id, sol, variables)
-        list_attributes = Set.new(@klass.attributes(:list))
-        struct = @klass.range(bnode_extraction).new
-        variables.each do |v|
-          next if v == :id
-
-          svalue = sol[v]
-          struct[v] = svalue.is_a?(RDF::Node) ? svalue : svalue.object
-        end
-        if list_attributes.include?(bnode_extraction)
-          pre = models_by_id[sol[:id]].instance_variable_get("@#{bnode_extraction}")
-          pre = pre ? (pre.dup << struct) : [struct]
-          struct = pre
-        end
-        struct
-      end
-
-      def create_class_model(id, klass, klass_struct)
-        klass_model = klass_struct ? klass_struct.new : klass.new
-        klass_model.id = id
-        klass_model.persistent = true unless klass_struct
-        klass_model.klass = klass if klass_struct
-        klass_model
-      end
-
-      def models_unmapped_to_array(models_by_id)
-        models_by_id.each do |_idm, m|
-          @lang_filter.models_unmapped_to_array(m)
-        end
-      end
-
-
-      def is_multiple_langs?
-        return true if @requested_lang.is_a?(Array) || @requested_lang.eql?(:ALL)
-        false
-      end
-
-      def include_bnodes(bnodes, models_by_id)
-        # group by attribute
-        attrs = bnodes.map { |_x, y| y.attribute }.uniq
-        attrs.each do |attr|
-          struct = @klass.range(attr)
-
-          # bnodes that are in a range of goo ground models
-          # for example parents and children in LD class models
-          # we skip this cases for the moment
-          next if struct.respond_to?(:model_name)
-
-          bnode_attrs = struct.new.to_h.keys
-          ids = bnodes.select { |_x, y| y.attribute == attr }.map { |_x, y| y.id }
-          @klass.where.models(models_by_id.select { |x, _y| ids.include?(x) }.values)
-                .in(@collection)
-                .include(bnode: { attr => bnode_attrs }).all
-        end
-      end
-
-      def include_embed_attributes(incl_embed, objects_new)
-        incl_embed.each do |attr, next_attrs|
-          # anything to join ?
-          attr_range = @klass.range(attr)
-          next if attr_range.nil?
-
-          range_objs = objects_new.select do |_id, obj|
-            obj.instance_of?(attr_range) || (obj.respond_to?(:klass) && obj[:klass] == attr_range)
-          end.values
-          next if range_objs.empty?
-
-          range_objs.uniq!
-          query = attr_range.where.models(range_objs).in(@collection).include(*next_attrs)
-          query = query.read_only if @read_only
-          query.all
-        end
-      end
-
-      def models_set_all_persistent(models_by_id)
-        return unless @ids
-
-        models_by_id.each do |_k, m|
-          m.persistent = true
-        end
-      end
-
-      def model_set_collection_attributes(models_by_id, objects_new)
-        collection_value = get_collection_value
-        return unless collection_value
-
-        collection_attribute = @klass.collection_opts
-        models_by_id.each do |_id, m|
-          m.send("#{collection_attribute}=", collection_value)
-        end
-        objects_new.each do |_id, obj_new|
-          if obj_new.respond_to?(:klass)
-            collection_attribute = obj_new[:klass].collection_opts
-            obj_new[collection_attribute] = collection_value
-          elsif obj_new.class.respond_to?(:collection_opts) &&
-                obj_new.class.collection_opts.instance_of?(Symbol)
-            collection_attribute = obj_new.class.collection_opts
-            obj_new.send("#{collection_attribute}=", collection_value)
-          end
-        end
-      end
-
-      def get_collection_value
-        collection_value = nil
-        if @klass.collection_opts.instance_of?(Symbol)
-          collection_value = @collection.first if @collection.is_a?(Array) && (@collection.length == 1)
-          collection_value = @collection if @collection.respond_to? :id
-        end
-        collection_value
-      end
-
-      def object_to_array(id, klass_struct, models_by_id, object, predicate)
-        pre = if klass_struct
-                models_by_id[id][predicate]
-              else
-                models_by_id[id].instance_variable_get("@#{predicate}")
-              end
-        if object.nil? && pre.nil?
-          object = []
-        elsif object.nil? && !pre.nil?
-          object = pre
-        elsif object
-          object = !pre ? [object] : (pre.dup << object)
-          object.uniq!
-        end
-        object
-      end
-
-      def dependent_model_creation(embed_struct, id, models_by_id, object, objects_new, v, options)
-        read_only = options[:read_only]
-        if object.is_a?(RDF::URI) && v != :id
-          range_for_v = @klass.range(v)
-          if range_for_v
-            if objects_new.include?(object)
-              object = objects_new[object]
-            elsif !range_for_v.inmutable?
-              pre_val = get_pre_val(id, models_by_id, object, v, read_only)
-              object = get_object_from_range(pre_val, embed_struct, object, objects_new, v, options)
-            else
-              object = range_for_v.find(object).first
-            end
-          end
-        end
-        object
-      end
-
-      def get_object_from_range(pre_val, embed_struct, object, objects_new, predicate)
-        range_for_v = @klass.range(predicate)
-        if !@read_only
-          object = pre_val || @klass.range_object(predicate, object)
-          objects_new[object.id] = object
-        else
-          # depedent read only
-          struct = pre_val || embed_struct[predicate].new
-          struct.id = object
-          struct.klass = range_for_v
-          objects_new[struct.id] = struct
-          object = struct
-        end
-        object
-      end
-
-      def get_pre_val(id, models_by_id, object, predicate)
-        pre_val = nil
-        if models_by_id[id] &&
-           ((models_by_id[id].respond_to?(:klass) && models_by_id[id]) ||
-             models_by_id[id].loaded_attributes.include?(predicate))
-          pre_val = if !@read_only
-                      models_by_id[id].instance_variable_get("@#{predicate}")
-                    else
-                      models_by_id[id][predicate]
-                    end
-
-          pre_val = pre_val.select { |x| x.id == object }.first if pre_val.is_a?(Array)
-        end
-        pre_val
-      end
-
-      def add_unmapped_to_model(sol)
-        predicate = sol[:attributeProperty].to_s.to_sym
-        return unless @properties_to_include[predicate]
-
-        id = sol[:id]
-        value = sol[:attributeObject]
-
-        @lang_filter.set_unmapped_value(@models_by_id[id], @properties_to_include[predicate][:uri], value)
-      end
-
-      def add_aggregations_to_model(sol)
-        id = sol[:id]
-        @aggregate_projections&.each do |aggregate_key, aggregate_val|
-          if @models_by_id[id].respond_to?(:add_aggregate)
-            @models_by_id[id].add_aggregate(aggregate_val[1], aggregate_val[0], sol[aggregate_key].object)
-          else
-            (@models_by_id[id].aggregates ||= []) << Goo::Base::AGGREGATE_VALUE.new(aggregate_val[1],
-                                                                                    aggregate_val[0],
-                                                                                    sol[aggregate_key].object)
-          end
-        end
-      end
-    end
-  end
-end
+module Goo
+  module SPARQL
+    class SolutionMapper
+      BNODES_TUPLES = Struct.new(:id, :attribute)
+
+      def initialize(aggregate_projections, bnode_extraction, embed_struct,
+                      incl_embed, klass_struct, models_by_id,
+                     properties_to_include, unmapped, variables, ids, options)
+
+        @aggregate_projections = aggregate_projections
+        @bnode_extraction = bnode_extraction
+        @embed_struct = embed_struct
+        @incl_embed = incl_embed
+        @klass_struct = klass_struct
+        @models_by_id = models_by_id
+        @properties_to_include = properties_to_include
+        @unmapped = unmapped
+        @variables = variables
+        @ids = ids
+        @klass = options[:klass]
+        @read_only = options[:read_only]
+        @incl = options[:include]
+        @count = options[:count]
+        @collection = options[:collection]
+        @options = options
+      end
+      
+      def map_each_solutions(select)
+        found = Set.new
+        objects_new = {}
+        list_attributes = Set.new(@klass.attributes(:list))
+        all_attributes = Set.new(@klass.attributes(:all))
+
+        @lang_filter = Goo::SPARQL::Solution::LanguageFilter.new(requested_lang: @options[:requested_lang].to_s, unmapped: @unmapped,
+           list_attributes: list_attributes)
+        
+        select.each_solution do |sol|
+          
+          next if sol[:some_type] && @klass.type_uri(@collection) != sol[:some_type]
+          return sol[:count_var].object if @count
+
+          found.add(sol[:id])
+          id = sol[:id]
+
+          create_model(id)
+
+          if @bnode_extraction
+            add_bnode_to_model(sol)
+            next
+          end
+
+          if @unmapped
+            add_unmapped_to_model(sol)
+            next
+          end
+
+          if @aggregate_projections
+            add_aggregations_to_model(sol)
+            next
+          end
+
+          predicate = sol[:attributeProperty].to_s.to_sym
+
+          next if predicate.nil? || !all_attributes.include?(predicate)
+
+          object = sol[:attributeObject]
+
+          # bnodes
+          if bnode_id?(object, predicate)
+            objects_new = bnode_id_tuple(id, object, objects_new, predicate)
+            next
+          end
+
+          objects, objects_new = get_value_object(id, objects_new, object, list_attributes, predicate)
+          add_object_to_model(id, objects, object, predicate)
+        end
+      
+        # for this moment we are not going to enrich models , maybe we will use it if the results are empty  
+        @lang_filter.fill_models_with_all_languages(@models_by_id)
+
+        init_unloaded_attributes(found, list_attributes)
+
+        return @models_by_id if @bnode_extraction
+
+        model_set_collection_attributes(@models_by_id, objects_new)
+
+        # remove from models_by_id elements that were not touched
+        @models_by_id.select! { |k, _m| found.include?(k) }
+
+        models_set_all_persistent(@models_by_id) unless @read_only
+
+        # next level of embed attributes
+        include_embed_attributes(@incl_embed, objects_new) if @incl_embed && !@incl_embed.empty?
+
+        # bnodes
+        blank_nodes = objects_new.select { |id, _obj| id.is_a?(RDF::Node) && id.anonymous? }
+        include_bnodes(blank_nodes, @models_by_id) unless blank_nodes.empty?
+
+        models_unmapped_to_array(@models_by_id) if @unmapped
+        
+       
+        @models_by_id
+      end
+
+      private
+
+      def init_unloaded_attributes(found, list_attributes)
+        return if @incl.nil?
+
+        # Here we are setting to nil all attributes that have been included but not found in the triplestore
+        found.uniq.each do |model_id|
+          m = @models_by_id[model_id]
+          @incl.each do |attr_to_incl|
+            is_handler = m.respond_to?(:handler?) && m.class.handler?(attr_to_incl)
+            next if attr_to_incl.to_s.eql?('unmapped') || is_handler
+
+            loaded = m.respond_to?('loaded_attributes') && m.loaded_attributes.include?(attr_to_incl)
+            is_list = list_attributes.include?(attr_to_incl)
+            is_struct = m.respond_to?(:klass)
+
+            # Go through all models queried
+            if is_struct
+              m[attr_to_incl] = [] if is_list && m[attr_to_incl].nil?
+            elsif is_list && (!loaded || m.send(attr_to_incl.to_s).nil?)
+              m.send("#{attr_to_incl}=", [], on_load: true)
+            elsif !loaded && !is_list && m.respond_to?("#{attr_to_incl}=")
+              m.send("#{attr_to_incl}=", nil, on_load: true)
+            end
+          end
+        end
+      end
+
+      def get_value_object(id, objects_new, object, list_attributes, predicate)
+        object = object.object if object && !(object.is_a? RDF::URI)
+        range_for_v = @klass.range(predicate)
+       
+
+        if object.is_a?(RDF::URI) && (predicate != :id) && !range_for_v.nil?
+          if objects_new.include?(object)
+            object = objects_new[object]
+          elsif !range_for_v.inmutable?
+            pre_val = get_preload_value(id, object, predicate)
+            object, objects_new = if !@read_only
+                                    preloaded_or_new_object(object, objects_new, pre_val, predicate)
+                                  else
+                                    # depedent read only
+                                    preloaded_or_new_struct(object, objects_new, pre_val, predicate)
+                                  end
+          else
+            object = range_for_v.find(object).first
+          end
+        end
+
+        if list_attributes.include?(predicate)
+          pre = @klass_struct ? @models_by_id[id][predicate] : @models_by_id[id].instance_variable_get("@#{predicate}")
+
+          if object.nil?
+            object = pre.nil? ? [] : pre
+          else            
+            object = pre.nil? ? [object] : (Array(pre).dup << object)
+            object.uniq!
+          end
+
+        end
+        [object, objects_new]
+      end
+
+      def add_object_to_model(id, objects, current_obj, predicate)
+
+        if @models_by_id[id].respond_to?(:klass)
+          @models_by_id[id][predicate] = objects unless objects.nil? && !@models_by_id[id][predicate].nil?
+        elsif !@models_by_id[id].class.handler?(predicate) &&
+              !(objects.nil? && !@models_by_id[id].instance_variable_get("@#{predicate}").nil?) &&
+              predicate != :id
+          @lang_filter.set_model_value(@models_by_id[id], predicate, objects, current_obj)
+        end
+      end
+
+      def get_preload_value(id, object, predicate)
+        pre_val = nil
+        if predicate_preloaded?(id, predicate)
+          pre_val = preloaded_value(id, predicate)
+          pre_val = pre_val.select { |x| x.respond_to?(:id) && (x.id == object) }.first if pre_val.is_a?(Array)
+        end
+        pre_val
+      end
+
+      def preloaded_or_new_object(object, objects_new, pre_val, predicate)
+        object = pre_val || @klass.range_object(predicate, object)
+        objects_new[object.id] = object
+        [object, objects_new]
+      end
+
+      def preloaded_or_new_struct(object, objects_new, pre_val, predicate)
+        struct = pre_val || @embed_struct[predicate].new
+        struct.id = object
+        struct.klass = @klass.range(predicate)
+        objects_new[struct.id] = struct
+        [struct, objects_new]
+      end
+
+      def preloaded_value(id, predicate)
+        if !@read_only
+          @models_by_id[id].instance_variable_get("@#{predicate}")
+
+        else
+          @models_by_id[id][predicate]
+        end
+      end
+
+      def predicate_preloaded?(id, predicate)
+        @models_by_id[id] &&
+          (@models_by_id[id].respond_to?(:klass) || @models_by_id[id].loaded_attributes.include?(predicate))
+      end
+
+      def bnode_id?(object, predicate)
+        object.is_a?(RDF::Node) && object.anonymous? && @incl.include?(predicate)
+      end
+
+      def bnode_id_tuple(id, object, objects_new, predicate)
+        range = @klass.range(predicate)
+        objects_new[object] = BNODES_TUPLES.new(id, predicate) if range.respond_to?(:new)
+        objects_new
+      end
+
+      def add_bnode_to_model(sol)
+        id = sol[:id]
+        struct = create_struct(@bnode_extraction, @models_by_id, sol, @variables)
+        @models_by_id[id].send("#{@bnode_extraction}=", struct)
+      end
+
+      def create_model(id)
+        @models_by_id[id] = create_class_model(id, @klass, @klass_struct) unless @models_by_id.include?(id)
+      end
+
+
+      def create_struct(bnode_extraction, models_by_id, sol, variables)
+        list_attributes = Set.new(@klass.attributes(:list))
+        struct = @klass.range(bnode_extraction).new
+        variables.each do |v|
+          next if v == :id
+
+          svalue = sol[v]
+          struct[v] = svalue.is_a?(RDF::Node) ? svalue : svalue.object
+        end
+        if list_attributes.include?(bnode_extraction)
+          pre = models_by_id[sol[:id]].instance_variable_get("@#{bnode_extraction}")
+          pre = pre ? (pre.dup << struct) : [struct]
+          struct = pre
+        end
+        struct
+      end
+
+      def create_class_model(id, klass, klass_struct)
+        klass_model = klass_struct ? klass_struct.new : klass.new
+        klass_model.id = id
+        klass_model.persistent = true unless klass_struct
+        klass_model.klass = klass if klass_struct
+        klass_model
+      end
+
+      def models_unmapped_to_array(models_by_id)
+        models_by_id.each do |_idm, m|
+          @lang_filter.models_unmapped_to_array(m)
+        end
+      end
+
+
+      def is_multiple_langs?
+        return true if @requested_lang.is_a?(Array) || @requested_lang.eql?(:ALL)
+        false
+      end
+
+      def include_bnodes(bnodes, models_by_id)
+        # group by attribute
+        attrs = bnodes.map { |_x, y| y.attribute }.uniq
+        attrs.each do |attr|
+          struct = @klass.range(attr)
+
+          # bnodes that are in a range of goo ground models
+          # for example parents and children in LD class models
+          # we skip this cases for the moment
+          next if struct.respond_to?(:model_name)
+
+          bnode_attrs = struct.new.to_h.keys
+          ids = bnodes.select { |_x, y| y.attribute == attr }.map { |_x, y| y.id }
+          @klass.where.models(models_by_id.select { |x, _y| ids.include?(x) }.values)
+                .in(@collection)
+                .include(bnode: { attr => bnode_attrs }).all
+        end
+      end
+
+      def include_embed_attributes(incl_embed, objects_new)
+        incl_embed.each do |attr, next_attrs|
+          # anything to join ?
+          attr_range = @klass.range(attr)
+          next if attr_range.nil?
+
+          range_objs = objects_new.select do |_id, obj|
+            obj.instance_of?(attr_range) || (obj.respond_to?(:klass) && obj[:klass] == attr_range)
+          end.values
+          next if range_objs.empty?
+
+          range_objs.uniq!
+          query = attr_range.where.models(range_objs).in(@collection).include(*next_attrs)
+          query = query.read_only if @read_only
+          query.all
+        end
+      end
+
+      def models_set_all_persistent(models_by_id)
+        return unless @ids
+
+        models_by_id.each do |_k, m|
+          m.persistent = true
+        end
+      end
+
+      def model_set_collection_attributes(models_by_id, objects_new)
+        collection_value = get_collection_value
+        return unless collection_value
+
+        collection_attribute = @klass.collection_opts
+        models_by_id.each do |_id, m|
+          m.send("#{collection_attribute}=", collection_value)
+        end
+        objects_new.each do |_id, obj_new|
+          if obj_new.respond_to?(:klass)
+            collection_attribute = obj_new[:klass].collection_opts
+            obj_new[collection_attribute] = collection_value
+          elsif obj_new.class.respond_to?(:collection_opts) &&
+                obj_new.class.collection_opts.instance_of?(Symbol)
+            collection_attribute = obj_new.class.collection_opts
+            obj_new.send("#{collection_attribute}=", collection_value)
+          end
+        end
+      end
+
+      def get_collection_value
+        collection_value = nil
+        if @klass.collection_opts.instance_of?(Symbol)
+          collection_value = @collection.first if @collection.is_a?(Array) && (@collection.length == 1)
+          collection_value = @collection if @collection.respond_to? :id
+        end
+        collection_value
+      end
+
+      def object_to_array(id, klass_struct, models_by_id, object, predicate)
+        pre = if klass_struct
+                models_by_id[id][predicate]
+              else
+                models_by_id[id].instance_variable_get("@#{predicate}")
+              end
+        if object.nil? && pre.nil?
+          object = []
+        elsif object.nil? && !pre.nil?
+          object = pre
+        elsif object
+          object = !pre ? [object] : (pre.dup << object)
+          object.uniq!
+        end
+        object
+      end
+
+      def dependent_model_creation(embed_struct, id, models_by_id, object, objects_new, v, options)
+        read_only = options[:read_only]
+        if object.is_a?(RDF::URI) && v != :id
+          range_for_v = @klass.range(v)
+          if range_for_v
+            if objects_new.include?(object)
+              object = objects_new[object]
+            elsif !range_for_v.inmutable?
+              pre_val = get_pre_val(id, models_by_id, object, v, read_only)
+              object = get_object_from_range(pre_val, embed_struct, object, objects_new, v, options)
+            else
+              object = range_for_v.find(object).first
+            end
+          end
+        end
+        object
+      end
+
+      def get_object_from_range(pre_val, embed_struct, object, objects_new, predicate)
+        range_for_v = @klass.range(predicate)
+        if !@read_only
+          object = pre_val || @klass.range_object(predicate, object)
+          objects_new[object.id] = object
+        else
+          # depedent read only
+          struct = pre_val || embed_struct[predicate].new
+          struct.id = object
+          struct.klass = range_for_v
+          objects_new[struct.id] = struct
+          object = struct
+        end
+        object
+      end
+
+      def get_pre_val(id, models_by_id, object, predicate)
+        pre_val = nil
+        if models_by_id[id] &&
+           ((models_by_id[id].respond_to?(:klass) && models_by_id[id]) ||
+             models_by_id[id].loaded_attributes.include?(predicate))
+          pre_val = if !@read_only
+                      models_by_id[id].instance_variable_get("@#{predicate}")
+                    else
+                      models_by_id[id][predicate]
+                    end
+
+          pre_val = pre_val.select { |x| x.id == object }.first if pre_val.is_a?(Array)
+        end
+        pre_val
+      end
+
+      def add_unmapped_to_model(sol)
+        predicate = sol[:attributeProperty].to_s.to_sym
+        return unless @properties_to_include[predicate]
+
+        id = sol[:id]
+        value = sol[:attributeObject]
+
+        @lang_filter.set_unmapped_value(@models_by_id[id], @properties_to_include[predicate][:uri], value)
+      end
+
+      def add_aggregations_to_model(sol)
+        id = sol[:id]
+        @aggregate_projections&.each do |aggregate_key, aggregate_val|
+          if @models_by_id[id].respond_to?(:add_aggregate)
+            @models_by_id[id].add_aggregate(aggregate_val[1], aggregate_val[0], sol[aggregate_key].object)
+          else
+            (@models_by_id[id].aggregates ||= []) << Goo::Base::AGGREGATE_VALUE.new(aggregate_val[1],
+                                                                                    aggregate_val[0],
+                                                                                    sol[aggregate_key].object)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/goo/validators/enforce.rb b/lib/goo/validators/enforce.rb
index 3c90e204..d6f3816d 100644
--- a/lib/goo/validators/enforce.rb
+++ b/lib/goo/validators/enforce.rb
@@ -41,6 +41,8 @@ def enforce(inst,attr,value)
               check Goo::Validators::DataType, inst, attr, value, opt, Float
             when :symmetric
               check Goo::Validators::Symmetric, inst, attr, value, opt
+            when :email
+              check Goo::Validators::Email, inst, attr, value, opt
             when /^distinct_of_/
               check Goo::Validators::DistinctOf, inst, attr, value, opt, opt
             when /^superior_equal_to_/
diff --git a/lib/goo/validators/implementations/data_type.rb b/lib/goo/validators/implementations/data_type.rb
index 0ea65ab3..04f46d0c 100644
--- a/lib/goo/validators/implementations/data_type.rb
+++ b/lib/goo/validators/implementations/data_type.rb
@@ -29,16 +29,16 @@ def enforce_type(type, value)
         return true if value.nil?
 
         if type == :boolean
-          return self.enforce_type_boolean(value)
+          self.enforce_type_boolean(value)
         elsif type.eql?(:uri) || type.eql?(RDF::URI)
-          return  self.enforce_type_uri(value)
+          self.enforce_type_uri(value)
         elsif type.eql?(:uri) || type.eql?(Array)
-          return value.is_a? Array
+          value.is_a? Array
         else
           if value.is_a? Array
-            return value.select{|x| !x.is_a?(type)}.empty?
+             value.select{|x| !x.is_a?(type)}.empty?
           else
-            return value.is_a? type
+             value.is_a? type
           end
         end
 
@@ -47,19 +47,28 @@ def enforce_type(type, value)
       def enforce_type_uri(value)
         return true  if value.nil?
 
-        value.is_a?(RDF::URI) && value.valid?
+        if value.kind_of? Array
+          value.select { |x| !is_a_uri?(x) }.empty?
+        else
+          is_a_uri?(value)
+        end
+
       end
 
       def enforce_type_boolean(value)
         if value.kind_of? Array
-          return value.select { |x| !is_a_boolean?(x) }.empty?
+          value.select { |x| !is_a_boolean?(x) }.empty?
         else
-          return  is_a_boolean?(value)
+          is_a_boolean?(value)
         end
       end
 
       def is_a_boolean?(value)
-        return (value.class == TrueClass) || (value.class == FalseClass)
+         (value.class == TrueClass) || (value.class == FalseClass)
+      end
+
+      def is_a_uri?(value)
+        value.is_a?(RDF::URI) && value.valid?
       end
     end
   end
diff --git a/lib/goo/validators/implementations/email.rb b/lib/goo/validators/implementations/email.rb
new file mode 100644
index 00000000..f8405714
--- /dev/null
+++ b/lib/goo/validators/implementations/email.rb
@@ -0,0 +1,22 @@
+module Goo
+  module Validators
+    class Email < ValidatorBase
+      include Validator
+      EMAIL_REGEXP = /\A[\w+\-.]+@[a-z\d\-]+(\.[a-z\d\-]+)*\.[a-z]+\z/i
+      key :email
+
+      error_message ->(obj) {
+        if @value.kind_of? Array
+          return "All values in attribute `#{@attr}` must be a valid emails"
+        else
+          return "Attribute `#{@attr}` with the value `#{@value}` must be a valid email"
+
+        end
+      }
+
+      validity_check -> (obj) do
+        @value.nil? || @value.match?(EMAIL_REGEXP)
+      end
+    end
+  end
+end
\ No newline at end of file
diff --git a/lib/goo/validators/implementations/superior_equal_to.rb b/lib/goo/validators/implementations/superior_equal_to.rb
index 91508f30..46676794 100644
--- a/lib/goo/validators/implementations/superior_equal_to.rb
+++ b/lib/goo/validators/implementations/superior_equal_to.rb
@@ -12,9 +12,9 @@ class SuperiorEqualTo < ValidatorBase
       validity_check -> (obj) do
         target_values = self.class.attr_value(@property, @inst)
 
-        return true if target_values.empty?
+        return true if target_values.nil? || target_values.empty?
 
-        return @value >= target_values.first
+        return  Array(@value).all? {|v| v.nil? || target_values.all?{|t_v| v >= t_v}}
       end
 
       def initialize(inst, attr, value, key)
diff --git a/lib/goo/validators/validator.rb b/lib/goo/validators/validator.rb
index e7db80a4..2536f985 100644
--- a/lib/goo/validators/validator.rb
+++ b/lib/goo/validators/validator.rb
@@ -75,6 +75,8 @@ def equivalent_value?(object1, object2)
         end
 
         def attr_value(attr, object)
+          object.bring attr if object.respond_to?(:bring?) && object.bring?(attr)
+
           Array(object.send(attr))
         end
 
diff --git a/test/test_validators.rb b/test/test_validators.rb
index 8795fccf..5110da80 100644
--- a/test/test_validators.rb
+++ b/test/test_validators.rb
@@ -12,6 +12,8 @@ class Person < Goo::Base::Resource
   attribute :birth_date, enforce: [ :date_time ]
   attribute :male, enforce: [:boolean]
   attribute :social, enforce: [:uri]
+  attribute :email, enforce: [:email]
+  attribute :socials, enforce: [:uri, :list]
   attribute :weight, enforce: [:float]
   attribute :friends, enforce: [Person, :list]
 end
@@ -143,9 +145,9 @@ def test_datatype_validators
     p.birth_date = 100
     p.male =  "ok"
     p.social =  100
+    p.socials =  [100]
     p.weight =  100
-
-
+    p.email = "test@test"
     #wrong types are not valid
     refute p.valid?
     assert p.errors[:last_name][:string]
@@ -155,14 +157,17 @@ def test_datatype_validators
     assert p.errors[:birth_date][:date_time]
     assert p.errors[:male][:boolean]
     assert p.errors[:social][:uri]
+    assert p.errors[:email][:email]
 
     p.last_name =  "hello"
     p.multiple_values = [22,11]
     p.one_number = 12
     p.birth_date = DateTime.parse('1978-01-01')
     p.male =  true
-    p.social =  RDF::URI.new('https://test.com/')
+    p.social = RDF::URI.new('https://test.com/')
+    p.socials = [RDF::URI.new('https://test.com/'), RDF::URI.new('https://test.com/')]
     p.weight =  100.0
+    p.email = "test@test.hi.com"
     #good types are  valid
     assert p.valid?
   end
diff --git a/test/test_where.rb b/test/test_where.rb
index 30d933e3..c80fed33 100644
--- a/test/test_where.rb
+++ b/test/test_where.rb
@@ -262,6 +262,19 @@ def test_embed_two_levels
     end
   end
 
+  def test_paging_with_filter_order
+
+    f = Goo::Filter.new(:birth_date) > DateTime.parse('1978-01-03')
+    total_count = Student.where.filter(f).count
+    page_1 = Student.where.include(:name, :birth_date).page(1, total_count - 1).filter(f).order_by(name: :asc).to_a
+    refute_empty page_1
+    assert page_1.next?
+    page_2 = Student.where.include(:name, :birth_date).page(page_1.next_page, total_count - 1).filter(f).order_by(name: :asc).to_a
+
+
+    refute_empty page_2
+    assert_equal total_count, page_1.size + page_2.size
+  end
 
   def test_unique_object_references
 

From 8b826abefabf7531a9b6871b5245a986f3032b35 Mon Sep 17 00:00:00 2001
From: Syphax bouazzouni <gs_bouazzouni@esi.dz>
Date: Thu, 14 Dec 2023 00:11:52 +0100
Subject: [PATCH 149/168] fix map attribute for  properties that contain a mix
 of string and URIs

---
 lib/goo/base/resource.rb | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/lib/goo/base/resource.rb b/lib/goo/base/resource.rb
index 02709f5e..2eaf17ad 100644
--- a/lib/goo/base/resource.rb
+++ b/lib/goo/base/resource.rb
@@ -379,17 +379,17 @@ def self.map_attributes(inst,equivalent_predicates=nil, include_languages: false
           attr_uri = klass.attribute_uri(attr,inst.collection).to_s
           if unmapped_string_keys.include?(attr_uri.to_s) ||
             (equivalent_predicates && equivalent_predicates.include?(attr_uri))
-            object = nil
             if !unmapped_string_keys.include?(attr_uri)
-              equivalent_predicates[attr_uri].each do |eq_attr|
-                if object.nil? and !unmapped_string_keys[eq_attr].nil?
-                  object = unmapped_string_keys[eq_attr].dup
-                else
-                  if object.is_a?Array
-                    object.concat(unmapped_string_keys[eq_attr]) if !unmapped_string_keys[eq_attr].nil?
-                  end
+              object = Array(equivalent_predicates[attr_uri].map { |eq_attr| unmapped_string_keys[eq_attr] }).flatten.compact
+              if include_languages && [RDF::URI, Hash].all?{|c| object.map(&:class).include?(c)}
+                object = object.reduce({})  do |all, new_v|
+                  new_v =  { none: [new_v] } if new_v.is_a?(RDF::URI)
+                  all.merge(new_v) {|_, a, b| a + b }
                 end
+              elsif include_languages
+                object = object.first
               end
+
               if object.nil?
                 inst.send("#{attr}=", list_attrs.include?(attr) ? [] : nil, on_load: true)
                 next

From fb89f9c4a679798c83d0e205a2194690834de4cd Mon Sep 17 00:00:00 2001
From: Syphax bouazzouni <gs_bouazzouni@esi.dz>
Date: Mon, 29 Jan 2024 13:48:43 +0100
Subject: [PATCH 150/168] simplify the test configuration init

---
 .gitignore                     |  2 +
 Rakefile                       |  1 -
 config/config.rb.sample        | 23 ++++++++++
 lib/goo.rb                     | 11 +----
 lib/goo/config/config.rb       | 78 ++++++++++++++++++++++++++++++++++
 test/app/bioportal.rb          |  2 -
 test/app/models.rb             |  2 -
 test/app/test_app.rb           |  2 -
 test/console.rb                |  1 -
 test/test_basic_persistence.rb |  2 -
 test/test_cache.rb             |  3 --
 test/test_case.rb              | 30 +------------
 test/test_chunks_write.rb      |  2 -
 test/test_collections.rb       |  2 -
 test/test_dsl_settings.rb      |  1 -
 test/test_enum.rb              |  2 -
 test/test_index.rb             |  2 -
 test/test_inmutable.rb         |  2 -
 test/test_inverse.rb           |  2 -
 test/test_model_complex.rb     |  2 -
 test/test_name_with.rb         |  2 -
 test/test_namespaces.rb        |  2 -
 test/test_read_only.rb         |  2 -
 test/test_schemaless.rb        |  2 +-
 test/test_search.rb            |  2 -
 test/test_validators.rb        |  1 -
 test/test_where.rb             |  3 --
 27 files changed, 106 insertions(+), 80 deletions(-)
 create mode 100644 config/config.rb.sample
 create mode 100644 lib/goo/config/config.rb

diff --git a/.gitignore b/.gitignore
index f887556d..5dcefa7c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -25,3 +25,5 @@ doc/
 
 .idea/*
 projectFilesBackup/*
+
+config/config.rb
\ No newline at end of file
diff --git a/Rakefile b/Rakefile
index e593ddce..42ddf39d 100644
--- a/Rakefile
+++ b/Rakefile
@@ -107,7 +107,6 @@ end
 desc "Console for working with data"
 task :console do
   require_relative "test/test_case"
-  GooTest.configure_goo
   binding.pry
 end
 
diff --git a/config/config.rb.sample b/config/config.rb.sample
new file mode 100644
index 00000000..12abdccb
--- /dev/null
+++ b/config/config.rb.sample
@@ -0,0 +1,23 @@
+Goo.config do |config|
+  # 4store
+  config.goo_backend_name    = '4store'
+  config.goo_port            = 8080
+  config.goo_host            = 'localhost'
+  config.goo_path_query      = '/sparql/'
+  config.goo_path_data       = '/data/'
+  config.goo_path_update     = '/update/'
+
+  # AllegroGraph
+  # config.goo_backend_name  = 'AG'
+  # config.goo_port          = 10035
+  # config.goo_host          = 'localhost'
+  # config.goo_path_query    = "/repositories/ontoportal"
+  # config.goo_path_data     = "/repositories/ontoportal/statements/"
+  # config.goo_path_update   = "/repositories/ontoportal/statements/"
+
+  config.search_server_url   = 'http://localhost:8983/solr/term_search_core1'
+  config.redis_host          = 'localhost'
+  config.redis_port          = 6379
+  config.bioportal_namespace = 'http://data.bioontology.org/'
+  config.queries_debug       = false
+end
\ No newline at end of file
diff --git a/lib/goo.rb b/lib/goo.rb
index ff0e6279..e34d7b96 100644
--- a/lib/goo.rb
+++ b/lib/goo.rb
@@ -12,6 +12,7 @@
 require 'uuid'
 require "cube"
 
+require_relative "goo/config/config"
 require_relative "goo/sparql/sparql"
 require_relative "goo/search/search"
 require_relative "goo/base/base"
@@ -110,16 +111,6 @@ def self.add_sparql_backend(name, *opts)
     @@sparql_backends.freeze
   end
 
-  def self.test_reset
-    if @@sparql_backends[:main][:query].url.to_s["localhost"].nil?
-      raise Exception, "only for testing"
-    end
-    @@sparql_backends[:main][:query]=Goo::SPARQL::Client.new("http://localhost:9000/sparql/",
-                 {protocol: "1.1", "Content-Type" => "application/x-www-form-urlencoded",
-                   read_timeout: 300,
-                  redis_cache: @@redis_client })
-  end
-
   def self.main_lang
     @@main_lang
   end
diff --git a/lib/goo/config/config.rb b/lib/goo/config/config.rb
new file mode 100644
index 00000000..ff51e8b7
--- /dev/null
+++ b/lib/goo/config/config.rb
@@ -0,0 +1,78 @@
+require 'ostruct'
+
+module Goo
+  extend self
+  attr_reader :settings
+
+  @settings = OpenStruct.new
+  @settings_run = false
+
+  def config(&block)
+    return if @settings_run
+    @settings_run = true
+
+    yield @settings if block_given?
+
+    # Set defaults
+    @settings.goo_backend_name    ||= ENV['GOO_BACKEND_NAME'] || '4store'
+    @settings.goo_port            ||= ENV['GOO_PORT'] || 9000
+    @settings.goo_host            ||= ENV['GOO_HOST'] || 'localhost'
+    @settings.goo_path_query      ||= ENV['GOO_PATH_QUERY'] || '/sparql/'
+    @settings.goo_path_data       ||= ENV['GOO_PATH_DATA'] || '/data/'
+    @settings.goo_path_update     ||= ENV['GOO_PATH_UPDATE'] || '/update/'
+    @settings.search_server_url   ||= ENV['SEARCH_SERVER_URL'] || 'http://localhost:8983/solr/term_search_core1'
+    @settings.redis_host          ||= ENV['REDIS_HOST'] || 'localhost'
+    @settings.redis_port          ||= ENV['REDIS_PORT'] || 6379
+    @settings.bioportal_namespace ||= ENV['BIOPORTAL_NAMESPACE'] || 'http://data.bioontology.org/'
+    @settings.queries_debug       ||= ENV['QUERIES_DEBUG'] || false
+    @settings.slice_loading_size  ||= ENV['GOO_SLICES'] || 500
+    puts "(GOO) >> Using RDF store (#{@settings.goo_backend_name}) #{@settings.goo_host}:#{@settings.goo_port}#{@settings.goo_path_query}"
+    puts "(GOO) >> Using term search server at #{@settings.search_server_url}"
+    puts "(GOO) >> Using Redis instance at #{@settings.redis_host}:#{@settings.redis_port}"
+
+    connect_goo
+  end
+
+  def connect_goo
+    begin
+      Goo.configure do |conf|
+        conf.queries_debug(@settings.queries_debug)
+        conf.add_sparql_backend(:main,
+                                backend_name: @settings.goo_backend_name,
+                                query: "http://#{@settings.goo_host}:#{@settings.goo_port}#{@settings.goo_path_query}",
+                                data: "http://#{@settings.goo_host}:#{@settings.goo_port}#{@settings.goo_path_data}",
+                                update: "http://#{@settings.goo_host}:#{@settings.goo_port}#{@settings.goo_path_update}",
+                                options: { rules: :NONE })
+        conf.add_search_backend(:main, service: @settings.search_server_url)
+        conf.add_redis_backend(host: @settings.goo_redis_host, port: @settings.goo_redis_port)
+
+        conf.add_namespace(:omv, RDF::Vocabulary.new("http://omv.org/ontology/"))
+        conf.add_namespace(:skos, RDF::Vocabulary.new("http://www.w3.org/2004/02/skos/core#"))
+        conf.add_namespace(:owl, RDF::Vocabulary.new("http://www.w3.org/2002/07/owl#"))
+        conf.add_namespace(:rdfs, RDF::Vocabulary.new("http://www.w3.org/2000/01/rdf-schema#"))
+        conf.add_namespace(:goo, RDF::Vocabulary.new("http://goo.org/default/"), default = true)
+        conf.add_namespace(:metadata, RDF::Vocabulary.new("http://goo.org/metadata/"))
+        conf.add_namespace(:foaf, RDF::Vocabulary.new("http://xmlns.com/foaf/0.1/"))
+        conf.add_namespace(:rdf, RDF::Vocabulary.new("http://www.w3.org/1999/02/22-rdf-syntax-ns#"))
+        conf.add_namespace(:tiger, RDF::Vocabulary.new("http://www.census.gov/tiger/2002/vocab#"))
+        conf.add_namespace(:nemo, RDF::Vocabulary.new("http://purl.bioontology.org/NEMO/ontology/NEMO_annotation_properties.owl#"))
+        conf.add_namespace(:bioportal, RDF::Vocabulary.new(@settings.bioportal_namespace))
+        conf.use_cache = false
+      end
+    rescue StandardError => e
+      abort("EXITING: Goo cannot connect to triplestore and/or search server:\n  #{e}\n#{e.backtrace.join("\n")}")
+    end
+  end
+
+  def self.test_reset
+    if @@sparql_backends[:main][:query].url.to_s["localhost"].nil?
+      raise Exception, "only for testing"
+    end
+    @@sparql_backends[:main][:query] = Goo::SPARQL::Client.new("http://#{@settings.goo_host}:#{@settings.goo_port}#{@settings.goo_path_query}",
+                                                             {protocol: "1.1", "Content-Type" => "application/x-www-form-urlencoded",
+                                                              read_timeout: 300,
+                                                              redis_cache: @@redis_client })
+  end
+
+
+end
diff --git a/test/app/bioportal.rb b/test/app/bioportal.rb
index 2cef2c2b..2b3c57b3 100644
--- a/test/app/bioportal.rb
+++ b/test/app/bioportal.rb
@@ -4,8 +4,6 @@
 require_relative '../test_case'
 require_relative './query_profiler'
 
-GooTest.configure_goo
-
 module Test
   module BioPortal 
     class Ontology < Goo::Base::Resource
diff --git a/test/app/models.rb b/test/app/models.rb
index 5aeb2a2e..876b70df 100644
--- a/test/app/models.rb
+++ b/test/app/models.rb
@@ -1,7 +1,5 @@
 require_relative '../test_case'
 
-GooTest.configure_goo
-
 module Test
   module Models
 
diff --git a/test/app/test_app.rb b/test/app/test_app.rb
index 26a88d60..4d444ad4 100644
--- a/test/app/test_app.rb
+++ b/test/app/test_app.rb
@@ -1,6 +1,4 @@
 require_relative '../test_case'
 require_relative 'bioportal'
 
-GooTest.configure_goo
-
 binding.pry
diff --git a/test/console.rb b/test/console.rb
index e64d4adf..39d19aa2 100644
--- a/test/console.rb
+++ b/test/console.rb
@@ -1,5 +1,4 @@
 require_relative "../lib/goo.rb"
 require_relative "./test_case.rb"
 
-GooTest.configure_goo
 binding.pry
diff --git a/test/test_basic_persistence.rb b/test/test_basic_persistence.rb
index 0cafdfbd..665a5d60 100644
--- a/test/test_basic_persistence.rb
+++ b/test/test_basic_persistence.rb
@@ -1,7 +1,5 @@
 require_relative 'test_case'
 
-GooTest.configure_goo
-
 module Dep
   class Ontology < Goo::Base::Resource
     model :ontology, name_with: :name
diff --git a/test/test_cache.rb b/test/test_cache.rb
index aecbdef3..be01f518 100644
--- a/test/test_cache.rb
+++ b/test/test_cache.rb
@@ -1,7 +1,4 @@
 require_relative 'test_case'
-
-GooTest.configure_goo
-
 require_relative 'models'
 
 class TestCache < MiniTest::Unit::TestCase
diff --git a/test/test_case.rb b/test/test_case.rb
index 82d9c50e..7073acd5 100644
--- a/test/test_case.rb
+++ b/test/test_case.rb
@@ -20,6 +20,7 @@
 MiniTest::Unit.autorun
 
 require_relative "../lib/goo.rb"
+require_relative '../config/config'
 
 class GooTest
 
@@ -59,35 +60,6 @@ def _run_suite(suite, type)
 
   MiniTest::Unit.runner = GooTest::Unit.new
 
-  def self.configure_goo
-    if not Goo.configure?
-      Goo.configure do |conf|
-        conf.add_redis_backend(host: "localhost")
-        conf.add_namespace(:omv, RDF::Vocabulary.new("http://omv.org/ontology/"))
-        conf.add_namespace(:skos, RDF::Vocabulary.new("http://www.w3.org/2004/02/skos/core#"))
-        conf.add_namespace(:owl, RDF::Vocabulary.new("http://www.w3.org/2002/07/owl#"))
-        conf.add_namespace(:rdfs, RDF::Vocabulary.new("http://www.w3.org/2000/01/rdf-schema#"))
-        conf.add_namespace(:goo, RDF::Vocabulary.new("http://goo.org/default/"),default=true)
-        conf.add_namespace(:metadata, RDF::Vocabulary.new("http://goo.org/metadata/"))
-        conf.add_namespace(:foaf, RDF::Vocabulary.new("http://xmlns.com/foaf/0.1/"))
-        conf.add_namespace(:rdf, RDF::Vocabulary.new("http://www.w3.org/1999/02/22-rdf-syntax-ns#"))
-        conf.add_namespace(:tiger, RDF::Vocabulary.new("http://www.census.gov/tiger/2002/vocab#"))
-        conf.add_namespace(:bioportal, RDF::Vocabulary.new("http://data.bioontology.org/"))
-        conf.add_namespace(:nemo, RDF::Vocabulary.new("http://purl.bioontology.org/NEMO/ontology/NEMO_annotation_properties.owl#"))
-        conf.add_sparql_backend(
-          :main, 
-          backend_name: "4store",
-          query: "http://localhost:9000/sparql/",
-          data: "http://localhost:9000/data/",
-          update: "http://localhost:9000/update/",
-          options: { rules: :NONE }
-        )
-        conf.add_search_backend(:main, service: "http://localhost:8983/solr/term_search_core1")
-        conf.use_cache = false
-      end
-    end
-  end
-
   def self.triples_for_subject(resource_id)
     rs = Goo.sparql_query_client.query("SELECT * WHERE { #{resource_id.to_ntriples} ?p ?o . }")
     count = 0
diff --git a/test/test_chunks_write.rb b/test/test_chunks_write.rb
index 4f0a8676..e2f82439 100644
--- a/test/test_chunks_write.rb
+++ b/test/test_chunks_write.rb
@@ -1,7 +1,5 @@
 require_relative 'test_case'
 
-GooTest.configure_goo
-
 module TestChunkWrite
   ONT_ID = "http:://example.org/data/nemo"
   ONT_ID_EXTRA = "http:://example.org/data/nemo/extra"
diff --git a/test/test_collections.rb b/test/test_collections.rb
index 2177c669..65d1f46d 100644
--- a/test/test_collections.rb
+++ b/test/test_collections.rb
@@ -1,7 +1,5 @@
 require_relative 'test_case'
 
-GooTest.configure_goo
-
 #collection on attribute
 class Issue < Goo::Base::Resource
   model :issue, collection: :owner, name_with: :description
diff --git a/test/test_dsl_settings.rb b/test/test_dsl_settings.rb
index 9a8f03df..52b0ac78 100644
--- a/test/test_dsl_settings.rb
+++ b/test/test_dsl_settings.rb
@@ -1,6 +1,5 @@
 require_relative 'test_case'
 
-GooTest.configure_goo
 
 class NewPersonModel < Goo::Base::Resource
   model :person_model_new, name_with: :name
diff --git a/test/test_enum.rb b/test/test_enum.rb
index db41c343..eaf13af2 100644
--- a/test/test_enum.rb
+++ b/test/test_enum.rb
@@ -1,7 +1,5 @@
 require_relative 'test_case'
 
-GooTest.configure_goo
-
 module TestEnum
   VALUES = ["uploaded","removed","archived"]
 
diff --git a/test/test_index.rb b/test/test_index.rb
index 4d781973..bf4b8937 100644
--- a/test/test_index.rb
+++ b/test/test_index.rb
@@ -1,8 +1,6 @@
 require_relative 'test_case'
 require_relative './app/models'
 
-GooTest.configure_goo
-
 module TestIndex
 
   class TestSchemaless < MiniTest::Unit::TestCase
diff --git a/test/test_inmutable.rb b/test/test_inmutable.rb
index 9d6037c0..0b1a8c2a 100644
--- a/test/test_inmutable.rb
+++ b/test/test_inmutable.rb
@@ -1,7 +1,5 @@
 require_relative 'test_case'
 
-GooTest.configure_goo
-
 module TestInmutable
   class Status < Goo::Base::Resource
     model :status, :inmutable, name_with: :code
diff --git a/test/test_inverse.rb b/test/test_inverse.rb
index e926a572..2fbb4479 100644
--- a/test/test_inverse.rb
+++ b/test/test_inverse.rb
@@ -1,7 +1,5 @@
 require_relative 'test_case'
 
-GooTest.configure_goo
-
 class Task < Goo::Base::Resource
   model :task, name_with: :description
   attribute :description, enforce: [ :existence, :unique]
diff --git a/test/test_model_complex.rb b/test/test_model_complex.rb
index 8f904d8b..0ee493e9 100644
--- a/test/test_model_complex.rb
+++ b/test/test_model_complex.rb
@@ -1,7 +1,5 @@
 require_relative 'test_case'
 
-GooTest.configure_goo
-
 module TestComplex
 
 class Submission < Goo::Base::Resource
diff --git a/test/test_name_with.rb b/test/test_name_with.rb
index 7ba4df42..c2f226a4 100644
--- a/test/test_name_with.rb
+++ b/test/test_name_with.rb
@@ -1,7 +1,5 @@
 require_relative 'test_case'
 
-GooTest.configure_goo
-
 class NameWith < Goo::Base::Resource
   model :name_with, name_with: lambda { |s| id_generator(s) } 
   attribute :name, enforce: [ :existence, :string, :unique ]
diff --git a/test/test_namespaces.rb b/test/test_namespaces.rb
index 78ba9a93..6c4bddc0 100644
--- a/test/test_namespaces.rb
+++ b/test/test_namespaces.rb
@@ -1,7 +1,5 @@
 require_relative 'test_case'
 
-GooTest.configure_goo
-
 class NamespacesModel < Goo::Base::Resource
   model :namespaces, namespace: :rdfs, name_with: :name
   attribute :name, enforce: [ :existence, :string, :unique ], namespace: :skos
diff --git a/test/test_read_only.rb b/test/test_read_only.rb
index 268f7c86..9855decf 100644
--- a/test/test_read_only.rb
+++ b/test/test_read_only.rb
@@ -1,8 +1,6 @@
 require_relative 'test_case'
 require_relative 'test_where'
 
-GooTest.configure_goo
-
 module TestReadOnly
 
   class TestReadOnlyWithStruct < TestWhere
diff --git a/test/test_schemaless.rb b/test/test_schemaless.rb
index f95a17d5..9e67e6e9 100644
--- a/test/test_schemaless.rb
+++ b/test/test_schemaless.rb
@@ -1,6 +1,6 @@
 require_relative 'test_case'
 
-GooTest.configure_goo
+module TestSchemaless
 
 module TestSChemaless
 
diff --git a/test/test_search.rb b/test/test_search.rb
index 433dee86..180062d1 100644
--- a/test/test_search.rb
+++ b/test/test_search.rb
@@ -1,7 +1,5 @@
 require_relative 'test_case'
 
-GooTest.configure_goo
-
 module TestSearch
 
   class TermSearch < Goo::Base::Resource
diff --git a/test/test_validators.rb b/test/test_validators.rb
index 5110da80..e5c3a9fe 100644
--- a/test/test_validators.rb
+++ b/test/test_validators.rb
@@ -1,6 +1,5 @@
 require_relative 'test_case'
 
-GooTest.configure_goo
 require_relative 'models'
 
 class Person < Goo::Base::Resource
diff --git a/test/test_where.rb b/test/test_where.rb
index c80fed33..a9087855 100644
--- a/test/test_where.rb
+++ b/test/test_where.rb
@@ -1,7 +1,4 @@
 require_relative 'test_case'
-
-GooTest.configure_goo
-
 require_relative 'models'
 
 class TestWhere < MiniTest::Unit::TestCase

From c06f10083bb63194d448db3f045f6df889824e49 Mon Sep 17 00:00:00 2001
From: Syphax bouazzouni <gs_bouazzouni@esi.dz>
Date: Mon, 29 Jan 2024 13:51:59 +0100
Subject: [PATCH 151/168] add docker based tests rake task to run test against
 4s, ag, gb, vo

---
 docker-compose.yml                | 101 +++++++++++--
 rakelib/docker_based_test.rake    | 107 +++++++++++++
 test/data/graphdb-repo-config.ttl |  33 ++++
 test/data/graphdb-test-load.nt    |   0
 test/data/virtuoso.init           | 240 ++++++++++++++++++++++++++++++
 5 files changed, 467 insertions(+), 14 deletions(-)
 create mode 100644 rakelib/docker_based_test.rake
 create mode 100644 test/data/graphdb-repo-config.ttl
 create mode 100644 test/data/graphdb-test-load.nt
 create mode 100644 test/data/virtuoso.init

diff --git a/docker-compose.yml b/docker-compose.yml
index fe29bc33..fd832341 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,16 +1,5 @@
-version: '3'
-
 services:
-  4store:
-    image: bde2020/4store
-    ports:
-      - 9000:9000
-    command: >
-      bash -c "4s-backend-setup --segments 4 ontoportal_kb
-      && 4s-backend ontoportal_kb
-      && 4s-httpd -D -s-1 -p 9000 ontoportal_kb"
-
-  redis:
+  redis-ut:
     image: redis
     ports:
       - 6379:6379
@@ -20,7 +9,91 @@ services:
       timeout: 3s
       retries: 30
 
-  solr:
-    image: ontoportal/solr-ut:0.1
+  solr-ut:
+    image: ontoportal/solr-ut:0.0.2
     ports:
       - 8983:8983
+    healthcheck:
+      test: ["CMD-SHELL", "curl -sf http://localhost:8983/solr/term_search_core1/admin/ping?wt=json | grep -iq '\"status\":\"OK\"}'  || exit 1"]
+      start_period: 10s
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  agraph-ut:
+    image: franzinc/agraph:v8.0.0.rc1
+    platform: linux/amd64
+    environment:
+      - AGRAPH_SUPER_USER=test
+      - AGRAPH_SUPER_PASSWORD=xyzzy
+    shm_size: 1g
+    ports:
+      #      - 10035:10035
+      - 10000-10035:10000-10035
+    volumes:
+      - agdata:/agraph/data
+      #      - ./agraph/etc:/agraph/etc
+    command: >
+      bash -c "/agraph/bin/agraph-control --config /agraph/etc/agraph.cfg start
+      ; agtool repos create ontoportal_test --supersede
+      ; agtool users add anonymous
+      ; agtool users grant anonymous root:ontoportal_test:rw
+      ; tail -f /agraph/data/agraph.log"
+    # healthcheck:
+    #   test: ["CMD-SHELL", "curl -sf http://127.0.0.1:10035/repositories/ontoportal_test/status | grep -iqE '(^running|^lingering)' || exit 1"]
+    #   start_period: 10s
+    #   interval: 10s
+    #   timeout: 5s
+    #   retries: 5
+    profiles:
+      - ag
+
+  4store-ut:
+    image: bde2020/4store
+    platform: linux/amd64
+    ports:
+      - 9000:9000
+    command: >
+      bash -c "4s-backend-setup --segments 4 ontoportal_kb
+      && 4s-backend ontoportal_kb
+      && 4s-httpd -D -s-1 -p 9000 ontoportal_kb"
+    profiles:
+      - fs
+  virtuoso-ut:
+    image: tenforce/virtuoso:virtuoso7.2.5
+    platform: linux/amd64
+    environment:
+      - SPARQL_UPDATE=true
+    ports:
+      - 1111:1111
+      - 8890:8890
+
+    profiles:
+      - vo
+
+  graphdb:
+    image: ontotext/graphdb:10.3.3
+    platform: linux/amd64
+    privileged: true
+    environment:
+      GDB_HEAP_SIZE: 5G
+      GDB_JAVA_OPTS: >-
+        -Xms5g -Xmx5g
+    ports:
+      - 7200:7200
+      - 7300:7300
+    volumes:
+      - ./test/data/graphdb-repo-config.ttl:/opt/graphdb/dist/configs/templates/data/graphdb-repo-config.ttl
+      - ./test/data/graphdb-test-load.nt:/opt/graphdb/dist/configs/templates/data/graphdb-test-load.nt
+
+    entrypoint: >
+      bash -c " importrdf load -f -c /opt/graphdb/dist/configs/templates/data/graphdb-repo-config.ttl -m parallel /opt/graphdb/dist/configs/templates/data/graphdb-test-load.nt ; graphdb -Ddefault.min.distinct.threshold=3000 "
+    profiles:
+      - gb
+
+volumes:
+  agdata:
+
+
+
+
diff --git a/rakelib/docker_based_test.rake b/rakelib/docker_based_test.rake
new file mode 100644
index 00000000..f3fa6c1a
--- /dev/null
+++ b/rakelib/docker_based_test.rake
@@ -0,0 +1,107 @@
+# Rake tasks for running unit tests with backend services running as docker containers
+
+desc 'Run unit tests with docker based backend'
+namespace :test do
+  namespace :docker do
+    task :up do
+      system("docker compose up -d") || abort("Unable to start docker containers")
+    end
+    task :down do
+      #system("docker compose --profile fs --profile ag stop")
+      #system("docker compose --profile fs --profile ag kill")
+    end
+    desc "run tests with docker AG backend"
+    task :ag do
+      ENV["GOO_BACKEND_NAME"]="AG"
+      ENV["GOO_PORT"]="10035"
+      ENV["GOO_PATH_QUERY"]="/repositories/ontoportal_test"
+      ENV["GOO_PATH_DATA"]="/repositories/ontoportal_test/statements"
+      ENV["GOO_PATH_UPDATE"]="/repositories/ontoportal_test/statements"
+      ENV["COMPOSE_PROFILES"]="ag"
+      Rake::Task["test:docker:up"].invoke
+      # AG takes some time to start and create databases/accounts
+      # TODO: replace system curl command with native ruby code
+      unless system("curl -sf http://127.0.0.1:10035/repositories/ontoportal_test/status | grep -iqE '(^running|^lingering)' || exit 1")
+        printf("waiting for AllegroGraph container to initialize")
+        sec = 0
+        until system("curl -sf http://127.0.0.1:10035/repositories/ontoportal_test/status | grep -iqE '(^running|^lingering)' || exit 1") do
+          sleep(1)
+          printf(".")
+          sec += 1
+        end
+      end
+      puts
+      system("docker compose ps") # TODO: remove after GH actions troubleshooting is complete
+      Rake::Task["test"].invoke
+      Rake::Task["test:docker:down"].invoke
+    end
+
+    desc "run tests with docker 4store backend"
+    task :fs do
+      ENV["GOO_PORT"]="9000"
+      ENV["COMPOSE_PROFILES"]='fs'
+      Rake::Task["test:docker:up"].invoke
+      Rake::Task["test"].invoke
+      Rake::Task["test:docker:down"].invoke
+    end
+
+    desc "run tests with docker Virtuoso backend"
+    task :vo do
+      ENV["GOO_BACKEND_NAME"]="VO"
+      ENV["GOO_PORT"]="8890"
+      ENV["GOO_PATH_QUERY"]="/sparql"
+      ENV["GOO_PATH_DATA"]="/sparql"
+      ENV["GOO_PATH_UPDATE"]="/sparql"
+      ENV["COMPOSE_PROFILES"]="vo"
+      Rake::Task["test:docker:up"].invoke
+      #
+      unless system("curl -sf  http://localhost:8890/sparql  || exit 1")
+        printf("waiting for Virtuoso container to initialize")
+        sec = 0
+        until system("curl -sf  http://localhost:8890/sparql || exit 1") do
+          sleep(1)
+          printf(".")
+          sec += 1
+          if sec > 30
+            system("docker compose logs virtuoso-ut")
+            abort("  Virtuoso container hasn't initialized properly")
+          end
+        end
+      end
+      Rake::Task["test"].invoke
+      Rake::Task["test:docker:down"].invoke
+    end
+
+
+    desc "run tests with docker GraphDb backend"
+    task :gb do
+      ENV["GOO_BACKEND_NAME"]="GB"
+      ENV["GOO_PORT"]="7200"
+      ENV["GOO_PATH_QUERY"]="/repositories/ontoportal"
+      ENV["GOO_PATH_DATA"]="/repositories/ontoportal/statements"
+      ENV["GOO_PATH_UPDATE"]="/repositories/ontoportal/statements"
+      ENV["COMPOSE_PROFILES"]="gb"
+      Rake::Task["test:docker:up"].invoke
+
+      #system("docker compose cp ./test/data/graphdb-repo-config.ttl graphdb:/opt/graphdb/dist/configs/templates/graphdb-repo-config.ttl")
+      #system("docker compose cp ./test/data/graphdb-test-load.nt graphdb:/opt/graphdb/dist/configs/templates/graphdb-test-load.nt")
+      #system('docker compose exec graphdb sh -c "importrdf load -f -c /opt/graphdb/dist/configs/templates/graphdb-repo-config.ttl -m parallel /opt/graphdb/dist/configs/templates/graphdb-test-load.nt ;"')
+      unless system("curl -sf http://localhost:7200/repositories || exit 1")
+        printf("waiting for Graphdb container to initialize")
+        sec = 0
+        until system("curl -sf http://localhost:7200/repositories || exit 1") do
+          sleep(1)
+          printf(".")
+          sec += 1
+          if sec > 30
+            system("docker compose logs graphdb")
+            abort("  Graphdb container hasn't initialized properly")
+          end
+        end
+      end
+      Rake::Task["test"].invoke
+      Rake::Task["test:docker:down"].invoke
+    end
+
+  end
+end
diff --git a/test/data/graphdb-repo-config.ttl b/test/data/graphdb-repo-config.ttl
new file mode 100644
index 00000000..9200da9a
--- /dev/null
+++ b/test/data/graphdb-repo-config.ttl
@@ -0,0 +1,33 @@
+@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
+@prefix rep: <http://www.openrdf.org/config/repository#> .
+@prefix sail: <http://www.openrdf.org/config/sail#> .
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
+
+<#ontoportal> a rep:Repository;
+  rep:repositoryID "ontoportal";
+  rep:repositoryImpl [
+      rep:repositoryType "graphdb:SailRepository";
+      <http://www.openrdf.org/config/repository/sail#sailImpl> [
+          <http://www.ontotext.com/trree/owlim#base-URL> "http://example.org/owlim#";
+          <http://www.ontotext.com/trree/owlim#check-for-inconsistencies> "false";
+          <http://www.ontotext.com/trree/owlim#defaultNS> "";
+          <http://www.ontotext.com/trree/owlim#disable-sameAs> "true";
+          <http://www.ontotext.com/trree/owlim#enable-context-index> "false";
+          <http://www.ontotext.com/trree/owlim#enable-literal-index> "true";
+          <http://www.ontotext.com/trree/owlim#enablePredicateList> "true";
+          <http://www.ontotext.com/trree/owlim#entity-id-size> "32";
+          <http://www.ontotext.com/trree/owlim#entity-index-size> "10000000";
+          <http://www.ontotext.com/trree/owlim#imports> "";
+          <http://www.ontotext.com/trree/owlim#in-memory-literal-properties> "true";
+          <http://www.ontotext.com/trree/owlim#owlim-license> "";
+          <http://www.ontotext.com/trree/owlim#query-limit-results> "0";
+          <http://www.ontotext.com/trree/owlim#query-timeout> "0";
+          <http://www.ontotext.com/trree/owlim#read-only> "false";
+          <http://www.ontotext.com/trree/owlim#repository-type> "file-repository";
+          <http://www.ontotext.com/trree/owlim#ruleset> "rdfsplus-optimized";
+          <http://www.ontotext.com/trree/owlim#storage-folder> "storage";
+          <http://www.ontotext.com/trree/owlim#throw-QueryEvaluationException-on-timeout> "false";
+          sail:sailType "owlim:Sail"
+        ]
+    ];
+  rdfs:label "" .
\ No newline at end of file
diff --git a/test/data/graphdb-test-load.nt b/test/data/graphdb-test-load.nt
new file mode 100644
index 00000000..e69de29b
diff --git a/test/data/virtuoso.init b/test/data/virtuoso.init
new file mode 100644
index 00000000..e5f4bd85
--- /dev/null
+++ b/test/data/virtuoso.init
@@ -0,0 +1,240 @@
+ ;
+ ;  virtuoso.ini
+ ;
+ ;  Configuration file for the OpenLink Virtuoso VDBMS Server
+ ;
+ ;  To learn more about this product, or any other product in our
+ ;  portfolio, please check out our web site at:
+ ;
+ ;      http://virtuoso.openlinksw.com/
+ ;
+ ;  or contact us at:
+ ;
+ ;      general.information@openlinksw.com
+ ;
+ ;  If you have any technical questions, please contact our support
+ ;  staff at:
+ ;
+ ;      technical.support@openlinksw.com
+ ;
+ ;
+ ;  Database setup
+ ;
+ [Database]
+ DatabaseFile       = ../database/virtuoso.db
+ ErrorLogFile       = ../database/virtuoso.log
+ LockFile           = ../database/virtuoso.lck
+ TransactionFile    = ../database/virtuoso.trx
+ xa_persistent_file = ../database/virtuoso.pxa
+ ErrorLogLevel      = 7
+ FileExtend         = 200
+ MaxCheckpointRemap = 2000
+ Striping           = 0
+ TempStorage        = TempDatabase
+
+ [TempDatabase]
+ DatabaseFile       = ../database/virtuoso-temp.db
+ TransactionFile    = ../database/virtuoso-temp.trx
+ MaxCheckpointRemap = 2000
+ Striping           = 0
+
+ ;
+ ;  Server parameters
+ ;
+ [Parameters]
+ ServerPort               = 1111
+ LiteMode                 = 0
+ DisableUnixSocket        = 1
+ DisableTcpSocket         = 0
+ ;SSLServerPort                  = 2111
+ ;SSLCertificate                 = cert.pem
+ ;SSLPrivateKey                  = pk.pem
+ ;X509ClientVerify               = 0
+ ;X509ClientVerifyDepth          = 0
+ ;X509ClientVerifyCAFile         = ca.pem
+ MaxClientConnections     = 10
+ CheckpointInterval       = 60
+ O_DIRECT                 = 0
+ CaseMode                 = 2
+ MaxStaticCursorRows      = 5000
+ CheckpointAuditTrail     = 0
+ AllowOSCalls             = 0
+ SchedulerInterval        = 10
+ DirsAllowed              = ., ../vad, /usr/share/proj
+ ThreadCleanupInterval    = 0
+ ThreadThreshold          = 10
+ ResourcesCleanupInterval = 0
+ FreeTextBatchSize        = 100000
+ SingleCPU                = 0
+ VADInstallDir            = ../vad/
+ PrefixResultNames        = 0
+ RdfFreeTextRulesSize     = 100
+ IndexTreeMaps            = 64
+ MaxMemPoolSize           = 200000000
+ PrefixResultNames        = 0
+ MacSpotlight             = 0
+ MaxQueryMem              = 2G   ; memory allocated to query processor
+ VectorSize               = 1000 ; initial parallel query vector (array of query operations) size
+ MaxVectorSize            = 1000000      ; query vector size threshold.
+ AdjustVectorSize         = 0
+ ThreadsPerQuery          = 4
+ AsyncQueueMaxThreads     = 10
+ ;;
+ ;; When running with large data sets, one should configure the Virtuoso
+ ;; process to use between 2/3 to 3/5 of free system memory and to stripe
+ ;; storage on all available disks.
+ ;;
+ ;; Uncomment next two lines if there is 2 GB system memory free
+ ;NumberOfBuffers          = 170000
+ ;MaxDirtyBuffers          = 130000
+ ;; Uncomment next two lines if there is 4 GB system memory free
+ ;NumberOfBuffers          = 340000
+ ; MaxDirtyBuffers          = 250000
+ ;; Uncomment next two lines if there is 8 GB system memory free
+ ;NumberOfBuffers          = 680000
+ ;MaxDirtyBuffers          = 500000
+ ;; Uncomment next two lines if there is 16 GB system memory free
+ ;NumberOfBuffers          = 1360000
+ ;MaxDirtyBuffers          = 1000000
+ ;; Uncomment next two lines if there is 32 GB system memory free
+ ;NumberOfBuffers          = 2720000
+ ;MaxDirtyBuffers          = 2000000
+ ;; Uncomment next two lines if there is 48 GB system memory free
+ ;NumberOfBuffers          = 4000000
+ ;MaxDirtyBuffers          = 3000000
+ ;; Uncomment next two lines if there is 64 GB system memory free
+ ;NumberOfBuffers          = 5450000
+ ;MaxDirtyBuffers          = 4000000
+ ;;
+ ;; Note the default settings will take very little memory
+ ;; but will not result in very good performance
+ ;;
+ NumberOfBuffers          = 10000
+ MaxDirtyBuffers          = 6000
+
+ [HTTPServer]
+ ServerPort                  = 8890
+ ServerRoot                  = ../vsp
+ MaxClientConnections        = 10
+ DavRoot                     = DAV
+ EnabledDavVSP               = 0
+ HTTPProxyEnabled            = 0
+ TempASPXDir                 = 0
+ DefaultMailServer           = localhost:25
+ MaxKeepAlives               = 10
+ KeepAliveTimeout            = 10
+ MaxCachedProxyConnections   = 10
+ ProxyConnectionCacheTimeout = 15
+ HTTPThreadSize              = 280000
+ HttpPrintWarningsInOutput   = 0
+ Charset                     = UTF-8
+ ;HTTPLogFile                    = logs/http.log
+ MaintenancePage             = atomic.html
+ EnabledGzipContent          = 1
+
+ [AutoRepair]
+ BadParentLinks = 0
+
+ [Client]
+ SQL_PREFETCH_ROWS  = 100
+ SQL_PREFETCH_BYTES = 16000
+ SQL_QUERY_TIMEOUT  = 0
+ SQL_TXN_TIMEOUT    = 0
+ ;SQL_NO_CHAR_C_ESCAPE           = 1
+ ;SQL_UTF8_EXECS                 = 0
+ ;SQL_NO_SYSTEM_TABLES           = 0
+ ;SQL_BINARY_TIMESTAMP           = 1
+ ;SQL_ENCRYPTION_ON_PASSWORD     = -1
+
+ [VDB]
+ ArrayOptimization           = 0
+ NumArrayParameters          = 10
+ VDBDisconnectTimeout        = 1000
+ KeepConnectionOnFixedThread = 0
+
+ [Replication]
+ ServerName   = db-BIONIC-PORT
+ ServerEnable = 1
+ QueueMax     = 50000
+
+ ;
+ ;  Striping setup
+ ;
+ ;  These parameters have only effect when Striping is set to 1 in the
+ ;  [Database] section, in which case the DatabaseFile parameter is ignored.
+ ;
+ ;  With striping, the database is spawned across multiple segments
+ ;  where each segment can have multiple stripes.
+ ;
+ ;  Format of the lines below:
+ ;    Segment<number> = <size>, <stripe file name> [, <stripe file name> .. ]
+ ;
+ ;  <number> must be ordered from 1 up.
+ ;
+ ;  The <size> is the total size of the segment which is equally divided
+ ;  across all stripes forming  the segment. Its specification can be in
+ ;  gigabytes (g), megabytes (m), kilobytes (k) or in database blocks
+ ;  (b, the default)
+ ;
+ ;  Note that the segment size must be a multiple of the database page size
+ ;  which is currently 8k. Also, the segment size must be divisible by the
+ ;  number of stripe files forming  the segment.
+ ;
+ ;  The example below creates a 200 meg database striped on two segments
+ ;  with two stripes of 50 meg and one of 100 meg.
+ ;
+ ;  You can always add more segments to the configuration, but once
+ ;  added, do not change the setup.
+ ;
+ [Striping]
+ Segment1 = 100M, db-seg1-1.db, db-seg1-2.db
+ Segment2 = 100M, db-seg2-1.db
+ ;...
+ ;[TempStriping]
+ ;Segment1                       = 100M, db-seg1-1.db, db-seg1-2.db
+ ;Segment2                       = 100M, db-seg2-1.db
+ ;...
+ ;[Ucms]
+ ;UcmPath                        = <path>
+ ;Ucm1                           = <file>
+ ;Ucm2                           = <file>
+ ;...
+
+ [Zero Config]
+ ServerName = virtuoso (BIONIC-PORT)
+ ;ServerDSN                      = ZDSN
+ ;SSLServerName                  =
+ ;SSLServerDSN                   =
+
+ [Mono]
+ ;MONO_TRACE                     = Off
+ ;MONO_PATH                      = <path_here>
+ ;MONO_ROOT                      = <path_here>
+ ;MONO_CFG_DIR                   = <path_here>
+ ;virtclr.dll                    =
+
+ [URIQA]
+ DynamicLocal = 0
+ DefaultHost  = localhost:8890
+
+ [SPARQL]
+ ;ExternalQuerySource            = 1
+ ;ExternalXsltSource             = 1
+ ;DefaultGraph                   = http://localhost:8890/dataspace
+ ;ImmutableGraphs                = http://localhost:8890/dataspace
+ ResultSetMaxRows           = 10000
+ MaxConstructTriples        = 10000
+ MaxQueryCostEstimationTime = 400        ; in seconds
+ MaxQueryExecutionTime      = 60 ; in seconds
+ DefaultQuery               = select distinct ?Concept where {[] a ?Concept} LIMIT 100
+ DeferInferenceRulesInit    = 0  ; controls inference rules loading
+ MaxMemInUse                = 0  ; limits the amount of memory for construct dict (0=unlimited)
+ ;LabelInferenceName             = facets        ; Only needed when using the Faceted Browser
+ ;PingService                    = http://rpc.pingthesemanticweb.com/
+
+ [Plugins]
+ LoadPath = ../hosting
+ Load1    = plain, geos
+ Load2    = plain, graphql
+ Load3    = plain, proj4
+ Load4    = plain, shapefileio
\ No newline at end of file

From 70fd74faf8aaa5e9279421d1de135f7cb5ebcdc2 Mon Sep 17 00:00:00 2001
From: Syphax bouazzouni <gs_bouazzouni@esi.dz>
Date: Mon, 29 Jan 2024 13:52:19 +0100
Subject: [PATCH 152/168] remove faraday gem usage

---
 Gemfile      |  1 -
 Gemfile.lock | 63 ++++++++++++++++++----------------------------------
 2 files changed, 22 insertions(+), 42 deletions(-)

diff --git a/Gemfile b/Gemfile
index 3564fe3b..49dd2b38 100644
--- a/Gemfile
+++ b/Gemfile
@@ -4,7 +4,6 @@ gemspec
 
 gem "activesupport"
 gem "cube-ruby", require: "cube"
-gem "faraday", '~> 1.9'
 gem "rake"
 gem "uuid"
 gem "request_store"
diff --git a/Gemfile.lock b/Gemfile.lock
index 9fe7bd02..f0c24969 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -30,41 +30,24 @@ GEM
       multi_json (~> 1.3)
       thread_safe (~> 0.1)
       tzinfo (~> 0.3.37)
-    addressable (2.8.1)
+    addressable (2.8.5)
       public_suffix (>= 2.0.2, < 6.0)
+    base64 (0.1.1)
     builder (3.2.4)
     coderay (1.1.3)
     concurrent-ruby (1.2.2)
-    connection_pool (2.3.0)
+    connection_pool (2.4.1)
     cube-ruby (0.0.3)
     daemons (1.4.1)
     docile (1.4.0)
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
     eventmachine (1.2.7)
-    faraday (1.10.3)
-      faraday-em_http (~> 1.0)
-      faraday-em_synchrony (~> 1.0)
-      faraday-excon (~> 1.1)
-      faraday-httpclient (~> 1.0)
-      faraday-multipart (~> 1.0)
-      faraday-net_http (~> 1.0)
-      faraday-net_http_persistent (~> 1.0)
-      faraday-patron (~> 1.0)
-      faraday-rack (~> 1.0)
-      faraday-retry (~> 1.0)
+    faraday (2.7.11)
+      base64
+      faraday-net_http (>= 2.0, < 3.1)
       ruby2_keywords (>= 0.0.4)
-    faraday-em_http (1.0.0)
-    faraday-em_synchrony (1.0.0)
-    faraday-excon (1.1.0)
-    faraday-httpclient (1.0.1)
-    faraday-multipart (1.0.4)
-      multipart-post (~> 2)
-    faraday-net_http (1.0.1)
-    faraday-net_http_persistent (1.2.0)
-    faraday-patron (1.0.0)
-    faraday-rack (1.0.0)
-    faraday-retry (1.0.3)
+    faraday-net_http (3.0.2)
     http-accept (1.7.0)
     http-cookie (1.0.5)
       domain_name (~> 0.5)
@@ -74,12 +57,11 @@ GEM
     macaddr (1.7.2)
       systemu (~> 2.6.5)
     method_source (1.0.0)
-    mime-types (3.4.1)
+    mime-types (3.5.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2023.0218.1)
+    mime-types-data (3.2023.1003)
     minitest (4.7.5)
     multi_json (1.15.0)
-    multipart-post (2.3.0)
     mustermann (3.0.0)
       ruby2_keywords (~> 0.0.1)
     net-http-persistent (2.9.4)
@@ -87,20 +69,20 @@ GEM
     pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    public_suffix (5.0.1)
-    rack (2.2.6.3)
+    public_suffix (5.0.3)
+    rack (2.2.8)
     rack-accept (0.4.5)
       rack (>= 0.4)
     rack-post-body-to-params (0.1.8)
       activesupport (>= 2.3)
-    rack-protection (3.0.5)
-      rack
+    rack-protection (3.1.0)
+      rack (~> 2.2, >= 2.2.4)
     rake (13.0.6)
     rdf (1.0.8)
       addressable (>= 2.2)
-    redis (5.0.6)
+    redis (5.0.7)
       redis-client (>= 0.9.0)
-    redis-client (0.13.0)
+    redis-client (0.17.0)
       connection_pool
     request_store (1.5.1)
       rack (>= 1.4)
@@ -109,7 +91,7 @@ GEM
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
-    rexml (3.2.5)
+    rexml (3.2.6)
     rsolr (2.5.0)
       builder (>= 2.1.2)
       faraday (>= 0.9, < 3, != 2.0.0)
@@ -123,19 +105,19 @@ GEM
       simplecov (~> 0.19)
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
-    sinatra (3.0.5)
+    sinatra (3.1.0)
       mustermann (~> 3.0)
       rack (~> 2.2, >= 2.2.4)
-      rack-protection (= 3.0.5)
+      rack-protection (= 3.1.0)
       tilt (~> 2.0)
     systemu (2.6.5)
-    thin (1.8.1)
+    thin (1.8.2)
       daemons (~> 1.0, >= 1.0.9)
       eventmachine (~> 1.0, >= 1.0.4)
       rack (>= 1, < 3)
     thread_safe (0.3.6)
-    tilt (2.1.0)
-    tzinfo (0.3.61)
+    tilt (2.3.0)
+    tzinfo (0.3.62)
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.8.2)
@@ -150,7 +132,6 @@ PLATFORMS
 DEPENDENCIES
   activesupport
   cube-ruby
-  faraday (~> 1.9)
   goo!
   minitest (< 5.0)
   pry
@@ -166,4 +147,4 @@ DEPENDENCIES
   uuid
 
 BUNDLED WITH
-   2.3.22
+   2.3.15

From bf6b3d17fafae9df18e5950c8c6addd0dffbba27 Mon Sep 17 00:00:00 2001
From: Syphax bouazzouni <gs_bouazzouni@esi.dz>
Date: Mon, 29 Jan 2024 13:53:23 +0100
Subject: [PATCH 153/168] update test CI to test against all the supported
 backends with diffirent slice sizes

---
 .github/workflows/ruby-unit-test.yml | 15 ++++++++++-----
 test/test_case.rb                    |  3 +--
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/ruby-unit-test.yml b/.github/workflows/ruby-unit-test.yml
index ccb161f3..ac40314b 100644
--- a/.github/workflows/ruby-unit-test.yml
+++ b/.github/workflows/ruby-unit-test.yml
@@ -12,21 +12,26 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby-version: ['2.7']
+        goo-slice: [ '20', '100', '500' ]
+        ruby-version: [ '2.7', '3.0' ]
+        triplestore: [ 'fs', 'ag', 'vo', 'gb' ]
 
     steps:
     - uses: actions/checkout@v3
     - name: Install Dependencies
-      run: sudo apt-get -y install raptor2-utils
+      run: sudo apt-get update && sudo apt-get -y install raptor2-utils
     - name: Set up Ruby
       uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby-version }}
         bundler-cache: true # runs 'bundle install' and caches installed gems automatically
-    - name: Start backend services via docker-compose
-      run: docker compose up -d
+    - name: Add config file
+      # tempoaray workaround for the config.rb file requirement
+      run:  echo 'Goo.config do |config| end' > config/config.rb
+    - name: List directory contents
+      run: ls -R ./test/data
     - name: Run tests
-      run: bundle exec rake test
+      run: GOO_SLICES=${{ matrix.goo-slice }} bundle exec rake test:docker:${{ matrix.triplestore }} TESTOPTS="-v"
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v3
 
diff --git a/test/test_case.rb b/test/test_case.rb
index 7073acd5..af7f2a84 100644
--- a/test/test_case.rb
+++ b/test/test_case.rb
@@ -42,9 +42,8 @@ def _run_suites(suites, type)
     end
 
     def _run_suite(suite, type)
-      %[1,5,10,20]
       ret = []
-      [1,5,10,20].each do |slice_size|
+      [Goo.slice_loading_size].each do |slice_size|
         puts "\nrunning test with slice_loading_size=#{slice_size}"
         Goo.slice_loading_size=slice_size
         begin

From f82890fd41054309cf88b8a64998b256cf314f87 Mon Sep 17 00:00:00 2001
From: Syphax bouazzouni <gs_bouazzouni@esi.dz>
Date: Mon, 29 Jan 2024 14:00:03 +0100
Subject: [PATCH 154/168] add high level helper to to know which backend we are
 currently using

---
 lib/goo.rb                     | 25 +++++++++++++++++++++++++
 rakelib/docker_based_test.rake |  6 +++---
 2 files changed, 28 insertions(+), 3 deletions(-)

diff --git a/lib/goo.rb b/lib/goo.rb
index e34d7b96..a63722d6 100644
--- a/lib/goo.rb
+++ b/lib/goo.rb
@@ -53,6 +53,31 @@ module Goo
   @@slice_loading_size = 500
 
 
+
+  def self.log_debug_file(str)
+    debug_file = "./queries.txt"
+    File.write(debug_file, str.to_s + "\n", mode: 'a')
+  end
+
+
+
+  def backend_4s?
+    sparql_backend_name.downcase.eql?("4store")
+  end
+
+  def backend_ag?
+    sparql_backend_name.downcase.eql?("allegrograph")
+  end
+
+  def backend_gb?
+    sparql_backend_name.downcase.eql?("graphdb")
+  end
+
+  def backend_vo?
+    sparql_backend_name.downcase.eql?("virtuoso")
+  end
+
+
   def self.main_languages
     @@main_languages
   end
diff --git a/rakelib/docker_based_test.rake b/rakelib/docker_based_test.rake
index f3fa6c1a..d9b334f4 100644
--- a/rakelib/docker_based_test.rake
+++ b/rakelib/docker_based_test.rake
@@ -12,7 +12,7 @@ namespace :test do
     end
     desc "run tests with docker AG backend"
     task :ag do
-      ENV["GOO_BACKEND_NAME"]="AG"
+      ENV["GOO_BACKEND_NAME"]="allegrograph"
       ENV["GOO_PORT"]="10035"
       ENV["GOO_PATH_QUERY"]="/repositories/ontoportal_test"
       ENV["GOO_PATH_DATA"]="/repositories/ontoportal_test/statements"
@@ -47,7 +47,7 @@ namespace :test do
 
     desc "run tests with docker Virtuoso backend"
     task :vo do
-      ENV["GOO_BACKEND_NAME"]="VO"
+      ENV["GOO_BACKEND_NAME"]="virtuoso"
       ENV["GOO_PORT"]="8890"
       ENV["GOO_PATH_QUERY"]="/sparql"
       ENV["GOO_PATH_DATA"]="/sparql"
@@ -75,7 +75,7 @@ namespace :test do
 
     desc "run tests with docker GraphDb backend"
     task :gb do
-      ENV["GOO_BACKEND_NAME"]="GB"
+      ENV["GOO_BACKEND_NAME"]="graphdb"
       ENV["GOO_PORT"]="7200"
       ENV["GOO_PATH_QUERY"]="/repositories/ontoportal"
       ENV["GOO_PATH_DATA"]="/repositories/ontoportal/statements"

From f68f96de259379e34860e6856a64578068a546cd Mon Sep 17 00:00:00 2001
From: Syphax bouazzouni <gs_bouazzouni@esi.dz>
Date: Mon, 29 Jan 2024 14:08:42 +0100
Subject: [PATCH 155/168] extract sparql processor module from where module

---
 lib/goo/base/where.rb       | 114 +-----------------------------
 lib/goo/sparql/processor.rb | 137 ++++++++++++++++++++++++++++++++++++
 lib/goo/sparql/sparql.rb    |   1 +
 3 files changed, 140 insertions(+), 112 deletions(-)
 create mode 100644 lib/goo/sparql/processor.rb

diff --git a/lib/goo/base/where.rb b/lib/goo/base/where.rb
index 81cd26ce..7aaad6ce 100644
--- a/lib/goo/base/where.rb
+++ b/lib/goo/base/where.rb
@@ -6,6 +6,7 @@ class Where
       AGGREGATE_PATTERN = Struct.new(:pattern,:aggregate)
 
       attr_accessor :where_options_load
+      include Goo::SPARQL::Processor
 
       def initialize(klass,*match_patterns)
         if Goo.queries_debug? && Thread.current[:ncbo_debug].nil?
@@ -122,118 +123,7 @@ def unmmaped_predicates()
       end
 
       def process_query(count=false)
-        if Goo.queries_debug? &&  Thread.current[:ncbo_debug]
-          tstart = Time.now
-          query_resp = process_query_intl(count=count)
-          (Thread.current[:ncbo_debug][:goo_process_query] ||= []) << (Time.now - tstart)
-          return query_resp
-        end
-        return process_query_intl(count=count)
-      end
-
-      def process_query_intl(count=false)
-        if @models == []
-          @result = []
-          return @result
-        end
-
-        @include << @include_embed if @include_embed.length > 0
-
-        @predicates = unmmaped_predicates()
-        @equivalent_predicates = retrieve_equivalent_predicates()
-
-        options_load = { models: @models, include: @include, ids: @ids,
-                         graph_match: @pattern, klass: @klass,
-                         filters: @filters, order_by: @order_by ,
-                         read_only: @read_only, rules: @rules,
-                         predicates: @predicates,
-                         no_graphs: @no_graphs,
-                         equivalent_predicates: @equivalent_predicates }
-
-        options_load.merge!(@where_options_load) if @where_options_load
-        if !@klass.collection_opts.nil? and !options_load.include?(:collection)
-          raise ArgumentError, "Collection needed call `#{@klass.name}`"
-        end
-
-        ids = nil
-        if @index_key
-          raise ArgumentError, "Redis is not configured" unless Goo.redis_client
-          rclient = Goo.redis_client
-          cache_key = cache_key_for_index(@index_key)
-          raise ArgumentError, "Index not found" unless rclient.exists(cache_key)
-          if @page_i
-            if !@count
-              @count = rclient.llen(cache_key)
-            end
-            rstart = (@page_i -1) * @page_size
-            rstop = (rstart + @page_size) -1
-            ids = rclient.lrange(cache_key,rstart,rstop)
-          else
-            ids = rclient.lrange(cache_key,0,-1)
-          end
-          ids = ids.map { |i| RDF::URI.new(i) }
-        end
-
-        if @page_i && !@index_key
-          page_options = options_load.dup
-          page_options.delete(:include)
-          page_options[:include_pagination] = @include
-          if not @pre_count.nil?
-            @count = @pre_count
-          else
-            if !@count && @do_count
-              page_options[:count] = :count
-              @count = Goo::SPARQL::Queries.model_load(page_options).to_i
-            end
-          end
-          page_options.delete :count
-          page_options[:query_options] = @query_options
-          page_options[:page] = { page_i: @page_i, page_size: @page_size }
-          models_by_id = Goo::SPARQL::Queries.model_load(page_options)
-          options_load[:models] = models_by_id.values
-
-          #models give the constraint
-          options_load.delete :graph_match
-        elsif count
-          count_options = options_load.dup
-          count_options.delete(:include)
-          count_options[:count] = :count
-          return Goo::SPARQL::Queries.model_load(count_options).to_i
-        end
-
-        if @indexing
-          #do not care about include values
-          @result = Goo::Base::Page.new(@page_i,@page_size,@count,models_by_id.values)
-          return @result
-        end
-
-        options_load[:ids] = ids if ids
-        models_by_id = {}
-        if @page_i && (options_load[:models].length > 0)
-          options_load.delete(:filters)
-          options_load.delete(:order_by)
-        end
-
-        if (@page_i && options_load[:models].length > 0) ||
-            (!@page_i && (@count.nil? || @count > 0))
-          models_by_id = Goo::SPARQL::Queries.model_load(options_load)
-          if @aggregate
-            if models_by_id.length > 0
-              options_load_agg = { models: models_by_id.values, klass: @klass,
-                             filters: @filters, read_only: @read_only,
-                             aggregate: @aggregate, rules: @rules }
-
-              options_load_agg.merge!(@where_options_load) if @where_options_load
-              Goo::SPARQL::Queries.model_load(options_load_agg)
-            end
-          end
-        end
-        unless @page_i
-          @result = @models ? @models : models_by_id.values
-        else
-          @result = Goo::Base::Page.new(@page_i,@page_size,@count,models_by_id.values)
-        end
-        @result
+        process_query_call(count = count)
       end
 
       def disable_rules
diff --git a/lib/goo/sparql/processor.rb b/lib/goo/sparql/processor.rb
new file mode 100644
index 00000000..c89778ed
--- /dev/null
+++ b/lib/goo/sparql/processor.rb
@@ -0,0 +1,137 @@
+module Goo
+  module SPARQL
+    module Processor
+      def process_query_call(count=false)
+        if Goo.queries_debug? &&  Thread.current[:ncbo_debug]
+          start = Time.now
+          query_resp = process_query_intl(count=count)
+          (Thread.current[:ncbo_debug][:goo_process_query] ||= []) << (Time.now - start)
+          return query_resp
+        end
+         process_query_init(count=count)
+      end
+
+      private
+      def process_query_init(count=false)
+        if @models == []
+          @result = []
+          return @result
+        end
+
+        @include << @include_embed if @include_embed.length > 0
+
+        @predicates = unmmaped_predicates()
+        @equivalent_predicates = retrieve_equivalent_predicates()
+
+        options_load = { models: @models, include: @include, ids: @ids,
+                         graph_match: @pattern, klass: @klass,
+                         filters: @filters, order_by: @order_by ,
+                         read_only: @read_only, rules: @rules,
+                         predicates: @predicates,
+                         no_graphs: @no_graphs,
+                         equivalent_predicates: @equivalent_predicates }
+
+        options_load.merge!(@where_options_load) if @where_options_load
+
+        if !@klass.collection_opts.nil? and !options_load.include?(:collection)
+          raise ArgumentError, "Collection needed call `#{@klass.name}`"
+        end
+
+        ids = nil
+
+
+        ids = redis_indexed_ids if use_redis_index?
+
+        if @page_i && !use_redis_index?
+          page_options = options_load.dup
+          page_options.delete(:include)
+          page_options[:include_pagination] = @include
+          page_options[:query_options] = @query_options
+
+          @count = run_count_query(page_options)
+          page_options[:page] = { page_i: @page_i, page_size: @page_size }
+
+          models_by_id = Goo::SPARQL::Queries.model_load(page_options)
+          options_load[:models] = models_by_id.values
+          #models give the constraint
+          options_load.delete :graph_match
+        elsif count
+          count_options = options_load.dup
+          count_options.delete(:include)
+          return run_count_query(count_options)
+        end
+
+        if @indexing
+          #do not care about include values
+          @result = Goo::Base::Page.new(@page_i,@page_size,@count,models_by_id.values)
+          return @result
+        end
+
+        options_load[:ids] = ids if ids
+        models_by_id = {}
+
+        if (@page_i && options_load[:models].nil?) ||
+          (@page_i && options_load[:models].length > 0) ||
+          (!@page_i && (@count.nil? || @count > 0))
+
+          models_by_id = Goo::SPARQL::Queries.model_load(options_load)
+          run_aggregate_query(models_by_id) if @aggregate && models_by_id.length > 0
+        end
+
+        if @page_i
+          @result = Goo::Base::Page.new(@page_i, @page_size, @count, models_by_id.values)
+        else
+          @result = @models ? @models : models_by_id.values
+        end
+        @result
+      end
+
+
+      def use_redis_index?
+        @index_key
+      end
+
+      def run_aggregate_query(models_by_id)
+        options_load_agg = { models: models_by_id.values, klass: @klass,
+                             filters: @filters, read_only: @read_only,
+                             aggregate: @aggregate, rules: @rules }
+        options_load_agg.merge!(@where_options_load) if @where_options_load
+        Goo::SPARQL::Queries.model_load(options_load_agg)
+      end
+      def run_count_query(page_options)
+        count = 0
+        if @pre_count
+          count = @pre_count
+        elsif !@count && @do_count
+          page_options[:count] = :count
+          r = Goo::SPARQL::Queries.model_load(page_options)
+          if r.is_a? Numeric
+            count = r.to_i
+          end
+        elsif @count
+          count = @count
+        end
+        page_options.delete :count
+        count
+      end
+
+      def redis_indexed_ids
+        raise ArgumentError, "Redis is not configured" unless Goo.redis_client
+        rclient = Goo.redis_client
+        cache_key = cache_key_for_index(@index_key)
+        raise ArgumentError, "Index not found" unless rclient.exists(cache_key)
+        if @page_i
+          if !@count
+            @count = rclient.llen(cache_key)
+          end
+          rstart = (@page_i -1) * @page_size
+          rstop = (rstart + @page_size) -1
+          ids = rclient.lrange(cache_key,rstart,rstop)
+        else
+          ids = rclient.lrange(cache_key,0,-1)
+        end
+        ids = ids.map { |i| RDF::URI.new(i) }
+      end
+    end
+  end
+end
diff --git a/lib/goo/sparql/sparql.rb b/lib/goo/sparql/sparql.rb
index 6fa1d582..d5315cde 100644
--- a/lib/goo/sparql/sparql.rb
+++ b/lib/goo/sparql/sparql.rb
@@ -8,3 +8,4 @@
 require_relative "triples"
 require_relative "loader"
 require_relative "queries"
+require_relative 'processor'

From a3f8a60f97f30f7e64ada86425626f213dda7cad Mon Sep 17 00:00:00 2001
From: Syphax bouazzouni <gs_bouazzouni@esi.dz>
Date: Mon, 29 Jan 2024 14:19:02 +0100
Subject: [PATCH 156/168] handle language_match? value to upcase by default

---
 lib/goo/sparql/mixins/solution_lang_filter.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/goo/sparql/mixins/solution_lang_filter.rb b/lib/goo/sparql/mixins/solution_lang_filter.rb
index 49d62c8e..3a751b13 100644
--- a/lib/goo/sparql/mixins/solution_lang_filter.rb
+++ b/lib/goo/sparql/mixins/solution_lang_filter.rb
@@ -95,7 +95,7 @@ def language_match?(language)
 
           return requested_lang.include?(language)  if requested_lang.is_a?(Array)
 
-          language.eql?(requested_lang)
+          language&.upcase.eql?(requested_lang)
         end
 
         def literal?(object)

From 8b59e8cb8501e917fc0734cdf3761018c8709ab7 Mon Sep 17 00:00:00 2001
From: Syphax bouazzouni <gs_bouazzouni@esi.dz>
Date: Mon, 29 Jan 2024 14:19:33 +0100
Subject: [PATCH 157/168] add support for virtuoso and graphdb sparql client

---
 lib/goo/sparql/client.rb | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/lib/goo/sparql/client.rb b/lib/goo/sparql/client.rb
index 8f7ad9e1..d5e32e04 100644
--- a/lib/goo/sparql/client.rb
+++ b/lib/goo/sparql/client.rb
@@ -14,7 +14,6 @@ class Client < RSPARQL::Client
         "text/x-nquads" => "nquads"
       }
 
-      BACKEND_4STORE = "4store"
 
       def status_based_sleep_time(operation)
         sleep(0.5)
@@ -39,16 +38,17 @@ def status_based_sleep_time(operation)
       end
 
       class DropGraph
-        def initialize(g)
+        def initialize(g, silent: false)
           @graph = g
           @caching_options = { :graph => @graph.to_s }
+          @silent = silent
         end
         def to_s
-          return "DROP GRAPH <#{@graph.to_s}>"
+          "DROP #{@silent ? 'SILENT' : ''} GRAPH <#{@graph.to_s}>"
         end
         def options
           #Returns the caching option
-          return @caching_options
+          @caching_options
         end
       end
 
@@ -77,7 +77,7 @@ def bnodes_filter_file(file_path,mime_type)
       end
 
       def delete_data_graph(graph)
-        Goo.sparql_update_client.update(DropGraph.new(graph))
+        Goo.sparql_update_client.update(DropGraph.new(graph, silent: Goo.backend_vo?))
       end
 
       def append_triples_no_bnodes(graph,file_path,mime_type_in)
@@ -184,8 +184,6 @@ def status
         resp
       end
 
-      private
-
       def execute_append_request(graph, data_file, mime_type_in)
         mime_type = "text/turtle"
 
@@ -195,9 +193,8 @@ def execute_append_request(graph, data_file, mime_type_in)
         end
 
         params = {method: :post, url: "#{url.to_s}", headers: {"content-type" => mime_type, "mime-type" => mime_type}, timeout: nil}
-        backend_name = Goo.sparql_backend_name
-
-        if backend_name == BACKEND_4STORE
+        
+        if Goo.backend_4s?
           params[:payload] = {
             graph: graph.to_s,
             data: data_file,
@@ -205,6 +202,9 @@ def execute_append_request(graph, data_file, mime_type_in)
           }
           #for some reason \\\\ breaks parsing
           params[:payload][:data] = params[:payload][:data].split("\n").map { |x| x.sub("\\\\","") }.join("\n")
+        elsif Goo.backend_vo?
+          params[:url] = "http://localhost:8890/sparql-graph-crud?graph=#{CGI.escape(graph.to_s)}"
+          params[:payload] = data_file
         else
           params[:url] << "?context=#{CGI.escape("<#{graph.to_s}>")}"
           params[:payload] = data_file

From 945bc811574fd03ab387ff504291105f54686725 Mon Sep 17 00:00:00 2001
From: Syphax bouazzouni <gs_bouazzouni@esi.dz>
Date: Mon, 29 Jan 2024 16:54:04 +0100
Subject: [PATCH 158/168] replace delete sparql query by delete graph in the
 model complex test

---
 test/test_model_complex.rb | 28 ++++++++++++++++++++++++++--
 1 file changed, 26 insertions(+), 2 deletions(-)

diff --git a/test/test_model_complex.rb b/test/test_model_complex.rb
index 0ee493e9..3074683c 100644
--- a/test/test_model_complex.rb
+++ b/test/test_model_complex.rb
@@ -76,12 +76,13 @@ def self.before_suite
     if GooTest.count_pattern("?s ?p ?o") > 100000
       raise Exception, "Too many triples in KB, does not seem right to run tests"
     end
-    Goo.sparql_update_client.update("DELETE {?s ?p ?o } WHERE { ?s ?p ?o }")
+
+    Goo.sparql_data_client.delete_graph(Submission.uri_type.to_s)
   end
 
   def self.after_suite
     Goo.use_cache = false
-    Goo.sparql_update_client.update("DELETE {?s ?p ?o } WHERE { ?s ?p ?o }")
+    Goo.sparql_data_client.delete_graph(Submission.uri_type.to_s)
   end
 
   def test_method_handler
@@ -183,6 +184,11 @@ def test_multiple_collection()
 
   def test_collection()
 
+    # This call is not usually necessary as it is usually covered by
+    # the model declaration above.  See the explanation in
+    # https://github.com/ncbo/goo/commit/0e09816b121750b3bb875a5c24cb79865287fcf4#commitcomment-90304626
+    Goo.add_model(:class, Term)
+
     submission = Submission.new(name: "submission1")
     unless submission.exist?
       submission.save
@@ -311,6 +317,11 @@ def test_two_resources_same_id
 
   def test_parents_inverse_children
 
+    # This call is not usually necessary as it is usually covered by
+    # the model declaration above.  See the explanation in
+    # https://github.com/ncbo/goo/commit/0e09816b121750b3bb875a5c24cb79865287fcf4#commitcomment-90304626
+    Goo.add_model(:class, Term)
+
     submission = Submission.new(name: "submission1")
     unless submission.exist?
       submission.save
@@ -318,6 +329,7 @@ def test_parents_inverse_children
       submission = Submission.find("submission1").first
     end
 
+
     terms = Term.in(submission)
     terms.each do |t|
       t.delete
@@ -651,6 +663,11 @@ def test_empty_pages
       assert_equal 0, GooTest.count_pattern("GRAPH #{submission.id.to_ntriples} { #{t.id.to_ntriples} ?p ?o . }")
     end
 
+    # This call is not usually necessary as it is usually covered by
+    # the model declaration above.  See the explanation in
+    # https://github.com/ncbo/goo/commit/0e09816b121750b3bb875a5c24cb79865287fcf4#commitcomment-90304626
+    Goo.add_model(:class, Term)
+
     terms = []
     10.times do |i|
       term = Term.new
@@ -677,6 +694,12 @@ def test_empty_pages
   end
 
   def test_readonly_pages_with_include
+
+    # This call is not usually necessary as it is usually covered by
+    # the model declaration above.  See the explanation in
+    # https://github.com/ncbo/goo/commit/0e09816b121750b3bb875a5c24cb79865287fcf4#commitcomment-90304626
+    Goo.add_model(:class, Term)
+
     submission = Submission.new(name: "submission1")
     unless submission.exist?
       submission.save
@@ -689,6 +712,7 @@ def test_readonly_pages_with_include
       assert_equal(0,
        GooTest.count_pattern("GRAPH #{submission.id.to_ntriples} { #{t.id.to_ntriples} ?p ?o . }"))
     end
+
     terms = []
     10.times do |i|
       term = Term.new

From 9d8b2992ab36ef5325e7a178b75ca7e4c57239ae Mon Sep 17 00:00:00 2001
From: Syphax bouazzouni <gs_bouazzouni@esi.dz>
Date: Mon, 29 Jan 2024 16:56:13 +0100
Subject: [PATCH 159/168] add some new edge cases tests t o test_where.rb and
 test_schemaless

---
 test/test_cache.rb      | 15 +++++---
 test/test_schemaless.rb | 19 ++++++++--
 test/test_where.rb      | 79 +++++++++++++++++++++++++----------------
 3 files changed, 75 insertions(+), 38 deletions(-)

diff --git a/test/test_cache.rb b/test/test_cache.rb
index be01f518..f5659ca4 100644
--- a/test/test_cache.rb
+++ b/test/test_cache.rb
@@ -100,7 +100,7 @@ def test_cache_models_back_door
     data = "<http://goo.org/default/student/Tim> " +
            "<http://goo.org/default/enrolled> " +
            "<http://example.org/program/Stanford/BioInformatics> ."
-    
+
     Goo.sparql_data_client.append_triples(Student.type_uri,data,"application/x-turtle")
     programs = Program.where(name: "BioInformatics", university: [ name: "Stanford"  ])
                           .include(:students).all
@@ -128,11 +128,16 @@ def x.response_backup *args
     def x.response *args
       raise Exception, "Should be a successful hit"
     end
-    programs = Program.where(name: "BioInformatics", university: [ name: "Stanford"  ])
-                        .include(:students).all
+    begin
+      programs = Program.where(name: "BioInformatics", university: [ name: "Stanford"  ])
+                          .include(:students).all
+    rescue  Exception
+      assert false, "should be cached"
+    end
+
     #from cache
-    assert programs.length == 1
-    assert_raises Exception do 
+    assert_equal 1, programs.length
+    assert_raises Exception do
       #different query
       programs = Program.where(name: "BioInformatics X", university: [ name: "Stanford"  ]).all
     end
diff --git a/test/test_schemaless.rb b/test/test_schemaless.rb
index 9e67e6e9..42084eb8 100644
--- a/test/test_schemaless.rb
+++ b/test/test_schemaless.rb
@@ -2,8 +2,6 @@
 
 module TestSchemaless
 
-module TestSChemaless
-
   ONT_ID = "http:://example.org/data/nemo"
 
   class Ontology < Goo::Base::Resource
@@ -118,6 +116,9 @@ def test_find_include_schemaless
       where = Klass.find(cognition_term).in(ontology).include(:unmapped)
       k =  where.first
       enter = 0
+
+      assert k.unmapped.keys.include?(Goo.vocabulary(:nemo)[:definition])
+
       k.unmapped.each do |p,vals|
         if p.to_s == Goo.vocabulary(:nemo)[:synonym].to_s
           enter += 1
@@ -185,7 +186,19 @@ def test_index_order_by
       end
 
     end
+    
+    def test_all_pages_loop
+      ontology = Ontology.find(RDF::URI.new(ONT_ID)).first
+      page = 1
+      count = 0
+      begin
+        paging = Klass.in(ontology).page(page,50).all
+        count += paging.size
+        page = paging.next_page if paging.next?
+      end while(paging.next?)
 
+      assert_equal count, Klass.in(ontology).count
+    end
     def test_page_reuse_predicates
       ontology = Ontology.find(RDF::URI.new(ONT_ID)).first
       paging = Klass.in(ontology).include(:unmapped).page(1,100)
@@ -208,7 +221,7 @@ def test_page_reuse_predicates
           all_ids << k.id
         end
         total += page.length
-        paging.page(page.next_page) if page.next?
+        paging.page(page.next_page, 100) if page.next?
         assert page.aggregate == 1713
       end while(page.next?)
       assert all_ids.length == all_ids.uniq.length
diff --git a/test/test_where.rb b/test/test_where.rb
index a9087855..a95131b6 100644
--- a/test/test_where.rb
+++ b/test/test_where.rb
@@ -70,7 +70,7 @@ def test_where_simple
       "http://example.org/program/Stanford/CompSci",
       "http://example.org/program/Stanford/Medicine"
     ]
-   assert_equal program_ids, st.programs.map { |x| x.id.to_s }.sort
+    assert_equal program_ids, st.programs.map { |x| x.id.to_s }.sort
   end
 
   def test_all
@@ -101,7 +101,7 @@ def test_where_2levels
     programs = Program.where(name: "BioInformatics", university: [ address: [ country: "UK" ]]).all
     assert programs.length == 1
     assert programs.first.id.to_s["Southampton/BioInformatics"]
-   
+
     #any program from universities in the US
     programs = Program.where(university: [ address: [ country: "US" ]]).include([:name]).all
     assert programs.length == 3
@@ -118,15 +118,15 @@ def test_where_2levels_inverse
 
     #equivalent
     unis = University.where(address: [country: "US"])
-                   .and(programs: [category: [code: "Biology"]]).all
+                     .and(programs: [category: [code: "Biology"]]).all
     assert unis.length == 1
     assert unis.first.id.to_s == "http://goo.org/default/university/Stanford"
   end
 
   def test_embed_include
     programs = Program.where.include(:name)
-                  .include(university: [:name])
-                  .include(category: [:code]).all
+                      .include(university: [:name])
+                      .include(category: [:code]).all
 
     assert programs.length == 9
     programs.each do |p|
@@ -177,7 +177,7 @@ def test_iterative_include_in_place
     #two levels
     unis = University.where.all
     unis_return = University.where.models(unis)
-                    .include(programs: [:name, students: [:name]]).to_a
+                            .include(programs: [:name, students: [:name]]).to_a
     assert unis_return.object_id == unis.object_id
     return_object_id = unis.map { |x| x.object_id }.uniq.sort
     unis_object_id = unis.map { |x| x.object_id }.uniq.sort
@@ -259,7 +259,18 @@ def test_embed_two_levels
     end
   end
 
+  def test_fetch_remaining
+    students = Student.where(enrolled:RDF::URI.new("http://example.org/program/Stanford/BioInformatics"))
+                      .include(:name, :birth_date, enrolled: [:name]).all
+
+
+    s = students.select { |x| x.name['Daniel'] }.first
+    refute_nil s
+    assert_equal 2, s.enrolled.size
+  end
+
   def test_paging_with_filter_order
+    skip('pagination with filter and order does not work in 4s') if Goo.backend_4s?
 
     f = Goo::Filter.new(:birth_date) > DateTime.parse('1978-01-03')
     total_count = Student.where.filter(f).count
@@ -273,6 +284,14 @@ def test_paging_with_filter_order
     assert_equal total_count, page_1.size + page_2.size
   end
 
+  def test_two_level_include
+    programs = Program.where.include(:name).all
+    r = Program.where.models(programs).include(students: [:name]).all
+    r.each do |p|
+      refute_nil p.students
+    end
+  end
+
   def test_unique_object_references
 
     #NOTE: unique references does not apply across different slice loading
@@ -332,7 +351,7 @@ def test_unique_object_references
   def test_complex_include
     #Students in a university by name
     students = Student.where(enrolled: [university: [name: "Stanford"]])
-                .include(:name)
+                      .include(:name)
                 .include(enrolled: [:name, university: [ :address ]]).all
 
     assert students.map { |x| x.name }.sort == ["Daniel","John","Susan"]
@@ -342,7 +361,7 @@ def test_complex_include
         assert_instance_of University, p.university
         assert_instance_of Array, p.university.addresses
         assert_instance_of Address, p.university.addresses.first
-        assert_raises Goo::Base::AttributeNotLoaded do 
+        assert_raises Goo::Base::AttributeNotLoaded do
           p.university.addresses.first.country
         end
       end
@@ -402,23 +421,23 @@ def test_where_union_pattern
 
   def test_where_direct_attributes
     st = Student.where(name: "Daniel")
-                  .or(name: "Louis")
-                  .or(name: "Lee")
-                  .or(name: "John").all
+                .or(name: "Louis")
+                .or(name: "Lee")
+                .or(name: "John").all
     assert st.length == 4
 
     st = Student.where(name: "Daniel")
-                  .and(name: "John").all
+                .and(name: "John").all
     assert st.length == 0
 
     st = Student.where(name: "Daniel")
-                  .and(birth_date: DateTime.parse('1978-01-04')).all
+                .and(birth_date: DateTime.parse('1978-01-04')).all
     assert st.length == 1
     assert st.first.id.to_s["Daniel"]
 
     st = Student.where(name: "Daniel")
-                  .or(name: "Louis")
-                  .and(birth_date: DateTime.parse('1978-01-04'))
+                .or(name: "Louis")
+                .and(birth_date: DateTime.parse('1978-01-04'))
     assert st.length == 1
     assert st.first.id.to_s["Daniel"]
 
@@ -454,8 +473,8 @@ def test_combine_where_patterns_with_include
     st.each do |p|
       assert (p.name == "Susan" || p.name == "Daniel")
       assert Array, p.enrolled
-      assert (p.name == "Susan" && p.enrolled.length == 1) || 
-        (p.name == "Daniel" && p.enrolled.length == 2) 
+      assert (p.name == "Susan" && p.enrolled.length == 1) ||
+               (p.name == "Daniel" && p.enrolled.length == 2)
       assert String, p.enrolled.first.university.address.first.country
     end
   end
@@ -467,31 +486,31 @@ def test_filter
     f = Goo::Filter.new(:birth_date) > DateTime.parse('1978-01-03')
     st = Student.where.filter(f).all
     assert st.map { |x| x.id.to_s }.sort == ["http://goo.org/default/student/Daniel",
- "http://goo.org/default/student/Lee",
- "http://goo.org/default/student/Louis",
- "http://goo.org/default/student/Robert"]
+                                             "http://goo.org/default/student/Lee",
+                                             "http://goo.org/default/student/Louis",
+                                             "http://goo.org/default/student/Robert"]
 
     f = (Goo::Filter.new(:birth_date) <= DateTime.parse('1978-01-01'))
           .or(Goo::Filter.new(:birth_date) >= DateTime.parse('1978-01-07'))
     st = Student.where.filter(f).all
     assert st.map { |x| x.id.to_s }.sort == [
- "http://goo.org/default/student/Robert",
- "http://goo.org/default/student/Susan"]
+      "http://goo.org/default/student/Robert",
+      "http://goo.org/default/student/Susan"]
 
     f = (Goo::Filter.new(:birth_date) <= DateTime.parse('1978-01-01'))
           .or(Goo::Filter.new(:name) == "Daniel")
     st = Student.where.filter(f).all
     assert st.map { |x| x.id.to_s }.sort == [
- "http://goo.org/default/student/Daniel",
- "http://goo.org/default/student/Susan"]
+      "http://goo.org/default/student/Daniel",
+      "http://goo.org/default/student/Susan"]
 
     f = (Goo::Filter.new(:birth_date) > DateTime.parse('1978-01-02'))
           .and(Goo::Filter.new(:birth_date) < DateTime.parse('1978-01-06'))
     st = Student.where.filter(f).all
     assert st.map { |x| x.id.to_s }.sort == [
- "http://goo.org/default/student/Daniel",
- "http://goo.org/default/student/Louis",
- "http://goo.org/default/student/Tim"]
+      "http://goo.org/default/student/Daniel",
+      "http://goo.org/default/student/Louis",
+      "http://goo.org/default/student/Tim"]
 
 
     f = Goo::Filter.new(enrolled: [ :credits ]) > 8
@@ -501,8 +520,8 @@ def test_filter
     #students without awards
     f = Goo::Filter.new(:awards).unbound
     st = Student.where.filter(f)
-                      .include(:name)
-                      .all
+                .include(:name)
+                .all
     assert st.map { |x| x.name }.sort == ["John","Tim","Louis","Lee","Robert"].sort
 
     #unbound on some non existing property
@@ -537,7 +556,7 @@ def test_aggregated
     sts = Student.where.include(:name).aggregate(:count, :enrolled).all
     sts.each do |st|
       assert (st.name == "Daniel" && st.aggregates.first.value == 2) ||
-                st.aggregates.first.value == 1
+               st.aggregates.first.value == 1
     end
 
     #students enrolled in more than 1 program and get the programs name

From deed0eba2c665543e0fcdfd0dd86ea053a095173 Mon Sep 17 00:00:00 2001
From: Syphax bouazzouni <gs_bouazzouni@esi.dz>
Date: Mon, 29 Jan 2024 17:42:23 +0100
Subject: [PATCH 160/168] make test_chunks_write.rb tests support multiple
 backends

---
 lib/goo/sparql/client.rb  | 11 ++++---
 test/test_chunks_write.rb | 66 +++++++++++++++++----------------------
 2 files changed, 35 insertions(+), 42 deletions(-)

diff --git a/lib/goo/sparql/client.rb b/lib/goo/sparql/client.rb
index d5e32e04..cf958398 100644
--- a/lib/goo/sparql/client.rb
+++ b/lib/goo/sparql/client.rb
@@ -184,7 +184,7 @@ def status
         resp
       end
 
-      def execute_append_request(graph, data_file, mime_type_in)
+      def params_for_backend(graph, data_file, mime_type_in, method = :post)
         mime_type = "text/turtle"
 
         if mime_type_in == "text/x-nquads"
@@ -192,8 +192,8 @@ def execute_append_request(graph, data_file, mime_type_in)
           graph = "http://data.bogus.graph/uri"
         end
 
-        params = {method: :post, url: "#{url.to_s}", headers: {"content-type" => mime_type, "mime-type" => mime_type}, timeout: nil}
-        
+        params = {method: method, url: "#{url.to_s}", headers: {"content-type" => mime_type, "mime-type" => mime_type}, timeout: nil}
+
         if Goo.backend_4s?
           params[:payload] = {
             graph: graph.to_s,
@@ -209,8 +209,11 @@ def execute_append_request(graph, data_file, mime_type_in)
           params[:url] << "?context=#{CGI.escape("<#{graph.to_s}>")}"
           params[:payload] = data_file
         end
+        params
+      end
 
-        RestClient::Request.execute(params)
+      def execute_append_request(graph, data_file, mime_type_in)
+        RestClient::Request.execute(params_for_backend(graph, data_file, mime_type_in))
       end
     end
   end
diff --git a/test/test_chunks_write.rb b/test/test_chunks_write.rb
index e2f82439..1a116aca 100644
--- a/test/test_chunks_write.rb
+++ b/test/test_chunks_write.rb
@@ -1,8 +1,8 @@
 require_relative 'test_case'
 
 module TestChunkWrite
-  ONT_ID = "http:://example.org/data/nemo"
-  ONT_ID_EXTRA = "http:://example.org/data/nemo/extra"
+  ONT_ID = "http://example.org/data/nemo"
+  ONT_ID_EXTRA = "http://example.org/data/nemo/extra"
 
   class TestChunkWrite < MiniTest::Unit::TestCase
 
@@ -18,14 +18,14 @@ def self.after_suite
       _delete
     end
 
+    def setup
+      self.class._delete
+    end
+
+
     def self._delete
-      graphs = [ONT_ID,ONT_ID_EXTRA]
-      url = Goo.sparql_data_client.url
-      graphs.each do |graph|
-        # This bypasses the chunks stuff
-        params = { method: :delete, url: "#{url.to_s}#{graph.to_s}", timeout: nil }
-        RestClient::Request.execute(params)
-      end
+      graphs = [ONT_ID, ONT_ID_EXTRA]
+      graphs.each { |graph| Goo.sparql_data_client.delete_graph(graph) }
     end
 
     def test_put_data
@@ -72,14 +72,7 @@ def test_reentrant_queries
       ntriples_file_path = "./test/data/nemo_ontology.ntriples"
 
       # Bypass in chunks
-      url = Goo.sparql_data_client.url
-      params = {
-        method: :put,
-        url: "#{url.to_s}#{ONT_ID}",
-        payload: File.read(ntriples_file_path),
-        headers: {content_type: "application/x-turtle"},
-        timeout: nil
-      }
+      params = self.class.params_for_backend(:post, ONT_ID, ntriples_file_path)
       RestClient::Request.execute(params)
 
       tput = Thread.new {
@@ -134,16 +127,7 @@ def test_reentrant_queries
 
     def test_query_flood
       ntriples_file_path = "./test/data/nemo_ontology.ntriples"
-
-      # Bypass in chunks
-      url = Goo.sparql_data_client.url
-      params = {
-        method: :put,
-        url: "#{url.to_s}#{ONT_ID}",
-        payload: File.read(ntriples_file_path),
-        headers: {content_type: "application/x-turtle"},
-        timeout: nil
-      }
+      params = self.class.params_for_backend(:post, ONT_ID, ntriples_file_path)
       RestClient::Request.execute(params)
 
       tput = Thread.new {
@@ -162,21 +146,27 @@ def test_query_flood
         }
       end
 
-      log_status = []
-      Thread.new {
-        10.times do |i|
-          log_status << Goo.sparql_query_client.status
-          sleep(1.2)
+      if Goo.backend_4s?
+        log_status = []
+        Thread.new {
+          10.times do |i|
+            log_status << Goo.sparql_query_client.status
+            sleep(1.2)
+          end
+        }
+
+        threads.each do |t|
+          t.join
         end
-      }
+        tput.join
 
-      threads.each do |t|
-        t.join
+        assert log_status.map { |x| x[:outstanding] }.max > 0
+        assert_equal 16, log_status.map { |x| x[:running] }.max
       end
-      tput.join
+    end
 
-      assert log_status.map { |x| x[:outstanding] }.max > 0
-      assert_equal 16, log_status.map { |x| x[:running] }.max
+    def self.params_for_backend(method, graph_name, ntriples_file_path = nil)
+      Goo.sparql_data_client.params_for_backend(graph_name, File.read(ntriples_file_path), "text/turtle", method)
     end
 
   end

From bc8a1e52379ae9fd67e77c5eb2fd1284ff32a043 Mon Sep 17 00:00:00 2001
From: Syphax bouazzouni <gs_bouazzouni@esi.dz>
Date: Mon, 29 Jan 2024 20:30:15 +0100
Subject: [PATCH 161/168] replace native insert_data with
 execute_append_request in model save

---
 lib/goo/base/resource.rb | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/goo/base/resource.rb b/lib/goo/base/resource.rb
index 2eaf17ad..3f18c677 100644
--- a/lib/goo/base/resource.rb
+++ b/lib/goo/base/resource.rb
@@ -289,7 +289,8 @@ def save(*opts)
               batch_file.write(lines.join(""))
               batch_file.flush()
             else
-              Goo.sparql_update_client.insert_data(graph_insert, graph: graph)
+              data = graph_insert.to_a.reduce("") { |acc, x| acc << x.to_s + " " }
+              Goo.sparql_data_client.execute_append_request(graph, data, "application/x-turtle")
             end
           rescue Exception => e
             raise e

From 402ce6df87ea5dce979fd765e86cb979d424687d Mon Sep 17 00:00:00 2001
From: Syphax bouazzouni <gs_bouazzouni@esi.dz>
Date: Mon, 29 Jan 2024 20:29:36 +0100
Subject: [PATCH 162/168] remove add_rules as it seems to no more be used

---
 lib/goo/sparql/loader.rb               |  1 -
 lib/goo/sparql/mixins/query_pattern.rb |  3 ---
 lib/goo/sparql/query_builder.rb        | 13 +------------
 3 files changed, 1 insertion(+), 16 deletions(-)

diff --git a/lib/goo/sparql/loader.rb b/lib/goo/sparql/loader.rb
index 094fbba2..bbd1e324 100644
--- a/lib/goo/sparql/loader.rb
+++ b/lib/goo/sparql/loader.rb
@@ -138,7 +138,6 @@ def get_includes(collection, graphs, incl, klass, query_options)
           properties_to_include = {}
           incl.each do |attr|
             graph, pattern = query_pattern(klass, attr, collection: collection)
-            add_rules(attr, klass, query_options)
             if klass.attributes(:all).include?(attr)
               properties_to_include[attr] = { uri: pattern[1], is_inverse: klass.inverse?(attr) } # [property_attr, property_uri , inverse: true]
             end
diff --git a/lib/goo/sparql/mixins/query_pattern.rb b/lib/goo/sparql/mixins/query_pattern.rb
index cc370795..9ee0df7d 100644
--- a/lib/goo/sparql/mixins/query_pattern.rb
+++ b/lib/goo/sparql/mixins/query_pattern.rb
@@ -3,9 +3,6 @@ module SPARQL
     module QueryPatterns
 
 
-      def add_rules(attr,klass,query_options)
-        (query_options[:rules] ||= []) << :SUBC  if klass.transitive?(attr)
-      end
 
       def query_pattern(klass,attr,**opts)
         value = opts[:value] || nil
diff --git a/lib/goo/sparql/query_builder.rb b/lib/goo/sparql/query_builder.rb
index 31880859..a839577f 100644
--- a/lib/goo/sparql/query_builder.rb
+++ b/lib/goo/sparql/query_builder.rb
@@ -14,7 +14,6 @@ def initialize(options)
         @unions = options[:unions] || []
         @aggregate = options[:aggregate]
         @collection = options[:collection]
-        @model_query_options = options[:query_options]
         @enable_rules = options[:rules]
         @order_by = options[:order_by]
         @internal_variables_map = {}
@@ -61,16 +60,6 @@ def build_select_query(ids, variables, graphs, patterns,
         count if @count
         paginate if @page
 
-        ## TODO see usage of rules and query_options
-        query_options.merge!(@model_query_options) if @model_query_options
-        query_options[:rules] = [:NONE] unless @enable_rules
-        query_options = nil if query_options.empty?
-        if query_options
-          query_options[:rules] = query_options[:rules]&.map { |x| x.to_s }.join('+')
-        else
-          query_options = { rules: ['NONE'] }
-        end
-        @query.options[:query_options] = query_options
         [@query, aggregate_projections]
       end
 
@@ -182,7 +171,7 @@ def patterns_for_match(klass, attr, value, graphs, patterns, unions,
           @internal_variables_map[new_internal_var] = value.empty? ? attr : {attr => value}
         end
 
-        add_rules(attr, klass, query_options)
+
         graph, pattern =
           query_pattern(klass, attr, value: new_internal_var, subject: subject, collection: collection)
         if pattern

From 5fa9dd3ee554d06a99d1d1dca7bb86c3a7f4ef2e Mon Sep 17 00:00:00 2001
From: Syphax bouazzouni <gs_bouazzouni@esi.dz>
Date: Mon, 29 Jan 2024 20:36:11 +0100
Subject: [PATCH 163/168] move expand_equivalent_predicates from loader to
 builder module

---
 lib/goo/sparql/loader.rb        | 10 ----------
 lib/goo/sparql/query_builder.rb | 19 +++++++++++++++++--
 2 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/lib/goo/sparql/loader.rb b/lib/goo/sparql/loader.rb
index bbd1e324..c8eb5edd 100644
--- a/lib/goo/sparql/loader.rb
+++ b/lib/goo/sparql/loader.rb
@@ -102,16 +102,6 @@ def model_load_sliced(*options)
         def set_request_lang(options)
           options[:requested_lang] = RequestStore.store[:requested_lang]
         end
-        def expand_equivalent_predicates(properties_to_include, eq_p)
-
-          return unless eq_p && !eq_p.empty?
-
-          properties_to_include&.each do |property_attr, property|
-            property_uri = property[:uri]
-            property[:equivalents] = eq_p[property_uri.to_s].to_a.map { |p| RDF::URI.new(p) } if eq_p.include?(property_uri.to_s)
-          end
-
-        end
 
         def predicate_map(predicates)
           predicates_map = nil
diff --git a/lib/goo/sparql/query_builder.rb b/lib/goo/sparql/query_builder.rb
index a839577f..bd63a218 100644
--- a/lib/goo/sparql/query_builder.rb
+++ b/lib/goo/sparql/query_builder.rb
@@ -17,12 +17,17 @@ def initialize(options)
         @enable_rules = options[:rules]
         @order_by = options[:order_by]
         @internal_variables_map = {}
+        @equivalent_predicates = options[:equivalent_predicates]
+        @properties_to_include = options[:properties_to_include]
         @query = get_client
       end
 
-      def build_select_query(ids, variables, graphs, patterns,
-                             query_options, properties_to_include)
+      def build_query(ids, variables, graphs, patterns)
+        query_options = {}
 
+        expand_equivalent_predicates(@properties_to_include, @equivalent_predicates)
+
+        properties_to_include = @properties_to_include
         patterns = graph_match(@collection, @graph_match, graphs, @klass, patterns, query_options, @unions)
         variables, patterns = add_some_type_to_id(patterns, query_options, variables)
         aggregate_projections, aggregate_vars, variables, optional_patterns = get_aggregate_vars(@aggregate, @collection, graphs, @klass, @unions, variables)
@@ -413,6 +418,16 @@ def add_some_type_to_id(patterns, query_options, variables)
       def internal_variables
         @internal_variables_map.keys
       end
+
+      def expand_equivalent_predicates(query_properties, eq_p)
+
+        return unless eq_p && !eq_p.empty?
+
+        query_properties&.each do |_, property|
+          property_uri = property[:uri]
+          property[:equivalents] = eq_p[property_uri.to_s].to_a.map { |p| RDF::URI.new(p) } if eq_p.include?(property_uri.to_s)
+        end
+      end
     end
   end
 end

From 45311c614fdc075f26eb328e9528cf884bc859e8 Mon Sep 17 00:00:00 2001
From: Syphax bouazzouni <gs_bouazzouni@esi.dz>
Date: Mon, 29 Jan 2024 20:39:51 +0100
Subject: [PATCH 164/168] build two diffirent queries depending on which
 backend used

---
 lib/goo/sparql/query_builder.rb | 66 +++++++++++++++++++++------------
 1 file changed, 42 insertions(+), 24 deletions(-)

diff --git a/lib/goo/sparql/query_builder.rb b/lib/goo/sparql/query_builder.rb
index bd63a218..1a6ea740 100644
--- a/lib/goo/sparql/query_builder.rb
+++ b/lib/goo/sparql/query_builder.rb
@@ -34,13 +34,11 @@ def build_query(ids, variables, graphs, patterns)
         query_filter_str, patterns, optional_patterns, filter_variables =
           filter_query_strings(@collection, graphs, @klass, optional_patterns, patterns, @query_filters)
         @order_by, variables, optional_patterns = init_order_by(@count, @klass, @order_by, optional_patterns, variables,patterns, query_options, graphs)
-        order_by_str, order_variables = order_by_string
-
 
         variables = [] if @count
         variables.delete :some_type
 
-        select_distinct(variables, aggregate_projections, filter_variables, order_variables)
+        select_distinct(variables, aggregate_projections, filter_variables)
           .from(graphs)
           .where(patterns)
           .union_bind_in_where(properties_to_include)
@@ -58,7 +56,7 @@ def build_query(ids, variables, graphs, patterns)
         ids_filter(ids) if ids
 
 
-        @query.order_by(*order_by_str) if @order_by
+        @query.order_by(*order_by_string) if @order_by
 
 
         put_query_aggregate_vars(aggregate_vars) if aggregate_vars
@@ -70,21 +68,31 @@ def build_query(ids, variables, graphs, patterns)
 
       def union_bind_in_where(properties)
         binding_as = []
-        properties.each do |property_attr, property|
-          predicates = [property[:uri]] + (property[:equivalents] || [])
-          options = {
-            binds: [{ value: property_attr, as: :attributeProperty }]
-          }
-          subject = property[:subject] || :id
-          predicates.uniq.each do |predicate_uri|
-            pattern = if property[:is_inverse]
-                        [:attributeObject, predicate_uri, subject]
-                      else
-                        [subject, predicate_uri, :attributeObject]
-                      end
-            binding_as << [[pattern], options]
+        if Goo.backend_4s? || Goo.backend_gb?
+          properties.each do |property_attr, property|
+            predicates = [property[:uri]] + (property[:equivalents] || [])
+            options = {
+              binds: [{ value: property_attr, as: :attributeProperty }]
+            }
+            subject = property[:subject] || :id
+            predicates.uniq.each do |predicate_uri|
+              pattern = if property[:is_inverse]
+                          [:attributeObject, predicate_uri, subject]
+                        else
+                          [subject, predicate_uri, :attributeObject]
+                        end
+              binding_as << [[pattern], options]
+            end
           end
+
+        else
+          direct_predicate, inverse_predicate = include_properties
+          direct_filter = direct_predicate.empty? ? [] : [{ values: direct_predicate, predicate: :attributeProperty }]
+          inverse_filter = inverse_predicate.empty? ? [] : [{ values: inverse_predicate, predicate: :attributeProperty }]
+          binding_as << [[[:id, :attributeProperty, :attributeObject]], { filters:  direct_filter}] unless direct_filter.empty?
+          binding_as << [[[:inverseAttributeObject, :attributeProperty, :id]], { filters: inverse_filter}] unless inverse_filter.empty?
         end
+
         @query.optional_union_with_bind_as(*binding_as) unless binding_as.empty?
         self
       end
@@ -121,7 +129,7 @@ def order_by_string
           order_variables << attr
           "#{order.to_s.upcase}(?#{attr})"
         end
-        [order_str,order_variables]
+        order_str
       end
 
       def from(graphs)
@@ -136,11 +144,11 @@ def from(graphs)
         self
       end
 
-      def select_distinct(variables, aggregate_variables, filter_variables, order_variables)
+      def select_distinct(variables, aggregate_patterns, filter_variables)
+        variables << :inverseAttributeObject if inverse_predicate?
         select_vars = variables.dup
-        reject_aggregations_from_vars(select_vars, aggregate_variables) if aggregate_variables
-        # Fix for 4store pagination with a filter https://github.com/ontoportal-lirmm/ontologies_api/issues/25
-        select_vars = (select_vars + filter_variables + order_variables).uniq  if @page
+        reject_aggregations_from_vars(select_vars, aggregate_patterns) if aggregate_patterns
+        select_vars = (select_vars + filter_variables).uniq  if @page && Goo.backend_4s? # Fix for 4store pagination with a filter
         @query = @query.select(*select_vars).distinct(true)
         self
       end
@@ -158,6 +166,16 @@ def ids_filter(ids)
 
       private
 
+      def include_properties
+        direct_predicates = @properties_to_include.select { |_, property| !property[:is_inverse] }.map { |_, property| [property[:uri]] + (property[:equivalents] || []) }.flatten
+        inverse_predicates = @properties_to_include.select { |_, property| property[:is_inverse] }.map { |_, property| [property[:uri]] + (property[:equivalents] || []) }.flatten
+        [direct_predicates, inverse_predicates]
+      end
+
+      def inverse_predicate?
+        @properties_to_include.any? { |_, property| property[:is_inverse] }
+      end
+
       def patterns_for_match(klass, attr, value, graphs, patterns, unions,
                              internal_variables, subject = :id, in_union = false,
                              in_aggregate = false, query_options = {}, collection = nil)
@@ -364,7 +382,7 @@ def query_filter_sparql(klass, filter, filter_patterns, filter_graphs,
             else
               value = RDF::Literal.new(filter_operation.value)
               if filter_operation.value.is_a? String
-                value = RDF::Literal.new(filter_operation.value, :datatype => RDF::XSD.string)
+                value = RDF::Literal.new(filter_operation.value)
               end
               filter_operations << (
                 "?#{filter_var.to_s} #{sparql_op_string(filter_operation.operator)} " +
@@ -397,7 +415,7 @@ def filter_query_strings(collection, graphs, klass,
               patterns.concat(filter_patterns)
             end
           end
-          filter_variables << inspected_patterns.values.last
+          #filter_variables << inspected_patterns.values.last
         end
         [query_filter_str, patterns, optional_patterns, filter_variables]
       end

From 4ff8361124f44a6613180529c4fb86a7054a4ede Mon Sep 17 00:00:00 2001
From: Syphax bouazzouni <gs_bouazzouni@esi.dz>
Date: Mon, 29 Jan 2024 20:41:27 +0100
Subject: [PATCH 165/168] update mapper to handle the two different queries
 depending on the backend used

---
 lib/goo/sparql/solutions_mapper.rb | 110 +++++++++++++++++------------
 1 file changed, 64 insertions(+), 46 deletions(-)

diff --git a/lib/goo/sparql/solutions_mapper.rb b/lib/goo/sparql/solutions_mapper.rb
index 64d258d5..d849be3a 100644
--- a/lib/goo/sparql/solutions_mapper.rb
+++ b/lib/goo/sparql/solutions_mapper.rb
@@ -3,39 +3,36 @@ module SPARQL
     class SolutionMapper
       BNODES_TUPLES = Struct.new(:id, :attribute)
 
-      def initialize(aggregate_projections, bnode_extraction, embed_struct,
-                      incl_embed, klass_struct, models_by_id,
-                     properties_to_include, unmapped, variables, ids, options)
+      def initialize(aggregate_projections, bnode_extraction, embed_struct,incl_embed, klass_struct, models_by_id, variables, options)
 
         @aggregate_projections = aggregate_projections
         @bnode_extraction = bnode_extraction
         @embed_struct = embed_struct
         @incl_embed = incl_embed
+        @incl = options[:include]
         @klass_struct = klass_struct
         @models_by_id = models_by_id
-        @properties_to_include = properties_to_include
-        @unmapped = unmapped
+        @properties_to_include = options[:properties_to_include]
+        @unmapped = options[:include] && options[:include].first.eql?(:unmapped)
         @variables = variables
-        @ids = ids
+        @ids = models_by_id.keys
         @klass = options[:klass]
         @read_only = options[:read_only]
-        @incl = options[:include]
         @count = options[:count]
         @collection = options[:collection]
         @options = options
       end
-      
+
       def map_each_solutions(select)
         found = Set.new
         objects_new = {}
         list_attributes = Set.new(@klass.attributes(:list))
-        all_attributes = Set.new(@klass.attributes(:all))
 
         @lang_filter = Goo::SPARQL::Solution::LanguageFilter.new(requested_lang: @options[:requested_lang].to_s, unmapped: @unmapped,
-           list_attributes: list_attributes)
-        
+                                                                 list_attributes: list_attributes)
+
         select.each_solution do |sol|
-          
+
           next if sol[:some_type] && @klass.type_uri(@collection) != sol[:some_type]
           return sol[:count_var].object if @count
 
@@ -59,26 +56,32 @@ def map_each_solutions(select)
             next
           end
 
-          predicate = sol[:attributeProperty].to_s.to_sym
+          predicates = find_predicate(sol[:attributeProperty], inverse: !sol[:inverseAttributeObject].nil?)
+          next if predicates.empty?
 
-          next if predicate.nil? || !all_attributes.include?(predicate)
+          object = if sol[:attributeObject]
+                     sol[:attributeObject]
+                   elsif sol[:inverseAttributeObject]
+                     sol[:inverseAttributeObject]
+                   end
 
-          object = sol[:attributeObject]
 
-          # bnodes
-          if bnode_id?(object, predicate)
-            objects_new = bnode_id_tuple(id, object, objects_new, predicate)
-            next
+          predicates.each do |predicate|
+            # bnodes
+            if bnode_id?(object, predicate)
+              objects_new = bnode_id_tuple(id, object, objects_new, predicate)
+              next
+            end
+
+            objects, objects_new = get_value_object(id, objects_new, object, list_attributes, predicate)
+            add_object_to_model(id, objects, object, predicate)
           end
 
-          objects, objects_new = get_value_object(id, objects_new, object, list_attributes, predicate)
-          add_object_to_model(id, objects, object, predicate)
         end
-      
-        # for this moment we are not going to enrich models , maybe we will use it if the results are empty  
+        # for this moment we are not going to enrich models , maybe we will use it if the results are empty
         @lang_filter.fill_models_with_all_languages(@models_by_id)
 
-        init_unloaded_attributes(found, list_attributes)
+        init_unloaded_attributes(list_attributes)
 
         return @models_by_id if @bnode_extraction
 
@@ -97,19 +100,30 @@ def map_each_solutions(select)
         include_bnodes(blank_nodes, @models_by_id) unless blank_nodes.empty?
 
         models_unmapped_to_array(@models_by_id) if @unmapped
-        
-       
+
+
         @models_by_id
       end
 
       private
 
-      def init_unloaded_attributes(found, list_attributes)
-        return if @incl.nil?
+      def find_predicate(predicate, unmapped: false, inverse: false)
+        if Goo.backend_4s? || Goo.backend_gb?
+          return [] if predicate.nil? || unmapped && @properties_to_include[predicate].nil?
+          predicate = predicate.to_s.to_sym
+        else
+          predicate = @properties_to_include.select { |x, v| v[:uri].to_s.eql?(predicate.to_s) || v[:equivalents]&.any? { |e| e.to_s.eql?(predicate.to_s) } }
+          return [] if predicate.empty?
 
+          predicate = predicate.select{|x, y| y[:is_inverse]&.eql?(inverse)}.keys
+        end
+        Array(predicate)
+      end
+
+      def init_unloaded_attributes(list_attributes)
+        return if @incl.nil? || @incl.empty?
         # Here we are setting to nil all attributes that have been included but not found in the triplestore
-        found.uniq.each do |model_id|
-          m = @models_by_id[model_id]
+        @models_by_id.each do |id, m|
           @incl.each do |attr_to_incl|
             is_handler = m.respond_to?(:handler?) && m.class.handler?(attr_to_incl)
             next if attr_to_incl.to_s.eql?('unmapped') || is_handler
@@ -133,7 +147,7 @@ def init_unloaded_attributes(found, list_attributes)
       def get_value_object(id, objects_new, object, list_attributes, predicate)
         object = object.object if object && !(object.is_a? RDF::URI)
         range_for_v = @klass.range(predicate)
-       
+
 
         if object.is_a?(RDF::URI) && (predicate != :id) && !range_for_v.nil?
           if objects_new.include?(object)
@@ -156,7 +170,7 @@ def get_value_object(id, objects_new, object, list_attributes, predicate)
 
           if object.nil?
             object = pre.nil? ? [] : pre
-          else            
+          else
             object = pre.nil? ? [object] : (Array(pre).dup << object)
             object.uniq!
           end
@@ -170,8 +184,8 @@ def add_object_to_model(id, objects, current_obj, predicate)
         if @models_by_id[id].respond_to?(:klass)
           @models_by_id[id][predicate] = objects unless objects.nil? && !@models_by_id[id][predicate].nil?
         elsif !@models_by_id[id].class.handler?(predicate) &&
-              !(objects.nil? && !@models_by_id[id].instance_variable_get("@#{predicate}").nil?) &&
-              predicate != :id
+          !(objects.nil? && !@models_by_id[id].instance_variable_get("@#{predicate}").nil?) &&
+          predicate != :id
           @lang_filter.set_model_value(@models_by_id[id], predicate, objects, current_obj)
         end
       end
@@ -329,7 +343,7 @@ def model_set_collection_attributes(models_by_id, objects_new)
             collection_attribute = obj_new[:klass].collection_opts
             obj_new[collection_attribute] = collection_value
           elsif obj_new.class.respond_to?(:collection_opts) &&
-                obj_new.class.collection_opts.instance_of?(Symbol)
+            obj_new.class.collection_opts.instance_of?(Symbol)
             collection_attribute = obj_new.class.collection_opts
             obj_new.send("#{collection_attribute}=", collection_value)
           end
@@ -370,8 +384,8 @@ def dependent_model_creation(embed_struct, id, models_by_id, object, objects_new
             if objects_new.include?(object)
               object = objects_new[object]
             elsif !range_for_v.inmutable?
-              pre_val = get_pre_val(id, models_by_id, object, v, read_only)
-              object = get_object_from_range(pre_val, embed_struct, object, objects_new, v, options)
+              pre_val = get_pre_val(id, models_by_id, object, v)
+              object = get_object_from_range(pre_val, embed_struct, object, objects_new, v)
             else
               object = range_for_v.find(object).first
             end
@@ -399,8 +413,8 @@ def get_object_from_range(pre_val, embed_struct, object, objects_new, predicate)
       def get_pre_val(id, models_by_id, object, predicate)
         pre_val = nil
         if models_by_id[id] &&
-           ((models_by_id[id].respond_to?(:klass) && models_by_id[id]) ||
-             models_by_id[id].loaded_attributes.include?(predicate))
+          ((models_by_id[id].respond_to?(:klass) && models_by_id[id]) ||
+            models_by_id[id].loaded_attributes.include?(predicate))
           pre_val = if !@read_only
                       models_by_id[id].instance_variable_get("@#{predicate}")
                     else
@@ -413,13 +427,17 @@ def get_pre_val(id, models_by_id, object, predicate)
       end
 
       def add_unmapped_to_model(sol)
-        predicate = sol[:attributeProperty].to_s.to_sym
-        return unless @properties_to_include[predicate]
-
-        id = sol[:id]
-        value = sol[:attributeObject]
-
-        @lang_filter.set_unmapped_value(@models_by_id[id], @properties_to_include[predicate][:uri], value)
+        predicates = find_predicate(sol[:attributeProperty])
+        predicates.each do |predicate|
+          if Goo.backend_4s? || Goo.backend_gb?
+            predicate = @properties_to_include[predicate][:uri]
+          else
+            predicate = sol[:attributeProperty]
+          end
+          id = sol[:id]
+          value = sol[:attributeObject]
+          @lang_filter.set_unmapped_value(@models_by_id[id], predicate, value)
+        end
       end
 
       def add_aggregations_to_model(sol)

From a57831b08f225808d2da60531ea267f3ad50b325 Mon Sep 17 00:00:00 2001
From: Syphax bouazzouni <gs_bouazzouni@esi.dz>
Date: Mon, 29 Jan 2024 20:42:12 +0100
Subject: [PATCH 166/168] simplify the loader code, by removing inferable
 variables

---
 lib/goo/sparql/loader.rb | 52 +++++++++++++++-------------------------
 1 file changed, 19 insertions(+), 33 deletions(-)

diff --git a/lib/goo/sparql/loader.rb b/lib/goo/sparql/loader.rb
index c8eb5edd..f3dcdb3d 100644
--- a/lib/goo/sparql/loader.rb
+++ b/lib/goo/sparql/loader.rb
@@ -36,63 +36,48 @@ def model_load(*options)
         ##
         def model_load_sliced(*options)
           options = options.last
-          ids = options[:ids]
           klass = options[:klass]
           incl = options[:include]
           models = options[:models]
-          aggregate = options[:aggregate]
-          read_only = options[:read_only]
           collection = options[:collection]
-          count = options[:count]
-          include_pagination = options[:include_pagination]
-          equivalent_predicates = options[:equivalent_predicates]
-          predicates = options[:predicates]
 
-          embed_struct, klass_struct = get_structures(aggregate, count, incl, include_pagination, klass, read_only)
-
-          raise_resource_must_persistent_error(models) if models
+          embed_struct, klass_struct = get_structures(options[:aggregate],  options[:count] , incl, options[:include_pagination], klass, options[:read_only])
+          raise_not_persistent_error(models) if models
 
           graphs = get_graphs(collection, klass)
-          ids, models_by_id = get_models_by_id_hash(ids, klass, klass_struct, models)
+          models_by_id = get_models_by_id_hash( options[:ids], klass, klass_struct, models)
 
-          query_options = {}
           #TODO: breaks the reasoner
           patterns = [[:id, RDF.type, klass.uri_type(collection)]]
 
           incl_embed = nil
-          unmapped = nil
           bnode_extraction = nil
           properties_to_include = []
           variables = [:id]
-          if incl
-            if incl.first && incl.first.is_a?(Hash) && incl.first.include?(:bnode)
+          if incl && !incl.empty?
+            if incl.first.is_a?(Hash) && incl.first.include?(:bnode)
               #limitation only one level BNODE
               bnode_extraction, patterns, variables = get_bnode_extraction(collection, incl, klass, patterns)
             else
               variables = %i[id attributeProperty attributeObject]
               if incl.first == :unmapped
-                unmapped = true
-                properties_to_include = predicate_map(predicates)
+                properties_to_include = predicate_map(options[:predicates])
               else
-                #make it deterministic
-                incl_embed = get_embed_includes(incl)
-                graphs, properties_to_include, query_options = get_includes(collection, graphs, incl,
-                                                             klass, query_options)
+                graphs, properties_to_include, incl_embed = get_includes(collection, graphs, incl, klass)
               end
             end
           end
 
-          expand_equivalent_predicates(properties_to_include, equivalent_predicates)
 
-          query_builder = Goo::SPARQL::QueryBuilder.new options
-          select, aggregate_projections = query_builder.build_select_query(ids, variables, graphs,
-                                                                           patterns, query_options,
-                                                                           properties_to_include)
+          options[:properties_to_include] = properties_to_include
+
+
+          select, aggregate_projections = Goo::SPARQL::QueryBuilder.new(options)
+                                                                   .build_query(models_by_id.keys, variables, graphs, patterns)
 
           solution_mapper = Goo::SPARQL::SolutionMapper.new aggregate_projections, bnode_extraction,
                                                             embed_struct, incl_embed, klass_struct, models_by_id,
-                                                            properties_to_include, unmapped,
-                                                            variables, ids, options
+                                                            variables, options
 
           solution_mapper.map_each_solutions(select)
         end
@@ -122,7 +107,8 @@ def predicate_map(predicates)
           predicates_map
         end
 
-        def get_includes(collection, graphs, incl, klass, query_options)
+        def get_includes(collection, graphs, incl, klass)
+          incl_embed ,incl = get_embed_includes(incl)
           incl = incl.to_a
           incl.delete_if { |a| !a.instance_of?(Symbol) }
           properties_to_include = {}
@@ -133,7 +119,7 @@ def get_includes(collection, graphs, incl, klass, query_options)
             end
             graphs << graph if graph && (!klass.collection_opts || klass.inverse?(attr))
           end
-          [graphs, properties_to_include,query_options]
+          [graphs, properties_to_include, incl_embed]
         end
 
         def get_bnode_extraction(collection, incl, klass, patterns)
@@ -170,7 +156,7 @@ def get_models_by_id_hash(ids, klass, klass_struct, models)
             #a where without models
 
           end
-          return ids, models_by_id
+          models_by_id
         end
 
         def get_graphs(collection, klass)
@@ -223,7 +209,7 @@ def get_structures(aggregate, count, incl, include_pagination, klass, read_only)
           [embed_struct, klass_struct]
         end
 
-        def raise_resource_must_persistent_error(models)
+        def raise_not_persistent_error(models)
           models.each do |m|
             if (not m.nil?) && !m.respond_to?(:klass) #read only
               raise ArgumentError,
@@ -241,7 +227,7 @@ def get_embed_includes(incl)
             #variables.concat(embed_variables)
             incl.concat(embed_variables)
           end
-          incl_embed
+          [incl_embed, incl]
         end
       end
 

From 970ee7827a46d7b132485426ff311ae1781bb338 Mon Sep 17 00:00:00 2001
From: Syphax bouazzouni <gs_bouazzouni@esi.dz>
Date: Mon, 29 Jan 2024 20:51:52 +0100
Subject: [PATCH 167/168] refactor and simplify map_attributes method

---
 lib/goo/base/resource.rb | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/lib/goo/base/resource.rb b/lib/goo/base/resource.rb
index 3f18c677..4d497f8b 100644
--- a/lib/goo/base/resource.rb
+++ b/lib/goo/base/resource.rb
@@ -378,25 +378,26 @@ def self.map_attributes(inst,equivalent_predicates=nil, include_languages: false
           next if inst.class.collection?(attr) #collection is already there
           next unless inst.respond_to?(attr)
           attr_uri = klass.attribute_uri(attr,inst.collection).to_s
-          if unmapped_string_keys.include?(attr_uri.to_s) ||
-            (equivalent_predicates && equivalent_predicates.include?(attr_uri))
-            if !unmapped_string_keys.include?(attr_uri)
-              object = Array(equivalent_predicates[attr_uri].map { |eq_attr| unmapped_string_keys[eq_attr] }).flatten.compact
-              if include_languages && [RDF::URI, Hash].all?{|c| object.map(&:class).include?(c)}
-                object = object.reduce({})  do |all, new_v|
-                  new_v =  { none: [new_v] } if new_v.is_a?(RDF::URI)
-                  all.merge(new_v) {|_, a, b| a + b }
+          if unmapped_string_keys.include?(attr_uri.to_s) || equivalent_predicates&.include?(attr_uri)
+            object = nil
+
+            if unmapped_string_keys.include?(attr_uri)
+              object = unmapped_string_keys[attr_uri]
+            else
+              equivalent_predicates[attr_uri].each do |eq_attr|
+                next if unmapped_string_keys[eq_attr].nil?
+
+                if object.nil?
+                  object = unmapped_string_keys[eq_attr].dup
+                elsif object.is_a?(Array)
+                  object.concat(unmapped_string_keys[eq_attr])
                 end
-              elsif include_languages
-                object = object.first
               end
 
               if object.nil?
                 inst.send("#{attr}=", list_attrs.include?(attr) ? [] : nil, on_load: true)
                 next
               end
-            else
-              object = unmapped_string_keys[attr_uri]
             end
 
             if object.is_a?(Hash)

From 09c944f08bb92507b528a95c56bf4b6e3da33291 Mon Sep 17 00:00:00 2001
From: Syphax bouazzouni <gs_bouazzouni@esi.dz>
Date: Mon, 29 Jan 2024 21:35:35 +0100
Subject: [PATCH 168/168] fix test chunks write concenrency issues

---
 test/test_chunks_write.rb | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/test/test_chunks_write.rb b/test/test_chunks_write.rb
index 1a116aca..afee11f1 100644
--- a/test/test_chunks_write.rb
+++ b/test/test_chunks_write.rb
@@ -77,8 +77,8 @@ def test_reentrant_queries
 
       tput = Thread.new {
         Goo.sparql_data_client.put_triples(ONT_ID_EXTRA, ntriples_file_path, mime_type="application/x-turtle")
-        sleep(1.5)
       }
+
       count_queries = 0
       tq = Thread.new {
        5.times do
@@ -94,16 +94,16 @@ def test_reentrant_queries
       assert_equal 5, count_queries
       tput.join
 
-      triples_no_bnodes = 25256
+
       count = "SELECT (count(?s) as ?c) WHERE { GRAPH <#{ONT_ID_EXTRA}> { ?s ?p ?o }}"
       Goo.sparql_query_client.query(count).each do |sol|
-        assert_equal triples_no_bnodes, sol[:c].object
+        assert_includes [25256, 50512], sol[:c].object
       end
 
       tdelete = Thread.new {
         Goo.sparql_data_client.delete_graph(ONT_ID_EXTRA)
-        sleep(1.5)
       }
+
       count_queries = 0
       tq = Thread.new {
        5.times do
@@ -115,9 +115,8 @@ def test_reentrant_queries
        end
       }
       tq.join
-      assert tdelete.alive?
-      assert_equal 5, count_queries
       tdelete.join
+      assert_equal 5, count_queries
 
       count = "SELECT (count(?s) as ?c) WHERE { GRAPH <#{ONT_ID_EXTRA}> { ?s ?p ?o }}"
       Goo.sparql_query_client.query(count).each do |sol|
@@ -140,18 +139,19 @@ def test_query_flood
           50.times do |j|
             oq = "SELECT (count(?s) as ?c) WHERE { ?s a ?o }"
             Goo.sparql_query_client.query(oq).each do |sol|
-              assert sol[:c].object > 0
+              refute_equal 0, sol[:c]
             end
           end
         }
       end
 
+      threads.join
+
       if Goo.backend_4s?
         log_status = []
         Thread.new {
           10.times do |i|
             log_status << Goo.sparql_query_client.status
-            sleep(1.2)
           end
         }