WIP lib: Better extensible attrsets

lib/customisation.nix

@@ -192,15 +192,13 @@ rec {
      provided by `newScope' and the set provides a `newScope' attribute
      which can form the parent scope for later package sets. */
   makeScope = newScope: f:
-    let self = f self // {
-          newScope = scope: newScope (self // scope);
-          callPackage = self.newScope {};
-          overrideScope = g: lib.warn
-            "`overrideScope` (from `lib.makeScope`) is deprecated. Do `overrideScope' (self: super: { … })` instead of `overrideScope (super: self: { … })`. All other overrides have the parameters in that order, including other definitions of `overrideScope`. This was the only definition violating the pattern."
-            (makeScope newScope (lib.fixedPoints.extends (lib.flip g) f));
-          overrideScope' = g: makeScope newScope (lib.fixedPoints.extends g f);
-          packages = f;
-        };
-    in self;
-
+    lib.overrides.makeExtensibleAttrset (self: super: f self // {
+      newScope = scope: newScope (self // scope);
+      callPackage = self.newScope {};
+      overrideScope = g: lib.warn
+        "`overrideScope` (from `lib.makeScope`) is deprecated. Do `overrideScope' (self: super: { … })` instead of `overrideScope (super: self: { … })`. All other overrides have the parameters in that order, including other definitions of `overrideScope`. This was the only definition violating the pattern."
+        (makeScope newScope (lib.overrides.extends (lib.flip g) f));
+      overrideScope' = g: makeScope newScope (lib.overrides.extends g f);
+      packages = f;
+    });
 }

lib/default.nix

@@ -4,10 +4,11 @@
  * for new functions in `./tests.nix'.
  */
 let
+  bootstrapLib = mkLib bootstrapLib;
 
-  inherit (import ./fixed-points.nix {}) makeExtensible;
+  lib = bootstrapLib.makeExtensible mkLib;
 
-  lib = makeExtensible (self: let
+  mkLib = self: let
     callLibs = file: import file { lib = self; };
   in with self; {
 
@@ -20,6 +21,8 @@ let
     lists = callLibs ./lists.nix;
     strings = callLibs ./strings.nix;
     stringsWithDeps = callLibs ./strings-with-deps.nix;
+    monoid = callLibs ./monoid.nix;
+    overrides = callLibs ./overrides.nix;
 
     # packaging
     customisation = callLibs ./customisation.nix;
@@ -61,8 +64,7 @@ let
       boolToString mergeAttrs flip mapNullable inNixShell min max
       importJSON warn info nixpkgsVersion version mod compare
       splitByAndCompare functionArgs setFunctionArgs isFunction;
-    inherit (fixedPoints) fix fix' extends composeExtensions
-      makeExtensible makeExtensibleWithCustomName;
+    inherit (fixedPoints) fix fix';
     inherit (attrsets) attrByPath hasAttrByPath setAttrByPath
       getAttrFromPath attrVals attrValues catAttrs filterAttrs
       filterAttrsRecursive foldAttrs collect nameValuePair mapAttrs
@@ -90,6 +92,7 @@ let
       toInt readPathsFromFile fileContents;
     inherit (stringsWithDeps) textClosureList textClosureMap
       noDepEntry fullDepEntry packEntry stringAfter;
+    inherit (overrides) extends composeExtensions makeExtensible;
     inherit (customisation) overrideDerivation makeOverridable
       callPackageWith callPackagesWith extendDerivation hydraJob
       makeScope;
@@ -134,5 +137,5 @@ let
       mergeAttrsNoOverride mergeAttrByFunc mergeAttrsByFuncDefaults
       mergeAttrsByFuncDefaultsClean mergeAttrBy prepareDerivationArgs
       nixType imap overridableDelayableArgs;
-  });
+  };
 in lib

lib/fixed-points.nix

@@ -23,69 +23,4 @@ rec {
   # implement deep overriding. See pkgs/development/haskell-modules/default.nix
   # for a concrete example.
   fix' = f: let x = f x // { __unfix__ = f; }; in x;
-
-  # Modify the contents of an explicitly recursive attribute set in a way that
-  # honors `self`-references. This is accomplished with a function
-  #
-  #     g = self: super: { foo = super.foo + " + "; }
-  #
-  # that has access to the unmodified input (`super`) as well as the final
-  # non-recursive representation of the attribute set (`self`). `extends`
-  # differs from the native `//` operator insofar as that it's applied *before*
-  # references to `self` are resolved:
-  #
-  #     nix-repl> fix (extends g f)
-  #     { bar = "bar"; foo = "foo + "; foobar = "foo + bar"; }
-  #
-  # The name of the function is inspired by object-oriented inheritance, i.e.
-  # think of it as an infix operator `g extends f` that mimics the syntax from
-  # Java. It may seem counter-intuitive to have the "base class" as the second
-  # argument, but it's nice this way if several uses of `extends` are cascaded.
-  #
-  # To get a better understanding how `extends` turns a function with a fix
-  # point (the package set we start with) into a new function with a different fix
-  # point (the desired packages set) lets just see, how `extends g f`
-  # unfolds with `g` and `f` defined above:
-  #
-  # extends g f = self: let super = f self; in super // g self super;
-  #             = self: let super = { foo = "foo"; bar = "bar"; foobar = self.foo + self.bar; }; in super // g self super
-  #             = self: { foo = "foo"; bar = "bar"; foobar = self.foo + self.bar; } // g self { foo = "foo"; bar = "bar"; foobar = self.foo + self.bar; }
-  #             = self: { foo = "foo"; bar = "bar"; foobar = self.foo + self.bar; } // { foo = "foo" + " + "; }
-  #             = self: { foo = "foo + "; bar = "bar"; foobar = self.foo + self.bar; }
-  #
-  extends = f: rattrs: self: let super = rattrs self; in super // f self super;
-
-  # Compose two extending functions of the type expected by 'extends'
-  # into one where changes made in the first are available in the
-  # 'super' of the second
-  composeExtensions =
-    f: g: self: super:
-      let fApplied = f self super;
-          super' = super // fApplied;
-      in fApplied // g self super';
-
-  # Create an overridable, recursive attribute set. For example:
-  #
-  #     nix-repl> obj = makeExtensible (self: { })
-  #
-  #     nix-repl> obj
-  #     { __unfix__ = «lambda»; extend = «lambda»; }
-  #
-  #     nix-repl> obj = obj.extend (self: super: { foo = "foo"; })
-  #
-  #     nix-repl> obj
-  #     { __unfix__ = «lambda»; extend = «lambda»; foo = "foo"; }
-  #
-  #     nix-repl> obj = obj.extend (self: super: { foo = super.foo + " + "; bar = "bar"; foobar = self.foo + self.bar; })
-  #
-  #     nix-repl> obj
-  #     { __unfix__ = «lambda»; bar = "bar"; extend = «lambda»; foo = "foo + "; foobar = "foo + bar"; }
-  makeExtensible = makeExtensibleWithCustomName "extend";
-
-  # Same as `makeExtensible` but the name of the extending attribute is
-  # customized.
-  makeExtensibleWithCustomName = extenderName: rattrs:
-    fix' rattrs // {
-      ${extenderName} = f: makeExtensibleWithCustomName extenderName (extends f rattrs);
-   };
 }

lib/monoid.nix

@@ -0,0 +1,23 @@
+{ lib }:
+
+# TODO say what monoid is I suppose
+
+{
+
+  list = {
+    identity = [];
+    append = lib.trivial.concat;
+  };
+
+  attrset = {
+    identity = {};
+    append = lib.trivial.mergeAttrs;
+  };
+
+  inherit (lib.overrides) monoidalExtension attrsetDeep extensibleAttrset;
+
+  # Monoidally left-fold a list. The monoid's `append` and `identity` become the
+  # other arguments besides the list to the `foldl`.
+  fold = { identity, append, ... }: lib.foldl append identity;
+
+}

lib/overrides.nix

@@ -0,0 +1,107 @@
+{ lib }:
+
+# Overlays, overrides, and other mechanisms for extensiblilty.
+
+let brand = "extensible-attrset"; in
+
+rec {
+  #
+  monoidalExtension = { append, identity }: {
+    identity = _: _: identity;
+    append = f: g: self: super:
+      let fApplied = f self super;
+        super' = append super fApplied;
+      in append fApplied (g self super');
+  };
+
+  #
+  fixMondoidalExtension = { append, identity }:
+    override: lib.fix (self: override self identity);
+
+  #
+  extendMonoidal = { append, identity }:
+    f: rattrs: self: let super = rattrs self; in append super (f self super);
+
+  # TODO need builtin to speed this up
+  mergeExtensibleRecursive = lib.zipAttrsWith (n: vs:
+    if builtins.length vs > 1 && lib.all (a: (a.type or "") == brand) vs
+      then lib.monoid.fold extensibleAttrset vs
+      else lib.last vs);
+
+  #
+  attrsetDeep = {
+    identity = {};
+    append = old: new: mergeExtensibleRecursive [old new];
+  };
+
+  #
+  extensibleAttrset = {
+    identity = { __override__ = self: super: {}; };
+    append = old: new: makeExtensibleAttrset
+      (composeExtensions old.__override__ new.__override__);
+  };
+
+  makeExtensibleAttrset = __override__:
+    fixMondoidalExtension attrsetDeep __override__ // {
+      type = brand;
+      inherit __override__;
+    };
+
+  # Modify the contents of an explicitly recursive attribute set in a way that
+  # honors `self`-references. This is accomplished with a function
+  #
+  #     g = self: super: { foo = super.foo + " + "; }
+  #
+  # that has access to the unmodified input (`super`) as well as the final
+  # non-recursive representation of the attribute set (`self`). `extends`
+  # differs from the native `//` operator insofar as that it's applied *before*
+  # references to `self` are resolved:
+  #
+  #     nix-repl> fix (extends g f)
+  #     { bar = "bar"; foo = "foo + "; foobar = "foo + bar"; }
+  #
+  # The name of the function is inspired by object-oriented inheritance, i.e.
+  # think of it as an infix operator `g extends f` that mimics the syntax from
+  # Java. It may seem counter-intuitive to have the "base class" as the second
+  # argument, but it's nice this way if several uses of `extends` are cascaded.
+  #
+  # To get a better understanding how `extends` turns a function with a fix
+  # point (the package set we start with) into a new function with a different fix
+  # point (the desired packages set) lets just see, how `extends g f`
+  # unfolds with `g` and `f` defined above:
+  #
+  # extends g f = self: let super = f self; in super // g self super;
+  #             = self: let super = { foo = "foo"; bar = "bar"; foobar = self.foo + self.bar; }; in super // g self super
+  #             = self: { foo = "foo"; bar = "bar"; foobar = self.foo + self.bar; } // g self { foo = "foo"; bar = "bar"; foobar = self.foo + self.bar; }
+  #             = self: { foo = "foo"; bar = "bar"; foobar = self.foo + self.bar; } // { foo = "foo" + " + "; }
+  #             = self: { foo = "foo + "; bar = "bar"; foobar = self.foo + self.bar; }
+  #
+  extends = extendMonoidal lib.monoid.attrsetDeep;
+
+  # Compose two extending functions of the type expected by 'extends'
+  # into one where changes made in the first are available in the
+  # 'super' of the second
+  composeExtensions = (monoidalExtension lib.monoid.attrsetDeep).append;
+
+  # Create an overridable, recursive attribute set. For example:
+  #
+  #     nix-repl> obj = makeExtensible (self: { })
+  #
+  #     nix-repl> obj
+  #     { __unfix__ = «lambda»; extend = «lambda»; }
+  #
+  #     nix-repl> obj = obj.extend (self: super: { foo = "foo"; })
+  #
+  #     nix-repl> obj
+  #     { __unfix__ = «lambda»; extend = «lambda»; foo = "foo"; }
+  #
+  #     nix-repl> obj = obj.extend (self: super: { foo = super.foo + " + "; bar = "bar"; foobar = self.foo + self.bar; })
+  #
+  #     nix-repl> obj
+  #     { __unfix__ = «lambda»; bar = "bar"; extend = «lambda»; foo = "foo + "; foobar = "foo + bar"; }
+  makeExtensible = rattrs:
+    makeExtensibleAttrset (self: super: rattrs self) // {
+      __unfix__ = rattrs;
+      extend = f: makeExtensible (extends f rattrs);
+    };
+}

pkgs/development/compilers/llvm/3.7/default.nix

@@ -56,4 +56,4 @@ let
     libcxxabi = callPackage ./libc++abi.nix {};
   });
 
-in { inherit tools libraries; } // libraries // tools
+in stdenv.lib.makeExtensible (_: _: { inherit tools libraries; } // tools // libraries)

pkgs/development/compilers/llvm/3.8/default.nix

@@ -56,4 +56,4 @@ let
     libcxxabi = callPackage ./libc++abi.nix {};
   });
 
-in { inherit tools libraries; } // libraries // tools
+in stdenv.lib.makeExtensible (_: _: { inherit tools libraries; } // tools // libraries)

pkgs/development/compilers/llvm/3.9/default.nix

@@ -56,4 +56,4 @@ let
     libcxxabi = callPackage ./libc++abi.nix {};
   });
 
-in { inherit tools libraries; } // libraries // tools
+in stdenv.lib.makeExtensible (_: _: { inherit tools libraries; } // tools // libraries)

pkgs/development/compilers/llvm/4/default.nix

@@ -71,4 +71,4 @@ let
     openmp = callPackage ./openmp.nix {};
   });
 
-in { inherit tools libraries; } // libraries // tools
+in stdenv.lib.makeExtensible (_: _: { inherit tools libraries; } // tools // libraries)

pkgs/development/compilers/llvm/5/default.nix

@@ -89,4 +89,4 @@ let
     openmp = callPackage ./openmp.nix {};
   });
 
-in { inherit tools libraries; } // libraries // tools
+in stdenv.lib.makeExtensible (_: _: { inherit tools libraries; } // tools // libraries)

pkgs/development/compilers/llvm/6/default.nix

@@ -89,4 +89,4 @@ let
     openmp = callPackage ./openmp.nix {};
   });
 
-in { inherit tools libraries; } // libraries // tools
+in stdenv.lib.makeExtensible (_: _: { inherit tools libraries; } // tools // libraries)

pkgs/development/compilers/llvm/7/default.nix

@@ -89,4 +89,4 @@ let
     openmp = callPackage ./openmp.nix {};
   });
 
-in { inherit tools libraries; } // libraries // tools
+in stdenv.lib.makeExtensible (_: _: { inherit tools libraries; } // tools // libraries)