tests.setup: Quick unit test infra for setup.sh

Since the previous commit is already a mass-rebuild, I've taken the
opportunity to add important documentation to setup.sh.

pkgs/stdenv/generic/default.nix

@@ -17,7 +17,7 @@ argsStdenv@{ name ? "stdenv", preHook ? "", initialPath
   # (see all-packages.nix).
   fetchurlBoot
 
-, setupScript ? ./setup.sh
+, setupScript ? config.setupScript or ./setup.sh
 
 , extraNativeBuildInputs ? []
 , extraBuildInputs ? []

pkgs/stdenv/generic/setup.sh

@@ -1,5 +1,21 @@
 # shellcheck shell=bash
 # shellcheck disable=1090,2154,2123,2034,2178,2048,2068,1091
+
+# TESTING
+
+# This file can be tested without mass-rebuild.
+# See `pkgs/test/setup/default.nix`.
+
+# DOCUMENTATION
+
+# This file is sourced by the derivation `builder` and is responsible for
+# some shell setup, loading scripts from inputs, and running phases and hooks.
+#
+# Comments in this file are implementation oriented. User facing documentation
+# is maintained outside of this file, so that it can be improved without mass
+# rebuilds.
+# See https://nixos.org/manual/nixpkgs/unstable/index.html#chap-stdenv
+
 __nixpkgs_setup_set_original=$-
 set -eu
 set -o pipefail

pkgs/test/default.nix

@@ -86,6 +86,29 @@ with pkgs;
   stdenv-inputs = callPackage ./stdenv-inputs { };
   stdenv = callPackage ./stdenv { };
 
+  # Unit tests for setup.sh
+  setup = import ./setup {
+    # NB: ^^^^^^
+    #     `import` and not `callPackage`, because we manipulate the whole `pkgs`
+    #     so that we can unit test without rebuilding everything; see setup/default.nix.
+    basePkgs =
+      if ! pkgs.config?baseCommit then pkgs
+      else import ../.. {
+        inherit (pkgs.stdenv.hostPlatform) system;
+        config = {
+          # Set the setup script to the version in the base commit
+          # We need setup.sh to be a single file with the same store path as "${./setup.sh}",
+          # so that stdenv equals stdenv for the base commit,
+          # which turns out to be a bit complicated when it comes from a fetched source.
+          setupScript = "${
+            let oldPkgsStr = builtins.fetchGit { url = ../..; ref = pkgs.config.baseCommit; };
+                oldPkgsPath = /. + builtins.unsafeDiscardStringContext (oldPkgsStr);
+            in oldPkgsPath + "/pkgs/stdenv/generic/setup.sh"
+          }";
+        };
+    };
+  };
+
   hardeningFlags = recurseIntoAttrs (callPackage ./cc-wrapper/hardening.nix {});
   hardeningFlags-gcc = recurseIntoAttrs (callPackage ./cc-wrapper/hardening.nix {
     stdenv = gccStdenv;

pkgs/test/setup/default.nix

@@ -0,0 +1,219 @@
+# To run these tests, without bootstrapping:
+#
+#   nix-build -A tests.setup --arg config "{ baseCommit = ''$(git merge-base HEAD upstream/master)''; }"
+#
+# Note that `tests.stdenv` currently also contains relevant tests, which are slower, but could be ported here.
+
+{
+  # A Nixpkgs which may use the old setup.sh, for which we have binaries in the cache.
+  basePkgs
+}:
+
+let
+  stdenv = basePkgs.stdenv.override {
+    # Restore setup.sh, but only for new derivations
+    setupScript = ../../stdenv/generic/setup.sh;
+  };
+  inherit (basePkgs) lib testers;
+in
+lib.recurseIntoAttrs rec {
+
+  # A manual testing tool.
+  # This works for a few packages. Could be developed further, but it's a tarpit.
+  # Example:
+  #   nix-build -A tests.setup.try-override-shallow.pkgs.hello --arg config "{ baseCommit = ''$(git merge-base HEAD upstream/master)''; }"
+  try-override-shallow = {
+    # no recurseIntoAttrs!
+    pkgs = lib.mapAttrs (name: old:
+      let new = old.override { inherit stdenv; };
+      in
+        assert lib.isDerivation old;
+        assert old != new;
+        new) basePkgs;
+  };
+
+  test-happy = stdenv.mkDerivation {
+    name = "hi";
+    dontUnpack = true;
+    buildPhase = ":";
+    installPhase = ''
+      mkdir -p $out/bin $out/empty
+      echo foo >$out/regular
+      echo bar >$out/bin/script
+      chmod a+x $out/bin/script
+      ln -s ../regular $out/symlink
+    '';
+    doInstallCheck = true;
+  };
+
+  test-installPhase-modified-regular =
+    basePkgs.runCommand "test-installPhase-modified-regular" {
+      failure = testers.testBuildFailure (
+        test-happy.overrideAttrs {
+          installCheckPhase = ''
+            echo baz >$out/regular
+          '';
+        }
+      );
+    } ''
+      echo $failure/testBuildFailure.log
+      (
+        set -x
+        grep 'ERROR: Files were changed during installCheckPhase' $failure/testBuildFailure.log >/dev/null
+        grep "$failure/regular" $failure/testBuildFailure.log >/dev/null
+      )
+      touch $out
+    '';
+
+  test-installPhase-modified-exe-bit =
+    basePkgs.runCommand "test-installPhase-modified-exe-bit" {
+      failure = testers.testBuildFailure (
+        test-happy.overrideAttrs {
+          installCheckPhase = ''
+            chmod a-x $out/bin/script
+          '';
+        }
+      );
+    } ''
+      echo $failure/testBuildFailure.log
+      (
+        set -x
+        grep 'ERROR: Files were changed during installCheckPhase' $failure/testBuildFailure.log >/dev/null
+        grep "$failure/bin/script" $failure/testBuildFailure.log >/dev/null
+      )
+      touch $out
+    '';
+
+  test-installPhase-modified-new-file =
+    basePkgs.runCommand "test-installPhase-modified-new-file" {
+      failure = testers.testBuildFailure (
+        test-happy.overrideAttrs {
+          installCheckPhase = ''
+            echo baz >$out/new
+          '';
+        }
+      );
+    } ''
+      echo $failure/testBuildFailure.log
+      (
+        set -x
+        grep 'ERROR: Files were changed during installCheckPhase' $failure/testBuildFailure.log >/dev/null
+        grep "$failure/new" $failure/testBuildFailure.log >/dev/null
+      )
+      touch $out
+    '';
+
+  test-installPhase-modified-remove-empty-dir =
+    basePkgs.runCommand "test-installPhase-modified-remove-empty-dir" {
+      failure = testers.testBuildFailure (
+        test-happy.overrideAttrs {
+          installCheckPhase = ''
+            rmdir $out/empty
+          '';
+        }
+      );
+    } ''
+      echo $failure/testBuildFailure.log
+      (
+        set -x
+        grep 'ERROR: Files were changed during installCheckPhase' $failure/testBuildFailure.log >/dev/null
+        grep "$failure/empty" $failure/testBuildFailure.log >/dev/null
+      )
+      touch $out
+    '';
+
+  test-installPhase-modified-remove-regular =
+    basePkgs.runCommand "test-installPhase-modified-remove-regular" {
+      failure = testers.testBuildFailure (
+        test-happy.overrideAttrs {
+          installCheckPhase = ''
+            rm $out/regular
+          '';
+        }
+      );
+    } ''
+      echo $failure/testBuildFailure.log
+      (
+        set -x
+        grep 'ERROR: Files were changed during installCheckPhase' $failure/testBuildFailure.log >/dev/null
+        grep "$failure/regular" $failure/testBuildFailure.log >/dev/null
+      )
+      touch $out
+    '';
+
+  test-installPhase-modified-remove-symlink =
+    basePkgs.runCommand "test-installPhase-modified-remove-symlink" {
+      failure = testers.testBuildFailure (
+        test-happy.overrideAttrs {
+          installCheckPhase = ''
+            rm $out/symlink
+          '';
+        }
+      );
+    } ''
+      echo $failure/testBuildFailure.log
+      (
+        set -x
+        grep 'ERROR: Files were changed during installCheckPhase' $failure/testBuildFailure.log >/dev/null
+        grep "$failure/symlink" $failure/testBuildFailure.log >/dev/null
+      )
+      touch $out
+    '';
+
+  test-installPhase-modified-remove-bin =
+    basePkgs.runCommand "test-installPhase-modified-remove-bin" {
+      failure = testers.testBuildFailure (
+        test-happy.overrideAttrs {
+          installCheckPhase = ''
+            rm $out/bin/script
+          '';
+        }
+      );
+    } ''
+      echo $failure/testBuildFailure.log
+      (
+        set -x
+        grep 'ERROR: Files were changed during installCheckPhase' $failure/testBuildFailure.log >/dev/null
+        grep "$failure/bin/script" $failure/testBuildFailure.log >/dev/null
+      )
+      touch $out
+    '';
+
+  test-installPhase-modified-symlink =
+    basePkgs.runCommand "test-installPhase-modified-symlink" {
+      failure = testers.testBuildFailure (
+        test-happy.overrideAttrs {
+          installCheckPhase = ''
+            ln -sf ../bin/script $out/symlink
+          '';
+        }
+      );
+    } ''
+      echo $failure/testBuildFailure.log
+      (
+        set -x
+        grep 'ERROR: Files were changed during installCheckPhase' $failure/testBuildFailure.log >/dev/null
+        grep "$failure/symlink" $failure/testBuildFailure.log >/dev/null
+      )
+      touch $out
+    '';
+
+  test-installPhase-modified-new-symlink =
+    basePkgs.runCommand "test-installPhase-modified-new-symlink" {
+      failure = testers.testBuildFailure (
+        test-happy.overrideAttrs {
+          installCheckPhase = ''
+            ln -sf ../bin/script $out/new
+          '';
+        }
+      );
+    } ''
+      echo $failure/testBuildFailure.log
+      (
+        set -x
+        grep 'ERROR: Files were changed during installCheckPhase' $failure/testBuildFailure.log >/dev/null
+        grep "$failure/new" $failure/testBuildFailure.log >/dev/null
+      )
+      touch $out
+    '';
+}