buildNpmPackage: init

.github/CODEOWNERS

@@ -289,3 +289,8 @@
 # Dotnet
 /pkgs/build-support/dotnet          @IvarWithoutBones
 /pkgs/development/compilers/dotnet  @IvarWithoutBones
+
+# Node.js
+/pkgs/build-support/node/build-npm-package      @winterqt
+/pkgs/build-support/node/fetch-npm-deps         @winterqt
+/doc/languages-frameworks/javascript.section.md @winterqt

doc/languages-frameworks/javascript.section.md

@@ -157,6 +157,61 @@ git config --global url."https://github.com/".insteadOf git://github.com/
 
 ## Tool specific instructions {#javascript-tool-specific}
 
+### buildNpmPackage {#javascript-buildNpmPackage}
+
+`buildNpmPackage` allows you to package npm-based projects in Nixpkgs without the use of an auto-generated dependencies file (as used in [node2nix](#javascript-node2nix)). It works by utilizing npm's cache functionality -- creating a reproducible cache that contains the dependencies of a project, and pointing npm to it.
+
+```nix
+{ lib, buildNpmPackage, fetchFromGitHub }:
+
+buildNpmPackage rec {
+  pname = "flood";
+  version = "4.7.0";
+
+  src = fetchFromGitHub {
+    owner = "jesec";
+    repo = pname;
+    rev = "v${version}";
+    hash = "sha256-BR+ZGkBBfd0dSQqAvujsbgsEPFYw/ThrylxUbOksYxM=";
+  };
+
+  patches = [ ./remove-prepack-script.patch ];
+
+  npmDepsHash = "sha256-s8SpZY/1tKZVd3vt7sA9vsqHvEaNORQBMrSyhWpj048=";
+
+  NODE_OPTIONS = "--openssl-legacy-provider";
+
+  meta = with lib; {
+    description = "A modern web UI for various torrent clients with a Node.js backend and React frontend";
+    homepage = "https://flood.js.org";
+    license = licenses.gpl3Only;
+    maintainers = with maintainers; [ winter ];
+  };
+}
+```
+
+#### Arguments {#javascript-buildNpmPackage-arguments}
+
+* `npmDepsHash`: The output hash of the dependencies for this project. Can be calculated in advance with [`prefetch-npm-deps`](#javascript-buildNpmPackage-prefetch-npm-deps).
+* `makeCacheWritable`: Whether to make the cache writable prior to installing dependencies. Don't set this unless npm tries to write to the cache directory, as it can slow down the build.
+* `npmBuildScript`: The script to run to build the project. Defaults to `"build"`.
+* `npmFlags`: Flags to pass to all npm commands.
+* `npmInstallFlags`: Flags to pass to `npm ci`.
+* `npmBuildFlags`: Flags to pass to `npm run ${npmBuildScript}`.
+* `npmPackFlags`: Flags to pass to `npm pack`.
+
+#### prefetch-npm-deps {#javascript-buildNpmPackage-prefetch-npm-deps}
+
+`prefetch-npm-deps` can calculate the hash of the dependencies of an npm project ahead of time.
+
+```console
+$ ls
+package.json package-lock.json index.js
+$ prefetch-npm-deps package-lock.json
+...
+sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
+```
+
 ### node2nix {#javascript-node2nix}
 
 #### Preparation {#javascript-node2nix-preparation}

pkgs/build-support/node/build-npm-package/default.nix

@@ -0,0 +1,54 @@
+{ lib, stdenv, fetchNpmDeps, npmHooks, nodejs }:
+
+{ name ? "${args.pname}-${args.version}"
+, src ? null
+, srcs ? null
+, sourceRoot ? null
+, patches ? [ ]
+, nativeBuildInputs ? [ ]
+, buildInputs ? [ ]
+  # The output hash of the dependencies for this project.
+  # Can be calculated in advance with prefetch-npm-deps.
+, npmDepsHash ? ""
+  # Whether to make the cache writable prior to installing dependencies.
+  # Don't set this unless npm tries to write to the cache directory, as it can slow down the build.
+, makeCacheWritable ? false
+  # The script to run to build the project.
+, npmBuildScript ? "build"
+  # Flags to pass to all npm commands.
+, npmFlags ? [ ]
+  # Flags to pass to `npm ci`.
+, npmInstallFlags ? [ ]
+  # Flags to pass to `npm rebuild`.
+, npmRebuildFlags ? [ ]
+  # Flags to pass to `npm run ${npmBuildScript}`.
+, npmBuildFlags ? [ ]
+  # Flags to pass to `npm pack`.
+, npmPackFlags ? [ ]
+, ...
+} @ args:
+
+let
+  npmDeps = fetchNpmDeps {
+    inherit src srcs sourceRoot patches;
+    name = "${name}-npm-deps";
+    hash = npmDepsHash;
+  };
+
+  inherit (npmHooks.override { inherit nodejs; }) npmConfigHook npmBuildHook npmInstallHook;
+in
+stdenv.mkDerivation (args // {
+  inherit npmDeps npmBuildScript;
+
+  nativeBuildInputs = nativeBuildInputs ++ [ nodejs npmConfigHook npmBuildHook npmInstallHook ];
+  buildInputs = buildInputs ++ [ nodejs ];
+
+  strictDeps = true;
+
+  # Stripping takes way too long with the amount of files required by a typical Node.js project.
+  dontStrip = args.dontStrip or true;
+
+  passthru = { inherit npmDeps; } // (args.passthru or { });
+
+  meta = (args.meta or { }) // { platforms = args.meta.platforms or nodejs.meta.platforms; };
+})

pkgs/build-support/node/build-npm-package/hooks/default.nix

@@ -0,0 +1,35 @@
+{ lib, makeSetupHook, nodejs, srcOnly, diffutils, jq, makeWrapper }:
+
+{
+  npmConfigHook = makeSetupHook
+    {
+      name = "npm-config-hook";
+      substitutions = {
+        nodeSrc = srcOnly nodejs;
+
+        # Specify the stdenv's `diff` and `jq` by abspath to ensure that the user's build
+        # inputs do not cause us to find the wrong binaries.
+        # The `.nativeDrv` stanza works like nativeBuildInputs and ensures cross-compiling has the right version available.
+        diff = "${diffutils.nativeDrv or diffutils}/bin/diff";
+        jq = "${jq.nativeDrv or jq}/bin/jq";
+
+        nodeVersion = nodejs.version;
+        nodeVersionMajor = lib.versions.major nodejs.version;
+      };
+    } ./npm-config-hook.sh;
+
+  npmBuildHook = makeSetupHook
+    {
+      name = "npm-build-hook";
+    } ./npm-build-hook.sh;
+
+  npmInstallHook = makeSetupHook
+    {
+      name = "npm-install-hook";
+      deps = [ makeWrapper ];
+      substitutions = {
+        hostNode = "${nodejs}/bin/node";
+        jq = "${jq.nativeDrv or jq}/bin/jq";
+      };
+    } ./npm-install-hook.sh;
+}

pkgs/build-support/node/build-npm-package/hooks/npm-build-hook.sh

@@ -0,0 +1,37 @@
+# shellcheck shell=bash
+
+npmBuildHook() {
+    echo "Executing npmBuildHook"
+
+    runHook preBuild
+
+    if [ -z "${npmBuildScript-}" ]; then
+        echo
+        echo "ERROR: no build script was specified"
+        echo 'Hint: set `npmBuildScript`, override `buildPhase`, or set `dontNpmBuild = true`.'
+        echo
+
+        exit 1
+    fi
+
+    if ! npm run "$npmBuildScript" $npmBuildFlags "${npmBuildFlagsArray[@]}" $npmFlags "${npmFlagsArray[@]}"; then
+        echo
+        echo 'ERROR: `npm build` failed'
+        echo
+        echo "Here are a few things you can try, depending on the error:"
+        echo "1. Make sure your build script ($npmBuildScript) exists"
+        echo '2. If the error being thrown is something similar to "error:0308010C:digital envelope routines::unsupported", add `NODE_OPTIONS = "--openssl-legacy-provider"` to your derivation'
+        echo "  See https://github.com/webpack/webpack/issues/14532 for more information."
+        echo
+
+        exit 1
+    fi
+
+    runHook postBuild
+
+    echo "Finished npmBuildHook"
+}
+
+if [ -z "${dontNpmBuild-}" ] && [ -z "${buildPhase-}" ]; then
+    buildPhase=npmBuildHook
+fi

pkgs/build-support/node/build-npm-package/hooks/npm-config-hook.sh

@@ -0,0 +1,102 @@
+# shellcheck shell=bash
+
+npmConfigHook() {
+    echo "Executing npmConfigHook"
+
+    echo "Configuring npm"
+
+    export HOME=$TMPDIR
+    export npm_config_nodedir="@nodeSrc@"
+
+    local -r cacheLockfile="$npmDeps/package-lock.json"
+    local -r srcLockfile="$PWD/package-lock.json"
+
+    echo "Validating consistency between $srcLockfile and $cacheLockfile"
+
+    if ! @diff@ "$srcLockfile" "$cacheLockfile"; then
+      # If the diff failed, first double-check that the file exists, so we can
+      # give a friendlier error msg.
+      if ! [ -e "$srcLockfile" ]; then
+        echo
+        echo "ERROR: Missing package-lock.json from src. Expected to find it at: $srcLockfile"
+        echo "Hint: You can use the patches attribute to add a package-lock.json manually to the build."
+        echo
+
+        exit 1
+      fi
+
+      if ! [ -e "$cacheLockfile" ]; then
+        echo
+        echo "ERROR: Missing lockfile from cache. Expected to find it at: $cacheLockfile"
+        echo
+
+        exit 1
+      fi
+
+      echo
+      echo "ERROR: npmDepsHash is out of date"
+      echo
+      echo "The package-lock.json in src is not the same as the in $npmDeps."
+      echo
+      echo "To fix the issue:"
+      echo '1. Use `lib.fakeHash` as the npmDepsHash value'
+      echo "2. Build the derivation and wait for it to fail with a hash mismatch"
+      echo "3. Copy the 'got: sha256-' value back into the npmDepsHash field"
+      echo
+
+      exit 1
+    fi
+
+    local cachePath
+
+    if [ -z "${makeCacheWritable-}" ]; then
+        cachePath=$npmDeps
+    else
+        echo "Making cache writable"
+        cp -r "$npmDeps" "$TMPDIR/cache"
+        chmod -R 700 "$TMPDIR/cache"
+        cachePath=$TMPDIR/cache
+    fi
+
+    npm config set cache "$cachePath"
+    npm config set offline true
+    npm config set progress false
+
+    echo "Installing dependencies"
+
+    if ! npm ci --ignore-scripts $npmInstallFlags "${npmInstallFlagsArray[@]}" $npmFlags "${npmFlagsArray[@]}"; then
+        echo
+        echo "ERROR: npm failed to install dependencies"
+        echo
+        echo "Here are a few things you can try, depending on the error:"
+        echo '1. Set `makeCacheWritable = true`'
+        echo "  Note that this won't help if npm is complaining about not being able to write to the logs directory -- look above that for the actual error."
+        echo '2. Set `npmInstallFlags = [ "--legacy-peer-deps" ]`'
+        echo
+
+        exit 1
+    fi
+
+    patchShebangs node_modules
+
+    local -r lockfileVersion="$(@jq@ .lockfileVersion package-lock.json)"
+
+    if (( lockfileVersion < 2 )); then
+      # This is required because npm consults a hidden lockfile in node_modules to figure out
+      # what to create bin links for. When using an old lockfile offline, this hidden lockfile
+      # contains insufficent data, making npm silently fail to create links. The hidden lockfile
+      # is bypassed when any file in node_modules is newer than it. Thus, we create a file when
+      # using an old lockfile, so bin links work as expected without having to downgrade Node or npm.
+      touch node_modules/.meow
+    fi
+
+    npm rebuild "${npmRebuildFlags[@]}" "${npmFlags[@]}"
+
+    if (( lockfileVersion < 2 )); then
+      rm node_modules/.meow
+    fi
+
+    echo "Finished npmConfigHook"
+}
+
+postPatchHooks+=(npmConfigHook)

pkgs/build-support/node/build-npm-package/hooks/npm-install-hook.sh

@@ -0,0 +1,43 @@
+# shellcheck shell=bash
+
+npmInstallHook() {
+    echo "Executing npmInstallHook"
+
+    runHook preInstall
+
+    # `npm pack` writes to cache
+    npm config delete cache
+
+    local -r packageOut="$out/lib/node_modules/$(@jq@ --raw-output '.name' package.json)"
+
+    while IFS= read -r file; do
+        local dest="$packageOut/$(dirname "$file")"
+        mkdir -p "$dest"
+        cp "$file" "$dest"
+    done < <(@jq@ --raw-output '.[0].files | map(.path) | join("\n")' <<< "$(npm pack --json --dry-run $npmPackFlags "${npmPackFlagsArray[@]}" $npmFlags "${npmFlagsArray[@]}")")
+
+    while IFS=" " read -ra bin; do
+        mkdir -p "$out/bin"
+        makeWrapper @hostNode@ "$out/bin/${bin[0]}" --add-flags "$packageOut/${bin[1]}"
+    done < <(@jq@ --raw-output '(.bin | type) as $typ | if $typ == "string" then
+        .name + " " + .bin
+        elif $typ == "object" then .bin | to_entries | map(.key + " " + .value) | join("\n")
+        else "invalid type " + $typ | halt_error end' package.json)
+
+    local -r nodeModulesPath="$packageOut/node_modules"
+
+    if [ ! -d "$nodeModulesPath" ]; then
+        npm prune --omit dev
+        find node_modules -maxdepth 1 -type d -empty -delete
+
+        cp -r node_modules "$nodeModulesPath"
+    fi
+
+    runHook postInstall
+
+    echo "Finished npmInstallHook"
+}
+
+if [ -z "${dontNpmInstall-}" ] && [ -z "${installPhase-}" ]; then
+    installPhase=npmInstallHook
+fi

pkgs/build-support/node/fetch-npm-deps/.gitignore

@@ -0,0 +1 @@
+/target