From 458233892eedd22819333fb7c22da80fd2a298a4 Mon Sep 17 00:00:00 2001 From: Adrian Gierakowski Date: Fri, 28 Jan 2022 17:05:09 +0000 Subject: [PATCH] separate deps from main project When used with pnp, we can avoid copying deps to the main project, so that rebuilds are faster and less spaces is used when only project source changes (but not deps). To make this work, we need to patch .pnp.cjs, since unfortunately symlinks don't work as packageLocations cannot be (see: https://github.com/yarnpkg/berry/issues/3514). We also cannot use absolute paths, so we end up using relative paths to store locations containing each deps. This also works for unplugged deps. --- src/generate.ts | 2 +- src/tmpl/yarn-project.nix.in | 189 +++++++++++++++++++++++++++++------ 2 files changed, 162 insertions(+), 29 deletions(-) diff --git a/src/generate.ts b/src/generate.ts index b7ca5ae..b677455 100644 --- a/src/generate.ts +++ b/src/generate.ts @@ -270,7 +270,7 @@ export default async (project: Project, cache: Cache, report: Report) => { const ident = project.topLevelWorkspace.manifest.name; const projectName = ident ? structUtils.stringifyIdent(ident) : `workspace`; const projectExpr = renderTmpl(projectExprTmpl, { - PROJECT_NAME: json(projectName), + PROJECT_NAME: projectName, YARN_PATH: yarnPathRel, LOCKFILE: lockfileRel, CACHE_FOLDER: json(cacheFolder), diff --git a/src/tmpl/yarn-project.nix.in b/src/tmpl/yarn-project.nix.in index 92d62e2..049709d 100644 --- a/src/tmpl/yarn-project.nix.in +++ b/src/tmpl/yarn-project.nix.in @@ -7,7 +7,18 @@ let yarnPath = ./@@YARN_PATH@@; - lockfile = ./@@LOCKFILE@@; + yarnRelativePathString = "./@@YARN_PATH@@"; + yarnLock = ./@@LOCKFILE@@; + packageJson = ./package.json; + yarnrcYaml = builtins.path { + path = ./.yarnrc.yml; + name = "yarnrc.yml"; + }; + yarnPlugins = builtins.path { + path = ./.yarn/plugins; + name = "yarnPlugins"; + }; + cacheFolder = @@CACHE_FOLDER@@; # Call overrideAttrs on a derivation if a function is provided. @@ -30,6 +41,21 @@ let ''; }; + setupProjectFiles = '' + # package.json cannot be symlinked since when executing "yarn run ...", yarn + # will follow the symlink and consider the location of the original file as + # the projects root directory. + cp ${packageJson} package.json + ln -s ${yarnLock} yarn.lock + ln -s ${yarnrcYaml} .yarnrc.yml + + mkdir -p .yarn + ln -s ${yarnPlugins} .yarn/plugins + + mkdir -p $(dirname ${yarnRelativePathString}) + ln -s ${yarnPath} ${yarnRelativePathString} + ''; + checkSandboxPathExists = writeShellScriptBin "check-sandbox-file-exists" '' set -ueo pipefail @@ -84,8 +110,6 @@ let ${exportEnvVarsFromFilesIfAny secretsEnvVars} home=$TMP - yarn_cache_folder=$home/cache - mkdir -p $yarn_cache_folder ${ if netrcFilePath != null @@ -93,16 +117,29 @@ let else "" } - cd "$src" - HOME="$home" yarn_cache_folder="$yarn_cache_folder" CI=1 \ - node '${yarnPath}' nixify fetch-one $locator + build_dir=$TMP/build + mkdir -p $build_dir + cd $build_dir + + ${setupProjectFiles} + + mkdir -p ${cacheFolder} + YARN_CACHE_FOLDER=$(pwd)/${cacheFolder} + + HOME="$home" \ + YARN_CACHE_FOLDER="$YARN_CACHE_FOLDER" \ + CI=1 \ + node '${yarnPath}' nixify fetch-one $locator + # Because we change the cache dir, Yarn may generate a different name. - mv "$yarn_cache_folder/$(sed 's/-[^-]*\.[^-]*$//' <<< "$outputFilename")"-* $out + output_filename_stripped=$(sed 's/-[^-]*\.[^-]*$//' <<< "$outputFilename") + + mv "$YARN_CACHE_FOLDER/$output_filename_stripped"-* $out ''; in lib.mapAttrs (locator: { filename, sha512 }: stdenv.mkDerivation { - inherit src builder locator; - name = lib.strings.sanitizeDerivationName locator; - buildInputs = [ nodejs git cacert ]; + inherit builder locator; + # We need .zip extension since without pnp will not look inside the archive. + name = lib.strings.sanitizeDerivationName locator + ".zip"; buildInputs = [ nodejs ]; nativeBuildInputs = [ git cacert linkNetrcFile checkSandboxPathExists ]; outputFilename = filename; @@ -112,9 +149,9 @@ let }) cacheEntries; # Create a shell snippet to copy dependencies from a list of derivations. - mkCacheBuilderForDrvs = drvs: + mkCacheBuilderForDrvs = symlinkPackages: drvs: writeText "collect-cache.sh" (lib.concatMapStrings (drv: '' - cp ${drv} '${drv.outputFilename}' + ${if symlinkPackages then "ln -s" else "cp"} ${drv} '${drv.outputFilename}' '') drvs); #@@ IF NEED_ISOLATED_BUILD_SUPPRORT @@ -122,7 +159,7 @@ let mkCacheBuilderForLocators = let pickCacheDrvs = map (locator: cacheDrvs.${locator}); in locators: - mkCacheBuilderForDrvs (pickCacheDrvs locators); + mkCacheBuilderForDrvs false (pickCacheDrvs locators); # Create a derivation that builds a node-pre-gyp module in isolation. mkIsolatedBuild = { pname, version, reference, locators }: stdenv.mkDerivation (drvCommon // { @@ -130,16 +167,21 @@ let phases = [ "buildPhase" "installPhase" ]; buildPhase = '' + runHook preBuild + mkdir -p .yarn/cache pushd .yarn/cache > /dev/null source ${mkCacheBuilderForLocators locators} popd > /dev/null echo '{ "dependencies": { "${pname}": "${reference}" } }' > package.json - install -m 0600 ${lockfile} ./yarn.lock - export yarn_global_folder="$TMP" - export YARN_ENABLE_IMMUTABLE_INSTALLS=false - yarn --immutable-cache + install -m 0600 ${yarnLock} ./yarn.lock + + yarn_global_folder="$TMP" \ + YARN_ENABLE_IMMUTABLE_INSTALLS=false \ + yarn --immutable-cache + + runHook postBuild ''; installPhase = '' @@ -154,19 +196,20 @@ let }); #@@ ENDIF NEED_ISOLATED_BUILD_SUPPRORT - # Main project derivation. - project = stdenv.mkDerivation (drvCommon // { - inherit src; - name = @@PROJECT_NAME@@; + # Derivation with content of .yarn/cache and .pnp.cjs + deps = stdenv.mkDerivation (drvCommon // { + name = "@@PROJECT_NAME@@-deps"; # Disable Nixify plugin to save on some unnecessary processing. yarn_enable_nixify = "false"; + nativeBuildInputs = [gnused]; configurePhase = '' + ${setupProjectFiles} + # Copy over the Yarn cache. - rm -fr '${cacheFolder}' - mkdir -p '${cacheFolder}' + mkdir -p ${cacheFolder} pushd '${cacheFolder}' > /dev/null - source ${mkCacheBuilderForDrvs (lib.attrValues cacheDrvs)} + source ${mkCacheBuilderForDrvs symlinkPackages (lib.attrValues cacheDrvs)} popd > /dev/null # Yarn may need a writable home directory. @@ -185,16 +228,105 @@ let @@ISOLATED_INTEGRATION@@ # Run normal Yarn install to complete dependency installation. - yarn install --immutable --immutable-cache + # YARN_VIRTUAL_FOLDER is set this way to make it easy to replace in + # installPhase below, so that in the end virtual paths resolve to + # packages in nix store. + YARN_CACHE_FOLDER=$(pwd)/${cacheFolder} \ + YARN_VIRTUAL_FOLDER=$(pwd)/__virtual__ \ + yarn install --immutable --immutable-cache runHook postConfigure ''; - buildPhase = '' - runHook preBuild - runHook postBuild + dontUnpack = true; + dontBuild = true; + + installPhase = '' + runHook preInstall + + # This needs nested under /nix/store at the same depth as the the location + # of the source in the output of project derivation so that + # relative_path_to_nix_store is valid from the final source. + output_dir=$out/libexec/deps + mkdir -p $output_dir + + mkdir -p $output_dir/.yarn + test -d .yarn/cache && mv .yarn/cache $output_dir/.yarn/cache + test -d .yarn/unplugged && mv .yarn/unplugged $output_dir/.yarn/unplugged + + mv .pnp.cjs $output_dir/.pnp.cjs + + cd $output_dir + + # Replace references from .pnp.cjs to symlinks in .yarn/cache with + # relative paths. Needed because of: https://github.com/yarnpkg/berry/issues/3514 + + # sed helpers + escape_sed_replacement () { + echo "$1" | sed -e 's/[\/&]/\\&/g' + } + + escape_sed_pattern () { + echo "$1" | sed -e 's/[]\/$*.^[]/\\&/g' + } + echo >&2 "fixup paths in .pnp.cjs" + + # TODO: this would be best done with a plugin which would resolve symlinks + # to actual store paths during yarn install. + relative_path_to_nix_store=$(realpath --relative-to=. /nix/store) + + unplugged_path_relative_to_nix_store=$(realpath --relative-to=/nix/store $output_dir/.yarn/unplugged) + echo unplugged_path_relative_to_nix_store: $unplugged_path_relative_to_nix_store + + sed -E -i \ + -e "s/$(escape_sed_pattern './.yarn/cache')/$(escape_sed_replacement "$relative_path_to_nix_store")/g" \ + -e "s/$(escape_sed_pattern './.yarn/unplugged')/$(escape_sed_replacement "''${relative_path_to_nix_store}/''${unplugged_path_relative_to_nix_store}")/g" \ + -e "s/$(escape_sed_pattern '0/.yarn/cache')/0/g" \ + -e "s/$(escape_sed_pattern './__virtual__')/$(escape_sed_replacement "$relative_path_to_nix_store/__virtual__")/g" \ + .pnp.cjs + + for path in .yarn/cache/*; do + # Skip empty + test -z "$path" && continue + + file_name_in_pnp=$(basename "$path") + file_name=$(basename $(realpath --relative-to=. "$path")) + + # echo >&2 "replace for path: $path" + # echo >&2 " file_name_in_pnp: $file_name_in_pnp" + # echo >&2 " with file_name: $file_name" + + sed -i "s/$(escape_sed_pattern "$file_name_in_pnp")/$(escape_sed_replacement "$file_name")/" .pnp.cjs + done + + mkdir ../pnp + mv .pnp.cjs ../pnp + runHook postInstall ''; + passthru = { + inherit nodejs; + }; + }); + + # Main project derivation. + project = stdenv.mkDerivation (drvCommon // { + inherit src; + name = "@@PROJECT_NAME@@"; + + configurePhase = '' + ${setupProjectFiles} + # We can't symlink this one since it doesn't work as a symlink due to + # packageLocations within it being relative path to this files locations + # real location, therefore it needs to be located at the root of the + # project for relative and workspace scoped imports to work. + cp ${deps}/libexec/pnp/.pnp.cjs .pnp.cjs + + runHook postConfigure + ''; + + dontBuild = true; + installPhase = '' runHook preInstall @@ -204,6 +336,7 @@ let mv $PWD "$out/libexec/$name" cd "$out/libexec/$name" + # Invoke a plugin internal command to setup binaries. yarn nixify install-bin $out/bin