Enhanced TurboSnap: Trace dependency changes instead of bailing out

bin-src/git/git.ts

@@ -1,5 +1,6 @@
 import execa from 'execa';
 import { EOL } from 'os';
+import { file as tmpFile } from 'tmp-promise';
 
 import { Context } from '../types';
 
@@ -211,6 +212,12 @@ export async function checkout(ref: string) {
   return execGitCommand(`git checkout ${ref}`);
 }
 
+export async function checkoutFile(ref: string, fileName: string) {
+  const { path: targetFileName } = await tmpFile();
+  await execGitCommand(`git show ${ref}:${fileName} > ${targetFileName}`);
+  return targetFileName;
+}
+
 export async function checkoutPrevious() {
   return execGitCommand(`git checkout -`);
 }
@@ -222,3 +229,8 @@ export async function discardChanges() {
 export async function getRepositoryRoot() {
   return execGitCommand(`git rev-parse --show-toplevel`);
 }
+
+export async function findFiles(pattern: string) {
+  const files = await execGitCommand(`git ls-files -z '${pattern}'`);
+  return files.split('\0').filter(Boolean);
+}

bin-src/lib/checkForUpdates.ts

@@ -40,6 +40,6 @@ export default async function checkForUpdates(ctx: Context) {
   }
 
   if (semver.major(ctx.pkg.version) < semver.major(latestVersion)) {
-    ctx.log.warn(outdatedPackage(ctx, latestVersion, hasYarn));
+    ctx.log.warn(outdatedPackage(ctx, latestVersion, hasYarn()));
   }
 }

bin-src/lib/findChangedDependencies.test.ts

@@ -0,0 +1,215 @@
+import { buildDepTreeFromFiles } from 'snyk-nodejs-lockfile-parser';
+import { findChangedDependencies } from './findChangedDependencies';
+import * as git from '../git/git';
+
+jest.mock('snyk-nodejs-lockfile-parser');
+jest.mock('yarn-or-npm');
+jest.mock('../git/git');
+
+const getRepositoryRoot = <jest.MockedFunction<typeof git.getRepositoryRoot>>git.getRepositoryRoot;
+const checkoutFile = <jest.MockedFunction<typeof git.checkoutFile>>git.checkoutFile;
+const findFiles = <jest.MockedFunction<typeof git.findFiles>>git.findFiles;
+const buildDepTree = <jest.MockedFunction<typeof buildDepTreeFromFiles>>buildDepTreeFromFiles;
+
+beforeEach(() => {
+  getRepositoryRoot.mockResolvedValue('/root');
+  findFiles.mockImplementation((file) => Promise.resolve(file.startsWith('**') ? [] : [file]));
+});
+afterEach(() => {
+  getRepositoryRoot.mockReset();
+  checkoutFile.mockReset();
+  findFiles.mockReset();
+  buildDepTree.mockReset();
+});
+
+describe('findChangedDependencies', () => {
+  it('returns nothing given no baselines', async () => {
+    buildDepTree.mockResolvedValue({ dependencies: {} });
+
+    await expect(findChangedDependencies([])).resolves.toEqual([]);
+  });
+
+  it('returns nothing when dependency tree is empty', async () => {
+    checkoutFile.mockResolvedValueOnce('A.package.json');
+    checkoutFile.mockResolvedValueOnce('A.package-lock.json');
+    buildDepTree.mockResolvedValue({ dependencies: {} });
+
+    await expect(findChangedDependencies(['A'])).resolves.toEqual([]);
+  });
+
+  it('returns nothing when dependency tree is unchanged', async () => {
+    checkoutFile.mockResolvedValueOnce('A.package.json');
+    checkoutFile.mockResolvedValueOnce('A.package-lock.json');
+    buildDepTree.mockResolvedValue({
+      dependencies: {
+        react: { name: 'react', version: '18.2.0', dependencies: {} },
+      },
+    });
+
+    await expect(findChangedDependencies(['A'])).resolves.toEqual([]);
+  });
+
+  it('returns updated dependencies', async () => {
+    // HEAD
+    buildDepTree.mockResolvedValueOnce({
+      dependencies: { react: { name: 'react', version: '18.2.0', dependencies: {} } },
+    });
+
+    // Baseline A
+    checkoutFile.mockResolvedValueOnce('A.package.json');
+    checkoutFile.mockResolvedValueOnce('A.package-lock.json');
+    buildDepTree.mockResolvedValueOnce({
+      dependencies: { react: { name: 'react', version: '18.3.0', dependencies: {} } },
+    });
+
+    await expect(findChangedDependencies(['A'])).resolves.toEqual(['react']);
+  });
+
+  it('returns added/removed dependencies', async () => {
+    // HEAD
+    buildDepTree.mockResolvedValueOnce({
+      dependencies: { react: { name: 'react', version: '18.2.0', dependencies: {} } },
+    });
+
+    // Baseline A
+    checkoutFile.mockResolvedValueOnce('A.package.json');
+    checkoutFile.mockResolvedValueOnce('A.package-lock.json');
+    buildDepTree.mockResolvedValueOnce({
+      dependencies: { vue: { name: 'vue', version: '3.2.0', dependencies: {} } },
+    });
+
+    await expect(findChangedDependencies(['A'])).resolves.toEqual(['vue', 'react']);
+  });
+
+  it('finds updated transient dependencies', async () => {
+    // HEAD
+    buildDepTree.mockResolvedValueOnce({
+      dependencies: {
+        react: {
+          name: 'react',
+          version: '18.2.0',
+          dependencies: {
+            'loose-envify': { name: 'loose-envify', version: '1.3.1', dependencies: {} },
+          },
+        },
+      },
+    });
+
+    // Baseline A
+    checkoutFile.mockResolvedValueOnce('A.package.json');
+    checkoutFile.mockResolvedValueOnce('A.package-lock.json');
+    buildDepTree.mockResolvedValueOnce({
+      dependencies: {
+        react: {
+          name: 'react',
+          version: '18.2.0',
+          dependencies: {
+            'loose-envify': { name: 'loose-envify', version: '1.4.0', dependencies: {} },
+          },
+        },
+      },
+    });
+
+    await expect(findChangedDependencies(['A'])).resolves.toEqual(['loose-envify']);
+  });
+
+  it('combines and dedupes changes for multiple baselines', async () => {
+    // HEAD
+    buildDepTree.mockResolvedValueOnce({
+      dependencies: {
+        react: { name: 'react', version: '18.2.0', dependencies: {} },
+        lodash: { name: 'lodash', version: '4.17.21', dependencies: {} },
+      },
+    });
+
+    // Baseline A
+    checkoutFile.mockResolvedValueOnce('A.package.json');
+    checkoutFile.mockResolvedValueOnce('A.package-lock.json');
+    buildDepTree.mockResolvedValueOnce({
+      dependencies: {
+        react: { name: 'react', version: '18.3.0', dependencies: {} },
+        lodash: { name: 'lodash', version: '4.17.21', dependencies: {} },
+      },
+    });
+
+    // Baseline B
+    checkoutFile.mockResolvedValueOnce('B.package.json');
+    checkoutFile.mockResolvedValueOnce('B.package-lock.json');
+    buildDepTree.mockResolvedValueOnce({
+      dependencies: {
+        react: { name: 'react', version: '18.3.0', dependencies: {} },
+        lodash: { name: 'lodash', version: '4.18.0', dependencies: {} },
+      },
+    });
+
+    await expect(findChangedDependencies(['A', 'B'])).resolves.toEqual(['react', 'lodash']);
+  });
+
+  it('looks for manifest and lock files in subpackages', async () => {
+    findFiles.mockImplementation((file) =>
+      Promise.resolve(file.startsWith('**') ? [file.replace('**', 'subdir')] : [file])
+    );
+
+    // HEAD root
+    buildDepTree.mockResolvedValueOnce({
+      dependencies: { react: { name: 'react', version: '18.2.0', dependencies: {} } },
+    });
+    // HEAD subdir
+    buildDepTree.mockResolvedValueOnce({
+      dependencies: { lodash: { name: 'lodash', version: '4.17.21', dependencies: {} } },
+    });
+
+    // Baseline A
+    checkoutFile.mockImplementation((commit, file) => Promise.resolve(`${commit}.${file}`));
+    buildDepTree.mockResolvedValueOnce({
+      dependencies: { react: { name: 'react', version: '18.3.0', dependencies: {} } },
+    });
+    buildDepTree.mockResolvedValueOnce({
+      dependencies: { lodash: { name: 'lodash', version: '4.18.0', dependencies: {} } },
+    });
+
+    await expect(findChangedDependencies(['A'])).resolves.toEqual(['react', 'lodash']);
+
+    // Root manifest and lock files are checked
+    expect(buildDepTree).toHaveBeenCalledWith('/root', 'package.json', 'package-lock.json', true);
+    expect(buildDepTree).toHaveBeenCalledWith(
+      '/root',
+      'A.package.json',
+      'A.package-lock.json',
+      true
+    );
+
+    // Subpackage manifest and lock files are checked
+    expect(buildDepTree).toHaveBeenCalledWith(
+      '/root',
+      'subdir/package.json',
+      'subdir/package-lock.json',
+      true
+    );
+    expect(buildDepTree).toHaveBeenCalledWith(
+      '/root',
+      'A.subdir/package.json',
+      'A.subdir/package-lock.json',
+      true
+    );
+  });
+
+  it('uses root lockfile when subpackage lockfile is missing', async () => {
+    findFiles.mockImplementation((file) => {
+      if (file === 'subdir/package-lock.json') return Promise.resolve([]);
+      return Promise.resolve(file.startsWith('**') ? [file.replace('**', 'subdir')] : [file]);
+    });
+
+    checkoutFile.mockImplementation((commit, file) => Promise.resolve(`${commit}.${file}`));
+    buildDepTree.mockResolvedValue({ dependencies: {} });
+
+    await expect(findChangedDependencies(['A'])).resolves.toEqual([]);
+
+    expect(buildDepTree).toHaveBeenCalledWith(
+      '/root',
+      'A.subdir/package.json',
+      'A.package-lock.json', // root lockfile
+      true
+    );
+  });
+});

bin-src/lib/findChangedDependencies.ts

@@ -0,0 +1,77 @@
+import path from 'path';
+import { buildDepTreeFromFiles, PkgTree } from 'snyk-nodejs-lockfile-parser';
+import { hasYarn } from 'yarn-or-npm';
+import { checkoutFile, findFiles, getRepositoryRoot } from '../git/git';
+
+const flattenDependencyTree = (
+  tree: PkgTree['dependencies'],
+  results = new Set<string>()
+): Set<string> =>
+  Object.values(tree).reduce((acc, dep) => {
+    acc.add(`${dep.name}@@${dep.version}`);
+    return flattenDependencyTree(dep.dependencies || {}, acc);
+  }, results);
+
+// Retrieve a set of values which is in either set, but not both.
+const xor = <T>(left: Set<T>, right: Set<T>) =>
+  Array.from(right.values()).reduce((acc, value) => {
+    if (acc.has(value)) acc.delete(value);
+    else acc.add(value);
+    return acc;
+  }, new Set(left));
+
+// Yields a list of dependency names which have changed since the baseline.
+// E.g. ['react', 'react-dom', '@storybook/react']
+export const findChangedDependencies = async (baselineCommits: string[]) => {
+  const manifestName = 'package.json';
+  const lockfileName = hasYarn() ? 'yarn.lock' : 'package-lock.json';
+
+  const rootPath = await getRepositoryRoot();
+  const [rootManifestPath] = await findFiles(manifestName);
+  const [rootLockfilePath] = await findFiles(lockfileName);
+  if (!rootManifestPath || !rootLockfilePath) {
+    throw new Error(`Could not find ${manifestName} or ${lockfileName} in the repository`);
+  }
+
+  // Handle monorepos with multiple package.json files.
+  const nestedManifestPaths = await findFiles(`**/${manifestName}`);
+  const pathPairs = await Promise.all(
+    nestedManifestPaths.map(async (manifestPath) => {
+      const dirname = path.dirname(manifestPath);
+      const [lockfilePath] = await findFiles(`${dirname}/${lockfileName}`);
+      // Fall back to the root lockfile if we can't find one in the same directory.
+      return [manifestPath, lockfilePath || rootLockfilePath];
+    })
+  );
+
+  // Use a Set so we only keep distinct package names.
+  const changedDependencyNames = new Set<string>();
+
+  await Promise.all(
+    [[rootManifestPath, rootLockfilePath], ...pathPairs].map(
+      async ([manifestPath, lockfilePath]) => {
+        const headTree = await buildDepTreeFromFiles(rootPath, manifestPath, lockfilePath, true);
+        const headDependencies = flattenDependencyTree(headTree.dependencies);
+
+        // Retrieve the union of dependencies which changed compared to each baseline.
+        // A change means either the version number is different or the dependency was added/removed.
+        // If a manifest or lockfile is missing on the baseline, this throws and we'll end up bailing.
+        await Promise.all(
+          baselineCommits.map(async (commit) => {
+            const manifest = await checkoutFile(commit, manifestPath);
+            const lockfile = await checkoutFile(commit, lockfilePath);
+            const baselineTree = await buildDepTreeFromFiles(rootPath, manifest, lockfile, true);
+            const baselineDependencies = flattenDependencyTree(baselineTree.dependencies);
+            // eslint-disable-next-line no-restricted-syntax
+            for (const dependency of xor(baselineDependencies, headDependencies)) {
+              // Strip the version number so we get a set of package names.
+              changedDependencyNames.add(dependency.split('@@')[0]);
+            }
+          })
+        );
+      }
+    )
+  );
+
+  return Array.from(changedDependencyNames);
+};