refactor: cache source urls

renovatebot · Mar 22, 2024 · 738418d · 738418d
1 parent 41aea6d
commit 738418d
Show file tree

Hide file tree

Showing 3 changed files with 125 additions and 17 deletions.
diff --git a/lib/modules/datasource/nuget/__fixtures__/nlog/NLog.4.7.3-no-repo.nupkg b/lib/modules/datasource/nuget/__fixtures__/nlog/NLog.4.7.3-no-repo.nupkg
diff --git a/lib/modules/datasource/nuget/index.spec.ts b/lib/modules/datasource/nuget/index.spec.ts
@@ -4,8 +4,9 @@ import { join } from 'upath';
 import { getPkgReleases } from '..';
 import { Fixtures } from '../../../../test/fixtures';
 import * as httpMock from '../../../../test/http-mock';
-import { logger } from '../../../../test/util';
+import { logger, mocked } from '../../../../test/util';
 import { GlobalConfig } from '../../../config/global';
+import * as _packageCache from '../../../util/cache/package';
 import * as _hostRules from '../../../util/host-rules';
 import { id as versioning } from '../../versioning/nuget';
 import { parseRegistryUrl } from './common';
@@ -17,6 +18,9 @@ const hostRules: any = _hostRules;
 
 jest.mock('../../../util/host-rules', () => mockDeep());
 
+jest.mock('../../../util/cache/package');
+const packageCache = mocked(_packageCache);
+
 const pkgInfoV3FromNuget = Fixtures.get('nunit/v3_nuget_org.xml');
 const pkgListV3Registration = Fixtures.get('nunit/v3_registration.json');
 
@@ -373,9 +377,86 @@ describe('modules/datasource/nuget/index', () => {
         },
         'Determined sourceUrl from nupkgUrl',
       );
+      expect(packageCache.set).toHaveBeenCalledWith(
+        'datasource-nuget',
+        'source-url:https://some-registry/v3-flatcontainer/nlog/4.7.3/nlog.4.7.3.nupkg',
+        'https://github.com/NLog/NLog.git',
+        60 * 24 * 7,
+      );
       expect(res?.sourceUrl).toBeDefined();
     });
 
+    it('can handle nupkg without repository metadata when PackageBaseAddress is missing', async () => {
+      GlobalConfig.set({
+        cacheDir: join('/tmp/cache'),
+      });
+
+      const nugetIndex = `
+        {
+          "version": "3.0.0",
+          "resources": [
+            {
+              "@id": "https://some-registry/v3/metadata",
+              "@type": "RegistrationsBaseUrl/3.0.0-beta",
+              "comment": "Get package metadata."
+            }
+          ]
+        }
+      `;
+      const nlogRegistration = `
+        {
+          "count": 1,
+          "items": [
+            {
+              "@id": "https://some-registry/v3/metadata/nlog/4.7.3.json",
+              "lower": "4.7.3",
+              "upper": "4.7.3",
+              "count": 1,
+              "items": [
+                {
+                  "@id": "foo",
+                  "catalogEntry": {
+                    "id": "NLog",
+                    "version": "4.7.3",
+                    "packageContent": "https://some-registry/v3-flatcontainer/nlog/4.7.3/nlog.4.7.3.nupkg"
+                  }
+                }
+              ]
+            }
+          ]
+        }
+      `;
+      httpMock
+        .scope('https://some-registry')
+        .get('/v3/index.json')
+        .twice()
+        .reply(200, nugetIndex)
+        .get('/v3/metadata/nlog/index.json')
+        .reply(200, nlogRegistration)
+        .get('/v3-flatcontainer/nlog/4.7.3/nlog.4.7.3.nupkg')
+        .reply(200, () => {
+          const readableStream = new Readable();
+          readableStream.push(
+            Fixtures.getBinary('nlog/NLog.4.7.3-no-repo.nupkg'),
+          );
+          readableStream.push(null);
+          return readableStream;
+        });
+      const res = await getPkgReleases({
+        datasource,
+        versioning,
+        packageName: 'NLog',
+        registryUrls: ['https://some-registry/v3/index.json'],
+      });
+      expect(packageCache.set).toHaveBeenCalledWith(
+        'datasource-nuget',
+        'source-url:https://some-registry/v3-flatcontainer/nlog/4.7.3/nlog.4.7.3.nupkg',
+        null,
+        60 * 24 * 7,
+      );
+      expect(res?.sourceUrl).toBeUndefined();
+    });
+
     it('returns null for non 200 (v3v2)', async () => {
       httpMock.scope('https://api.nuget.org').get('/v3/index.json').reply(500);
       httpMock

diff --git a/lib/modules/datasource/nuget/v3.ts b/lib/modules/datasource/nuget/v3.ts
@@ -224,7 +224,7 @@ export async function getReleases(
         dep.sourceUrl = massageUrl(sourceUrl);
       }
     } else if (latestNupkgUrl) {
-      const sourceUrl = await getSourceUrlFromNupkg(
+      const sourceUrl = await getSourceUrlFromNupkgCached(
         http,
         pkgName,
         latestStable,
@@ -268,37 +268,64 @@ export async function getReleases(
   return dep;
 }
 
+async function getSourceUrlFromNupkgCached(
+  http: Http,
+  packageName: string,
+  packageVersion: string | null,
+  nupkgUrl: string,
+): Promise<string | null> {
+  const cacheKey = `source-url:${nupkgUrl}`;
+  const sourceUrlFromCache = await packageCache.get<string | null>(
+    cacheNamespace,
+    cacheKey,
+  );
+  // istanbul ignore if
+  if (sourceUrlFromCache !== undefined) {
+    return sourceUrlFromCache;
+  }
+  const sourceUrl = await getSourceUrlFromNupkg(
+    http,
+    packageName,
+    packageVersion,
+    nupkgUrl,
+  );
+  const cacheTtl = 10080; // 1 week
+  await packageCache.set(cacheNamespace, cacheKey, sourceUrl, cacheTtl);
+  return sourceUrl;
+}
+
 async function getSourceUrlFromNupkg(
   http: Http,
   packageName: string,
   packageVersion: string | null,
   nupkgUrl: string,
-): Promise<string | undefined> {
+): Promise<string | null> {
   // istanbul ignore if: experimental feature
   if (process.env.RENOVATE_X_NUGET_DISABLE_NUPKG_DOWNLOAD) {
     logger.debug(
       `Skipping nupkg download because RENOVATE_X_NUGET_DISABLE_NUPKG_DOWNLOAD is set.`,
     );
-    return undefined;
+    return null;
   }
-  const cacheDir = await ensureCacheDir(`nuget`);
+  const cacheDir = await ensureCacheDir('nuget');
+  const nupkgFile = upath.join(
+    cacheDir,
+    `${packageName}.${packageVersion}.nupkg`,
+  );
+  const nupkgContentsDir = upath.join(
+    cacheDir,
+    `${packageName}.${packageVersion}`,
+  );
   const readStream = http.stream(nupkgUrl);
   try {
-    const nupkgFile = upath.join(
-      cacheDir,
-      `${packageName}.${packageVersion}.nupkg`,
-    );
     const writeStream = fs.createCacheWriteStream(nupkgFile);
     await fs.pipeline(readStream, writeStream);
-    const contentsDir = upath.join(
-      cacheDir,
-      `${packageName}.${packageVersion}`,
-    );
-    await extract(nupkgFile, { dir: contentsDir });
-    const nuspecFile = upath.join(contentsDir, `${packageName}.nuspec`);
+    await extract(nupkgFile, { dir: nupkgContentsDir });
+    const nuspecFile = upath.join(nupkgContentsDir, `${packageName}.nuspec`);
     const nuspec = new XmlDocument(await fs.readCacheFile(nuspecFile, 'utf8'));
-    return nuspec.valueWithPath('metadata.repository@url');
+    return nuspec.valueWithPath('metadata.repository@url') ?? null;
   } finally {
-    await fs.rmCache(cacheDir);
+    await fs.rmCache(nupkgFile);
+    await fs.rmCache(nupkgContentsDir);
   }
 }