feat(docker): Long-term cache for Docker Hub tags (#28489)

Co-authored-by: Michael Kriese <[email protected]>

lib/modules/datasource/docker/dockerhub-cache.spec.ts

@@ -0,0 +1,176 @@
+import { mocked } from '../../../../test/util';
+import * as _packageCache from '../../../util/cache/package';
+import { DockerHubCache, DockerHubCacheData } from './dockerhub-cache';
+import type { DockerHubTag } from './schema';
+
+jest.mock('../../../util/cache/package');
+const packageCache = mocked(_packageCache);
+
+function oldCacheData(): DockerHubCacheData {
+  return {
+    items: {
+      1: {
+        id: 1,
+        last_updated: '2022-01-01',
+        name: '1',
+        tag_last_pushed: '2022-01-01',
+        digest: 'sha256:111',
+      },
+      2: {
+        id: 2,
+        last_updated: '2022-01-02',
+        name: '2',
+        tag_last_pushed: '2022-01-02',
+        digest: 'sha256:222',
+      },
+      3: {
+        id: 3,
+        last_updated: '2022-01-03',
+        name: '3',
+        tag_last_pushed: '2022-01-03',
+        digest: 'sha256:333',
+      },
+    },
+    updatedAt: '2022-01-01',
+  };
+}
+
+function newItem(): DockerHubTag {
+  return {
+    id: 4,
+    last_updated: '2022-01-04',
+    name: '4',
+    tag_last_pushed: '2022-01-04',
+    digest: 'sha256:444',
+  };
+}
+
+function newCacheData(): DockerHubCacheData {
+  const { items } = oldCacheData();
+  const item = newItem();
+  return {
+    items: {
+      ...items,
+      [item.id]: item,
+    },
+    updatedAt: '2022-01-04',
+  };
+}
+
+describe('modules/datasource/docker/dockerhub-cache', () => {
+  beforeEach(() => {
+    jest.resetAllMocks();
+  });
+
+  const dockerRepository = 'foo/bar';
+
+  it('initializes empty cache', async () => {
+    packageCache.get.mockResolvedValue(undefined);
+
+    const res = await DockerHubCache.init(dockerRepository);
+
+    expect(res).toEqual({
+      dockerRepository,
+      cache: {
+        items: {},
+        updatedAt: null,
+      },
+      isChanged: false,
+    });
+  });
+
+  it('initializes cache with data', async () => {
+    const oldCache = oldCacheData();
+    packageCache.get.mockResolvedValue(oldCache);
+
+    const res = await DockerHubCache.init(dockerRepository);
+
+    expect(res).toEqual({
+      dockerRepository,
+      cache: oldCache,
+      isChanged: false,
+    });
+  });
+
+  it('reconciles new items', async () => {
+    const oldCache = oldCacheData();
+    const newCache = newCacheData();
+
+    packageCache.get.mockResolvedValue(oldCache);
+    const cache = await DockerHubCache.init(dockerRepository);
+    const newItems: DockerHubTag[] = [newItem()];
+
+    const needNextPage = cache.reconcile(newItems);
+
+    expect(needNextPage).toBe(true);
+    expect(cache).toEqual({
+      cache: newCache,
+      dockerRepository: 'foo/bar',
+      isChanged: true,
+    });
+
+    const res = cache.getItems();
+    expect(res).toEqual(Object.values(newCache.items));
+
+    await cache.save();
+    expect(packageCache.set).toHaveBeenCalledWith(
+      'datasource-docker-hub-cache',
+      'foo/bar',
+      newCache,
+      3 * 60 * 24 * 30,
+    );
+  });
+
+  it('reconciles existing items', async () => {
+    const oldCache = oldCacheData();
+
+    packageCache.get.mockResolvedValue(oldCache);
+    const cache = await DockerHubCache.init(dockerRepository);
+    const items: DockerHubTag[] = Object.values(oldCache.items);
+
+    const needNextPage = cache.reconcile(items);
+
+    expect(needNextPage).toBe(false);
+    expect(cache).toEqual({
+      cache: oldCache,
+      dockerRepository: 'foo/bar',
+      isChanged: false,
+    });
+
+    const res = cache.getItems();
+    expect(res).toEqual(items);
+
+    await cache.save();
+    expect(packageCache.set).not.toHaveBeenCalled();
+  });
+
+  it('reconciles from empty cache', async () => {
+    const item = newItem();
+    const expectedCache = {
+      items: {
+        [item.id]: item,
+      },
+      updatedAt: item.last_updated,
+    };
+    const cache = await DockerHubCache.init(dockerRepository);
+
+    const needNextPage = cache.reconcile([item]);
+    expect(needNextPage).toBe(true);
+    expect(cache).toEqual({
+      cache: expectedCache,
+      dockerRepository: 'foo/bar',
+      isChanged: true,
+    });
+
+    const res = cache.getItems();
+    expect(res).toEqual([item]);
+
+    await cache.save();
+    expect(packageCache.set).toHaveBeenCalledWith(
+      'datasource-docker-hub-cache',
+      'foo/bar',
+      expectedCache,
+      3 * 60 * 24 * 30,
+    );
+  });
+});

lib/modules/datasource/docker/dockerhub-cache.ts

@@ -0,0 +1,78 @@
+import { dequal } from 'dequal';
+import { DateTime } from 'luxon';
+import * as packageCache from '../../../util/cache/package';
+import type { DockerHubTag } from './schema';
+
+export interface DockerHubCacheData {
+  items: Record<number, DockerHubTag>;
+  updatedAt: string | null;
+}
+
+const cacheNamespace = 'datasource-docker-hub-cache';
+
+export class DockerHubCache {
+  private isChanged = false;
+
+  private constructor(
+    private dockerRepository: string,
+    private cache: DockerHubCacheData,
+  ) {}
+
+  static async init(dockerRepository: string): Promise<DockerHubCache> {
+    let repoCache = await packageCache.get<DockerHubCacheData>(
+      cacheNamespace,
+      dockerRepository,
+    );
+
+    repoCache ??= {
+      items: {},
+      updatedAt: null,
+    };
+
+    return new DockerHubCache(dockerRepository, repoCache);
+  }
+
+  reconcile(items: DockerHubTag[]): boolean {
+    let needNextPage = true;
+
+    let { updatedAt } = this.cache;
+    let latestDate = updatedAt ? DateTime.fromISO(updatedAt) : null;
+
+    for (const newItem of items) {
+      const id = newItem.id;
+      const oldItem = this.cache.items[id];
+
+      if (dequal(oldItem, newItem)) {
+        needNextPage = false;
+        continue;
+      }
+
+      this.cache.items[newItem.id] = newItem;
+      const newItemDate = DateTime.fromISO(newItem.last_updated);
+      if (!latestDate || latestDate < newItemDate) {
+        updatedAt = newItem.last_updated;
+        latestDate = newItemDate;
+      }
+
+      this.isChanged = true;
+    }
+
+    this.cache.updatedAt = updatedAt;
+    return needNextPage;
+  }
+
+  async save(): Promise<void> {
+    if (this.isChanged) {
+      await packageCache.set(
+        cacheNamespace,
+        this.dockerRepository,
+        this.cache,
+        3 * 60 * 24 * 30,
+      );
+    }
+  }
+
+  getItems(): DockerHubTag[] {
+    return Object.values(this.cache.items);
+  }
+}

lib/modules/datasource/docker/index.spec.ts

@@ -1796,21 +1796,25 @@ describe('modules/datasource/docker/index', () => {
       process.env.RENOVATE_X_DOCKER_HUB_TAGS = 'true';
       httpMock
         .scope(dockerHubUrl)
-        .get('/library/node/tags?page_size=1000')
+        .get('/library/node/tags?page_size=1000&ordering=last_updated')
         .reply(200, {
-          next: `${dockerHubUrl}/library/node/tags?page=2&page_size=1000`,
+          next: `${dockerHubUrl}/library/node/tags?page=2&page_size=1000&ordering=last_updated`,
           results: [
             {
+              id: 2,
+              last_updated: '2021-01-01T00:00:00.000Z',
               name: '1.0.0',
               tag_last_pushed: '2021-01-01T00:00:00.000Z',
               digest: 'aaa',
             },
           ],
         })
-        .get('/library/node/tags?page=2&page_size=1000')
+        .get('/library/node/tags?page=2&page_size=1000&ordering=last_updated')
         .reply(200, {
           results: [
             {
+              id: 1,
+              last_updated: '2020-01-01T00:00:00.000Z',
               name: '0.9.0',
               tag_last_pushed: '2020-01-01T00:00:00.000Z',
               digest: 'bbb',
@@ -1838,7 +1842,7 @@ describe('modules/datasource/docker/index', () => {
       const tags = ['1.0.0'];
       httpMock
         .scope(dockerHubUrl)
-        .get('/library/node/tags?page_size=1000')
+        .get('/library/node/tags?page_size=1000&ordering=last_updated')
         .reply(404);
       httpMock
         .scope(baseUrl)
@@ -1866,21 +1870,25 @@ describe('modules/datasource/docker/index', () => {
       process.env.RENOVATE_X_DOCKER_HUB_TAGS = 'true';
       httpMock
         .scope(dockerHubUrl)
-        .get('/library/node/tags?page_size=1000')
+        .get('/library/node/tags?page_size=1000&ordering=last_updated')
         .reply(200, {
-          next: `${dockerHubUrl}/library/node/tags?page=2&page_size=1000`,
+          next: `${dockerHubUrl}/library/node/tags?page=2&page_size=1000&ordering=last_updated`,
           results: [
             {
+              id: 2,
+              last_updated: '2021-01-01T00:00:00.000Z',
               name: '1.0.0',
               tag_last_pushed: '2021-01-01T00:00:00.000Z',
               digest: 'aaa',
             },
           ],
         })
-        .get('/library/node/tags?page=2&page_size=1000')
+        .get('/library/node/tags?page=2&page_size=1000&ordering=last_updated')
         .reply(200, {
           results: [
             {
+              id: 1,
+              last_updated: '2020-01-01T00:00:00.000Z',
               name: '0.9.0',
               tag_last_pushed: '2020-01-01T00:00:00.000Z',
               digest: 'bbb',

lib/modules/datasource/docker/index.ts

@@ -37,6 +37,7 @@ import {
   sourceLabel,
   sourceLabels,
 } from './common';
+import { DockerHubCache } from './dockerhub-cache';
 import { ecrPublicRegex, ecrRegex, isECRMaxResultsError } from './ecr';
 import {
   DistributionManifest,
@@ -927,10 +928,11 @@ export class DockerDatasource extends Datasource {
     key: (dockerRepository: string) => `${dockerRepository}`,
   })
   async getDockerHubTags(dockerRepository: string): Promise<Release[] | null> {
-    const result: Release[] = [];
-    let url: null | string =
-      `https://hub.docker.com/v2/repositories/${dockerRepository}/tags?page_size=1000`;
-    while (url) {
+    let url = `https://hub.docker.com/v2/repositories/${dockerRepository}/tags?page_size=1000&ordering=last_updated`;
+
+    const cache = await DockerHubCache.init(dockerRepository);
+    let needNextPage: boolean = true;
+    while (needNextPage) {
       const { val, err } = await this.http
         .getJsonSafe(url, DockerHubTagsPage)
         .unwrap();
@@ -940,11 +942,39 @@ export class DockerDatasource extends Datasource {
         return null;
       }
 
-      result.push(...val.items);
-      url = val.nextPage;
+      const { results, next } = val;
+
+      needNextPage = cache.reconcile(results);
+
+      if (!next) {
+        break;
+      }
+
+      url = next;
     }
 
-    return result;
+    await cache.save();
+
+    const items = cache.getItems();
+    return items.map(
+      ({
+        name: version,
+        tag_last_pushed: releaseTimestamp,
+        digest: newDigest,
+      }) => {
+        const release: Release = { version };
+
+        if (releaseTimestamp) {
+          release.releaseTimestamp = releaseTimestamp;
+        }
+
+        if (newDigest) {
+          release.newDigest = newDigest;
+        }
+
+        return release;
+      },
+    );
   }
 
   /**