From 9912662953531648ef7df2f991beb229565d1666 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Wed, 17 Apr 2024 20:56:04 -0300 Subject: [PATCH 1/5] feat(docker): Long-term cache for Docker Hub tags --- .../datasource/docker/dockerhub-cache.spec.ts | 176 ++++++++++++++++++ .../datasource/docker/dockerhub-cache.ts | 76 ++++++++ lib/modules/datasource/docker/index.spec.ts | 22 ++- lib/modules/datasource/docker/index.ts | 44 ++++- lib/modules/datasource/docker/schema.ts | 56 ++---- lib/util/cache/package/types.ts | 1 + 6 files changed, 325 insertions(+), 50 deletions(-) create mode 100644 lib/modules/datasource/docker/dockerhub-cache.spec.ts create mode 100644 lib/modules/datasource/docker/dockerhub-cache.ts diff --git a/lib/modules/datasource/docker/dockerhub-cache.spec.ts b/lib/modules/datasource/docker/dockerhub-cache.spec.ts new file mode 100644 index 00000000000000..18bccde555b128 --- /dev/null +++ b/lib/modules/datasource/docker/dockerhub-cache.spec.ts @@ -0,0 +1,176 @@ +import { mocked } from '../../../../test/util'; +import * as _packageCache from '../../../util/cache/package'; +import { DockerHubCache, DockerHubCacheData } from './dockerhub-cache'; +import type { DockerHubTag } from './schema'; + +jest.mock('../../../util/cache/package'); +const packageCache = mocked(_packageCache); + +function oldCacheData(): DockerHubCacheData { + return { + items: { + 1: { + id: 1, + last_updated: '2022-01-01', + name: '1', + tag_last_pushed: '2022-01-01', + digest: 'sha256:111', + }, + 2: { + id: 2, + last_updated: '2022-01-02', + name: '2', + tag_last_pushed: '2022-01-02', + digest: 'sha256:222', + }, + 3: { + id: 3, + last_updated: '2022-01-03', + name: '3', + tag_last_pushed: '2022-01-03', + digest: 'sha256:333', + }, + }, + updatedAt: '2022-01-01', + }; +} + +function newItem(): DockerHubTag { + return { + id: 4, + last_updated: '2022-01-04', + name: '4', + tag_last_pushed: '2022-01-04', + digest: 'sha256:444', + }; +} + +function newCacheData(): DockerHubCacheData { + const { items } = oldCacheData(); + const item = newItem(); + return { + items: { + ...items, + [item.id]: item, + }, + updatedAt: '2022-01-04', + }; +} + +describe('modules/datasource/docker/dockerhub-cache', () => { + beforeEach(() => { + jest.resetAllMocks(); + }); + + const dockerRepository = 'foo/bar'; + + it('initializes empty cache', async () => { + packageCache.get.mockResolvedValue(undefined); + + const res = await DockerHubCache.init(dockerRepository); + + expect(res).toEqual({ + dockerRepository, + cache: { + items: {}, + updatedAt: null, + }, + isChanged: false, + }); + }); + + it('initializes cache with data', async () => { + const oldCache = oldCacheData(); + packageCache.get.mockResolvedValue(oldCache); + + const res = await DockerHubCache.init(dockerRepository); + + expect(res).toEqual({ + dockerRepository, + cache: oldCache, + isChanged: false, + }); + }); + + it('reconciles new items', async () => { + const oldCache = oldCacheData(); + const newCache = newCacheData(); + + packageCache.get.mockResolvedValue(oldCache); + const cache = await DockerHubCache.init(dockerRepository); + const newItems: DockerHubTag[] = [newItem()]; + + const needNextPage = cache.reconcile(newItems); + + expect(needNextPage).toBe(true); + expect(cache).toEqual({ + cache: newCache, + dockerRepository: 'foo/bar', + isChanged: true, + }); + + const res = cache.getItems(); + expect(res).toEqual(Object.values(newCache.items)); + + await cache.save(); + expect(packageCache.set).toHaveBeenCalledWith( + 'datasource-docker-cache', + 'foo/bar', + newCache, + 3 * 60 * 24 * 30, + ); + }); + + it('reconciles existing items', async () => { + const oldCache = oldCacheData(); + + packageCache.get.mockResolvedValue(oldCache); + const cache = await DockerHubCache.init(dockerRepository); + const items: DockerHubTag[] = Object.values(oldCache.items); + + const needNextPage = cache.reconcile(items); + + expect(needNextPage).toBe(false); + expect(cache).toEqual({ + cache: oldCache, + dockerRepository: 'foo/bar', + isChanged: false, + }); + + const res = cache.getItems(); + expect(res).toEqual(items); + + await cache.save(); + expect(packageCache.set).not.toHaveBeenCalled(); + }); + + it('reconciles from empty cache', async () => { + const item = newItem(); + const expectedCache = { + items: { + [item.id]: item, + }, + updatedAt: item.last_updated, + }; + const cache = await DockerHubCache.init(dockerRepository); + + const needNextPage = cache.reconcile([item]); + expect(needNextPage).toBe(true); + expect(cache).toEqual({ + cache: expectedCache, + dockerRepository: 'foo/bar', + isChanged: true, + }); + + const res = cache.getItems(); + expect(res).toEqual([item]); + + await cache.save(); + expect(packageCache.set).toHaveBeenCalledWith( + 'datasource-docker-cache', + 'foo/bar', + expectedCache, + 3 * 60 * 24 * 30, + ); + }); +}); diff --git a/lib/modules/datasource/docker/dockerhub-cache.ts b/lib/modules/datasource/docker/dockerhub-cache.ts new file mode 100644 index 00000000000000..b798879586216f --- /dev/null +++ b/lib/modules/datasource/docker/dockerhub-cache.ts @@ -0,0 +1,76 @@ +import { dequal } from 'dequal'; +import { DateTime } from 'luxon'; +import * as packageCache from '../../../util/cache/package'; +import type { DockerHubTag } from './schema'; + +export interface DockerHubCacheData { + items: Record; + updatedAt: string | null; +} + +export class DockerHubCache { + private isChanged = false; + + private constructor( + private dockerRepository: string, + private cache: DockerHubCacheData, + ) {} + + static async init(dockerRepository: string): Promise { + let repoCache = await packageCache.get( + 'datasource-docker-cache', + dockerRepository, + ); + + repoCache ??= { + items: {}, + updatedAt: null, + }; + + return new DockerHubCache(dockerRepository, repoCache); + } + + reconcile(items: DockerHubTag[]): boolean { + let needNextPage = true; + + let { updatedAt } = this.cache; + let latestDate = updatedAt ? DateTime.fromISO(updatedAt) : null; + + for (const newItem of items) { + const id = newItem.id; + const oldItem = this.cache.items[id]; + + if (dequal(oldItem, newItem)) { + needNextPage = false; + continue; + } + + this.cache.items[newItem.id] = newItem; + const newItemDate = DateTime.fromISO(newItem.last_updated); + if (!latestDate || latestDate < newItemDate) { + updatedAt = newItem.last_updated; + latestDate = newItemDate; + } + + this.isChanged = true; + } + + this.cache.updatedAt = updatedAt; + return needNextPage; + } + + async save(): Promise { + if (this.isChanged) { + await packageCache.set( + 'datasource-docker-cache', + this.dockerRepository, + this.cache, + 3 * 60 * 24 * 30, + ); + } + } + + getItems(): DockerHubTag[] { + return Object.values(this.cache.items); + } +} diff --git a/lib/modules/datasource/docker/index.spec.ts b/lib/modules/datasource/docker/index.spec.ts index d450cbad183722..e16d42ed533b86 100644 --- a/lib/modules/datasource/docker/index.spec.ts +++ b/lib/modules/datasource/docker/index.spec.ts @@ -1796,21 +1796,25 @@ describe('modules/datasource/docker/index', () => { process.env.RENOVATE_X_DOCKER_HUB_TAGS = 'true'; httpMock .scope(dockerHubUrl) - .get('/library/node/tags?page_size=1000') + .get('/library/node/tags?page_size=1000&ordering=last_updated') .reply(200, { - next: `${dockerHubUrl}/library/node/tags?page=2&page_size=1000`, + next: `${dockerHubUrl}/library/node/tags?page=2&page_size=1000&ordering=last_updated`, results: [ { + id: 2, + last_updated: '2021-01-01T00:00:00.000Z', name: '1.0.0', tag_last_pushed: '2021-01-01T00:00:00.000Z', digest: 'aaa', }, ], }) - .get('/library/node/tags?page=2&page_size=1000') + .get('/library/node/tags?page=2&page_size=1000&ordering=last_updated') .reply(200, { results: [ { + id: 1, + last_updated: '2020-01-01T00:00:00.000Z', name: '0.9.0', tag_last_pushed: '2020-01-01T00:00:00.000Z', digest: 'bbb', @@ -1838,7 +1842,7 @@ describe('modules/datasource/docker/index', () => { const tags = ['1.0.0']; httpMock .scope(dockerHubUrl) - .get('/library/node/tags?page_size=1000') + .get('/library/node/tags?page_size=1000&ordering=last_updated') .reply(404); httpMock .scope(baseUrl) @@ -1866,21 +1870,25 @@ describe('modules/datasource/docker/index', () => { process.env.RENOVATE_X_DOCKER_HUB_TAGS = 'true'; httpMock .scope(dockerHubUrl) - .get('/library/node/tags?page_size=1000') + .get('/library/node/tags?page_size=1000&ordering=last_updated') .reply(200, { - next: `${dockerHubUrl}/library/node/tags?page=2&page_size=1000`, + next: `${dockerHubUrl}/library/node/tags?page=2&page_size=1000&ordering=last_updated`, results: [ { + id: 2, + last_updated: '2021-01-01T00:00:00.000Z', name: '1.0.0', tag_last_pushed: '2021-01-01T00:00:00.000Z', digest: 'aaa', }, ], }) - .get('/library/node/tags?page=2&page_size=1000') + .get('/library/node/tags?page=2&page_size=1000&ordering=last_updated') .reply(200, { results: [ { + id: 1, + last_updated: '2020-01-01T00:00:00.000Z', name: '0.9.0', tag_last_pushed: '2020-01-01T00:00:00.000Z', digest: 'bbb', diff --git a/lib/modules/datasource/docker/index.ts b/lib/modules/datasource/docker/index.ts index 5d896ed774dee7..5404ad09e1d9a5 100644 --- a/lib/modules/datasource/docker/index.ts +++ b/lib/modules/datasource/docker/index.ts @@ -37,6 +37,7 @@ import { sourceLabel, sourceLabels, } from './common'; +import { DockerHubCache } from './dockerhub-cache'; import { ecrPublicRegex, ecrRegex, isECRMaxResultsError } from './ecr'; import { DistributionManifest, @@ -927,10 +928,11 @@ export class DockerDatasource extends Datasource { key: (dockerRepository: string) => `${dockerRepository}`, }) async getDockerHubTags(dockerRepository: string): Promise { - const result: Release[] = []; - let url: null | string = - `https://hub.docker.com/v2/repositories/${dockerRepository}/tags?page_size=1000`; - while (url) { + let url: string = `https://hub.docker.com/v2/repositories/${dockerRepository}/tags?page_size=1000&ordering=last_updated`; + + const cache = await DockerHubCache.init(dockerRepository); + let needNextPage: boolean = true; + while (needNextPage) { const { val, err } = await this.http .getJsonSafe(url, DockerHubTagsPage) .unwrap(); @@ -940,11 +942,39 @@ export class DockerDatasource extends Datasource { return null; } - result.push(...val.items); - url = val.nextPage; + const { results, next } = val; + + needNextPage = cache.reconcile(results); + + if (!next) { + break; + } + + url = next; } - return result; + await cache.save(); + + const items = cache.getItems(); + return items.map( + ({ + name: version, + tag_last_pushed: releaseTimestamp, + digest: newDigest, + }) => { + const release: Release = { version }; + + if (releaseTimestamp) { + release.releaseTimestamp = releaseTimestamp; + } + + if (newDigest) { + release.newDigest = newDigest; + } + + return release; + }, + ); } /** diff --git a/lib/modules/datasource/docker/schema.ts b/lib/modules/datasource/docker/schema.ts index 1af87da24b49d4..68146b4e56ccdf 100644 --- a/lib/modules/datasource/docker/schema.ts +++ b/lib/modules/datasource/docker/schema.ts @@ -155,39 +155,23 @@ export const Manifest = ManifestObject.passthrough() export type Manifest = z.infer; export const ManifestJson = Json.pipe(Manifest); -export const DockerHubTag = z - .object({ - name: z.string(), - tag_last_pushed: z.string().datetime().nullable().catch(null), - digest: z.string().nullable().catch(null), - }) - .transform(({ name, tag_last_pushed, digest }) => { - const release: Release = { version: name }; - - if (tag_last_pushed) { - release.releaseTimestamp = tag_last_pushed; - } - - if (digest) { - release.newDigest = digest; - } - - return release; - }); - -export const DockerHubTagsPage = z - .object({ - next: z.string().nullable().catch(null), - results: LooseArray(DockerHubTag, { - onError: /* istanbul ignore next */ ({ error }) => { - logger.debug( - { error }, - 'Docker: Failed to parse some tags from Docker Hub', - ); - }, - }), - }) - .transform(({ next, results }) => ({ - nextPage: next, - items: results, - })); +export const DockerHubTag = z.object({ + id: z.number(), + last_updated: z.string().datetime(), + name: z.string(), + tag_last_pushed: z.string().datetime().nullable().catch(null), + digest: z.string().nullable().catch(null), +}); +export type DockerHubTag = z.infer; + +export const DockerHubTagsPage = z.object({ + next: z.string().nullable().catch(null), + results: LooseArray(DockerHubTag, { + onError: /* istanbul ignore next */ ({ error }) => { + logger.debug( + { error }, + 'Docker: Failed to parse some tags from Docker Hub', + ); + }, + }), +}); diff --git a/lib/util/cache/package/types.ts b/lib/util/cache/package/types.ts index bb1673c399d8eb..3b45a5a0970a56 100644 --- a/lib/util/cache/package/types.ts +++ b/lib/util/cache/package/types.ts @@ -45,6 +45,7 @@ export type PackageCacheNamespace = | 'datasource-deno-versions' | 'datasource-deno' | 'datasource-docker-architecture' + | 'datasource-docker-cache' | 'datasource-docker-digest' | 'datasource-docker-hub-tags' | 'datasource-docker-imageconfig' From 72d7c7402a18d6ac246bbfc899849da3c5eff10e Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Wed, 17 Apr 2024 21:02:48 -0300 Subject: [PATCH 2/5] Fix lint --- lib/modules/datasource/docker/schema.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/modules/datasource/docker/schema.ts b/lib/modules/datasource/docker/schema.ts index 68146b4e56ccdf..6a58e08abc513a 100644 --- a/lib/modules/datasource/docker/schema.ts +++ b/lib/modules/datasource/docker/schema.ts @@ -1,7 +1,6 @@ import { z } from 'zod'; import { logger } from '../../../logger'; import { Json, LooseArray } from '../../../util/schema-utils'; -import type { Release } from '../types'; // OCI manifests From 1224ea0d668aac73af6f9833635b661b2659545a Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Thu, 18 Apr 2024 07:10:37 -0300 Subject: [PATCH 3/5] Update lib/modules/datasource/docker/index.ts Co-authored-by: Michael Kriese --- lib/modules/datasource/docker/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/modules/datasource/docker/index.ts b/lib/modules/datasource/docker/index.ts index 5404ad09e1d9a5..0f5563551d38c8 100644 --- a/lib/modules/datasource/docker/index.ts +++ b/lib/modules/datasource/docker/index.ts @@ -928,7 +928,7 @@ export class DockerDatasource extends Datasource { key: (dockerRepository: string) => `${dockerRepository}`, }) async getDockerHubTags(dockerRepository: string): Promise { - let url: string = `https://hub.docker.com/v2/repositories/${dockerRepository}/tags?page_size=1000&ordering=last_updated`; + let url = `https://hub.docker.com/v2/repositories/${dockerRepository}/tags?page_size=1000&ordering=last_updated`; const cache = await DockerHubCache.init(dockerRepository); let needNextPage: boolean = true; From fb3551f348c63d987f48df0ad2a5291022669407 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Thu, 18 Apr 2024 10:55:56 -0300 Subject: [PATCH 4/5] Change cache key --- lib/modules/datasource/docker/dockerhub-cache.ts | 6 ++++-- lib/util/cache/package/types.ts | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/modules/datasource/docker/dockerhub-cache.ts b/lib/modules/datasource/docker/dockerhub-cache.ts index b798879586216f..0e97726fc01fb2 100644 --- a/lib/modules/datasource/docker/dockerhub-cache.ts +++ b/lib/modules/datasource/docker/dockerhub-cache.ts @@ -8,6 +8,8 @@ export interface DockerHubCacheData { updatedAt: string | null; } +const cacheNamespace = 'datasource-docker-hub-cache'; + export class DockerHubCache { private isChanged = false; @@ -18,7 +20,7 @@ export class DockerHubCache { static async init(dockerRepository: string): Promise { let repoCache = await packageCache.get( - 'datasource-docker-cache', + cacheNamespace, dockerRepository, ); @@ -62,7 +64,7 @@ export class DockerHubCache { async save(): Promise { if (this.isChanged) { await packageCache.set( - 'datasource-docker-cache', + cacheNamespace, this.dockerRepository, this.cache, 3 * 60 * 24 * 30, diff --git a/lib/util/cache/package/types.ts b/lib/util/cache/package/types.ts index 3b45a5a0970a56..6d600df7e14e70 100644 --- a/lib/util/cache/package/types.ts +++ b/lib/util/cache/package/types.ts @@ -45,7 +45,7 @@ export type PackageCacheNamespace = | 'datasource-deno-versions' | 'datasource-deno' | 'datasource-docker-architecture' - | 'datasource-docker-cache' + | 'datasource-docker-hub-cache' | 'datasource-docker-digest' | 'datasource-docker-hub-tags' | 'datasource-docker-imageconfig' From f36707b131fe0992f5b1ec2ec41a2f2394b9924e Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Thu, 18 Apr 2024 11:02:34 -0300 Subject: [PATCH 5/5] Fix tests --- lib/modules/datasource/docker/dockerhub-cache.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/modules/datasource/docker/dockerhub-cache.spec.ts b/lib/modules/datasource/docker/dockerhub-cache.spec.ts index 18bccde555b128..fe8ad65504229e 100644 --- a/lib/modules/datasource/docker/dockerhub-cache.spec.ts +++ b/lib/modules/datasource/docker/dockerhub-cache.spec.ts @@ -114,7 +114,7 @@ describe('modules/datasource/docker/dockerhub-cache', () => { await cache.save(); expect(packageCache.set).toHaveBeenCalledWith( - 'datasource-docker-cache', + 'datasource-docker-hub-cache', 'foo/bar', newCache, 3 * 60 * 24 * 30, @@ -167,7 +167,7 @@ describe('modules/datasource/docker/dockerhub-cache', () => { await cache.save(); expect(packageCache.set).toHaveBeenCalledWith( - 'datasource-docker-cache', + 'datasource-docker-hub-cache', 'foo/bar', expectedCache, 3 * 60 * 24 * 30,