From c22519662bc9085eab04e75ee9cd175a9eb29dba Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Tue, 6 Aug 2024 05:00:50 -0300 Subject: [PATCH] feat(docker): Reconcile deleted releases in cache (#30491) Co-authored-by: Michael Kriese --- .../datasource/docker/dockerhub-cache.spec.ts | 53 +++++++++++++++++-- .../datasource/docker/dockerhub-cache.ts | 48 ++++++++++++++--- lib/modules/datasource/docker/index.spec.ts | 4 ++ lib/modules/datasource/docker/index.ts | 4 +- lib/modules/datasource/docker/schema.ts | 1 + 5 files changed, 98 insertions(+), 12 deletions(-) diff --git a/lib/modules/datasource/docker/dockerhub-cache.spec.ts b/lib/modules/datasource/docker/dockerhub-cache.spec.ts index fe8ad65504229e..1a97be3488a0ea 100644 --- a/lib/modules/datasource/docker/dockerhub-cache.spec.ts +++ b/lib/modules/datasource/docker/dockerhub-cache.spec.ts @@ -76,6 +76,7 @@ describe('modules/datasource/docker/dockerhub-cache', () => { updatedAt: null, }, isChanged: false, + reconciledIds: new Set(), }); }); @@ -89,6 +90,7 @@ describe('modules/datasource/docker/dockerhub-cache', () => { dockerRepository, cache: oldCache, isChanged: false, + reconciledIds: new Set(), }); }); @@ -100,13 +102,14 @@ describe('modules/datasource/docker/dockerhub-cache', () => { const cache = await DockerHubCache.init(dockerRepository); const newItems: DockerHubTag[] = [newItem()]; - const needNextPage = cache.reconcile(newItems); + const needNextPage = cache.reconcile(newItems, 4); expect(needNextPage).toBe(true); expect(cache).toEqual({ cache: newCache, dockerRepository: 'foo/bar', isChanged: true, + reconciledIds: new Set([4]), }); const res = cache.getItems(); @@ -128,13 +131,14 @@ describe('modules/datasource/docker/dockerhub-cache', () => { const cache = await DockerHubCache.init(dockerRepository); const items: DockerHubTag[] = Object.values(oldCache.items); - const needNextPage = cache.reconcile(items); + const needNextPage = cache.reconcile(items, 3); expect(needNextPage).toBe(false); expect(cache).toEqual({ cache: oldCache, dockerRepository: 'foo/bar', isChanged: false, + reconciledIds: new Set([1, 2, 3]), }); const res = cache.getItems(); @@ -144,6 +148,48 @@ describe('modules/datasource/docker/dockerhub-cache', () => { expect(packageCache.set).not.toHaveBeenCalled(); }); + it('asks for the next page if the expected count does not match cached items', async () => { + const oldCache = oldCacheData(); + + packageCache.get.mockResolvedValue(oldCache); + const cache = await DockerHubCache.init(dockerRepository); + const items: DockerHubTag[] = Object.values(oldCache.items); + + const needNextPage = cache.reconcile(items, 0); + + expect(needNextPage).toBe(true); + expect(cache).toEqual({ + cache: oldCache, + dockerRepository: 'foo/bar', + isChanged: false, + reconciledIds: new Set([1, 2, 3]), + }); + + const res = cache.getItems(); + expect(res).toEqual(items); + + await cache.save(); + expect(packageCache.set).not.toHaveBeenCalled(); + }); + + it('reconciles deleted items', async () => { + const oldCache = oldCacheData(); + + packageCache.get.mockResolvedValue(oldCache); + const cache = await DockerHubCache.init(dockerRepository); + const items: DockerHubTag[] = [oldCache.items[1], oldCache.items[3]]; + + const needNextPage = cache.reconcile(items, 2); + + expect(needNextPage).toBe(false); + + const res = cache.getItems(); + expect(res).toEqual(items); + + await cache.save(); + expect(packageCache.set).toHaveBeenCalled(); + }); + it('reconciles from empty cache', async () => { const item = newItem(); const expectedCache = { @@ -154,12 +200,13 @@ describe('modules/datasource/docker/dockerhub-cache', () => { }; const cache = await DockerHubCache.init(dockerRepository); - const needNextPage = cache.reconcile([item]); + const needNextPage = cache.reconcile([item], 1); expect(needNextPage).toBe(true); expect(cache).toEqual({ cache: expectedCache, dockerRepository: 'foo/bar', isChanged: true, + reconciledIds: new Set([4]), }); const res = cache.getItems(); diff --git a/lib/modules/datasource/docker/dockerhub-cache.ts b/lib/modules/datasource/docker/dockerhub-cache.ts index 0e97726fc01fb2..a90703e97b1b22 100644 --- a/lib/modules/datasource/docker/dockerhub-cache.ts +++ b/lib/modules/datasource/docker/dockerhub-cache.ts @@ -12,6 +12,7 @@ const cacheNamespace = 'datasource-docker-hub-cache'; export class DockerHubCache { private isChanged = false; + private reconciledIds = new Set(); private constructor( private dockerRepository: string, @@ -32,32 +33,65 @@ export class DockerHubCache { return new DockerHubCache(dockerRepository, repoCache); } - reconcile(items: DockerHubTag[]): boolean { + reconcile(items: DockerHubTag[], expectedCount: number): boolean { let needNextPage = true; + let earliestDate = null; + let { updatedAt } = this.cache; let latestDate = updatedAt ? DateTime.fromISO(updatedAt) : null; for (const newItem of items) { const id = newItem.id; + this.reconciledIds.add(id); + const oldItem = this.cache.items[id]; + const itemDate = DateTime.fromISO(newItem.last_updated); + + if (!earliestDate || earliestDate > itemDate) { + earliestDate = itemDate; + } + + if (!latestDate || latestDate < itemDate) { + latestDate = itemDate; + updatedAt = newItem.last_updated; + } + if (dequal(oldItem, newItem)) { needNextPage = false; continue; } this.cache.items[newItem.id] = newItem; - const newItemDate = DateTime.fromISO(newItem.last_updated); - if (!latestDate || latestDate < newItemDate) { - updatedAt = newItem.last_updated; - latestDate = newItemDate; - } - this.isChanged = true; } this.cache.updatedAt = updatedAt; + + if (earliestDate && latestDate) { + for (const [key, item] of Object.entries(this.cache.items)) { + const id = parseInt(key, 10); + + const itemDate = DateTime.fromISO(item.last_updated); + + if ( + itemDate < earliestDate || + itemDate > latestDate || + this.reconciledIds.has(id) + ) { + continue; + } + + delete this.cache.items[id]; + this.isChanged = true; + } + + if (Object.keys(this.cache.items).length > expectedCount) { + return true; + } + } + return needNextPage; } diff --git a/lib/modules/datasource/docker/index.spec.ts b/lib/modules/datasource/docker/index.spec.ts index 3b84b910e43d34..0380dbffdbf5e7 100644 --- a/lib/modules/datasource/docker/index.spec.ts +++ b/lib/modules/datasource/docker/index.spec.ts @@ -1887,6 +1887,7 @@ describe('modules/datasource/docker/index', () => { .scope(dockerHubUrl) .get('/library/node/tags?page_size=1000&ordering=last_updated') .reply(200, { + count: 2, next: `${dockerHubUrl}/library/node/tags?page=2&page_size=1000&ordering=last_updated`, results: [ { @@ -1900,6 +1901,7 @@ describe('modules/datasource/docker/index', () => { }) .get('/library/node/tags?page=2&page_size=1000&ordering=last_updated') .reply(200, { + count: 2, results: [ { id: 1, @@ -1960,6 +1962,7 @@ describe('modules/datasource/docker/index', () => { .get('/library/node/tags?page_size=1000&ordering=last_updated') .reply(200, { next: `${dockerHubUrl}/library/node/tags?page=2&page_size=1000&ordering=last_updated`, + count: 2, results: [ { id: 2, @@ -1972,6 +1975,7 @@ describe('modules/datasource/docker/index', () => { }) .get('/library/node/tags?page=2&page_size=1000&ordering=last_updated') .reply(200, { + count: 2, results: [ { id: 1, diff --git a/lib/modules/datasource/docker/index.ts b/lib/modules/datasource/docker/index.ts index 37863452698756..f318e3d2c11c80 100644 --- a/lib/modules/datasource/docker/index.ts +++ b/lib/modules/datasource/docker/index.ts @@ -988,9 +988,9 @@ export class DockerDatasource extends Datasource { return null; } - const { results, next } = val; + const { results, next, count } = val; - needNextPage = cache.reconcile(results); + needNextPage = cache.reconcile(results, count); if (!next) { break; diff --git a/lib/modules/datasource/docker/schema.ts b/lib/modules/datasource/docker/schema.ts index a7f494acfa6d03..1cfc4c0b521ea7 100644 --- a/lib/modules/datasource/docker/schema.ts +++ b/lib/modules/datasource/docker/schema.ts @@ -165,6 +165,7 @@ export const DockerHubTag = z.object({ export type DockerHubTag = z.infer; export const DockerHubTagsPage = z.object({ + count: z.number(), next: z.string().nullable().catch(null), results: LooseArray(DockerHubTag, { onError: /* istanbul ignore next */ ({ error }) => {