Skip to content

Commit

Permalink
feat(docker): Reconcile deleted releases in cache (#30491)
Browse files Browse the repository at this point in the history 
Co-authored-by: Michael Kriese <[email protected]>
  • Loading branch information
@zharinov @viceice
zharinov and viceice authored Aug 6, 2024
1 parent 33d8d58 commit c225196
Show file tree
Hide file tree
Showing 5 changed files with 98 additions and 12 deletions.
53 changes: 50 additions & 3 deletions lib/modules/datasource/docker/dockerhub-cache.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ describe('modules/datasource/docker/dockerhub-cache', () => {
updatedAt: null,
},
isChanged: false,
reconciledIds: new Set(),
});
});

Expand All @@ -89,6 +90,7 @@ describe('modules/datasource/docker/dockerhub-cache', () => {
dockerRepository,
cache: oldCache,
isChanged: false,
reconciledIds: new Set(),
});
});

Expand All @@ -100,13 +102,14 @@ describe('modules/datasource/docker/dockerhub-cache', () => {
const cache = await DockerHubCache.init(dockerRepository);
const newItems: DockerHubTag[] = [newItem()];

const needNextPage = cache.reconcile(newItems);
const needNextPage = cache.reconcile(newItems, 4);

expect(needNextPage).toBe(true);
expect(cache).toEqual({
cache: newCache,
dockerRepository: 'foo/bar',
isChanged: true,
reconciledIds: new Set([4]),
});

const res = cache.getItems();
Expand All @@ -128,13 +131,14 @@ describe('modules/datasource/docker/dockerhub-cache', () => {
const cache = await DockerHubCache.init(dockerRepository);
const items: DockerHubTag[] = Object.values(oldCache.items);

const needNextPage = cache.reconcile(items);
const needNextPage = cache.reconcile(items, 3);

expect(needNextPage).toBe(false);
expect(cache).toEqual({
cache: oldCache,
dockerRepository: 'foo/bar',
isChanged: false,
reconciledIds: new Set([1, 2, 3]),
});

const res = cache.getItems();
Expand All @@ -144,6 +148,48 @@ describe('modules/datasource/docker/dockerhub-cache', () => {
expect(packageCache.set).not.toHaveBeenCalled();
});

it('asks for the next page if the expected count does not match cached items', async () => {
const oldCache = oldCacheData();

packageCache.get.mockResolvedValue(oldCache);
const cache = await DockerHubCache.init(dockerRepository);
const items: DockerHubTag[] = Object.values(oldCache.items);

const needNextPage = cache.reconcile(items, 0);

expect(needNextPage).toBe(true);
expect(cache).toEqual({
cache: oldCache,
dockerRepository: 'foo/bar',
isChanged: false,
reconciledIds: new Set([1, 2, 3]),
});

const res = cache.getItems();
expect(res).toEqual(items);

await cache.save();
expect(packageCache.set).not.toHaveBeenCalled();
});

it('reconciles deleted items', async () => {
const oldCache = oldCacheData();

packageCache.get.mockResolvedValue(oldCache);
const cache = await DockerHubCache.init(dockerRepository);
const items: DockerHubTag[] = [oldCache.items[1], oldCache.items[3]];

const needNextPage = cache.reconcile(items, 2);

expect(needNextPage).toBe(false);

const res = cache.getItems();
expect(res).toEqual(items);

await cache.save();
expect(packageCache.set).toHaveBeenCalled();
});

it('reconciles from empty cache', async () => {
const item = newItem();
const expectedCache = {
Expand All @@ -154,12 +200,13 @@ describe('modules/datasource/docker/dockerhub-cache', () => {
};
const cache = await DockerHubCache.init(dockerRepository);

const needNextPage = cache.reconcile([item]);
const needNextPage = cache.reconcile([item], 1);
expect(needNextPage).toBe(true);
expect(cache).toEqual({
cache: expectedCache,
dockerRepository: 'foo/bar',
isChanged: true,
reconciledIds: new Set([4]),
});

const res = cache.getItems();
Expand Down
48 changes: 41 additions & 7 deletions lib/modules/datasource/docker/dockerhub-cache.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ const cacheNamespace = 'datasource-docker-hub-cache';

export class DockerHubCache {
private isChanged = false;
private reconciledIds = new Set<number>();

private constructor(
private dockerRepository: string,
Expand All @@ -32,32 +33,65 @@ export class DockerHubCache {
return new DockerHubCache(dockerRepository, repoCache);
}

reconcile(items: DockerHubTag[]): boolean {
reconcile(items: DockerHubTag[], expectedCount: number): boolean {
let needNextPage = true;

let earliestDate = null;

let { updatedAt } = this.cache;
let latestDate = updatedAt ? DateTime.fromISO(updatedAt) : null;

for (const newItem of items) {
const id = newItem.id;
this.reconciledIds.add(id);

const oldItem = this.cache.items[id];

const itemDate = DateTime.fromISO(newItem.last_updated);

if (!earliestDate || earliestDate > itemDate) {
earliestDate = itemDate;
}

if (!latestDate || latestDate < itemDate) {
latestDate = itemDate;
updatedAt = newItem.last_updated;
}

if (dequal(oldItem, newItem)) {
needNextPage = false;
continue;
}

this.cache.items[newItem.id] = newItem;
const newItemDate = DateTime.fromISO(newItem.last_updated);
if (!latestDate || latestDate < newItemDate) {
updatedAt = newItem.last_updated;
latestDate = newItemDate;
}

this.isChanged = true;
}

this.cache.updatedAt = updatedAt;

if (earliestDate && latestDate) {
for (const [key, item] of Object.entries(this.cache.items)) {
const id = parseInt(key, 10);

const itemDate = DateTime.fromISO(item.last_updated);

if (
itemDate < earliestDate ||
itemDate > latestDate ||
this.reconciledIds.has(id)
) {
continue;
}

delete this.cache.items[id];
this.isChanged = true;
}

if (Object.keys(this.cache.items).length > expectedCount) {
return true;
}
}

return needNextPage;
}

Expand Down
4 changes: 4 additions & 0 deletions lib/modules/datasource/docker/index.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1887,6 +1887,7 @@ describe('modules/datasource/docker/index', () => {
.scope(dockerHubUrl)
.get('/library/node/tags?page_size=1000&ordering=last_updated')
.reply(200, {
count: 2,
next: `${dockerHubUrl}/library/node/tags?page=2&page_size=1000&ordering=last_updated`,
results: [
{
Expand All @@ -1900,6 +1901,7 @@ describe('modules/datasource/docker/index', () => {
})
.get('/library/node/tags?page=2&page_size=1000&ordering=last_updated')
.reply(200, {
count: 2,
results: [
{
id: 1,
Expand Down Expand Up @@ -1960,6 +1962,7 @@ describe('modules/datasource/docker/index', () => {
.get('/library/node/tags?page_size=1000&ordering=last_updated')
.reply(200, {
next: `${dockerHubUrl}/library/node/tags?page=2&page_size=1000&ordering=last_updated`,
count: 2,
results: [
{
id: 2,
Expand All @@ -1972,6 +1975,7 @@ describe('modules/datasource/docker/index', () => {
})
.get('/library/node/tags?page=2&page_size=1000&ordering=last_updated')
.reply(200, {
count: 2,
results: [
{
id: 1,
Expand Down
4 changes: 2 additions & 2 deletions lib/modules/datasource/docker/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -988,9 +988,9 @@ export class DockerDatasource extends Datasource {
return null;
}

const { results, next } = val;
const { results, next, count } = val;

needNextPage = cache.reconcile(results);
needNextPage = cache.reconcile(results, count);

if (!next) {
break;
Expand Down
1 change: 1 addition & 0 deletions lib/modules/datasource/docker/schema.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,7 @@ export const DockerHubTag = z.object({
export type DockerHubTag = z.infer<typeof DockerHubTag>;

export const DockerHubTagsPage = z.object({
count: z.number(),
next: z.string().nullable().catch(null),
results: LooseArray(DockerHubTag, {
onError: /* istanbul ignore next */ ({ error }) => {
Expand Down

0 comments on commit c225196

Please sign in to comment.