From bf89487fc8504ed4751eda0b5c87fb64e03f2d30 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Tue, 28 Jan 2025 14:34:13 -0300 Subject: [PATCH 1/9] feat(http): Create memory cache provider --- .../cache/memory-http-cache-provider.spec.ts | 39 +++++++++++++++++++ .../http/cache/memory-http-cache-provider.ts | 21 ++++++++++ 2 files changed, 60 insertions(+) create mode 100644 lib/util/http/cache/memory-http-cache-provider.spec.ts create mode 100644 lib/util/http/cache/memory-http-cache-provider.ts diff --git a/lib/util/http/cache/memory-http-cache-provider.spec.ts b/lib/util/http/cache/memory-http-cache-provider.spec.ts new file mode 100644 index 00000000000000..365a7aa8a17776 --- /dev/null +++ b/lib/util/http/cache/memory-http-cache-provider.spec.ts @@ -0,0 +1,39 @@ +import { Http } from '..'; +import * as httpMock from '../../../../test/http-mock'; +import * as memCache from '../../cache/memory'; +import { memCacheProvider as cacheProvider } from './memory-http-cache-provider'; + +describe('util/http/cache/memory-http-cache-provider', () => { + beforeEach(() => { + memCache.init(); + }); + + afterEach(() => { + memCache.reset(); + }); + + const http = new Http('test'); + + it('reuses data with etag', async () => { + const scope = httpMock.scope('https://example.com'); + + scope.get('/foo/bar').reply(200, { msg: 'Hello, world!' }, { etag: '123' }); + const res1 = await http.getJsonUnchecked('https://example.com/foo/bar', { + cacheProvider, + }); + expect(res1).toMatchObject({ + statusCode: 200, + body: { msg: 'Hello, world!' }, + authorization: false, + }); + + const res2 = await http.getJsonUnchecked('https://example.com/foo/bar', { + cacheProvider, + }); + expect(res2).toMatchObject({ + statusCode: 200, + body: { msg: 'Hello, world!' }, + authorization: false, + }); + }); +}); diff --git a/lib/util/http/cache/memory-http-cache-provider.ts b/lib/util/http/cache/memory-http-cache-provider.ts new file mode 100644 index 00000000000000..86ca660733d980 --- /dev/null +++ b/lib/util/http/cache/memory-http-cache-provider.ts @@ -0,0 +1,21 @@ +import * as memCache from '../../cache/memory'; +import { AbstractHttpCacheProvider } from './abstract-http-cache-provider'; +import type { HttpCache } from './schema'; + +export class MemoryHttpCacheProvider extends AbstractHttpCacheProvider { + private cacheKey(url: string): string { + return `memory-cache-http-provider:${url}`; + } + + protected override load(url: string): Promise { + const data = memCache.get(this.cacheKey(url)); + return Promise.resolve(data); + } + + protected override persist(url: string, data: HttpCache): Promise { + memCache.set(this.cacheKey(url), data); + return Promise.resolve(); + } +} + +export const memCacheProvider = new MemoryHttpCacheProvider(); From 91b3a1d5b18ec2945b3ef173c3b946b4ad693387 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Tue, 28 Jan 2025 15:25:09 -0300 Subject: [PATCH 2/9] feat(http): Switch to memory cache provider --- .../platform/bitbucket-server/index.ts | 20 +++++++++---- lib/modules/platform/bitbucket/index.ts | 20 ++++++------- lib/modules/platform/gerrit/client.ts | 11 +++++-- lib/modules/platform/gitea/gitea-helper.ts | 16 +++------- lib/modules/platform/gitea/index.ts | 24 +++++++-------- lib/modules/platform/gitea/pr-cache.ts | 5 +--- lib/modules/platform/github/branch.ts | 1 - lib/modules/platform/github/index.ts | 14 ++++----- lib/modules/platform/gitlab/index.ts | 29 ++++++++++--------- lib/util/http/index.ts | 25 +--------------- lib/util/http/types.ts | 2 -- 11 files changed, 72 insertions(+), 95 deletions(-) diff --git a/lib/modules/platform/bitbucket-server/index.ts b/lib/modules/platform/bitbucket-server/index.ts index 7f1b2accebc809..f05de92615795c 100644 --- a/lib/modules/platform/bitbucket-server/index.ts +++ b/lib/modules/platform/bitbucket-server/index.ts @@ -17,6 +17,7 @@ import { BitbucketServerHttp, setBaseUrl, } from '../../../util/http/bitbucket-server'; +import { memCacheProvider } from '../../../util/http/cache/memory-http-cache-provider'; import type { HttpOptions, HttpResponse } from '../../../util/http/types'; import { newlineRegex, regEx } from '../../../util/regex'; import { sanitize } from '../../../util/sanitize'; @@ -136,9 +137,7 @@ export async function initPlatform({ if (!gitAuthor && username) { logger.debug(`Attempting to confirm gitAuthor from username`); - const options: HttpOptions = { - memCache: false, - }; + const options: HttpOptions = {}; if (token) { options.token = token; @@ -327,9 +326,14 @@ export async function getPr( return null; } + const opts: HttpOptions = {}; + if (!refreshCache) { + opts.cacheProvider = memCacheProvider; + } + const res = await bitbucketServerHttp.getJsonUnchecked( `./rest/api/1.0/projects/${config.projectKey}/repos/${config.repositorySlug}/pull-requests/${prNo}`, - { memCache: !refreshCache }, + opts, ); const pr: BbsPr = { @@ -457,11 +461,13 @@ async function getStatus( ): Promise { const branchCommit = git.getBranchCommit(branchName); + const opts: HttpOptions = memCache ? { cacheProvider: memCacheProvider } : {}; + return ( await bitbucketServerHttp.getJsonUnchecked( // TODO: types (#22198) `./rest/build-status/1.0/commits/stats/${branchCommit!}`, - { memCache }, + opts, ) ).body; } @@ -503,11 +509,13 @@ function getStatusCheck( ): Promise { const branchCommit = git.getBranchCommit(branchName); + const opts: HttpOptions = memCache ? { cacheProvider: memCacheProvider } : {}; + return utils.accumulateValues( // TODO: types (#22198) `./rest/build-status/1.0/commits/${branchCommit!}`, 'get', - { memCache }, + opts, ); } diff --git a/lib/modules/platform/bitbucket/index.ts b/lib/modules/platform/bitbucket/index.ts index 0d562c7b5d61f7..e4db46ca5d51b9 100644 --- a/lib/modules/platform/bitbucket/index.ts +++ b/lib/modules/platform/bitbucket/index.ts @@ -6,7 +6,9 @@ import type { BranchStatus } from '../../../types'; import { parseJson } from '../../../util/common'; import * as git from '../../../util/git'; import * as hostRules from '../../../util/host-rules'; +import type { BitbucketHttpOptions } from '../../../util/http/bitbucket'; import { BitbucketHttp, setBaseUrl } from '../../../util/http/bitbucket'; +import { memCacheProvider } from '../../../util/http/cache/memory-http-cache-provider'; import { repoCacheProvider } from '../../../util/http/cache/repository-http-cache-provider'; import type { HttpOptions } from '../../../util/http/types'; import { regEx } from '../../../util/regex'; @@ -82,9 +84,7 @@ export async function initPlatform({ } setBaseUrl(defaults.endpoint); renovateUserUuid = null; - const options: HttpOptions = { - memCache: false, - }; + const options: HttpOptions = {}; if (token) { options.token = token; } else { @@ -166,7 +166,6 @@ export async function getRawFile( `/${path}`; const res = await bitbucketHttp.get(url, { cacheProvider: repoCacheProvider, - memCache: true, }); return res.body; } @@ -423,13 +422,14 @@ async function getStatus( memCache = true, ): Promise { const sha = await getBranchCommit(branchName); + const opts: BitbucketHttpOptions = { paginate: true }; + if (memCache) { + opts.cacheProvider = memCacheProvider; + } return ( await bitbucketHttp.getJsonUnchecked>( `/2.0/repositories/${config.repository}/commit/${sha!}/statuses`, - { - paginate: true, - memCache, - }, + opts, ) ).body.values; } @@ -790,7 +790,7 @@ async function sanitizeReviewers( const reviewerUser = ( await bitbucketHttp.getJsonUnchecked( `/2.0/users/${reviewer.uuid}`, - { memCache: true }, + { cacheProvider: memCacheProvider }, ) ).body; @@ -845,7 +845,7 @@ async function isAccountMemberOfWorkspace( try { await bitbucketHttp.get( `/2.0/workspaces/${workspace}/members/${reviewer.uuid}`, - { memCache: true }, + { cacheProvider: memCacheProvider }, ); return true; diff --git a/lib/modules/platform/gerrit/client.ts b/lib/modules/platform/gerrit/client.ts index e058f0b703f791..7ad9dec90c6a80 100644 --- a/lib/modules/platform/gerrit/client.ts +++ b/lib/modules/platform/gerrit/client.ts @@ -1,6 +1,8 @@ import { REPOSITORY_ARCHIVED } from '../../../constants/error-messages'; import { logger } from '../../../logger'; +import { memCacheProvider } from '../../../util/http/cache/memory-http-cache-provider'; import { GerritHttp } from '../../../util/http/gerrit'; +import type { HttpOptions } from '../../../util/http/types'; import { regEx } from '../../../util/regex'; import type { GerritAccountInfo, @@ -60,12 +62,17 @@ class GerritClient { findPRConfig: GerritFindPRConfig, refreshCache?: boolean, ): Promise { + const opts: HttpOptions = {}; + if (!refreshCache) { + opts.cacheProvider = memCacheProvider; + } + const filters = GerritClient.buildSearchFilters(repository, findPRConfig); const changes = await this.gerritHttp.getJsonUnchecked( `a/changes/?q=` + filters.join('+') + this.requestDetails.map((det) => `&o=${det}`).join(''), - { memCache: !refreshCache }, + opts, ); logger.trace( `findChanges(${filters.join(', ')}) => ${changes.body.length}`, @@ -103,7 +110,7 @@ class GerritClient { async getMessages(changeNumber: number): Promise { const messages = await this.gerritHttp.getJsonUnchecked< GerritChangeMessageInfo[] - >(`a/changes/${changeNumber}/messages`, { memCache: false }); + >(`a/changes/${changeNumber}/messages`); return messages.body; } diff --git a/lib/modules/platform/gitea/gitea-helper.ts b/lib/modules/platform/gitea/gitea-helper.ts index 53e5a54abf4e0d..2cd3b4d612bb6d 100644 --- a/lib/modules/platform/gitea/gitea-helper.ts +++ b/lib/modules/platform/gitea/gitea-helper.ts @@ -276,23 +276,15 @@ export async function getIssue( return res.body; } -export async function getRepoLabels( - repoPath: string, - options?: GiteaHttpOptions, -): Promise { +export async function getRepoLabels(repoPath: string): Promise { const url = `${API_PATH}/repos/${repoPath}/labels`; - const res = await giteaHttp.getJsonUnchecked(url, options); - + const res = await giteaHttp.getJsonUnchecked(url); return res.body; } -export async function getOrgLabels( - orgName: string, - options?: GiteaHttpOptions, -): Promise { +export async function getOrgLabels(orgName: string): Promise { const url = `${API_PATH}/orgs/${orgName}/labels`; - const res = await giteaHttp.getJsonUnchecked(url, options); - + const res = await giteaHttp.getJsonUnchecked(url); return res.body; } diff --git a/lib/modules/platform/gitea/index.ts b/lib/modules/platform/gitea/index.ts index 369edaad369b7b..dffee92ec3a3e8 100644 --- a/lib/modules/platform/gitea/index.ts +++ b/lib/modules/platform/gitea/index.ts @@ -13,6 +13,8 @@ import type { BranchStatus } from '../../../types'; import { deduplicateArray } from '../../../util/array'; import { parseJson } from '../../../util/common'; import * as git from '../../../util/git'; +import { memCacheProvider } from '../../../util/http/cache/memory-http-cache-provider'; +import type { GiteaHttpOptions } from '../../../util/http/gitea'; import { setBaseUrl } from '../../../util/http/gitea'; import { map } from '../../../util/promises'; import { sanitize } from '../../../util/sanitize'; @@ -124,18 +126,14 @@ function findCommentByContent( function getLabelList(): Promise { if (config.labelList === null) { const repoLabels = helper - .getRepoLabels(config.repository, { - memCache: false, - }) + .getRepoLabels(config.repository) .then((labels) => { logger.debug(`Retrieved ${labels.length} repo labels`); return labels; }); const orgLabels = helper - .getOrgLabels(config.repository.split('/')[0], { - memCache: false, - }) + .getOrgLabels(config.repository.split('/')[0]) .then((labels) => { logger.debug(`Retrieved ${labels.length} org labels`); return labels; @@ -398,9 +396,7 @@ const platform: Platform = { }); // Refresh caches by re-fetching commit status for branch - await helper.getCombinedCommitStatus(config.repository, branchName, { - memCache: false, - }); + await helper.getCombinedCommitStatus(config.repository, branchName); } catch (err) { logger.warn({ err }, 'Failed to set branch status'); } @@ -701,7 +697,7 @@ const platform: Platform = { } if (config.issueList === null) { config.issueList = helper - .searchIssues(config.repository, { state: 'all' }, { memCache: false }) + .searchIssues(config.repository, { state: 'all' }) .then((issues) => { const issueList = issues.map(toRenovateIssue); logger.debug(`Retrieved ${issueList.length} Issues`); @@ -717,9 +713,11 @@ const platform: Platform = { return null; } try { - const body = ( - await helper.getIssue(config.repository, number, { memCache }) - ).body; + const opts: GiteaHttpOptions = memCache + ? { cacheProvider: memCacheProvider } + : {}; + const body = (await helper.getIssue(config.repository, number, opts)) + .body; return { number, body, diff --git a/lib/modules/platform/gitea/pr-cache.ts b/lib/modules/platform/gitea/pr-cache.ts index 82d935de312186..86ec5c41c1893c 100644 --- a/lib/modules/platform/gitea/pr-cache.ts +++ b/lib/modules/platform/gitea/pr-cache.ts @@ -138,10 +138,7 @@ export class GiteaPrCache { // TODO: use zod, typescript can't infer the type of the response #22198 const res: HttpResponse<(PR | null)[]> = await http.getJsonUnchecked( url, - { - memCache: false, - paginate: false, - }, + { paginate: false }, ); const needNextPage = this.reconcile(res.body); diff --git a/lib/modules/platform/github/branch.ts b/lib/modules/platform/github/branch.ts index 43bbf4f8aab54b..4f4b4aabf38ad0 100644 --- a/lib/modules/platform/github/branch.ts +++ b/lib/modules/platform/github/branch.ts @@ -13,7 +13,6 @@ async function matchingBranches( ): Promise { const { body: branches } = await githubApi.getJson( `/repos/${repo}/git/matching-refs/heads/${branchName}`, - { memCache: false }, MatchingRef, ); return branches; diff --git a/lib/modules/platform/github/index.ts b/lib/modules/platform/github/index.ts index 2ef5eb87dac3f3..5a939639e99bc5 100644 --- a/lib/modules/platform/github/index.ts +++ b/lib/modules/platform/github/index.ts @@ -33,6 +33,7 @@ import type { LongCommitSha, } from '../../../util/git/types'; import * as hostRules from '../../../util/host-rules'; +import { memCacheProvider } from '../../../util/http/cache/memory-http-cache-provider'; import { repoCacheProvider } from '../../../util/http/cache/repository-http-cache-provider'; import * as githubHttp from '../../../util/http/github'; import type { GithubHttpOptions } from '../../../util/http/github'; @@ -906,7 +907,7 @@ async function ensureBranchSha( const repository = config.repository!; try { const commitUrl = `/repos/${repository}/git/commits/${sha}`; - await githubApi.head(commitUrl, { memCache: false }); + await githubApi.head(commitUrl); } catch (err) { logger.error({ err, sha, branchName }, 'Commit not found'); throw err; @@ -1001,7 +1002,6 @@ async function getStatus( const { body: status } = await githubApi.getJsonUnchecked(url, { - memCache: useCache, cacheProvider: repoCacheProvider, }); @@ -1119,11 +1119,11 @@ async function getStatusCheck( const url = `repos/${config.repository}/commits/${branchCommit}/statuses`; - return ( - await githubApi.getJsonUnchecked(url, { - memCache: useCache, - }) - ).body; + const opts: GithubHttpOptions = useCache + ? { cacheProvider: memCacheProvider } + : {}; + + return (await githubApi.getJsonUnchecked(url, opts)).body; } interface GithubToRenovateStatusMapping { diff --git a/lib/modules/platform/gitlab/index.ts b/lib/modules/platform/gitlab/index.ts index 64ed26afd3a3d1..0ed275e70af31c 100644 --- a/lib/modules/platform/gitlab/index.ts +++ b/lib/modules/platform/gitlab/index.ts @@ -21,6 +21,8 @@ import { coerceArray } from '../../../util/array'; import { noLeadingAtSymbol, parseJson } from '../../../util/common'; import * as git from '../../../util/git'; import * as hostRules from '../../../util/host-rules'; +import { memCacheProvider } from '../../../util/http/cache/memory-http-cache-provider'; +import type { GitlabHttpOptions } from '../../../util/http/gitlab'; import { setBaseUrl } from '../../../util/http/gitlab'; import type { HttpResponse } from '../../../util/http/types'; import { parseInteger } from '../../../util/number'; @@ -434,12 +436,13 @@ async function getStatus( config.repository }/repository/commits/${branchSha!}/statuses`; - return ( - await gitlabApi.getJsonUnchecked(url, { - paginate: true, - memCache: useCache, - }) - ).body; + const opts: GitlabHttpOptions = { paginate: true }; + if (useCache) { + opts.cacheProvider = memCacheProvider; + } + + return (await gitlabApi.getJsonUnchecked(url, opts)) + .body; } catch (err) /* istanbul ignore next */ { logger.debug({ err }, 'Error getting commit status'); if (err.response?.statusCode === 404) { @@ -666,9 +669,7 @@ async function tryPrAutomerge( pipeline: { status: string; }; - }>(`projects/${config.repository}/merge_requests/${pr}`, { - memCache: false, - }); + }>(`projects/${config.repository}/merge_requests/${pr}`); // detailed_merge_status is available with Gitlab >=15.6.0 const use_detailed_merge_status = !!body.detailed_merge_status; const detailed_merge_status_check = @@ -1061,10 +1062,7 @@ export async function getIssueList(): Promise { const query = getQueryString(searchParams); const res = await gitlabApi.getJsonUnchecked< { iid: number; title: string; labels: string[] }[] - >(`projects/${config.repository}/issues?${query}`, { - memCache: false, - paginate: true, - }); + >(`projects/${config.repository}/issues?${query}`, { paginate: true }); // istanbul ignore if if (!is.array(res.body)) { logger.warn({ responseBody: res.body }, 'Could not retrieve issue list'); @@ -1084,10 +1082,13 @@ export async function getIssue( useCache = true, ): Promise { try { + const opts: GitlabHttpOptions = useCache + ? { cacheProvider: memCacheProvider } + : {}; const issueBody = ( await gitlabApi.getJsonUnchecked<{ description: string }>( `projects/${config.repository}/issues/${number}`, - { memCache: useCache }, + opts, ) ).body.description; return { diff --git a/lib/util/http/index.ts b/lib/util/http/index.ts index e05e745cae6a4f..857ee94cada224 100644 --- a/lib/util/http/index.ts +++ b/lib/util/http/index.ts @@ -10,8 +10,6 @@ import { HOST_DISABLED } from '../../constants/error-messages'; import { pkg } from '../../expose.cjs'; import { logger } from '../../logger'; import { ExternalHostError } from '../../types/errors/external-host-error'; -import * as memCache from '../cache/memory'; -import { hash } from '../hash'; import { type AsyncResult, Result } from '../result'; import { type HttpRequestStatsDataPoint, HttpStats } from '../stats'; import { resolveBaseUrl } from '../url'; @@ -186,25 +184,8 @@ export class Http { return cachedResponse; } - const memCacheKey = - options.memCache !== false && - (options.method === 'get' || options.method === 'head') - ? hash( - `got-${JSON.stringify({ - url, - headers: options.headers, - method: options.method, - })}`, - ) - : null; - let resPromise: Promise> | null = null; - // Cache GET requests unless memCache=false - if (memCacheKey) { - resPromise = memCache.get(memCacheKey); - } - // istanbul ignore else: no cache tests if (!resPromise) { if (cacheProvider) { @@ -229,15 +210,11 @@ export class Http { const { maxRetryAfter = 60 } = hostRule; resPromise = wrapWithRetry(queuedTask, url, getRetryAfter, maxRetryAfter); - - if (memCacheKey) { - memCache.set(memCacheKey, resPromise); - } } try { const res = await resPromise; - const deepCopyNeeded = !!memCacheKey && res.statusCode !== 304; + const deepCopyNeeded = res.statusCode !== 304; const resCopy = copyResponse(res, deepCopyNeeded); resCopy.authorization = !!options?.headers?.authorization; diff --git a/lib/util/http/types.ts b/lib/util/http/types.ts index 66f260a48fb295..2b699a954d1854 100644 --- a/lib/util/http/types.ts +++ b/lib/util/http/types.ts @@ -21,7 +21,6 @@ export type GotExtraOptions = { token?: string; hostType?: string; enabled?: boolean; - memCache?: boolean; noAuth?: boolean; context?: GotContextOptions; }; @@ -66,7 +65,6 @@ export interface HttpOptions { throwHttpErrors?: boolean; token?: string; - memCache?: boolean; cacheProvider?: HttpCacheProvider; readOnly?: boolean; } From 32a83c9f3756cd8981b576b1c9258920ecbd4346 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Mon, 10 Feb 2025 12:31:05 -0300 Subject: [PATCH 3/9] feat(cache): Relax requirements for cacheable response headers --- .../http/cache/abstract-http-cache-provider.ts | 14 ++++++++------ .../cache/repository-http-cache-provider.spec.ts | 13 ------------- lib/util/http/cache/schema.ts | 4 ---- 3 files changed, 8 insertions(+), 23 deletions(-) diff --git a/lib/util/http/cache/abstract-http-cache-provider.ts b/lib/util/http/cache/abstract-http-cache-provider.ts index 46ab61e258466d..a1c3f7dcd72ed1 100644 --- a/lib/util/http/cache/abstract-http-cache-provider.ts +++ b/lib/util/http/cache/abstract-http-cache-provider.ts @@ -65,15 +65,17 @@ export abstract class AbstractHttpCacheProvider implements HttpCacheProvider { httpResponse, timestamp, }); - if (newHttpCache) { - logger.debug( - `http cache: saving ${url} (etag=${etag}, lastModified=${lastModified})`, - ); - await this.persist(url, newHttpCache as HttpCache); - } else { + + // istanbul ignore if: should never happen + if (!newHttpCache) { logger.debug(`http cache: failed to persist cache for ${url}`); + return resp; } + logger.debug( + `http cache: saving ${url} (etag=${etag}, lastModified=${lastModified})`, + ); + await this.persist(url, newHttpCache as HttpCache); return resp; } diff --git a/lib/util/http/cache/repository-http-cache-provider.spec.ts b/lib/util/http/cache/repository-http-cache-provider.spec.ts index 5b070114cee82d..a702f8171dbf5b 100644 --- a/lib/util/http/cache/repository-http-cache-provider.spec.ts +++ b/lib/util/http/cache/repository-http-cache-provider.spec.ts @@ -59,19 +59,6 @@ describe('util/http/cache/repository-http-cache-provider', () => { }); }); - it('reports if cache could not be persisted', async () => { - httpMock - .scope('https://example.com') - .get('/foo/bar') - .reply(200, { msg: 'Hello, world!' }); - - await http.getJsonUnchecked('https://example.com/foo/bar'); - - expect(logger.logger.debug).toHaveBeenCalledWith( - 'http cache: failed to persist cache for https://example.com/foo/bar', - ); - }); - it('handles abrupt cache reset', async () => { const scope = httpMock.scope('https://example.com'); diff --git a/lib/util/http/cache/schema.ts b/lib/util/http/cache/schema.ts index c9b9db1e291d92..fe69554ddc4833 100644 --- a/lib/util/http/cache/schema.ts +++ b/lib/util/http/cache/schema.ts @@ -7,10 +7,6 @@ export const HttpCacheSchema = z httpResponse: z.unknown(), timestamp: z.string(), }) - .refine( - ({ etag, lastModified }) => etag ?? lastModified, - 'Cache object should have `etag` or `lastModified` fields', - ) .nullable() .catch(null); export type HttpCache = z.infer; From 5271fbbdd3d18af63aa4b7190ccc71065433e7f4 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Mon, 10 Feb 2025 12:35:41 -0300 Subject: [PATCH 4/9] fix --- lib/util/http/cache/repository-http-cache-provider.spec.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/util/http/cache/repository-http-cache-provider.spec.ts b/lib/util/http/cache/repository-http-cache-provider.spec.ts index a702f8171dbf5b..a50121cced822a 100644 --- a/lib/util/http/cache/repository-http-cache-provider.spec.ts +++ b/lib/util/http/cache/repository-http-cache-provider.spec.ts @@ -1,6 +1,5 @@ import { Http } from '..'; import * as httpMock from '../../../../test/http-mock'; -import { logger } from '../../../../test/util'; import { resetCache } from '../../cache/repository'; import { repoCacheProvider } from './repository-http-cache-provider'; From cc4c5ddee64bfd2f7c9ba71fda1c6462102270d9 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Mon, 10 Feb 2025 12:44:22 -0300 Subject: [PATCH 5/9] Fix --- .../http/cache/memory-http-cache-provider.spec.ts | 14 +++++++------- lib/util/http/cache/memory-http-cache-provider.ts | 10 ++++++++++ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/lib/util/http/cache/memory-http-cache-provider.spec.ts b/lib/util/http/cache/memory-http-cache-provider.spec.ts index 365a7aa8a17776..d283705888cda8 100644 --- a/lib/util/http/cache/memory-http-cache-provider.spec.ts +++ b/lib/util/http/cache/memory-http-cache-provider.spec.ts @@ -1,7 +1,7 @@ import { Http } from '..'; import * as httpMock from '../../../../test/http-mock'; import * as memCache from '../../cache/memory'; -import { memCacheProvider as cacheProvider } from './memory-http-cache-provider'; +import { memCacheProvider } from './memory-http-cache-provider'; describe('util/http/cache/memory-http-cache-provider', () => { beforeEach(() => { @@ -15,25 +15,25 @@ describe('util/http/cache/memory-http-cache-provider', () => { const http = new Http('test'); it('reuses data with etag', async () => { - const scope = httpMock.scope('https://example.com'); + httpMock + .scope('https://example.com') + .get('/foo/bar') + .reply(200, { msg: 'Hello, world!' }); - scope.get('/foo/bar').reply(200, { msg: 'Hello, world!' }, { etag: '123' }); const res1 = await http.getJsonUnchecked('https://example.com/foo/bar', { - cacheProvider, + cacheProvider: memCacheProvider, }); expect(res1).toMatchObject({ statusCode: 200, body: { msg: 'Hello, world!' }, - authorization: false, }); const res2 = await http.getJsonUnchecked('https://example.com/foo/bar', { - cacheProvider, + cacheProvider: memCacheProvider, }); expect(res2).toMatchObject({ statusCode: 200, body: { msg: 'Hello, world!' }, - authorization: false, }); }); }); diff --git a/lib/util/http/cache/memory-http-cache-provider.ts b/lib/util/http/cache/memory-http-cache-provider.ts index 86ca660733d980..87c38af0cbe481 100644 --- a/lib/util/http/cache/memory-http-cache-provider.ts +++ b/lib/util/http/cache/memory-http-cache-provider.ts @@ -1,4 +1,5 @@ import * as memCache from '../../cache/memory'; +import { HttpResponse } from '../types'; import { AbstractHttpCacheProvider } from './abstract-http-cache-provider'; import type { HttpCache } from './schema'; @@ -16,6 +17,15 @@ export class MemoryHttpCacheProvider extends AbstractHttpCacheProvider { memCache.set(this.cacheKey(url), data); return Promise.resolve(); } + + override async bypassServer(url: string): Promise | null> { + const cached = await this.get(url); + if (!cached) { + return null; + } + + return cached.httpResponse as HttpResponse; + } } export const memCacheProvider = new MemoryHttpCacheProvider(); From d5cc1b879624f69bd18c84253b97366acb0aa594 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Mon, 10 Feb 2025 13:17:18 -0300 Subject: [PATCH 6/9] Fixes --- lib/modules/platform/bitbucket/index.ts | 2 ++ lib/modules/platform/bitbucket/pr-cache.spec.ts | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/modules/platform/bitbucket/index.ts b/lib/modules/platform/bitbucket/index.ts index e4db46ca5d51b9..4fd2e46883f728 100644 --- a/lib/modules/platform/bitbucket/index.ts +++ b/lib/modules/platform/bitbucket/index.ts @@ -364,6 +364,7 @@ export async function getPr(prNo: number): Promise { const pr = ( await bitbucketHttp.getJsonUnchecked( `/2.0/repositories/${config.repository}/pullrequests/${prNo}`, + { cacheProvider: memCacheProvider }, ) ).body; @@ -398,6 +399,7 @@ async function getBranchCommit( `/2.0/repositories/${config.repository}/refs/branches/${escapeHash( branchName, )}`, + { cacheProvider: memCacheProvider }, ) ).body; return branch.target.hash; diff --git a/lib/modules/platform/bitbucket/pr-cache.spec.ts b/lib/modules/platform/bitbucket/pr-cache.spec.ts index 5bf1d30da056f4..49db64d31d65a6 100644 --- a/lib/modules/platform/bitbucket/pr-cache.spec.ts +++ b/lib/modules/platform/bitbucket/pr-cache.spec.ts @@ -74,7 +74,7 @@ describe('modules/platform/bitbucket/pr-cache', () => { }, ]); expect(cache).toEqual({ - httpCache: {}, + httpCache: expect.toBeNonEmptyObject(), platform: { bitbucket: { pullRequestsCache: { @@ -123,7 +123,7 @@ describe('modules/platform/bitbucket/pr-cache', () => { }, ]); expect(cache).toEqual({ - httpCache: {}, + httpCache: expect.toBeNonEmptyObject(), platform: { bitbucket: { pullRequestsCache: { @@ -170,7 +170,7 @@ describe('modules/platform/bitbucket/pr-cache', () => { { number: 1, title: 'title' }, ]); expect(cache).toEqual({ - httpCache: {}, + httpCache: expect.toBeNonEmptyObject(), platform: { bitbucket: { pullRequestsCache: { From 371f6479d7c86f471831f239242d22d11014c9c1 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Mon, 10 Feb 2025 13:22:10 -0300 Subject: [PATCH 7/9] Fix --- lib/modules/platform/gitlab/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/modules/platform/gitlab/index.ts b/lib/modules/platform/gitlab/index.ts index 5517a3c1b12ae1..e886eb0a852ece 100644 --- a/lib/modules/platform/gitlab/index.ts +++ b/lib/modules/platform/gitlab/index.ts @@ -1026,7 +1026,7 @@ export async function setBranchStatus({ for (let attempt = 1; attempt <= retryTimes + 1; attempt += 1) { const commitUrl = `projects/${config.repository}/repository/commits/${branchSha}`; await gitlabApi - .getJsonSafe(commitUrl, { memCache: false }, LastPipelineId) + .getJsonSafe(commitUrl, LastPipelineId) .onValue((pipelineId) => { options.pipeline_id = pipelineId; }); From 847fbbbc1c1c072e1de47f058a7e903e86c3ef09 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Mon, 10 Feb 2025 13:36:34 -0300 Subject: [PATCH 8/9] Fix --- lib/util/http/cache/memory-http-cache-provider.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/util/http/cache/memory-http-cache-provider.ts b/lib/util/http/cache/memory-http-cache-provider.ts index 87c38af0cbe481..aab5095d0a6350 100644 --- a/lib/util/http/cache/memory-http-cache-provider.ts +++ b/lib/util/http/cache/memory-http-cache-provider.ts @@ -1,5 +1,5 @@ import * as memCache from '../../cache/memory'; -import { HttpResponse } from '../types'; +import type { HttpResponse } from '../types'; import { AbstractHttpCacheProvider } from './abstract-http-cache-provider'; import type { HttpCache } from './schema'; From f0364cc4144f0f315188acd283f59722d6492cfc Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Mon, 10 Feb 2025 13:58:31 -0300 Subject: [PATCH 9/9] Fix coverage --- lib/modules/platform/gitea/index.ts | 7 ++++--- lib/modules/platform/gitlab/index.ts | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/modules/platform/gitea/index.ts b/lib/modules/platform/gitea/index.ts index 3ed5a5a61f320c..1475b235a7dda2 100644 --- a/lib/modules/platform/gitea/index.ts +++ b/lib/modules/platform/gitea/index.ts @@ -721,9 +721,10 @@ const platform: Platform = { return null; } try { - const opts: GiteaHttpOptions = memCache - ? { cacheProvider: memCacheProvider } - : {}; + const opts: GiteaHttpOptions = {}; + if (memCache) { + opts.cacheProvider = memCacheProvider; + } const body = (await helper.getIssue(config.repository, number, opts)) .body; return { diff --git a/lib/modules/platform/gitlab/index.ts b/lib/modules/platform/gitlab/index.ts index e886eb0a852ece..ab5500a041abd2 100644 --- a/lib/modules/platform/gitlab/index.ts +++ b/lib/modules/platform/gitlab/index.ts @@ -1102,9 +1102,10 @@ export async function getIssue( useCache = true, ): Promise { try { - const opts: GitlabHttpOptions = useCache - ? { cacheProvider: memCacheProvider } - : {}; + const opts: GitlabHttpOptions = {}; + if (useCache) { + opts.cacheProvider = memCacheProvider; + } const issueBody = ( await gitlabApi.getJsonUnchecked<{ description: string }>( `projects/${config.repository}/issues/${number}`,