Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(http): Switch to memory cache provider #33901

Open
wants to merge 14 commits into
base: main
Choose a base branch
from
20 changes: 14 additions & 6 deletions lib/modules/platform/bitbucket-server/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import {
BitbucketServerHttp,
setBaseUrl,
} from '../../../util/http/bitbucket-server';
import { memCacheProvider } from '../../../util/http/cache/memory-http-cache-provider';
import type { HttpOptions, HttpResponse } from '../../../util/http/types';
import { newlineRegex, regEx } from '../../../util/regex';
import { sanitize } from '../../../util/sanitize';
Expand Down Expand Up @@ -136,9 +137,7 @@ export async function initPlatform({

if (!gitAuthor && username) {
logger.debug(`Attempting to confirm gitAuthor from username`);
const options: HttpOptions = {
memCache: false,
};
const options: HttpOptions = {};

if (token) {
options.token = token;
Expand Down Expand Up @@ -327,9 +326,14 @@ export async function getPr(
return null;
}

const opts: HttpOptions = {};
if (!refreshCache) {
opts.cacheProvider = memCacheProvider;
}

const res = await bitbucketServerHttp.getJsonUnchecked<BbsRestPr>(
`./rest/api/1.0/projects/${config.projectKey}/repos/${config.repositorySlug}/pull-requests/${prNo}`,
{ memCache: !refreshCache },
opts,
);

const pr: BbsPr = {
Expand Down Expand Up @@ -457,11 +461,13 @@ async function getStatus(
): Promise<utils.BitbucketCommitStatus> {
const branchCommit = git.getBranchCommit(branchName);

const opts: HttpOptions = memCache ? { cacheProvider: memCacheProvider } : {};

return (
await bitbucketServerHttp.getJsonUnchecked<utils.BitbucketCommitStatus>(
// TODO: types (#22198)
`./rest/build-status/1.0/commits/stats/${branchCommit!}`,
{ memCache },
opts,
)
).body;
}
Expand Down Expand Up @@ -503,11 +509,13 @@ function getStatusCheck(
): Promise<utils.BitbucketStatus[]> {
const branchCommit = git.getBranchCommit(branchName);

const opts: HttpOptions = memCache ? { cacheProvider: memCacheProvider } : {};

return utils.accumulateValues(
// TODO: types (#22198)
`./rest/build-status/1.0/commits/${branchCommit!}`,
'get',
{ memCache },
opts,
);
}

Expand Down
22 changes: 12 additions & 10 deletions lib/modules/platform/bitbucket/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,9 @@ import type { BranchStatus } from '../../../types';
import { parseJson } from '../../../util/common';
import * as git from '../../../util/git';
import * as hostRules from '../../../util/host-rules';
import type { BitbucketHttpOptions } from '../../../util/http/bitbucket';
import { BitbucketHttp, setBaseUrl } from '../../../util/http/bitbucket';
import { memCacheProvider } from '../../../util/http/cache/memory-http-cache-provider';
import { repoCacheProvider } from '../../../util/http/cache/repository-http-cache-provider';
import type { HttpOptions } from '../../../util/http/types';
import { regEx } from '../../../util/regex';
Expand Down Expand Up @@ -82,9 +84,7 @@ export async function initPlatform({
}
setBaseUrl(defaults.endpoint);
renovateUserUuid = null;
const options: HttpOptions = {
memCache: false,
};
const options: HttpOptions = {};
if (token) {
options.token = token;
} else {
Expand Down Expand Up @@ -166,7 +166,6 @@ export async function getRawFile(
`/${path}`;
const res = await bitbucketHttp.get(url, {
cacheProvider: repoCacheProvider,
memCache: true,
});
return res.body;
}
Expand Down Expand Up @@ -365,6 +364,7 @@ export async function getPr(prNo: number): Promise<Pr | null> {
const pr = (
await bitbucketHttp.getJsonUnchecked<PrResponse>(
`/2.0/repositories/${config.repository}/pullrequests/${prNo}`,
{ cacheProvider: memCacheProvider },
)
).body;

Expand Down Expand Up @@ -399,6 +399,7 @@ async function getBranchCommit(
`/2.0/repositories/${config.repository}/refs/branches/${escapeHash(
branchName,
)}`,
{ cacheProvider: memCacheProvider },
)
).body;
return branch.target.hash;
Expand All @@ -423,13 +424,14 @@ async function getStatus(
memCache = true,
): Promise<BitbucketStatus[]> {
const sha = await getBranchCommit(branchName);
const opts: BitbucketHttpOptions = { paginate: true };
if (memCache) {
opts.cacheProvider = memCacheProvider;
}
return (
await bitbucketHttp.getJsonUnchecked<PagedResult<BitbucketStatus>>(
`/2.0/repositories/${config.repository}/commit/${sha!}/statuses`,
{
paginate: true,
memCache,
},
opts,
)
).body.values;
}
Expand Down Expand Up @@ -790,7 +792,7 @@ async function sanitizeReviewers(
const reviewerUser = (
await bitbucketHttp.getJsonUnchecked<Account>(
`/2.0/users/${reviewer.uuid}`,
{ memCache: true },
{ cacheProvider: memCacheProvider },
)
).body;

Expand Down Expand Up @@ -845,7 +847,7 @@ async function isAccountMemberOfWorkspace(
try {
await bitbucketHttp.get(
`/2.0/workspaces/${workspace}/members/${reviewer.uuid}`,
{ memCache: true },
{ cacheProvider: memCacheProvider },
);

return true;
Expand Down
11 changes: 9 additions & 2 deletions lib/modules/platform/gerrit/client.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
import { REPOSITORY_ARCHIVED } from '../../../constants/error-messages';
import { logger } from '../../../logger';
import { memCacheProvider } from '../../../util/http/cache/memory-http-cache-provider';
import { GerritHttp } from '../../../util/http/gerrit';
import type { HttpOptions } from '../../../util/http/types';
import { regEx } from '../../../util/regex';
import type {
GerritAccountInfo,
Expand Down Expand Up @@ -60,12 +62,17 @@ class GerritClient {
findPRConfig: GerritFindPRConfig,
refreshCache?: boolean,
): Promise<GerritChange[]> {
const opts: HttpOptions = {};
if (!refreshCache) {
opts.cacheProvider = memCacheProvider;
}

const filters = GerritClient.buildSearchFilters(repository, findPRConfig);
const changes = await this.gerritHttp.getJsonUnchecked<GerritChange[]>(
`a/changes/?q=` +
filters.join('+') +
this.requestDetails.map((det) => `&o=${det}`).join(''),
{ memCache: !refreshCache },
opts,
);
logger.trace(
`findChanges(${filters.join(', ')}) => ${changes.body.length}`,
Expand Down Expand Up @@ -103,7 +110,7 @@ class GerritClient {
async getMessages(changeNumber: number): Promise<GerritChangeMessageInfo[]> {
const messages = await this.gerritHttp.getJsonUnchecked<
GerritChangeMessageInfo[]
>(`a/changes/${changeNumber}/messages`, { memCache: false });
>(`a/changes/${changeNumber}/messages`);
return messages.body;
}

Expand Down
16 changes: 4 additions & 12 deletions lib/modules/platform/gitea/gitea-helper.ts
Original file line number Diff line number Diff line change
Expand Up @@ -276,23 +276,15 @@ export async function getIssue(
return res.body;
}

export async function getRepoLabels(
repoPath: string,
options?: GiteaHttpOptions,
): Promise<Label[]> {
export async function getRepoLabels(repoPath: string): Promise<Label[]> {
const url = `${API_PATH}/repos/${repoPath}/labels`;
const res = await giteaHttp.getJsonUnchecked<Label[]>(url, options);

const res = await giteaHttp.getJsonUnchecked<Label[]>(url);
return res.body;
}

export async function getOrgLabels(
orgName: string,
options?: GiteaHttpOptions,
): Promise<Label[]> {
export async function getOrgLabels(orgName: string): Promise<Label[]> {
const url = `${API_PATH}/orgs/${orgName}/labels`;
const res = await giteaHttp.getJsonUnchecked<Label[]>(url, options);

const res = await giteaHttp.getJsonUnchecked<Label[]>(url);
return res.body;
}

Expand Down
25 changes: 12 additions & 13 deletions lib/modules/platform/gitea/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ import type { BranchStatus } from '../../../types';
import { deduplicateArray } from '../../../util/array';
import { parseJson } from '../../../util/common';
import * as git from '../../../util/git';
import { memCacheProvider } from '../../../util/http/cache/memory-http-cache-provider';
import type { GiteaHttpOptions } from '../../../util/http/gitea';
import { setBaseUrl } from '../../../util/http/gitea';
import { map } from '../../../util/promises';
import { sanitize } from '../../../util/sanitize';
Expand Down Expand Up @@ -124,18 +126,14 @@ function findCommentByContent(
function getLabelList(): Promise<Label[]> {
if (config.labelList === null) {
const repoLabels = helper
.getRepoLabels(config.repository, {
memCache: false,
})
.getRepoLabels(config.repository)
.then((labels) => {
logger.debug(`Retrieved ${labels.length} repo labels`);
return labels;
});

const orgLabels = helper
.getOrgLabels(config.repository.split('/')[0], {
memCache: false,
})
.getOrgLabels(config.repository.split('/')[0])
.then((labels) => {
logger.debug(`Retrieved ${labels.length} org labels`);
return labels;
Expand Down Expand Up @@ -406,9 +404,7 @@ const platform: Platform = {
});

// Refresh caches by re-fetching commit status for branch
await helper.getCombinedCommitStatus(config.repository, branchName, {
memCache: false,
});
await helper.getCombinedCommitStatus(config.repository, branchName);
} catch (err) {
logger.warn({ err }, 'Failed to set branch status');
}
Expand Down Expand Up @@ -709,7 +705,7 @@ const platform: Platform = {
}
if (config.issueList === null) {
config.issueList = helper
.searchIssues(config.repository, { state: 'all' }, { memCache: false })
.searchIssues(config.repository, { state: 'all' })
.then((issues) => {
const issueList = issues.map(toRenovateIssue);
logger.debug(`Retrieved ${issueList.length} Issues`);
Expand All @@ -725,9 +721,12 @@ const platform: Platform = {
return null;
}
try {
const body = (
await helper.getIssue(config.repository, number, { memCache })
).body;
const opts: GiteaHttpOptions = {};
if (memCache) {
opts.cacheProvider = memCacheProvider;
}
const body = (await helper.getIssue(config.repository, number, opts))
.body;
return {
number,
body,
Expand Down
5 changes: 1 addition & 4 deletions lib/modules/platform/gitea/pr-cache.ts
Original file line number Diff line number Diff line change
Expand Up @@ -138,10 +138,7 @@ export class GiteaPrCache {
// TODO: use zod, typescript can't infer the type of the response #22198
const res: HttpResponse<(PR | null)[]> = await http.getJsonUnchecked(
url,
{
memCache: false,
paginate: false,
},
{ paginate: false },
);

const needNextPage = this.reconcile(res.body);
Expand Down
1 change: 0 additions & 1 deletion lib/modules/platform/github/branch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ async function matchingBranches(
): Promise<string[]> {
const { body: branches } = await githubApi.getJson(
`/repos/${repo}/git/matching-refs/heads/${branchName}`,
{ memCache: false },
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you explain why the memCache=false parts are no longer needed?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

memCache=false is the equivalent of the absent cacheProvider option

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

And yes, no memory cache is the default now

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How can we be confident that we haven't missed anything?

MatchingRef,
);
return branches;
Expand Down
14 changes: 7 additions & 7 deletions lib/modules/platform/github/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import type {
LongCommitSha,
} from '../../../util/git/types';
import * as hostRules from '../../../util/host-rules';
import { memCacheProvider } from '../../../util/http/cache/memory-http-cache-provider';
import { repoCacheProvider } from '../../../util/http/cache/repository-http-cache-provider';
import * as githubHttp from '../../../util/http/github';
import type { GithubHttpOptions } from '../../../util/http/github';
Expand Down Expand Up @@ -906,7 +907,7 @@ async function ensureBranchSha(
const repository = config.repository!;
try {
const commitUrl = `/repos/${repository}/git/commits/${sha}`;
await githubApi.head(commitUrl, { memCache: false });
await githubApi.head(commitUrl);
} catch (err) {
logger.error({ err, sha, branchName }, 'Commit not found');
throw err;
Expand Down Expand Up @@ -1001,7 +1002,6 @@ async function getStatus(

const { body: status } =
await githubApi.getJsonUnchecked<CombinedBranchStatus>(url, {
memCache: useCache,
cacheProvider: repoCacheProvider,
});

Expand Down Expand Up @@ -1119,11 +1119,11 @@ async function getStatusCheck(

const url = `repos/${config.repository}/commits/${branchCommit}/statuses`;

return (
await githubApi.getJsonUnchecked<GhBranchStatus[]>(url, {
memCache: useCache,
})
).body;
const opts: GithubHttpOptions = useCache
? { cacheProvider: memCacheProvider }
: {};

return (await githubApi.getJsonUnchecked<GhBranchStatus[]>(url, opts)).body;
}

interface GithubToRenovateStatusMapping {
Expand Down
Loading
Loading