Skip to content

Commit

Permalink
feat(pypi): support GCloud credentials for Google Artifact Registry (#…
Browse files Browse the repository at this point in the history 
…31262)
  • Loading branch information
@maxbrunet
maxbrunet authored Sep 14, 2024
1 parent 7399b6d commit 0049a94
Show file tree
Hide file tree
Showing 2 changed files with 182 additions and 3 deletions.
156 changes: 156 additions & 0 deletions lib/modules/datasource/pypi/index.spec.ts
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,75 @@
import { GoogleAuth as _googleAuth } from 'google-auth-library';
import { getPkgReleases } from '..';
import { Fixtures } from '../../../../test/fixtures';
import * as httpMock from '../../../../test/http-mock';
import { mocked } from '../../../../test/util';
import * as hostRules from '../../../util/host-rules';
import { PypiDatasource } from '.';

const googleAuth = mocked(_googleAuth);
jest.mock('google-auth-library');

const res1 = Fixtures.get('azure-cli-monitor.json');
const htmlResponse = Fixtures.get('versions-html.html');
const mixedCaseResponse = Fixtures.get('versions-html-mixed-case.html');
const withPeriodsResponse = Fixtures.get('versions-html-with-periods.html');

const azureCliMonitorReleases = [
{ releaseTimestamp: '2017-04-03T16:55:14.000Z', version: '0.0.1' },
{ releaseTimestamp: '2017-04-17T20:32:30.000Z', version: '0.0.2' },
{ releaseTimestamp: '2017-04-28T21:18:54.000Z', version: '0.0.3' },
{ releaseTimestamp: '2017-05-09T21:36:51.000Z', version: '0.0.4' },
{ releaseTimestamp: '2017-05-30T23:13:49.000Z', version: '0.0.5' },
{ releaseTimestamp: '2017-06-13T22:21:05.000Z', version: '0.0.6' },
{ releaseTimestamp: '2017-06-21T22:12:36.000Z', version: '0.0.7' },
{ releaseTimestamp: '2017-07-07T16:22:26.000Z', version: '0.0.8' },
{ releaseTimestamp: '2017-08-28T20:14:33.000Z', version: '0.0.9' },
{ releaseTimestamp: '2017-09-22T23:47:59.000Z', version: '0.0.10' },
{ releaseTimestamp: '2017-10-24T02:14:07.000Z', version: '0.0.11' },
{ releaseTimestamp: '2017-11-14T18:31:57.000Z', version: '0.0.12' },
{ releaseTimestamp: '2017-12-05T18:57:54.000Z', version: '0.0.13' },
{ releaseTimestamp: '2018-01-05T21:26:03.000Z', version: '0.0.14' },
{ releaseTimestamp: '2018-01-17T18:36:39.000Z', version: '0.1.0' },
{ releaseTimestamp: '2018-01-31T18:05:22.000Z', version: '0.1.1' },
{ releaseTimestamp: '2018-02-13T18:17:52.000Z', version: '0.1.2' },
{ releaseTimestamp: '2018-03-13T17:08:20.000Z', version: '0.1.3' },
{ releaseTimestamp: '2018-03-27T17:55:25.000Z', version: '0.1.4' },
{ releaseTimestamp: '2018-04-10T17:25:47.000Z', version: '0.1.5' },
{ releaseTimestamp: '2018-05-07T17:59:09.000Z', version: '0.1.6' },
{ releaseTimestamp: '2018-05-22T17:25:23.000Z', version: '0.1.7' },
{ releaseTimestamp: '2018-07-03T16:18:06.000Z', version: '0.1.8' },
{ releaseTimestamp: '2018-07-18T16:20:01.000Z', version: '0.2.0' },
{ releaseTimestamp: '2018-07-31T15:32:28.000Z', version: '0.2.1' },
{ releaseTimestamp: '2018-08-14T14:55:32.000Z', version: '0.2.2' },
{ releaseTimestamp: '2018-08-28T15:35:01.000Z', version: '0.2.3' },
{ releaseTimestamp: '2018-10-09T18:09:08.000Z', version: '0.2.4' },
{ releaseTimestamp: '2018-10-23T16:54:38.000Z', version: '0.2.5' },
{ releaseTimestamp: '2018-11-06T16:34:51.000Z', version: '0.2.6' },
{ releaseTimestamp: '2018-11-20T20:16:03.000Z', version: '0.2.7' },
{ releaseTimestamp: '2019-01-15T21:08:09.000Z', version: '0.2.8' },
{ releaseTimestamp: '2019-01-30T01:51:15.000Z', version: '0.2.9' },
{ releaseTimestamp: '2019-02-12T18:09:43.000Z', version: '0.2.10' },
{ releaseTimestamp: '2019-03-26T17:57:43.000Z', version: '0.2.11' },
{ releaseTimestamp: '2019-04-09T17:01:09.000Z', version: '0.2.12' },
{ releaseTimestamp: '2019-04-23T17:00:58.000Z', version: '0.2.13' },
{ releaseTimestamp: '2019-05-21T18:43:17.000Z', version: '0.2.14' },
{ releaseTimestamp: '2019-06-18T13:58:55.000Z', version: '0.2.15' },
];

const djDatabaseUrlSimpleReleases = [
{ version: '0.1.2' },
{ version: '0.1.3' },
{ version: '0.1.4' },
{ version: '0.2.0' },
{ version: '0.2.1' },
{ version: '0.2.2' },
{ version: '0.3.0' },
{ version: '0.4.0' },
{ version: '0.4.1' },
{ version: '0.4.2' },
{ isDeprecated: true, version: '0.5.0' },
];

const baseUrl = 'https://pypi.org/pypi';
const datasource = PypiDatasource.id;

Expand Down Expand Up @@ -125,6 +186,58 @@ describe('modules/datasource/pypi/index', () => {
});
});

it('supports Google Auth', async () => {
httpMock
.scope('https://someregion-python.pkg.dev/some-project/some-repo/')
.get('/azure-cli-monitor/json')
.matchHeader(
'authorization',
'Basic b2F1dGgyYWNjZXNzdG9rZW46c29tZS10b2tlbg==',
)
.reply(200, Fixtures.get('azure-cli-monitor-updated.json'));
const config = {
registryUrls: [
'https://someregion-python.pkg.dev/some-project/some-repo',
],
};
googleAuth.mockImplementationOnce(
jest.fn().mockImplementationOnce(() => ({
getAccessToken: jest.fn().mockResolvedValue('some-token'),
})),
);
const res = await getPkgReleases({
...config,
datasource,
packageName: 'azure-cli-monitor',
});
expect(res).toMatchObject({ releases: azureCliMonitorReleases });
expect(googleAuth).toHaveBeenCalledTimes(1);
});

it('supports Google Auth not being configured', async () => {
httpMock
.scope('https://someregion-python.pkg.dev/some-project/some-repo/')
.get('/azure-cli-monitor/json')
.reply(200, Fixtures.get('azure-cli-monitor-updated.json'));
const config = {
registryUrls: [
'https://someregion-python.pkg.dev/some-project/some-repo',
],
};
googleAuth.mockImplementation(
jest.fn().mockImplementation(() => ({
getAccessToken: jest.fn().mockResolvedValue(undefined),
})),
);
const res = await getPkgReleases({
...config,
datasource,
packageName: 'azure-cli-monitor',
});
expect(res).toMatchObject({ releases: azureCliMonitorReleases });
expect(googleAuth).toHaveBeenCalledTimes(1);
});

it('returns non-github home_page', async () => {
httpMock
.scope(baseUrl)
Expand Down Expand Up @@ -643,6 +756,49 @@ describe('modules/datasource/pypi/index', () => {
});
});

it('supports Google Auth with simple endpoint', async () => {
httpMock
.scope('https://someregion-python.pkg.dev/some-project/some-repo/simple/')
.get('/dj-database-url/')
.reply(200, htmlResponse);
const config = {
registryUrls: [
'https://someregion-python.pkg.dev/some-project/some-repo/simple/',
],
};
googleAuth.mockImplementationOnce(
jest.fn().mockImplementationOnce(() => ({
getAccessToken: jest.fn().mockResolvedValue('some-token'),
})),
);
expect(
await getPkgReleases({
datasource,
...config,
constraints: { python: '2.7' },
packageName: 'dj-database-url',
}),
).toMatchObject({
isPrivate: true,
registryUrl:
'https://someregion-python.pkg.dev/some-project/some-repo/simple',
releases: djDatabaseUrlSimpleReleases,
});
expect(googleAuth).toHaveBeenCalledTimes(1);
});

it('ignores an invalid URL when checking for auth headers', async () => {
const config = {
registryUrls: ['not-a-url/simple/'],
};
const res = await getPkgReleases({
...config,
datasource,
packageName: 'azure-cli-monitor',
});
expect(res).toBeNil();
});

it('uses https://pypi.org/pypi/ instead of https://pypi.org/simple/', async () => {
httpMock.scope(baseUrl).get('/azure-cli-monitor/json').reply(200, res1);
const config = {
Expand Down
29 changes: 26 additions & 3 deletions lib/modules/datasource/pypi/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,13 @@ import changelogFilenameRegex from 'changelog-filename-regex';
import { logger } from '../../../logger';
import { coerceArray } from '../../../util/array';
import { parse } from '../../../util/html';
import type { OutgoingHttpHeaders } from '../../../util/http/types';
import { regEx } from '../../../util/regex';
import { ensureTrailingSlash } from '../../../util/url';
import { ensureTrailingSlash, parseUrl } from '../../../util/url';
import * as pep440 from '../../versioning/pep440';
import { Datasource } from '../datasource';
import type { GetReleasesConfig, Release, ReleaseResult } from '../types';
import { getGoogleAuthToken } from '../util';
import { isGitHubRepo, normalizePythonDepName } from './common';
import type { PypiJSON, PypiJSONRelease, Releases } from './types';

Expand Down Expand Up @@ -82,6 +84,25 @@ export class PypiDatasource extends Datasource {
return dependency;
}

private async getAuthHeaders(
lookupUrl: string,
): Promise<OutgoingHttpHeaders> {
const parsedUrl = parseUrl(lookupUrl);
if (!parsedUrl) {
logger.once.debug({ lookupUrl }, 'Failed to parse URL');
return {};
}
if (parsedUrl.hostname.endsWith('.pkg.dev')) {
const auth = await getGoogleAuthToken();
if (auth) {
return { authorization: `Basic ${auth}` };
}
logger.once.debug({ lookupUrl }, 'Could not get Google access token');
return {};
}
return {};
}

private async getDependency(
packageName: string,
hostUrl: string,
Expand All @@ -92,7 +113,8 @@ export class PypiDatasource extends Datasource {
);
const dependency: ReleaseResult = { releases: [] };
logger.trace({ lookupUrl }, 'Pypi api got lookup');
const rep = await this.http.getJson<PypiJSON>(lookupUrl);
const headers = await this.getAuthHeaders(lookupUrl);
const rep = await this.http.getJson<PypiJSON>(lookupUrl, { headers });
const dep = rep?.body;
if (!dep) {
logger.trace({ dependency: packageName }, 'pip package not found');
Expand Down Expand Up @@ -237,7 +259,8 @@ export class PypiDatasource extends Datasource {
ensureTrailingSlash(normalizePythonDepName(packageName)),
);
const dependency: ReleaseResult = { releases: [] };
const response = await this.http.get(lookupUrl);
const headers = await this.getAuthHeaders(lookupUrl);
const response = await this.http.get(lookupUrl, { headers });
const dep = response?.body;
if (!dep) {
logger.trace({ dependency: packageName }, 'pip package not found');
Expand Down

0 comments on commit 0049a94

Please sign in to comment.