-
Notifications
You must be signed in to change notification settings - Fork 125
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add AwsSigV4 signing functionality #279
Changes from 6 commits
d78ae1c
59bbbf4
caf4d74
7faaae1
6a67179
90a443d
e50e8fc
f58e21e
f1e5a16
93f347b
6d7b79c
8505f85
562d683
341df4f
f629fa1
c952ba0
4abf7ea
48e5b3a
e376ef4
1475bf0
52ab9ad
d31eb6f
f036b30
ef2b3c7
13d30de
b1308b9
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -13,6 +13,7 @@ OpenSearch Node.js client | |
- [Example use](#example-use) | ||
- [Setup](#setup) | ||
- [Sample code](#sample-code) | ||
- [With AWS SigV4 signing](#with-aws-sigv4-signing) | ||
- [Project Resources](#project-resources) | ||
- [Code of Conduct](#code-of-conduct) | ||
- [License](#license) | ||
|
@@ -152,6 +153,94 @@ async function search() { | |
search().catch(console.log); | ||
``` | ||
|
||
### With AWS SigV4 signing | ||
```javascript | ||
const endpoint = ""; // OpenSearch domain URL e.g. https://search-xxx.region.es.amazonaws.com | ||
const { Client } = require('@opensearch-project/opensearch'); | ||
const { AwsSigv4Signer } = require('@opensearch-project/opensearch/aws'); | ||
const { defaultProvider } = require("@aws-sdk/credential-provider-node"); | ||
|
||
async function getClient() { | ||
const credentials = await defaultProvider()(); | ||
var client = new Client({ | ||
...AwsV4Signer({ | ||
credentials: credentials, | ||
region: "us-west-2", | ||
}), | ||
node: endpoint, | ||
}); | ||
return client; | ||
} | ||
|
||
async function search() { | ||
|
||
var client = await getClient(); | ||
|
||
var index_name = "books-test-1"; | ||
var settings = { | ||
settings: { | ||
index: { | ||
number_of_shards: 4, | ||
number_of_replicas: 3, | ||
}, | ||
}, | ||
}; | ||
|
||
var response = await client.indices.create({ | ||
index: index_name, | ||
body: settings, | ||
}); | ||
|
||
console.log("Creating index:"); | ||
console.log(response.body); | ||
|
||
// Add a document to the index. | ||
var document = { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. All of the examples here look very similar to the already existing ones. I see that initializing the client is different, can we provide may be just one example here so we don't duplicate all the test data? Also, consider moving this to a USER_GUIDE since the number of examples is increasing, might make the README very long. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. +1, but I also think we can break this up in this PR or do it in another PR, so no hard ask from me. FYI I do like what they have done in the .NET client: https://github.com/opensearch-project/opensearch-net/blob/main/USER_GUIDE.md There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @VachaShah @dblock I've split the code snippets into a new USER_GUIDE. wdyt? |
||
title: "The Outsider", | ||
author: "Stephen King", | ||
year: "2018", | ||
genre: "Crime fiction", | ||
}; | ||
|
||
var id = "1"; | ||
|
||
var response = await client.index({ | ||
id: id, | ||
index: index_name, | ||
body: document, | ||
refresh: true, | ||
}); | ||
|
||
console.log("Adding document:"); | ||
console.log(response.body); | ||
|
||
var response = await client.bulk({ body: bulk_documents }); | ||
console.log(response.body); | ||
|
||
// Search for the document. | ||
var query = { | ||
query: { | ||
match: { | ||
title: { | ||
query: "The Outsider", | ||
}, | ||
}, | ||
}, | ||
}; | ||
|
||
var response = await client.search({ | ||
index: index_name, | ||
body: query, | ||
}); | ||
|
||
console.log("Search results:"); | ||
console.log(response.body.hits); | ||
} | ||
|
||
search().catch(console.log); | ||
``` | ||
|
||
|
||
## Project Resources | ||
|
||
- [Project Website](https://opensearch.org/) | ||
|
Large diffs are not rendered by default.
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,43 @@ | ||
/* | ||
* SPDX-License-Identifier: Apache-2.0 | ||
* | ||
* The OpenSearch Contributors require contributions made to | ||
* this file be licensed under the Apache-2.0 license or a | ||
* compatible open source license. | ||
* | ||
* Modifications Copyright OpenSearch Contributors. See | ||
* GitHub history for details. | ||
*/ | ||
|
||
'use strict'; | ||
const Connection = require('../Connection'); | ||
const aws4 = require('aws4'); | ||
const AwsSigv4SignerError = require('./errors'); | ||
|
||
function AwsSigv4Signer(opts) { | ||
if (opts && (!opts.region || opts.region === null || opts.region === '')) { | ||
throw new AwsSigv4SignerError('Region cannot be empty'); | ||
} | ||
if (opts && (!opts.credentials || opts.credentials === null || opts.credentials === '')) { | ||
throw new AwsSigv4SignerError('Credentials cannot be empty'); | ||
} | ||
|
||
function buildSignedRequestObject(request = {}) { | ||
request.service = 'es'; | ||
request.region = opts.region; | ||
request.headers = request.headers || {}; | ||
request.headers['host'] = request.hostname; | ||
return aws4.sign(request, opts.credentials); | ||
} | ||
class AwsSigv4SignerConnection extends Connection { | ||
buildRequestObject(params) { | ||
const request = super.buildRequestObject(params); | ||
return buildSignedRequestObject(request); | ||
} | ||
} | ||
return { | ||
Connection: AwsSigv4SignerConnection, | ||
buildSignedRequestObject, | ||
}; | ||
} | ||
module.exports = AwsSigv4Signer; |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,25 @@ | ||
/* | ||
* SPDX-License-Identifier: Apache-2.0 | ||
* | ||
* The OpenSearch Contributors require contributions made to | ||
* this file be licensed under the Apache-2.0 license or a | ||
* compatible open source license. | ||
* | ||
* Modifications Copyright OpenSearch Contributors. See | ||
* GitHub history for details. | ||
*/ | ||
|
||
'use strict'; | ||
const { OpenSearchClientError } = require('../errors'); | ||
|
||
class AwsSigv4SignerError extends OpenSearchClientError { | ||
constructor(message, data) { | ||
super(message, data); | ||
Error.captureStackTrace(this, AwsSigv4SignerError); | ||
this.name = 'AwsSigv4SignerError'; | ||
this.message = message || 'AwsSigv4Signer Error'; | ||
this.data = data; | ||
} | ||
} | ||
|
||
module.exports = AwsSigv4SignerError; |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,38 @@ | ||
/* | ||
* SPDX-License-Identifier: Apache-2.0 | ||
* | ||
* The OpenSearch Contributors require contributions made to | ||
* this file be licensed under the Apache-2.0 license or a | ||
* compatible open source license. | ||
* | ||
* Modifications Copyright OpenSearch Contributors. See | ||
* GitHub history for details. | ||
*/ | ||
|
||
/// <reference types="node" /> | ||
|
||
import { Credentials } from '@aws-sdk/types'; | ||
import Connection from '../Connection'; | ||
import * as http from 'http'; | ||
import { OpenSearchClientError } from '../errors'; | ||
|
||
interface AwsSigv4SignerOptions { | ||
credentials: Credentials; | ||
region: string; | ||
} | ||
|
||
interface AwsSigv4SignerResponse { | ||
Connection: Connection; | ||
buildSignedRequestObject(request: any): http.ClientRequestArgs; | ||
} | ||
|
||
declare function AwsSigv4Signer(opts: AwsSigv4SignerOptions): AwsSigv4SignerResponse; | ||
|
||
declare class AwsSigv4SignerError extends OpenSearchClientError { | ||
name: string; | ||
message: string; | ||
data: any; | ||
constructor(message: string, data: any); | ||
} | ||
|
||
export { AwsSigv4Signer, AwsSigv4SignerOptions, AwsSigv4SignerResponse, AwsSigv4SignerError }; |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,20 @@ | ||
/* | ||
* SPDX-License-Identifier: Apache-2.0 | ||
* | ||
* The OpenSearch Contributors require contributions made to | ||
* this file be licensed under the Apache-2.0 license or a | ||
* compatible open source license. | ||
* | ||
* Modifications Copyright OpenSearch Contributors. See | ||
* GitHub history for details. | ||
*/ | ||
|
||
'use strict'; | ||
|
||
const AwsSigv4Signer = require('./AwsSigv4Signer'); | ||
const AwsSigv4SignerError = require('./errors'); | ||
|
||
module.exports = { | ||
AwsSigv4Signer, | ||
AwsSigv4SignerError, | ||
}; |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -8,10 +8,21 @@ | |
"require": "./index.js", | ||
"import": "./index.mjs" | ||
}, | ||
"./aws": "./lib/aws/index.js", | ||
"./": "./" | ||
}, | ||
"typesVersions": { | ||
"*": { | ||
".": [ | ||
"index.d.ts" | ||
], | ||
"aws": [ | ||
"./lib/aws/index.d.ts" | ||
] | ||
} | ||
}, | ||
"homepage": "https://www.opensearch.org/", | ||
"version": "2.0.0", | ||
"version": "2.1.0", | ||
"versionCanary": "7.10.0-canary.6", | ||
"keywords": [ | ||
"opensearch", | ||
|
@@ -77,6 +88,8 @@ | |
"xmlbuilder2": "^2.4.1" | ||
}, | ||
"dependencies": { | ||
"@aws-sdk/credential-provider-node": "^3.154.0", | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. So does this become a hard dependency for this package? Meaning if I am not doing AWS sigv4, am I still dragging this module in? Can it be avoided? Asking again because I'd like not to drag any vendor specific components in by default. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Removed this dependency like I mentioned below. We are still depending on the aws4 library which we cannot do away(unless we want to re-write all of that implementation) if we want to support signing natively in the client. |
||
"aws4": "^1.11.0", | ||
"debug": "^4.3.1", | ||
"hpagent": "^0.1.1", | ||
"ms": "^2.1.3", | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,29 @@ | ||
/* | ||
* SPDX-License-Identifier: Apache-2.0 | ||
* | ||
* The OpenSearch Contributors require contributions made to | ||
* this file be licensed under the Apache-2.0 license or a | ||
* compatible open source license. | ||
* | ||
* Modifications Copyright OpenSearch Contributors. See | ||
* GitHub history for details. | ||
*/ | ||
|
||
import { expectType } from 'tsd'; | ||
const { v4: uuidv4 } = require('uuid'); | ||
import { AwsSigv4SignerResponse, AwsSigv4Signer } from '../../lib/aws'; | ||
|
||
const mockCreds = { | ||
accessKeyId: uuidv4(), | ||
secretAccessKey: uuidv4(), | ||
}; | ||
|
||
const mockRegion = 'us-west-2'; | ||
|
||
{ | ||
const AwsSigv4SignerOptions = { credentials: mockCreds, region: mockRegion }; | ||
|
||
const auth = AwsSigv4Signer(AwsSigv4SignerOptions); | ||
|
||
expectType<AwsSigv4SignerResponse>(auth); | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,81 @@ | ||
/* | ||
harshavamsi marked this conversation as resolved.
Show resolved
Hide resolved
|
||
* SPDX-License-Identifier: Apache-2.0 | ||
* | ||
* The OpenSearch Contributors require contributions made to | ||
* this file be licensed under the Apache-2.0 license or a | ||
* compatible open source license. | ||
* | ||
* Modifications Copyright OpenSearch Contributors. See | ||
* GitHub history for details. | ||
*/ | ||
const { test } = require('tap'); | ||
const { URL } = require('url'); | ||
const { v4: uuidv4 } = require('uuid'); | ||
const AwsSigv4Signer = require('../../lib/aws/AwsSigv4Signer'); | ||
const AwsSigv4SignerError = require('../../lib/aws/errors'); | ||
const { Connection } = require('../../index'); | ||
|
||
test('Sign with SigV4', (t) => { | ||
harshavamsi marked this conversation as resolved.
Show resolved
Hide resolved
|
||
t.plan(2); | ||
|
||
const mockCreds = { | ||
accessKeyId: uuidv4(), | ||
secretAccessKey: uuidv4(), | ||
}; | ||
|
||
const mockRegion = 'us-west-2'; | ||
|
||
const AwsSigv4SignerOptions = { credentials: mockCreds, region: mockRegion }; | ||
|
||
const auth = AwsSigv4Signer(AwsSigv4SignerOptions); | ||
|
||
const connection = new Connection({ | ||
url: new URL('https://localhost:9200'), | ||
}); | ||
|
||
const request = connection.buildRequestObject({ | ||
path: '/hello', | ||
method: 'GET', | ||
headers: { | ||
'X-Custom-Test': true, | ||
}, | ||
}); | ||
const signedRequest = auth.buildSignedRequestObject(request); | ||
t.hasProp(signedRequest.headers, 'X-Amz-Date'); | ||
t.hasProp(signedRequest.headers, 'Authorization'); | ||
}); | ||
|
||
test('Sign with SigV4 failure (with empty region)', (t) => { | ||
t.plan(2); | ||
|
||
const mockCreds = { | ||
accessKeyId: uuidv4(), | ||
secretAccessKey: uuidv4(), | ||
}; | ||
|
||
const AwsSigv4SignerOptions = { credentials: mockCreds }; | ||
|
||
try { | ||
AwsSigv4Signer(AwsSigv4SignerOptions); | ||
t.fail('Should fail'); | ||
} catch (err) { | ||
t.ok(err instanceof AwsSigv4SignerError); | ||
t.equal(err.message, 'Region cannot be empty'); | ||
} | ||
}); | ||
|
||
test('Sign with SigV4 failure (with empty credentials)', (t) => { | ||
t.plan(2); | ||
|
||
const mockRegion = 'us-west-2'; | ||
|
||
const AwsSigv4SignerOptions = { region: mockRegion }; | ||
|
||
try { | ||
AwsSigv4Signer(AwsSigv4SignerOptions); | ||
t.fail('Should fail'); | ||
} catch (err) { | ||
t.ok(err instanceof AwsSigv4SignerError); | ||
t.equal(err.message, 'Credentials cannot be empty'); | ||
} | ||
}); |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This is identical to the samples above? Maybe we should break up the examples to specific sections (indexing, searching, AWS Sigv4 Signing...?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The reason why it lives in a separate code block is just because of the difference in the imports and the way the client is initialized. Because the signer needs credentials, it's an async call. Felt like maybe the two should be written separately.