From 64be3987cf38bcc49be24262b111e8ee2af7b3a7 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 29 Nov 2021 12:53:28 +0000 Subject: [PATCH 1/8] feat: swap js implementation of Ed25519 for wasm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the DHT is enabled, the amount of network traffic nodes send is vastly increased. At that point our crypto implmenetations fast become a performance bottleneck as we are signing and verifying a greatly increased number of messages due to the increased number of peers we have. Pure-js implementations of Ed25519 are just not fast enough for use. Switching to native/WASM versions drops ambient CPU usage for js-IPFS from ~80% to ~2% while DHT table refreshes are ongoing in my local testing. Uses [ed25519](https://www.npmjs.com/package/ed25519) in node, falling back to [ed25519-wasm-pro](https://www.npmjs.com/package/ed25519-wasm-pro) in the browser. Benchmarks: ``` @noble/ed25519 x 191 ops/sec ±24.37% (78 runs sampled) @stablelib/ed25519 x 62.10 ops/sec ±2.14% (73 runs sampled) node-forge/ed25519 x 63.00 ops/sec ±2.62% (71 runs sampled) supercop.wasm x 3,433 ops/sec ±1.77% (83 runs sampled) ed25519-wasm-pro x 3,562 ops/sec ±1.07% (80 runs sampled) ed25519 (native module) x 4,625 ops/sec ±0.96% (84 runs sampled) node.js web-crypto x 3,344 ops/sec ±2.61% (78 runs sampled) ``` --- benchmarks/ed25519/compat.js | 27 ++++++++++ benchmarks/ed25519/index.js | 40 +++++++++++---- package.json | 7 ++- src/keys/ed25519-class.js | 9 ++-- src/keys/ed25519.js | 68 ------------------------- src/keys/ed25519/ed25519-native.js | 40 +++++++++++++++ src/keys/ed25519/ed25519-wasm.js | 54 ++++++++++++++++++++ src/keys/ed25519/index.js | 81 ++++++++++++++++++++++++++++++ 8 files changed, 244 insertions(+), 82 deletions(-) delete mode 100644 src/keys/ed25519.js create mode 100644 src/keys/ed25519/ed25519-native.js create mode 100644 src/keys/ed25519/ed25519-wasm.js create mode 100644 src/keys/ed25519/index.js diff --git a/benchmarks/ed25519/compat.js b/benchmarks/ed25519/compat.js index 479d7859..cb99b2eb 100644 --- a/benchmarks/ed25519/compat.js +++ b/benchmarks/ed25519/compat.js @@ -26,6 +26,7 @@ require('node-forge/lib/ed25519') const forge = require('node-forge/lib/forge') const stable = require('@stablelib/ed25519') const supercopWasm = require('supercop.wasm') +const ed25519WasmPro = require('ed25519-wasm-pro') const ALGORITHM = 'NODE-ED25519' const ED25519_PKCS8_PREFIX = fromString('302e020100300506032b657004220420', 'hex') @@ -97,6 +98,32 @@ const implementations = [{ verify: (message, signature, keyPair) => { return supercopWasm.verify(signature, message, keyPair.publicKey) } +}, { + name: 'ed25519-wasm-pro', + before: () => { + return new Promise(resolve => { + ed25519WasmPro.ready(() => { + resolve() + }) + }) + }, + generateKeyPair: async () => { + const seed = ed25519WasmPro.createSeed() + const key = ed25519WasmPro.createKeyPair(seed) + + return { + privateKey: seed, + publicKey: key.publicKey + } + }, + sign: (message, keyPair) => { + const key = ed25519WasmPro.createKeyPair(keyPair.privateKey) + + return ed25519WasmPro.sign(message, key.publicKey, key.secretKey) + }, + verify: (message, signature, keyPair) => { + return ed25519WasmPro.verify(signature, message, keyPair.publicKey) + } }, { name: 'native Ed25519', generateKeyPair: async () => { diff --git a/benchmarks/ed25519/index.js b/benchmarks/ed25519/index.js index 5d07d67c..0e038662 100644 --- a/benchmarks/ed25519/index.js +++ b/benchmarks/ed25519/index.js @@ -10,6 +10,7 @@ require('node-forge/lib/ed25519') const forge = require('node-forge/lib/forge') const stable = require('@stablelib/ed25519') const supercopWasm = require('supercop.wasm') +const ed25519WasmPro = require('ed25519-wasm-pro') const suite = new Benchmark.Suite('ed25519 implementations') @@ -62,7 +63,21 @@ suite.add('supercop.wasm', async (d) => { const isSigned = await supercopWasm.verify(signature, message, keys.publicKey) if (!isSigned) { - throw new Error('could not verify noble signature') + throw new Error('could not verify supercop.wasm signature') + } + + d.resolve() +}, { defer: true }) + +suite.add('ed25519-wasm-pro', async (d) => { + const message = Buffer.from('hello world ' + Math.random()) + const seed = ed25519WasmPro.createSeed() + const keys = ed25519WasmPro.createKeyPair(seed) + const signature = ed25519WasmPro.sign(message, keys.publicKey, keys.secretKey) + const isSigned = await ed25519WasmPro.verify(signature, message, keys.publicKey) + + if (!isSigned) { + throw new Error('could not verify ed25519-wasm-pro signature') } d.resolve() @@ -100,14 +115,21 @@ suite.add('node.js web-crypto', async (d) => { }, { defer: true }) async function main () { - supercopWasm.ready(() => { - suite - .on('cycle', (event) => console.log(String(event.target))) - .on('complete', function () { - console.log('fastest is ' + this.filter('fastest').map('name')) - }) - .run({ async: true }) - }) + await Promise.all([ + new Promise((resolve) => { + supercopWasm.ready(() => resolve()) + }), + new Promise((resolve) => { + ed25519WasmPro.ready(() => resolve()) + }) + ]) + + suite + .on('cycle', (event) => console.log(String(event.target))) + .on('complete', function () { + console.log('fastest is ' + this.filter('fastest').map('name')) + }) + .run({ async: true }) } main() diff --git a/package.json b/package.json index ca5641cd..dd7f809b 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,10 @@ "./src/ciphers/aes-gcm.js": "./src/ciphers/aes-gcm.browser.js", "./src/hmac/index.js": "./src/hmac/index-browser.js", "./src/keys/ecdh.js": "./src/keys/ecdh-browser.js", - "./src/keys/rsa.js": "./src/keys/rsa-browser.js" + "./src/keys/rsa.js": "./src/keys/rsa-browser.js", + "./src/keys/ed25519/ed25519-native.js": "./src/keys/ed25519/ed25519-wasm.js", + "crypto": false, + "fs": false }, "files": [ "src", @@ -42,6 +45,8 @@ "dependencies": { "@noble/ed25519": "^1.3.0", "@noble/secp256k1": "^1.3.0", + "ed25519": "^0.0.5", + "ed25519-wasm-pro": "1.1.1", "err-code": "^3.0.1", "iso-random-stream": "^2.0.0", "keypair": "^1.0.4", diff --git a/src/keys/ed25519-class.js b/src/keys/ed25519-class.js index a75be769..bc0bf57f 100644 --- a/src/keys/ed25519-class.js +++ b/src/keys/ed25519-class.js @@ -2,6 +2,7 @@ const errcode = require('err-code') const { equals: uint8ArrayEquals } = require('uint8arrays/equals') +const { concat: uint8ArrayConcat } = require('uint8arrays/concat') const { sha256 } = require('multiformats/hashes/sha2') const { base58btc } = require('multiformats/bases/base58') const { identity } = require('multiformats/hashes/identity') @@ -41,7 +42,7 @@ class Ed25519PublicKey { } class Ed25519PrivateKey { - // key - 64 byte Uint8Array containing private key + // key - 32 byte Uint8Array containing private key // publicKey - 32 byte Uint8Array containing public key constructor (key, publicKey) { this._key = ensureKey(key, crypto.privateKeyLength) @@ -57,7 +58,7 @@ class Ed25519PrivateKey { } marshal () { - return this._key + return uint8ArrayConcat([this._key, this._publicKey], crypto.privateKeyLength + crypto.publicKeyLength) } get bytes () { @@ -139,10 +140,10 @@ async function generateKeyPairFromSeed (seed) { function ensureKey (key, length) { key = Uint8Array.from(key || []) - if (key.length !== length) { + if (key.length < length) { throw errcode(new Error(`Key must be a Uint8Array of length ${length}, got ${key.length}`), 'ERR_INVALID_KEY_TYPE') } - return key + return key.subarray(0, length) } module.exports = { diff --git a/src/keys/ed25519.js b/src/keys/ed25519.js deleted file mode 100644 index a552d626..00000000 --- a/src/keys/ed25519.js +++ /dev/null @@ -1,68 +0,0 @@ -'use strict' - -const ed = require('@noble/ed25519') - -const PUBLIC_KEY_BYTE_LENGTH = 32 -const PRIVATE_KEY_BYTE_LENGTH = 64 // private key is actually 32 bytes but for historical reasons we concat private and public keys -const KEYS_BYTE_LENGTH = 32 - -exports.publicKeyLength = PUBLIC_KEY_BYTE_LENGTH -exports.privateKeyLength = PRIVATE_KEY_BYTE_LENGTH - -exports.generateKey = async function () { - // the actual private key (32 bytes) - const privateKeyRaw = ed.utils.randomPrivateKey() - const publicKey = await ed.getPublicKey(privateKeyRaw) - - // concatenated the public key to the private key - const privateKey = concatKeys(privateKeyRaw, publicKey) - - return { - privateKey, - publicKey - } -} - -/** - * Generate keypair from a seed - * - * @param {Uint8Array} seed - seed should be a 32 byte uint8array - * @returns - */ -exports.generateKeyFromSeed = async function (seed) { - if (seed.length !== KEYS_BYTE_LENGTH) { - throw new TypeError('"seed" must be 32 bytes in length.') - } else if (!(seed instanceof Uint8Array)) { - throw new TypeError('"seed" must be a node.js Buffer, or Uint8Array.') - } - - // based on node forges algorithm, the seed is used directly as private key - const privateKeyRaw = seed - const publicKey = await ed.getPublicKey(privateKeyRaw) - - const privateKey = concatKeys(privateKeyRaw, publicKey) - - return { - privateKey, - publicKey - } -} - -exports.hashAndSign = function (privateKey, msg) { - const privateKeyRaw = privateKey.slice(0, KEYS_BYTE_LENGTH) - - return ed.sign(msg, privateKeyRaw) -} - -exports.hashAndVerify = function (publicKey, sig, msg) { - return ed.verify(sig, msg, publicKey) -} - -function concatKeys (privateKeyRaw, publicKey) { - const privateKey = new Uint8Array(exports.privateKeyLength) - for (let i = 0; i < KEYS_BYTE_LENGTH; i++) { - privateKey[i] = privateKeyRaw[i] - privateKey[KEYS_BYTE_LENGTH + i] = publicKey[i] - } - return privateKey -} diff --git a/src/keys/ed25519/ed25519-native.js b/src/keys/ed25519/ed25519-native.js new file mode 100644 index 00000000..6245fe98 --- /dev/null +++ b/src/keys/ed25519/ed25519-native.js @@ -0,0 +1,40 @@ +'use strict' + +const native = require('ed25519') + +/** + * Generate keypair from a seed + * + * @param {Uint8Array} seed - seed should be a 32 byte uint8array + */ +async function generateKeyFromSeed (seed) { + const key = native.MakeKeypair(seed) + + return { + privateKey: key.privateKey.subarray(0, 32), + publicKey: key.publicKey + } +} + +/** + * @param {Uint8Array} privateKey + * @param {Uint8Array} message + */ +async function hashAndSign (privateKey, message) { + return native.Sign(message, privateKey) +} + +/** + * @param {Uint8Array} publicKey + * @param {Uint8Array} signature + * @param {Uint8Array} message + */ +async function hashAndVerify (publicKey, signature, message) { + return native.Verify(message, signature, publicKey) +} + +module.exports = { + generateKeyFromSeed, + hashAndSign, + hashAndVerify +} diff --git a/src/keys/ed25519/ed25519-wasm.js b/src/keys/ed25519/ed25519-wasm.js new file mode 100644 index 00000000..3c3450bc --- /dev/null +++ b/src/keys/ed25519/ed25519-wasm.js @@ -0,0 +1,54 @@ +'use strict' + +const ed25519 = require('ed25519-wasm-pro') + +const ready = new Promise((resolve) => { + ed25519.ready(() => { + resolve() + }) +}) + +/** + * Generate keypair from a seed + * + * @param {Uint8Array} seed - seed should be a 32 byte uint8array + */ +async function generateKeyFromSeed (seed) { + await ready + + const key = ed25519.createKeyPair(seed) + + return { + privateKey: seed, + publicKey: key.publicKey + } +} + +/** + * @param {Uint8Array} privateKey + * @param {Uint8Array} message + */ +async function hashAndSign (privateKey, message) { + await ready + + const key = ed25519.createKeyPair(privateKey) + + return ed25519.sign(message, key.publicKey, key.secretKey) +} + +/** + * @param {Uint8Array} publicKey + * @param {Uint8Array} signature + * @param {Uint8Array} message + */ +async function hashAndVerify (publicKey, signature, message) { + await ready + + return ed25519.verify(signature, message, publicKey) +} + +module.exports = { + generateKeyFromSeed, + hashAndSign, + hashAndVerify +} diff --git a/src/keys/ed25519/index.js b/src/keys/ed25519/index.js new file mode 100644 index 00000000..dfb631c2 --- /dev/null +++ b/src/keys/ed25519/index.js @@ -0,0 +1,81 @@ +'use strict' + +const randomBytes = require('../../random-bytes') + +let impl + +try { + impl = require('./ed25519-native') +} catch { + impl = require('./ed25519-wasm') +} + +const PUBLIC_KEY_BYTE_LENGTH = 32 +const PRIVATE_KEY_BYTE_LENGTH = 32 +const SEED_BYTE_LENGTH = 32 +const SIGNATURE_BYTE_LENGTH = 64 + +async function generateKey () { + return generateKeyFromSeed(randomBytes(SEED_BYTE_LENGTH)) +} + +/** + * Generate keypair from a seed + * + * @param {Uint8Array} seed - seed should be a 32 byte uint8array + */ +async function generateKeyFromSeed (seed) { + assertBytes('seed', seed, SEED_BYTE_LENGTH) + + return impl.generateKeyFromSeed(seed) +} + +/** + * @param {Uint8Array} privateKey + * @param {Uint8Array} message + */ +async function hashAndSign (privateKey, message) { + assertBytes('privateKey', privateKey, PUBLIC_KEY_BYTE_LENGTH) + assertBytes('message', message) + + return impl.hashAndSign(privateKey, message) +} + +/** + * @param {Uint8Array} publicKey + * @param {Uint8Array} signature + * @param {Uint8Array} message + */ +async function hashAndVerify (publicKey, signature, message) { + assertBytes('publicKey', publicKey, PUBLIC_KEY_BYTE_LENGTH) + assertBytes('signature', signature, SIGNATURE_BYTE_LENGTH) + assertBytes('message', message) + + return impl.hashAndVerify(publicKey, signature, message) +} + +function assertBytes (name, value, length) { + if (!(value instanceof Uint8Array)) { + throw new TypeError(`"${name}" must be a Uint8Array`) + } + + if (length != null) { + if (value.length !== length) { + throw new TypeError(`"${name}" must be ${length} bytes in length`) + } + } else { + if (!value.length) { + throw new TypeError(`"${name}" must be have a length`) + } + } +} + +module.exports = { + publicKeyLength: PUBLIC_KEY_BYTE_LENGTH, + privateKeyLength: PRIVATE_KEY_BYTE_LENGTH, + + generateKey, + generateKeyFromSeed, + hashAndSign, + hashAndVerify +} From df551a7a1ed968c99fd6ca9b31c391b2483ebca4 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 29 Nov 2021 14:00:42 +0000 Subject: [PATCH 2/8] chore: update config --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 82349c5f..6c2ec303 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,8 @@ "./src/keys/rsa.js": "./src/keys/rsa-browser.js", "./src/keys/ed25519/ed25519-native.js": "./src/keys/ed25519/ed25519-wasm.js", "crypto": false, - "fs": false + "fs": false, + "path": false }, "files": [ "src", From e795cd2c1e8000759aff37acfdca5515fe433f18 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 29 Nov 2021 14:06:00 +0000 Subject: [PATCH 3/8] chore: increase bundle size --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 6c2ec303..b1845a35 100644 --- a/package.json +++ b/package.json @@ -67,7 +67,7 @@ }, "aegir": { "build": { - "bundlesizeMax": "71kB" + "bundlesizeMax": "121kB" } }, "engines": { From ed1872584d5f9469cc6c5ebf032f3a3377d5304f Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 29 Nov 2021 14:13:24 +0000 Subject: [PATCH 4/8] chore: make native ed25519 optional as we will fall back to wasm version anyway --- package.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index b1845a35..6321da14 100644 --- a/package.json +++ b/package.json @@ -46,7 +46,6 @@ "dependencies": { "@noble/ed25519": "^1.3.0", "@noble/secp256k1": "^1.3.0", - "ed25519": "^0.0.5", "ed25519-wasm-pro": "1.1.1", "err-code": "^3.0.1", "iso-random-stream": "^2.0.0", @@ -58,6 +57,9 @@ "uint8arrays": "^3.0.0", "ursa-optional": "^0.10.1" }, + "optionalDependencies": { + "ed25519": "^0.0.5" + }, "devDependencies": { "@types/mocha": "^9.0.0", "aegir": "^36.0.2", From 1ac021f37ccd87cdb2abb14c35494a5828a25ec2 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 29 Nov 2021 14:21:25 +0000 Subject: [PATCH 5/8] chore: install node-pre-gyp --- .github/workflows/main.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0a575aab..48ee255d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,6 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + - run: npm install -g @mapbox/node-pre-gyp - run: npm install - run: npm run lint - run: npm run build @@ -33,6 +34,7 @@ jobs: - uses: actions/setup-node@v1 with: node-version: ${{ matrix.node }} + - run: npm install -g @mapbox/node-pre-gyp - run: npm install - run: npx nyc --reporter=lcov aegir test -t node -- --bail - uses: codecov/codecov-action@v1 @@ -41,6 +43,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + - run: npm install -g @mapbox/node-pre-gyp - run: npm install - run: npm run test -- -t browser -t webworker --bail test-firefox: @@ -48,6 +51,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + - run: npm install -g @mapbox/node-pre-gyp - run: npm install - run: npm run test -- -t browser -t webworker --bail -- --browsers FirefoxHeadless test-electron-main: @@ -55,6 +59,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + - run: npm install -g @mapbox/node-pre-gyp - run: npm install - run: npx xvfb-maybe aegir test -t electron-main --bail test-electron-renderer: @@ -62,5 +67,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + - run: npm install -g @mapbox/node-pre-gyp - run: npm install - run: npx xvfb-maybe aegir test -t electron-renderer --bail From 1f3290cf101ad0e87ef1c76d2ba311ebb54b1290 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 29 Nov 2021 15:02:02 +0000 Subject: [PATCH 6/8] chore: try making it non-optional --- package.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/package.json b/package.json index 6321da14..6a7a378c 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,6 @@ ], "license": "MIT", "dependencies": { - "@noble/ed25519": "^1.3.0", "@noble/secp256k1": "^1.3.0", "ed25519-wasm-pro": "1.1.1", "err-code": "^3.0.1", @@ -55,7 +54,7 @@ "pem-jwk": "^2.0.0", "protobufjs": "^6.11.2", "uint8arrays": "^3.0.0", - "ursa-optional": "^0.10.1" + "ursa": "^0.9.4" }, "optionalDependencies": { "ed25519": "^0.0.5" From e1ce4cca6589f3261b307453209eb59d2022840d Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 29 Nov 2021 15:09:04 +0000 Subject: [PATCH 7/8] chore: revert previous commit --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 6a7a378c..de915d7c 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,7 @@ "pem-jwk": "^2.0.0", "protobufjs": "^6.11.2", "uint8arrays": "^3.0.0", - "ursa": "^0.9.4" + "ursa-optional": "^0.10.1" }, "optionalDependencies": { "ed25519": "^0.0.5" From ae5b51a8a4d7e8c804131b3ccdbd2e67c8897f94 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 29 Nov 2021 16:10:12 +0000 Subject: [PATCH 8/8] chore: fall back to js for rsa on windows --- test/keys/rsa-crypto-libs.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/keys/rsa-crypto-libs.js b/test/keys/rsa-crypto-libs.js index 289445a9..b92cc453 100644 --- a/test/keys/rsa-crypto-libs.js +++ b/test/keys/rsa-crypto-libs.js @@ -4,9 +4,16 @@ /* eslint max-nested-callbacks: ["error", 8] */ const { expect } = require('aegir/utils/chai') +const os = require('os') const LIBS = ['ursa', 'keypair'] +if (os.platform() === 'win32') { + // usra is broken on windows + // TODO: only use webcrypto for RSA + LIBS.shift() +} + describe('RSA crypto libs', function () { this.timeout(20 * 1000)