From e76c2479de9e3ad9d235d09f870e98701fe2f58e Mon Sep 17 00:00:00 2001 From: Katie Byers Date: Wed, 25 May 2022 07:22:10 -0700 Subject: [PATCH] ref(build): Use rollup to build AWS lambda layer (#5146) Our current system for building the AWS lambda layer involves a script which manually traces the serverless package's dependencies, including other monorepo packages, and symlinks them into a `node_modules` folder in a directory set up to mimic an installation from npm. There are a few disadvantages to this system: - The signals we rely on to figure out what is and isn't a dependency aren't exhaustive, and we're not experts at this, both of which have made getting the right files and only the right files into the layer a brittle process, which has broken down more than once and caused issue for our users. - The script is complicated, which makes it harder to figure out exactly what's causing it when something does go wrong. - We symlink in entire packages at a time, regardless of whether or not we're using most of the code. This is true even of the serverless package itself, where we include all of the GCP code along with the AWS code. This refactors our lambda layer build process to use the new bundling config functions we use for CDN bundles, to create a bundle which can take the place of the the index file, but which contains everything it needs internally. This has the following advantages: - This puts Rollup in charge of figuring out dependencies instead of us. - The config is in line with existing configs and is therefore much easier to reason about and maintain. - It lets us easily exclude anything GCP-related in the SDK. Between that, Rollup's treeshaking, and terser's minifying, the layer will now take up much less of the finite size allotted to each lambda function. - Removing extraneous files means less to cache and retrieve from the cache in each GHA job. Key changes: - The layer now builds in `packages/serverless/build/aws` rather than at the top level of the repo. (It is now moved to the top level only in our GHA workflow creating the lambda zip.) - In that workflow, the process to determine the SDK version has been simplified. - The bundle builds based not on the main index file but on a new bundle-specific index file, which only includes AWS code. - There is new rollup config just for the layer, which uses the bundle-building functions to make the main bundle and the npm-package-building functions to create a separate module for the optional script which automatically starts up the SDK in AWS. - The old build script has been replaced by one which runs rollup with that config, and then symlinks the built auto-startup script into its legacy location, so as to be backwards compatible with folks who run it using an environment variable pointing to the old path. - The building of the layer has temporarily been shifted from `build:awslambda-layer` to `build:bundle`, so that it will run during the first phase of the repo-level `yarn build`. (The goal is to eventually do everything in one phase, for greater parallelization and shorter overall build time.) h/t to @antonpirker for all his help testing and thinking through this with me. --- .github/workflows/build.yml | 82 ++++---- .gitignore | 3 + packages/core/.npmignore | 15 -- packages/hub/.npmignore | 15 -- packages/node/.npmignore | 15 -- packages/serverless/.eslintrc.js | 14 ++ packages/serverless/.gitignore | 1 - packages/serverless/.npmignore | 13 -- packages/serverless/package.json | 13 +- packages/serverless/rollup.aws.config.js | 44 ++++ packages/serverless/rollup.npm.config.js | 5 + .../scripts/build-awslambda-layer.js | 194 ------------------ .../serverless/scripts/buildLambdaLayer.ts | 62 ++++++ packages/serverless/src/index.awslambda.ts | 8 + packages/serverless/tsconfig.types.json | 6 +- packages/tracing/.npmignore | 15 -- packages/types/.npmignore | 13 -- packages/utils/.npmignore | 15 -- scripts/aws-deploy-local-layer.sh | 66 ++++++ scripts/ensure-bundle-deps.ts | 5 +- 20 files changed, 258 insertions(+), 346 deletions(-) delete mode 100644 packages/core/.npmignore delete mode 100644 packages/hub/.npmignore delete mode 100644 packages/node/.npmignore delete mode 100644 packages/serverless/.gitignore delete mode 100644 packages/serverless/.npmignore create mode 100644 packages/serverless/rollup.aws.config.js delete mode 100644 packages/serverless/scripts/build-awslambda-layer.js create mode 100644 packages/serverless/scripts/buildLambdaLayer.ts create mode 100644 packages/serverless/src/index.awslambda.ts delete mode 100644 packages/tracing/.npmignore delete mode 100644 packages/types/.npmignore delete mode 100644 packages/utils/.npmignore create mode 100755 scripts/aws-deploy-local-layer.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 35e2c3e95ccf..9765416a614c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,7 +31,7 @@ env: ${{ github.workspace }}/packages/ember/instance-initializers ${{ github.workspace }}/packages/gatsby/*.d.ts ${{ github.workspace }}/packages/core/src/version.ts - ${{ github.workspace }}/dist-serverless + ${{ github.workspace }}/packages/serverless ${{ github.workspace }}/packages/utils/cjs ${{ github.workspace }}/packages/utils/esm @@ -124,52 +124,50 @@ jobs: # this file) to a constant and skip rebuilding all of the packages each time CI runs. if: steps.cache_built_packages.outputs.cache-hit == '' run: yarn build - - name: Save SDK version for later - run: | - echo "Saving SDK_VERSION for later" - cat packages/core/src/version.ts | awk -F"'" '{print $2}' > dist-serverless/version - [ ! -z $(cat dist-serverless/version) ] && echo SDK_VERSION=$(cat dist-serverless/version) || (echo "Version extraction failed" && exit 1) outputs: # this needs to be passed on, because the `needs` context only looks at direct ancestors (so steps which depend on # `job_build` can't see `job_install_deps` and what it returned) dependency_cache_key: ${{ needs.job_install_deps.outputs.dependency_cache_key }} - # job_build_aws_lambda_layer: - # name: Build AWS Lambda Layer - # needs: job_build - # runs-on: ubuntu-latest - # steps: - # - name: Check out current commit (${{ env.HEAD_COMMIT }}) - # uses: actions/checkout@v2 - # with: - # ref: ${{ env.HEAD_COMMIT }} - # - name: Set up Node - # uses: actions/setup-node@v1 - # with: - # node-version: ${{ env.DEFAULT_NODE_VERSION }} - # - name: Check dependency cache - # uses: actions/cache@v2 - # with: - # path: ${{ env.CACHED_DEPENDENCY_PATHS }} - # key: ${{ needs.job_build.outputs.dependency_cache_key }} - # - name: Check build cache - # uses: actions/cache@v2 - # with: - # path: ${{ env.CACHED_BUILD_PATHS }} - # key: ${{ env.BUILD_CACHE_KEY }} - # - name: Get SDK version - # run: | - # export SDK_VERSION=$(cat dist-serverless/version) - # echo "SDK_VERSION=$SDK_VERSION" | tee -a $GITHUB_ENV - # - uses: actions/upload-artifact@v3 - # with: - # name: ${{ env.HEAD_COMMIT }} - # path: | - # dist-serverless/* - # - uses: getsentry/action-build-aws-lambda-extension@v1 - # with: - # artifact_name: ${{ env.HEAD_COMMIT }} - # zip_file_name: sentry-node-serverless-${{ env.SDK_VERSION }}.zip + job_pack_aws_lambda_layer: + name: Pack and Upload AWS Lambda Layer + needs: [job_get_metadata, job_build] + runs-on: ubuntu-latest + steps: + - name: Check out current commit (${{ needs.job_get_metadata.outputs.commit_label }}) + uses: actions/checkout@v2 + with: + ref: ${{ env.HEAD_COMMIT }} + - name: Set up Node + uses: actions/setup-node@v1 + with: + node-version: ${{ env.DEFAULT_NODE_VERSION }} + - name: Check dependency cache + uses: actions/cache@v2 + with: + path: ${{ env.CACHED_DEPENDENCY_PATHS }} + key: ${{ needs.job_build.outputs.dependency_cache_key }} + - name: Check build cache + uses: actions/cache@v2 + with: + path: ${{ env.CACHED_BUILD_PATHS }} + key: ${{ env.BUILD_CACHE_KEY }} + - name: Get SDK version + # `jq` reads JSON files, and `tee` pipes its input to the given location and to stdout. (Adding `-a` is the + # equivalent of using >> rather than >.) + run: | + export SDK_VERSION=$(cat packages/core/package.json | jq --raw-output '.version') + echo "SDK_VERSION=$SDK_VERSION" | tee -a $GITHUB_ENV + - name: Move dist-serverless to root directory (requirement for zipping action) + run: | + mv ./packages/serverless/build/aws/dist-serverless . + - name: Create and upload final zip file + uses: getsentry/action-build-aws-lambda-extension@v1 + with: + artifact_name: ${{ env.HEAD_COMMIT }} + zip_file_name: sentry-node-serverless-${{ env.SDK_VERSION }}.zip + build_cache_paths: ${{ env.CACHED_BUILD_PATHS }} + build_cache_key: ${{ env.BUILD_CACHE_KEY }} job_size_check: name: Size Check diff --git a/.gitignore b/.gitignore index 3c490212dcc2..bc3f92fa4202 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,9 @@ scratch/ *.js.map *.pyc *.tsbuildinfo +# side effects of running AWS lambda layer zip action locally +dist-serverless/ +sentry-node-serverless-*.zip # transpiled transformers jest/transformers/*.js # node tarballs diff --git a/packages/core/.npmignore b/packages/core/.npmignore deleted file mode 100644 index 329293958886..000000000000 --- a/packages/core/.npmignore +++ /dev/null @@ -1,15 +0,0 @@ -# The paths in this file are specified so that they align with the file structure in `./build` after this file is copied -# into it by the prepack script `scripts/prepack.ts`. - -* - -!/cjs/**/* -!/esm/**/* -!/types/**/* - -# These paths are necessary for Node AWS Lambda layer creation -# This package is a (transitive) dependency of @sentry/serverless and thus it is pulled into -# the lambda layer zip file. -!/build/cjs/**/* -!/build/esm/**/* -!/build/types/**/* diff --git a/packages/hub/.npmignore b/packages/hub/.npmignore deleted file mode 100644 index 329293958886..000000000000 --- a/packages/hub/.npmignore +++ /dev/null @@ -1,15 +0,0 @@ -# The paths in this file are specified so that they align with the file structure in `./build` after this file is copied -# into it by the prepack script `scripts/prepack.ts`. - -* - -!/cjs/**/* -!/esm/**/* -!/types/**/* - -# These paths are necessary for Node AWS Lambda layer creation -# This package is a (transitive) dependency of @sentry/serverless and thus it is pulled into -# the lambda layer zip file. -!/build/cjs/**/* -!/build/esm/**/* -!/build/types/**/* diff --git a/packages/node/.npmignore b/packages/node/.npmignore deleted file mode 100644 index 79d2576ebbb5..000000000000 --- a/packages/node/.npmignore +++ /dev/null @@ -1,15 +0,0 @@ -# The paths in this file are specified so that they align with the file structure in `./build` after this file is copied -# into it by the prepack script `scripts/prepack.ts`. - -* - -!/cjs/**/* -!/esm/**/* -!/types/**/* - -# These paths are necessary for Node AWS Lambda creation -# This package is a (transitive) dependency of @sentry/serverless and thus it is pulled into -# the lambda layer zip file. By specifying these paths, we -!/build/cjs/**/* -!/build/esm/**/* -!/build/types/**/* diff --git a/packages/serverless/.eslintrc.js b/packages/serverless/.eslintrc.js index ce28fd3a0514..b341e7ca3056 100644 --- a/packages/serverless/.eslintrc.js +++ b/packages/serverless/.eslintrc.js @@ -6,4 +6,18 @@ module.exports = { rules: { '@sentry-internal/sdk/no-async-await': 'off', }, + overrides: [ + { + files: ['scripts/**/*.ts'], + parserOptions: { + project: ['../../tsconfig.dev.json'], + }, + }, + { + files: ['test/**'], + parserOptions: { + sourceType: 'module', + }, + }, + ], }; diff --git a/packages/serverless/.gitignore b/packages/serverless/.gitignore deleted file mode 100644 index 466d272fde97..000000000000 --- a/packages/serverless/.gitignore +++ /dev/null @@ -1 +0,0 @@ -dist-serverless/ diff --git a/packages/serverless/.npmignore b/packages/serverless/.npmignore deleted file mode 100644 index 53b2a9d51a37..000000000000 --- a/packages/serverless/.npmignore +++ /dev/null @@ -1,13 +0,0 @@ -# The paths in this file are specified so that they align with the file structure in `./build` after this file is copied -# into it by the prepack script `scripts/prepack.ts`. - -* - -!/cjs/**/* -!/esm/**/* -!/types/**/* - -# These paths are necessary for Node AWS Lambda layer creation -!/build/cjs/**/* -!/build/esm/**/* -!/build/types/**/* diff --git a/packages/serverless/package.json b/packages/serverless/package.json index af75c1b74272..5350d7396571 100644 --- a/packages/serverless/package.json +++ b/packages/serverless/package.json @@ -9,9 +9,9 @@ "engines": { "node": ">=10" }, - "main": "build/cjs/index.js", - "module": "build/esm/index.js", - "types": "build/types/index.d.ts", + "main": "build/npm/cjs/index.js", + "module": "build/npm/esm/index.js", + "types": "build/npm/types/index.d.ts", "publishConfig": { "access": "public" }, @@ -38,8 +38,9 @@ "read-pkg": "^5.2.0" }, "scripts": { - "build": "run-p build:rollup build:types && yarn build:extras", - "build:awslambda-layer": "node scripts/build-awslambda-layer.js", + "build": "run-p build:rollup build:types build:bundle && yarn build:extras", + "build:awslambda-layer": "echo 'WARNING: AWS lambda layer build emporarily moved to \\`build:bundle\\`.'", + "build:bundle": "yarn ts-node scripts/buildLambdaLayer.ts", "build:dev": "run-p build:rollup build:types", "build:extras": "yarn build:awslambda-layer", "build:rollup": "rollup -c rollup.npm.config.js", @@ -48,7 +49,7 @@ "build:dev:watch": "run-s build:watch", "build:rollup:watch": "rollup -c rollup.npm.config.js --watch", "build:types:watch": "tsc -p tsconfig.types.json --watch", - "build:npm": "ts-node ../../scripts/prepack.ts && npm pack ./build", + "build:npm": "ts-node ../../scripts/prepack.ts --bundles && npm pack ./build/npm", "circularDepCheck": "madge --circular src/index.ts", "clean": "rimraf build dist-awslambda-layer coverage sentry-serverless-*.tgz", "fix": "run-s fix:eslint fix:prettier", diff --git a/packages/serverless/rollup.aws.config.js b/packages/serverless/rollup.aws.config.js new file mode 100644 index 000000000000..23c10b757f33 --- /dev/null +++ b/packages/serverless/rollup.aws.config.js @@ -0,0 +1,44 @@ +import { makeBaseBundleConfig, makeBundleConfigVariants, makeBaseNPMConfig } from '../../rollup/index.js'; + +export default [ + // The SDK + ...makeBundleConfigVariants( + makeBaseBundleConfig({ + // this automatically sets it to be CJS + bundleType: 'node', + entrypoints: ['src/index.awslambda.ts'], + jsVersion: 'es6', + licenseTitle: '@sentry/serverless', + outputFileBase: () => 'index', + packageSpecificConfig: { + output: { + dir: 'build/aws/dist-serverless/nodejs/node_modules/@sentry/serverless/build/npm/cjs', + sourcemap: false, + }, + }, + }), + // We only need one copy of the SDK, and we pick the minified one because there's a cap on how big a lambda function + // plus its dependencies can be, and we might as well take up as little of that space as is necessary. We'll rename + // it to be `index.js` in the build script, since it's standing in for the index file of the npm package. + { variants: ['.min.js'] }, + ), + + // This builds a wrapper file, which our lambda layer integration automatically sets up to run as soon as node + // launches (via the `NODE_OPTIONS="-r @sentry/serverless/dist/awslambda-auto"` variable). Note the inclusion in this + // path of the legacy `dist` folder; for backwards compatibility, in the build script we'll copy the file there. + makeBaseNPMConfig({ + entrypoints: ['src/awslambda-auto.ts'], + packageSpecificConfig: { + // Normally `makeNPMConfigVariants` sets both of these values for us, but we don't actually want the ESM variant, + // and the directory structure is different than normal, so we have to do it ourselves. + output: { + format: 'cjs', + dir: 'build/aws/dist-serverless/nodejs/node_modules/@sentry/serverless/build/npm/cjs', + sourcemap: false, + }, + // We only want `awslambda-auto.js`, not the modules that it imports, because they're all included in the bundle + // we generate above + external: ['./index'], + }, + }), +]; diff --git a/packages/serverless/rollup.npm.config.js b/packages/serverless/rollup.npm.config.js index 9b3074be8002..4e9641d5879e 100644 --- a/packages/serverless/rollup.npm.config.js +++ b/packages/serverless/rollup.npm.config.js @@ -2,6 +2,11 @@ import { makeBaseNPMConfig, makeNPMConfigVariants } from '../../rollup/index.js' export default makeNPMConfigVariants( makeBaseNPMConfig({ + // TODO: `awslambda-auto.ts` is a file which the lambda layer uses to automatically init the SDK. Does it need to be + // in the npm package? Is it possible that some people are using it themselves in the same way the layer uses it (in + // which case removing it would be a breaking change)? Should it stay here or not? entrypoints: ['src/index.ts', 'src/awslambda-auto.ts'], + // packages with bundles have a different build directory structure + hasBundles: true, }), ); diff --git a/packages/serverless/scripts/build-awslambda-layer.js b/packages/serverless/scripts/build-awslambda-layer.js deleted file mode 100644 index 8fbc493558e0..000000000000 --- a/packages/serverless/scripts/build-awslambda-layer.js +++ /dev/null @@ -1,194 +0,0 @@ -/* eslint-disable no-console */ -const path = require('path'); -const process = require('process'); -const fs = require('fs'); - -const findUp = require('find-up'); -const packList = require('npm-packlist'); -const readPkg = require('read-pkg'); - -const serverlessPackageJson = require('../package.json'); - -if (!process.env.GITHUB_ACTIONS) { - console.log('Skipping build-awslambda-layer script in local environment.'); - process.exit(0); -} - -// AWS Lambda layer are being uploaded as zip archive, whose content is then being unpacked to the /opt -// directory in the lambda environment. -// -// So this script does the following: it builds a 'dist-awslambda-layer/nodejs/node_modules/@sentry/serverless' -// directory with a special index.js and with all necessary @sentry packages symlinked as node_modules. -// Then, this directory is compressed with zip. -// -// The tricky part about it is that one cannot just symlink the entire package directories into node_modules because -// all the src/ contents and other unnecessary files will end up in the zip archive. So, we need to symlink only -// individual files from package and it must be only those of them that are distributable. -// There exists a `npm-packlist` library for such purpose. So we need to traverse all the dependencies, -// execute `npm-packlist` on them and symlink the files into 'dist-awslambda-layer/.../@sentry/serverless/node_modules'. -// I didn't find any way to achieve this goal using standard command-line tools so I have to write this script. -// -// Another, and much simpler way to assemble such zip bundle is install all the dependencies from npm registry and -// just bundle the entire node_modules. -// It's easier and looks more stable but it's inconvenient if one wants build a zip bundle out of current source tree. -// -// And yet another way is to bundle everything with webpack into a single file. I tried and it seems to be error-prone -// so I think it's better to have a classic package directory with node_modules file structure. - -/** - * Recursively traverses all the dependencies of @param pkg and collects all the info to the map - * The map ultimately contains @sentry/serverless itself, its direct dependencies and - * its transitive dependencies. - * - * @param cwd the root directory of the package - * @param packages the map accumulating all packages - */ -async function collectPackages(cwd, packages = {}) { - const packageJson = await readPkg({ cwd }); - - packages[packageJson.name] = { cwd, packageJson }; - - if (!packageJson.dependencies) { - return packages; - } - - await Promise.all( - Object.keys(packageJson.dependencies).map(async dep => { - // We are interested only in 'external' dependencies which are strictly upper than current directory. - // Internal deps aka local node_modules folder of each package is handled differently. - const searchPath = path.resolve(cwd, '..'); - const depPath = fs.realpathSync( - await findUp(path.join('node_modules', dep), { type: 'directory', cwd: searchPath }), - ); - if (packages[dep]) { - if (packages[dep].cwd != depPath) { - throw new Error(`${packageJson.name}'s dependency ${dep} maps to both ${packages[dep].cwd} and ${depPath}`); - } - return; - } - await collectPackages(depPath, packages); - }), - ); - - return packages; -} - -async function main() { - const baseDir = path.resolve(__dirname, '../../../') - const serverlessDir = path.resolve(__dirname, '..'); // packages/serverless directory - - const cjsBuildDir = path.resolve(serverlessDir, 'build', 'cjs'); - if (!fs.existsSync(cjsBuildDir)) { - console.log(`The path ${cjsBuildDir} must exist.`); - return; - } - - const packages = await collectPackages(serverlessDir); - - // the build directory of the Lambda layer - const layerBuildDir = path.resolve(baseDir, 'dist-serverless'); - - // the root directory in which the Lambda layer files + dependencies are copied to - // this structure resembles the structure where Lambda expects to find @sentry/serverless - const destRootRelative = 'nodejs/node_modules/@sentry/serverless'; - const destRootDir = path.resolve(layerBuildDir, destRootRelative); - - // this is where all the (transitive) dependencies of @sentry/serverless go - const destRootNodeModulesDir = path.resolve(destRootDir, 'node_modules'); - - try { - // Setting `force: true` ignores exceptions when paths don't exist. - fs.rmSync(destRootDir, { force: true, recursive: true, maxRetries: 1 }); - fs.mkdirSync(destRootDir, { recursive: true }); - } catch (error) { - // Ignore errors. - } - - await Promise.all( - Object.entries(packages).map(async ([name, pkg]) => { - const isServelessPkg = name == serverlessPackageJson.name; - const destDir = isServelessPkg ? destRootDir : path.resolve(destRootNodeModulesDir, name); - - // Scan over the "distributable" files of `pkg` and symlink all of them. - // `packList` returns all files it deems "distributable" from `pkg.cwd`. - // "Distributable" means in this case that the file would end up in the NPM tarball of `pkg`. - // To find out which files are distributable, packlist scans for NPM file configurations in the following order: - // 1. if `files` section present in package.json, take everything* from there - // 2. if `.npmignore` present, take everything* except what's ignored there - // 3. if `.gitignore` present, take everything* except what's ignored there - // 4. else take everything* - // In our case, rule 2 applies. - // * everything except certain unimportant files similarly to what `npm pack` does when packing a tarball. - // For more information on the rules see: https://github.com/npm/npm-packlist#readme - const sourceFiles = await packList({ path: pkg.cwd }); - - await Promise.all( - sourceFiles.map(async filename => { - const sourceFilePath = path.resolve(pkg.cwd, filename); - const destFilePath = path.resolve(destDir, filename); - - try { - fs.mkdirSync(path.dirname(destFilePath), { recursive: true }); - fs.symlinkSync(sourceFilePath, destFilePath); - } catch (error) { - // Ignore errors. - } - }), - ); - - // Now we deal with the `pkg`'s dependencies in its local `node_modules` directory - const pkgNodeModulesDir = path.resolve(pkg.cwd, 'node_modules'); - - // First, check if `pkg` has node modules. If not, we're done with this `pkg`. - // `fs.constants.F_OK` indicates whether the file is visible to the current process, but it doesn't check - // its permissions. For more information, refer to https://nodejs.org/api/fs.html#fs_file_access_constants. - try { - fs.accessSync(path.resolve(pkgNodeModulesDir), fs.constants.F_OK); - } catch (error) { - return; - } - - // Then, scan over local node_modules folder of `pkg` and symlink its non-dev dependencies. - const pkgNodeModules = fs.readdirSync(pkgNodeModulesDir); - await Promise.all( - pkgNodeModules.map(async nodeModule => { - if (!pkg.packageJson.dependencies || !pkg.packageJson.dependencies[nodeModule]) { - return; - } - - const sourceModulePath = path.resolve(pkgNodeModulesDir, nodeModule); - const destModulePath = path.resolve(destDir, 'node_modules', nodeModule); - - try { - fs.mkdirSync(path.dirname(destModulePath), { recursive: true }); - fs.symlinkSync(sourceModulePath, destModulePath); - } catch (error) { - // Ignore errors. - } - }), - ); - }), - ); - - // link from `./build/cjs` to `./dist` - // This needs to be done to satisfy the NODE_OPTIONS environment variable path that is set in - // AWS lambda functions when connecting them to Sentry. On initialization, the layer preloads a js - // file specified in NODE_OPTIONS to initialize the SDK. - // Hence we symlink everything from `.build/cjs` to `.dist`. - // This creates duplication but it's not too bad file size wise. - try { - fs.symlinkSync(path.resolve(destRootDir, 'build', 'cjs'), path.resolve(destRootDir, 'dist')); - } catch (error) { - console.error(error); - } -} - -main().then( - () => { - process.exit(0); - }, - err => { - console.error(err); - process.exit(-1); - }, -); diff --git a/packages/serverless/scripts/buildLambdaLayer.ts b/packages/serverless/scripts/buildLambdaLayer.ts new file mode 100644 index 000000000000..1a04aafde8aa --- /dev/null +++ b/packages/serverless/scripts/buildLambdaLayer.ts @@ -0,0 +1,62 @@ +/* eslint-disable no-console */ +import * as childProcess from 'child_process'; +import * as fs from 'fs'; +import * as rimraf from 'rimraf'; + +import { ensureBundleBuildPrereqs } from '../../../scripts/ensure-bundle-deps'; + +/** + * Run the given shell command, piping the shell process's `stdin`, `stdout`, and `stderr` to that of the current + * process. Returns contents of `stdout`. + */ +function run(cmd: string, options?: childProcess.ExecSyncOptions): string { + return String(childProcess.execSync(cmd, { stdio: 'inherit', ...options })); +} + +async function buildLambdaLayer(): Promise { + // Create the main SDK bundle + await ensureBundleBuildPrereqs({ + dependencies: ['@sentry/utils', '@sentry/hub', '@sentry/core', '@sentry/tracing', '@sentry/node'], + }); + run('yarn rollup --config rollup.aws.config.js'); + + // We build a minified bundle, but it's standing in for the regular `index.js` file listed in `package.json`'s `main` + // property, so we have to rename it so it's findable. + fs.renameSync( + 'build/aws/dist-serverless/nodejs/node_modules/@sentry/serverless/build/npm/cjs/index.min.js', + 'build/aws/dist-serverless/nodejs/node_modules/@sentry/serverless/build/npm/cjs/index.js', + ); + + // We're creating a bundle for the SDK, but still using it in a Node context, so we need to copy in `package.json`, + // purely for its `main` property. + console.log('Copying `package.json` into lambda layer.'); + fs.copyFileSync('package.json', 'build/aws/dist-serverless/nodejs/node_modules/@sentry/serverless/package.json'); + + // The layer also includes `awslambda-auto.js`, a helper file which calls `Sentry.init()` and wraps the lambda + // handler. It gets run when Node is launched inside the lambda, using the environment variable + // + // `NODE_OPTIONS="-r @sentry/serverless/dist/awslambda-auto"`. + // + // (The`-r` is what runs the script on startup.) The `dist` directory is no longer where we emit our built code, so + // for backwards compatibility, we create a symlink. + console.log('Creating symlink for `awslambda-auto.js` in legacy `dist` directory.'); + fsForceMkdirSync('build/aws/dist-serverless/nodejs/node_modules/@sentry/serverless/dist'); + fs.symlinkSync( + '../build/npm/cjs/awslambda-auto.js', + 'build/aws/dist-serverless/nodejs/node_modules/@sentry/serverless/dist/awslambda-auto.js', + ); +} + +void buildLambdaLayer(); + +/** + * Make a directory synchronously, overwriting the old directory if necessary. + * + * This is what `fs.mkdirSync(path, { force: true })` would be, if it existed. Primarily useful for local building and + * testing, where scripts are often run more than once (and so the directory you're trying to create may already be + * there), but also harmless when used in CI. + */ +function fsForceMkdirSync(path: string): void { + rimraf.sync(path); + fs.mkdirSync(path); +} diff --git a/packages/serverless/src/index.awslambda.ts b/packages/serverless/src/index.awslambda.ts new file mode 100644 index 000000000000..c097591ab9dc --- /dev/null +++ b/packages/serverless/src/index.awslambda.ts @@ -0,0 +1,8 @@ +/** This file is used as the entrypoint for the lambda layer bundle, and is not included in the npm package. */ + +// https://medium.com/unsplash/named-namespace-imports-7345212bbffb +import * as AWSLambda from './awslambda'; +export { AWSLambda }; + +export * from './awsservices'; +export * from '@sentry/node'; diff --git a/packages/serverless/tsconfig.types.json b/packages/serverless/tsconfig.types.json index 65455f66bd75..4c51bd21e64b 100644 --- a/packages/serverless/tsconfig.types.json +++ b/packages/serverless/tsconfig.types.json @@ -1,10 +1,14 @@ { "extends": "./tsconfig.json", + // We don't ship this in the npm package (it exists purely for controlling what ends up in the AWS lambda layer), so + // no need to build types for it + "exclude": ["src/index.awslambda.ts"], + "compilerOptions": { "declaration": true, "declarationMap": true, "emitDeclarationOnly": true, - "outDir": "build/types" + "outDir": "build/npm/types" } } diff --git a/packages/tracing/.npmignore b/packages/tracing/.npmignore deleted file mode 100644 index 37cb0bef4d30..000000000000 --- a/packages/tracing/.npmignore +++ /dev/null @@ -1,15 +0,0 @@ -# The paths in this file are specified so that they align with the file structure in `./build` after this file is copied -# into it by the prepack script `scripts/prepack.ts`. - -* - -!/cjs/**/* -!/esm/**/* -!/types/**/* - -# These paths are necessary for Node AWS Lambda layer creation -# This package is a (transitive) dependency of @sentry/serverless and thus it is pulled into -# the lambda layer zip file. -!/build/npm/cjs/**/* -!/build/npm/esm/**/* -!/build/npm/types/**/* diff --git a/packages/types/.npmignore b/packages/types/.npmignore deleted file mode 100644 index 7e33b95686a8..000000000000 --- a/packages/types/.npmignore +++ /dev/null @@ -1,13 +0,0 @@ -# The paths in this file are specified so that they align with the file structure in `./build` after this file is copied -# into it by the prepack script `scripts/prepack.ts`. - -* - -!/cjs/**/* -!/esm/**/* -!/types/**/* - -# These paths are necessary for @sentry/serverless AWS Lambda Layer creation -!/build/cjs/**/* -!/build/esm/**/* -!/build/types/**/* diff --git a/packages/utils/.npmignore b/packages/utils/.npmignore deleted file mode 100644 index 329293958886..000000000000 --- a/packages/utils/.npmignore +++ /dev/null @@ -1,15 +0,0 @@ -# The paths in this file are specified so that they align with the file structure in `./build` after this file is copied -# into it by the prepack script `scripts/prepack.ts`. - -* - -!/cjs/**/* -!/esm/**/* -!/types/**/* - -# These paths are necessary for Node AWS Lambda layer creation -# This package is a (transitive) dependency of @sentry/serverless and thus it is pulled into -# the lambda layer zip file. -!/build/cjs/**/* -!/build/esm/**/* -!/build/types/**/* diff --git a/scripts/aws-deploy-local-layer.sh b/scripts/aws-deploy-local-layer.sh new file mode 100755 index 000000000000..b86dcd344d78 --- /dev/null +++ b/scripts/aws-deploy-local-layer.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# +# Builds and deploys the Sentry AWS Lambda layer (including the Sentry SDK and the Sentry Lambda Extension) +# +# The currently checked out version of the SDK in your local directory is used. +# The latest version of the Lambda Extension is fetched from the Sentry Release Registry. +# +# Note: While we normally try to write all of our scripts in TS, this is in bash because it's meant to exactly mirror +# what the lambda-zipping GHA is doing (see https://github.com/getsentry/action-build-aws-lambda-extension) + +set -euo pipefail + +# Cleanup +echo "Preparing local directories for new build..." +rm -rf dist-serverless/ +rm -rf ./packages/serverless/build +rm -rf ./packages/serverless/dist +rm -rf ./packages/serverless/node_modules +rm -f sentry-node-serverless-*.zip + +# Creating Lambda layer +echo "Creating Lambda layer in ./packages/serverless/build/aws/dist-serverless..." +cd packages/serverless +yarn build +cd ../../ +echo "Done creating Lambda layer in ./packages/serverless/build/aws/dist-serverless." + +# Move dist-serverless/ to the root folder for the action to pick it up. +# This is only needed in this script, because in GitHub workflow +# this is done with the upload-artifact/download-artifact actions +echo "Copying Lambda layer in ./packages/serverless/build/aws/dist-serverless to working directory..." +mv ./packages/serverless/build/aws/dist-serverless . +echo "Done copying Lambda layer in ./packages/serverless/build/aws/dist-serverless to working directory." + +# IMPORTANT: +# Please make sure that this does the same as the GitHub action that +# is building the Lambda layer in production! +# see: https://github.com/getsentry/action-build-aws-lambda-extension/blob/main/action.yml#L23-L40 + +# Adding Sentry Lambda extension to Lambda layer +echo "Adding Sentry Lambda extension to Lambda layer in ./dist-serverless..." +mkdir -p dist-serverless/extensions +curl -0 --silent --output dist-serverless/extensions/sentry-lambda-extension $(curl -s https://release-registry.services.sentry.io/apps/sentry-lambda-extension/latest | jq -r .files.\"sentry-lambda-extension\".url) +chmod +x dist-serverless/extensions/sentry-lambda-extension +echo "Done adding Sentry Lambda extension to Lambda layer in ./dist-serverless." + +# Zip Lambda layer and included Lambda extension +echo "Zipping Lambda layer and included Lambda extension..." +cd dist-serverless/ +zip -r -y ../sentry-node-serverless-x.x.x-dev.zip . +cd .. +echo "Done Zipping Lambda layer and included Lambda extension to ./sentry-node-serverless-x.x.x-dev.zip." + +# Deploying zipped Lambda layer to AWS +echo "Deploying zipped Lambda layer to AWS..." + +aws lambda publish-layer-version \ + --layer-name "SentryNodeServerlessSDK-local-dev" \ + --region "eu-central-1" \ + --zip-file "fileb://sentry-node-serverless-x.x.x-dev.zip" \ + --description "Local test build of SentryNodeServerlessSDK (can be deleted)" \ + --no-cli-pager + +echo "Done deploying zipped Lambda layer to AWS as 'SentryNodeServerlessSDK-local-dev'." + +echo "All done. Have a nice day!" diff --git a/scripts/ensure-bundle-deps.ts b/scripts/ensure-bundle-deps.ts index 61e724fc0029..06bdaf863582 100644 --- a/scripts/ensure-bundle-deps.ts +++ b/scripts/ensure-bundle-deps.ts @@ -7,7 +7,10 @@ import * as util from 'util'; /** * Ensure that `build:bundle` has all of the dependencies it needs to run. Works at both the repo and package level. */ -async function ensureBundleBuildPrereqs(options: { dependencies: string[]; maxRetries?: number }): Promise { +export async function ensureBundleBuildPrereqs(options: { + dependencies: string[]; + maxRetries?: number; +}): Promise { const { maxRetries = 12, dependencies } = options; const {