From 19556d2c69b0063ddd8d484014417683256a4e42 Mon Sep 17 00:00:00 2001 From: Trivikram Kamat <16024985+trivikr@users.noreply.github.com> Date: Tue, 2 Jul 2024 09:19:28 -0700 Subject: [PATCH] metrics: add documentation for updating Cloud Run (#3779) * chore: add .gitignore with metrics-processor-service-key.json * metrics(process-cloudflare): add update instructions * metrics(summaries): add update instructions * metrics(index-generator): add update instructions * chore: remove copy from secrets/build/infra * metrics: remove .gitignore * metrics: remove metrics-processor-service-key.json from client params * metrics: remove step for copying credentials file * metrics: attach service account in gcloud run deploy command * metrics: remove Storage Legacy Bucket Writer permission * metrics: sort roles whose permissions are required * metrics: provide correct service-account details * chore: move region after at the end of gcloud run deploy Since the region is generic, and can be set as default in gcloud CLI --- .../metrics/files/index-generator/README.md | 32 +++++++++++++++++++ .../files/index-generator/index-generator.js | 4 +-- .../files/process-cloudflare/README.md | 32 +++++++++++++++++++ .../process-cloudflare/process-cloudflare.js | 4 +-- .../roles/metrics/files/summaries/README.md | 32 +++++++++++++++++++ .../metrics/files/summaries/summaries.js | 8 ++--- 6 files changed, 100 insertions(+), 12 deletions(-) create mode 100644 ansible/roles/metrics/files/index-generator/README.md create mode 100644 ansible/roles/metrics/files/process-cloudflare/README.md create mode 100644 ansible/roles/metrics/files/summaries/README.md diff --git a/ansible/roles/metrics/files/index-generator/README.md b/ansible/roles/metrics/files/index-generator/README.md new file mode 100644 index 000000000..849f4152c --- /dev/null +++ b/ansible/roles/metrics/files/index-generator/README.md @@ -0,0 +1,32 @@ +# index-generator + +## Prerequisites + +- Ensure that you have following roles on GCP: + - `Cloud Run Developer` + - `Service Account User` + - `Storage Object User` + - Example request: [nodejs/build#3774][cloud-run-roles-request] +- Install [Node.js][install-nodejs]. +- Install [gcloud CLI][install-gcloud-cli]. + - Authenticate using [`gcloud auth`][gcloud-auth]. +- Install [Docker Desktop][install-docker-desktop]. + - Authenticate using [`gcloud auth configure-docker`][gcloud-auth-docker]. + +## Setup + +- Build Dockerfile + - `docker buildx build --platform linux/amd64 -t gcr.io/nodejs-org/index-generator:latest .` +- Push Docker image to GCR + - `docker push gcr.io/nodejs-org/index-generator:latest` +- Deploy Cloud Run service + - `gcloud run deploy index-generator --image gcr.io/nodejs-org/index-generator:latest --service-account metrics-processor@nodejs-org.iam.gserviceaccount.com --no-allow-unauthenticated --region us-central1` +- If traffic is not routed to the new revision, update the service to send all traffic to the new revision + - `gcloud run services update-traffic index-generator --to-latest --region us-central1` + +[cloud-run-roles-request]: https://github.com/nodejs/build/issues/3774 +[gcloud-auth]: https://cloud.google.com/sdk/gcloud/reference/auth +[gcloud-auth-docker]: https://cloud.google.com/sdk/gcloud/reference/auth/configure-docker +[install-docker-desktop]: https://www.docker.com/products/docker-desktop +[install-gcloud-cli]: https://cloud.google.com/sdk/docs/install +[install-nodejs]: https://nodejs.org/en/learn/getting-started/how-to-install-nodejs diff --git a/ansible/roles/metrics/files/index-generator/index-generator.js b/ansible/roles/metrics/files/index-generator/index-generator.js index f168124da..412acd3d8 100644 --- a/ansible/roles/metrics/files/index-generator/index-generator.js +++ b/ansible/roles/metrics/files/index-generator/index-generator.js @@ -1,7 +1,5 @@ const { Storage } = require('@google-cloud/storage') -const storage = new Storage({ - keyFilename: "metrics-processor-service-key.json", - }); +const storage = new Storage(); const express = require('express') const app = express() app.use(express.json()) diff --git a/ansible/roles/metrics/files/process-cloudflare/README.md b/ansible/roles/metrics/files/process-cloudflare/README.md new file mode 100644 index 000000000..7152dff16 --- /dev/null +++ b/ansible/roles/metrics/files/process-cloudflare/README.md @@ -0,0 +1,32 @@ +# process-cloudflare + +## Prerequisites + +- Ensure that you have following roles on GCP: + - `Cloud Run Developer` + - `Service Account User` + - `Storage Object User` + - Example request: [nodejs/build#3774][cloud-run-roles-request] +- Install [Node.js][install-nodejs]. +- Install [gcloud CLI][install-gcloud-cli]. + - Authenticate using [`gcloud auth`][gcloud-auth]. +- Install [Docker Desktop][install-docker-desktop]. + - Authenticate using [`gcloud auth configure-docker`][gcloud-auth-docker]. + +## Setup + +- Build Dockerfile + - `docker buildx build --platform linux/amd64 -t gcr.io/nodejs-org/processcloudflare:latest .` +- Push Docker image to GCR + - `docker push gcr.io/nodejs-org/processcloudflare:latest` +- Deploy Cloud Run service + - `gcloud run deploy processlogs --image gcr.io/nodejs-org/processcloudflare:latest --service-account metrics-processor@nodejs-org.iam.gserviceaccount.com --no-allow-unauthenticated --region us-central1` +- If traffic is not routed to the new revision, update the service to send all traffic to the new revision + - `gcloud run services update-traffic processlogs --to-latest --region us-central1` + +[cloud-run-roles-request]: https://github.com/nodejs/build/issues/3774 +[gcloud-auth]: https://cloud.google.com/sdk/gcloud/reference/auth +[gcloud-auth-docker]: https://cloud.google.com/sdk/gcloud/reference/auth/configure-docker +[install-docker-desktop]: https://www.docker.com/products/docker-desktop +[install-gcloud-cli]: https://cloud.google.com/sdk/docs/install +[install-nodejs]: https://nodejs.org/en/learn/getting-started/how-to-install-nodejs diff --git a/ansible/roles/metrics/files/process-cloudflare/process-cloudflare.js b/ansible/roles/metrics/files/process-cloudflare/process-cloudflare.js index 3428205e7..a4bd017ea 100755 --- a/ansible/roles/metrics/files/process-cloudflare/process-cloudflare.js +++ b/ansible/roles/metrics/files/process-cloudflare/process-cloudflare.js @@ -141,9 +141,7 @@ async function processLogs (bucket, filename) { } async function createPipeline (bucket, filename, processedFile) { - const storage = new Storage({ - keyFilename: 'metrics-processor-service-key.json' - }) + const storage = new Storage() console.log('INSIDE CREATE PIPELINE') const readBucket = storage.bucket(bucket) diff --git a/ansible/roles/metrics/files/summaries/README.md b/ansible/roles/metrics/files/summaries/README.md new file mode 100644 index 000000000..da79fe02a --- /dev/null +++ b/ansible/roles/metrics/files/summaries/README.md @@ -0,0 +1,32 @@ +# summaries + +## Prerequisites + +- Ensure that you have following roles on GCP: + - `Cloud Run Developer` + - `Service Account User` + - `Storage Object User` + - Example request: [nodejs/build#3774][cloud-run-roles-request] +- Install [Node.js][install-nodejs]. +- Install [gcloud CLI][install-gcloud-cli]. + - Authenticate using [`gcloud auth`][gcloud-auth]. +- Install [Docker Desktop][install-docker-desktop]. + - Authenticate using [`gcloud auth configure-docker`][gcloud-auth-docker]. + +## Setup + +- Build Dockerfile + - `docker buildx build --platform linux/amd64 -t gcr.io/nodejs-org/producesummaries:latest .` +- Push Docker image to GCR + - `docker push gcr.io/nodejs-org/producesummaries:latest` +- Deploy Cloud Run service + - `gcloud run deploy produce-summaries --image gcr.io/nodejs-org/producesummaries:latest --service-account metrics-processor@nodejs-org.iam.gserviceaccount.com --no-allow-unauthenticated --region us-central1` +- If traffic is not routed to the new revision, update the service to send all traffic to the new revision + - `gcloud run services update-traffic produce-summaries --to-latest --region us-central1` + +[cloud-run-roles-request]: https://github.com/nodejs/build/issues/3774 +[gcloud-auth]: https://cloud.google.com/sdk/gcloud/reference/auth +[gcloud-auth-docker]: https://cloud.google.com/sdk/gcloud/reference/auth/configure-docker +[install-docker-desktop]: https://www.docker.com/products/docker-desktop +[install-gcloud-cli]: https://cloud.google.com/sdk/docs/install +[install-nodejs]: https://nodejs.org/en/learn/getting-started/how-to-install-nodejs diff --git a/ansible/roles/metrics/files/summaries/summaries.js b/ansible/roles/metrics/files/summaries/summaries.js index 325da2491..df7ff6a2d 100755 --- a/ansible/roles/metrics/files/summaries/summaries.js +++ b/ansible/roles/metrics/files/summaries/summaries.js @@ -64,9 +64,7 @@ function summary (chunk) { } async function collectData (date) { - const storage = new Storage({ - keyFilename: "metrics-processor-service-key.json", - }) + const storage = new Storage() const filePrefix = date.toString().concat('/') console.log(filePrefix) const [files] = await storage.bucket('processed-logs-nodejs').getFiles({ prefix: `${filePrefix}`}) @@ -84,9 +82,7 @@ async function collectData (date) { } async function produceSummaries (date) { - const storage = new Storage({ - keyFilename: "metrics-processor-service-key.json", - }) + const storage = new Storage() await collectData(date) prepare()