migrate code from googleapis/nodejs-dataproc

.github/workflows/dataproc.yaml

@@ -0,0 +1,68 @@
+name: dataproc
+on:
+  push:
+    branches:
+    - main
+    paths:
+    - 'dataproc/**'
+  pull_request:
+    paths:
+    - 'dataproc/**'
+  pull_request_target:
+    types: [labeled]
+    paths:
+    - 'dataproc/**'
+  schedule:
+  - cron:  '0 0 * * 0'
+jobs:
+  test:
+    if: ${{ github.event.action != 'labeled' || github.event.label.name == 'actions:force-run' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    permissions:
+      contents: 'write'
+      pull-requests: 'write'
+      id-token: 'write'
+    steps:
+    - uses: actions/[email protected]
+      with:
+        ref: ${{github.event.pull_request.head.sha}}
+    - uses: 'google-github-actions/[email protected]'
+      with:
+        workload_identity_provider: 'projects/1046198160504/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-provider'
+        service_account: '[email protected]'
+        create_credentials_file: 'true'
+        access_token_lifetime: 600s
+    - uses: actions/[email protected]
+      with:
+        node-version: 16
+    - run: npm install
+      working-directory: dataproc
+    - run: npm test
+      working-directory: dataproc
+      env:
+        MOCHA_REPORTER_SUITENAME: dataproc
+        MOCHA_REPORTER_OUTPUT: dataproc_sponge_log.xml
+        MOCHA_REPORTER: xunit
+    - if: ${{ github.event.action == 'labeled' && github.event.label.name == 'actions:force-run' }}
+      uses: actions/github-script@v6
+      with:
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        script: |
+          try {
+            await github.rest.issues.removeLabel({
+              name: 'actions:force-run',
+              owner: 'GoogleCloudPlatform',
+              repo: 'nodejs-docs-samples',
+              issue_number: context.payload.pull_request.number
+            });
+          } catch (e) {
+            if (!e.message.includes('Label does not exist')) {
+              throw e;
+            }
+          }
+    - if: ${{ github.event_name == 'schedule'}}
+      run: |
+        curl https://github.com/googleapis/repo-automation-bots/releases/download/flakybot-1.1.0/flakybot -o flakybot -s -L
+        chmod +x ./flakybot
+        ./flakybot --repo GoogleCloudPlatform/nodejs-docs-samples --commit_hash ${{github.sha}} --build_url https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}

.github/workflows/workflows.json

@@ -30,6 +30,7 @@
   "datacatalog/cloud-client",
   "datacatalog/snippets",
   "datalabeling",
+  "dataproc",
   "datastore/functions",
   "datacatalog/quickstart",
   "dialogflow",

dataproc/.eslintrc.yml

@@ -0,0 +1,3 @@
+---
+rules:
+  no-console: off

dataproc/createCluster.js

@@ -0,0 +1,74 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Create a Dataproc cluster with the Node.js Client Library.
+
+// sample-metadata:
+//   title: Create Cluster
+//   usage: node createCluster.js <PROJECT_ID> <REGION> <CLUSTER_NAME>
+
+/*eslint no-warning-comments: [0, { "terms": ["todo", "fixme"], "location": "anywhere" }]*/
+
+function main(
+  projectId = 'YOUR_PROJECT_ID',
+  region = 'YOUR_CLUSTER_REGION',
+  clusterName = 'YOUR_CLUSTER_NAME'
+) {
+  // [START dataproc_create_cluster]
+  const dataproc = require('@google-cloud/dataproc');
+
+  // TODO(developer): Uncomment and set the following variables
+  // projectId = 'YOUR_PROJECT_ID'
+  // region = 'YOUR_CLUSTER_REGION'
+  // clusterName = 'YOUR_CLUSTER_NAME'
+
+  // Create a client with the endpoint set to the desired cluster region
+  const client = new dataproc.v1.ClusterControllerClient({
+    apiEndpoint: `${region}-dataproc.googleapis.com`,
+    projectId: projectId,
+  });
+
+  async function createCluster() {
+    // Create the cluster config
+    const request = {
+      projectId: projectId,
+      region: region,
+      cluster: {
+        clusterName: clusterName,
+        config: {
+          masterConfig: {
+            numInstances: 1,
+            machineTypeUri: 'n1-standard-2',
+          },
+          workerConfig: {
+            numInstances: 2,
+            machineTypeUri: 'n1-standard-2',
+          },
+        },
+      },
+    };
+
+    // Create the cluster
+    const [operation] = await client.createCluster(request);
+    const [response] = await operation.promise();
+
+    // Output a success message
+    console.log(`Cluster created successfully: ${response.clusterName}`);
+    // [END dataproc_create_cluster]
+  }
+
+  createCluster();
+}
+
+main(...process.argv.slice(2));

dataproc/instantiateInlineWorkflowTemplate.js

@@ -0,0 +1,97 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This sample instantiates an inline workflow template using the client
+// library for Dataproc.
+
+// sample-metadata:
+//   title: Instantiate an inline workflow template
+//   usage: node instantiateInlineWorkflowTemplate.js <PROJECT_ID> <REGION>
+
+/*eslint no-warning-comments: [0, { "terms": ["todo", "fixme"], "location": "anywhere" }]*/
+
+async function main(projectId = 'YOUR_PROJECT_ID', region = 'YOUR_REGION') {
+  // [START dataproc_instantiate_inline_workflow_template]
+  const dataproc = require('@google-cloud/dataproc');
+
+  // TODO(developer): Uncomment and set the following variables
+  // projectId = 'YOUR_PROJECT_ID'
+  // region = 'YOUR_REGION'
+
+  // Create a client with the endpoint set to the desired region
+  const client = new dataproc.v1.WorkflowTemplateServiceClient({
+    apiEndpoint: `${region}-dataproc.googleapis.com`,
+    projectId: projectId,
+  });
+
+  async function instantiateInlineWorkflowTemplate() {
+    // Create the formatted parent.
+    const parent = client.regionPath(projectId, region);
+
+    // Create the template
+    const template = {
+      jobs: [
+        {
+          hadoopJob: {
+            mainJarFileUri:
+              'file:///usr/lib/hadoop-mapreduce/hadoop-mapreduce-examples.jar',
+            args: ['teragen', '1000', 'hdfs:///gen/'],
+          },
+          stepId: 'teragen',
+        },
+        {
+          hadoopJob: {
+            mainJarFileUri:
+              'file:///usr/lib/hadoop-mapreduce/hadoop-mapreduce-examples.jar',
+            args: ['terasort', 'hdfs:///gen/', 'hdfs:///sort/'],
+          },
+          stepId: 'terasort',
+          prerequisiteStepIds: ['teragen'],
+        },
+      ],
+      placement: {
+        managedCluster: {
+          clusterName: 'my-managed-cluster',
+          config: {
+            gceClusterConfig: {
+              // Leave 'zoneUri' empty for 'Auto Zone Placement'
+              // zoneUri: ''
+              zoneUri: 'us-central1-a',
+            },
+          },
+        },
+      },
+    };
+
+    const request = {
+      parent: parent,
+      template: template,
+    };
+
+    // Submit the request to instantiate the workflow from an inline template.
+    const [operation] = await client.instantiateInlineWorkflowTemplate(request);
+    await operation.promise();
+
+    // Output a success message
+    console.log('Workflow ran successfully.');
+    // [END dataproc_instantiate_inline_workflow_template]
+  }
+
+  instantiateInlineWorkflowTemplate();
+}
+
+main(...process.argv.slice(2)).catch(err => {
+  console.error(err);
+  process.exitCode = 1;
+});

dataproc/package.json

@@ -0,0 +1,25 @@
+{
+  "name": "nodejs-docs-samples-dataproc",
+  "license": "Apache-2.0",
+  "author": "Google Inc.",
+  "files": [
+    "*.js"
+  ],
+  "engines": {
+    "node": ">=12.0.0"
+  },
+  "repository": "googleapis/nodejs-dataproc",
+  "private": true,
+  "scripts": {
+    "test": "mocha system-test --timeout 600000"
+  },
+  "dependencies": {
+    "@google-cloud/dataproc": "^4.1.1",
+    "@google-cloud/storage": "^6.0.0"
+  },
+  "devDependencies": {
+    "chai": "^4.2.0",
+    "mocha": "^8.0.0",
+    "uuid": "^9.0.0"
+  }
+}