From 37cdd4fb3895e07356086d3c6817bbdb94a51887 Mon Sep 17 00:00:00 2001 From: Eric Schmidt Date: Fri, 7 May 2021 14:39:37 -0700 Subject: [PATCH] samples: adds batch prediction samples for text (#119) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * samples: adds batch prediction samples for text * samples: added tests * 🦉 Updates from OwlBot * fix: lint * fix: lint * fix: docs test Co-authored-by: Owl Bot --- ...atch-prediction-job-text-classification.js | 93 +++++++++++++++++++ ...h-prediction-job-text-entity-extraction.js | 93 +++++++++++++++++++ ...-prediction-job-text-sentiment-analysis.js | 93 +++++++++++++++++++ ...prediction-job-text-classification.test.js | 74 +++++++++++++++ ...diction-job-text-entity-extraction.test.js | 74 +++++++++++++++ ...iction-job-text-sentiment-analysis.test.js | 74 +++++++++++++++ 6 files changed, 501 insertions(+) create mode 100644 ai-platform/snippets/create-batch-prediction-job-text-classification.js create mode 100644 ai-platform/snippets/create-batch-prediction-job-text-entity-extraction.js create mode 100644 ai-platform/snippets/create-batch-prediction-job-text-sentiment-analysis.js create mode 100644 ai-platform/snippets/test/create-batch-prediction-job-text-classification.test.js create mode 100644 ai-platform/snippets/test/create-batch-prediction-job-text-entity-extraction.test.js create mode 100644 ai-platform/snippets/test/create-batch-prediction-job-text-sentiment-analysis.test.js diff --git a/ai-platform/snippets/create-batch-prediction-job-text-classification.js b/ai-platform/snippets/create-batch-prediction-job-text-classification.js new file mode 100644 index 0000000000..faa35338f4 --- /dev/null +++ b/ai-platform/snippets/create-batch-prediction-job-text-classification.js @@ -0,0 +1,93 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +async function main( + batchPredictionDisplayName, + modelId, + gcsSourceUri, + gcsDestinationOutputUriPrefix, + project, + location = 'us-central1' +) { + // [START aiplatform_create_batch_prediction_job_text_classification] + /** + * TODO(developer): Uncomment these variables before running the sample.\ + * (Not necessary if passing values as arguments) + */ + + // const batchPredictionDisplayName = 'YOUR_BATCH_PREDICTION_DISPLAY_NAME'; + // const modelId = 'YOUR_MODEL_ID'; + // const gcsSourceUri = 'YOUR_GCS_SOURCE_URI'; + // const gcsDestinationOutputUriPrefix = 'YOUR_GCS_DEST_OUTPUT_URI_PREFIX'; + // eg. "gs:///destination_path" + // const project = 'YOUR_PROJECT_ID'; + // const location = 'YOUR_PROJECT_LOCATION'; + + // Imports the Google Cloud Job Service Client library + const {JobServiceClient} = require('@google-cloud/aiplatform').v1; + + // Specifies the location of the api endpoint + const clientOptions = { + apiEndpoint: 'us-central1-aiplatform.googleapis.com', + }; + + // Instantiates a client + const jobServiceClient = new JobServiceClient(clientOptions); + + async function createBatchPredictionJobTextClassification() { + // Configure the parent resource + const parent = `projects/${project}/locations/${location}`; + const modelName = `projects/${project}/locations/${location}/models/${modelId}`; + + const inputConfig = { + instancesFormat: 'jsonl', + gcsSource: {uris: [gcsSourceUri]}, + }; + const outputConfig = { + predictionsFormat: 'jsonl', + gcsDestination: {outputUriPrefix: gcsDestinationOutputUriPrefix}, + }; + const batchPredictionJob = { + displayName: batchPredictionDisplayName, + model: modelName, + inputConfig, + outputConfig, + }; + const request = { + parent, + batchPredictionJob, + }; + + // Create batch prediction job request + const [response] = await jobServiceClient.createBatchPredictionJob(request); + + console.log('Create batch prediction job text classification response'); + console.log(`Name : ${response.name}`); + console.log('Raw response:'); + console.log(JSON.stringify(response, null, 2)); + } + createBatchPredictionJobTextClassification(); + // [END aiplatform_create_batch_prediction_job_text_classification] +} + +process.on('unhandledRejection', err => { + console.error(err.message); + process.exitCode = 1; +}); + +main(...process.argv.slice(2)); diff --git a/ai-platform/snippets/create-batch-prediction-job-text-entity-extraction.js b/ai-platform/snippets/create-batch-prediction-job-text-entity-extraction.js new file mode 100644 index 0000000000..85c20e67da --- /dev/null +++ b/ai-platform/snippets/create-batch-prediction-job-text-entity-extraction.js @@ -0,0 +1,93 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +async function main( + batchPredictionDisplayName, + modelId, + gcsSourceUri, + gcsDestinationOutputUriPrefix, + project, + location = 'us-central1' +) { + // [START aiplatform_create_batch_prediction_job_text_entity_extraction] + /** + * TODO(developer): Uncomment these variables before running the sample.\ + * (Not necessary if passing values as arguments) + */ + + // const batchPredictionDisplayName = 'YOUR_BATCH_PREDICTION_DISPLAY_NAME'; + // const modelId = 'YOUR_MODEL_ID'; + // const gcsSourceUri = 'YOUR_GCS_SOURCE_URI'; + // const gcsDestinationOutputUriPrefix = 'YOUR_GCS_DEST_OUTPUT_URI_PREFIX'; + // eg. "gs:///destination_path" + // const project = 'YOUR_PROJECT_ID'; + // const location = 'YOUR_PROJECT_LOCATION'; + + // Imports the Google Cloud Job Service Client library + const {JobServiceClient} = require('@google-cloud/aiplatform').v1; + + // Specifies the location of the api endpoint + const clientOptions = { + apiEndpoint: 'us-central1-aiplatform.googleapis.com', + }; + + // Instantiates a client + const jobServiceClient = new JobServiceClient(clientOptions); + + async function createBatchPredictionJobTextEntityExtraction() { + // Configure the parent resource + const parent = `projects/${project}/locations/${location}`; + const modelName = `projects/${project}/locations/${location}/models/${modelId}`; + + const inputConfig = { + instancesFormat: 'jsonl', + gcsSource: {uris: [gcsSourceUri]}, + }; + const outputConfig = { + predictionsFormat: 'jsonl', + gcsDestination: {outputUriPrefix: gcsDestinationOutputUriPrefix}, + }; + const batchPredictionJob = { + displayName: batchPredictionDisplayName, + model: modelName, + inputConfig, + outputConfig, + }; + const request = { + parent, + batchPredictionJob, + }; + + // Create batch prediction job request + const [response] = await jobServiceClient.createBatchPredictionJob(request); + + console.log('Create batch prediction job text entity extraction response'); + console.log(`Name : ${response.name}`); + console.log('Raw response:'); + console.log(JSON.stringify(response, null, 2)); + } + createBatchPredictionJobTextEntityExtraction(); + // [END aiplatform_create_batch_prediction_job_text_entity_extraction] +} + +process.on('unhandledRejection', err => { + console.error(err.message); + process.exitCode = 1; +}); + +main(...process.argv.slice(2)); diff --git a/ai-platform/snippets/create-batch-prediction-job-text-sentiment-analysis.js b/ai-platform/snippets/create-batch-prediction-job-text-sentiment-analysis.js new file mode 100644 index 0000000000..269ec73444 --- /dev/null +++ b/ai-platform/snippets/create-batch-prediction-job-text-sentiment-analysis.js @@ -0,0 +1,93 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +async function main( + batchPredictionDisplayName, + modelId, + gcsSourceUri, + gcsDestinationOutputUriPrefix, + project, + location = 'us-central1' +) { + // [START aiplatform_create_batch_prediction_job_text_sentiment_analysis] + /** + * TODO(developer): Uncomment these variables before running the sample.\ + * (Not necessary if passing values as arguments) + */ + + // const batchPredictionDisplayName = 'YOUR_BATCH_PREDICTION_DISPLAY_NAME'; + // const modelId = 'YOUR_MODEL_ID'; + // const gcsSourceUri = 'YOUR_GCS_SOURCE_URI'; + // const gcsDestinationOutputUriPrefix = 'YOUR_GCS_DEST_OUTPUT_URI_PREFIX'; + // eg. "gs:///destination_path" + // const project = 'YOUR_PROJECT_ID'; + // const location = 'YOUR_PROJECT_LOCATION'; + + // Imports the Google Cloud Job Service Client library + const {JobServiceClient} = require('@google-cloud/aiplatform').v1; + + // Specifies the location of the api endpoint + const clientOptions = { + apiEndpoint: 'us-central1-aiplatform.googleapis.com', + }; + + // Instantiates a client + const jobServiceClient = new JobServiceClient(clientOptions); + + async function createBatchPredictionJobTextSentimentAnalysis() { + // Configure the parent resource + const parent = `projects/${project}/locations/${location}`; + const modelName = `projects/${project}/locations/${location}/models/${modelId}`; + + const inputConfig = { + instancesFormat: 'jsonl', + gcsSource: {uris: [gcsSourceUri]}, + }; + const outputConfig = { + predictionsFormat: 'jsonl', + gcsDestination: {outputUriPrefix: gcsDestinationOutputUriPrefix}, + }; + const batchPredictionJob = { + displayName: batchPredictionDisplayName, + model: modelName, + inputConfig, + outputConfig, + }; + const request = { + parent, + batchPredictionJob, + }; + + // Create batch prediction job request + const [response] = await jobServiceClient.createBatchPredictionJob(request); + + console.log('Create batch prediction job text sentiment analysis response'); + console.log(`Name : ${response.name}`); + console.log('Raw response:'); + console.log(JSON.stringify(response, null, 2)); + } + createBatchPredictionJobTextSentimentAnalysis(); + // [END aiplatform_create_batch_prediction_job_text_sentiment_analysis] +} + +process.on('unhandledRejection', err => { + console.error(err.message); + process.exitCode = 1; +}); + +main(...process.argv.slice(2)); diff --git a/ai-platform/snippets/test/create-batch-prediction-job-text-classification.test.js b/ai-platform/snippets/test/create-batch-prediction-job-text-classification.test.js new file mode 100644 index 0000000000..ac285ad4c4 --- /dev/null +++ b/ai-platform/snippets/test/create-batch-prediction-job-text-classification.test.js @@ -0,0 +1,74 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const {assert} = require('chai'); +const {after, describe, it} = require('mocha'); +const uuid = require('uuid').v4; +const cp = require('child_process'); +const execSync = cmd => cp.execSync(cmd, {encoding: 'utf-8'}); + +const aiplatform = require('@google-cloud/aiplatform'); +const clientOptions = { + apiEndpoint: 'us-central1-aiplatform.googleapis.com', +}; + +const jobServiceClient = new aiplatform.v1.JobServiceClient(clientOptions); + +const batchPredictionDisplayName = `temp_create_batch_prediction_text_classification_test${uuid()}`; +const modelId = '7827432074230366208'; +const gcsSourceUri = + 'gs://ucaip-samples-test-output/inputs/batch_predict_TCN/tcn_inputs.jsonl'; +const gcsDestinationOutputUriPrefix = 'gs://ucaip-samples-test-output/'; +const location = 'us-central1'; +const project = process.env.CAIP_PROJECT_ID; + +let batchPredictionJobId; + +describe('AI platform create batch prediction job text classification', () => { + it('should create a text classification batch prediction job', async () => { + const stdout = execSync( + `node ./create-batch-prediction-job-text-classification.js ${batchPredictionDisplayName} ${modelId} ${gcsSourceUri} ${gcsDestinationOutputUriPrefix} ${project} ${location}` + ); + assert.match( + stdout, + /Create batch prediction job text classification response/ + ); + batchPredictionJobId = stdout + .split('/locations/us-central1/batchPredictionJobs/')[1] + .split('\n')[0]; + }); + after('should cancel delete the batch prediction job', async () => { + const name = jobServiceClient.batchPredictionJobPath( + project, + location, + batchPredictionJobId + ); + + const cancelRequest = { + name, + }; + + jobServiceClient.cancelBatchPredictionJob(cancelRequest).then(() => { + const deleteRequest = { + name, + }; + + return jobServiceClient.deleteBatchPredictionJob(deleteRequest); + }); + }); +}); diff --git a/ai-platform/snippets/test/create-batch-prediction-job-text-entity-extraction.test.js b/ai-platform/snippets/test/create-batch-prediction-job-text-entity-extraction.test.js new file mode 100644 index 0000000000..bb630905bd --- /dev/null +++ b/ai-platform/snippets/test/create-batch-prediction-job-text-entity-extraction.test.js @@ -0,0 +1,74 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const {assert} = require('chai'); +const {after, describe, it} = require('mocha'); +const uuid = require('uuid').v4; +const cp = require('child_process'); +const execSync = cmd => cp.execSync(cmd, {encoding: 'utf-8'}); + +const aiplatform = require('@google-cloud/aiplatform'); +const clientOptions = { + apiEndpoint: 'us-central1-aiplatform.googleapis.com', +}; + +const jobServiceClient = new aiplatform.v1.JobServiceClient(clientOptions); + +const batchPredictionDisplayName = `temp_create_batch_prediction_text_entity_extraction_test${uuid()}`; +const modelId = '6305215400179138560'; +const gcsSourceUri = + 'gs://ucaip-samples-test-output/inputs/batch_predict_TEN/ten_inputs.jsonl'; +const gcsDestinationOutputUriPrefix = 'gs://ucaip-samples-test-output/'; +const location = 'us-central1'; +const project = process.env.CAIP_PROJECT_ID; + +let batchPredictionJobId; + +describe('AI platform create batch prediction job text entity extraction', () => { + it('should create a text entity extraction batch prediction job', async () => { + const stdout = execSync( + `node ./create-batch-prediction-job-text-entity-extraction.js ${batchPredictionDisplayName} ${modelId} ${gcsSourceUri} ${gcsDestinationOutputUriPrefix} ${project} ${location}` + ); + assert.match( + stdout, + /Create batch prediction job text entity extraction response/ + ); + batchPredictionJobId = stdout + .split('/locations/us-central1/batchPredictionJobs/')[1] + .split('\n')[0]; + }); + after('should cancel delete the batch prediction job', async () => { + const name = jobServiceClient.batchPredictionJobPath( + project, + location, + batchPredictionJobId + ); + + const cancelRequest = { + name, + }; + + jobServiceClient.cancelBatchPredictionJob(cancelRequest).then(() => { + const deleteRequest = { + name, + }; + + return jobServiceClient.deleteBatchPredictionJob(deleteRequest); + }); + }); +}); diff --git a/ai-platform/snippets/test/create-batch-prediction-job-text-sentiment-analysis.test.js b/ai-platform/snippets/test/create-batch-prediction-job-text-sentiment-analysis.test.js new file mode 100644 index 0000000000..c9d1832615 --- /dev/null +++ b/ai-platform/snippets/test/create-batch-prediction-job-text-sentiment-analysis.test.js @@ -0,0 +1,74 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const {assert} = require('chai'); +const {after, describe, it} = require('mocha'); +const uuid = require('uuid').v4; +const cp = require('child_process'); +const execSync = cmd => cp.execSync(cmd, {encoding: 'utf-8'}); + +const aiplatform = require('@google-cloud/aiplatform'); +const clientOptions = { + apiEndpoint: 'us-central1-aiplatform.googleapis.com', +}; + +const jobServiceClient = new aiplatform.v1.JobServiceClient(clientOptions); + +const batchPredictionDisplayName = `temp_create_batch_prediction_text_sentiment_analysis_test${uuid()}`; +const modelId = '4792568875336073216'; +const gcsSourceUri = + 'gs://ucaip-samples-test-output/inputs/batch_predict_TSN/tsn_inputs.jsonl'; +const gcsDestinationOutputUriPrefix = 'gs://ucaip-samples-test-output/'; +const location = 'us-central1'; +const project = process.env.CAIP_PROJECT_ID; + +let batchPredictionJobId; + +describe('AI platform create batch prediction job text sentiment analysis', () => { + it('should create a text sentiment analysis batch prediction job', async () => { + const stdout = execSync( + `node ./create-batch-prediction-job-text-sentiment-analysis.js ${batchPredictionDisplayName} ${modelId} ${gcsSourceUri} ${gcsDestinationOutputUriPrefix} ${project} ${location}` + ); + assert.match( + stdout, + /Create batch prediction job text sentiment analysis response/ + ); + batchPredictionJobId = stdout + .split('/locations/us-central1/batchPredictionJobs/')[1] + .split('\n')[0]; + }); + after('should cancel delete the batch prediction job', async () => { + const name = jobServiceClient.batchPredictionJobPath( + project, + location, + batchPredictionJobId + ); + + const cancelRequest = { + name, + }; + + jobServiceClient.cancelBatchPredictionJob(cancelRequest).then(() => { + const deleteRequest = { + name, + }; + + return jobServiceClient.deleteBatchPredictionJob(deleteRequest); + }); + }); +});