Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add cli functionality to dataproc quickstart #2047

Merged
merged 7 commits into from
Feb 3, 2020
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Changed quickstart to be an executable program
  • Loading branch information
bradmiro committed Jan 30, 2020
commit bc4865870490d13359198989fea0c0120da8d7c0
15 changes: 14 additions & 1 deletion dataproc/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,10 @@
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.source>1.8</maven.compiler.source>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<dataproc.project.id/>
<dataproc.region/>
<dataproc.cluster.name/>
<dataproc.job.file.path/>
</properties>

<build>
Expand All @@ -34,6 +38,16 @@
</plugin>
</plugins>
</pluginManagement>
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<version>1.6.0</version>
<configuration>
<mainClass>Quickstart</mainClass>
</configuration>
</plugin>
</plugins>
</build>

<dependencyManagement>
Expand Down Expand Up @@ -65,5 +79,4 @@
<scope>test</scope>
</dependency>
</dependencies>

</project>
50 changes: 36 additions & 14 deletions dataproc/src/main/java/Quickstart.java
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,17 @@
*/

// [START dataproc_quickstart]
/* This quickstart sample walks a user through creating a Cloud Dataproc
* cluster, submitting a PySpark job from Google Cloud Storage to the
* cluster, reading the output of the job and deleting the cluster, all
* using the Java client library.
*
* Usage:
* mvn clean package -DskipTests
*
* mvn exec:java -Dexec.args="<PROJECT_ID> <REGION> <CLUSTER_NAME> <GCS_JOB_FILE_PATH>"
*/

import com.google.api.gax.longrunning.OperationFuture;
import com.google.cloud.dataproc.v1.Cluster;
import com.google.cloud.dataproc.v1.ClusterConfig;
Expand Down Expand Up @@ -60,15 +71,6 @@ public static Job waitForJobCompletion(
}
}

public static void quickstart() throws IOException, InterruptedException {
// TODO(developer): Replace these variables before running the sample.
String projectId = "your-project-id";
String region = "your-project-region";
String clusterName = "your-cluster-name";
String jobFilePath = "your-job-file-path";
quickstart(projectId, region, clusterName, jobFilePath);
}

public static void quickstart(
String projectId, String region, String clusterName, String jobFilePath)
throws IOException, InterruptedException {
Expand All @@ -82,9 +84,12 @@ public static void quickstart(
JobControllerSettings jobControllerSettings =
JobControllerSettings.newBuilder().setEndpoint(myEndpoint).build();

// Create both a cluster controller client and job controller client with the configured
// settings. The client only needs to be created once and can be reused for multiple requests.
// Using a try-with-resources closes the client, but this can also be done manually with
// Create both a cluster controller client and job controller client with the
// configured
// settings. The client only needs to be created once and can be reused for
// multiple requests.
// Using a try-with-resources closes the client, but this can also be done
// manually with
// the .close() method.
try (ClusterControllerClient clusterControllerClient =
ClusterControllerClient.create(clusterControllerSettings);
Expand Down Expand Up @@ -114,7 +119,8 @@ public static void quickstart(
OperationFuture<Cluster, ClusterOperationMetadata> createClusterAsyncRequest =
clusterControllerClient.createClusterAsync(projectId, region, cluster);
Cluster response = createClusterAsyncRequest.get();
System.out.printf("Cluster created successfully: %s", response.getClusterName());
System.out.println(
String.format("Cluster created successfully: %s", response.getClusterName()));

// Configure the settings for our job.
JobPlacement jobPlacement = JobPlacement.newBuilder().setClusterName(clusterName).build();
Expand All @@ -133,7 +139,7 @@ public static void quickstart(
int timeout = 10;
try {
Job jobInfo = finishedJobFuture.get(timeout, TimeUnit.MINUTES);
System.out.printf("Job %s finished successfully.", jobId);
System.out.println(String.format("Job %s finished successfully.", jobId));

// Cloud Dataproc job output gets saved to a GCS bucket allocated to it.
Cluster clusterInfo = clusterControllerClient.getCluster(projectId, region, clusterName);
Expand Down Expand Up @@ -163,5 +169,21 @@ public static void quickstart(
System.err.println(String.format("Error executing quickstart: %s ", e.getMessage()));
}
}

public static void main(String... args) throws IOException, InterruptedException {
if (args.length != 4) {
System.err.println(
"Insufficient number of parameters provided. Please make sure a "
+ "PROJECT_ID, REGION, CLUSTER_NAME and JOB_FILE_PATH are provided, in this order.");
return;
}

String projectId = args[0];
String region = args[1];
String clusterName = args[2];
String jobFilePath = args[3];

quickstart(projectId, region, clusterName, jobFilePath);
}
}
// [END dataproc_quickstart]
4 changes: 4 additions & 0 deletions dataproc/src/test/java/QuickstartTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,11 @@ public void setUp() {

@Test
public void quickstartTest() throws IOException, InterruptedException {
<<<<<<< HEAD
Quickstart.main(PROJECT_ID, REGION, CLUSTER_NAME, JOB_FILE_PATH);
=======
Quickstart.quickstart(PROJECT_ID, REGION, CLUSTER_NAME, JOB_FILE_PATH);
>>>>>>> f4816860c1d21ec5d2de8e50d8f31ce8775e5b74
String output = bout.toString();

assertThat(output, CoreMatchers.containsString("Cluster created successfully"));
Expand Down