Created EmptyQuickstart command

pinot-tools/src/main/java/org/apache/pinot/tools/EmptyQuickstart.java

@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.tools;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import org.apache.pinot.tools.admin.PinotAdministrator;
+import org.apache.pinot.tools.admin.command.QuickstartRunner;
+
+
+public class EmptyQuickstart extends QuickStartBase {
+  @Override
+  public List<String> types() {
+    return Arrays.asList("EMPTY", "DEFAULT");
+  }
+
+  public String getAuthToken() {
+    return null;
+  }
+
+  public Map<String, Object> getConfigOverrides() {
+    return null;
+  }
+
+  public void execute()
+      throws Exception {
+    File quickstartTmpDir = new File(_dataDir.getAbsolutePath());
+    File dataDir = new File(quickstartTmpDir, "rawdata");
+    if (!dataDir.mkdirs()) {
+      printStatus(Quickstart.Color.YELLOW, "***** Bootstrapping data from existing directory *****");
+    } else {
+      printStatus(Quickstart.Color.YELLOW, "***** Creating new data directory for fresh installation *****");
+    }
+
+    QuickstartRunner runner =
+        new QuickstartRunner(new ArrayList<>(), 1, 1, 1, 0,
+            dataDir, true, getAuthToken(), getConfigOverrides(), _zkExternalAddress, false);
+
+    if (_zkExternalAddress != null) {
+      printStatus(Quickstart.Color.CYAN, "***** Starting controller, broker and server *****");
+    } else {
+      printStatus(Quickstart.Color.CYAN, "***** Starting Zookeeper, controller, broker and server *****");
+    }
+
+    runner.startAll();
+
+    Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+      try {
+        printStatus(Quickstart.Color.GREEN, "***** Shutting down empty quick start *****");
+        runner.stop();
+      } catch (Exception e) {
+        e.printStackTrace();
+      }
+    }));
+
+    waitForBootstrapToComplete(runner);
+
+    printStatus(Quickstart.Color.YELLOW, "***** Empty quickstart setup complete *****");
+    printStatus(Quickstart.Color.GREEN,
+        "You can always go to http://localhost:9000 to play around in the query console");
+  }
+
+  public static void main(String[] args)
+      throws Exception {
+    List<String> arguments = new ArrayList<>();
+    arguments.addAll(Arrays.asList("QuickStart", "-type", "EMPTY"));
+    arguments.addAll(Arrays.asList(args));
+    PinotAdministrator.main(arguments.toArray(new String[arguments.size()]));
+  }
+}

pinot-tools/src/main/java/org/apache/pinot/tools/GenericQuickstart.java

@@ -86,7 +86,7 @@ public void execute()
     File tempDir = new File(FileUtils.getTempDirectory(), String.valueOf(System.currentTimeMillis()));
     Preconditions.checkState(tempDir.mkdirs());
     QuickstartTableRequest request = new QuickstartTableRequest(_tableDirectory.getAbsolutePath());
-    final QuickstartRunner runner = new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1, tempDir);
+    final QuickstartRunner runner = new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1, 0, tempDir);
 
     printStatus(Color.CYAN, "***** Starting Kafka *****");
     startKafka();

pinot-tools/src/main/java/org/apache/pinot/tools/GitHubEventsQuickstart.java

@@ -84,7 +84,7 @@ public void execute(String personalAccessToken)
     File tempDir = new File(FileUtils.getTempDirectory(), String.valueOf(System.currentTimeMillis()));
     Preconditions.checkState(tempDir.mkdirs());
     QuickstartTableRequest request = new QuickstartTableRequest(quickStartDataDir.getAbsolutePath());
-    final QuickstartRunner runner = new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1, tempDir);
+    final QuickstartRunner runner = new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1, 0, tempDir);
 
     printStatus(Color.CYAN, "***** Starting Kafka *****");
     startKafka();

pinot-tools/src/main/java/org/apache/pinot/tools/HybridQuickstart.java

@@ -41,7 +41,6 @@
 import org.apache.pinot.tools.utils.KafkaStarterUtils;
 
 import static org.apache.pinot.tools.Quickstart.prettyPrintResponse;
-import static org.apache.pinot.tools.Quickstart.printStatus;
 
 
 public class HybridQuickstart extends QuickStartBase {
@@ -110,12 +109,13 @@ private void startKafka() {
 
   public void execute()
       throws Exception {
-    File quickstartTmpDir = new File(_tmpDir, String.valueOf(System.currentTimeMillis()));
+    File quickstartTmpDir = new File(_dataDir, String.valueOf(System.currentTimeMillis()));
     File baseDir = new File(quickstartTmpDir, "airlineStats");
     File dataDir = new File(baseDir, "data");
     Preconditions.checkState(dataDir.mkdirs());
     QuickstartTableRequest bootstrapTableRequest = prepareTableRequest(baseDir);
-    final QuickstartRunner runner = new QuickstartRunner(Lists.newArrayList(bootstrapTableRequest), 1, 1, 1, dataDir);
+    final QuickstartRunner runner = new QuickstartRunner(Lists.newArrayList(bootstrapTableRequest),
+        1, 1, 1, 0, dataDir);
     printStatus(Color.YELLOW, "***** Starting Kafka  *****");
     startKafka();
     printStatus(Color.YELLOW, "***** Starting airline data stream and publishing to Kafka *****");

pinot-tools/src/main/java/org/apache/pinot/tools/JoinQuickStart.java

@@ -31,7 +31,6 @@
 import org.apache.pinot.tools.admin.command.QuickstartRunner;
 
 import static org.apache.pinot.tools.Quickstart.prettyPrintResponse;
-import static org.apache.pinot.tools.Quickstart.printStatus;
 
 
 public class JoinQuickStart extends QuickStartBase {
@@ -43,7 +42,7 @@ public List<String> types() {
 
   public void execute()
       throws Exception {
-    File quickstartTmpDir = new File(_tmpDir, String.valueOf(System.currentTimeMillis()));
+    File quickstartTmpDir = new File(_dataDir, String.valueOf(System.currentTimeMillis()));
 
     // Baseball stat table
     File baseBallStatsBaseDir = new File(quickstartTmpDir, "baseballStats");
@@ -81,7 +80,7 @@ public void execute()
 
     File tempDir = new File(quickstartTmpDir, "tmp");
     FileUtils.forceMkdir(tempDir);
-    QuickstartRunner runner = new QuickstartRunner(Lists.newArrayList(request, dimTableRequest), 1, 1, 3, tempDir);
+    QuickstartRunner runner = new QuickstartRunner(Lists.newArrayList(request, dimTableRequest), 1, 1, 3, 0, tempDir);
 
     printStatus(Quickstart.Color.CYAN, "***** Starting Zookeeper, controller, broker and server *****");
     runner.startAll();
@@ -97,9 +96,7 @@ public void execute()
     printStatus(Quickstart.Color.CYAN, "***** Bootstrap baseballStats table *****");
     runner.bootstrapTable();
 
-    printStatus(Quickstart.Color.CYAN,
-        "***** Waiting for 5 seconds for the server to fetch the assigned segment *****");
-    Thread.sleep(5000);
+    waitForBootstrapToComplete(null);
 
     printStatus(Quickstart.Color.YELLOW, "***** Offline quickstart setup complete *****");
 

pinot-tools/src/main/java/org/apache/pinot/tools/JsonIndexQuickStart.java

@@ -31,7 +31,6 @@
 import org.apache.pinot.tools.admin.command.QuickstartRunner;
 
 import static org.apache.pinot.tools.Quickstart.prettyPrintResponse;
-import static org.apache.pinot.tools.Quickstart.printStatus;
 
 
 public class JsonIndexQuickStart extends QuickStartBase {
@@ -43,7 +42,7 @@ public List<String> types() {
 
   public void execute()
       throws Exception {
-    File quickstartTmpDir = new File(_tmpDir, String.valueOf(System.currentTimeMillis()));
+    File quickstartTmpDir = new File(_dataDir, String.valueOf(System.currentTimeMillis()));
     File baseDir = new File(quickstartTmpDir, "githubEvents");
     File dataDir = new File(quickstartTmpDir, "rawdata");
     Preconditions.checkState(dataDir.mkdirs());
@@ -64,7 +63,7 @@ public void execute()
     FileUtils.copyURLToFile(resource, ingestionJobSpecFile);
 
     QuickstartTableRequest request = new QuickstartTableRequest(baseDir.getAbsolutePath());
-    final QuickstartRunner runner = new QuickstartRunner(Collections.singletonList(request), 1, 1, 1, dataDir);
+    final QuickstartRunner runner = new QuickstartRunner(Collections.singletonList(request), 1, 1, 1, 0, dataDir);
 
     printStatus(Color.CYAN, "***** Starting Zookeeper, controller, broker and server *****");
     runner.startAll();
@@ -80,8 +79,7 @@ public void execute()
     printStatus(Color.CYAN, "***** Bootstrap githubEvents table *****");
     runner.bootstrapTable();
 
-    printStatus(Color.CYAN, "***** Waiting for 5 seconds for the server to fetch the assigned segment *****");
-    Thread.sleep(5000);
+    waitForBootstrapToComplete(null);
 
     printStatus(Color.YELLOW, "***** Offline json-index quickstart setup complete *****");
 

pinot-tools/src/main/java/org/apache/pinot/tools/OfflineComplexTypeHandlingQuickStart.java

@@ -31,7 +31,6 @@
 import org.apache.pinot.tools.admin.command.QuickstartRunner;
 
 import static org.apache.pinot.tools.Quickstart.prettyPrintResponse;
-import static org.apache.pinot.tools.Quickstart.printStatus;
 
 
 public class OfflineComplexTypeHandlingQuickStart extends QuickStartBase {
@@ -42,7 +41,7 @@ public List<String> types() {
 
   public void execute()
       throws Exception {
-    File quickstartTmpDir = new File(_tmpDir, String.valueOf(System.currentTimeMillis()));
+    File quickstartTmpDir = new File(_dataDir, String.valueOf(System.currentTimeMillis()));
     File baseDir = new File(quickstartTmpDir, "githubEvents");
     File dataDir = new File(quickstartTmpDir, "rawdata");
     Preconditions.checkState(dataDir.mkdirs());
@@ -66,7 +65,7 @@ public void execute()
     FileUtils.copyURLToFile(resource, ingestionJobSpecFile);
 
     QuickstartTableRequest request = new QuickstartTableRequest(baseDir.getAbsolutePath());
-    final QuickstartRunner runner = new QuickstartRunner(Collections.singletonList(request), 1, 1, 1, dataDir);
+    final QuickstartRunner runner = new QuickstartRunner(Collections.singletonList(request), 1, 1, 1, 0, dataDir);
 
     printStatus(Color.CYAN, "***** Starting Zookeeper, controller, broker and server *****");
     runner.startAll();
@@ -82,8 +81,7 @@ public void execute()
     printStatus(Color.CYAN, "***** Bootstrap githubEvents table *****");
     runner.bootstrapTable();
 
-    printStatus(Color.CYAN, "***** Waiting for 5 seconds for the server to fetch the assigned segment *****");
-    Thread.sleep(5000);
+    waitForBootstrapToComplete(null);
 
     printStatus(Color.YELLOW, "***** Offline complex-type-handling quickstart setup complete *****");
 

pinot-tools/src/main/java/org/apache/pinot/tools/PartialUpsertQuickStart.java

@@ -66,7 +66,7 @@ public void execute()
     FileUtils.copyURLToFile(resource, tableConfigFile);
 
     QuickstartTableRequest request = new QuickstartTableRequest(bootstrapTableDir.getAbsolutePath());
-    final QuickstartRunner runner = new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1, dataDir);
+    final QuickstartRunner runner = new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1, 0, dataDir);
 
     printStatus(Color.CYAN, "***** Starting Kafka *****");
     final ZkStarter.ZookeeperInstance zookeeperInstance = ZkStarter.startLocalZkServer();

pinot-tools/src/main/java/org/apache/pinot/tools/QuickStartBase.java

@@ -21,18 +21,36 @@
 import java.io.File;
 import java.util.List;
 import org.apache.commons.io.FileUtils;
+import org.apache.pinot.tools.admin.command.QuickstartRunner;
 
 
 public abstract class QuickStartBase {
-  protected File _tmpDir = FileUtils.getTempDirectory();
+  protected File _dataDir = FileUtils.getTempDirectory();
+  protected String _zkExternalAddress;
 
-  public QuickStartBase setTmpDir(String tmpDir) {
-    _tmpDir = new File(tmpDir);
+  public QuickStartBase setDataDir(String dataDir) {
+    _dataDir = new File(dataDir);
+    return this;
+  }
+
+  public QuickStartBase setZkExternalAddress(String zkExternalAddress) {
+    _zkExternalAddress = zkExternalAddress;
     return this;
   }
 
   public abstract List<String> types();
 
+  protected void waitForBootstrapToComplete(QuickstartRunner runner)
+      throws Exception {
+    QuickStartBase.printStatus(Quickstart.Color.CYAN,
+        "***** Waiting for 5 seconds for the server to fetch the assigned segment *****");
+    Thread.sleep(5000);
+  }
+
+  public static void printStatus(Quickstart.Color color, String message) {
+    System.out.println(color.getCode() + message + Quickstart.Color.RESET.getCode());
+  }
+
   public abstract void execute()
       throws Exception;
 }

pinot-tools/src/main/java/org/apache/pinot/tools/Quickstart.java

@@ -44,7 +44,11 @@ public List<String> types() {
   public enum Color {
     RESET("\u001B[0m"), GREEN("\u001B[32m"), YELLOW("\u001B[33m"), CYAN("\u001B[36m");
 
-    private String _code;
+    private final String _code;
+
+    public String getCode() {
+      return _code;
+    }
 
     Color(String code) {
       _code = code;
@@ -67,16 +71,6 @@ public Map<String, Object> getConfigOverrides() {
     return null;
   }
 
-  protected void waitForBootstrapToComplete(QuickstartRunner runner)
-      throws Exception {
-    printStatus(Color.CYAN, "***** Waiting for 5 seconds for the server to fetch the assigned segment *****");
-    Thread.sleep(5000);
-  }
-
-  public static void printStatus(Color color, String message) {
-    System.out.println(color._code + message + Color.RESET._code);
-  }
-
   public static String prettyPrintResponse(JsonNode response) {
     StringBuilder responseBuilder = new StringBuilder();
 
@@ -163,7 +157,7 @@ public static String prettyPrintResponse(JsonNode response) {
 
   public void execute()
       throws Exception {
-    File quickstartTmpDir = new File(_tmpDir, String.valueOf(System.currentTimeMillis()));
+    File quickstartTmpDir = new File(_dataDir, String.valueOf(System.currentTimeMillis()));
     File baseDir = new File(quickstartTmpDir, "baseballStats");
     File dataDir = new File(baseDir, "rawdata");
     Preconditions.checkState(dataDir.mkdirs());
@@ -190,8 +184,9 @@ public void execute()
 
     QuickstartTableRequest request = new QuickstartTableRequest(baseDir.getAbsolutePath());
     QuickstartRunner runner =
-        new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1, getNumMinions(), dataDir, true, getAuthToken(),
-            getConfigOverrides());
+        new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1,
+            getNumMinions(), dataDir, true, getAuthToken(),
+            getConfigOverrides(), null, true);
 
     printStatus(Color.CYAN, "***** Starting Zookeeper, controller, broker and server *****");
     runner.startAll();

pinot-tools/src/main/java/org/apache/pinot/tools/RealtimeComplexTypeHandlingQuickStart.java

@@ -36,7 +36,6 @@
 import org.apache.pinot.tools.utils.KafkaStarterUtils;
 
 import static org.apache.pinot.tools.Quickstart.prettyPrintResponse;
-import static org.apache.pinot.tools.Quickstart.printStatus;
 
 
 public class RealtimeComplexTypeHandlingQuickStart extends QuickStartBase {
@@ -57,7 +56,7 @@ public static void main(String[] args)
 
   public void execute()
       throws Exception {
-    File quickstartTmpDir = new File(_tmpDir, String.valueOf(System.currentTimeMillis()));
+    File quickstartTmpDir = new File(_dataDir, String.valueOf(System.currentTimeMillis()));
     File baseDir = new File(quickstartTmpDir, "meetupRsvp");
     File dataDir = new File(baseDir, "data");
     Preconditions.checkState(dataDir.mkdirs());
@@ -75,7 +74,7 @@ public void execute()
     FileUtils.copyURLToFile(resource, tableConfigFile);
 
     QuickstartTableRequest request = new QuickstartTableRequest(baseDir.getAbsolutePath());
-    QuickstartRunner runner = new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1, dataDir);
+    QuickstartRunner runner = new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1, 0, dataDir);
 
     printStatus(Color.CYAN, "***** Starting Kafka *****");
     ZkStarter.ZookeeperInstance zookeeperInstance = ZkStarter.startLocalZkServer();

pinot-tools/src/main/java/org/apache/pinot/tools/RealtimeJsonIndexQuickStart.java

@@ -36,7 +36,6 @@
 import org.apache.pinot.tools.utils.KafkaStarterUtils;
 
 import static org.apache.pinot.tools.Quickstart.prettyPrintResponse;
-import static org.apache.pinot.tools.Quickstart.printStatus;
 
 
 public class RealtimeJsonIndexQuickStart extends QuickStartBase {
@@ -57,7 +56,7 @@ public static void main(String[] args)
 
   public void execute()
       throws Exception {
-    File quickstartTmpDir = new File(_tmpDir, String.valueOf(System.currentTimeMillis()));
+    File quickstartTmpDir = new File(_dataDir, String.valueOf(System.currentTimeMillis()));
     File baseDir = new File(quickstartTmpDir, "meetupRsvp");
     File dataDir = new File(baseDir, "data");
     Preconditions.checkState(dataDir.mkdirs());
@@ -74,7 +73,7 @@ public void execute()
     FileUtils.copyURLToFile(resource, tableConfigFile);
 
     QuickstartTableRequest request = new QuickstartTableRequest(baseDir.getAbsolutePath());
-    QuickstartRunner runner = new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1, dataDir);
+    QuickstartRunner runner = new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1, 0, dataDir);
 
     printStatus(Color.CYAN, "***** Starting Kafka *****");
     ZkStarter.ZookeeperInstance zookeeperInstance = ZkStarter.startLocalZkServer();