parallelize start of multiple workloads and misc updates

Signed-off-by: Benjamin Hoflich <[email protected]>

runHarness/AppInstance/AppInstance.pm

@@ -270,14 +270,6 @@ sub getUsers {
 	return $self->users;
 }
 
-sub getMaxLoadedUsers {
-	my ($self)        = @_;
-	my $dbServicesRef = $self->getAllServicesByType("dbServer");
-	my $dbServer      = $dbServicesRef->[0];
-
-	return $dbServer->getMaxLoadedUsers();
-}
-
 sub getLoadPath {
 	my ($self) = @_;
 	return $self->getParamValue('userLoadPath');
@@ -600,11 +592,6 @@ sub startServices {
 	my ( $self, $serviceTier, $setupLogDir, $forked ) = @_;
 	my $logger = get_logger("Weathervane::AppInstance::AppInstance");
 	my $impl         = $self->getParamValue('workloadImpl');
-
-	my $appInstanceName = $self->name;
-	my $logName         = "$setupLogDir/start-$serviceTier-$appInstanceName.log";
-	my $logFile;
-	open( $logFile, " > $logName " ) or die " Error opening $logName: $!";
 
 	my $users    = $self->dataManager->getParamValue('maxUsers');
 	$logger->debug(
@@ -632,14 +619,7 @@ sub startServices {
 				next;
 			}
 		}
-		# Don't return until all services are ready
-		my $allIsRunningAndUp = $self->isRunningAndUpServices($serviceTier, $logFile, $forked);
-		if ( !$allIsRunningAndUp ) {
-			close $logFile;
-			return 0;
-		}
 	}
-	close $logFile;
 	return 1;
 }
 
@@ -695,7 +675,12 @@ sub stopDataManager {
 }
 
 sub isRunningAndUpServices {
-	my ( $self, $serviceTier, $logFile, $forked ) = @_;
+	my ( $self, $serviceTier, $setupLogDir, $forked ) = @_;
+	my $appInstanceName = $self->name;
+	my $logName         = "$setupLogDir/start-$serviceTier-$appInstanceName.log";
+	my $logFile;
+	open( $logFile, " > $logName " ) or die " Error opening $logName: $!";
+
 	my $logger         = get_logger("Weathervane::DataManager::AuctionKubernetesDataManager");
 	my $console_logger = get_logger("Console");
 
@@ -717,6 +702,7 @@ sub isRunningAndUpServices {
 		if ($forked) {
 			exit;
 		} else {
+			close $logFile;
 			return 0;
 		}
 	}
@@ -729,10 +715,12 @@ sub isRunningAndUpServices {
 		if ($forked) {
 			exit;
 		} else {
+			close $logFile;
 			return 0;
 		}
 	}
 	$logger->debug( "All $serviceTier services are up for appInstance $appInstanceNum of workload $workloadNum." );
+	close $logFile;
 	return 1;
 }
 
@@ -912,8 +900,7 @@ sub isUp {
 			$isUp = $service->isUp($log);
 			if ( !$isUp ) {
 				$allUp = 0;
-				if ( ( $retriesRemaining == 0 ) && !$isUp ) {
-
+				if ( ( $retriesRemaining == 1 ) && !$isUp ) {
 					# no more retries so give an error
 					my $hostname = $service->host->name;
 					$console_logger->error( "Couldn't start $serviceType "

runHarness/AppInstance/AuctionKubernetesAppInstance.pm

@@ -64,11 +64,6 @@ override 'startServices' => sub {
 	my $users  = $self->dataManager->getParamValue('maxUsers');
 	my $impl         = $self->getParamValue('workloadImpl');
 
-	my $appInstanceName = $self->name;
-	my $logName         = "$setupLogDir/start-$serviceTier-$appInstanceName.log";
-	my $logFile;
-	open( $logFile, " > $logName " ) or die " Error opening $logName: $!";
-
 	$logger->debug(
 		"startServices for serviceTier $serviceTier, workload ",
 		$self->workload->instanceNum,
@@ -94,15 +89,7 @@ override 'startServices' => sub {
 				next;
 			}		
 		}
-
-		# Don't return until all services are ready
-		my $allIsRunningAndUp = $self->isRunningAndUpServices($serviceTier, $logFile, $forked);
-		if ( !$allIsRunningAndUp ) {
-			close $logFile;
-			return 0;
-		}
 	}
-	close $logFile;
 	return 1;
 };
 

runHarness/ComputeResources/DockerHost.pm

@@ -235,6 +235,7 @@ sub dockerPort {
 	foreach my $line (@lines) {
 		if ($line =~ /(\d+)\/.*\:(\d+)\s*$/) {
 			$portMap{$1} = $2;
+			last;
 		}
 	}
 
@@ -581,7 +582,7 @@ sub dockerCopyFrom {
 	my $cmdString = "$dockerHostString docker cp $name:$sourceFile $destFile";
 	my ($cmdFailed, $out) = runCmd($cmdString, 0);
 	if ($cmdFailed) {
-		$logger->error("dockerCopyFrom failed: $cmdFailed");
+		$logger->debug("dockerCopyFrom failed: $cmdFailed");
 	}
 	print $logFileHandle "$cmdString\n";
 	$logger->debug("docker cp output: $out");

runHarness/DataManagers/AuctionDataManager.pm

@@ -17,6 +17,7 @@ use Log::Log4perl qw(get_logger);
 with Storage( 'format' => 'JSON', 'io' => 'File' );
 
 use namespace::autoclean;
+use Utils qw(runCmd);
 
 extends 'DataManager';
 
@@ -69,7 +70,7 @@ sub startDataManagerContainer {
 	foreach my $nosqlServer (@$nosqlServicesRef) {
 		$cassandraContactpoints .= $self->getHostnameForUsedService($nosqlServer) . ",";
 	}
-	$cassandraContactpoints =~ s/,$//;		
+	$cassandraContactpoints =~ s/,$//;
 	$envVarMap{"CASSANDRA_CONTACTPOINTS"} = $cassandraContactpoints;
 	$envVarMap{"CASSANDRA_PORT"} = $cassandraPort;
 
@@ -139,7 +140,8 @@ sub prepareDataServices {
 		$appInstance->clearDataServicesBeforeStart($logPath);
 	}
 
-	my $allIsStarted = $appInstance->startServices("data", $logPath, 0);
+	$appInstance->startServices("data", $logPath, 0);
+	my $allIsStarted = $appInstance->isRunningAndUpServices("data", $logPath, 0);
 	if ( !$allIsStarted ) {
 		close $logHandle;
 		return $allIsStarted;
@@ -155,7 +157,8 @@ sub prepareDataServices {
 		# Need to stop and restart services so that we can clear out any old data
 		$appInstance->stopServices("data", $logPath);
 		$appInstance->clearDataServicesBeforeStart($logPath);
-		$allIsStarted = $appInstance->startServices("data", $logPath, 0);
+		$appInstance->startServices("data", $logPath, 0);
+		$allIsStarted = $appInstance->isRunningAndUpServices("data", $logPath, 0);
 		if ( !$allIsStarted ) {
 			close $logHandle;
 			return $allIsStarted;
@@ -213,6 +216,11 @@ sub prepareData {
 	$logger->debug( "All data services are up for appInstance $appInstanceNum of workload $workloadNum." );
 
 	if ($reloadDb || !$self->isDataLoaded( $users, $logPath )) {
+		if (!$reloadDb) {
+			$console_logger->info(
+				    "Data is not loaded for $maxUsers maxUsers for appInstance "
+				  . "$appInstanceNum of workload $workloadNum. Loading data." );
+		}
 		$appInstance->clearDataServicesAfterStart($logPath);
 		$retVal = $self->loadData( $users, $logPath );
 		if ( !$retVal ) { return 0; }
@@ -226,12 +234,11 @@ sub prepareData {
 				  . "$appInstanceNum of workload $workloadNum." );
 	print $logHandle "Exec-ing perl /prepareData.pl  in container $name\n";
 	$logger->debug("Exec-ing perl /prepareData.pl  in container $name");
-	my $dockerHostString  = $self->host->dockerHostString;	
-	my $cmdOut = `$dockerHostString docker exec $name perl /prepareData.pl`;
+
+	my ($cmdFailed, $cmdOut) = $self->host->dockerExec($logHandle, $name, "perl /prepareData.pl");
 	print $logHandle "Output: $cmdOut, \$? = $?\n";
-	$logger->debug("Output: $cmdOut, \$? = $?");
-	if ($?) {
-		$console_logger->error( "Data preparation process failed.  Check PrepareData.log for more information." );
+	if ($cmdFailed) {
+		$console_logger->error( "Data preparation process failed.  Check PrepareData.log for more information. \$cmdOut = $cmdOut" );
 		$self->stopDataManagerContainer($logHandle);
 		return 0;
 	}
@@ -278,7 +285,6 @@ sub loadData {
 	$logger->debug("Exec-ing perl /loadData.pl in container $name");
 	print $applog "Exec-ing perl /loadData.pl in container $name\n";
 	my $dockerHostString  = $self->host->dockerHostString;
-
 	open my $pipe, "$dockerHostString docker exec $name perl /loadData.pl  |"   or die "Couldn't execute program: $!";
  	while ( defined( my $line = <$pipe> )  ) {
 		chomp($line);
@@ -327,14 +333,17 @@ sub isDataLoaded {
 
 	print $applog "Exec-ing perl /isDataLoaded.pl  in container $name\n";
 	$logger->debug("Exec-ing perl /isDataLoaded.pl  in container $name");
-	my $dockerHostString  = $self->host->dockerHostString;	
-	my $cmdOut = `$dockerHostString docker exec $name perl /isDataLoaded.pl`;
+
+	my ($cmdFailed, $cmdOut) = $self->host->dockerExec($applog, $name, "perl /isDataLoaded.pl");
 	print $applog "Output: $cmdOut, \$? = $?\n";
-	$logger->debug("Output: $cmdOut, \$? = $?");
 	close $applog;
-	if ($?) {
-		$logger->debug( "Data is not loaded for workload $workloadNum, appInstance $appInstanceNum. \$cmdOut = $cmdOut" );
-		return 0;
+	if ($cmdFailed) {
+		if ( $cmdOut =~ /sh: line 1:    16 Killed/ ) {
+			die "AuctionDataManager container Killed -- the VM might not be big enough for the container.\n";
+		} else {
+			$logger->debug( "Data is not loaded for workload $workloadNum, appInstance $appInstanceNum." );
+			return 0;
+		}
 	}
 	else {
 		$logger->debug( "Data is loaded for workload $workloadNum, appInstance $appInstanceNum. \$cmdOut = $cmdOut" );
@@ -348,7 +357,11 @@ sub cleanData {
 	my $logger         = get_logger("Weathervane::DataManager::AuctionDataManager");
 	my $nosqlServersRef = $self->appInstance->getAllServicesByType('nosqlServer');
 	foreach my $nosqlServerRef (@$nosqlServersRef) {
-#		$nosqlServerRef->cleanData($users, $logHandle);
+		$nosqlServerRef->cleanData($users, $logHandle);
+	}
+	my $dbServersRef = $self->appInstance->getAllServicesByType('dbServer');
+	foreach my $dbServerRef (@$dbServersRef) {
+		$dbServerRef->cleanData($users, $logHandle);
 	}
 }
 

runHarness/DataManagers/AuctionKubernetesDataManager.pm

@@ -30,7 +30,7 @@ override 'initialize' => sub {
 
 	my $workloadNum = $self->appInstance->workload->instanceNum;
 	my $appInstanceNum = $self->appInstance->instanceNum;
-	
+
 	$self->name("auctiondatamanagerW${workloadNum}A${appInstanceNum}");
 
 	super();
@@ -205,8 +205,9 @@ sub prepareDataServices {
 		# we are reloading the data
 		$appInstance->clearDataServicesBeforeStart($logPath);
 	}
-	my $allIsStarted = $appInstance->startServices("data", $logPath, 0);
-
+	$appInstance->startServices("data", $logPath, 0);
+	my $allIsStarted = $appInstance->isRunningAndUpServices("data", $logPath, 0);
+
 	if (!$allIsStarted) {
 		$appInstance->getDataServiceLogFiles($logPath . "/prepareDataServicesFailure");
 		if ($self->getParamValue("reloadOnFailure")) {
@@ -216,7 +217,8 @@ sub prepareDataServices {
 			$appInstance->stopServices("data", $logPath);
 			my $cluster = $self->host;
 			$cluster->kubernetesDeleteAllWithLabelAndResourceType("app=auction", "pvc", $self->appInstance->namespace );
-			$allIsStarted = $appInstance->startServices("data", $logPath, 0);
+			$appInstance->startServices("data", $logPath, 0);
+			$allIsStarted = $appInstance->isRunningAndUpServices("data", $logPath, 0);
 		} else {
 			$console_logger->info(
 				"Couldn't start data services for appInstance $appInstanceNum of workload $workloadNum.\n" . 
@@ -288,6 +290,7 @@ sub prepareData {
 			$appInstance->stopServices("data", $logPath);
 			$appInstance->clearDataServicesBeforeStart($logPath);
 			$appInstance->startServices("data", $logPath, 0);
+			$appInstance->isRunningAndUpServices("data", $logPath, 0);
 			$appInstance->clearDataServicesAfterStart($logPath);
 
 			$retVal = $self->loadData( $users, $logPath );
@@ -328,6 +331,7 @@ sub prepareData {
 			$appInstance->stopServices("data", $logPath);
 			$cluster->kubernetesDeleteAllWithLabelAndResourceType("app=auction", "pvc", $self->appInstance->namespace );
 			$appInstance->startServices("data", $logPath, 0);
+			$appInstance->isRunningAndUpServices("data", $logPath, 0);
 			return $self->prepareData( $users, $logPath, 1);
 		} else {
 			$console_logger->error( 

runHarness/RunManagers/FixedRunManager.pm

@@ -62,6 +62,10 @@ override 'start' => sub {
 	elsif ( $runProcedureType eq 'stop' ) {
 		$console_logger->info("Run stopped");
 	}
+	elsif ( $runProcedureType eq 'loadOnly' ) {
+		$console_logger->info($runResult->toString());
+		$console_logger->info("Application loaded.");
+	}
 	else {
 		$self->printCsv( $runResult->resultsSummaryHashRef, 1 );
 		$console_logger->info($runResult->toString());

runHarness/RunProcedures/LoadOnlyRunProcedure.pm

@@ -9,7 +9,7 @@ use WeathervaneTypes;
 use RunResults::RunResult;
 use JSON;
 use Log::Log4perl qw(get_logger);
-use Utils qw(callMethodOnObjectsParamListParallel1 callMethodOnObjectsParallel2 callMethodOnObjectsParallel callMethodsOnObjectParallel callMethodsOnObjectParallel1 callMethodOnObjectsParallel1);
+use Utils qw(callMethodOnObjectsParamListParallel1 callMethodOnObjectsParallel callMethodsOnObjectParallel callMethodsOnObjectParallel1 callMethodOnObjectsParallel1 callMethodOnObjectsParallel2);
 
 use Parameters qw(getParamValue setParamValue);
 
@@ -86,7 +86,8 @@ sub run {
 	if ( !( -e $tmpDir ) ) {
 		`mkdir $tmpDir`;
 	}
-
+	$self->tmpDir($tmpDir);
+
 	# directory for logs related to start/stop/etc
 	my $setupLogDir = $tmpDir . "/setupLogs";
 	if ( !( -e $setupLogDir ) ) {
@@ -104,14 +105,14 @@ sub run {
 	$appender->layout($layout);
 	$console_logger->add_appender($appender);
 
-	$console_logger->info("Stopping running services and cleaning up old log and stats files.\n");
+	$console_logger->info("Stopping services from previous runs.\n");
 
 	# Make sure that no previous Benchmark processes are still running
 	$debug_logger->debug("killOldWorkloadDrivers");
 	$self->killOldWorkloadDrivers($setupLogDir);
 
 	$debug_logger->debug("stop services");
-	$self->stopDataManager($setupLogDir);		
+	$self->stopDataManager($setupLogDir);
 	$self->stopServicesInClusters();
 	my @tiers = qw(frontend);
 	callMethodOnObjectsParamListParallel1( "stopServices", [$self], \@tiers, $setupLogDir );
@@ -130,10 +131,13 @@ sub run {
 		$self->setParamValue( 'redeploy', 0 );
 	}
 
-	# Prepare the data for this run and start the data services
+	my $allUp;
 	# Start the data services for all AppInstances.  This happens serially so
 	# that we don't have to spawn processes and lose port number info.
-	$self->prepareDataServices($setupLogDir);	
+	$allUp = $self->prepareDataServices($setupLogDir);
+	if ( !$allUp ) {
+		$self->cleanupAfterFailure( "Could not properly start the data services for run $seqnum.  Exiting.", $seqnum, $tmpDir );
+	}
 	# Prepare the data for this run.  This happens in parallel on all appInstances
 	$console_logger->info("Preparing data for use in current run.\n");
 	my $dataPrepared = $self->prepareData($setupLogDir);