Adaptive Load FakeStepController doom update

test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller.cc

@@ -15,9 +15,10 @@ absl::Status StatusFromProtoRpcStatus(const google::rpc::Status& status_proto) {
 } // namespace
 
 FakeStepController::FakeStepController(
-    const nighthawk::adaptive_load::FakeStepControllerConfig& config,
+    nighthawk::adaptive_load::FakeStepControllerConfig config,
     nighthawk::client::CommandLineOptions command_line_options_template)
-    : is_converged_{false}, is_doomed_{false}, fixed_rps_value_{config.fixed_rps_value()},
+    : input_setting_failure_countdown_{config.artificial_input_setting_failure_countdown()},
+      config_{std::move(config)}, is_converged_{false}, is_doomed_{false},
       command_line_options_template_{std::move(command_line_options_template)} {}
 
 bool FakeStepController::IsConverged() const { return is_converged_; }
@@ -33,20 +34,31 @@ bool FakeStepController::IsDoomed(std::string& doomed_reason) const {
 
 absl::StatusOr<nighthawk::client::CommandLineOptions>
 FakeStepController::GetCurrentCommandLineOptions() const {
+  if (config_.has_artificial_input_setting_failure() && input_setting_failure_countdown_ <= 0) {
+    return StatusFromProtoRpcStatus(config_.artificial_input_setting_failure());
+  }
   nighthawk::client::CommandLineOptions options = command_line_options_template_;
-  options.mutable_requests_per_second()->set_value(fixed_rps_value_);
+  options.mutable_requests_per_second()->set_value(config_.fixed_rps_value());
   return options;
 }
 
 void FakeStepController::UpdateAndRecompute(
     const nighthawk::adaptive_load::BenchmarkResult& benchmark_result) {
+  if (input_setting_failure_countdown_ > 0) {
+    --input_setting_failure_countdown_;
+  }
   // "Convergence" is defined as the latest benchmark reporting any score > 0.0.
+  // "Doom" is defined as any score < 0.0. Neutral is all scores equal to 0.0.
   is_converged_ = false;
+  is_doomed_ = false;
+  doomed_reason_ = "";
   for (const nighthawk::adaptive_load::MetricEvaluation& metric_evaluation :
        benchmark_result.metric_evaluations()) {
-    if (metric_evaluation.threshold_score() > 0.0) {
+    if (metric_evaluation.threshold_score() < 0.0) {
+      is_doomed_ = true;
+      doomed_reason_ = "artificial doom triggered by negative score";
+    } else if (metric_evaluation.threshold_score() > 0.0) {
       is_converged_ = true;
-      break;
     }
   }
 }
@@ -110,4 +122,20 @@ envoy::config::core::v3::TypedExtensionConfig MakeFakeStepControllerPluginConfig
   return outer_config;
 }
 
+envoy::config::core::v3::TypedExtensionConfig
+MakeFakeStepControllerPluginConfigWithInputSettingError(
+    int fixed_rps_value, const absl::Status& artificial_input_setting_failure, int countdown) {
+  envoy::config::core::v3::TypedExtensionConfig outer_config;
+  outer_config.set_name("nighthawk.fake_step_controller");
+  nighthawk::adaptive_load::FakeStepControllerConfig config;
+  config.set_fixed_rps_value(fixed_rps_value);
+  config.mutable_artificial_input_setting_failure()->set_code(
+      static_cast<int>(artificial_input_setting_failure.code()));
+  config.mutable_artificial_input_setting_failure()->set_message(
+      std::string(artificial_input_setting_failure.message()));
+  config.set_artificial_input_setting_failure_countdown(countdown);
+  outer_config.mutable_typed_config()->PackFrom(config);
+  return outer_config;
+}
+
 } // namespace Nighthawk

test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller.h

@@ -13,6 +13,8 @@ namespace Nighthawk {
 
 /**
  * StepController for testing: Configurable convergence and doom countdowns, fixed RPS value.
+ *
+ * This class is not thread-safe.
  */
 class FakeStepController : public StepController {
 public:
@@ -22,7 +24,7 @@ class FakeStepController : public StepController {
    * @param config FakeStepControllerConfig proto for setting the fixed RPS value.
    * @param command_line_options_template A template for producing Nighthawk input.
    */
-  FakeStepController(const nighthawk::adaptive_load::FakeStepControllerConfig& config,
+  FakeStepController(nighthawk::adaptive_load::FakeStepControllerConfig config,
                      nighthawk::client::CommandLineOptions command_line_options_template);
   /**
    * @return bool The current value of |is_converged_|.
@@ -42,20 +44,23 @@ class FakeStepController : public StepController {
   absl::StatusOr<nighthawk::client::CommandLineOptions>
   GetCurrentCommandLineOptions() const override;
   /**
-   * Updates |is_converged_| to reflect whether |benchmark_result| contains any score >0. Sets
-   * |is_doomed_| based whether the status in |benchmark_result| is OK; copies the status message
-   * into |doomed_reason_| only when the status is not OK.
+   * Updates |is_converged_| to reflect whether |benchmark_result| contains any score >0. Updates
+   * |is_doomed_| to reflect whether |benchmark_result| contains any score <0. A non-converged,
+   * non-doomed input has scores all equal to 0.
    *
    * @param benchmark_result A Nighthawk benchmark result proto.
    */
   void
   UpdateAndRecompute(const nighthawk::adaptive_load::BenchmarkResult& benchmark_result) override;
 
 private:
+  // Counts down UpdateAndRecompute() calls. When this reaches zero, GetCurrentCommandLineOptions()
+  // starts to return an artificial input value setting failure if one is specified in the config.
+  int input_setting_failure_countdown_;
+  const nighthawk::adaptive_load::FakeStepControllerConfig config_;
   bool is_converged_;
   bool is_doomed_;
   std::string doomed_reason_;
-  const int fixed_rps_value_;
   const nighthawk::client::CommandLineOptions command_line_options_template_;
 };
 
@@ -91,7 +96,8 @@ MakeFakeStepControllerPluginConfig(int fixed_rps_value);
  * Creates a valid TypedExtensionConfig proto that activates a FakeStepController with a
  * FakeInputVariableSetterConfig that fails validation.
  *
- * @param artificial_validation_error An error status.
+ * @param artificial_validation_error An artificial error status to be returned by
+ * FakeStepControllerConfigFactory::ValidateConfig() when attempting LoadStepControllerPlugin().
  *
  * @return TypedExtensionConfig A proto that activates FakeStepController by name and includes
  * a FakeStepControllerConfig proto wrapped in an Any. This proto will fail validation when
@@ -100,4 +106,21 @@ MakeFakeStepControllerPluginConfig(int fixed_rps_value);
 envoy::config::core::v3::TypedExtensionConfig MakeFakeStepControllerPluginConfigWithValidationError(
     const absl::Status& artificial_validation_error);
 
+/**
+ * Creates a valid TypedExtensionConfig proto that activates a FakeStepController with a
+ * FakeInputVariableSetterConfig that returns an error from GetCurrentCommandLineOptions().
+ *
+ * @param fixed_rps_value Value for RPS to set in the FakeStepControllerConfig proto until the
+ * countdown reaches zero.
+ * @param artificial_input_setting_failure An error status.
+ * @param countdown Number of times UpdateAndRecompute() must be called before
+ * GetCurrentCommandLineOptions() starts to return the input error status.
+ *
+ * @return TypedExtensionConfig A proto that activates FakeStepController by name and includes
+ * a FakeStepControllerConfig proto wrapped in an Any.
+ */
+envoy::config::core::v3::TypedExtensionConfig
+MakeFakeStepControllerPluginConfigWithInputSettingError(
+    int fixed_rps_value, const absl::Status& artificial_input_setting_failure, int countdown);
+
 } // namespace Nighthawk

test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller.proto

@@ -5,12 +5,25 @@ package nighthawk.adaptive_load;
 import "envoy/config/core/v3/extension.proto";
 import "google/rpc/status.proto";
 
-// Configuration for FakeStepController (plugin name: "nighthawk.fake_step_controller") that always
-// returns a fixed RPS value and changes converged and doomed states based on the latest reported
-// BenchmarkResult.
+// Configuration for FakeStepController (plugin name: "nighthawk.fake_step_controller") that returns
+// a fixed RPS value and changes converged and doomed states based on the latest reported
+// BenchmarkResult. Can also be programmed to return a proto validation failure, return an error
+// from input value setting every time, or return an error after some number of UpdateAndRecompute()
+// iterations.
 message FakeStepControllerConfig {
-  // RPS that should always be returned. Optional, default 0.
+  // RPS that should always be returned, except when artificial errors are configured. Optional,
+  // default 0.
   int32 fixed_rps_value = 1;
   // Artificial error that the plugin factory should return during validation. Optional.
   google.rpc.Status artificial_validation_failure = 2;
+  // Artificial error that should be returned from GetCurrentCommandLineOptions(). Optional. May be
+  // used in conjunction with |artificial_input_setting_failure_countdown| to activate error
+  // behavior after a delay.
+  google.rpc.Status artificial_input_setting_failure = 3;
+  // Relevant only when |artificial_input_setting_failure| is set. Number of calls to
+  // UpdateAndRecompute() the controller must receive before it starts to return
+  // |artificial_input_setting_failure|. Before this total is reached, |fixed_rps_value| is
+  // returned. Optional, default 0, meaning the failure is returned regardless of calls to
+  // UpdateAndRecompute().
+  int32 artificial_input_setting_failure_countdown = 4;
 }

test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller_test.cc

@@ -111,19 +111,66 @@ TEST(FakeStepController, GetCurrentCommandLineOptionsReturnsRpsFromConfig) {
             kExpectedValue);
 }
 
+TEST(FakeStepController, GetCurrentCommandLineOptionsReturnsArtificialErrorImmediately) {
+  FakeStepControllerConfig config;
+  const int kExpectedCode = ::grpc::DEADLINE_EXCEEDED;
+  const std::string kExpectedMessage = "artificial input setting error";
+  config.mutable_artificial_input_setting_failure()->set_code(kExpectedCode);
+  config.mutable_artificial_input_setting_failure()->set_message(kExpectedMessage);
+  // Not setting countdown.
+
+  FakeStepController step_controller(config, CommandLineOptions());
+  absl::StatusOr<nighthawk::client::CommandLineOptions> command_line_options_or =
+      step_controller.GetCurrentCommandLineOptions();
+  ASSERT_FALSE(command_line_options_or.ok());
+  EXPECT_EQ(static_cast<int>(command_line_options_or.status().code()), kExpectedCode);
+  EXPECT_EQ(command_line_options_or.status().message(), kExpectedMessage);
+}
+
+TEST(FakeStepController, GetCurrentCommandLineOptionsReturnsArtificialErrorAfterCountdown) {
+  FakeStepControllerConfig config;
+  const int kExpectedCode = ::grpc::DEADLINE_EXCEEDED;
+  const std::string kExpectedMessage = "artificial input setting error";
+  config.mutable_artificial_input_setting_failure()->set_code(kExpectedCode);
+  config.mutable_artificial_input_setting_failure()->set_message(kExpectedMessage);
+  config.set_artificial_input_setting_failure_countdown(2);
+
+  FakeStepController step_controller(config, CommandLineOptions());
+  absl::StatusOr<nighthawk::client::CommandLineOptions> command_line_options_or1 =
+      step_controller.GetCurrentCommandLineOptions();
+  EXPECT_TRUE(command_line_options_or1.ok());
+
+  step_controller.UpdateAndRecompute(nighthawk::adaptive_load::BenchmarkResult());
+  // Countdown should now be 1.
+
+  absl::StatusOr<nighthawk::client::CommandLineOptions> command_line_options_or2 =
+      step_controller.GetCurrentCommandLineOptions();
+  EXPECT_TRUE(command_line_options_or2.ok());
+
+  step_controller.UpdateAndRecompute(nighthawk::adaptive_load::BenchmarkResult());
+  // Countdown should now have reached 0.
+
+  // This should now return the artificial input setting failure:
+  absl::StatusOr<nighthawk::client::CommandLineOptions> command_line_options_or3 =
+      step_controller.GetCurrentCommandLineOptions();
+  ASSERT_FALSE(command_line_options_or3.ok());
+  EXPECT_EQ(static_cast<int>(command_line_options_or3.status().code()), kExpectedCode);
+  EXPECT_EQ(command_line_options_or3.status().message(), kExpectedMessage);
+}
+
 TEST(FakeStepController, IsConvergedInitiallyReturnsFalse) {
   FakeStepController step_controller(FakeStepControllerConfig{}, CommandLineOptions{});
   EXPECT_FALSE(step_controller.IsConverged());
 }
 
-TEST(FakeStepController, IsConvergedReturnsFalseAfterBenchmarkResultWithoutPositiveScore) {
+TEST(FakeStepController, IsConvergedReturnsFalseAfterNeutralBenchmarkResult) {
   FakeStepController step_controller(FakeStepControllerConfig{}, CommandLineOptions{});
   BenchmarkResult benchmark_result;
   step_controller.UpdateAndRecompute(benchmark_result);
   EXPECT_FALSE(step_controller.IsConverged());
 }
 
-TEST(FakeStepController, IsConvergedReturnsTrueAfterBenchmarkResultWithPositiveScore) {
+TEST(FakeStepController, IsConvergedReturnsTrueAfterPositiveBenchmarkResultScore) {
   FakeStepController step_controller(FakeStepControllerConfig{}, CommandLineOptions{});
   BenchmarkResult benchmark_result;
   MetricEvaluation* evaluation = benchmark_result.mutable_metric_evaluations()->Add();
@@ -132,6 +179,35 @@ TEST(FakeStepController, IsConvergedReturnsTrueAfterBenchmarkResultWithPositiveS
   EXPECT_TRUE(step_controller.IsConverged());
 }
 
+TEST(FakeStepController, IsDoomedReturnsFalseAfterNeutralBenchmarkResult) {
+  FakeStepController step_controller(FakeStepControllerConfig{}, CommandLineOptions{});
+  BenchmarkResult benchmark_result;
+  step_controller.UpdateAndRecompute(benchmark_result);
+  std::string doomed_reason;
+  EXPECT_FALSE(step_controller.IsDoomed(doomed_reason));
+}
+
+TEST(FakeStepController,
+     IsDoomedReturnsFalseAndLeavesDoomedReasonUntouchedAfterNeutralBenchmarkResult) {
+  FakeStepController step_controller(FakeStepControllerConfig{}, CommandLineOptions{});
+  BenchmarkResult benchmark_result;
+  step_controller.UpdateAndRecompute(benchmark_result);
+  std::string variable_that_should_not_be_written = "original value";
+  EXPECT_FALSE(step_controller.IsDoomed(variable_that_should_not_be_written));
+  EXPECT_EQ(variable_that_should_not_be_written, "original value");
+}
+
+TEST(FakeStepController, IsDoomedReturnsTrueAndSetsDoomedReasonAfterNegativeBenchmarkResultScore) {
+  FakeStepController step_controller(FakeStepControllerConfig{}, CommandLineOptions{});
+  BenchmarkResult benchmark_result;
+  MetricEvaluation* evaluation = benchmark_result.mutable_metric_evaluations()->Add();
+  evaluation->set_threshold_score(-1.0);
+  step_controller.UpdateAndRecompute(benchmark_result);
+  std::string doomed_reason;
+  EXPECT_TRUE(step_controller.IsDoomed(doomed_reason));
+  EXPECT_EQ(doomed_reason, "artificial doom triggered by negative score");
+}
+
 TEST(MakeFakeStepControllerPluginConfig, ActivatesFakeStepControllerPlugin) {
   absl::StatusOr<StepControllerPtr> plugin_or = LoadStepControllerPlugin(
       MakeFakeStepControllerPluginConfig(0), nighthawk::client::CommandLineOptions{});
@@ -153,7 +229,7 @@ TEST(MakeFakeStepControllerPluginConfig, ProducesFakeStepControllerPluginWithCon
 }
 
 TEST(MakeFakeStepControllerPluginConfigWithValidationError,
-     ProducesFakeStepControllerPluginWithConfiguredValue) {
+     ProducesFakeStepControllerPluginWithConfiguredError) {
   std::string kValidationErrorMessage = "artificial validation error";
   absl::StatusOr<StepControllerPtr> plugin_or =
       LoadStepControllerPlugin(MakeFakeStepControllerPluginConfigWithValidationError(
@@ -163,5 +239,29 @@ TEST(MakeFakeStepControllerPluginConfigWithValidationError,
   EXPECT_EQ(plugin_or.status().message(), kValidationErrorMessage);
 }
 
+TEST(MakeFakeStepControllerPluginConfigWithInputSettingError,
+     ProducesFakeStepControllerPluginWithConfiguredErrorAndCountdown) {
+  const int kExpectedRpsValue = 123;
+  const std::string kInputSettingErrorMessage = "artificial input setting error";
+  absl::StatusOr<StepControllerPtr> plugin_or = LoadStepControllerPlugin(
+      MakeFakeStepControllerPluginConfigWithInputSettingError(
+          kExpectedRpsValue, absl::DeadlineExceededError(kInputSettingErrorMessage),
+          /*countdown=*/1),
+      nighthawk::client::CommandLineOptions{});
+  ASSERT_TRUE(plugin_or.ok());
+  auto* plugin = dynamic_cast<FakeStepController*>(plugin_or.value().get());
+  ASSERT_NE(plugin, nullptr);
+  absl::StatusOr<nighthawk::client::CommandLineOptions> command_line_options_or1 =
+      plugin->GetCurrentCommandLineOptions();
+  ASSERT_TRUE(command_line_options_or1.ok());
+  EXPECT_EQ(command_line_options_or1.value().requests_per_second().value(), kExpectedRpsValue);
+  plugin->UpdateAndRecompute(BenchmarkResult());
+  absl::StatusOr<nighthawk::client::CommandLineOptions> command_line_options_or2 =
+      plugin->GetCurrentCommandLineOptions();
+  ASSERT_FALSE(command_line_options_or2.ok());
+  EXPECT_EQ(command_line_options_or2.status().code(), absl::StatusCode::kDeadlineExceeded);
+  EXPECT_EQ(command_line_options_or2.status().message(), kInputSettingErrorMessage);
+}
+
 } // namespace
 } // namespace Nighthawk