Add ability to configure GRPC max connection age and max connection a…

…ge grace
triton-inference-server · Nov 28, 2023 · 238198e · 238198e
1 parent 4b34a48
commit 238198e
Show file tree

Hide file tree

Showing 3 changed files with 40 additions and 0 deletions.
diff --git a/src/command_line_parser.cc b/src/command_line_parser.cc
@@ -310,6 +310,8 @@ enum TritonOptionId {
   OPTION_GRPC_ARG_HTTP2_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS,
   OPTION_GRPC_ARG_HTTP2_MAX_PING_STRIKES,
   OPTION_GRPC_RESTRICTED_PROTOCOL,
+  OPTION_GRPC_ARG_MAX_CONNECTION_AGE_MS,
+  OPTION_GRPC_ARG_MAX_CONNECTION_AGE_GRACE_MS,
 #endif  // TRITON_ENABLE_GRPC
 #if defined(TRITON_ENABLE_SAGEMAKER)
   OPTION_ALLOW_SAGEMAKER,
@@ -568,6 +570,16 @@ TritonParser::SetupOptions()
        "Maximum number of bad pings that the server will tolerate before "
        "sending an HTTP2 GOAWAY frame and closing the transport. Setting it to "
        "0 allows the server to accept any number of bad pings. Default is 2."});
+  grpc_options_.push_back(
+      {OPTION_GRPC_ARG_MAX_CONNECTION_AGE_MS, "grpc-max-connection-age",
+       Option::ArgInt,
+       "Maximum time that a channel may exist in milliseconds. "
+       "Default is INT_MAX (infinite)."});
+  grpc_options_.push_back(
+      {OPTION_GRPC_ARG_MAX_CONNECTION_AGE_GRACE_MS, "grpc-max-connection-age-grace",
+       Option::ArgInt,
+       "Grace period after the channel reaches its max age. "
+       "Default is INT_MAX (infinite)."});
   grpc_options_.push_back(
       {OPTION_GRPC_RESTRICTED_PROTOCOL, "grpc-restricted-protocol",
        "<string>:<string>=<string>",
@@ -1436,6 +1448,14 @@ TritonParser::Parse(int argc, char** argv)
           lgrpc_options.keep_alive_.http2_max_ping_strikes_ =
               ParseOption<int>(optarg);
           break;
+        case OPTION_GRPC_ARG_MAX_CONNECTION_AGE_MS:
+          lgrpc_options.keep_alive_.max_connection_age_ms_ =
+              ParseOption<int>(optarg);
+          break;
+        case OPTION_GRPC_ARG_MAX_CONNECTION_AGE_GRACE_MS:
+          lgrpc_options.keep_alive_.max_connection_age_grace_ms_ =
+              ParseOption<int>(optarg);
+          break;
         case OPTION_GRPC_RESTRICTED_PROTOCOL: {
           ParseRestrictedFeatureOption(
               optarg, long_options[option_index].name,

diff --git a/src/grpc/grpc_server.cc b/src/grpc/grpc_server.cc
@@ -2366,6 +2366,12 @@ Server::Server(
     builder_.AddChannelArgument(
         GRPC_ARG_HTTP2_MAX_PING_STRIKES,
         keepalive_options.http2_max_ping_strikes_);
+    builder_.AddChannelArgument(
+        GRPC_ARG_MAX_CONNECTION_AGE_MS,
+        keepalive_options.max_connection_age_ms_);
+    builder_.AddChannelArgument(
+        GRPC_ARG_MAX_CONNECTION_AGE_GRACE_MS,
+        keepalive_options.max_connection_age_grace_ms_);
 
     std::vector<std::string> headers{"GRPC KeepAlive Option", "Value"};
     triton::common::TablePrinter table_printer(headers);
@@ -2399,6 +2405,17 @@ Server::Server(
         "http2_max_ping_strikes",
         std::to_string(keepalive_options.http2_max_ping_strikes_)};
     table_printer.InsertRow(row);
+
+    row = {
+        "max_connection_age_ms",
+        std::to_string(
+            keepalive_options.max_connection_age_ms_)};
+    table_printer.InsertRow(row);
+
+    row = {
+        "max_connection_age_grace_ms",
+        std::to_string(keepalive_options.max_connection_age_grace_ms_)};
+    table_printer.InsertRow(row);
     LOG_VERBOSE(1) << table_printer.PrintTable();
   }
 

diff --git a/src/grpc/grpc_server.h b/src/grpc/grpc_server.h
@@ -66,13 +66,16 @@ struct SslOptions {
 };
 
 // GRPC KeepAlive: https://grpc.github.io/grpc/cpp/md_doc_keepalive.html
+// https://grpc.io/docs/guides/keepalive/
 struct KeepAliveOptions {
   int keepalive_time_ms_{7200000};
   int keepalive_timeout_ms_{20000};
   bool keepalive_permit_without_calls_{false};
   int http2_max_pings_without_data_{2};
   int http2_min_recv_ping_interval_without_data_ms_{300000};
   int http2_max_ping_strikes_{2};
+  int max_connection_age_ms_{INT_MAX};
+  int max_connection_age_grace_ms_{INT_MAX};
 };
 
 struct Options {