From aaa65137f64217791d8506101783e1c905ef33d5 Mon Sep 17 00:00:00 2001 From: Yingge He Date: Thu, 15 Feb 2024 19:55:58 -0800 Subject: [PATCH 01/15] Modify "header_forward_pattern" to match headers case-insensitively. Add unit tests. --- qa/L0_parameters/parameters_test.py | 17 +++++++++-------- qa/L0_parameters/test.sh | 6 ++++-- src/command_line_parser.cc | 11 +++++++++-- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/qa/L0_parameters/parameters_test.py b/qa/L0_parameters/parameters_test.py index 190326fc2c..8c28675b97 100755 --- a/qa/L0_parameters/parameters_test.py +++ b/qa/L0_parameters/parameters_test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2023-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -54,13 +54,14 @@ async def asyncSetUp(self): self.async_grpc = asyncgrpcclient.InferenceServerClient(url="localhost:8001") self.parameter_list = [] - self.parameter_list.append({"key1": "value1", "key2": "value2"}) - self.parameter_list.append({"key1": 1, "key2": 2}) - self.parameter_list.append({"key1": 123.123, "key2": 321.321}) - self.parameter_list.append({"key1": True, "key2": "value2"}) - self.parameter_list.append({"triton_": True, "key2": "value2"}) - - if TEST_HEADER == "1": + if TEST_HEADER in ["0","1"]: + self.parameter_list.append({"key1": "value1", "key2": "value2"}) + self.parameter_list.append({"key1": 1, "key2": 2}) + self.parameter_list.append({"key1": 123.123, "key2": 321.321}) + self.parameter_list.append({"key1": True, "key2": "value2"}) + self.parameter_list.append({"triton_": True, "key2": "value2"}) + + if TEST_HEADER in ["1","2"]: self.headers = { "header_1": "value_1", "header_2": "value_2", diff --git a/qa/L0_parameters/test.sh b/qa/L0_parameters/test.sh index 967ead15c7..9d2396be07 100755 --- a/qa/L0_parameters/test.sh +++ b/qa/L0_parameters/test.sh @@ -54,12 +54,14 @@ mkdir -p "${MODELDIR}/ensemble/1" # https://jirasw.nvidia.com/browse/DLIS-4673 RET=0 -for i in {0..1}; do +for i in {0..2}; do # TEST_HEADER is a parameter used by `parameters_test.py` that controls # whether the script will test for inclusion of headers in parameters or not. if [ $i == 1 ]; then SERVER_ARGS="--model-repository=${MODELDIR} --exit-timeout-secs=120 --grpc-header-forward-pattern my_header.* --http-header-forward-pattern my_header.*" + elif [ $i == 2 ]; then + SERVER_ARGS="--model-repository=${MODELDIR} --exit-timeout-secs=120 --grpc-header-forward-pattern MY_HEADER.* --http-header-forward-pattern MY_HEADER.*" else SERVER_ARGS="--model-repository=${MODELDIR} --exit-timeout-secs=120" fi @@ -73,7 +75,7 @@ for i in {0..1}; do set +e TEST_HEADER=$i python3 $TEST_SCRIPT_PY >$CLIENT_LOG 2>&1 if [ $? -ne 0 ]; then - cat $CLIENT_LOG + cat $CLIENT_LOG echo -e "\n***\n*** Test Failed\n***" RET=1 fi diff --git a/src/command_line_parser.cc b/src/command_line_parser.cc index 5b490a45f6..1ba9fe2175 100644 --- a/src/command_line_parser.cc +++ b/src/command_line_parser.cc @@ -1235,6 +1235,11 @@ TritonParser::Parse(int argc, char** argv) triton::server::grpc::Options& lgrpc_options = lparams.grpc_options_; #endif // TRITON_ENABLE_GRPC +#if defined TRITON_ENABLE_HTTP || defined TRITON_ENABLE_GRPC + // According to HTTP specification header names are case-insensitive. + const std::string case_insensitive_prefix{"(?i)"}; +#endif // TRITON_ENABLE_HTTP || TRITON_ENABLE_GRPC + #ifdef TRITON_ENABLE_VERTEX_AI // Set different default value if specific flag is set { @@ -1345,7 +1350,8 @@ TritonParser::Parse(int argc, char** argv) lparams.http_address_ = optarg; break; case OPTION_HTTP_HEADER_FORWARD_PATTERN: - lparams.http_forward_header_pattern_ = optarg; + lparams.http_forward_header_pattern_ = + std::move(case_insensitive_prefix + optarg); break; break; case OPTION_HTTP_THREAD_COUNT: @@ -1484,7 +1490,8 @@ TritonParser::Parse(int argc, char** argv) break; } case OPTION_GRPC_HEADER_FORWARD_PATTERN: - lgrpc_options.forward_header_pattern_ = optarg; + lgrpc_options.forward_header_pattern_ = + std::move(case_insensitive_prefix + optarg); break; #endif // TRITON_ENABLE_GRPC From dcf2c94989ea36a983691d0a7c00b50d2e6c23e2 Mon Sep 17 00:00:00 2001 From: Yingge He Date: Thu, 15 Feb 2024 20:13:50 -0800 Subject: [PATCH 02/15] fix indentation --- qa/L0_parameters/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/L0_parameters/test.sh b/qa/L0_parameters/test.sh index 9d2396be07..fce504ff55 100755 --- a/qa/L0_parameters/test.sh +++ b/qa/L0_parameters/test.sh @@ -75,7 +75,7 @@ for i in {0..2}; do set +e TEST_HEADER=$i python3 $TEST_SCRIPT_PY >$CLIENT_LOG 2>&1 if [ $? -ne 0 ]; then - cat $CLIENT_LOG + cat $CLIENT_LOG echo -e "\n***\n*** Test Failed\n***" RET=1 fi From 95244bf5f27bc81bd20b541ca1af92f4280db96d Mon Sep 17 00:00:00 2001 From: Yingge He Date: Thu, 15 Feb 2024 23:19:36 -0800 Subject: [PATCH 03/15] fix pre-comiit errors --- qa/L0_parameters/parameters_test.py | 4 ++-- qa/L0_parameters/test.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/qa/L0_parameters/parameters_test.py b/qa/L0_parameters/parameters_test.py index 8c28675b97..a18b0ea86e 100755 --- a/qa/L0_parameters/parameters_test.py +++ b/qa/L0_parameters/parameters_test.py @@ -54,14 +54,14 @@ async def asyncSetUp(self): self.async_grpc = asyncgrpcclient.InferenceServerClient(url="localhost:8001") self.parameter_list = [] - if TEST_HEADER in ["0","1"]: + if TEST_HEADER in ["0", "1"]: self.parameter_list.append({"key1": "value1", "key2": "value2"}) self.parameter_list.append({"key1": 1, "key2": 2}) self.parameter_list.append({"key1": 123.123, "key2": 321.321}) self.parameter_list.append({"key1": True, "key2": "value2"}) self.parameter_list.append({"triton_": True, "key2": "value2"}) - if TEST_HEADER in ["1","2"]: + if TEST_HEADER in ["1", "2"]: self.headers = { "header_1": "value_1", "header_2": "value_2", diff --git a/qa/L0_parameters/test.sh b/qa/L0_parameters/test.sh index fce504ff55..01e45f6db2 100755 --- a/qa/L0_parameters/test.sh +++ b/qa/L0_parameters/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions From 08793a46bbcc3f2d29b98e953dcc352c97d9e10c Mon Sep 17 00:00:00 2001 From: Yingge He Date: Tue, 20 Feb 2024 16:19:27 -0800 Subject: [PATCH 04/15] Update doc --- docs/protocol/extension_parameters.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/protocol/extension_parameters.md b/docs/protocol/extension_parameters.md index 4cdb60cf38..843d3e9472 100644 --- a/docs/protocol/extension_parameters.md +++ b/docs/protocol/extension_parameters.md @@ -98,6 +98,11 @@ value. For example to forward all the headers that start with 'PREFIX_' from both HTTP and GRPC, you should add `--http-header-forward-pattern PREFIX_.* --grpc-header-forward-pattern PREFIX_.*` to your `tritonserver` command. +By default, the regular expression pattern matches headers with case-insensitive +mode according to the HTTP protocol. If you want to enforce case-sensitive mode, +simplying adding the `(?-i)` prefix which turns off case-insensitive mode, e.g. +`--http-header-forward-pattern (?-i)PREFIX_.*`. + The forwarded headers can be accessed using the [Python](https://github.com/triton-inference-server/python_backend#inference-request-parameters) or C Backend APIs as inference request parameters. From b6579f99877bf6bca36f18479128562e625ff05e Mon Sep 17 00:00:00 2001 From: Yingge He Date: Tue, 20 Feb 2024 16:22:05 -0800 Subject: [PATCH 05/15] Update copyright --- docs/protocol/extension_parameters.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/protocol/extension_parameters.md b/docs/protocol/extension_parameters.md index 843d3e9472..c02ec3cfd2 100644 --- a/docs/protocol/extension_parameters.md +++ b/docs/protocol/extension_parameters.md @@ -1,5 +1,5 @@