Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Flexible Checksums Updates #3845

Merged
merged 33 commits into from
Oct 2, 2024
Merged
Changes from 1 commit
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
65cd999
Added request_checksum_calculation to SdkConfig
landonxjames Aug 25, 2024
d0a5c29
Working version of flexible checksums for requests
landonxjames Aug 28, 2024
6cac2a9
Add S3 MultiPartUpload decorator
landonxjames Aug 29, 2024
9c7ae97
Add response_checksum_validation to sdk_config
landonxjames Aug 29, 2024
fd8dc48
Cleaning up response_checksum_validation logic
landonxjames Aug 29, 2024
a40952a
Finished HttpResponseChecksum functionality
landonxjames Sep 1, 2024
3d71421
First working test of flexible checksums
landonxjames Sep 4, 2024
ffd530b
Adding first suite of working checksum tests
landonxjames Sep 4, 2024
9881f5a
Added tests for successful response checksums
landonxjames Sep 5, 2024
5557ea3
Add tests of failed response checksum validations
landonxjames Sep 5, 2024
738f305
Add default crc32 to requestAlgorithmMember shapes
landonxjames Sep 6, 2024
ab6fbf9
Update docs for checksum config enums
landonxjames Sep 6, 2024
ea410c4
Interceptor to set default request checksum algo
landonxjames Sep 9, 2024
3025611
Cleaning up some comments
landonxjames Sep 9, 2024
5c123a5
Add interceptor for defaulting S3 MPU checksum
landonxjames Sep 11, 2024
80f48ca
Refactor http request checksum interceptor
landonxjames Sep 11, 2024
24f0024
Refactor http response checksum mutation intercept
landonxjames Sep 11, 2024
50728c4
Add streaming checksum tests
landonxjames Sep 22, 2024
e9d4c89
Remove S3MPU Decorator
landonxjames Sep 22, 2024
6ae5120
Add changelog
landonxjames Sep 22, 2024
ae0a7b9
Cleaning up
landonxjames Sep 22, 2024
d1c6aad
Move checksum configs to aws_smithy_types
landonxjames Sep 22, 2024
b9d6ee0
Exclude x-amz-checksum-mode header from signing
landonxjames Sep 22, 2024
06b384f
PR feedback
landonxjames Sep 24, 2024
c4b41bc
Add comment requested in PR feedback
landonxjames Sep 24, 2024
13d1626
Update serviceHasHttpChecksumOperation
landonxjames Sep 25, 2024
1f25cb4
Update some s3 integ and protocol tests
landonxjames Sep 25, 2024
d682c8d
Moving request checksum mutator to a closure
landonxjames Sep 30, 2024
6c9c3da
Move response mutator to closure
landonxjames Sep 30, 2024
2027715
Adding user-agent metrics for flexible checksums
landonxjames Sep 27, 2024
aec2180
Add tests for checksum user-agent business metrics
landonxjames Sep 28, 2024
d714eaa
Update HttpChecksumTest to use UA test utils
landonxjames Oct 1, 2024
1339593
Adding user-agent metrics for flexible checksums (#3851)
landonxjames Oct 2, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Add streaming checksum tests
  • Loading branch information
landonxjames committed Sep 27, 2024
commit 50728c40b44759d27d0833559f773502135abcb7
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,8 @@ internal class HttpChecksumTest {
checksumResponseSuccTests.map { createResponseChecksumValidationSuccessTest(it, context) }.join("\n")
val checksumResponseFailTestWritables =
checksumResponseFailTests.map { createResponseChecksumValidationFailureTest(it, context) }.join("\n")
val checksumStreamingRequestTestWritables =
streamingRequestTests.map { createStreamingRequestChecksumCalculationTest(it, context) }.join("\n")

// Shared imports for all test types
val testBase =
Expand All @@ -152,6 +154,8 @@ internal class HttpChecksumTest {
use #{Region};
use #{pretty_assertions}::assert_eq;
use #{SdkBody};
use std::io::Write;
use http_body::Body;
""",
*preludeScope,
"Blob" to RuntimeType.smithyTypes(rc).resolve("Blob"),
Expand All @@ -173,6 +177,10 @@ internal class HttpChecksumTest {
rustCrate.integrationTest("response_checksums_fail") {
testBase.plus(checksumResponseFailTestWritables)()
}

rustCrate.integrationTest("streaming_request_checksums") {
testBase.plus(checksumStreamingRequestTestWritables)()
}
}
}

Expand Down Expand Up @@ -233,6 +241,85 @@ internal class HttpChecksumTest {
}
}

/**
* Generate tests where the request is streaming and checksum is calculated correctly
*/
private fun createStreamingRequestChecksumCalculationTest(
testDef: StreamingRequestChecksumCalculationTest,
context: ClientCodegenContext,
): Writable {
val rc = context.runtimeConfig
val moduleName = context.moduleUseName()
val algoLower = testDef.checksumAlgorithm.lowercase()
// If the algo is Crc32 don't explicitly set it to test that the default is correctly set
val setChecksumAlgo =
if (testDef.checksumAlgorithm != "Crc32") {
".checksum_algorithm($moduleName::types::ChecksumAlgorithm::${testDef.checksumAlgorithm})"
} else {
""
}
return writable {
rustTemplate(
"""
//${testDef.docs}
##[#{tokio}::test]
async fn ${algoLower}_request_checksums_work() {
let (http_client, rx) = #{capture_request}(None);
let config = $moduleName::Config::builder()
.region(Region::from_static("doesntmatter"))
.with_test_defaults()
.http_client(http_client)
.build();

let client = $moduleName::Client::from_conf(config);

let mut file = tempfile::NamedTempFile::new().unwrap();
file.as_file_mut()
.write_all("${testDef.requestPayload}".as_bytes())
.unwrap();

let streaming_body = aws_smithy_types::byte_stream::ByteStream::read_from()
.path(&file)
.build()
.await
.unwrap();

let _operation = client
.some_streaming_operation()
.body(streaming_body)
$setChecksumAlgo
.send()
.await;


let request = rx.expect_request();

let headers = request.headers();

assert_eq!(
headers.get("x-amz-trailer").unwrap(),
"x-amz-checksum-$algoLower",
);
assert_eq!(headers.get("content-encoding").unwrap(), "aws-chunked");

let mut body = request.body().try_clone().expect("body is retryable");

let mut body_data = bytes::BytesMut::new();
while let Some(data) = body.data().await {
body_data.extend_from_slice(&data.unwrap())
}

let body_string = std::str::from_utf8(&body_data).unwrap();
assert!(body_string.contains("x-amz-checksum-$algoLower:${testDef.trailerChecksum}"));
}
""",
*preludeScope,
"tokio" to CargoDependency.Tokio.toType(),
"capture_request" to RuntimeType.captureRequest(rc),
)
}
}

/**
* Generate tests where the response checksum validates successfully
*/
Expand Down Expand Up @@ -397,11 +484,43 @@ data class StreamingRequestChecksumCalculationTest(
val docs: String,
val requestPayload: String,
val checksumAlgorithm: String,
val contentHeader: String,
val trailerHeader: String,
val trailerChecksum: String,
)

val streamingRequestTests =
listOf(
StreamingRequestChecksumCalculationTest(
"CRC32 streaming checksum calculation works.",
"Hello world",
"Crc32",
"i9aeUg==",
),
StreamingRequestChecksumCalculationTest(
"CRC32C streaming checksum calculation works.",
"Hello world",
"Crc32C",
"crUfeA==",
),
// StreamingRequestChecksumCalculationTest(
// "CRC64NVME streaming checksum calculation works.",
// "Hello world",
// "Crc64Nvme",
// "uc8X9yrZrD4=",
// ),
StreamingRequestChecksumCalculationTest(
"SHA1 streaming checksum calculation works.",
"Hello world",
"Sha1",
"e1AsOh9IyGCa4hLN+2Od7jlnP14=",
),
StreamingRequestChecksumCalculationTest(
"SHA256 streaming checksum calculation works.",
"Hello world",
"Sha256",
"ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=",
),
)

data class ResponseChecksumValidationSuccessTest(
val docs: String,
val responsePayload: String,
Expand Down