From 9b570f37e39009e5c2cbbfcabe32dfac2bfbd6e6 Mon Sep 17 00:00:00 2001 From: Manuel Reis Date: Tue, 9 Jul 2024 13:32:21 +0200 Subject: [PATCH 1/3] Recognize _unknown_ S3 server type --- include/ncs3sdk.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/ncs3sdk.h b/include/ncs3sdk.h index 50f2ad2708..adc7e456be 100644 --- a/include/ncs3sdk.h +++ b/include/ncs3sdk.h @@ -15,7 +15,7 @@ /* Track the server type, if known */ typedef enum NCS3SVC {NCS3UNK=0, /* unknown */ NCS3=1, /* s3.amazon.aws */ - NCS3GS=0 /* storage.googleapis.com */ + NCS3GS=2 /* storage.googleapis.com */ } NCS3SVC; typedef struct NCS3INFO { From e6a26a28b3dea0885acc2d96c2f5bf5bf2ddf5f6 Mon Sep 17 00:00:00 2001 From: Manuel Reis Date: Tue, 9 Jul 2024 13:39:04 +0200 Subject: [PATCH 2/3] Allow region and bucket inference with NCS3UNK S3 server type --- libdispatch/ds3util.c | 50 ++++++++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/libdispatch/ds3util.c b/libdispatch/ds3util.c index aab8fe987b..5ccd01f9e7 100644 --- a/libdispatch/ds3util.c +++ b/libdispatch/ds3util.c @@ -110,17 +110,24 @@ NC_s3urlrebuild(NCURI* url, NCS3INFO* s3, NCURI** newurlp) /* split the path by "/" */ if((stat = NC_split_delim(url->path,'/',pathsegments))) goto done; - /* Distinguish path-style from virtual-host style from s3: and from other. - Virtual: https://.s3..amazonaws.com/ (1) - or: https://.s3.amazonaws.com/ -- region defaults (to us-east-1) (2) - Path: https://s3..amazonaws.com// (3) - or: https://s3.amazonaws.com// -- region defaults to us-east-1 (4) - S3: s3:/// (5) - Google: https://storage.googleapis.com// (6) - or: gs3:/// (7) - Other: https://// (8) - */ - if(url->host == NULL || strlen(url->host) == 0) + /* Distinguish path-style from virtual-host style from s3: and from other. + Virtual: + (1) https://.s3..amazonaws.com/ + (2) https://.s3.amazonaws.com/ -- region defaults (to us-east-1) + Path: + (3) https://s3..amazonaws.com// + (4) https://s3.amazonaws.com// -- region defaults to us-east-1 + S3: + (5) s3:/// + Google: + (6) https://storage.googleapis.com// + (7) gs3:/// + Other: + (8) https://// + (9) https://.s3..domain.example.com/ + (10)https://s3..example.com// + */ + if(url->host == NULL || strlen(url->host) == 0) {stat = NC_EURL; goto done;} /* Reduce the host to standard form such as s3.amazonaws.com by pulling out the @@ -168,12 +175,21 @@ NC_s3urlrebuild(NCURI* url, NCS3INFO* s3, NCURI** newurlp) /* region is unknown */ /* bucket is unknown at this point */ svc = NCS3GS; - } else { /* Presume Format (8) */ - if((host = strdup(url->host))==NULL) - {stat = NC_ENOMEM; goto done;} - /* region is unknown */ - /* bucket is unknown */ - } + } else { /* Presume Formats (8),(9),(10) */ + if (nclistlength(hostsegments) > 3 && strcasecmp(nclistget(hostsegments, 1), "s3") == 0){ + bucket = nclistremove(hostsegments, 0); + region = nclistremove(hostsegments, 2); + host = strdup(url->host + sizeof(bucket) + 1); + }else{ + if (nclistlength(hostsegments) > 2 && strcasecmp(nclistget(hostsegments, 0), "s3") == 0){ + region = nclistremove(hostsegments, 1); + } + if ((host = strdup(url->host)) == NULL){ + stat = NC_ENOMEM; + goto done; + } + } + } /* region = (1) from url, (2) s3->region, (3) default */ if(region == NULL && s3 != NULL) From b010c57e10304110eef4f9ae682437b6b07e8b18 Mon Sep 17 00:00:00 2001 From: Manuel Reis Date: Tue, 9 Jul 2024 13:39:43 +0200 Subject: [PATCH 3/3] Test inferred S3 url rebuild --- unit_test/CMakeLists.txt | 1 + unit_test/test_ncuri.c | 68 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/unit_test/CMakeLists.txt b/unit_test/CMakeLists.txt index 59f04127c3..c08055d93e 100644 --- a/unit_test/CMakeLists.txt +++ b/unit_test/CMakeLists.txt @@ -14,6 +14,7 @@ # run on Windows. SET(UNIT_TESTS test_ncuri) +add_bin_test(unit_test test_ncuri) IF(USE_X_GETOPT) SET(XGETOPTSRC "${CMAKE_CURRENT_SOURCE_DIR}/../libdispatch/XGetopt.c") diff --git a/unit_test/test_ncuri.c b/unit_test/test_ncuri.c index 56d1a98f14..dd0dbf7891 100644 --- a/unit_test/test_ncuri.c +++ b/unit_test/test_ncuri.c @@ -12,6 +12,9 @@ Test the ncuri parsing #include #include "netcdf.h" #include "ncuri.h" +#ifdef NETCDF_ENABLE_S3 +#include "ncpathmgr.h" // to initialize global state needed by NC_s3urlrebuild +#endif typedef struct Test { char* url; @@ -47,6 +50,34 @@ static Test TESTS[] = { {NULL,NULL} }; +#if defined(NETCDF_ENABLE_S3) +static Test S3TESTS[] = { +//Virtual + {"https://.s3..amazonaws.com/","https://s3..amazonaws.com//#mode=s3"}, + {"https://.s3.amazonaws.com/","https://s3.us-east-1.amazonaws.com//#mode=s3"}, +//Path + {"https://s3..amazonaws.com//","https://s3..amazonaws.com//#mode=s3"}, + {"https://s3.amazonaws.com//","https://s3.us-east-1.amazonaws.com//#mode=s3"}, +//s3 + {"s3:///","https://s3.us-east-1.amazonaws.com//#mode=s3"}, +//Google + {"https://storage.googleapis.com//","https://storage.googleapis.com//#mode=s3"}, + {"gs3:///","https://storage.googleapis.com//#mode=s3"}, +//Other +// (8) https://// + {"https:////path/2/resource/#mode=s3,zarr", "https:////path/2/resource#mode=s3,zarr"}, +// (9) https://.s3..domain.example.com/ + {"https://.s3../path/2/resource/#mode=s3,zarr", "https://s3..//path/2/resource#mode=s3,zarr"}, +// (10)https://s3..example.com// + {"https://s3..example.com/bucket/path/2/resource/#mode=s3,zarr", "https://s3..example.com/bucket/path/2/resource#mode=s3,zarr"}, + {"https://s3.example.com/bucket/path/2/resource/#mode=s3,zarr", "https://s3.example.com/bucket/path/2/resource#mode=s3,zarr"}, + {"https://server.example.com/bucket/path/2/resource#mode=s3", "https://server.example.com/bucket/path/2/resource#mode=s3"}, + {"https://prefix..s3.localhost.example.cloud/path/2/resource#mode=s3", "https://prefix..s3.localhost.example.cloud/path/2/resource#mode=s3"}, + {"https://.s3..localhost/path/2/resource#mode=s3", "https://s3..localhost//path/2/resource#mode=s3"}, + {NULL,NULL} +}; +#endif + /* Tests that should fail */ static char* XTESTS[] = { "[dap4http://localhost:8081/x", @@ -92,6 +123,43 @@ main(int argc, char** argv) } } +#ifdef NETCDF_ENABLE_S3 + nc_initialize(); + for(index=0, test=S3TESTS;test->url;test++,index++) { + int ret = 0; + NCURI* uri = NULL; + ret = ncuriparse(test->url,&uri); + if(ret != NC_NOERR) { + fprintf(stderr,"Parse fail: %s\n",test->url); + failcount++; + } else { + int iss3 = NC_iss3(uri,NULL); + if(iss3 != 0){ + NCURI * newuri = NULL; + if(NC_s3urlrebuild(uri,NULL,&newuri)){ + fprintf(stderr, "Could not reinterpret url [%d] with s3urlrebuild: %s\n",index,test->url); + fprintf(stderr,"Mismatch: [%d] expected=|%s| actual=|%s|\n",index,test->expected,"NULL"); + failcount ++; + }else{ + char* built = ncuribuild(newuri,NULL,NULL,NCURIALL); + if(built == NULL) { + fprintf(stderr,"Build fail: %s\n",test->url); + failcount++; + } else { + if(strcmp(test->expected,built) != 0) { + fprintf(stderr,"Mismatch: [%d] expected=|%s| actual=|%s|\n",index,test->expected,built); + failcount++; + } + free(built); + } + } + ncurifree(newuri); + } + } + ncurifree(uri); + } + nc_finalize(); +#endif fprintf(stderr,"%s test_ncuri\n",failcount > 0 ? "***FAIL":"***PASS"); return (failcount > 0 ? 1 : 0); }