diff --git a/src/itkOMEZarrNGFFImageIO.cxx b/src/itkOMEZarrNGFFImageIO.cxx index 3ccf1d1..fe5aef0 100644 --- a/src/itkOMEZarrNGFFImageIO.cxx +++ b/src/itkOMEZarrNGFFImageIO.cxx @@ -78,6 +78,29 @@ ReadFromStore(const tensorstore::TensorStore<> & store, const ImageIORegion & io } } +// Update an existing "read" specification for an "http" driver to retrieve remote files. +// Note that an "http" driver specification may operate on an HTTP or HTTPS connection. +void +MakeKVStoreHTTPDriverSpec(nlohmann::json & spec, const std::string & fullPath) +{ + // Decompose path into a base URL and reference subpath according to TensorStore HTTP KVStore driver spec + // https://google.github.io/tensorstore/kvstore/http/index.html + spec["kvstore"] = { { "driver", "http" } }; + + // Naively decompose the URL into "base" and "resource" components. + // Generally assumes that the spec will only be used once to access a specific resource. + // For example, the URL "http://localhost/path/to/resource.json" will be split + // into components "http://localhost/path/to" and "resource.json". + // + // Could be revisited for a better root "base_url" at the top level allowing acces + // to multiple subpaths. For instance, decomposing the example above into + // "http://localhost/" and "path/to/resource.json" would allow for a given HTTP spec + // to be more easily reused with different subpaths. + // + spec["kvstore"]["base_url"] = fullPath.substr(0, fullPath.find_last_of("/")); + spec["kvstore"]["path"] = fullPath.substr(fullPath.find_last_of("/") + 1); +} + } // namespace thread_local tensorstore::Context tsContext = tensorstore::Context::Default(); @@ -229,6 +252,10 @@ getKVstoreDriver(std::string path) { return "file"; } + if (path.substr(0, 4) == "http") + { // http or https + return "http"; + } if (path.substr(path.size() - 4) == ".zip" || path.substr(path.size() - 7) == ".memory") { return "zip"; @@ -262,10 +289,13 @@ bool jsonRead(const std::string path, nlohmann::json & result, std::string driver) { // Reading JSON via TensorStore allows it to be in the cloud - auto attrs_store = tensorstore::Open( - { { "driver", "json" }, { "kvstore", { { "driver", driver }, { "path", path } } } }, tsContext) - .result() - .value(); + nlohmann::json readSpec = { { "driver", "json" }, { "kvstore", { { "driver", driver }, { "path", path } } } }; + if (driver == "http") + { + MakeKVStoreHTTPDriverSpec(readSpec, path); + } + + auto attrs_store = tensorstore::Open(readSpec, tsContext).result().value(); auto attrs_array_result = tensorstore::Read(attrs_store).result(); @@ -336,12 +366,17 @@ thread_local tensorstore::TensorStore<> store; // initialized by ReadImageInform void OMEZarrNGFFImageIO::ReadArrayMetadata(std::string path, std::string driver) { - auto openFuture = - tensorstore::Open({ { "driver", "zarr" }, { "kvstore", { { "driver", driver }, { "path", path } } } }, - tsContext, - tensorstore::OpenMode::open, - tensorstore::RecheckCached{ false }, - tensorstore::ReadWriteMode::read); + nlohmann::json readSpec = { { "driver", "zarr" }, { "kvstore", { { "driver", driver }, { "path", path } } } }; + if (driver == "http") + { + MakeKVStoreHTTPDriverSpec(readSpec, path); + } + + auto openFuture = tensorstore::Open(readSpec, + tsContext, + tensorstore::OpenMode::open, + tensorstore::RecheckCached{ false }, + tensorstore::ReadWriteMode::read); TS_EVAL_CHECK(openFuture); store = openFuture.value(); auto shape_span = store.domain().shape(); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 91fe558..bad4a6c 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,8 +1,10 @@ itk_module_test() set(IOOMEZarrNGFFTests + itkOMEZarrNGFFHTTPTest.cxx itkOMEZarrNGFFImageIOTest.cxx itkOMEZarrNGFFInMemoryTest.cxx + itkOMEZarrNGFFReadTest.cxx itkOMEZarrNGFFReadSubregionTest.cxx ) @@ -102,3 +104,17 @@ COMMAND IOOMEZarrNGFFTestDriver ${ITK_TEST_OUTPUT_DIR}/cthead1.zarr ${ITK_TEST_OUTPUT_DIR}/cthead1Subregion.mha ) + +# HTTP test with encoded test cases +itk_add_test( + NAME IOOMEZarrNGFFHTTP_2D + COMMAND IOOMEZarrNGFFTestDriver + itkOMEZarrNGFFHTTPTest + 0 +) +itk_add_test( + NAME IOOMEZarrNGFFHTTP_3D + COMMAND IOOMEZarrNGFFTestDriver + itkOMEZarrNGFFHTTPTest + 1 +) diff --git a/test/itkOMEZarrNGFFHTTPTest.cxx b/test/itkOMEZarrNGFFHTTPTest.cxx new file mode 100644 index 0000000..3cc1a5d --- /dev/null +++ b/test/itkOMEZarrNGFFHTTPTest.cxx @@ -0,0 +1,185 @@ +/*========================================================================= + * + * Copyright NumFOCUS + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0.txt + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + *=========================================================================*/ + +// Read an OME-Zarr image from a remote store. +// Example data is available at https://github.com/ome/ome-ngff-prototypes + +#include +#include "itkImageFileReader.h" +#include "itkImageFileWriter.h" +#include "itkOMEZarrNGFFImageIO.h" +#include "itkOMEZarrNGFFImageIOFactory.h" +#include "itkTestingMacros.h" +#include "itkImageIOBase.h" + +namespace +{ +bool +test2DImage() +{ + using ImageType = itk::Image; + const std::string resourceURL = "https://s3.embl.de/i2k-2020/ngff-example-data/v0.4/yx.ome.zarr"; + + itk::OMEZarrNGFFImageIOFactory::RegisterOneFactory(); + + // Baseline image metadata + auto baselineImage0 = ImageType::New(); + baselineImage0->SetRegions(itk::MakeSize(1024, 930)); + // unit origin, spacing, direction + + auto baselineImage1 = ImageType::New(); + baselineImage1->SetRegions(itk::MakeSize(512, 465)); + const float spacing1[2] = { 2.0, 2.0 }; + baselineImage1->SetSpacing(spacing1); + // unit origin, direction + + auto baselineImage2 = ImageType::New(); + baselineImage2->SetRegions(itk::MakeSize(256, 232)); + const float spacing2[2] = { 4.0, 4.0 }; + baselineImage2->SetSpacing(spacing2); + // unit origin, direction + + // Resolution 0 + auto image = itk::ReadImage(resourceURL); + image->Print(std::cout); + + ITK_TEST_EXPECT_EQUAL(image->GetLargestPossibleRegion(), baselineImage0->GetLargestPossibleRegion()); + ITK_TEST_EXPECT_EQUAL(image->GetBufferedRegion(), baselineImage0->GetLargestPossibleRegion()); + ITK_TEST_EXPECT_EQUAL(image->GetSpacing(), baselineImage0->GetSpacing()); + ITK_TEST_EXPECT_EQUAL(image->GetOrigin(), baselineImage0->GetOrigin()); + + // Resolution 1 + auto imageIO = itk::OMEZarrNGFFImageIO::New(); + imageIO->SetDatasetIndex(1); + auto reader1 = itk::ImageFileReader::New(); + reader1->SetFileName(resourceURL); + reader1->SetImageIO(imageIO); + reader1->Update(); + image = reader1->GetOutput(); + image->Print(std::cout); + ITK_TEST_EXPECT_EQUAL(image->GetLargestPossibleRegion(), baselineImage1->GetLargestPossibleRegion()); + ITK_TEST_EXPECT_EQUAL(image->GetBufferedRegion(), baselineImage1->GetLargestPossibleRegion()); + ITK_TEST_EXPECT_EQUAL(image->GetSpacing(), baselineImage1->GetSpacing()); + ITK_TEST_EXPECT_EQUAL(image->GetOrigin(), baselineImage1->GetOrigin()); + + // Resolution 2 + imageIO->SetDatasetIndex(2); + auto reader2 = itk::ImageFileReader::New(); + reader2->SetFileName(resourceURL); + reader2->SetImageIO(imageIO); + reader2->Update(); + image = reader2->GetOutput(); + image->Print(std::cout); + ITK_TEST_EXPECT_EQUAL(image->GetLargestPossibleRegion(), baselineImage2->GetLargestPossibleRegion()); + ITK_TEST_EXPECT_EQUAL(image->GetBufferedRegion(), baselineImage2->GetLargestPossibleRegion()); + ITK_TEST_EXPECT_EQUAL(image->GetSpacing(), baselineImage2->GetSpacing()); + ITK_TEST_EXPECT_EQUAL(image->GetOrigin(), baselineImage2->GetOrigin()); + + return EXIT_SUCCESS; +} + +bool +test3DImage() +{ + using ImageType = itk::Image; + const std::string resourceURL = "https://s3.embl.de/i2k-2020/ngff-example-data/v0.4/zyx.ome.zarr"; + + // Baseline image metadata + auto baselineImage0 = ImageType::New(); + baselineImage0->SetRegions(itk::MakeSize(483, 393, 603)); + const float spacing0[3] = { 64, 64, 64 }; + baselineImage0->SetSpacing(spacing0); + + auto baselineImage1 = ImageType::New(); + baselineImage1->SetRegions(itk::MakeSize(242, 196, 302)); + const float spacing1[3] = { 128, 128, 128 }; + baselineImage1->SetSpacing(spacing1); + + auto baselineImage2 = ImageType::New(); + baselineImage2->SetRegions(itk::MakeSize(121, 98, 151)); + const float spacing2[3] = { 256, 256, 256 }; + baselineImage2->SetSpacing(spacing2); + + itk::OMEZarrNGFFImageIOFactory::RegisterOneFactory(); + + // Resolution 0 + auto image = itk::ReadImage(resourceURL); + image->Print(std::cout); + + ITK_TEST_EXPECT_EQUAL(image->GetLargestPossibleRegion(), baselineImage0->GetLargestPossibleRegion()); + ITK_TEST_EXPECT_EQUAL(image->GetBufferedRegion(), baselineImage0->GetLargestPossibleRegion()); + ITK_TEST_EXPECT_EQUAL(image->GetSpacing(), baselineImage0->GetSpacing()); + ITK_TEST_EXPECT_EQUAL(image->GetOrigin(), baselineImage0->GetOrigin()); + + // Resolution 1 + auto imageIO = itk::OMEZarrNGFFImageIO::New(); + imageIO->SetDatasetIndex(1); + auto reader1 = itk::ImageFileReader::New(); + reader1->SetFileName(resourceURL); + reader1->SetImageIO(imageIO); + reader1->Update(); + image = reader1->GetOutput(); + image->Print(std::cout); + ITK_TEST_EXPECT_EQUAL(image->GetLargestPossibleRegion(), baselineImage1->GetLargestPossibleRegion()); + ITK_TEST_EXPECT_EQUAL(image->GetBufferedRegion(), baselineImage1->GetLargestPossibleRegion()); + ITK_TEST_EXPECT_EQUAL(image->GetSpacing(), baselineImage1->GetSpacing()); + ITK_TEST_EXPECT_EQUAL(image->GetOrigin(), baselineImage1->GetOrigin()); + + // Resolution 2 + imageIO->SetDatasetIndex(2); + auto reader2 = itk::ImageFileReader::New(); + reader2->SetFileName(resourceURL); + reader2->SetImageIO(imageIO); + reader2->Update(); + image = reader2->GetOutput(); + image->Print(std::cout); + ITK_TEST_EXPECT_EQUAL(image->GetLargestPossibleRegion(), baselineImage2->GetLargestPossibleRegion()); + ITK_TEST_EXPECT_EQUAL(image->GetBufferedRegion(), baselineImage2->GetLargestPossibleRegion()); + ITK_TEST_EXPECT_EQUAL(image->GetSpacing(), baselineImage2->GetSpacing()); + ITK_TEST_EXPECT_EQUAL(image->GetOrigin(), baselineImage2->GetOrigin()); + + return EXIT_SUCCESS; +} +} // namespace + +int +itkOMEZarrNGFFHTTPTest(int argc, char * argv[]) +{ + if (argc < 2) + { + std::cerr << "Missing parameters." << std::endl; + std::cerr << "Usage: " << std::endl; + std::cerr << itkNameOfTestExecutableMacro(argv) << " " << std::endl; + return EXIT_FAILURE; + } + size_t testCase = std::atoi(argv[1]); + + switch (testCase) + { + case 0: + return test2DImage(); + break; + case 1: + return test3DImage(); + break; + default: + throw std::invalid_argument("Invalid test case ID: " + std::to_string(testCase)); + } + + return EXIT_FAILURE; +} diff --git a/test/itkOMEZarrNGFFReadTest.cxx b/test/itkOMEZarrNGFFReadTest.cxx new file mode 100644 index 0000000..a65ff33 --- /dev/null +++ b/test/itkOMEZarrNGFFReadTest.cxx @@ -0,0 +1,101 @@ +/*========================================================================= + * + * Copyright NumFOCUS + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0.txt + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + *=========================================================================*/ + +/// This test utility may be used to validate that an OME-Zarr image can be +/// read from disk or from an HTTP source. +/// +/// No attempt is made to validate input data. A summary of the retrieved image +/// is printed to `std::cout`. +/// +/// Does not currently support multichannel sources. +/// https://github.com/InsightSoftwareConsortium/ITKIOOMEZarrNGFF/issues/32 + +#include +#include + +#include "itkImageRegionConstIteratorWithIndex.h" +#include "itkImageFileReader.h" +#include "itkImageFileWriter.h" +#include "itkExtractImageFilter.h" +#include "itkOMEZarrNGFFImageIO.h" +#include "itkOMEZarrNGFFImageIOFactory.h" +#include "itkTestingMacros.h" +#include "itkImageIOBase.h" + +namespace +{ +template +void +doRead(const std::string & path, const int datasetIndex) +{ + using ImageType = itk::Image; + auto imageReader = itk::ImageFileReader::New(); + imageReader->SetFileName(path); + + auto imageIO = itk::OMEZarrNGFFImageIO::New(); + imageIO->SetDatasetIndex(datasetIndex); + imageReader->SetImageIO(imageIO); + + imageReader->UpdateOutputInformation(); + auto output = imageReader->GetOutput(); + output->Print(std::cout); + + imageReader->Update(); + output->Print(std::cout); +} +} // namespace + +int +itkOMEZarrNGFFReadTest(int argc, char * argv[]) +{ + if (argc < 2) + { + std::cerr << "Missing parameters." << std::endl; + std::cerr << "Usage: " << std::endl; + std::cerr << itkNameOfTestExecutableMacro(argv) << " Input [NumChannels]" + << std::endl; + return EXIT_FAILURE; + } + const char * inputFileName = argv[1]; + const size_t imageDimension = (argc > 2 ? std::atoi(argv[2]) : 3); + const size_t datasetIndex = (argc > 3 ? std::atoi(argv[3]) : 0); + const size_t numChannels = (argc > 4 ? std::atoi(argv[4]) : 1); + + if (numChannels != 1) + { + throw std::runtime_error("Multichannel image reading is not currently supported"); + } + else if (imageDimension == 2) + { + doRead(inputFileName, datasetIndex); + } + else if (imageDimension == 3) + { + doRead(inputFileName, datasetIndex); + } + else if (imageDimension == 4) + { + doRead(inputFileName, datasetIndex); + } + else + { + throw std::invalid_argument("Received an invalid test case"); + } + + return EXIT_SUCCESS; +} diff --git a/wrapping/test/CMakeLists.txt b/wrapping/test/CMakeLists.txt index eda3476..5627d18 100644 --- a/wrapping/test/CMakeLists.txt +++ b/wrapping/test/CMakeLists.txt @@ -15,3 +15,22 @@ itk_python_add_test(NAME itkOMEZarrNGFFImageIOReadConvertTestPython ${ITK_TEST_OUTPUT_DIR}/cthead1py.zarr ${ITK_TEST_OUTPUT_DIR}/cthead1.png ) + +itk_python_add_test( + NAME itkOMEZarrNGFFHTTPReadLocalTestPython + COMMAND itkOMEZarrNGFFHTTPReadLocalTestPython.py + DATA{${test_input_dir}/cthead1.mha} + ${ITK_TEST_OUTPUT_DIR}/cthead1.zarr +) + +itk_python_add_test( + NAME itkOMEZarrNGFFHTTPReadRemoteTest2DPython + COMMAND itkOMEZarrNGFFHTTPReadRemoteTestPython.py + https://s3.embl.de/i2k-2020/ngff-example-data/v0.4/yx.ome.zarr +) + +itk_python_add_test( + NAME itkOMEZarrNGFFHTTPReadRemoteTest3DPython + COMMAND itkOMEZarrNGFFHTTPReadRemoteTestPython.py + https://s3.embl.de/i2k-2020/ngff-example-data/v0.4/zyx.ome.zarr +) diff --git a/wrapping/test/itkOMEZarrNGFFHTTPReadLocalTestPython.py b/wrapping/test/itkOMEZarrNGFFHTTPReadLocalTestPython.py new file mode 100644 index 0000000..06dcd0c --- /dev/null +++ b/wrapping/test/itkOMEZarrNGFFHTTPReadLocalTestPython.py @@ -0,0 +1,75 @@ +#========================================================================== +# +# Copyright NumFOCUS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0.txt +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +#==========================================================================*/ + +# Test reading OME-Zarr NGFF chunked data over HTTP. +# +# This test spawns a Python web server child process to serve from the +# test data directory on port 9999, then reads the local OME-Zarr file +# as if it were remotely served. + +import os +import subprocess +import sys + +import itk +import numpy as np + +TEST_PORT = 9999 +LOCALHOST_BINDING = '127.0.0.1' + +itk.auto_progress(2) + +imageio = itk.OMEZarrNGFFImageIO.New() + +if(len(sys.argv) < 2): + raise ValueError('Expected arguments: ') + +# Test setup: create OME-Zarr store on local disk +print(f"Reading {sys.argv[1]}") +image = itk.imread(sys.argv[1], pixel_type=itk.F) + +# Assign arbitrary spacing and origin to assess metadata I/O +# Note that direction is not supported under the OME-Zarr v0.4 spec +image.SetSpacing(np.arange(0.2, 0.1 * image.GetImageDimension() + 0.19, 0.1)) +image.SetOrigin(np.arange(0.6, 0.1 * image.GetImageDimension() + 0.59, 0.1)) + +print(f"Writing {sys.argv[2]}") +itk.imwrite(image, sys.argv[2], imageio=imageio, compression=False) + +# Serve files on "localhost" in the background +p = subprocess.Popen([sys.executable, + '-m', 'http.server', str(TEST_PORT), + '--directory', os.path.dirname(sys.argv[2]), + '--bind', LOCALHOST_BINDING]) + +url = f'http://localhost:{TEST_PORT}/{os.path.basename(sys.argv[2])}' + +print(f"Reading {url}") +image2 = itk.imread(url, imageio=imageio) +print(image2) + +# Compare metadata +assert np.all(np.array(itk.size(image2)) == np.array(itk.size(image))), 'Image size mismatch' +assert np.all(np.array(itk.spacing(image2)) == np.array(itk.spacing(image))), 'Image spacing mismatch' +assert np.all(np.array(itk.origin(image2)) == np.array(itk.origin(image))), 'Image origin mismatch' + +# Compare data +assert np.all(itk.array_view_from_image(image2) == itk.array_view_from_image(image)), 'Image data mismatch' + +# Clean up +p.kill() diff --git a/wrapping/test/itkOMEZarrNGFFHTTPReadRemoteTestPython.py b/wrapping/test/itkOMEZarrNGFFHTTPReadRemoteTestPython.py new file mode 100644 index 0000000..963593d --- /dev/null +++ b/wrapping/test/itkOMEZarrNGFFHTTPReadRemoteTestPython.py @@ -0,0 +1,48 @@ +#========================================================================== +# +# Copyright NumFOCUS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0.txt +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +#==========================================================================*/ + +# Test reading OME-Zarr NGFF chunked data over HTTP. +# +# This test accepts a remote OME-Zarr endpoint and reads +# its data over HTTP/HTTPS. + +import sys + +import itk + +itk.auto_progress(2) + +if(len(sys.argv) < 2): + raise ValueError('Expected arguments: [datasetIndex]') + +assert sys.argv[1].startswith('http'), 'OME-Zarr remote store location must be an HTTP endpoint' +assert sys.argv[1].endswith('.zarr'), 'Expected `.zarr` store remote location' + +imageio = None +if len(sys.argv) > 2: + dataset_index = int(sys.argv[2]) + imageio = itk.OMEZarrNGFFImageIO.New(dataset_index=dataset_index) + +# Read dataset from remote server +# TODO: Add support for 4D images and multichannel images +# https://github.com/InsightSoftwareConsortium/ITKIOOMEZarrNGFF/issues/32 +print(f"Reading {sys.argv[1]}") +image = itk.imread(sys.argv[1], pixel_type=itk.F, imageio=imageio) + +print(f'Read in image:') +print(image)