From 2fc9b4149f2f09a4b72290148a7cded568287afa Mon Sep 17 00:00:00 2001 From: Stella TAN Date: Wed, 10 Aug 2022 12:27:01 +0200 Subject: [PATCH 1/5] Add OpenCV function NLMeans add associated parameters --- src/CMakeLists.txt | 2 +- src/aliceVision/image/convertionOpenCV.hpp | 30 ++++++- src/software/utils/main_imageProcessing.cpp | 95 ++++++++++++++++++--- 3 files changed, 111 insertions(+), 16 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1d72b6865e..cdcab81b7f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -474,7 +474,7 @@ set(ALICEVISION_HAVE_OCVSIFT 0) if(ALICEVISION_BUILD_SFM) if(NOT ALICEVISION_USE_OPENCV STREQUAL "OFF") - find_package(OpenCV COMPONENTS core imgproc video imgcodecs videoio features2d) + find_package(OpenCV COMPONENTS core imgproc video imgcodecs videoio features2d photo) if(OpenCV_FOUND) # We do not set the minimal version directly in find_package diff --git a/src/aliceVision/image/convertionOpenCV.hpp b/src/aliceVision/image/convertionOpenCV.hpp index cd0c700f71..cde891f47e 100644 --- a/src/aliceVision/image/convertionOpenCV.hpp +++ b/src/aliceVision/image/convertionOpenCV.hpp @@ -81,13 +81,35 @@ inline cv::Mat imageRGBAToCvMatBGR(const image::Image& img, i */ inline void cvMatBGRToImageRGBA(const cv::Mat& img, image::Image& imageOut) { + const float charToFloat = 1.0f / 255.0f; for(int row = 0; row < imageOut.Height(); row++) { - const cv::Vec3f* rowPtr = img.ptr(row); - for(int col = 0; col < imageOut.Width(); col++) + switch(img.type()) { - const cv::Vec3f& matPixel = rowPtr[col]; - imageOut(row, col) = image::RGBAfColor(matPixel[2], matPixel[1], matPixel[0], imageOut(row, col).a()); + case CV_32FC3: + { + const cv::Vec3f* rowPtr = img.ptr(row); + for(int col = 0; col < imageOut.Width(); col++) + { + const cv::Vec3f& matPixel = rowPtr[col]; + imageOut(row, col) = + image::RGBAfColor(matPixel[2], matPixel[1], matPixel[0], imageOut(row, col).a()); + } + break; + } + case CV_8UC3: + { + const cv::Vec3b* rowPtr = img.ptr(row); + for(int col = 0; col < imageOut.Width(); col++) + { + const cv::Vec3b& matPixel = rowPtr[col]; + imageOut(row, col) = image::RGBAfColor(matPixel[2] * charToFloat, matPixel[1] * charToFloat, + matPixel[0] * charToFloat, imageOut(row, col).a()); + } + break; + } + default: + std::runtime_error("Cannot handle OpenCV matrix type '" + std::to_string(img.type()) + "'."); } } } diff --git a/src/software/utils/main_imageProcessing.cpp b/src/software/utils/main_imageProcessing.cpp index c9b9941736..2a7f721b62 100644 --- a/src/software/utils/main_imageProcessing.cpp +++ b/src/software/utils/main_imageProcessing.cpp @@ -1,4 +1,3 @@ - #include #include #include @@ -18,6 +17,9 @@ #if ALICEVISION_IS_DEFINED(ALICEVISION_HAVE_OPENCV) #include +#include +#include +#include #endif #include @@ -213,6 +215,39 @@ inline std::istream& operator>>(std::istream& in, EImageFormat& e) return in; } +struct NLMeansFilterParams +{ + bool enabled; + float filterStrength; + float filterStrengthColor; + int templateWindowSize; + int searchWindowSize; +}; + +std::istream& operator>>(std::istream& in, NLMeansFilterParams& nlmParams) +{ + std::string token; + in >> token; + std::vector splitParams; + boost::split(splitParams, token, boost::algorithm::is_any_of(":")); + if(splitParams.size() != 5) + throw std::invalid_argument("Failed to parse NLMeansFilterParams from: " + token); + nlmParams.enabled = boost::to_lower_copy(splitParams[0]) == "true"; + nlmParams.filterStrength = boost::lexical_cast(splitParams[1]); + nlmParams.filterStrengthColor = boost::lexical_cast(splitParams[2]); + nlmParams.templateWindowSize = boost::lexical_cast(splitParams[3]); + nlmParams.searchWindowSize = boost::lexical_cast(splitParams[4]); + + return in; +} + +inline std::ostream& operator<<(std::ostream& os, const NLMeansFilterParams& nlmParams) +{ + os << nlmParams.enabled << ":" << nlmParams.filterStrength << ":" << nlmParams.filterStrengthColor << ":" + << nlmParams.templateWindowSize << ":" << nlmParams.searchWindowSize; + return os; +} + std::string getColorProfileDatabaseFolder() { const char* value = std::getenv("ALICEVISION_COLOR_PROFILE_DB"); @@ -262,6 +297,14 @@ struct ProcessingParams true // mono }; + NLMeansFilterParams nlmFilter = + { + false, // enable + 5.0f, // filterStrength + 5.0f, // filterStrengthColor + 7, // templateWindowSize + 21 // searchWindowSize + }; }; @@ -408,6 +451,27 @@ void processImage(image::Image& image, const ProcessingParams oiio::ImageBuf inBuf(oiio::ImageSpec(image.Width(), image.Height(), nchannels, oiio::TypeDesc::FLOAT), image.data()); oiio::ImageBufAlgo::noise(inBuf, ENoiseMethod_enumToString(pParams.noise.method), pParams.noise.A, pParams.noise.B, pParams.noise.mono); } + + if(pParams.nlmFilter.enabled) + { +#if ALICEVISION_IS_DEFINED(ALICEVISION_HAVE_OPENCV) + + // Create temporary OpenCV Mat (keep only 3 Channels) to handled Eigen data of our image + cv::Mat openCVMatIn = image::imageRGBAToCvMatBGR(image, CV_8UC3); + cv::Mat openCVMatOut(image.Width(), image.Height(), CV_8UC3); + + cv::fastNlMeansDenoisingColored(openCVMatIn, openCVMatOut, pParams.nlmFilter.filterStrength, + pParams.nlmFilter.filterStrengthColor, pParams.nlmFilter.templateWindowSize, + pParams.nlmFilter.searchWindowSize); + + // Copy filtered data from openCV Mat(3 channels) to our image(keep the alpha channel unfiltered) + image::cvMatBGRToImageRGBA(openCVMatOut, image); + +#else + throw std::invalid_argument( + "Unsupported mode! If you intended to use a non-local means filter, please add OpenCV support."); +#endif + } } void saveImage(image::Image& image, const std::string& inputPath, const std::string& outputPath, std::map inputMetadata, @@ -582,14 +646,22 @@ int aliceVision_main(int argc, char * argv[]) " * TileGridSize: Sets Size Of Grid For Histogram Equalization. Input Image Will Be Divided Into Equally Sized Rectangular Tiles.") ("noiseFilter", po::value(&pParams.noise)->default_value(pParams.noise), - "Noise Filter parameters:\n" - " * Enabled: Add Noise.\n" - " * method: There are several noise types to choose from:\n" - " - uniform: adds noise values uninformly distributed on range [A,B).\n" - " - gaussian: adds Gaussian (normal distribution) noise values with mean value A and standard deviation B.\n" - " - salt: changes to value A a portion of pixels given by B.\n" - " * A, B: parameters that have a different interpretation depending on the method chosen.\n" - " * mono: If is true, a single noise value will be applied to all channels otherwise a separate noise value will be computed for each channel.") + "Noise Filter parameters:\n" + " * Enabled: Add Noise.\n" + " * method: There are several noise types to choose from:\n" + " - uniform: adds noise values uninformly distributed on range [A,B).\n" + " - gaussian: adds Gaussian (normal distribution) noise values with mean value A and standard deviation B.\n" + " - salt: changes to value A a portion of pixels given by B.\n" + " * A, B: parameters that have a different interpretation depending on the method chosen.\n" + " * mono: If is true, a single noise value will be applied to all channels otherwise a separate noise value will be computed for each channel.") + + ("nlmFilter", po::value(&pParams.nlmFilter)->default_value(pParams.nlmFilter), + "Non local means Filter parameters:\n" + " * Enabled: Use non local means Filter.\n" + " * H: Parameter regulating filter strength. Bigger H value perfectly removes noise but also removes image details, smaller H value preserves details but also preserves some noise.\n" + " * HColor: Parameter regulating filter strength for color images only. Normally same as Filtering Parameter H. Not necessary for grayscale images\n " + " * templateWindowSize: Size in pixels of the template patch that is used to compute weights. Should be odd. \n" + " * searchWindowSize:Size in pixels of the window that is used to compute weighted average for given pixel. Should be odd. Affect performance linearly: greater searchWindowsSize - greater denoising time.") ("workingColorSpace", po::value(&workingColorSpace)->default_value(workingColorSpace), ("Working color space: " + image::EImageColorSpace_informations()).c_str()) @@ -635,9 +707,10 @@ int aliceVision_main(int argc, char * argv[]) } #if !ALICEVISION_IS_DEFINED(ALICEVISION_HAVE_OPENCV) - if(pParams.bilateralFilter.enabled || pParams.claheFilter.enabled) + if(pParams.bilateralFilter.enabled || pParams.claheFilter.enabled || pParams.nlmFilter.enabled) { - ALICEVISION_LOG_ERROR("Invalid option: BilateralFilter and claheFilter can't be used without openCV !"); + ALICEVISION_LOG_ERROR( + "Invalid option: BilateralFilter, claheFilter and nlmFilter can't be used without openCV !"); return EXIT_FAILURE; } #endif From c00ebf2d25db36441934dc2ea1cfdf7b4620daa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Candice=20Bent=C3=A9jac?= Date: Fri, 2 Dec 2022 00:45:32 +0100 Subject: [PATCH 2/5] [utils] ImageProcessing: Remove unnecessary imports OpenCV's core/core.hpp and photo/cuda.hpp are not needed to use the implementation of fast non-local means denoising. --- src/software/utils/main_imageProcessing.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/software/utils/main_imageProcessing.cpp b/src/software/utils/main_imageProcessing.cpp index 2a7f721b62..15700c401e 100644 --- a/src/software/utils/main_imageProcessing.cpp +++ b/src/software/utils/main_imageProcessing.cpp @@ -17,9 +17,7 @@ #if ALICEVISION_IS_DEFINED(ALICEVISION_HAVE_OPENCV) #include -#include #include -#include #endif #include From ab2ded901ed172dee7cde53ca92238271492ed75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Candice=20Bent=C3=A9jac?= Date: Fri, 2 Dec 2022 01:13:11 +0100 Subject: [PATCH 3/5] [utils] ImageProcessing: fix NL Means comments and default values This commit corrects a few typos in the comments dedicated to the NL Means and it updates the default value of the "filterStrengthColor" parameter to 10.f (instead of 5.f), as recommended by OpenCV's documentation. --- src/software/utils/main_imageProcessing.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/software/utils/main_imageProcessing.cpp b/src/software/utils/main_imageProcessing.cpp index 15700c401e..3e0af00abe 100644 --- a/src/software/utils/main_imageProcessing.cpp +++ b/src/software/utils/main_imageProcessing.cpp @@ -299,7 +299,7 @@ struct ProcessingParams { false, // enable 5.0f, // filterStrength - 5.0f, // filterStrengthColor + 10.0f, // filterStrengthColor 7, // templateWindowSize 21 // searchWindowSize }; @@ -453,8 +453,7 @@ void processImage(image::Image& image, const ProcessingParams if(pParams.nlmFilter.enabled) { #if ALICEVISION_IS_DEFINED(ALICEVISION_HAVE_OPENCV) - - // Create temporary OpenCV Mat (keep only 3 Channels) to handled Eigen data of our image + // Create temporary OpenCV Mat (keep only 3 channels) to handle Eigen data of our image cv::Mat openCVMatIn = image::imageRGBAToCvMatBGR(image, CV_8UC3); cv::Mat openCVMatOut(image.Width(), image.Height(), CV_8UC3); @@ -462,7 +461,7 @@ void processImage(image::Image& image, const ProcessingParams pParams.nlmFilter.filterStrengthColor, pParams.nlmFilter.templateWindowSize, pParams.nlmFilter.searchWindowSize); - // Copy filtered data from openCV Mat(3 channels) to our image(keep the alpha channel unfiltered) + // Copy filtered data from OpenCV Mat(3 channels) to our image (keep the alpha channel unfiltered) image::cvMatBGRToImageRGBA(openCVMatOut, image); #else From 62b3f484352fbed07dfa5274bac98cb75d2f02b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Candice=20Bent=C3=A9jac?= Date: Fri, 2 Dec 2022 01:01:43 +0100 Subject: [PATCH 4/5] [image] Add a template implementation for BGR to RGBA conversion This commit moves the implementation of cvMatBGRToImageRGBA into a dedicated template function (cvMatBGRToImageRGBAImpl). The documentation for both methods is added and updated where needed. --- src/aliceVision/image/convertionOpenCV.hpp | 66 ++++++++++++---------- 1 file changed, 35 insertions(+), 31 deletions(-) diff --git a/src/aliceVision/image/convertionOpenCV.hpp b/src/aliceVision/image/convertionOpenCV.hpp index cde891f47e..50d2b64695 100644 --- a/src/aliceVision/image/convertionOpenCV.hpp +++ b/src/aliceVision/image/convertionOpenCV.hpp @@ -73,47 +73,51 @@ inline cv::Mat imageRGBAToCvMatBGR(const image::Image& img, i /** - * @brief Converts an openCv image (cv::Mat) in BGR to an aliceVision image + * @brief Implements the conversion of an OpenCV image (cv::Mat) in BGR to an aliceVision image * Keeps the alpha channel of the output image unchanged - * @param[in] img - Input openCV image (cv::Mat) - * @param[out] img - output RGBA aliceVision image - * @return the resulting regex + * @tparam VecType - OpenCV vector type to interpret the img values with + * @param[in] img - input OpenCV image (supported OpenCV image types: CV_32FC3, CV_8UC3) + * @param[inout] imageOut - output RGBA aliceVision image + * @param[in] factor - optional scale factor */ -inline void cvMatBGRToImageRGBA(const cv::Mat& img, image::Image& imageOut) +template +inline void cvMatBGRToImageRGBAImpl(const cv::Mat& img, image::Image& imageOut, float factor = 1.f) { - const float charToFloat = 1.0f / 255.0f; for(int row = 0; row < imageOut.Height(); row++) { - switch(img.type()) + const VecType* rowPtr = img.ptr(row); + for(int col = 0; col < imageOut.Width(); col++) { - case CV_32FC3: - { - const cv::Vec3f* rowPtr = img.ptr(row); - for(int col = 0; col < imageOut.Width(); col++) - { - const cv::Vec3f& matPixel = rowPtr[col]; - imageOut(row, col) = - image::RGBAfColor(matPixel[2], matPixel[1], matPixel[0], imageOut(row, col).a()); - } - break; - } - case CV_8UC3: - { - const cv::Vec3b* rowPtr = img.ptr(row); - for(int col = 0; col < imageOut.Width(); col++) - { - const cv::Vec3b& matPixel = rowPtr[col]; - imageOut(row, col) = image::RGBAfColor(matPixel[2] * charToFloat, matPixel[1] * charToFloat, - matPixel[0] * charToFloat, imageOut(row, col).a()); - } - break; - } - default: - std::runtime_error("Cannot handle OpenCV matrix type '" + std::to_string(img.type()) + "'."); + const VecType& matPixel = rowPtr[col]; + imageOut(row, col) = image::RGBAfColor(matPixel[2] * factor, matPixel[1] * factor, + matPixel[0] * factor, imageOut(row, col).a()); } } } + +/** + * @brief Converts an OpenCV image (cv::Mat) in BGR to an aliceVision image + * Keeps the alpha channel of the output image unchanged + * @param[in] img - input OpenCV image (supported OpenCV image types: CV_32FC3, CV_8UC3) + * @param[inout] imageOut - output RGBA aliceVision image + * @return the resulting aliceVision image + */ +inline void cvMatBGRToImageRGBA(const cv::Mat& img, image::Image& imageOut) +{ + switch(img.type()) + { + case CV_32FC3: + cvMatBGRToImageRGBAImpl(img, imageOut); + break; + case CV_8UC3: + cvMatBGRToImageRGBAImpl(img, imageOut, 1.0f / 255.0f); + break; + default: + std::runtime_error("Cannot handle OpenCV matrix type '" + std::to_string(img.type()) + "'."); + } +} + } // namespace image } // namespace aliceVision From 7e029a71707115fae2293ae3eab8a2c9821dc7c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Candice=20Bent=C3=A9jac?= Date: Fri, 2 Dec 2022 01:03:43 +0100 Subject: [PATCH 5/5] [image] ConvertionOpenCV: improve the documentation This commit harmonizes the descriptions of existing and unmodified functions, and corrects an error with the Doxygen tags. --- src/aliceVision/image/convertionOpenCV.hpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/aliceVision/image/convertionOpenCV.hpp b/src/aliceVision/image/convertionOpenCV.hpp index 50d2b64695..c129d7034a 100644 --- a/src/aliceVision/image/convertionOpenCV.hpp +++ b/src/aliceVision/image/convertionOpenCV.hpp @@ -29,7 +29,6 @@ namespace image * @param[in] color - value to set * @param[in] factor - optional scale factor * @param[in] delta - optional delta added to the scaled values - * @return the resulting openCV image */ template inline void setValueCvMatBGR(cv::Mat& mat, int i, int j, const image::RGBAfColor& color, float factor = 1.f, @@ -42,11 +41,11 @@ inline void setValueCvMatBGR(cv::Mat& mat, int i, int j, const image::RGBAfColor /** - * @brief Converts an aliceVision image to an openCv image (cv::Mat) in BGR + * @brief Converts an aliceVision image to an OpenCV image (cv::Mat) in BGR * Ignores the alpha channel of the source image - * @param[in] img - Input RGBA aliceVision image + * @param[in] img - input RGBA aliceVision image * @param[in] cvtype - OpenCV mat type (supported values: CV_32FC3, CV_8UC3) - * @return the resulting openCV image + * @return the resulting OpenCV image */ inline cv::Mat imageRGBAToCvMatBGR(const image::Image& img, int cvtype = CV_32FC3) {