Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

opencv4: cuda: fix libstdc++ errors downstream, plus upkeep #265070

Merged
merged 5 commits into from
Nov 13, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions pkgs/development/libraries/opencv/3.x.nix
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
{ lib, stdenv
, fetchFromGitHub
, fetchpatch
, callPackage
, cmake, pkg-config, unzip, zlib, pcre, hdf5
, glog, boost, gflags, protobuf3_21
, config
Expand All @@ -14,8 +15,7 @@
, enableOpenblas ? true, openblas, blas, lapack
, enableContrib ? true

, enableCuda ? config.cudaSupport &&
stdenv.hostPlatform.isx86_64
, enableCuda ? config.cudaSupport
, cudaPackages ? { }
, enableUnfree ? false
, enableIpp ? false
Expand Down Expand Up @@ -289,7 +289,11 @@ stdenv.mkDerivation {

hardeningDisable = [ "bindnow" "relro" ];

passthru = lib.optionalAttrs enablePython { pythonPath = []; };
passthru = lib.optionalAttrs enablePython { pythonPath = []; } // {
tests = lib.optionalAttrs enableCuda {
no-libstdcxx-errors = callPackage ./libstdcxx-test.nix { attrName = "opencv3"; };
};
};

meta = with lib; {
description = "Open Computer Vision Library with more than 500 algorithms";
Expand Down
89 changes: 41 additions & 48 deletions pkgs/development/libraries/opencv/4.x.nix
Original file line number Diff line number Diff line change
Expand Up @@ -37,12 +37,11 @@
, blas
, enableContrib ? true

, enableCuda ? config.cudaSupport && stdenv.hostPlatform.isx86_64
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh I dropped this without thinking. Why do we ignore config on non-x86_64 in the first place?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

c4adedd

config.cudaSupport can be true and cudatoolkit doesn't work with i686,
which can happen inside pkgsi686Linux wrapping.

OK, so this wasn't about aarch64 😅

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also, why would anyone set config.cudaSupport = true on an i686 device?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I should move this to a separate commit though

, enableCuda ? config.cudaSupport
, enableCublas ? enableCuda
, enableCudnn ? false # NOTE: CUDNN has a large impact on closure size so we disable it by default
, enableCufft ? enableCuda
, cudaPackages ? {}
, symlinkJoin
, nvidia-optical-flow-sdk

, enableLto ? true
Expand Down Expand Up @@ -83,11 +82,16 @@
, Accelerate
, bzip2
, callPackage
}:
}@inputs:

let
version = "4.7.0";

# It's necessary to consistently use backendStdenv when building with CUDA
# support, otherwise we get libstdc++ errors downstream
stdenv = throw "Use effectiveStdenv instead";
effectiveStdenv = if enableCuda then cudaPackages.backendStdenv else inputs.stdenv;

src = fetchFromGitHub {
owner = "opencv";
repo = "opencv";
Expand Down Expand Up @@ -121,11 +125,11 @@ let
sha256 = "1msbkc3zixx61rcg6a04i1bcfhw1phgsrh93glq1n80hgsk3nbjq";
} + "/ippicv";
files = let name = platform: "ippicv_2019_${platform}_general_20180723.tgz"; in
if stdenv.hostPlatform.system == "x86_64-linux" then
if effectiveStdenv.hostPlatform.system == "x86_64-linux" then
{ ${name "lnx_intel64"} = "c0bd78adb4156bbf552c1dfe90599607"; }
else if stdenv.hostPlatform.system == "i686-linux" then
else if effectiveStdenv.hostPlatform.system == "i686-linux" then
{ ${name "lnx_ia32"} = "4f38432c30bfd6423164b7a24bbc98a0"; }
else if stdenv.hostPlatform.system == "x86_64-darwin" then
else if effectiveStdenv.hostPlatform.system == "x86_64-darwin" then
{ ${name "mac_intel64"} = "fe6b2bb75ae0e3f19ad3ae1a31dfa4a2"; }
else
throw "ICV is not available for this platform (or not yet supported by this package)";
Expand Down Expand Up @@ -232,35 +236,12 @@ let
#https://github.com/xianyi/OpenBLAS/wiki/Faq/4bded95e8dc8aadc70ce65267d1093ca7bdefc4c#multi-threaded
openblas_ = blas.provider.override { singleThreaded = true; };

inherit (cudaPackages) backendStdenv cudaFlags cudaVersion;
inherit (cudaPackages) cudaFlags cudaVersion;
inherit (cudaFlags) cudaCapabilities;

cuda-common-redist = with cudaPackages; [
cuda_cccl # <thrust/*>
libnpp # npp.h
] ++ lib.optionals enableCublas [
libcublas # cublas_v2.h
] ++ lib.optionals enableCudnn [
cudnn # cudnn.h
] ++ lib.optionals enableCufft [
libcufft # cufft.h
];

cuda-native-redist = symlinkJoin {
name = "cuda-native-redist-${cudaVersion}";
paths = with cudaPackages; [
cuda_cudart # cuda_runtime.h
cuda_nvcc
] ++ cuda-common-redist;
};

cuda-redist = symlinkJoin {
name = "cuda-redist-${cudaVersion}";
paths = cuda-common-redist;
};
in

stdenv.mkDerivation {
effectiveStdenv.mkDerivation {
pname = "opencv";
inherit version src;

Expand Down Expand Up @@ -319,7 +300,7 @@ stdenv.mkDerivation {

buildInputs = [ zlib pcre boost gflags protobuf3_21 ]
++ lib.optional enablePython pythonPackages.python
++ lib.optional (stdenv.buildPlatform == stdenv.hostPlatform) hdf5
++ lib.optional (effectiveStdenv.buildPlatform == effectiveStdenv.hostPlatform) hdf5
++ lib.optional enableGtk2 gtk2
++ lib.optional enableGtk3 gtk3
++ lib.optional enableVtk vtk
Expand All @@ -330,7 +311,7 @@ stdenv.mkDerivation {
++ lib.optionals enableEXR [ openexr ilmbase ]
++ lib.optional enableJPEG2000 openjpeg
++ lib.optional enableFfmpeg ffmpeg
++ lib.optionals (enableFfmpeg && stdenv.isDarwin)
++ lib.optionals (enableFfmpeg && effectiveStdenv.isDarwin)
[ VideoDecodeAcceleration bzip2 ]
++ lib.optionals enableGStreamer (with gst_all_1; [ gstreamer gst-plugins-base gst-plugins-good ])
++ lib.optional enableOvis ogre
Expand All @@ -343,11 +324,21 @@ stdenv.mkDerivation {
# tesseract & leptonica.
++ lib.optionals enableTesseract [ tesseract leptonica ]
++ lib.optional enableTbb tbb
++ lib.optionals stdenv.isDarwin [
++ lib.optionals effectiveStdenv.isDarwin [
bzip2 AVFoundation Cocoa VideoDecodeAcceleration CoreMedia MediaToolbox Accelerate
]
++ lib.optionals enableDocs [ doxygen graphviz-nox ]
++ lib.optionals enableCuda [ cuda-redist ];
++ lib.optionals enableCuda (with cudaPackages; [
cuda_cudart
cuda_cccl # <thrust/*>
libnpp # npp.h
] ++ lib.optionals enableCublas [
libcublas # cublas_v2.h
] ++ lib.optionals enableCudnn [
cudnn # cudnn.h
] ++ lib.optionals enableCufft [
libcufft # cufft.h
]);

propagatedBuildInputs = lib.optional enablePython pythonPackages.numpy
++ lib.optionals enableCuda [ nvidia-optical-flow-sdk ];
Expand All @@ -357,7 +348,9 @@ stdenv.mkDerivation {
pythonPackages.pip
pythonPackages.wheel
pythonPackages.setuptools
] ++ lib.optionals enableCuda [ cuda-native-redist ];
] ++ lib.optionals enableCuda [
cudaPackages.cuda_nvcc
];

env.NIX_CFLAGS_COMPILE = lib.optionalString enableEXR "-I${ilmbase.dev}/include/OpenEXR";

Expand Down Expand Up @@ -400,17 +393,14 @@ stdenv.mkDerivation {
(opencvFlag "ENABLE_LTO" enableLto)
(opencvFlag "ENABLE_THIN_LTO" (
enableLto && (
# Only clang supports thin LTO, so we must either be using clang through the stdenv,
stdenv.cc.isClang ||
# or through the backend stdenv.
(enableCuda && backendStdenv.cc.isClang)
# Only clang supports thin LTO, so we must either be using clang through the effectiveStdenv,
effectiveStdenv.cc.isClang ||
# or through the backend effectiveStdenv.
(enableCuda && effectiveStdenv.cc.isClang)
)
))
] ++ lib.optionals enableCuda [
"-DCUDA_FAST_MATH=ON"
# We need to set the C and C++ host compilers for CUDA to the same compiler.
"-DCMAKE_C_COMPILER=${backendStdenv.cc}/bin/cc"
"-DCMAKE_CXX_COMPILER=${backendStdenv.cc}/bin/c++"
"-DCUDA_NVCC_FLAGS=--expt-relaxed-constexpr"

# OpenCV respects at least three variables:
Expand All @@ -421,7 +411,7 @@ stdenv.mkDerivation {
"-DCUDA_ARCH_PTX=${lib.last cudaCapabilities}"

"-DNVIDIA_OPTICAL_FLOW_2_0_HEADERS_PATH=${nvidia-optical-flow-sdk}"
] ++ lib.optionals stdenv.isDarwin [
] ++ lib.optionals effectiveStdenv.isDarwin [
"-DWITH_OPENCL=OFF"
"-DWITH_LAPACK=OFF"

Expand All @@ -435,7 +425,7 @@ stdenv.mkDerivation {
"-DBUILD_JPEG=OFF"
"-DBUILD_PNG=OFF"
"-DBUILD_WEBP=OFF"
] ++ lib.optionals (!stdenv.isDarwin) [
] ++ lib.optionals (!effectiveStdenv.isDarwin) [
"-DOPENCL_LIBRARY=${ocl-icd}/lib/libOpenCL.so"
] ++ lib.optionals enablePython [
"-DOPENCV_SKIP_PYTHON_LOADER=ON"
Expand Down Expand Up @@ -489,14 +479,17 @@ stdenv.mkDerivation {
tests = {
inherit (gst_all_1) gst-plugins-bad;
}
// lib.optionalAttrs (!stdenv.isDarwin) { inherit qimgv; }
// lib.optionalAttrs (!effectiveStdenv.isDarwin) { inherit qimgv; }
// lib.optionalAttrs (!enablePython) { pythonEnabled = pythonPackages.opencv4; }
// lib.optionalAttrs (stdenv.buildPlatform != "x86_64-darwin") {
// lib.optionalAttrs (effectiveStdenv.buildPlatform != "x86_64-darwin") {
opencv4-tests = callPackage ./tests.nix {
inherit enableGStreamer enableGtk2 enableGtk3 runAccuracyTests runPerformanceTests testDataSrc;
inherit opencv4;
};
};
}
// lib.optionalAttrs (enableCuda) {
no-libstdcxx-errors = callPackage ./libstdcxx-test.nix { attrName = "opencv4"; };
};
} // lib.optionalAttrs enablePython { pythonPath = [ ]; };

meta = with lib; {
Expand Down
17 changes: 17 additions & 0 deletions pkgs/development/libraries/opencv/libstdcxx-test.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
{ python3Packages, runCommand, attrName }:

runCommand "${python3Packages.${attrName}.name}-libstdcxx-test"
{
nativeBuildInputs = [
(python3Packages.python.withPackages (ps: [
(ps.${attrName}.override { enableCuda = true; })
ps.scikit-image
]))
];
} ''
python << EOF
import cv2
from skimage.transform import pyramid_reduce
EOF
touch $out
''