From 9b9a8c4bc223804fae8f1045be78e6215bd43018 Mon Sep 17 00:00:00 2001 From: Shahzad Malik Muzaffar Date: Tue, 11 Jun 2024 18:13:43 +0200 Subject: [PATCH 01/21] Update TF v2.16.1 --- pip/numpy.file | 1 + pip/requirements.txt | 10 +++++----- tensorflow-sources.spec | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/pip/numpy.file b/pip/numpy.file index c74ac8c8402..0b308938f5c 100644 --- a/pip/numpy.file +++ b/pip/numpy.file @@ -1,3 +1,4 @@ +BuildRequires: py3-meson-python Requires: py3-cython Requires: zlib OpenBLAS diff --git a/pip/requirements.txt b/pip/requirements.txt index de98affea6b..34d2071ea0f 100644 --- a/pip/requirements.txt +++ b/pip/requirements.txt @@ -84,7 +84,7 @@ cryptography==42.0.5 cx-Oracle==8.3.0 cycler==0.12.1 # NO_AUTO_UPDATE: numpy 1.24.3 doesn't support cython 3.x -cython==0.29.35 +cython==3.0.3 dask==2024.4.2 dask-awkward==2024.3.0 decorator==5.1.1 @@ -244,7 +244,7 @@ nvidia-ml-py==12.550.52 numexpr==2.8.4 # setuptools version <64 is needed by numpy: https://github.com/pypa/setuptools/issues/3549 # NO_AUTO_UPDATE: update together with tensorflow -numpy==1.24.3 +numpy==1.26.4 # NO_AUTO_UPDATE:1 onnx==1.16.0 onnxmltools==1.12.0 @@ -376,9 +376,9 @@ tables==3.8.0 tblib==1.7.0 tabulate==0.9.0 tenacity==8.2.3 -#NO_AUTO_UPDATE:1: Force to use tensorflow 2.15x; this should match the version in tensorflow-sources.spec -tensorflow==2.15.0 -tensorboard==2.16.2 +#NO_AUTO_UPDATE:1: Force to use tensorflow 2.16.1; this should match the version in tensorflow-sources.spec +tensorflow==2.16.1 +tensorboard==2.17.0 tensorflow-io-gcs-filesystem==0.37.0 tensorflow-estimator==2.15.0 tensorboard-data-server==0.7.2 diff --git a/tensorflow-sources.spec b/tensorflow-sources.spec index 9e8d769b80b..bdc0e80f315 100644 --- a/tensorflow-sources.spec +++ b/tensorflow-sources.spec @@ -1,4 +1,4 @@ -### RPM external tensorflow-sources 2.15.0 +### RPM external tensorflow-sources 2.16.1 %define tag 76adab72b874c49eaaca0ea18e246d9309aa2b82 %define branch cms/v%{realversion} %define github_user cms-externals From e7f25dc57628c24c33390bb51893eef42ef252ba Mon Sep 17 00:00:00 2001 From: Shahzad Malik Muzaffar Date: Tue, 11 Jun 2024 18:25:53 +0200 Subject: [PATCH 02/21] update abseil-cpp 20230802.2 --- abseil-cpp.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/abseil-cpp.spec b/abseil-cpp.spec index fdfeac30c02..e8734740e62 100644 --- a/abseil-cpp.spec +++ b/abseil-cpp.spec @@ -1,4 +1,4 @@ -### RPM external abseil-cpp 20230125.3 +### RPM external abseil-cpp 20230802.2 ## INCLUDE cpp-standard Source: https://github.com/abseil/abseil-cpp/archive/%{realversion}.tar.gz From a5ab9c5e50984bf72dc87db2bd37b17411abda5a Mon Sep 17 00:00:00 2001 From: Shahzad Malik Muzaffar Date: Tue, 11 Jun 2024 18:32:50 +0200 Subject: [PATCH 03/21] update bazel-absl patch --- bazel-absl.patch | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bazel-absl.patch b/bazel-absl.patch index edf826c227a..d613a0c8794 100644 --- a/bazel-absl.patch +++ b/bazel-absl.patch @@ -8,20 +8,20 @@ index b896f1b..0646f73 100755 "com_google_absl": { - "archive": "20211102.0.tar.gz", - "sha256": "dcf71b9cba8dc0ca9940c4b316a0c796be8fab42b070bb6b7cab62b48f0e66c4", -+ "archive": "20230125.3.tar.gz", -+ "sha256": "5366d7e7fa7ba0d915014d387b66d0d002c03236448e1ba9ef98122c13b35c36", ++ "archive": "20230802.2.tar.gz", ++ "sha256": "7c11539617af1f332f0854a6fb21e296a1b29c27d03f23c7b49d4adefcd102cc", "urls": [ - "https://mirror.bazel.build/github.com/abseil/abseil-cpp/archive/refs/tags/20211102.0.tar.gz", - "https://github.com/abseil/abseil-cpp/archive/refs/tags/20211102.0.tar.gz", -+ "https://mirror.bazel.build/github.com/abseil/abseil-cpp/archive/refs/tags/20230125.3.tar.gz", -+ "https://github.com/abseil/abseil-cpp/archive/refs/tags/20230125.3.tar.gz", ++ "https://mirror.bazel.build/github.com/abseil/abseil-cpp/archive/refs/tags/20230802.2.tar.gz", ++ "https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.2.tar.gz", ], "used_in": [ "additional_distfiles", "test_WORKSPACE_files", ], - "strip_prefix": "abseil-cpp-20211102.0", -+ "strip_prefix": "abseil-cpp-20230125.3", ++ "strip_prefix": "abseil-cpp-20230802.2", }, "zstd-jni": { "archive": "v1.5.2-3.zip", From 85957cb76d2d42aeb7274ca343d9b7b0575cfba6 Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Mon, 5 Aug 2024 11:23:34 +0200 Subject: [PATCH 04/21] TF2.16: Apply abseil aarch64 patch --- abseil-cpp.spec | 2 ++ tensorflow-sources.spec | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/abseil-cpp.spec b/abseil-cpp.spec index e8734740e62..7c8a9066843 100644 --- a/abseil-cpp.spec +++ b/abseil-cpp.spec @@ -2,11 +2,13 @@ ## INCLUDE cpp-standard Source: https://github.com/abseil/abseil-cpp/archive/%{realversion}.tar.gz +Source2: https://patch-diff.githubusercontent.com/raw/abseil/abseil-cpp/pull/1732.diff BuildRequires: cmake gmake %prep %setup -n %{n}-%{realversion} +patch -p1 <%{_sourcedir}/1732.diff %build rm -rf ../build diff --git a/tensorflow-sources.spec b/tensorflow-sources.spec index bdc0e80f315..fca3d427109 100644 --- a/tensorflow-sources.spec +++ b/tensorflow-sources.spec @@ -1,5 +1,5 @@ ### RPM external tensorflow-sources 2.16.1 -%define tag 76adab72b874c49eaaca0ea18e246d9309aa2b82 +%define tag 6edb11fa48921474f9476c070c02c7bac37c02cd %define branch cms/v%{realversion} %define github_user cms-externals %define build_type opt From f3b3548ee8236d236e250dd14d3230078d34f2ab Mon Sep 17 00:00:00 2001 From: iarspider Date: Tue, 6 Aug 2024 09:17:08 +0200 Subject: [PATCH 05/21] Update py3-scipy --- pip/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pip/requirements.txt b/pip/requirements.txt index cd3179e0758..510891098c5 100644 --- a/pip/requirements.txt +++ b/pip/requirements.txt @@ -349,7 +349,7 @@ scikit-build==0.16.7 scikit-build-core==0.9.3 scikit-learn==1.2.2 scinum==2.1.0 -scipy==1.10.0 +scipy==1.12.0 secretstorage==3.3.3 semantic-version==2.10.0 threadpoolctl==3.5.0 From a819169a2d4e934f9e7c595a9c8683c84aee0c86 Mon Sep 17 00:00:00 2001 From: iarspider Date: Tue, 6 Aug 2024 15:02:21 +0200 Subject: [PATCH 06/21] Update cython as well --- pip/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pip/requirements.txt b/pip/requirements.txt index 510891098c5..e9e1fe31945 100644 --- a/pip/requirements.txt +++ b/pip/requirements.txt @@ -84,7 +84,7 @@ cryptography==42.0.5 cx-Oracle==8.3.0 cycler==0.12.1 # NO_AUTO_UPDATE: numpy 1.24.3 doesn't support cython 3.x -cython==3.0.3 +cython==3.0.10 dask==2024.4.2 dask-awkward==2024.3.0 decorator==5.1.1 From 81be62055226286fd004041125bcbf096f56044a Mon Sep 17 00:00:00 2001 From: iarspider Date: Tue, 6 Aug 2024 13:01:18 -0700 Subject: [PATCH 07/21] Try updating py3-blosc2 --- pip/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pip/requirements.txt b/pip/requirements.txt index e9e1fe31945..28c5cc7f534 100644 --- a/pip/requirements.txt +++ b/pip/requirements.txt @@ -45,7 +45,7 @@ beautifulsoup4==4.12.3 beniget==0.4.1 bleach==6.1.0 # NO_AUTO_UPDATE: needs Python 3.10 -blosc2==2.2.0 +blosc2==2.5.1 bokeh==3.4.1 boost-histogram==1.4.1 bottle==0.12.25 From f03065f80f2ce22fe38d8e0b85d8bb4c4760d0c9 Mon Sep 17 00:00:00 2001 From: iarspider Date: Wed, 7 Aug 2024 09:17:35 +0200 Subject: [PATCH 08/21] Update py3-tables --- pip/requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pip/requirements.txt b/pip/requirements.txt index 28c5cc7f534..ed7b3c4a832 100644 --- a/pip/requirements.txt +++ b/pip/requirements.txt @@ -371,8 +371,7 @@ stack-data==0.6.3 stevedore==5.2.0 subprocess32==3.5.4 sympy==1.12 -# NO_AUTO_UPDATE: Needs newer blosc2 -tables==3.8.0 +tables==3.9.0 tblib==1.7.0 tabulate==0.9.0 tenacity==8.2.3 From fd4d07d5bb106628dfeb42102e6122426655cffa Mon Sep 17 00:00:00 2001 From: iarspider Date: Mon, 12 Aug 2024 09:33:39 +0200 Subject: [PATCH 09/21] Add symlink for cudnn_frontend_archive --- tensorflow-sources.file | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow-sources.file b/tensorflow-sources.file index 05d8f970d49..c42ee5242b5 100644 --- a/tensorflow-sources.file +++ b/tensorflow-sources.file @@ -225,7 +225,7 @@ popd #FIXME: Create missing externals links pushd bazel-tensorflow-%{realversion}/external srcdir=$(dirname $(readlink zlib)) - for e in ducc farmhash_gpu_archive stablehlo ; do + for e in ducc farmhash_gpu_archive stablehlo cudnn_frontend_archive ; do echo "Check external link: $e" if [ -e ${srcdir}/$e ] ; then if [ ! -e $e ] ; then From 49ca37a90873c958a34206c84b56d061ab3c96fb Mon Sep 17 00:00:00 2001 From: iarspider Date: Mon, 12 Aug 2024 15:13:17 +0200 Subject: [PATCH 10/21] Update tensorflow.spec --- tensorflow.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow.spec b/tensorflow.spec index 79a18c59bbf..58d46a6fa92 100644 --- a/tensorflow.spec +++ b/tensorflow.spec @@ -1,4 +1,4 @@ -### RPM external tensorflow 2.15.0 +### RPM external tensorflow 2.16.1 %if "%{?vectorized_package:set}" != "set" %define source_package tensorflow-sources %else From 9abe3d254e711d5a12523a1e3f24cd900e7e54a8 Mon Sep 17 00:00:00 2001 From: iarspider Date: Tue, 13 Aug 2024 13:57:45 +0200 Subject: [PATCH 11/21] Remove more files with broken symbols --- tensorflow-xla-runtime.spec | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorflow-xla-runtime.spec b/tensorflow-xla-runtime.spec index fa3f783e6a4..e562621ad1f 100644 --- a/tensorflow-xla-runtime.spec +++ b/tensorflow-xla-runtime.spec @@ -25,6 +25,8 @@ CXXFLAGS="-fPIC -Wl,-z,defs %{arch_build_flags} ${CMS_EIGEN_CXX_FLAGS} %{selecte pushd tensorflow/xla_aot_runtime_src # remove unnecessary implementations that use symbols that are not even existing find . -type f -path '*/service/cpu/runtime_fork_join.cc' | xargs rm -f + find . -type f -path '*/service/cpu/runtime_fft.cc' | xargs rm -f + find . -type f -path '*/service/cpu/runtime_single_threaded_fft.cc' | xargs rm -f cmake . \ -DCMAKE_CXX_FLAGS="${CXXFLAGS}" \ From d7f70e3af5cb04d889b08c104056c415a821c182 Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Fri, 16 Aug 2024 10:42:15 +0200 Subject: [PATCH 12/21] Update keras --- pip/requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pip/requirements.txt b/pip/requirements.txt index ed7b3c4a832..6122220c961 100644 --- a/pip/requirements.txt +++ b/pip/requirements.txt @@ -185,8 +185,7 @@ jupyter-server-mathjax==0.2.6 jupyter-server-terminals==0.5.3 jupyterlab==4.1.8 jupyterlab-widgets==3.0.10 -#NO_AUTO_UPDATE: needs newer TF -keras==2.15.0 +keras==3.5.0 keras-applications==1.0.8 keras-preprocessing==1.1.2 keras2onnx==1.7.0 From f38a19438c4731f350ad1df7594d3afec91ddfac Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Fri, 16 Aug 2024 11:43:09 +0200 Subject: [PATCH 13/21] Add missing keras dependencies --- pip/keras.file | 1 + pip/optree.file | 1 + pip/requirements.txt | 2 ++ 3 files changed, 4 insertions(+) create mode 100644 pip/optree.file diff --git a/pip/keras.file b/pip/keras.file index 6dd53aaf7f3..a8e76c157d6 100644 --- a/pip/keras.file +++ b/pip/keras.file @@ -1,3 +1,4 @@ %define PipDownloadSourceType none Requires: py3-PyYAML py3-six Requires: py3-h5py py3-keras-applications py3-keras-preprocessing +Requires: py3-absl-py py3-ml_dtypes py3-namex py3-optree py3-rich diff --git a/pip/optree.file b/pip/optree.file new file mode 100644 index 00000000000..005b7c73656 --- /dev/null +++ b/pip/optree.file @@ -0,0 +1 @@ +Requires: py3-typing-extensions diff --git a/pip/requirements.txt b/pip/requirements.txt index 6122220c961..7c5de7c6207 100644 --- a/pip/requirements.txt +++ b/pip/requirements.txt @@ -223,6 +223,7 @@ mplhep-data==0.0.3 mpmath==1.3.0 msgpack==1.0.8 multidict==6.0.5 +namex==0.0.8 nbclassic==1.0.0 nbclient==0.10.0 nbconvert==7.16.4 @@ -249,6 +250,7 @@ onnx==1.16.0 onnxmltools==1.12.0 onnxconverter-common==1.14.0 oauthlib==3.2.2 +optree==0.12.1 opt-einsum==3.3.0 pkginfo==1.10.0 packaging==24.0 From 302fd5e98737ea6e3ff9b31d0982d05754a14694 Mon Sep 17 00:00:00 2001 From: iarspider Date: Tue, 3 Sep 2024 13:25:05 -0700 Subject: [PATCH 14/21] Update numpy.file --- pip/numpy.file | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pip/numpy.file b/pip/numpy.file index 0b308938f5c..8b2badb15df 100644 --- a/pip/numpy.file +++ b/pip/numpy.file @@ -29,8 +29,6 @@ EOF ln -s ${numpy_core} %{i}/c-api/core %define PipPostPost \ - %{relocateConfig}lib/python*/site-packages/numpy/__config__.py \ - %{relocateConfig}lib/python*/site-packages/numpy/distutils/__config__.py \ - %{relocateConfig}lib/python*/site-packages/numpy/distutils/site.cfg + %{relocateConfig}lib/python*/site-packages/numpy/__config__.py %define PipPostBuildPy rm -f %{i}/bin/f2py From 84be8329fe6357df8c1ecf3e7bae311cc7c25a82 Mon Sep 17 00:00:00 2001 From: iarspider Date: Tue, 3 Sep 2024 13:25:41 -0700 Subject: [PATCH 15/21] Update blosc2.file --- pip/blosc2.file | 3 --- 1 file changed, 3 deletions(-) diff --git a/pip/blosc2.file b/pip/blosc2.file index 08a9425f105..9ba16ea095a 100644 --- a/pip/blosc2.file +++ b/pip/blosc2.file @@ -1,5 +1,2 @@ Requires: py3-scikit-build py3-cython py3-numpy py3-msgpack py3-ndindex py3-py-cpuinfo py3-rich BuildRequires: cmake ninja - -%define PipPostPost \ - %{relocateConfig}lib64/pkgconfig/blosc2.pc \ No newline at end of file From 8df8a959f85d050f6c9802d1085c550bdcc55ddf Mon Sep 17 00:00:00 2001 From: iarspider Date: Wed, 4 Sep 2024 12:36:53 +0200 Subject: [PATCH 16/21] Update keras.file --- pip/keras.file | 1 + 1 file changed, 1 insertion(+) diff --git a/pip/keras.file b/pip/keras.file index a8e76c157d6..52ed88edd1f 100644 --- a/pip/keras.file +++ b/pip/keras.file @@ -2,3 +2,4 @@ Requires: py3-PyYAML py3-six Requires: py3-h5py py3-keras-applications py3-keras-preprocessing Requires: py3-absl-py py3-ml_dtypes py3-namex py3-optree py3-rich +Requires: py3-theano From 0509280715e282ea38e20b5894937e2fabb06d13 Mon Sep 17 00:00:00 2001 From: iarspider Date: Wed, 4 Sep 2024 12:44:23 +0200 Subject: [PATCH 17/21] Update keras.file --- pip/keras.file | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pip/keras.file b/pip/keras.file index 52ed88edd1f..f24526d7179 100644 --- a/pip/keras.file +++ b/pip/keras.file @@ -2,4 +2,4 @@ Requires: py3-PyYAML py3-six Requires: py3-h5py py3-keras-applications py3-keras-preprocessing Requires: py3-absl-py py3-ml_dtypes py3-namex py3-optree py3-rich -Requires: py3-theano +Requires: py3-Theano From fdfef333f1bdb55ecff0420ee31038298db1a4b5 Mon Sep 17 00:00:00 2001 From: iarspider Date: Wed, 4 Sep 2024 13:00:02 +0200 Subject: [PATCH 18/21] Update keras.file --- pip/keras.file | 1 - 1 file changed, 1 deletion(-) diff --git a/pip/keras.file b/pip/keras.file index f24526d7179..a8e76c157d6 100644 --- a/pip/keras.file +++ b/pip/keras.file @@ -2,4 +2,3 @@ Requires: py3-PyYAML py3-six Requires: py3-h5py py3-keras-applications py3-keras-preprocessing Requires: py3-absl-py py3-ml_dtypes py3-namex py3-optree py3-rich -Requires: py3-Theano From 1ba91fcf806f9fd4068e54ff36832bc0d0624482 Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Wed, 4 Sep 2024 15:36:52 +0200 Subject: [PATCH 19/21] Force Keras to use TensorFlow backend --- pip/keras.file | 1 + 1 file changed, 1 insertion(+) diff --git a/pip/keras.file b/pip/keras.file index a8e76c157d6..3968d0f0dc6 100644 --- a/pip/keras.file +++ b/pip/keras.file @@ -1,3 +1,4 @@ +## INITENV SET KERAS_BACKEND tensorflow %define PipDownloadSourceType none Requires: py3-PyYAML py3-six Requires: py3-h5py py3-keras-applications py3-keras-preprocessing From 3a27d69a2cc9ef7303195330fbe87e4cc3f8f0a3 Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Mon, 9 Sep 2024 11:41:06 +0200 Subject: [PATCH 20/21] Set KERAS_BACKEND in toolfile --- scram-tools.file/tools/tensorflow/tensorflow.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/scram-tools.file/tools/tensorflow/tensorflow.xml b/scram-tools.file/tools/tensorflow/tensorflow.xml index 5cdd52deef3..3e82c7fe5c3 100644 --- a/scram-tools.file/tools/tensorflow/tensorflow.xml +++ b/scram-tools.file/tools/tensorflow/tensorflow.xml @@ -3,6 +3,7 @@ + From 34311da36d5769e4cc2373c8549ce6b225f0cbfb Mon Sep 17 00:00:00 2001 From: Ivan Razumov Date: Mon, 9 Sep 2024 11:54:09 +0200 Subject: [PATCH 21/21] KERAS_BACKEND should be runtime --- scram-tools.file/tools/tensorflow/tensorflow.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scram-tools.file/tools/tensorflow/tensorflow.xml b/scram-tools.file/tools/tensorflow/tensorflow.xml index 3e82c7fe5c3..fb7e447acbe 100644 --- a/scram-tools.file/tools/tensorflow/tensorflow.xml +++ b/scram-tools.file/tools/tensorflow/tensorflow.xml @@ -3,8 +3,8 @@ - +