Merge pull request #2821 from ROCm/develop-upstream-sync-250121

Develop upstream sync 250121

.bazelrc

@@ -132,6 +132,10 @@ build --incompatible_enforce_config_setting_visibility
 # TODO: also enable this flag after fixing the visibility violations
 # build --incompatible_config_setting_private_default_visibility
 
+# Print a stacktrace when a test is killed
+test --test_env="GTEST_INSTALL_FAILURE_SIGNAL_HANDLER=1"
+
+
 # Default options should come above this line.
 
 # Android configs. Bazel needs to have --cpu and --fat_apk_cpu both set to the
@@ -295,6 +299,8 @@ build:tpu --define=enable_mlir_bridge=true
 build:rocm_base --crosstool_top=@local_config_rocm//crosstool:toolchain
 build:rocm_base --define=using_rocm_hipcc=true
 build:rocm_base --define=tensorflow_mkldnn_contraction_kernel=0
+build:rocm_base --define=xnn_enable_avxvnniint8=false
+build:rocm_base --define=xnn_enable_avx512fp16=false
 build:rocm_base --repo_env TF_NEED_ROCM=1
 
 build:rocm --config=rocm_base

ci/devinfra/docker/windows/Dockerfile

@@ -1,3 +1,7 @@
+# NOTE: This Dockerfile is no longer in use.
+# It is kept just in case, but it's recommended to use the 2022 version,
+# and that is what internal CI uses as well.
+
 # This Dockerfile creates an image that has:
 # - the correct MTU setting for networking from inside the container to work.
 # - Visual Studio 2022 Build Tools

ci/devinfra/docker/windows2022/Dockerfile

@@ -0,0 +1,207 @@
+# This Dockerfile creates an image that has:
+# - the correct MTU setting for networking from inside the container to work.
+# - Visual Studio 2022 Build Tools
+# - MSVC 14.39
+# - LLVM/Clang 18.1.4
+# - MSYS2 + curl, git, patch, vim, unzip, zip
+# - Python 3.9 - 3.13
+# - Bazelisk 1.22.1
+# - JDK 21 (Azul Zulu)
+
+FROM mcr.microsoft.com/windows/servercore:ltsc2022
+
+SHELL ["powershell.exe", "-ExecutionPolicy", "Bypass", "-Command", \
+       "$ErrorActionPreference='Stop'; $ProgressPreference='SilentlyContinue';$VerbosePreference = 'Continue';"]
+
+# Enable long paths
+RUN New-ItemProperty -Path "HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem" -Name "LongPathsEnabled" -Value 1 -PropertyType DWORD -Force
+
+RUN md C:\TEMP
+RUN md C:\TMP
+ENV TMP "C:/TMP"
+ENV TEMP "C:/TEMP"
+
+# Install 7-Zip.
+RUN (New-Object Net.WebClient).DownloadFile('https://www.7-zip.org/a/7z2201-x64.msi', '7z.msi'); \
+    Start-Process msiexec.exe -ArgumentList \"/i 7z.msi /qn /norestart /log C:\\TEMP\\7z_install_log.txt\" -wait; \
+    Remove-Item .\7z.msi;
+
+# Download the Visual Studio 2022 Installer.
+RUN (New-Object Net.WebClient).DownloadFile('https://aka.ms/vs/17/release/vs_community.exe', 'C:\TEMP\vs_community.exe');
+# Install Visual Studio 2022 Build Tools + Compiler
+SHELL ["cmd", "/S", "/C"]
+# Packages, and component versions, can be found here:
+# https://learn.microsoft.com/en-us/visualstudio/install/workload-component-id-vs-build-tools
+RUN C:\TEMP\vs_community.exe \
+        --quiet --wait --norestart --nocache \
+        --add Microsoft.VisualStudio.Component.VC.Tools.x86.x64 \
+        --add Microsoft.VisualStudio.Workload.NativeDesktop \
+        --add Microsoft.VisualStudio.Component.VC.14.39.17.9.x86.64 \
+        --add Microsoft.VisualStudio.Component.Windows11SDK.22621 \
+ || IF "%ERRORLEVEL%"=="3010" EXIT 0
+
+SHELL ["powershell.exe", "-ExecutionPolicy", "Bypass", "-Command", \
+       "$ErrorActionPreference='Stop'; $ProgressPreference='SilentlyContinue'; $VerbosePreference = 'Continue';"]
+
+# Install Clang.
+RUN (New-Object Net.WebClient).DownloadFile( \
+         'https://github.com/llvm/llvm-project/releases/download/llvmorg-18.1.4/LLVM-18.1.4-win64.exe', \
+         'LLVM.exe'); \
+    Start-Process -FilePath \"C:\Program Files\7-Zip\7z.exe\" -ArgumentList 'x LLVM.exe -oC:\tools\LLVM' -Wait; \
+    $env:PATH = [Environment]::GetEnvironmentVariable('PATH', 'Machine') + ';C:\tools\LLVM\bin'; \
+    [Environment]::SetEnvironmentVariable('PATH', $env:PATH, 'Machine');
+
+# Install MSYS2.
+RUN (New-Object Net.WebClient).DownloadFile( \
+         'https://repo.msys2.org/distrib/x86_64/msys2-base-x86_64-20240727.tar.xz', \
+         'msys2.tar.xz'); \
+    Start-Process -FilePath \"C:\Program Files\7-Zip\7z.exe\" -ArgumentList 'x msys2.tar.xz -oC:\TEMP\msys2.tar' -Wait; \
+    Start-Process -FilePath \"C:\Program Files\7-Zip\7z.exe\" -ArgumentList 'x C:\TEMP\msys2.tar -oC:\tools' -Wait; \
+    $env:PATH = [Environment]::GetEnvironmentVariable('PATH', 'Machine') + ';C:\tools\msys64;C:\tools\msys64\usr\bin\'; \
+    [Environment]::SetEnvironmentVariable('PATH', $env:PATH, 'Machine');
+
+# Disable signature checking on pacman because we cannot initialize the keyring.
+RUN Add-Content -Path C:\tools\msys64\etc\pacman.d\mirrorlist.mingw32 -Value 'SigLevel = Never'
+RUN Add-Content -Path C:\tools\msys64\etc\pacman.d\mirrorlist.mingw64 -Value 'SigLevel = Never'
+RUN Add-Content -Path C:\tools\msys64\etc\pacman.d\mirrorlist.msys -Value 'SigLevel = Never'
+
+# Install pacman packages.
+RUN C:\tools\msys64\usr\bin\bash.exe -lc \
+  'pacman --noconfirm -Syy curl git patch vim unzip zip'
+
+# Install multiple Pythons, but only add one of them to PATH.
+RUN function Install-Python { \
+        param( \
+            [string]$version, \
+            [int]$prependPath \
+        ) \
+        $url = ('https://www.python.org/ftp/python/{0}/python-{0}-amd64.exe' -f $version); \
+        Write-Host ('Downloading {0}...' -f $url); \
+        [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; \
+        (New-Object Net.WebClient).DownloadFile($url, 'C:\tmp\pyinstall.exe'); \
+        \
+        # Without the patch version \
+        $truncatedVersion = $($version -replace '\.\d+$', ''); \
+        $installDir = 'C:\Python' + $truncatedVersion; \
+        Write-Host ('Installing into {0} (PrependPath: {1})...' -f $installDir, $($prependPath -eq 1)); \
+        $argumentList = ('/quiet InstallAllUsers=1 PrependPath={0} TargetDir={1}' -f $prependPath, $installDir); \
+        Start-Process -FilePath 'C:\tmp\pyinstall.exe' -ArgumentList $argumentList -Wait; \
+        \
+        Write-Host 'Verifying install...'; \
+        Write-Host \"  python --version $version\"; & $installDir\python.exe --version; \
+        \
+        Write-Host 'Verifying pip install...'; \
+        & $installDir\python.exe -m pip --version; \
+        \
+        Write-Host 'Updating pip...'; \
+        & $installDir\python.exe -m pip install --upgrade pip; \
+        \
+        Write-Host 'Installing/updating packages...'; \
+        & $installDir\python.exe -m pip install --upgrade setuptools packaging; \
+        \
+        Write-Host 'Removing installation binary...'; \
+        Remove-Item C:\tmp\pyinstall.exe -Force; \
+    }; \
+    Write-Host 'Installing multiple Python versions...'; \
+    $versions = @( \
+        @{ version = '3.9.13'; prependPath = 0 }, \
+        @{ version = '3.10.11'; prependPath = 0 }, \
+        @{ version = '3.11.9'; prependPath = 0 }, \
+        @{ version = '3.12.8'; prependPath = 0 }, \
+        @{ version = '3.13.1'; prependPath = 1 } \
+    ); \
+    foreach ($v in $versions) { \
+        Install-Python -version $v.version -prependPath $v.prependPath; \
+    }; \
+    Write-Host 'Python installations complete.';
+
+# Add a python3 symlink for the Python in PATH.
+# It's not feasible to add one for each version, as on Windows,
+# Python uses PATH and the binary's (symlink's) location, to launch itself.
+RUN C:\tools\msys64\usr\bin\bash.exe -lc 'ln -s /c/Python3.13/python.exe /usr/bin/python3';
+
+# Install JDK 21.
+RUN \
+  Add-Type -AssemblyName \"System.IO.Compression.FileSystem\"; \
+  $zulu_pkg = \"zulu21.34.19-ca-jdk21.0.3-win_x64.zip\"; \
+  $zulu_url = \"https://cdn.azul.com/zulu/bin/${zulu_pkg}\"; \
+  $zulu_zip = \"c:\\temp\\${zulu_pkg}\"; \
+  $zulu_extracted_path = \"c:\\temp\\\" + [IO.Path]::GetFileNameWithoutExtension($zulu_zip); \
+  $zulu_root = \"c:\\openjdk\"; \
+  (New-Object Net.WebClient).DownloadFile($zulu_url, $zulu_zip); \
+  [System.IO.Compression.ZipFile]::ExtractToDirectory($zulu_zip, \"c:\\temp\"); \
+  Move-Item $zulu_extracted_path -Destination $zulu_root; \
+  Remove-Item $zulu_zip; \
+  $env:PATH = [Environment]::GetEnvironmentVariable(\"PATH\", \"Machine\") + \";${zulu_root}\\bin\"; \
+  [Environment]::SetEnvironmentVariable(\"PATH\", $env:PATH, \"Machine\"); \
+  $env:JAVA_HOME = $zulu_root; \
+  [Environment]::SetEnvironmentVariable(\"JAVA_HOME\", $env:JAVA_HOME, \"Machine\")
+
+# Point to the LLVM installation.
+# The Bazel Windows guide claims it can find LLVM automatically,
+# but it likely only works if it's installed somewhere inside C:\Program Files.
+ENV BAZEL_LLVM "C:\tools\LLVM"
+
+# These variables may be useful, but so far haven't been. Keeping for posterity.
+# ENV CLANG_COMPILER_PATH "C:\tools\llvm\bin\clang.exe"
+# ENV CC "C:\tools\llvm\bin\clang.exe"
+# ENV BAZEL_COMPILER "C:\tools\llvm\bin\clang.exe"
+
+ENV BAZEL_SH "C:\tools\msys64\usr\bin\bash.exe"
+ENV BAZEL_VS "C:\Program Files\Microsoft Visual Studio\2022\BuildTools"
+ENV BAZEL_VC "C:\Program Files\Microsoft Visual Studio\2022\Community\VC"
+
+# Environment variables to prevent auto-conversion of Linux-like paths to Windows paths.
+# This is necessary as some paths end up invalid/mangled.
+ENV MSYS_NO_PATHCONV 1
+ENV MSYS2_ARG_CONV_EXCL *
+
+# This should only be necessary if there are multiple, differently-versioned
+# MSVC compilers installed, and a particular one should be used.
+# To find exact versions available:
+# - Navigate to the relevant folder, e.g.
+#     C:\Program Files\Microsoft Visual Studio\2022
+# - Search for the `cl.exe` file: `gci -r -fi cl.exe`
+# - The version will be part of the found path, e.g.
+#     2022\Community\VC\Tools\MSVC\14.39.33519\bin\Hostx64\x64
+# ENV BAZEL_VC_FULL_VERSION 14.39.33519
+
+# Install Bazelisk.
+RUN md C:\tools\bazel
+RUN (New-Object Net.WebClient).DownloadFile( \
+         'https://github.com/bazelbuild/bazelisk/releases/download/v1.22.1/bazelisk-windows-amd64.exe', \
+         'C:\tools\bazel\bazel.exe'); \
+    $env:PATH = [Environment]::GetEnvironmentVariable('PATH', 'Machine') + ';C:\tools\bazel'; \
+    [Environment]::SetEnvironmentVariable('PATH', $env:PATH, 'Machine');
+
+# Install gcloud, and add it to PATH
+ENV CLOUDSDK_CORE_DISABLE_PROMPTS 1
+RUN (New-Object Net.WebClient).DownloadFile('https://dl.google.com/dl/cloudsdk/channels/rapid/google-cloud-sdk.zip', 'C:\Temp\google-cloud-sdk.zip'); \
+    Expand-Archive -Path 'C:\Temp\google-cloud-sdk.zip' -DestinationPath $env:ProgramFiles -Verbose:$false
+RUN & \"$env:ProgramFiles\\google-cloud-sdk\\install.bat\" --path-update false
+RUN $env:Path += \";$env:ProgramFiles\\google-cloud-sdk\\bin\"; \
+    [Environment]::SetEnvironmentVariable('Path', $env:Path, [EnvironmentVariableTarget]::Machine);
+# Re-enable prompts for interactive use.
+ENV CLOUDSDK_CORE_DISABLE_PROMPTS=""
+
+# MSYS attempts to use non-cmd versions, which aren't meant for Windows
+RUN Add-Content -Path C:\tools\msys64\.bashrc -Value 'alias gcloud=gcloud.cmd'
+RUN Add-Content -Path C:\tools\msys64\.bashrc -Value 'alias gsutil=gsutil.cmd'
+RUN Add-Content -Path C:\tools\msys64\.bashrc -Value 'alias bq=bq.cmd'
+
+# Symlink a directory, to have it pretend be the T:\ drive.
+# This drive letter is used by internal CI,
+# and part of it is mounted to the container during the container's creation.
+#
+# While the mount argument (`-v host_path:container_path`) still requires
+# `container_path` to be a legitimate C:\ path, in this case, 'C:\drive_t',
+# this symlink does allow for the convenience of passing unedited paths
+# to `docker exec` commands, e.g., 'T:\path', instead of 'C:\path',
+# without having to replace the drive letter with C:\ every time.
+# Such a workaround is not required on Linux, since it
+# can create arbitrary paths within the container, e.g., '/t'.
+# Note: This does not affect/work for `docker cp` commands.
+RUN New-Item -ItemType directory -Path C:\drive_t; \
+    New-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Session Manager\DOS Devices' -Name 'T:'  -Value '\??\C:\drive_t' -PropertyType String;
+
+CMD ["bash", "-i"]

ci/official/envs/linux_x86

@@ -25,6 +25,5 @@ TFCI_OUTPUT_DIR=build_output
 TFCI_WHL_AUDIT_ENABLE=1
 TFCI_WHL_AUDIT_PLAT=manylinux2014_x86_64
 TFCI_WHL_BAZEL_TEST_ENABLE=1
-# TODO: Set back to 240M once the wheel size is fixed.
-TFCI_WHL_SIZE_LIMIT=270M
+TFCI_WHL_SIZE_LIMIT=260M
 TFCI_WHL_SIZE_LIMIT_ENABLE=1

ci/official/envs/linux_x86_cuda

@@ -18,4 +18,5 @@ TFCI_BAZEL_TARGET_SELECTING_CONFIG_PREFIX=linux_cuda
 TFCI_BUILD_PIP_PACKAGE_ARGS="--repo_env=WHEEL_NAME=tensorflow"
 TFCI_DOCKER_ARGS="--gpus all"
 TFCI_LIB_SUFFIX="-gpu-linux-x86_64"
-TFCI_WHL_SIZE_LIMIT=610M
+# TODO: Set back to 610M once the wheel size is fixed.
+TFCI_WHL_SIZE_LIMIT=620M

ci/official/envs/macos_arm64

@@ -21,8 +21,7 @@ TFCI_MACOS_BAZEL_TEST_DIR_ENABLE=1
 TFCI_MACOS_BAZEL_TEST_DIR_PATH="/Volumes/BuildData/bazel_output"
 TFCI_OUTPUT_DIR=build_output
 TFCI_WHL_BAZEL_TEST_ENABLE=1
-# TODO: Set back to 240M once the wheel size is fixed.
-TFCI_WHL_SIZE_LIMIT=270M
+TFCI_WHL_SIZE_LIMIT=240M
 TFCI_WHL_SIZE_LIMIT_ENABLE=1
 
 # 3.11 is the system python on our images

ci/official/envs/windows_x86_2022

@@ -31,14 +31,15 @@ TFCI_WHL_SIZE_LIMIT_ENABLE=1
 TFCI_WHL_IMPORT_TEST_ENABLE=1
 TFCI_PYTHON_VERIFY_PIP_INSTALL_ARGS=""
 
-# TODO(belitskiy): Add a link to the Dockerfile comment that explains this more.
 # Used to simulate a T:\ drive within the container, to a limited extent,
 # via a symlink.
 # Helpful since the internal CI utilizes a T:\ drive, part of which is mounted
 # to the container, and would result in C:\<path> != T:\<path> mismatches,
 # when using variables like `TFCI_OUTPUT_DIR` in `docker exec commands,
 # requiring conditional path adjustments throughout the CI scripts.
 # Note: This does not work for `docker cp` commands.
+# For a little more detail, see:
+# https://github.com/tensorflow/tensorflow/blob/cc3006af6fea8d2a1360b3b5e65f9d14bbdb7094/ci/devinfra/docker/windows2022/Dockerfile#L196
 TFCI_OUTPUT_WIN_DOCKER_DIR='C:/drive_t'
 
 # Docker on Windows doesn't support the `host` networking mode, and so

ci/official/utilities/repack_libtensorflow.sh

@@ -66,15 +66,18 @@ if [[ $(uname -s) != MSYS_NT* ]]; then
   cp_normalized_srcjar bazel-bin/tensorflow/java/libtensorflow-src.jar "${DIR}/libtensorflow-src.jar"
   cp bazel-bin/tensorflow/tools/lib_package/libtensorflow_proto.zip "${DIR}"
 else
-  LIB_PKG="$1/lib_package"
+  # Temporary directory for package zipping
+  LIB_PKG="$DIR/lib_package"
   mkdir -p ${LIB_PKG}
 
   # Zip up the .dll and the LICENSE for the JNI library.
   cp bazel-bin/tensorflow/java/tensorflow_jni.dll ${LIB_PKG}/tensorflow_jni.dll
-  zip -j ${LIB_PKG}/libtensorflow_jni-cpu-windows-$(uname -m).zip \
+  libtensorflow_jni_zip="libtensorflow_jni${TARBALL_SUFFIX}.zip"
+  zip -j "$libtensorflow_jni_zip" \
     ${LIB_PKG}/tensorflow_jni.dll \
     bazel-bin/tensorflow/tools/lib_package/include/tensorflow/THIRD_PARTY_TF_JNI_LICENSES \
     LICENSE
+  mv "$libtensorflow_jni_zip" "$DIR"
   rm -f ${LIB_PKG}/tensorflow_jni.dll
 
   # Zip up the .dll, LICENSE and include files for the C library.
@@ -110,9 +113,8 @@ else
   cp third_party/xla/third_party/tsl/tsl/platform/ctstring.h \
      third_party/xla/third_party/tsl/tsl/platform/ctstring_internal.h \
      ${LIB_PKG}/include/tsl/platform
-  cp LICENSE ${LIB_PKG}/LICENSE
-  cp bazel-bin/tensorflow/tools/lib_package/THIRD_PARTY_TF_C_LICENSES ${LIB_PKG}/
   cd ${LIB_PKG}
+  libtensorflow_zip="libtensorflow${TARBALL_SUFFIX}.zip"
   zip libtensorflow-cpu-windows-$(uname -m).zip \
     lib/tensorflow.dll \
     lib/tensorflow.lib \
@@ -138,10 +140,9 @@ else
     include/tsl/platform/ctstring_internal.h \
     LICENSE \
     THIRD_PARTY_TF_C_LICENSES
-  rm -rf lib include
+  mv "$LIB_PKG/$libtensorflow_zip" "$DIR"
 
-  cd ..
-  tar -zcvf windows_cpu_libtensorflow_binaries.tar.gz $LIB_PKG
+  cd "$DIR"
   rm -rf $LIB_PKG
 
 fi

tensorflow/c/BUILD

@@ -698,10 +698,10 @@ tf_cc_test(
     deps = [
         ":tf_status",
         ":tf_status_helper",
+        "@com_google_googletest//:gtest_main",
         "@local_tsl//tsl/platform:errors",
         "@local_tsl//tsl/platform:status",
         "@local_tsl//tsl/platform:test",
-        "@local_tsl//tsl/platform:test_main",
     ],
 )
 

tensorflow/c/env.cc

@@ -110,7 +110,7 @@ void TF_AppendWritableFile(TF_WritableFileHandle* handle, const char* data,
   auto* cc_file = reinterpret_cast<::tensorflow::WritableFile*>(handle);
   TF_SetStatus(status, TF_OK, "");
   ::tensorflow::Set_TF_Status_from_Status(
-      status, cc_file->Append(::tensorflow::StringPiece{data, length}));
+      status, cc_file->Append(absl::string_view{data, length}));
 }
 
 void TF_DeleteFile(const char* filename, TF_Status* status) {

tensorflow/c/experimental/filesystem/modular_filesystem.h

@@ -148,9 +148,9 @@ class ModularRandomAccessFile final : public RandomAccessFile {
 
   ~ModularRandomAccessFile() override { ops_->cleanup(file_.get()); }
 
-  absl::Status Read(uint64 offset, size_t n, StringPiece* result,
+  absl::Status Read(uint64 offset, size_t n, absl::string_view* result,
                     char* scratch) const override;
-  absl::Status Name(StringPiece* result) const override;
+  absl::Status Name(absl::string_view* result) const override;
 
  private:
   std::string filename_;
@@ -169,11 +169,11 @@ class ModularWritableFile final : public WritableFile {
 
   ~ModularWritableFile() override { ops_->cleanup(file_.get()); }
 
-  absl::Status Append(StringPiece data) override;
+  absl::Status Append(absl::string_view data) override;
   absl::Status Close() override;
   absl::Status Flush() override;
   absl::Status Sync() override;
-  absl::Status Name(StringPiece* result) const override;
+  absl::Status Name(absl::string_view* result) const override;
   absl::Status Tell(int64_t* position) override;
 
  private:

tensorflow/c/experimental/filesystem/plugins/posix/copy_file_linux.cc

@@ -16,6 +16,8 @@ limitations under the License.
 #include <stdint.h>
 #include <sys/sendfile.h>
 
+#include <cstddef>
+
 #include "tensorflow/c/experimental/filesystem/plugins/posix/copy_file.h"
 
 namespace tf_posix_filesystem {

tensorflow/c/experimental/stream_executor/BUILD

@@ -45,7 +45,10 @@ cc_library(
         "//tensorflow/core/common_runtime/device:device_utils",
         "//tensorflow/core/platform:strcat",
         "@com_google_absl//absl/functional:any_invocable",
+        "@com_google_absl//absl/log",
+        "@com_google_absl//absl/log:check",
         "@com_google_absl//absl/status",
+        "@com_google_absl//absl/status:statusor",
         "@com_google_absl//absl/strings:str_format",
         "@com_google_absl//absl/types:optional",
         "@local_tsl//tsl/platform:status",