From 80d3d692b8c7a781668a48f90599e709eb4992a4 Mon Sep 17 00:00:00 2001
From: Ethan Stewart <ethan.stewart@amd.com>
Date: Fri, 19 Apr 2024 17:13:01 -0500
Subject: [PATCH 1/3] Revamp how libomp.so is found

An upstream llvm change enables
LLVM_ENABLE_PER_TARGET_RUNTIME_DIR by default for the openmp
build. This installs the openmp libraries into
/opt/rocm-ver/llvm/lib/x86_64-unknown-linux-gnu instead of
/opt/rocm-ver/llvm/lib. Currenty, hipBLAS only uses /lib.

Since hipBLAS uses gcc by default, find_package(OpenMP) will
locate libgomp from the gnu installation. We would prefer
to use libomp from the ROCm install. Query amdclang to find
LLVM_TARGET_TRIPLE and use as a suffix for find_library(omp).

On the other hand, the user can choose to use hipcc, which
allows find_package(OpenMP) to properly locate libomp
inside ROCm llvm.

To include legacy support, I left the hardcoded -L with
use of HIP_CLANG_ROOT as a fallback.

This would be much simpler if hipBLAS defaulted to hipcc
as the cmake compiler.
---
 clients/CMakeLists.txt | 37 ++++++++++++++++++++++++++++++++-----
 1 file changed, 32 insertions(+), 5 deletions(-)

diff --git a/clients/CMakeLists.txt b/clients/CMakeLists.txt
index 7cad5e13..16b5a5e1 100644
--- a/clients/CMakeLists.txt
+++ b/clients/CMakeLists.txt
@@ -116,13 +116,40 @@ if( BUILD_CLIENTS_BENCHMARKS OR BUILD_CLIENTS_TESTS)
 
   # if it fails to find OpenMP compile and link flags in strange configurations it can just use non-parallel reference computation
   # if there is no omp.h to find the client compilation will fail and this should be obvious, used to be REQUIRED
-  find_package(OpenMP)
+  if( CMAKE_CXX_COMPILER MATCHES ".*/hipcc$" )
+    find_package(OpenMP)
+    if (TARGET OpenMP::OpenMP_CXX)
+      set( LIBOMP_FOUND OpenMP::OpenMP_CXX)
+      get_filename_component(LIBOMP_PATH "${OpenMP_omp_LIBRARY}" PATH)
+    endif()
+  # Support for using gnu as cmake compiler.
+  else()
+    # libomp.so may be in a target subdirectory directory.
+    execute_process(
+      OUTPUT_STRIP_TRAILING_WHITESPACE
+      COMMAND bash -c "${HIP_CLANG_ROOT}/bin/amdclang --version | grep -oP '(?<=Target: ).+'"
+      OUTPUT_VARIABLE LLVM_TARGET_TRIPLE
+    )
+    find_library(LIBOMP_FOUND omp PATHS ${HIP_CLANG_ROOT}/lib PATH_SUFFIXES ${LLVM_TARGET_TRIPLE} NO_DEFAULT_PATH)
+    if (LIBOMP_FOUND)
+      get_filename_component(LIBOMP_PATH "${LIBOMP_FOUND}" PATH)
+    endif()
+  endif()
 
-  if (TARGET OpenMP::OpenMP_CXX)
-    set( COMMON_LINK_LIBS "OpenMP::OpenMP_CXX")
-    if(HIP_PLATFORM STREQUAL amd)
-      list( APPEND COMMON_LINK_LIBS "-L\"${HIP_CLANG_ROOT}/lib\"")
+  if(HIP_PLATFORM STREQUAL amd)
+    if(LIBOMP_FOUND)
+      set( COMMON_LINK_LIBS ${LIBOMP_FOUND})
       if (NOT WIN32)
+        list( APPEND COMMON_LINK_LIBS "-Wl,-rpath=${LIBOMP_PATH}")
+      endif()
+    # If cmake cannot find the proper libomp.so, use HIP_CLANG_ROOT as fallback.
+    else()
+      # Add library search paths.
+      list( APPEND COMMON_LINK_LIBS "-L\"${HIP_CLANG_ROOT}/lib/${LLVM_TARGET_TRIPLE}\"")
+      list( APPEND COMMON_LINK_LIBS "-L\"${HIP_CLANG_ROOT}/lib\"")
+      if(NOT WIN32)
+        # Add rpath.
+        list( APPEND COMMON_LINK_LIBS "-Wl,-rpath=${HIP_CLANG_ROOT}/lib/${LLVM_TARGET_TRIPLE}")
         list( APPEND COMMON_LINK_LIBS "-Wl,-rpath=${HIP_CLANG_ROOT}/lib -lomp")
       else()
         list( APPEND COMMON_LINK_LIBS "libomp")

From 26d2fb2c0af40379ab2ea7ddb19765a42e7879a3 Mon Sep 17 00:00:00 2001
From: Ethan Stewart <ethan.stewart@amd.com>
Date: Mon, 22 Apr 2024 14:41:05 -0500
Subject: [PATCH 2/3] Add libomp.so if found to both amd/cuda builds

---
 clients/CMakeLists.txt | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/clients/CMakeLists.txt b/clients/CMakeLists.txt
index 16b5a5e1..4c7f7d76 100644
--- a/clients/CMakeLists.txt
+++ b/clients/CMakeLists.txt
@@ -136,14 +136,15 @@ if( BUILD_CLIENTS_BENCHMARKS OR BUILD_CLIENTS_TESTS)
     endif()
   endif()
 
-  if(HIP_PLATFORM STREQUAL amd)
-    if(LIBOMP_FOUND)
-      set( COMMON_LINK_LIBS ${LIBOMP_FOUND})
-      if (NOT WIN32)
-        list( APPEND COMMON_LINK_LIBS "-Wl,-rpath=${LIBOMP_PATH}")
-      endif()
-    # If cmake cannot find the proper libomp.so, use HIP_CLANG_ROOT as fallback.
-    else()
+  if(LIBOMP_FOUND)
+    # LIBOMP_FOUND will contain full path to libomp.so
+    set( COMMON_LINK_LIBS ${LIBOMP_FOUND})
+    if(HIP_PLATFORM STREQUAL amd AND NOT WIN32)
+      list( APPEND COMMON_LINK_LIBS "-Wl,-rpath=${LIBOMP_PATH}")
+    endif()
+  # If cmake cannot find the proper libomp.so, use HIP_CLANG_ROOT as fallback.
+  else()
+    if(HIP_PLATFORM STREQUAL amd)
       # Add library search paths.
       list( APPEND COMMON_LINK_LIBS "-L\"${HIP_CLANG_ROOT}/lib/${LLVM_TARGET_TRIPLE}\"")
       list( APPEND COMMON_LINK_LIBS "-L\"${HIP_CLANG_ROOT}/lib\"")

From e1e41f2b59f32512e5f87b7887d3dfdf8bf4f1da Mon Sep 17 00:00:00 2001
From: Ethan Stewart <ethan.stewart@amd.com>
Date: Mon, 22 Apr 2024 17:51:29 -0500
Subject: [PATCH 3/3] Remove usage of HIP_CLANG_ROOT as cuda builds do not have
 this set

Remove amd check for rpath addtion. If we use the libomp.so from
ROCm we need the rpath.
---
 clients/CMakeLists.txt | 30 ++++++++++++++++--------------
 1 file changed, 16 insertions(+), 14 deletions(-)

diff --git a/clients/CMakeLists.txt b/clients/CMakeLists.txt
index 4c7f7d76..db0a0d7e 100644
--- a/clients/CMakeLists.txt
+++ b/clients/CMakeLists.txt
@@ -124,13 +124,17 @@ if( BUILD_CLIENTS_BENCHMARKS OR BUILD_CLIENTS_TESTS)
     endif()
   # Support for using gnu as cmake compiler.
   else()
+    # Discover path to amdclang
+    get_filename_component(AMDCLANG_PATH "${HIP_HIPCC_EXECUTABLE}/../amdclang" REALPATH)
+    get_filename_component(AMDCLANG_DIR "${AMDCLANG_PATH}" DIRECTORY)
+    set(AMDCLANG_LIBDIR "${AMDCLANG_DIR}/../lib")
     # libomp.so may be in a target subdirectory directory.
     execute_process(
       OUTPUT_STRIP_TRAILING_WHITESPACE
-      COMMAND bash -c "${HIP_CLANG_ROOT}/bin/amdclang --version | grep -oP '(?<=Target: ).+'"
+      COMMAND bash -c "${AMDCLANG_DIR}/amdclang --version | grep -oP '(?<=Target: ).+'"
       OUTPUT_VARIABLE LLVM_TARGET_TRIPLE
     )
-    find_library(LIBOMP_FOUND omp PATHS ${HIP_CLANG_ROOT}/lib PATH_SUFFIXES ${LLVM_TARGET_TRIPLE} NO_DEFAULT_PATH)
+    find_library(LIBOMP_FOUND omp PATHS ${AMDCLANG_LIBDIR} PATH_SUFFIXES ${LLVM_TARGET_TRIPLE} NO_DEFAULT_PATH)
     if (LIBOMP_FOUND)
       get_filename_component(LIBOMP_PATH "${LIBOMP_FOUND}" PATH)
     endif()
@@ -139,22 +143,20 @@ if( BUILD_CLIENTS_BENCHMARKS OR BUILD_CLIENTS_TESTS)
   if(LIBOMP_FOUND)
     # LIBOMP_FOUND will contain full path to libomp.so
     set( COMMON_LINK_LIBS ${LIBOMP_FOUND})
-    if(HIP_PLATFORM STREQUAL amd AND NOT WIN32)
+    if(NOT WIN32)
       list( APPEND COMMON_LINK_LIBS "-Wl,-rpath=${LIBOMP_PATH}")
     endif()
   # If cmake cannot find the proper libomp.so, use HIP_CLANG_ROOT as fallback.
   else()
-    if(HIP_PLATFORM STREQUAL amd)
-      # Add library search paths.
-      list( APPEND COMMON_LINK_LIBS "-L\"${HIP_CLANG_ROOT}/lib/${LLVM_TARGET_TRIPLE}\"")
-      list( APPEND COMMON_LINK_LIBS "-L\"${HIP_CLANG_ROOT}/lib\"")
-      if(NOT WIN32)
-        # Add rpath.
-        list( APPEND COMMON_LINK_LIBS "-Wl,-rpath=${HIP_CLANG_ROOT}/lib/${LLVM_TARGET_TRIPLE}")
-        list( APPEND COMMON_LINK_LIBS "-Wl,-rpath=${HIP_CLANG_ROOT}/lib -lomp")
-      else()
-        list( APPEND COMMON_LINK_LIBS "libomp")
-      endif()
+    # Add library search paths.
+    list( APPEND COMMON_LINK_LIBS "-L\"${AMDCLANG_LIBDIR}/${LLVM_TARGET_TRIPLE}\"")
+    list( APPEND COMMON_LINK_LIBS "-L\"${AMDCLANG_LIBDIR}\"")
+    if(NOT WIN32)
+      # Add rpath.
+      list( APPEND COMMON_LINK_LIBS "-Wl,-rpath=${AMDCLANG_LIBDIR}/${LLVM_TARGET_TRIPLE}")
+      list( APPEND COMMON_LINK_LIBS "-Wl,-rpath=${AMDCLANG_LIBDIR} -lomp")
+    else()
+      list( APPEND COMMON_LINK_LIBS "libomp")
     endif()
   endif()