New alpaka backend rules

SCRAM/GMake/Makefile

@@ -8,7 +8,7 @@ ifndef SCRAM_SCRIPT_EXT
 SCRAM_SCRIPT_EXT:=.py
 endif
 ifndef SCRAM_TOOLS_DIR
-SCRAM_TOOLS_DIR:=$(SCRAM_ADMIN_DIR)/$(if $(strip $(filter V2_%,$(SCRAM_VERSION))),timestamps,tools)
+SCRAM_TOOLS_DIR:=$(SCRAM_ADMIN_DIR)/tools
 endif
 EXTERNAL_LINK_SCRIPT:=$(LOCALTOP)/$(SCRAM_CONFIGDIR)/SCRAM/linkexternal$(SCRAM_SCRIPT_EXT)
 TOOL_MAKE_SCRIPT:=$(LOCALTOP)/$(SCRAM_CONFIGDIR)/SCRAM/updateToolMK$(SCRAM_SCRIPT_EXT)

SCRAM/GMake/Makefile.rules

@@ -1604,7 +1604,7 @@ project_clean clean_$(SCRAM_SOURCEDIR) vclean cache_clean distclean:
 	fi
 	$(CMD_rm) -rf $(SCRAM_INTwork) $(COMMON_WORKINGDIR)/cache/packs
 
-project_help help_$(SCRAM_SOURCEDIR):
+project_help help_$(SCRAM_SOURCEDIR)::
 	@$(CMD_echo) "------------ Help for Project-level Builds rules/targets ------------"
 	@$(CMD_echo) ""
 	@$(CMD_echo) "help"
@@ -2067,6 +2067,52 @@ else
 dxr:
 	@$(CMD_echo) "**** ERROR: py2-dxr tool not available."
 endif
+.PHONY: enable-cuda enable-rocm disable-cuda disable-rocm enable-cuda\$(colon)sm_% enable-rocm\$(colon)gfx%
+.PHONY: enable-cuda\$(colon)native enable-cuda\$(colon)reset enable-rocm\$(colon)native enable-rocm\$(colon)reset
+.PHONY: enable-gpus\$(colon)native
+project_help help_$(SCRAM_SOURCEDIR)::
+	@$(CMD_echo) -e "\
+enable-cuda enable-rocm disable-cuda disable-rocm\n\
+  Enable/Disable Cuda/Rocm Alpaka backends build rules.\n\
+\n\
+enable-cuda:comma-separated-compute-capabilities enable-rocm:comma-separated-compute-capabilities\n\
+  Set cuda/rocm compute capabilities and enable cuda/rocm alpaka backend if disabled.\n\
+     enable-cuda:sm_75,sm_89\n\
+     enable-rocm:gfx1100\n\
+\n\
+enable-cuda:native enable-rocm:native\n\
+  Find out native compute capabilities of cuda/rocm and set those.\n\
+  Also enable cuda/rocm alpaka backend if it was disabled.\n\
+\n\
+enable-cuda:reset enable-rocm:reset\n\
+  Reset cuda/rocm compute capabilities and set them from the project release area.\n\
+  Also enable cuda/rocm alpaka backend if it was disabled.\n\
+\n\
+enable-gpus:native\n\
+  Enable/Disable alpaka gpus backends if not support on local host. It also calls\n\
+  Also set native compute capabilities for the enabled alpaka backends.\n"
+
+enable-cuda enable-rocm disable-cuda disable-rocm:
+	@$(SCRAM_CONFIGDIR)/SCRAM/utils/alpaka-backends.sh "$@"
+enable-cuda\$(colon)sm_%: $(if $(strip $(filter cuda,$(ALPAKA_SELECTED_BACKENDS))),,enable-cuda)
+	@$(SCRAM_CONFIGDIR)/SCRAM/utils/backend-setup.sh "$@"
+enable-rocm\$(color)gfx%: $(if $(strip $(filter rocm,$(ALPAKA_SELECTED_BACKENDS))),,enable-rocm)
+	@$(SCRAM_CONFIGDIR)/SCRAM/utils/backend-setup.sh "$@"
+enable-cuda\$(colon)native enable-cuda\$(colon)reset: $(if $(strip $(filter cuda,$(ALPAKA_SELECTED_BACKENDS))),,enable-cuda)
+	@$(SCRAM_CONFIGDIR)/SCRAM/utils/backend-setup.sh "$@"
+enable-rocm\$(colon)native enable-rocm\$(colon)reset: $(if $(strip $(filter rocm,$(ALPAKA_SELECTED_BACKENDS))),,enable-rocm)
+	@$(SCRAM_CONFIGDIR)/SCRAM/utils/backend-setup.sh "$@"
+enable-gpus\$(colon)native:
+	@for t in cuda rocm ; do \
+          $(CMD_echo) ">> Checking $$t" &&\
+          OUT="$$($(SCRAM_CONFIGDIR)/SCRAM/utils/backend-setup.sh "enable-$$t:native" 2>&1)" &&\
+          $(CMD_echo) "$$OUT" &&\
+          if [ $$($(CMD_echo) "$$OUT" | grep 'Warning: Unable to find.*compute capabilities' | $(CMD_wc) -l) -gt 0 ] ; then \
+            $(SCRAM_CONFIGDIR)/SCRAM/utils/alpaka-backends.sh disable-$$t ;\
+          else \
+            $(SCRAM_CONFIGDIR)/SCRAM/utils/alpaka-backends.sh enable-$$t ;\
+          fi;\
+        done
 disable-multi-targets:
 	@$(CMD_rm) -f $(SCRAM_ADMIN_DIR)/multi-targets && $(CMD_echo) "Building with multi-targets is disabled."
 enable-multi-targets:

SCRAM/utils/alpaka-backends.sh

@@ -0,0 +1,29 @@
+#!/bin/bash -e
+backend=$(echo $1 | cut -d- -f2)
+req_type=$(echo $1 | cut -d- -f1)
+config_changed=false
+SELF_TOOL=${LOCALTOP}/config/Self.xml
+ALL_BACKENDS=$(grep ALPAKA_BACKENDS= ${SELF_TOOL} | sed 's|.*=.||;s|".*||')
+if [ "${req_type}" = "enable" ]; then
+  if [ $(echo " $ALL_BACKENDS " | grep " ${backend} " | wc -l) -gt 0 ] ; then
+    echo "Alpaka backend ${backend} already enabled."
+    exit 0
+  else
+    sed -i -e "s|ALPAKA_BACKENDS=\"|ALPAKA_BACKENDS=\"${backend} |" ${SELF_TOOL}
+    config_changed=true
+  fi
+elif [ "${req_type}" = "disable" ]; then
+  if [ $(echo " $ALL_BACKENDS " | grep " ${backend} " | wc -l) -eq 0 ] ; then
+    echo "Alpaka backend ${backend} already disabled."
+    exit 0
+  else
+    SEL_BACKEND=$(echo ${ALL_BACKENDS} | tr ' ' '\n' | grep -v "${backend}" | tr '\n' ' ' | sed 's| *$||')
+    sed -i -e "s|ALPAKA_BACKENDS=.*\"|ALPAKA_BACKENDS=\"${SEL_BACKEND}\"|" ${SELF_TOOL}
+    config_changed=true
+  fi
+fi
+if $self_updated ; then
+  scram setup self        >/dev/null 2>&1
+  scram build -r echo_CXX >/dev/null 2>&1
+  echo "Alpaka backend ${backend} is ${req_type}."
+fi

SCRAM/utils/backend-setup.sh

@@ -0,0 +1,69 @@
+#!/bin/bash -e
+backend=$(echo $1 | cut -d- -f2 | cut -d: -f1)
+req_type=$(echo $1 | cut -d- -f1)
+CAPS=$(echo $1 | cut -d- -f2 | cut -d: -f2)
+TOOL=config/toolbox/${SCRAM_ARCH}/tools/selected/${backend}.xml
+
+if [ ! -e ${TOOL} ] ; then
+  echo "Error: Unknown backend ${backend}" >&2
+  exit 1
+fi
+
+if [ "${CAPS}" = "reset" ] ; then
+  cp -f $CMSSW_RELEASE_BASE/${TOOL} ${TOOL}.tmp
+elif [ "${backend}" = "cuda" ] ; then
+  if [ "${CAPS}" = "native" ] ; then
+    DOTS=$(cudaComputeCapabilities | awk '{ print $2 }' | sort -u)
+    CAPS=$(echo $DOTS | sed -e 's#\.*##g')
+    if [ "${CAPS}" = "" ] ; then
+      echo "Warning: Unable to find cuda compute capabilities." >&2
+      exit 0
+    fi
+  else
+    CAPS=$(echo ${CAPS} | tr ',' '\n' | sed 's|^sm_||' | tr '\n' ' ' | sed 's| *$||')
+  fi
+  cp -f ${TOOL} ${TOOL}.tmp
+  # remove existing capabilities
+  sed -i -e "s# *-gencode arch=compute_..,code=sm_.. *# #g" ${TOOL}.tmp
+  sed -i -e "s# *-gencode arch=compute_..,code=\[sm_..,compute_..\] *# #g" ${TOOL}.tmp
+
+  # add support for the capabilities found on this machine
+  for CAP in $CAPS; do
+    sed -i -e "/flags CUDA_FLAGS=/s#=\"#=\"-gencode arch=compute_$CAP,code=[sm_$CAP,compute_$CAP] #" ${TOOL}.tmp
+  done
+  CAPS=$(echo $CAPS | tr ' ' '\n' | sed 's|^|sm_|' | tr '\n' ',' | sed 's|,$||')
+elif [ "${backend}" = "rocm" ] ; then
+  if [ "${CAPS}" = "native" ] ; then
+    CAPS=$(rocmComputeCapabilities | awk '{ print $2 }' | sort -u)
+    if [ "${CAPS}" = "" ] ; then
+      echo "Warning: Unable to find rocm compute capabilities." >&2
+      exit 0
+    fi
+  else
+    CAPS=$(echo ${CAPS} | tr ',' ' ')
+  fi
+  cp -f ${TOOL} ${TOOL}.tmp
+  #Remove existing capabilities flag
+  sed -r -i -e '/flags ROCM_FLAGS=.*gfx[0-9a-f]+/d' ${TOOL}.tmp
+
+  #add support for the capabilities found on this machine
+  for CAP in $CAPS; do
+    sed -i -e "s#</client>#</client>\n  <flags ROCM_FLAGS=\"--offload-arch=${CAP}\"/>#" ${TOOL}.tmp
+  done
+  CAPS=$(echo $CAPS | tr ' ' ',' | sed 's|,$||')
+fi
+
+if [ $(diff -w ${TOOL}.tmp ${TOOL} | wc -l) -gt 0 ] ; then
+  mv ${TOOL}.tmp ${TOOL}
+  scram setup ${backend} >/dev/null 2>&1
+  echo -n "Compute capabilities for ${backend} are "
+  if [ "${CAPS}" = "reset" ] ; then
+    echo "reset."
+  else
+    echo "set to ${CAPS}"
+  fi
+else
+  [ -f .SCRAM/${SCRAM_ARCH}/tools/${backend} ] && touch .SCRAM/${SCRAM_ARCH}/tools/${backend}
+  rm -f ${TOOL}.tmp
+  echo "No change, compute capabilities for ${backend} are already set."
+fi