Add nightly CI check for SYCL

.github/workflows/main.yml

@@ -21,7 +21,7 @@ jobs:
           git fetch &> /dev/null
 
           # For every file changed, apply clang-format
-          for file in $(git diff --name-only origin/$GITHUB_BASE_REF | egrep '.*\.cpp|.*\.hpp|.*\.h'); do
+          for file in $(git diff --name-only origin/$GITHUB_BASE_REF | egrep '.*\.cpp$|.*\.hpp$|.*\.h$'); do
             clang-format-8 -i -style=file $file
             git add $file
           done

.jenkins/nightly.groovy

@@ -1,40 +1,86 @@
 pipeline {
     agent none
 
+    options {
+        timeout(time: 3, unit: 'HOURS')
+    }
+
     stages {
-        stage('HIP-ROCm-4.2-C++14') {
-            agent {
-                dockerfile {
-                    filename 'Dockerfile.hip'
-                    dir 'scripts/docker'
-                    additionalBuildArgs '--build-arg BASE=rocm/dev-ubuntu-20.04:4.2'
-                    label 'rocm-docker && vega'
-                    args '-v /tmp/ccache.kokkos:/tmp/ccache --device=/dev/kfd --device=/dev/dri --security-opt seccomp=unconfined --group-add video --env HIP_VISIBLE_DEVICES=$HIP_VISIBLE_DEVICES'
+        stage('Build & Run') {
+            parallel {
+                stage('SYCL-OneAPI') {
+                    agent {
+                        dockerfile {
+                            filename 'Dockerfile.sycl'
+                            dir 'scripts/docker'
+                            label 'nvidia-docker && volta'
+                            args '-v /tmp/ccache.kokkos:/tmp/ccache'
+                        }
+                    }
+                    steps {
+                        sh '''rm -rf kokkos &&
+                              git clone -b develop https://github.com/kokkos/kokkos.git && cd kokkos && \
+                              mkdir build && cd build && \
+                              cmake \
+                                -DCMAKE_BUILD_TYPE=Release \
+                                -DCMAKE_CXX_COMPILER=clang++ \
+                                -DKokkos_ARCH_VOLTA70=ON \
+                                -DKokkos_ENABLE_DEPRECATED_CODE_3=OFF \
+                                -DKokkos_ENABLE_SYCL=ON \
+                                -DKokkos_ENABLE_UNSUPPORTED_ARCHS=ON \
+                                -DCMAKE_CXX_STANDARD=17 \
+                              .. && \
+                              make -j8 && make install && \
+                              cd ../.. && rm -rf kokkos'''
+                        sh '''rm -rf build && mkdir -p build && cd build && \
+                              cmake \
+                                -DCMAKE_BUILD_TYPE=Release \
+                                -DCMAKE_CXX_COMPILER=clang++ \
+                                -DKokkosKernels_ENABLE_TESTS=ON \
+                                -DKokkosKernels_ENABLE_EXAMPLES=ON \
+                                -DKokkosKernels_INST_DOUBLE=ON \
+                                -DKokkosKernels_INST_ORDINAL_INT=ON \
+                                -DKokkosKernels_INST_OFFSET_INT=ON \
+                              .. && \
+                              make -j8'''
+                    }
+                }
+
+                stage('HIP-ROCm-4.2-C++14') {
+                    agent {
+                        dockerfile {
+                            filename 'Dockerfile.hip'
+                            dir 'scripts/docker'
+                            additionalBuildArgs '--build-arg BASE=rocm/dev-ubuntu-20.04:4.2'
+                            label 'rocm-docker && vega'
+                            args '-v /tmp/ccache.kokkos:/tmp/ccache --device=/dev/kfd --device=/dev/dri --security-opt seccomp=unconfined --group-add video --env HIP_VISIBLE_DEVICES=$HIP_VISIBLE_DEVICES'
+                        }
+                    }
+                    steps {
+                        sh '''rm -rf kokkos &&
+                              git clone -b develop https://github.com/kokkos/kokkos.git && cd kokkos && \
+                              mkdir build && cd build && \
+                              cmake \
+                                -DCMAKE_CXX_COMPILER=hipcc \
+                                -DCMAKE_CXX_EXTENSIONS=OFF \
+                                -DKokkos_ENABLE_HIP=ON \
+                              .. && \
+                              make -j8 && make install && \
+                              cd ../.. && rm -rf kokkos'''
+                        sh '''rm -rf build && mkdir -p build && cd build && \
+                              cmake \
+                                -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+                                -DCMAKE_CXX_COMPILER=hipcc \
+                                -DCMAKE_CXX_EXTENSIONS=OFF \
+                                -DKokkosKernels_ENABLE_TESTS=ON \
+                                -DKokkosKernels_ENABLE_EXAMPLES=ON \
+                                -DKokkosKernels_INST_DOUBLE=ON \
+                                -DKokkosKernels_INST_ORDINAL_INT=ON \
+                                -DKokkosKernels_INST_OFFSET_INT=ON \
+                              .. && \
+                              make -j8 && ctest --verbose'''
+                    }
                 }
-            }
-            steps {
-                sh '''rm -rf kokkos &&
-                      git clone -b develop https://github.com/kokkos/kokkos.git && cd kokkos && \
-                      mkdir build && cd build && \
-                      cmake \
-                        -DCMAKE_CXX_COMPILER=hipcc \
-                        -DCMAKE_CXX_EXTENSIONS=OFF \
-                        -DKokkos_ENABLE_HIP=ON \
-                        .. && \
-                       make -j8 && make install && \
-                       cd ../.. && rm -rf kokkos'''
-                sh '''rm -rf build && mkdir -p build && cd build && \
-                      cmake \
-                        -DCMAKE_BUILD_TYPE=RelWithDebInfo \
-                        -DCMAKE_CXX_COMPILER=hipcc \
-                        -DCMAKE_CXX_EXTENSIONS=OFF \
-                        -DKokkosKernels_ENABLE_TESTS=ON \
-                        -DKokkosKernels_ENABLE_EXAMPLES=ON \
-                        -DKokkosKernels_INST_DOUBLE=ON \
-                        -DKokkosKernels_INST_ORDINAL_INT=ON \
-                        -DKokkosKernels_INST_OFFSET_INT=ON \
-                      .. && \
-                      make -j8 && ctest --verbose'''
             }
         }
     }

example/gmres/CMakeLists.txt

@@ -6,18 +6,24 @@ KOKKOSKERNELS_ADD_EXECUTABLE(
   SOURCES ex_real_A.cpp
   )
 
+# FIXME_SYCL CUDA_ERROR_INVALID_ADDRESS_SPACE
+IF(NOT KOKKOS_ENABLE_SYCL)
 KOKKOSKERNELS_ADD_EXECUTABLE_AND_TEST(
   gmres_test_real_A
   SOURCES test_real_A.cpp
   )
+ENDIF()
 
 KOKKOSKERNELS_ADD_EXECUTABLE(
   gmres_test_cmplx_A
   SOURCES test_cmplx_A.cpp
   )
 
+# FIXME_SYCL CUDA_ERROR_INVALID_ADDRESS_SPACE
+IF(NOT KOKKOS_ENABLE_SYCL)
 KOKKOSKERNELS_ADD_EXECUTABLE_AND_TEST(
   gmres_test_prec
   SOURCES test_prec.cpp
   )
+ENDIF()
 

example/wiki/sparse/CMakeLists.txt

@@ -25,10 +25,13 @@ KOKKOSKERNELS_ADD_EXECUTABLE_AND_TEST(
   SOURCES KokkosSparse_wiki_spadd.cpp
   )
 
+# FIXME_SYCL SYCL does not support querying free/total memory
+IF (NOT KOKKOS_ENABLE_SYCL)
 KOKKOSKERNELS_ADD_EXECUTABLE_AND_TEST(
   wiki_spgemm
   SOURCES KokkosSparse_wiki_spgemm.cpp
   )
+ENDIF()
 
 KOKKOSKERNELS_ADD_EXECUTABLE_AND_TEST(
   wiki_gauss_seidel

scripts/docker/Dockerfile.hip

@@ -10,6 +10,14 @@ RUN apt-get update && apt-get install -y \
 
 ENV PATH=/opt/rocm/bin:$PATH
 
+RUN KEYDUMP_URL=https://cloud.cees.ornl.gov/download && \
+    KEYDUMP_FILE=keydump && \
+    wget --quiet ${KEYDUMP_URL}/${KEYDUMP_FILE} && \
+    wget --quiet ${KEYDUMP_URL}/${KEYDUMP_FILE}.sig && \
+    gpg --import ${KEYDUMP_FILE} && \
+    gpg --verify ${KEYDUMP_FILE}.sig ${KEYDUMP_FILE} && \
+    rm ${KEYDUMP_FILE}*
+
 ARG CMAKE_VERSION=3.18.5
 ENV CMAKE_DIR=/opt/cmake
 RUN CMAKE_KEY=2D2CEF1034921684 && \
@@ -19,7 +27,6 @@ RUN CMAKE_KEY=2D2CEF1034921684 && \
     wget --quiet ${CMAKE_URL}/${CMAKE_SHA256} && \
     wget --quiet ${CMAKE_URL}/${CMAKE_SHA256}.asc && \
     wget --quiet ${CMAKE_URL}/${CMAKE_SCRIPT} && \
-    gpg --keyserver pool.sks-keyservers.net --recv-keys ${CMAKE_KEY} && \
     gpg --verify ${CMAKE_SHA256}.asc ${CMAKE_SHA256} && \
     grep ${CMAKE_SCRIPT} ${CMAKE_SHA256} | sha256sum --check && \
     mkdir -p ${CMAKE_DIR} && \

scripts/docker/Dockerfile.sycl

@@ -0,0 +1,53 @@
+ARG BASE=nvidia/cuda:10.2-devel
+FROM $BASE
+
+RUN apt-get update && apt-get install -y \
+        bc \
+        wget \
+        ccache \
+        ninja-build \
+        python3 \
+        git \
+        && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN KEYDUMP_URL=https://cloud.cees.ornl.gov/download && \
+    KEYDUMP_FILE=keydump && \
+    wget --quiet ${KEYDUMP_URL}/${KEYDUMP_FILE} && \
+    wget --quiet ${KEYDUMP_URL}/${KEYDUMP_FILE}.sig && \
+    gpg --import ${KEYDUMP_FILE} && \
+    gpg --verify ${KEYDUMP_FILE}.sig ${KEYDUMP_FILE} && \
+    rm ${KEYDUMP_FILE}*
+
+ARG CMAKE_VERSION=3.18.5
+ENV CMAKE_DIR=/opt/cmake
+RUN CMAKE_URL=https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION} && \
+    CMAKE_SCRIPT=cmake-${CMAKE_VERSION}-Linux-x86_64.sh && \
+    CMAKE_SHA256=cmake-${CMAKE_VERSION}-SHA-256.txt && \
+    wget --quiet ${CMAKE_URL}/${CMAKE_SHA256} && \
+    wget --quiet ${CMAKE_URL}/${CMAKE_SHA256}.asc && \
+    wget --quiet ${CMAKE_URL}/${CMAKE_SCRIPT} && \
+    gpg --verify ${CMAKE_SHA256}.asc ${CMAKE_SHA256} && \
+    grep ${CMAKE_SCRIPT} ${CMAKE_SHA256} | sha256sum --check && \
+    mkdir -p ${CMAKE_DIR} && \
+    sh ${CMAKE_SCRIPT} --skip-license --prefix=${CMAKE_DIR} && \
+    rm cmake*
+ENV PATH=${CMAKE_DIR}/bin:$PATH
+
+ENV SYCL_DIR=/opt/sycl
+RUN SYCL_VERSION=20210903 && \
+    SYCL_URL=https://github.com/intel/llvm/archive/sycl-nightly && \
+    SYCL_ARCHIVE=${SYCL_VERSION}.tar.gz && \
+    SCRATCH_DIR=/scratch && mkdir -p ${SCRATCH_DIR} && cd ${SCRATCH_DIR} && \
+    wget --quiet ${SYCL_URL}/${SYCL_ARCHIVE} && \
+    mkdir llvm && \
+    tar -xf ${SYCL_ARCHIVE} -C llvm --strip-components=1 && \
+    cd llvm && \
+    python3 buildbot/configure.py --cuda && \
+    python3 buildbot/compile.py && \
+    mkdir -p ${SYCL_DIR} && \
+    mv ${SCRATCH_DIR}/llvm/build/install/* ${SYCL_DIR} && \
+    echo "${SYCL_DIR}/lib" > /etc/ld.so.conf.d/sycl.conf && ldconfig && \
+    rm -rf ${SCRATCH_DIR}
+ENV PATH=${SYCL_DIR}/bin:$PATH