Add Ubuntu packaging scripts and GHA testing (#5754)

* Fix CMake & packaging glitches for Ubuntu package.

* Add Ubuntu packaging scripts and presets.

* Add GHA workflow to test packaging and usage on Ubuntu

* Address review comments.

.github/workflows/packaging.yml

@@ -0,0 +1,51 @@
+name: Packaging
+on: [ 'pull_request' ]
+jobs:
+  package-ubuntu:
+    name: Package for Ubuntu
+    runs-on: ubuntu-20.04
+    env:
+      CMAKE_CXX_COMPILER_LAUNCHER: ccache
+      CMAKE_C_COMPILER_LAUNCHER: ccache
+      LLVM_ROOT: /usr/lib/llvm-11
+    steps:
+      - name: Install dependencies
+        run: |
+          wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null \
+            | gpg --dearmor - | sudo tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null
+          sudo apt-add-repository 'deb https://apt.kitware.com/ubuntu/ focal main'
+          sudo apt update
+          sudo apt install cmake ninja-build doxygen ccache
+          sudo apt install llvm-11-dev liblld-11-dev clang-11 libclang-11-dev libjpeg-dev libpng-dev
+          sudo apt install lintian dpkg-dev
+      - name: Check out sources
+        uses: actions/checkout@v2
+      - name: Set up ccache
+        uses: hendrikmuhs/ccache-action@v1
+      - name: Run Ubuntu packaging script
+        run: ./packaging/ubuntu/package.sh . ubuntu
+      - name: Upload packages
+        uses: actions/upload-artifact@v2
+        with:
+          name: packages
+          path: ubuntu/*.deb
+  test-ubuntu:
+    name: Test Ubuntu package
+    needs: package-ubuntu
+    runs-on: ubuntu-20.04
+    steps:
+      # Specifically use the CMake version that comes with Ubuntu.
+      - name: Install dependencies
+        run: sudo apt install cmake ninja-build libc6-dev-arm64-cross gcc-aarch64-linux-gnu g++-aarch64-linux-gnu qemu-user
+      - name: Check out sources
+        uses: actions/checkout@v2
+      - name: Download Halide Ubuntu packages
+        uses: actions/download-artifact@v2
+        with:
+          name: packages
+      - name: Install Halide Ubuntu packages
+        run: sudo apt install ./*.deb
+      - name: Test integration
+        run: |
+          cmake -S test/integration -B build
+          cd build && ctest -j$(nproc) --output-on-failure

CMakePresets.json

@@ -98,7 +98,7 @@
         "BUILD_SHARED_LIBS": "YES",
         "CMAKE_INSTALL_BINDIR": "bin/$<CONFIG>",
         "CMAKE_INSTALL_LIBDIR": "lib/$<CONFIG>",
-        "HALIDE_INSTALL_CMAKEDIR": "lib/cmake/Halide"
+        "Halide_INSTALL_CMAKEDIR": "lib/cmake/Halide"
       }
     },
     {
@@ -127,6 +127,35 @@
         "BUILD_SHARED_LIBS": "NO",
         "Halide_BUNDLE_LLVM": "YES"
       }
+    },
+    {
+      "name": "package-ubuntu-shared",
+      "inherits": "package-unix-shared",
+      "displayName": "Package shared Halide for Ubuntu",
+      "description": "Package shared Halide for Ubuntu, using system packages.",
+      "binaryDir": "shared-release",
+      "cacheVariables": {
+        "Halide_SHARED_LLVM": "YES",
+        "LLVM_DIR": "$env{LLVM_ROOT}/lib/cmake/llvm",
+        "Clang_DIR": "$env{LLVM_ROOT}/lib/cmake/clang",
+        "LLD_DIR": "$env{LLVM_ROOT}/lib/cmake/lld",
+        "CMAKE_INSTALL_INCLUDEDIR": "include/Halide",
+        "CMAKE_INSTALL_LIBDIR": "lib/x86_64-linux-gnu",
+        "Halide_INSTALL_PLUGINDIR": "lib/x86_64-linux-gnu/Halide",
+        "Halide_INSTALL_HELPERSDIR": "lib/cmake/HalideHelpers",
+        "CMAKE_STRIP": "${sourceDir}/packaging/ubuntu/extra-strip.sh"
+      }
+    },
+    {
+      "name": "package-ubuntu-static",
+      "inherits": "package-ubuntu-shared",
+      "displayName": "Package static Halide for Ubuntu",
+      "description": "Package static Halide for Ubuntu, using system packages.",
+      "binaryDir": "static-release",
+      "cacheVariables": {
+        "BUILD_SHARED_LIBS": "NO",
+        "WITH_DOCS": "NO"
+      }
     }
   ]
 }

cmake/HalideGeneratorHelpers.cmake

@@ -258,14 +258,31 @@ endfunction()
 ##
 
 function(_Halide_place_dll GEN)
+    if (NOT WIN32)
+        return()
+    endif ()
+
+    # Short circuit so that Halide::Halide isn't checked when importing a generator from another CMake project
     get_property(is_imported TARGET ${GEN} PROPERTY IMPORTED)
+    if (is_imported)
+        return()
+    endif ()
+
     get_property(has_post_build TARGET ${GEN} PROPERTY Halide_GENERATOR_HAS_POST_BUILD)
+    if (has_post_build)
+        return()
+    endif ()
+
+    # Here GEN is not IMPORTED, which means that it must be linked
+    # to Halide::Halide and therefore Halide::Halide must exist.
     get_property(halide_type TARGET Halide::Halide PROPERTY TYPE)
-    if (WIN32 AND NOT is_imported AND NOT has_post_build AND halide_type STREQUAL "SHARED_LIBRARY")
-        add_custom_command(TARGET ${GEN} POST_BUILD
-                           COMMAND ${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:Halide::Halide> $<TARGET_FILE_DIR:${GEN}>)
-        set_property(TARGET ${GEN} PROPERTY Halide_GENERATOR_HAS_POST_BUILD 1)
+    if (NOT halide_type STREQUAL "SHARED_LIBRARY")
+        return()
     endif ()
+
+    add_custom_command(TARGET ${GEN} POST_BUILD
+                       COMMAND ${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:Halide::Halide> $<TARGET_FILE_DIR:${GEN}>)
+    set_property(TARGET ${GEN} PROPERTY Halide_GENERATOR_HAS_POST_BUILD 1)
 endfunction()
 
 ##

cmake/toolchain.linux-aarch64.cmake

@@ -14,3 +14,8 @@ set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
 set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
 set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
 set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)
+
+find_program(QEMU_AARCH64 qemu-aarch64)
+if (QEMU_AARCH64 AND EXISTS "/usr/aarch64-linux-gnu")
+    set(CMAKE_CROSSCOMPILING_EMULATOR ${QEMU_AARCH64} -L /usr/aarch64-linux-gnu)
+endif ()

packaging/common/HalideConfig.cmake

@@ -55,8 +55,6 @@ set(Halide_VERSION_MINOR @Halide_VERSION_MINOR@)
 set(Halide_VERSION_PATCH @Halide_VERSION_PATCH@)
 set(Halide_VERSION_TWEAK @Halide_VERSION_TWEAK@)
 
-set(Halide_HOST_TARGET @Halide_HOST_TARGET@)
-
 set(Halide_ENABLE_EXCEPTIONS @Halide_ENABLE_EXCEPTIONS@)
 set(Halide_ENABLE_RTTI @Halide_ENABLE_RTTI@)
 
@@ -67,7 +65,6 @@ set(Halide_ENABLE_RTTI @Halide_ENABLE_RTTI@)
 include(CMakeFindDependencyMacro)
 
 find_dependency(HalideHelpers "${Halide_VERSION}" EXACT)
-find_dependency(Threads)
 
 if (Halide_comp_PNG)
     Halide_find_component_dependency(PNG PNG)

packaging/common/HalideHelpersConfig.cmake

@@ -1,5 +1,12 @@
 cmake_minimum_required(VERSION 3.16)
 
+set(Halide_HOST_TARGET @Halide_HOST_TARGET@)
+
+include(CMakeFindDependencyMacro)
+
+set(THREADS_PREFER_PTHREAD_FLAG TRUE)
+find_dependency(Threads)
+
 include(${CMAKE_CURRENT_LIST_DIR}/Halide-Interfaces.cmake)
 include(${CMAKE_CURRENT_LIST_DIR}/HalideTargetHelpers.cmake)
 include(${CMAKE_CURRENT_LIST_DIR}/HalideGeneratorHelpers.cmake)

packaging/ubuntu/changelog

@@ -0,0 +1,5 @@
+@package_name@ (@CPACK_PACKAGE_VERSION@) UNRELEASED; urgency=low
+
+  * Initial package release.
+
+ -- @CPACK_PACKAGE_CONTACT@  @timestamp@ -0000

packaging/ubuntu/config.cmake

@@ -0,0 +1,137 @@
+cmake_minimum_required(VERSION 3.19)
+
+include("shared-Release/CPackConfig.cmake")
+
+## General setup
+
+set(CPACK_PACKAGE_CONTACT "Alex Reinking <[email protected]>")
+set(CPACK_STRIP_FILES TRUE)
+set(CPACK_PRE_BUILD_SCRIPTS "${CMAKE_CURRENT_LIST_DIR}/pre_build.cmake")
+
+##############################
+## Components configuration ##
+##############################
+
+# This is a mapping from CPack component names to CMake install() components.
+# We use the identity mapping here for simplicity; some advanced configurations
+# with GUI installers require these to diverge.
+set(CPACK_COMPONENTS_HALIDE_RUNTIME Halide_Runtime)
+set(CPACK_COMPONENTS_HALIDE_DEVELOPMENT Halide_Development)
+set(CPACK_COMPONENTS_HALIDE_DOCUMENTATION Halide_Documentation)
+
+set(CPACK_COMPONENTS_ALL Halide_Runtime Halide_Development Halide_Documentation)
+
+set(CPACK_INSTALL_CMAKE_PROJECTS
+    static-Release Halide ALL /
+    shared-Release Halide ALL /)
+
+###################################
+## Ubuntu-specific configuration ##
+###################################
+
+# We set every variable documented here: https://cmake.org/cmake/help/latest/cpack_gen/deb.html
+# even if it's just to the default. That way there are no surprises.
+
+set(CPACK_DEB_COMPONENT_INSTALL YES)
+
+set(CPACK_DEBIAN_HALIDE_RUNTIME_PACKAGE_NAME libHalide${CPACK_PACKAGE_VERSION_MAJOR})
+set(CPACK_DEBIAN_HALIDE_DEVELOPMENT_PACKAGE_NAME libHalide${CPACK_PACKAGE_VERSION_MAJOR}-dev)
+set(CPACK_DEBIAN_HALIDE_DOCUMENTATION_PACKAGE_NAME libHalide${CPACK_PACKAGE_VERSION_MAJOR}-doc)
+
+set(CPACK_DEBIAN_HALIDE_RUNTIME_FILE_NAME DEB-DEFAULT)
+set(CPACK_DEBIAN_HALIDE_DEVELOPMENT_FILE_NAME DEB-DEFAULT)
+set(CPACK_DEBIAN_HALIDE_DOCUMENTATION_FILE_NAME DEB-DEFAULT)
+
+# Debian package versions look like: <epoch>:<version>-<release>
+# <epoch>   is a number that increases when changing the whole versioning schema.
+#           We would ideally _never_ have to set this since we're using semver.
+# <version> is the version number of the actual software being packaged.
+# <release> is the version number of the _package_. Set/increment this when fixing
+#           bugs in the package itself. This should also not be incremented too
+#           frequently. It's always safe to bump the patch version when in doubt.
+unset(CPACK_DEBIAN_PACKAGE_EPOCH)
+set(CPACK_DEBIAN_PACKAGE_VERSION "${CPACK_PACKAGE_VERSION}")
+unset(CPACK_DEBIAN_PACKAGE_RELEASE)
+
+# The default here is the host system architecture. It will generally be best
+# to package for ARM on ARM, for x86 on x86, etc. The documentation gets the
+# pseudo-architecture "all" to indicate that it has no binaries (ie. is arch
+# independent).
+unset(CPACK_DEBIAN_PACKAGE_ARCHITECTURE)
+set(CPACK_DEBIAN_HALIDE_DOCUMENTATION_PACKAGE_ARCHITECTURE all)
+
+# Package dependencies.
+# TODO: figure out how to get LLVM major version piped in here.
+set(CPACK_DEBIAN_HALIDE_RUNTIME_PACKAGE_DEPENDS "llvm-11 (>= 11.0.0)")
+set(CPACK_DEBIAN_HALIDE_DEVELOPMENT_PACKAGE_DEPENDS "llvm-11-dev (>= 11.0.0), liblld-11-dev (>= 11.0.0)")
+set(CPACK_DEBIAN_HALIDE_DOCUMENTATION_PACKAGE_DEPENDS "")
+
+# Sets up package dependencies based on CPack component dependencies
+set(CPACK_DEBIAN_ENABLE_COMPONENT_DEPENDS ON)
+
+# Uses CPACK_PACKAGE_CONTACT as default
+unset(CPACK_DEBIAN_PACKAGE_MAINTAINER)
+
+# These inherit their values from cpack cpack_add_component
+unset(CPACK_DEBIAN_HALIDE_RUNTIME_DESCRIPTION)
+unset(CPACK_DEBIAN_HALIDE_DEVELOPMENT_DESCRIPTION)
+unset(CPACK_DEBIAN_HALIDE_DOCUMENTATION_DESCRIPTION)
+
+# The Debian repository package section.
+# See: https://packages.debian.org/unstable/
+# libs     = Libraries to make other programs work. They provide special features to developers.
+# libdevel = Libraries necessary for developers to write programs that use them.
+# doc      = FAQs, HOWTOs and other documents trying to explain everything related to
+#            Debian, and software needed to browse documentation (man, info, etc).
+set(CPACK_DEBIAN_HALIDE_RUNTIME_PACKAGE_SECTION libs)
+set(CPACK_DEBIAN_HALIDE_DEVELOPMENT_PACKAGE_SECTION libdevel)
+set(CPACK_DEBIAN_HALIDE_DOCUMENTATION_PACKAGE_SECTION doc)
+
+# Deprecated: do not use
+unset(CPACK_DEBIAN_ARCHIVE_TYPE)
+
+# Could also choose from lzma, xz, or bzip2 if one gave a better ratio.
+set(CPACK_DEBIAN_COMPRESSION_TYPE "gzip")
+
+# Optional just means that it is optional for the safe running of
+# a Debian system to have our package installed. The other categories
+# do not apply to us: required (won't boot without), important (core
+# system utils), and standard (basic niceties for a character-mode
+# system).
+set(CPACK_DEBIAN_PACKAGE_PRIORITY "optional")
+
+# Uses CMAKE_PROJECT_HOMEPAGE_URL as default.
+unset(CPACK_DEBIAN_PACKAGE_HOMEPAGE)
+
+# Call dpkg-shlibdeps to get dependencies on system libraries.
+set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON)
+unset(CPACK_DEBIAN_PACKAGE_SHLIBDEPS_PRIVATE_DIRS)  # CMake 3.20+ only
+
+# Disable debug messaging
+unset(CPACK_DEBIAN_PACKAGE_DEBUG)
+
+# Special variables for package constraints. We don't have any yet.
+unset(CPACK_DEBIAN_PACKAGE_PREDEPENDS)
+unset(CPACK_DEBIAN_PACKAGE_ENHANCES)
+unset(CPACK_DEBIAN_PACKAGE_BREAKS)
+unset(CPACK_DEBIAN_PACKAGE_CONFLICTS)
+unset(CPACK_DEBIAN_PACKAGE_PROVIDES)
+unset(CPACK_DEBIAN_PACKAGE_REPLACES)
+unset(CPACK_DEBIAN_PACKAGE_RECOMMENDS)
+unset(CPACK_DEBIAN_PACKAGE_SUGGESTS)
+
+# Generate debian/shlibs control file; require exact versions.
+set(CPACK_DEBIAN_PACKAGE_GENERATE_SHLIBS YES)
+set(CPACK_DEBIAN_PACKAGE_GENERATE_SHLIBS_POLICY "=")
+
+# Add custom scripts to package. Used to ensure ldconfig runs.
+unset(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA)
+set(CPACK_DEBIAN_HALIDE_RUNTIME_PACKAGE_CONTROL_EXTRA
+    "${CMAKE_CURRENT_LIST_DIR}/triggers")
+set(CPACK_DEBIAN_PACKAGE_CONTROL_STRICT_PERMISSION TRUE)
+
+# Name the source package for this one. TODO?
+unset(CPACK_DEBIAN_PACKAGE_SOURCE)
+
+# Name the package containing debug symbols for this one. TODO?
+unset(CPACK_DEBIAN_DEBUGINFO_PACKAGE)

packaging/ubuntu/copyright

@@ -0,0 +1,26 @@
+Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: @CPACK_PACKAGE_NAME@
+Upstream-Contact: @CPACK_PACKAGE_CONTACT@
+Source: @CPACK_PACKAGE_HOMEPAGE_URL@
+
+Files: *
+Copyright: @copyright_line@
+License: MIT
+  Permission is hereby granted, free of charge, to any person obtaining
+  a copy of this software and associated documentation files (the
+  "Software"), to deal in the Software without restriction, including
+  without limitation the rights to use, copy, modify, merge, publish,
+  distribute, sublicense, and/or sell copies of the Software, and to
+  permit persons to whom the Software is furnished to do so, subject
+  to the following conditions:
+  .
+  The above copyright notice and this permission notice shall be included
+  in all copies or substantial portions of the Software.
+  .
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+  OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+  CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+  TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+  SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

packaging/ubuntu/extra-strip.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# See https://github.com/Debian/debhelper/blob/5d1bb29841043d8e47ebbdd043e6cd086cad508e/dh_strip#L362-L384
+# for what dh_strip removes.
+
+strip --remove-section=.comment --remove-section=.note "$@"

packaging/ubuntu/package.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+set -e -o pipefail
+
+halide_source=$(readlink -f "$1")
+halide_build_root=$(readlink -f "$2")
+
+[ -z "$halide_source" ] && echo "Usage: $0 <source-dir> <build-dir>" && exit
+[ -z "$halide_build_root" ] && echo "Usage: $0 <source-dir> <build-dir>" && exit
+[ -z "$LLVM_ROOT" ] && echo "Must set LLVM_ROOT to /usr/lib/llvm-VERSION" && exit
+
+function group() {
+  [[ -n "${GITHUB_ACTIONS}" && -n "${SEEN_GROUP}" ]] && echo "::endgroup::"
+  [[ -n "${GITHUB_ACTIONS}" ]] && echo "::group::$*"
+  export SEEN_GROUP=1
+}
+
+group "Configure shared Halide build"
+cmake --preset=package-ubuntu-shared -S "$halide_source" -B "$halide_build_root/shared-Release"
+
+group "Configure static Halide build"
+cmake --preset=package-ubuntu-static -S "$halide_source" -B "$halide_build_root/static-Release"
+
+group "Build shared Halide"
+cmake --build "$halide_build_root/shared-Release" -- -v
+
+group "Build static Halide"
+cmake --build "$halide_build_root/static-Release" -- -v
+
+group "Create Ubuntu packages"
+cd "$halide_build_root"
+rm -rf ./_CPack_Packages ./*.deb lintian.log
+umask 0022
+export LD_LIBRARY_PATH="$halide_build_root/shared-Release/src"
+
+cpack -G DEB -C Release --config "$halide_source/packaging/ubuntu/config.cmake"
+
+# Lintian: https://lintian.debian.org/tags
+
+group "Run strict Lintian checks"
+lintian --no-tag-display-limit -i ./*.deb
+
+group "Run extra Lintian checks"
+lintian --no-tag-display-limit -L "=info" -i ./*.deb
+
+echo "Success!"