diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..0c205de --- /dev/null +++ b/.dockerignore @@ -0,0 +1,7 @@ +# .dockerignore file for DAOS. + +# Firstly deny everything and then allow only directories and files that we're +# interested in. Other files will not be required for the build and they +# just generate noise and extra work for docker. +* +!packaging/scripts diff --git a/Jenkinsfile b/Jenkinsfile index b588d90..d4e8490 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,10 +1,12 @@ #!/usr/bin/groovy // SPDX-License-Identifier: BSD-2-Clause-Patent -// Copyright (c) 2019-2024 Intel Corporation +// Copyright 2019-2024 Intel Corporation +// Copyright 2025 Hewlett Packard Enterprise Development LP // To use a test branch (i.e. PR) until it lands to master // I.e. for testing library changes //@Library(value="pipeline-lib@your_branch") _ +@Library(value='pipeline-lib@malmberg/sre-2832') _ /* groovylint-disable-next-line CompileStatic */ packageBuildingPipelineDAOSTest(['distros': ['el8', 'el9', 'leap15', 'ubuntu20.04'], diff --git a/README.md b/README.md index 1c64873..457cba9 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,6 @@ # libfabric + libfabric RPM packaging + +This project just contains code to package a distribution from its +official repository. diff --git a/libfabric.spec b/libfabric.spec index ba6bd42..28c5193 100644 --- a/libfabric.spec +++ b/libfabric.spec @@ -234,7 +234,7 @@ find %{buildroot} -name '*.la' -exec rm -f {} ';' * Wed May 4 2022 Brian J. Murrell - 1.15.0~rc3-2 - Add _hardened_build flag to build PIE binaries on CentOS 7 -- Add optoins to C*FLAGS to build PIE binaries on Leap 15 +- Add options to C*FLAGS to build PIE binaries on Leap 15 * Tue Apr 19 2022 Lei Huang - 1.15.0~rc3-1 - Update to v1.15.0rc3 diff --git a/packaging/Dockerfile.coverity b/packaging/Dockerfile.coverity index 7eed2c3..f8171c9 100755 --- a/packaging/Dockerfile.coverity +++ b/packaging/Dockerfile.coverity @@ -1,12 +1,13 @@ # # Copyright 2018-2020, Intel Corporation +# Copyright 2025 Hewlett Packard Enterprise Development LP # # 'recipe' for Docker to build for a Coverity scan. # # Pull base image FROM fedora:latest -MAINTAINER daos-stack +LABEL maintainer="daos-stack "" # use same UID as host and default value of 1000 if not specified ARG UID=1000 diff --git a/packaging/Dockerfile.mockbuild b/packaging/Dockerfile.mockbuild index 6bad37d..4576583 100644 --- a/packaging/Dockerfile.mockbuild +++ b/packaging/Dockerfile.mockbuild @@ -1,5 +1,6 @@ # # Copyright 2018-2024 Intel Corporation +# Copyright 2025 Hewlett Packard Enterprise Development LP # # 'recipe' for Docker to build an RPM # @@ -13,15 +14,16 @@ LABEL maintainer="daos@daos.groups.io" # Use local repo server if present ARG REPO_FILE_URL -RUN if [ -n "$REPO_FILE_URL" ]; then \ - cd /etc/yum.repos.d/ && \ - curl -k -f -o daos_ci-fedora-artifactory.repo.tmp \ - "$REPO_FILE_URL"daos_ci-fedora-artifactory.repo && \ - for file in *.repo; do \ - true > $file; \ - done; \ - mv daos_ci-fedora-artifactory.repo{.tmp,}; \ - fi +ARG DAOS_LAB_CA_FILE_URL +ARG REPOSITORY_NAME +# script to install OS updates basic tools and daos dependencies +# COPY ./utils/scripts/install-fedora.sh /tmp/install.sh +# script to setup local repo if available +COPY ./packaging/scripts/repo-helper-fedora.sh /tmp/repo-helper.sh + +RUN chmod +x /tmp/repo-helper.sh && \ + /tmp/repo-helper.sh && \ + rm -f /tmp/repo-helper.sh # Install basic tools RUN dnf -y install mock make \ @@ -33,8 +35,8 @@ RUN dnf -y install mock make \ ARG UID=1000 # Add build user (to keep rpmbuild happy) -ENV USER build -ENV PASSWD build +ENV USER=build +ENV PASSWD=build # add the user to the mock group so it can run mock RUN if [ $UID != 0 ]; then \ useradd -u $UID -ms /bin/bash $USER; \ diff --git a/packaging/Dockerfile.ubuntu b/packaging/Dockerfile.ubuntu new file mode 100644 index 0000000..233f17c --- /dev/null +++ b/packaging/Dockerfile.ubuntu @@ -0,0 +1,51 @@ +# Keep Dockerfile.ubuntu the same as this file until all packaging +# jobs are fixed to have a Dockerfile.ubuntu, and then the common +# Jenkinsfile will be changed to use Dockerfile.ubuntu. +# +# Copyright 2019-2021, Intel Corporation +# Copyright 2025 Hewlett Packard Enterprise Development LP +# +# 'recipe' for Docker to build an Debian package +# +# Pull base image +ARG BASE_DISTRO=ubuntu:20.04 +FROM $BASE_DISTRO +LABEL org.opencontainers.image.authors="daos@daos.groups.io" +# Needed for later use of BASE_DISTRO +ARG BASE_DISTRO + +ARG REPO_FILE_URL +ARG DAOS_LAB_CA_FILE_URL +ARG REPOSITORY_NAME +# script to setup local repo if available +COPY ./packaging/scripts/repo-helper-ubuntu.sh /tmp/repo-helper.sh + +RUN chmod +x /tmp/repo-helper.sh && \ + /tmp/repo-helper.sh && \ + rm -f /tmp/repo-helper.sh + +# Install basic tools +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + autoconf bash ca-certificates curl debhelper dh-make \ + dpkg-dev dh-python doxygen gcc git git-buildpackage \ + javahelper locales make patch pbuilder pkg-config \ + python3-dev python3-distro python3-distutils rpm scons wget \ + cmake valgrind rpmdevtools + +# use same UID as host and default value of 1000 if not specified +ARG UID=1000 + +# Add build user (to keep chrootbuild happy) +ENV USER=build +RUN useradd -u $UID -ms /bin/bash $USER + +# need to run the build command as root, as it needs to chroot +RUN if ! grep "^#includedir /etc/sudoers.d" /etc/sudoers; then \ + echo "#includedir /etc/sudoers.d" >> /etc/sudoers; \ + fi; \ + echo "Defaults env_keep += \"DPKG_GENSYMBOLS_CHECK_LEVEL\"" > /etc/sudoers.d/build; \ + echo "build ALL=(ALL) NOPASSWD: /usr/bin/tee /root/.pbuilderrc" >> /etc/sudoers.d/build; \ + echo "build ALL=(ALL) NOPASSWD: /usr/sbin/pbuilder" >> /etc/sudoers.d/build; \ + chmod 0440 /etc/sudoers.d/build; \ + visudo -c; \ + sudo -l -U build diff --git a/packaging/Dockerfile.ubuntu.20.04 b/packaging/Dockerfile.ubuntu.20.04 index 4d05464..bd5363b 100644 --- a/packaging/Dockerfile.ubuntu.20.04 +++ b/packaging/Dockerfile.ubuntu.20.04 @@ -1,54 +1,42 @@ +# Keep Dockerfile.ubuntu the same as this file until all packaging +# jobs are fixed to have a Dockerfile.ubuntu, and then the common +# Jenkinsfile will be changed to use Dockerfile.ubuntu. # # Copyright 2019-2021, Intel Corporation +# Copyright 2025 Hewlett Packard Enterprise Development LP # # 'recipe' for Docker to build an Debian package # # Pull base image -FROM ubuntu:20.04 +ARG BASE_DISTRO=ubuntu:20.04 +FROM $BASE_DISTRO LABEL org.opencontainers.image.authors="daos@daos.groups.io" - -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ - curl gpg +# Needed for later use of BASE_DISTRO +ARG BASE_DISTRO ARG REPO_FILE_URL -RUN if [ -n "$REPO_FILE_URL" ]; then \ - cd /etc/apt/sources.list.d && \ - curl -f -o daos_ci-ubuntu20.04-artifactory.list.tmp \ - "$REPO_FILE_URL"daos_ci-ubuntu20.04-artifactory.list && \ - true > ../sources.list && \ - mv daos_ci-ubuntu20.04-artifactory.list.tmp \ - daos_ci-ubuntu20.04-artifactory.list; \ - url="${REPO_FILE_URL%/*/}/hpe-ilorest-ubuntu-bionic-proxy/"; \ - else \ - url="https://downloads.linux.hpe.com/SDR/repo/ilorest/"; \ - fi; \ - cd -; \ - mkdir -p /usr/local/share/keyrings/; \ - curl -f -O "$url"GPG-KEY-hprest; \ - gpg --no-default-keyring --keyring ./temp-keyring.gpg \ - --import GPG-KEY-hprest; \ - gpg --no-default-keyring --keyring ./temp-keyring.gpg --export \ - --output /usr/local/share/keyrings/hpe-sdr-public.gpg; \ - rm ./temp-keyring.gpg; \ - curl -f -O "$REPO_FILE_URL"esad_repo.key; \ - gpg --no-default-keyring --keyring ./temp-keyring.gpg \ - --import esad_repo.key; \ - gpg --no-default-keyring --keyring ./temp-keyring.gpg --export \ - --output /usr/local/share/keyrings/daos-stack-public.gpg +ARG DAOS_LAB_CA_FILE_URL +ARG REPOSITORY_NAME +# script to setup local repo if available +COPY ./packaging/scripts/repo-helper-ubuntu.sh /tmp/repo-helper.sh + +RUN chmod +x /tmp/repo-helper.sh && \ + /tmp/repo-helper.sh && \ + rm -f /tmp/repo-helper.sh -# Install basic tools +# Install basic tools - rpmdevtools temporary commented out. RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ autoconf bash ca-certificates curl debhelper dh-make \ dpkg-dev dh-python doxygen gcc git git-buildpackage \ javahelper locales make patch pbuilder pkg-config \ - python3-dev python3-distro python3-distutils rpm scons wget \ - cmake valgrind rpmdevtools + python3-dev python3-distro python3-distutils rpm scons sudo \ + wget cmake valgrind # rpmdevtools # use same UID as host and default value of 1000 if not specified ARG UID=1000 # Add build user (to keep chrootbuild happy) -ENV USER build +ENV USER=build RUN useradd -u $UID -ms /bin/bash $USER # need to run the build command as root, as it needs to chroot @@ -56,7 +44,7 @@ RUN if ! grep "^#includedir /etc/sudoers.d" /etc/sudoers; then echo "#includedir /etc/sudoers.d" >> /etc/sudoers; \ fi; \ echo "Defaults env_keep += \"DPKG_GENSYMBOLS_CHECK_LEVEL\"" > /etc/sudoers.d/build; \ - echo "build ALL=(ALL) NOPASSWD: /usr/bin/tee /root/.pbuilderrc" >> /etc/sudoers.d/build; \ + echo "build ALL=(ALL) NOPASSWD: /usr/bin/tee /root/.pbuilderrc" >> /etc/sudoers.d/build; \ echo "build ALL=(ALL) NOPASSWD: /usr/sbin/pbuilder" >> /etc/sudoers.d/build; \ chmod 0440 /etc/sudoers.d/build; \ visudo -c; \ diff --git a/packaging/Dockerfile.ubuntu.rolling b/packaging/Dockerfile.ubuntu.rolling index 02aca45..ed5d363 100644 --- a/packaging/Dockerfile.ubuntu.rolling +++ b/packaging/Dockerfile.ubuntu.rolling @@ -1,11 +1,12 @@ # # Copyright 2019, Intel Corporation +# Copyright 2025 Hewlett Packard Enterprise Development LP # # 'recipe' for Docker to build an Debian package # # Pull base image FROM ubuntu:rolling -Maintainer daos-stack +LABEL org.opencontainers.image.authors="daos@daos.groups.io" # use same UID as host and default value of 1000 if not specified ARG UID=1000 diff --git a/packaging/Makefile_packaging.mk b/packaging/Makefile_packaging.mk index 75038c1..f7a5117 100644 --- a/packaging/Makefile_packaging.mk +++ b/packaging/Makefile_packaging.mk @@ -168,6 +168,7 @@ endif $(notdir $(SOURCE) $(OTHER_SOURCES) $(REAL_SOURCE)): $(SPEC) $(CALLING_MAKEFILE) # TODO: need to clean up old ones + export $(SPECTOOL) $(COMMON_RPM_ARGS) -g $(SPEC) $(DEB_TOP)/%: % | $(DEB_TOP)/ diff --git a/packaging/debian_chrootbuild b/packaging/debian_chrootbuild index cc2cc96..e28e856 100755 --- a/packaging/debian_chrootbuild +++ b/packaging/debian_chrootbuild @@ -2,8 +2,21 @@ set -uex +: "${REPO_FILE_URL:=}" + +# Currently not working if [ -n "${ARTIFACTORY_URL:-}" ] && "$LOCAL_REPOS"; then - echo "MIRRORSITE=${ARTIFACTORY_URL}artifactory/ubuntu-proxy" | sudo tee /root/.pbuilderrc + pbuilderrc="./pbuilder_rc.txt" + rm -f "$pbuilderrc" + #echo "MIRRORSITE=${ARTIFACTORY_URL}/ubuntu-proxy/ubuntu" > "$pbuilderrc" + echo "export http_proxy=\"${HTTPS_PROXY}\"" >> "$pbuilderrc" + #if [ -n "$REPO_FILE_URL" ]; then + # direct="${REPO_FILE_URL##*//}" + # direct="${direct%%/*}" + # echo "no_proxy=\"${direct}\"" >> "$pbuilderrc" + #fi + # shellcheck disable=SC2002 + cat "$pbuilderrc" | sudo tee /root/.pbuilderrc fi # shellcheck disable=SC2086 diff --git a/packaging/rpm_chrootbuild b/packaging/rpm_chrootbuild index d6443b7..fe28ff3 100755 --- a/packaging/rpm_chrootbuild +++ b/packaging/rpm_chrootbuild @@ -2,6 +2,11 @@ set -uex +: "${HTTPS_PROXY:=}" +: "${REPO_FILE_URL:=}" +: "${ARCH:=$(arch)}" +: "${REPOSITORY_NAME:=artifactory}" + cp /etc/mock/"$CHROOT_NAME".cfg mock.cfg # Enable mock ccache plugin @@ -10,8 +15,20 @@ config_opts['plugin_conf']['ccache_enable'] = True config_opts['plugin_conf']['ccache_opts']['dir'] = "%(cache_topdir)s/%(root)s/ccache/" EOF +# Optionally add a proxy to mock +if [ -n "$HTTPS_PROXY" ];then + yum_proxy="http://${HTTPS_PROXY##*//}" + echo "config_opts['https_proxy'] = '$yum_proxy'" >> mock.cfg +fi + +# No proxy for local mirrors +if [ -n "$REPO_FILE_URL" ]; then + direct="${REPO_FILE_URL##*//}" + direct="${direct%%/*}" + echo "config_opts['no_proxy'] = '${direct}'" >> mock.cfg +fi -if [[ $CHROOT_NAME == *epel-8-x86_64 ]]; then +if [[ $CHROOT_NAME == *"epel-8-${ARCH}" ]]; then cat <> mock.cfg config_opts['module_setup_commands'] = [ ('enable', 'javapackages-tools:201801'), @@ -21,7 +38,7 @@ EOF fi # Use dnf on CentOS 7 -if [[ $CHROOT_NAME == *epel-7-x86_64 ]]; then +if [[ $CHROOT_NAME == *"epel-7-${ARCH}" ]]; then MOCK_OPTIONS="--dnf --no-bootstrap-chroot${MOCK_OPTIONS:+ }$MOCK_OPTIONS" fi @@ -61,7 +78,7 @@ if [ -n "${ARTIFACTORY_URL:-}" ] && "$LOCAL_REPOS"; then fi fi curl -sSf "$REPO_FILE_URL"daos_ci-"${CHROOT_NAME%-*}".repo >> mock.cfg - repo_adds+=("--enablerepo *-artifactory") + repo_adds+=("--enablerepo *-${REPOSITORY_NAME}") fi fi @@ -127,7 +144,7 @@ if ! eval time mock -r mock.cfg ${repo_dels[*]} ${repo_adds[*]} --no-clean \ fi # Save the ccache -if [ -d /scratch/ ]; then +if [ -d /scratch/mock ]; then mkdir -p "$bs_dir"/ if ! flock "$bs_dir" -c "tar -czf $bs_dir/ccache-$CHROOT_NAME-$PACKAGE.tar.gz /var/cache/mock/${CHROOT_NAME}/ccache"; then echo "Failed to save ccache. Plowing onward." diff --git a/packaging/scripts/repo-helper-fedora.sh b/packaging/scripts/repo-helper-fedora.sh new file mode 100644 index 0000000..baa7026 --- /dev/null +++ b/packaging/scripts/repo-helper-fedora.sh @@ -0,0 +1,75 @@ +#!/bin/bash +set -uex + +# This script is used by Dockerfiles to optionally use +# a local repository instead of a distro provided repository. + +: "${REPO_FILE_URL:=}" +: "${DAOS_LAB_CA_FILE_URL:=}" +: "${FVERSION:=latest}" +: "${REPOSITORY_NAME:=artifactory}" +: "${archive:=}" +if [ "$FVERSION" != "latest" ]; then + archive="-archive" +fi + +# shellcheck disable=SC2120 +disable_repos () { + local repos_dir="$1" + shift + local save_repos + IFS=" " read -r -a save_repos <<< "${*:-} daos_ci-fedora${archive}-${REPOSITORY_NAME}" + if [ -n "$REPO_FILE_URL" ]; then + pushd "$repos_dir" + local repo + for repo in "${save_repos[@]}"; do + mv "$repo".repo{,.tmp} + done + for file in *.repo; do + true > "$file" + done + for repo in "${save_repos[@]}"; do + mv "$repo".repo{.tmp,} + done + popd + fi +} + +# Use local repo server if present +install_curl() { + : +} + +# Use local repo server if present +install_optional_ca() { + ca_storage="/etc/pki/ca-trust/source/anchors/" + if [ -n "$DAOS_LAB_CA_FILE_URL" ]; then + curl -k --noproxy '*' -sSf -o "${ca_storage}lab_ca_file.crt" \ + "$DAOS_LAB_CA_FILE_URL" + update-ca-trust + fi +} + +# Use local repo server if present +# if a local repo server is present and the distro repo server can not +# be reached, have to bootstrap in an environment to get curl installed +# to then install the pre-built repo file. + +if [ -n "$REPO_FILE_URL" ]; then + install_curl + install_optional_ca + mkdir -p /etc/yum.repos.d + pushd /etc/yum.repos.d/ + curl -k --noproxy '*' -sSf \ + -o "daos_ci-fedora${archive}-${REPOSITORY_NAME}.repo" \ + "{$REPO_FILE_URL}daos_ci-fedora${archive}-${REPOSITORY_NAME}.repo" + disable_repos /etc/yum.repos.d/ + popd +fi +dnf -y install dnf-plugins-core +# This does not work in fedora/41 anymore -- needs investigation +# dnf -y config-manager --save --setopt=assumeyes=True +# dnf config-manager --save --setopt=install_weak_deps=False +dnf clean all + +disable_repos /etc/yum.repos.d/ "${save_repos[@]}" diff --git a/packaging/scripts/repo-helper-ubuntu.sh b/packaging/scripts/repo-helper-ubuntu.sh new file mode 100644 index 0000000..f62451a --- /dev/null +++ b/packaging/scripts/repo-helper-ubuntu.sh @@ -0,0 +1,103 @@ +#!/bin/bash +set -uex + +# This script is used by Dockerfiles to optionally use +# a local repository instead of a distro provided repository. +# It will also optionally allow running a /tmp/install script +# for custom packages if present. + +: "${REPO_FILE_URL:=}" +: "${HTTPS_PROXY:=}" +: "${DAOS_LAB_CA_FILE_UR:=}" +: "${REPOSITORY_NAME:=artifactory}" + +disable_repos () { + if [ -e /etc/apt/sources.list.d/ubuntu.sources ];then + mv /etc/apt/sources.list.d/ubuntu.sources \ + etc/apt/sources.list.d/ubuntu.sources.disabled + elif [ -e /etc/apt/sources.list ];then + mv /etc/apt/sources.list \ + etc/apt/sources.list.disabled + fi +} + +# Use local repo server if present +install_curl() { + + if command -v curl; then + echo "found curl!" + return + else + apt-get update + apt-get install curl ca-certificates gpg gpg-agent \ + software-properties-common + fi + + if command -v wget; then + echo "found wget!" + return + fi + # If we don't find one of these, we are basically sunk for using + # a local repository mirror. +} + +# Use local repo server if present +install_optional_ca() { + ca_storage="/usr/local/share/ca-certificates/" + if [ -n "$DAOS_LAB_CA_FILE_URL" ]; then + curl -k --noproxy '*' -sSf -o "${ca_storage}lab_ca_file.crt" \ + "$DAOS_LAB_CA_FILE_URL" + update-ca-certificates + fi +} + +echo "APT::Get::Assume-Yes \"true\";" > /etc/apt/apt.conf.d/no-prompt +echo "APT::Install-Recommends \"false\";" > /etc/apt/apt.conf.d/no-recommends +if [ -n "$HTTPS_PROXY" ];then + apt_proxy="http://${HTTPS_PROXY##*//}" + echo "Acquire::http::Proxy \"$apt_proxy\";" > \ + /etc/apt/apt.conf.d/local_proxy + if [ -n "$REPO_FILE_URL" ]; then + direct="${REPO_FILE_URL##*//}" + direct="${direct%%/*}" + echo "Acquire::http::Proxy { $direct DIRECT; };" >> \ + /etc/apt/apt.conf.d/local_proxy + fi +fi + +# Use local repo server if present +# if a local repo server is present and the distro repo server can not +# be reached, have to bootstrap in an environment to get curl installed +# to then install the pre-built repo file. +DISTRO_VERSION="${BASE_DISTRO##*:}" +if [ -n "$REPO_FILE_URL" ]; then + install_curl + install_optional_ca + # Ubuntu local repo mirror is not working + # curl -k --noproxy '*' -sSf \ + # -o "daos_ci-ubuntu${DISTRO_VERSION}-${REPOSITORY_NAME}.list" \ + # "${REPO_FILE_URL}daos_ci-ubuntu${DISTRO_VERSION}-${REPOSITORY_NAME}.list" + # disable_repos + # temp hack until we can debug the Ubuntu repos. + REPO_BASE="${REPO_FILE_URL%repo-files/}" + curl -k --noproxy '*' -sSf \ + -o "rpmdevtools_8.10-10_amd64.deb" \ + "${REPO_BASE}daos-stack-deps-ubuntu-${DISTRO_VERSION}-x86_64-stable-local/pool/rpmdevtools_8.10-10_amd64.deb" + apt-get install "./rpmdevtools_8.10-10_amd64.deb" + mkdir -p /usr/local/share/keyrings/ + curl --noproxy '*' -sSf -O "${REPO_FILE_URL}esad_repo.key" + gpg --no-default-keyring --keyring ./temp-keyring.gpg \ + --import esad_repo.key + gpg --no-default-keyring --keyring ./temp-keyring.gpg --export \ + --output /usr/local/share/keyrings/daos-stack-public.gpg +fi + +apt-get update +apt-get upgrade +add-apt-repository ppa:longsleep/golang-backports +apt-get update +if [ -e /tmp/install.sh ]; then + chmod +x /tmp/install.sh + /tmp/install.sh +fi +apt-get clean all