SRE-2832 Packaging update part 1

.dockerignore

@@ -0,0 +1,7 @@
+# .dockerignore file for DAOS.
+
+# Firstly deny everything and then allow only directories and files that we're
+# interested in.  Other files will not be required for the build and they
+# just generate noise and extra work for docker.
+*
+!packaging/scripts

Jenkinsfile

@@ -1,10 +1,12 @@
 #!/usr/bin/groovy
 // SPDX-License-Identifier: BSD-2-Clause-Patent
-// Copyright (c) 2019-2024 Intel Corporation
+// Copyright 2019-2024 Intel Corporation
+// Copyright 2025 Hewlett Packard Enterprise Development LP
 
 // To use a test branch (i.e. PR) until it lands to master
 // I.e. for testing library changes
 //@Library(value="pipeline-lib@your_branch") _
+@Library(value='pipeline-lib@malmberg/sre-2832') _
 
 /* groovylint-disable-next-line CompileStatic */
 packageBuildingPipelineDAOSTest(['distros': ['el8', 'el9', 'leap15', 'ubuntu20.04'],

README.md

@@ -1,2 +1,6 @@
 # libfabric
+
 libfabric RPM packaging
+
+This project just contains code to package a distribution from its
+official repository.

libfabric.spec

@@ -234,7 +234,7 @@ find %{buildroot} -name '*.la' -exec rm -f {} ';'
 
 * Wed May  4 2022 Brian J. Murrell <[email protected]> - 1.15.0~rc3-2
 - Add _hardened_build flag to build PIE binaries on CentOS 7
-- Add optoins to C*FLAGS to build PIE binaries on Leap 15
+- Add options to C*FLAGS to build PIE binaries on Leap 15
 
 * Tue Apr 19 2022 Lei Huang <[email protected]> - 1.15.0~rc3-1
 - Update to v1.15.0rc3

packaging/Dockerfile.coverity

@@ -1,12 +1,13 @@
 #
 # Copyright 2018-2020, Intel Corporation
+# Copyright 2025 Hewlett Packard Enterprise Development LP
 #
 # 'recipe' for Docker to build for a Coverity scan.
 #
 
 # Pull base image
 FROM fedora:latest
-MAINTAINER daos-stack <[email protected]>
+LABEL maintainer="daos-stack <[email protected]>""
 
 # use same UID as host and default value of 1000 if not specified
 ARG UID=1000

packaging/Dockerfile.mockbuild

@@ -1,5 +1,6 @@
 #
 # Copyright 2018-2024 Intel Corporation
+# Copyright 2025 Hewlett Packard Enterprise Development LP
 #
 # 'recipe' for Docker to build an RPM
 #
@@ -13,15 +14,16 @@ LABEL maintainer="[email protected]"
 
 # Use local repo server if present
 ARG REPO_FILE_URL
-RUN if [ -n "$REPO_FILE_URL" ]; then                            \
-        cd /etc/yum.repos.d/ &&                                 \
-        curl -k -f -o daos_ci-fedora-artifactory.repo.tmp       \
-             "$REPO_FILE_URL"daos_ci-fedora-artifactory.repo && \
-        for file in *.repo; do                                  \
-            true > $file;                                       \
-        done;                                                   \
-        mv daos_ci-fedora-artifactory.repo{.tmp,};              \
-    fi
+ARG DAOS_LAB_CA_FILE_URL
+ARG REPOSITORY_NAME
+# script to install OS updates basic tools and daos dependencies
+# COPY ./utils/scripts/install-fedora.sh /tmp/install.sh
+# script to setup local repo if available
+COPY ./packaging/scripts/repo-helper-fedora.sh /tmp/repo-helper.sh
+
+RUN chmod +x /tmp/repo-helper.sh &&                 \
+    /tmp/repo-helper.sh &&                          \
+    rm -f /tmp/repo-helper.sh
 
 # Install basic tools
 RUN dnf -y install mock make                                        \
@@ -33,8 +35,8 @@ RUN dnf -y install mock make                                        \
 ARG UID=1000
 
 # Add build user (to keep rpmbuild happy)
-ENV USER build
-ENV PASSWD build
+ENV USER=build
+ENV PASSWD=build
 # add the user to the mock group so it can run mock
 RUN if [ $UID != 0 ]; then                   \
         useradd -u $UID -ms /bin/bash $USER; \

packaging/Dockerfile.ubuntu

@@ -0,0 +1,51 @@
+# Keep Dockerfile.ubuntu the same as this file until all packaging
+# jobs are fixed to have a Dockerfile.ubuntu, and then the common
+# Jenkinsfile will be changed to use Dockerfile.ubuntu.
+#
+# Copyright 2019-2021, Intel Corporation
+# Copyright 2025 Hewlett Packard Enterprise Development LP
+#
+# 'recipe' for Docker to build an Debian package
+#
+# Pull base image
+ARG BASE_DISTRO=ubuntu:20.04
+FROM $BASE_DISTRO
+LABEL org.opencontainers.image.authors="[email protected]"
+# Needed for later use of BASE_DISTRO
+ARG BASE_DISTRO
+
+ARG REPO_FILE_URL
+ARG DAOS_LAB_CA_FILE_URL
+ARG REPOSITORY_NAME
+# script to setup local repo if available
+COPY ./packaging/scripts/repo-helper-ubuntu.sh /tmp/repo-helper.sh
+
+RUN chmod +x /tmp/repo-helper.sh &&                 \
+    /tmp/repo-helper.sh &&                          \
+    rm -f /tmp/repo-helper.sh
+
+# Install basic tools
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+            autoconf bash ca-certificates curl debhelper dh-make        \
+            dpkg-dev dh-python doxygen gcc git git-buildpackage         \
+            javahelper locales make patch pbuilder pkg-config           \
+            python3-dev python3-distro python3-distutils rpm scons wget \
+            cmake valgrind rpmdevtools
+
+# use same UID as host and default value of 1000 if not specified
+ARG UID=1000
+
+# Add build user (to keep chrootbuild happy)
+ENV USER=build
+RUN useradd -u $UID -ms /bin/bash $USER
+
+# need to run the build command as root, as it needs to chroot
+RUN if ! grep "^#includedir /etc/sudoers.d" /etc/sudoers; then                                     \
+        echo "#includedir /etc/sudoers.d" >> /etc/sudoers;                                         \
+    fi;                                                                                            \
+    echo "Defaults env_keep += \"DPKG_GENSYMBOLS_CHECK_LEVEL\"" > /etc/sudoers.d/build;            \
+    echo "build ALL=(ALL) NOPASSWD: /usr/bin/tee /root/.pbuilderrc" >> /etc/sudoers.d/build;       \
+    echo "build ALL=(ALL) NOPASSWD: /usr/sbin/pbuilder" >> /etc/sudoers.d/build;                   \
+    chmod 0440 /etc/sudoers.d/build;                                                               \
+    visudo -c;                                                                                     \
+    sudo -l -U build

packaging/Dockerfile.ubuntu.20.04

@@ -1,62 +1,50 @@
+# Keep Dockerfile.ubuntu the same as this file until all packaging
+# jobs are fixed to have a Dockerfile.ubuntu, and then the common
+# Jenkinsfile will be changed to use Dockerfile.ubuntu.
 #
 # Copyright 2019-2021, Intel Corporation
+# Copyright 2025 Hewlett Packard Enterprise Development LP
 #
 # 'recipe' for Docker to build an Debian package
 #
 # Pull base image
-FROM ubuntu:20.04
+ARG BASE_DISTRO=ubuntu:20.04
+FROM $BASE_DISTRO
 LABEL org.opencontainers.image.authors="[email protected]"
-
-RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
-    curl gpg
+# Needed for later use of BASE_DISTRO
+ARG BASE_DISTRO
 
 ARG REPO_FILE_URL
-RUN if [ -n "$REPO_FILE_URL" ]; then                                 \
-        cd /etc/apt/sources.list.d &&                                \
-        curl -f -o daos_ci-ubuntu20.04-artifactory.list.tmp          \
-             "$REPO_FILE_URL"daos_ci-ubuntu20.04-artifactory.list && \
-        true > ../sources.list &&                                    \
-        mv daos_ci-ubuntu20.04-artifactory.list.tmp                  \
-           daos_ci-ubuntu20.04-artifactory.list;                     \
-        url="${REPO_FILE_URL%/*/}/hpe-ilorest-ubuntu-bionic-proxy/"; \
-    else                                                             \
-        url="https://downloads.linux.hpe.com/SDR/repo/ilorest/";     \
-    fi;                                                              \
-    cd -;                                                            \
-    mkdir -p /usr/local/share/keyrings/;                             \
-    curl -f -O "$url"GPG-KEY-hprest;                                 \
-    gpg --no-default-keyring --keyring ./temp-keyring.gpg            \
-        --import GPG-KEY-hprest;                                     \
-    gpg --no-default-keyring --keyring ./temp-keyring.gpg --export   \
-        --output /usr/local/share/keyrings/hpe-sdr-public.gpg;       \
-    rm ./temp-keyring.gpg;                                           \
-    curl -f -O "$REPO_FILE_URL"esad_repo.key;                        \
-    gpg --no-default-keyring --keyring ./temp-keyring.gpg            \
-        --import esad_repo.key;                                      \
-    gpg --no-default-keyring --keyring ./temp-keyring.gpg --export   \
-        --output /usr/local/share/keyrings/daos-stack-public.gpg
+ARG DAOS_LAB_CA_FILE_URL
+ARG REPOSITORY_NAME
+# script to setup local repo if available
+COPY ./packaging/scripts/repo-helper-ubuntu.sh /tmp/repo-helper.sh
+
+RUN chmod +x /tmp/repo-helper.sh &&                 \
+    /tmp/repo-helper.sh &&                          \
+    rm -f /tmp/repo-helper.sh
 
-# Install basic tools
+# Install basic tools - rpmdevtools temporary commented out.
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
             autoconf bash ca-certificates curl debhelper dh-make        \
             dpkg-dev dh-python doxygen gcc git git-buildpackage         \
             javahelper locales make patch pbuilder pkg-config           \
-            python3-dev python3-distro python3-distutils rpm scons wget \
-            cmake valgrind rpmdevtools
+            python3-dev python3-distro python3-distutils rpm scons sudo \
+            wget cmake valgrind # rpmdevtools
 
 # use same UID as host and default value of 1000 if not specified
 ARG UID=1000
 
 # Add build user (to keep chrootbuild happy)
-ENV USER build
+ENV USER=build
 RUN useradd -u $UID -ms /bin/bash $USER
 
 # need to run the build command as root, as it needs to chroot
 RUN if ! grep "^#includedir /etc/sudoers.d" /etc/sudoers; then                                     \
         echo "#includedir /etc/sudoers.d" >> /etc/sudoers;                                         \
     fi;                                                                                            \
     echo "Defaults env_keep += \"DPKG_GENSYMBOLS_CHECK_LEVEL\"" > /etc/sudoers.d/build;            \
-    echo "build ALL=(ALL) NOPASSWD: /usr/bin/tee /root/.pbuilderrc" >> /etc/sudoers.d/build; \
+    echo "build ALL=(ALL) NOPASSWD: /usr/bin/tee /root/.pbuilderrc" >> /etc/sudoers.d/build;       \
     echo "build ALL=(ALL) NOPASSWD: /usr/sbin/pbuilder" >> /etc/sudoers.d/build;                   \
     chmod 0440 /etc/sudoers.d/build;                                                               \
     visudo -c;                                                                                     \

packaging/Dockerfile.ubuntu.rolling

@@ -1,11 +1,12 @@
 #
 # Copyright 2019, Intel Corporation
+# Copyright 2025 Hewlett Packard Enterprise Development LP
 #
 # 'recipe' for Docker to build an Debian package
 #
 # Pull base image
 FROM ubuntu:rolling
-Maintainer daos-stack <daos@daos.groups.io>
+LABEL org.opencontainers.image.authors="[email protected]"
 
 # use same UID as host and default value of 1000 if not specified
 ARG UID=1000

packaging/Makefile_packaging.mk

@@ -168,6 +168,7 @@ endif
 
 $(notdir $(SOURCE) $(OTHER_SOURCES) $(REAL_SOURCE)): $(SPEC) $(CALLING_MAKEFILE)
 	# TODO: need to clean up old ones
+	export
 	$(SPECTOOL) $(COMMON_RPM_ARGS) -g $(SPEC)
 
 $(DEB_TOP)/%: % | $(DEB_TOP)/

packaging/debian_chrootbuild

@@ -2,8 +2,21 @@
 
 set -uex
 
+: "${REPO_FILE_URL:=}"
+
+# Currently not working
 if [ -n "${ARTIFACTORY_URL:-}" ] && "$LOCAL_REPOS"; then
-    echo "MIRRORSITE=${ARTIFACTORY_URL}artifactory/ubuntu-proxy" | sudo tee /root/.pbuilderrc
+    pbuilderrc="./pbuilder_rc.txt"
+    rm -f "$pbuilderrc"
+    #echo "MIRRORSITE=${ARTIFACTORY_URL}/ubuntu-proxy/ubuntu" > "$pbuilderrc"
+    echo "export http_proxy=\"${HTTPS_PROXY}\"" >> "$pbuilderrc"
+    #if [ -n "$REPO_FILE_URL" ]; then
+    #    direct="${REPO_FILE_URL##*//}"
+    #    direct="${direct%%/*}"
+    #    echo "no_proxy=\"${direct}\"" >> "$pbuilderrc"
+    #fi
+    # shellcheck disable=SC2002
+    cat "$pbuilderrc" | sudo tee /root/.pbuilderrc
 fi
 
 # shellcheck disable=SC2086

packaging/rpm_chrootbuild

@@ -2,6 +2,11 @@
 
 set -uex
 
+: "${HTTPS_PROXY:=}"
+: "${REPO_FILE_URL:=}"
+: "${ARCH:=$(arch)}"
+: "${REPOSITORY_NAME:=artifactory}"
+
 cp /etc/mock/"$CHROOT_NAME".cfg mock.cfg
 
 # Enable mock ccache plugin
@@ -10,8 +15,20 @@ config_opts['plugin_conf']['ccache_enable'] = True
 config_opts['plugin_conf']['ccache_opts']['dir'] = "%(cache_topdir)s/%(root)s/ccache/"
 EOF
 
+# Optionally add a proxy to mock
+if [ -n "$HTTPS_PROXY" ];then
+    yum_proxy="http://${HTTPS_PROXY##*//}"
+    echo "config_opts['https_proxy'] = '$yum_proxy'" >> mock.cfg
+fi
+
+# No proxy for local mirrors
+if [ -n "$REPO_FILE_URL" ]; then
+    direct="${REPO_FILE_URL##*//}"
+    direct="${direct%%/*}"
+    echo "config_opts['no_proxy'] = '${direct}'" >> mock.cfg
+fi
 
-if [[ $CHROOT_NAME == *epel-8-x86_64 ]]; then
+if [[ $CHROOT_NAME == *"epel-8-${ARCH}" ]]; then
     cat <<EOF >> mock.cfg
 config_opts['module_setup_commands'] = [
   ('enable', 'javapackages-tools:201801'),
@@ -21,7 +38,7 @@ EOF
 fi
 
 # Use dnf on CentOS 7
-if [[ $CHROOT_NAME == *epel-7-x86_64 ]]; then
+if [[ $CHROOT_NAME == *"epel-7-${ARCH}" ]]; then
     MOCK_OPTIONS="--dnf --no-bootstrap-chroot${MOCK_OPTIONS:+ }$MOCK_OPTIONS"
 fi
 
@@ -61,7 +78,7 @@ if [ -n "${ARTIFACTORY_URL:-}" ] && "$LOCAL_REPOS"; then
             fi
         fi
         curl -sSf "$REPO_FILE_URL"daos_ci-"${CHROOT_NAME%-*}".repo >> mock.cfg
-        repo_adds+=("--enablerepo *-artifactory")
+        repo_adds+=("--enablerepo *-${REPOSITORY_NAME}")
     fi
 fi
 
@@ -127,7 +144,7 @@ if ! eval time mock -r mock.cfg ${repo_dels[*]} ${repo_adds[*]} --no-clean \
 fi
 
 # Save the ccache
-if [ -d /scratch/ ]; then
+if [ -d /scratch/mock ]; then
     mkdir -p "$bs_dir"/
     if ! flock "$bs_dir" -c "tar -czf $bs_dir/ccache-$CHROOT_NAME-$PACKAGE.tar.gz /var/cache/mock/${CHROOT_NAME}/ccache"; then
         echo "Failed to save ccache.  Plowing onward."