SRE-2832 Build: Update for larger Jenkinsfiles

Update Jenkinsfiles to support SCRIPT_SPLITTING_TRANSFORMATION per
https://issues.jenkins.io/browse/JENKINS-56500

utils/githooks/README.md: Fix markuplint flagged issues.

utils/scripts/helpers/install_maldet.ch :
  Fix defaults to be more implementation neutral.

utils/scripts/helpers/repo-helper-*.sh :
  Optionally install CA needed to validate local repository mirrors.

Dockerfiles modified to support intalling the CA.

Dockerfile.code_scanning modified to allow specifying the fedora
version used.

Signed-off-by: John E. Malmberg <[email protected]>

.dockerignore

@@ -15,6 +15,7 @@
 !utils/setup_daos_server_helper.sh
 !utils/sl/setup_local.sh
 !utils/scripts
+!utils/rpms
 !utils/run_utest.py
 !utils/utest.yaml
 !utils/unit

ci/docker/Dockerfile.maldet.el.8

@@ -1,4 +1,5 @@
 # Copyright 2018-2022 Intel Corporation
+# Copyright 2025 Hewlett Packard Enterprise Development LP
 # All rights reserved.
 #
 # 'recipe' for Docker to build an environment scan DAOS for malware.
@@ -17,6 +18,7 @@ ARG CB0
 ARG REPO_FILE_URL
 ARG JENKINS_URL
 ARG REPOS
+ARG DAOS_LAB_CA_FILE_URL
 # script to translate lookup distro names to commonly used names
 COPY ./utils/scripts/helpers/distro_info.sh /tmp/distro_info.sh
 # script to install OS updates basic tools and maldet

ci/docker/Dockerfile.maldet.leap.15

@@ -1,4 +1,5 @@
 # Copyright 2018-2022 Intel Corporation
+# Copyright 2025 Hewlett Packard Enterprise Development LP
 # All rights reserved.
 #
 # 'recipe' for Docker to build an environment scan DAOS for malware.
@@ -17,6 +18,7 @@ ARG CB0
 ARG REPO_FILE_URL
 ARG JENKINS_URL
 ARG REPOS
+ARG DAOS_LAB_CA_FILE_URL
 # script to translate lookup distro names to commonly used names
 COPY ./utils/scripts/helpers/distro_info.sh /tmp/distro_info.sh
 # script to install OS updates basic tools and maldet

ci/rpm/build.sh

@@ -21,7 +21,7 @@ fi
 mydir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
 ci_envs="$mydir/../parse_ci_envs.sh"
 if [ -e "${ci_envs}" ]; then
-  # shellcheck source=parse_ci_envs.sh
+  # shellcheck source=parse_ci_envs.sh disable=SC1091
   source "${ci_envs}"
 fi
 
@@ -52,6 +52,8 @@ if ! mkdir -p "artifacts/${TARGET}/"; then
     exit 1
 fi
 
+export
+
 # shellcheck disable=SC2086
 DEBEMAIL="$DAOS_EMAIL" DEBFULLNAME="$DAOS_FULLNAME"               \
 TOPDIR=$PWD make CHROOT_NAME="${CHROOT_NAME}" ${JOB_REPOS}        \

utils/docker/Dockerfile.code_scanning

@@ -4,15 +4,34 @@
 # 'recipe' for Docker for code scanning.
 #
 
-# Pull base image
-FROM fedora:latest
+ARG FVERSION=latest
+FROM fedora:$FVERSION
+# Needed for later use of FVERSION
+ARG FVERSION
 LABEL maintainer="[email protected]"
 
 # Intermittent cache-bust.  Used to reduce load on the actual CACHEBUST later.
 ARG CB0
 
+# Use local repo server if present
+ARG REPO_FILE_URL
+ARG DAOS_LAB_CA_FILE_URL
+# script to install OS updates basic tools and daos dependencies
+# COPY ./utils/scripts/install-fedora.sh /tmp/install.sh
+# script to setup local repo if available
+COPY ./utils/scripts/helpers/repo-helper-fedora.sh /tmp/repo-helper.sh
+
+RUN chmod +x /tmp/repo-helper.sh &&                 \
+    /tmp/repo-helper.sh &&                          \
+    rm -f /tmp/repo-helper.sh
+
 # Install Python Bandit scanner.
-RUN dnf -y upgrade && dnf -y install bandit && dnf clean all
+# The unset commands are currently needed for the combination of running
+# with a local repository, yet needing a proxy to reach outside repositories.
+# This needs to be moved to a shell script like above in the future to
+# properly only remove the proxy variables only when they need to be removed
+RUN unset HTTPS_PROXY && unset https_proxy && \
+    dnf -y upgrade && dnf -y install bandit && dnf clean all
 
 ARG CB1
-RUN dnf -y upgrade && dnf clean all
+RUN unset HTTPS_PROXY && unset https_proxy && dnf -y upgrade && dnf clean all

utils/docker/Dockerfile.el.8

@@ -1,4 +1,5 @@
 # Copyright 2018-2024 Intel Corporation
+# Copyright 2025 Hewlett Packard Enterprise Development LP
 # All rights reserved.
 #
 # 'recipe' for Docker to build an image of EL 8 based
@@ -8,7 +9,7 @@
 # Pull base image
 ARG POINT_RELEASE=
 ARG BASE_DISTRO=rockylinux/rockylinux:8$POINT_RELEASE
-FROM $BASE_DISTRO as basic
+FROM $BASE_DISTRO AS basic
 LABEL maintainer="[email protected]"
 # Needed for later use of BASE_DISTRO
 ARG BASE_DISTRO
@@ -19,6 +20,7 @@ ARG CB0
 ARG REPO_FILE_URL
 ARG JENKINS_URL
 ARG REPOS
+ARG DAOS_LAB_CA_FILE_URL
 # script to install OS updates basic tools and daos dependencies
 COPY ./utils/scripts/install-el8.sh /tmp/install.sh
 # script to setup local repo if available

utils/docker/Dockerfile.el.9

@@ -1,4 +1,5 @@
 # Copyright 2022-2024 Intel Corporation
+# Copyright 2025 Hewlett Packard Enterprise Development LP
 # All rights reserved.
 #
 # 'recipe' for Docker to build an image of EL 9 based
@@ -8,7 +9,7 @@
 # Pull base image
 ARG POINT_RELEASE=
 ARG BASE_DISTRO=almalinux:9$POINT_RELEASE
-FROM $BASE_DISTRO as basic
+FROM $BASE_DISTRO AS basic
 LABEL maintainer="[email protected]"
 # Needed for later use of BASE_DISTRO
 ARG BASE_DISTRO
@@ -19,6 +20,7 @@ ARG CB0
 ARG REPO_FILE_URL
 ARG JENKINS_URL
 ARG REPOS
+ARG DAOS_LAB_CA_FILE_URL
 # script to install OS updates basic tools and daos dependencies
 COPY ./utils/scripts/install-el9.sh /tmp/install.sh
 # script to setup local repo if available

utils/docker/Dockerfile.leap.15

@@ -1,4 +1,5 @@
 # Copyright 2018-2024 Intel Corporation
+# Copyright 2025 Hewlett Packard Enterprise Development LP
 # All rights reserved.
 #
 # 'recipe' for Docker to build an image of Leap based
@@ -8,7 +9,7 @@
 # Pull base image
 ARG POINT_RELEASE=
 ARG BASE_DISTRO=registry.opensuse.org/opensuse/leap-dnf:15$POINT_RELEASE
-FROM $BASE_DISTRO as basic
+FROM $BASE_DISTRO AS basic
 LABEL maintainer="[email protected]"
 # Needed for later use of BASE_DISTRO
 ARG BASE_DISTRO
@@ -19,6 +20,7 @@ ARG CB0
 ARG REPO_FILE_URL
 ARG JENKINS_URL
 ARG REPOS
+ARG DAOS_LAB_CA_FILE_URL
 # script to install OS updates basic tools and daos dependencies
 COPY ./utils/scripts/install-leap15.sh /tmp/install.sh
 # script to setup local repo if available

utils/docker/Dockerfile.ubuntu

@@ -1,4 +1,5 @@
 # Copyright 2018-2024 Intel Corporation
+# Copyright 2025 Hewlett Packard Enterprise Development LP
 # All rights reserved.
 #
 # 'recipe' for Docker to build an image of Ubuntu-based environment for building the DAOS project.
@@ -10,24 +11,22 @@
 ARG BASE_DISTRO=ubuntu:22.04
 FROM $BASE_DISTRO
 LABEL maintainer="[email protected]"
+# Needed for later use of BASE_DISTRO
+ARG BASE_DISTRO
 
 # Intermittent cache-bust.  Used to reduce load on the actual CB1 later.
 ARG CB0
 
 ENV DEBIAN_FRONTEND=noninteractive
 
 # Install basic tools
-COPY ./utils/scripts/install-ubuntu.sh /tmp/install.sh
-RUN echo "APT::Get::Assume-Yes \"true\";" > /etc/apt/apt.conf.d/no-prompt
-RUN echo "APT::Install-Recommends \"false\";" > /etc/apt/apt.conf.d/no-recommends
-RUN apt-get update && \
-    apt-get upgrade && \
-    apt-get install gpg-agent software-properties-common && \
-    add-apt-repository ppa:longsleep/golang-backports && \
-    apt-get update && \
-    chmod +x /tmp/install.sh && \
-    /tmp/install.sh && \
-    apt-get clean all
+ARG DAOS_LAB_CA_FILE_URL
+# script to setup local repo if available
+COPY ./utils/scripts/helpers/repo-helper-ubuntu.sh /tmp/repo-helper.sh
+
+RUN chmod +x /tmp/repo-helper.sh &&                 \
+    /tmp/repo-helper.sh &&                          \
+    rm -f /tmp/repo-helper.sh
 
 RUN locale-gen en_US.UTF-8
 

utils/githooks/README.md

@@ -1,7 +1,7 @@
 # About DAOS Git hooks
 
 Githooks are a [well documented](https://git-scm.com/docs/githooks) feature
-of git that enable various local exectubles to be run during various stages of
+of git that enable various local executables to be run during various stages of
 the git workflow.
 
 The DAOS repo contains several built-in githooks that are intended
@@ -18,6 +18,7 @@ Recommended: Configure your `core.hookspath`.
 Any new githooks added to the repository will automatically run,
 but possibly require additional software to produce the desired effect.
 Additionally, as the branch changes, the githooks change with it.
+
 ```sh
 git config core.hookspath utils/githooks
 ```
@@ -33,10 +34,13 @@ effectively a noop if such is not installed.
 Requirements come from a combination of `pip` and system packages and can usually be installed through standard means.  
 To install `pip` packages specified in [utils/cq/requirements.txt](../../utils/cq/requirements.txt) it is recommended to setup a virtual environment and install with pip.  
 If you already have a [virtual environment for building](../../docs/QSG/build_from_scratch.md#python-packages) you can simply install the requirements:
+
 ```sh
 python3 -m pip install -r utils/cq/requirements.txt
 ```
+
 Install system packages with your package manager - for example:
+
 ```sh
 sudo dnf install git-clang-format -y
 ```
@@ -52,7 +56,7 @@ If the check is unable to parse the version output, it will fail. Try running
 3. `flake8`
 4. `isort`
 5. `yamllint`
-6. `gofmt`
+6. `gofmt` or `golang-go`
 7. `codespell`
 
 #### Optional tools
@@ -79,11 +83,13 @@ any one of the required tools is missing.
 allowing the user to inspect the changes and retry the commit.
    - See [.clang-format](../../.clang-format) for configuration
    - In some cases unwanted formatting changes are made. To disable formatting, for example:
-     ```
+
+     ```c
      /* clang-format off */
      ...
      /* clang-format on */
      ```
+
 6. gofmt - Automatically formats for modified GO files
 7. isort - Linter for python imports on modified python files
 8. flake - Linter for python files

utils/rpms/packaging/Dockerfile.coverity

@@ -6,7 +6,7 @@
 
 # Pull base image
 FROM fedora:latest
-MAINTAINER daos-stack <[email protected]>
+LABEL maintainer="daos-stack <[email protected]>"
 
 # use same UID as host and default value of 1000 if not specified
 ARG UID=1000

utils/rpms/packaging/Dockerfile.mockbuild

@@ -1,5 +1,6 @@
 #
 # Copyright 2018-2024 Intel Corporation
+# Copyright 2025 Hewlett Packard Enterprise Development LP
 #
 # 'recipe' for Docker to build an RPM
 #
@@ -13,15 +14,15 @@ LABEL maintainer="[email protected]"
 
 # Use local repo server if present
 ARG REPO_FILE_URL
-RUN if [ -n "$REPO_FILE_URL" ]; then                            \
-        cd /etc/yum.repos.d/ &&                                 \
-        curl -k -f -o daos_ci-fedora-artifactory.repo.tmp       \
-             "$REPO_FILE_URL"daos_ci-fedora-artifactory.repo && \
-        for file in *.repo; do                                  \
-            true > $file;                                       \
-        done;                                                   \
-        mv daos_ci-fedora-artifactory.repo{.tmp,};              \
-    fi
+ARG DAOS_LAB_CA_FILE_URL
+# script to install OS updates basic tools and daos dependencies
+# COPY ./utils/scripts/install-fedora.sh /tmp/install.sh
+# script to setup local repo if available
+COPY ./utils/scripts/helpers/repo-helper-fedora.sh /tmp/repo-helper.sh
+
+RUN chmod +x /tmp/repo-helper.sh &&                 \
+    /tmp/repo-helper.sh &&                          \
+    rm -f /tmp/repo-helper.sh
 
 # Install basic tools
 RUN dnf -y install mock make                                        \
@@ -50,7 +51,7 @@ RUN dnf -y upgrade && \
 # RPM doesn't wipe out our patch
 # Ditto for the patch to zero and display ccache stats
 # https://github.com/rpm-software-management/mock/pull/1299
-ARG PACKAGINGDIR=packaging
+ARG PACKAGINGDIR=./utils/rpms/packaging
 COPY ${PACKAGINGDIR}/*.patch ./
 RUN (cd $(python3 -c 'import site; print(site.getsitepackages()[-1])') &&                      \
      if ! grep -e --ignore-unused-rpmlintrc rpmlint/cli.py; then                               \

utils/rpms/packaging/Dockerfile.ubuntu.20.04

@@ -4,38 +4,24 @@
 # 'recipe' for Docker to build an Debian package
 #
 # Pull base image
-FROM ubuntu:20.04
+ARG BASE_DISTRO=ubuntu:20.04
+FROM $BASE_DISTRO
 LABEL org.opencontainers.image.authors="[email protected]"
+# Needed for later use of BASE_DISTRO
+ARG BASE_DISTRO
+
+ARG REPO_FILE_URL
+ARG DAOS_LAB_CA_FILE_URL
+# script to setup local repo if available
+COPY ./utils/scripts/helpers/repo-helper-ubuntu.sh /tmp/repo-helper.sh
+
+RUN chmod +x /tmp/repo-helper.sh &&                 \
+    /tmp/repo-helper.sh                             \
+    rm -f /tmp/repo-helper.sh
 
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
     curl gpg
 
-ARG REPO_FILE_URL
-RUN if [ -n "$REPO_FILE_URL" ]; then                                 \
-        cd /etc/apt/sources.list.d &&                                \
-        curl -f -o daos_ci-ubuntu20.04-artifactory.list.tmp          \
-             "$REPO_FILE_URL"daos_ci-ubuntu20.04-artifactory.list && \
-        true > ../sources.list &&                                    \
-        mv daos_ci-ubuntu20.04-artifactory.list.tmp                  \
-           daos_ci-ubuntu20.04-artifactory.list;                     \
-        url="${REPO_FILE_URL%/*/}/hpe-ilorest-ubuntu-bionic-proxy/"; \
-    else                                                             \
-        url="https://downloads.linux.hpe.com/SDR/repo/ilorest/";     \
-    fi;                                                              \
-    cd -;                                                            \
-    mkdir -p /usr/local/share/keyrings/;                             \
-    curl -f -O "$url"GPG-KEY-hprest;                                 \
-    gpg --no-default-keyring --keyring ./temp-keyring.gpg            \
-        --import GPG-KEY-hprest;                                     \
-    gpg --no-default-keyring --keyring ./temp-keyring.gpg --export   \
-        --output /usr/local/share/keyrings/hpe-sdr-public.gpg;       \
-    rm ./temp-keyring.gpg;                                           \
-    curl -f -O "$REPO_FILE_URL"esad_repo.key;                        \
-    gpg --no-default-keyring --keyring ./temp-keyring.gpg            \
-        --import esad_repo.key;                                      \
-    gpg --no-default-keyring --keyring ./temp-keyring.gpg --export   \
-        --output /usr/local/share/keyrings/daos-stack-public.gpg
-
 # Install basic tools
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
             autoconf bash ca-certificates curl debhelper dh-make        \
@@ -45,10 +31,10 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
             cmake valgrind rpmdevtools
 
 # use same UID as host and default value of 1000 if not specified
-ARG UID=1000
+ARG UID 1000
 
 # Add build user (to keep chrootbuild happy)
-ENV USER build
+ENV USER=build
 RUN useradd -u $UID -ms /bin/bash $USER
 
 # need to run the build command as root, as it needs to chroot

utils/rpms/packaging/Dockerfile.ubuntu.rolling

@@ -5,7 +5,7 @@
 #
 # Pull base image
 FROM ubuntu:rolling
-Maintainer daos-stack <[email protected]>
+LABEL maintainer="daos-stack <[email protected]>"
 
 # use same UID as host and default value of 1000 if not specified
 ARG UID=1000