Dockerfile.QA

# Copyright 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#  * Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#  * Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#  * Neither the name of NVIDIA CORPORATION nor the names of its
#    contributors may be used to endorse or promote products derived
#    from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

ARG BASE_IMAGE=tritonserver
ARG CIBASE_IMAGE=tritonserver_cibase
ARG SDK_IMAGE=tritonserver_sdk
ARG TRITON_REPO_ORGANIZATION=http://github.com/triton-inference-server
ARG TRITON_COMMON_REPO_TAG=main
ARG TRITON_CORE_REPO_TAG=main
ARG TRITON_THIRD_PARTY_REPO_TAG=main
ARG TRITON_BACKEND_REPO_TAG=main
ARG TRITONTMP_DIR=/tmp
ARG IGPU_BUILD=0

############################################################################
## Test artifacts built as part of the tritonserver build are
## available in CIBASE_IMAGE. Copy these artifacts into the QA area.
############################################################################
FROM ${CIBASE_IMAGE} AS cibase

ARG TRITONTMP_DIR
ARG TRITON_REPO_ORGANIZATION
ARG TRITON_COMMON_REPO_TAG
ARG TRITON_CORE_REPO_TAG
ARG TRITON_THIRD_PARTY_REPO_TAG
ARG TRITON_BACKEND_REPO_TAG
ARG IGPU_BUILD

# Ensure apt-get won't prompt for selecting options
ENV DEBIAN_FRONTEND=noninteractive

RUN apt-get update && \
    apt-get install -y --no-install-recommends \
            build-essential \
            libarchive-dev \
            libboost-dev \
            python3-dev \
            python3-pip \
            rapidjson-dev \
            software-properties-common && \
    rm -rf /var/lib/apt/lists/*

RUN pip3 install --upgrade pip && \
    pip3 install --upgrade wheel setuptools

RUN apt update -q=2 \
    && apt install -y gpg wget \
    && wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - |  tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null \
    && . /etc/os-release \
    && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $UBUNTU_CODENAME main" | tee /etc/apt/sources.list.d/kitware.list >/dev/null \
    && apt-get update -q=2 \
    && apt-get install -y --no-install-recommends cmake=3.27.7* cmake-data=3.27.7*

# Add inception_graphdef model to example repo
WORKDIR /workspace/docs/examples/model_repository
RUN mkdir -p inception_graphdef/1 && \
    wget -O ${TRITONTMP_DIR}/inception_v3_2016_08_28_frozen.pb.tar.gz \
        https://storage.googleapis.com/download.tensorflow.org/models/inception_v3_2016_08_28_frozen.pb.tar.gz && \
    (cd ${TRITONTMP_DIR} && tar xzf inception_v3_2016_08_28_frozen.pb.tar.gz) && \
    mv ${TRITONTMP_DIR}/inception_v3_2016_08_28_frozen.pb inception_graphdef/1/model.graphdef

# Update the qa/ directory with test executables, models, etc.
WORKDIR /workspace
RUN mkdir -p qa/common && \
    cp -r /workspace/src/test/models/repeat_int32 qa/L0_decoupled/models/ && \
    cp -r /workspace/src/test/models/square_int32 qa/L0_decoupled/models/ && \
    mkdir qa/L0_simple_example/models && \
    cp -r docs/examples/model_repository/simple qa/L0_simple_example/models/. && \
    mkdir qa/L0_simple_go_client/models && \
    cp -r docs/examples/model_repository/simple qa/L0_simple_go_client/models/. && \
    mkdir qa/L0_backend_release/simple_models && \
    cp -r docs/examples/model_repository/simple qa/L0_backend_release/simple_models/. && \
    mkdir qa/L0_simple_nodejs_client/models && \
    cp -r docs/examples/model_repository/simple qa/L0_simple_nodejs_client/models/. && \
    mkdir qa/L0_backend_release/simple_seq_models && \
    cp -r /workspace/docs/examples/model_repository/simple_sequence qa/L0_backend_release/simple_seq_models/. && \
    mkdir qa/L0_shared_memory/models && \
    cp -r docs/examples/model_repository/simple qa/L0_shared_memory/models/. && \
    mkdir qa/L0_cuda_shared_memory/models && \
    cp -r docs/examples/model_repository/simple qa/L0_cuda_shared_memory/models/. && \
    mkdir qa/L0_client_java/models && \
    cp -r docs/examples/model_repository/simple qa/L0_client_java/models && \
    mkdir qa/L0_grpc/models && \
    cp -r docs/examples/model_repository/simple qa/L0_grpc/models && \
    cp -r docs/examples/model_repository/simple_dyna_sequence qa/L0_grpc/models && \
    cp -r docs/examples/model_repository/simple_int8 qa/L0_grpc/models && \
    cp -r docs/examples/model_repository/simple_identity qa/L0_grpc/models && \
    cp -r docs/examples/model_repository/simple_sequence qa/L0_grpc/models && \
    cp -r docs/examples/model_repository/simple_string qa/L0_grpc/models && \
    cp -r docs/examples/model_repository/inception_graphdef qa/L0_grpc/models && \
    mkdir qa/L0_grpc_state_cleanup/models && \
    cp -r /workspace/src/test/models/repeat_int32 qa/L0_grpc_state_cleanup/models/ && \
    mkdir qa/L0_http/models && \
    cp -r docs/examples/model_repository/simple qa/L0_http/models && \
    cp -r docs/examples/model_repository/simple_dyna_sequence qa/L0_http/models && \
    cp -r docs/examples/model_repository/simple_identity qa/L0_http/models && \
    cp -r docs/examples/model_repository/simple_sequence qa/L0_http/models && \
    cp -r docs/examples/model_repository/simple_string qa/L0_http/models && \
    cp -r docs/examples/model_repository/inception_graphdef qa/L0_http/models && \
    mkdir qa/L0_https/models && \
    cp -r docs/examples/model_repository/simple qa/L0_https/models/. && \
    mkdir qa/L0_secure_grpc/models && \
    cp -r docs/examples/model_repository/simple qa/L0_secure_grpc/models/. && \
    cp bin/simple qa/L0_simple_lib/. && \
    cp bin/memory_alloc qa/L0_io/. && \
    cp bin/multi_server qa/L0_multi_server/. && \
    cp bin/memory_test qa/L0_memory/. && \
    cp bin/pinned_memory_manager_test qa/L0_memory/. && \
    cp bin/repo_agent_test qa/L0_triton_repo_agent/. && \
    cp lib/libtritonrepoagent_relocation.so qa/L0_triton_repo_agent/. && \
    mkdir qa/L0_query/models/query/1 && \
    cp tritonbuild/tritonserver/backends/query/libtriton_query.so qa/L0_query/models/query/1/. && \
    cp bin/query_test qa/L0_query/. && \
    mkdir qa/L0_iterative_sequence/models/iterative_sequence/1 && \
    cp tritonbuild/tritonserver/backends/iterative_sequence/libtriton_iterative_sequence.so qa/L0_iterative_sequence/models/iterative_sequence/1/. && \
    cp bin/register_api_test qa/L0_register/. && \
    cp bin/async_work_queue_test qa/L0_async_work_queue/. && \
    cp tritonbuild/tritonserver/backends/implicit_state/libtriton_implicit_state.so \
       qa/L0_implicit_state/. && \
    mkdir qa/L0_data_compression/models && \
    cp -r docs/examples/model_repository/simple qa/L0_data_compression/models && \
    cp bin/data_compressor_test qa/L0_data_compression/. && \
    cp bin/metrics_api_test qa/L0_metrics/. && \
    cp bin/response_cache_test qa/L0_response_cache/. && \
    cp bin/request_cancellation_test qa/L0_request_cancellation/. && \
    cp bin/triton_json_test qa/L0_json/. && \
    cp bin/backend_output_detail_test qa/L0_backend_output_detail/. && \
    cp -r deploy/mlflow-triton-plugin qa/L0_mlflow/.

RUN mkdir -p qa/pkgs && \
    cp python/triton*.whl qa/pkgs/. && \
    cp -rf python/test/. qa/L0_python_api/.

# caffe2plan will not exist if the build was done without TensorRT enabled
RUN if [ -f bin/caffe2plan ]; then \
       cp bin/caffe2plan qa/common/.; \
    fi

RUN mkdir -p qa/L0_simple_ensemble/models/simple/1 && \
    cp docs/examples/model_repository/simple/1/model.graphdef \
        qa/L0_simple_ensemble/models/simple/1/. && \
    mkdir -p qa/L0_simple_ensemble/models/simple/2 && \
    cp docs/examples/model_repository/simple/1/model.graphdef \
        qa/L0_simple_ensemble/models/simple/2/. && \
    mkdir -p qa/L0_socket/models/simple/1 && \
    cp docs/examples/model_repository/simple/1/model.graphdef \
        qa/L0_socket/models/simple/1/.

RUN mkdir -p qa/L0_backend_identity/models && \
    cp -r src/test/models/identity_fp32 qa/L0_backend_identity/models/. && \
    mkdir -p qa/L0_backend_identity/models/identity_fp32/1

RUN mkdir -p qa/custom_models/custom_sequence_int32/1 && \
    cp tritonbuild/tritonserver/backends/sequence/libtriton_sequence.so \
        qa/custom_models/custom_sequence_int32/1/. && \
    mkdir -p qa/custom_models/custom_dyna_sequence_int32/1 && \
    cp tritonbuild/tritonserver/backends/dyna_sequence/libtriton_dyna_sequence.so \
        qa/custom_models/custom_dyna_sequence_int32/1/.

# L0_lifecycle needs No-GPU build of identity backend.
RUN cd tritonbuild/identity && \
    rm -rf install build && mkdir build && cd build && \
    cmake -DTRITON_ENABLE_GPU=OFF \
        -DCMAKE_INSTALL_PREFIX:PATH=/workspace/tritonbuild/identity/install \
        -DTRITON_REPO_ORGANIZATION:STRING=${TRITON_REPO_ORGANIZATION} \
        -DTRITON_COMMON_REPO_TAG:STRING=${TRITON_COMMON_REPO_TAG} \
        -DTRITON_CORE_REPO_TAG:STRING=${TRITON_CORE_REPO_TAG} \
        -DTRITON_THIRD_PARTY_REPO_TAG:STRING=${TRITON_THIRD_PARTY_REPO_TAG} \
        -DTRITON_BACKEND_REPO_TAG:STRING=${TRITON_BACKEND_REPO_TAG} .. && \
    make -j16 install

# L0_backend_python test require triton_shm_monitor
RUN cd tritonbuild/python && \
    rm -rf install build && mkdir build && cd build && \
    cmake -DCMAKE_INSTALL_PREFIX:PATH=/workspace/tritonbuild/python/install \
        -DTRITON_REPO_ORGANIZATION:STRING=${TRITON_REPO_ORGANIZATION} \
        -DTRITON_COMMON_REPO_TAG:STRING=${TRITON_COMMON_REPO_TAG} \
        -DTRITON_CORE_REPO_TAG:STRING=${TRITON_CORE_REPO_TAG} \
        -DTRITON_BACKEND_REPO_TAG:STRING=${TRITON_BACKEND_REPO_TAG} .. && \
    make -j16 triton-shm-monitor install

RUN cp tritonbuild/identity/install/backends/identity/libtriton_identity.so \
        qa/L0_lifecycle/. && \
    cp tritonbuild/python/install/backends/python/triton_shm_monitor*.so \
        qa/common/. && \
    mkdir -p qa/L0_perf_nomodel/custom_models/custom_zero_1_float32/1 && \
    mkdir -p qa/L0_perf_pyclients/custom_models/custom_zero_1_int32/1 && \
    mkdir -p qa/L0_infer_shm && \
    cp -r qa/L0_infer/. qa/L0_infer_shm && \
    mkdir -p qa/L0_infer_cudashm && \
    cp -r qa/L0_infer/. qa/L0_infer_cudashm && \
    mkdir -p qa/L0_infer_valgrind && \
    cp -r qa/L0_infer/. qa/L0_infer_valgrind && \
    mkdir -p qa/L0_trt_shape_tensors_shm && \
    cp -r qa/L0_trt_shape_tensors/. qa/L0_trt_shape_tensors_shm && \
    mkdir -p qa/L0_trt_shape_tensors_cudashm && \
    cp -r qa/L0_trt_shape_tensors/. qa/L0_trt_shape_tensors_cudashm && \
    mkdir -p qa/L0_batcher_shm && \
    cp -r qa/L0_batcher/. qa/L0_batcher_shm && \
    mkdir -p qa/L0_batcher_cudashm && \
    cp -r qa/L0_batcher/. qa/L0_batcher_cudashm && \
    mkdir -p qa/L0_batcher_valgrind && \
    cp -r qa/L0_batcher/. qa/L0_batcher_valgrind && \
    mkdir -p qa/L0_sequence_batcher_shm && \
    cp -r qa/L0_sequence_batcher/. qa/L0_sequence_batcher_shm && \
    mkdir -p qa/L0_sequence_batcher_cudashm && \
    cp -r qa/L0_sequence_batcher/. qa/L0_sequence_batcher_cudashm && \
    mkdir -p qa/L0_sequence_batcher_valgrind && \
    cp -r qa/L0_sequence_batcher/. qa/L0_sequence_batcher_valgrind && \
    mkdir -p qa/L0_perf_nomodel_shm && \
    cp -r qa/L0_perf_nomodel/. qa/L0_perf_nomodel_shm && \
    mkdir -p qa/L0_perf_nomodel_cudashm && \
    cp -r qa/L0_perf_nomodel/. qa/L0_perf_nomodel_cudashm

# L0_model_control_stress will not be present if gitlab tests are not available
RUN if [ -d qa/L0_model_control_stress ]; then \
        mkdir -p qa/L0_model_control_stress_valgrind && \
            cp -r qa/L0_model_control_stress/. qa/L0_model_control_stress_valgrind && \
            mkdir -p qa/L0_model_control_stress_valgrind_massif && \
            cp -r qa/L0_model_control_stress/. qa/L0_model_control_stress_valgrind_massif; \
    fi

RUN mkdir -p qa/L0_decoupled/models/repeat_int32/1 && \
    mkdir -p qa/L0_decoupled/models/square_int32/1 && \
    mkdir -p qa/L0_decoupled/models/identity_int32/1 && \
    mkdir -p qa/L0_decoupled/models/simple_repeat/1 && \
    mkdir -p qa/L0_decoupled/models/fan_repeat/1 && \
    mkdir -p qa/L0_decoupled/models/sequence_repeat/1 && \
    mkdir -p qa/L0_decoupled/models/repeat_square/1 && \
    mkdir -p qa/L0_decoupled/models/nested_square/1 && \
    mkdir -p qa/L0_grpc_state_cleanup/models/repeat_int32/1

RUN if [ "$IGPU_BUILD" == "0" ]; then \
        cp backends/repeat/libtriton_repeat.so qa/L0_model_config && \
        cp backends/repeat/libtriton_repeat.so qa/L0_decoupled/models/repeat_int32/1 && \
        cp backends/repeat/libtriton_repeat.so qa/L0_grpc_state_cleanup/models/repeat_int32/1/. && \
        cp backends/square/libtriton_square.so qa/L0_decoupled/models/square_int32/1; \
    fi

RUN cp -r qa/L0_decoupled/models qa/L0_decoupled/python_models/ && \
    cp /workspace/tritonbuild/python/examples/decoupled/repeat_model.py \
        qa/L0_decoupled/python_models/repeat_int32/1/. && \
    cp /workspace/tritonbuild/python/examples/decoupled/repeat_config.pbtxt \
        qa/L0_decoupled/python_models/repeat_int32/. && \
    cp /workspace/tritonbuild/python/examples/decoupled/square_model.py \
        qa/L0_decoupled/python_models/square_int32/1/. && \
    cp /workspace/tritonbuild/python/examples/decoupled/square_config.pbtxt \
        qa/L0_decoupled/python_models/square_int32/.

RUN mkdir -p qa/L0_repoagent_checksum/models/identity_int32/1 && \
    cp tritonbuild/identity/install/backends/identity/libtriton_identity.so \
        qa/L0_repoagent_checksum/models/identity_int32/1/.
RUN mkdir -p qa/L0_passive_instance/models/distributed_int32_int32_int32/1 && \
    cp tritonbuild/tritonserver/backends/distributed_addsub/libtriton_distributed_addsub.so \
        qa/L0_passive_instance/models/distributed_int32_int32_int32/1/.

############################################################################
## Copy artifacts from sdk container
############################################################################
FROM ${SDK_IMAGE} AS sdk

ARG TARGETPLATFORM
WORKDIR /workspace
COPY --from=cibase /workspace/qa/ qa/
RUN mkdir -p qa/clients && mkdir -p qa/pkgs && \
    cp -a install/bin/* qa/clients/. && \
    cp install/lib/libgrpcclient.so qa/clients/. && \
    cp install/lib/libhttpclient.so qa/clients/. && \
    cp install/python/*.py qa/clients/. && \
    cp install/python/triton*.whl qa/pkgs/. && \
    cp install/java/examples/*.jar qa/clients/.
RUN cp client/src/grpc_generated/go/*.go qa/L0_simple_go_client/. && \
    cp client/src/grpc_generated/javascript/*.js qa/L0_simple_nodejs_client/. && \
    cp client/src/grpc_generated/javascript/*.json qa/L0_simple_nodejs_client/. && \
    cp -r client/src/grpc_generated/java qa/L0_client_java/.

############################################################################
## Create CI enabled image
############################################################################
FROM $BASE_IMAGE

ARG TARGETPLATFORM

# Ensure apt-get won't prompt for selecting options
ENV DEBIAN_FRONTEND=noninteractive

# install platform specific packages
RUN if [ $(cat /etc/os-release | grep 'VERSION_ID="20.04"' | wc -l) -ne 0 ]; then \
        apt-get update && \
        apt-get install -y --no-install-recommends \
                libpng-dev; \
    elif [ $(cat /etc/os-release | grep 'VERSION_ID="22.04"' | wc -l) -ne 0 ]; then \
        apt-get update && \
        apt-get install -y --no-install-recommends \
                libpng-dev; \
    elif [ $(cat /etc/os-release | grep 'VERSION_ID="18.04"' | wc -l) -ne 0 ]; then \
        apt-get update && \
        apt-get install -y --no-install-recommends \
                libpng-dev; \
    else \
        echo "Ubuntu version must be either 18.04, 20.04 or 22.04" && \
        exit 1; \
    fi

# CI/QA for memcheck requires valgrind
# libarchive-dev is required by Python backend
RUN apt-get update && apt-get install -y --no-install-recommends \
                              curl \
                              gdb \
                              libopencv-dev \
                              libarchive-dev \
                              libopencv-core-dev \
                              libzmq3-dev \
                              maven \
                              openjdk-11-jdk \
                              nginx \
                              npm \
                              protobuf-compiler \
                              python3-dev \
                              python3-pip \
                              python3-protobuf \
                              python3-setuptools \
                              swig \
                              valgrind && \
    rm -rf /var/lib/apt/lists/*

# CI/QA expects "python" executable (not python3).
RUN rm -f /usr/bin/python && \
    ln -s /usr/bin/python3 /usr/bin/python

RUN pip3 install --upgrade wheel setuptools && \
    pip3 install --upgrade numpy pillow attrdict future grpcio requests gsutil \
                           awscli six grpcio-channelz prettytable virtualenv \
                           check-jsonschema

# go needed for example go client test.
RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
      wget https://golang.org/dl/go1.22.3.linux-arm64.tar.gz && \
      rm -rf /usr/local/go && tar -C /usr/local -xzf go1.22.3.linux-arm64.tar.gz && \
      rm -f go1.22.3.linux-arm64.tar.gz; \
    else \
      wget https://golang.org/dl/go1.22.3.linux-amd64.tar.gz && \
      rm -rf /usr/local/go && tar -C /usr/local -xzf go1.22.3.linux-amd64.tar.gz && \
      rm -f go1.22.3.linux-amd64.tar.gz; \
    fi
ENV GOPATH /root/go
ENV PATH $PATH:/usr/local/go/bin:$GOPATH/bin
RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@latest && \
    go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest
# CI expects tests in /opt/tritonserver/qa. The triton-server (1000)
# user should own all artifacts in case CI is run using triton-server
# user.
WORKDIR /opt/tritonserver
COPY --chown=1000:1000 --from=sdk /workspace/qa/ qa/

# Remove CI tests that are meant to run only on build image and
# install the tritonserver/triton python client APIs.
RUN rm -fr qa/L0_copyrights qa/L0_build_variants && \
    find qa/pkgs/ -maxdepth 1 -type f -name \
    "tritonclient-*linux*.whl" | xargs printf -- '%s[all]' | \
    xargs pip3 install --upgrade

# Install Triton Python API
RUN find qa/pkgs/ -maxdepth 1 -type f -name \
    "tritonserver-*.whl" | xargs -I {} pip3 install --upgrade {}[all]

ENV LD_LIBRARY_PATH /opt/tritonserver/qa/clients:${LD_LIBRARY_PATH}

# DLIS-3631: Needed to run Perf Analyzer CI tests correctly
ENV LD_LIBRARY_PATH /opt/hpcx/ompi/lib:${LD_LIBRARY_PATH}

# Required for PyTorch to pickup the correct HPCX libraries
ENV LD_LIBRARY_PATH /opt/hpcx/ucc/lib/:/opt/hpcx/ucx/lib/:${LD_LIBRARY_PATH}