Merge remote-tracking branch 'upstream/main'

.github/ISSUE_TEMPLATE/component_release_template.md

@@ -30,7 +30,7 @@ If including changes in this release, increment the version on `{{RELEASE_VERSIO
 ### Preparation
 
 - [ ] Assign this issue to a release owner.
-- [ ] Finalize scope and feature set and update [the Public Roadmap](https://github.com/orgs/opensearch-project/projects/1).
+- [ ] Finalize scope and feature set and update [the Public Roadmap](https://github.com/orgs/opensearch-project/projects/206).
 - [ ] All the tasks in this issue have been reviewed by the release owner.
 - [ ] Create, update, triage and label all features and issues targeted for this release with `v{{RELEASE_VERSION}}`.
 - [ ] Finalize the code and create the the release branch `{{RELEASE_BRANCH}}` from the `{{RELEASE_VERSION_X}}` branch.

.github/ISSUE_TEMPLATE/release_template.md

@@ -81,11 +81,13 @@ Release blog is ready | :red_circle: |   |
 - [ ] [Core Release Branch](https://github.com/opensearch-project/opensearch-build/wiki/Releasing-the-Distribution#core).
 - [ ] [Components Release Branch](https://github.com/opensearch-project/opensearch-build/wiki/Releasing-the-Distribution#components).
 - [ ] [Generate Release Candidate](https://github.com/opensearch-project/opensearch-build/wiki/Releasing-the-Distribution#release-candidate).
-- [ ] [Integ Test TAR](https://github.com/opensearch-project/opensearch-build/wiki/Releasing-the-Distribution#integ-test-tar).
-- [ ] [Integ Test RPM](https://github.com/opensearch-project/opensearch-build/wiki/Releasing-the-Distribution#integ-test-rpm).
-- [ ] [Docker Build and Scan](https://github.com/opensearch-project/opensearch-build/wiki/Releasing-the-Distribution#docker-build-and-scan).
+- [ ] [Docker Scan](https://github.com/opensearch-project/opensearch-build/wiki/Releasing-the-Distribution#docker-scan).
+- [ ] [Integ Test](https://github.com/opensearch-project/opensearch-build/wiki/Releasing-the-Distribution#integration-test).
+   - [ ] TAR
+   - [ ] RPM
+   - [ ] DEB
+   - [ ] Windows
 - [ ] [Backwards Compatibility Tests](https://github.com/opensearch-project/opensearch-build/wiki/Releasing-the-Distribution#backwards-compatibility-tests).
-- [ ] [Windows Integration Test](https://github.com/opensearch-project/opensearch-build/wiki/Releasing-the-Distribution#windows-integration-test).
 - [ ] [Broadcast and Communication](https://github.com/opensearch-project/opensearch-build/wiki/Releasing-the-Distribution#broadcast-and-communication).
 - [ ] [Release Candidate Lock](https://github.com/opensearch-project/opensearch-build/wiki/Releasing-the-Distribution#release-candidate-lock).
 - [ ] [Stop Release Candidate periodic auto builds](https://github.com/opensearch-project/opensearch-build/wiki/Releasing-the-Distribution#stop-periodic-auto-builds)
@@ -102,7 +104,7 @@ Release blog is ready | :red_circle: |   |
 
 ### [Release](https://github.com/opensearch-project/opensearch-build/wiki/Releasing-the-Distribution#main-release) - _Ends {__REPLACE_RELEASE-day}_
 
-- [ ] [Central Promotion Workflow](https://github.com/opensearch-project/opensearch-build/wiki/Central-Release-Promotion-Workflow).
+- [ ] [Central Promotion Workflow](https://github.com/opensearch-project/opensearch-build/wiki/Releasing-the-Distribution#central-release-promotion-workflow).
 - [ ] [Collaboration with the Project Management Team](https://github.com/opensearch-project/opensearch-build/wiki/Releasing-the-Distribution#collaboration-with-the-project-management-team).
 
 ### [Release Checklist](https://github.com/opensearch-project/opensearch-build/wiki/Releasing-the-Distribution#release-checklist).

.github/workflows/groovy-tests.yml

@@ -15,7 +15,3 @@ jobs:
       - name: Run Tests
         run: |
           ./gradlew test --info
-      - name: Upload Coverage Report
-        uses: codecov/codecov-action@v2
-        with:
-          files: ./build/reports/jacoco/test/jacocoTestReport.xml

.github/workflows/manifests.yml

@@ -3,15 +3,15 @@ name: manifests-ci-check
 
 on:
   pull_request:
-    types: [labeled, synchronize]
+    types: [opened, synchronize]
     paths:
       - 'manifests/**/*.yml'
       - '!manifests/templates/**/'
       - 'legacy-manifests/**/*.yml'
 
 jobs:
   list-changed-manifests:
-    if: ${{ github.repository == 'opensearch-project/opensearch-build' && github.event.label.name == 'manifest-ci-check' }}
+    if: ${{ github.repository == 'opensearch-project/opensearch-build' }}
     runs-on: ubuntu-latest
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}

.github/workflows/os-increment-plugin-versions.yml

@@ -50,14 +50,14 @@ jobs:
           - {repo: skills}
           - {repo: query-insights}
           - {repo: opensearch-system-templates}
+          - {repo: opensearch-remote-metadata-sdk}
+          - {repo: opensearch-learning-to-rank-base}
         branch:
-          - 1.x
-          - '1.3'
           - 2.x
           - main
-          - '2.17'
           - '2.18'
           - '2.19'
+          - '2.20'
         exclude:
           - {entry: {repo: geospatial}, branch: '1.3'}
           - {entry: {repo: neural-search}, branch: '1.3'}

.github/workflows/osd-increment-plugin-versions.yml

@@ -41,14 +41,13 @@ jobs:
           - {repo: security-dashboards-plugin}
           - {repo: dashboards-search-relevance}
           - {repo: opensearch-dashboards-functional-test}
+          - {repo: query-insights-dashboards}
         branch:
-          - 1.x
-          - '1.3'
           - 2.x
           - main
-          - '2.17'
           - '2.18'
           - '2.19'
+          - '2.20'
     steps:
       - name: Check out OpenSearch Dashboards repo
         uses: actions/checkout@v3

.github/workflows/osd-release-issues.yml

@@ -54,6 +54,7 @@ jobs:
           - {repo: security-dashboards-plugin}
           - {repo: dashboards-search-relevance}
           - {repo: opensearch-dashboards-functional-test}
+          - {repo: query-insights-dashboards}
         release_version: ${{ fromJson(needs.list-manifest-versions.outputs.matrix) }}
     steps:
       - name: GitHub App token

.github/workflows/python-tests.yml

@@ -5,15 +5,16 @@ on: [push, pull_request]
 
 jobs:
   Get-CI-Image-Tag:
-    uses: ./.github/workflows/get-ci-image-tag.yml
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
     with:
       product: opensearch
 
   python-tests:
     strategy:
       matrix:
         os:
-          - macos-latest
+          - macos-14  # arm64
+          - macos-13  # x64
           - windows-latest
     runs-on: ${{ matrix.os }}
     env:
@@ -40,14 +41,15 @@ jobs:
       - name: Run Tests with Coverage
         run: |
           pipenv run coverage run -m pytest --cov=./src --cov-report=xml
-      - name: Upload Coverage Report
-        uses: codecov/codecov-action@v2
-        with:
-          files: ./coverage.xml
 
   python-tests-linux:
     needs: Get-CI-Image-Tag
-    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        os:
+          - ubuntu-24.04-arm  # arm64-preview
+          - ubuntu-24.04  # x64
+    runs-on: ${{ matrix.os }}
     container:
       # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
       # this image tag is subject to change as more dependencies and updates will arrive over time
@@ -74,7 +76,21 @@ jobs:
       - name: Run Tests with Coverage
         run: |
           pipenv run coverage run -m pytest --cov=./src --cov-report=xml
+      - uses: actions/upload-artifact@v4
+        with:
+          name: coverage-report-${{ matrix.os }}
+          path: ./coverage.xml
+
+  python-tests-codecov:
+    needs: python-tests-linux
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: coverage-report-ubuntu-24.04
+          path: ./
       - name: Upload Coverage Report
-        uses: codecov/codecov-action@v2
+        uses: codecov/codecov-action@v5
         with:
+          token: ${{ secrets.CODECOV_TOKEN }}
           files: ./coverage.xml

TRIAGING.md

@@ -33,6 +33,7 @@ Meetings are 45 minutes and structured as follows:
     1. [ansible-playbook](https://github.com/opensearch-project/ansible-playbook)
     1. [opensearch-cluster-cdk](https://github.com/opensearch-project/opensearch-cluster-cdk)
     1. [opensearch-devops](https://github.com/opensearch-project/opensearch-devops)
+    1. [automation-app](https://github.com/opensearch-project/automation-app)
 4. Member Requests: Opportunity for any meeting member to ask for consideration of an issue or pull request.
 5. Untriaged Items: Review any [issues](https://github.com/orgs/opensearch-project/projects/208/views/2) that might have had the 'untriaged' label removed but require additional triage discussion.
 6. Open Discussion: Allow for members of the meeting to surface any topics without issues filed or pull request created.

docker/ci/dockerfiles/current/build.al2.opensearch.x64.arm64.dockerfile

@@ -67,7 +67,7 @@ RUN curl -sSL https://rvm.io/mpapis.asc | gpg2 --import - && \
 SHELL ["/bin/bash", "-lc"]
 CMD ["/bin/bash", "-l"]
 
-# Install ruby / rpm / fpm related dependencies
+# Install ruby / rpm / fpm / openssl / gcc / binutils related dependencies
 RUN . /etc/profile.d/rvm.sh && rvm install 2.6.0 && rvm --default use 2.6.0 && yum install -y rpm-build createrepo && yum clean all
 
 ENV RUBY_HOME=/usr/local/rvm/rubies/ruby-2.6.0/bin
@@ -82,14 +82,14 @@ ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/lib64:/usr/lib
 RUN yum install -y curl libcurl-devel libfaketime perl-core pcre-devel && yum remove -y openssl-devel && yum clean all && \
     mkdir -p /tmp/openssl && cd /tmp/openssl && \
     curl -sSL -o- https://www.openssl.org/source/openssl-1.1.1g.tar.gz | tar -xz --strip-components 1 && \
-    ./config --prefix=/usr --openssldir=/etc/ssl --libdir=lib shared zlib-dynamic && make && make install && \
+    ./config --prefix=/usr --openssldir=/etc/ssl --libdir=lib shared zlib-dynamic && make -j$(nproc) && make install && \
     echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/lib64:/usr/lib" > /etc/profile.d/openssl.sh && openssl version
 
 # Install Python binary
 RUN curl https://www.python.org/ftp/python/3.9.7/Python-3.9.7.tgz | tar xzvf - && \
     cd Python-3.9.7 && \
     env LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/lib64:/usr/lib ./configure --enable-optimizations --with-openssl=/usr --prefix=/usr/local && \
-    make altinstall && cd ../ && rm -rf Python-3.9.7.tgz Python-3.9.7 && \
+    make -j$(nproc) altinstall && cd ../ && rm -rf Python-3.9.7.tgz Python-3.9.7 && \
     cp -v /etc/ssl/certs/ca-bundle.crt /etc/ssl/cert.pem
 
 # Setup Python links
@@ -99,28 +99,60 @@ RUN ln -sfn /usr/local/bin/python3.9 /usr/bin/python3 && \
     ln -sfn /usr/local/bin/pip3.9 /usr/bin/pip3 && \
     pip3 install pip==23.1.2 && pip3 install pipenv==2023.6.12 awscli==1.32.17
 
-# Upgrade gcc
-RUN yum install -y gcc10* && \
-    mv -v /usr/bin/gcc /usr/bin/gcc7-gcc && \
-    mv -v /usr/bin/g++ /usr/bin/gcc7-g++ && \
-    mv -v /usr/bin/gfortran /usr/bin/gcc7-gfortran && \
-    update-alternatives --install /usr/bin/gcc gcc $(which gcc10-gcc) 1 && \
-    update-alternatives --install /usr/bin/g++ g++ $(which gcc10-g++) 1 && \
-    update-alternatives --install /usr/bin/gfortran gfortran $(which gcc10-gfortran) 1
+# Upgrade gcc, while keep libstdc++.so to older 6.0.24 version for backward compatibility
+# Only x64 requires gcc 12+ for k-NN avx512_spr fp16 feature
+# https://github.com/opensearch-project/opensearch-build/issues/5226
+# Due to cross-compilation being too slow on arm64, it will stay on gcc 10 for the time being
+RUN if [ `uname -m` = "x86_64" ]; then \
+        curl -SL https://ci.opensearch.org/ci/dbc/tools/gcc/gcc-12.4.0.tar.gz -o gcc12.tgz && \
+        tar -xzf gcc12.tgz && cd gcc-12.4.0 && \
+        sed -i 's@base_url=.*@base_url=https://ci.opensearch.org/ci/dbc/tools/gcc/@g' ./contrib/download_prerequisites && \
+        ./contrib/download_prerequisites && \
+        mkdir build && cd build && \
+        ../configure --enable-languages=all --prefix=/usr --disable-multilib --disable-bootstrap && \
+        make -j$(nproc) && make install && gcc --version && g++ --version && gfortran --version && \
+        cd  ../../ && rm -rf gcc12.tgz gcc-12.4.0 && cd /lib64/ && \
+        ln -sfn libstdc++.so.6 libstdc++.so && \
+        ln -sfn libstdc++.so.6.0.24 libstdc++.so.6 && \
+        rm -v libstdc++.so.6.0.30* ; \
+    else \
+        yum install -y gcc10* && \
+        mv -v /usr/bin/gcc /usr/bin/gcc7-gcc && \
+        mv -v /usr/bin/g++ /usr/bin/gcc7-g++ && \
+        mv -v /usr/bin/gfortran /usr/bin/gcc7-gfortran && \
+        update-alternatives --install /usr/bin/gcc gcc $(which gcc10-gcc) 1 && \
+        update-alternatives --install /usr/bin/g++ g++ $(which gcc10-g++) 1 && \
+        update-alternatives --install /usr/bin/gfortran gfortran $(which gcc10-gfortran) 1; \
+    fi
+
+# Upgrade binutils
+# This is only required if gcc upgrade to 12 or above
+RUN if [ `uname -m` = "x86_64" ]; then \
+        yum install -y texinfo && \
+        curl -SLO https://ci.opensearch.org/ci/dbc/tools/gcc/binutils-2.42.90.tar.xz && \
+        tar -xf binutils-2.42.90.tar.xz && cd binutils-2.42.90 && \
+        mkdir build && cd build && \
+        ../configure --prefix=/usr && \
+        make -j$(nproc) && make install && ld --version && \
+        cd ../../ && rm -rf binutils-2.42.90.tar.xz binutils-2.42.90 && \
+        yum remove -y texinfo; \
+    fi
+
 ENV FC=gfortran
 ENV CXX=g++
 
 # Add k-NN Library dependencies
-RUN yum repolist && yum install lapack -y
-RUN git clone -b v0.3.27 --single-branch https://github.com/xianyi/OpenBLAS.git && \
+RUN yum repolist && yum install lapack -y && yum clean all && rm -rf /var/cache/yum/*
+RUN git clone -b v0.3.27 --single-branch https://github.com/OpenMathLib/OpenBLAS.git && \
     cd OpenBLAS && \
     if [ "$(uname -m)" = "x86_64" ]; then \
         echo "Machine is x86_64. Adding DYNAMIC_ARCH=1 to openblas make command."; \
-        make USE_OPENMP=1 FC=gfortran DYNAMIC_ARCH=1; \
+        make -j$(nproc) USE_OPENMP=1 FC=gfortran DYNAMIC_ARCH=1; \
     else \
-        make USE_OPENMP=1 FC=gfortran; \
+        make -j$(nproc) USE_OPENMP=1 FC=gfortran; \
     fi && \
-    make PREFIX=/usr/local install
+    make PREFIX=/usr/local install && \
+    cd ../ && rm -rf OpenBLAS
 ENV LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH"
 RUN pip3 install cmake==3.26.4
 
@@ -131,19 +163,22 @@ RUN pip3 install cmake==3.26.4
 # GitHub enforce nodejs 20 official build in runner 2.317.0 of their actions and CentOS7/AL2 would fail due to having older glibc versions
 # Until https://github.com/actions/runner/pull/3128 is merged or AL2 is deprecated (2025/06) this is a quick fix with unofficial builds support glibc 2.17
 # With changes done similar to this PR (https://github.com/opensearch-project/job-scheduler/pull/702) alongside the image here
-# Only linux x64 is supported in unofficial build until https://github.com/nodejs/unofficial-builds/pull/91 is merged
+# Only linux x64 glibc217 is supported in unofficial build until https://github.com/nodejs/unofficial-builds/pull/91 is merged for pre-compiled arm64 binaries
+# The linux arm64 glibc226 tarball here is directly compiled from the source code on AL2 host for the time being
 RUN if [ `uname -m` = "x86_64" ]; then \
-        curl -SL https://unofficial-builds.nodejs.org/download/release/v20.10.0/node-v20.10.0-linux-x64-glibc-217.tar.xz -o /node20.tar.xz; \
-        mkdir /node_al2; \
-        tar -xf /node20.tar.xz --strip-components 1 -C /node_al2; \
-        rm -v /node20.tar.xz; \
-    fi
+        curl -SL https://ci.opensearch.org/ci/dbc/tools/node/node-v20.18.0-linux-x64-glibc-217.tar.xz -o /node20.tar.xz; \
+    else \
+        curl -SL https://ci.opensearch.org/ci/dbc/tools/node/node-v20.18.0-linux-arm64-glibc-226-libstdcpp-6.0.24.tar.xz -o /node20.tar.xz; \
+    fi; \
+    mkdir /node_al2 && \
+    tar -xf /node20.tar.xz --strip-components 1 -C /node_al2 && \
+    rm -v /node20.tar.xz
 
 # Change User
 USER $CONTAINER_USER
 WORKDIR $CONTAINER_USER_HOME
 
 # Install fpm for opensearch dashboards core
-RUN gem install dotenv -v 2.8.1 && gem install public_suffix -v 5.1.1 && gem install fpm -v 1.14.2
+RUN gem install dotenv -v 2.8.1 && gem install public_suffix -v 5.1.1 && gem install rchardet -v 1.8.0 && gem install fpm -v 1.14.2
 ENV PATH=$CONTAINER_USER_HOME/.gem/gems/fpm-1.14.2/bin:$PATH
 RUN fpm -v

docker/ci/dockerfiles/current/build.almalinux8.opensearch-dashboards.x64.arm64.ppc64le.dockerfile

@@ -58,7 +58,7 @@ RUN curl -sSL https://rvm.io/mpapis.asc | gpg2 --import - && \
 SHELL ["/bin/bash", "-lc"]
 
 # Install ruby / rpm / fpm related dependencies
-RUN . /etc/profile.d/rvm.sh && rvm install 2.6.0 && rvm --default use 2.6.0 && dnf install -y rpm-build createrepo && dnf clean all
+RUN . /etc/profile.d/rvm.sh && rvm install 2.6.0 && rvm --default use 2.6.0 && gem install rchardet -v 1.8.0 && dnf install -y rpm-build createrepo && dnf clean all
 
 ENV RUBY_HOME=/usr/local/rvm/rubies/ruby-2.6.0/bin
 ENV RVM_HOME=/usr/local/rvm/bin

docker/ci/dockerfiles/current/build.almalinux8.opensearch.x64.arm64.ppc64le.dockerfile

@@ -102,6 +102,6 @@ USER $CONTAINER_USER
 WORKDIR $CONTAINER_USER_HOME
 
 # Install fpm for opensearch dashboards core
-RUN gem install dotenv -v 2.8.1 && gem install public_suffix -v 5.1.1 && gem install fpm -v 1.14.2
+RUN gem install dotenv -v 2.8.1 && gem install public_suffix -v 5.1.1 && gem install rchardet -v 1.8.0&& gem install fpm -v 1.14.2
 ENV PATH=$CONTAINER_USER_HOME/.gem/gems/fpm-1.14.2/bin:$PATH
 RUN fpm -v

docker/release/dockerfiles/opensearch.al2023.dockerfile

@@ -83,7 +83,7 @@ ENV JAVA_HOME=$OPENSEARCH_HOME/jdk
 ENV PATH=$PATH:$JAVA_HOME/bin:$OPENSEARCH_HOME/bin
 
 # Add k-NN lib directory to library loading path variable
-ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$OPENSEARCH_HOME/plugins/opensearch-knn/lib"
+ENV LD_LIBRARY_PATH="$OPENSEARCH_HOME/plugins/opensearch-knn/lib"
 
 # Change user
 USER $UID