diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index f6bb354bc5a9..e73ddae35823 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -3,10 +3,10 @@ name: CUDA Version on: push: branches: - - master + - master pull_request: branches: - - master + - master # Run manually by clicking a button in the UI workflow_dispatch: inputs: @@ -43,9 +43,12 @@ jobs: software-properties-common # set up nvidia-docker curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" -y + sudo add-apt-repository -y \ + "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" curl -sL https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add - - curl -sL https://nvidia.github.io/nvidia-docker/$(. /etc/os-release;echo $ID$VERSION_ID)/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list + curl -sL \ + https://nvidia.github.io/nvidia-docker/$(. /etc/os-release;echo $ID$VERSION_ID)/nvidia-docker.list \ + | sudo tee /etc/apt/sources.list.d/nvidia-docker.list sudo apt-get update sudo apt-get install --no-install-recommends -y \ containerd.io \ @@ -58,6 +61,7 @@ jobs: run: | exit 0 test: + # yamllint disable-line rule:line-length name: ${{ matrix.task }} ${{ matrix.cuda_version }} ${{ matrix.method }} (${{ matrix.linux_version }}, ${{ matrix.compiler }}, Python ${{ matrix.python_version }}) runs-on: [self-hosted, linux] needs: [restart-docker] @@ -130,7 +134,7 @@ jobs: runs-on: ubuntu-latest needs: [test] steps: - - name: Note that all tests succeeded - uses: re-actors/alls-green@v1.2.2 - with: - jobs: ${{ toJSON(needs) }} + - name: Note that all tests succeeded + uses: re-actors/alls-green@v1.2.2 + with: + jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml index 195fd5f1c8f1..2552f022c32e 100644 --- a/.github/workflows/lock.yml +++ b/.github/workflows/lock.yml @@ -32,12 +32,18 @@ jobs: remove-pr-labels: 'awaiting response,awaiting review,blocking,in progress' # what message should be posted prior to locking? issue-comment: > - This issue has been automatically locked since there has not been any recent activity since it was closed. - To start a new related discussion, open a new issue at https://github.com/microsoft/LightGBM/issues + This issue has been automatically locked + since there has not been any recent activity since it was closed. + + To start a new related discussion, + open a new issue at https://github.com/microsoft/LightGBM/issues including a reference to this. pr-comment: > - This pull request has been automatically locked since there has not been any recent activity since it was closed. - To start a new related discussion, open a new issue at https://github.com/microsoft/LightGBM/issues + This pull request has been automatically locked + since there has not been any recent activity since it was closed. + + To start a new related discussion, + open a new issue at https://github.com/microsoft/LightGBM/issues including a reference to this. # what should the locking status be? issue-lock-reason: 'resolved' diff --git a/.github/workflows/no-response.yml b/.github/workflows/no-response.yml index 051d44fba20c..65c226978ee9 100644 --- a/.github/workflows/no-response.yml +++ b/.github/workflows/no-response.yml @@ -18,9 +18,14 @@ jobs: - uses: lee-dohm/no-response@v0.5.0 with: closeComment: > - This issue has been automatically closed because it has been awaiting a response for too long. - When you have time to to work with the maintainers to resolve this issue, please post a new comment and it will be re-opened. - If the issue has been locked for editing by the time you return to it, please open a new issue and reference this one. + This issue has been automatically closed + because it has been awaiting a response for too long. + + When you have time to to work with the maintainers to resolve this issue, + please post a new comment and it will be re-opened. + If the issue has been locked for editing by the time you return to it, + please open a new issue and reference this one. + Thank you for taking the time to improve LightGBM! daysUntilClose: 30 responseRequiredLabel: awaiting response diff --git a/.github/workflows/optional_checks.yml b/.github/workflows/optional_checks.yml index f794c38e770c..702ec99ad51a 100644 --- a/.github/workflows/optional_checks.yml +++ b/.github/workflows/optional_checks.yml @@ -22,8 +22,9 @@ jobs: ) for i in "${workflows[@]}"; do workflow_name=${i%;*} + comment="The last reported status from workflow \"$workflow_name\" is failure." + comment+=" Commit fixes and rerun the workflow." trigger_phrase=${i#*;} python "$GITHUB_WORKSPACE/.ci/get-workflow-status.py" "$trigger_phrase" \ - || { echo "The last reported status from workflow \"$workflow_name\" is failure. Commit fixes and rerun the workflow."; \ - exit 1; } + || { echo ${comment}; exit 1; } done diff --git a/.github/workflows/python_package.yml b/.github/workflows/python_package.yml index acc40026f9b8..154baf67b7ee 100644 --- a/.github/workflows/python_package.yml +++ b/.github/workflows/python_package.yml @@ -3,10 +3,10 @@ name: Python-package on: push: branches: - - master + - master pull_request: branches: - - master + - master # automatically cancel in-progress builds if another commit is pushed concurrency: @@ -147,7 +147,7 @@ jobs: runs-on: ubuntu-latest needs: [test, test-latest-versions, test-oldest-versions] steps: - - name: Note that all tests succeeded - uses: re-actors/alls-green@v1.2.2 - with: - jobs: ${{ toJSON(needs) }} + - name: Note that all tests succeeded + uses: re-actors/alls-green@v1.2.2 + with: + jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/r_package.yml b/.github/workflows/r_package.yml index 66e05a18ba1f..1c70ca567319 100644 --- a/.github/workflows/r_package.yml +++ b/.github/workflows/r_package.yml @@ -3,10 +3,10 @@ name: R-package on: push: branches: - - master + - master pull_request: branches: - - master + - master # automatically cancel in-progress builds if another commit is pushed concurrency: @@ -33,6 +33,7 @@ env: jobs: test: + # yamllint disable-line rule:line-length name: ${{ matrix.task }} (${{ matrix.os }}, ${{ matrix.compiler }}, R ${{ matrix.r_version }}, ${{ matrix.build_type }}) runs-on: ${{ matrix.os }} container: ${{ matrix.container }} @@ -230,7 +231,10 @@ jobs: - name: Install packages shell: bash run: | - RDscript${{ matrix.r_customization }} -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'RhpcBLASctl', 'testthat'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())" + R_LIBS="c('R6', 'data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'RhpcBLASctl', 'testthat')" + CRAN_MIRROR="https://cran.rstudio.com" + RDscript${{ matrix.r_customization }} \ + -e "install.packages(${R_LIBS}, repos = '${CRAN_MIRROR}', Ncpus = parallel::detectCores())" sh build-cran-package.sh --r-executable=RD${{ matrix.r_customization }} RD${{ matrix.r_customization }} CMD INSTALL lightgbm_*.tar.gz || exit 1 - name: Run tests with sanitizers @@ -295,7 +299,10 @@ jobs: - name: Install packages and run tests shell: bash run: | - Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'RhpcBLASctl'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())" + R_LIBS="c('R6', 'data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'RhpcBLASctl')" + CRAN_MIRROR="https://cran.rstudio.com" + Rscript \ + -e "install.packages(${R_LIBS}, repos = '${CRAN_MIRROR}', Ncpus = parallel::detectCores())" sh build-cran-package.sh # 'rchk' isn't run through 'R CMD check', use the approach documented at @@ -344,7 +351,7 @@ jobs: runs-on: ubuntu-latest needs: [test, test-r-sanitizers, test-r-extra-checks] steps: - - name: Note that all tests succeeded - uses: re-actors/alls-green@v1.2.2 - with: - jobs: ${{ toJSON(needs) }} + - name: Note that all tests succeeded + uses: re-actors/alls-green@v1.2.2 + with: + jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/r_valgrind.yml b/.github/workflows/r_valgrind.yml index 8593831eed56..961fe20942cb 100644 --- a/.github/workflows/r_valgrind.yml +++ b/.github/workflows/r_valgrind.yml @@ -33,17 +33,25 @@ jobs: - name: Send init status if: ${{ always() }} run: | - $GITHUB_WORKSPACE/.ci/set-commit-status.sh "${{ github.workflow }}" "pending" "${{ github.event.client_payload.pr_sha }}" + $GITHUB_WORKSPACE/.ci/set-commit-status.sh \ + "${{ github.workflow }}" \ + "pending" \ + "${{ github.event.client_payload.pr_sha }}" + comment="Workflow **${{ github.workflow }}** has been triggered! 🚀\r\n" + comment+="${GITHUB_SERVER_URL}/microsoft/LightGBM/actions/runs/${GITHUB_RUN_ID}" $GITHUB_WORKSPACE/.ci/append-comment.sh \ "${{ github.event.client_payload.comment_number }}" \ - "Workflow **${{ github.workflow }}** has been triggered! 🚀\r\n${GITHUB_SERVER_URL}/microsoft/LightGBM/actions/runs/${GITHUB_RUN_ID}" + "${comment}" - name: Run tests with valgrind shell: bash run: ./.ci/test-r-package-valgrind.sh - name: Send final status if: ${{ always() }} run: | - $GITHUB_WORKSPACE/.ci/set-commit-status.sh "${{ github.workflow }}" "${{ job.status }}" "${{ github.event.client_payload.pr_sha }}" + $GITHUB_WORKSPACE/.ci/set-commit-status.sh \ + "${{ github.workflow }}" \ + "${{ job.status }}" \ + "${{ github.event.client_payload.pr_sha }}" $GITHUB_WORKSPACE/.ci/append-comment.sh \ "${{ github.event.client_payload.comment_number }}" \ "Status: ${{ job.status }}." diff --git a/.github/workflows/static_analysis.yml b/.github/workflows/static_analysis.yml index 34e573e0eea6..2a61923bd5aa 100644 --- a/.github/workflows/static_analysis.yml +++ b/.github/workflows/static_analysis.yml @@ -5,10 +5,10 @@ name: Static Analysis on: push: branches: - - master + - master pull_request: branches: - - master + - master # automatically cancel in-progress builds if another commit is pushed concurrency: @@ -64,7 +64,10 @@ jobs: - name: Install packages shell: bash run: | - Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'RhpcBLASctl', 'roxygen2', 'testthat'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())" + R_LIBS="c('R6', 'data.table', 'jsonlite', 'knitr', 'markdown'" + R_LIBS+=", 'Matrix', 'RhpcBLASctl', 'roxygen2', 'testthat')" + CRAN_MIRROR="https://cran.rstudio.com" + Rscript -e "install.packages(${R_LIBS}, repos = '${CRAN_MIRROR}', Ncpus = parallel::detectCores())" sh build-cran-package.sh || exit 1 R CMD INSTALL --with-keep.source lightgbm_*.tar.gz || exit 1 - name: Test documentation @@ -88,7 +91,7 @@ jobs: runs-on: ubuntu-latest needs: [test, r-check-docs] steps: - - name: Note that all tests succeeded - uses: re-actors/alls-green@v1.2.2 - with: - jobs: ${{ toJSON(needs) }} + - name: Note that all tests succeeded + uses: re-actors/alls-green@v1.2.2 + with: + jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/triggering_comments.yml b/.github/workflows/triggering_comments.yml index be33a0e81bf9..452d487c42e5 100644 --- a/.github/workflows/triggering_comments.yml +++ b/.github/workflows/triggering_comments.yml @@ -6,29 +6,32 @@ on: jobs: triggering-tests: - if: github.event.issue.pull_request && contains('OWNER,MEMBER,COLLABORATOR', github.event.comment.author_association) && startsWith(github.event.comment.body, '/gha run') + if: | + github.event.issue.pull_request && + contains('OWNER,MEMBER,COLLABORATOR', github.event.comment.author_association) && + startsWith(github.event.comment.body, '/gha run') runs-on: ubuntu-latest env: SECRETS_WORKFLOW: ${{ secrets.WORKFLOW }} steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 5 - submodules: false + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 5 + submodules: false - - name: Trigger R valgrind tests - if: github.event.comment.body == '/gha run r-valgrind' - run: | - $GITHUB_WORKSPACE/.ci/trigger-dispatch-run.sh \ - "${{ github.event.issue.pull_request.url }}" \ - "${{ github.event.comment.id }}" \ - "gha_run_r_valgrind" + - name: Trigger R valgrind tests + if: github.event.comment.body == '/gha run r-valgrind' + run: | + $GITHUB_WORKSPACE/.ci/trigger-dispatch-run.sh \ + "${{ github.event.issue.pull_request.url }}" \ + "${{ github.event.comment.id }}" \ + "gha_run_r_valgrind" - - name: Trigger update R configure - if: github.event.comment.body == '/gha run r-configure' - run: | - $GITHUB_WORKSPACE/.ci/trigger-dispatch-run.sh \ - "${{ github.event.issue.pull_request.url }}" \ - "${{ github.event.comment.id }}" \ - "gha_run_r_configure" + - name: Trigger update R configure + if: github.event.comment.body == '/gha run r-configure' + run: | + $GITHUB_WORKSPACE/.ci/trigger-dispatch-run.sh \ + "${{ github.event.issue.pull_request.url }}" \ + "${{ github.event.comment.id }}" \ + "gha_run_r_configure" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0edab8df1be6..56dfec23b088 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,6 +17,11 @@ repos: hooks: - id: end-of-file-fixer - id: trailing-whitespace + - repo: https://github.com/adrienverge/yamllint + rev: v1.35.1 + hooks: + - id: yamllint + args: ["--strict"] - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. rev: v0.8.3 diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 3a111e10898e..0a23e955d593 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -1,12 +1,12 @@ trigger: branches: include: - - master + - master tags: include: - - v* + - v* pr: -- master + - master variables: AZURE: 'true' CMAKE_BUILD_PARALLEL_LEVEL: 4 @@ -26,456 +26,481 @@ resources: # to minimize the risk of side effects from one run affecting future runs. # ref: https://learn.microsoft.com/en-us/azure/devops/pipelines/yaml-schema/resources-containers-container containers: - - container: linux-artifact-builder - image: lightgbm/vsts-agent:manylinux_2_28_x86_64 - mountReadOnly: - work: false - externals: true - tools: true - tasks: true - - container: ubuntu-latest - image: 'ubuntu:22.04' - options: "--name ci-container -v /usr/bin/docker:/tmp/docker:ro" - mountReadOnly: - work: false - externals: true - tools: true - tasks: true - - container: rbase - image: wch1/r-debug - mountReadOnly: - work: false - externals: true - tools: true - tasks: true + - container: linux-artifact-builder + image: lightgbm/vsts-agent:manylinux_2_28_x86_64 + mountReadOnly: + work: false + externals: true + tools: true + tasks: true + - container: ubuntu-latest + image: 'ubuntu:22.04' + options: "--name ci-container -v /usr/bin/docker:/tmp/docker:ro" + mountReadOnly: + work: false + externals: true + tools: true + tasks: true + - container: rbase + image: wch1/r-debug + mountReadOnly: + work: false + externals: true + tools: true + tasks: true + jobs: -########################################### -- job: Maintenance -########################################### - pool: mariner-20240410-0 - container: ubuntu-latest - # routine maintenance (like periodically deleting old files), - # to be run on 1 random CI runner in the self-hosted pool each runner - steps: - - script: | - print-diagnostics(){ - echo "---- df -h -m ----" - df -h -m - echo "---- docker system df ----" - /tmp/docker system df - echo "---- docker images ----" - /tmp/docker images - } - # check disk usage - print-diagnostics - # remove old containers, container images, volumes - # ref: https://stackoverflow.com/a/32723127/3986677) - echo "---- running 'docker system prune' ----" - /tmp/docker system prune \ - --all \ - --force \ - --filter until=720h - # check disk usage again - print-diagnostics - displayName: clean -########################################### -- job: Linux -########################################### - variables: - COMPILER: gcc - SETUP_CONDA: 'false' - OS_NAME: 'linux' - PRODUCES_ARTIFACTS: 'true' - pool: mariner-20240410-0 - container: linux-artifact-builder - strategy: - matrix: - regular: - TASK: regular - PYTHON_VERSION: '3.10' - sdist: - TASK: sdist - PYTHON_VERSION: '3.8' - bdist: - TASK: bdist - PYTHON_VERSION: '3.9' - inference: - TASK: if-else - mpi_source: - TASK: mpi - METHOD: source - PYTHON_VERSION: '3.9' - gpu_source: - TASK: gpu - METHOD: source - swig: - TASK: swig - steps: - - script: | - echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" - echo "##vso[task.prependpath]/usr/lib64/openmpi/bin" - echo "##vso[task.prependpath]$CONDA/bin" - displayName: 'Set variables' - - script: | - git clean -d -f -x - displayName: 'Clean source directory' - - script: | - echo '$(Build.SourceVersion)' > '$(Build.ArtifactStagingDirectory)/commit.txt' - displayName: 'Add commit hash to artifacts archive' - - task: Bash@3 - displayName: Setup - inputs: - filePath: $(Build.SourcesDirectory)/.ci/setup.sh - targetType: filePath - - task: Bash@3 - displayName: Test - inputs: - filePath: $(Build.SourcesDirectory)/.ci/test.sh - targetType: filePath - - task: PublishBuildArtifacts@1 - condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) - inputs: - pathtoPublish: '$(Build.ArtifactStagingDirectory)' - artifactName: PackageAssets - artifactType: container -########################################### -- job: Linux_latest -########################################### - variables: - COMPILER: clang-17 - DEBIAN_FRONTEND: 'noninteractive' - IN_UBUNTU_BASE_CONTAINER: 'true' - OS_NAME: 'linux' - SETUP_CONDA: 'true' - pool: mariner-20240410-0 - container: ubuntu-latest - strategy: - matrix: - regular: - TASK: regular - sdist: - TASK: sdist - bdist: - TASK: bdist - PYTHON_VERSION: '3.10' - inference: - TASK: if-else - mpi_source: - TASK: mpi - METHOD: source - mpi_pip: - TASK: mpi - METHOD: pip - PYTHON_VERSION: '3.11' - mpi_wheel: - TASK: mpi - METHOD: wheel - PYTHON_VERSION: '3.9' - gpu_source: - TASK: gpu - METHOD: source - PYTHON_VERSION: '3.11' - gpu_pip: - TASK: gpu - METHOD: pip - PYTHON_VERSION: '3.10' - gpu_wheel: - TASK: gpu - METHOD: wheel - PYTHON_VERSION: '3.9' - cpp_tests: - TASK: cpp-tests - METHOD: with-sanitizers - steps: - - script: | - echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" - CONDA=$HOME/miniforge - echo "##vso[task.setvariable variable=CONDA]$CONDA" - echo "##vso[task.prependpath]$CONDA/bin" - displayName: 'Set variables' - # https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-687983301 - - script: | - /tmp/docker exec -t -u 0 ci-container \ - sh -c "apt-get update && apt-get -o Dpkg::Options::="--force-confold" -y install sudo" - displayName: 'Install sudo' - - script: | - sudo apt-get update - sudo apt-get install -y --no-install-recommends git - git clean -d -f -x - displayName: 'Clean source directory' - - task: Bash@3 - displayName: Setup - inputs: - filePath: $(Build.SourcesDirectory)/.ci/setup.sh - targetType: 'filePath' - - task: Bash@3 - displayName: Test - inputs: - filePath: $(Build.SourcesDirectory)/.ci/test.sh - targetType: 'filePath' -########################################### -- job: QEMU_multiarch -########################################### - variables: - BUILD_DIRECTORY: /LightGBM - COMPILER: gcc - PRODUCES_ARTIFACTS: 'true' - pool: - vmImage: ubuntu-22.04 - timeoutInMinutes: 180 - strategy: - matrix: - bdist: - TASK: bdist - ARCH: aarch64 - steps: - - script: | - sudo apt-get update - sudo apt-get install --no-install-recommends -y \ - binfmt-support \ - qemu \ - qemu-user \ - qemu-user-static - displayName: 'Install QEMU' - - script: | - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - displayName: 'Enable Docker multi-architecture support' - - script: | - git clean -d -f -x - displayName: 'Clean source directory' - - script: | - cat > docker-script.sh < '$(Build.ArtifactStagingDirectory)/commit.txt' + displayName: 'Add commit hash to artifacts archive' + - task: Bash@3 + displayName: Setup + inputs: + filePath: $(Build.SourcesDirectory)/.ci/setup.sh + targetType: filePath + - task: Bash@3 + displayName: Test + inputs: + filePath: $(Build.SourcesDirectory)/.ci/test.sh + targetType: filePath + - task: PublishBuildArtifacts@1 + condition: > + and( + succeeded(), + in(variables['TASK'], 'regular', 'sdist', 'bdist', 'swig'), + not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')) + ) + inputs: + pathtoPublish: '$(Build.ArtifactStagingDirectory)' + artifactName: PackageAssets + artifactType: container + + - job: Linux_latest + ########################################### + variables: + COMPILER: clang-17 + DEBIAN_FRONTEND: 'noninteractive' + IN_UBUNTU_BASE_CONTAINER: 'true' + OS_NAME: 'linux' + SETUP_CONDA: 'true' + pool: mariner-20240410-0 + container: ubuntu-latest + strategy: + matrix: + regular: + TASK: regular + sdist: + TASK: sdist + bdist: + TASK: bdist + PYTHON_VERSION: '3.10' + inference: + TASK: if-else + mpi_source: + TASK: mpi + METHOD: source + mpi_pip: + TASK: mpi + METHOD: pip + PYTHON_VERSION: '3.11' + mpi_wheel: + TASK: mpi + METHOD: wheel + PYTHON_VERSION: '3.9' + gpu_source: + TASK: gpu + METHOD: source + PYTHON_VERSION: '3.11' + gpu_pip: + TASK: gpu + METHOD: pip + PYTHON_VERSION: '3.10' + gpu_wheel: + TASK: gpu + METHOD: wheel + PYTHON_VERSION: '3.9' + cpp_tests: + TASK: cpp-tests + METHOD: with-sanitizers + steps: + - script: | + echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" + CONDA=$HOME/miniforge + echo "##vso[task.setvariable variable=CONDA]$CONDA" + echo "##vso[task.prependpath]$CONDA/bin" + displayName: 'Set variables' + # https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-687983301 + - script: | + /tmp/docker exec -t -u 0 ci-container \ + sh -c "apt-get update && apt-get -o Dpkg::Options::="--force-confold" -y install sudo" + displayName: 'Install sudo' + - script: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends git + git clean -d -f -x + displayName: 'Clean source directory' + - task: Bash@3 + displayName: Setup + inputs: + filePath: $(Build.SourcesDirectory)/.ci/setup.sh + targetType: 'filePath' + - task: Bash@3 + displayName: Test + inputs: + filePath: $(Build.SourcesDirectory)/.ci/test.sh + targetType: 'filePath' + + - job: QEMU_multiarch + ########################################### + variables: + BUILD_DIRECTORY: /LightGBM + COMPILER: gcc + PRODUCES_ARTIFACTS: 'true' + pool: + vmImage: ubuntu-22.04 + timeoutInMinutes: 180 + strategy: + matrix: + bdist: + TASK: bdist + ARCH: aarch64 + steps: + - script: | + sudo apt-get update + sudo apt-get install --no-install-recommends -y \ + binfmt-support \ + qemu \ + qemu-user \ + qemu-user-static + displayName: 'Install QEMU' + - script: | + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + displayName: 'Enable Docker multi-architecture support' + - script: | + git clean -d -f -x + displayName: 'Clean source directory' + - script: | + cat > docker-script.sh < + and( + succeeded(), + in(variables['TASK'], 'bdist'), + not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')) + ) + inputs: + pathtoPublish: '$(Build.ArtifactStagingDirectory)' + artifactName: PackageAssets + artifactType: container + + - job: macOS + ########################################### + variables: + COMPILER: clang + OS_NAME: 'macos' + PRODUCES_ARTIFACTS: 'true' + pool: + vmImage: 'macOS-12' + strategy: + matrix: + regular: + TASK: regular + PYTHON_VERSION: '3.10' + sdist: + TASK: sdist + PYTHON_VERSION: '3.9' + bdist: + TASK: bdist + swig: + TASK: swig + cpp_tests: + TASK: cpp-tests + METHOD: with-sanitizers + SANITIZERS: "address;undefined" + steps: + - script: | + echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" + CONDA=$AGENT_HOMEDIRECTORY/miniforge + echo "##vso[task.setvariable variable=CONDA]$CONDA" + echo "##vso[task.prependpath]$CONDA/bin" + echo "##vso[task.setvariable variable=JAVA_HOME]$JAVA_HOME_8_X64" + displayName: 'Set variables' + - script: | + git clean -d -f -x + displayName: 'Clean source directory' + - task: Bash@3 + displayName: Setup + inputs: + filePath: $(Build.SourcesDirectory)/.ci/setup.sh + targetType: filePath + - task: Bash@3 + displayName: Test + inputs: + filePath: $(Build.SourcesDirectory)/.ci/test.sh + targetType: filePath + - task: PublishBuildArtifacts@1 + condition: > + and( + succeeded(), + in(variables['TASK'], 'regular', 'bdist', 'swig'), + not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')) + ) + inputs: + pathtoPublish: '$(Build.ArtifactStagingDirectory)' + artifactName: PackageAssets + artifactType: container + + - job: Windows + ########################################### + pool: + vmImage: 'windows-2019' + strategy: + matrix: + regular: + TASK: regular + PYTHON_VERSION: '3.10' + sdist: + TASK: sdist + PYTHON_VERSION: '3.9' + bdist: + TASK: bdist + swig: + TASK: swig + cpp_tests: + TASK: cpp-tests + steps: + - powershell: | + Write-Host "##vso[task.prependpath]$env:CONDA\Scripts" + displayName: 'Set Variables' + - script: | + git clean -d -f -x + displayName: 'Clean source directory' + - script: | + cmd /c "powershell -ExecutionPolicy Bypass -File %BUILD_SOURCESDIRECTORY%/.ci/install-opencl.ps1" + condition: eq(variables['TASK'], 'bdist') + displayName: 'Install OpenCL' + - script: | + cmd /c "conda config --remove channels defaults" + cmd /c "conda config --add channels nodefaults" + cmd /c "conda config --add channels conda-forge" + cmd /c "conda config --set channel_priority strict" + cmd /c "conda init powershell" + cmd /c "powershell -ExecutionPolicy Bypass -File %BUILD_SOURCESDIRECTORY%/.ci/test-windows.ps1" + displayName: Test + - task: PublishBuildArtifacts@1 + condition: > + and( + succeeded(), + in(variables['TASK'], 'regular', 'bdist', 'swig'), + not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')) + ) + inputs: + pathtoPublish: '$(Build.ArtifactStagingDirectory)' + artifactName: PackageAssets + artifactType: container + + - job: R_artifact + ########################################### + condition: not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')) + pool: + vmImage: 'ubuntu-22.04' + container: rbase + steps: + - script: | + git clean -d -f -x + displayName: 'Clean source directory' + - script: | + LGB_VER=$(head -n 1 VERSION.txt | sed "s/rc/-/g") + R_LIB_PATH=~/Rlib + R_LIBS="c('R6', 'data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'RhpcBLASctl')" + R_DEPS="c('Depends', 'Imports', 'LinkingTo')" + CRAN_MIRROR="https://cran.rstudio.com" + INSTALLATION_STR="install.packages(${R_LIBS}, lib = '${R_LIB_PATH}', " + INSTALLATION_STR+="dependencies = ${R_DEPS}, repos = '${CRAN_MIRROR}', Ncpus = parallel::detectCores())" + export R_LIBS=${R_LIB_PATH} + mkdir -p ${R_LIB_PATH} + RDscript -e "${INSTALLATION_STR}" || exit 1 + sh build-cran-package.sh --r-executable=RD || exit 1 + mv lightgbm_${LGB_VER}.tar.gz $(Build.ArtifactStagingDirectory)/lightgbm-${LGB_VER}-r-cran.tar.gz + displayName: 'Build CRAN R-package' + - task: PublishBuildArtifacts@1 + condition: succeeded() + inputs: + pathtoPublish: $(Build.ArtifactStagingDirectory) + artifactName: R-package + artifactType: container + -########################################### -- job: Package -########################################### - dependsOn: - - Linux - - Linux_latest - - QEMU_multiarch - - macOS - - Windows - - R_artifact - condition: and(succeeded(), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) - pool: - vmImage: 'ubuntu-22.04' - steps: - # Create archives with complete source code included (with git submodules) - - task: ArchiveFiles@2 - displayName: Create zip archive - condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v')) - inputs: - rootFolderOrFile: $(Build.SourcesDirectory) - includeRootFolder: false - archiveType: zip - archiveFile: '$(Build.ArtifactStagingDirectory)/archives/LightGBM-complete_source_code_zip.zip' - replaceExistingArchive: true - - task: ArchiveFiles@2 - displayName: Create tar.gz archive - condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v')) - inputs: - rootFolderOrFile: $(Build.SourcesDirectory) - includeRootFolder: false - archiveType: tar - tarCompression: gz - archiveFile: '$(Build.ArtifactStagingDirectory)/archives/LightGBM-complete_source_code_tar_gz.tar.gz' - replaceExistingArchive: true - # Download all agent packages from all previous phases - - task: DownloadBuildArtifacts@0 - displayName: Download package assets - inputs: - artifactName: PackageAssets - downloadPath: $(Build.SourcesDirectory)/binaries - - task: DownloadBuildArtifacts@0 - displayName: Download R-package - condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v')) - inputs: - artifactName: R-package - downloadPath: $(Build.SourcesDirectory)/R - - script: | - python "$(Build.SourcesDirectory)/.ci/create-nuget.py" "$(Build.SourcesDirectory)/binaries/PackageAssets" - displayName: 'Create NuGet configuration files' - - task: NuGetCommand@2 - inputs: - command: pack - packagesToPack: '$(Build.SourcesDirectory)/.ci/nuget/*.nuspec' - packDestination: '$(Build.ArtifactStagingDirectory)/nuget' - - task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: '$(Build.ArtifactStagingDirectory)/nuget' - artifactName: NuGet - artifactType: container - - task: GitHubRelease@0 - displayName: 'Create GitHub Release' - condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v')) - inputs: - gitHubConnection: guolinke - repositoryName: '$(Build.Repository.Name)' - action: 'create' - target: '$(Build.SourceVersion)' - tagSource: 'auto' - title: '$(Build.SourceBranchName)' - assets: | - $(Build.SourcesDirectory)/binaries/PackageAssets/* - $(Build.SourcesDirectory)/R/R-package/* - $(Build.ArtifactStagingDirectory)/nuget/*.nupkg - $(Build.ArtifactStagingDirectory)/archives/* - assetUploadMode: 'delete' - isDraft: true - isPreRelease: false - addChangeLog: false + - job: Package + ########################################### + dependsOn: + - Linux + - Linux_latest + - QEMU_multiarch + - macOS + - Windows + - R_artifact + condition: and(succeeded(), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) + pool: + vmImage: 'ubuntu-22.04' + steps: + # Create archives with complete source code included (with git submodules) + - task: ArchiveFiles@2 + displayName: Create zip archive + condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v')) + inputs: + rootFolderOrFile: $(Build.SourcesDirectory) + includeRootFolder: false + archiveType: zip + archiveFile: '$(Build.ArtifactStagingDirectory)/archives/LightGBM-complete_source_code_zip.zip' + replaceExistingArchive: true + - task: ArchiveFiles@2 + displayName: Create tar.gz archive + condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v')) + inputs: + rootFolderOrFile: $(Build.SourcesDirectory) + includeRootFolder: false + archiveType: tar + tarCompression: gz + archiveFile: '$(Build.ArtifactStagingDirectory)/archives/LightGBM-complete_source_code_tar_gz.tar.gz' + replaceExistingArchive: true + # Download all agent packages from all previous phases + - task: DownloadBuildArtifacts@0 + displayName: Download package assets + inputs: + artifactName: PackageAssets + downloadPath: $(Build.SourcesDirectory)/binaries + - task: DownloadBuildArtifacts@0 + displayName: Download R-package + condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v')) + inputs: + artifactName: R-package + downloadPath: $(Build.SourcesDirectory)/R + - script: | + python "$(Build.SourcesDirectory)/.ci/create-nuget.py" "$(Build.SourcesDirectory)/binaries/PackageAssets" + displayName: 'Create NuGet configuration files' + - task: NuGetCommand@2 + inputs: + command: pack + packagesToPack: '$(Build.SourcesDirectory)/.ci/nuget/*.nuspec' + packDestination: '$(Build.ArtifactStagingDirectory)/nuget' + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: '$(Build.ArtifactStagingDirectory)/nuget' + artifactName: NuGet + artifactType: container + - task: GitHubRelease@0 + displayName: 'Create GitHub Release' + condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v')) + inputs: + gitHubConnection: guolinke + repositoryName: '$(Build.Repository.Name)' + action: 'create' + target: '$(Build.SourceVersion)' + tagSource: 'auto' + title: '$(Build.SourceBranchName)' + assets: | + $(Build.SourcesDirectory)/binaries/PackageAssets/* + $(Build.SourcesDirectory)/R/R-package/* + $(Build.ArtifactStagingDirectory)/nuget/*.nupkg + $(Build.ArtifactStagingDirectory)/archives/* + assetUploadMode: 'delete' + isDraft: true + isPreRelease: false + addChangeLog: false diff --git a/.yamllint.yml b/.yamllint.yml new file mode 100644 index 000000000000..7bb0ae95ec74 --- /dev/null +++ b/.yamllint.yml @@ -0,0 +1,10 @@ +# default config: https://yamllint.readthedocs.io/en/stable/configuration.html#default-configuration +extends: default + +rules: + document-start: disable + line-length: + max: 120 + truthy: + # prevent treating GitHub Workflow "on" key as boolean value + check-keys: false diff --git a/R-package/pkgdown/_pkgdown.yml b/R-package/pkgdown/_pkgdown.yml index c2d6718a2926..258a27a2ea31 100644 --- a/R-package/pkgdown/_pkgdown.yml +++ b/R-package/pkgdown/_pkgdown.yml @@ -37,70 +37,70 @@ navbar: title: LightGBM type: default left: - - icon: fa-reply fa-lg - href: ../ - - icon: fa-home fa-lg - href: index.html - - text: Articles - href: articles/index.html - - text: Reference - href: reference/index.html + - icon: fa-reply fa-lg + href: ../ + - icon: fa-home fa-lg + href: index.html + - text: Articles + href: articles/index.html + - text: Reference + href: reference/index.html right: - - icon: fa-github fa-lg - href: https://github.com/microsoft/LightGBM/tree/master/R-package + - icon: fa-github fa-lg + href: https://github.com/microsoft/LightGBM/tree/master/R-package reference: - title: Datasets desc: Datasets included with the R-package contents: - - '`agaricus.train`' - - '`agaricus.test`' - - '`bank`' + - '`agaricus.train`' + - '`agaricus.test`' + - '`bank`' - title: Data Input / Output desc: Data I/O required for LightGBM contents: - - '`dim.lgb.Dataset`' - - '`dimnames.lgb.Dataset`' - - '`get_field`' - - '`set_field`' - - '`lgb.Dataset`' - - '`lgb.Dataset.construct`' - - '`lgb.Dataset.create.valid`' - - '`lgb.Dataset.save`' - - '`lgb.Dataset.set.categorical`' - - '`lgb.Dataset.set.reference`' - - '`lgb.convert_with_rules`' - - '`lgb.slice.Dataset`' + - '`dim.lgb.Dataset`' + - '`dimnames.lgb.Dataset`' + - '`get_field`' + - '`set_field`' + - '`lgb.Dataset`' + - '`lgb.Dataset.construct`' + - '`lgb.Dataset.create.valid`' + - '`lgb.Dataset.save`' + - '`lgb.Dataset.set.categorical`' + - '`lgb.Dataset.set.reference`' + - '`lgb.convert_with_rules`' + - '`lgb.slice.Dataset`' - title: Machine Learning desc: Train models with LightGBM and then use them to make predictions on new data contents: - - '`lightgbm`' - - '`lgb.train`' - - '`predict.lgb.Booster`' - - '`lgb.cv`' - - '`lgb.configure_fast_predict`' + - '`lightgbm`' + - '`lgb.train`' + - '`predict.lgb.Booster`' + - '`lgb.cv`' + - '`lgb.configure_fast_predict`' - title: Saving / Loading Models desc: Save and load LightGBM models contents: - - '`lgb.dump`' - - '`lgb.save`' - - '`lgb.load`' - - '`lgb.model.dt.tree`' - - '`lgb.drop_serialized`' - - '`lgb.make_serializable`' - - '`lgb.restore_handle`' + - '`lgb.dump`' + - '`lgb.save`' + - '`lgb.load`' + - '`lgb.model.dt.tree`' + - '`lgb.drop_serialized`' + - '`lgb.make_serializable`' + - '`lgb.restore_handle`' - title: Model Interpretation desc: Analyze your models contents: - - '`lgb.get.eval.result`' - - '`lgb.importance`' - - '`lgb.interprete`' - - '`lgb.plot.importance`' - - '`lgb.plot.interpretation`' - - '`print.lgb.Booster`' - - '`summary.lgb.Booster`' + - '`lgb.get.eval.result`' + - '`lgb.importance`' + - '`lgb.interprete`' + - '`lgb.plot.importance`' + - '`lgb.plot.interpretation`' + - '`print.lgb.Booster`' + - '`summary.lgb.Booster`' - title: Multithreading Control desc: Manage degree of parallelism used by LightGBM contents: - - '`getLGBMThreads`' - - '`setLGBMThreads`' + - '`getLGBMThreads`' + - '`setLGBMThreads`'