[ci] Test auto benchmark

.github/workflows/presubmit.yml

@@ -1,9 +1,12 @@
+
+
 name: Presubmit Checks
 on:
   pull_request:
     types: [opened, synchronize, reopened]
 
 jobs:
+
   title_format:
     name: Check PR Title
     if: ${{ github.event.pull_request }}
@@ -40,7 +43,6 @@ jobs:
           git remote add upstream https://github.com/taichi-dev/taichi.git
           git fetch upstream master
           sudo apt install clang-format-10
-
       - name: Cache PIP
         uses: actions/cache@v2
         with:
@@ -50,240 +52,39 @@ jobs:
       - name: Install requirements
         run: |
           python3 -m pip install --user -r requirements_dev.txt
-
       - name: Check code format
         run: |
           python3 misc/code_format.py
           git checkout -b _enforced_format
           git commit -am "enforce code format" || true
           # exit with 1 if there were differences:
           git diff _fake_squash _enforced_format --exit-code
-
-      - name: Pylint
-        run: |
-          # Make sure pylint doesn't regress
-          pylint python/taichi/ --disable=all --enable=C0121,C0415
-          if [ $? -eq 0 ]
-          then
-            echo "PASSED: pylint is happy"
-            exit 0
-          else
-            echo "FAILED: please run the pylint command above and make sure it passes"
-            exit 1
-          fi
-
-  build_and_test_cpu_required:
-    # This job will be required to pass before merging to master branch.
-    name: Required Build and Test (CPU)
-    needs: check_code_format
+
+  performance_monitoring:
+    name: Performance monitoring (NVGPU)
     timeout-minutes: 60
-    strategy:
-      matrix:
-        include:
-          - os: ubuntu-latest
-            python: 3.6
-    runs-on: ${{ matrix.os }}
+    runs-on: [self-hosted, x64, cuda, linux, benchmark]
     steps:
       - uses: actions/checkout@v2
         with:
-          submodules: "recursive"
-
-      - uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python }}
-
-      - name: Download Pre-Built LLVM 10.0.0
-        run: python misc/ci_download.py
-        env:
-          CI_PLATFORM: ${{ matrix.os }}
+          submodules: 'recursive'
 
       - name: Build & Install
-        run: .github/workflows/scripts/unix_build.sh
-        env:
-          CI_SETUP_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CC:BOOL=ON -DTI_WITH_VULKAN:BOOL=OFF -DTI_BUILD_TESTS:BOOL=ON
-          CXX: clang++
-
-      - name: Test
-        run: .github/workflows/scripts/unix_test.sh
-        env:
-          RUN_CPP_TESTS: ON
-
-  build_and_test_cpu:
-    name: Build and Test (CPU)
-    needs: build_and_test_cpu_required
-    timeout-minutes: 60
-    strategy:
-      matrix:
-        include:
-          - os: macos-latest
-            python: 3.7
-            with_cc: OFF
-            with_cpp_tests: ON
-          - os: ubuntu-latest
-            python: 3.9
-            with_cc: OFF
-            with_cpp_tests: OFF
-          - os: ubuntu-latest
-            python: 3.8
-            with_cc: ON
-            with_cpp_tests: OFF
-    runs-on: ${{ matrix.os }}
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: "recursive"
-
-      - uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python }}
-
-      - name: Download Pre-Built LLVM 10.0.0
-        run: python misc/ci_download.py
-        env:
-          CI_PLATFORM: ${{ matrix.os }}
-
-      - name: Build & Install
-        run: .github/workflows/scripts/unix_build.sh
-        env:
-          CI_SETUP_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CC:BOOL=${{ matrix.with_cc }} -DTI_WITH_VULKAN:BOOL=OFF -DTI_BUILD_TESTS:BOOL=${{ matrix.with_cpp_tests }}
-          CXX: clang++
-      # [DEBUG] Copy this step around to enable debugging inside Github Action instances.
-      #- name: Setup tmate session
-      #  uses: mxschmitt/action-tmate@v3
-      #  with:
-      #    limit-access-to-actor: true
-
-      - name: Test
-        run: .github/workflows/scripts/unix_test.sh
-        env:
-          RUN_CPP_TESTS: ${{ matrix.with_cpp_tests }}
-
-  build_and_test_gpu_linux:
-    name: Build and Test (GPU)
-    needs: check_code_format
-    runs-on: [self-hosted, cuda, vulkan, cn]
-    timeout-minutes: 60
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: "recursive"
-
-      - name: Build
         run: |
           export PATH=$PATH:/usr/local/cuda/bin
           .github/workflows/scripts/unix_build.sh
         env:
           LLVM_LIB_ROOT_DIR: /opt/taichi-llvm-10.0.0
-          CI_SETUP_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=ON -DTI_WITH_CC:BOOL=OFF -DTI_WITH_VULKAN:BOOL=ON -DTI_BUILD_TESTS:BOOL=ON
-          BUILD_NUM_THREADS: 8
           LLVM_PATH: /opt/taichi-llvm-10.0.0/bin
           LLVM_DIR: /opt/taichi-llvm-10.0.0/lib/cmake/llvm
-          CXX: clang++-8
-
-      - name: Test
-        run: .github/workflows/scripts/unix_test.sh
-        env:
-          DISPLAY: :1
-          GPU_TEST: ON
-          RUN_CPP_TESTS: ON
-
-  build_and_test_windows:
-    name: Build and Test (Windows)
-    needs: check_code_format
-    runs-on: windows-latest
-    timeout-minutes: 90
-    steps:
-      - name: Install 7Zip PowerShell
-        shell: powershell
-        run: Install-Module 7Zip4PowerShell -Force -Verbose
-
-      - uses: actions/checkout@v2
-        with:
-          submodules: "recursive"
-
-      - uses: actions/setup-python@v2
-        with:
-          python-version: 3.7
-
-      - name: Add msbuild to PATH
-        uses: microsoft/[email protected]
-
-      - name: Download And Install Vulkan
-        shell: powershell
-        run: |
-          Invoke-WebRequest -Uri "https://sdk.lunarg.com/sdk/download/1.2.189.0/windows/VulkanSDK-1.2.189.0-Installer.exe" -OutFile VulkanSDK.exe
-          $installer = Start-Process -FilePath VulkanSDK.exe -Wait -PassThru -ArgumentList @("/S");
-          $installer.WaitForExit();
-
-      - name: Build
-        shell: powershell
-        run: |
-          $env:Path += ";C:/VulkanSDK/1.2.189.0/Bin"
-          cd C:\
-          Remove-item alias:curl
-          curl --retry 10 --retry-delay 5 https://github.com/taichi-dev/taichi_assets/releases/download/llvm10/taichi-llvm-10.0.0-msvc2019.zip -LO
-          7z x taichi-llvm-10.0.0-msvc2019.zip -otaichi_llvm
-          curl --retry 10 --retry-delay 5 https://github.com/taichi-dev/taichi_assets/releases/download/llvm10/clang-10.0.0-win.zip -LO
-          7z x clang-10.0.0-win.zip -otaichi_clang
-          $env:PATH = ";C:\taichi_llvm\bin;C:\taichi_clang\bin;" + $env:PATH
-          clang --version
-          cd D:\a\taichi\taichi
-          python -m pip install -r requirements_dev.txt
-          cd python
-          git fetch origin master
-          $env:TAICHI_CMAKE_ARGS = $env:CI_SETUP_CMAKE_ARGS
-          python build.py build
-          cd ..\dist
-          $env:WHL = $(dir *.whl)
-          python -m pip install $env:WHL
-        env:
-          PYTHON: C:\hostedtoolcache\windows\Python\3.7.9\x64\python.exe
-          CI_SETUP_CMAKE_ARGS: -G "Visual Studio 16 2019" -A x64 -DLLVM_DIR=C:\taichi_llvm\lib\cmake\llvm -DTI_WITH_VULKAN:BOOL=ON
-          VULKAN_SDK: C:/VulkanSDK/1.2.189.0
-
-      - name: Test
-        shell: powershell
-        run: |
-          $env:PATH = ";C:\taichi_llvm\bin;C:\taichi_clang\bin;" + $env:PATH
-          python -c "import taichi"
-          python examples/algorithm/laplace.py
-          python bin/taichi diagnose
-          python bin/taichi changelog
-          python bin/taichi test -vr2 -t2
-        env:
-          PYTHON: C:\hostedtoolcache\windows\Python\3.7.9\x64\python.exe
-
-  build_and_test_m1:
-    name: Build and Test (Apple M1)
-    needs: check_code_format
-    timeout-minutes: 60
-    strategy:
-      matrix:
-        include:
-          - os: macos-latest
-            python: 3.8
-    defaults:
-      run:
-        # https://github.com/actions/runner/issues/805#issuecomment-844426478
-        shell: "/usr/bin/arch -arch arm64e /bin/bash --noprofile --norc -eo pipefail {0}"
-    runs-on: [self-hosted, m1]
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: "recursive"
+          CUDA_TOOLKIT_ROOT_DIR: /usr/local/cuda/
+          CI_SETUP_CMAKE_ARGS: -DTI_WITH_CUDA_TOOLKIT:BOOL=ON
+          BUILD_NUM_THREADS: 8
+          CXX: clang++-10
 
-      - name: Build
-        run: |
-          rm -rf $HOME/Library/Python/3.8/lib/python/site-packages/taichi
-          .github/workflows/scripts/unix_build.sh
+      - name: Dump GitHub context
         env:
-          CI_SETUP_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CUDA:BOOL=OFF -DTI_WITH_CC:BOOL=OFF -DTI_WITH_VULKAN:BOOL=OFF -DTI_BUILD_TESTS:BOOL=ON
-          CXX: clang++
-
-      - name: Test
+          GITHUB_CONTEXT: ${{ toJson(github) }}
         run: |
-          export PATH=$PATH:$HOME/Library/Python/3.8/bin
-          python3 examples/algorithm/laplace.py
-          TI_LIB_DIR=`python3 -c "import taichi;print(taichi.__path__[0])" | tail -1`
-          TI_LIB_DIR="$TI_LIB_DIR/lib" ./build/taichi_cpp_tests
-          ti test -vr2 -t4 -x
+          python3 benchmarks/misc/test.py
+          python3 benchmarks/misc/print.py