From a510804b3ec31c8bdc7fc7b7f9d70f2c5cffb288 Mon Sep 17 00:00:00 2001 From: cgu Date: Wed, 3 Feb 2021 22:23:25 -0500 Subject: [PATCH 01/11] ci: use nondocker to run pytest and jest --- .github/workflows/tests.yml | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b49c89c52..fc71e5bb8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,19 +4,11 @@ on: branches: - master pull_request: -# A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: - # This workflow contains a single job called "build" lint: - # The type of runner that the job will run on runs-on: ubuntu-latest - - # Steps represent a sequence of tasks that will be executed as part of the job steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v2 - with: - fetch-depth: 1 - name: Set up Python uses: actions/setup-python@v1 with: @@ -39,5 +31,17 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 1 - - name: Run docker test - run: make test + - uses: actions/setup-python@v2 + with: + python-version: '3.7.9' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pytest + pip install -r requirements/prod.txt + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v2 + with: + node-version: '12' + - run: yarn install --frozen-lockfile + - run: PYTHONPATH=querybook/server ./querybook/scripts/run_test From f32dd64409dc4c9d04b9173928edd590b4ffa01f Mon Sep 17 00:00:00 2001 From: cgu Date: Wed, 3 Feb 2021 22:29:59 -0500 Subject: [PATCH 02/11] Fix testing --- .github/workflows/tests.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index fc71e5bb8..2a7cbe335 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -34,14 +34,13 @@ jobs: - uses: actions/setup-python@v2 with: python-version: '3.7.9' - - name: Install dependencies + - name: Install Python dependencies run: | python -m pip install --upgrade pip pip install pytest - pip install -r requirements/prod.txt - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v2 + pip install -r requirements/base.txt + - uses: actions/setup-node@v2 with: - node-version: '12' + node-version: '14' - run: yarn install --frozen-lockfile - run: PYTHONPATH=querybook/server ./querybook/scripts/run_test From 2d0969935501a7c760f8993aafc42c9cf7450ad6 Mon Sep 17 00:00:00 2001 From: cgu Date: Wed, 3 Feb 2021 22:37:50 -0500 Subject: [PATCH 03/11] add apt-get install --- .github/workflows/tests.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2a7cbe335..90a10650c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -31,6 +31,15 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 1 + - name: Install libs + run: + DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y --allow-downgrades --allow-remove-essential --allow-change-held-packages \ + libsasl2-dev \ + libsasl2-modules \ + build-essential \ + python-dev \ + libssl-dev \ + libldap2-dev - uses: actions/setup-python@v2 with: python-version: '3.7.9' From 59bd0050b99b7e20b1ac3b8047eebb9d2455ab87 Mon Sep 17 00:00:00 2001 From: cgu Date: Wed, 3 Feb 2021 22:38:52 -0500 Subject: [PATCH 04/11] sudo --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 90a10650c..c555fad1b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -33,7 +33,7 @@ jobs: fetch-depth: 1 - name: Install libs run: - DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y --allow-downgrades --allow-remove-essential --allow-change-held-packages \ + DEBIAN_FRONTEND=noninteractive sudo apt-get install --no-install-recommends -y --allow-downgrades --allow-remove-essential --allow-change-held-packages \ libsasl2-dev \ libsasl2-modules \ build-essential \ From a107afda6d56e7aa1dafc79b8ba33c7b2e2aef6b Mon Sep 17 00:00:00 2001 From: cgu Date: Wed, 3 Feb 2021 23:00:20 -0500 Subject: [PATCH 05/11] try to solve package not found --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c555fad1b..c392c2c0b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -33,7 +33,7 @@ jobs: fetch-depth: 1 - name: Install libs run: - DEBIAN_FRONTEND=noninteractive sudo apt-get install --no-install-recommends -y --allow-downgrades --allow-remove-essential --allow-change-held-packages \ + apt-get update && DEBIAN_FRONTEND=noninteractive sudo apt-get install --no-install-recommends -y --allow-downgrades --allow-remove-essential --allow-change-held-packages \ libsasl2-dev \ libsasl2-modules \ build-essential \ From aff85c8a19fea32ee3c7a96eade7f49809424468 Mon Sep 17 00:00:00 2001 From: cgu Date: Thu, 4 Feb 2021 00:23:52 -0500 Subject: [PATCH 06/11] split up tests, fix install --- .github/workflows/tests.yml | 37 ++++++++++++++++++++----------------- querybook/scripts/run_test | 31 ++++++++++++++++++++++++++++++- 2 files changed, 50 insertions(+), 18 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c392c2c0b..9477771ee 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,9 +10,9 @@ jobs: steps: - uses: actions/checkout@v2 - name: Set up Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: - python-version: 3.7 + python-version: 3.7.9 - name: Set PY run: | echo 'PY<> $GITHUB_ENV @@ -25,31 +25,34 @@ jobs: - uses: pre-commit/action@v1.0.1 with: token: ${{ secrets.GITHUB_TOKEN }} - tests: + nodetests: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + - uses: actions/setup-node@v2 with: - fetch-depth: 1 + node-version: '14' + - run: yarn install --frozen-lockfile + - run: PYTHONPATH=querybook/server ./querybook/scripts/run_test --node + pythontests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 - name: Install libs - run: - apt-get update && DEBIAN_FRONTEND=noninteractive sudo apt-get install --no-install-recommends -y --allow-downgrades --allow-remove-essential --allow-change-held-packages \ - libsasl2-dev \ - libsasl2-modules \ - build-essential \ - python-dev \ - libssl-dev \ + run: > + sudo apt-get update && DEBIAN_FRONTEND=noninteractive sudo apt-get install --no-install-recommends -y --allow-downgrades --allow-remove-essential --allow-change-held-packages + libsasl2-dev + libsasl2-modules + build-essential + python-dev + libssl-dev libldap2-dev - uses: actions/setup-python@v2 with: - python-version: '3.7.9' + python-version: 3.7.9 - name: Install Python dependencies run: | python -m pip install --upgrade pip pip install pytest pip install -r requirements/base.txt - - uses: actions/setup-node@v2 - with: - node-version: '14' - - run: yarn install --frozen-lockfile - - run: PYTHONPATH=querybook/server ./querybook/scripts/run_test + - run: PYTHONPATH=querybook/server ./querybook/scripts/run_test --python diff --git a/querybook/scripts/run_test b/querybook/scripts/run_test index c475fc0d6..baefbdc46 100755 --- a/querybook/scripts/run_test +++ b/querybook/scripts/run_test @@ -1,5 +1,20 @@ #!/bin/bash +usage() { + echo "Runs querybook tests" + echo '' + echo 'If not specified, then both python and node tests' + echo 'will be executed.' + echo '' + echo 'Flags:' + echo ' --python only run python tests' + echo ' --node only run node js tests' + echo + echo 'Usage:' + echo ' $0 [--python]' + exit 1 +} + run_python_unit_test() { echo 'Start running python unit tests >>>>>>>>>>>>>>>>>>>>>>>>>>>>' py.test querybook/tests || exit 1 @@ -29,14 +44,28 @@ run_eslint_validation() { npm run lint } -tests=( +PYTHON_TESTS=( "run_python_unit_test" +) + +NODE_TESTS=( "run_ts_validation" "run_js_unit_test" "run_eslint_validation" "run_webpack_test" ) +tests=("${PYTHON_TESTS[@]}" "${NODE_TESTS[@]}") +while [[ "$#" -gt 0 ]]; do + case $1 in + --python) tests=("${PYTHON_TESTS[@]}");; + --node) tests=("${NODE_TESTS[@]}");; + --help) usage;; + *) echo "Unknown parameter passed: $1"; usage;; + esac; + shift; +done + children_pids=() clen=`expr "${#tests[@]}" - 1` From 42a4cbcda598ae9805a1780cfa72d4b4d1c72d80 Mon Sep 17 00:00:00 2001 From: cgu Date: Thu, 4 Feb 2021 00:41:48 -0500 Subject: [PATCH 07/11] add caching --- .github/workflows/tests.yml | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9477771ee..46ac58cec 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -32,8 +32,19 @@ jobs: - uses: actions/setup-node@v2 with: node-version: '14' + - name: Get yarn cache directory path + id: yarn-cache-dir-path + run: echo "::set-output name=dir::$(yarn cache dir)" + - uses: actions/cache@v2 + id: yarn-cache + with: + path: ${{ steps.yarn-cache-dir-path.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn- - run: yarn install --frozen-lockfile - - run: PYTHONPATH=querybook/server ./querybook/scripts/run_test --node + if: steps.yarn-cache.outputs.cache-hit != 'true' + - run: ./querybook/scripts/run_test --node pythontests: runs-on: ubuntu-latest steps: @@ -50,7 +61,15 @@ jobs: - uses: actions/setup-python@v2 with: python-version: 3.7.9 + - uses: actions/cache@v2 + id: pip-cache + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/base.txt') }} + restore-keys: | + ${{ runner.os }}-pip- - name: Install Python dependencies + if: steps.pip-cache.outputs.cache-hit != 'true' run: | python -m pip install --upgrade pip pip install pytest From ad3fd338df1291f30528dc0389b01f802b105cc6 Mon Sep 17 00:00:00 2001 From: cgu Date: Thu, 4 Feb 2021 01:09:18 -0500 Subject: [PATCH 08/11] remove cache if since instlalation is still needed --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 46ac58cec..250f333ab 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -43,7 +43,7 @@ jobs: restore-keys: | ${{ runner.os }}-yarn- - run: yarn install --frozen-lockfile - if: steps.yarn-cache.outputs.cache-hit != 'true' + # if: steps.yarn-cache.outputs.cache-hit != 'true' - run: ./querybook/scripts/run_test --node pythontests: runs-on: ubuntu-latest @@ -69,7 +69,7 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - name: Install Python dependencies - if: steps.pip-cache.outputs.cache-hit != 'true' + # if: steps.pip-cache.outputs.cache-hit != 'true' run: | python -m pip install --upgrade pip pip install pytest From c3e0fe52eea7c3e395ed3d412a0eb7151f9d8065 Mon Sep 17 00:00:00 2001 From: cgu Date: Thu, 4 Feb 2021 01:59:20 -0500 Subject: [PATCH 09/11] Update pre-commited hooks --- .github/workflows/tests.yml | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 250f333ab..c6b3cf7bf 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -9,22 +9,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.7.9 - - name: Set PY - run: | - echo 'PY<> $GITHUB_ENV - echo "$(python -c 'import hashlib, sys;print(hashlib.sha256(sys.version.encode()+sys.executable.encode()).hexdigest())')" >> $GITHUB_ENV - echo 'EOF' >> $GITHUB_ENV - - uses: actions/cache@v1 - with: - path: ~/.cache/pre-commit - key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }} - - uses: pre-commit/action@v1.0.1 - with: - token: ${{ secrets.GITHUB_TOKEN }} + - uses: actions/setup-python@v2 + - uses: pre-commit/action@v2.0.0 nodetests: runs-on: ubuntu-latest steps: From 6a310129b5bb1da82e63c2e7e49a2beb35426b37 Mon Sep 17 00:00:00 2001 From: cgu Date: Thu, 4 Feb 2021 12:57:28 -0500 Subject: [PATCH 10/11] Split out task into different steps --- .github/workflows/tests.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c6b3cf7bf..58ca9bddf 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -30,7 +30,10 @@ jobs: ${{ runner.os }}-yarn- - run: yarn install --frozen-lockfile # if: steps.yarn-cache.outputs.cache-hit != 'true' - - run: ./querybook/scripts/run_test --node + - run: NODE_ENV=test ./node_modules/.bin/jest + - run: npm run tsc-check + - run: npm run lint + - run: ./node_modules/.bin/webpack --mode=production pythontests: runs-on: ubuntu-latest steps: From 01781642febaf1f4f22f696d79963a814508674b Mon Sep 17 00:00:00 2001 From: cgu Date: Thu, 4 Feb 2021 14:44:39 -0500 Subject: [PATCH 11/11] faster yarn install --- .github/workflows/tests.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 58ca9bddf..b451e4f8d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -28,12 +28,9 @@ jobs: key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} restore-keys: | ${{ runner.os }}-yarn- - - run: yarn install --frozen-lockfile + - run: yarn install --frozen-lockfile --prefer-offline # if: steps.yarn-cache.outputs.cache-hit != 'true' - - run: NODE_ENV=test ./node_modules/.bin/jest - - run: npm run tsc-check - - run: npm run lint - - run: ./node_modules/.bin/webpack --mode=production + - run: ./querybook/scripts/run_test --node pythontests: runs-on: ubuntu-latest steps: