From 1b9ef639e79670929695925570b708b22cab4cfb Mon Sep 17 00:00:00 2001 From: Marc Garcia Date: Fri, 30 Nov 2018 21:33:42 +0000 Subject: [PATCH] TST/CI: Unify CI test scripts (single and multi) and simplify the system (#23924) * WIP: unifying script_single and script_multi * Unified scripts that run the tests (single and multi) * pending fixes * logging when we upload the coverage * Restoring pytest --skip-slow... options * Not running tests on doc build --- .circleci/config.yml | 4 +-- .travis.yml | 24 ++++++++---------- ci/azure/linux.yml | 10 +++----- ci/azure/macos.yml | 6 ++--- ci/azure/windows-py27.yml | 2 +- ci/azure/windows.yml | 2 +- ci/run_tests.sh | 52 +++++++++++++++++++++++++++++++++++++++ ci/script_multi.sh | 49 ------------------------------------ ci/script_single.sh | 39 ----------------------------- ci/upload_coverage.sh | 11 --------- 10 files changed, 72 insertions(+), 127 deletions(-) create mode 100755 ci/run_tests.sh delete mode 100755 ci/script_multi.sh delete mode 100755 ci/script_single.sh delete mode 100755 ci/upload_coverage.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index dc4162a0674fdf..6b516b21722acb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -34,5 +34,5 @@ jobs: command: | export PATH="$MINICONDA_DIR/bin:$PATH" source activate pandas-dev - echo "pytest --strict --durations=10 --color=no --junitxml=$CIRCLE_TEST_REPORTS/reports/junit.xml --skip-slow --skip-network pandas" - pytest --strict --durations=10 --color=no --junitxml=$CIRCLE_TEST_REPORTS/reports/junit.xml --skip-slow --skip-network pandas + echo "pytest -m "not slow and not network" --strict --durations=10 --color=no --junitxml=$CIRCLE_TEST_REPORTS/reports/junit.xml pandas" + pytest -m "not slow and not network" --strict --durations=10 --color=no --junitxml=$CIRCLE_TEST_REPORTS/reports/junit.xml pandas diff --git a/.travis.yml b/.travis.yml index 3217fc5aa1ed6b..6bbc44fba864af 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,28 +34,28 @@ matrix: include: - dist: trusty env: - - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" TEST_ARGS="--skip-slow --skip-network" + - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" PATTERN="not slow and not network" - dist: trusty env: - - JOB="2.7, locale, slow, old NumPy" ENV_FILE="ci/deps/travis-27-locale.yaml" LOCALE_OVERRIDE="zh_CN.UTF-8" SLOW=true + - JOB="2.7, locale, slow, old NumPy" ENV_FILE="ci/deps/travis-27-locale.yaml" LOCALE_OVERRIDE="zh_CN.UTF-8" PATTERN="slow" addons: apt: packages: - language-pack-zh-hans - dist: trusty env: - - JOB="2.7" ENV_FILE="ci/deps/travis-27.yaml" TEST_ARGS="--skip-slow" + - JOB="2.7" ENV_FILE="ci/deps/travis-27.yaml" PATTERN="not slow" addons: apt: packages: - python-gtk2 - dist: trusty env: - - JOB="3.6, lint, coverage" ENV_FILE="ci/deps/travis-36.yaml" TEST_ARGS="--skip-slow --skip-network" PANDAS_TESTING_MODE="deprecate" COVERAGE=true LINT=true + - JOB="3.6, lint, coverage" ENV_FILE="ci/deps/travis-36.yaml" PATTERN="not slow and not network" PANDAS_TESTING_MODE="deprecate" COVERAGE=true LINT=true - dist: trusty env: - - JOB="3.7, NumPy dev" ENV_FILE="ci/deps/travis-37-numpydev.yaml" TEST_ARGS="--skip-slow --skip-network -W error" PANDAS_TESTING_MODE="deprecate" + - JOB="3.7, NumPy dev" ENV_FILE="ci/deps/travis-37-numpydev.yaml" PATTERN="not slow and not network" TEST_ARGS="-W error" PANDAS_TESTING_MODE="deprecate" addons: apt: packages: @@ -64,7 +64,7 @@ matrix: # In allow_failures - dist: trusty env: - - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" SLOW=true + - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" PATTERN="slow" # In allow_failures - dist: trusty @@ -73,7 +73,7 @@ matrix: allow_failures: - dist: trusty env: - - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" SLOW=true + - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" PATTERN="slow" - dist: trusty env: - JOB="3.6, doc" ENV_FILE="ci/deps/travis-36-doc.yaml" DOC=true @@ -107,20 +107,16 @@ script: - echo "script start" - source activate pandas-dev - ci/run_build_docs.sh - - ci/script_single.sh - - ci/script_multi.sh + - ci/run_tests.sh - ci/code_checks.sh -after_success: - - ci/upload_coverage.sh - after_script: - echo "after_script start" - source activate pandas-dev && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd - if [ -e test-data-single.xml ]; then - ci/print_skipped.py test-data-single.xml; + ci/print_skipped.py test-data-single.xml; fi - if [ -e test-data-multiple.xml ]; then - ci/print_skipped.py test-data-multiple.xml; + ci/print_skipped.py test-data-multiple.xml; fi - echo "after_script done" diff --git a/ci/azure/linux.yml b/ci/azure/linux.yml index 7fa8a9a1783f90..fe64307e9d08f9 100644 --- a/ci/azure/linux.yml +++ b/ci/azure/linux.yml @@ -12,18 +12,18 @@ jobs: py27_np_120: ENV_FILE: ci/deps/azure-27-compat.yaml CONDA_PY: "27" - TEST_ARGS: "--skip-slow --skip-network" + PATTERN: "not slow and not network" py37_locale: ENV_FILE: ci/deps/azure-37-locale.yaml CONDA_PY: "37" - TEST_ARGS: "--skip-slow --skip-network" + PATTERN: "not slow and not network" LOCALE_OVERRIDE: "zh_CN.UTF-8" py36_locale_slow: ENV_FILE: ci/deps/azure-36-locale_slow.yaml CONDA_PY: "36" - TEST_ARGS: "--only-slow --skip-network" + PATTERN: "not slow and not network" LOCALE_OVERRIDE: "it_IT.UTF-8" steps: @@ -43,9 +43,7 @@ jobs: - script: | export PATH=$HOME/miniconda3/bin:$PATH source activate pandas-dev - ci/script_single.sh - ci/script_multi.sh - echo "[Test done]" + ci/run_tests.sh displayName: 'Test' - script: | export PATH=$HOME/miniconda3/bin:$PATH diff --git a/ci/azure/macos.yml b/ci/azure/macos.yml index d537f0c70cbec8..98409576a5a875 100644 --- a/ci/azure/macos.yml +++ b/ci/azure/macos.yml @@ -12,7 +12,7 @@ jobs: py35_np_120: ENV_FILE: ci/deps/azure-macos-35.yaml CONDA_PY: "35" - TEST_ARGS: "--skip-slow --skip-network" + PATTERN: "not slow and not network" steps: - script: | @@ -31,9 +31,7 @@ jobs: - script: | export PATH=$HOME/miniconda3/bin:$PATH source activate pandas-dev - ci/script_single.sh - ci/script_multi.sh - echo "[Test done]" + ci/run_tests.sh displayName: 'Test' - script: | export PATH=$HOME/miniconda3/bin:$PATH diff --git a/ci/azure/windows-py27.yml b/ci/azure/windows-py27.yml index ac918f3becd2eb..0d9aea816c4adc 100644 --- a/ci/azure/windows-py27.yml +++ b/ci/azure/windows-py27.yml @@ -37,7 +37,7 @@ jobs: displayName: 'Build' - script: | call activate pandas-dev - pytest --junitxml=test-data.xml --skip-slow --skip-network pandas -n 2 -r sxX --strict --durations=10 %* + pytest -m "not slow and not network" --junitxml=test-data.xml pandas -n 2 -r sxX --strict --durations=10 %* displayName: 'Test' - task: PublishTestResults@2 inputs: diff --git a/ci/azure/windows.yml b/ci/azure/windows.yml index f0ebba509e4419..b69c210ca27ba4 100644 --- a/ci/azure/windows.yml +++ b/ci/azure/windows.yml @@ -28,7 +28,7 @@ jobs: displayName: 'Build' - script: | call activate pandas-dev - pytest --junitxml=test-data.xml --skip-slow --skip-network pandas -n 2 -r sxX --strict --durations=10 %* + pytest -m "not slow and not network" --junitxml=test-data.xml pandas -n 2 -r sxX --strict --durations=10 %* displayName: 'Test' - task: PublishTestResults@2 inputs: diff --git a/ci/run_tests.sh b/ci/run_tests.sh new file mode 100755 index 00000000000000..77efc60a8cf972 --- /dev/null +++ b/ci/run_tests.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +if [ "$DOC" ]; then + echo "We are not running pytest as this is a doc-build" + exit 0 +fi + +# Workaround for pytest-xdist flaky collection order +# https://github.com/pytest-dev/pytest/issues/920 +# https://github.com/pytest-dev/pytest/issues/1075 +export PYTHONHASHSEED=$(python -c 'import random; print(random.randint(1, 4294967295))') + +if [ -n "$LOCALE_OVERRIDE" ]; then + export LC_ALL="$LOCALE_OVERRIDE" + export LANG="$LOCALE_OVERRIDE" + PANDAS_LOCALE=`python -c 'import pandas; pandas.get_option("display.encoding")'` + if [[ "$LOCALE_OVERIDE" != "$PANDAS_LOCALE" ]]; then + echo "pandas could not detect the locale. System locale: $LOCALE_OVERRIDE, pandas detected: $PANDAS_LOCALE" + # TODO Not really aborting the tests until https://github.com/pandas-dev/pandas/issues/23923 is fixed + # exit 1 + fi +fi +if [[ "not network" == *"$PATTERN"* ]]; then + export http_proxy=http://1.2.3.4 https_proxy=http://1.2.3.4; +fi + + +if [ -n "$PATTERN" ]; then + PATTERN=" and $PATTERN" +fi + +for TYPE in single multiple +do + if [ "$COVERAGE" ]; then + COVERAGE_FNAME="/tmp/coc-$TYPE.xml" + COVERAGE="-s --cov=pandas --cov-report=xml:$COVERAGE_FNAME" + fi + + TYPE_PATTERN=$TYPE + NUM_JOBS=1 + if [[ "$TYPE_PATTERN" == "multiple" ]]; then + TYPE_PATTERN="not single" + NUM_JOBS=2 + fi + + pytest -m "$TYPE_PATTERN$PATTERN" -n $NUM_JOBS -s --strict --durations=10 --junitxml=test-data-$TYPE.xml $TEST_ARGS $COVERAGE pandas + + if [[ "$COVERAGE" && $? == 0 ]]; then + echo "uploading coverage for $TYPE tests" + bash <(curl -s https://codecov.io/bash) -Z -c -F $TYPE -f $COVERAGE_FNAME + fi +done diff --git a/ci/script_multi.sh b/ci/script_multi.sh deleted file mode 100755 index fba0c7ba19dd4b..00000000000000 --- a/ci/script_multi.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -e - -echo "[script multi]" - -if [ -n "$LOCALE_OVERRIDE" ]; then - export LC_ALL="$LOCALE_OVERRIDE"; - export LANG="$LOCALE_OVERRIDE"; - echo "Setting LC_ALL to $LOCALE_OVERRIDE" - - pycmd='import pandas; print("pandas detected console encoding: %s" % pandas.get_option("display.encoding"))' - python -c "$pycmd" -fi - -# Enforce absent network during testing by faking a proxy -if echo "$TEST_ARGS" | grep -e --skip-network -q; then - export http_proxy=http://1.2.3.4 https_proxy=http://1.2.3.4; -fi - -# Workaround for pytest-xdist flaky collection order -# https://github.com/pytest-dev/pytest/issues/920 -# https://github.com/pytest-dev/pytest/issues/1075 -export PYTHONHASHSEED=$(python -c 'import random; print(random.randint(1, 4294967295))') -echo PYTHONHASHSEED=$PYTHONHASHSEED - -if [ "$DOC" ]; then - echo "We are not running pytest as this is a doc-build" - -elif [ "$COVERAGE" ]; then - echo pytest -s -n 2 -m "not single" --durations=10 --cov=pandas --cov-report xml:/tmp/cov-multiple.xml --junitxml=test-data-multiple.xml --strict $TEST_ARGS pandas - pytest -s -n 2 -m "not single" --durations=10 --cov=pandas --cov-report xml:/tmp/cov-multiple.xml --junitxml=test-data-multiple.xml --strict $TEST_ARGS pandas - -elif [ "$SLOW" ]; then - TEST_ARGS="--only-slow --skip-network" - # The `-m " and slow"` is redundant here, as `--only-slow` is already used (via $TEST_ARGS). But is needed, because with - # `--only-slow` fast tests are skipped, but each of them is printed in the log (which can be avoided with `-q`), - # and also added to `test-data-multiple.xml`, and then printed in the log in the call to `ci/print_skipped.py`. - # Printing them to the log makes the log exceed the maximum size allowed by Travis and makes the build fail. - echo pytest -n 2 -m "not single and slow" --durations=10 --junitxml=test-data-multiple.xml --strict $TEST_ARGS pandas - pytest -n 2 -m "not single and slow" --durations=10 --junitxml=test-data-multiple.xml --strict $TEST_ARGS pandas - -else - echo pytest -n 2 -m "not single" --durations=10 --junitxml=test-data-multiple.xml --strict $TEST_ARGS pandas - pytest -n 2 -m "not single" --durations=10 --junitxml=test-data-multiple.xml --strict $TEST_ARGS pandas # TODO: doctest - -fi - -RET="$?" - -exit "$RET" diff --git a/ci/script_single.sh b/ci/script_single.sh deleted file mode 100755 index cbbb7a49541c21..00000000000000 --- a/ci/script_single.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -echo "[script_single]" - -if [ -n "$LOCALE_OVERRIDE" ]; then - echo "Setting LC_ALL and LANG to $LOCALE_OVERRIDE" - export LC_ALL="$LOCALE_OVERRIDE"; - export LANG="$LOCALE_OVERRIDE"; - - pycmd='import pandas; print("pandas detected console encoding: %s" % pandas.get_option("display.encoding"))' - python -c "$pycmd" -fi - -if [ "$SLOW" ]; then - TEST_ARGS="--only-slow --skip-network" -fi - -# Enforce absent network during testing by faking a proxy -if echo "$TEST_ARGS" | grep -e --skip-network -q; then - export http_proxy=http://1.2.3.4 https_proxy=http://1.2.3.4; -fi - -if [ "$DOC" ]; then - echo "We are not running pytest as this is a doc-build" - -elif [ "$COVERAGE" ]; then - echo pytest -s -m "single" --durations=10 --strict --cov=pandas --cov-report xml:/tmp/cov-single.xml --junitxml=test-data-single.xml $TEST_ARGS pandas - pytest -s -m "single" --durations=10 --strict --cov=pandas --cov-report xml:/tmp/cov-single.xml --junitxml=test-data-single.xml $TEST_ARGS pandas - echo pytest -s --strict scripts - pytest -s --strict scripts -else - echo pytest -m "single" --durations=10 --junitxml=test-data-single.xml --strict $TEST_ARGS pandas - pytest -m "single" --durations=10 --junitxml=test-data-single.xml --strict $TEST_ARGS pandas - -fi - -RET="$?" - -exit "$RET" diff --git a/ci/upload_coverage.sh b/ci/upload_coverage.sh deleted file mode 100755 index 88aca20590505b..00000000000000 --- a/ci/upload_coverage.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -if [ -z "$COVERAGE" ]; then - echo "coverage is not selected for this build" - exit 0 -fi - - -echo "uploading coverage" -bash <(curl -s https://codecov.io/bash) -Z -c -F single -f /tmp/cov-single.xml -bash <(curl -s https://codecov.io/bash) -Z -c -F multiple -f /tmp/cov-multiple.xml