TST/CI: Unify CI test scripts (single and multi) and simplify the sys…

…tem (pandas-dev#23924)

* WIP: unifying script_single and script_multi

* Unified scripts that run the tests (single and multi)

* pending fixes

* logging when we upload the coverage

* Restoring pytest --skip-slow... options

* Not running tests on doc build

.circleci/config.yml

@@ -34,5 +34,5 @@ jobs:
           command: |
             export PATH="$MINICONDA_DIR/bin:$PATH"
             source activate pandas-dev
-            echo "pytest --strict --durations=10 --color=no --junitxml=$CIRCLE_TEST_REPORTS/reports/junit.xml --skip-slow --skip-network pandas"
-            pytest       --strict --durations=10 --color=no --junitxml=$CIRCLE_TEST_REPORTS/reports/junit.xml --skip-slow --skip-network pandas
+            echo "pytest -m "not slow and not network" --strict --durations=10 --color=no --junitxml=$CIRCLE_TEST_REPORTS/reports/junit.xml pandas"
+            pytest       -m "not slow and not network" --strict --durations=10 --color=no --junitxml=$CIRCLE_TEST_REPORTS/reports/junit.xml pandas

.travis.yml

@@ -34,28 +34,28 @@ matrix:
     include:
     - dist: trusty
       env:
-        - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" TEST_ARGS="--skip-slow --skip-network"
+        - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" PATTERN="not slow and not network"
 
     - dist: trusty
       env:
-        - JOB="2.7, locale, slow, old NumPy" ENV_FILE="ci/deps/travis-27-locale.yaml" LOCALE_OVERRIDE="zh_CN.UTF-8" SLOW=true
+        - JOB="2.7, locale, slow, old NumPy" ENV_FILE="ci/deps/travis-27-locale.yaml" LOCALE_OVERRIDE="zh_CN.UTF-8" PATTERN="slow"
       addons:
         apt:
           packages:
           - language-pack-zh-hans
     - dist: trusty
       env:
-        - JOB="2.7" ENV_FILE="ci/deps/travis-27.yaml" TEST_ARGS="--skip-slow"
+        - JOB="2.7" ENV_FILE="ci/deps/travis-27.yaml" PATTERN="not slow"
       addons:
         apt:
           packages:
           - python-gtk2
     - dist: trusty
       env:
-        - JOB="3.6, lint, coverage" ENV_FILE="ci/deps/travis-36.yaml" TEST_ARGS="--skip-slow --skip-network" PANDAS_TESTING_MODE="deprecate" COVERAGE=true LINT=true
+        - JOB="3.6, lint, coverage" ENV_FILE="ci/deps/travis-36.yaml" PATTERN="not slow and not network" PANDAS_TESTING_MODE="deprecate" COVERAGE=true LINT=true
     - dist: trusty
       env:
-        - JOB="3.7, NumPy dev" ENV_FILE="ci/deps/travis-37-numpydev.yaml" TEST_ARGS="--skip-slow --skip-network -W error" PANDAS_TESTING_MODE="deprecate"
+        - JOB="3.7, NumPy dev" ENV_FILE="ci/deps/travis-37-numpydev.yaml" PATTERN="not slow and not network" TEST_ARGS="-W error" PANDAS_TESTING_MODE="deprecate"
       addons:
         apt:
           packages:
@@ -64,7 +64,7 @@ matrix:
     # In allow_failures
     - dist: trusty
       env:
-        - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" SLOW=true
+        - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" PATTERN="slow"
 
     # In allow_failures
     - dist: trusty
@@ -73,7 +73,7 @@ matrix:
     allow_failures:
       - dist: trusty
         env:
-          - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" SLOW=true
+          - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" PATTERN="slow"
       - dist: trusty
         env:
           - JOB="3.6, doc" ENV_FILE="ci/deps/travis-36-doc.yaml" DOC=true
@@ -107,20 +107,16 @@ script:
   - echo "script start"
   - source activate pandas-dev
   - ci/run_build_docs.sh
-  - ci/script_single.sh
-  - ci/script_multi.sh
+  - ci/run_tests.sh
   - ci/code_checks.sh
 
-after_success:
-  -  ci/upload_coverage.sh
-
 after_script:
   - echo "after_script start"
   - source activate pandas-dev && pushd /tmp && python -c "import pandas; pandas.show_versions();" && popd
   - if [ -e test-data-single.xml ]; then
-    ci/print_skipped.py test-data-single.xml;
+        ci/print_skipped.py test-data-single.xml;
     fi
   - if [ -e test-data-multiple.xml ]; then
-    ci/print_skipped.py test-data-multiple.xml;
+        ci/print_skipped.py test-data-multiple.xml;
     fi
   - echo "after_script done"

ci/azure/linux.yml

@@ -12,18 +12,18 @@ jobs:
       py27_np_120:
         ENV_FILE: ci/deps/azure-27-compat.yaml
         CONDA_PY: "27"
-        TEST_ARGS: "--skip-slow --skip-network"
+        PATTERN: "not slow and not network"
 
       py37_locale:
         ENV_FILE: ci/deps/azure-37-locale.yaml
         CONDA_PY: "37"
-        TEST_ARGS: "--skip-slow --skip-network"
+        PATTERN: "not slow and not network"
         LOCALE_OVERRIDE: "zh_CN.UTF-8"
 
       py36_locale_slow:
         ENV_FILE: ci/deps/azure-36-locale_slow.yaml
         CONDA_PY: "36"
-        TEST_ARGS: "--only-slow --skip-network"
+        PATTERN: "not slow and not network"
         LOCALE_OVERRIDE: "it_IT.UTF-8"
 
   steps:
@@ -43,9 +43,7 @@ jobs:
     - script: |
         export PATH=$HOME/miniconda3/bin:$PATH
         source activate pandas-dev
-        ci/script_single.sh
-        ci/script_multi.sh
-        echo "[Test done]"
+        ci/run_tests.sh
       displayName: 'Test'
     - script: |
         export PATH=$HOME/miniconda3/bin:$PATH

ci/azure/macos.yml

@@ -12,7 +12,7 @@ jobs:
       py35_np_120:
         ENV_FILE: ci/deps/azure-macos-35.yaml
         CONDA_PY: "35"
-        TEST_ARGS: "--skip-slow --skip-network"
+        PATTERN: "not slow and not network"
 
   steps:
     - script: |
@@ -31,9 +31,7 @@ jobs:
     - script: |
         export PATH=$HOME/miniconda3/bin:$PATH
         source activate pandas-dev
-        ci/script_single.sh
-        ci/script_multi.sh
-        echo "[Test done]"
+        ci/run_tests.sh
       displayName: 'Test'
     - script: |
         export PATH=$HOME/miniconda3/bin:$PATH

ci/azure/windows-py27.yml

@@ -37,7 +37,7 @@ jobs:
       displayName: 'Build'
     - script: |
         call activate pandas-dev
-        pytest --junitxml=test-data.xml --skip-slow --skip-network pandas -n 2 -r sxX --strict --durations=10 %*
+        pytest -m "not slow and not network" --junitxml=test-data.xml pandas -n 2 -r sxX --strict --durations=10 %*
       displayName: 'Test'
     - task: PublishTestResults@2
       inputs:

ci/azure/windows.yml

@@ -28,7 +28,7 @@ jobs:
       displayName: 'Build'
     - script: |
         call activate pandas-dev
-        pytest --junitxml=test-data.xml --skip-slow --skip-network pandas -n 2 -r sxX --strict --durations=10 %*
+        pytest -m "not slow and not network" --junitxml=test-data.xml pandas -n 2 -r sxX --strict --durations=10 %*
       displayName: 'Test'
     - task: PublishTestResults@2
       inputs:

ci/run_tests.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+if [ "$DOC" ]; then
+    echo "We are not running pytest as this is a doc-build"
+    exit 0
+fi
+
+# Workaround for pytest-xdist flaky collection order
+# https://github.com/pytest-dev/pytest/issues/920
+# https://github.com/pytest-dev/pytest/issues/1075
+export PYTHONHASHSEED=$(python -c 'import random; print(random.randint(1, 4294967295))')
+
+if [ -n "$LOCALE_OVERRIDE" ]; then
+    export LC_ALL="$LOCALE_OVERRIDE"
+    export LANG="$LOCALE_OVERRIDE"
+    PANDAS_LOCALE=`python -c 'import pandas; pandas.get_option("display.encoding")'`
+    if [[ "$LOCALE_OVERIDE" != "$PANDAS_LOCALE" ]]; then
+        echo "pandas could not detect the locale. System locale: $LOCALE_OVERRIDE, pandas detected: $PANDAS_LOCALE"
+        # TODO Not really aborting the tests until https://github.com/pandas-dev/pandas/issues/23923 is fixed
+        # exit 1
+    fi
+fi
+if [[ "not network" == *"$PATTERN"* ]]; then
+    export http_proxy=http://1.2.3.4 https_proxy=http://1.2.3.4;
+fi
+
+
+if [ -n "$PATTERN" ]; then
+    PATTERN=" and $PATTERN"
+fi
+
+for TYPE in single multiple
+do
+    if [ "$COVERAGE" ]; then
+        COVERAGE_FNAME="/tmp/coc-$TYPE.xml"
+        COVERAGE="-s --cov=pandas --cov-report=xml:$COVERAGE_FNAME"
+    fi
+
+    TYPE_PATTERN=$TYPE
+    NUM_JOBS=1
+    if [[ "$TYPE_PATTERN" == "multiple" ]]; then
+        TYPE_PATTERN="not single"
+        NUM_JOBS=2
+    fi
+
+    pytest -m "$TYPE_PATTERN$PATTERN" -n $NUM_JOBS -s --strict --durations=10 --junitxml=test-data-$TYPE.xml $TEST_ARGS $COVERAGE pandas
+
+    if [[ "$COVERAGE" && $? == 0 ]]; then
+        echo "uploading coverage for $TYPE tests"
+        bash <(curl -s https://codecov.io/bash) -Z -c -F $TYPE -f $COVERAGE_FNAME
+    fi
+done