Zero code change automation (aws#128)

* changes for automatic zero_code_changes_testing. This is first step towards automated release

config/buildspec_zero_code_change.yml

@@ -0,0 +1,43 @@
+# Build Spec for AWS CodeBuild CI
+
+version: 0.2
+env:
+  variables:
+    ## below enviornment variables are overridden in respective code build.
+    # for example for pytorch build run_pytest_pytorch and run_integration_pytest_pytorch will be enabled
+    run_pytest_pytorch: "disable"
+    run_pytest_mxnet: "enable"
+    run_pytest_tensorflow: "disable"
+    run_pytest_xgboost: "disable"
+    run_integration_pytest_pytorch: "disable"
+    run_integration_pytest_mxnet: "enable"
+    run_integration_pytest_tensorflow: "disable"
+    run_integration_pytest_xgboost: "disable"
+    # below needs to be enabled
+    zero_code_change_test: "enable"
+phases:
+  install:
+    commands:
+        -  . config/change_branch.sh
+        - su && apt-get update
+        - apt-get install sudo -qq -o=Dpkg::Use-Pty=0 # silence output: https://askubuntu.com/a/668859/724247
+        - sudo apt-get update -qq -o=Dpkg::Use-Pty=0
+        - sudo apt-get install unzip -qq -o=Dpkg::Use-Pty=0
+        - cd $CODEBUILD_SRC_DIR  && chmod +x config/protoc_downloader.sh && ./config/protoc_downloader.sh
+        - pip install -U pip
+        - pip install -q pytest wheel pyYaml pytest-html pre-commit
+
+  build:
+    commands:
+      - cd $CODEBUILD_SRC_DIR && chmod +x config/install_smdebug.sh && chmod +x config/check_smdebug_install.sh && ./config/install_smdebug.sh && cd ..
+      - cd $RULES_CODEBUILD_SRC_DIR && python setup.py bdist_wheel --universal && pip install dist/*.whl && cd ..
+      - cd $CODEBUILD_SRC_DIR  && chmod +x config/tests.sh && ./config/tests.sh  && mkdir -p upload/$CURRENT_COMMIT_PATH/wheels && cp ./dist/*.whl upload/$CURRENT_COMMIT_PATH/wheels && cd ..
+      - cd $RULES_CODEBUILD_SRC_DIR && chmod +x config/tests.sh && mkdir -p upload/$CURRENT_COMMIT_PATH/wheels && ./config/tests.sh  && cp ./dist/*.whl upload/$CURRENT_COMMIT_PATH/wheels && cd ..
+
+  post_build:
+    commands:
+      - . $CODEBUILD_SRC_DIR/config/upload_on_end.sh
+      - rm -rf $CODEBUILD_SRC_DIR/upload/$CURRENT_COMMIT_PATH
+      - rm -rf $RULES_CODEBUILD_SRC_DIR/upload/$CURRENT_COMMIT_PATH
+      - if [ "$CODEBUILD_BUILD_SUCCEEDING" -eq 0 ]; then echo "ERROR BUILD FAILED " && exit 1 ; fi
+      - if [ "$CODEBUILD_BUILD_SUCCEEDING" -eq 1 ]; then echo "INFO BUILD SUCCEEDED !!! " ; fi

config/check_smdebug_install.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+set -ex
+set -o pipefail
+python -c "import smdebug"
+res="$?"
+echo "output of import smdebug is: $res"
+
+#version=python -c "exec(\"import smdebug\nprint(smdebug.__version__)\")`
+version="$(python -c "exec(\"import smdebug\nprint(smdebug.__version__)\")")"
+# force check version, you can set this env variable in build env when releasing
+
+if [ "$force_check_smdebug_version" ]  && [ "$force_check_smdebug_version" != "${version}" ]; then
+  echo "force_check_version $force_check_smdebug_version doesn't match version found: ${version}"
+  exit 1
+fi
+
+if [ $1 ]; then
+  python -c "import $1"
+  res="$?"
+  echo "output of import $1 is: $res"
+  exit $res
+fi
+exit $res

config/install_smdebug.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+set -ex
+set -o pipefail
+
+
+cd $CODEBUILD_SRC_DIR
+# you can provide bip binary as s3 path in the build environment
+if [ "$SMDEBUG_S3_BINARY" ]; then
+  mkdir -p s3_pip_binary
+  aws s3 cp "$SMDEBUG_S3_BINARY" s3_pip_binary
+  pip install --force-reinstall --upgrade s3_pip_binary/*.whl
+else
+  python setup.py bdist_wheel --universal && pip install --upgrade --force-reinstall dist/*.whl
+fi
+
+
+if [ "$run_pytest_mxnet" == 'enable' ]; then
+  ./config/check_smdebug_install.sh mxnet
+fi
+if [ "$run_pytest_tensorflow" == 'enable' ]; then
+  ./config/check_smdebug_install.sh tensorflow
+fi
+if [ "$run_pytest_pytorch" == 'enable' ]; then
+  ./config/check_smdebug_install.sh torch
+fi

config/tests.sh

@@ -15,7 +15,20 @@ check_logs() {
 }
 
 run_for_framework() {
-    python -m pytest --durations=50 --html=$REPORT_DIR/report_$1.html -v -s --self-contained-html tests/$1
+    if [ "$zero_code_change_test" = "enable" ] ; then
+      # ignoring some test becuase they require multiple frmaeworks to be installed, these tests need to be broken down
+      python -m pytest --durations=50 --html=$REPORT_DIR/report_$1.html -v -s --self-contained-html --ignore=tests/core/test_hook_save_scalar.py --ignore=tests/core/test_paths.py --ignore=tests/core/test_index_utils.py --ignore=tests/core/test_collections.py tests/$1
+      if [ "$1" = "mxnet" ] ; then
+        python  tests/zero_code_change/mxnet_gluon_integration_test.py
+      elif [ "$1" = "pytorch" ] ; then
+        python tests/zero_code_change/pytorch_integration_tests.py
+      elif [ "$1" = "tensorflow" ] ; then
+        python -m pytest tests/zero_code_change/tensorflow_integration_tests.py
+      fi
+
+    else
+      python -m pytest --durations=50 --html=$REPORT_DIR/report_$1.html -v -s --self-contained-html tests/$1
+    fi
 }
 
 export TF_CPP_MIN_LOG_LEVEL=1
@@ -26,7 +39,8 @@ export SMDEBUG_LOG_LEVEL=info
 export OUT_DIR=upload/$CURRENT_COMMIT_PATH
 export REPORT_DIR=$OUT_DIR/pytest_reports
 python -m pytest -v -W=ignore --durations=50 --html=$REPORT_DIR/report_analysis.html --self-contained-html tests/analysis
-python -m pytest -v -W=ignore --durations=50 --html=$REPORT_DIR/report_core.html --self-contained-html tests/core
+
+run_for_framework core
 
 if [ "$run_pytest_xgboost" = "enable" ] ; then
     run_for_framework xgboost
@@ -48,6 +62,9 @@ check_logs $REPORT_DIR/*
 
 # Only look at newly added files
 if [ -n "$(git status --porcelain | grep ^?? | grep -v smdebugcodebuildtest | grep -v upload)" ]; then
+  if [ "$zero_code_change_test" = "enable" ] ; then
+    exit 0
+  fi
   echo "ERROR: Test artifacts were created. Please place these in /tmp."
   exit 1
 fi