Port Travis fix from develop branch to release branch, and improve scanning with Sonar

programs/build_helpers/build_and_test

@@ -4,13 +4,21 @@ set -e
 programs/build_helpers/buildstep -s 3500
 ccache -s
 programs/build_helpers/buildstep Prepare 1 "sed -i '/tests/d' libraries/fc/CMakeLists.txt"
-programs/build_helpers/buildstep cmake 5 "cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_FLAGS=--coverage -DCMAKE_CXX_FLAGS=--coverage -DCMAKE_CXX_FLAGS_DEBUG=-DTRAVIS_BUILD -DBoost_USE_STATIC_LIBS=OFF -DCMAKE_CXX_OUTPUT_EXTENSION_REPLACE=ON ."
+programs/build_helpers/buildstep cmake 5 "cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_FLAGS=--coverage -DCMAKE_CXX_FLAGS=--coverage -DBoost_USE_STATIC_LIBS=OFF -DCMAKE_CXX_OUTPUT_EXTENSION_REPLACE=ON ."
 programs/build_helpers/buildstep make.everything 2400 "programs/build_helpers/make_with_sonar bw-output -j 2 witness_node chain_test cli_test"
 set -o pipefail
 programs/build_helpers/buildstep run.chain_test 240 "libraries/fc/tests/run-parallel-tests.sh tests/chain_test"
 programs/build_helpers/buildstep run.cli_test 60 "libraries/fc/tests/run-parallel-tests.sh tests/cli_test"
-programs/build_helpers/buildstep prepare.sonar 20 "find libraries/[acdenptuw]*/CMakeFiles/*.dir programs/[cdgjsw]*/CMakeFiles/*.dir -name graphene_protocol_custom_auths.dir -prune -o -name custom_authorities -prune -o -type d -print | while read d; do gcov -o \"\$d\" \"\${d/CMakeFiles*.dir//}\"/*.cpp; done >/dev/null; programs/build_helpers/set_sonar_branch sonar-project.properties" || true
-programs/build_helpers/buildstep run.sonar 1200 "which sonar-scanner && sonar-scanner" || true
+programs/build_helpers/buildstep prepare.sonar 20 "find libraries/[acdenptuw]*/CMakeFiles/*.dir programs/[cdgjsw]*/CMakeFiles/*.dir -type d -print | while read d; do gcov -o \"\$d\" \"\${d/CMakeFiles*.dir//}\"/*.cpp; done >/dev/null; programs/build_helpers/set_sonar_branch sonar-project.properties"
+du -hs sonar_cache
+# The first pass, skip some files. This will remove the skipped files from the cache, but is an acceptable trade-off
+programs/build_helpers/buildstep prepare.sonar.part 1 "cp sonar-project.properties sonar-project.properties.bak; sed -i '/sonar\.exclusions=/d;s/#sonar\.exclusions.part/sonar.exclusions/' sonar-project.properties"
+programs/build_helpers/buildstep run.sonar.part 1500 "which sonar-scanner && sonar-scanner"
+du -hs sonar_cache
+# The second pass, scan all files
+programs/build_helpers/buildstep prepare.sonar.full 1 "cp sonar-project.properties.bak sonar-project.properties"
+programs/build_helpers/buildstep run.sonar.full 2000 "which sonar-scanner && sonar-scanner"
+du -hs sonar_cache
 programs/build_helpers/buildstep end 0
 ccache -s
 

programs/build_helpers/build_protocol

@@ -4,9 +4,13 @@ set -e
 programs/build_helpers/buildstep -s 3500
 ccache -s
 programs/build_helpers/buildstep Prepare 1 "sed -i '/tests/d' libraries/fc/CMakeLists.txt"
-programs/build_helpers/buildstep cmake 5 "cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_FLAGS=--coverage -DCMAKE_CXX_FLAGS=--coverage -DCMAKE_CXX_FLAGS_DEBUG=-DTRAVIS_BUILD -DBoost_USE_STATIC_LIBS=OFF -DCMAKE_CXX_OUTPUT_EXTENSION_REPLACE=ON ."
-programs/build_helpers/buildstep make.fc 200 "make -j 2 fc"
-programs/build_helpers/buildstep make.custom_auths 1300 "make -j 1 graphene_protocol_custom_auths"
-programs/build_helpers/buildstep make.protocol 400 "make -j 2 graphene_protocol"
+programs/build_helpers/buildstep cmake 5 "cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_FLAGS=--coverage -DCMAKE_CXX_FLAGS=--coverage -DBoost_USE_STATIC_LIBS=OFF -DCMAKE_CXX_OUTPUT_EXTENSION_REPLACE=ON ."
+programs/build_helpers/buildstep make.fc 230 "make -j 2 fc"
+programs/build_helpers/buildstep make.custom_auths 700 "make -j 1 graphene_protocol_custom_auths"
+programs/build_helpers/buildstep make.protocol 250 "make -j 2 graphene_protocol"
+programs/build_helpers/buildstep make.chain 450 "make -j 2 graphene_chain"
+programs/build_helpers/buildstep make.node 600 "make -j 2 witness_node"
+programs/build_helpers/buildstep make.cli 500 "make -j 2 cli_wallet"
+programs/build_helpers/buildstep make.chain_test 750 "make -j 2 chain_test"
 programs/build_helpers/buildstep end 0
 ccache -s

programs/build_helpers/set_sonar_branch

@@ -22,55 +22,58 @@ clear_branch () {
     sed -i '/sonar\.branch/d' "$1"
 }
 
-TARGET=
+ORIGINAL_TARGET="$( grep 'sonar\.branch\.target' "$1" | sed 's=^.*[:=] *==' )"
+
+TARGET="$ORIGINAL_TARGET"
 FETCH=
 
 if [ -n "$TRAVIS_PULL_REQUEST" -a "$TRAVIS_PULL_REQUEST" != false ]; then
-    # PRs work per default
+    # PRs work per default, remove sonar.branch.* since they only work with sonar.pullrequest.*
     echo "Detected PR '$TRAVIS_PULL_REQUEST'"
-    clear_branch "$1"
+    TARGET=
     FETCH="$TRAVIS_BRANCH"
 elif [ -n "$TRAVIS_TAG" ]; then
     # Tag build is either master or testnet
     echo "Detected tag '$TRAVIS_TAG'"
-    clear_branch "$1"
     case "$TRAVIS_TAG" in
 	*test*) TARGET=testnet; FETCH=testnet; ;;
-	*)	FETCH=master; ;;
+	*)	TARGET=master;  FETCH=master; ;;
     esac
 else
     case "$TRAVIS_BRANCH" in
-	master|develop|testnet|next_hardfork)
+	master|develop|testnet|hardfork)
 	    # Long-lived branches stand for themselves
 	    echo "Detected long-lived branch '$TRAVIS_BRANCH'"
-	    clear_branch "$1"
+	    TARGET="$TRAVIS_BRANCH"
 	    FETCH="$TRAVIS_BRANCH"
 	    ;;
 	*test*release*)
 	    # Testnet release branch will be merged into testnet
 	    echo "Detected testnet release branch '$TRAVIS_BRANCH'"
-	    clear_branch "$1"
 	    TARGET=testnet
 	    FETCH=testnet
 	    ;;
 	*release*)
 	    # Release branch will be merged into default (master)
 	    echo "Detected release branch '$TRAVIS_BRANCH'"
-	    clear_branch "$1"
+	    TARGET=master
 	    FETCH=master
 	    ;;
 	*)
 	    # All other branches should have sonar.branch.target in their
 	    # sonar.properties, leave it unchanged
 	    echo "Detected normal branch '$TRAVIS_BRANCH'"
-	    FETCH="$( grep 'sonar\.branch\.target' "$1" | sed 's=^.*[:=] *==' )"
+	    FETCH="$TARGET"
     esac
 fi
 
 echo "Branch target '$TARGET', fetch target '$FETCH'"
 
-if [ -n "$TARGET" ]; then
-    echo "sonar.branch.target=$TARGET" >>"$1"
+if [ "$TARGET" != "$ORIGINAL_TARGET" ]; then
+    clear_branch "$1"
+    if [ -n "$TARGET" ]; then
+        echo "sonar.branch.target=$TARGET" >>"$1"
+    fi
 fi
 #if [ -n "$FETCH" ]; then
     # Unfortunately this leads to sonar failing. Apparently it needs a full

sonar-project.properties

@@ -7,6 +7,8 @@ sonar.links.issue=https://github.com/bitshares/bitshares-core/issues
 sonar.links.scm=https://github.com/bitshares/bitshares-core/tree/master
 
 sonar.tests=tests
+# Used by the `build_and_test` script for the first pass when building with Travis CI, to skip some files
+#sonar.exclusions.part=programs/build_helper/**/*,libraries/fc/**/*,libraries/egenesis/egenesis_full.cpp,libraries/chain/**/*,libraries/protocol/**/*
 sonar.exclusions=programs/build_helper/**/*,libraries/fc/**/*,libraries/egenesis/egenesis_full.cpp
 sonar.sources=libraries,programs
 sonar.cfamily.build-wrapper-output=bw-output
@@ -15,5 +17,6 @@ sonar.cfamily.threads=2
 sonar.cfamily.cache.enabled=true
 sonar.cfamily.cache.path=sonar_cache
 
-# should be changed in hardfork branch and removed in release branches
+# Decide which tree the current build belongs to in SonarCloud.
+# Managed by the `set_sonar_branch` script when building with Travis CI.
 sonar.branch.target=develop