[#491] Add test coverage reports into the runtime documentation

mikir · mikir · commit 564c5ca6cef0 · 2023-03-24T15:20:06.000+01:00
diff --git a/cmake/coverage_utils.cmake b/cmake/coverage_utils.cmake
@@ -28,6 +28,7 @@ function(create_coverage_target)
     endif ()
 
     set(cov_binary_dir "${PROJECT_BINARY_DIR}/${cov_tgt_name}")
+    set(cov_install_dir "${CMAKE_INSTALL_PREFIX}/cpp/zserio_doc/coverage")
 
     if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
         set(cov_fail)
@@ -49,7 +50,8 @@ function(create_coverage_target)
                 -r ${PROJECT_SOURCE_DIR}
                 --object-directory=${PROJECT_BINARY_DIR}
                 ${cov_fail}
-                ${cov_exclude}
+                ${cov_exclude} > ${cov_binary_dir}/coverage_report.txt
+            COMMAND ${CMAKE_COMMAND} -E copy_directory ${cov_binary_dir} ${cov_install_dir}/gcc
             WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
             VERBATIM
             COMMENT "Generating html code coverage report in ${cov_binary_dir}/index.html")
@@ -67,16 +69,20 @@ function(create_coverage_target)
             COMMAND ${LLVM_PROFDATA_BIN} merge --sparse default.profraw -o ${cov_binary_dir}/runtime.profdata
             COMMAND ${LLVM_COV_BIN} show ${cov_test_exectable}
                 -instr-profile=${cov_binary_dir}/runtime.profdata
-                --format=html --show-instantiations=false -output-dir=${cov_binary_dir}
+                -format=html -show-instantiations=false -output-dir=${cov_binary_dir}
                 ${cov_exclude}
             COMMAND ${LLVM_COV_BIN} report ${cov_test_exectable}
                 -instr-profile=${cov_binary_dir}/runtime.profdata
                 ${cov_exclude}
+            COMMAND bash -c "${LLVM_COV_BIN} report ${cov_test_exectable} \
+                    -instr-profile=${cov_binary_dir}/runtime.profdata ${cov_exclude} \
+                    > ${cov_binary_dir}/coverage_report.txt"
             COMMAND bash -c "(( \
                 `${LLVM_COV_BIN} report ${cov_test_exectable} \
                     -instr-profile=${cov_binary_dir}/runtime.profdata ${cov_exclude} | grep TOTAL | \
                     tr -s ' ' | cut -d' ' -f 10 | cut -d. -f 1` >= ${cov_INCOMPLETE_COVERAGE_FAIL} \
                 ))"
+            COMMAND ${CMAKE_COMMAND} -E copy_directory ${cov_binary_dir} ${cov_install_dir}/clang
             WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
             VERBATIM
             COMMENT "Generating html code coverage report in ${cov_binary_dir}/index.html")
diff --git a/compiler/extensions/java/runtime/build.xml b/compiler/extensions/java/runtime/build.xml
@@ -54,6 +54,8 @@ spotbugs.home_dir               - Location of the spotbugs tool. If not set, spo
             value="${zserio_java_runtime.test.build_dir}/coverage"/>
     <property name="zserio_java_runtime.test.coverage_file"
             value="${zserio_java_runtime.test.coverage_dir}/jacoco_coverage.bin"/>
+    <property name="zserio_java_runtime.test.coverage_jar_file"
+            value="${zserio_java_runtime.install_dir}/zserio_runtime_coverage.jar"/>
 
     <property name="zserio_java_runtime.jar_dir" value="${zserio_java_runtime.build_dir}/jar"/>
     <property name="zserio_java_runtime.jar_file" value="${zserio_java_runtime.jar_dir}/zserio_runtime.jar"/>
@@ -311,13 +313,23 @@ spotbugs.home_dir               - Location of the spotbugs tool. If not set, spo
             </structure>
 
             <html destdir="${zserio_java_runtime.test.coverage_dir}"/>
+            <xml destfile="${zserio_java_runtime.test.coverage_dir}/jacoco_report.xml"/>
 
             <check>
                 <rule element="BUNDLE">
                     <limit counter="INSTRUCTION" value="COVEREDRATIO" minimum="92%"/>
                 </rule>
             </check>
         </jacoco:report>
+
+        <jar destfile="${zserio_java_runtime.test.coverage_jar_file}">
+            <fileset dir="${zserio_java_runtime.test.coverage_dir}">
+                <include name="**"/>
+            </fileset>
+            <manifest>
+                <attribute name="Implementation-Version" value="${zserio_java_runtime.version}"/>
+            </manifest>
+        </jar>
     </target>
 
     <target name="test" depends="test.run">
diff --git a/scripts/build.sh b/scripts/build.sh
@@ -38,6 +38,7 @@ test_python_runtime()
     ZSERIO_CPP_DIR=$(ls -d1 "${BUILD_DIR}/zserio_cpp/lib"*)
     if [ $? -ne 0 ] ; then
         stderr_echo "Failed to locate C++ runtime binding to Python!"
+        popd > /dev/null
         return 1
     fi
 
@@ -66,6 +67,14 @@ test_python_runtime()
         return 1
     fi
 
+    python -m coverage xml -o "coverage/coverage_report.xml" --fail-under=100 --omit="*test_object*"
+    local COVERAGE_RESULT=$?
+    if [ ${COVERAGE_RESULT} -ne 0 ] ; then
+        stderr_echo "Running python coverage report failed with return code ${COVERAGE_RESULT}!"
+        popd > /dev/null
+        return 1
+    fi
+
     popd > /dev/null
     echo
 
@@ -176,6 +185,15 @@ install_python_runtime()
         return 1
     fi
 
+    # install test coverage report
+    echo "Installing test coverage report"
+    cp -r "${PYTHON_RUNTIME_BUILD_DIR}/coverage" "${PYTHON_RUNTIME_DISTR_DIR}/zserio_doc/coverage"
+    if [ $? -ne 0 ] ; then
+        stderr_echo "Failed to install documentation!"
+        return 1
+    fi
+    rm -f "${PYTHON_RUNTIME_DISTR_DIR}/zserio_doc/coverage/.gitignore"
+
     return 0
 }
 
diff --git a/scripts/release.sh b/scripts/release.sh
@@ -52,6 +52,13 @@ set_release_global_variables()
             return 1
         fi
 
+        # XMLLINT to use, defaults to "xmllint" if not set
+        XMLLINT="${XMLLINT:-xmllint}"
+        if [ ! -f "`which "${XMLLINT}"`" ] ; then
+            stderr_echo "Cannot find xmllint! Set XMLLINT environment variable."
+            return 1
+        fi
+
         # GPG to use, defaults to "gpg" if not set
         GPG="${GPG:-gpg}"
         if [ ! -f "`which "${GPG}"`" ] ; then
@@ -140,6 +147,7 @@ Uses the following environment variables for releasing:
     MVN      Mvn executable to use. Default is "mvn".
     GIT      Git executable to use. Default is "git".
     UNZIP    Unzip executable to use. Default is "unzip".
+    XMLLINT  Xmllint executable to use. Default is "xmllint".
     GPG      Gpg executable to use. Default is "gpg".
     JAVA_BIN Java executable to use. Default is "java".
     PYTHON   Python executable to use. Default is "python3".
@@ -534,6 +542,12 @@ update_web_pages()
         stderr_echo "Cannot unzip zserio runtime javadocs jar!"
         return 1
     fi
+    "${UNZIP}" -q "${WEB_PAGES_BUILD_DIR}"/runtime_libs/java/zserio_runtime_coverage.jar \
+            -d "${WEB_PAGES_BUILD_DIR}"/runtime_libs/java/zserio_doc/coverage -x META-INF/* 
+    if [ $? -ne 0 ] ; then
+        stderr_echo "Cannot unzip zserio runtime javadocs jar!"
+        return 1
+    fi
     echo "Done"
 
     echo -ne "Copying Zserio runtime libraries..."
@@ -554,6 +568,42 @@ update_web_pages()
     fi
     echo "Done"
 
+    echo -ne "Creating Zserio runtime library GitHub badges..."
+    local CLANG_COVERAGE_DIR="${ZSERIO_PROJECT_ROOT}"/doc/runtime/cpp/coverage/clang
+    local CLANG_LINES_COVERAGE=`cat "${CLANG_COVERAGE_DIR}"/coverage_report.txt | grep TOTAL | \
+            tr -s ' ' | cut -d' ' -f 10`
+    create_github_badge_json "${CLANG_COVERAGE_DIR}"/coverage_github_badge.json \
+            "C++ clang runtime ${ZSERIO_VERSION} coverage" "${CLANG_LINES_COVERAGE}"
+
+    local GCC_COVERAGE_DIR="${ZSERIO_PROJECT_ROOT}"/doc/runtime/cpp/coverage/gcc
+    local GCC_LINES_COVERAGE=`cat "${GCC_COVERAGE_DIR}"/coverage_report.txt | grep lines: | \
+            tr -s ' ' | cut -d' ' -f 2`
+    create_github_badge_json "${GCC_COVERAGE_DIR}"/coverage_github_badge.json \
+            "C++ gcc runtime ${ZSERIO_VERSION} coverage" "${GCC_LINES_COVERAGE}"
+
+    local JAVA_COVERAGE_DIR="${ZSERIO_PROJECT_ROOT}"/doc/runtime/java/coverage
+    local JAVA_LINES_MISSED=`${XMLLINT} --format "${JAVA_COVERAGE_DIR}"/jacoco_report.xml | \
+            grep '<counter type="INSTRUCTION"' | tail -1 | tr -s ' ' | cut -d' ' -f4 | cut -d= -f2 | tr -d \"`
+    local JAVA_LINES_COVERED=`${XMLLINT} --format "${JAVA_COVERAGE_DIR}"/jacoco_report.xml | \
+            grep '<counter type="INSTRUCTION"' | tail -1 | tr -s ' ' | cut -d' ' -f5 | cut -d= -f2 | \
+            tr -d \"\/\>`
+    local JAVA_LINES_VALID=$((${JAVA_LINES_COVERED} - ${JAVA_LINES_MISSED}))
+    local JAVA_LINES_COVERAGE=$((10000 * ${JAVA_LINES_VALID} / ${JAVA_LINES_COVERED}))
+    create_github_badge_json "${JAVA_COVERAGE_DIR}"/coverage_github_badge.json \
+            "Java runtime ${ZSERIO_VERSION} coverage" \
+            "${JAVA_LINES_COVERAGE:0:-2}.${JAVA_LINES_COVERAGE: -2}%"
+
+    local PYTHON_COVERAGE_DIR="${ZSERIO_PROJECT_ROOT}"/doc/runtime/python/coverage
+    local PYTHON_LINES_VALID=`cat "${PYTHON_COVERAGE_DIR}"/coverage_report.xml | grep lines-covered | \
+            cut -d' ' -f 4 | cut -d= -f2 | tr -d \"`
+    local PYTHON_LINES_COVERED=`cat "${PYTHON_COVERAGE_DIR}"/coverage_report.xml | grep lines-covered | \
+            cut -d' ' -f 5 | cut -d= -f2 | tr -d \"`
+    local PYTHON_LINES_COVERAGE=$((10000 * ${PYTHON_LINES_VALID} / ${PYTHON_LINES_COVERED}))
+    create_github_badge_json "${PYTHON_COVERAGE_DIR}"/coverage_github_badge.json \
+            "Python runtime ${ZSERIO_VERSION} coverage" \
+            "${PYTHON_LINES_COVERAGE:0:-2}.${PYTHON_LINES_COVERAGE: -2}%"
+    echo "Done"
+
     # This is necessary because Jekyll ignores Python runtime doc directories that start with underscores.
     echo -ne "Creating Jekyll configuration file..."
     echo "theme: jekyll-theme-slate" > "${ZSERIO_PROJECT_ROOT}"/_config.yml
@@ -590,6 +640,24 @@ update_web_pages()
     return 0
 }
 
+# Create JSON configuration file for GitHub badge
+create_github_badge_json()
+{
+    exit_if_argc_ne $# 3
+    local BADGE_JSON_FILE="$1"; shift
+    local BADGE_LABEL="$1"; shift
+    local BADGE_MESSAGE="$1"; shift
+
+    cat > "${BADGE_JSON_FILE}" << EOF
+{
+    "schemaVersion": 1,
+    "label": "${BADGE_LABEL}",
+    "message": "${BADGE_MESSAGE}",
+    "color": "green"
+}
+EOF
+}
+
 # Print help message.
 print_help()
 {
