[#491] Add test coverage reports into the runtime documentation

mikir · mikir · commit 47b0361e5e7a · 2023-03-29T10:00:47.000+02:00
diff --git a/cmake/coverage_utils.cmake b/cmake/coverage_utils.cmake
@@ -27,7 +27,7 @@ function(create_coverage_target)
         set(cov_tgt_name "coverage")
     endif ()
 
-    set(cov_binary_dir "${PROJECT_BINARY_DIR}/${cov_tgt_name}")
+    set(cov_html_dir "${PROJECT_BINARY_DIR}/zserio_doc/${cov_tgt_name}")
 
     if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
         set(cov_fail)
@@ -40,25 +40,33 @@ function(create_coverage_target)
             list(APPEND cov_exclude "--exclude=${cov_EXCLUDE_SOURCES}")
         endif ()
 
+        set(cov_html_gcc_dir "${cov_html_dir}/gcc")
         add_custom_target(
             ${cov_tgt_name}
-            COMMAND ${CMAKE_COMMAND} -E make_directory ${cov_binary_dir}
+            COMMAND ${CMAKE_COMMAND} -E make_directory ${cov_html_gcc_dir}
             COMMAND ${GCOVR_BIN}
                 -s
-                --html --html-details -o "${cov_tgt_name}/index.html"
                 -r ${PROJECT_SOURCE_DIR}
                 --object-directory=${PROJECT_BINARY_DIR}
-                ${cov_fail}
                 ${cov_exclude}
+            COMMAND ${GCOVR_BIN}
+                -s
+                --html --html-details -o "${cov_html_gcc_dir}/index.html"
+                -r ${PROJECT_SOURCE_DIR}
+                --object-directory=${PROJECT_BINARY_DIR}
+                ${cov_fail}
+                ${cov_exclude} > ${cov_html_gcc_dir}/coverage_report.txt
             WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
             VERBATIM
-            COMMENT "Generating html code coverage report in ${cov_binary_dir}/index.html")
+            COMMENT "Generating html code coverage report in ${cov_html_gcc_dir}/index.html")
     elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
-        set(cov_test_exectable "${ZserioCppRuntimeTest_BINARY_DIR}/ZserioCppRuntimeTest")
         set(cov_exclude)
         if (cov_EXCLUDE_SOURCES)
             list(APPEND cov_exclude "--ignore-filename-regex=${cov_EXCLUDE_SOURCES}")
         endif ()
+        set(cov_test_exectable "${ZserioCppRuntimeTest_BINARY_DIR}/ZserioCppRuntimeTest")
+        set(cov_binary_dir "${PROJECT_BINARY_DIR}/${cov_tgt_name}")
+        set(cov_html_clang_dir "${cov_html_dir}/clang")
         add_custom_target(
             ${cov_tgt_name}
             COMMAND ${CMAKE_COMMAND} -E make_directory ${cov_binary_dir}
@@ -67,19 +75,22 @@ function(create_coverage_target)
             COMMAND ${LLVM_PROFDATA_BIN} merge --sparse default.profraw -o ${cov_binary_dir}/runtime.profdata
             COMMAND ${LLVM_COV_BIN} show ${cov_test_exectable}
                 -instr-profile=${cov_binary_dir}/runtime.profdata
-                --format=html --show-instantiations=false -output-dir=${cov_binary_dir}
+                -format=html -show-instantiations=false -show-branches=count -output-dir=${cov_html_clang_dir}
                 ${cov_exclude}
             COMMAND ${LLVM_COV_BIN} report ${cov_test_exectable}
                 -instr-profile=${cov_binary_dir}/runtime.profdata
                 ${cov_exclude}
+            COMMAND bash -c "${LLVM_COV_BIN} report ${cov_test_exectable} \
+                    -instr-profile=${cov_binary_dir}/runtime.profdata ${cov_exclude} \
+                    > ${cov_html_clang_dir}/coverage_report.txt"
             COMMAND bash -c "(( \
                 `${LLVM_COV_BIN} report ${cov_test_exectable} \
                     -instr-profile=${cov_binary_dir}/runtime.profdata ${cov_exclude} | grep TOTAL | \
                     tr -s ' ' | cut -d' ' -f 10 | cut -d. -f 1` >= ${cov_INCOMPLETE_COVERAGE_FAIL} \
                 ))"
             WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
             VERBATIM
-            COMMENT "Generating html code coverage report in ${cov_binary_dir}/index.html")
+            COMMENT "Generating html code coverage report in ${cov_html_clang_dir}/index.html")
     else ()
         message(FATAL_ERROR "Coverage reports are not supported for target compiler ${CMAKE_CXX_COMPILER_ID}!")
     endif ()
diff --git a/compiler/extensions/cpp/runtime/src/zserio/MainPage.dox b/compiler/extensions/cpp/runtime/src/zserio/MainPage.dox
@@ -7,4 +7,10 @@
  *
  * - Serialization (`SerializeUtil.h` module)
  * - Debug string output (`DebugStringUtil.h` module)
+ *
+ * Test coverage report of this library is available for <a href="coverage/gcc/index.html">gcc</a> and for
+ * <a href="coverage/clang/index.html">clang</a>. Please note that <a href="coverage/gcc/index.html">gcc</a>
+ * coverage report has many uncovered branches because of
+ * <a href="https://gcovr.com/en/stable/faq.html#why-does-c-code-have-so-many-uncovered-branches">
+ * different measurement approach</a>.
  */
diff --git a/compiler/extensions/java/runtime/build.xml b/compiler/extensions/java/runtime/build.xml
@@ -54,6 +54,8 @@ spotbugs.home_dir               - Location of the spotbugs tool. If not set, spo
             value="${zserio_java_runtime.test.build_dir}/coverage"/>
     <property name="zserio_java_runtime.test.coverage_file"
             value="${zserio_java_runtime.test.coverage_dir}/jacoco_coverage.bin"/>
+    <property name="zserio_java_runtime.test.coverage_html_dir"
+            value="${zserio_java_runtime.javadocs_dir}/coverage"/>
 
     <property name="zserio_java_runtime.jar_dir" value="${zserio_java_runtime.build_dir}/jar"/>
     <property name="zserio_java_runtime.jar_file" value="${zserio_java_runtime.jar_dir}/zserio_runtime.jar"/>
@@ -310,14 +312,23 @@ spotbugs.home_dir               - Location of the spotbugs tool. If not set, spo
                 </sourcefiles>
             </structure>
 
-            <html destdir="${zserio_java_runtime.test.coverage_dir}"/>
+            <html destdir="${zserio_java_runtime.test.coverage_html_dir}"/>
+            <xml destfile="${zserio_java_runtime.test.coverage_html_dir}/jacoco_report.xml"/>
 
             <check>
                 <rule element="BUNDLE">
                     <limit counter="INSTRUCTION" value="COVEREDRATIO" minimum="92%"/>
                 </rule>
             </check>
         </jacoco:report>
+
+        <jar destfile="${zserio_java_runtime.javadocs_jar_file}" update="true">
+            <fileset dir="${zserio_java_runtime.javadocs_dir}">
+                <include name="**"/>
+            </fileset>
+        </jar>
+        <copy file="${zserio_java_runtime.javadocs_jar_file}" todir="${zserio_java_runtime.install_dir}"
+               overwrite="true"/>
     </target>
 
     <target name="test" depends="test.run">
diff --git a/compiler/extensions/java/runtime/src/zserio/runtime/overview.html b/compiler/extensions/java/runtime/src/zserio/runtime/overview.html
@@ -11,5 +11,7 @@
     <li>Debug string output (<code><a class="el" href="zserio/runtime/DebugStringUtil.html">zserio.runtime.DebugStringUtil.java</a></code> module)</li>
 </ul>
 
+<p><a href="coverage/index.html">Test coverage report</a> of this library is available as well.
+
 </body>
 </html>
diff --git a/compiler/extensions/python/runtime/src/zserio/__init__.py b/compiler/extensions/python/runtime/src/zserio/__init__.py
@@ -8,6 +8,8 @@
 
 - Serialization (:obj:`zserio.serialization` module)
 - Debug string output (:obj:`zserio.debugstring` module)
+
+`Test coverage report <coverage/index.html>`_ of this library is available as well.
 """
 
 from zserio.bitbuffer import BitBuffer
diff --git a/scripts/build.sh b/scripts/build.sh
@@ -38,6 +38,7 @@ test_python_runtime()
     ZSERIO_CPP_DIR=$(ls -d1 "${BUILD_DIR}/zserio_cpp/lib"*)
     if [ $? -ne 0 ] ; then
         stderr_echo "Failed to locate C++ runtime binding to Python!"
+        popd > /dev/null
         return 1
     fi
 
@@ -57,6 +58,13 @@ test_python_runtime()
     echo
 
     python -m coverage combine --keep coverage_cpp.data coverage_python.data
+    python -m coverage xml -o "coverage/coverage_report.xml" --omit="*test_object*"
+    local COVERAGE_RESULT=$?
+    if [ ${COVERAGE_RESULT} -ne 0 ] ; then
+        stderr_echo "Running python coverage report failed with return code ${COVERAGE_RESULT}!"
+        popd > /dev/null
+        return 1
+    fi
     python -m coverage html --directory="coverage" --fail-under=100 \
             --omit="*test_object*" --title="Zserio Python Runtime Library"
     local COVERAGE_RESULT=$?
@@ -120,8 +128,12 @@ install_python_runtime()
     local PYTHON_RUNTIME_DOC_BUILD_DIR="${PYTHON_RUNTIME_BUILD_DIR}/doc"
     local ZSERIO_LOGO="${ZSERIO_PROJECT_ROOT}/doc/Zserio.png"
 
-    # build doc
-    echo "Building documentation for Zserio Python runtime library."
+    echo "Purging python distr dir ${PYTHON_RUNTIME_DISTR_DIR}."
+    rm -rf "${PYTHON_RUNTIME_DISTR_DIR}/"
+    mkdir -p "${PYTHON_RUNTIME_DISTR_DIR}"
+
+    # install doc together with test coverage report
+    echo "Installing documentation for Zserio Python runtime library."
     echo
     rm -rf "${PYTHON_RUNTIME_DOC_BUILD_DIR}/"
     mkdir -p "${PYTHON_RUNTIME_DOC_BUILD_DIR}"
@@ -137,22 +149,24 @@ install_python_runtime()
     sphinx-apidoc --module-first --force --separate -o . "${PYTHON_RUNTIME_SOURCES}/zserio/"
     rm modules.rst
     PYTHONPATH="${PYTHON_RUNTIME_SOURCES}" \
-    sphinx-build -Wa -b html -d . -Dhtml_logo="${ZSERIO_LOGO}" . zserio_doc
+    sphinx-build -Wa -b html -d . -Dhtml_logo="${ZSERIO_LOGO}" . "${PYTHON_RUNTIME_DISTR_DIR}/zserio_doc"
     if [ $? -ne 0 ] ; then
         popd > /dev/null
         return 1
     fi
-
     popd > /dev/null
 
+    cp -r "${PYTHON_RUNTIME_BUILD_DIR}/coverage" "${PYTHON_RUNTIME_DISTR_DIR}/zserio_doc/coverage"
+    if [ $? -ne 0 ] ; then
+        stderr_echo "Failed to install documentation!"
+        return 1
+    fi
+    rm -f "${PYTHON_RUNTIME_DISTR_DIR}/zserio_doc/coverage/.gitignore"
+
     echo
     echo "Installing Zserio Python runtime library."
     echo
 
-    echo "Purging python distr dir: ${PYTHON_RUNTIME_DISTR_DIR}"
-    rm -rf "${PYTHON_RUNTIME_DISTR_DIR}/"
-    mkdir -p "${PYTHON_RUNTIME_DISTR_DIR}"
-
     # install sources
     pushd "${PYTHON_RUNTIME_SOURCES}" > /dev/null
 
@@ -168,14 +182,6 @@ install_python_runtime()
 
     popd > /dev/null
 
-    # install doc
-    echo "Installing API documentation"
-    cp -r "${PYTHON_RUNTIME_DOC_BUILD_DIR}/zserio_doc" "${PYTHON_RUNTIME_DISTR_DIR}/"
-    if [ $? -ne 0 ] ; then
-        stderr_echo "Failed to install documentation!"
-        return 1
-    fi
-
     return 0
 }
 
diff --git a/scripts/common_tools.sh b/scripts/common_tools.sh
@@ -724,13 +724,17 @@ compile_cpp_for_target()
     mkdir -p "${BUILD_DIR}"
     pushd "${BUILD_DIR}" > /dev/null
 
-    local CMAKE_BUILD_TARGET=${MAKE_TARGET}
+    if [[ ${MAKE_TARGET} == "install" ]] ; then
+        local CMAKE_BUILD_TARGET="all"
+    else
+        local CMAKE_BUILD_TARGET="${MAKE_TARGET}"
+    fi
 
     # resolve CMake generator
     if [[ ${TARGET} == *"-msvc" ]] ; then
         local CMAKE_BUILD_CONFIG="--config ${BUILD_TYPE}"
         CTEST_ARGS+=("-C ${BUILD_TYPE}")
-        if [[ ${MAKE_TARGET} == "all" ]] ; then
+        if [[ ${CMAKE_BUILD_TARGET} == "all" ]] ; then
             CMAKE_BUILD_TARGET="ALL_BUILD" # all target doesn't exist in MSVC solution
         fi
         local CMAKE_GENERATOR="${MSVC_CMAKE_GENERATOR}";
@@ -782,6 +786,17 @@ compile_cpp_for_target()
         fi
     fi
 
+    # install it running cmake
+    if [[ ${MAKE_TARGET} == "install" ]] ; then
+        "${CMAKE}" --build . --target install ${CMAKE_BUILD_CONFIG} -- ${CMAKE_BUILD_OPTIONS}
+        local CMAKE_RESULT=$?
+        if [ ${CMAKE_RESULT} -ne 0 ] ; then
+            stderr_echo "Running CMake failed with return code ${CMAKE_RESULT}!"
+            popd > /dev/null
+            return 1
+        fi
+    fi
+
     popd > /dev/null
 
     return 0
diff --git a/scripts/release.sh b/scripts/release.sh
@@ -554,14 +554,13 @@ update_web_pages()
     fi
     echo "Done"
 
+    echo -ne "Creating Zserio runtime library GitHub badges..."
+    create_github_badge_jsons "${ZSERIO_PROJECT_ROOT}" "${ZSERIO_VERSION}"
+    echo "Done"
+
     # This is necessary because Jekyll ignores Python runtime doc directories that start with underscores.
     echo -ne "Creating Jekyll configuration file..."
-    echo "theme: jekyll-theme-slate" > "${ZSERIO_PROJECT_ROOT}"/_config.yml
-    echo "exclude: 3rdparty" >> "${ZSERIO_PROJECT_ROOT}"/_config.yml
-    echo "include:" >> "${ZSERIO_PROJECT_ROOT}"/_config.yml
-    echo "  - _images" >> "${ZSERIO_PROJECT_ROOT}"/_config.yml
-    echo "  - _modules" >> "${ZSERIO_PROJECT_ROOT}"/_config.yml
-    echo "  - _static" >> "${ZSERIO_PROJECT_ROOT}"/_config.yml
+    create_jekyll_config_file "${ZSERIO_PROJECT_ROOT}"
     echo "Done"
 
     echo
@@ -590,6 +589,78 @@ update_web_pages()
     return 0
 }
 
+# Create JSON configuration files for all GitHub badges
+create_github_badge_jsons()
+{
+    exit_if_argc_ne $# 2
+    local ZSERIO_PROJECT_ROOT="$1"; shift
+    local ZSERIO_VERSION="$1"; shift
+
+    local CLANG_COVERAGE_DIR="${ZSERIO_PROJECT_ROOT}"/doc/runtime/cpp/coverage/clang
+    local CLANG_LINES_COVERAGE=`cat "${CLANG_COVERAGE_DIR}"/coverage_report.txt | grep TOTAL | \
+            tr -s ' ' | cut -d' ' -f 10`
+    create_github_badge_json "${CLANG_COVERAGE_DIR}"/coverage_github_badge.json \
+            "C++ clang runtime ${ZSERIO_VERSION} coverage" "${CLANG_LINES_COVERAGE}"
+
+    local GCC_COVERAGE_DIR="${ZSERIO_PROJECT_ROOT}"/doc/runtime/cpp/coverage/gcc
+    local GCC_LINES_COVERAGE=`cat "${GCC_COVERAGE_DIR}"/coverage_report.txt | grep lines: | \
+            tr -s ' ' | cut -d' ' -f 2`
+    create_github_badge_json "${GCC_COVERAGE_DIR}"/coverage_github_badge.json \
+            "C++ gcc runtime ${ZSERIO_VERSION} coverage" "${GCC_LINES_COVERAGE}"
+
+    local JAVA_COVERAGE_DIR="${ZSERIO_PROJECT_ROOT}"/doc/runtime/java/coverage
+    local JAVA_COVERAGE_REPORT=`cat "${JAVA_COVERAGE_DIR}"/jacoco_report.xml`
+    local JAVA_LINES_MISSED=`echo ${JAVA_COVERAGE_REPORT##*INSTRUCTION} | cut -d'"' -f3`
+    local JAVA_LINES_COVERED=`echo ${JAVA_COVERAGE_REPORT##*INSTRUCTION} | cut -d'"' -f5`
+    local JAVA_LINES_VALID=$((${JAVA_LINES_COVERED} - ${JAVA_LINES_MISSED}))
+    local JAVA_LINES_COVERAGE=$((10000 * ${JAVA_LINES_VALID} / ${JAVA_LINES_COVERED}))
+    create_github_badge_json "${JAVA_COVERAGE_DIR}"/coverage_github_badge.json \
+            "Java runtime ${ZSERIO_VERSION} coverage" \
+            "${JAVA_LINES_COVERAGE:0:-2}.${JAVA_LINES_COVERAGE: -2}%"
+
+    local PYTHON_COVERAGE_DIR="${ZSERIO_PROJECT_ROOT}"/doc/runtime/python/coverage
+    local PYTHON_LINES_VALID=`cat "${PYTHON_COVERAGE_DIR}"/coverage_report.xml | grep lines-covered | \
+            cut -d' ' -f 4 | cut -d= -f2 | tr -d \"`
+    local PYTHON_LINES_COVERED=`cat "${PYTHON_COVERAGE_DIR}"/coverage_report.xml | grep lines-covered | \
+            cut -d' ' -f 5 | cut -d= -f2 | tr -d \"`
+    local PYTHON_LINES_COVERAGE=$((10000 * ${PYTHON_LINES_VALID} / ${PYTHON_LINES_COVERED}))
+    create_github_badge_json "${PYTHON_COVERAGE_DIR}"/coverage_github_badge.json \
+            "Python runtime ${ZSERIO_VERSION} coverage" \
+            "${PYTHON_LINES_COVERAGE:0:-2}.${PYTHON_LINES_COVERAGE: -2}%"
+}
+
+# Create JSON configuration file for GitHub badge
+create_github_badge_json()
+{
+    exit_if_argc_ne $# 3
+    local BADGE_JSON_FILE="$1"; shift
+    local BADGE_LABEL="$1"; shift
+    local BADGE_MESSAGE="$1"; shift
+
+    cat > "${BADGE_JSON_FILE}" << EOF
+{
+    "schemaVersion": 1,
+    "label": "${BADGE_LABEL}",
+    "message": "${BADGE_MESSAGE}",
+    "color": "green"
+}
+EOF
+}
+
+# Create Jekyll configuration file
+create_jekyll_config_file()
+{
+    exit_if_argc_ne $# 1
+    local ZSERIO_PROJECT_ROOT="$1"; shift
+
+    echo "theme: jekyll-theme-slate" > "${ZSERIO_PROJECT_ROOT}"/_config.yml
+    echo "exclude: 3rdparty" >> "${ZSERIO_PROJECT_ROOT}"/_config.yml
+    echo "include:" >> "${ZSERIO_PROJECT_ROOT}"/_config.yml
+    echo "  - _images" >> "${ZSERIO_PROJECT_ROOT}"/_config.yml
+    echo "  - _modules" >> "${ZSERIO_PROJECT_ROOT}"/_config.yml
+    echo "  - _static" >> "${ZSERIO_PROJECT_ROOT}"/_config.yml
+}
+
 # Print help message.
 print_help()
 {
