Beau null test driver

TESTING_test_apps.md

@@ -36,3 +36,14 @@ Use of the *TestAppBase* is optional.
 
 Test apps are built as part of the default build CMAKE build process. In order to stop test apps from building, set the
 **GFXRECON_INCLUDE_TEST_APPS** CMake variable to OFF, e.g. provide `-DGFXRECON_INCLUDE_TEST_APPS=OFF` in your cmake command line.
+
+## **Test App Verification**
+
+To run the test apps and validate output against known good '.gfxr' files, build the project using ./scripts/build.py,
+and then run the test script from within the 'test' install directory.
+
+|Operating System|Test Directory|Test Script|
+|---------------|---------------|------------|
+|Windows|build/windows/x64/output/test|run-tests.ps1|
+|Linux|build/linux/x64/output/test|run-tests.sh|
+|MacOs|build/darwin/universal/output/test|run-tests_macos.sh|

cmake/FindGoogleTest.cmake

@@ -0,0 +1,8 @@
+include(FetchContent)
+
+FetchContent_Declare(
+        googletest
+        GIT_REPOSITORY https://github.com/google/googletest.git
+        GIT_TAG v1.15.2
+)
+FetchContent_MakeAvailable(googletest)

cmake/FindSDL3.cmake

@@ -3,6 +3,9 @@ include(FetchContent)
 set(SDL_GIT "https://github.com/libsdl-org/SDL.git")
 set(SDL_GIT_TAG "preview-3.1.3")
 message(STATUS "Fetching SDL3 files from ${SDL_GIT} ${SDL_GIT_TAG}")
+
+set(SDL_STATIC ON)
+
 FetchContent_Declare(
         sdl
         GIT_REPOSITORY ${SDL_GIT}

layer/json/VkLayer_gfxreconstruct.json.in

@@ -18,6 +18,9 @@
                 "entrypoints": [ "vkGetPhysicalDeviceToolPropertiesEXT" ]
             }
         ],
+        "disable_environment": {
+            "GFXRECON_DISABLE": ""
+        },
         "features": {
             "presets": [
                 {

test/CMakeLists.txt

@@ -26,6 +26,46 @@
 
 
 cmake_minimum_required(VERSION 3.24.0)
-add_subdirectory(test_apps)
 
-# TODO: runner goes here
+if (NOT DEFINED ENV{GFXRECON_NO_TEST_APPS})
+    set(GFXRECON_INSTALL_TESTDIR ${CMAKE_INSTALL_PREFIX}/test)
+
+    add_subdirectory(icd)
+    add_subdirectory(test_apps)
+
+    enable_testing()
+    set(INSTALL_GTEST OFF)
+    find_package(GoogleTest REQUIRED)
+    include(GoogleTest)
+    add_executable(gfxrecon-testapp-runner
+            test_cases/triangle.cpp
+            test_cases/multisample-depth.cpp
+            test_cases/shader-objects.cpp
+            test_cases/pipeline-binaries.cpp
+            verify-gfxr.cpp
+    )
+    add_dependencies(gfxrecon-testapp-runner gfxrecon-testapps VkICD_mock_icd gfxrecon-convert VkLayer_gfxreconstruct)
+    target_include_directories(gfxrecon-testapp-runner PRIVATE ${CMAKE_CURRENT_LIST_DIR})
+    target_link_libraries(gfxrecon-testapp-runner
+            nlohmann_json::nlohmann_json
+            GTest::gtest GTest::gtest_main)
+    gtest_discover_tests(gfxrecon-testapp-runner)
+
+    install(TARGETS gfxrecon-testapp-runner RUNTIME DESTINATION ${GFXRECON_INSTALL_TESTDIR})
+    install(DIRECTORY known_good DESTINATION ${GFXRECON_INSTALL_TESTDIR})
+    install(TARGETS gfxrecon-convert RUNTIME DESTINATION ${GFXRECON_INSTALL_TESTDIR})
+    if (APPLE)
+        install(PROGRAMS run-tests_macos.sh DESTINATION ${GFXRECON_INSTALL_TESTDIR})
+        install(FILES $<TARGET_FILE_DIR:VkLayer_gfxreconstruct>/staging-json/VkLayer_gfxreconstruct.json DESTINATION ${GFXRECON_INSTALL_TESTDIR}/vulkan/explicit_layer.d)
+        install(TARGETS VkLayer_gfxreconstruct RUNTIME DESTINATION ${GFXRECON_INSTALL_TESTDIR}/vulkan/explicit_layer.d LIBRARY DESTINATION ${GFXRECON_INSTALL_TESTDIR}/vulkan/explicit_layer.d)
+    elseif (UNIX)
+        install(PROGRAMS run-tests.sh DESTINATION ${GFXRECON_INSTALL_TESTDIR})
+        install(FILES $<TARGET_FILE_DIR:VkLayer_gfxreconstruct>/staging-json/VkLayer_gfxreconstruct.json DESTINATION ${GFXRECON_INSTALL_TESTDIR}/vulkan/explicit_layer.d)
+        install(TARGETS VkLayer_gfxreconstruct RUNTIME DESTINATION ${GFXRECON_INSTALL_TESTDIR}/vulkan/explicit_layer.d LIBRARY DESTINATION ${GFXRECON_INSTALL_TESTDIR}/vulkan/explicit_layer.d)
+    else ()
+        install(PROGRAMS run-tests.ps1 DESTINATION ${GFXRECON_INSTALL_TESTDIR})
+        install(FILES $<TARGET_FILE_DIR:VkLayer_gfxreconstruct>/VkLayer_gfxreconstruct.json DESTINATION ${GFXRECON_INSTALL_TESTDIR})
+        install(TARGETS VkLayer_gfxreconstruct RUNTIME DESTINATION ${GFXRECON_INSTALL_TESTDIR} LIBRARY DESTINATION ${GFXRECON_INSTALL_TESTDIR})
+    endif ()
+
+endif ()

test/icd/CMakeLists.txt

@@ -0,0 +1,132 @@
+# ~~~
+# Copyright (c) 2014-2018 Valve Corporation
+# Copyright (c) 2014-2018 LunarG, Inc.
+# Copyright (c) 2023-2023 RasterGrid Kft.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ~~~
+
+if (ANDROID OR MINGW)
+    return()
+endif()
+
+# These variables enable downstream users to customize the CMake targets
+# based on the target API variant (e.g. Vulkan SC)
+set(MOCK_ICD_NAME VkICD_mock_icd)
+set(GENERATED generated)
+
+option(BUILD_MOCK_ANDROID_SUPPORT "Build with Android Platform headers" OFF)
+
+if(WIN32)
+    add_definitions(-DVK_USE_PLATFORM_WIN32_KHR -DVK_USE_PLATFORM_WIN32_KHX -DWIN32_LEAN_AND_MEAN)
+elseif(APPLE)
+    add_definitions(-DVK_USE_PLATFORM_MACOS_MVK)
+elseif(BUILD_MOCK_ANDROID_SUPPORT)
+    add_definitions(-DVK_USE_PLATFORM_ANDROID_KHR)
+elseif(CMAKE_SYSTEM_NAME MATCHES "Linux|BSD|GNU")
+    if(BUILD_WSI_XCB_SUPPORT)
+        add_definitions(-DVK_USE_PLATFORM_XCB_KHR -DVK_USE_PLATFORM_XCB_KHX)
+    endif()
+
+    if(BUILD_WSI_XLIB_SUPPORT)
+        add_definitions(-DVK_USE_PLATFORM_XLIB_KHR -DVK_USE_PLATFORM_XLIB_KHX -DVK_USE_PLATFORM_XLIB_XRANDR_EXT)
+    endif()
+
+    if(BUILD_WSI_WAYLAND_SUPPORT)
+        add_definitions(-DVK_USE_PLATFORM_WAYLAND_KHR -DVK_USE_PLATFORM_WAYLAND_KHX)
+    endif()
+else()
+    message(FATAL_ERROR "Unsupported Platform!")
+endif()
+
+add_library(VkICD_mock_icd MODULE)
+target_sources(VkICD_mock_icd PRIVATE mock_icd.cpp)
+target_link_libraries(VkICD_mock_icd PRIVATE Vulkan::Headers)
+
+target_include_directories(VkICD_mock_icd PUBLIC
+    ${GENERATED}
+    .
+)
+
+if(${CMAKE_CXX_COMPILER_ID} MATCHES "GNU|Clang")
+    target_compile_options(VkICD_mock_icd PRIVATE
+        -Wpointer-arith
+        -Wno-unused-function
+        -Wno-sign-compare
+    )
+endif()
+
+if(MSVC)
+    target_compile_options(VkICD_mock_icd PRIVATE /bigobj)
+    target_compile_definitions(VkICD_mock_icd PRIVATE _CRT_SECURE_NO_WARNINGS)
+    target_link_options(VkICD_mock_icd PRIVATE /DEF:${CMAKE_CURRENT_SOURCE_DIR}/${MOCK_ICD_NAME}.def)
+else()
+    if(APPLE)
+        set_target_properties(VkICD_mock_icd PROPERTIES SUFFIX ".dylib")
+    endif()
+    message(DEBUG "Mock ICD Functions are exported via EXPORT")
+endif()
+
+set_target_properties(VkICD_mock_icd PROPERTIES OUTPUT_NAME ${MOCK_ICD_NAME})
+
+if (DEFINED GIT_BRANCH_NAME AND DEFINED GIT_TAG_INFO)
+    target_compile_definitions(VkICD_mock_icd PRIVATE GIT_BRANCH_NAME="${GIT_BRANCH_NAME}" GIT_TAG_INFO="${GIT_TAG_INFO}")
+endif()
+
+# There are 2 primary deliverables for the mock driver.
+# - The actual library (lib)VkICD_mock_icd.(dll|so|dylib)
+# - The respective json file, VkICD_mock_icd.json
+# This code generates the appropriate json for both local testing and the installation.
+# NOTE: For WIN32 the JSON and dll MUST be placed in the same location, due to Win32 using a relative path for installation.
+set(INPUT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/${MOCK_ICD_NAME}.json.in")
+set(INTERMEDIATE_FILE "${CMAKE_CURRENT_BINARY_DIR}/json/mock_icd.json")
+set(OUTPUT_FILE_FINAL_NAME "${MOCK_ICD_NAME}.json")
+set(LAYER_INSTALL_DIR ${GFXRECON_INSTALL_TESTDIR}/test_apps)
+if (WIN32)
+    set(LAYER_INSTALL_DIR ${GFXRECON_INSTALL_TESTDIR}/test_apps) # WIN32 expect the dll in the `bin` dir, this matches our WIN32 SDK process
+endif()
+
+if (WIN32)
+    set(JSON_LIBRARY_PATH ".\\\\${MOCK_ICD_NAME}.dll")
+elseif(APPLE)
+    set(JSON_LIBRARY_PATH "./lib${MOCK_ICD_NAME}.dylib")
+else()
+    set(JSON_LIBRARY_PATH "./lib${MOCK_ICD_NAME}.so")
+endif()
+
+configure_file(${INPUT_FILE} ${INTERMEDIATE_FILE} @ONLY)
+
+# To support both multi/single configuration generators just copy the json to the correct directory
+add_custom_command(TARGET VkICD_mock_icd POST_BUILD
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different ${INTERMEDIATE_FILE} $<TARGET_FILE_DIR:VkICD_mock_icd>/${OUTPUT_FILE_FINAL_NAME}
+)
+
+# For UNIX-based systems, `library_path` should not contain a relative path (indicated by "./") before installing to system directories
+# This json isn't used for regular local development, it's used for installation
+if (UNIX)
+    set(UNIX_INTERMEDIATE_FILE "${CMAKE_CURRENT_BINARY_DIR}/json/unix_install_mock_icd.json")
+
+    if(APPLE)
+        set(JSON_LIBRARY_PATH "lib${MOCK_ICD_NAME}.dylib")
+    else()
+        set(JSON_LIBRARY_PATH "lib${MOCK_ICD_NAME}.so")
+    endif()
+
+    configure_file(${INPUT_FILE} ${UNIX_INTERMEDIATE_FILE} @ONLY)
+
+    install(FILES ${UNIX_INTERMEDIATE_FILE} DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/vulkan/icd.d RENAME ${OUTPUT_FILE_FINAL_NAME})
+endif()
+
+install(FILES ${INTERMEDIATE_FILE} DESTINATION ${LAYER_INSTALL_DIR} RENAME ${OUTPUT_FILE_FINAL_NAME})
+
+install(TARGETS VkICD_mock_icd DESTINATION ${LAYER_INSTALL_DIR})

test/icd/README.md

@@ -0,0 +1,80 @@
+# Vulkan Mock ICD
+
+This directory contains a mock ICD driver designed for validation layer testing.
+
+## Introduction
+
+The mock ICD is focused on enabling validation testing apart from an actual device. Because the validation layers
+sit on top of the ICD and don't depend upon the results of Vulkan rendering, they can be tested without having actual
+GPU hardware backing the ICD. The final mock driver will be composed of three main features: a null driver, flexible
+device configuration, and entrypoint tracking & verification.
+
+### Null Driver
+The intial mock driver features just the null driver capability. This allows all of the validation tests to be run
+on a fixed device configuration that is hard-coded into the ICD.
+
+### Entrypoint Tracking & Verification
+Entrypoint tracking and verification will be added to the mock layer as a later feature. The idea is that all expected
+Vulkan function calls and their parameters can be stored in the ICD and then a separate call can be made to verify that
+the exepected calls and parameters actually entered the ICD. This allows verification that the validation layers are
+correctly passing calls and their parameters through to the ICD unchanged.
+
+## Using the Mock ICD
+
+To enable the mock ICD, set VK\_ICD\_FILENAMES environment variable to point to your {BUILD_DIR}/icd/VkICD\_mock\_icd.json.
+
+## Plans
+
+The initial mock ICD is just the null driver which can be used to test validation layers on
+simulated devices. Here's a rough sequence of tasks planned for the mock driver going forward:
+- [X] Get all LVL tests passing on the bare null driver
+- [X] Get failing tests passing
+- [X] Get skipped tests passing as able
+- [ ] Get all LVL tests to run without unexpected errors
+- [X] Develop automated test flow using mock ICD (alternative to or replacement for run\_all\_tests.sh)
+- [ ] Update LVL tests with device dependencies to target specific device profiles
+- [ ] Add entrypoint tracking & verification
+- [ ] Initially track expected calls
+- [ ] Update some tests to verify expected capability
+- [ ] Expand tracking to include parameters
+
+## Beyond Validation Layer Testing
+
+The focus of the mock icd is for validation testing, but the code is available to use and enhance for anyone wishing to apply it for alternative
+purposes.
+With the following enhancements, the mock driver state available to the app should very closely mimic an actual ICD:
+- Update various function return codes
+- Simulated synchronization objects
+- Simulated query with mock data
+- Basic command buffer state tracking to note synch object transitions and query state updates
+
+Beyond that it's certainly possible that the mock icd could be hooked up to a SW renderer and serve as a virtual GPU with complete rendering/compute
+capabilities.
+
+## Status
+
+This is a temporary section used for tracking as the mock icd is being developed. Once all tests are passing with the mock, this section can be removed.
+Currently 333/333 tests are passing with the mock icd, but many passing tests have unexpected validation errors that need to be cleaned up.
+
+### Failing Tests
+
+NONE
+
+### Passing Tests With Unexpected Errors
+
+- VkLayerTest.RenderPassInUseDestroyedSignaled
+- VkLayerTest.RenderPassIncompatible
+
+### Skipped Tests
+
+- VkLayerTest.BindImageInvalidMemoryType
+- VkLayerTest.CreatePipelineBadVertexAttributeFormat
+- VkLayerTest.MiscBlitImageTests
+- VkLayerTest.TemporaryExternalSemaphore
+- VkLayerTest.TemporaryExternalFence
+- VkLayerTest.InvalidBarriers
+- VkLayerTest.CommandQueueFlags
+- VkPositiveLayerTest.TwoQueuesEnsureCorrectRetirementWithWorkStolen
+- VkPositiveLayerTest.ExternalSemaphore
+- VkPositiveLayerTest.ExternalFence
+- VkPositiveLayerTest.ExternalMemory

test/icd/VkICD_mock_icd.def

@@ -0,0 +1,37 @@
+
+;;;; Begin Copyright Notice ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;
+; Copyright (c) 2015-2017 The Khronos Group Inc.
+; Copyright (c) 2015-2017 Valve Corporation
+; Copyright (c) 2015-2017 LunarG, Inc.
+; Copyright (c) 2015-2017 Google Inc.
+;
+; Licensed under the Apache License, Version 2.0 (the "License");
+; you may not use this file except in compliance with the License.
+; You may obtain a copy of the License at
+;
+;     http://www.apache.org/licenses/LICENSE-2.0
+;
+; Unless required by applicable law or agreed to in writing, software
+; distributed under the License is distributed on an "AS IS" BASIS,
+; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+; See the License for the specific language governing permissions and
+; limitations under the License.
+;
+;  Author: Tobin Ehlis <[email protected]>
+;
+;;;;  End Copyright Notice ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+LIBRARY VkICD_mock_icd
+EXPORTS
+vk_icdGetInstanceProcAddr
+vk_icdGetPhysicalDeviceProcAddr
+vk_icdNegotiateLoaderICDInterfaceVersion
+vkDestroySurfaceKHR
+vkGetPhysicalDeviceSurfaceSupportKHR
+vkGetPhysicalDeviceSurfaceCapabilitiesKHR
+vkGetPhysicalDeviceSurfaceFormatsKHR
+vkGetPhysicalDeviceSurfacePresentModesKHR
+vkCreateDisplayPlaneSurfaceKHR
+vkCreateWin32SurfaceKHR
+mockICD_getTestConfig

test/icd/VkICD_mock_icd.json.in

@@ -0,0 +1,7 @@
+{
+    "file_format_version": "1.0.1",
+    "ICD": {
+        "library_path": "@JSON_LIBRARY_PATH@",
+        "api_version": "1.3.296"
+    }
+}

test/icd/generated/.clang-format

@@ -0,0 +1,5 @@
+---
+# Disable clang-format for generated code
+DisableFormat: true
+SortIncludes: false
+...