External Texture implementation for Vulkan

Add support for the new createTextureExternalImage2R
API instead of createTextureExternalImageR.

Add the support to import AHardwareBuffer objects
into Vulkan in Android.

Author: rafadevai

filament/backend/CMakeLists.txt

@@ -233,7 +233,10 @@ if (FILAMENT_SUPPORTS_VULKAN)
     elseif (APPLE OR IOS)
         list(APPEND SRCS src/vulkan/platform/VulkanPlatformApple.mm)
     elseif (ANDROID)
-        list(APPEND SRCS src/vulkan/platform/VulkanPlatformAndroid.cpp)
+        list(APPEND SRCS
+            include/backend/platforms/VulkanPlatformAndroid.h
+            src/vulkan/platform/VulkanPlatformAndroid.cpp
+        )
     endif()
 endif()
 

filament/backend/include/backend/platforms/VulkanPlatform.h

@@ -347,18 +347,20 @@ class VulkanPlatform : public Platform, utils::PrivateImplementation<VulkanPlatf
          */
         uint32_t memoryTypeBits;
     };
-    virtual ExternalImageMetadata getExternalImageMetadata(void* externalImage);
+    virtual ExternalImageMetadata getExternalImageMetadata(ExternalImageHandleRef externalImage);
 
     using ImageData = std::pair<VkImage, VkDeviceMemory>;
-    virtual ImageData createExternalImage(void* externalImage,
+    virtual ImageData createExternalImageData(ExternalImageHandleRef externalImage,
             const ExternalImageMetadata& metadata);
 
 private:
     static ExtensionSet getSwapchainInstanceExtensions();
-    static ExternalImageMetadata getExternalImageMetadataImpl(void* externalImage,
+
+    static ExternalImageMetadata getExternalImageMetadataImpl(ExternalImageHandleRef externalImage,
             VkDevice device);
-    static ImageData createExternalImageImpl(void* externalImage, VkDevice device,
-            const VkAllocationCallbacks* allocator, const ExternalImageMetadata& metadata);
+
+    static ImageData createExternalImageDataImpl(ExternalImageHandleRef externalImage,
+            VkDevice device, const ExternalImageMetadata& metadata);
 
     // Platform dependent helper methods
     using SurfaceBundle = std::tuple<VkSurfaceKHR, VkExtent2D>;

filament/backend/include/backend/platforms/VulkanPlatformAndroid.h

@@ -0,0 +1,43 @@
+/*
+ * Copyright (C) 2025 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef TNT_FILAMENT_BACKEND_PLATFORMS_VULKAN_PLATFORM_ANDROID_H
+#define TNT_FILAMENT_BACKEND_PLATFORMS_VULKAN_PLATFORM_ANDROID_H
+
+#include <backend/Platform.h>
+
+#include <android/hardware_buffer.h>
+
+namespace filament::backend::fvkandroid {
+
+struct ExternalImageVulkanAndroid : public Platform::ExternalImage {
+    AHardwareBuffer *aHardwareBuffer = nullptr;
+    bool sRGB = false;
+    unsigned int width;   // Texture width
+    unsigned int height;  // Texture height
+    TextureFormat format; // Texture format
+    TextureUsage usage;   // Texture usage flags
+
+protected:
+    ~ExternalImageVulkanAndroid() override;
+};
+
+Platform::ExternalImageHandle UTILS_PUBLIC createExternalImage(
+    AHardwareBuffer const *buffer, bool sRGB) noexcept;
+
+} // namespace filament::backend::fvkandroid
+
+#endif // TNT_FILAMENT_BACKEND_PLATFORMS_VULKAN_PLATFORM_ANDROID_H

filament/backend/src/vulkan/VulkanDriver.cpp

@@ -560,14 +560,6 @@ void VulkanDriver::createTextureExternalImage2R(Handle<HwTexture> th,
         Platform::ExternalImageHandleRef externalImage) {
     FVK_SYSTRACE_SCOPE();
 
-    // FIXME: implement createTextureExternalImage2R
-}
-
-void VulkanDriver::createTextureExternalImageR(Handle<HwTexture> th, backend::SamplerType target,
-        backend::TextureFormat format, uint32_t width, uint32_t height, backend::TextureUsage usage,
-        void* externalImage) {
-    FVK_SYSTRACE_SCOPE();
-
     const auto& metadata = mPlatform->getExternalImageMetadata(externalImage);
     if (metadata.isProtected) {
         usage |= backend::TextureUsage::PROTECTED;
@@ -577,15 +569,23 @@ void VulkanDriver::createTextureExternalImageR(Handle<HwTexture> th, backend::Sa
     assert_invariant(height == metadata.height);
     assert_invariant(fvkutils::getVkFormat(format) == metadata.format);
 
-    const auto& data = mPlatform->createExternalImage(externalImage, metadata);
+    const auto& data = mPlatform->createExternalImageData(externalImage, metadata);
 
-    auto texture = resource_ptr<VulkanTexture>::make(&mResourceManager, th, mPlatform->getDevice(),
-        mAllocator, &mResourceManager, &mCommands, data.first, data.second, metadata.format,
-        1, metadata.width, metadata.height, /*depth=*/1, usage, mStagePool);
+    auto texture = resource_ptr<VulkanTexture>::make(&mResourceManager, th,
+            mPlatform->getDevice(), mAllocator, &mResourceManager, &mCommands, data.first, data.second, metadata.format,
+            1, metadata.width, metadata.height, /*depth=*/1, usage, mStagePool);
 
     texture.inc();
 }
 
+void VulkanDriver::createTextureExternalImageR(Handle<HwTexture> th, backend::SamplerType target,
+        backend::TextureFormat format, uint32_t width, uint32_t height, backend::TextureUsage usage,
+        void* externalImage) {
+    FVK_SYSTRACE_SCOPE();
+
+    // not supported in this backend
+}
+
 void VulkanDriver::createTextureExternalImagePlaneR(Handle<HwTexture> th,
         backend::TextureFormat format, uint32_t width, uint32_t height, backend::TextureUsage usage,
         void* image, uint32_t plane) {

filament/backend/src/vulkan/platform/VulkanPlatform.cpp

@@ -978,13 +978,13 @@ VkQueue VulkanPlatform::getProtectedGraphicsQueue() const noexcept {
 }
 
 VulkanPlatform::ExternalImageMetadata VulkanPlatform::getExternalImageMetadata(
-        void* externalImage) {
+        ExternalImageHandleRef externalImage) {
     return getExternalImageMetadataImpl(externalImage, mImpl->mDevice);
 }
 
-VulkanPlatform::ImageData VulkanPlatform::createExternalImage(void* externalImage,
-            const ExternalImageMetadata& metadata) {
-    return createExternalImageImpl(externalImage, mImpl->mDevice, nullptr, metadata);
+VulkanPlatform::ImageData VulkanPlatform::createExternalImageData(
+        ExternalImageHandleRef externalImage, const ExternalImageMetadata& metadata) {
+    return createExternalImageDataImpl(externalImage, mImpl->mDevice, metadata);
 }
 
 #undef SWAPCHAIN_RET_FUNC

filament/backend/src/vulkan/platform/VulkanPlatformAndroid.cpp

@@ -15,6 +15,9 @@
  */
 #include <backend/platforms/VulkanPlatform.h>
 
+#include <backend/DriverEnums.h>
+#include <backend/platforms/VulkanPlatformAndroid.h>
+
 #include "vulkan/VulkanConstants.h"
 
 #include <utils/Panic.h>
@@ -24,17 +27,43 @@
 #include <android/hardware_buffer.h>
 #include <android/native_window.h>
 
+#include <utility>
+
 using namespace bluevk;
 
 namespace filament::backend {
 
 namespace {
-void getVKFormatAndUsage(const AHardwareBuffer_Desc& desc, VkFormat& format,
-        VkImageUsageFlags& usage, bool& isProtected) {
+
+VkFormat transformVkFormat(VkFormat format, bool sRGB) {
+    if (!sRGB) {
+        return format;
+    }    
+    
+    switch (format) {
+        case VK_FORMAT_R8G8B8A8_UNORM:
+            format = VK_FORMAT_R8G8B8A8_SRGB;
+            break;
+        case VK_FORMAT_R8G8B8_UNORM:
+            format = VK_FORMAT_R8G8B8_SRGB;
+            break;
+        default:
+            break;
+    }
+
+    return format;
+}
+
+bool isProtectedFromUsage(uint64_t usage) {
+    return (usage & AHARDWAREBUFFER_USAGE_PROTECTED_CONTENT) ? true : false;
+}
+
+std::pair<VkFormat, VkImageUsageFlags> getVKFormatAndUsage(const AHardwareBuffer_Desc& desc, bool sRGB) {
+    VkFormat format = VK_FORMAT_UNDEFINED;
+    VkImageUsageFlags usage = 0;
     // Refer to "11.2.17. External Memory Handle Types" in the spec, and
     // Tables 13/14 for how the following derivation works.
     bool isDepthFormat = false;
-    isProtected = false;
     switch (desc.format) {
         case AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM:
             format = VK_FORMAT_R8G8B8A8_UNORM;
@@ -82,6 +111,8 @@ void getVKFormatAndUsage(const AHardwareBuffer_Desc& desc, VkFormat& format,
             format = VK_FORMAT_UNDEFINED;
     }
 
+    format = transformVkFormat(format, sRGB);
+
     // The following only concern usage flags derived from Table 14.
     usage = 0;
     if (desc.usage & AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE) {
@@ -98,16 +129,89 @@ void getVKFormatAndUsage(const AHardwareBuffer_Desc& desc, VkFormat& format,
     if (desc.usage & AHARDWAREBUFFER_USAGE_GPU_DATA_BUFFER) {
         usage = VK_IMAGE_USAGE_STORAGE_BIT;
     }
-    if (desc.usage & AHARDWAREBUFFER_USAGE_PROTECTED_CONTENT) {
-        isProtected = true;
+
+    return {format, usage};
+}
+
+TextureFormat mapToFilamentFormat(unsigned int format, bool isSrgbTransfer) noexcept {
+    if (isSrgbTransfer) {
+        switch (format) {
+            case AHARDWAREBUFFER_FORMAT_R8G8B8_UNORM:
+                return TextureFormat::SRGB8;
+            default:
+                return TextureFormat::SRGB8_A8;
+        }
+    }
+    switch (format) {
+        case AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM:
+        case AHARDWAREBUFFER_FORMAT_R8G8B8X8_UNORM:
+            return TextureFormat::RGBA8;
+        case AHARDWAREBUFFER_FORMAT_R8G8B8_UNORM:
+        case AHARDWAREBUFFER_FORMAT_Y8Cb8Cr8_420:
+            return TextureFormat::RGB8;
+        case AHARDWAREBUFFER_FORMAT_R5G6B5_UNORM:
+            return TextureFormat::RGB565;
+        case AHARDWAREBUFFER_FORMAT_R16G16B16A16_FLOAT:
+            return TextureFormat::RGBA16F;
+        case AHARDWAREBUFFER_FORMAT_R10G10B10A2_UNORM:
+            return TextureFormat::RGB10_A2;
+        case AHARDWAREBUFFER_FORMAT_D16_UNORM:
+            return TextureFormat::DEPTH16;
+        case AHARDWAREBUFFER_FORMAT_D24_UNORM:
+            return TextureFormat::DEPTH24;
+        case AHARDWAREBUFFER_FORMAT_D24_UNORM_S8_UINT:
+            return TextureFormat::DEPTH24_STENCIL8;
+        case AHARDWAREBUFFER_FORMAT_D32_FLOAT:
+            return TextureFormat::DEPTH32F;
+        case AHARDWAREBUFFER_FORMAT_D32_FLOAT_S8_UINT:
+            return TextureFormat::DEPTH32F_STENCIL8;
+        case AHARDWAREBUFFER_FORMAT_S8_UINT:
+            return TextureFormat::STENCIL8;
+        case AHARDWAREBUFFER_FORMAT_R8_UNORM:
+            return TextureFormat::R8;
+        default:
+            return TextureFormat::UNUSED;
+    }
+}
+
+TextureUsage mapToFilamentUsage(unsigned int usage, TextureFormat format) noexcept {
+    TextureUsage usageFlags = TextureUsage::NONE;
+
+    if (usage & AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE) {
+        usageFlags |= TextureUsage::SAMPLEABLE;
+    }
+
+    if (usage & AHARDWAREBUFFER_USAGE_GPU_FRAMEBUFFER) {
+        if (isDepthFormat(format)) {
+            usageFlags |= TextureUsage::DEPTH_ATTACHMENT;
+        }
+        if (isStencilFormat(format)) {
+            usageFlags |= TextureUsage::STENCIL_ATTACHMENT;
+        }
+        if (isColorFormat(format)) {
+            usageFlags |= TextureUsage::COLOR_ATTACHMENT;
+        }
+    }
+
+    if (usage & AHARDWAREBUFFER_USAGE_GPU_DATA_BUFFER) {
+        usageFlags |= TextureUsage::UPLOADABLE;
+    }
+
+    if (usage & AHARDWAREBUFFER_USAGE_PROTECTED_CONTENT) {
+        usageFlags |= TextureUsage::PROTECTED;
+    }
+
+    if (usageFlags == TextureUsage::NONE) {
+        usageFlags = TextureUsage::COLOR_ATTACHMENT | TextureUsage::SAMPLEABLE;
     }
+
+    return usageFlags;
 }
 
-VulkanPlatform::ImageData allocateExternalImage(void* externalBuffer,
+VulkanPlatform::ImageData allocateExternalImage(AHardwareBuffer* buffer,
         VkDevice device, const VkAllocationCallbacks* allocator,
         VulkanPlatform::ExternalImageMetadata const& metadata) {
     VulkanPlatform::ImageData data;
-    AHardwareBuffer* buffer = static_cast<AHardwareBuffer*>(externalBuffer);
 
     // if external format we need to specifiy it in the allocation
     const bool useExternalFormat = metadata.format == VK_FORMAT_UNDEFINED;
@@ -167,20 +271,50 @@ VulkanPlatform::ImageData allocateExternalImage(void* externalBuffer,
     return data;
 }
 
-}// namespace
+} // namespace
+
+namespace fvkandroid {
+
+ExternalImageVulkanAndroid::~ExternalImageVulkanAndroid() = default;
+
+Platform::ExternalImageHandle createExternalImage(AHardwareBuffer const *buffer,
+        bool sRGB) noexcept {
+    if (__builtin_available(android 26, *)) {
+        AHardwareBuffer_Desc hardwareBufferDescription = {};
+        AHardwareBuffer_describe(buffer, &hardwareBufferDescription);
+
+        auto* const p = new (std::nothrow) ExternalImageVulkanAndroid;
+        p->aHardwareBuffer = const_cast<AHardwareBuffer*>(buffer);
+        p->sRGB = sRGB;
+        p->height = hardwareBufferDescription.height;
+        p->width = hardwareBufferDescription.width;
+        TextureFormat textureFormat = mapToFilamentFormat(hardwareBufferDescription.format, sRGB);
+        p->format = textureFormat;
+        p->usage = mapToFilamentUsage(hardwareBufferDescription.usage, textureFormat);
+        return Platform::ExternalImageHandle{p};
+    }
+    
+    return Platform::ExternalImageHandle{};
+}
+
+}
+
+VulkanPlatform::ExternalImageMetadata VulkanPlatform::getExternalImageMetadataImpl(
+        ExternalImageHandleRef externalImage, VkDevice device) {
+    auto const *fvkExternalImage =
+            static_cast<fvkandroid::ExternalImageVulkanAndroid const*>(externalImage.get());
 
-VulkanPlatform::ExternalImageMetadata VulkanPlatform::getExternalImageMetadataImpl(void* externalImage,
-        VkDevice device) {
     ExternalImageMetadata metadata;
-    AHardwareBuffer* buffer = static_cast<AHardwareBuffer*>(externalImage);
+    AHardwareBuffer* buffer = fvkExternalImage->aHardwareBuffer;
     if (__builtin_available(android 26, *)) {
         AHardwareBuffer_Desc bufferDesc;
         AHardwareBuffer_describe(buffer, &bufferDesc);
         metadata.width = bufferDesc.width;
         metadata.height = bufferDesc.height;
         metadata.layers = bufferDesc.layers;
-
-        getVKFormatAndUsage(bufferDesc, metadata.format, metadata.usage, metadata.isProtected);
+        metadata.isProtected = isProtectedFromUsage(bufferDesc.usage);
+        std::tie(metadata.format, metadata.usage) = getVKFormatAndUsage(bufferDesc,
+                fvkExternalImage->sRGB);
     }
     // In the unprotected case add R/W capabilities
     if (metadata.isProtected == false) {
@@ -197,20 +331,26 @@ VulkanPlatform::ExternalImageMetadata VulkanPlatform::getExternalImageMetadataIm
     };
     VkResult result = vkGetAndroidHardwareBufferPropertiesANDROID(device, buffer, &properties);
     FILAMENT_CHECK_POSTCONDITION(result == VK_SUCCESS)
-            << "vkGetAndroidHardwareBufferProperties failed with error="
-            << static_cast<int32_t>(result);
-    FILAMENT_CHECK_POSTCONDITION(metadata.format == formatInfo.format)
-            << "mismatched image format for external image (AHB)";
+        << "vkGetAndroidHardwareBufferProperties failed with error="
+        << static_cast<int32_t>(result);
+                            
+    VkFormat bufferPropertiesFormat = transformVkFormat(formatInfo.format, fvkExternalImage->sRGB);
+    FILAMENT_CHECK_POSTCONDITION(metadata.format == bufferPropertiesFormat)
+            << "mismatched image format( " << metadata.format << ") and queried format("
+            << bufferPropertiesFormat << ") for external image (AHB)";
     metadata.externalFormat = formatInfo.externalFormat;
     metadata.allocationSize = properties.allocationSize;
     metadata.memoryTypeBits = properties.memoryTypeBits;
     return metadata;
 }
 
-VulkanPlatform::ImageData VulkanPlatform::createExternalImageImpl(void* externalImage,
-        VkDevice device, const VkAllocationCallbacks* allocator,
+VulkanPlatform::ImageData VulkanPlatform::createExternalImageDataImpl(
+        ExternalImageHandleRef externalImage, VkDevice device,
         const ExternalImageMetadata& metadata) {
-    ImageData data = allocateExternalImage(externalImage, device, allocator, metadata);
+    auto const* fvkExternalImage =
+            static_cast<fvkandroid::ExternalImageVulkanAndroid const*>(externalImage.get());
+    ImageData data = allocateExternalImage(fvkExternalImage->aHardwareBuffer,
+            device, VKALLOC, metadata);
     VkResult result = vkBindImageMemory(device, data.first, data.second, 0);
     FILAMENT_CHECK_POSTCONDITION(result == VK_SUCCESS)
             << "vkBindImageMemory error=" << static_cast<int32_t>(result);