[Impeller] Add fenced command buffer that waits on-submit (#37750)

* [Impeller] Add fenced command buffer that waits on-submit

this isn't ideal for performance long term but aligns well with what
the other backends do. This also lets us remove the dependence on frame
numbers for renderpasses setting up for implementing MSAA.

* fix licenses

ci/licenses_golden/licenses_flutter

@@ -1460,10 +1460,14 @@ FILE: ../../../flutter/impeller/renderer/backend/vulkan/command_pool_vk.cc
 FILE: ../../../flutter/impeller/renderer/backend/vulkan/command_pool_vk.h
 FILE: ../../../flutter/impeller/renderer/backend/vulkan/context_vk.cc
 FILE: ../../../flutter/impeller/renderer/backend/vulkan/context_vk.h
+FILE: ../../../flutter/impeller/renderer/backend/vulkan/deletion_queue_vk.cc
+FILE: ../../../flutter/impeller/renderer/backend/vulkan/deletion_queue_vk.h
 FILE: ../../../flutter/impeller/renderer/backend/vulkan/descriptor_pool_vk.cc
 FILE: ../../../flutter/impeller/renderer/backend/vulkan/descriptor_pool_vk.h
 FILE: ../../../flutter/impeller/renderer/backend/vulkan/device_buffer_vk.cc
 FILE: ../../../flutter/impeller/renderer/backend/vulkan/device_buffer_vk.h
+FILE: ../../../flutter/impeller/renderer/backend/vulkan/fenced_command_buffer_vk.cc
+FILE: ../../../flutter/impeller/renderer/backend/vulkan/fenced_command_buffer_vk.h
 FILE: ../../../flutter/impeller/renderer/backend/vulkan/formats_vk.cc
 FILE: ../../../flutter/impeller/renderer/backend/vulkan/formats_vk.h
 FILE: ../../../flutter/impeller/renderer/backend/vulkan/pipeline_library_vk.cc

impeller/renderer/backend/vulkan/BUILD.gn

@@ -18,10 +18,14 @@ impeller_component("vulkan") {
     "command_pool_vk.h",
     "context_vk.cc",
     "context_vk.h",
+    "deletion_queue_vk.cc",
+    "deletion_queue_vk.h",
     "descriptor_pool_vk.cc",
     "descriptor_pool_vk.h",
     "device_buffer_vk.cc",
     "device_buffer_vk.h",
+    "fenced_command_buffer_vk.cc",
+    "fenced_command_buffer_vk.h",
     "formats_vk.cc",
     "formats_vk.h",
     "pipeline_library_vk.cc",

impeller/renderer/backend/vulkan/command_buffer_vk.cc

@@ -4,11 +4,13 @@
 
 #include "impeller/renderer/backend/vulkan/command_buffer_vk.h"
 
+#include <memory>
 #include <utility>
 
 #include "flutter/fml/logging.h"
 #include "impeller/base/validation.h"
 #include "impeller/renderer/backend/vulkan/context_vk.h"
+#include "impeller/renderer/backend/vulkan/fenced_command_buffer_vk.h"
 #include "impeller/renderer/backend/vulkan/formats_vk.h"
 #include "impeller/renderer/backend/vulkan/render_pass_vk.h"
 #include "impeller/renderer/command_buffer.h"
@@ -17,37 +19,30 @@
 namespace impeller {
 
 std::shared_ptr<CommandBufferVK> CommandBufferVK::Create(
-    const std::weak_ptr<const Context>& context,
+    const std::weak_ptr<const Context>& context_arg,
     vk::Device device,
-    vk::CommandPool command_pool,
-    SurfaceProducerVK* surface_producer) {
-  vk::CommandBufferAllocateInfo allocate_info;
-  allocate_info.setLevel(vk::CommandBufferLevel::ePrimary);
-  allocate_info.setCommandBufferCount(1);
-  allocate_info.setCommandPool(command_pool);
-
-  auto res = device.allocateCommandBuffersUnique(allocate_info);
-  if (res.result != vk::Result::eSuccess) {
-    VALIDATION_LOG << "Failed to allocate command buffer: "
-                   << vk::to_string(res.result);
+    vk::CommandPool command_pool) {
+  if (auto context = context_arg.lock()) {
+    auto queue =
+        reinterpret_cast<const ContextVK*>(context.get())->GetGraphicsQueue();
+    auto fenced_command_buffer =
+        std::make_shared<FencedCommandBufferVK>(device, queue, command_pool);
+    return std::make_shared<CommandBufferVK>(context, device, command_pool,
+                                             fenced_command_buffer);
+  } else {
     return nullptr;
   }
-
-  vk::UniqueCommandBuffer cmd = std::move(res.value[0]);
-  return std::make_shared<CommandBufferVK>(context, device, surface_producer,
-                                           command_pool, std::move(cmd));
 }
 
-CommandBufferVK::CommandBufferVK(std::weak_ptr<const Context> context,
-                                 vk::Device device,
-                                 SurfaceProducerVK* surface_producer,
-                                 vk::CommandPool command_pool,
-                                 vk::UniqueCommandBuffer command_buffer)
+CommandBufferVK::CommandBufferVK(
+    std::weak_ptr<const Context> context,
+    vk::Device device,
+    vk::CommandPool command_pool,
+    std::shared_ptr<FencedCommandBufferVK> command_buffer)
     : CommandBuffer(std::move(context)),
       device_(device),
       command_pool_(command_pool),
-      command_buffer_(std::move(command_buffer)),
-      surface_producer_(surface_producer) {
+      fenced_command_buffer_(std::move(command_buffer)) {
   is_valid_ = true;
 }
 
@@ -56,7 +51,7 @@ CommandBufferVK::~CommandBufferVK() = default;
 void CommandBufferVK::SetLabel(const std::string& label) const {
   if (auto context = context_.lock()) {
     reinterpret_cast<const ContextVK*>(context.get())
-        ->SetDebugName(*command_buffer_, label);
+        ->SetDebugName(fenced_command_buffer_->Get(), label);
   }
 }
 
@@ -65,16 +60,12 @@ bool CommandBufferVK::IsValid() const {
 }
 
 bool CommandBufferVK::OnSubmitCommands(CompletionCallback callback) {
-  // TODO(https://github.com/flutter/flutter/issues/112387)
-  // This needs to be the place where the command buffer, renderpass,
-  // and the various descriptor sets in use by the command buffer are
-  // disposed of.
-
+  bool submit = fenced_command_buffer_->Submit();
   if (callback) {
-    callback(CommandBuffer::Status::kCompleted);
+    callback(submit ? CommandBuffer::Status::kCompleted
+                    : CommandBuffer::Status::kError);
   }
-
-  return true;
+  return submit;
 }
 
 std::shared_ptr<RenderPass> CommandBufferVK::OnCreateRenderPass(
@@ -115,17 +106,16 @@ std::shared_ptr<RenderPass> CommandBufferVK::OnCreateRenderPass(
   render_pass_create.setSubpassCount(1);
   render_pass_create.setPSubpasses(&subpass_desc);
 
-  auto render_pass_create_res =
-      device_.createRenderPassUnique(render_pass_create);
+  auto render_pass_create_res = device_.createRenderPass(render_pass_create);
   if (render_pass_create_res.result != vk::Result::eSuccess) {
     VALIDATION_LOG << "Failed to create render pass: "
                    << vk::to_string(render_pass_create_res.result);
     return nullptr;
   }
 
-  return std::make_shared<RenderPassVK>(
-      context_, device_, std::move(target), std::move(command_buffer_),
-      std::move(render_pass_create_res.value), surface_producer_);
+  vk::RenderPass render_pass = render_pass_create_res.value;
+  return std::make_shared<RenderPassVK>(context_, device_, std::move(target),
+                                        fenced_command_buffer_, render_pass);
 }
 
 std::shared_ptr<BlitPass> CommandBufferVK::OnCreateBlitPass() const {

impeller/renderer/backend/vulkan/command_buffer_vk.h

@@ -5,6 +5,7 @@
 #pragma once
 
 #include "flutter/fml/macros.h"
+#include "impeller/renderer/backend/vulkan/fenced_command_buffer_vk.h"
 #include "impeller/renderer/backend/vulkan/surface_producer_vk.h"
 #include "impeller/renderer/backend/vulkan/vk.h"
 #include "impeller/renderer/command_buffer.h"
@@ -16,14 +17,12 @@ class CommandBufferVK final : public CommandBuffer {
   static std::shared_ptr<CommandBufferVK> Create(
       const std::weak_ptr<const Context>& context,
       vk::Device device,
-      vk::CommandPool command_pool,
-      SurfaceProducerVK* surface_producer);
+      vk::CommandPool command_pool);
 
   CommandBufferVK(std::weak_ptr<const Context> context,
                   vk::Device device,
-                  SurfaceProducerVK* surface_producer,
                   vk::CommandPool command_pool,
-                  vk::UniqueCommandBuffer command_buffer);
+                  std::shared_ptr<FencedCommandBufferVK> command_buffer);
 
   // |CommandBuffer|
   ~CommandBufferVK() override;
@@ -33,9 +32,8 @@ class CommandBufferVK final : public CommandBuffer {
 
   vk::Device device_;
   vk::CommandPool command_pool_;
-  vk::UniqueCommandBuffer command_buffer_;
   vk::UniqueRenderPass render_pass_;
-  SurfaceProducerVK* surface_producer_;
+  std::shared_ptr<FencedCommandBufferVK> fenced_command_buffer_;
   bool is_valid_ = false;
 
   // |CommandBuffer|

impeller/renderer/backend/vulkan/context_vk.cc

@@ -21,6 +21,7 @@
 #include "impeller/renderer/backend/vulkan/allocator_vk.h"
 #include "impeller/renderer/backend/vulkan/capabilities_vk.h"
 #include "impeller/renderer/backend/vulkan/command_buffer_vk.h"
+#include "impeller/renderer/backend/vulkan/deletion_queue_vk.h"
 #include "impeller/renderer/backend/vulkan/formats_vk.h"
 #include "impeller/renderer/backend/vulkan/surface_producer_vk.h"
 #include "impeller/renderer/backend/vulkan/swapchain_details_vk.h"
@@ -492,7 +493,7 @@ ContextVK::ContextVK(
       device_->getQueue(transfer_queue->family, transfer_queue->index);
   graphics_command_pool_ =
       CommandPoolVK::Create(*device_, graphics_queue->index);
-  descriptor_pool_ = std::make_shared<DescriptorPoolVK>(*device_);
+
   is_valid_ = true;
 }
 
@@ -525,8 +526,7 @@ std::shared_ptr<WorkQueue> ContextVK::GetWorkQueue() const {
 
 std::shared_ptr<CommandBuffer> ContextVK::CreateCommandBuffer() const {
   return CommandBufferVK::Create(weak_from_this(), *device_,
-                                 graphics_command_pool_->Get(),
-                                 surface_producer_.get());
+                                 graphics_command_pool_->Get());
 }
 
 vk::Instance ContextVK::GetInstance() const {
@@ -589,8 +589,8 @@ bool ContextVK::SupportsOffscreenMSAA() const {
   return true;
 }
 
-std::shared_ptr<DescriptorPoolVK> ContextVK::GetDescriptorPool() const {
-  return descriptor_pool_;
+std::unique_ptr<DescriptorPoolVK> ContextVK::CreateDescriptorPool() const {
+  return std::make_unique<DescriptorPoolVK>(*device_);
 }
 
 PixelFormat ContextVK::GetColorAttachmentPixelFormat() const {
@@ -601,4 +601,8 @@ const BackendFeatures& ContextVK::GetBackendFeatures() const {
   return kModernBackendFeatures;
 }
 
+vk::Queue ContextVK::GetGraphicsQueue() const {
+  return graphics_queue_;
+}
+
 }  // namespace impeller

impeller/renderer/backend/vulkan/context_vk.h

@@ -11,6 +11,7 @@
 #include "flutter/fml/mapping.h"
 #include "impeller/base/backend_cast.h"
 #include "impeller/renderer/backend/vulkan/command_pool_vk.h"
+#include "impeller/renderer/backend/vulkan/deletion_queue_vk.h"
 #include "impeller/renderer/backend/vulkan/descriptor_pool_vk.h"
 #include "impeller/renderer/backend/vulkan/pipeline_library_vk.h"
 #include "impeller/renderer/backend/vulkan/sampler_library_vk.h"
@@ -85,12 +86,14 @@ class ContextVK final : public Context, public BackendCast<ContextVK, Context> {
 
   std::unique_ptr<Surface> AcquireSurface(size_t current_frame);
 
-  std::shared_ptr<DescriptorPoolVK> GetDescriptorPool() const;
+  std::unique_ptr<DescriptorPoolVK> CreateDescriptorPool() const;
 
 #ifdef FML_OS_ANDROID
   vk::UniqueSurfaceKHR CreateAndroidSurface(ANativeWindow* window) const;
 #endif  // FML_OS_ANDROID
 
+  vk::Queue GetGraphicsQueue() const;
+
  private:
   std::shared_ptr<fml::ConcurrentTaskRunner> worker_task_runner_;
   vk::UniqueInstance instance_;
@@ -111,7 +114,6 @@ class ContextVK final : public Context, public BackendCast<ContextVK, Context> {
   std::unique_ptr<CommandPoolVK> graphics_command_pool_;
   std::unique_ptr<SurfaceProducerVK> surface_producer_;
   std::shared_ptr<WorkQueue> work_queue_;
-  std::shared_ptr<DescriptorPoolVK> descriptor_pool_;
   bool is_valid_ = false;
 
   ContextVK(

impeller/renderer/backend/vulkan/deletion_queue_vk.cc

@@ -0,0 +1,27 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "impeller/renderer/backend/vulkan/deletion_queue_vk.h"
+
+namespace impeller {
+
+DeletionQueueVK::DeletionQueueVK() = default;
+
+DeletionQueueVK::~DeletionQueueVK() {
+  Flush();
+}
+
+void DeletionQueueVK::Flush() {
+  for (auto it = deletors_.rbegin(); it != deletors_.rend(); ++it) {
+    (*it)();
+  }
+
+  deletors_.clear();
+}
+
+void DeletionQueueVK::Push(Deletor&& deletor) {
+  deletors_.push_back(std::move(deletor));
+}
+
+}  // namespace impeller

impeller/renderer/backend/vulkan/deletion_queue_vk.h

@@ -0,0 +1,32 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#pragma once
+
+#include <deque>
+#include <functional>
+
+#include "flutter/fml/macros.h"
+
+namespace impeller {
+
+class DeletionQueueVK {
+ public:
+  using Deletor = std::function<void()>;
+
+  explicit DeletionQueueVK();
+
+  ~DeletionQueueVK();
+
+  void Flush();
+
+  void Push(Deletor&& deletor);
+
+ private:
+  std::deque<Deletor> deletors_;
+
+  FML_DISALLOW_COPY_AND_ASSIGN(DeletionQueueVK);
+};
+
+}  // namespace impeller

impeller/renderer/backend/vulkan/descriptor_pool_vk.cc

@@ -10,7 +10,7 @@
 
 namespace impeller {
 
-DescriptorPoolVK::DescriptorPoolVK(vk::Device device) : device_(device) {
+DescriptorPoolVK::DescriptorPoolVK(vk::Device device) {
   constexpr size_t kPoolSize = 1024;
 
   std::vector<vk::DescriptorPoolSize> pool_sizes = {
@@ -48,10 +48,6 @@ vk::DescriptorPool DescriptorPoolVK::GetPool() {
   return pool_;
 }
 
-DescriptorPoolVK::~DescriptorPoolVK() {
-  if (is_valid_) {
-    device_.destroyDescriptorPool(pool_);
-  }
-}
+DescriptorPoolVK::~DescriptorPoolVK() = default;
 
 }  // namespace impeller

impeller/renderer/backend/vulkan/descriptor_pool_vk.h

@@ -22,7 +22,6 @@ class DescriptorPoolVK {
   vk::DescriptorPool GetPool();
 
  private:
-  vk::Device device_;
   vk::DescriptorPool pool_;
   bool is_valid_ = false;