Add sparse binding support

vulkano/src/acceleration_structure.rs

@@ -85,7 +85,7 @@
 //! [`WriteDescriptorSet::acceleration_structure`]: crate::descriptor_set::WriteDescriptorSet::acceleration_structure
 
 use crate::{
-    buffer::{BufferUsage, IndexBuffer, Subbuffer},
+    buffer::{BufferCreateFlags, BufferUsage, IndexBuffer, Subbuffer},
     device::{Device, DeviceOwned},
     format::{Format, FormatFeatures},
     instance::InstanceOwnedDebugWrapper,
@@ -379,6 +379,19 @@ impl AccelerationStructureCreateInfo {
             }));
         }
 
+        if buffer
+            .buffer()
+            .flags()
+            .intersects(BufferCreateFlags::SPARSE_RESIDENCY)
+        {
+            return Err(Box::new(ValidationError {
+                context: "buffer.buffer().flags()".into(),
+                problem: "contains `BufferCreateFlags::SPARSE_RESIDENCY`".into(),
+                vuids: &["VUID-VkAccelerationStructureCreateInfoKHR-buffer-03615"],
+                ..Default::default()
+            }));
+        }
+
         // VUID-VkAccelerationStructureCreateInfoKHR-offset-03616
         // Ensured by the definition of `Subbuffer`.
 

vulkano/src/buffer/mod.rs

@@ -377,8 +377,9 @@ impl Buffer {
         allocation_info: AllocationCreateInfo,
         layout: DeviceLayout,
     ) -> Result<Arc<Self>, Validated<AllocateBufferError>> {
-        // TODO: Enable once sparse binding materializes
-        // assert!(!allocate_info.flags.contains(BufferCreateFlags::SPARSE_BINDING));
+        assert!(!create_info
+            .flags
+            .contains(BufferCreateFlags::SPARSE_BINDING));
 
         assert_eq!(
             create_info.size, 0,
@@ -802,25 +803,24 @@ vulkan_bitflags! {
     /// Flags specifying additional properties of a buffer.
     BufferCreateFlags = BufferCreateFlags(u32);
 
-    /* TODO: enable
-    /// The buffer will be backed by sparse memory binding (through queue commands) instead of
-    /// regular binding (through [`bind_memory`]).
+    /// The buffer will be backed by sparse memory binding (through the [`bind_sparse`] queue
+    /// command) instead of regular binding (through [`bind_memory`]).
     ///
     /// The [`sparse_binding`] feature must be enabled on the device.
     ///
+    /// [`bind_sparse`]: crate::device::queue::QueueGuard::bind_sparse
     /// [`bind_memory`]: sys::RawBuffer::bind_memory
     /// [`sparse_binding`]: crate::device::DeviceFeatures::sparse_binding
-    SPARSE_BINDING = SPARSE_BINDING,*/
+    SPARSE_BINDING = SPARSE_BINDING,
 
-    /* TODO: enable
     /// The buffer can be used without being fully resident in memory at the time of use.
     ///
-    /// This requires the `sparse_binding` flag as well.
+    /// This requires the [`BufferCreateFlags::SPARSE_BINDING`] flag as well.
     ///
     /// The [`sparse_residency_buffer`] feature must be enabled on the device.
     ///
     /// [`sparse_residency_buffer`]: crate::device::DeviceFeatures::sparse_residency_buffer
-    SPARSE_RESIDENCY = SPARSE_RESIDENCY,*/
+    SPARSE_RESIDENCY = SPARSE_RESIDENCY,
 
     /* TODO: enable
     /// The buffer's memory can alias with another buffer or a different part of the same buffer.

vulkano/src/buffer/sys.rs

@@ -69,6 +69,11 @@ impl RawBuffer {
             .validate(device)
             .map_err(|err| err.add_context("create_info"))?;
 
+        // TODO: sparse_address_space_size and extended_sparse_address_space_size limits
+        // VUID-vkCreateBuffer-flags-00911
+        // VUID-vkCreateBuffer-flags-09383
+        // VUID-vkCreateBuffer-flags-09384
+
         Ok(())
     }
 
@@ -245,6 +250,7 @@ impl RawBuffer {
     }
 
     /// Binds device memory to this buffer.
+    #[inline]
     pub fn bind_memory(
         self,
         allocation: ResourceMemory,
@@ -261,6 +267,15 @@ impl RawBuffer {
         &self,
         allocation: &ResourceMemory,
     ) -> Result<(), Box<ValidationError>> {
+        if self.flags().intersects(BufferCreateFlags::SPARSE_BINDING) {
+            return Err(Box::new(ValidationError {
+                context: "self.flags()".into(),
+                problem: "contains `BufferCreateFlags::SPARSE_BINDING`".into(),
+                vuids: &["VUID-VkBindBufferMemoryInfo-buffer-01030"],
+                ..Default::default()
+            }));
+        }
+
         assert_ne!(allocation.allocation_type(), AllocationType::NonLinear);
 
         let physical_device = self.device().physical_device();
@@ -277,10 +292,6 @@ impl RawBuffer {
         // VUID-VkBindBufferMemoryInfo-buffer-07459
         // Ensured by taking ownership of `RawBuffer`.
 
-        // VUID-VkBindBufferMemoryInfo-buffer-01030
-        // Currently ensured by not having sparse binding flags, but this needs to be checked once
-        // those are enabled.
-
         // VUID-VkBindBufferMemoryInfo-memoryOffset-01031
         // Assume that `allocation` was created correctly.
 
@@ -511,13 +522,31 @@ impl RawBuffer {
         Ok(Buffer::from_raw(self, BufferMemory::Normal(allocation)))
     }
 
-    /// Assume this buffer has memory bound to it.
+    /// Converts a raw buffer into a full buffer without binding any memory.
     ///
     /// # Safety
     ///
-    /// - The buffer must have memory bound to it.
+    /// If `self.flags()` does not contain [`BufferCreateFlags::SPARSE_BINDING`]:
+    ///
+    /// - The buffer must already have a suitable memory allocation bound to it.
+    ///
+    /// If `self.flags()` does contain [`BufferCreateFlags::SPARSE_BINDING`]:
+    ///
+    /// - If `self.flags()` does not contain [`BufferCreateFlags::SPARSE_RESIDENCY`], then the
+    ///   buffer must be fully bound with memory before its memory is accessed by the device.
+    /// - If `self.flags()` contains [`BufferCreateFlags::SPARSE_RESIDENCY`], then you must ensure
+    ///   that any reads from the buffer are prepared to handle unexpected or inconsistent values,
+    ///   as determined by the [`Properties::residency_non_resident_strict`] device property.
+    ///
+    /// [`Properties::residency_non_resident_strict`]: crate::device::Properties::residency_non_resident_strict
     pub unsafe fn assume_bound(self) -> Buffer {
-        Buffer::from_raw(self, BufferMemory::External)
+        let memory = if self.flags().intersects(BufferCreateFlags::SPARSE_BINDING) {
+            BufferMemory::Sparse
+        } else {
+            BufferMemory::External
+        };
+
+        Buffer::from_raw(self, memory)
     }
 
     /// Returns the memory requirements for this buffer.
@@ -678,45 +707,6 @@ impl BufferCreateInfo {
             }));
         }
 
-        /* Enable when sparse binding is properly handled
-        if let Some(sparse_level) = sparse {
-            if !device.enabled_features().sparse_binding {
-                return Err(Box::new(ValidationError {
-                    context: "sparse".into(),
-                    problem: "is `Some`".into(),
-                    requires_one_of: RequiresOneOf(&[RequiresAllOf(&[Requires::Feature(
-                        "sparse_binding",
-                    )])]),
-                    vuids: &["VUID-VkBufferCreateInfo-flags-00915"],
-                }));
-            }
-
-            if sparse_level.sparse_residency && !device.enabled_features().sparse_residency_buffer {
-                return Err(Box::new(ValidationError {
-                    context: "sparse".into(),
-                    problem: "contains `BufferCreateFlags::SPARSE_RESIDENCY`".into(),
-                    requires_one_of: RequiresOneOf(&[RequiresAllOf(&[Requires::Feature(
-                        "sparse_residency_buffer",
-                    )])]),
-                    vuids: &["VUID-VkBufferCreateInfo-flags-00916"],
-                }));
-            }
-
-            if sparse_level.sparse_aliased && !device.enabled_features().sparse_residency_aliased {
-                return Err(Box::new(ValidationError {
-                    context: "sparse".into(),
-                    problem: "contains `BufferCreateFlags::SPARSE_ALIASED`".into(),
-                    requires_one_of: RequiresOneOf(&[RequiresAllOf(&[Requires::Feature(
-                        "sparse_residency_aliased",
-                    )])]),
-                    vuids: &["VUID-VkBufferCreateInfo-flags-00917"],
-                }));
-            }
-
-            // TODO:
-            // VUID-VkBufferCreateInfo-flags-00918
-        }*/
-
         match sharing {
             Sharing::Exclusive => (),
             Sharing::Concurrent(queue_family_indices) => {
@@ -761,6 +751,43 @@ impl BufferCreateInfo {
             }
         }
 
+        if flags.intersects(BufferCreateFlags::SPARSE_BINDING) {
+            if !device.enabled_features().sparse_binding {
+                return Err(Box::new(ValidationError {
+                    context: "flags".into(),
+                    problem: "contains `BufferCreateFlags::SPARSE_BINDING`".into(),
+                    requires_one_of: RequiresOneOf(&[RequiresAllOf(&[Requires::DeviceFeature(
+                        "sparse_binding",
+                    )])]),
+                    vuids: &["VUID-VkBufferCreateInfo-flags-00915"],
+                }));
+            }
+        }
+
+        if flags.intersects(BufferCreateFlags::SPARSE_RESIDENCY) {
+            if !flags.intersects(BufferCreateFlags::SPARSE_BINDING) {
+                return Err(Box::new(ValidationError {
+                    context: "flags".into(),
+                    problem: "contains `BufferCreateFlags::SPARSE_RESIDENCY`, but does not also \
+                        contain `BufferCreateFlags::SPARSE_BINDING`"
+                        .into(),
+                    vuids: &["VUID-VkBufferCreateInfo-flags-00918"],
+                    ..Default::default()
+                }));
+            }
+
+            if !device.enabled_features().sparse_residency_buffer {
+                return Err(Box::new(ValidationError {
+                    context: "flags".into(),
+                    problem: "contains `BufferCreateFlags::SPARSE_RESIDENCY`".into(),
+                    requires_one_of: RequiresOneOf(&[RequiresAllOf(&[Requires::DeviceFeature(
+                        "sparse_residency_buffer",
+                    )])]),
+                    vuids: &["VUID-VkBufferCreateInfo-flags-00916"],
+                }));
+            }
+        }
+
         if let Some(max_buffer_size) = device.physical_device().properties().max_buffer_size {
             if size > max_buffer_size {
                 return Err(Box::new(ValidationError {

vulkano/src/device/queue.rs

@@ -3,7 +3,7 @@ use crate::{
     command_buffer::{CommandBufferSubmitInfo, SemaphoreSubmitInfo, SubmitInfo},
     instance::{debug::DebugUtilsLabel, InstanceOwnedDebugWrapper},
     macros::vulkan_bitflags,
-    memory::BindSparseInfo,
+    memory::sparse::BindSparseInfo,
     swapchain::{PresentInfo, SwapchainPresentInfo},
     sync::{fence::Fence, PipelineStages},
     Requires, RequiresAllOf, RequiresOneOf, Validated, ValidationError, Version, VulkanError,
@@ -112,6 +112,10 @@ impl Queue {
             _state: self.state.lock(),
         })
     }
+
+    fn queue_family_properties(&self) -> &QueueFamilyProperties {
+        &self.device().physical_device().queue_family_properties()[self.queue_family_index as usize]
+    }
 }
 
 impl Drop for Queue {
@@ -216,6 +220,85 @@ impl QueueGuard<'_> {
             .map_err(VulkanError::from)
     }
 
+    /// Bind or unbind memory to resources with sparse memory.
+    ///
+    /// # Safety
+    ///
+    /// For every semaphore in the `wait_semaphores` elements of every `bind_infos` element:
+    /// - The semaphore must be kept alive while the command is being executed.
+    /// - The semaphore must be already in the signaled state, or there must be a previously
+    ///   submitted operation that will signal it.
+    /// - When the wait operation is executed, no other queue must be waiting on the same
+    ///   semaphore.
+    ///
+    /// For every element in the `buffer_binds`, `image_opaque_binds` and `image_binds`
+    /// elements of every `bind_infos` element:
+    /// - The buffers and images must be kept alive while the command is being executed.
+    /// - The memory allocations must be kept alive while they are bound to a buffer or image.
+    /// - Access to the affected regions of each buffer or image must be synchronized.
+    ///
+    /// For every semaphore in the `signal_semaphores` elements of every `bind_infos` element:
+    /// - The semaphore must be kept alive while the command is being executed.
+    /// - When the signal operation is executed, the semaphore must be in the unsignaled state.
+    ///
+    /// If `fence` is `Some`:
+    /// - The fence must be kept alive while the command is being executed.
+    /// - The fence must be unsignaled and must not be associated with any other command that is
+    ///   still executing.
+    #[inline]
+    pub unsafe fn bind_sparse(
+        &mut self,
+        bind_infos: &[BindSparseInfo],
+        fence: Option<&Arc<Fence>>,
+    ) -> Result<(), Validated<VulkanError>> {
+        self.validate_bind_sparse(bind_infos, fence)?;
+
+        Ok(self.bind_sparse_unchecked(bind_infos, fence)?)
+    }
+
+    fn validate_bind_sparse(
+        &self,
+        bind_infos: &[BindSparseInfo],
+        fence: Option<&Arc<Fence>>,
+    ) -> Result<(), Box<ValidationError>> {
+        if !self
+            .queue
+            .queue_family_properties()
+            .queue_flags
+            .intersects(QueueFlags::SPARSE_BINDING)
+        {
+            return Err(Box::new(ValidationError {
+                problem: "the queue family of this queue does not support \
+                    sparse binding operations"
+                    .into(),
+                vuids: &["VUID-vkQueueBindSparse-queuetype"],
+                ..Default::default()
+            }));
+        }
+
+        let device = self.queue.device();
+
+        if let Some(fence) = fence {
+            // VUID-vkQueueBindSparse-commonparent
+            assert_eq!(device, fence.device());
+        }
+
+        for (index, bind_sparse_info) in bind_infos.iter().enumerate() {
+            bind_sparse_info
+                .validate(device)
+                .map_err(|err| err.add_context(format!("bind_infos[{}]", index)))?;
+        }
+
+        // unsafe
+        // VUID-vkQueueBindSparse-fence-01113
+        // VUID-vkQueueBindSparse-fence-01114
+        // VUID-vkQueueBindSparse-pSignalSemaphores-01115
+        // VUID-vkQueueBindSparse-pWaitSemaphores-01116
+        // VUID-vkQueueBindSparse-pWaitSemaphores-03245
+
+        Ok(())
+    }
+
     #[cfg_attr(not(feature = "document_unchecked"), doc(hidden))]
     pub unsafe fn bind_sparse_unchecked(
         &mut self,