From: Joshua Simmons Date: Tue, 11 Jul 2023 20:27:23 +0000 (+0200) Subject: narcissus-gpu: Tidy up transient allocators X-Git-Url: https://git.nega.tv//gitweb.cgi?a=commitdiff_plain;h=459ab3c69bccc86efe58674612c9709d840f3ab6;p=josh%2Fnarcissus narcissus-gpu: Tidy up transient allocators --- diff --git a/libs/narcissus-gpu/src/backend/vulkan/mod.rs b/libs/narcissus-gpu/src/backend/vulkan/mod.rs index cecf0bc..3822310 100644 --- a/libs/narcissus-gpu/src/backend/vulkan/mod.rs +++ b/libs/narcissus-gpu/src/backend/vulkan/mod.rs @@ -893,6 +893,35 @@ struct VulkanBoundPipeline { pipeline_bind_point: vk::PipelineBindPoint, } +#[derive(Clone, Copy)] +enum VulkanTransientBufferType { + Index, + Storage, + Uniform, +} + +impl VulkanTransientBufferType { + const fn count() -> usize { + 3 + } + + const fn index(self) -> usize { + match self { + VulkanTransientBufferType::Index => 0, + VulkanTransientBufferType::Storage => 1, + VulkanTransientBufferType::Uniform => 2, + } + } + + const fn usage(self) -> vk::BufferUsageFlags { + match self { + VulkanTransientBufferType::Index => vk::BufferUsageFlags::INDEX_BUFFER, + VulkanTransientBufferType::Storage => vk::BufferUsageFlags::STORAGE_BUFFER, + VulkanTransientBufferType::Uniform => vk::BufferUsageFlags::UNIFORM_BUFFER, + } + } +} + #[derive(Clone)] struct VulkanTransientBuffer { buffer: vk::Buffer, @@ -900,7 +929,6 @@ struct VulkanTransientBuffer { } struct VulkanTransientAllocator { - usage: vk::BufferUsageFlags, min_align: u64, offset: u64, current: Option, @@ -908,9 +936,8 @@ struct VulkanTransientAllocator { } impl VulkanTransientAllocator { - fn new(usage: vk::BufferUsageFlags, min_align: u64) -> Self { + fn new(min_align: u64) -> Self { Self { - usage, min_align, offset: 0, current: None, @@ -922,45 +949,6 @@ impl VulkanTransientAllocator { self.current = None; self.offset = 0; } - - fn alloc<'a>(&mut self, device: &VulkanDevice, size: u64, align: u64) -> TransientBuffer<'a> { - assert!(size <= VULKAN_CONSTANTS.transient_buffer_size); - assert!( - align != 0 - && align.is_power_of_two() - && align <= VULKAN_CONSTANTS.transient_buffer_max_align - ); - - let align = align.max(self.min_align); - - if self.offset < size || self.current.is_none() { - let transient_buffer = - device.request_transient_buffer(VULKAN_CONSTANTS.transient_buffer_size, self.usage); - - self.used_buffers.push(transient_buffer.clone()); - self.current = Some(transient_buffer); - self.offset = VULKAN_CONSTANTS.transient_buffer_size; - } - - let current = self.current.as_ref().unwrap(); - - self.offset = self.offset.wrapping_sub(size); - self.offset &= !(align - 1); - - TransientBuffer { - ptr: NonNull::new( - current - .memory - .mapped_ptr() - .wrapping_offset(self.offset as isize), - ) - .unwrap(), - len: size as usize, - buffer: current.buffer.as_raw(), - offset: self.offset, - _phantom: &PhantomData, - } - } } struct VulkanCmdBuffer { @@ -989,9 +977,7 @@ struct VulkanCmdBufferPool { struct VulkanPerThread { cmd_buffer_pool: RefCell, descriptor_pool: Cell, - transient_index_allocator: RefCell, - transient_storage_allocator: RefCell, - transient_uniform_allocator: RefCell, + transient_allocators: RefCell<[VulkanTransientAllocator; VulkanTransientBufferType::count()]>, arena: Arena, } @@ -1060,9 +1046,8 @@ pub(crate) struct VulkanDevice { recycled_semaphores: Mutex>, recycled_descriptor_pools: Mutex>, - recycled_transient_index_buffers: Mutex>, - recycled_transient_storage_buffers: Mutex>, - recycled_transient_uniform_buffers: Mutex>, + recycled_transient_buffers: + [Mutex>; VulkanTransientBufferType::count()], allocators: [Option>; vk::MAX_MEMORY_TYPES as usize], @@ -1346,17 +1331,14 @@ impl VulkanDevice { command_buffers: Vec::new(), next_free_index: 0, }; - let transient_index_allocator = - VulkanTransientAllocator::new(vk::BufferUsageFlags::INDEX_BUFFER, 1); + let transient_index_allocator = VulkanTransientAllocator::new(1); let transient_storage_allocator = VulkanTransientAllocator::new( - vk::BufferUsageFlags::STORAGE_BUFFER, physical_device_properties .properties .limits .min_storage_buffer_offset_alignment, ); let transient_uniform_allocator = VulkanTransientAllocator::new( - vk::BufferUsageFlags::UNIFORM_BUFFER, physical_device_properties .properties .limits @@ -1366,9 +1348,11 @@ impl VulkanDevice { VulkanPerThread { cmd_buffer_pool: RefCell::new(cmd_buffer_pool), descriptor_pool: Cell::new(vk::DescriptorPool::null()), - transient_index_allocator: RefCell::new(transient_index_allocator), - transient_storage_allocator: RefCell::new(transient_storage_allocator), - transient_uniform_allocator: RefCell::new(transient_uniform_allocator), + transient_allocators: RefCell::new([ + transient_index_allocator, + transient_storage_allocator, + transient_uniform_allocator, + ]), arena: Arena::new(), } }); @@ -1431,9 +1415,7 @@ impl VulkanDevice { recycled_semaphores: default(), recycled_descriptor_pools: default(), - recycled_transient_index_buffers: default(), - recycled_transient_storage_buffers: default(), - recycled_transient_uniform_buffers: default(), + recycled_transient_buffers: default(), allocators, @@ -2446,12 +2428,13 @@ impl Device for VulkanDevice { size: usize, align: usize, ) -> TransientBuffer<'a> { - let frame = self.frame(frame); - let per_thread = frame.per_thread.get(thread_token); - per_thread - .transient_index_allocator - .borrow_mut() - .alloc(self, size as u64, align as u64) + self.request_transient_buffer( + frame, + thread_token, + size as u64, + align as u64, + VulkanTransientBufferType::Index, + ) } fn request_transient_storage_buffer<'a>( @@ -2461,12 +2444,13 @@ impl Device for VulkanDevice { size: usize, align: usize, ) -> TransientBuffer<'a> { - let frame = self.frame(frame); - let per_thread = frame.per_thread.get(thread_token); - per_thread - .transient_storage_allocator - .borrow_mut() - .alloc(self, size as u64, align as u64) + self.request_transient_buffer( + frame, + thread_token, + size as u64, + align as u64, + VulkanTransientBufferType::Storage, + ) } fn request_transient_uniform_buffer<'a>( @@ -2476,12 +2460,13 @@ impl Device for VulkanDevice { size: usize, align: usize, ) -> TransientBuffer<'a> { - let frame = self.frame(frame); - let per_thread = frame.per_thread.get(thread_token); - per_thread - .transient_uniform_allocator - .borrow_mut() - .alloc(self, size as u64, align as u64) + self.request_transient_buffer( + frame, + thread_token, + size as u64, + align as u64, + VulkanTransientBufferType::Uniform, + ) } fn create_cmd_buffer<'a, 'thread>( @@ -3211,43 +3196,21 @@ impl Device for VulkanDevice { cmd_buffer_pool.next_free_index = 0; } - let used_index_buffers = - &mut per_thread.transient_index_allocator.get_mut().used_buffers; - - if !used_index_buffers.is_empty() { - self.recycled_transient_index_buffers - .lock() - .extend(used_index_buffers.drain(..)) - } - - per_thread.transient_index_allocator.get_mut().reset(); - - let used_storage_buffers = &mut per_thread - .transient_storage_allocator + for (index, transient_allocator) in per_thread + .transient_allocators .get_mut() - .used_buffers; - - if !used_storage_buffers.is_empty() { - self.recycled_transient_storage_buffers - .lock() - .extend(used_storage_buffers.drain(..)) - } - - per_thread.transient_storage_allocator.get_mut().reset(); - - let used_uniform_buffers = &mut per_thread - .transient_uniform_allocator - .get_mut() - .used_buffers; + .iter_mut() + .enumerate() + { + if !transient_allocator.used_buffers.is_empty() { + self.recycled_transient_buffers[index] + .lock() + .extend(transient_allocator.used_buffers.drain(..)) + } - if !used_uniform_buffers.is_empty() { - self.recycled_transient_uniform_buffers - .lock() - .extend(used_uniform_buffers.drain(..)) + transient_allocator.reset() } - per_thread.transient_uniform_allocator.get_mut().reset(); - per_thread.arena.reset() } @@ -3344,28 +3307,72 @@ impl Device for VulkanDevice { } impl VulkanDevice { - fn request_transient_buffer( + fn request_transient_buffer<'a>( &self, + frame: &'a Frame, + thread_token: &'a ThreadToken, size: u64, - usage: vk::BufferUsageFlags, + align: u64, + transient_buffer_type: VulkanTransientBufferType, + ) -> TransientBuffer<'a> { + let frame = self.frame(frame); + let per_thread = frame.per_thread.get(thread_token); + let mut transient_allocators = per_thread.transient_allocators.borrow_mut(); + let allocator = &mut transient_allocators[transient_buffer_type.index()]; + + assert!(size <= VULKAN_CONSTANTS.transient_buffer_size); + assert!( + align != 0 + && align.is_power_of_two() + && align <= VULKAN_CONSTANTS.transient_buffer_max_align + ); + + let align = align.max(allocator.min_align); + + if allocator.offset < size || allocator.current.is_none() { + let transient_buffer = self.allocate_transient_buffer(transient_buffer_type); + allocator.used_buffers.push(transient_buffer.clone()); + allocator.current = Some(transient_buffer); + allocator.offset = VULKAN_CONSTANTS.transient_buffer_size; + } + + let current = allocator.current.as_ref().unwrap(); + + allocator.offset = allocator.offset.wrapping_sub(size); + allocator.offset &= !(align - 1); + + TransientBuffer { + ptr: NonNull::new( + current + .memory + .mapped_ptr() + .wrapping_offset(allocator.offset as isize), + ) + .unwrap(), + len: size as usize, + buffer: current.buffer.as_raw(), + offset: allocator.offset, + _phantom: &PhantomData, + } + } + + fn allocate_transient_buffer( + &self, + transient_buffer_type: VulkanTransientBufferType, ) -> VulkanTransientBuffer { - if let Some(transient_buffer) = if usage == vk::BufferUsageFlags::INDEX_BUFFER { - self.recycled_transient_index_buffers.lock().pop_back() - } else if usage == vk::BufferUsageFlags::STORAGE_BUFFER { - self.recycled_transient_storage_buffers.lock().pop_back() - } else if usage == vk::BufferUsageFlags::UNIFORM_BUFFER { - self.recycled_transient_uniform_buffers.lock().pop_back() - } else { - panic!() - } { + if let Some(transient_buffer) = self.recycled_transient_buffers + [transient_buffer_type.index()] + .lock() + .pop_back() + { return transient_buffer; } let queue_family_indices = &[self.universal_queue_family_index]; let create_info = vk::BufferCreateInfo { - size, - usage, + size: VULKAN_CONSTANTS.transient_buffer_size, + usage: transient_buffer_type.usage(), queue_family_indices: queue_family_indices.into(), sharing_mode: vk::SharingMode::Exclusive, ..default() @@ -3459,51 +3466,18 @@ impl Drop for VulkanDevice { .destroy_command_pool(device, cmd_buffer_pool.command_pool, None) } - for VulkanTransientBuffer { buffer, memory: _ } in per_thread - .transient_index_allocator - .get_mut() - .used_buffers - .iter() - { - unsafe { self.device_fn.destroy_buffer(device, *buffer, None) } - } - - for VulkanTransientBuffer { buffer, memory: _ } in per_thread - .transient_storage_allocator - .get_mut() - .used_buffers - .iter() - { - unsafe { self.device_fn.destroy_buffer(device, *buffer, None) } - } - - for VulkanTransientBuffer { buffer, memory: _ } in per_thread - .transient_uniform_allocator - .get_mut() - .used_buffers - .iter() - { - unsafe { self.device_fn.destroy_buffer(device, *buffer, None) } + for transient_allocator in per_thread.transient_allocators.get_mut() { + for buffer in transient_allocator.used_buffers.iter() { + unsafe { self.device_fn.destroy_buffer(device, buffer.buffer, None) } + } } } } - for VulkanTransientBuffer { buffer, memory: _ } in - self.recycled_transient_index_buffers.get_mut() - { - unsafe { self.device_fn.destroy_buffer(device, *buffer, None) } - } - - for VulkanTransientBuffer { buffer, memory: _ } in - self.recycled_transient_storage_buffers.get_mut() - { - unsafe { self.device_fn.destroy_buffer(device, *buffer, None) } - } - - for VulkanTransientBuffer { buffer, memory: _ } in - self.recycled_transient_uniform_buffers.get_mut() - { - unsafe { self.device_fn.destroy_buffer(device, *buffer, None) } + for recycled_transient_buffers in &mut self.recycled_transient_buffers { + for buffer in recycled_transient_buffers.get_mut() { + unsafe { self.device_fn.destroy_buffer(device, buffer.buffer, None) } + } } for buffer in self.buffer_pool.get_mut().values() {