Move buffers GPU side

src/main.rs

@@ -32,13 +32,13 @@ use simplelog::{CombinedLogger, Config, TermLogger, WriteLogger};
 use vulkano::{
     Validated, Version, VulkanError, VulkanLibrary,
     buffer::{
-        Buffer, BufferContents, BufferCreateInfo, BufferUsage, Subbuffer,
+        BufferContents, BufferUsage, Subbuffer,
         allocator::{SubbufferAllocator, SubbufferAllocatorCreateInfo},
     },
     command_buffer::{
-        AutoCommandBufferBuilder, CommandBufferUsage, CopyBufferInfo, PrimaryAutoCommandBuffer,
+        AutoCommandBufferBuilder, CommandBufferExecFuture, CommandBufferUsage, CopyBufferInfo,
-        PrimaryCommandBufferAbstract, RenderPassBeginInfo, SubpassBeginInfo, SubpassContents,
+        PrimaryAutoCommandBuffer, PrimaryCommandBufferAbstract, RenderPassBeginInfo,
-        allocator::StandardCommandBufferAllocator,
+        SubpassBeginInfo, SubpassContents, allocator::StandardCommandBufferAllocator,
     },
     descriptor_set::{
         DescriptorSet, WriteDescriptorSet, allocator::StandardDescriptorSetAllocator,
@@ -54,9 +54,7 @@ use vulkano::{
         view::{ImageView, ImageViewCreateInfo},
     },
     instance::{Instance, InstanceCreateInfo, InstanceExtensions},
-    memory::allocator::{
+    memory::allocator::{AllocationCreateInfo, MemoryTypeFilter, StandardMemoryAllocator},
-        AllocationCreateInfo, MemoryAllocatePreference, MemoryTypeFilter, StandardMemoryAllocator,
-    },
     pipeline::{
         DynamicState, GraphicsPipeline, Pipeline, PipelineBindPoint, PipelineCreateFlags,
         PipelineLayout, PipelineShaderStageCreateInfo,
@@ -78,7 +76,10 @@ use vulkano::{
         PresentMode, Surface, SurfaceInfo, Swapchain, SwapchainCreateInfo, SwapchainPresentInfo,
         acquire_next_image,
     },
-    sync::{self, GpuFuture, future::FenceSignalFuture},
+    sync::{
+        self, GpuFuture,
+        future::{FenceSignalFuture, NowFuture},
+    },
 };
 use winit::{
     application::ApplicationHandler,
@@ -196,8 +197,9 @@ struct App {
     memory_allocator: Arc<StandardMemoryAllocator>,
     descriptor_set_allocator: Arc<StandardDescriptorSetAllocator>,
     command_buffer_allocator: Arc<StandardCommandBufferAllocator>,
-    uniform_buffer_allocator: SubbufferAllocator,
+    uniform_buffer_allocator: Arc<Mutex<SubbufferAllocator>>,
     block_enable_allocator: Arc<Mutex<SubbufferAllocator>>,
+    host_visible_allocator: Arc<Mutex<SubbufferAllocator>>,
     pipeline_cache: Arc<PipelineCache>,
     draw_gui: bool,
     gstate: GState,
@@ -205,10 +207,10 @@ struct App {
     previous_debug: PreviousDebug,
     cstate: CState,
     time: f32,
-    vertex_buffer: Subbuffer<[IVertex; VERTEX_COUNT]>,
+    vertex_buffer: Subbuffer<[IVertex]>,
     trace_module: Module,
     normals_module: Module,
-    threads: Vec<JoinHandle<()>>,
+    _threads: Vec<JoinHandle<()>>,
     thread_work_creation: mpmc::Sender<WorkItem>,
     thread_work_completion: mpsc::Receiver<WorkComplete>,
     rcx: Arc<Mutex<Option<RenderContext>>>,
@@ -222,25 +224,27 @@ struct ShaderModules {
 }
 
 struct RenderContext {
-    window:             Arc<Window>,
+    window: Arc<Window>,
-    swapchain:          Arc<Swapchain>,
+    swapchain: Arc<Swapchain>,
-    render_pass:        Arc<RenderPass>,
+    render_pass: Arc<RenderPass>,
-    framebuffers:       Vec<Arc<Framebuffer>>,
+    framebuffers: Vec<Arc<Framebuffer>>,
-    color_buffer:       Arc<ImageView>,
+    color_buffer: Arc<ImageView>,
-    material_buffer:    Arc<ImageView>,
+    material_buffer: Arc<ImageView>,
-    ao_r_m_buffer:      Arc<ImageView>,
+    ao_r_m_buffer: Arc<ImageView>,
-    normal_buffer:      Arc<ImageView>,
+    normal_buffer: Arc<ImageView>,
-    depth_buffer:       Arc<ImageView>,
+    depth_buffer: Arc<ImageView>,
-    camera_buffers:     Vec<Subbuffer<Camera>>,
+    camera_buffers: Vec<Subbuffer<Camera>>,
-    lights_buffers:     Vec<Subbuffer<Lights>>,
+    camera_buffers_host_visible: Vec<Subbuffer<Camera>>,
-    shader_modules:     ShaderModules,
+    lights_buffers: Vec<Subbuffer<Lights>>,
-    lighting_pipeline:  Arc<GraphicsPipeline>,
+    lights_buffers_host_visible: Vec<Subbuffer<Lights>>,
-    viewport:           Viewport,
+    shader_modules: ShaderModules,
+    lighting_pipeline: Arc<GraphicsPipeline>,
+    viewport: Viewport,
     recreate_swapchain: bool,
     recreate_pipelines: bool,
     previous_frame_end: Vec<Option<FenceSignalFuture<Box<dyn GpuFuture>>>>,
-    gui:                Gui,
+    gui: Gui,
-    render_start:       Instant,
+    render_start: Instant,
 }
 
 fn app_version() -> Version {
@@ -398,24 +402,32 @@ impl App {
             Default::default(),
         ));
 
-        let uniform_buffer_allocator = SubbufferAllocator::new(
+        let uniform_buffer_allocator = Arc::new(Mutex::new(SubbufferAllocator::new(
             memory_allocator.clone(),
             SubbufferAllocatorCreateInfo {
                 buffer_usage: BufferUsage::UNIFORM_BUFFER
                     | BufferUsage::STORAGE_BUFFER
-                    | BufferUsage::VERTEX_BUFFER,
+                    | BufferUsage::VERTEX_BUFFER
-                memory_type_filter: MemoryTypeFilter::PREFER_DEVICE
+                    | BufferUsage::TRANSFER_DST,
-                    | MemoryTypeFilter::HOST_SEQUENTIAL_WRITE,
+                memory_type_filter: MemoryTypeFilter::PREFER_DEVICE,
                 ..Default::default()
             },
-        );
+        )));
 
         let block_enable_allocator = Arc::new(Mutex::new(SubbufferAllocator::new(
             memory_allocator.clone(),
             SubbufferAllocatorCreateInfo {
-                buffer_usage: BufferUsage::UNIFORM_BUFFER,
+                buffer_usage: BufferUsage::UNIFORM_BUFFER | BufferUsage::TRANSFER_DST,
-                memory_type_filter: MemoryTypeFilter::PREFER_DEVICE
+                memory_type_filter: MemoryTypeFilter::PREFER_DEVICE,
-                    | MemoryTypeFilter::HOST_SEQUENTIAL_WRITE,
+                ..Default::default()
+            },
+        )));
+
+        let host_visible_allocator = Arc::new(Mutex::new(SubbufferAllocator::new(
+            memory_allocator.clone(),
+            SubbufferAllocatorCreateInfo {
+                buffer_usage: BufferUsage::TRANSFER_SRC,
+                memory_type_filter: MemoryTypeFilter::HOST_SEQUENTIAL_WRITE,
                 ..Default::default()
             },
         )));
@@ -441,10 +453,29 @@ impl App {
             .lights
             .push(Light::new([-4., 6., -8.], [8., 4., 1.], 0.05));
 
-        let vertex_buffer: Subbuffer<[IVertex; VERTEX_COUNT]> =
+        let vertex_buffer: Subbuffer<[IVertex]> = uniform_buffer_allocator
-            uniform_buffer_allocator.allocate_sized().unwrap();
+            .lock()
+            .unwrap()
+            .allocate_slice(CUBE_VERTEX.len() as _)
+            .unwrap();
 
-        *vertex_buffer.write().unwrap() = CUBE_VERTEX;
+        let vertex_buffer_visible: Subbuffer<[IVertex]> = host_visible_allocator
+            .lock()
+            .unwrap()
+            .allocate_slice(CUBE_VERTEX.len() as _)
+            .unwrap();
+
+        gpu_upload_slice::<IVertex>(
+            &CUBE_VERTEX,
+            vertex_buffer.clone(),
+            vertex_buffer_visible.clone(),
+            command_buffer_allocator.clone(),
+            transfer_queue.clone(),
+        )
+        .then_signal_fence_and_flush()
+        .unwrap()
+        .wait(None)
+        .unwrap();
 
         let trace_spv_code = include_bytes!("trace.recompiled.spv");
         let trace_spv_code_u32 = trace_spv_code
@@ -487,6 +518,7 @@ impl App {
             descriptor_set_allocator,
             command_buffer_allocator,
             uniform_buffer_allocator,
+            host_visible_allocator,
             block_enable_allocator,
             pipeline_cache,
             csg_count: gstate.csg.len(),
@@ -498,7 +530,7 @@ impl App {
             vertex_buffer,
             trace_module,
             normals_module,
-            threads,
+            _threads: threads,
             thread_work_creation: thread_work_creation_sender,
             thread_work_completion: thread_work_completion_receiver,
             rcx: Arc::new(Mutex::new(None)),
@@ -748,12 +780,29 @@ impl ApplicationHandler for App {
             &self.gstate.debug,
         );
 
-        let camera_buffers: Vec<Subbuffer<Camera>> = (0..swapchain.image_count())
+        let (
-            .map(|_| self.uniform_buffer_allocator.allocate_sized().unwrap())
+            camera_buffers,
-            .collect();
+            camera_buffers_host_visible,
-        let lights_buffers: Vec<Subbuffer<Lights>> = (0..swapchain.image_count())
+            lights_buffers,
-            .map(|_| self.uniform_buffer_allocator.allocate_sized().unwrap())
+            lights_buffers_host_visible,
-            .collect();
+        ) = {
+            let device_lock = self.uniform_buffer_allocator.lock().unwrap();
+            let host_lock = self.host_visible_allocator.lock().unwrap();
+            (
+                (0..swapchain.image_count())
+                    .map(|_| device_lock.allocate_sized().unwrap())
+                    .collect::<Vec<Subbuffer<Camera>>>(),
+                (0..swapchain.image_count())
+                    .map(|_| host_lock.allocate_sized().unwrap())
+                    .collect::<Vec<Subbuffer<Camera>>>(),
+                (0..swapchain.image_count())
+                    .map(|_| device_lock.allocate_sized().unwrap())
+                    .collect::<Vec<Subbuffer<Lights>>>(),
+                (0..swapchain.image_count())
+                    .map(|_| host_lock.allocate_sized().unwrap())
+                    .collect::<Vec<Subbuffer<Lights>>>(),
+            )
+        };
         let previous_frame_end = (0..swapchain.image_count()).map(|_| None).collect();
 
         // Create an egui GUI
@@ -781,7 +830,9 @@ impl ApplicationHandler for App {
             normal_buffer,
             depth_buffer,
             camera_buffers,
+            camera_buffers_host_visible,
             lights_buffers,
+            lights_buffers_host_visible,
             shader_modules,
             lighting_pipeline,
             viewport,
@@ -879,6 +930,7 @@ impl ApplicationHandler for App {
                                 rcx.shader_modules.clone(),
                                 self.previous_debug.clone(),
                                 self.block_enable_allocator.clone(),
+                                self.host_visible_allocator.clone(),
                                 self.descriptor_set_allocator.clone(),
                                 rcx.swapchain.image_count(),
                                 DEFAULT_SUBDIVISION,
@@ -952,7 +1004,11 @@ impl App {
         }
     }
 
-    fn update_camera_uniform(&self, rcx: &mut RenderContext, index: usize) {
+    fn update_camera_uniform(
+        &self,
+        rcx: &mut RenderContext,
+        index: usize,
+    ) -> CommandBufferExecFuture<NowFuture> {
         let near = 0.01;
 
         let aspect_ratio =
@@ -974,17 +1030,27 @@ impl App {
             ],
         };
 
-        *rcx.camera_buffers[index].write().unwrap() = uniform_data;
-
         if self.cstate.looking {
             trace!(
                 "campos: {:?} camforward: {:?}",
                 self.cstate.position, self.cstate.forward
             );
         }
+
+        gpu_upload(
+            uniform_data,
+            rcx.camera_buffers[index].clone(),
+            rcx.camera_buffers_host_visible[index].clone(),
+            self.command_buffer_allocator.clone(),
+            self.transfer_queue.clone(),
+        )
     }
 
-    fn update_lights_uniform(&self, rcx: &mut RenderContext, index: usize) {
+    fn update_lights_uniform(
+        &self,
+        rcx: &mut RenderContext,
+        index: usize,
+    ) -> CommandBufferExecFuture<NowFuture> {
         let mut pos = [[0f32; 4]; 32];
         let mut col = [[0f32; 4]; 32];
 
@@ -1003,7 +1069,13 @@ impl App {
             light_count: self.gstate.lights.len() as u32,
         };
 
-        *rcx.lights_buffers[index].write().unwrap() = uniform_data;
+        gpu_upload(
+            uniform_data,
+            rcx.lights_buffers[index].clone(),
+            rcx.lights_buffers_host_visible[index].clone(),
+            self.command_buffer_allocator.clone(),
+            self.transfer_queue.clone(),
+        )
     }
 
     fn get_descriptor_sets(
@@ -1308,6 +1380,7 @@ impl App {
                         rcx.shader_modules.clone(),
                         self.previous_debug.clone(),
                         self.block_enable_allocator.clone(),
+                        self.host_visible_allocator.clone(),
                         self.descriptor_set_allocator.clone(),
                         rcx.swapchain.image_count(),
                         DEFAULT_SUBDIVISION,
@@ -1389,7 +1462,7 @@ impl App {
             while csg_left > 0 {
                 for work in self.thread_work_completion.try_iter() {
                     match work {
-                        WorkComplete::RecompilePipelines(index) => {
+                        WorkComplete::RecompilePipelines(_index) => {
                             csg_left -= 1;
                         },
                         other => work_for_later.push(other),
@@ -1421,7 +1494,7 @@ impl App {
             while csg_left > 0 {
                 for work in self.thread_work_completion.try_iter() {
                     match work {
-                        WorkComplete::RecompilePipelines(index) => {
+                        WorkComplete::RecompilePipelines(_index) => {
                             csg_left -= 1;
                         },
                         other => work_for_later.push(other),
@@ -1455,6 +1528,8 @@ impl App {
                     csg.clone(),
                     self.time,
                     image_index,
+                    self.command_buffer_allocator.clone(),
+                    self.transfer_queue.clone(),
                     i,
                 ))
                 .unwrap();
@@ -1465,8 +1540,13 @@ impl App {
             future.cleanup_finished();
         }
 
-        self.update_camera_uniform(rcx, image_index);
+        let camera_semaphore = self.update_camera_uniform(rcx, image_index);
-        self.update_lights_uniform(rcx, image_index);
+        let lights_semaphore = self.update_lights_uniform(rcx, image_index);
+
+        let camera_and_lights = camera_semaphore
+            .join(lights_semaphore)
+            .then_signal_semaphore_and_flush()
+            .unwrap();
 
         let (trace_set, normals_set1, normals_set2, lighting_set1, lighting_set2) =
             self.get_descriptor_sets(rcx, image_index);
@@ -1500,11 +1580,18 @@ impl App {
             .set_viewport(0, [rcx.viewport.clone()].into_iter().collect())
             .unwrap();
 
+        let mut push_constant_future: Option<Box<dyn GpuFuture>> = None;
+
         while push_constants_left > 0 {
             for work in self.thread_work_completion.try_iter() {
                 match work {
-                    WorkComplete::GetPushConstants(pc, index) => {
+                    WorkComplete::GetPushConstants(pc, future, index) => {
                         push_constants[index] = pc;
+                        push_constant_future = if push_constant_future.is_some() {
+                            Some(push_constant_future.unwrap().join(future).boxed())
+                        } else {
+                            Some(future.boxed())
+                        };
                         push_constants_left -= 1;
                     },
                     other => work_for_later.push(other),
@@ -1512,6 +1599,15 @@ impl App {
             }
         }
 
+        let futures = if let Some(f) = push_constant_future {
+            f.then_signal_semaphore_and_flush()
+                .unwrap()
+                .join(camera_and_lights)
+                .boxed()
+        } else {
+            camera_and_lights.boxed()
+        };
+
         if self.gstate.csg.len() > 0 {
             self.add_commands_depth_pass(
                 &mut builder,
@@ -1550,6 +1646,7 @@ impl App {
             .map(|f| f.boxed())
             .unwrap_or(sync::now(self.device.clone()).boxed())
             .join(acquire_future)
+            .join(futures)
             .then_execute(self.graphics_queue.clone(), command_buffer)
             .unwrap()
             .then_swapchain_present(
@@ -1864,63 +1961,74 @@ fn pipeline_recompile(
     lighting_pipeline
 }
 
-fn gpu_buffer<T>(
+fn gpu_upload<T>(
-    input: &[&[T]],
+    input: T,
-    allocator: Arc<StandardMemoryAllocator>,
+    device_local: Subbuffer<T>,
-    sub_allocator: &SubbufferAllocator,
+    host_visible: Subbuffer<T>,
     command_allocator: Arc<StandardCommandBufferAllocator>,
     transfer_queue: Arc<Queue>,
-) -> Subbuffer<[T]>
+) -> CommandBufferExecFuture<NowFuture>
+where
+    T: BufferContents,
+{
+    {
+        let mut writer = host_visible.write().unwrap();
+        *writer = input;
+    }
+
+    gpu_upload_command_buffer(
+        device_local,
+        host_visible,
+        command_allocator,
+        transfer_queue,
+    )
+}
+
+fn gpu_upload_slice<T>(
+    input: &[T],
+    device_local: Subbuffer<[T]>,
+    host_visible: Subbuffer<[T]>,
+    command_allocator: Arc<StandardCommandBufferAllocator>,
+    transfer_queue: Arc<Queue>,
+) -> CommandBufferExecFuture<NowFuture>
 where
     T: BufferContents + Copy,
 {
-    let total_len = input.iter().map(|i| i.len() as u64).sum::<u64>();
+    {
-
+        let mut writer = host_visible.write().unwrap();
-    let buffer = Buffer::new_slice(
+        writer.copy_from_slice(input);
-        allocator,
-        BufferCreateInfo {
-            usage: BufferUsage::STORAGE_BUFFER | BufferUsage::TRANSFER_DST,
-            ..Default::default()
-        },
-        AllocationCreateInfo {
-            memory_type_filter: MemoryTypeFilter::PREFER_DEVICE,
-            allocate_preference: MemoryAllocatePreference::AlwaysAllocate,
-            ..Default::default()
-        },
-        total_len,
-    )
-    .unwrap();
-
-    let staging = sub_allocator.allocate_slice(total_len).unwrap();
-
-    let mut writer = staging.write().unwrap();
-    let mut pointer = 0;
-    for input in input {
-        writer[pointer..(pointer + input.len())].copy_from_slice(&input[..]);
-        pointer += input.len();
     }
-    drop(writer);
 
+    gpu_upload_command_buffer(
+        device_local,
+        host_visible,
+        command_allocator,
+        transfer_queue,
+    )
+}
+
+fn gpu_upload_command_buffer<T>(
+    device_local: Subbuffer<T>,
+    host_visible: Subbuffer<T>,
+    command_allocator: Arc<StandardCommandBufferAllocator>,
+    transfer_queue: Arc<Queue>,
+) -> CommandBufferExecFuture<NowFuture>
+where
+    T: BufferContents + ?Sized,
+{
     let mut builder = AutoCommandBufferBuilder::primary(
         command_allocator,
         transfer_queue.queue_family_index(),
         CommandBufferUsage::OneTimeSubmit,
     )
     .unwrap();
+
     builder
-        .copy_buffer(CopyBufferInfo::buffers(staging, buffer.clone()))
+        .copy_buffer(CopyBufferInfo::buffers(host_visible, device_local))
         .unwrap();
     let commands = builder.build().unwrap();
 
-    commands
+    commands.execute(transfer_queue).unwrap()
-        .execute(transfer_queue)
-        .unwrap()
-        .then_signal_fence_and_flush()
-        .unwrap()
-        .wait(None)
-        .unwrap();
-
-    buffer
 }
 
 fn dump_pipeline_cache(cache: Arc<PipelineCache>) {

src/objects.rs

@@ -38,19 +38,20 @@ pub(crate) struct Mesh {
 
 #[derive(Debug)]
 pub(crate) struct CSG {
-    pub(crate) name:                   String,
+    pub(crate) name: String,
-    pub(crate) parts:                  SSATape,
+    pub(crate) parts: SSATape,
-    pub(crate) pos:                    Vec3,
+    pub(crate) pos: Vec3,
-    pub(crate) rot:                    Vec3,
+    pub(crate) rot: Vec3,
-    pub(crate) scale:                  Vec3,
+    pub(crate) scale: Vec3,
-    pub(crate) new_pipelines_needed:   bool,
+    pub(crate) new_pipelines_needed: bool,
-    pub(crate) trace_shader_module:    Arc<ShaderModule>,
+    pub(crate) trace_shader_module: Arc<ShaderModule>,
-    pub(crate) normals_shader_module:  Arc<ShaderModule>,
+    pub(crate) normals_shader_module: Arc<ShaderModule>,
-    pub(crate) trace_pipeline:         Arc<GraphicsPipeline>,
+    pub(crate) trace_pipeline: Arc<GraphicsPipeline>,
-    pub(crate) normals_pipeline:       Arc<GraphicsPipeline>,
+    pub(crate) normals_pipeline: Arc<GraphicsPipeline>,
-    pub(crate) subdivision:            u32,
+    pub(crate) subdivision: u32,
-    pub(crate) enable_buffer:          Vec<Subbuffer<Object>>,
+    pub(crate) enable_buffer: Vec<Subbuffer<Object>>,
-    pub(crate) trace_descriptor_set:   Vec<Arc<DescriptorSet>>,
+    pub(crate) enable_buffer_host_visible: Vec<Subbuffer<Object>>,
+    pub(crate) trace_descriptor_set: Vec<Arc<DescriptorSet>>,
     pub(crate) normals_descriptor_set: Vec<Arc<DescriptorSet>>,
 }
 

src/threads.rs

@@ -11,33 +11,24 @@ use glam::{EulerRot, Mat4, Vec3};
 use log::info;
 use rspirv::{binary::Assemble, dr::Module};
 use vulkano::{
-    buffer::{Subbuffer, allocator::SubbufferAllocator},
+    buffer::{allocator::SubbufferAllocator, Subbuffer},
+    command_buffer::{allocator::StandardCommandBufferAllocator, CommandBufferExecFuture},
     descriptor_set::{
-        DescriptorSet, WriteDescriptorSet, allocator::StandardDescriptorSetAllocator,
+        allocator::StandardDescriptorSetAllocator, DescriptorSet, WriteDescriptorSet
     },
-    device::Device,
+    device::{Device, Queue},
     pipeline::{
-        DynamicState, GraphicsPipeline, Pipeline, PipelineCreateFlags, PipelineLayout,
+        cache::PipelineCache, graphics::{
-        PipelineShaderStageCreateInfo,
+            color_blend::{ColorBlendAttachmentState, ColorBlendState}, depth_stencil::{DepthState, DepthStencilState}, input_assembly::InputAssemblyState, multisample::MultisampleState, rasterization::{CullMode, FrontFace, PolygonMode, RasterizationState}, vertex_input::{Vertex, VertexDefinition, VertexInputState}, GraphicsPipelineCreateInfo
-        cache::PipelineCache,
+        }, layout::PipelineDescriptorSetLayoutCreateInfo, DynamicState, GraphicsPipeline, Pipeline, PipelineCreateFlags, PipelineLayout, PipelineShaderStageCreateInfo
-        graphics::{
-            GraphicsPipelineCreateInfo,
-            color_blend::{ColorBlendAttachmentState, ColorBlendState},
-            depth_stencil::{DepthState, DepthStencilState},
-            input_assembly::InputAssemblyState,
-            multisample::MultisampleState,
-            rasterization::{CullMode, FrontFace, PolygonMode, RasterizationState},
-            vertex_input::{Vertex, VertexDefinition, VertexInputState},
-        },
-        layout::PipelineDescriptorSetLayoutCreateInfo,
     },
     render_pass::{RenderPass, Subpass},
-    shader::{ShaderModule, ShaderModuleCreateInfo},
+    shader::{ShaderModule, ShaderModuleCreateInfo}, sync::future::NowFuture,
 };
 
 use crate::{
     DUMP_SPV_TO_FILE, IVertex, MAXIMUM_SUBDIVISION, MINUMUM_SUBDIVISION, ShaderModules,
-    get_spec_constants,
+    get_spec_constants, gpu_upload,
     gui::PreviousDebug,
     interpreter::{self, IntervalInterpreter, PointInterpreter, VALUE_0},
     objects::CSG,
@@ -57,12 +48,20 @@ pub enum WorkItem {
         ShaderModules,
         PreviousDebug,
         Arc<Mutex<SubbufferAllocator>>,
+        Arc<Mutex<SubbufferAllocator>>,
         Arc<StandardDescriptorSetAllocator>,
         u32,
         u32,
         isize,
     ),
-    GetPushConstants(Arc<RwLock<CSG>>, f32, usize, usize),
+    GetPushConstants(
+        Arc<RwLock<CSG>>,
+        f32,
+        usize,
+        Arc<StandardCommandBufferAllocator>,
+        Arc<Queue>,
+        usize,
+    ),
     RecompilePipelines(
         Arc<RwLock<CSG>>,
         Arc<Device>,
@@ -74,10 +73,9 @@ pub enum WorkItem {
     ),
 }
 
-#[derive(Debug)]
 pub enum WorkComplete {
     CreateCSG(Arc<RwLock<CSG>>, isize),
-    GetPushConstants(PushConstantData, usize),
+    GetPushConstants(PushConstantData, CommandBufferExecFuture<NowFuture>, usize),
     RecompilePipelines(usize),
 }
 
@@ -93,6 +91,7 @@ pub fn thread_loop(recv: mpmc::Receiver<WorkItem>, send: mpsc::SyncSender<WorkCo
                 modules,
                 debug,
                 block_enable_allocator,
+                host_visible_allocator,
                 descriptor_set_allocator,
                 frames,
                 subdivision,
@@ -130,6 +129,16 @@ pub fn thread_loop(recv: mpmc::Receiver<WorkItem>, send: mpsc::SyncSender<WorkCo
                             .unwrap()
                     })
                     .collect();
+                let enable_buffer_host_visible: Vec<Subbuffer<Object>> = (0..frames)
+                    .into_iter()
+                    .map(|_| {
+                        host_visible_allocator
+                            .lock()
+                            .unwrap()
+                            .allocate_sized()
+                            .unwrap()
+                    })
+                    .collect();
 
                 let trace_layout = trace_pipeline.layout().set_layouts()[1].clone();
                 let trace_descriptor_set = (0..frames)
@@ -178,6 +187,7 @@ pub fn thread_loop(recv: mpmc::Receiver<WorkItem>, send: mpsc::SyncSender<WorkCo
                     normals_pipeline,
                     subdivision,
                     enable_buffer,
+                    enable_buffer_host_visible,
                     trace_descriptor_set,
                     normals_descriptor_set,
                 }));
@@ -190,7 +200,14 @@ pub fn thread_loop(recv: mpmc::Receiver<WorkItem>, send: mpsc::SyncSender<WorkCo
                     (csg_end - csg_start).as_secs_f64() * 1000.0
                 );
             },
-            WorkItem::GetPushConstants(csg, time, frame_index, index) => {
+            WorkItem::GetPushConstants(
+                csg,
+                time,
+                frame_index,
+                command_allocator,
+                transfer_queue,
+                index,
+            ) => {
                 let csg = csg.read().unwrap();
 
                 let world = Mat4::from_translation(csg.pos * 0.01)
@@ -207,9 +224,16 @@ pub fn thread_loop(recv: mpmc::Receiver<WorkItem>, send: mpsc::SyncSender<WorkCo
                     inv_world: world.inverse().to_cols_array_2d(),
                 };
 
-                interval_check(&csg, index as u32 + 1, time, frame_index);
+                let future = interval_check(
+                    &csg,
+                    index as u32 + 1,
+                    time,
+                    frame_index,
+                    command_allocator,
+                    transfer_queue,
+                );
 
-                send.send(WorkComplete::GetPushConstants(push_constants, index))
+                send.send(WorkComplete::GetPushConstants(push_constants, future, index))
                     .unwrap();
             },
             WorkItem::RecompilePipelines(
@@ -441,7 +465,14 @@ fn create_csg() -> SSATape {
     tape
 }
 
-fn interval_check(csg: &CSG, material: u32, time: f32, frame_index: usize) {
+fn interval_check(
+    csg: &CSG,
+    material: u32,
+    time: f32,
+    frame_index: usize,
+    command_allocator: Arc<StandardCommandBufferAllocator>,
+    transfer_queue: Arc<Queue>,
+) -> CommandBufferExecFuture<NowFuture> {
     const INTERPRET_INPUT_X: interpreter::Value =
         interpreter::Value::from_array([10000.0, 0.0, 0.0, -10000.0, 0.0, 0.0, 0.0, 0.0]);
     const INTERPRET_INPUT_Y: interpreter::Value =
@@ -572,7 +603,13 @@ fn interval_check(csg: &CSG, material: u32, time: f32, frame_index: usize) {
         }
     }
 
-    *csg.enable_buffer[frame_index].write().unwrap() = obj;
+    gpu_upload(
+        obj,
+        csg.enable_buffer[frame_index].clone(),
+        csg.enable_buffer_host_visible[frame_index].clone(),
+        command_allocator,
+        transfer_queue,
+    )
 }
 
 fn sdf_specialize_module(