entirely rewrite how shader loading works

README.md

@@ -4,4 +4,4 @@ A realtime renderer for implicit surfaces that JIT compiles the bytecode represe
 
 ## Building
 
-`python3 assemble_trace.py && cargo build --release`
+`cargo build --release`

tape-drive/build.rs

@@ -4,9 +4,12 @@ fn main() {
     //let mut output = std::fs::File::create("output.txt").unwrap();
     println!("cargo::rerun-if-changed=build.rs");
     println!("cargo::rerun-if-changed=Cargo.lock");
-    println!("cargo::rerun-if-changed=assemble_trace.py");
+    println!("cargo::rerun-if-changed=compile_shaders.py");
 
-    let paths = read_dir("src/shaders").unwrap();
+    let paths = read_dir("shaders/normal")
+        .unwrap()
+        .chain(read_dir("shaders/include").unwrap())
+        .chain(read_dir("shaders/replacement").unwrap());
     for path in paths {
         if let Ok(path) = path
             && path
@@ -22,7 +25,7 @@ fn main() {
     }
 
     Command::new("python3")
-        .arg("assemble_trace.py")
+        .arg("compile_shaders.py")
         .status()
         .expect("Failed to assemble header shaders");
 }

tape-drive/compile_shaders.py

@@ -6,64 +6,59 @@ import sys
 import subprocess
 import re
 
-SHADERS_OUT = "shaders_out"
-SHADERS_IN = "src/shaders"
 TARGET_ENV = "vulkan1.3"
+SHADERS_OUT = "shaders_out"
+SHADERS_BASE = "shaders"
+SHADERS_INCLUDE = f"{SHADERS_BASE}/include"
+SHADERS_IN_NORMAL = f"{SHADERS_BASE}/normal"
+SHADERS_IN_REPLACEMENT = f"{SHADERS_BASE}/replacement"
+SHADERS_OUT_NORMAL = f"{SHADERS_OUT}/normal"
+SHADERS_OUT_REPLACEMENT = f"{SHADERS_OUT}/replacement"
 
 DUMP_DISASSEMBLIES = True
 
 if os.path.isdir(SHADERS_OUT):
     shutil.rmtree(SHADERS_OUT)
 os.mkdir(SHADERS_OUT)
+os.mkdir(SHADERS_OUT_NORMAL)
+os.mkdir(SHADERS_OUT_REPLACEMENT)
 
-normal_shaders = [
+normal_shaders = [f for f in os.listdir(SHADERS_IN_NORMAL) if f.endswith(".glsl")]
-    ("fullscreen.vert", "vert"),
-    ("lighting.frag", "frag"),
-    ("trace.vert", "vert"),
-    ("rgen", "rgen"),
-    ("rchit", "rchit"),
-    ("rmiss", "rmiss"),
-]
 
-for shader, ty in normal_shaders:
+for shader in normal_shaders:
     result = subprocess.run(
         [
             "glslangValidator",
+            f"-I{SHADERS_INCLUDE}",
             "--spirv-val",
             "-gVS",
-            "-S",
-            ty,
             "--target-env",
             TARGET_ENV,
             "-o",
-            f"{SHADERS_OUT}/{shader}.spv",
+            f"{SHADERS_OUT_NORMAL}/{shader}.spv",
-            f"{SHADERS_IN}/{shader}.glsl",
+            f"{SHADERS_IN_NORMAL}/{shader}",
         ]
     )
     if result.returncode != 0:
         sys.exit(result.returncode)
 
 replacement_shaders = [
-    ("trace.frag", "frag"),
+    f for f in os.listdir(SHADERS_IN_REPLACEMENT) if f.endswith(".glsl")
-    ("normals.frag", "frag"),
-    ("fuzz.comp", "comp"),
-    ("enable_gen.comp", "comp"),
 ]
 
-for shader, ty in replacement_shaders:
+for shader in replacement_shaders:
-    print(f"{SHADERS_IN}/{shader}.glsl")
+    print(f"{SHADERS_IN_REPLACEMENT}/{shader}")
     result = subprocess.run(
         [
             "glslangValidator",
+            f"-I{SHADERS_INCLUDE}",
             "--spirv-val",
             "--spirv-dis",
             "-g",
-            "-S",
-            ty,
             "--target-env",
             TARGET_ENV,
             "--quiet",
-            f"{SHADERS_IN}/{shader}.glsl",
+            f"{SHADERS_IN_REPLACEMENT}/{shader}",
         ],
         capture_output=True,
         text=True,
@@ -74,7 +69,7 @@ for shader, ty in replacement_shaders:
     asm = result.stdout
 
     if DUMP_DISASSEMBLIES:
-        with open(f"{SHADERS_OUT}/{shader}.stage0.spv-dis", "w") as f:
+        with open(f"{SHADERS_OUT_REPLACEMENT}/{shader}.stage0.spv-dis", "w") as f:
             f.write(asm)
 
     asm = re.sub(r"(?ms)%scene_vf4_ =.*?OpFunctionEnd", "", asm)
@@ -85,7 +80,7 @@ for shader, ty in replacement_shaders:
     asm = re.sub(r"%gradient_scene_vf4_", "%30000", asm)
 
     if DUMP_DISASSEMBLIES:
-        with open(f"{SHADERS_OUT}/{shader}.stage1.spv-dis", "w") as f:
+        with open(f"{SHADERS_OUT_REPLACEMENT}/{shader}.stage1.spv-dis", "w") as f:
             f.write(asm)
 
     result = subprocess.run(
@@ -95,7 +90,7 @@ for shader, ty in replacement_shaders:
             "--target-env",
             "vulkan1.3",
             "-o",
-            f"{SHADERS_OUT}/{shader}.spv",
+            f"{SHADERS_OUT_REPLACEMENT}/{shader}.spv",
             "-",
         ],
         input=asm,
@@ -103,4 +98,8 @@ for shader, ty in replacement_shaders:
     )
     if result.returncode != 0:
         sys.exit(result.returncode)
-    os.remove(f"{ty}.spv")
+
+for f in os.listdir("."):
+    if f.endswith(".spv"):
+        print(f"removing {f}")
+        os.remove(f)

tape-drive/shaders/normal/fullscreen.vert.glsl

@@ -13,5 +13,5 @@ const vec2 full_screen_triangle[3] = {
 void main() {
     vec2 tri = (full_screen_triangle[gl_VertexIndex - gl_BaseVertex] * 2.0) - 1.0;
     v_pos = tri;
-    gl_Position = vec4(v_pos, gl_BaseVertex / 65536.0 , 1.0);
+    gl_Position = vec4(v_pos, gl_BaseVertex / 65536.0, 1.0);
 }

tape-drive/shaders/replacement/normals.rchit.glsl

@@ -1,6 +1,5 @@
-//  global fragment shader
-
 #version 460
+#extension GL_EXT_ray_tracing : require
 #extension GL_GOOGLE_include_directive:require
 #include "include.glsl"
 #include "implicit_include.glsl"
@@ -23,7 +22,7 @@ vec3 getNormal(vec3 p) {
 
 //Implicit Surface Entrypoint
 void main() {
-    vec3 position = gl_ObjectRayOriginEXT + (gl_ObjectRayDirectionEXT * gl_HitTEXT)
+    vec3 position = gl_ObjectRayOriginEXT + (gl_ObjectRayDirectionEXT * gl_HitTEXT);
 
     vec3 n = getNormal(position);
     hit_value = n * 0.5 + 0.5;

tape-drive/src/main.rs

@@ -14,8 +14,8 @@ const DEFAULT_SUBDIVISION: u32 = 16;
 const MAXIMUM_SUBDIVISION: u32 = 64;
 const MINUMUM_SUBDIVISION: u32 = 8;
 
-const RUN_STANDARD_PIPELINE: bool = false;
+const RUN_STANDARD_PIPELINE: bool = true;
-const RUN_RT_PIPELINE: bool = true;
+const RUN_RT_PIPELINE: bool = false;
 
 use std::{
     error::Error,
@@ -23,7 +23,7 @@ use std::{
     io::{Cursor, Read, Write},
     path::{Path, PathBuf},
     simd::{f32x8, num::SimdFloat},
-    sync::{Arc, Mutex, atomic::AtomicBool, mpmc, mpsc},
+    sync::{Arc, Mutex, RwLock, atomic::AtomicBool, mpmc, mpsc},
     thread::{self, JoinHandle},
     time::Instant,
 };
@@ -227,10 +227,7 @@ struct App {
     cstate: CState,
     time: f32,
     vertex_buffer: Subbuffer<[IVertex]>,
-    trace_module: Module,
+    replacement_modules: Arc<RwLock<HashMap<String, Module>>>,
-    normals_module: Module,
-    fuzz_module: Module,
-    enable_gen_module: Module,
     _threads: Vec<JoinHandle<()>>,
     thread_work_creation: mpmc::Sender<WorkItem>,
     thread_work_completion: mpsc::Receiver<WorkComplete>,
@@ -240,16 +237,6 @@ struct App {
     application_start: Instant,
 }
 
-#[derive(Debug, Clone)]
-struct ShaderModules {
-    trace_vs:      Arc<ShaderModule>,
-    fullscreen_vs: Arc<ShaderModule>,
-    lighting_fs:   Arc<ShaderModule>,
-    raygen:        Arc<ShaderModule>,
-    closest_hit:   Arc<ShaderModule>,
-    miss:          Arc<ShaderModule>,
-}
-
 struct RenderContext {
     window: Arc<Window>,
     swapchain: Arc<Swapchain>,
@@ -264,7 +251,7 @@ struct RenderContext {
     camera_buffers_host_visible: Vec<Subbuffer<Camera>>,
     lights_buffers: Vec<Subbuffer<Lights>>,
     lights_buffers_host_visible: Vec<Subbuffer<Lights>>,
-    shader_modules: ShaderModules,
+    shader_modules: Arc<RwLock<HashMap<String, Arc<ShaderModule>>>>,
     lighting_pipeline: Arc<GraphicsPipeline>,
     rt_pipeline: Arc<RayTracingPipeline>,
     rt_shader_binding_table: ShaderBindingTable,
@@ -676,37 +663,28 @@ impl App {
         .wait(None)
         .unwrap();
 
-        let trace_spv_code = include_bytes!("../shaders_out/trace.frag.spv");
+        let mut replacement_modules = HashMap::new();
-        let trace_spv_code_u32 = vulkano::shader::spirv::bytes_to_words(trace_spv_code)
-            .unwrap()
-            .into_owned();
-        let mut loader = rspirv::dr::Loader::new();
-        rspirv::binary::parse_words(trace_spv_code_u32, &mut loader).unwrap();
-        let trace_module = loader.module();
 
-        let normals_spv_code = include_bytes!("../shaders_out/normals.frag.spv");
+        macro_rules! define_replacement_module {
-        let normals_spv_code_u32 = vulkano::shader::spirv::bytes_to_words(normals_spv_code)
+            ($path:literal) => {
-            .unwrap()
+                let spv_code =
-            .into_owned();
+                    include_bytes!(concat!("../shaders_out/replacement/", $path, ".glsl.spv"));
-        let mut loader = rspirv::dr::Loader::new();
+                let spv_code_u32 = vulkano::shader::spirv::bytes_to_words(spv_code)
-        rspirv::binary::parse_words(normals_spv_code_u32, &mut loader).unwrap();
+                    .unwrap()
-        let normals_module = loader.module();
+                    .into_owned();
+                let mut loader = rspirv::dr::Loader::new();
+                rspirv::binary::parse_words(spv_code_u32, &mut loader).unwrap();
+                let module = loader.module();
+                replacement_modules.insert($path.to_string(), module);
+            };
+        }
 
-        let fuzz_spv_code = include_bytes!("../shaders_out/fuzz.comp.spv");
+        define_replacement_module!("trace.frag");
-        let fuzz_spv_code_u32 = vulkano::shader::spirv::bytes_to_words(fuzz_spv_code)
+        define_replacement_module!("normals.frag");
-            .unwrap()
+        define_replacement_module!("fuzz.comp");
-            .into_owned();
+        define_replacement_module!("enable_gen.comp");
-        let mut loader = rspirv::dr::Loader::new();
-        rspirv::binary::parse_words(fuzz_spv_code_u32, &mut loader).unwrap();
-        let fuzz_module = loader.module();
 
-        let enable_gen_spv_code = include_bytes!("../shaders_out/enable_gen.comp.spv");
+        let replacement_modules = Arc::new(RwLock::new(replacement_modules));
-        let enable_gen_spv_code_u32 = vulkano::shader::spirv::bytes_to_words(enable_gen_spv_code)
-            .unwrap()
-            .into_owned();
-        let mut loader = rspirv::dr::Loader::new();
-        rspirv::binary::parse_words(enable_gen_spv_code_u32, &mut loader).unwrap();
-        let enable_gen_module = loader.module();
 
         let (thread_work_creation_sender, thread_work_creation_receiver) =
             mpmc::sync_channel::<WorkItem>(256);
@@ -739,20 +717,22 @@ impl App {
         .ok();
 
         if watcher.is_some()
-            && let Ok(paths) = std::fs::read_dir("tape-drive/src/shaders")
+            && let Ok(normal_paths) = std::fs::read_dir("tape-drive/shaders/normal")
+            && let Ok(replacement_paths) = std::fs::read_dir("tape-drive/shaders/replacement")
+            && let Ok(include_paths) = std::fs::read_dir("tape-drive/shaders/include")
         {
-            for path in paths {
+            for path in normal_paths.chain(replacement_paths).chain(include_paths) {
                 if let Ok(path) = path
                     && path.path().extension().is_some_and(|ext| ext == "glsl")
                     && let Some(watcher) = &mut watcher
                 {
                     let _ = watcher
-                        .watch(&path.path(), notify::RecursiveMode::NonRecursive)
+                        .watch(&path.path(), notify::RecursiveMode::Recursive)
-                        .inspect_err(|f| {
+                        .inspect_err(|f| warn!("Failed to watch {}: {}", path.path().display(), f));
-                            error!("Failed to watch {}: {}", path.path().display(), f)
-                        });
                 }
             }
+        } else {
+            warn!("disabling shader hot reload");
         }
 
         App {
@@ -778,10 +758,7 @@ impl App {
             cstate,
             time: 0.0,
             vertex_buffer,
-            trace_module,
+            replacement_modules,
-            normals_module,
-            fuzz_module,
-            enable_gen_module,
             _threads: threads,
             thread_work_creation: thread_work_creation_sender,
             thread_work_completion: thread_work_completion_receiver,
@@ -795,52 +772,7 @@ impl App {
 
 mod trace_vs {
     vulkano_shaders::shader! {
-        bytes: "shaders_out/trace.vert.spv",
+        bytes: "shaders_out/replacement/trace.frag.glsl.spv",
-        vulkan_version: "1.3",
-        spirv_version: "1.6",
-        custom_derives: [Debug, Clone, Copy],
-    }
-}
-
-mod fullscreen_vs {
-    vulkano_shaders::shader! {
-        bytes: "shaders_out/fullscreen.vert.spv",
-        vulkan_version: "1.3",
-        spirv_version: "1.6",
-        custom_derives: [Debug, Clone, Copy],
-    }
-}
-
-mod lighting_fs {
-    vulkano_shaders::shader! {
-        bytes: "shaders_out/lighting.frag.spv",
-        vulkan_version: "1.3",
-        spirv_version: "1.6",
-        custom_derives: [Debug, Clone, Copy],
-    }
-}
-
-mod raygen {
-    vulkano_shaders::shader! {
-        bytes: "shaders_out/rgen.spv",
-        vulkan_version: "1.3",
-        spirv_version: "1.6",
-        custom_derives: [Debug, Clone, Copy],
-    }
-}
-
-mod closest_hit {
-    vulkano_shaders::shader! {
-        bytes: "shaders_out/rchit.spv",
-        vulkan_version: "1.3",
-        spirv_version: "1.6",
-        custom_derives: [Debug, Clone, Copy],
-    }
-}
-
-mod miss {
-    vulkano_shaders::shader! {
-        bytes: "shaders_out/rmiss.spv",
         vulkan_version: "1.3",
         spirv_version: "1.6",
         custom_derives: [Debug, Clone, Copy],
@@ -849,16 +781,7 @@ mod miss {
 
 mod fuzz_cs {
     vulkano_shaders::shader! {
-        bytes: "shaders_out/fuzz.comp.spv",
+        bytes: "shaders_out/replacement/fuzz.comp.glsl.spv",
-        vulkan_version: "1.3",
-        spirv_version: "1.6",
-        custom_derives: [Debug, Clone, Copy],
-    }
-}
-
-mod enable_gen_cs {
-    vulkano_shaders::shader! {
-        bytes: "shaders_out/enable_gen.comp.spv",
         vulkan_version: "1.3",
         spirv_version: "1.6",
         custom_derives: [Debug, Clone, Copy],
@@ -964,72 +887,34 @@ impl ApplicationHandler for App {
             .unwrap()
         };
 
-        let loaders: Vec<
+        let mut shader_modules = HashMap::new();
-            fn(
-                ::std::sync::Arc<::vulkano::device::Device>,
-            ) -> Result<
-                ::std::sync::Arc<::vulkano::shader::ShaderModule>,
-                ::vulkano::Validated<::vulkano::VulkanError>,
-            >,
-        > = vec![
-            ((trace_vs::load)
-                as fn(
-                    ::std::sync::Arc<::vulkano::device::Device>,
-                ) -> Result<
-                    ::std::sync::Arc<::vulkano::shader::ShaderModule>,
-                    ::vulkano::Validated<::vulkano::VulkanError>,
-                >),
-            ((fullscreen_vs::load)
-                as fn(
-                    ::std::sync::Arc<::vulkano::device::Device>,
-                ) -> Result<
-                    ::std::sync::Arc<::vulkano::shader::ShaderModule>,
-                    ::vulkano::Validated<::vulkano::VulkanError>,
-                >),
-            ((lighting_fs::load)
-                as fn(
-                    ::std::sync::Arc<::vulkano::device::Device>,
-                ) -> Result<
-                    ::std::sync::Arc<::vulkano::shader::ShaderModule>,
-                    ::vulkano::Validated<::vulkano::VulkanError>,
-                >),
-            ((raygen::load)
-                as fn(
-                    ::std::sync::Arc<::vulkano::device::Device>,
-                ) -> Result<
-                    ::std::sync::Arc<::vulkano::shader::ShaderModule>,
-                    ::vulkano::Validated<::vulkano::VulkanError>,
-                >),
-            ((closest_hit::load)
-                as fn(
-                    ::std::sync::Arc<::vulkano::device::Device>,
-                ) -> Result<
-                    ::std::sync::Arc<::vulkano::shader::ShaderModule>,
-                    ::vulkano::Validated<::vulkano::VulkanError>,
-                >),
-            ((miss::load)
-                as fn(
-                    ::std::sync::Arc<::vulkano::device::Device>,
-                ) -> Result<
-                    ::std::sync::Arc<::vulkano::shader::ShaderModule>,
-                    ::vulkano::Validated<::vulkano::VulkanError>,
-                >),
-        ];
 
-        let mut pariter = loaders
+        macro_rules! define_normal_module {
-            .iter()
+            ($path:literal) => {
-            .map(|load| load(self.device.clone()).unwrap());
+                let spv_code =
+                    include_bytes!(concat!("../shaders_out/normal/", $path, ".glsl.spv"));
+                let spv_code_u32 = vulkano::shader::spirv::bytes_to_words(spv_code)
+                    .unwrap()
+                    .into_owned();
+                let shader = unsafe {
+                    ::vulkano::shader::ShaderModule::new(
+                        self.device.clone(),
+                        ShaderModuleCreateInfo::new(&spv_code_u32),
+                    )
+                    .unwrap()
+                };
+                shader_modules.insert($path.to_string(), shader);
+            };
+        }
 
-        let shader_modules = ShaderModules {
+        define_normal_module!("fullscreen.vert");
-            trace_vs:      pariter.next().unwrap(),
+        define_normal_module!("lighting.frag");
-            fullscreen_vs: pariter.next().unwrap(),
+        define_normal_module!("mesh.rchit");
-            lighting_fs:   pariter.next().unwrap(),
+        define_normal_module!("mesh.rgen");
-            raygen:        pariter.next().unwrap(),
+        define_normal_module!("mesh.rmiss");
-            closest_hit:   pariter.next().unwrap(),
+        define_normal_module!("trace.vert");
-            miss:          pariter.next().unwrap(),
-        };
 
-        drop(pariter);
+        let shader_modules = Arc::new(RwLock::new(shader_modules));
 
         let render_pass = vulkano::ordered_passes_renderpass!(
             self.device.clone(),
@@ -1289,10 +1174,7 @@ impl ApplicationHandler for App {
                         self.thread_work_creation
                             .send(WorkItem::CreateCSG(
                                 self.device.clone(),
-                                self.trace_module.clone(),
+                                self.replacement_modules.clone(),
-                                self.normals_module.clone(),
-                                self.fuzz_module.clone(),
-                                self.enable_gen_module.clone(),
                                 rcx.render_pass.clone(),
                                 self.pipeline_cache.clone(),
                                 rcx.shader_modules.clone(),
@@ -1837,149 +1719,107 @@ impl App {
     }
 
     fn load_all_spv_from_file(&mut self, rcx: &mut RenderContext) {
-        let trace_frag = read_spirv_words_from_file("trace.frag.spv");
+        let mut normal_modules = HashMap::new();
-        if trace_frag.is_err() {
+
-            error!("Could not read trace frag file");
+        let dir_read = std::fs::read_dir("tape-drive/shaders_out/normal");
+        if dir_read.is_err() {
+            error!(
+                "Could not read directory tape-drive/shaders_out/normal: {}",
+                dir_read.unwrap_err()
+            );
             return;
         }
-        let trace_frag = trace_frag.unwrap();
 
-        let trace_vert = read_spirv_words_from_file("trace.vert.spv");
+        for path in dir_read.unwrap() {
-        if trace_vert.is_err() {
+            if let Ok(path) = path
-            error!("Could not read trace vert file");
+                && path
+                    .path()
+                    .extension()
+                    .map(|ext| ext == "spv")
+                    .unwrap_or(false)
+            {
+                let words = read_spirv_words_from_file(path.path());
+                if words.is_err() {
+                    error!(
+                        "Could not read file {}: {}",
+                        path.path().display(),
+                        words.unwrap_err()
+                    );
+                    return;
+                }
+                let words = words.unwrap();
+                let module = unsafe {
+                    ::vulkano::shader::ShaderModule::new(
+                        self.device.clone(),
+                        ShaderModuleCreateInfo::new(&words),
+                    )
+                    .unwrap()
+                };
+                normal_modules.insert(
+                    Path::new(
+                        Path::new(path.path().file_stem().unwrap())
+                            .file_stem()
+                            .unwrap(),
+                    )
+                    .file_name()
+                    .unwrap()
+                    .to_string_lossy()
+                    .to_string(),
+                    module,
+                );
+                info!("Loaded normal shader {}", path.path().display());
+            }
+        }
+
+        let mut replacement_modules = HashMap::new();
+
+        let dir_read = std::fs::read_dir("tape-drive/shaders_out/replacement");
+        if dir_read.is_err() {
+            error!(
+                "Could not read directory tape-drive/shaders_out/replacement: {}",
+                dir_read.unwrap_err()
+            );
             return;
         }
-        let trace_vert = trace_vert.unwrap();
 
-        let normals_frag = read_spirv_words_from_file("normals.frag.spv");
+        for path in dir_read.unwrap() {
-        if normals_frag.is_err() {
+            if let Ok(path) = path
-            error!("Could not read normals frag file");
+                && path
-            return;
+                    .path()
+                    .extension()
+                    .map(|ext| ext == "spv")
+                    .unwrap_or(false)
+            {
+                let words = read_spirv_words_from_file(path.path());
+                if words.is_err() {
+                    error!(
+                        "Could not read file {}: {}",
+                        path.path().display(),
+                        words.unwrap_err()
+                    );
+                    return;
+                }
+                let words = words.unwrap();
+                let mut loader = rspirv::dr::Loader::new();
+                rspirv::binary::parse_words(words, &mut loader).unwrap();
+                replacement_modules.insert(
+                    (Path::new(
+                        Path::new(path.path().file_stem().unwrap())
+                            .file_stem()
+                            .unwrap(),
+                    )
+                    .file_name()
+                    .unwrap()
+                    .to_string_lossy())
+                    .to_string(),
+                    loader.module(),
+                );
+                info!("Loaded replacement shader {}", path.path().display());
+            }
         }
-        let normals_frag = normals_frag.unwrap();
 
-        let fullscreen_vert = read_spirv_words_from_file("fullscreen.vert.spv");
+        *rcx.shader_modules.write().unwrap() = normal_modules;
-        if fullscreen_vert.is_err() {
+        *self.replacement_modules.write().unwrap() = replacement_modules;
-            error!("Could not read fullscreen vert file");
-            return;
-        }
-        let fullscreen_vert = fullscreen_vert.unwrap();
-
-        let lighting_frag = read_spirv_words_from_file("lighting.frag.spv");
-        if lighting_frag.is_err() {
-            error!("Could not read lighting frag file");
-            return;
-        }
-        let lighting_frag = lighting_frag.unwrap();
-
-        let raygen = read_spirv_words_from_file("rgen.spv");
-        if raygen.is_err() {
-            error!("Could not read raygen file");
-            return;
-        }
-        let raygen = raygen.unwrap();
-
-        let closest_hit = read_spirv_words_from_file("rchit.spv");
-        if closest_hit.is_err() {
-            error!("Could not read closest hit file");
-            return;
-        }
-        let closest_hit = closest_hit.unwrap();
-
-        let miss = read_spirv_words_from_file("rmiss.spv");
-        if miss.is_err() {
-            error!("Could not read miss file");
-            return;
-        }
-        let miss = miss.unwrap();
-
-        let fuzz_comp = read_spirv_words_from_file("fuzz.comp.spv");
-        if fuzz_comp.is_err() {
-            error!("Could not read fuzz comp file");
-            return;
-        }
-        let fuzz_comp = fuzz_comp.unwrap();
-
-        let enable_gen_comp = read_spirv_words_from_file("enable_gen.comp.spv");
-        if enable_gen_comp.is_err() {
-            error!("Could not read enable_gen comp file");
-            return;
-        }
-        let enable_gen_comp = enable_gen_comp.unwrap();
-
-        let mut trace_loader = rspirv::dr::Loader::new();
-        rspirv::binary::parse_words(trace_frag, &mut trace_loader).unwrap();
-
-        let mut normals_loader = rspirv::dr::Loader::new();
-        rspirv::binary::parse_words(normals_frag, &mut normals_loader).unwrap();
-
-        let mut fuzz_loader = rspirv::dr::Loader::new();
-        rspirv::binary::parse_words(fuzz_comp, &mut fuzz_loader).unwrap();
-
-        let mut enable_gen_loader = rspirv::dr::Loader::new();
-        rspirv::binary::parse_words(enable_gen_comp, &mut enable_gen_loader).unwrap();
-
-        let trace_vert = unsafe {
-            ::vulkano::shader::ShaderModule::new(
-                self.device.clone(),
-                ShaderModuleCreateInfo::new(&trace_vert),
-            )
-            .unwrap()
-        };
-
-        let fullscreen_vert = unsafe {
-            ::vulkano::shader::ShaderModule::new(
-                self.device.clone(),
-                ShaderModuleCreateInfo::new(&fullscreen_vert),
-            )
-            .unwrap()
-        };
-
-        let lighting_frag = unsafe {
-            ::vulkano::shader::ShaderModule::new(
-                self.device.clone(),
-                ShaderModuleCreateInfo::new(&lighting_frag),
-            )
-            .unwrap()
-        };
-
-        let raygen = unsafe {
-            ::vulkano::shader::ShaderModule::new(
-                self.device.clone(),
-                ShaderModuleCreateInfo::new(&raygen),
-            )
-            .unwrap()
-        };
-
-        let closest_hit = unsafe {
-            ::vulkano::shader::ShaderModule::new(
-                self.device.clone(),
-                ShaderModuleCreateInfo::new(&closest_hit),
-            )
-            .unwrap()
-        };
-
-        let miss = unsafe {
-            ::vulkano::shader::ShaderModule::new(
-                self.device.clone(),
-                ShaderModuleCreateInfo::new(&miss),
-            )
-            .unwrap()
-        };
-
-        rcx.shader_modules = ShaderModules {
-            trace_vs: trace_vert,
-            fullscreen_vs: fullscreen_vert,
-            lighting_fs: lighting_frag,
-            raygen,
-            closest_hit,
-            miss,
-        };
-
-        self.trace_module = trace_loader.module();
-        self.normals_module = normals_loader.module();
-        self.fuzz_module = fuzz_loader.module();
-        self.enable_gen_module = enable_gen_loader.module();
     }
 
     fn recreate_pipelines(
@@ -1995,10 +1835,7 @@ impl App {
                 .send(WorkItem::RecompilePipelines(
                     csg.clone(),
                     self.device.clone(),
-                    self.trace_module.clone(),
+                    self.replacement_modules.clone(),
-                    self.normals_module.clone(),
-                    self.fuzz_module.clone(),
-                    self.enable_gen_module.clone(),
                     rcx.render_pass.clone(),
                     self.pipeline_cache.clone(),
                     rcx.shader_modules.clone(),
@@ -2037,10 +1874,7 @@ impl App {
             self.thread_work_creation
                 .send(WorkItem::CreateCSG(
                     self.device.clone(),
-                    self.trace_module.clone(),
+                    self.replacement_modules.clone(),
-                    self.normals_module.clone(),
-                    self.fuzz_module.clone(),
-                    self.enable_gen_module.clone(),
                     rcx.render_pass.clone(),
                     self.pipeline_cache.clone(),
                     rcx.shader_modules.clone(),
@@ -2247,7 +2081,7 @@ impl App {
             self.glsl_changed
                 .store(false, std::sync::atomic::Ordering::Relaxed);
             if std::process::Command::new("python3")
-                .arg("assemble_trace.py")
+                .arg("compile_shaders.py")
                 .current_dir("tape-drive")
                 .status()
                 .is_err()
@@ -2271,10 +2105,7 @@ impl App {
             self.thread_work_creation
                 .send(WorkItem::CreateCSG(
                     self.device.clone(),
-                    self.trace_module.clone(),
+                    self.replacement_modules.clone(),
-                    self.normals_module.clone(),
-                    self.fuzz_module.clone(),
-                    self.enable_gen_module.clone(),
                     rcx.render_pass.clone(),
                     self.pipeline_cache.clone(),
                     rcx.shader_modules.clone(),
@@ -2356,10 +2187,7 @@ impl App {
                         .send(WorkItem::RecompilePipelines(
                             csg.clone(),
                             self.device.clone(),
-                            self.trace_module.clone(),
+                            self.replacement_modules.clone(),
-                            self.normals_module.clone(),
-                            self.fuzz_module.clone(),
-                            self.enable_gen_module.clone(),
                             rcx.render_pass.clone(),
                             self.pipeline_cache.clone(),
                             rcx.shader_modules.clone(),
@@ -2890,7 +2718,7 @@ fn pipeline_recompile(
     memory_allocator: &Arc<dyn MemoryAllocator>,
     render_pass: &Arc<RenderPass>,
     cache: &Arc<PipelineCache>,
-    shader_modules: &ShaderModules,
+    shader_modules: &Arc<RwLock<HashMap<String, Arc<ShaderModule>>>>,
     debug: &PreviousDebug,
 ) -> (
     Arc<GraphicsPipeline>,
@@ -2903,8 +2731,9 @@ fn pipeline_recompile(
 
     let dynamic_state = [DynamicState::Viewport].into_iter().collect::<HashSet<_>>();
 
-    let fullscreen_vs_entry = shader_modules
+    let shader_modules = shader_modules.read().unwrap();
-        .fullscreen_vs
+
+    let fullscreen_vs_entry = shader_modules["fullscreen.vert"]
         .specialize(specs.clone())
         .unwrap()
         .single_entry_point()
@@ -2914,8 +2743,7 @@ fn pipeline_recompile(
 
     let fullscreen_vs_info = PipelineShaderStageCreateInfo::new(fullscreen_vs_entry);
     let lighting_fs_info = PipelineShaderStageCreateInfo::new(
-        shader_modules
+        shader_modules["lighting.frag"]
-            .lighting_fs
             .specialize(specs.clone())
             .unwrap()
             .single_entry_point()
@@ -2967,20 +2795,17 @@ fn pipeline_recompile(
     };
 
     let rt_pipeline = {
-        let raygen = shader_modules
+        let raygen = shader_modules["mesh.rgen"]
-            .raygen
             .specialize(specs.clone())
             .unwrap()
             .single_entry_point()
             .unwrap();
-        let closest_hit = shader_modules
+        let closest_hit = shader_modules["mesh.rchit"]
-            .closest_hit
             .specialize(specs.clone())
             .unwrap()
             .single_entry_point()
             .unwrap();
-        let miss = shader_modules
+        let miss = shader_modules["mesh.rmiss"]
-            .miss
             .specialize(specs.clone())
             .unwrap()
             .single_entry_point()
@@ -3088,8 +2913,8 @@ fn relpath(path: &str) -> PathBuf {
     Path::new(env!("CARGO_MANIFEST_DIR")).join(path)
 }
 
-fn read_spirv_words_from_file(name: &str) -> std::io::Result<Vec<u32>> {
+fn read_spirv_words_from_file(path: PathBuf) -> std::io::Result<Vec<u32>> {
-    let mut file = std::fs::File::open(format!("tape-drive/shaders_out/{name}"))?;
+    let mut file = std::fs::File::open(path)?;
     let mut bytes = vec![];
     file.read_to_end(&mut bytes).unwrap();
 

tape-drive/src/objects.rs

@@ -1,6 +1,10 @@
-use std::{collections::HashMap, io::Read, sync::Arc};
+use std::{
+    io::Read,
+    sync::{Arc, RwLock},
+};
 
 use bytemuck::{Pod, Zeroable};
+use foldhash::{HashMap, HashMapExt};
 use glam::{self, Vec3};
 use obj::{LoadConfig, ObjData, ObjError};
 use vulkano::{
@@ -15,7 +19,8 @@ use vulkano::{
 
 use crate::{ssa::SSATape, trace_vs::Object};
 
-pub(crate) const PLATONIC_SOLIDS: [(&str, &[u8]); 1] = [("Buny", include_bytes!("data/bunny.obj"))];
+pub(crate) const PLATONIC_SOLIDS: [(&str, &[u8]); 1] =
+    [("Buny", include_bytes!("../data/bunny.obj"))];
 
 #[repr(C)]
 #[repr(align(16))]
@@ -49,10 +54,7 @@ pub(crate) struct CSG {
     pub(crate) roughness: f32,
     pub(crate) metallic: f32,
     pub(crate) new_pipelines_needed: bool,
-    pub(crate) trace_shader_module: Arc<ShaderModule>,
+    pub(crate) replacement_modules: Arc<RwLock<HashMap<String, Arc<ShaderModule>>>>,
-    pub(crate) normals_shader_module: Arc<ShaderModule>,
-    pub(crate) fuzz_shader_module: Arc<ShaderModule>,
-    pub(crate) enable_gen_shader_module: Arc<ShaderModule>,
     pub(crate) trace_pipeline: Arc<GraphicsPipeline>,
     pub(crate) normals_pipeline: Arc<GraphicsPipeline>,
     pub(crate) fuzz_pipeline: Arc<ComputePipeline>,

tape-drive/src/threads.rs

@@ -6,7 +6,7 @@ use std::{
     time::Instant,
 };
 
-use foldhash::HashSet;
+use foldhash::{HashMap, HashMapExt, HashSet};
 use glam::{EulerRot, Mat4, Vec3};
 use log::info;
 use rand::{Rng, SeedableRng, rngs::SmallRng};
@@ -39,8 +39,8 @@ use vulkano::{
 };
 
 use crate::{
-    DUMP_SPV_TO_FILE, IVertex, MAXIMUM_SUBDIVISION, MINUMUM_SUBDIVISION, ShaderModules,
+    DUMP_SPV_TO_FILE, IVertex, MAXIMUM_SUBDIVISION, MINUMUM_SUBDIVISION, get_spec_constants,
-    get_spec_constants, gpu_upload,
+    gpu_upload,
     gui::PreviousDebug,
     interpreters::{VALUE_0, interval::IntervalInterpreter, point::PointInterpreter},
     objects::CSG,
@@ -53,13 +53,10 @@ use crate::{
 pub enum WorkItem {
     CreateCSG(
         Arc<Device>,
-        Module,
+        Arc<RwLock<HashMap<String, Module>>>,
-        Module,
-        Module,
-        Module,
         Arc<RenderPass>,
         Arc<PipelineCache>,
-        ShaderModules,
+        Arc<RwLock<HashMap<String, Arc<ShaderModule>>>>,
         PreviousDebug,
         Arc<Mutex<SubbufferAllocator>>,
         Arc<Mutex<SubbufferAllocator>>,
@@ -82,13 +79,10 @@ pub enum WorkItem {
     RecompilePipelines(
         Arc<RwLock<CSG>>,
         Arc<Device>,
-        Module,
+        Arc<RwLock<HashMap<String, Module>>>,
-        Module,
-        Module,
-        Module,
         Arc<RenderPass>,
         Arc<PipelineCache>,
-        ShaderModules,
+        Arc<RwLock<HashMap<String, Arc<ShaderModule>>>>,
         PreviousDebug,
         usize,
     ),
@@ -105,13 +99,10 @@ pub fn thread_loop(recv: mpmc::Receiver<WorkItem>, send: mpsc::SyncSender<WorkCo
         match work {
             WorkItem::CreateCSG(
                 device,
-                trace_module,
+                replacement_modules,
-                normals_module,
-                fuzz_module,
-                enable_gen_module,
                 render_pass,
                 cache,
-                modules,
+                normal_modules,
                 debug,
                 block_enable_allocator,
                 aabb_allocator,
@@ -125,25 +116,23 @@ pub fn thread_loop(recv: mpmc::Receiver<WorkItem>, send: mpsc::SyncSender<WorkCo
                 let csg_start = Instant::now();
                 let parts = create_csg(seed);
 
-                let trace_shader_module =
+                let mut new_replacement_modules = HashMap::new();
-                    sdf_specialize_module(device.clone(), &parts, trace_module, "trace");
+
-                let normals_shader_module =
+                for (name, module) in replacement_modules.read().unwrap().iter() {
-                    sdf_specialize_module(device.clone(), &parts, normals_module, "normals");
+                    let specialised_module =
-                let fuzz_shader_module =
+                        sdf_specialize_module(device.clone(), &parts, module.clone(), name);
-                    sdf_specialize_module(device.clone(), &parts, fuzz_module, "fuzz");
+                    new_replacement_modules.insert(name.clone(), specialised_module);
-                let enable_gen_shader_module =
+                }
-                    sdf_specialize_module(device.clone(), &parts, enable_gen_module, "enable_gen");
+
+                let replacement_modules = Arc::new(RwLock::new(new_replacement_modules));
 
                 let (trace_pipeline, normals_pipeline, fuzz_pipeline, enable_gen_pipeline) =
                     deferred_pipelines_recompile(
                         device,
                         render_pass,
                         cache,
-                        modules,
+                        normal_modules,
-                        trace_shader_module.clone(),
+                        replacement_modules.clone(),
-                        normals_shader_module.clone(),
-                        fuzz_shader_module.clone(),
-                        enable_gen_shader_module.clone(),
                         debug,
                     );
 
@@ -236,10 +225,7 @@ pub fn thread_loop(recv: mpmc::Receiver<WorkItem>, send: mpsc::SyncSender<WorkCo
                     rot: Vec3::ZERO,
                     scale: Vec3::ONE,
                     new_pipelines_needed: false,
-                    trace_shader_module,
+                    replacement_modules,
-                    normals_shader_module,
-                    fuzz_shader_module,
-                    enable_gen_shader_module,
                     trace_pipeline,
                     normals_pipeline,
                     fuzz_pipeline,
@@ -309,30 +295,26 @@ pub fn thread_loop(recv: mpmc::Receiver<WorkItem>, send: mpsc::SyncSender<WorkCo
             WorkItem::RecompilePipelines(
                 csg,
                 device,
-                trace_module,
+                replacement_modules,
-                normals_module,
-                fuzz_module,
-                enable_gen_module,
                 render_pass,
                 cache,
-                modules,
+                normal_modules,
                 debug,
                 index,
             ) => {
                 let csg_start = Instant::now();
                 let mut csg = csg.write().unwrap();
-                csg.trace_shader_module =
+
-                    sdf_specialize_module(device.clone(), &csg.parts, trace_module, "trace");
+                let mut new_replacement_modules = csg.replacement_modules.write().unwrap();
-                csg.normals_shader_module =
+                new_replacement_modules.clear();
-                    sdf_specialize_module(device.clone(), &csg.parts, normals_module, "normals");
+
-                csg.fuzz_shader_module =
+                for (name, module) in replacement_modules.read().unwrap().iter() {
-                    sdf_specialize_module(device.clone(), &csg.parts, fuzz_module, "fuzz");
+                    let specialised_module =
-                csg.enable_gen_shader_module = sdf_specialize_module(
+                        sdf_specialize_module(device.clone(), &csg.parts, module.clone(), name);
-                    device.clone(),
+                    new_replacement_modules.insert(name.clone(), specialised_module);
-                    &csg.parts,
+                }
-                    enable_gen_module,
+
-                    "enable_gen",
+                drop(new_replacement_modules);
-                );
 
                 (
                     csg.trace_pipeline,
@@ -343,13 +325,11 @@ pub fn thread_loop(recv: mpmc::Receiver<WorkItem>, send: mpsc::SyncSender<WorkCo
                     device,
                     render_pass,
                     cache,
-                    modules,
+                    normal_modules,
-                    csg.trace_shader_module.clone(),
+                    csg.replacement_modules.clone(),
-                    csg.normals_shader_module.clone(),
-                    csg.fuzz_shader_module.clone(),
-                    csg.enable_gen_shader_module.clone(),
                     debug,
                 );
+
                 csg.new_pipelines_needed = false;
                 send.send(WorkComplete::RecompilePipelines(index)).unwrap();
 
@@ -367,11 +347,8 @@ fn deferred_pipelines_recompile(
     device: Arc<Device>,
     render_pass: Arc<RenderPass>,
     cache: Arc<PipelineCache>,
-    shader_modules: ShaderModules,
+    shader_modules: Arc<RwLock<HashMap<String, Arc<ShaderModule>>>>,
-    trace_shader_module: Arc<ShaderModule>,
+    replacement_modules: Arc<RwLock<HashMap<String, Arc<ShaderModule>>>>,
-    normals_shader_module: Arc<ShaderModule>,
-    fuzz_shader_module: Arc<ShaderModule>,
-    enable_gen_shader_module: Arc<ShaderModule>,
     debug: PreviousDebug,
 ) -> (
     Arc<GraphicsPipeline>,
@@ -383,8 +360,10 @@ fn deferred_pipelines_recompile(
 
     let dynamic_state = [DynamicState::Viewport].into_iter().collect::<HashSet<_>>();
 
-    let trace_vs_entry = shader_modules
+    let shader_modules = shader_modules.read().unwrap();
-        .trace_vs
+    let replacement_modules = replacement_modules.read().unwrap();
+
+    let trace_vs_entry = shader_modules["trace.vert"]
         .specialize(specs.clone())
         .unwrap()
         .single_entry_point()
@@ -395,7 +374,7 @@ fn deferred_pipelines_recompile(
     let trace_vs_info = PipelineShaderStageCreateInfo::new(trace_vs_entry);
 
     let trace_fs_info = PipelineShaderStageCreateInfo::new(
-        trace_shader_module
+        replacement_modules["trace.frag"]
             .specialize(specs.clone())
             .unwrap()
             .single_entry_point()
@@ -445,8 +424,7 @@ fn deferred_pipelines_recompile(
     )
     .unwrap();
 
-    let fullscreen_vs_entry = shader_modules
+    let fullscreen_vs_entry = shader_modules["fullscreen.vert"]
-        .fullscreen_vs
         .specialize(specs.clone())
         .unwrap()
         .single_entry_point()
@@ -457,7 +435,7 @@ fn deferred_pipelines_recompile(
     let fullscreen_vs_info = PipelineShaderStageCreateInfo::new(fullscreen_vs_entry);
 
     let normals_fs_info = PipelineShaderStageCreateInfo::new(
-        normals_shader_module
+        replacement_modules["normals.frag"]
             .specialize(specs.clone())
             .unwrap()
             .single_entry_point()
@@ -506,7 +484,7 @@ fn deferred_pipelines_recompile(
     .unwrap();
 
     let fuzz_cs_info = PipelineShaderStageCreateInfo::new(
-        fuzz_shader_module
+        replacement_modules["fuzz.comp"]
             .specialize(specs.clone())
             .unwrap()
             .single_entry_point()
@@ -529,7 +507,7 @@ fn deferred_pipelines_recompile(
     .expect("failed to create fuzz pipeline");
 
     let enable_gen_cs_info = PipelineShaderStageCreateInfo::new(
-        enable_gen_shader_module
+        replacement_modules["enable_gen.comp"]
             .specialize(specs.clone())
             .unwrap()
             .single_entry_point()