Start rendering work

.gitignore

@@ -1,4 +1,6 @@
 /target
 /Cargo.lock
 
+*.spv
+
 vk_functions

build.rs

@@ -9,7 +9,7 @@ const VK_HEADER: &[&str] = &[
 ];
 const FN_PREFIX: &str = "PFN_";
 
-fn main() {
+fn query_vulkan_function_typedefs() {
     let mut fns = Vec::new();
 
     for header in VK_HEADER {
@@ -41,3 +41,31 @@ fn main() {
         file.write_all(format!("{}\n", func).as_bytes()).unwrap();
     }
 }
+
+fn compile_shader() {
+    Command::new("glslangValidator")
+        .arg("--help")
+        .output()
+        .expect("Failed to execute glslangValidator. Maybe you need to install it first?");
+
+    Command::new("glslangValidator")
+        .arg("-V")
+        .arg("src/overlay/shader/single_color.vert")
+        .arg("-o")
+        .arg("src/overlay/shader/single_color.vert.spv")
+        .output()
+        .expect("Failed to compile single_color.vert");
+
+    Command::new("glslangValidator")
+        .arg("-V")
+        .arg("src/overlay/shader/single_color.frag")
+        .arg("-o")
+        .arg("src/overlay/shader/single_color.frag.spv")
+        .output()
+        .expect("Failed to compile single_color.frag");
+}
+
+fn main() {
+    query_vulkan_function_typedefs();
+    compile_shader();
+}

src/enums.rs

@@ -27,6 +27,8 @@ pub enum Functions {
     DestroyDevice(PFN_vkDestroyDevice),
     CreateSwapchain(PFN_vkCreateSwapchainKHR),
     QueueSubmit(PFN_vkQueueSubmit),
+    GetDeviceQueue(PFN_vkGetDeviceQueue),
+    AcquireNextImageKHR(PFN_vkAcquireNextImageKHR),
 }
 
 impl Functions {
@@ -41,6 +43,8 @@ impl Functions {
             Functions::DestroyDevice(func) => unsafe { mem::transmute(func) },
             Functions::CreateSwapchain(func) => unsafe { mem::transmute(func) },
             Functions::QueueSubmit(func) => unsafe { mem::transmute(func) },
+            Functions::GetDeviceQueue(func) => unsafe { mem::transmute(func) },
+            Functions::AcquireNextImageKHR(func) => unsafe { mem::transmute(func) },
         }
     }
 }

src/lib.rs

@@ -283,19 +283,30 @@ extern "system" fn create_swapchain(
 ) -> VkResult {
     write_log(" ================== vulkan layer create swapchain ==================");
 
-    let swapchain = match unsafe { Swapchain::from_ci(OVERLAY.device(), &*create_info) } {
-        Ok(swapchain) => swapchain,
-        Err(err) => {
-            write_log(format!("create swapchain failed: {:?}", err));
-            return VK_ERROR_INITIALIZATION_FAILED;
-        }
-    };
+    let create_swapchain: PFN_vkCreateSwapchainKHR =
+        match vk_handles().handle("vkCreateSwapchainKHR") {
+            Some(create_swapchain) => unsafe { mem::transmute(create_swapchain) },
+            None => {
+                write_log("failed querying vkCreateSwapchainKHR");
+                return VK_ERROR_INITIALIZATION_FAILED;
+            }
+        };
 
-    unsafe { *p_swapchain = swapchain.vk_handle() };
-    if let Err(err) = unsafe { OVERLAY.create_rendering(swapchain) } {
-        write_log(format!("create overlay rendering struct failed: {:?}", err));
-        return VK_ERROR_INITIALIZATION_FAILED;
-    }
+    create_swapchain(_device, create_info, _allocator, p_swapchain);
+
+    let swapchain =
+        match unsafe { Swapchain::from_raw(OVERLAY.device(), &*create_info, *p_swapchain) } {
+            Ok(swapchain) => swapchain,
+            Err(err) => {
+                write_log(format!("create swapchain failed: {:?}", err));
+                return VK_ERROR_INITIALIZATION_FAILED;
+            }
+        };
+
+    // if let Err(err) = unsafe { OVERLAY.create_rendering(swapchain) } {
+    //     write_log(format!("create overlay rendering struct failed: {:?}", err));
+    //     return VK_ERROR_INITIALIZATION_FAILED;
+    // }
 
     VK_SUCCESS
 }
@@ -306,11 +317,15 @@ extern "system" fn submit_queue(
     submits: *const VkSubmitInfo,
     fence: VkFence,
 ) -> VkResult {
-    // unsafe { return QUEUE_SUBMIT(queue, submit_count, submits, fence): }
+    write_log(" ================== vulkan layer submit queue ==================");
+
+    unsafe {
+        return QUEUE_SUBMIT(queue, submit_count, submits, fence);
+    }
 
     unsafe {
         let input_submit_info = slice::from_raw_parts(submits, submit_count as usize);
-        let overlay_submit = match OVERLAY.render() {
+        let overlay_submit = match OVERLAY.render(queue) {
             Ok(submit) => submit,
             Err(err) => {
                 write_log(format!("overlay rendering failed: {:?}", err));
@@ -329,6 +344,67 @@ extern "system" fn submit_queue(
     }
 }
 
+extern "system" fn get_device_queue(
+    device: VkDevice,
+    queue_family_index: u32,
+    queue_index: u32,
+    p_queue: *mut VkQueue,
+) {
+    write_log(" ================== vulkan layer get device queue ==================");
+
+    let get_device_queue: PFN_vkGetDeviceQueue = match vk_handles().handle("vkGetDeviceQueue") {
+        Some(get_queue) => unsafe { mem::transmute(get_queue) },
+        None => {
+            write_log("failed querying vkGetDeviceQueue");
+            panic!("failed querying vkGetDeviceQueue");
+        }
+    };
+
+    get_device_queue(device, queue_family_index, queue_index, p_queue);
+
+    unsafe {
+        let arc_device = OVERLAY.device();
+        OVERLAY.add_queue(Queue::new(
+            arc_device,
+            *p_queue,
+            queue_family_index,
+            queue_index,
+        ));
+    }
+}
+
+extern "system" fn acquire_next_image(
+    _device: VkDevice,
+    swapchain: VkSwapchainKHR,
+    timeout: u64,
+    semaphore: VkSemaphore,
+    fence: VkFence,
+    image_index: *mut u32,
+) -> VkResult {
+    write_log(" ================== vulkan layer acquire next image ==================");
+
+    unsafe {
+        *image_index = match OVERLAY.device().acquire_next_image(
+            swapchain,
+            timeout,
+            Some(semaphore),
+            Some(fence),
+        ) {
+            Ok(res) => match res {
+                OutOfDate::Ok(index) => index,
+                OutOfDate::OutOfDate => return VK_ERROR_OUT_OF_DATE_KHR,
+                OutOfDate::TimeOut => return VK_TIMEOUT,
+            },
+            Err(err) => {
+                write_log(format!("Failed acquiring next image: {:?}", err));
+                return VK_ERROR_DEVICE_LOST;
+            }
+        }
+    }
+
+    VK_SUCCESS
+}
+
 pub fn write_log(msg: impl ToString) {
     if let Ok(mut file) = OpenOptions::new().append(true).create(true).open(LOG_FILE) {
         if let Err(_) = file.write_all(format!("{}\n", msg.to_string()).as_bytes()) {}
@@ -343,7 +419,8 @@ pub fn get_vk_func(s: &str) -> Option<Functions> {
         "vkDestroyInstance" => Some(Functions::DestroyInstance(destroy_instance)),
         "vkCreateSwapchainKHR" => Some(Functions::CreateSwapchain(create_swapchain)),
         "vkQueueSubmit" => Some(Functions::QueueSubmit(submit_queue)),
-
+        "vkGetDeviceQueue" => Some(Functions::GetDeviceQueue(get_device_queue)),
+        // "vkAcquireNextImageKHR" => Some(Functions::AcquireNextImageKHR(acquire_next_image)),
         _ => None,
     }
 }

src/overlay/mod.rs

@@ -1,16 +1,18 @@
 use self::rendering::Rendering;
 
+mod pipeline;
 mod rendering;
 
 use anyhow::Result;
-use std::sync::Arc;
+use std::sync::{Arc, Mutex};
 use vulkan_rs::prelude::*;
 
 #[derive(Default)]
 pub struct Overlay {
     instance: Option<Arc<Instance>>,
     device: Option<Arc<Device>>,
-    rendering: Option<Rendering>,
+    queues: Vec<Arc<Mutex<Queue>>>,
+    renderings: Vec<Rendering>,
 }
 
 impl Overlay {
@@ -18,7 +20,8 @@ impl Overlay {
         Self {
             instance: None,
             device: None,
-            rendering: None,
+            queues: Vec::new(),
+            renderings: Vec::new(),
         }
     }
 
@@ -38,13 +41,44 @@ impl Overlay {
         self.device.as_ref().unwrap().clone()
     }
 
-    pub fn create_rendering(&mut self, swapchain: Arc<Swapchain>) -> Result<()> {
-        self.rendering = Some(Rendering::new(swapchain));
+    pub fn add_queue(&mut self, queue: Arc<Mutex<Queue>>) {
+        self.queues.push(queue);
+    }
+
+    pub fn queue(&self, queue: VkQueue) -> Arc<Mutex<Queue>> {
+        match self
+            .queues
+            .iter()
+            .find(|q| q.lock().unwrap().vk_handle() == queue)
+        {
+            Some(q) => q.clone(),
+            None => panic!(),
+        }
+    }
+
+    pub fn swapchain(&self, swapchain: VkSwapchainKHR) -> &Arc<Swapchain> {
+        match self
+            .renderings
+            .iter()
+            .find(|r| r.swapchain().vk_handle() == swapchain)
+        {
+            Some(s) => s.swapchain(),
+            None => panic!(),
+        }
+    }
+
+    pub fn add_rendering(&mut self, swapchain: Arc<Swapchain>) -> Result<()> {
+        // self.renderings
+        //     .push(Rendering::new(self.device(), self.queue(), swapchain)?);
 
         Ok(())
     }
 
-    pub fn render(&mut self) -> Result<VkSubmitInfo> {
-        self.rendering.as_mut().unwrap().render()
+    pub fn render(&mut self, queue: VkQueue) -> Result<VkSubmitInfo> {
+        // for rendering in self.renderings {
+        //     rendering.render()
+        // }
+
+        todo!()
     }
 }

src/overlay/pipeline.rs

@@ -0,0 +1,65 @@
+use anyhow::Result;
+use vulkan_rs::prelude::*;
+
+use std::{mem, sync::Arc};
+
+pub struct SingleColorPipeline {
+    pipeline: Arc<Pipeline>,
+    pipeline_layout: Arc<PipelineLayout>,
+
+    vertex_shader: Arc<ShaderModule>,
+    fragment_shader: Arc<ShaderModule>,
+}
+
+impl SingleColorPipeline {
+    pub fn new(device: Arc<Device>, renderpass: &Arc<RenderPass>) -> Result<Self> {
+        let vertex_shader = ShaderModule::from_slice(
+            device.clone(),
+            include_bytes!("shader/single_color.vert.spv"),
+            ShaderType::Vertex,
+        )?;
+        let fragment_shader = ShaderModule::from_slice(
+            device.clone(),
+            include_bytes!("shader/single_color.frag.spv"),
+            ShaderType::Fragment,
+        )?;
+
+        let pipeline_layout = PipelineLayout::builder().build(device.clone())?;
+
+        let pipeline = Pipeline::new_graphics()
+            .set_vertex_shader(
+                vertex_shader.clone(),
+                vec![VkVertexInputBindingDescription {
+                    binding: 0,
+                    stride: mem::size_of::<[f32; 4]>() as u32,
+                    inputRate: VK_VERTEX_INPUT_RATE_VERTEX,
+                }],
+                vec![
+                    // position
+                    VkVertexInputAttributeDescription {
+                        location: 0,
+                        binding: 0,
+                        format: VK_FORMAT_R32G32B32A32_SFLOAT,
+                        offset: 0,
+                    },
+                ],
+            )
+            .set_fragment_shader(fragment_shader.clone())
+            .input_assembly(VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST, false)
+            .default_depth_stencil(false, false)
+            .default_rasterization(VK_CULL_MODE_NONE, VK_FRONT_FACE_COUNTER_CLOCKWISE)
+            .default_multisample(VK_SAMPLE_COUNT_1_BIT)
+            .build(device, &pipeline_layout, &renderpass, 0)?;
+
+        Ok(Self {
+            vertex_shader,
+            fragment_shader,
+            pipeline,
+            pipeline_layout,
+        })
+    }
+
+    pub fn pipeline(&self) -> &Arc<Pipeline> {
+        &self.pipeline
+    }
+}

src/overlay/rendering.rs

@@ -1,15 +1,43 @@
 use anyhow::Result;
 use vulkan_rs::prelude::*;
 
-use std::sync::Arc;
+use std::sync::{Arc, Mutex};
+
+use super::pipeline::SingleColorPipeline;
 
 pub struct Rendering {
     swapchain: Arc<Swapchain>,
+    pipeline: SingleColorPipeline,
+    render_target: RenderTarget,
+    images: Vec<Arc<Image>>,
 }
 
 impl Rendering {
-    pub fn new(swapchain: Arc<Swapchain>) -> Self {
-        Self { swapchain }
+    pub fn new(
+        device: Arc<Device>,
+        queue: Arc<Mutex<Queue>>,
+        swapchain: Arc<Swapchain>,
+    ) -> Result<Self> {
+        let images = swapchain.wrap_images(&swapchain.vk_images()?, &queue)?;
+
+        let render_target = RenderTarget::builder()
+            .add_sub_pass(
+                SubPass::builder(swapchain.width(), swapchain.height())
+                    .set_prepared_targets(&images, 0, [0.0, 0.0, 0.0, 1.0], false)
+                    .build(&device, &queue)?,
+            )
+            .build(&device)?;
+
+        Ok(Self {
+            swapchain,
+            pipeline: SingleColorPipeline::new(device, render_target.render_pass())?,
+            render_target,
+            images,
+        })
+    }
+
+    pub fn swapchain(&self) -> &Arc<Swapchain> {
+        &self.swapchain
     }
 
     pub fn render(&mut self) -> Result<VkSubmitInfo> {

src/overlay/shader/single_color.frag

@@ -0,0 +1,12 @@
+#version 450
+
+layout (set = 0, binding = 0) uniform Color {
+	vec4 val;
+} color;
+
+layout (location = 0) out vec4 out_color;
+
+void main()
+{
+	out_color = vec4(color.val);
+}

src/overlay/shader/single_color.vert

@@ -0,0 +1,8 @@
+#version 450
+
+layout (location = 0) in vec4 position;
+
+void main()
+{
+	gl_Position = position;
+}