From e860b2413a7661cb38ad226fb3dd52b7bef31fa3 Mon Sep 17 00:00:00 2001
From: hodasemi <michaelh.95@t-online.de>
Date: Fri, 1 Dec 2023 14:51:47 +0100
Subject: [PATCH] Fix dead lock

---
 src/lib.rs          | 21 +++++----------------
 src/vk_layer/mod.rs | 28 ++++++++++++++++------------
 2 files changed, 21 insertions(+), 28 deletions(-)

diff --git a/src/lib.rs b/src/lib.rs
index 234447d..a1512cd 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -5,7 +5,7 @@ use std::{
     fs::{self, File, OpenOptions},
     io::Write,
     path::Path,
-    sync::{RwLock, RwLockReadGuard, RwLockWriteGuard},
+    sync::{Mutex, MutexGuard},
 };
 
 use lazy_static::lazy_static;
@@ -14,25 +14,14 @@ use overlay::{Overlay, OverlayConfig};
 static mut LOG_ENABLED: bool = true;
 static mut LOG_FILE: String = String::new();
 lazy_static! {
-    static ref OVERLAY: RwLock<Overlay> = RwLock::new(Overlay::new());
+    static ref OVERLAY: Mutex<Overlay> = Mutex::new(Overlay::new());
 }
 
-pub(crate) fn overlay() -> RwLockReadGuard<'static, Overlay> {
-    match OVERLAY.try_read() {
+pub(crate) fn overlay() -> MutexGuard<'static, Overlay> {
+    match OVERLAY.lock() {
         Ok(overlay) => overlay,
         Err(_) => {
-            write_log!("failed to get read handle from OVERLAY");
-            write_log!(std::backtrace::Backtrace::force_capture());
-            panic!();
-        }
-    }
-}
-
-pub(crate) fn overlay_mut() -> RwLockWriteGuard<'static, Overlay> {
-    match OVERLAY.try_write() {
-        Ok(overlay) => overlay,
-        Err(_) => {
-            write_log!("failed to get write handle from OVERLAY");
+            write_log!("failed to lock OVERLAY");
             write_log!(std::backtrace::Backtrace::force_capture());
             panic!();
         }
diff --git a/src/vk_layer/mod.rs b/src/vk_layer/mod.rs
index 405f7da..4236d0f 100644
--- a/src/vk_layer/mod.rs
+++ b/src/vk_layer/mod.rs
@@ -13,7 +13,7 @@ use std::{ffi::c_void, mem, os::raw::c_char, ptr};
 static mut ACQUIRE_NEXT_IMAGE: PFN_vkAcquireNextImageKHR =
     unsafe { mem::transmute(vkVoidFunction as *const c_void) };
 
-use crate::{get_config, logging, overlay, overlay_mut, write_log};
+use crate::{get_config, logging, overlay, write_log};
 
 #[no_mangle]
 #[allow(non_snake_case)]
@@ -24,7 +24,7 @@ pub(crate) extern "C" fn vkNegotiateLoaderLayerInterfaceVersion(
 
     check_logging(&home);
 
-    overlay_mut().set_config(get_config(&home));
+    overlay().set_config(get_config(&home));
 
     unsafe {
         *pVersionStruct = VkNegotiateLayerInterface {
@@ -200,7 +200,7 @@ pub(crate) extern "system" fn create_instance(
                 }
             };
 
-            overlay_mut().add_instance(ins);
+            overlay().add_instance(ins);
         }
 
         write_log!("-> created local instance handle");
@@ -220,7 +220,7 @@ pub(crate) extern "system" fn destroy_instance(
     unsafe {
         if let Some(vk_fn) = vk_handles().handle("vkDestroyInstance") {
             let destroy_instance: PFN_vkDestroyInstance = mem::transmute(vk_fn);
-            overlay_mut().remove_instance(instance);
+            overlay().remove_instance(instance);
 
             destroy_instance(instance, allocator);
         }
@@ -277,7 +277,9 @@ pub(crate) extern "system" fn create_device(
         };
     };
 
-    let instance = overlay().instance_by_physical_device(physical_device);
+    let mut overlay = overlay();
+
+    let instance = overlay.instance_by_physical_device(physical_device);
 
     let pdev = match PhysicalDevice::from_raw(instance, physical_device) {
         Ok(pdev) => pdev,
@@ -333,8 +335,8 @@ pub(crate) extern "system" fn create_device(
 
     write_log!("got queue from device");
 
-    overlay_mut().set_device(device.clone());
-    overlay_mut().set_queue(queue);
+    overlay.set_device(device.clone());
+    overlay.set_queue(queue);
 
     VK_SUCCESS
 }
@@ -377,12 +379,14 @@ pub(crate) extern "system" fn create_swapchain(
         *p_swapchain
     }));
 
-    overlay_mut().insert_swapchain(unsafe {
-        Swapchain::from_raw(overlay().device(_device), &*create_info, *p_swapchain)
-    });
+    let mut overlay = overlay();
+
+    let swapchain =
+        unsafe { Swapchain::from_raw(overlay.device(_device), &*create_info, *p_swapchain) };
+    overlay.insert_swapchain(swapchain);
 
     {
-        let instances = &overlay().instances;
+        let instances = &overlay.instances;
 
         write_log!(format!("number of instances: {}", instances.len()));
 
@@ -412,7 +416,7 @@ pub(crate) extern "system" fn create_swapchain(
 
     write_log!("-> created Arc<Swapchain>");
 
-    if let Err(err) = overlay_mut().create_rendering(_device) {
+    if let Err(err) = overlay.create_rendering(_device) {
         write_log!(format!("create overlay rendering struct failed: {:?}", err));
         return VK_ERROR_INITIALIZATION_FAILED;
     }