Vulkan debug messager (yay)

doesn't deal yet with the corner case of reporting errors to do with instance creation and destruction

Force-Push: yes
Change-Id: I6094d960081d8cd3123ae6fe46f77cb133948f39

3 files changed, 187 insertions, 23 deletions

diff --git a/Cargo.toml b/Cargo.toml
index 26435be..684023d 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -4,6 +4,10 @@ version = "0.1.0"
 authors = ["Irene Knapp <ireneista@irenes.space>"]
 edition = "2024"
 
+[features]
+# Validation is turned on by debug assertions, and also by this feature.
+vulkan-validation = [ ]
+
 [dependencies]
 libloading = "0.8.9"
 winit = "0.30.13"
diff --git a/flake.nix b/flake.nix
index 702c355..a883414 100644
--- a/flake.nix
+++ b/flake.nix
@@ -26,6 +26,10 @@
         libxi
         libxkbcommon
         vulkan-loader
+
+        #   It might be nice to make this conditional; see also the patchelf
+        # call below.
+        vulkan-validation-layers
       ];
   in {
     packages = forAllSystems (system: let pkgs = nixpkgsFor.${system}; in {
@@ -39,6 +43,11 @@
         # This needs to apply only to the top-level derivation, not to the
         # dependencies.
         preFixup = ''
+          #   For whatever reason, Vulkan layers need to be added in a
+          # separate call before Vulkan itself is.
+          patchelf --add-needed libVkLayer_khronos_validation.so \
+                   $out/bin/surreality
+
           patchelf --add-needed libxkbcommon-x11.so \
                    --add-needed libvulkan.so.1 \
                    $out/bin/surreality
@@ -56,7 +65,12 @@
 
         #   This makes cargo run work; mind that you don't let it mask a
         # problem with the nix build.
-        LD_LIBRARY_PATH = "${pkgs.libxkbcommon}/lib:${pkgs.vulkan-loader}/lib";
+        LD_LIBRARY_PATH = pkgs.lib.join ":"
+            (pkgs.lib.map (pkg: "${pkg}/lib") (with pkgs; [
+              libxkbcommon
+              vulkan-loader
+              vulkan-validation-layers
+            ]));
       };
     });
   };
diff --git a/src/main.rs b/src/main.rs
index c7e12a6..0312eb3 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -1,10 +1,13 @@
 #![deny(unsafe_code)]
 use std::cell::OnceCell;
+use std::collections::HashSet;
+use std::ffi::{ c_void, CStr };
 use vulkanalia::{ Entry, Instance, Version };
 use vulkanalia::loader::{ LibloadingLoader, LIBRARY };
 use vulkanalia::vk::{ self, HasBuilder,
                       ApplicationInfo, InstanceCreateInfo,
-                      DeviceV1_4, EntryV1_4, InstanceV1_4 };
+                      DeviceV1_4, EntryV1_0, InstanceV1_0,
+                      ExtDebugUtilsExtensionInstanceCommands };
 use winit::dpi::LogicalSize;
 use winit::application::ApplicationHandler;
 use winit::event::WindowEvent;
@@ -101,6 +104,10 @@ struct Surreality {
   window: OnceCell<Window>,
   entry: OnceCell<Entry>,
   instance: OnceCell<Instance>,
+
+  //   Vulkan spells "messager" as "messenger", but this is absurd
+  // over-formality and we don't indulge it.
+  debug_messager: OnceCell<vk::DebugUtilsMessengerEXT>,
 }
 
 impl Surreality {
@@ -109,6 +116,7 @@ impl Surreality {
       window: OnceCell::new(),
       entry: OnceCell::new(),
       instance: OnceCell::new(),
+      debug_messager: OnceCell::new(),
     }
   }
 
@@ -118,17 +126,20 @@ impl Surreality {
     }
 
     if self.entry.get().is_none() {
-      self.init_entry()?;
+      self.init_vulkan_entry()?;
     }
 
     if self.instance.get().is_none() {
-      self.init_instance()?;
+      self.init_vulkan_instance()?;
     }
 
     Ok(())
   }
 
   fn init_window(&mut self, event_loop: &ActiveEventLoop) -> Result<()> {
+    //   Notice that we do this before having a Vulkan instance. The window is
+    // actually a parameter needed to create the instance; see
+    // init_vulkan_instance(), below.
     let window_attributes = WindowAttributes::default()
             .with_title("Love, Curiosity, Justice")
             .with_inner_size(LogicalSize::new(1024, 768));
@@ -140,32 +151,70 @@ impl Surreality {
     Ok(())
   }
 
-  fn init_entry(&mut self) -> Result<()> {
+  fn init_vulkan_entry(&mut self) -> Result<()> {
+    //   Okay, so, a Vulkan "entry" is a small set of functions which are used
+    // to dynamically load all the rest of Vulkan. It's our responsibility to
+    // know how to load the entry, then it will take care of the rest. At
+    // least, that's the theory, but also see flake.nix for all the
+    // FHS-centric assumptions it makes that we have to correct.
+    //
+    //   Anyway, Vulkanalia offers an integration with libloading, which is a
+    // crate that wraps POSIX dlopen(). We use that; it's enabled by
+    // Vulkanalia's "libloading" feature.
     #[allow(unsafe_code)]
-    let loader = unsafe { LibloadingLoader::new(LIBRARY)? };
+    let loader = unsafe { LibloadingLoader::new(LIBRARY) }?;
     #[allow(unsafe_code)]
-    let entry = unsafe { Entry::new(loader)? };
+    let entry = unsafe { Entry::new(loader) }?;
 
     let _ = self.entry.set(entry);
 
     Ok(())
   }
 
-  fn init_instance(&mut self) -> Result<()> {
+  fn init_vulkan_instance(&mut self) -> Result<()> {
     let entry = self.entry.get().unwrap();
 
-    let application_info = ApplicationInfo::builder()
-      .application_name(b"Surreality\0")
-      .application_version(vk::make_version(1, 0, 0))
-      .engine_name(b"Surreality\0")
-      .engine_version(vk::make_version(1, 0, 0))
-      .api_version(vk::make_version(1, 0, 0));
+    let enable_validation = cfg!(feature = "vulkan-validation")
+                            || cfg!(debug_assertions);
+
+    //   Since there's a lot of factors going into our instance creation
+    // request, we'll build up the parameters mutably.
+    let mut flags = vk::InstanceCreateFlags::empty();
+    let mut extensions = Vec::new();
+    let mut layers = Vec::new();
+
+    //   Before we go any further, use Vulkan's introspection to list off
+    // what's available.
+    let mut available_extensions = HashSet::new();
+    #[allow(unsafe_code)]
+    for extension in
+            unsafe { entry.enumerate_instance_extension_properties(None) }?
+    {
+      available_extensions.insert(extension.extension_name);
+    }
+    let available_extensions = available_extensions;
 
-    let mut extensions = vulkanalia::window::get_required_instance_extensions(
+    let mut available_layers = HashSet::new();
+    #[allow(unsafe_code)]
+    for layer in unsafe { entry.enumerate_instance_layer_properties() }? {
+      available_layers.insert(layer.layer_name);
+    }
+    let available_layers = available_layers;
+
+    //   There are certain extensions which are required by the nature of our
+    // windowing system. Happily, vulanaklia knows how to deal with that based
+    // on the type of window we give it.
+    //
+    //   This is possible because of an integration between Vulkanalia and
+    // winit, which is enabled by Vulkanalia's "window" feature.
+    for extension in vulkanalia::window::get_required_instance_extensions(
                          self.window.get().unwrap())
-        .iter().map(|item| item.as_ptr()).collect::<Vec<_>>();
+    {
+      extensions.push(extension.as_ptr());
+    }
 
-    let mut flags = vk::InstanceCreateFlags::empty();
+    //   Deal with Vulkan's thing about opting in to non-conforming
+    // implementations.
     if entry.version()? >= VULKAN_FIRST_PORTABILITY_VERSION {
       if cfg!(target_os = "macos") {
         // Vulkan on the Mac is not fully conforming.
@@ -177,15 +226,65 @@ impl Surreality {
       }
     }
 
+    // Request the LunarG validation layer, when appropriate.
+    if enable_validation {
+      let layer_name = vk::ExtensionName::from_bytes(
+                           b"VK_LAYER_KHRONOS_validation");
+      if available_layers.contains(&layer_name) {
+        layers.push(layer_name.as_ptr());
+      } else {
+        eprintln!("Vulkan validation requested at build time, \
+                   but no validation layer available.");
+      }
+    }
+
+    //   Request the debug extension. This cooperates with code below, which
+    // runs after the instance is created.
+    let debug_extension_name = vk::EXT_DEBUG_UTILS_EXTENSION.name;
+    if available_extensions.contains(&debug_extension_name) {
+      extensions.push(debug_extension_name.as_ptr());
+    } else {
+      eprintln!("Vulkan debug extension not available; \
+                 this may mean other messages don't show up.");
+    }
+
+    let application_info = ApplicationInfo::builder()
+            .application_name(b"Surreality\0")
+            .application_version(vk::make_version(1, 0, 0))
+            .engine_name(b"Surreality\0")
+            .engine_version(vk::make_version(1, 0, 0))
+            .api_version(vk::make_version(1, 0, 0));
+
     let instance_create_info = InstanceCreateInfo::builder()
-      .application_info(&application_info)
-      .enabled_extension_names(&extensions)
-      .flags(flags);
+            .application_info(&application_info)
+            .flags(flags)
+            .enabled_extension_names(&extensions)
+            .enabled_layer_names(&layers);
 
     #[allow(unsafe_code)]
     let instance = unsafe {
-      entry.create_instance(&instance_create_info, None)?
-    };
+      entry.create_instance(&instance_create_info, None)
+    }?;
+
+    //   Configure the debug extension. This cooperates with code above, which
+    // requests the extension.
+    if available_extensions.contains(&debug_extension_name)
+       && self.debug_messager.get().is_none()
+    {
+      let debug_info = vk::DebugUtilsMessengerCreateInfoEXT::builder()
+              .message_severity(vk::DebugUtilsMessageSeverityFlagsEXT::all())
+              .message_type(vk::DebugUtilsMessageTypeFlagsEXT::GENERAL
+                            | vk::DebugUtilsMessageTypeFlagsEXT::VALIDATION
+                            | vk::DebugUtilsMessageTypeFlagsEXT::PERFORMANCE)
+              .user_callback(Some(debug_messager_callback));
+
+      #[allow(unsafe_code)]
+      let debug_messager = unsafe {
+        instance.create_debug_utils_messenger_ext(&debug_info, None)
+      }?;
+
+      let _ = self.debug_messager.set(debug_messager);
+    }
 
     let _ = self.instance.set(instance);
 
@@ -205,10 +304,27 @@ impl Surreality {
   }
 }
 
+impl Drop for Surreality {
+  fn drop(&mut self) {
+    if let Some(debug_messager) = self.debug_messager.get()
+       && let Some(instance) = self.instance.get()
+    {
+      #[allow(unsafe_code)]
+      unsafe {
+        instance.destroy_debug_utils_messenger_ext(*debug_messager, None);
+      }
+    }
+
+    if let Some(instance) = self.instance.get() {
+      #[allow(unsafe_code)]
+      unsafe { instance.destroy_instance(None) };
+    }
+  }
+}
+
 impl ApplicationHandler for Surreality {
   fn resumed(&mut self, event_loop: &ActiveEventLoop) {
     ignore_errors(move || {
-      println!("resumed");
       self.init(event_loop)?;
 
       Ok(())
@@ -254,3 +370,33 @@ fn main() -> std::process::ExitCode {
   }
 }
 
+
+extern "system" fn debug_messager_callback(
+    severity: vk::DebugUtilsMessageSeverityFlagsEXT,
+    flags: vk::DebugUtilsMessageTypeFlagsEXT,
+    data: *const vk::DebugUtilsMessengerCallbackDataEXT,
+    _context: *mut c_void) -> vk::Bool32
+{
+  #[allow(unsafe_code)]
+  let data = unsafe { *data };
+  #[allow(unsafe_code)]
+  let text = unsafe { CStr::from_ptr(data.message) }.to_string_lossy();
+
+  let severity = if severity >= vk::DebugUtilsMessageSeverityFlagsEXT::ERROR {
+    "error"
+  } else if severity >= vk::DebugUtilsMessageSeverityFlagsEXT::WARNING {
+    "warning"
+  } else if severity >= vk::DebugUtilsMessageSeverityFlagsEXT::INFO {
+    "informational message"
+  } else {
+    "message of unknown, very minor severity"
+  };
+
+  eprintln!("Vulkan {}: {} (flags {:?})", severity, text, flags);
+
+  //   A return value of true would tell the validation layer we're unhappy
+  // with it, for the sake of conformance testing. We're not a conformance
+  // test so anything it does is fine with us.
+  vk::FALSE
+}
+