Add OXR swapchain interop

apps/viewer/src/main.cpp

@@ -52,20 +52,21 @@ int main(int argc, char** argv)
     auto pre = env::prefix("VKGS");
     auto validationLayers = pre.register_variable<bool>("VALIDATION_LAYERS");
     auto physicalDeviceId = pre.register_variable<uint8_t>("PHYSICAL_DEVICE");
-    auto immediateSwapchain = pre.register_variable<bool>("IMMEDIATE_SWAPCHAIN");
+    auto immediateSwapchainEnv = pre.register_variable<bool>("IMMEDIATE_SWAPCHAIN");
     auto envVars = pre.parse_and_validate();
 
     if (args::get(verboseFlag))
     {
         spdlog::set_level(spdlog::level::debug);
     }
 
+    bool immediateSwapchain = envVars.get_or(immediateSwapchainEnv, false);
+
     VulkanSplatting::RendererConfiguration config{
         envVars.get_or(validationLayers, false),
         envVars.get(physicalDeviceId).has_value()
             ? std::make_optional(envVars.get(physicalDeviceId).value())
             : std::nullopt,
-        envVars.get_or(immediateSwapchain, false),
         args::get(scenePath)
     };
 
@@ -88,7 +89,7 @@ int main(int argc, char** argv)
 
     if (immediateSwapchainFlag)
     {
-        config.immediateSwapchain = args::get(immediateSwapchainFlag);
+        immediateSwapchain = args::get(immediateSwapchainFlag);
     }
 
     if (noGuiFlag)
@@ -106,7 +107,7 @@ int main(int argc, char** argv)
 #ifndef DEBUG
     try {
 #endif
-    auto renderer = VulkanSplatting(std::move(config), VulkanSplatting::createGlfwWindow("Vulkan Splatting", width, height, config.immediateSwapchain));
+    auto renderer = VulkanSplatting(std::move(config), VulkanSplatting::createGlfwWindow("Vulkan Splatting", width, height, immediateSwapchain));
     renderer.start();
 #ifndef DEBUG
     } catch (const std::exception& e) {

apps/vr_viewer/src/VRViewer.cpp

@@ -1,5 +1,6 @@
 #include "VRViewer.h"
 
+#include <complex>
 #include <memory>
 #include <openxr/openxr_platform.h>
 
@@ -16,22 +17,35 @@ void VRViewer::run() {
     VulkanSplatting::OpenXRConfiguration configuration;
     configuration.instanceExtensions = context->getRequiredVulkanInstanceExtensions();
     configuration.deviceExtensions = context->getRequiredVulkanDeviceExtensions();
-    configuration.getPhysicalDevice = std::bind(&OXRContext::getPhysicalDevice, context.get(), std::placeholders::_1);
+    configuration.getPhysicalDevice = std::bind(&OXRContext::getPhysicalDevice, context, std::placeholders::_1);
     configuration.postVulkanInit = std::bind(&VRViewer::finishSetup, this, std::placeholders::_1, std::placeholders::_2,
         std::placeholders::_3, std::placeholders::_4, std::placeholders::_5);
+    configuration.getCurrentExtent = std::bind(&VRViewer::getCurrentExtent, this);
+    configuration.acquireNextImage = std::bind(&Layer::acquireNextImage, projectionLayer, std::placeholders::_1);
+    configuration.present = std::bind(&Layer::present, projectionLayer, std::placeholders::_1, std::placeholders::_2);
 
     auto renderingTarget = VulkanSplatting::createOpenXRRenderTarget(configuration);
+    VulkanSplatting::RendererConfiguration rendererConfiguration;
+    rendererConfiguration.scene = config.scenePath;
+    rendererConfiguration.enableGui = false;
+    VulkanSplatting vulkanSplatting(rendererConfiguration, renderingTarget);
+    vulkanSplatting.initialize();
 }
 
 void VRViewer::finishSetup(void *vkInstance, void *vkPhysicalDevice, void *vkDevice, uint32_t vkQueueFamilyIndex,
     uint32_t vkQueueIndex) {
+    spdlog::info("Finishing setup");
     context->createSession(vkInstance, vkPhysicalDevice, vkDevice, vkQueueFamilyIndex, vkQueueIndex);
     context->createReferenceSpace();
     context->beginSession();
     createProjectionLayer();
 }
 
-void VRViewer::createProjectionLayer() {
+std::pair<uint32_t, uint32_t> VRViewer::getCurrentExtent() {
+    return {projectionLayer->views[0].subImage.imageRect.extent.width, projectionLayer->views[0].subImage.imageRect.extent.height};
+}
 
+void VRViewer::createProjectionLayer() {
+    projectionLayer = std::make_shared<Layer>(context, XR_COMPOSITION_LAYER_UNPREMULTIPLIED_ALPHA_BIT, false);
 }
 

apps/vr_viewer/src/VRViewer.h

@@ -5,6 +5,7 @@
 #include <memory>
 #include <utility>
 
+#include "oxr/Layer.h"
 #include "oxr/OXRContext.h"
 
 class VRViewer {
@@ -21,10 +22,15 @@ class VRViewer {
 
     void finishSetup(void *vkInstance, void *vkPhysicalDevice, void *vkDevice, uint32_t vkQueueFamilyIndex, uint32_t vkQueueIndex);
 
+    std::pair<uint32_t, uint32_t> getCurrentExtent();
+
 private:
     Configuration config;
+
     std::shared_ptr<OXR::OXRContext> context = nullptr;
 
+    std::shared_ptr<OXR::Layer> projectionLayer = nullptr;
+
     void createProjectionLayer();
 };
 

apps/vr_viewer/src/oxr/Layer.cpp

@@ -9,21 +9,46 @@ namespace OXR {
         depth_enabled) {
         createSwapchains(context);
         createProjectionViews(context, flags);
+        spdlog::info("Layer created");
+    }
+
+    std::pair<std::optional<uint32_t>, bool> Layer::acquireNextImage(int8_t swapchain) {
+        XrSwapchainImageAcquireInfo acquireInfo = {XR_TYPE_SWAPCHAIN_IMAGE_ACQUIRE_INFO};
+        auto ret = xrAcquireSwapchainImage(swapchains[swapchain], &acquireInfo, &lastImageAcquired[swapchain]);
+        XR_CHECK(ret, "Failed to acquire swapchain image");
+
+        XrSwapchainImageWaitInfo waitInfo = {XR_TYPE_SWAPCHAIN_IMAGE_WAIT_INFO};
+        waitInfo.timeout = INT64_MAX;
+        ret = xrWaitSwapchainImage(swapchains[swapchain], &waitInfo);
+        XR_CHECK(ret, "Failed to wait for swapchain image");
+
+        return {{lastImageAcquired[swapchain]}, false};
+    }
+
+    bool Layer::present(int8_t swapchain, uint32_t imageIndex) {
+        assert(imageIndex == lastImageAcquired[swapchain]); // TODO: Remove this check
+
+        XrSwapchainImageReleaseInfo releaseInfo = {XR_TYPE_SWAPCHAIN_IMAGE_RELEASE_INFO};
+        XR_CHECK(xrReleaseSwapchainImage(swapchains[swapchain], &releaseInfo), "Failed to release swapchain image");
     }
 
     void Layer::createSwapchains(std::shared_ptr<OXRContext> context) {
         // Print the swapchain formats
         spdlog::info("Swapchain formats:");
         auto swapchainFormats = getVulkanFormatsForSwapchain(context);
+        vk::Format selectedFormat = swapchainFormats[0];
         for (vk::Format format: swapchainFormats) {
-            spdlog::info("{}", string_VkFormat(static_cast<VkFormat>(format)));
+            spdlog::info("\t{}", string_VkFormat(static_cast<VkFormat>(format)));
+            if (format == vk::Format::eB8G8R8A8Unorm) {
+                selectedFormat = format;
+            }
         }
 
         std::vector<XrSwapchainImageVulkanKHR> swapchainImageVectors[2];
         for (int i = 0; i < 2; i++) {
             XrSwapchainCreateInfo swapchainCreateInfo = {XR_TYPE_SWAPCHAIN_CREATE_INFO};
             swapchainCreateInfo.arraySize = 1;
-            swapchainCreateInfo.format = static_cast<int64_t>(swapchainFormats[0]);
+            swapchainCreateInfo.format = static_cast<int64_t>(selectedFormat);
             swapchainCreateInfo.width = context->views[i].recommendedImageRectWidth;
             swapchainCreateInfo.height = context->views[i].recommendedImageRectHeight;
             swapchainCreateInfo.mipCount = 1;
@@ -132,17 +157,21 @@ namespace OXR {
 
     void Layer::createProjectionViews(std::shared_ptr<OXRContext> context, uint64_t flags) {
         for (int i = 0; i < 2; i++) {
-            views[i].pose = {.orientation = { 0.0f, 0.0f, 0.0f, 1.0f }, .position = { 0.0f, 0.0f, 0.0f }};
+            views[i].pose = {.orientation = {0.0f, 0.0f, 0.0f, 1.0f}, .position = {0.0f, 0.0f, 0.0f}};
             views[i].subImage.swapchain = swapchains[i];
             views[i].subImage.imageRect.offset = {0, 0};
-            views[i].subImage.imageRect.extent = {static_cast<int32_t>(context->views[i].recommendedImageRectWidth),
-                                                  static_cast<int32_t>(context->views[i].recommendedImageRectHeight)};
+            views[i].subImage.imageRect.extent = {
+                static_cast<int32_t>(context->views[i].recommendedImageRectWidth),
+                static_cast<int32_t>(context->views[i].recommendedImageRectHeight)
+            };
             if (depthEnabled) {
                 views[i].next = &depthLayer;
                 depthLayer.subImage.swapchain = depthSwapchains[i];
                 depthLayer.subImage.imageRect.offset = {0, 0};
-                depthLayer.subImage.imageRect.extent = {static_cast<int32_t>(context->views[i].recommendedImageRectWidth),
-                                                        static_cast<int32_t>(context->views[i].recommendedImageRectHeight)};
+                depthLayer.subImage.imageRect.extent = {
+                    static_cast<int32_t>(context->views[i].recommendedImageRectWidth),
+                    static_cast<int32_t>(context->views[i].recommendedImageRectHeight)
+                };
             }
         }
 

apps/vr_viewer/src/oxr/Layer.h

@@ -13,11 +13,16 @@ class Layer {
 public:
     explicit Layer(std::shared_ptr<OXRContext> context, XrCompositionLayerFlags flags, bool depth_enabled);
 
+    std::pair<std::optional<uint32_t>, bool> acquireNextImage(int8_t swapchain);
+
+    bool present(int8_t swapchain, uint32_t imageIndex);
+
+
+    XrCompositionLayerProjectionView views[2] = {{XR_TYPE_COMPOSITION_LAYER_PROJECTION_VIEW}, {XR_TYPE_COMPOSITION_LAYER_PROJECTION_VIEW}};
 
 private:
     XrCompositionLayerProjection layer = {XR_TYPE_COMPOSITION_LAYER_PROJECTION};
     XrCompositionLayerDepthInfoKHR depthLayer = {XR_TYPE_COMPOSITION_LAYER_DEPTH_INFO_KHR};
-    XrCompositionLayerProjectionView views[2] = {{XR_TYPE_COMPOSITION_LAYER_PROJECTION_VIEW}, {XR_TYPE_COMPOSITION_LAYER_PROJECTION_VIEW}};
     XrSwapchain swapchains[2] = {XR_NULL_HANDLE, XR_NULL_HANDLE};
     uint32_t swapchainSize[2] = {0, 0};
     uint32_t lastImageAcquired[2] = {0, 0};

apps/vr_viewer/src/oxr/OXRContext.cpp

@@ -21,8 +21,6 @@ void OXRContext::setup() {
     createInstance();
     createSystem();
     setupViews();
-    std::ignore = getRequiredVulkanInstanceExtensions();
-    std::ignore = getRequiredVulkanDeviceExtensions();
 }
 
 void OXRContext::createInstance() {
@@ -111,8 +109,6 @@ std::vector<std::string> OXRContext::getRequiredVulkanInstanceExtensions() const
         throw std::runtime_error("Failed to get Vulkan instance extensions");
     }
 
-    spdlog::debug("Vulkan instance extensions: {}", extensions);
-
     // split extensions by space
     std::vector<std::string> extensionList;
     std::string extension;
@@ -147,8 +143,6 @@ std::vector<std::string> OXRContext::getRequiredVulkanDeviceExtensions() const {
         throw std::runtime_error("Failed to get Vulkan device extensions");
     }
 
-    spdlog::debug("Vulkan device extensions: {}", extensions);
-
     // split extensions by space
     std::vector<std::string> extensionList;
     std::string extension;
@@ -165,6 +159,9 @@ std::vector<std::string> OXRContext::getRequiredVulkanDeviceExtensions() const {
 }
 
 void * OXRContext::getPhysicalDevice(void *instance) const {
+    checkVulkanInstance(instance);
+
+    spdlog::debug("Getting Vulkan graphics device");
     PFN_xrGetVulkanGraphicsDeviceKHR xrGetVulkanGraphicsDeviceKHR;
     auto result = xrGetInstanceProcAddr(oxrInstance, "xrGetVulkanGraphicsDeviceKHR", reinterpret_cast<PFN_xrVoidFunction*>(&xrGetVulkanGraphicsDeviceKHR));
     if (XR_FAILED(result)) {
@@ -180,8 +177,25 @@ void * OXRContext::getPhysicalDevice(void *instance) const {
     return physicalDevice;
 }
 
+void OXRContext::checkVulkanInstance(void *instance) const {
+    XrGraphicsRequirementsVulkanKHR requirements = {XR_TYPE_GRAPHICS_REQUIREMENTS_VULKAN_KHR};
+    PFN_xrGetVulkanGraphicsRequirementsKHR xrGetVulkanGraphicsRequirementsKHR;
+    auto result = xrGetInstanceProcAddr(oxrInstance, "xrGetVulkanGraphicsRequirementsKHR", reinterpret_cast<PFN_xrVoidFunction*>(&xrGetVulkanGraphicsRequirementsKHR));
+    XR_CHECK(result, "Failed to get xrGetVulkanGraphicsRequirementsKHR");
+
+    result = xrGetVulkanGraphicsRequirementsKHR(oxrInstance, systemId, &requirements);
+    XR_CHECK(result, "Failed to get Vulkan graphics requirements");
+
+
+    spdlog::debug("Vulkan graphics requirements:");
+    spdlog::debug("  minApiVersionSupported: {}.{}.{}", XR_VERSION_MAJOR(requirements.minApiVersionSupported),
+                  XR_VERSION_MINOR(requirements.minApiVersionSupported), XR_VERSION_PATCH(requirements.minApiVersionSupported));
+    spdlog::debug("  maxApiVersionSupported: {}.{}.{}", XR_VERSION_MAJOR(requirements.maxApiVersionSupported),
+                  XR_VERSION_MINOR(requirements.maxApiVersionSupported), XR_VERSION_PATCH(requirements.maxApiVersionSupported));
+}
+
 void OXRContext::createSession(void *vkInstance, void *vkPhysicalDevice, void *vkDevice, uint32_t vkQueueFamilyIndex,
-    uint32_t vkQueueIndex) {
+                               uint32_t vkQueueIndex) {
     XrGraphicsBindingVulkanKHR graphicsBinding = {XR_TYPE_GRAPHICS_BINDING_VULKAN_KHR};
     graphicsBinding.instance = static_cast<VkInstance>(vkInstance);
     graphicsBinding.physicalDevice = static_cast<VkPhysicalDevice>(vkPhysicalDevice);

apps/vr_viewer/src/oxr/OXRContext.h

@@ -8,8 +8,11 @@ namespace OXR {
     class OXRContext {
     public:
         void setup();
+
         void* getPhysicalDevice(void *instance) const;
 
+        void checkVulkanInstance(void * instance) const;
+
         void createSession(void *vkInstance, void *vkPhysicalDevice, void *vkDevice, uint32_t vkQueueFamilyIndex,
                            uint32_t vkQueueIndex);
 

include/3dgs/3dgs.h

@@ -15,7 +15,6 @@ class VulkanSplatting {
     struct RendererConfiguration {
         bool enableVulkanValidationLayers = false;
         std::optional<uint8_t> physicalDeviceId = std::nullopt;
-        bool immediateSwapchain = false;
         std::string scene;
 
         float fov = 45.0f;
@@ -36,11 +35,18 @@ class VulkanSplatting {
 
 #ifdef VKGS_ENABLE_OPENXR
     struct OpenXRConfiguration {
+        struct PostVulkanSetup {
+            std::vector<void*> swapchainImages;
+        };
         std::vector<std::string> instanceExtensions;
         std::vector<std::string> deviceExtensions;
 
         std::function<void*(void*)> getPhysicalDevice;
         std::function<void(void*, void*, void*, uint32_t, uint32_t)> postVulkanInit;
+        std::function<std::pair<uint32_t, uint32_t>(void)> getCurrentExtent;
+
+        std::function<std::pair<std::optional<uint32_t>, bool> (uint8_t)> acquireNextImage;
+        std::function<bool(uint32_t)> present;
     };
     static std::shared_ptr<RenderTarget> createOpenXRRenderTarget(OpenXRConfiguration configuration);
 #endif

src/Renderer.cpp

@@ -116,7 +116,11 @@ void Renderer::recreatePipelines() {
 void Renderer::initializeVulkan() {
     spdlog::debug("Initializing Vulkan");
 
-    context = std::make_shared<VulkanContext>(renderTarget->getRequiredInstanceExtensions(), renderTarget->getRequiredDeviceExtensions(),
+    auto deviceExtensions = renderTarget->getRequiredDeviceExtensions();
+    if (configuration.enableGui) {
+        deviceExtensions.push_back(VK_KHR_DYNAMIC_RENDERING_EXTENSION_NAME);
+    }
+    context = std::make_shared<VulkanContext>(renderTarget->getRequiredInstanceExtensions(), deviceExtensions,
                                               configuration.enableVulkanValidationLayers);
 
     context->createInstance();
@@ -363,7 +367,11 @@ void Renderer::createRenderPipeline() {
     auto outputSet = std::make_shared<DescriptorSet>(context, 1);
     for (auto &image: renderTarget->swapchainImages) {
         outputSet->bindImageToDescriptorSet(0, vk::DescriptorType::eStorageImage, vk::ShaderStageFlagBits::eCompute,
-                                            image);
+                                            image[0]);
+        if (image.isStereo) {
+            outputSet->bindImageToDescriptorSet(0, vk::DescriptorType::eStorageImage, vk::ShaderStageFlagBits::eCompute,
+                                            image[1]);
+        }
     }
     outputSet->build();
     renderPipeline->addDescriptorSet(0, inputSet);

src/vulkan/ImguiManager.cpp

@@ -168,8 +168,12 @@ void ImguiManager::draw(vk::CommandBuffer commandBuffer, uint32_t currentImageIn
     imguiFunction();
     ImGui::Render();
 
+    if (window->swapchainImages[currentImageIndex].isStereo) {
+        throw std::runtime_error("Stereo rendering is not supported");
+    }
+
     vk::RenderingAttachmentInfoKHR attachment_info{
-        window->swapchainImages[currentImageIndex]->imageView.get(), vk::ImageLayout::eColorAttachmentOptimal
+        window->swapchainImages[currentImageIndex].image->imageView.get(), vk::ImageLayout::eColorAttachmentOptimal
     };
     vk::RenderingInfoKHR rendering_info{{}, vk::Rect2D{{0, 0}, window->currentExtent()}, 1, {}, 1, &attachment_info};
     commandBuffer.beginRenderingKHR(rendering_info);

src/vulkan/Swapchain.h

@@ -15,7 +15,9 @@ class Swapchain {
 
     virtual ~Swapchain() = default;
 
-    std::vector<std::shared_ptr<Image>> swapchainImages;
+    virtual bool isStereo() { return false; }
+
+    std::vector<ImageProxy> swapchainImages;
     std::vector<vk::UniqueSemaphore> imageAvailableSemaphores;
     vk::Format swapchainFormat;
     uint32_t imageCount;