feature/bundled-shaders: bundle compiled SPIR-V in binary (#15)

* Bundled shaders * Platform independent commands * Read compiled SPIR-V in binary mode * Various fixes
shg8 · Mar 14, 2024 · 11bbd84 · 11bbd84
1 parent 3371de7
commit 11bbd84
Show file tree

Hide file tree

Showing 10 changed files with 148 additions and 47 deletions.
diff --git a/3dgs/GSScene.cpp b/3dgs/GSScene.cpp
@@ -6,6 +6,7 @@
 #include "GSScene.h"
 
 #include <random>
+#include "shaders.h"
 
 #include "../vulkan/Utils.h"
 #include "../vulkan/DescriptorSet.h"
@@ -155,7 +156,9 @@ std::shared_ptr<Buffer> GSScene::createBuffer(const std::shared_ptr<VulkanContex
 void GSScene::precomputeCov3D(const std::shared_ptr<VulkanContext>&context) {
     cov3DBuffer = createBuffer(context, header.numVertices * sizeof(float) * 6);
 
-    auto pipeline = std::make_shared<ComputePipeline>(context, "precomp_cov3d");
+    auto pipeline = std::make_shared<ComputePipeline>(
+        context, std::make_shared<Shader>(context, "precomp_cov3d", SPV_PRECOMP_COV3D, SPV_PRECOMP_COV3D_len));
+
     auto descriptorSet = std::make_shared<DescriptorSet>(context, FRAMES_IN_FLIGHT);
     descriptorSet->bindBufferToDescriptorSet(0, vk::DescriptorType::eStorageBuffer, vk::ShaderStageFlagBits::eCompute,
                                              vertexBuffer);

diff --git a/3dgs/GUIManager.cpp b/3dgs/GUIManager.cpp
@@ -33,9 +33,9 @@ void GUIManager::buildGui() {
         ImPlot::SetupAxisLimits(ImAxis_Y1, 0, 1);
         ImPlot::SetNextFillStyle(IMPLOT_AUTO_COL, 0.5f);
         for (auto& [name, values]: *metricsMap) {
-            if (!values.Data.empty()) {
-                ImPlot::PlotLine(name.c_str(), &values.Data[0].x, &values.Data[0].y, values.Data.size(), 0,
-                                 values.Offset, 2 * sizeof(float));
+            if (!values.data.empty()) {
+                ImPlot::PlotLine(name.c_str(), &values.data[0].x, &values.data[0].y, values.data.size(), 0,
+                                 values.offset, 2 * sizeof(float));
             }
         }
         ImPlot::EndPlot();
@@ -79,7 +79,7 @@ void GUIManager::pushTextMetric(const std::string& name, float value) {
 void GUIManager::pushMetric(const std::string& name, float value) {
     int maxSize = 600;
     if (!metricsMap->contains(name)) {
-        metricsMap->insert({name, ScrollingBuffer(maxSize)});
+        metricsMap->insert({name, ScrollingBuffer{}});
     }
     metricsMap->at(name).addPoint(ImGui::GetTime(), value);
 }

diff --git a/3dgs/GUIManager.h b/3dgs/GUIManager.h
@@ -9,26 +9,26 @@
 #include "implot/implot.h"
 
 struct ScrollingBuffer {
-    int MaxSize;
-    int Offset;
-    ImVector<ImVec2> Data;
-    ScrollingBuffer(int max_size = 2000) {
-        MaxSize = max_size;
-        Offset  = 0;
-        Data.reserve(MaxSize);
+    int maxSize;
+    int offset;
+    ImVector<ImVec2> data;
+    explicit ScrollingBuffer(const int max_size = 10000) {
+        maxSize = max_size;
+        offset  = 0;
+        data.reserve(maxSize);
     }
     void addPoint(float x, float y) {
-        if (Data.size() < MaxSize)
-            Data.push_back(ImVec2(x,y));
+        if (data.size() < maxSize)
+            data.push_back(ImVec2(x,y));
         else {
-            Data[Offset] = ImVec2(x,y);
-            Offset =  (Offset + 1) % MaxSize;
+            data[offset] = ImVec2(x,y);
+            offset =  (offset + 1) % maxSize;
         }
     }
     void clear() {
-        if (Data.size() > 0) {
-            Data.shrink(0);
-            Offset  = 0;
+        if (data.size() > 0) {
+            data.shrink(0);
+            offset  = 0;
         }
     }
 };

diff --git a/3dgs/Renderer.cpp b/3dgs/Renderer.cpp
@@ -5,6 +5,7 @@
 #include "../vulkan/Swapchain.h"
 
 #include <memory>
+#include <shaders.h>
 #include <utility>
 
 #include <glm/glm.hpp>
@@ -99,7 +100,7 @@ void Renderer::retrieveTimestamps() {
 
 void Renderer::initializeVulkan() {
     spdlog::debug("Initializing Vulkan");
-    window = std::make_shared<Window>("Vulkan Splatting", 800, 600);
+    window = std::make_shared<Window>("Vulkan Splatting", 1920, 1080);
     context = std::make_shared<VulkanContext>(Window::getRequiredInstanceExtensions(), std::vector<std::string>{},
                                               configuration.enableVulkanValidationLayers);
 
@@ -148,7 +149,7 @@ void Renderer::createPreprocessPipeline() {
     vertexAttributeBuffer = Buffer::storage(context, scene->getNumVertices() * sizeof(VertexAttributeBuffer), false);
     tileOverlapBuffer = Buffer::storage(context, scene->getNumVertices() * sizeof(uint32_t), false);
 
-    preprocessPipeline = std::make_shared<ComputePipeline>(context, "preprocess");
+    preprocessPipeline = std::make_shared<ComputePipeline>(context,  std::make_shared<Shader>(context, "preprocess", SPV_PREPROCESS, SPV_PREPROCESS_len));
     inputSet = std::make_shared<DescriptorSet>(context, FRAMES_IN_FLIGHT);
     inputSet->bindBufferToDescriptorSet(0, vk::DescriptorType::eStorageBuffer, vk::ShaderStageFlagBits::eCompute,
                                         scene->vertexBuffer);
@@ -193,7 +194,7 @@ void Renderer::createPrefixSumPipeline() {
     prefixSumPongBuffer = Buffer::storage(context, scene->getNumVertices() * sizeof(uint32_t), false);
     totalSumBufferHost = Buffer::staging(context, sizeof(uint32_t));
 
-    prefixSumPipeline = std::make_shared<ComputePipeline>(context, "prefix_sum");
+    prefixSumPipeline = std::make_shared<ComputePipeline>(context, std::make_shared<Shader>(context, "prefix_sum", SPV_PREFIX_SUM, SPV_PREFIX_SUM_len));
     auto descriptorSet = std::make_shared<DescriptorSet>(context, FRAMES_IN_FLIGHT);
     descriptorSet->bindBufferToDescriptorSet(0, vk::DescriptorType::eStorageBuffer, vk::ShaderStageFlagBits::eCompute,
                                              prefixSumPingBuffer);
@@ -225,8 +226,8 @@ void Renderer::createRadixSortPipeline() {
 
     sortHistBuffer = Buffer::storage(context, numWorkgroups * 256 * sizeof(uint32_t), false);
 
-    sortHistPipeline = std::make_shared<ComputePipeline>(context, "hist");
-    sortPipeline = std::make_shared<ComputePipeline>(context, "sort");
+    sortHistPipeline = std::make_shared<ComputePipeline>(context, std::make_shared<Shader>(context, "hist", SPV_HIST, SPV_HIST_len));
+    sortPipeline = std::make_shared<ComputePipeline>(context, std::make_shared<Shader>(context, "sort", SPV_SORT, SPV_SORT_len));
 
     auto descriptorSet = std::make_shared<DescriptorSet>(context, FRAMES_IN_FLIGHT);
     descriptorSet->bindBufferToDescriptorSet(0, vk::DescriptorType::eStorageBuffer, vk::ShaderStageFlagBits::eCompute,
@@ -267,7 +268,7 @@ void Renderer::createRadixSortPipeline() {
 
 void Renderer::createPreprocessSortPipeline() {
     spdlog::debug("Creating preprocess sort pipeline");
-    preprocessSortPipeline = std::make_shared<ComputePipeline>(context, "preprocess_sort");
+    preprocessSortPipeline = std::make_shared<ComputePipeline>(context, std::make_shared<Shader>(context, "preprocess_sort", SPV_PREPROCESS_SORT, SPV_PREPROCESS_SORT_len));
     auto descriptorSet = std::make_shared<DescriptorSet>(context, FRAMES_IN_FLIGHT);
     descriptorSet->bindBufferToDescriptorSet(0, vk::DescriptorType::eStorageBuffer, vk::ShaderStageFlagBits::eCompute,
                                              vertexAttributeBuffer);
@@ -293,7 +294,7 @@ void Renderer::createTileBoundaryPipeline() {
     auto tileY = (height + 16 - 1) / 16;
     tileBoundaryBuffer = Buffer::storage(context, tileX * tileY * sizeof(uint32_t) * 2, false);
 
-    tileBoundaryPipeline = std::make_shared<ComputePipeline>(context, "tile_boundary");
+    tileBoundaryPipeline = std::make_shared<ComputePipeline>(context, std::make_shared<Shader>(context, "tile_boundary", SPV_TILE_BOUNDARY, SPV_TILE_BOUNDARY_len));
     auto descriptorSet = std::make_shared<DescriptorSet>(context, FRAMES_IN_FLIGHT);
     descriptorSet->bindBufferToDescriptorSet(0, vk::DescriptorType::eStorageBuffer, vk::ShaderStageFlagBits::eCompute,
                                              sortKBufferEven);
@@ -310,7 +311,7 @@ void Renderer::createTileBoundaryPipeline() {
 
 void Renderer::createRenderPipeline() {
     spdlog::debug("Creating render pipeline");
-    renderPipeline = std::make_shared<ComputePipeline>(context, "render");
+    renderPipeline = std::make_shared<ComputePipeline>(context, std::make_shared<Shader>(context, "render", SPV_RENDER, SPV_RENDER_len));
     auto inputSet = std::make_shared<DescriptorSet>(context, FRAMES_IN_FLIGHT);
     inputSet->bindBufferToDescriptorSet(0, vk::DescriptorType::eStorageBuffer, vk::ShaderStageFlagBits::eCompute,
                                         vertexAttributeBuffer);
@@ -696,7 +697,7 @@ void Renderer::updateUniforms() {
     auto view = glm::inverse(translation * rotation);
 
     data.view_mat = view;
-    data.proj_mat = glm::perspective(glm::radians(camera.fov), static_cast<float>(width) / static_cast<float>(height),
+    data.proj_mat = glm::perspective(glm::radians(camera.fov) / 2.0f, static_cast<float>(width) / static_cast<float>(height),
                                      camera.nearPlane,
                                      camera.farPlane) * view;
 

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -54,7 +54,11 @@ FetchContent_Declare(spdlog
         GIT_REPOSITORY https://github.com/gabime/spdlog.git
         GIT_TAG v1.13.0
 )
-FetchContent_MakeAvailable(spdlog)
+FetchContent_GetProperties(spdlog)
+if (NOT spdlog_POPULATED)
+    FetchContent_Populate(spdlog)
+    add_subdirectory(${spdlog_SOURCE_DIR} ${spdlog_BINARY_DIR})
+endif()
 
 FetchContent_Declare(imgui
         GIT_REPOSITORY https://github.com/ocornut/imgui.git
@@ -89,21 +93,45 @@ if (APPLE)
     list(APPEND GLSLC_DEFINE "-DAPPLE")
 endif ()
 
+add_executable(embedfile cmake/embedfile.c)
+
+set(SHADER_HEADER "${PROJECT_BINARY_DIR}/shaders/shaders.h")
+message(STATUS "Shader header file: ${SHADER_HEADER}")
+
+# Delete old header file
+add_custom_command(
+        OUTPUT ${SHADER_HEADER}
+        COMMAND ${CMAKE_COMMAND} -E remove ${SHADER_HEADER}
+        DEPENDS ${GLSL_SOURCE_FILES}
+)
+
 foreach (GLSL ${GLSL_SOURCE_FILES})
-    get_filename_component(FILE_NAME ${GLSL} NAME)
+    get_filename_component(FILE_NAME ${GLSL} NAME_WE)
+    string(TOUPPER ${FILE_NAME} FILE_NAME_UPPER)
+    set(FILE_NAME_UPPER "SPV_${FILE_NAME_UPPER}")
     set(SPIRV "${PROJECT_BINARY_DIR}/shaders/${FILE_NAME}.spv")
+    SET(TEMP_HEADER "${PROJECT_BINARY_DIR}/shaders/${FILE_NAME}.h")
     add_custom_command(
             OUTPUT ${SPIRV}
             COMMAND ${CMAKE_COMMAND} -E make_directory "${PROJECT_BINARY_DIR}/shaders/"
             COMMAND ${Vulkan_GLSLANG_VALIDATOR_EXECUTABLE} "--target-env" "vulkan1.2" -V ${GLSL} -o ${SPIRV} ${GLSLC_DEFINE}
             DEPENDS ${GLSL})
     list(APPEND SPIRV_BINARY_FILES ${SPIRV})
+
+    add_custom_command(
+            OUTPUT ${SHADER_HEADER}
+            COMMAND embedfile ${FILE_NAME_UPPER} ${SPIRV} ${SHADER_HEADER}
+            DEPENDS ${SPIRV}
+            APPEND)
+
+    list(APPEND TEMP_HEADERS ${TEMP_HEADER})
 endforeach (GLSL)
 
 add_custom_target(
         Shaders
-        DEPENDS ${SPIRV_BINARY_FILES}
+        DEPENDS ${SPIRV_BINARY_FILES} ${SHADER_HEADER}
 )
+include_directories(${PROJECT_BINARY_DIR}/shaders)
 
 include_directories(third_party)
 
@@ -154,7 +182,7 @@ target_include_directories(vulkan_splatting PUBLIC
         ${spdlog_SOURCE_DIR}/include
 )
 
-target_link_libraries(vulkan_splatting PUBLIC glfw libenvpp::libenvpp)
+target_link_libraries(vulkan_splatting PUBLIC glfw libenvpp::libenvpp spdlog::spdlog)
 target_link_libraries(vulkan_splatting PUBLIC Vulkan::Vulkan)
 if (UNIX)
     target_link_libraries(vulkan_splatting PUBLIC ${CMAKE_DL_LIBS})

diff --git a/cmake/embedfile.c b/cmake/embedfile.c
@@ -0,0 +1,47 @@
+#include <stdlib.h>
+#include <stdio.h>
+
+FILE* open_or_exit(const char* fname, const char* mode)
+{
+    FILE* f = fopen(fname, mode);
+    if (f == NULL) {
+        perror(fname);
+        exit(EXIT_FAILURE);
+    }
+    return f;
+}
+
+int main(int argc, char** argv)
+{
+    if (argc < 3) {
+        fprintf(stderr, "USAGE: %s {sym} {rsrc}\n\n"
+            "  Creates {sym}.c from the contents of {rsrc}\n",
+            argv[0]);
+        return EXIT_FAILURE;
+    }
+
+    const char* sym = argv[1];
+    FILE* in = open_or_exit(argv[2], "rb");
+    FILE* out = open_or_exit(argv[3], "a");
+    fprintf(out, "static const unsigned char %s[] = {\n", sym);
+
+    unsigned char buf[256];
+    size_t nread = 0;
+    size_t linecount = 0;
+    do {
+        nread = fread(buf, 1, sizeof(buf), in);
+        size_t i;
+        for (i=0; i < nread; i++) {
+            fprintf(out, "0x%02x, ", buf[i]);
+            if (++linecount == 10) { fprintf(out, "\n"); linecount = 0; }
+        }
+    } while (nread > 0);
+    if (linecount > 0) fprintf(out, "\n");
+    fprintf(out, "};\n");
+    fprintf(out, "static const size_t %s_len = sizeof(%s);\n\n",sym,sym);
+
+    fclose(in);
+    fclose(out);
+
+    return EXIT_SUCCESS;
+}
diff --git a/vulkan/Shader.cpp b/vulkan/Shader.cpp
@@ -2,13 +2,18 @@
 #include "Utils.h"
 
 void Shader::load() {
-    auto filename = "shaders/" + name + ".comp.spv";
-    auto shader_code = Utils::readFile(filename);
-    if (shader_code.empty()) {
-        throw std::runtime_error("Failed to load shader: " + filename);
-    }
     vk::ShaderModuleCreateInfo create_info;
-    create_info.codeSize = shader_code.size();
-    create_info.pCode = reinterpret_cast<const uint32_t *>(shader_code.data());
+    if (data == nullptr) {
+        auto fn = "shaders/" + filename + ".spv";
+        auto shader_code = Utils::readFile(fn);
+        if (shader_code.empty()) {
+            throw std::runtime_error("Failed to load shader: " + fn);
+        }
+        create_info.codeSize = shader_code.size();
+        create_info.pCode = reinterpret_cast<const uint32_t *>(shader_code.data());
+    } else {
+        create_info.codeSize = size;
+        create_info.pCode = reinterpret_cast<const uint32_t *>(data);
+    }
     shader = context->device->createShaderModuleUnique(create_info);
 }
diff --git a/vulkan/Shader.h b/vulkan/Shader.h
@@ -9,17 +9,33 @@
 
 class Shader {
 public:
-    Shader(const std::shared_ptr<VulkanContext>& _context, std::string  name)
+    Shader(const std::shared_ptr<VulkanContext>& _context, std::string filename)
         : context(_context),
-          name(std::move(name)) {
+          filename(std::move(filename)) {
+    }
+
+    Shader(const std::shared_ptr<VulkanContext>& _context, const unsigned char * data, size_t size)
+        : context(_context),
+          filename(""),
+          data(data),
+          size(size) {
+    }
+
+    Shader(const std::shared_ptr<VulkanContext>& context, const std::string& filename, const unsigned char * data, size_t size)
+        : filename(filename),
+          context(context),
+          data(data),
+          size(size) {
     }
 
     void load();
 
     vk::UniqueShaderModule shader;
 private:
-    const std::string name;
+    const std::string filename;
     std::shared_ptr<VulkanContext> context;
+    const unsigned char* data = nullptr;
+    size_t size;
 };
 
 

diff --git a/vulkan/pipelines/ComputePipeline.cpp b/vulkan/pipelines/ComputePipeline.cpp
@@ -2,14 +2,14 @@
 
 #include "ComputePipeline.h"
 
-ComputePipeline::ComputePipeline(const std::shared_ptr<VulkanContext> &context, const std::string &shaderFile) : Pipeline(context), shader(context, shaderFile) {
-    shader.load();
+ComputePipeline::ComputePipeline(const std::shared_ptr<VulkanContext>& context, std::shared_ptr<Shader> shader): Pipeline(context), shader(std::move(shader)) {
+    this->shader->load();
 }
 
 void ComputePipeline::build() {
     buildPipelineLayout();
 
-    vk::PipelineShaderStageCreateInfo pipelineShaderStageCreateInfo({}, vk::ShaderStageFlagBits::eCompute, shader.shader.get(), "main");
+    vk::PipelineShaderStageCreateInfo pipelineShaderStageCreateInfo({}, vk::ShaderStageFlagBits::eCompute, shader->shader.get(), "main");
     vk::ComputePipelineCreateInfo computePipelineCreateInfo({}, pipelineShaderStageCreateInfo, pipelineLayout.get());
     pipeline = context->device->createComputePipelineUnique(nullptr, computePipelineCreateInfo).value;
 }
diff --git a/vulkan/pipelines/ComputePipeline.h b/vulkan/pipelines/ComputePipeline.h
@@ -5,14 +5,15 @@
 
 
 #include "Pipeline.h"
+#include <memory>
 
 class ComputePipeline : public Pipeline {
 public:
-    explicit ComputePipeline(const std::shared_ptr<VulkanContext> &context, const std::string& shader);
+    explicit ComputePipeline(const std::shared_ptr<VulkanContext> &context, std::shared_ptr<Shader> shader);;
 
     void build() override;
 private:
-    Shader shader;
+    std::shared_ptr<Shader> shader;
 };