ilikecats/stk-code_catmod
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add GEVulkanCommandLoader with multithreading support

Browse Source
This commit is contained in:
Benau 2022-04-17 14:08:29 +08:00
parent 993073b7dc
commit da665fc4fd
5 changed files with 359 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
lib/graphics_engine/CMakeLists.txt
View File

@ -26,6 +26,7 @@ set(GE_SOURCES
src/ge_texture.cpp
src/ge_dx9_texture.cpp
src/ge_vulkan_2d_renderer.cpp
src/ge_vulkan_command_loader.cpp
src/ge_vulkan_driver.cpp
src/ge_vulkan_dynamic_buffer.cpp
src/ge_vulkan_features.cpp

11
lib/graphics_engine/include/ge_vulkan_driver.hpp
View File

@ -13,6 +13,7 @@
#include "SColor.h"
#include <array>
#include <memory>
#include <mutex>
#include <string>
#include <vector>
@ -327,6 +328,10 @@ namespace GE
destroySwapChainRelated(false/*handle_surface*/);
createSwapChainRelated(false/*handle_surface*/);
}
uint32_t getGraphicsFamily() const { return m_graphics_family; }
unsigned getGraphicsQueueCount() const
{ return m_graphics_queue_count; }
std::unique_lock<std::mutex> getGraphicsQueue(VkQueue* queue) const;
private:
struct SwapChainSupportDetails
{
@ -430,11 +435,13 @@ namespace GE
VkSurfaceCapabilitiesKHR m_surface_capabilities;
std::vector<VkSurfaceFormatKHR> m_surface_formats;
std::vector<VkPresentModeKHR> m_present_modes;
VkQueue m_graphics_queue;
std::vector<VkQueue> m_graphics_queue;
VkQueue m_present_queue;
mutable std::vector<std::mutex*> m_graphics_queue_mutexes;
uint32_t m_graphics_family;
uint32_t m_present_family;
unsigned m_graphics_queue_count;
VkPhysicalDeviceProperties m_properties;
VkPhysicalDeviceFeatures m_features;
@ -453,7 +460,7 @@ namespace GE
void createInstance(SDL_Window* window);
void findPhysicalDevice();
bool checkDeviceExtensions(VkPhysicalDevice device);
bool findQueueFamilies(VkPhysicalDevice device, uint32_t* graphics_family, uint32_t* present_family);
bool findQueueFamilies(VkPhysicalDevice device, uint32_t* graphics_family, unsigned* graphics_queue_count, uint32_t* present_family);
bool updateSurfaceInformation(VkPhysicalDevice device,
VkSurfaceCapabilitiesKHR* surface_capabilities,
std::vector<VkSurfaceFormatKHR>* surface_formats,

228
lib/graphics_engine/src/ge_vulkan_command_loader.cpp Normal file
View File

@ -0,0 +1,228 @@
#include "ge_vulkan_command_loader.hpp"
#include "ge_vulkan_driver.hpp"
#include <atomic>
#include <condition_variable>
#include <cstdio>
#include <deque>
#include <mutex>
#include <thread>
#include "../source/Irrlicht/os.h"
#ifndef thread_local
# if __STDC_VERSION__ >= 201112 && !defined __STDC_NO_THREADS__
# define thread_local _Thread_local
# elif defined _WIN32 && ( \
defined _MSC_VER || \
defined __ICL || \
defined __DMC__ || \
defined __BORLANDC__ )
# define thread_local __declspec(thread)
/* note that ICC (linux) and Clang are covered by __GNUC__ */
# elif defined __GNUC__ || \
defined __SUNPRO_C || \
defined __xlC__
# define thread_local __thread
# else
# error "Cannot define thread_local"
# endif
#endif
namespace GE
{
namespace GEVulkanCommandLoader
{
// ============================================================================
GEVulkanDriver* g_vk = NULL;
std::mutex g_loaders_mutex;
std::condition_variable g_loaders_cv;
std::vector<std::thread> g_loaders;
std::deque<std::function<void()> > g_threaded_commands;
thread_local int g_loader_id = 0;
std::atomic_uint g_loader_count(0);
std::vector<VkCommandPool> g_command_pools;
std::vector<VkFence> g_command_fences;
} // GEVulkanCommandLoader
// ============================================================================
void GEVulkanCommandLoader::init(GEVulkanDriver* vk)
{
g_vk = vk;
unsigned thread_count = std::thread::hardware_concurrency();
if (thread_count == 0)
thread_count = 3;
else
thread_count += 3;
g_command_pools.resize(thread_count);
g_command_fences.resize(thread_count);
for (unsigned i = 0; i < thread_count; i++)
{
VkCommandPoolCreateInfo pool_info = {};
pool_info.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO;
pool_info.queueFamilyIndex = g_vk->getGraphicsFamily();
pool_info.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT;
VkResult result = vkCreateCommandPool(g_vk->getDevice(), &pool_info,
NULL, &g_command_pools[i]);
if (result != VK_SUCCESS)
{
throw std::runtime_error(
"GEVulkanCommandLoader: vkCreateCommandPool failed");
}
VkFenceCreateInfo fence_info = {};
fence_info.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
result = vkCreateFence(g_vk->getDevice(), &fence_info, NULL,
&g_command_fences[i]);
if (result != VK_SUCCESS)
{
throw std::runtime_error(
"GEVulkanCommandLoader: vkCreateFence failed");
}
}
g_loader_count.store(thread_count - 1);
for (unsigned i = 0; i < thread_count - 1; i++)
{
g_loaders.emplace_back(
[i]()->void
{
g_loader_id = i + 1;
while (true)
{
std::unique_lock<std::mutex> ul(g_loaders_mutex);
g_loaders_cv.wait(ul, []
{
return !g_threaded_commands.empty();
});
if (g_loader_count.load() == 0)
return;
std::function<void()> copied = g_threaded_commands.front();
g_threaded_commands.pop_front();
ul.unlock();
copied();
}
});
}
char thread_count_str[40] = {};
snprintf(thread_count_str, 40, "%d threads used, %d graphics queue(s)",
thread_count - 1, vk->getGraphicsQueueCount());
os::Printer::log("Vulkan command loader", thread_count_str);
} // init
// ----------------------------------------------------------------------------
void GEVulkanCommandLoader::destroy()
{
g_loader_count.store(0);
if (!g_loaders.empty())
{
std::unique_lock<std::mutex> ul(g_loaders_mutex);
g_threaded_commands.push_back([](){});
g_loaders_cv.notify_all();
ul.unlock();
for (std::thread& t : g_loaders)
t.join();
g_loaders.clear();
}
for (auto& f : g_threaded_commands)
f();
g_threaded_commands.clear();
for (VkCommandPool& pool : g_command_pools)
vkDestroyCommandPool(g_vk->getDevice(), pool, NULL);
g_command_pools.clear();
for (VkFence& fence : g_command_fences)
vkDestroyFence(g_vk->getDevice(), fence, NULL);
g_command_fences.clear();
} // destroy
// ----------------------------------------------------------------------------
bool GEVulkanCommandLoader::multiThreadingEnabled()
{
return g_loader_count.load() != 0;
} // enabled
// ----------------------------------------------------------------------------
bool GEVulkanCommandLoader::isUsingMultiThreadingNow()
{
return g_loader_id != 0;
} // isUsingMultiThreadingNow
// ----------------------------------------------------------------------------
int GEVulkanCommandLoader::getLoaderId()
{
return g_loader_id;
} // getLoaderId
// ----------------------------------------------------------------------------
VkCommandPool GEVulkanCommandLoader::getCurrentCommandPool()
{
return g_command_pools[g_loader_id];
} // getCurrentCommandPool
// ----------------------------------------------------------------------------
VkFence GEVulkanCommandLoader::getCurrentFence()
{
return g_command_fences[g_loader_id];
} // getCurrentFence
// ----------------------------------------------------------------------------
void GEVulkanCommandLoader::addMultiThreadingCommand(std::function<void()> cmd)
{
if (g_loaders.empty())
return;
std::lock_guard<std::mutex> lock(g_loaders_mutex);
g_threaded_commands.push_back(cmd);
g_loaders_cv.notify_one();
} // addMultiThreadingCommand
// ----------------------------------------------------------------------------
VkCommandBuffer GEVulkanCommandLoader::beginSingleTimeCommands()
{
VkCommandBufferAllocateInfo alloc_info = {};
alloc_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO;
alloc_info.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
alloc_info.commandPool = g_command_pools[g_loader_id];
alloc_info.commandBufferCount = 1;
VkCommandBuffer command_buffer;
vkAllocateCommandBuffers(g_vk->getDevice(), &alloc_info, &command_buffer);
VkCommandBufferBeginInfo begin_info = {};
begin_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
begin_info.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
vkBeginCommandBuffer(command_buffer, &begin_info);
return command_buffer;
} // beginSingleTimeCommands
// ----------------------------------------------------------------------------
void GEVulkanCommandLoader::endSingleTimeCommands(VkCommandBuffer command_buffer,
VkQueueFlagBits bit)
{
vkEndCommandBuffer(command_buffer);
VkSubmitInfo submit_info = {};
submit_info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO;
submit_info.commandBufferCount = 1;
submit_info.pCommandBuffers = &command_buffer;
const int loader_id = g_loader_id;
VkQueue queue = VK_NULL_HANDLE;
std::unique_lock<std::mutex> lock = g_vk->getGraphicsQueue(&queue);
vkQueueSubmit(queue, 1, &submit_info, g_command_fences[loader_id]);
lock.unlock();
vkWaitForFences(g_vk->getDevice(), 1, &g_command_fences[loader_id],
VK_TRUE, std::numeric_limits<uint64_t>::max());
vkResetFences(g_vk->getDevice(), 1, &g_command_fences[loader_id]);
vkFreeCommandBuffers(g_vk->getDevice(), g_command_pools[loader_id], 1,
&command_buffer);
} // endSingleTimeCommands
}

38
lib/graphics_engine/src/ge_vulkan_command_loader.hpp Normal file
View File

@ -0,0 +1,38 @@
#ifndef HEADER_GE_VULKAN_COMMAND_LOADER_HPP
#define HEADER_GE_VULKAN_COMMAND_LOADER_HPP
#include "vulkan_wrapper.h"
#include <functional>
namespace GE
{
class GEVulkanDriver;
namespace GEVulkanCommandLoader
{
// ----------------------------------------------------------------------------
void init(GEVulkanDriver*);
// ----------------------------------------------------------------------------
void destroy();
// ----------------------------------------------------------------------------
bool multiThreadingEnabled();
// ----------------------------------------------------------------------------
bool isUsingMultiThreadingNow();
// ----------------------------------------------------------------------------
int getLoaderId();
// ----------------------------------------------------------------------------
VkCommandPool getCurrentCommandPool();
// ----------------------------------------------------------------------------
VkFence getCurrentFence();
// ----------------------------------------------------------------------------
void addMultiThreadingCommand(std::function<void()> cmd);
// ----------------------------------------------------------------------------
VkCommandBuffer beginSingleTimeCommands();
// ----------------------------------------------------------------------------
void endSingleTimeCommands(VkCommandBuffer command_buffer,
VkQueueFlagBits bit = VK_QUEUE_GRAPHICS_BIT);
}; // GEVulkanCommandLoader
}
#endif

102
lib/graphics_engine/src/ge_vulkan_driver.cpp
View File

@ -1,10 +1,12 @@
#include "ge_vulkan_driver.hpp"
#include "ge_main.hpp"
#include "ge_vulkan_2d_renderer.hpp"
#include "ge_vulkan_features.hpp"
#include "ge_main.hpp"
#include "ge_vulkan_shader_manager.hpp"
#include "ge_vulkan_texture.hpp"
#include "ge_vulkan_command_loader.hpp"
#ifdef _IRR_COMPILE_WITH_VULKAN_
#include "SDL_vulkan.h"
@ -454,9 +456,9 @@ GEVulkanDriver::GEVulkanDriver(const SIrrlichtCreationParameters& params,
{
m_vk.reset(new VK());
m_physical_device = VK_NULL_HANDLE;
m_graphics_queue = VK_NULL_HANDLE;
m_present_queue = VK_NULL_HANDLE;
m_graphics_family = m_present_family = 0;
m_graphics_queue_count = 0;
m_properties = {};
m_features = {};
@ -528,21 +530,24 @@ GEVulkanDriver::GEVulkanDriver(const SIrrlichtCreationParameters& params,
createSwapChain();
createSyncObjects();
createCommandPool();
createCommandBuffers();
createSamplers();
createRenderPass();
createFramebuffers();
GEVulkanShaderManager::init(this);
// For GEVulkanDynamicBuffer
GE::setVideoDriver(this);
GEVulkan2dRenderer::init(this);
createUnicolorTextures();
os::Printer::log("Vulkan version", getVulkanVersionString().c_str());
os::Printer::log("Vulkan vendor", getVendorInfo().c_str());
os::Printer::log("Vulkan renderer", m_properties.deviceName);
os::Printer::log("Vulkan driver version", getDriverVersionString().c_str());
for (const char* ext : m_device_extensions)
os::Printer::log("Vulkan enabled extension", ext);
GEVulkanCommandLoader::init(this);
createCommandBuffers();
GEVulkanShaderManager::init(this);
// For GEVulkanDynamicBuffer
GE::setVideoDriver(this);
GEVulkan2dRenderer::init(this);
createUnicolorTextures();
GEVulkanFeatures::printStats();
} // GEVulkanDriver
@ -568,6 +573,12 @@ void GEVulkanDriver::destroyVulkan()
GEVulkan2dRenderer::destroy();
GEVulkanShaderManager::destroy();
GEVulkanCommandLoader::destroy();
for (std::mutex* m : m_graphics_queue_mutexes)
delete m;
m_graphics_queue_mutexes.clear();
delete m_vk.get();
m_vk.release();
} // destroyVulkan
@ -667,8 +678,10 @@ void GEVulkanDriver::findPhysicalDevice()
{
uint32_t graphics_family = 0;
uint32_t present_family = 0;
unsigned graphics_queue_count = 0;
bool success = findQueueFamilies(device, &graphics_family, &present_family);
bool success = findQueueFamilies(device, &graphics_family,
&graphics_queue_count, &present_family);
if (!success)
continue;
@ -690,6 +703,7 @@ void GEVulkanDriver::findPhysicalDevice()
vkGetPhysicalDeviceFeatures(device, &m_features);
m_graphics_family = graphics_family;
m_graphics_queue_count = graphics_queue_count;
m_present_family = present_family;
m_surface_capabilities = surface_capabilities;
m_surface_formats = surface_formats;
@ -755,6 +769,7 @@ bool GEVulkanDriver::updateSurfaceInformation(VkPhysicalDevice device,
// ----------------------------------------------------------------------------
bool GEVulkanDriver::findQueueFamilies(VkPhysicalDevice device,
uint32_t* graphics_family,
unsigned* graphics_queue_count,
uint32_t* present_family)
{
uint32_t queue_family_count = 0;
@ -775,6 +790,7 @@ bool GEVulkanDriver::findQueueFamilies(VkPhysicalDevice device,
queue_families[i].queueFlags & VK_QUEUE_GRAPHICS_BIT)
{
*graphics_family = i;
*graphics_queue_count = queue_families[i].queueCount;
found_graphics_family = true;
break;
}
@ -800,18 +816,20 @@ bool GEVulkanDriver::findQueueFamilies(VkPhysicalDevice device,
void GEVulkanDriver::createDevice()
{
std::vector<VkDeviceQueueCreateInfo> queue_create_infos;
float queue_priority = 1.0f;
std::vector<float> queue_priority;
queue_priority.resize(m_graphics_queue_count, 1.0f);
VkDeviceQueueCreateInfo queue_create_info = {};
queue_create_info.sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO;
queue_create_info.queueFamilyIndex = m_graphics_family;
queue_create_info.queueCount = 1;
queue_create_info.pQueuePriorities = &queue_priority;
queue_create_info.queueCount = m_graphics_queue_count;
queue_create_info.pQueuePriorities = queue_priority.data();
queue_create_infos.push_back(queue_create_info);
if (m_present_family != m_graphics_family)
{
queue_create_info.queueFamilyIndex = m_present_family;
queue_create_info.queueCount = 1;
queue_create_infos.push_back(queue_create_info);
}
@ -846,10 +864,43 @@ void GEVulkanDriver::createDevice()
if (result != VK_SUCCESS)
throw std::runtime_error("vkCreateDevice failed");
vkGetDeviceQueue(m_vk->device, m_graphics_family, 0, &m_graphics_queue);
vkGetDeviceQueue(m_vk->device, m_present_family, 0, &m_present_queue);
m_graphics_queue.resize(m_graphics_queue_count);
for (unsigned i = 0; i < m_graphics_queue_count; i++)
{
vkGetDeviceQueue(m_vk->device, m_graphics_family, i,
&m_graphics_queue[i]);
m_graphics_queue_mutexes.push_back(new std::mutex());
}
if (m_present_family != m_graphics_family)
vkGetDeviceQueue(m_vk->device, m_present_family, 0, &m_present_queue);
} // createDevice
// ----------------------------------------------------------------------------
std::unique_lock<std::mutex> GEVulkanDriver::getGraphicsQueue(VkQueue* queue) const
{
if (m_graphics_queue_count == 0)
throw std::runtime_error("No graphics queue created");
if (m_graphics_queue_count == 1)
{
*queue = m_graphics_queue[0];
return std::unique_lock<std::mutex>(*m_graphics_queue_mutexes[0]);
}
while (true)
{
for (unsigned i = 0; i < m_graphics_queue_count; i++)
{
std::unique_lock<std::mutex> lock(*m_graphics_queue_mutexes[i],
std::defer_lock);
if (lock.try_lock())
{
*queue = m_graphics_queue[i];
return lock;
}
}
}
} // getGraphicsQueue
// ----------------------------------------------------------------------------
std::string GEVulkanDriver::getVulkanVersionString() const
{
@ -1360,8 +1411,11 @@ void GEVulkanDriver::endSingleTimeCommands(VkCommandBuffer command_buffer)
submit_info.commandBufferCount = 1;
submit_info.pCommandBuffers = &command_buffer;
vkQueueSubmit(m_graphics_queue, 1, &submit_info, VK_NULL_HANDLE);
vkQueueWaitIdle(m_graphics_queue);
VkQueue queue = VK_NULL_HANDLE;
std::unique_lock<std::mutex> ul = getGraphicsQueue(&queue);
vkQueueSubmit(queue, 1, &submit_info, VK_NULL_HANDLE);
vkQueueWaitIdle(queue);
ul.unlock();
vkFreeCommandBuffers(m_vk->device, m_vk->command_pool, 1, &command_buffer);
} // beginSingleTimeCommands
@ -1432,8 +1486,11 @@ bool GEVulkanDriver::endScene()
submit_info.signalSemaphoreCount = 1;
submit_info.pSignalSemaphores = signal_semaphores;
VkResult result = vkQueueSubmit(m_graphics_queue, 1, &submit_info,
VkQueue queue = VK_NULL_HANDLE;
std::unique_lock<std::mutex> ul = getGraphicsQueue(&queue);
VkResult result = vkQueueSubmit(queue, 1, &submit_info,
m_vk->in_flight_fences[m_current_frame]);
ul.unlock();
if (result != VK_SUCCESS)
throw std::runtime_error("vkQueueSubmit failed");
@ -1457,8 +1514,15 @@ bool GEVulkanDriver::endScene()
m_current_frame = (m_current_frame + 1) % getMaxFrameInFlight();
result = vkQueuePresentKHR(m_present_queue, &present_info);
if (m_present_queue)
result = vkQueuePresentKHR(m_present_queue, &present_info);
else
{
VkQueue present_queue = VK_NULL_HANDLE;
std::unique_lock<std::mutex> ul = getGraphicsQueue(&present_queue);
result = vkQueuePresentKHR(present_queue, &present_info);
ul.unlock();
}
if (!video::CNullDriver::endScene())
return false;