Skip to content

Commit

Permalink
Add compute
Browse files Browse the repository at this point in the history
  • Loading branch information
@solidpixel
solidpixel committed Dec 12, 2024
1 parent 47a7180 commit 829a39f
Show file tree
Hide file tree
Showing 10 changed files with 246 additions and 39 deletions.
1 change: 1 addition & 0 deletions layer_gpu_timeline/source/layer_device_functions_command_buffer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ VKAPI_ATTR VkResult layer_vkBeginCommandBuffer<user_tag>(
auto& tracker = layer->getStateTracker();
auto& cmdBuffer = tracker.getCommandBuffer(commandBuffer);
cmdBuffer.reset();
cmdBuffer.begin(pBeginInfo->flags & VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT);

// Release the lock to call into the driver
lock.unlock();
Expand Down
80 changes: 71 additions & 9 deletions layer_gpu_timeline/source/layer_device_functions_dispatch.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,13 +32,16 @@

extern std::mutex g_vulkanLock;

static void registerDispatch(
static uint64_t registerDispatch(
Device* layer,
VkCommandBuffer commandBuffer
VkCommandBuffer commandBuffer,
int64_t groupX,
int64_t groupY,
int64_t groupZ
) {
auto& state = layer->getStateTracker();
auto& stats = state.getCommandBuffer(commandBuffer).getStats();
stats.incDispatchCount();
auto& tracker = layer->getStateTracker();
auto& cb = tracker.getCommandBuffer(commandBuffer);
return cb.dispatch(groupX, groupY, groupZ);
}

/* See Vulkan API for documentation. */
Expand All @@ -55,11 +58,27 @@ VKAPI_ATTR void VKAPI_CALL layer_vkCmdDispatch<user_tag>(
std::unique_lock<std::mutex> lock { g_vulkanLock };
auto* layer = Device::retrieve(commandBuffer);

registerDispatch(layer, commandBuffer);
uint64_t tagID = registerDispatch(
layer,
commandBuffer,
static_cast<int64_t>(groupCountX),
static_cast<int64_t>(groupCountY),
static_cast<int64_t>(groupCountZ));

// Emit the unique workload tag into the command stream
std::string tagLabel = formatString("t%" PRIu64, tagID);
VkDebugUtilsLabelEXT tagInfo {
.sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_LABEL_EXT,
.pNext = nullptr,
.pLabelName = tagLabel.c_str(),
.color = { 0.0f, 0.0f, 0.0f, 0.0f }
};

// Release the lock to call into the driver
lock.unlock();
layer->driver.vkCmdBeginDebugUtilsLabelEXT(commandBuffer, &tagInfo);
layer->driver.vkCmdDispatch(commandBuffer, groupCountX, groupCountY, groupCountZ);
layer->driver.vkCmdEndDebugUtilsLabelEXT(commandBuffer);
}

/* See Vulkan API for documentation. */
Expand All @@ -79,11 +98,27 @@ VKAPI_ATTR void VKAPI_CALL layer_vkCmdDispatchBase<user_tag>(
std::unique_lock<std::mutex> lock { g_vulkanLock };
auto* layer = Device::retrieve(commandBuffer);

registerDispatch(layer, commandBuffer);
uint64_t tagID = registerDispatch(
layer,
commandBuffer,
static_cast<int64_t>(groupCountX),
static_cast<int64_t>(groupCountY),
static_cast<int64_t>(groupCountZ));

// Emit the unique workload tag into the command stream
std::string tagLabel = formatString("t%" PRIu64, tagID);
VkDebugUtilsLabelEXT tagInfo {
.sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_LABEL_EXT,
.pNext = nullptr,
.pLabelName = tagLabel.c_str(),
.color = { 0.0f, 0.0f, 0.0f, 0.0f }
};

// Release the lock to call into the driver
lock.unlock();
layer->driver.vkCmdBeginDebugUtilsLabelEXT(commandBuffer, &tagInfo);
layer->driver.vkCmdDispatchBase(commandBuffer, baseGroupX, baseGroupY, baseGroupZ, groupCountX, groupCountY, groupCountZ);
layer->driver.vkCmdEndDebugUtilsLabelEXT(commandBuffer);
}

/* See Vulkan API for documentation. */
Expand All @@ -103,11 +138,27 @@ VKAPI_ATTR void VKAPI_CALL layer_vkCmdDispatchBaseKHR<user_tag>(
std::unique_lock<std::mutex> lock { g_vulkanLock };
auto* layer = Device::retrieve(commandBuffer);

registerDispatch(layer, commandBuffer);
uint64_t tagID = registerDispatch(
layer,
commandBuffer,
static_cast<int64_t>(groupCountX),
static_cast<int64_t>(groupCountY),
static_cast<int64_t>(groupCountZ));

// Emit the unique workload tag into the command stream
std::string tagLabel = formatString("t%" PRIu64, tagID);
VkDebugUtilsLabelEXT tagInfo {
.sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_LABEL_EXT,
.pNext = nullptr,
.pLabelName = tagLabel.c_str(),
.color = { 0.0f, 0.0f, 0.0f, 0.0f }
};

// Release the lock to call into the driver
lock.unlock();
layer->driver.vkCmdBeginDebugUtilsLabelEXT(commandBuffer, &tagInfo);
layer->driver.vkCmdDispatchBaseKHR(commandBuffer, baseGroupX, baseGroupY, baseGroupZ, groupCountX, groupCountY, groupCountZ);
layer->driver.vkCmdEndDebugUtilsLabelEXT(commandBuffer);
}

/* See Vulkan API for documentation. */
Expand All @@ -123,9 +174,20 @@ VKAPI_ATTR void VKAPI_CALL layer_vkCmdDispatchIndirect<user_tag>(
std::unique_lock<std::mutex> lock { g_vulkanLock };
auto* layer = Device::retrieve(commandBuffer);

registerDispatch(layer, commandBuffer);
uint64_t tagID = registerDispatch(layer, commandBuffer, -1, -1, -1);

// Emit the unique workload tag into the command stream
std::string tagLabel = formatString("t%" PRIu64, tagID);
VkDebugUtilsLabelEXT tagInfo {
.sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_LABEL_EXT,
.pNext = nullptr,
.pLabelName = tagLabel.c_str(),
.color = { 0.0f, 0.0f, 0.0f, 0.0f }
};

// Release the lock to call into the driver
lock.unlock();
layer->driver.vkCmdBeginDebugUtilsLabelEXT(commandBuffer, &tagInfo);
layer->driver.vkCmdDispatchIndirect(commandBuffer, buffer, offset);
layer->driver.vkCmdEndDebugUtilsLabelEXT(commandBuffer);
}
2 changes: 1 addition & 1 deletion layer_gpu_timeline/source/layer_device_functions_render_pass.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,7 @@ VKAPI_ATTR void VKAPI_CALL layer_vkCmdBeginRenderPass<user_tag>(

// Emit the unique workload tag into the command stream
std::string tagLabel = formatString("t%" PRIu64, tagID);
[[maybe_unused]] VkDebugUtilsLabelEXT tagInfo {
VkDebugUtilsLabelEXT tagInfo {
.sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_LABEL_EXT,
.pNext = nullptr,
.pLabelName = tagLabel.c_str(),
Expand Down
23 changes: 16 additions & 7 deletions lglpy/service_gpu_timeline.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,7 @@ class GPUTimelineService:
def __init__(self):
self.frame = {
"frame": 0,
"workloads": [

]
"workloads": []
}

# TODO: Make file name configurable
Expand All @@ -45,8 +43,6 @@ def get_service_name(self) -> str:
return 'GPUTimeline'

def handle_frame(self, msg):
print(json.dumps(self.frame, indent=4))

# Write frame packet to the file
lastFrame = json.dumps(self.frame).encode('utf-8')
length = struct.pack('<I', len(lastFrame))
Expand All @@ -55,11 +51,15 @@ def handle_frame(self, msg):
self.fileHandle.write(lastFrame)

# Reset the local frame state for the next frame
nextFrame = msg['fid']
self.frame = {
'frame': msg['fid'],
'frame': nextFrame,
'workloads': []
}

if nextFrame % 100 == 0:
print(f'Starting frame {nextFrame} ...')

def handle_renderpass(self, msg):
# Find the last workload
lastRenderPass = None
Expand All @@ -70,11 +70,17 @@ def handle_renderpass(self, msg):

# Continuation
if lastRenderPass and lastRenderPass['tid'] == msg['tid']:
lastRenderPass['drawCallCount'] += msg['drawCallCount']
# Don't accumulate if tagID is not unique metadata tag
if lastRenderPass['drawCallCount'] != -1:
lastRenderPass['drawCallCount'] += msg['drawCallCount']
# New render pass
else:
self.frame['workloads'].append(msg)

def handle_dispatch(self, msg):
# Find the last workload
self.frame['workloads'].append(msg)

def handle_message(self, message: Message):
payload = message.payload.decode('utf-8')
parsedPayload = json.loads(payload)
Expand All @@ -87,6 +93,9 @@ def handle_message(self, message: Message):
elif payloadType == 'renderpass':
self.handle_renderpass(parsedPayload)

elif payloadType == 'dispatch':
self.handle_dispatch(parsedPayload)

else:
assert False, f'Unknown payload type {payloadType}'

Expand Down
51 changes: 37 additions & 14 deletions source_common/trackers/command_buffer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,14 +43,21 @@ CommandBuffer::CommandBuffer(
/* See header for documentation. */
void CommandBuffer::reset()
{
oneTimeSubmit = false;
stats.reset();
workloads.clear();
workloadCommandStream.clear();
}

/**
* @brief Begin a user debug marker range.
*/
/* See header for documentation. */
void CommandBuffer::begin(
bool _oneTimeSubmit
)
{
oneTimeSubmit = _oneTimeSubmit;
}

/* See header for documentation. */
void CommandBuffer::debugMarkerBegin(
std::string marker
) {
Expand All @@ -63,9 +70,7 @@ void CommandBuffer::debugMarkerBegin(
workloadCommandStream.push_back(instr);
}

/**
* @brief End a user debug marker range.
*/
/* See header for documentation. */
void CommandBuffer::debugMarkerEnd()
{
// Add command with empty workload to update queue debug stack on submit
Expand All @@ -74,9 +79,7 @@ void CommandBuffer::debugMarkerEnd()
workloadCommandStream.push_back(instr);
}

/**
* @brief End a user render pass.
*/
/* See header for documentation. */
uint64_t CommandBuffer::renderPassBegin(
const RenderPass& renderPass,
uint32_t width,
Expand All @@ -99,7 +102,7 @@ uint64_t CommandBuffer::renderPassBegin(
renderPassStartDrawCount = stats.getDrawCallCount();

auto workload = std::make_shared<LCSRenderPass>(
tagID, renderPass, width, height, suspending);
tagID, renderPass, width, height, suspending, oneTimeSubmit);

currentRenderPass = workload;
workloads.push_back(workload);
Expand All @@ -111,9 +114,7 @@ uint64_t CommandBuffer::renderPassBegin(
return tagID;
}

/**
* @brief End a user render pass.
*/
/* See header for documentation. */
bool CommandBuffer::renderPassEnd()
{
assert(currentRenderPass);
Expand All @@ -130,6 +131,28 @@ bool CommandBuffer::renderPassEnd()
return suspending;
}

/* See header for documentation. */
uint64_t CommandBuffer::dispatch(
int64_t xGroups,
int64_t yGroups,
int64_t zGroups
) {
LAYER_LOG("Creating LCSDispatch workload");
uint64_t tagID = Tracker::LCSWorkload::assignTagID();
stats.incDispatchCount();

// Add a workload to the render pass
auto workload = std::make_shared<LCSDispatch>(
tagID, xGroups, yGroups, zGroups);
workloads.push_back(workload);

// Add a command to the layer-side command stream
auto instr = std::make_pair(LCSOpcode::DISPATCH, workload);
workloadCommandStream.push_back(instr);

return tagID;
}

/* See header for documentation. */
void CommandBuffer::executeCommands(
CommandBuffer& secondary
Expand All @@ -142,7 +165,7 @@ void CommandBuffer::executeCommands(
vecAppend(workloadCommandStream, secondary.workloadCommandStream);
}


/* See header for documentation. */
CommandPool::CommandPool(
VkCommandPool _handle) :
handle(_handle)
Expand Down
27 changes: 27 additions & 0 deletions source_common/trackers/command_buffer.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,20 @@ class CommandBuffer
*/
bool renderPassEnd();

/**
* @brief Capture a compute dispatch.
*
* @param xGroups Number of groups in X dimension, or -1 if unknown.
* @param yGroups Number of groups in Y dimension, or -1 if unknown.
* @param zGroups Number of groups in Z dimension, or -1 if unknown.
*
* @return Returns the tagID assigned to this workload.
*/
uint64_t dispatch(
int64_t xGroups,
int64_t yGroups,
int64_t zGroups);

/**
* @brief Begin a user debug marker range.
*/
Expand All @@ -136,12 +150,25 @@ class CommandBuffer
*/
void reset();

/**
* @brief Begin recording back into the @a Recording state.
*
* @param oneTimeSubmit Is this a one-time submit recording.
*/
void begin(
bool oneTimeSubmit);

private:
/**
* @brief The Vulkan API handle of this command buffer.
*/
const VkCommandBuffer handle;

/**
* @brief Is this command buffer recording one-time-submit?
*/
bool oneTimeSubmit { false };

/**
* @brief The command buffer draw count at the start of the render pass.
*/
Expand Down
Loading

0 comments on commit 829a39f

Please sign in to comment.