diff --git a/Source/Game-Lib/Game-Lib/ECS/Scheduler.cpp b/Source/Game-Lib/Game-Lib/ECS/Scheduler.cpp index fbf2ea3d..65c82a20 100644 --- a/Source/Game-Lib/Game-Lib/ECS/Scheduler.cpp +++ b/Source/Game-Lib/Game-Lib/ECS/Scheduler.cpp @@ -13,7 +13,6 @@ #include "Game-Lib/ECS/Systems/Animation.h" #include "Game-Lib/ECS/Systems/UpdateAreaLights.h" #include "Game-Lib/ECS/Systems/CalculateCameraMatrices.h" -#include "Game-Lib/ECS/Systems/CalculateShadowCameraMatrices.h" #include "Game-Lib/ECS/Systems/CalculateTransformMatrices.h" #include "Game-Lib/ECS/Systems/UpdateDayNightCycle.h" #include "Game-Lib/ECS/Systems/DrawDebugMesh.h" @@ -109,7 +108,6 @@ namespace ECS Systems::OrbitalCamera::Update(gameRegistry, clampedDeltaTime); Systems::CalculateCameraMatrices::Update(gameRegistry, clampedDeltaTime); Systems::CharacterControllerInput::UpdateHoveredUnit(gameRegistry, clampedDeltaTime); - Systems::CalculateShadowCameraMatrices::Update(gameRegistry, clampedDeltaTime); Systems::UpdateSkyboxes::Update(gameRegistry, clampedDeltaTime); Systems::UpdateAreaLights::Update(gameRegistry, clampedDeltaTime); Systems::Editor::EditorTools::Update(gameRegistry, clampedDeltaTime); diff --git a/Source/Game-Lib/Game-Lib/ECS/Systems/CalculateShadowCameraMatrices.cpp b/Source/Game-Lib/Game-Lib/ECS/Systems/CalculateShadowCameraMatrices.cpp deleted file mode 100644 index e368a428..00000000 --- a/Source/Game-Lib/Game-Lib/ECS/Systems/CalculateShadowCameraMatrices.cpp +++ /dev/null @@ -1,294 +0,0 @@ -#include "CalculateShadowCameraMatrices.h" - -#include "Game-Lib/ECS/Components/Camera.h" -#include "Game-Lib/ECS/Singletons/ActiveCamera.h" -#include "Game-Lib/ECS/Singletons/DayNightCycle.h" -#include "Game-Lib/ECS/Systems/UpdateAreaLights.h" -#include "Game-Lib/ECS/Util/Transforms.h" -#include "Game-Lib/Rendering/GameRenderer.h" -#include "Game-Lib/Util/ServiceLocator.h" - -#include - -#include - -#include -#include -#include -#include - -AutoCVar_Int CVAR_ShadowsStable(CVarCategory::Client | CVarCategory::Rendering, "shadowStable", "stable shadows", 1, CVarFlags::EditCheckbox); - -AutoCVar_Int CVAR_ShadowCascadeNum(CVarCategory::Client | CVarCategory::Rendering, "shadowCascadeNum", "number of shadow cascades", 0); -AutoCVar_Float CVAR_ShadowCascadeSplitLambda(CVarCategory::Client | CVarCategory::Rendering, "shadowCascadeSplitLambda", "split lambda for cascades, between 0.0f and 1.0f", 0.8f); - -AutoCVar_Int CVAR_ShadowCascadeTextureSize(CVarCategory::Client | CVarCategory::Rendering, "shadowCascadeSize", "size of biggest cascade (per side)", 4096); - -namespace ECS::Systems -{ - inline vec4 EncodePlane(vec3 position, vec3 normal) - { - vec3 normalizedNormal = glm::normalize(normal); - vec4 result = vec4(normalizedNormal, glm::dot(normalizedNormal, position)); - return result; - } - - void CalculateShadowCameraMatrices::Update(entt::registry& registry, f32 deltaTime) - { - ZoneScopedN("ECS::CalculateShadowCameraMatrices"); - - GameRenderer* gameRenderer = ServiceLocator::GetGameRenderer(); - Renderer::Renderer* renderer = gameRenderer->GetRenderer(); - - RenderResources& renderResources = gameRenderer->GetRenderResources(); - - u32 numCascades = CVAR_ShadowCascadeNum.GetU32(); - i32 cascadeTextureSize = CVAR_ShadowCascadeTextureSize.Get(); - bool stableShadows = CVAR_ShadowsStable.Get() == 1; - - // Initialize any new shadow cascades - if (numCascades != renderResources.shadowDepthCascades.size()) - { - u32 numCameras = renderResources.cameras.Count(); - u32 numCamerasToAdd = numCascades - numCameras; - renderResources.cameras.AddCount(numCamerasToAdd); - - while (numCascades > renderResources.shadowDepthCascades.size()) - { - u32 cascadeIndex = static_cast(renderResources.shadowDepthCascades.size()); - - // Shadow depth rendertarget - Renderer::DepthImageDesc shadowDepthDesc; - shadowDepthDesc.dimensions = vec2(cascadeTextureSize, cascadeTextureSize); - shadowDepthDesc.dimensionType = Renderer::ImageDimensionType::DIMENSION_ABSOLUTE; - shadowDepthDesc.format = Renderer::DepthImageFormat::D32_FLOAT; - shadowDepthDesc.sampleCount = Renderer::SampleCount::SAMPLE_COUNT_1; - shadowDepthDesc.depthClearValue = 0.0f; - shadowDepthDesc.debugName = "ShadowDepthCascade" + std::to_string(cascadeIndex); - - Renderer::DepthImageID cascadeDepthImage = renderer->CreateDepthImage(shadowDepthDesc); - renderResources.shadowDepthCascades.push_back(cascadeDepthImage); - } - } - - entt::registry::context& ctx = registry.ctx(); - auto& dayNightCycle = ctx.get(); - - // Get light settings - vec3 lightDirection = UpdateAreaLights::GetLightDirection(dayNightCycle.GetTimeInSecondsF32()); - - // Get active render camera - auto& activeCamera = ctx.get(); - - auto& cameraTransform = registry.get(activeCamera.entity); - auto& camera = registry.get(activeCamera.entity); - - vec3 cameraPos = cameraTransform.GetWorldPosition(); - const mat4x4& cameraViewProj = camera.worldToClip; - - // Calculate frustum split depths and matrices for the shadow map cascades - f32 cascadeSplitLambda = CVAR_ShadowCascadeSplitLambda.GetFloat(); - - f32 cascadeSplits[Renderer::Settings::MAX_SHADOW_CASCADES]; - - f32 nearClip = camera.nearClip; - f32 farClip = camera.farClip; - - f32 clipRange = farClip - nearClip; - - f32 minZ = nearClip; - f32 maxZ = nearClip + clipRange; - - f32 range = maxZ - minZ; - f32 ratio = maxZ / minZ; - - // Calculate split depths based on view camera frustum - // Based on method presented in https://developer.nvidia.com/gpugems/GPUGems3/gpugems3_ch10.html - for (u32 i = 0; i < numCascades; i++) - { - f32 p = (i + 1) / static_cast(numCascades); - f32 log = minZ * std::pow(ratio, p); - f32 uniform = minZ + range * p; - f32 d = cascadeSplitLambda * (log - uniform) + uniform; - - cascadeSplits[i] = 1.0f - ((d - nearClip) / clipRange); - } - - // Calculate orthographic projection matrix for each cascade - f32 lastSplitDist = 1.0f; - for (u32 i = 0; i < numCascades; i++) - { - f32 splitDist = cascadeSplits[i]; - - vec3 frustumCorners[8] = - { - vec3(-1.0f, 1.0f, 0.0f), - vec3(1.0f, 1.0f, 0.0f), - vec3(1.0f, -1.0f, 0.0f), - vec3(-1.0f, -1.0f, 0.0f), - vec3(-1.0f, 1.0f, 1.0f), - vec3(1.0f, 1.0f, 1.0f), - vec3(1.0f, -1.0f, 1.0f), - vec3(-1.0f, -1.0f, 1.0f) - }; - - mat4x4 invViewProj = glm::inverse(cameraViewProj); - for (u32 j = 0; j < 8; j++) - { - vec4 invCorner = invViewProj * vec4(frustumCorners[j], 1.0f); - frustumCorners[j] = invCorner / invCorner.w; - } - - // Get the corners of the current cascsade slice of the view frustum - for (u32 j = 0; j < 4; j++) - { - vec3 cornerRay = frustumCorners[j + 4] - frustumCorners[j]; - vec3 nearCornerRay = cornerRay * lastSplitDist; - vec3 farCornerRay = cornerRay * splitDist; - - frustumCorners[j + 4] = frustumCorners[j] + farCornerRay; // TODO this looks sus - frustumCorners[j] = frustumCorners[j] + nearCornerRay; - } - - // Get frustum center - vec3 frustumCenter = vec3(0.0f); - - for (u32 j = 0; j < 8; j++) - { - frustumCenter += frustumCorners[j]; - } - frustumCenter /= 8.0f; - - vec3 upDir = -cameraTransform.GetLocalRight(); // This was GetLeft, unsure about this, double check if there are issues - - vec3 minExtents; - vec3 maxExtents; - if (stableShadows) - { - // This needs to be constant for it to be stable - upDir = vec3(0.0f, 1.0f, 0.0f); - - // Calculate the radius of a bounding sphere surrounding the frustum corners - f32 sphereRadius = 0.0f; - for (u32 j = 0; j < 8; j++) - { - f32 dist = glm::length(frustumCorners[j] - frustumCenter); - sphereRadius = Math::Max(sphereRadius, dist); - } - - sphereRadius = std::ceil(sphereRadius * 16.0f) / 16.0f; - - maxExtents = vec3(sphereRadius, sphereRadius, sphereRadius); - minExtents = -maxExtents; - } - else - { - // Create a temporary view matrix for the light - vec3 lightCameraPos = frustumCenter; - vec3 lookAt = frustumCenter - lightDirection; - mat4x4 lightView = glm::lookAt(lightCameraPos, lookAt, upDir); - - // Calculate an AABB around the frustum corners - vec3 mins = vec3(std::numeric_limits::max()); - vec3 maxes = vec3(std::numeric_limits::lowest()); - for (u32 j = 0; j < 8; j++) - { - vec4 corner = lightView * vec4(frustumCorners[j], 1.0f); - - mins = glm::min(mins, vec3(corner / corner.w)); - maxes = glm::max(maxes, vec3(corner / corner.w)); - } - - minExtents = mins; - maxExtents = maxes; - - // Adjust the min/max to accommodate the filtering size - const u32 fixedFilterKernelSize = 3; // TODO: Figure this out from the MJP sample - - f32 scale = (cascadeTextureSize + fixedFilterKernelSize) / static_cast(cascadeTextureSize); - minExtents.x *= scale; - minExtents.y *= scale; - maxExtents.x *= scale; - maxExtents.y *= scale; - } - - vec3 cascadeExtents = maxExtents - minExtents; - - // Get postion of the shadow camera - vec3 shadowCameraPos = frustumCenter + lightDirection * -minExtents.z; - - // Store the far and near planes - f32 farPlane = maxExtents.z; - f32 nearPlane = minExtents.z; - - // Come up with a new orthographic projection matrix for the shadow caster - mat4x4 projMatrix = glm::ortho(minExtents.x, maxExtents.x, minExtents.y, maxExtents.y, farPlane - nearPlane, 0.0f); - mat4x4 viewMatrix = glm::lookAt(shadowCameraPos, frustumCenter, upDir); - - if (stableShadows) - { - // Create the rounding matrix, by projecting the world-space origin and determining the fractional offset in texel space - mat4x4 shadowMatrix = projMatrix * viewMatrix; - vec4 shadowOrigin = vec4(0.0f, 0.0f, 0.0f, 1.0f); - shadowOrigin = shadowMatrix * shadowOrigin; - shadowOrigin *= static_cast(cascadeTextureSize) / 2.0f; - - vec4 roundedOrigin = glm::round(shadowOrigin); - vec4 roundOffset = roundedOrigin - shadowOrigin; - roundOffset *= 2.0f / static_cast(cascadeTextureSize); - roundOffset.z = 0.0f; - roundOffset.w = 0.0f; - - projMatrix[3] += roundOffset; - } - - // Store split distance and matrix in cascade camera - Camera& cascadeCamera = renderResources.cameras[i + 1]; // +1 because the first camera is the main camera - - cascadeCamera.worldToView = viewMatrix; - cascadeCamera.viewToClip = projMatrix; - - cascadeCamera.viewToWorld = glm::inverse(cascadeCamera.worldToView); - cascadeCamera.clipToView = glm::inverse(cascadeCamera.viewToClip); - - cascadeCamera.worldToClip = cascadeCamera.viewToClip * cascadeCamera.worldToView; - cascadeCamera.clipToWorld = cascadeCamera.viewToWorld * cascadeCamera.clipToView; - - f32 splitDepth = (farClip - (nearClip + splitDist * clipRange));// *-1.0f; - cascadeCamera.eyePosition = vec4(shadowCameraPos, splitDepth); // w holds split depth - - vec3 scale; - quat rotation; - vec3 translation; - vec3 skew; - vec4 perspective; - glm::decompose(viewMatrix, scale, rotation, translation, skew, perspective); - vec3 eulerAngles = glm::eulerAngles(rotation); - - f32 pitch = eulerAngles.x; - f32 yaw = eulerAngles.y; - f32 roll = eulerAngles.z; - - cascadeCamera.eyeRotation = vec4(roll, pitch, yaw, 0.0f); // Is this order correct? - - // Calculate frustum planes - glm::vec3 front = glm::vec3(0, 0, 1); - glm::vec3 right = glm::vec3(1, 0, 0); - glm::vec3 up = glm::vec3(0, 1, 0); - - front = vec3(viewMatrix * vec4(front, 0.0f)); - right = vec3(viewMatrix * vec4(right, 0.0f)); - up = vec3(viewMatrix * vec4(up, 0.0f)); - - cascadeCamera.frustum[(size_t)FrustumPlane::Near] = EncodePlane(shadowCameraPos + front * nearPlane, front); - cascadeCamera.frustum[(size_t)FrustumPlane::Far] = EncodePlane(shadowCameraPos + front * farPlane , -front); - cascadeCamera.frustum[(size_t)FrustumPlane::Right] = EncodePlane(shadowCameraPos - right * cascadeExtents.x, right); - cascadeCamera.frustum[(size_t)FrustumPlane::Left] = EncodePlane(shadowCameraPos + right * cascadeExtents.x, -right); - cascadeCamera.frustum[(size_t)FrustumPlane::Top] = EncodePlane(shadowCameraPos + up * cascadeExtents.z, -up); - cascadeCamera.frustum[(size_t)FrustumPlane::Bottom] = EncodePlane(shadowCameraPos - up * cascadeExtents.z, up); - renderResources.cameras.SetDirtyElement(i+1); - - lastSplitDist = cascadeSplits[i]; - } - } -} diff --git a/Source/Game-Lib/Game-Lib/ECS/Systems/CalculateShadowCameraMatrices.h b/Source/Game-Lib/Game-Lib/ECS/Systems/CalculateShadowCameraMatrices.h deleted file mode 100644 index b036e206..00000000 --- a/Source/Game-Lib/Game-Lib/ECS/Systems/CalculateShadowCameraMatrices.h +++ /dev/null @@ -1,12 +0,0 @@ -#pragma once -#include -#include - -namespace ECS::Systems -{ - class CalculateShadowCameraMatrices - { - public: - static void Update(entt::registry& registry, f32 deltaTime); - }; -} \ No newline at end of file diff --git a/Source/Game-Lib/Game-Lib/ECS/Systems/UpdateAreaLights.cpp b/Source/Game-Lib/Game-Lib/ECS/Systems/UpdateAreaLights.cpp index c652ab9e..67de43e6 100644 --- a/Source/Game-Lib/Game-Lib/ECS/Systems/UpdateAreaLights.cpp +++ b/Source/Game-Lib/Game-Lib/ECS/Systems/UpdateAreaLights.cpp @@ -17,9 +17,24 @@ #include #include +#include + +AutoCVar_Int CVAR_SunFullRotation(CVarCategory::Client | CVarCategory::Rendering, "sunFullRotation", "sun does a full rotation per day instead of the authored wobble, the sun sets at night", 1, CVarFlags::EditCheckbox); +AutoCVar_Float CVAR_ShadowSunUpdateInterval(CVarCategory::Client | CVarCategory::Rendering, "shadowSunUpdateInterval", "game seconds between shadow sun direction updates, a continuously rotating sun re-renders every shadow texel each frame and shimmers", 120.0f); namespace ECS::Systems { + // The shadow sun steps in discrete intervals, the visual sun stays smooth. A continuously + // rotating light invalidates the whole texel grid every frame, which snapping cannot hide + f32 GetShadowTimeOfDay(f32 timeOfDay) + { + f32 interval = CVAR_ShadowSunUpdateInterval.GetFloat(); + if (interval <= 0.0f) + return timeOfDay; + + return glm::floor(timeOfDay / interval) * interval; + } + vec3 UnpackU32BGRToColor(u32 bgr) { vec3 result; @@ -314,7 +329,7 @@ namespace ECS::Systems const vec3& ambientColor = areaLightInfo.finalColorData.ambientColor; vec3 groundAmbientColor = ambientColor * 1.0f; vec3 skyAmbientColor = ambientColor * 1.0f; - vec3 shadowColor = vec3(77.f/255.f, 77.f/255.f, 77.f/255.f); + const vec3& shadowColor = areaLightInfo.finalColorData.shadowColor; // Per-area authored tint, multiplied onto the shadowed directional term constexpr f32 ambientIntensity = 1.0f; if (!materialRenderer->SetDirectionalLight(0, direction, diffuseColor, 1.0f, groundAmbientColor, ambientIntensity, skyAmbientColor, ambientIntensity, shadowColor)) @@ -324,8 +339,14 @@ namespace ECS::Systems SkyboxRenderer* skyboxRenderer = ServiceLocator::GetGameRenderer()->GetSkyboxRenderer(); skyboxRenderer->SetSkybandColors(areaLightInfo.finalColorData.skybandTopColor, areaLightInfo.finalColorData.skybandMiddleColor, areaLightInfo.finalColorData.skybandBottomColor, areaLightInfo.finalColorData.skybandAboveHorizonColor, areaLightInfo.finalColorData.skybandHorizonColor); + skyboxRenderer->SetSunDirection(direction); // direction points toward the sun + + // Fade shadows out as the sun approaches the horizon, below it the shadow views would project the underside of the world + f32 sunElevationSin = direction.y; // Positive while the sun is above the horizon + f32 shadowStrength = glm::clamp(sunElevationSin / 0.1f, 0.0f, 1.0f); + *CVarSystem::Get()->GetFloatCVar(CVarCategory::Client | CVarCategory::Rendering, "shadowStrength"_h) = shadowStrength; - *CVarSystem::Get()->GetVecFloatCVar(CVarCategory::Client | CVarCategory::Rendering, "fogColor"_h) = vec4(areaLightInfo.finalColorData.fogColor, 1.0f); + *CVarSystem::Get()->GetVecFloatCVar(CVarCategory::Client | CVarCategory::Rendering, "fogColor"_h) = vec4(areaLightInfo.finalColorData.fogColor, 1.0f); *CVarSystem::Get()->GetFloatCVar(CVarCategory::Client | CVarCategory::Rendering, "fogBlendBegin"_h) = areaLightInfo.finalColorData.fogEnd * areaLightInfo.finalColorData.fogScaler; *CVarSystem::Get()->GetFloatCVar(CVarCategory::Client | CVarCategory::Rendering, "fogBlendEnd"_h) = areaLightInfo.finalColorData.fogEnd; } @@ -343,24 +364,33 @@ namespace ECS::Systems }; f32 progressDayAndNight = timeOfDay / 86400.0f; - u32 currentPhiIndex = static_cast(progressDayAndNight / 0.25f); - u32 nextPhiIndex = 0; - - if (currentPhiIndex < 3) - nextPhiIndex = currentPhiIndex + 1; - // Lerp between the current value of phi and the next value of phi + if (CVAR_SunFullRotation.Get()) { - f32 currentTimestamp = currentPhiIndex * 0.25f; - f32 nextTimestamp = nextPhiIndex * 0.25f; + // Full rotation per day, midnight (progress 0) puts the sun straight down, noon straight up + phiValue = progressDayAndNight * glm::two_pi(); + } + else + { + u32 currentPhiIndex = static_cast(progressDayAndNight / 0.25f); + u32 nextPhiIndex = 0; - f32 transitionTime = 0.25f; - f32 transitionProgress = (progressDayAndNight / 0.25f) - currentPhiIndex; + if (currentPhiIndex < 3) + nextPhiIndex = currentPhiIndex + 1; - f32 currentPhiValue = phiTable[currentPhiIndex]; - f32 nextPhiValue = phiTable[nextPhiIndex]; + // Lerp between the current value of phi and the next value of phi + { + f32 currentTimestamp = currentPhiIndex * 0.25f; + f32 nextTimestamp = nextPhiIndex * 0.25f; - phiValue = glm::mix(currentPhiValue, nextPhiValue, transitionProgress); + f32 transitionTime = 0.25f; + f32 transitionProgress = (progressDayAndNight / 0.25f) - currentPhiIndex; + + f32 currentPhiValue = phiTable[currentPhiIndex]; + f32 nextPhiValue = phiTable[nextPhiIndex]; + + phiValue = glm::mix(currentPhiValue, nextPhiValue, transitionProgress); + } } // Convert from Spherical Position to Cartesian coordinates @@ -374,6 +404,8 @@ namespace ECS::Systems f32 lightDirZ = sinPhi * sinTheta; f32 lightDirY = cosPhi; + // Points toward the sun (the shading convention); SVSM consumers negate this to get the + // direction the light travels return vec3(lightDirX, -lightDirY, -lightDirZ); } } \ No newline at end of file diff --git a/Source/Game-Lib/Game-Lib/ECS/Systems/UpdateAreaLights.h b/Source/Game-Lib/Game-Lib/ECS/Systems/UpdateAreaLights.h index c48f4ec7..2985c768 100644 --- a/Source/Game-Lib/Game-Lib/ECS/Systems/UpdateAreaLights.h +++ b/Source/Game-Lib/Game-Lib/ECS/Systems/UpdateAreaLights.h @@ -4,6 +4,9 @@ namespace ECS::Systems { + // Quantizes the time of day used for the shadow sun direction, see shadowSunUpdateInterval + f32 GetShadowTimeOfDay(f32 timeOfDay); + class UpdateAreaLights { public: diff --git a/Source/Game-Lib/Game-Lib/Editor/PerformanceDiagnostics.cpp b/Source/Game-Lib/Game-Lib/Editor/PerformanceDiagnostics.cpp index a468f2ba..d4e453f5 100644 --- a/Source/Game-Lib/Game-Lib/Editor/PerformanceDiagnostics.cpp +++ b/Source/Game-Lib/Game-Lib/Editor/PerformanceDiagnostics.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -79,7 +80,141 @@ namespace Editor const std::string rightHeaderText = "Survived / Total (%)"; static ImGuiTableFlags flags = ImGuiTableFlags_SizingFixedFit /*| ImGuiTableFlags_BordersOuter*/ | ImGuiTableFlags_BordersV | ImGuiTableFlags_BordersH | ImGuiTableFlags_ContextMenuInBody; - u32 numCascades = *CVarSystem::Get()->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "shadowCascadeNum"_h); + const u32 numClipmaps = static_cast(*CVarSystem::Get()->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "svsmNumClipmaps"_h)); + + // SVSM stat block + { + GameRenderer* gameRenderer = ServiceLocator::GetGameRenderer(); + ShadowRenderer* shadowRenderer = gameRenderer->GetShadowRenderer(); + + // Copy the shadow stat block (SVSM pool/dynamic/instances, per-clipmap table) as + // semicolon-separated lines, same rule as the render pass CSV + if (shadowRenderer) + { + const char* copyStatsLabel = "Copy Stats"; + ImGui::SameLine(ImGui::GetContentRegionAvail().x + ImGui::GetCursorPosX() - ImGui::CalcTextSize(copyStatsLabel).x - ImGui::GetStyle().FramePadding.x * 2.0f); + if (ImGui::SmallButton(copyStatsLabel)) + { + std::string csv; + char line[512]; + + { + u32 freePages = 0; + u32 totalPages = 0; + u32 overflow = 0; + u32 invalidationCause = 0; + shadowRenderer->GetSVSMGlobalStats(freePages, totalPages, overflow, invalidationCause); + snprintf(line, sizeof(line), "SVSMPool;used;%u;total;%u;overflow;%u;invalidationCause;%u\n", totalPages - glm::min(freePages, totalPages), totalPages, overflow, invalidationCause); + csv += line; + + u32 dynamicLive = 0; + u32 dynamicTotal = 0; + u32 dynamicOverflow = 0; + shadowRenderer->GetSVSMDynamicStats(dynamicLive, dynamicTotal, dynamicOverflow); + + u32 dynamicCasters = 0; + u32 transitionsIn = 0; + u32 transitionsOut = 0; + u32 droppedAABBs = 0; + shadowRenderer->GetSVSMCasterStats(dynamicCasters, transitionsIn, transitionsOut, droppedAABBs); + snprintf(line, sizeof(line), "SVSMDynamic;live;%u;total;%u;overflow;%u;dynamicCasters;%u;transitionsIn;%u;transitionsOut;%u;droppedAABBs;%u\n", dynamicLive, dynamicTotal, dynamicOverflow, dynamicCasters, transitionsIn, transitionsOut, droppedAABBs); + csv += line; + + snprintf(line, sizeof(line), "SVSMBudgetUsed;%u\n", shadowRenderer->GetSVSMBudgetUsed()); + csv += line; + + if (ModelRenderer* statsModelRenderer = gameRenderer->GetModelRenderer()) + { + std::string dynInstances = "SVSMDynInstances"; + for (u32 view = 1; view <= numClipmaps && view < Renderer::Settings::MAX_VIEWS; view++) + { + dynInstances += ";" + std::to_string(statsModelRenderer->GetNumSVSMDynamicInstances(view)); + } + csv += dynInstances + "\n"; + } + + csv += "Clipmap;extent;marked;resident;dirty;invalidated;evicted;dynamic;deferred\n"; + for (u32 i = 0; i < numClipmaps; i++) + { + ShadowRenderer::SVSMClipmapStats stats; + if (!shadowRenderer->GetSVSMClipmapStats(i, stats)) + break; + + snprintf(line, sizeof(line), "P%u;%.0f;%u;%u;%u;%u;%u;%u;%u\n", i, stats.extent, stats.marked, stats.resident, stats.dirty, stats.invalidated, stats.evicted, stats.dynamicLive, stats.deferred); + csv += line; + } + } + + ImGui::SetClipboardText(csv.c_str()); + } + } + + // SVSM page table state, one frame old + if (shadowRenderer) + { + u32 freePages = 0; + u32 totalPages = 0; + u32 overflow = 0; + u32 invalidationCause = 0; + shadowRenderer->GetSVSMGlobalStats(freePages, totalPages, overflow, invalidationCause); + + // Every line renders unconditionally: one-frame events must never add or + // remove lines, a layout that jumps for a frame is unreadable + ImGui::Spacing(); + ImGui::Text("SVSM Pool: %u / %u pages used%s", totalPages - glm::min(freePages, totalPages), totalPages, overflow > 0 ? " (request overflow!)" : ""); + + u32 dynamicLive = 0; + u32 dynamicTotal = 0; + u32 dynamicOverflow = 0; + shadowRenderer->GetSVSMDynamicStats(dynamicLive, dynamicTotal, dynamicOverflow); + + u32 dynamicCasters = 0; + u32 transitionsIn = 0; + u32 transitionsOut = 0; + u32 droppedAABBs = 0; + shadowRenderer->GetSVSMCasterStats(dynamicCasters, transitionsIn, transitionsOut, droppedAABBs); + ImGui::Text("SVSM Dynamic: %u / %u pages live%s | %u casters (+%u / -%u total) | %u spilled", dynamicLive, dynamicTotal, dynamicOverflow > 0 ? " (overflowing!)" : "", dynamicCasters, transitionsIn, transitionsOut, droppedAABBs); + + // Per-clipmap surviving dynamic instances of the SVSM dynamic fills, one + // frame old. A bouncing count here = the fill/cull chain loses casters + if (ModelRenderer* svsmModelRenderer = gameRenderer->GetModelRenderer()) + { + std::string dynDraws = "SVSM Dyn instances:"; + for (u32 view = 1; view <= numClipmaps && view < Renderer::Settings::MAX_VIEWS; view++) + { + dynDraws += " " + std::to_string(svsmModelRenderer->GetNumSVSMDynamicInstances(view)); + } + ImGui::Text("%s", dynDraws.c_str()); + } + + i32 renderBudget = *CVarSystem::Get()->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "svsmRenderBudget"_h); + if (renderBudget > 0) + { + ImGui::Text("SVSM Budget: %u rendered / %d allowed", shadowRenderer->GetSVSMBudgetUsed(), renderBudget); + } + + // Invalidations happen on a single frame, latch the last cause so the line is + // stable and readable + if (invalidationCause != 0) + { + _svsmLastInvalidationCause.clear(); + if (invalidationCause & 1) _svsmLastInvalidationCause += "sun step "; + if (invalidationCause & 2) _svsmLastInvalidationCause += "z range "; + if (invalidationCause & 4) _svsmLastInvalidationCause += "manual "; + if (invalidationCause & 8) _svsmLastInvalidationCause += "aabb overflow "; + } + ImGui::Text("SVSM Last Invalidation: %s", _svsmLastInvalidationCause.empty() ? "-" : _svsmLastInvalidationCause.c_str()); + + for (u32 i = 0; i < numClipmaps; i++) + { + ShadowRenderer::SVSMClipmapStats stats; + if (!shadowRenderer->GetSVSMClipmapStats(i, stats)) + break; + + ImGui::Text("P%u (%.0fm): %u marked, %u resident, %u dirty, %u inv, %u evict, %u dyn, %u defer", i, stats.extent, stats.marked, stats.resident, stats.dirty, stats.invalidated, stats.evicted, stats.dynamicLive, stats.deferred); + } + } + } ImGui::Spacing(); @@ -131,10 +266,10 @@ namespace Editor switch (i) { case 0: - DrawSurvivingDrawCalls(heightConstraint, rightHeaderText, numCascades); + DrawSurvivingDrawCalls(heightConstraint, rightHeaderText, numClipmaps); break; case 1: - DrawSurvivingTriangles(heightConstraint, rightHeaderText, numCascades); + DrawSurvivingTriangles(heightConstraint, rightHeaderText, numClipmaps); break; case 2: DrawFrameTimes(heightConstraint, average, flags); @@ -180,10 +315,10 @@ namespace Editor switch (i) { case 0: - DrawSurvivingDrawCalls(heightConstraint, rightHeaderText, numCascades, widthConstraint); + DrawSurvivingDrawCalls(heightConstraint, rightHeaderText, numClipmaps, widthConstraint); break; case 1: - DrawSurvivingTriangles(heightConstraint, rightHeaderText, numCascades, widthConstraint); + DrawSurvivingTriangles(heightConstraint, rightHeaderText, numClipmaps, widthConstraint); break; case 2: DrawFrameTimes(heightConstraint, average, flags, widthConstraint); @@ -241,9 +376,9 @@ namespace Editor { ImGui::TableNextRow(); ImGui::TableSetColumnIndex(0); - DrawSurvivingDrawCalls(newHeightProportions[0], rightHeaderText, numCascades); + DrawSurvivingDrawCalls(newHeightProportions[0], rightHeaderText, numClipmaps); ImGui::TableSetColumnIndex(1); - DrawSurvivingTriangles(newHeightProportions[1], rightHeaderText, numCascades); + DrawSurvivingTriangles(newHeightProportions[1], rightHeaderText, numClipmaps); ImGui::TableNextRow(); ImGui::TableSetColumnIndex(0); @@ -268,8 +403,8 @@ namespace Editor height *= ImGui::GetContentRegionAvail().y; } - DrawSurvivingDrawCalls(newHeightProportions[0], rightHeaderText, numCascades); - DrawSurvivingTriangles(newHeightProportions[1], rightHeaderText, numCascades); + DrawSurvivingDrawCalls(newHeightProportions[0], rightHeaderText, numClipmaps); + DrawSurvivingTriangles(newHeightProportions[1], rightHeaderText, numClipmaps); DrawFrameTimes(newHeightProportions[2], average, flags); DrawRenderPass(newHeightProportions[3], renderer, stats, flags); DrawFrameTimesGraph(newHeightProportions[4], stats); @@ -321,7 +456,7 @@ namespace Editor } } - void PerformanceDiagnostics::DrawSurvivingDrawCalls(f32 constraint, const std::string& text, u32 numCascades, f32 widthConstraint) + void PerformanceDiagnostics::DrawSurvivingDrawCalls(f32 constraint, const std::string& text, u32 numClipmaps, f32 widthConstraint) { if (!_showSurvivingDrawCalls) return; @@ -356,7 +491,7 @@ namespace Editor if (!_drawCallStatsOnlyForMainView) { - for (u32 i = 1; i < numCascades + 1; i++) + for (u32 i = 1; i < numClipmaps + 1; i++) { DrawCullingDrawCallStatsView(i, textPos, totalDrawCalls, totalDrawCallsSurvived); } @@ -369,7 +504,7 @@ namespace Editor } } - void PerformanceDiagnostics::DrawSurvivingTriangles(f32 constraint, const std::string& text, u32 numCascades, f32 widthConstraint) + void PerformanceDiagnostics::DrawSurvivingTriangles(f32 constraint, const std::string& text, u32 numClipmaps, f32 widthConstraint) { if (!_showSurvivingTriangle) return; @@ -404,7 +539,7 @@ namespace Editor if (!_drawCallStatsOnlyForMainView) { - for (u32 i = 1; i < numCascades + 1; i++) + for (u32 i = 1; i < numClipmaps + 1; i++) { DrawCullingTriangleStatsView(i, textPos, totalTriangles, totalTrianglesSurvived); } @@ -480,6 +615,29 @@ namespace Editor if (frameTimeQueries.size() > 0) { ImGui::Text("Render Passes (GPU)"); + + // Copy the pass list as "name;ms" lines. Semicolon separator so a comma decimal + // separator can never collide with it + const char* copyLabel = "Copy CSV"; + ImGui::SameLine(ImGui::GetContentRegionAvail().x + ImGui::GetCursorPosX() - ImGui::CalcTextSize(copyLabel).x - ImGui::GetStyle().FramePadding.x * 2.0f); + if (ImGui::SmallButton(copyLabel)) + { + std::string csv = "Pass;Milliseconds\n"; + for (u32 i = 0; i < frameTimeQueries.size(); i++) + { + const std::string& name = renderer->GetTimeQueryName(frameTimeQueries[i]); + + f32 averageMS = 0.0f; + if (stats.AverageNamed(name, 240, averageMS)) + { + char line[256]; + snprintf(line, sizeof(line), "%s;%.3f\n", name.c_str(), averageMS); + csv += line; + } + } + ImGui::SetClipboardText(csv.c_str()); + } + if (ImGui::BeginTable("passtimes", 2, flags)) { for (u32 i = 0; i < frameTimeQueries.size(); i++) @@ -574,7 +732,7 @@ namespace Editor std::string viewName = "Main View Instances"; if (viewID > 0) { - viewName = "Shadow Cascade " + std::to_string(viewID - 1) + " Drawcalls"; + viewName = "Clipmap " + std::to_string(viewID - 1) + " Drawcalls"; } if (!_drawCallStatsOnlyForMainView) @@ -594,11 +752,11 @@ namespace Editor ImGui::Text("%s", rightHeaderText.c_str()); ImGui::Separator(); } - + u32 viewDrawCalls = 0; u32 viewDrawCallsSurvived = 0; - bool viewSupportsTerrainOcclusionCulling = true; + bool viewSupportsTerrainOcclusionCulling = viewID == 0; // Clipmap views draw their full surviving set in one phase, no occluders bool viewSupportsModelsOcclusionCulling = viewID == 0; bool viewRendersTerrainCulling = true; @@ -683,7 +841,7 @@ namespace Editor std::string viewName = "Main View Triangles"; if (viewID > 0) { - viewName = "Shadow Cascade " + std::to_string(viewID - 1) + " Triangles"; + viewName = "Clipmap " + std::to_string(viewID - 1) + " Triangles"; } if (!_drawCallStatsOnlyForMainView) @@ -707,7 +865,7 @@ namespace Editor u32 viewTriangles = 0; u32 viewTrianglesSurvived = 0; - bool viewSupportsTerrainOcclusionCulling = true; + bool viewSupportsTerrainOcclusionCulling = viewID == 0; // Clipmap views draw their full surviving set in one phase, no occluders bool viewSupportsModelsOcclusionCulling = viewID == 0; bool viewRendersTerrainCulling = true; // Only main view supports terrain culling so far diff --git a/Source/Game-Lib/Game-Lib/Editor/PerformanceDiagnostics.h b/Source/Game-Lib/Game-Lib/Editor/PerformanceDiagnostics.h index 69fd0676..d321842f 100644 --- a/Source/Game-Lib/Game-Lib/Editor/PerformanceDiagnostics.h +++ b/Source/Game-Lib/Game-Lib/Editor/PerformanceDiagnostics.h @@ -39,8 +39,8 @@ namespace Editor void DrawCullingStatsEntry(std::string_view name, u32 drawCalls, u32 survivedDrawCalls); - void DrawSurvivingDrawCalls(f32 constraint, const std::string& text, u32 numCascades, f32 widthConstraint = -1.f); - void DrawSurvivingTriangles(f32 constraint, const std::string& text, u32 numCascades, f32 widthConstraint = -1.f); + void DrawSurvivingDrawCalls(f32 constraint, const std::string& text, u32 numClipmaps, f32 widthConstraint = -1.f); + void DrawSurvivingTriangles(f32 constraint, const std::string& text, u32 numClipmaps, f32 widthConstraint = -1.f); void DrawFrameTimes(f32 constraint, const ECS::Singletons::FrameTimes& average, const ImGuiTableFlags& flags, f32 widthConstraint = -1.f); void DrawRenderPass(f32 constraint, Renderer::Renderer* renderer, ECS::Singletons::EngineStats& stats, const ImGuiTableFlags& flags, f32 widthConstraint = -1.f); void DrawFrameTimesGraph(f32 constraint, const ECS::Singletons::EngineStats& stats, f32 widthConstraint = -1.f); @@ -52,6 +52,8 @@ namespace Editor private: bool _drawCallStatsOnlyForMainView = true; + std::string _svsmLastInvalidationCause; // Latched, invalidations only exist for one frame + bool _showSurvivingDrawCalls = true; bool _showSurvivingTriangle = true; bool _showFrameTime = true; diff --git a/Source/Game-Lib/Game-Lib/Rendering/CulledRenderer.cpp b/Source/Game-Lib/Game-Lib/Rendering/CulledRenderer.cpp index cdbf3b95..b8237fa1 100644 --- a/Source/Game-Lib/Game-Lib/Rendering/CulledRenderer.cpp +++ b/Source/Game-Lib/Game-Lib/Rendering/CulledRenderer.cpp @@ -4,8 +4,13 @@ #include "Game-Lib/Rendering/GameRenderer.h" #include "Game-Lib/Rendering/RenderUtils.h" +#include + +AutoCVar_Int CVAR_ShadowDebugCullingView(CVarCategory::Client | CVarCategory::Rendering, "shadowDebugCullingView", "debug draw culling results for a view as AABBs (green = kept, red = culled), 0 = main view, 1..N = clipmaps, -1 = off", -1); + bool CulledRenderer::_pipelinesCreated = false; Renderer::ComputePipelineID CulledRenderer::_fillInstancedDrawCallsFromBitmaskPipeline[2]; // [0] = non-indexed, [1] = indexed +Renderer::ComputePipelineID CulledRenderer::_fillInstancedDrawCallsFilteredPipeline[2]; // Same, with the SVSM dynamic-mask filter Renderer::ComputePipelineID CulledRenderer::_fillDrawCallsFromBitmaskPipeline[2]; // [0] = non-indexed, [1] = indexed Renderer::ComputePipelineID CulledRenderer::_createIndirectAfterCullingPipeline[2]; // [0] = non-indexed, [1] = indexed Renderer::ComputePipelineID CulledRenderer::_createIndirectAfterCullingOrderedPipeline[2]; // [0] = non-indexed, [1] = indexed @@ -36,6 +41,11 @@ void CulledRenderer::InitCullingResources(CullingResourcesBase& cullingResources Renderer::DescriptorSet& geometryFillDescriptorSet = cullingResources.GetGeometryFillDescriptorSet(); geometryFillDescriptorSet.RegisterPipeline(_renderer, fillPipeline); + if (isInstanced) + { + // SVSM caster-split fills, only ever dispatched for resources that bind the dynamic mask + geometryFillDescriptorSet.RegisterPipeline(_renderer, _fillInstancedDrawCallsFilteredPipeline[isIndexed]); + } geometryFillDescriptorSet.Init(_renderer); } @@ -62,6 +72,104 @@ void CulledRenderer::Clear() } +// One thread per instance against a view's bitmask slice, appending survivors to the shared +// per-drawcall counts/lookup buffers. An indirect args buffer routes the dispatch through +// Finalize-written per-view group counts instead of covering every instance +void CulledRenderer::DispatchInstancedFill(PassParams& params, const std::string& markerName, Renderer::DescriptorSetResource& fillSet, bool filtered, u32 currentBitmaskIndex, u32 bitmaskOffset, bool keepDynamic, u32 baseInstanceLookupOffset, u32 drawCallDataSize, Renderer::BufferResource indirectArgsBuffer, u32 indirectArgsByteOffset) +{ + const u32 numInstances = params.cullingResources->GetNumInstances(); + + params.commandList->PushMarker(markerName, Color::White); + + Renderer::ComputePipelineID pipeline = filtered ? _fillInstancedDrawCallsFilteredPipeline[params.cullingResources->IsIndexed()] + : _fillInstancedDrawCallsFromBitmaskPipeline[params.cullingResources->IsIndexed()]; + params.commandList->BeginPipeline(pipeline); + + struct FillDrawCallConstants + { + u32 numTotalInstances; + u32 baseInstanceLookupOffset; // Byte offset into drawCallDatas where the baseInstanceLookup is stored + u32 drawCallDataSize; + u32 currentBitmaskIndex; + u32 bitmaskOffset; + u32 keepDynamic; + }; + + FillDrawCallConstants* fillConstants = params.graphResources->FrameNew(); + fillConstants->numTotalInstances = numInstances; + fillConstants->baseInstanceLookupOffset = baseInstanceLookupOffset; + fillConstants->drawCallDataSize = drawCallDataSize; + fillConstants->currentBitmaskIndex = currentBitmaskIndex; + fillConstants->bitmaskOffset = bitmaskOffset; + fillConstants->keepDynamic = keepDynamic; + params.commandList->PushConstant(fillConstants, 0, sizeof(FillDrawCallConstants)); + + params.commandList->BindDescriptorSet(fillSet, params.frameIndex); + + if (indirectArgsBuffer != Renderer::BufferResource::Invalid()) + { + // Finalize zeroed the group count for rings with no page work this frame + params.commandList->DispatchIndirect(indirectArgsBuffer, indirectArgsByteOffset); + } + else + { + params.commandList->Dispatch((numInstances + 31) / 32, 1, 1); + } + + params.commandList->EndPipeline(pipeline); + params.commandList->PopMarker(); +} + +// Per-drawcall instance counts become indirect draw args, each count cleared after consumption +// (the counts buffer is always zero between uses). An indirect args buffer applies the same +// Finalize gate as the fill that produced the counts +void CulledRenderer::DispatchCreateIndirect(PassParams& params, const std::string& markerName, Renderer::DescriptorSetResource& createIndirectSet, Renderer::DescriptorSetResource* debugSet, u32 baseInstanceLookupOffset, u32 drawCallDataSize, Renderer::BufferResource indirectArgsBuffer, u32 indirectArgsByteOffset) +{ + const u32 numDrawCalls = params.cullingResources->GetDrawCallCount(); + const bool debugOrdered = false; // Single-group ordered variant for determinism debugging + + params.commandList->PushMarker(markerName, Color::Yellow); + + Renderer::ComputePipelineID pipeline = debugOrdered ? _createIndirectAfterCullingOrderedPipeline[params.cullingResources->IsIndexed()] + : _createIndirectAfterCullingPipeline[params.cullingResources->IsIndexed()]; + params.commandList->BeginPipeline(pipeline); + + struct CreateIndirectConstants + { + u32 numTotalDrawCalls; + u32 baseInstanceLookupOffset; + u32 drawCallDataSize; + }; + CreateIndirectConstants* createIndirectConstants = params.graphResources->FrameNew(); + createIndirectConstants->numTotalDrawCalls = numDrawCalls; + createIndirectConstants->baseInstanceLookupOffset = baseInstanceLookupOffset; + createIndirectConstants->drawCallDataSize = drawCallDataSize; + params.commandList->PushConstant(createIndirectConstants, 0, sizeof(CreateIndirectConstants)); + + if (debugSet != nullptr) + { + params.commandList->BindDescriptorSet(*debugSet, params.frameIndex); + } + params.commandList->BindDescriptorSet(createIndirectSet, params.frameIndex); + + if (indirectArgsBuffer != Renderer::BufferResource::Invalid()) + { + // Zero groups for views with no page work this frame, same gate as the fill + params.commandList->DispatchIndirect(indirectArgsBuffer, indirectArgsByteOffset); + } + else if (debugOrdered) + { + params.commandList->Dispatch(1, 1, 1); + } + else + { + params.commandList->Dispatch((numDrawCalls + 31) / 32, 1, 1); + } + + params.commandList->EndPipeline(pipeline); + params.commandList->PopMarker(); +} + void CulledRenderer::OccluderPass(OccluderPassParams& params) { NC_ASSERT(params.drawCallback != nullptr, "CulledRenderer : OccluderPass got params with invalid drawCallback"); @@ -83,101 +191,28 @@ void CulledRenderer::OccluderPass(OccluderPassParams& params) if (params.cullingResources->IsInstanced()) { - const bool debugOrdered = false; - params.commandList->FillBuffer(params.culledInstanceCountsBuffer, 0, sizeof(u32) * numDrawCalls, 0); + params.commandList->FillBuffer(params.drawCountBuffer, 0, sizeof(u32), 0); // CreateIndirect accumulates the surviving counts, reset or the readback stats carry over from last frame + params.commandList->FillBuffer(params.triangleCountBuffer, 0, sizeof(u32), 0); params.commandList->BufferBarrier(params.culledInstanceCountsBuffer, Renderer::BufferPassUsage::TRANSFER); + params.commandList->BufferBarrier(params.drawCountBuffer, Renderer::BufferPassUsage::TRANSFER); + params.commandList->BufferBarrier(params.triangleCountBuffer, Renderer::BufferPassUsage::TRANSFER); if (params.disableTwoStepCulling) { - params.commandList->FillBuffer(params.culledDrawCallsBitMaskBuffer, 0, RenderUtils::CalcCullingBitmaskSize(numInstances), 0); + u32 sizePerView = params.cullingResources->GetBitMaskBufferSizePerView(); + params.commandList->FillBuffer(params.culledDrawCallsBitMaskBuffer, 0, sizePerView * Renderer::Settings::MAX_VIEWS, 0); params.commandList->BufferBarrier(params.culledDrawCallsBitMaskBuffer, Renderer::BufferPassUsage::TRANSFER); } - // Fill the occluders to draw - { - std::string debugName = params.passName + " Instanced Occlusion Fill"; - params.commandList->PushMarker(debugName, Color::White); - - Renderer::ComputePipelineID pipeline = _fillInstancedDrawCallsFromBitmaskPipeline[params.cullingResources->IsIndexed()]; - params.commandList->BeginPipeline(pipeline); - - struct FillDrawCallConstants - { - u32 numTotalInstances; - u32 baseInstanceLookupOffset; // Byte offset into drawCallDatas where the baseInstanceLookup is stored - u32 drawCallDataSize; - u32 currentBitmaskIndex; - }; - - FillDrawCallConstants* fillConstants = params.graphResources->FrameNew(); - fillConstants->numTotalInstances = numInstances; - fillConstants->baseInstanceLookupOffset = params.baseInstanceLookupOffset; - fillConstants->drawCallDataSize = params.drawCallDataSize; - fillConstants->currentBitmaskIndex = !params.frameIndex; // Occluders consume last frame's culling output - - params.commandList->PushConstant(fillConstants, 0, sizeof(FillDrawCallConstants)); - - // Bind descriptorset - params.commandList->BindDescriptorSet(params.occluderFillDescriptorSet, params.frameIndex); - - params.commandList->Dispatch((numInstances + 31) / 32, 1, 1); - - params.commandList->EndPipeline(pipeline); - - params.commandList->PopMarker(); - } + // Fill the occluders to draw: last frame's main-view culling output + DispatchInstancedFill(params, params.passName + " Instanced Occlusion Fill", params.occluderFillDescriptorSet, false, !params.frameIndex, 0, false, params.baseInstanceLookupOffset, params.drawCallDataSize, Renderer::BufferResource::Invalid(), 0); params.commandList->FillBuffer(params.culledDrawCallCountBuffer, 0, sizeof(u32), 0); params.commandList->BufferBarrier(params.culledDrawCallCountBuffer, Renderer::BufferPassUsage::TRANSFER); // Create indirect argument buffer - { - std::string debugName = params.passName + " Create Indirect"; - params.commandList->PushMarker(debugName, Color::Yellow); - - Renderer::ComputePipelineDesc cullingPipelineDesc; - cullingPipelineDesc.debugName = debugName; - params.graphResources->InitializePipelineDesc(cullingPipelineDesc); - - Renderer::ComputePipelineID pipeline; - if (debugOrdered) - { - pipeline = _createIndirectAfterCullingOrderedPipeline[params.cullingResources->IsIndexed()]; - } - else - { - pipeline = _createIndirectAfterCullingPipeline[params.cullingResources->IsIndexed()]; - } - params.commandList->BeginPipeline(pipeline); - - struct CullConstants - { - u32 numTotalDrawCalls; - u32 baseInstanceLookupOffset; - u32 drawCallDataSize; - }; - CullConstants* cullConstants = params.graphResources->FrameNew(); - - cullConstants->numTotalDrawCalls = numDrawCalls; - cullConstants->baseInstanceLookupOffset = params.baseInstanceLookupOffset; - cullConstants->drawCallDataSize = params.drawCallDataSize; - params.commandList->PushConstant(cullConstants, 0, sizeof(CullConstants)); - - params.commandList->BindDescriptorSet(params.createIndirectDescriptorSet, params.frameIndex); - - if (debugOrdered) - { - params.commandList->Dispatch(1, 1, 1); - } - else - { - params.commandList->Dispatch((numDrawCalls + 31) / 32, 1, 1); - } - - params.commandList->EndPipeline(pipeline); - params.commandList->PopMarker(); - } + DispatchCreateIndirect(params, params.passName + " Create Indirect", params.createIndirectDescriptorSet, nullptr, params.baseInstanceLookupOffset, params.drawCallDataSize, Renderer::BufferResource::Invalid(), 0); params.commandList->BufferBarrier(params.culledDrawCallsBuffer, Renderer::BufferPassUsage::COMPUTE); @@ -188,11 +223,10 @@ void CulledRenderer::OccluderPass(OccluderPassParams& params) DrawParams drawParams; drawParams.cullingEnabled = true; // The occuder pass only makes sense if culling is enabled - drawParams.shadowPass = false; drawParams.viewIndex = 0; drawParams.rt0 = params.rt0; drawParams.rt1 = params.rt1; - drawParams.depth = params.depth[0]; + drawParams.depth = params.depth; drawParams.argumentBuffer = params.culledDrawCallsBuffer; drawParams.drawCountBuffer = params.culledDrawCallCountBuffer; drawParams.drawCountIndex = 0; @@ -223,110 +257,153 @@ void CulledRenderer::OccluderPass(OccluderPassParams& params) params.commandList->BufferBarrier(params.culledDrawCallsBitMaskBuffer, Renderer::BufferPassUsage::TRANSFER); } - for (u32 i = 0; i < params.numCascades + 1; i++) + // Reset the counters { - std::string markerName = (i == 0) ? "Main" : "Cascade " + std::to_string(i - 1); - params.commandList->PushMarker(markerName, Color::PastelYellow); + params.commandList->BufferBarrier(params.drawCountBuffer, Renderer::BufferPassUsage::TRANSFER); + params.commandList->BufferBarrier(params.triangleCountBuffer, Renderer::BufferPassUsage::TRANSFER); - // Reset the counters - { - params.commandList->BufferBarrier(params.drawCountBuffer, Renderer::BufferPassUsage::TRANSFER); - params.commandList->BufferBarrier(params.triangleCountBuffer, Renderer::BufferPassUsage::TRANSFER); + params.commandList->FillBuffer(params.drawCountBuffer, 0, sizeof(u32), 0); + params.commandList->FillBuffer(params.triangleCountBuffer, 0, sizeof(u32), 0); - // Reset the counters - params.commandList->FillBuffer(params.drawCountBuffer, 0, sizeof(u32), 0); - params.commandList->FillBuffer(params.triangleCountBuffer, 0, sizeof(u32), 0); + params.commandList->BufferBarrier(params.drawCountBuffer, Renderer::BufferPassUsage::TRANSFER); + params.commandList->BufferBarrier(params.triangleCountBuffer, Renderer::BufferPassUsage::TRANSFER); + } - params.commandList->BufferBarrier(params.drawCountBuffer, Renderer::BufferPassUsage::TRANSFER); - params.commandList->BufferBarrier(params.triangleCountBuffer, Renderer::BufferPassUsage::TRANSFER); - } + // Fill the occluders to draw + { + std::string debugName = params.passName + " Occlusion Fill"; + params.commandList->PushMarker(debugName, Color::White); + + Renderer::ComputePipelineID pipeline = _fillDrawCallsFromBitmaskPipeline[params.cullingResources->IsIndexed()]; + params.commandList->BeginPipeline(pipeline); - if (i == 1) + struct FillDrawCallConstants { - uvec2 shadowDepthDimensions = params.graphResources->GetImageDimensions(params.depth[1]); + u32 numTotalDraws; + u32 bitmaskOffset; + u32 diffAgainstPrev; + u32 currentBitmaskIndex; + }; - params.commandList->SetViewport(0, 0, static_cast(shadowDepthDimensions.x), static_cast(shadowDepthDimensions.y), 0.0f, 1.0f); - params.commandList->SetScissorRect(0, shadowDepthDimensions.x, 0, shadowDepthDimensions.y); + FillDrawCallConstants* fillConstants = params.graphResources->FrameNew(); + fillConstants->numTotalDraws = numDrawCalls; + fillConstants->bitmaskOffset = 0; // Occluders only draw the main view + fillConstants->diffAgainstPrev = 0; // Occluders should not diff against prev + fillConstants->currentBitmaskIndex = !params.frameIndex; // Occluders consume last frame's culling output + params.commandList->PushConstant(fillConstants, 0, sizeof(FillDrawCallConstants)); - params.commandList->SetDepthBias(params.biasConstantFactor, params.biasClamp, params.biasSlopeFactor); - } + // Bind descriptorset + params.commandList->BindDescriptorSet(params.globalDescriptorSet, params.frameIndex); + params.commandList->BindDescriptorSet(params.occluderFillDescriptorSet, params.frameIndex); - // Fill the occluders to draw - { - std::string debugName = params.passName + " Occlusion Fill"; - params.commandList->PushMarker(debugName, Color::White); + params.commandList->Dispatch((numDrawCalls + 31) / 32, 1, 1); - Renderer::ComputePipelineID pipeline = _fillDrawCallsFromBitmaskPipeline[params.cullingResources->IsIndexed()]; - params.commandList->BeginPipeline(pipeline); + params.commandList->EndPipeline(pipeline); - struct FillDrawCallConstants - { - u32 numTotalDraws; - u32 bitmaskOffset; - u32 diffAgainstPrev; - u32 currentBitmaskIndex; - }; + params.commandList->PopMarker(); + } - FillDrawCallConstants* fillConstants = params.graphResources->FrameNew(); - fillConstants->numTotalDraws = numDrawCalls; - fillConstants->bitmaskOffset = i * params.cullingResources->GetBitMaskBufferUintsPerView(); - fillConstants->diffAgainstPrev = 0; // Occluders should not diff against prev - fillConstants->currentBitmaskIndex = !params.frameIndex; // Occluders consume last frame's culling output - params.commandList->PushConstant(fillConstants, 0, sizeof(FillDrawCallConstants)); + params.commandList->BufferBarrier(params.culledDrawCallsBuffer, Renderer::BufferPassUsage::COMPUTE); + params.commandList->BufferBarrier(params.drawCountBuffer, Renderer::BufferPassUsage::COMPUTE); + params.commandList->BufferBarrier(params.triangleCountBuffer, Renderer::BufferPassUsage::COMPUTE); - // Bind descriptorset - params.commandList->BindDescriptorSet(params.globalDescriptorSet, params.frameIndex); - params.commandList->BindDescriptorSet(params.occluderFillDescriptorSet, params.frameIndex); + if (params.enableDrawing) + { + // Draw Occluders + params.commandList->PushMarker(params.passName + " Occlusion Draw " + std::to_string(numDrawCalls), Color::White); - params.commandList->Dispatch((numDrawCalls + 31) / 32, 1, 1); + DrawParams drawParams; + drawParams.cullingEnabled = true; // The occuder pass only makes sense if culling is enabled + drawParams.viewIndex = 0; + drawParams.rt0 = params.rt0; + drawParams.rt1 = params.rt1; + drawParams.depth = params.depth; + drawParams.argumentBuffer = params.culledDrawCallsBuffer; + drawParams.drawCountBuffer = params.drawCountBuffer; + drawParams.drawCountIndex = 0; + drawParams.numMaxDrawCalls = numDrawCalls; - params.commandList->EndPipeline(pipeline); + params.drawCallback(drawParams); + } - params.commandList->PopMarker(); - } + // Copy from our draw count buffer to the readback buffer + params.commandList->CopyBuffer(params.drawCountReadBackBuffer, 0, params.drawCountBuffer, 0, sizeof(u32)); + params.commandList->CopyBuffer(params.triangleCountReadBackBuffer, 0, params.triangleCountBuffer, 0, sizeof(u32)); - params.commandList->BufferBarrier(params.culledDrawCallsBuffer, Renderer::BufferPassUsage::COMPUTE); - params.commandList->BufferBarrier(params.drawCountBuffer, Renderer::BufferPassUsage::COMPUTE); - params.commandList->BufferBarrier(params.triangleCountBuffer, Renderer::BufferPassUsage::COMPUTE); + if (params.enableDrawing) + { + params.commandList->PopMarker(); + } + } +} - if (params.enableDrawing) - { - // Draw Occluders - params.commandList->PushMarker(params.passName + " Occlusion Draw " + std::to_string(numDrawCalls), Color::White); - - DrawParams drawParams; - drawParams.cullingEnabled = true; // The occuder pass only makes sense if culling is enabled - drawParams.shadowPass = i > 0; - drawParams.viewIndex = i; - drawParams.rt0 = params.rt0; - drawParams.rt1 = params.rt1; - drawParams.depth = params.depth[i]; - drawParams.argumentBuffer = params.culledDrawCallsBuffer; - drawParams.drawCountBuffer = params.drawCountBuffer; - drawParams.drawCountIndex = 0; - drawParams.numMaxDrawCalls = numDrawCalls; +// The instanced cull dispatch shared by CullingPass and ClipmapCullingPass: one thread per +// instance against the per-view frustums, writing bitmask slices and instance counts. +// occlusionCull/cullMainView/debugDrawColliders come from params; the clipmap pass overrides +// them before calling +void CulledRenderer::RunInstancedCullingDispatch(CullingPassParams& params, bool useBitmasks, bool bindDepthPyramid) +{ + const u32 numInstances = params.cullingResources->GetNumInstances(); - params.drawCallback(drawParams); - } + Renderer::ComputePipelineID pipeline = _cullingInstancedPipeline[useBitmasks]; + params.commandList->BeginPipeline(pipeline); - // Copy from our draw count buffer to the readback buffer - params.commandList->CopyBuffer(params.drawCountReadBackBuffer, sizeof(u32) * i, params.drawCountBuffer, 0, sizeof(u32)); - params.commandList->CopyBuffer(params.triangleCountReadBackBuffer, sizeof(u32) * i, params.triangleCountBuffer, 0, sizeof(u32)); + vec2 viewportSize = _renderer->GetRenderSize(); - if (params.enableDrawing) - { - params.commandList->PopMarker(); - } + struct CullConstants + { + u32 viewportSizeX; + u32 viewportSizeY; + u32 numTotalInstances; + u32 occlusionCull; + u32 instanceCountOffset; // Byte offset into drawCalls where the instanceCount is stored + u32 drawCallSize; + u32 baseInstanceLookupOffset; // Byte offset into drawCallDatas where the baseInstanceLookup is stored + u32 modelIDOffset; // Byte offset into drawCallDatas where the modelID is stored + u32 drawCallDataSize; + u32 cullingDataIsWorldspace; // TODO: This controls two things, are both needed? I feel like one counters the other but I'm not sure... + u32 debugDrawColliders; + u32 currentBitmaskIndex; + u32 numShadowViews; + u32 bitMaskBufferUintsPerView; + u32 debugDrawView; + u32 cullMainView; + }; + CullConstants* cullConstants = params.graphResources->FrameNew(); + cullConstants->viewportSizeX = u32(viewportSize.x); + cullConstants->viewportSizeY = u32(viewportSize.y); + cullConstants->numTotalInstances = numInstances; + cullConstants->occlusionCull = params.occlusionCull; + + u32 instanceCountOffset = params.cullingResources->IsIndexed() ? offsetof(Renderer::IndexedIndirectDraw, Renderer::IndexedIndirectDraw::instanceCount) : offsetof(Renderer::IndirectDraw, Renderer::IndirectDraw::instanceCount); + cullConstants->instanceCountOffset = instanceCountOffset; + cullConstants->drawCallSize = params.cullingResources->IsIndexed() ? sizeof(Renderer::IndexedIndirectDraw) : sizeof(Renderer::IndirectDraw); + + cullConstants->baseInstanceLookupOffset = params.baseInstanceLookupOffset; + cullConstants->modelIDOffset = params.modelIDOffset; + cullConstants->drawCallDataSize = params.drawCallDataSize; + + cullConstants->cullingDataIsWorldspace = params.cullingDataIsWorldspace; + cullConstants->debugDrawColliders = params.debugDrawColliders; + cullConstants->currentBitmaskIndex = params.frameIndex; + cullConstants->numShadowViews = params.numShadowViews; + cullConstants->bitMaskBufferUintsPerView = params.cullingResources->GetBitMaskBufferUintsPerView(); + cullConstants->debugDrawView = static_cast(CVAR_ShadowDebugCullingView.Get()); + cullConstants->cullMainView = params.cullMainView; + params.commandList->PushConstant(cullConstants, 0, sizeof(CullConstants)); + + if (bindDepthPyramid) + { + params.cullingDescriptorSet.Bind("_depthPyramid"_h, params.depthPyramid); + } - params.commandList->PopMarker(); - } + params.commandList->BindDescriptorSet(params.debugDescriptorSet, params.frameIndex); + params.commandList->BindDescriptorSet(params.globalDescriptorSet, params.frameIndex); + params.commandList->BindDescriptorSet(params.cullingDescriptorSet, params.frameIndex); - // Finish by resetting the viewport, scissor and depth bias - vec2 renderSize = _renderer->GetRenderSize(); - params.commandList->SetViewport(0, 0, renderSize.x, renderSize.y, 0.0f, 1.0f); - params.commandList->SetScissorRect(0, static_cast(renderSize.x), 0, static_cast(renderSize.y)); - params.commandList->SetDepthBias(0, 0, 0); - } + params.commandList->Dispatch((numInstances + 31) / 32, 1, 1); + + params.commandList->EndPipeline(pipeline); } void CulledRenderer::CullingPass(CullingPassParams& params) @@ -340,8 +417,6 @@ void CulledRenderer::CullingPass(CullingPassParams& params) { if (params.cullingResources->IsInstanced()) { - const bool debugOrdered = false; - params.commandList->PushMarker(params.passName + " Culling", Color::Yellow); // Reset the counters @@ -360,104 +435,15 @@ void CulledRenderer::CullingPass(CullingPassParams& params) std::string debugName = params.passName + " Instanced Culling"; params.commandList->PushMarker(debugName, Color::Yellow); - Renderer::ComputePipelineID pipeline = _cullingInstancedPipeline[!params.disableTwoStepCulling]; - params.commandList->BeginPipeline(pipeline); - - vec2 viewportSize = _renderer->GetRenderSize(); + RunInstancedCullingDispatch(params, !params.disableTwoStepCulling, true); - struct CullConstants - { - u32 viewportSizeX; - u32 viewportSizeY; - u32 numTotalInstances; - u32 occlusionCull; - u32 instanceCountOffset; // Byte offset into drawCalls where the instanceCount is stored - u32 drawCallSize; - u32 baseInstanceLookupOffset; // Byte offset into drawCallDatas where the baseInstanceLookup is stored - u32 modelIDOffset; // Byte offset into drawCallDatas where the modelID is stored - u32 drawCallDataSize; - u32 cullingDataIsWorldspace; // TODO: This controls two things, are both needed? I feel like one counters the other but I'm not sure... - u32 debugDrawColliders; - u32 currentBitmaskIndex; - }; - CullConstants* cullConstants = params.graphResources->FrameNew(); - cullConstants->viewportSizeX = u32(viewportSize.x); - cullConstants->viewportSizeY = u32(viewportSize.y); - cullConstants->numTotalInstances = numInstances; - cullConstants->occlusionCull = params.occlusionCull; - - u32 instanceCountOffset = params.cullingResources->IsIndexed() ? offsetof(Renderer::IndexedIndirectDraw, Renderer::IndexedIndirectDraw::instanceCount) : offsetof(Renderer::IndirectDraw, Renderer::IndirectDraw::instanceCount); - cullConstants->instanceCountOffset = instanceCountOffset; - cullConstants->drawCallSize = params.cullingResources->IsIndexed() ? sizeof(Renderer::IndexedIndirectDraw) : sizeof(Renderer::IndirectDraw); - - cullConstants->baseInstanceLookupOffset = params.baseInstanceLookupOffset; - cullConstants->modelIDOffset = params.modelIDOffset; - cullConstants->drawCallDataSize = params.drawCallDataSize; - - cullConstants->cullingDataIsWorldspace = params.cullingDataIsWorldspace; - cullConstants->debugDrawColliders = params.debugDrawColliders; - cullConstants->currentBitmaskIndex = params.frameIndex; - params.commandList->PushConstant(cullConstants, 0, sizeof(CullConstants)); - - params.cullingDescriptorSet.Bind("_depthPyramid"_h, params.depthPyramid); - - params.commandList->BindDescriptorSet(params.debugDescriptorSet, params.frameIndex); - params.commandList->BindDescriptorSet(params.globalDescriptorSet, params.frameIndex); - params.commandList->BindDescriptorSet(params.cullingDescriptorSet, params.frameIndex); - - params.commandList->Dispatch((numInstances + 31) / 32, 1, 1); - - params.commandList->EndPipeline(pipeline); params.commandList->PopMarker(); } params.commandList->BufferBarrier(params.culledDrawCallCountBuffer, Renderer::BufferPassUsage::COMPUTE); // Create indirect argument buffer - { - std::string debugName = params.passName + " Create Indirect"; - params.commandList->PushMarker(debugName, Color::Yellow); - - - Renderer::ComputePipelineID pipeline; - if (debugOrdered) - { - pipeline = _createIndirectAfterCullingOrderedPipeline[params.cullingResources->IsIndexed()]; - } - else - { - pipeline = _createIndirectAfterCullingPipeline[params.cullingResources->IsIndexed()]; - } - params.commandList->BeginPipeline(pipeline); - - struct CullConstants - { - u32 numTotalDrawCalls; - u32 baseInstanceLookupOffset; - u32 drawCallDataSize; - }; - CullConstants* cullConstants = params.graphResources->FrameNew(); - - cullConstants->numTotalDrawCalls = numDrawCalls; - cullConstants->baseInstanceLookupOffset = params.baseInstanceLookupOffset; - cullConstants->drawCallDataSize = params.drawCallDataSize; - params.commandList->PushConstant(cullConstants, 0, sizeof(CullConstants)); - - params.commandList->BindDescriptorSet(params.debugDescriptorSet, params.frameIndex); - params.commandList->BindDescriptorSet(params.createIndirectAfterCullSet, params.frameIndex); - - if (debugOrdered) - { - params.commandList->Dispatch(1, 1, 1); - } - else - { - params.commandList->Dispatch((numDrawCalls + 31) / 32, 1, 1); - } - - params.commandList->EndPipeline(pipeline); - params.commandList->PopMarker(); - } + DispatchCreateIndirect(params, params.passName + " Create Indirect", params.createIndirectAfterCullSet, ¶ms.debugDescriptorSet, params.baseInstanceLookupOffset, params.drawCallDataSize, Renderer::BufferResource::Invalid(), 0); params.commandList->PopMarker(); } @@ -484,7 +470,7 @@ void CulledRenderer::CullingPass(CullingPassParams& params) u32 viewportSizeX; u32 viewportSizeY; u32 maxDrawCount; - u32 numCascades; + u32 numShadowViews; u32 occlusionCull; u32 instanceIDOffset; u32 modelIDOffset; @@ -499,7 +485,7 @@ void CulledRenderer::CullingPass(CullingPassParams& params) cullConstants->viewportSizeX = u32(viewportSize.x); cullConstants->viewportSizeY = u32(viewportSize.y); cullConstants->maxDrawCount = numDrawCalls; - cullConstants->numCascades = params.numCascades; + cullConstants->numShadowViews = params.numShadowViews; cullConstants->occlusionCull = params.occlusionCull; cullConstants->instanceIDOffset = params.instanceIDOffset; cullConstants->modelIDOffset = params.modelIDOffset; @@ -544,33 +530,123 @@ void CulledRenderer::CullingPass(CullingPassParams& params) } } +void CulledRenderer::ClipmapCullingPass(CullingPassParams& params) +{ + NC_ASSERT(params.drawCallDataSize > 0, "CulledRenderer : ClipmapCullingPass params provided an invalid drawCallDataSize"); + NC_ASSERT(params.cullingResources->IsInstanced(), "CulledRenderer : ClipmapCullingPass only supports instanced culling resources"); + + const u32 numDrawCalls = params.cullingResources->GetDrawCallCount(); + u32 numInstances = params.cullingResources->GetNumInstances(); + + if (numDrawCalls == 0 || numInstances == 0 || params.numShadowViews == 0) + return; + + // Frustum-only cull of the clipmap views into their bitmask slices. No counter resets (the + // per-clipmap fill in the geometry pass resets and rebuilds the shared draw sets), no occlusion + std::string debugName = params.passName + " Clipmap Culling"; + params.commandList->PushMarker(debugName, Color::Yellow); + + params.occlusionCull = false; + params.cullMainView = false; + params.debugDrawColliders = false; + + // Clipmaps require the bitmask permutation; _depthPyramid stays bound from the main culling + // pass, rebinding here would rewrite an already-bound set + RunInstancedCullingDispatch(params, true, false); + + params.commandList->PopMarker(); +} + +// Rebuilds the shared instance counts/lookup/argument buffers from a view's bitmask slice. +// filtered selects the SVSM caster-split fill variant, keepDynamic its instance class +void CulledRenderer::RunInstancedGeometryFill(GeometryPassParams& params, u32 viewIndex, bool filtered, bool keepDynamic, const std::string& fillMarkerName, const std::string& createIndirectMarkerName) +{ + const u32 numDrawCalls = params.cullingResources->GetDrawCallCount(); + + // SVSM fills gate their drawcall-granularity setup (instance-count clear + CreateIndirect) + // on the Finalize-written per-view args; the counts buffer stays always-zero between uses + // because CreateIndirect clears after consumption. The args byte layout comes through params + // (the ShadowRenderer's stride/offset constants) + const bool gatedSetup = params.svsmPass && params.svsmFillArgsBuffer != Renderer::BufferResource::Invalid(); + Renderer::BufferResource fillArgsBuffer = gatedSetup ? params.svsmFillArgsBuffer : Renderer::BufferResource::Invalid(); + const u32 fillArgsOffset = gatedSetup ? (viewIndex - 1) * params.svsmFillArgsViewStride + (keepDynamic ? params.svsmFillArgsDynamicOffset : 0) : 0; + const u32 overheadArgsOffset = gatedSetup ? (viewIndex - 1) * params.svsmFillArgsViewStride + (keepDynamic ? params.svsmFillArgsDynamicOverheadOffset : params.svsmFillArgsStaticOverheadOffset) : 0; + + // Reset the counters. The per-drawcall instance counts skip their FillBuffer in the gated + // path: CreateIndirect clears each count after consuming it, so the buffer is always zero + // between uses (and a gated-off CreateIndirect means nothing read the counts this view). + // The previous view's draws consumed the culled args at DRAW_INDIRECT — this view's + // CreateIndirect rewrite needs the execution dependency against that read (WAR) + params.commandList->BufferBarrier(params.culledDrawCallsBuffer, Renderer::BufferPassUsage::GRAPHICS); + params.commandList->BufferBarrier(params.drawCountBuffer, Renderer::BufferPassUsage::TRANSFER); + params.commandList->BufferBarrier(params.triangleCountBuffer, Renderer::BufferPassUsage::TRANSFER); + if (!gatedSetup) + { + params.commandList->BufferBarrier(params.culledInstanceCountsBuffer, Renderer::BufferPassUsage::TRANSFER); + params.commandList->FillBuffer(params.culledInstanceCountsBuffer, 0, sizeof(u32) * numDrawCalls, 0); + params.commandList->BufferBarrier(params.culledInstanceCountsBuffer, Renderer::BufferPassUsage::TRANSFER); + } + params.commandList->FillBuffer(params.drawCountBuffer, 0, sizeof(u32), 0); + params.commandList->FillBuffer(params.triangleCountBuffer, 0, sizeof(u32), 0); + params.commandList->BufferBarrier(params.drawCountBuffer, Renderer::BufferPassUsage::TRANSFER); + params.commandList->BufferBarrier(params.triangleCountBuffer, Renderer::BufferPassUsage::TRANSFER); + + // Fill the instances visible in this view + DispatchInstancedFill(params, fillMarkerName, params.fillDescriptorSet, filtered, params.frameIndex, viewIndex * params.cullingResources->GetBitMaskBufferUintsPerView(), keepDynamic, params.baseInstanceLookupOffset, params.drawCallDataSize, fillArgsBuffer, fillArgsOffset); + + params.commandList->BufferBarrier(params.culledInstanceCountsBuffer, Renderer::BufferPassUsage::COMPUTE); + + // The draws consume this count, a zero-work view must still draw zero. The previous view's + // draws read it at DRAW_INDIRECT (and CreateIndirect wrote it in compute) — wait on both + params.commandList->BufferBarrier(params.culledDrawCallCountBuffer, Renderer::BufferPassUsage::GRAPHICS | Renderer::BufferPassUsage::COMPUTE); + params.commandList->FillBuffer(params.culledDrawCallCountBuffer, 0, sizeof(u32), 0); + params.commandList->BufferBarrier(params.culledDrawCallCountBuffer, Renderer::BufferPassUsage::TRANSFER); + + // Create indirect argument buffer + DispatchCreateIndirect(params, createIndirectMarkerName, params.createIndirectDescriptorSet, nullptr, params.baseInstanceLookupOffset, params.drawCallDataSize, fillArgsBuffer, overheadArgsOffset); + + params.commandList->BufferBarrier(params.culledDrawCallsBuffer, Renderer::BufferPassUsage::COMPUTE); + params.commandList->BufferBarrier(params.culledDrawCallCountBuffer, Renderer::BufferPassUsage::COMPUTE); + params.commandList->BufferBarrier(params.drawCountBuffer, Renderer::BufferPassUsage::COMPUTE); + params.commandList->BufferBarrier(params.triangleCountBuffer, Renderer::BufferPassUsage::COMPUTE); +} + void CulledRenderer::GeometryPass(GeometryPassParams& params) { NC_ASSERT(params.drawCallback != nullptr, "CulledRenderer : GeometryPass got params with invalid drawCallback"); const u32 numDrawCalls = params.cullingResources->GetDrawCallCount(); - for (u32 i = 0; i < params.numCascades + 1; i++) + // svsmProfileGeometry: per-view fill/draw GPU timings, they surface in the perf editor's + // render pass list + const bool profileSVSM = params.svsmPass && *CVarSystem::Get()->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "svsmProfileGeometry"_h) != 0; + RenderUtils::SVSMGeometryProfiler Profiled(_renderer, *params.commandList, "SVSM", profileSVSM); + + // View-invariant marker strings and cvar reads, the loop body runs up to 8 views x 2 fills + const std::string fillMarkerName = params.passName + " Instanced Geometry Fill"; + const std::string createIndirectMarkerName = params.passName + " Create Indirect"; + const std::string drawMarkerName = params.passName + " " + std::to_string(numDrawCalls); + const bool svsmClipRectsEnabled = params.svsmPass && *CVarSystem::Get()->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "svsmClipRects"_h) != 0; + + for (u32 i = params.firstViewIndex; i < params.numShadowViews + 1; i++) { - std::string markerName = (i == 0) ? "Main" : "Cascade " + std::to_string(i - 1); + std::string markerName = (i == 0) ? "Main" : "Clipmap " + std::to_string(i - 1); params.commandList->PushMarker(markerName, Color::PastelYellow); + if (i == 1) + { + // The viewport spans the virtual texture so SV_Position.xy is the virtual texel. + // No depth bias: there is no depth attachment for it to apply to + params.commandList->SetViewport(0, 0, static_cast(params.svsmExtent.x), static_cast(params.svsmExtent.y), 0.0f, 1.0f); + params.commandList->SetScissorRect(0, params.svsmExtent.x, 0, params.svsmExtent.y); + } + // Reset the counters if (!params.cullingResources->IsInstanced() && params.cullingResources->HasSupportForTwoStepCulling()) { params.commandList->BufferBarrier(params.drawCountBuffer, Renderer::BufferPassUsage::TRANSFER); params.commandList->FillBuffer(params.drawCountBuffer, 0, sizeof(u32), 0); params.commandList->BufferBarrier(params.drawCountBuffer, Renderer::BufferPassUsage::TRANSFER); - - if (i == 1) - { - uvec2 shadowDepthDimensions = params.graphResources->GetImageDimensions(params.depth[1]); - - params.commandList->SetViewport(0, 0, static_cast(shadowDepthDimensions.x), static_cast(shadowDepthDimensions.y), 0.0f, 1.0f); - params.commandList->SetScissorRect(0, shadowDepthDimensions.x, 0, shadowDepthDimensions.y); - - params.commandList->SetDepthBias(params.biasConstantFactor, params.biasClamp, params.biasSlopeFactor); - } // Fill the geometry to draw { @@ -610,6 +686,16 @@ void CulledRenderer::GeometryPass(GeometryPassParams& params) params.commandList->BufferBarrier(params.drawCountBuffer, Renderer::BufferPassUsage::COMPUTE); params.commandList->BufferBarrier(params.triangleCountBuffer, Renderer::BufferPassUsage::COMPUTE); } + else if (params.cullingResources->IsInstanced() && params.cullingResources->HasSupportForTwoStepCulling() && i > 0) + { + // The main view (i == 0) consumes the culling pass's output directly, cascades rebuild the + // shared instance counts/lookup/argument buffers from their bitmask slice before drawing. + // Always the FILTERED fill: the geometry fill set's Vk layout is the union of both fill + // permutations (it includes the dynamic mask binding), and the unfiltered pipeline's own + // layout is incompatible with it (VUID-00358). keepDynamic 0 against an all-zero mask + // keeps every instance, so this is the unfiltered behavior whenever the split is idle + Profiled("Fill", i, [&] { RunInstancedGeometryFill(params, i, true, false, fillMarkerName, createIndirectMarkerName); }); + } if (!params.cullingEnabled) { @@ -621,13 +707,14 @@ void CulledRenderer::GeometryPass(GeometryPassParams& params) if (params.enableDrawing) { - params.commandList->PushMarker(params.passName + " " + std::to_string(numDrawCalls), Color::White); + params.commandList->PushMarker(drawMarkerName, Color::White); const u32 debugDrawCallBufferIndex = 0;//CVAR_ComplexModelDebugShadowDraws.Get(); DrawParams drawParams; drawParams.cullingEnabled = params.cullingEnabled; - drawParams.shadowPass = i > 0; + drawParams.svsmPass = params.svsmPass; + drawParams.svsmExtent = params.svsmExtent; drawParams.viewIndex = i; drawParams.cullingResources = params.cullingResources; drawParams.rt0 = params.rt0; @@ -665,7 +752,8 @@ void CulledRenderer::GeometryPass(GeometryPassParams& params) } - params.drawCallback(drawParams); + const bool svsmClipRects = svsmClipRectsEnabled && i > 0; + Profiled("Draw", i, [&] { RenderUtils::DrawSVSMClipRects(svsmClipRects, drawParams, [&](DrawParams& rectDrawParams) { params.drawCallback(rectDrawParams); }); }); } // Copy from our draw count buffer to the readback buffer @@ -677,14 +765,44 @@ void CulledRenderer::GeometryPass(GeometryPassParams& params) params.commandList->PopMarker(); } + // SVSM caster split: rebuild the shared buffers with only dynamic instances and draw them + // into the dynamic pool. Runs after the static readback copies so the perf stats show the + // static (dominant) counts. Rings without resident dynamic pages cost a zero-group + // indirect fill and a zero-count draw (Finalize's fill args, same-frame GPU truth) + if (params.svsmSplitFills && params.enableDrawing && i > 0) + { + params.commandList->PushMarker("Dynamic", Color::PastelOrange); + + Profiled("DynFill", i, [&] { RunInstancedGeometryFill(params, i, true, true, fillMarkerName, createIndirectMarkerName); }); + + DrawParams drawParams; + drawParams.cullingEnabled = params.cullingEnabled; + drawParams.svsmPass = params.svsmPass; + drawParams.svsmDynamicPass = true; + drawParams.svsmExtent = params.svsmExtent; + drawParams.viewIndex = i; + drawParams.cullingResources = params.cullingResources; + drawParams.argumentBuffer = params.culledDrawCallsBuffer; + drawParams.drawCountBuffer = params.culledDrawCallCountBuffer; + drawParams.drawCountIndex = 0; + drawParams.numMaxDrawCalls = params.cullingResources->GetDrawCallCount(); + + Profiled("DynDraw", i, [&] { params.drawCallbackDynamic(drawParams); }); + + // Dynamic surviving-count readback: the fill wrote this view's dynamic instance count + // to drawCountBuffer, snapshot it so missing dynamic draws are visible in the perf editor + params.commandList->CopyBuffer(params.svsmDynamicDrawCountReadBackBuffer, sizeof(u32) * i, params.drawCountBuffer, 0, sizeof(u32)); + + params.commandList->PopMarker(); + } + params.commandList->PopMarker(); } - // Finish by resetting the viewport, scissor and depth bias + // Finish by resetting the viewport and scissor vec2 renderSize = _renderer->GetRenderSize(); params.commandList->SetViewport(0, 0, renderSize.x, renderSize.y, 0.0f, 1.0f); params.commandList->SetScissorRect(0, static_cast(renderSize.x), 0, static_cast(renderSize.y)); - params.commandList->SetDepthBias(0, 0, 0); } void CulledRenderer::BindCullingResource(CullingResourcesBase& resources) @@ -723,22 +841,33 @@ void CulledRenderer::CreatePipelines() // Fill Drawcalls From Bitmask pipelines Renderer::ComputePipelineDesc pipelineDesc; { - pipelineDesc.debugName = "FillInstancedDrawcallsFromBitmask"; - for (u32 i = 0; i < 2; i++) { - std::vector permutationFields = + for (u32 filtered = 0; filtered < 2; filtered++) { - { "IS_INDEXED", std::to_string(i) } - }; - u32 shaderEntryNameHash = Renderer::GetShaderEntryNameHash("Utils/FillInstancedDrawCallsFromBitmask.cs", permutationFields); + pipelineDesc.debugName = (filtered == 0) ? "FillInstancedDrawcallsFromBitmask" : "FillInstancedDrawcallsFromBitmaskFiltered"; - Renderer::ComputeShaderDesc shaderDesc; - shaderDesc.shaderEntry = shaderDesc.shaderEntry = _gameRenderer->GetShaderEntry(shaderEntryNameHash, "Utils/FillInstancedDrawCallsFromBitmask.cs"); + std::vector permutationFields = + { + { "IS_INDEXED", std::to_string(i) }, + { "DYNAMIC_FILTER", std::to_string(filtered) } + }; + u32 shaderEntryNameHash = Renderer::GetShaderEntryNameHash("Utils/FillInstancedDrawCallsFromBitmask.cs", permutationFields); - pipelineDesc.computeShader = _renderer->LoadShader(shaderDesc); + Renderer::ComputeShaderDesc shaderDesc; + shaderDesc.shaderEntry = _gameRenderer->GetShaderEntry(shaderEntryNameHash, "Utils/FillInstancedDrawCallsFromBitmask.cs"); + + pipelineDesc.computeShader = _renderer->LoadShader(shaderDesc); - _fillInstancedDrawCallsFromBitmaskPipeline[i] = _renderer->CreatePipeline(pipelineDesc); + if (filtered == 0) + { + _fillInstancedDrawCallsFromBitmaskPipeline[i] = _renderer->CreatePipeline(pipelineDesc); + } + else + { + _fillInstancedDrawCallsFilteredPipeline[i] = _renderer->CreatePipeline(pipelineDesc); + } + } } } { diff --git a/Source/Game-Lib/Game-Lib/Rendering/CulledRenderer.h b/Source/Game-Lib/Game-Lib/Rendering/CulledRenderer.h index f845033b..8601ea92 100644 --- a/Source/Game-Lib/Game-Lib/Rendering/CulledRenderer.h +++ b/Source/Game-Lib/Game-Lib/Rendering/CulledRenderer.h @@ -37,14 +37,17 @@ class CulledRenderer { public: bool cullingEnabled = false; - bool shadowPass = false; + bool svsmPass = false; // Attachment-less page render, needs an explicit render area + bool svsmDynamicPass = false; // Caster-split dynamic instances into the dynamic pool + u32 svsmRectIndex = 0xFFFFFFFFu; // Clip rect this draw renders (0-2), SVSM_CLIP_RECT_DISABLED = no clipping u32 viewIndex = 0; CullingResourcesBase* cullingResources; - + Renderer::ImageMutableResource rt0; Renderer::ImageMutableResource rt1; Renderer::DepthImageMutableResource depth; + uvec2 svsmExtent = uvec2(0, 0); Renderer::BufferMutableResource argumentBuffer; Renderer::BufferMutableResource drawCountBuffer; @@ -125,7 +128,7 @@ class CulledRenderer public: Renderer::ImageMutableResource rt0; Renderer::ImageMutableResource rt1; - Renderer::DepthImageMutableResource depth[Renderer::Settings::MAX_VIEWS]; + Renderer::DepthImageMutableResource depth; Renderer::BufferMutableResource culledDrawCallsBuffer; Renderer::BufferMutableResource culledDrawCallCountBuffer; @@ -149,12 +152,6 @@ class CulledRenderer u32 baseInstanceLookupOffset = 0; u32 drawCallDataSize = 0; - u32 numCascades = 0; - - f32 biasConstantFactor = 0.0f; - f32 biasClamp = 0.0f; - f32 biasSlopeFactor = 0.0f; - bool enableDrawing = false; // Allows us to do everything but the actual drawcall, for debugging bool disableTwoStepCulling = false; }; @@ -224,7 +221,8 @@ class CulledRenderer Renderer::DescriptorSetResource cullingDescriptorSet; Renderer::DescriptorSetResource createIndirectAfterCullSet; - u32 numCascades = 0; + u32 numShadowViews = 0; + bool cullMainView = true; bool occlusionCull = true; bool disableTwoStepCulling = false; @@ -246,15 +244,15 @@ class CulledRenderer data.drawCallsBuffer = builder.Write(cullingResources->GetDrawCalls().GetBuffer(), BufferUsage::GRAPHICS | BufferUsage::COMPUTE); data.culledDrawCallsBuffer = builder.Write(cullingResources->GetCulledDrawsBuffer(), BufferUsage::GRAPHICS | BufferUsage::COMPUTE); - data.culledDrawCallCountBuffer = builder.Write(cullingResources->GetCulledDrawCallCountBuffer(), BufferUsage::GRAPHICS); + data.culledDrawCallCountBuffer = builder.Write(cullingResources->GetCulledDrawCallCountBuffer(), BufferUsage::TRANSFER | BufferUsage::GRAPHICS | BufferUsage::COMPUTE); data.drawCountBuffer = builder.Write(cullingResources->GetDrawCountBuffer(), BufferUsage::TRANSFER | BufferUsage::GRAPHICS | BufferUsage::COMPUTE); data.triangleCountBuffer = builder.Write(cullingResources->GetTriangleCountBuffer(), BufferUsage::TRANSFER | BufferUsage::GRAPHICS | BufferUsage::COMPUTE); data.drawCountReadBackBuffer = builder.Write(cullingResources->GetDrawCountReadBackBuffer(), BufferUsage::TRANSFER); data.triangleCountReadBackBuffer = builder.Write(cullingResources->GetTriangleCountReadBackBuffer(), BufferUsage::TRANSFER); - builder.Read(cullingResources->GetInstanceRefs().GetBuffer(), BufferUsage::GRAPHICS); - builder.Read(cullingResources->GetCulledInstanceLookupTableBuffer(), BufferUsage::GRAPHICS); + builder.Read(cullingResources->GetInstanceRefs().GetBuffer(), BufferUsage::GRAPHICS | BufferUsage::COMPUTE); + builder.Write(cullingResources->GetCulledInstanceLookupTableBuffer(), BufferUsage::COMPUTE | BufferUsage::GRAPHICS); data.drawSet = builder.Use(cullingResources->GetGeometryPassDescriptorSet()); } @@ -291,6 +289,8 @@ class CulledRenderer Renderer::BufferMutableResource culledDrawCallCountBuffer; + Renderer::BufferMutableResource culledInstanceCountsBuffer; // Instanced only, used to rebuild per-cascade draw sets + Renderer::BufferMutableResource drawCountBuffer; Renderer::BufferMutableResource triangleCountBuffer; Renderer::BufferMutableResource drawCountReadBackBuffer; @@ -298,20 +298,61 @@ class CulledRenderer Renderer::DescriptorSetResource globalDescriptorSet; Renderer::DescriptorSetResource fillDescriptorSet; + Renderer::DescriptorSetResource createIndirectDescriptorSet; // Instanced only Renderer::DescriptorSetResource drawDescriptorSet; std::function drawCallback; - u32 numCascades = 0; + u32 baseInstanceLookupOffset = 0; // Instanced only + u32 drawCallDataSize = 0; // Instanced only - f32 biasConstantFactor = 0.0f; - f32 biasClamp = 0.0f; - f32 biasSlopeFactor = 0.0f; + u32 firstViewIndex = 0; // 0 = main view first, 1 = clipmap views only + u32 numShadowViews = 0; bool enableDrawing = false; // Allows us to do everything but the actual drawcall, for debugging bool cullingEnabled = false; + + bool svsmPass = false; // SVSM page render: no depth targets, attachment-less draws into svsmExtent + uvec2 svsmExtent = uvec2(0, 0); + + // SVSM caster split: per view, fill+draw static instances (drawCallback) then dynamic + // instances (drawCallbackDynamic), filtered by the dynamic instance mask + bool svsmSplitFills = false; + std::function drawCallbackDynamic; + Renderer::BufferMutableResource svsmDynamicDrawCountReadBackBuffer; // Per-view dynamic surviving counts for the perf editor + + // Finalize-written per-view fill dispatch args: rings with no dirty static pages / no + // resident dynamic pages this frame get zero-group fills. Same-frame GPU truth — a + // CPU/readback gate here is a frame late and flickers freshly acquired dynamic pages + Renderer::BufferResource svsmFillArgsBuffer; // Finalize-written dispatch args, consumed read-only via DispatchIndirect + + // Byte layout of svsmFillArgsBuffer (the ShadowRenderer's stride/offset constants), + // passed in by the pass owner so this shared base never depends on ShadowRenderer + u32 svsmFillArgsViewStride = 0; + u32 svsmFillArgsDynamicOffset = 0; + u32 svsmFillArgsStaticOverheadOffset = 0; + u32 svsmFillArgsDynamicOverheadOffset = 0; }; void GeometryPass(GeometryPassParams& params); + void RunInstancedGeometryFill(GeometryPassParams& params, u32 viewIndex, bool filtered, bool keepDynamic, const std::string& fillMarkerName, const std::string& createIndirectMarkerName); // Shared-buffer rebuild from a view's bitmask slice; markers prebuilt by the caller, this runs per view + void ClipmapCullingPass(CullingPassParams& params); // Instanced-only frustum cull of cascade views, no occlusion, no counter resets + + // The instanced cull dispatch shared by CullingPass and ClipmapCullingPass: one thread per + // instance against the per-view frustums, writing bitmask slices and instance counts. + // occlusionCull/cullMainView/debugDrawColliders come from params; the clipmap pass overrides + // them before calling + void RunInstancedCullingDispatch(CullingPassParams& params, bool useBitmasks, bool bindDepthPyramid); + + // The bitmask-driven instanced fill dispatch shared by OccluderPass (last frame's main-view + // bits) and RunInstancedGeometryFill (this frame's per-view slices, optionally gated through + // Finalize-written indirect args) + void DispatchInstancedFill(PassParams& params, const std::string& markerName, Renderer::DescriptorSetResource& fillSet, bool filtered, u32 currentBitmaskIndex, u32 bitmaskOffset, bool keepDynamic, u32 baseInstanceLookupOffset, u32 drawCallDataSize, Renderer::BufferResource indirectArgsBuffer, u32 indirectArgsByteOffset); + + // The CreateIndirectAfterCulling dispatch shared by the occluder, culling and geometry-fill + // paths: per-drawcall instance counts become indirect draw args (each count is cleared after + // consumption, keeping the counts buffer always-zero between uses). debugSet is optional. + // Both helpers take the full marker string prebuilt so per-view callers pay it once + void DispatchCreateIndirect(PassParams& params, const std::string& markerName, Renderer::DescriptorSetResource& createIndirectSet, Renderer::DescriptorSetResource* debugSet, u32 baseInstanceLookupOffset, u32 drawCallDataSize, Renderer::BufferResource indirectArgsBuffer, u32 indirectArgsByteOffset); void SyncToGPU(); void BindCullingResource(CullingResourcesBase& resources); @@ -329,6 +370,7 @@ class CulledRenderer static bool _pipelinesCreated; static Renderer::ComputePipelineID _fillInstancedDrawCallsFromBitmaskPipeline[2]; // [0] = non-indexed, [1] = indexed + static Renderer::ComputePipelineID _fillInstancedDrawCallsFilteredPipeline[2]; // Same, with the SVSM dynamic-mask filter static Renderer::ComputePipelineID _fillDrawCallsFromBitmaskPipeline[2]; // [0] = non-indexed, [1] = indexed static Renderer::ComputePipelineID _createIndirectAfterCullingPipeline[2]; // [0] = non-indexed, [1] = indexed static Renderer::ComputePipelineID _createIndirectAfterCullingOrderedPipeline[2]; // [0] = non-indexed, [1] = indexed diff --git a/Source/Game-Lib/Game-Lib/Rendering/CullingResources.cpp b/Source/Game-Lib/Game-Lib/Rendering/CullingResources.cpp index cbd4d622..759ce266 100644 --- a/Source/Game-Lib/Game-Lib/Rendering/CullingResources.cpp +++ b/Source/Game-Lib/Game-Lib/Rendering/CullingResources.cpp @@ -157,8 +157,8 @@ bool CullingResourcesBase::SyncToGPU(bool forceRecount) { _occluderFillDescriptorSet.Bind("_instanceRefTable"_h, _instanceRefs.GetBuffer()); _cullingDescriptorSet.Bind("_instanceRefTable"_h, _instanceRefs.GetBuffer()); + _geometryFillDescriptorSet.Bind("_instanceRefTable"_h, _instanceRefs.GetBuffer()); } - //_geometryFillDescriptorSet.Bind2("_instanceRefTable"_h, _instanceRefs.GetBuffer()); _geometryPassDescriptorSet.Bind("_instanceRefTable"_h, _instanceRefs.GetBuffer(), true); if (_materialPassDescriptorSet != nullptr) { @@ -180,8 +180,8 @@ bool CullingResourcesBase::SyncToGPU(bool forceRecount) { _occluderFillDescriptorSet.Bind("_culledInstanceLookupTable"_h, _culledInstanceLookupTableBuffer); _cullingDescriptorSet.Bind("_culledInstanceLookupTable"_h, _culledInstanceLookupTableBuffer); + _geometryFillDescriptorSet.Bind("_culledInstanceLookupTable"_h, _culledInstanceLookupTableBuffer); } - //_geometryFillDescriptorSet.Bind("_culledInstanceLookupTable"_h, _culledInstanceLookupTableBuffer); _geometryPassDescriptorSet.Bind("_culledInstanceLookupTable"_h, _culledInstanceLookupTableBuffer, true); if (_materialPassDescriptorSet != nullptr) { diff --git a/Source/Game-Lib/Game-Lib/Rendering/CullingResources.h b/Source/Game-Lib/Game-Lib/Rendering/CullingResources.h index 3199dbad..e58a8754 100644 --- a/Source/Game-Lib/Game-Lib/Rendering/CullingResources.h +++ b/Source/Game-Lib/Game-Lib/Rendering/CullingResources.h @@ -117,11 +117,11 @@ class CullingResourcesBase u32 _numInstances = 0; u32 _numSurvivingOccluderInstances = 0; - u32 _numSurvivingInstances[Renderer::Settings::MAX_VIEWS] = { 0 }; // One for the main view, then one per shadow cascade + u32 _numSurvivingInstances[Renderer::Settings::MAX_VIEWS] = { 0 }; // One for the main view, then one per shadow clipmap view u32 _numTriangles = 0; u32 _numSurvivingOccluderTriangles = 0; - u32 _numSurvivingTriangles[Renderer::Settings::MAX_VIEWS] = { 0 }; // One for the main view, then one per shadow cascade + u32 _numSurvivingTriangles[Renderer::Settings::MAX_VIEWS] = { 0 }; // One for the main view, then one per shadow clipmap view }; class CullingResourcesIndexedBase : public CullingResourcesBase diff --git a/Source/Game-Lib/Game-Lib/Rendering/Debug/JoltDebugRenderer.cpp b/Source/Game-Lib/Game-Lib/Rendering/Debug/JoltDebugRenderer.cpp index b9c1ddb5..77bd099a 100644 --- a/Source/Game-Lib/Game-Lib/Rendering/Debug/JoltDebugRenderer.cpp +++ b/Source/Game-Lib/Game-Lib/Rendering/Debug/JoltDebugRenderer.cpp @@ -111,7 +111,7 @@ void JoltDebugRenderer::AddOccluderPass(Renderer::RenderGraph* renderGraph, Rend //if (!CVAR_JoltDebugCullingEnabled.Get()) // return; - u32 numCascades = 0; // Debug meshes should never cast shadows + u32 numShadowViews = 0; // Debug meshes should never cast shadows struct Data { @@ -173,7 +173,7 @@ void JoltDebugRenderer::AddOccluderPass(Renderer::RenderGraph* renderGraph, Rend params.frameIndex = frameIndex; params.rt0 = data.visibilityBuffer; - params.depth[0] = data.depth; + params.depth = data.depth; params.culledDrawCallsBuffer = data.culledDrawCallsBuffer; params.culledDrawCallCountBuffer = data.culledDrawCallCountBuffer; @@ -246,7 +246,7 @@ void JoltDebugRenderer::AddOccluderPass(Renderer::RenderGraph* renderGraph, Rend params.frameIndex = frameIndex; params.rt0 = data.visibilityBuffer; - params.depth[0] = data.depth; + params.depth = data.depth; params.culledDrawCallsBuffer = data.culledDrawCallsBuffer; params.culledDrawCallCountBuffer = data.culledDrawCallCountBuffer; @@ -291,7 +291,7 @@ void JoltDebugRenderer::AddCullingPass(Renderer::RenderGraph* renderGraph, Rende //if (!CVAR_JoltDebugCullingEnabled.Get()) // return; - u32 numCascades = 0; // Debug meshes should never cast shadows + u32 numShadowViews = 0; // Debug meshes should never cast shadows struct Data { @@ -363,7 +363,7 @@ void JoltDebugRenderer::AddCullingPass(Renderer::RenderGraph* renderGraph, Rende params.cullingDescriptorSet = data.cullingSet; params.createIndirectAfterCullSet = data.createIndirectAfterCullSet; - params.numCascades = 0; + params.numShadowViews = 0; params.occlusionCull = CVAR_JoltDebugOcclusionCullingEnabled.Get(); params.cullingDataIsWorldspace = false; @@ -424,7 +424,7 @@ void JoltDebugRenderer::AddCullingPass(Renderer::RenderGraph* renderGraph, Rende params.globalDescriptorSet = data.globalSet; params.cullingDescriptorSet = data.cullingSet; - params.numCascades = 0; + params.numShadowViews = 0; params.occlusionCull = CVAR_JoltDebugOcclusionCullingEnabled.Get(); params.cullingDataIsWorldspace = false; @@ -523,7 +523,7 @@ void JoltDebugRenderer::AddGeometryPass(Renderer::RenderGraph* renderGraph, Rend params.enableDrawing = CVAR_JoltDebugDrawGeometry.Get(); params.cullingEnabled = cullingEnabled; - params.numCascades = 0; + params.numShadowViews = 0; GeometryPass(params); @@ -603,7 +603,7 @@ void JoltDebugRenderer::AddGeometryPass(Renderer::RenderGraph* renderGraph, Rend params.enableDrawing = CVAR_JoltDebugDrawGeometry.Get(); params.cullingEnabled = cullingEnabled; - params.numCascades = 0; + params.numShadowViews = 0; GeometryPass(params); diff --git a/Source/Game-Lib/Game-Lib/Rendering/GameRenderer.cpp b/Source/Game-Lib/Game-Lib/Rendering/GameRenderer.cpp index 1fbb3920..a6edb369 100644 --- a/Source/Game-Lib/Game-Lib/Rendering/GameRenderer.cpp +++ b/Source/Game-Lib/Game-Lib/Rendering/GameRenderer.cpp @@ -229,7 +229,7 @@ GameRenderer::GameRenderer() _canvasRenderer = new CanvasRenderer(_renderer, this, _debugRenderer); _uiRenderer = new UIRenderer(_renderer); _effectRenderer = new EffectRenderer(_renderer, this); - _shadowRenderer = new ShadowRenderer(_renderer, this, _debugRenderer, _terrainRenderer, _modelRenderer, _resources); + _shadowRenderer = new ShadowRenderer(_renderer, this, _terrainRenderer, _modelRenderer, _resources); _pixelQuery = new PixelQuery(_renderer, this); _nameHashToCursor.reserve(128); @@ -317,7 +317,10 @@ f32 GameRenderer::Render() if (_resources.cameras.SyncToGPU(_renderer)) { + // Should never fire: the vector is prefilled to MAX_VIEWS at init. Kept as a safety net, + // note the rebinds only reach the shaders a frame-cycle later _resources.globalDescriptorSet.Bind("_cameras", _resources.cameras.GetBuffer()); + _shadowRenderer->BindCameraBuffers(_resources); } // Create rendergraph @@ -390,8 +393,6 @@ f32 GameRenderer::Render() _skyboxRenderer->AddSkyboxPass(&renderGraph, _resources, _frameIndex); _modelRenderer->AddSkyboxPass(&renderGraph, _resources, _frameIndex); - _shadowRenderer->AddShadowPass(&renderGraph, _resources, _frameIndex); - // Occluder passes _terrainRenderer->AddOccluderPass(&renderGraph, _resources, _frameIndex); _modelRenderer->AddOccluderPass(&renderGraph, _resources, _frameIndex); @@ -458,6 +459,21 @@ f32 GameRenderer::Render() _liquidRenderer->AddCullingPass(&renderGraph, _resources, _frameIndex); _liquidRenderer->AddGeometryPass(&renderGraph, _resources, _frameIndex); + // SVSM block, runs after the main depth is complete so page marking can analyze the visible + // samples, then allocates, culls per clipmap view and renders the dirty pages the same frame. + // The night gate skips the whole producer side while the sun is below the horizon (the + // material pass already samples nothing at strength 0); the bind pass stays, bindings must + // remain valid + _shadowRenderer->AddSVSMUpdatePass(&renderGraph, _resources, _frameIndex); + _shadowRenderer->AddSVSMBindPass(&renderGraph, _resources, _frameIndex); + if (_shadowRenderer->IsSVSMActive()) + { + _terrainRenderer->AddClipmapCullingPass(&renderGraph, _resources, _frameIndex); + _modelRenderer->AddClipmapCullingPass(&renderGraph, _resources, _frameIndex); + _terrainRenderer->AddSVSMGeometryPass(&renderGraph, _resources, _frameIndex, _shadowRenderer); + _modelRenderer->AddSVSMGeometryPass(&renderGraph, _resources, _frameIndex, _shadowRenderer); + } + _lightRenderer->AddClassificationPass(&renderGraph, _resources, _frameIndex); _materialRenderer->AddPreEffectsPass(&renderGraph, _resources, _frameIndex); @@ -474,6 +490,7 @@ f32 GameRenderer::Render() _debugRenderer->Add2DPass(&renderGraph, _resources, _frameIndex); _lightRenderer->AddDebugPass(&renderGraph, _resources, _frameIndex); + _shadowRenderer->AddSVSMDebugOverlayPass(&renderGraph, _resources, _frameIndex); Renderer::ImageID finalTarget = isEditorMode ? _resources.finalColor : _resources.sceneColor; _uiRenderer->AddImguiPass(&renderGraph, _resources, _frameIndex, finalTarget); @@ -702,6 +719,13 @@ void GameRenderer::CreatePermanentResources() _resources.cameras.SetDebugName("Cameras"); _resources.cameras.SetUsage(Renderer::BufferUsage::STORAGE_BUFFER); _resources.cameras.Add(Camera()); + + // Prefill to the runtime cap and create the buffer now: descriptor binds of it happen at init + // (mid-frame buffer binds only reach the canonical descriptor copies at a later FlipFrame), + // so the buffer must exist before the renderers bind it and must never resize + _resources.cameras.AddCount(Renderer::Settings::MAX_VIEWS - 1); + _resources.cameras.SyncToGPU(_renderer); + _resources.globalDescriptorSet.Bind("_cameras", _resources.cameras.GetBuffer()); } void GameRenderer::CreateRenderTargets() diff --git a/Source/Game-Lib/Game-Lib/Rendering/GameRenderer.h b/Source/Game-Lib/Game-Lib/Rendering/GameRenderer.h index 3df7cf70..3bc4b12e 100644 --- a/Source/Game-Lib/Game-Lib/Rendering/GameRenderer.h +++ b/Source/Game-Lib/Game-Lib/Rendering/GameRenderer.h @@ -78,6 +78,7 @@ class GameRenderer LiquidRenderer* GetLiquidRenderer() { return _liquidRenderer; } MaterialRenderer* GetMaterialRenderer() { return _materialRenderer; } ModelRenderer* GetModelRenderer() { return _modelRenderer; } + ShadowRenderer* GetShadowRenderer() { return _shadowRenderer; } TerrainRenderer* GetTerrainRenderer() { return _terrainRenderer; } TextureRenderer* GetTextureRenderer() { return _textureRenderer; } SkyboxRenderer* GetSkyboxRenderer() { return _skyboxRenderer; } diff --git a/Source/Game-Lib/Game-Lib/Rendering/Liquid/LiquidRenderer.cpp b/Source/Game-Lib/Game-Lib/Rendering/Liquid/LiquidRenderer.cpp index 241593bd..54f6f597 100644 --- a/Source/Game-Lib/Game-Lib/Rendering/Liquid/LiquidRenderer.cpp +++ b/Source/Game-Lib/Game-Lib/Rendering/Liquid/LiquidRenderer.cpp @@ -272,7 +272,7 @@ void LiquidRenderer::AddCullingPass(Renderer::RenderGraph* renderGraph, RenderRe if (_cullingResources.GetDrawCalls().Count() == 0) return; - u32 numCascades = 0;// *CVarSystem::Get()->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "numShadowCascades"_h); + u32 numShadowViews = 0; struct Data { @@ -344,7 +344,7 @@ void LiquidRenderer::AddCullingPass(Renderer::RenderGraph* renderGraph, RenderRe params.cullingDescriptorSet = data.cullingSet; params.createIndirectAfterCullSet = data.cullingSet; - params.numCascades = 0;// *CVarSystem::Get()->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "numShadowCascades"_h); + params.numShadowViews = 0; params.occlusionCull = CVAR_LiquidOcclusionCullingEnabled.Get(); params.disableTwoStepCulling = true; // Transparent objects don't write depth, so we don't need to two step cull them @@ -500,7 +500,7 @@ void LiquidRenderer::AddGeometryPass(Renderer::RenderGraph* renderGraph, RenderR params.enableDrawing = true;//CVAR_ModelDrawGeometry.Get(); params.cullingEnabled = cullingEnabled; - params.numCascades = 0;// *CVarSystem::Get()->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "numShadowCascades"_h); + params.numShadowViews = 0; GeometryPass(params); }); diff --git a/Source/Game-Lib/Game-Lib/Rendering/Material/MaterialRenderer.cpp b/Source/Game-Lib/Game-Lib/Rendering/Material/MaterialRenderer.cpp index 75d4cfeb..40878363 100644 --- a/Source/Game-Lib/Game-Lib/Rendering/Material/MaterialRenderer.cpp +++ b/Source/Game-Lib/Game-Lib/Rendering/Material/MaterialRenderer.cpp @@ -8,6 +8,7 @@ #include "Game-Lib/Rendering/Light/LightRenderer.h" #include "Game-Lib/Rendering/Model/ModelRenderer.h" #include "Game-Lib/Rendering/RenderResources.h" +#include "Game-Lib/Rendering/Shadow/ShadowRenderer.h" #include "Game-Lib/Rendering/Terrain/TerrainRenderer.h" #include "Game-Lib/Util/PhysicsUtil.h" #include "Game-Lib/Util/ServiceLocator.h" @@ -19,7 +20,7 @@ #include -AutoCVar_Int CVAR_VisibilityBufferDebugID(CVarCategory::Client | CVarCategory::Rendering, "visibilityBufferDebugID", "Debug visualizers: 0 - Off, 1 - TypeID, 2 - ObjectID, 3 - TriangleID, 4 - ShadowCascade", 0); +AutoCVar_Int CVAR_VisibilityBufferDebugID(CVarCategory::Client | CVarCategory::Rendering, "visibilityBufferDebugID", "Debug visualizers: 0 - Off, 1 - TypeID, 2 - ObjectID, 3 - TriangleID, 4 - SVSM Compare Margin", 0); AutoCVar_ShowFlag CVAR_DrawTerrainWireframe(CVarCategory::Client | CVarCategory::Rendering, "drawTerrainWireframe", "Draw terrain wireframe", ShowFlag::DISABLED); AutoCVar_ShowFlag CVAR_EnableFog(CVarCategory::Client | CVarCategory::Rendering, "enableFog", "Toggle fog", ShowFlag::DISABLED); AutoCVar_VecFloat CVAR_FogColor(CVarCategory::Client | CVarCategory::Rendering, "fogColor", "Change fog color", vec4(0.33f, 0.2f, 0.38f, 1.0f), CVarFlags::None); @@ -129,6 +130,9 @@ void MaterialRenderer::AddMaterialPass(Renderer::RenderGraph* renderGraph, Rende Renderer::ImageResource ambientOcclusion; + Renderer::ImageResource svsmPagePool; + Renderer::ImageResource svsmDynamicPagePool; + Renderer::DescriptorSetResource debugSet; Renderer::DescriptorSetResource globalSet; Renderer::DescriptorSetResource lightSet; @@ -153,6 +157,16 @@ void MaterialRenderer::AddMaterialPass(Renderer::RenderGraph* renderGraph, Rende builder.Read(resources.cameras.GetBuffer(), Renderer::BufferPassUsage::COMPUTE); builder.Read(_directionalLights.GetBuffer(), Renderer::BufferPassUsage::COMPUTE); + // SVSM sampling reads the page pools the geometry passes wrote this frame, the graph + // needs the dependencies declared for the barriers. The pool bindings must always be + // valid, a placeholder stands in until the real pools are lazily created + ShadowRenderer* shadowRenderer = _gameRenderer->GetShadowRenderer(); + data.svsmPagePool = builder.Read(shadowRenderer->GetSVSMPagePoolOrPlaceholder(), Renderer::PipelineType::COMPUTE); + data.svsmDynamicPagePool = builder.Read(shadowRenderer->GetSVSMDynamicPagePoolOrPlaceholder(), Renderer::PipelineType::COMPUTE); + builder.Read(shadowRenderer->GetSVSMDataBuffer(), Renderer::BufferPassUsage::COMPUTE); + builder.Read(shadowRenderer->GetSVSMPageTableBuffer(), Renderer::BufferPassUsage::COMPUTE); + builder.Read(shadowRenderer->GetSVSMDynamicPageTableBuffer(), Renderer::BufferPassUsage::COMPUTE); + data.debugSet = builder.Use(resources.debugDescriptorSet); data.globalSet = builder.Use(resources.globalDescriptorSet); data.lightSet = builder.Use(resources.lightDescriptorSet); @@ -181,20 +195,44 @@ void MaterialRenderer::AddMaterialPass(Renderer::RenderGraph* renderGraph, Rende data.materialSet.Bind("_ambientOcclusion", data.ambientOcclusion); - // Bind descriptorset - commandList.BindDescriptorSet(data.debugSet, frameIndex); - commandList.BindDescriptorSet(data.globalSet, frameIndex); - commandList.BindDescriptorSet(data.lightSet, frameIndex); - commandList.BindDescriptorSet(data.terrainSet, frameIndex); - commandList.BindDescriptorSet(data.modelSet, frameIndex); - commandList.BindDescriptorSet(data.materialSet, frameIndex); + data.lightSet.Bind("_svsmPagePool"_h, data.svsmPagePool); + data.lightSet.Bind("_svsmDynamicPagePool"_h, data.svsmDynamicPagePool); + + // The debug permutations dead-strip descriptor set usage, so each variant binds + // exactly what it statically references or the set-usage validation fires + switch (visibilityBufferDebugID) + { + case 1: // TypeID, pure math on the visibility buffer + case 3: // TriangleID + commandList.BindDescriptorSet(data.materialSet, frameIndex); + break; + case 2: // ObjectID reads terrain instance data + commandList.BindDescriptorSet(data.terrainSet, frameIndex); + commandList.BindDescriptorSet(data.materialSet, frameIndex); + break; + case 4: // SVSM compare margin reconstructs vertex data and samples the virtual shadow map + commandList.BindDescriptorSet(data.globalSet, frameIndex); + commandList.BindDescriptorSet(data.lightSet, frameIndex); + commandList.BindDescriptorSet(data.terrainSet, frameIndex); + commandList.BindDescriptorSet(data.modelSet, frameIndex); + commandList.BindDescriptorSet(data.materialSet, frameIndex); + break; + default: // Full shading + commandList.BindDescriptorSet(data.debugSet, frameIndex); + commandList.BindDescriptorSet(data.globalSet, frameIndex); + commandList.BindDescriptorSet(data.lightSet, frameIndex); + commandList.BindDescriptorSet(data.terrainSet, frameIndex); + commandList.BindDescriptorSet(data.modelSet, frameIndex); + commandList.BindDescriptorSet(data.materialSet, frameIndex); + break; + } //if (CVAR_DrawTerrainWireframe.Get() == ShowFlag::ENABLED) { struct Constants { - vec4 renderInfo; // x = Render Width, y = Render Height, z = 1/Width, w = 1/Height - uvec4 lightInfo; // x = Directional Light Count, Y = Point Light Count, Z = Cascade Count, W = Shadows Enabled + vec4 renderInfo; // x = Render Width, y = Render Height, z = 1/Width, w = 1/Height + uvec4 lightInfo; // x = Directional Light Count, y = Shadows Ready (enabled, strength > 0, pool created), zw = UNUSED uvec4 tileInfo; // xy = Num Tiles, zw = UNUSED vec4 fogColor; vec4 fogSettings; // x = Enabled, y = Begin Fog Blend Dist, z = End Fog Blend Dist, w = UNUSED @@ -205,6 +243,7 @@ void MaterialRenderer::AddMaterialPass(Renderer::RenderGraph* renderGraph, Rende Color patchEdgeColor; Color vertexColor; Color brushColor; + vec4 shadowSettings; // x = Shadow Strength, y = Normal Offset Bias, z = SVSM Constant Bias (world meters), w = UNUSED }; Constants* constants = graphResources.FrameNew(); @@ -213,9 +252,14 @@ void MaterialRenderer::AddMaterialPass(Renderer::RenderGraph* renderGraph, Rende constants->renderInfo = vec4(outputSize, 1.0f / outputSize); CVarSystem* cvarSystem = CVarSystem::Get(); - const u32 numCascades = static_cast(*cvarSystem->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "shadowCascadeNum")); - const u32 shadowEnabled = static_cast(*cvarSystem->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "shadowEnabled")); - constants->lightInfo = uvec4(static_cast(_directionalLights.Count()), 0, numCascades, shadowEnabled); + const f32 shadowStrength = static_cast(*cvarSystem->GetFloatCVar(CVarCategory::Client | CVarCategory::Rendering, "shadowStrength")); + // Shadows sample only once the (lazily created) page pool exists, the placeholder + // pool bound before then must never be read + ShadowRenderer* shadowRenderer = _gameRenderer->GetShadowRenderer(); + const bool shadowsReady = static_cast(*cvarSystem->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "shadowEnabled")) != 0 + && shadowStrength > 0.0f + && shadowRenderer != nullptr && shadowRenderer->GetSVSMPagePool() != Renderer::ImageID::Invalid(); + constants->lightInfo = uvec4(static_cast(_directionalLights.Count()), shadowsReady ? 1 : 0, 0, 0); constants->tileInfo = uvec4(_lightRenderer->CalculateNumTiles2D(outputSize), 0, 0); @@ -237,6 +281,10 @@ void MaterialRenderer::AddMaterialPass(Renderer::RenderGraph* renderGraph, Rende constants->vertexColor = terrainTools->GetVertexColor(); constants->brushColor = terrainTools->GetBrushColor(); + f32 shadowNormalOffsetBias = static_cast(*cvarSystem->GetFloatCVar(CVarCategory::Client | CVarCategory::Rendering, "shadowNormalOffsetBias")); + f32 svsmConstantBias = static_cast(*cvarSystem->GetFloatCVar(CVarCategory::Client | CVarCategory::Rendering, "svsmConstantBias")); + constants->shadowSettings = vec4(shadowStrength, shadowNormalOffsetBias, svsmConstantBias, 0.0f); + commandList.PushConstant(constants, 0, sizeof(Constants)); } @@ -306,11 +354,6 @@ void MaterialRenderer::CreatePermanentResources() CreateMaterialPipeline(); }); - CVarSystem::Get()->AddOnIntValueChanged(CVarCategory::Client | CVarCategory::Rendering, "shadowFilterMode"_h, [this](const i32& val) - { - CreateMaterialPipeline(); - }); - // Register pipelines with descriptor sets and init _preEffectsPassDescriptorSet.RegisterPipeline(_renderer, _preEffectsPipeline); _preEffectsPassDescriptorSet.Init(_renderer); @@ -322,7 +365,7 @@ void MaterialRenderer::CreatePermanentResources() samplerDesc.enabled = true; samplerDesc.filter = Renderer::SamplerFilter::ANISOTROPIC; samplerDesc.maxAnisotropy = 16; - samplerDesc.mipLODBias = -0.5f; + //samplerDesc.mipLODBias = -0.5f; samplerDesc.addressU = Renderer::TextureAddressMode::WRAP; samplerDesc.addressV = Renderer::TextureAddressMode::WRAP; samplerDesc.addressW = Renderer::TextureAddressMode::CLAMP; @@ -353,12 +396,9 @@ void MaterialRenderer::CreatePermanentResources() void MaterialRenderer::CreateMaterialPipeline() { - const i32 shadowFilterMode = *CVarSystem::Get()->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "shadowFilterMode"_h); - - std::vector permutationFields = + std::vector permutationFields = { { "DEBUG_ID", std::to_string(CVAR_VisibilityBufferDebugID.Get()) }, - { "SHADOW_FILTER_MODE", std::to_string(shadowFilterMode) }, { "EDITOR_MODE", CVAR_DrawTerrainWireframe.Get() == ShowFlag::ENABLED ? "1" : "0" } }; u32 shaderEntryNameHash = Renderer::GetShaderEntryNameHash("Material/MaterialPass.cs", permutationFields); diff --git a/Source/Game-Lib/Game-Lib/Rendering/Model/ModelRenderer.cpp b/Source/Game-Lib/Game-Lib/Rendering/Model/ModelRenderer.cpp index 1209ae31..68a0d152 100644 --- a/Source/Game-Lib/Game-Lib/Rendering/Model/ModelRenderer.cpp +++ b/Source/Game-Lib/Game-Lib/Rendering/Model/ModelRenderer.cpp @@ -3,6 +3,7 @@ #include "Game-Lib/Application/EnttRegistries.h" #include "Game-Lib/ECS/Components/DisplayInfo.h" #include "Game-Lib/ECS/Components/Model.h" +#include "Game-Lib/ECS/Singletons/ActiveCamera.h" #include "Game-Lib/ECS/Components/Tags.h" #include "Game-Lib/ECS/Components/UnitCustomization.h" #include "Game-Lib/ECS/Singletons/Database/ClientDBSingleton.h" @@ -15,6 +16,7 @@ #include "Game-Lib/Rendering/CullUtils.h" #include "Game-Lib/Rendering/RenderUtils.h" #include "Game-Lib/Rendering/GameRenderer.h" +#include "Game-Lib/Rendering/Shadow/ShadowRenderer.h" #include "Game-Lib/Rendering/RenderResources.h" #include "Game-Lib/Rendering/Debug/DebugRenderer.h" #include "Game-Lib/Rendering/Model/ModelLoader.h" @@ -63,10 +65,15 @@ ModelRenderer::ModelRenderer(Renderer::Renderer* renderer, GameRenderer* gameRen , _renderer(renderer) , _gameRenderer(gameRenderer) , _debugRenderer(debugRenderer) + , _svsmDrawDescriptorSet(Renderer::DescriptorSetSlot::PER_PASS) + , _svsmDynamicDrawDescriptorSet(Renderer::DescriptorSetSlot::PER_PASS) , _ownerThreadID(std::this_thread::get_id()) { ZoneScoped; + _dynamicInstanceMask.SetDebugName("ModelDynamicInstanceMask"); + _dynamicInstanceMask.SetUsage(Renderer::BufferUsage::STORAGE_BUFFER); + CreatePermanentResources(); if (CVAR_ModelValidateTransfers.Get()) @@ -119,11 +126,244 @@ void ModelRenderer::Update(f32 deltaTime) mat4x4& matrix = _instanceMatrices[instanceID]; + // Transitioning into the dynamic class: the shadow baked at the PREVIOUS position must + // come out of the static pages too, a single-frame move can be arbitrarily large. + // Already-classified instances only refresh their cached shadow AABB, the dynamic + // pool is transient per frame + auto casterIt = _dynamicCasterStates.find(instanceID); + if (casterIt == _dynamicCasterStates.end()) + { + QueueShadowInvalidation(instanceID, matrix); // Pre-move matrix + } + matrix = transform.GetMatrix(); _instanceMatrices.SetDirtyElement(instanceID); + + if (casterIt != _dynamicCasterStates.end()) + { + ComputeInstanceShadowAABB(instanceID, matrix, casterIt->second.aabbMin, casterIt->second.aabbMax); + } + + // Moved this frame -> dynamic shadow caster this frame + _dynamicInstanceQueue.enqueue(instanceID); }); } + CVarSystem* cvarSystem = CVarSystem::Get(); + const bool shadowsEnabled = *cvarSystem->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "shadowEnabled"_h) == 1; + const bool split = *cvarSystem->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "svsmDynamicSplit"_h) == 1; + + // Dynamic shadow casters: one classifier for everything that moved or pushed bone matrices, + // at instance granularity. Producers enqueue signals from any thread (the transform view + // above, the instanced bone-push path, and the in-range placements of uninstanced animated + // models below); draining them here stamps a per-instance last-signal time, and an instance + // stays classified for a grace period after its last signal. Entering pulls the baked pose + // out of the static pages, expiring bakes the final pose back — so a death pose persists and + // an idle pose never ghosts under a mover. Signals raised after this block runs are seen next + // Update, one frame of latency the grace swallows + { + ZoneScopedN("Dynamic Shadow Casters"); + + const f32 range = shadowsEnabled ? static_cast(glm::max(*cvarSystem->GetFloatCVar(CVarCategory::Client | CVarCategory::Rendering, "svsmAnimatedCasterRange"_h), 0.0)) : 0.0f; + + _dynamicCasterTime += deltaTime; + _dynamicCasterLiveIDs.clear(); + _dynamicCasterAABBs.clear(); + _dynamicCastersDropped = 0; + // Transition counters accumulate (reset on Clear) — per-frame events are unreadable in + // the perf editor and made its layout jump + + // Brief animation pauses don't churn the static cache + constexpr f32 dynamicGraceSeconds = 0.5f; + + // A first-time stamp is a transition IN: the instance's baked pose must come out of the + // static pages (movers additionally invalidated their pre-move footprint above). The + // shadow AABB caches in the entry — it only changes with the transform, which the + // dirty-transform view refreshes, so re-stamps just bump the time + auto Stamp = [&](u32 instanceID) + { + auto [it, inserted] = _dynamicCasterStates.try_emplace(instanceID); + DynamicCasterState& state = it->second; + state.lastSignal = _dynamicCasterTime; + if (inserted) + { + _dynamicCasterTransitionsIn++; + ComputeInstanceShadowAABB(instanceID, _instanceMatrices[instanceID], state.aabbMin, state.aabbMax); + QueueShadowInvalidation(state.aabbMin, state.aabbMax); + } + }; + + u32 instanceID; + while (_dynamicInstanceQueue.try_dequeue(instanceID)) + { + Stamp(instanceID); + } + + // Uninstanced animated models (windmills, flags: static position, moving geometry) push + // bones per MODEL; the range scan promotes their in-camera-range placements to + // per-instance signals. Beyond the range the pose stays baked, where coarse rings can't + // resolve the motion anyway + u32 modelID; + while (_uninstancedAnimatedModelQueue.try_dequeue(modelID)) + { + _animatedModelLastPushTime[modelID].lastPushTime = _dynamicCasterTime; + } + + vec3 cameraPos = vec3(0.0f); + bool hasCamera = false; + if (auto* activeCamera = gameRegistry->ctx().find()) + { + if (activeCamera->entity != entt::null && gameRegistry->all_of(activeCamera->entity)) + { + cameraPos = gameRegistry->get(activeCamera->entity).GetWorldPosition(); + hasCamera = true; + } + } + + if (range > 0.0f && hasCamera) + { + const f32 enterDistSq = range * range; + const f32 leaveDist = range * 1.15f; // Hysteresis against camera jitter at the boundary + const f32 leaveDistSq = leaveDist * leaveDist; + + // Rescan slack: a placement beyond leaveDist at scan time cannot reach the enter + // radius until the camera has moved at least the hysteresis margin + const f32 rescanDist = leaveDist - range; + const f32 rescanDistSq = rescanDist * rescanDist; + + for (auto it = _animatedModelLastPushTime.begin(); it != _animatedModelLastPushTime.end();) + { + AnimatedModelState& state = it->second; + if (_dynamicCasterTime - state.lastPushTime > dynamicGraceSeconds) + { + it = _animatedModelLastPushTime.erase(it); + continue; + } + + u32 animatedModelID = it->first; + + // The full placement scan only runs when the cached near-camera subset can be + // stale: first scan, camera moved past the hysteresis slack, or the instance-set + // epoch advanced (any model's add/remove — spare rescans are cheap and rare). + // The manifest mutex is only taken for the scan itself; the steady-state tick + // walks the cached subset lock-free + const u32 instanceSetEpoch = _instanceSetEpoch.load(std::memory_order_acquire); + vec3 toScanCamera = state.scanCameraPos - cameraPos; + if (!state.scanned || glm::dot(toScanCamera, toScanCamera) > rescanDistSq || state.lastSeenInstanceEpoch != instanceSetEpoch) + { + state.scanned = true; + state.scanCameraPos = cameraPos; + state.lastSeenInstanceEpoch = instanceSetEpoch; + state.nearInstances.clear(); + + std::scoped_lock lock(*_modelManifestsInstancesMutexes[animatedModelID]); + const ModelManifest& manifest = _modelManifests[animatedModelID]; + for (u32 placementID : manifest.instances) + { + vec3 toCamera = vec3(_instanceMatrices[placementID][3]) - cameraPos; + if (glm::dot(toCamera, toCamera) < leaveDistSq) + { + state.nearInstances.push_back(placementID); + } + } + } + + for (u32 placementID : state.nearInstances) + { + vec3 toCamera = vec3(_instanceMatrices[placementID][3]) - cameraPos; + f32 distSq = glm::dot(toCamera, toCamera); + + // One lookup serves both the hysteresis limit and the re-stamp + auto casterIt = _dynamicCasterStates.find(placementID); + f32 limitSq = casterIt != _dynamicCasterStates.end() ? leaveDistSq : enterDistSq; + if (distSq < limitSq) + { + if (casterIt != _dynamicCasterStates.end()) + { + casterIt->second.lastSignal = _dynamicCasterTime; + } + else + { + Stamp(placementID); + } + } + } + ++it; + } + } + + // Oversized casters would exceed the dynamic marker's 1024-page cutoff in EVERY clipmap + // ring (span quarters per coarser ring) and must never rely on the dynamic pool: extent + // beyond 32 pages of the coarsest ring routes to the static path instead + const u32 numClipmaps = static_cast(glm::clamp(*cvarSystem->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "svsmNumClipmaps"_h), 1, 8)); + const f32 clipmap0Extent = static_cast(*cvarSystem->GetFloatCVar(CVarCategory::Client | CVarCategory::Rendering, "svsmClipmap0Extent"_h)); + const f32 oversizeLimit = clipmap0Extent * static_cast(1u << (numClipmaps - 1)) * 0.5f; + + // Tick the classifier: expired entries transition OUT (their final pose bakes back into + // the static pages — a despawn-recycled ID costs one spurious page refresh), live entries + // emit this frame's mask feed and dynamic AABB list. Overflow and oversize SPILL to the + // static path instead of dropping: a spilled caster re-bakes every frame (page churn) but + // is never excluded from both pools + for (auto it = _dynamicCasterStates.begin(); it != _dynamicCasterStates.end();) + { + u32 liveInstanceID = it->first; + const DynamicCasterState& state = it->second; + + if (_dynamicCasterTime - state.lastSignal > dynamicGraceSeconds) + { + _dynamicCasterTransitionsOut++; + QueueShadowInvalidation(state.aabbMin, state.aabbMax); + it = _dynamicCasterStates.erase(it); + continue; + } + + if (split) + { + vec3 extent = state.aabbMax - state.aabbMin; + const bool oversized = glm::max(extent.x, glm::max(extent.y, extent.z)) > oversizeLimit; + const bool overCap = _dynamicCasterAABBs.size() >= ShadowRenderer::SVSM_MAX_DYNAMIC_AABBS * 2; + if (oversized || overCap) + { + _dynamicCastersDropped++; + QueueShadowInvalidation(state.aabbMin, state.aabbMax); + } + else + { + _dynamicCasterLiveIDs.push_back(liveInstanceID); + _dynamicCasterAABBs.push_back(vec4(state.aabbMin, 0.0f)); + _dynamicCasterAABBs.push_back(vec4(state.aabbMax, 0.0f)); + } + } + else + { + // Split off: v1-style per-frame static invalidation + QueueShadowInvalidation(state.aabbMin, state.aabbMax); + } + + ++it; + } + } + + // Diagnostics only, nothing to read back while the dynamic draws don't run + if (shadowsEnabled && split) + { + ZoneScopedN("SVSM Dynamic Draw Count ReadBack"); + + u32* counts = static_cast(_renderer->MapBuffer(_svsmDynamicDrawCountReadBackBuffer)); + if (counts != nullptr) + { + memcpy(_numSvsmDynamicInstances, counts, sizeof(u32) * Renderer::Settings::MAX_VIEWS); + } + _renderer->UnmapBuffer(_svsmDynamicDrawCountReadBackBuffer); + + _numSvsmDynamicInstancesZeroed = false; + } + else if (!_numSvsmDynamicInstancesZeroed) + { + memset(_numSvsmDynamicInstances, 0, sizeof(_numSvsmDynamicInstances)); + _numSvsmDynamicInstancesZeroed = true; + } + { ZoneScopedN("Update Culling Resources"); @@ -448,6 +688,34 @@ void ModelRenderer::Clear() ChangeSkyboxRequest changeSkyboxRequest; while (_changeSkyboxRequests.try_dequeue(changeSkyboxRequest)) {} + // Queued modelIDs/instanceIDs are invalid after a clear + u32 drainedID; + while (_uninstancedAnimatedModelQueue.try_dequeue(drainedID)) {} + while (_dynamicInstanceQueue.try_dequeue(drainedID)) {} + ShadowInvalidation drainedInvalidation; + while (_shadowInvalidationQueue.try_dequeue(drainedInvalidation)) {} + _animatedModelLastPushTime.clear(); + _dynamicCasterStates.clear(); + _dynamicCasterLiveIDs.clear(); + _dynamicCasterAABBs.clear(); + _dynamicCasterTransitionsIn = 0; + _dynamicCasterTransitionsOut = 0; + + // Zero the mask bits set last frame BEFORE the instance capacity shrinks under the word + // count — recycled IDs on the next map must not start masked out of the static pages. Sized + // against the mask's own count, the trailing SyncToGPU shrinks nothing + u32 maskWordCount = static_cast(_dynamicInstanceMask.Count()); + for (u32 instanceID : _dynamicInstanceIDs) + { + u32 word = instanceID / 32; + if (word >= maskWordCount) + continue; + + _dynamicInstanceMask[word] &= ~(1u << (instanceID & 31)); + _dynamicInstanceMask.SetDirtyElement(word); + } + _dynamicInstanceIDs.clear(); + _renderer->UnloadTexturesInArray(_textures, 1); _instancesDirty = true; @@ -470,16 +738,10 @@ void ModelRenderer::AddOccluderPass(Renderer::RenderGraph* renderGraph, RenderRe CVarSystem* cvarSystem = CVarSystem::Get(); - u32 numCascades = 0; - if (CVAR_ModelsCastShadow.Get() == 1) - { - numCascades = *cvarSystem->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "shadowCascadeNum"); - } - struct Data { Renderer::ImageMutableResource visibilityBuffer; - Renderer::DepthImageMutableResource depth[Renderer::Settings::MAX_VIEWS]; + Renderer::DepthImageMutableResource depth; Renderer::BufferMutableResource culledDrawCallsBuffer; Renderer::BufferMutableResource culledDrawCallCountBuffer; @@ -502,16 +764,12 @@ void ModelRenderer::AddOccluderPass(Renderer::RenderGraph* renderGraph, RenderRe }; renderGraph->AddPass("Model (O) Occluders", - [this, &resources, frameIndex, numCascades](Data& data, Renderer::RenderGraphBuilder& builder) + [this, &resources, frameIndex](Data& data, Renderer::RenderGraphBuilder& builder) { using BufferUsage = Renderer::BufferPassUsage; data.visibilityBuffer = builder.Write(resources.visibilityBuffer, Renderer::PipelineType::GRAPHICS, Renderer::LoadMode::LOAD); - data.depth[0] = builder.Write(resources.depth, Renderer::PipelineType::GRAPHICS, Renderer::LoadMode::LOAD); - for (u32 i = 1; i < numCascades + 1; i++) - { - data.depth[i] = builder.Write(resources.shadowDepthCascades[i - 1], Renderer::PipelineType::GRAPHICS, Renderer::LoadMode::LOAD); - } + data.depth = builder.Write(resources.depth, Renderer::PipelineType::GRAPHICS, Renderer::LoadMode::LOAD); builder.Read(resources.cameras.GetBuffer(), BufferUsage::GRAPHICS | BufferUsage::COMPUTE); builder.Read(_vertices.GetBuffer(), BufferUsage::GRAPHICS); @@ -536,7 +794,7 @@ void ModelRenderer::AddOccluderPass(Renderer::RenderGraph* renderGraph, RenderRe return true; // Return true from setup to enable this pass, return false to disable it }, - [this, &resources, frameIndex, numCascades, cvarSystem](Data& data, Renderer::RenderGraphResources& graphResources, Renderer::CommandList& commandList) // Execute + [this, &resources, frameIndex, cvarSystem](Data& data, Renderer::RenderGraphResources& graphResources, Renderer::CommandList& commandList) // Execute { GPU_SCOPED_PROFILER_ZONE(commandList, ModelOccluders); @@ -548,10 +806,7 @@ void ModelRenderer::AddOccluderPass(Renderer::RenderGraph* renderGraph, RenderRe params.frameIndex = frameIndex; params.rt0 = data.visibilityBuffer; - for (u32 i = 0; i < numCascades + 1; i++) - { - params.depth[i] = data.depth[i]; - } + params.depth = data.depth; params.culledDrawCallsBuffer = data.culledDrawCallsBuffer; params.culledDrawCallCountBuffer = data.culledDrawCallCountBuffer; @@ -579,12 +834,6 @@ void ModelRenderer::AddOccluderPass(Renderer::RenderGraph* renderGraph, RenderRe params.baseInstanceLookupOffset = offsetof(DrawCallData, DrawCallData::baseInstanceLookupOffset); params.drawCallDataSize = sizeof(DrawCallData); - - params.numCascades = numCascades; - - params.biasConstantFactor = static_cast(*cvarSystem->GetFloatCVar(CVarCategory::Client | CVarCategory::Rendering, "shadowDepthBiasConstant")); - params.biasClamp = static_cast(*cvarSystem->GetFloatCVar(CVarCategory::Client | CVarCategory::Rendering, "shadowDepthBiasClamp")); - params.biasSlopeFactor = static_cast(*cvarSystem->GetFloatCVar(CVarCategory::Client | CVarCategory::Rendering, "shadowDepthBiasSlope")); params.enableDrawing = CVAR_ModelDrawOccluders.Get(); params.disableTwoStepCulling = CVAR_ModelDisableTwoStepCulling.Get(); @@ -672,7 +921,8 @@ void ModelRenderer::AddCullingPass(Renderer::RenderGraph* renderGraph, RenderRes params.cullingDescriptorSet = data.cullingSet; params.createIndirectAfterCullSet = data.createIndirectAfterCullSet; - params.numCascades = *CVarSystem::Get()->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "shadowCascadeNum"_h); + params.numShadowViews = 0; // Main view only, cascades are culled in their own block late in the frame + params.cullMainView = true; params.occlusionCull = CVAR_ModelOcclusionCullingEnabled.Get(); params.disableTwoStepCulling = CVAR_ModelDisableTwoStepCulling.Get(); @@ -700,11 +950,7 @@ void ModelRenderer::AddGeometryPass(Renderer::RenderGraph* renderGraph, RenderRe CVarSystem* cvarSystem = CVarSystem::Get(); const bool cullingEnabled = CVAR_ModelCullingEnabled.Get(); - u32 numCascades = 0; - if (CVAR_ModelsCastShadow.Get() == 1) - { - numCascades = *cvarSystem->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "shadowCascadeNum"); - } + const u32 numShadowViews = 0; // Main view only, cascades render in their own block late in the frame struct Data { @@ -727,16 +973,12 @@ void ModelRenderer::AddGeometryPass(Renderer::RenderGraph* renderGraph, RenderRe }; renderGraph->AddPass("Model (O) Geometry", - [this, &resources, frameIndex, numCascades](Data& data, Renderer::RenderGraphBuilder& builder) + [this, &resources, frameIndex, numShadowViews](Data& data, Renderer::RenderGraphBuilder& builder) { using BufferUsage = Renderer::BufferPassUsage; data.visibilityBuffer = builder.Write(resources.visibilityBuffer, Renderer::PipelineType::GRAPHICS, Renderer::LoadMode::LOAD); data.depth[0] = builder.Write(resources.depth, Renderer::PipelineType::GRAPHICS, Renderer::LoadMode::LOAD); - for (u32 i = 1; i < numCascades + 1; i++) - { - data.depth[i] = builder.Write(resources.shadowDepthCascades[i - 1], Renderer::PipelineType::GRAPHICS, Renderer::LoadMode::LOAD); - } builder.Read(resources.cameras.GetBuffer(), BufferUsage::GRAPHICS | BufferUsage::COMPUTE); builder.Read(_vertices.GetBuffer(), BufferUsage::GRAPHICS); @@ -754,7 +996,12 @@ void ModelRenderer::AddGeometryPass(Renderer::RenderGraph* renderGraph, RenderRe builder.Write(_opaqueCullingResources.GetCulledDrawCallsBitMaskBuffer(frameIndex), BufferUsage::TRANSFER | BufferUsage::GRAPHICS | BufferUsage::COMPUTE); builder.Write(_opaqueCullingResources.GetCulledDrawCallsBitMaskBuffer(!frameIndex), BufferUsage::TRANSFER | BufferUsage::GRAPHICS | BufferUsage::COMPUTE); - builder.Read(_opaqueCullingResources.GetDrawCallDatas().GetBuffer(), BufferUsage::GRAPHICS); + // No culled-instance-counts / createIndirect registrations: the main-view-only pass + // consumes the culling pass's output directly and never runs the per-view fill — + // registering them created a false render-graph dependency against the SVSM passes + // sharing those buffers + + builder.Read(_opaqueCullingResources.GetDrawCallDatas().GetBuffer(), BufferUsage::GRAPHICS | BufferUsage::COMPUTE); data.globalSet = builder.Use(resources.globalDescriptorSet); data.modelSet = builder.Use(resources.modelDescriptorSet); @@ -762,7 +1009,7 @@ void ModelRenderer::AddGeometryPass(Renderer::RenderGraph* renderGraph, RenderRe return true; // Return true from setup to enable this pass, return false to disable it }, - [this, &resources, frameIndex, cullingEnabled, numCascades, cvarSystem](Data& data, Renderer::RenderGraphResources& graphResources, Renderer::CommandList& commandList) + [this, &resources, frameIndex, cullingEnabled, numShadowViews, cvarSystem](Data& data, Renderer::RenderGraphResources& graphResources, Renderer::CommandList& commandList) { GPU_SCOPED_PROFILER_ZONE(commandList, ModelGeometry); @@ -774,7 +1021,7 @@ void ModelRenderer::AddGeometryPass(Renderer::RenderGraph* renderGraph, RenderRe params.frameIndex = frameIndex; params.rt0 = data.visibilityBuffer; - for (u32 i = 0; i < numCascades + 1; i++) + for (u32 i = 0; i < numShadowViews + 1; i++) { params.depth[i] = data.depth[i]; } @@ -801,11 +1048,10 @@ void ModelRenderer::AddGeometryPass(Renderer::RenderGraph* renderGraph, RenderRe Draw(resources, frameIndex, graphResources, commandList, drawParams); }; - params.numCascades = numCascades; + params.baseInstanceLookupOffset = offsetof(DrawCallData, DrawCallData::baseInstanceLookupOffset); + params.drawCallDataSize = sizeof(DrawCallData); - params.biasConstantFactor = static_cast(*cvarSystem->GetFloatCVar(CVarCategory::Client | CVarCategory::Rendering, "shadowDepthBiasConstant")); - params.biasClamp = static_cast(*cvarSystem->GetFloatCVar(CVarCategory::Client | CVarCategory::Rendering, "shadowDepthBiasClamp")); - params.biasSlopeFactor = static_cast(*cvarSystem->GetFloatCVar(CVarCategory::Client | CVarCategory::Rendering, "shadowDepthBiasSlope")); + params.numShadowViews = numShadowViews; params.enableDrawing = CVAR_ModelDrawGeometry.Get(); params.cullingEnabled = cullingEnabled; @@ -814,6 +1060,288 @@ void ModelRenderer::AddGeometryPass(Renderer::RenderGraph* renderGraph, RenderRe }); } +void ModelRenderer::AddClipmapCullingPass(Renderer::RenderGraph* renderGraph, RenderResources& resources, u8 frameIndex) +{ + ZoneScoped; + + if (!CVAR_ModelRendererEnabled.Get()) + return; + + if (!CVAR_ModelCullingEnabled.Get()) + return; + + if (*CVarSystem::Get()->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "shadowEnabled"_h) == 0) + return; + + if (CVAR_ModelsCastShadow.Get() != 1) + return; + + if (_opaqueCullingResources.GetDrawCalls().Count() == 0) + return; + + // The same per-view culling the main view uses, run against the clipmap cameras + const u32 numShadowViews = static_cast(glm::clamp(*CVarSystem::Get()->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "svsmNumClipmaps"_h), i32(1), i32(ShadowRenderer::SVSM_MAX_CLIPMAPS))); + + struct Data + { + Renderer::ImageResource depthPyramid; + + Renderer::DescriptorSetResource debugSet; + Renderer::DescriptorSetResource globalSet; + Renderer::DescriptorSetResource cullingSet; + }; + + renderGraph->AddPass("Model (O) Clipmap Culling", + [this, &resources, frameIndex](Data& data, Renderer::RenderGraphBuilder& builder) // Setup + { + using BufferUsage = Renderer::BufferPassUsage; + + data.depthPyramid = builder.Read(resources.depthPyramid, Renderer::PipelineType::COMPUTE); + + builder.Read(resources.cameras.GetBuffer(), BufferUsage::COMPUTE); + builder.Read(_cullingDatas.GetBuffer(), BufferUsage::COMPUTE); + builder.Read(_instanceMatrices.GetBuffer(), BufferUsage::COMPUTE); + builder.Read(_opaqueCullingResources.GetDrawCalls().GetBuffer(), BufferUsage::COMPUTE); + builder.Read(_opaqueCullingResources.GetDrawCallDatas().GetBuffer(), BufferUsage::COMPUTE); + builder.Read(_opaqueCullingResources.GetInstanceRefs().GetBuffer(), BufferUsage::COMPUTE); + + builder.Write(_opaqueCullingResources.GetCulledDrawCallsBitMaskBuffer(frameIndex), BufferUsage::COMPUTE); + builder.Write(_opaqueCullingResources.GetCulledDrawCallsBitMaskBuffer(!frameIndex), BufferUsage::COMPUTE); // Both bitmask bindings are RW in the shader + + // Not written by the cascade dispatch (cullMainView == 0), but bound RW in the culling set + builder.Write(_opaqueCullingResources.GetCulledInstanceCountsBuffer(), BufferUsage::COMPUTE); + builder.Write(_opaqueCullingResources.GetCulledInstanceLookupTableBuffer(), BufferUsage::COMPUTE); + + data.debugSet = builder.Use(_debugRenderer->GetDebugDescriptorSet()); + data.globalSet = builder.Use(resources.globalDescriptorSet); + data.cullingSet = builder.Use(_opaqueCullingResources.GetCullingDescriptorSet()); + + _debugRenderer->RegisterCullingPassBufferUsage(builder); + + return true; // Return true from setup to enable this pass, return false to disable it + }, + [this, frameIndex, numShadowViews](Data& data, Renderer::RenderGraphResources& graphResources, Renderer::CommandList& commandList) // Execute + { + GPU_SCOPED_PROFILER_ZONE(commandList, ModelClipmapCulling); + + CulledRenderer::CullingPassParams params; + params.passName = "Opaque"; + params.graphResources = &graphResources; + params.commandList = &commandList; + params.cullingResources = &_opaqueCullingResources; + params.frameIndex = frameIndex; + + params.depthPyramid = data.depthPyramid; + + params.debugDescriptorSet = data.debugSet; + params.globalDescriptorSet = data.globalSet; + params.cullingDescriptorSet = data.cullingSet; + + params.numShadowViews = numShadowViews; + params.cullMainView = false; + params.occlusionCull = false; + + params.cullingDataIsWorldspace = false; + + params.baseInstanceLookupOffset = offsetof(DrawCallData, baseInstanceLookupOffset); + params.modelIDOffset = offsetof(DrawCallData, modelID); + params.drawCallDataSize = sizeof(DrawCallData); + + ClipmapCullingPass(params); + }); +} + +void ModelRenderer::AddSVSMGeometryPass(Renderer::RenderGraph* renderGraph, RenderResources& resources, u8 frameIndex, ShadowRenderer* shadowRenderer) +{ + ZoneScoped; + + if (!CVAR_ModelRendererEnabled.Get()) + return; + + if (!CVAR_ModelCullingEnabled.Get()) // Clipmaps are driven by the culled bitmask slices + return; + + CVarSystem* cvarSystem = CVarSystem::Get(); + + if (*cvarSystem->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "shadowEnabled"_h) == 0) + return; + + if (CVAR_ModelsCastShadow.Get() != 1) + return; + + if (shadowRenderer->GetSVSMPagePool() == Renderer::ImageID::Invalid()) + return; + + if (_opaqueCullingResources.GetDrawCalls().Count() == 0) + return; + + const u32 numClipmaps = static_cast(glm::clamp(*cvarSystem->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "svsmNumClipmaps"_h), i32(1), i32(ShadowRenderer::SVSM_MAX_CLIPMAPS))); + const u32 virtualSize = static_cast(*cvarSystem->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "svsmVirtualSize"_h)); + + // No dynamic casters this frame -> skip the dynamic fills/draws outright. The instance mask + // is all zero then, so the unfiltered static fill is equivalent to the filtered one + const bool splitFills = *cvarSystem->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "svsmDynamicSplit"_h) == 1 + && shadowRenderer->GetSVSMDynamicPagePool() != Renderer::ImageID::Invalid() + && shadowRenderer->HasSVSMDynamicCasters(); + + struct Data + { + Renderer::ImageMutableResource pagePool; + Renderer::ImageMutableResource dynamicPagePool; + Renderer::BufferResource svsmData; + Renderer::BufferResource pageTable; + Renderer::BufferResource dynamicPageTable; + + Renderer::BufferMutableResource drawCallsBuffer; + Renderer::BufferMutableResource culledDrawCallsBuffer; + Renderer::BufferMutableResource culledDrawCallCountBuffer; + + Renderer::BufferMutableResource culledInstanceCountsBuffer; + + Renderer::BufferMutableResource drawCountBuffer; + Renderer::BufferMutableResource triangleCountBuffer; + Renderer::BufferMutableResource drawCountReadBackBuffer; + Renderer::BufferMutableResource triangleCountReadBackBuffer; + + Renderer::BufferMutableResource svsmDynamicDrawCountReadBackBuffer; + Renderer::BufferResource svsmFillArgsBuffer; + + Renderer::DescriptorSetResource globalSet; + Renderer::DescriptorSetResource modelSet; + Renderer::DescriptorSetResource fillSet; + Renderer::DescriptorSetResource createIndirectSet; + Renderer::DescriptorSetResource drawSet; + Renderer::DescriptorSetResource svsmSet; + Renderer::DescriptorSetResource svsmDynamicSet; + }; + + renderGraph->AddPass("Model (O) SVSM Geometry", + [this, &resources, frameIndex, shadowRenderer, splitFills](Data& data, Renderer::RenderGraphBuilder& builder) + { + using BufferUsage = Renderer::BufferPassUsage; + + data.pagePool = builder.Write(shadowRenderer->GetSVSMPagePool(), Renderer::PipelineType::GRAPHICS, Renderer::LoadMode::LOAD); + data.svsmData = builder.Read(shadowRenderer->GetSVSMDataBuffer(), BufferUsage::GRAPHICS); + data.pageTable = builder.Read(shadowRenderer->GetSVSMPageTableBuffer(), BufferUsage::GRAPHICS); + data.svsmFillArgsBuffer = builder.Read(shadowRenderer->GetSVSMFillArgsBuffer(), BufferUsage::COMPUTE); // Consumed read-only by DispatchIndirect in the per-view fills + builder.Read(_dynamicInstanceMask.GetBuffer(), BufferUsage::COMPUTE); // The static fills always run the filtered pipeline against the mask, split live or not + if (splitFills) + { + data.dynamicPagePool = builder.Write(shadowRenderer->GetSVSMDynamicPagePool(), Renderer::PipelineType::GRAPHICS, Renderer::LoadMode::LOAD); + data.dynamicPageTable = builder.Read(shadowRenderer->GetSVSMDynamicPageTableBuffer(), BufferUsage::GRAPHICS); + data.svsmDynamicDrawCountReadBackBuffer = builder.Write(_svsmDynamicDrawCountReadBackBuffer, BufferUsage::TRANSFER); + data.svsmDynamicSet = builder.Use(_svsmDynamicDrawDescriptorSet); + } + + builder.Read(resources.cameras.GetBuffer(), BufferUsage::GRAPHICS | BufferUsage::COMPUTE); + builder.Read(_vertices.GetBuffer(), BufferUsage::GRAPHICS); + builder.Read(_indices.GetBuffer(), BufferUsage::GRAPHICS); + builder.Read(_textureDatas.GetBuffer(), BufferUsage::GRAPHICS); + builder.Read(_textureUnits.GetBuffer(), BufferUsage::GRAPHICS); + builder.Read(_instanceDatas.GetBuffer(), BufferUsage::GRAPHICS); + builder.Read(_instanceMatrices.GetBuffer(), BufferUsage::GRAPHICS); + builder.Read(_boneMatrices.GetBuffer(), BufferUsage::GRAPHICS | BufferUsage::COMPUTE); + builder.Read(_textureTransformMatrices.GetBuffer(), BufferUsage::GRAPHICS | BufferUsage::COMPUTE); + + builder.Write(_animatedVertices.GetBuffer(), BufferUsage::GRAPHICS | BufferUsage::COMPUTE); + + GeometryPassSetup(data, builder, &_opaqueCullingResources, frameIndex); + builder.Write(_opaqueCullingResources.GetCulledDrawCallsBitMaskBuffer(frameIndex), BufferUsage::TRANSFER | BufferUsage::GRAPHICS | BufferUsage::COMPUTE); + builder.Write(_opaqueCullingResources.GetCulledDrawCallsBitMaskBuffer(!frameIndex), BufferUsage::TRANSFER | BufferUsage::GRAPHICS | BufferUsage::COMPUTE); + + data.culledInstanceCountsBuffer = builder.Write(_opaqueCullingResources.GetCulledInstanceCountsBuffer(), BufferUsage::TRANSFER | BufferUsage::COMPUTE); + + builder.Read(_opaqueCullingResources.GetDrawCallDatas().GetBuffer(), BufferUsage::GRAPHICS | BufferUsage::COMPUTE); + + data.globalSet = builder.Use(resources.globalDescriptorSet); + data.modelSet = builder.Use(resources.modelDescriptorSet); + data.fillSet = builder.Use(_opaqueCullingResources.GetGeometryFillDescriptorSet()); + data.createIndirectSet = builder.Use(_opaqueCullingResources.GetCreateIndirectAfterCullDescriptorSet()); + data.svsmSet = builder.Use(_svsmDrawDescriptorSet); + + return true; // Return true from setup to enable this pass, return false to disable it + }, + [this, &resources, frameIndex, numClipmaps, virtualSize, splitFills, shadowRenderer](Data& data, Renderer::RenderGraphResources& graphResources, Renderer::CommandList& commandList) + { + GPU_SCOPED_PROFILER_ZONE(commandList, ModelSVSMGeometry); + + // _svsmData and the page tables are bound once at init through BindSVSMBuffers, only + // the lazily created pools bind per frame (image binds write the descriptor immediately) + data.svsmSet.Bind("_pagePool"_h, data.pagePool); + if (splitFills) + { + data.svsmDynamicSet.Bind("_pagePool"_h, data.dynamicPagePool); + } + + CulledRenderer::GeometryPassParams params; + params.passName = "Opaque"; + params.graphResources = &graphResources; + params.commandList = &commandList; + params.cullingResources = &_opaqueCullingResources; + + params.frameIndex = frameIndex; + + params.drawCallsBuffer = data.drawCallsBuffer; + params.culledDrawCallsBuffer = data.culledDrawCallsBuffer; + params.culledDrawCallCountBuffer = data.culledDrawCallCountBuffer; + params.culledInstanceCountsBuffer = data.culledInstanceCountsBuffer; + + params.drawCountBuffer = data.drawCountBuffer; + params.triangleCountBuffer = data.triangleCountBuffer; + params.drawCountReadBackBuffer = data.drawCountReadBackBuffer; + params.triangleCountReadBackBuffer = data.triangleCountReadBackBuffer; + + params.globalDescriptorSet = data.globalSet; + params.fillDescriptorSet = data.fillSet; + params.createIndirectDescriptorSet = data.createIndirectSet; + params.drawDescriptorSet = data.drawSet; + + params.drawCallback = [&](DrawParams& drawParams) + { + drawParams.descriptorSets = { + &data.globalSet, + &data.modelSet, + &data.svsmSet + }; + Draw(resources, frameIndex, graphResources, commandList, drawParams); + }; + + params.baseInstanceLookupOffset = offsetof(DrawCallData, DrawCallData::baseInstanceLookupOffset); + params.drawCallDataSize = sizeof(DrawCallData); + + params.firstViewIndex = 1; + params.numShadowViews = numClipmaps; + + params.svsmPass = true; + params.svsmExtent = uvec2(virtualSize, virtualSize); + + params.svsmSplitFills = splitFills; + params.svsmFillArgsBuffer = data.svsmFillArgsBuffer; + params.svsmFillArgsViewStride = ShadowRenderer::SVSM_FILL_ARGS_VIEW_STRIDE; + params.svsmFillArgsDynamicOffset = ShadowRenderer::SVSM_FILL_ARGS_DYNAMIC_OFFSET; + params.svsmFillArgsStaticOverheadOffset = ShadowRenderer::SVSM_FILL_ARGS_STATIC_OVERHEAD_OFFSET; + params.svsmFillArgsDynamicOverheadOffset = ShadowRenderer::SVSM_FILL_ARGS_DYNAMIC_OVERHEAD_OFFSET; + if (splitFills) + { + params.svsmDynamicDrawCountReadBackBuffer = data.svsmDynamicDrawCountReadBackBuffer; + params.drawCallbackDynamic = [&](DrawParams& drawParams) + { + drawParams.descriptorSets = { + &data.globalSet, + &data.modelSet, + &data.svsmDynamicSet + }; + Draw(resources, frameIndex, graphResources, commandList, drawParams); + }; + } + + params.enableDrawing = CVAR_ModelDrawGeometry.Get(); + params.cullingEnabled = true; + + GeometryPass(params); + }); +} + void ModelRenderer::AddTransparencyCullingPass(Renderer::RenderGraph* renderGraph, RenderResources& resources, u8 frameIndex) { ZoneScoped; @@ -824,7 +1352,7 @@ void ModelRenderer::AddTransparencyCullingPass(Renderer::RenderGraph* renderGrap if (!CVAR_ModelCullingEnabled.Get()) return; - u32 numCascades = 0;// *CVarSystem::Get()->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "numShadowCascades"_h); + u32 numShadowViews = 0;// *CVarSystem::Get()->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "numShadowCascades"_h); struct Data { @@ -894,7 +1422,7 @@ void ModelRenderer::AddTransparencyCullingPass(Renderer::RenderGraph* renderGrap params.cullingDescriptorSet = data.cullingSet; params.createIndirectAfterCullSet = data.createIndirectAfterCullSet; - params.numCascades = 0;// *CVarSystem::Get()->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "numShadowCascades"_h); + params.numShadowViews = 0;// *CVarSystem::Get()->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "numShadowCascades"_h); params.occlusionCull = CVAR_ModelOcclusionCullingEnabled.Get(); params.disableTwoStepCulling = true; // Transparent objects don't write depth, so we don't need to two step cull them @@ -1010,7 +1538,7 @@ void ModelRenderer::AddTransparencyGeometryPass(Renderer::RenderGraph* renderGra DrawTransparent(resources, frameIndex, graphResources, commandList, drawParams); }; - params.numCascades = 0; + params.numShadowViews = 0; params.enableDrawing = CVAR_ModelDrawGeometry.Get(); params.cullingEnabled = cullingEnabled; @@ -1117,7 +1645,7 @@ void ModelRenderer::AddSkyboxPass(Renderer::RenderGraph* renderGraph, RenderReso params.enableDrawing = CVAR_ModelDrawGeometry.Get(); params.cullingEnabled = false; - params.numCascades = 0;// *CVarSystem::Get()->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "numShadowCascades"_h); + params.numShadowViews = 0;// *CVarSystem::Get()->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "numShadowCascades"_h); GeometryPass(params); }); @@ -1217,7 +1745,7 @@ void ModelRenderer::AddSkyboxPass(Renderer::RenderGraph* renderGraph, RenderReso params.enableDrawing = CVAR_ModelDrawGeometry.Get(); params.cullingEnabled = false; - params.numCascades = 0;// *CVarSystem::Get()->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "numShadowCascades"_h); + params.numShadowViews = 0;// *CVarSystem::Get()->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "numShadowCascades"_h); GeometryPass(params); }); @@ -1590,6 +2118,63 @@ u32 ModelRenderer::AddPlacementInstance(entt::entity entityID, u32 modelID, u64 return instanceIndex; } +void ModelRenderer::ComputeInstanceShadowAABB(u32 instanceID, const mat4x4& transformMatrix, vec3& outMin, vec3& outMax) +{ + const InstanceData& instanceData = _instanceDatas[instanceID]; + const Model::ComplexModel::CullingData& cullingData = _cullingDatas[instanceData.modelID]; + + vec3 center = vec3(transformMatrix * vec4(vec3(cullingData.center), 1.0f)); + mat3x3 absRotation = mat3x3(transformMatrix); + absRotation[0] = glm::abs(absRotation[0]); + absRotation[1] = glm::abs(absRotation[1]); + absRotation[2] = glm::abs(absRotation[2]); + vec3 extents = absRotation * vec3(cullingData.extents); + + outMin = center - extents; + outMax = center + extents; +} + +void ModelRenderer::QueueShadowInvalidation(u32 instanceID, const mat4x4& transformMatrix) +{ + vec3 aabbMin, aabbMax; + ComputeInstanceShadowAABB(instanceID, transformMatrix, aabbMin, aabbMax); + QueueShadowInvalidation(aabbMin, aabbMax); +} + +void ModelRenderer::QueueShadowInvalidation(const vec3& aabbMin, const vec3& aabbMax) +{ + ShadowInvalidation invalidation; + invalidation.min = vec4(aabbMin, 0.0f); + invalidation.max = vec4(aabbMax, 0.0f); + _shadowInvalidationQueue.enqueue(invalidation); +} + +void ModelRenderer::BindSVSMBuffers(Renderer::BufferID svsmDataBuffer, Renderer::BufferID pageTableBuffer, Renderer::BufferID dynamicPageTableBuffer) +{ + _svsmDrawDescriptorSet.Bind("_svsmData"_h, svsmDataBuffer); + _svsmDrawDescriptorSet.Bind("_pageTable"_h, pageTableBuffer); + _svsmDynamicDrawDescriptorSet.Bind("_svsmData"_h, svsmDataBuffer); + _svsmDynamicDrawDescriptorSet.Bind("_pageTable"_h, dynamicPageTableBuffer); +} + +u32 ModelRenderer::DrainShadowInvalidations(std::vector& outMinMaxPairs, u32 maxPairs) +{ + u32 numDrained = 0; + + ShadowInvalidation invalidation; + while (_shadowInvalidationQueue.try_dequeue(invalidation)) + { + if (numDrained < maxPairs) + { + outMinMaxPairs.push_back(invalidation.min); + outMinMaxPairs.push_back(invalidation.max); + } + numDrained++; // Keeps draining past the cap so the queue can't grow unbounded + } + + return numDrained; +} + u32 ModelRenderer::AddInstance(entt::entity entityID, u32 modelID, Model::ComplexModel* model, const mat4x4& transformMatrix, u64 displayInfoPacked) { ZoneScopedN("ModelRenderer::AddInstance"); @@ -1625,6 +2210,7 @@ u32 ModelRenderer::AddInstance(entt::entity entityID, u32 modelID, Model::Comple { std::scoped_lock lock(*_modelManifestsInstancesMutexes[modelID]); manifest.instances.insert(instanceOffsets.instanceIndex); + _instanceSetEpoch.fetch_add(1, std::memory_order_release); } // Set up InstanceManifest @@ -1651,6 +2237,9 @@ u32 ModelRenderer::AddInstance(entt::entity entityID, u32 modelID, Model::Comple RequestChangeSkybox(instanceOffsets.instanceIndex, true); } + // A spawned static caster must invalidate the cached static shadow pages under it + QueueShadowInvalidation(instanceOffsets.instanceIndex, transformMatrix); + _instancesDirty = true; return instanceOffsets.instanceIndex; @@ -1664,6 +2253,9 @@ void ModelRenderer::RemoveInstance(u32 instanceID) const u32 removedModelID = instanceData.modelID; ModelManifest& manifest = _modelManifests[removedModelID]; + // Capture the shadow footprint before the instance data dies, its baked static shadow must go + QueueShadowInvalidation(instanceID, _instanceMatrices[instanceID]); + // TODO: We need to change _animatedVerticesIndex so we can free up between instanceData.animatedVertexOffset + manifest.numVertices // Remove Instance from ModelManifest @@ -1671,6 +2263,7 @@ void ModelRenderer::RemoveInstance(u32 instanceID) std::scoped_lock lock(*_modelManifestsInstancesMutexes[instanceData.modelID]); manifest.instances.erase(instanceID); manifest.skyboxInstances.erase(instanceID); + _instanceSetEpoch.fetch_add(1, std::memory_order_release); } std::scoped_lock lock(_instanceOffsetsMutex); @@ -1702,6 +2295,10 @@ void ModelRenderer::ModifyInstance(entt::entity entityID, u32 instanceID, u32 mo InstanceData& instanceData = _instanceDatas[instanceID]; + // The old shadow footprint's cached static pages must refresh, the new one queues below once + // the model ID is swapped + QueueShadowInvalidation(instanceID, _instanceMatrices[instanceID]); + u32 oldModelID = instanceData.modelID; ModelManifest& oldManifest = _modelManifests[oldModelID]; @@ -1710,6 +2307,7 @@ void ModelRenderer::ModifyInstance(entt::entity entityID, u32 instanceID, u32 mo std::scoped_lock lock(*_modelManifestsInstancesMutexes[oldModelID]); oldManifest.instances.erase(instanceID); oldManifest.skyboxInstances.erase(instanceID); + _instanceSetEpoch.fetch_add(1, std::memory_order_release); } // Deallocate old animation data @@ -1724,6 +2322,7 @@ void ModelRenderer::ModifyInstance(entt::entity entityID, u32 instanceID, u32 mo { std::scoped_lock lock(*_modelManifestsInstancesMutexes[modelID]); newManifest.instances.insert(instanceID); + _instanceSetEpoch.fetch_add(1, std::memory_order_release); } // Modify InstanceData @@ -1748,6 +2347,9 @@ void ModelRenderer::ModifyInstance(entt::entity entityID, u32 instanceID, u32 mo instanceMatrix = transformMatrix; } + // New model + transform in place, refresh the new shadow footprint too + QueueShadowInvalidation(instanceID, transformMatrix); + InstanceManifest& instanceManifest = _instanceManifests[instanceID]; instanceManifest.modelID = modelID; @@ -2336,6 +2938,10 @@ bool ModelRenderer::SetUninstancedBoneMatricesAsDirty(u32 modelID, u32 boneMatri if (endGlobalBoneIndex > boneMatrixOffset + modelManifest.numBones) return false; + // This model's shared animation advanced, Update classifies its in-range placements as + // dynamic shadow casters + _uninstancedAnimatedModelQueue.enqueue(modelID); + if (count == 1) { _boneMatrices[globalBoneIndex] = *boneMatrixArray; @@ -2464,6 +3070,10 @@ bool ModelRenderer::SetBoneMatricesAsDirty(u32 instanceID, u32 localBoneIndex, u _boneMatrices.SetDirtyElements(globalBoneIndex, count); } + // Pushed bone matrices this frame -> dynamic shadow caster this frame. The static-baked + // animation path (SetUninstancedBoneMatricesAsDirty) intentionally does not do this + _dynamicInstanceQueue.enqueue(instanceID); + return true; } @@ -2513,6 +3123,13 @@ void ModelRenderer::CreatePermanentResources() RenderResources& resources = _gameRenderer->GetRenderResources(); + Renderer::BufferDesc svsmDynamicCountDesc; + svsmDynamicCountDesc.name = "ModelSVSMDynamicDrawCountRBBuffer"; + svsmDynamicCountDesc.size = sizeof(u32) * Renderer::Settings::MAX_VIEWS; + svsmDynamicCountDesc.usage = Renderer::BufferUsage::STORAGE_BUFFER | Renderer::BufferUsage::TRANSFER_DESTINATION; + svsmDynamicCountDesc.cpuAccess = Renderer::BufferCPUAccess::ReadOnly; + _svsmDynamicDrawCountReadBackBuffer = _renderer->CreateBuffer(_svsmDynamicDrawCountReadBackBuffer, svsmDynamicCountDesc); + Renderer::TextureArrayDesc textureArrayDesc; textureArrayDesc.size = Renderer::Settings::MAX_TEXTURES; @@ -2710,54 +3327,61 @@ void ModelRenderer::CreateModelPipelines() std::vector vertexPermutationFields = { { "EDITOR_PASS", "0" }, - { "SHADOW_PASS", "0"} + { "SVSM_PASS", "0"} }; u32 shaderEntryNameHash = Renderer::GetShaderEntryNameHash("Model/Draw.vs", vertexPermutationFields); vertexShaderDesc.shaderEntry = _gameRenderer->GetShaderEntry(shaderEntryNameHash, "Model/Draw.vs"); pipelineDesc.states.vertexShader = _renderer->LoadShader(vertexShaderDesc); Renderer::PixelShaderDesc pixelShaderDesc; - std::vector pixelPermutationFields = - { - { "SHADOW_PASS", "0" } - }; - shaderEntryNameHash = Renderer::GetShaderEntryNameHash("Model/Draw.ps", pixelPermutationFields); - pixelShaderDesc.shaderEntry = _gameRenderer->GetShaderEntry(shaderEntryNameHash, "Model/Draw.ps"); + pixelShaderDesc.shaderEntry = _gameRenderer->GetShaderEntry("Model/Draw.ps"_h, "Model/Draw.ps"); pipelineDesc.states.pixelShader = _renderer->LoadShader(pixelShaderDesc); _drawPipeline = _renderer->CreatePipeline(pipelineDesc); } - // Shadows + // SVSM pages: attachment-less, the pixel shader keeps the alpha-key discard and writes depth + // into the page pool with image atomics. Depth clamp stays on so pancaked casters rasterize { Renderer::GraphicsPipelineDesc pipelineDesc; - // Rasterizer state + pipelineDesc.debugName = "Model Draw SVSM"; + + // Rasterizer state, no depth or color attachments pipelineDesc.states.rasterizerState.cullMode = Renderer::CullMode::NONE; - pipelineDesc.states.rasterizerState.depthBiasEnabled = true; pipelineDesc.states.rasterizerState.depthClampEnabled = true; - pipelineDesc.states.depthStencilFormat = Renderer::DepthImageFormat::D32_FLOAT; - Renderer::VertexShaderDesc vertexShaderDesc; std::vector vertexPermutationFields = { { "EDITOR_PASS", "0" }, - { "SHADOW_PASS", "1"} + { "SVSM_PASS", "1"} }; u32 shaderEntryNameHash = Renderer::GetShaderEntryNameHash("Model/Draw.vs", vertexPermutationFields); vertexShaderDesc.shaderEntry = _gameRenderer->GetShaderEntry(shaderEntryNameHash, "Model/Draw.vs"); pipelineDesc.states.vertexShader = _renderer->LoadShader(vertexShaderDesc); - - Renderer::PixelShaderDesc pixelShaderDesc; - std::vector pixelPermutationFields = + + for (u32 dynamic = 0; dynamic < 2; dynamic++) { - { "SHADOW_PASS", "1" } - }; - shaderEntryNameHash = Renderer::GetShaderEntryNameHash("Model/Draw.ps", pixelPermutationFields); - pixelShaderDesc.shaderEntry = _gameRenderer->GetShaderEntry(shaderEntryNameHash, "Model/Draw.ps"); - pipelineDesc.states.pixelShader = _renderer->LoadShader(pixelShaderDesc); - - // Draw - _drawShadowPipeline = _renderer->CreatePipeline(pipelineDesc); + std::vector pixelPermutationFields = + { + { "SVSM_DYNAMIC", std::to_string(dynamic) } + }; + u32 pixelShaderEntryNameHash = Renderer::GetShaderEntryNameHash("Model/DrawSVSM.ps", pixelPermutationFields); + + Renderer::PixelShaderDesc pixelShaderDesc; + pixelShaderDesc.shaderEntry = _gameRenderer->GetShaderEntry(pixelShaderEntryNameHash, "Model/DrawSVSM.ps"); + pipelineDesc.states.pixelShader = _renderer->LoadShader(pixelShaderDesc); + + if (dynamic == 0) + { + pipelineDesc.debugName = "Model Draw SVSM"; + _drawSVSMPipeline = _renderer->CreatePipeline(pipelineDesc); + } + else + { + pipelineDesc.debugName = "Model Draw SVSM Dynamic"; + _drawSVSMDynamicPipeline = _renderer->CreatePipeline(pipelineDesc); + } + } } // Transparencies { @@ -2920,6 +3544,16 @@ void ModelRenderer::InitDescriptorSets() geometryPassDescriptorSet.RegisterPipeline(_renderer, _drawSkyboxTransparentPipeline); geometryPassDescriptorSet.Init(_renderer); } + + // SVSM page render + { + _svsmDrawDescriptorSet.RegisterPipeline(_renderer, _drawSVSMPipeline); + _svsmDrawDescriptorSet.Init(_renderer); + + _svsmDynamicDrawDescriptorSet.RegisterPipeline(_renderer, _drawSVSMDynamicPipeline); + _svsmDynamicDrawDescriptorSet.Init(_renderer); + } + } void ModelRenderer::AssertOwnerThread() const @@ -3175,6 +3809,7 @@ void ModelRenderer::MakeInstanceSkybox(u32 instanceID, InstanceManifest& instanc // Remove the non skybox instance modelManifest.instances.erase(instanceID); + _instanceSetEpoch.fetch_add(1, std::memory_order_release); } else { @@ -3183,6 +3818,7 @@ void ModelRenderer::MakeInstanceSkybox(u32 instanceID, InstanceManifest& instanc // Add the non skybox instance modelManifest.instances.insert(instanceID); + _instanceSetEpoch.fetch_add(1, std::memory_order_release); } } @@ -3474,6 +4110,57 @@ void ModelRenderer::SyncToGPU() BindCullingResource(_opaqueSkyboxCullingResources); BindCullingResource(_transparentSkyboxCullingResources); } + + // Rebuild the SVSM dynamic instance mask from the classifier's live set (Update drained the + // raw signals into it earlier this frame). The buffer always exists and stays bound — the + // SVSM fills always run the filtered pipeline against it (an all-zero mask with keepDynamic 0 + // keeps everything); only the bit updates skip when nothing is live and nothing was masked + { + ZoneScopedN("Sync Dynamic Instance Mask"); + + u32 wordsNeeded = (static_cast(_instanceDatas.Capacity()) + 31) / 32; + u32 wordCount = static_cast(_dynamicInstanceMask.Count()); + if (wordsNeeded > wordCount) + { + _dynamicInstanceMask.AddCount(wordsNeeded - wordCount); + for (u32 i = wordCount; i < wordsNeeded; i++) + { + _dynamicInstanceMask[i] = 0; + } + } + + if (!_dynamicCasterLiveIDs.empty() || !_dynamicInstanceIDs.empty()) + { + // Clear last frame's bits, then set this frame's. The bit test doubles as dedupe + for (u32 instanceID : _dynamicInstanceIDs) + { + u32 word = instanceID / 32; + _dynamicInstanceMask[word] &= ~(1u << (instanceID & 31)); + _dynamicInstanceMask.SetDirtyElement(word); + } + _dynamicInstanceIDs.clear(); + + for (u32 instanceID : _dynamicCasterLiveIDs) + { + u32 word = instanceID / 32; + if (word >= wordsNeeded) + continue; + + u32 bit = 1u << (instanceID & 31); + if ((_dynamicInstanceMask[word] & bit) == 0) + { + _dynamicInstanceMask[word] |= bit; + _dynamicInstanceMask.SetDirtyElement(word); + _dynamicInstanceIDs.push_back(instanceID); + } + } + } + + if (_dynamicInstanceMask.SyncToGPU(_renderer)) + { + _opaqueCullingResources.GetGeometryFillDescriptorSet().Bind("_dynamicInstanceMask"_h, _dynamicInstanceMask.GetBuffer()); + } + } } void ModelRenderer::Draw(const RenderResources& resources, u8 frameIndex, Renderer::RenderGraphResources& graphResources, Renderer::CommandList& commandList, const DrawParams& params) @@ -3482,29 +4169,41 @@ void ModelRenderer::Draw(const RenderResources& resources, u8 frameIndex, Render graphResources.InitializeRenderPassDesc(renderPassDesc); // Render targets - if (!params.shadowPass) + if (params.svsmPass) + { + // Attachment-less: with no render targets BeginRenderPass has no extent fallback, + // the render area must be explicit or it collapses to zero + renderPassDesc.extent = params.svsmExtent; + } + else { renderPassDesc.renderTargets[0] = params.rt0; if (params.rt1 != Renderer::ImageMutableResource::Invalid()) { renderPassDesc.renderTargets[1] = params.rt1; } + renderPassDesc.depthStencil = params.depth; } - renderPassDesc.depthStencil = params.depth; commandList.BeginRenderPass(renderPassDesc); - Renderer::GraphicsPipelineID pipeline = params.shadowPass ? _drawShadowPipeline : _drawPipeline; + Renderer::GraphicsPipelineID pipeline = params.svsmPass ? (params.svsmDynamicPass ? _drawSVSMDynamicPipeline : _drawSVSMPipeline) + : _drawPipeline; commandList.BeginPipeline(pipeline); struct PushConstants { u32 viewIndex; + u32 svsmRectIndex; }; PushConstants* constants = graphResources.FrameNew(); constants->viewIndex = params.viewIndex; - commandList.PushConstant(constants, 0, sizeof(PushConstants)); + constants->svsmRectIndex = params.svsmRectIndex; + + // Only the SVSM vertex permutation declares the rect index, the other pipelines' push range + // is a single uint + commandList.PushConstant(constants, 0, params.svsmPass ? sizeof(PushConstants) : sizeof(u32)); for (auto& descriptorSet : params.descriptorSets) { diff --git a/Source/Game-Lib/Game-Lib/Rendering/Model/ModelRenderer.h b/Source/Game-Lib/Game-Lib/Rendering/Model/ModelRenderer.h index cb9fb105..1195848e 100644 --- a/Source/Game-Lib/Game-Lib/Rendering/Model/ModelRenderer.h +++ b/Source/Game-Lib/Game-Lib/Rendering/Model/ModelRenderer.h @@ -22,6 +22,7 @@ class DebugRenderer; class GameRenderer; +class ShadowRenderer; struct RenderResources; namespace Renderer @@ -106,6 +107,7 @@ class ModelRenderer : CulledRenderer robin_hood::unordered_set instances; robin_hood::unordered_set skyboxInstances; robin_hood::unordered_set originallyTransparentDrawIDs; + }; struct InstanceManifest @@ -334,6 +336,12 @@ class ModelRenderer : CulledRenderer void AddOccluderPass(Renderer::RenderGraph* renderGraph, RenderResources& resources, u8 frameIndex); void AddCullingPass(Renderer::RenderGraph* renderGraph, RenderResources& resources, u8 frameIndex); void AddGeometryPass(Renderer::RenderGraph* renderGraph, RenderResources& resources, u8 frameIndex); + void AddClipmapCullingPass(Renderer::RenderGraph* renderGraph, RenderResources& resources, u8 frameIndex); // Per-clipmap-view frustum culling into the bitmask slices + void AddSVSMGeometryPass(Renderer::RenderGraph* renderGraph, RenderResources& resources, u8 frameIndex, ShadowRenderer* shadowRenderer); + + // Called once by ShadowRenderer at init, buffer binds must happen before the first frame + // (they only reach the canonical descriptor copies at a later FlipFrame) + void BindSVSMBuffers(Renderer::BufferID svsmDataBuffer, Renderer::BufferID pageTableBuffer, Renderer::BufferID dynamicPageTableBuffer); void AddTransparencyCullingPass(Renderer::RenderGraph* renderGraph, RenderResources& resources, u8 frameIndex); void AddTransparencyGeometryPass(Renderer::RenderGraph* renderGraph, RenderResources& resources, u8 frameIndex); @@ -343,6 +351,20 @@ class ModelRenderer : CulledRenderer Renderer::GPUVector& GetInstanceMatrices() { return _instanceMatrices; } const std::vector& GetModelManifests() { return _modelManifests; } + // SVSM: appends world (min, max) pairs of spawned/despawned instances, returns pairs appended + u32 DrainShadowInvalidations(std::vector& outMinMaxPairs, u32 maxPairs); + + // SVSM dynamic-caster classifier outputs, rebuilt each Update: world (min, max) pairs of the + // live set, its size, spill count, and this frame's enter/leave transitions + const std::vector& GetDynamicCasterAABBs() const { return _dynamicCasterAABBs; } + u32 GetNumDynamicCasters() const { return static_cast(_dynamicCasterLiveIDs.size()); } + u32 GetNumDynamicCastersDropped() const { return _dynamicCastersDropped; } + void GetDynamicCasterTransitions(u32& outTransitionsIn, u32& outTransitionsOut) const + { + outTransitionsIn = _dynamicCasterTransitionsIn; + outTransitionsOut = _dynamicCasterTransitionsOut; + } + CullingResourcesIndexed& GetOpaqueCullingResources() { return _opaqueCullingResources; } CullingResourcesIndexed& GetTransparentCullingResources() { return _transparentCullingResources; } @@ -350,6 +372,7 @@ class ModelRenderer : CulledRenderer u32 GetNumDrawCalls() { return 0; } u32 GetNumOccluderDrawCalls() { return _numOccluderDrawCalls; } u32 GetNumSurvivingDrawCalls(u32 viewID) { return _numSurvivingDrawCalls[viewID]; } + u32 GetNumSVSMDynamicInstances(u32 viewID) { return _numSvsmDynamicInstances[viewID]; } // One frame old, view 0 unused // Triangle stats u32 GetNumTriangles() { return 0; } @@ -375,6 +398,10 @@ class ModelRenderer : CulledRenderer void MakeInstanceTransparent(u32 instanceID, InstanceManifest& instanceManifest, ModelManifest& modelManifest); void MakeInstanceSkybox(u32 instanceID, InstanceManifest& instanceManifest, bool skybox); + void ComputeInstanceShadowAABB(u32 instanceID, const mat4x4& transformMatrix, vec3& outMin, vec3& outMax); + void QueueShadowInvalidation(u32 instanceID, const mat4x4& transformMatrix); + void QueueShadowInvalidation(const vec3& aabbMin, const vec3& aabbMax); // For callers holding a cached AABB + void CompactInstanceRefs(); void SyncToGPU(); @@ -428,11 +455,81 @@ class ModelRenderer : CulledRenderer CullingResourcesIndexed _transparentSkyboxCullingResources; Renderer::GraphicsPipelineID _drawPipeline; - Renderer::GraphicsPipelineID _drawShadowPipeline; + Renderer::GraphicsPipelineID _drawSVSMPipeline; + Renderer::GraphicsPipelineID _drawSVSMDynamicPipeline; Renderer::GraphicsPipelineID _drawTransparentPipeline; Renderer::GraphicsPipelineID _drawSkyboxOpaquePipeline; Renderer::GraphicsPipelineID _drawSkyboxTransparentPipeline; + Renderer::DescriptorSet _svsmDrawDescriptorSet; + Renderer::DescriptorSet _svsmDynamicDrawDescriptorSet; + + // SVSM static/dynamic caster split: instances that moved or pushed bone matrices this frame. + // Producers enqueue from any thread; Update drains them into the classifier below, SyncToGPU + // rebuilds the bit mask once per frame from the classifier's live set + moodycamel::ConcurrentQueue _dynamicInstanceQueue; + std::vector _dynamicInstanceIDs; // Last frame's masked set, backs the incremental bit updates + Renderer::GPUVector _dynamicInstanceMask; + + // The single dynamic-caster classifier: instanceID -> accumulated-seconds stamp of its last + // dynamic signal (move or bone push). Instances stay classified for a grace period after the + // last signal so brief pauses don't churn the static cache; entering pulls the baked pose out + // of the static pages, expiring bakes it back. A despawned instance's stale entry expires + // within the grace and costs one spurious page refresh at worst (RemoveInstance already + // invalidated its real footprint). + // The shadow AABB is cached in the entry and recomputed only when it can change (classifier + // entry, transform update) — the per-frame tick reads it instead of redoing the glm math + struct DynamicCasterState + { + f32 lastSignal = 0.0f; + vec3 aabbMin = vec3(0.0f); + vec3 aabbMax = vec3(0.0f); + }; + robin_hood::unordered_map _dynamicCasterStates; + std::vector _dynamicCasterLiveIDs; // This frame's live set, feeds the mask sync + std::vector _dynamicCasterAABBs; // World (min, max) pairs of the live set, rebuilt each frame + f32 _dynamicCasterTime = 0.0f; + u32 _dynamicCastersDropped = 0; // Spilled to static this frame (cap or oversize), never dropped from both pools + u32 _dynamicCasterTransitionsIn = 0; // Running totals since map load, per-frame ticks are unreadable in the perf editor + u32 _dynamicCasterTransitionsOut = 0; + + // Spawned/despawned/re-modeled instances must invalidate the cached static shadow pages under + // them, ShadowRenderer drains this each frame + struct ShadowInvalidation + { + vec4 min; + vec4 max; + }; + moodycamel::ConcurrentQueue _shadowInvalidationQueue; + + // Per-view surviving dynamic-class instance counts of the SVSM dynamic fills, the instrument + // for diagnosing missing dynamic shadow draws + Renderer::BufferID _svsmDynamicDrawCountReadBackBuffer; + u32 _numSvsmDynamicInstances[Renderer::Settings::MAX_VIEWS] = { 0 }; + bool _numSvsmDynamicInstancesZeroed = false; // One-shot zero of the stats when the readback is gated off + + // Animated doodads (windmills, flags, swaying vegetation) share one uninstanced bone stream + // per model; the queue signals which models advanced their animation this frame so Update can + // promote their placements within svsmAnimatedCasterRange to per-instance dynamic signals. + // The full placement scan is expensive (large vegetation models have tens of thousands of + // placements), so each in-grace model caches its near-camera subset and rescans only when the + // camera moves past the hysteresis slack or the placement set changes + struct AnimatedModelState + { + f32 lastPushTime = 0.0f; // Accumulated seconds of the last bone push + vec3 scanCameraPos = vec3(0.0f); // Camera position the cached subset was scanned from + bool scanned = false; + u32 lastSeenInstanceEpoch = 0; // _instanceSetEpoch value the cached subset was scanned at + std::vector nearInstances; // Placements within leaveDist at scan time + }; + moodycamel::ConcurrentQueue _uninstancedAnimatedModelQueue; + robin_hood::unordered_map _animatedModelLastPushTime; // Keyed by modelID + + // Bumped on every instance add/remove/re-model (any model, any thread). The animated-caster + // scan caches compare against it so the steady-state tick never takes a manifest mutex; an + // epoch bump from an unrelated model costs the few in-grace models one spare rescan + std::atomic _instanceSetEpoch = 0; + // GPU-only workbuffers Renderer::BufferID _occluderArgumentBuffer; Renderer::BufferID _argumentBuffer; diff --git a/Source/Game-Lib/Game-Lib/Rendering/RenderResources.h b/Source/Game-Lib/Game-Lib/Rendering/RenderResources.h index e4253829..58044a9d 100644 --- a/Source/Game-Lib/Game-Lib/Rendering/RenderResources.h +++ b/Source/Game-Lib/Game-Lib/Rendering/RenderResources.h @@ -51,8 +51,6 @@ struct RenderResources Renderer::ImageID depthColorCopy; Renderer::DepthImageID skyboxDepth; - std::vector shadowDepthCascades; - Renderer::DepthImageID debugRendererDepth; Renderer::SemaphoreID sceneRenderedSemaphore; // This semaphore tells the present function when the scene is ready to be blitted and presented diff --git a/Source/Game-Lib/Game-Lib/Rendering/RenderUtils.cpp b/Source/Game-Lib/Game-Lib/Rendering/RenderUtils.cpp index ee634559..38e122b3 100644 --- a/Source/Game-Lib/Game-Lib/Rendering/RenderUtils.cpp +++ b/Source/Game-Lib/Game-Lib/Rendering/RenderUtils.cpp @@ -33,6 +33,20 @@ void RenderUtils::Init(Renderer::Renderer* renderer, GameRenderer* gameRenderer) } } +Renderer::TimeQueryID RenderUtils::SVSMGeometryProfiler::BeginStage(const char* stageName, u32 viewIndex) const +{ + Renderer::TimeQueryDesc timeQueryDesc; + timeQueryDesc.name = _namePrefix + " " + stageName + " v" + std::to_string(viewIndex); + Renderer::TimeQueryID timeQuery = _renderer->CreateTimeQuery(timeQueryDesc); + _commandList->BeginTimeQuery(timeQuery); + return timeQuery; +} + +void RenderUtils::SVSMGeometryProfiler::EndStage(Renderer::TimeQueryID timeQuery) const +{ + _commandList->EndTimeQuery(timeQuery); +} + void RenderUtils::Blit(Renderer::RenderGraphResources& graphResources, Renderer::CommandList& commandList, u32 frameIndex, const BlitParams& params) { commandList.PushMarker("Blit", Color::White); diff --git a/Source/Game-Lib/Game-Lib/Rendering/RenderUtils.h b/Source/Game-Lib/Game-Lib/Rendering/RenderUtils.h index 5d751004..046e2cbd 100644 --- a/Source/Game-Lib/Game-Lib/Rendering/RenderUtils.h +++ b/Source/Game-Lib/Game-Lib/Rendering/RenderUtils.h @@ -5,8 +5,11 @@ #include #include #include +#include #include +#include + namespace Renderer { class Renderer; @@ -143,6 +146,65 @@ class RenderUtils static void CopyDepthToColor(Renderer::RenderGraphResources& graphResources, Renderer::CommandList& commandList, u32 frameIndex, const CopyDepthToColorParams& params); static Renderer::ComputePipelineID GetCopyDepthToColorPipeline() { return _copyDepthToColorPipeline; } + + // svsmProfileGeometry: wraps a per-view SVSM fill/draw stage in a named GPU time query so it + // surfaces in the perf editor's render pass list. Off by default, queries around tiny + // dispatches serialize slightly + class SVSMGeometryProfiler + { + public: + SVSMGeometryProfiler(Renderer::Renderer* renderer, Renderer::CommandList& commandList, std::string namePrefix, bool enabled) + : _renderer(renderer) + , _commandList(&commandList) + , _namePrefix(std::move(namePrefix)) + , _enabled(enabled) + { + } + + template + void operator()(const char* stageName, u32 viewIndex, Work&& work) const + { + if (!_enabled) + { + work(); + return; + } + + Renderer::TimeQueryID timeQuery = BeginStage(stageName, viewIndex); + work(); + EndStage(timeQuery); + } + + private: + Renderer::TimeQueryID BeginStage(const char* stageName, u32 viewIndex) const; + void EndStage(Renderer::TimeQueryID timeQuery) const; + + Renderer::Renderer* _renderer = nullptr; + Renderer::CommandList* _commandList = nullptr; + std::string _namePrefix; + bool _enabled = false; + }; + + // One draw per SVSM clip rect (new X columns, new Y rows, other): the vertex shader clips + // fragments to the rect, so an L-shaped toroidal update rasterizes two thin stripes instead + // of its whole-window bounding box. svsmClipRects 0: single draw, the rect index stays at the + // disabled sentinel + template + static void DrawSVSMClipRects(bool clipRectsEnabled, DrawParamsT& drawParams, DrawFn&& draw) + { + if (clipRectsEnabled) + { + for (u32 rectIndex = 0; rectIndex < 3; rectIndex++) + { + drawParams.svsmRectIndex = rectIndex; + draw(drawParams); + } + } + else + { + draw(drawParams); + } + } private: static Renderer::Renderer* _renderer; static GameRenderer* _gameRenderer; diff --git a/Source/Game-Lib/Game-Lib/Rendering/Shadow/ShadowRenderer.cpp b/Source/Game-Lib/Game-Lib/Rendering/Shadow/ShadowRenderer.cpp index fe13b5ac..9fa35bb8 100644 --- a/Source/Game-Lib/Game-Lib/Rendering/Shadow/ShadowRenderer.cpp +++ b/Source/Game-Lib/Game-Lib/Rendering/Shadow/ShadowRenderer.cpp @@ -1,43 +1,229 @@ #include "ShadowRenderer.h" +#include +#include #include -#include +#include +#include +#include + +#include +#include #include #include #include #include +#include + +#include #include #include #include -#include -#include #include -AutoCVar_Int CVAR_ShadowEnabled(CVarCategory::Client | CVarCategory::Rendering, "shadowEnabled", "enable shadows", 0, CVarFlags::EditCheckbox); +#include +#include + +AutoCVar_Int CVAR_ShadowEnabled(CVarCategory::Client | CVarCategory::Rendering, "shadowEnabled", "enable shadows", 1, CVarFlags::EditCheckbox); +AutoCVar_Float CVAR_ShadowStrength(CVarCategory::Client | CVarCategory::Rendering, "shadowStrength", "directional shadow strength, overwritten each frame from the sun elevation", 1.0f, CVarFlags::EditReadOnly); +AutoCVar_Float CVAR_ShadowNormalOffsetBias(CVarCategory::Client | CVarCategory::Rendering, "shadowNormalOffsetBias", "receiver offset along the surface normal in shadow texels, fights acne on hard angles", 1.0f); +AutoCVar_Float CVAR_ShadowCasterMargin(CVarCategory::Client | CVarCategory::Rendering, "shadowCasterMargin", "extends clipmap culling toward the sun so far-away casters with long shadows are not culled, depth clamp pancakes them onto the near plane", 2500.0f); +AutoCVar_Int CVAR_SVSMNumClipmaps(CVarCategory::Client | CVarCategory::Rendering, "svsmNumClipmaps", "number of SVSM clipmap rings, each ring doubles the covered area", 6); +AutoCVar_Float CVAR_SVSMClipmap0Extent(CVarCategory::Client | CVarCategory::Rendering, "svsmClipmap0Extent", "world extent of the finest SVSM clipmap window in meters, finer rings multiply the near-field page demand", 64.0f); +AutoCVar_Int CVAR_SVSMVirtualSize(CVarCategory::Client | CVarCategory::Rendering, "svsmVirtualSize", "virtual texture resolution per clipmap", 8192); +AutoCVar_Int CVAR_SVSMPageSize(CVarCategory::Client | CVarCategory::Rendering, "svsmPageSize", "texels per page", 128); +AutoCVar_Int CVAR_SVSMPoolSize(CVarCategory::Client | CVarCategory::Rendering, "svsmPoolSize", "physical page pool texture resolution, restart-only once shadows have been enabled", 8192); +AutoCVar_Int CVAR_SVSMPageEvictAge(CVarCategory::Client | CVarCategory::Rendering, "svsmPageEvictAge", "frames without a visible sample before a cached page returns to the pool, the age counter caps at 254", 240); +AutoCVar_Float CVAR_SVSMMarkBorderTexels(CVarCategory::Client | CVarCategory::Rendering, "svsmMarkBorderTexels", "filter footprint margin in texels, samples near a page border also mark the neighbor page", 4.0f); +AutoCVar_Float CVAR_SVSMResolutionScale(CVarCategory::Client | CVarCategory::Rendering, "svsmResolutionScale", "eye-distance clipmap floor: skip rings finer than the sample's screen footprint times this, 0 disables, lower = sharper distant shadows for more pages. Below ~0.25 the pool pressure-evicts and churns", 1.0f); +AutoCVar_Int CVAR_SVSMDynamicSplit(CVarCategory::Client | CVarCategory::Rendering, "svsmDynamicSplit", "static/dynamic caster split: animated and moving casters render into a transient dynamic pool instead of churning the static cache, 0 reverts to v1 behavior", 1, CVarFlags::EditCheckbox); +AutoCVar_Int CVAR_SVSMDynamicPoolSize(CVarCategory::Client | CVarCategory::Rendering, "svsmDynamicPoolSize", "dynamic page pool texture resolution, restart-only once shadows have been enabled", 2048); +AutoCVar_Int CVAR_SVSMRenderBudget(CVarCategory::Client | CVarCategory::Rendering, "svsmRenderBudget", "static pages rendered per frame, 0 = unlimited; overflow refines over following frames coarse-to-fine, the coarsest two rings are exempt", 0); +AutoCVar_Float CVAR_SVSMAnimatedCasterRange(CVarCategory::Client | CVarCategory::Rendering, "svsmAnimatedCasterRange", "camera range in meters within which animated doodads (windmills, flags) cast dynamic shadows, beyond it their pose bakes static, 0 disables", 128.0f); +AutoCVar_Int CVAR_SVSMFreeze(CVarCategory::Client | CVarCategory::Rendering, "svsmFreeze", "freeze SVSM page marking and lifecycle to inspect the cached state. Stale dirty state stays live and pages never clear, so dynamic content ghost-accumulates while frozen; unfreezing re-bakes the cache if anything spawned/despawned meanwhile", 0, CVarFlags::EditCheckbox); +AutoCVar_Int CVAR_SVSMInvalidateAll(CVarCategory::Client | CVarCategory::Rendering, "svsmInvalidateAll", "one shot, invalidate every cached SVSM page", 0, CVarFlags::EditCheckbox); +AutoCVar_Int CVAR_SVSMValidateDynamic(CVarCategory::Client | CVarCategory::Rendering, "svsmValidateDynamic", "one shot, read back the dynamic page table and free list and validate pool invariants (aliasing, leaks)", 0, CVarFlags::EditCheckbox); +AutoCVar_Int CVAR_SVSMDebugClipmap(CVarCategory::Client | CVarCategory::Rendering, "svsmDebugClipmap", "draw this clipmap's page table as an overlay, -1 disables", -1); +AutoCVar_Int CVAR_SVSMDebugShowPool(CVarCategory::Client | CVarCategory::Rendering, "svsmDebugShowPool", "draw a downsampled view of a physical page pool as an overlay: 1 = static pool, 2 = dynamic pool", 0); +AutoCVar_Float CVAR_SVSMZHalfRange(CVarCategory::Client | CVarCategory::Rendering, "svsmZHalfRange", "half depth range of the clipmap windows around the camera in light space, changes invalidate all pages", 2048.0f); +AutoCVar_Float CVAR_SVSMConstantBias(CVarCategory::Client | CVarCategory::Rendering, "svsmConstantBias", "SVSM compare bias toward the sun in world meters, the software depth path has no hardware bias", 0.15f); +AutoCVar_Int CVAR_SVSMProfileGeometry(CVarCategory::Client | CVarCategory::Rendering, "svsmProfileGeometry", "debug: per-view fill/draw GPU time queries in the SVSM geometry passes, shown in the render pass list", 0, CVarFlags::EditCheckbox); +AutoCVar_Int CVAR_SVSMClipRects(CVarCategory::Client | CVarCategory::Rendering, "svsmClipRects", "clip the static page draws to the classified dirty rects (3 draws per view), 0 reverts to one unclipped draw for A/B", 1, CVarFlags::EditCheckbox); +AutoCVar_Int CVAR_SVSMNightGate(CVarCategory::Client | CVarCategory::Rendering, "svsmNightGate", "skip all SVSM update/render work while the sun is below the horizon (shadow strength 0), dawn resumes with a full re-bake", 1, CVarFlags::EditCheckbox); + +// CPU mirror of SVSMData in Shadows/SVSM.inc.slang (scalar arrays only, so std430 matches this +// exactly). Never read as a struct: it exists so the readback offsets below are compiler-derived +// and the size tripwire fires when the two sides drift instead of the readback silently +// misreporting. Insert/rename fields here in lockstep with the Slang struct +struct SVSMDataMirror +{ + f32 prevLightDirection[4]; + + f32 zRangeMin; + f32 zRangeMax; + u32 frameInvalidationFlags; + u32 padding0; + + i32 anchorPageMinX[8]; + i32 anchorPageMinY[8]; + i32 prevAnchorPageMinX[8]; + i32 prevAnchorPageMinY[8]; + f32 camMinusWindowMinX[8]; + f32 camMinusWindowMinY[8]; + f32 extent[8]; + f32 pageWorld[8]; + u32 clipmapInvalidate[8]; + + i32 dirtyRectMinX[8]; + i32 dirtyRectMinY[8]; + i32 dirtyRectMaxX[8]; + i32 dirtyRectMaxY[8]; + + u32 statsMarked[8]; + u32 statsResident[8]; + u32 statsDirty[8]; + u32 statsEvicted[8]; + u32 statsInvalidated[8]; + u32 statsOverflow; + u32 statsInvalidationCause; + u32 statsFreeListCount; + + u32 configPageTableSize; + u32 configPageSize; + u32 configPoolPagesPerRow; + u32 configNumClipmaps; + f32 configMarkBorderTexels; + f32 configPadding0; + f32 configCamMinusZAnchor; + f32 configResolutionScale; + u32 padding3; + + i32 dynamicRectMinX[8]; + i32 dynamicRectMinY[8]; + i32 dynamicRectMaxX[8]; + i32 dynamicRectMaxY[8]; + + u32 statsDynamicLive[8]; + u32 statsDynamicOverflow; + u32 statsDynamicTotal; + u32 configDynamicPoolPagesPerRow; + u32 padding4; + + u32 statsDeferred[8]; + u32 statsBudgetUsed; + u32 padding5; + u32 padding6; + u32 padding7; + + i32 clipRectMinX[24]; + i32 clipRectMinY[24]; + i32 clipRectMaxX[24]; + i32 clipRectMaxY[24]; +}; +static_assert(sizeof(SVSMDataMirror) == ShadowRenderer::SVSM_DATA_UINT_COUNT * sizeof(u32), "SVSMDataMirror drifted from SVSMData in Shadows/SVSM.inc.slang, update both sides and the uint count together"); + +// u32 indices into the flat SVSMData readback, derived from the mirror so they cannot drift +// from the layout independently +namespace SVSMDataOffsets +{ + constexpr u32 Extent = offsetof(SVSMDataMirror, extent) / sizeof(u32); + constexpr u32 StatsMarked = offsetof(SVSMDataMirror, statsMarked) / sizeof(u32); + constexpr u32 StatsResident = offsetof(SVSMDataMirror, statsResident) / sizeof(u32); + constexpr u32 StatsDirty = offsetof(SVSMDataMirror, statsDirty) / sizeof(u32); + constexpr u32 StatsEvicted = offsetof(SVSMDataMirror, statsEvicted) / sizeof(u32); + constexpr u32 StatsInvalidated = offsetof(SVSMDataMirror, statsInvalidated) / sizeof(u32); + constexpr u32 StatsOverflow = offsetof(SVSMDataMirror, statsOverflow) / sizeof(u32); + constexpr u32 StatsInvalidationCause = offsetof(SVSMDataMirror, statsInvalidationCause) / sizeof(u32); + constexpr u32 StatsFreeListCount = offsetof(SVSMDataMirror, statsFreeListCount) / sizeof(u32); + constexpr u32 StatsDynamicLive = offsetof(SVSMDataMirror, statsDynamicLive) / sizeof(u32); + constexpr u32 StatsDynamicOverflow = offsetof(SVSMDataMirror, statsDynamicOverflow) / sizeof(u32); + constexpr u32 StatsDynamicTotal = offsetof(SVSMDataMirror, statsDynamicTotal) / sizeof(u32); + constexpr u32 StatsDeferred = offsetof(SVSMDataMirror, statsDeferred) / sizeof(u32); + constexpr u32 StatsBudgetUsed = offsetof(SVSMDataMirror, statsBudgetUsed) / sizeof(u32); +} + +// Page table entry bits, mirrors Shadows/SVSM.inc.slang +namespace SVSMPageEntry +{ + constexpr u32 PhysMask = 0xFFFu; // SVSM_PAGE_PHYS_MASK + constexpr u32 Resident = 1u << 25; // SVSM_PAGE_RESIDENT +} + +// SVSM invalidation cause bits, shared with Shadows/SVSM.inc.slang +namespace SVSMCause +{ + constexpr u32 Manual = 4; + constexpr u32 AABBOverflow = 8; +} -AutoCVar_Int CVAR_ShadowDebugMatrices(CVarCategory::Client | CVarCategory::Rendering, "shadowDebugMatrices", "debug shadow matrices by applying them to the camera", 0, CVarFlags::EditCheckbox); -AutoCVar_Int CVAR_ShadowDebugMatrixIndex(CVarCategory::Client | CVarCategory::Rendering, "shadowDebugMatricesIndex", "index of the cascade to debug", 0); +// pageTableSize must be a power of two in [16, maxPageTableSize] (WrapPageSlot masks with +// pageTableSize - 1), and every consumer — the geometry passes' viewport extents included — +// derives it from these cvars, so invalid values are corrected at the source. Returns true when +// anything was corrected: the pre-snap values were live for a few frames and mis-addressed the +// cached pages, so the caller must invalidate the cache +static bool SanitizeSVSMConfigCVars(u32 maxPageTableSize) +{ + const u32 pageSize = static_cast(glm::max(CVAR_SVSMPageSize.Get(), 16)); + const u32 virtualSize = static_cast(glm::max(CVAR_SVSMVirtualSize.Get(), 1)); -AutoCVar_Int CVAR_ShadowDrawMatrices(CVarCategory::Client | CVarCategory::Rendering, "shadowDrawMatrices", "debug shadow matrices by debug drawing them", 0, CVarFlags::EditCheckbox); + const u32 pageTableSize = glm::clamp(std::bit_floor(virtualSize / pageSize), 16u, maxPageTableSize); + const u32 correctedVirtualSize = pageTableSize * pageSize; -AutoCVar_Int CVAR_ShadowFilterMode(CVarCategory::Client | CVarCategory::Rendering, "shadowFilterMode", "0: No filtering, 1: Percentage Closer Filtering, 2: Percentage Closer Soft Shadows", 1); -AutoCVar_Float CVAR_ShadowFilterSize(CVarCategory::Client | CVarCategory::Rendering, "shadowFilterSize", "size of the filter used for shadow sampling", 3.0f); -AutoCVar_Float CVAR_ShadowFilterPenumbraSize(CVarCategory::Client | CVarCategory::Rendering, "shadowFilterPenumbraSize", "size of the filter used for penumbra sampling", 3.0f); + bool corrected = false; + if (correctedVirtualSize != static_cast(CVAR_SVSMVirtualSize.Get())) + { + NC_LOG_WARNING("SVSM: svsmVirtualSize {0} is not a power-of-two multiple of the page size in [16, {1}] pages, corrected to {2}", CVAR_SVSMVirtualSize.Get(), maxPageTableSize, correctedVirtualSize); + CVAR_SVSMVirtualSize.Set(static_cast(correctedVirtualSize)); + corrected = true; + } + if (pageSize != static_cast(CVAR_SVSMPageSize.Get())) + { + NC_LOG_WARNING("SVSM: svsmPageSize {0} is below the 16 texel minimum, corrected to {1}", CVAR_SVSMPageSize.Get(), pageSize); + CVAR_SVSMPageSize.Set(static_cast(pageSize)); + corrected = true; + } + return corrected; +} -AutoCVar_Float CVAR_ShadowDepthBiasConstantFactor(CVarCategory::Client | CVarCategory::Rendering, "shadowDepthBiasConstant", "constant factor of depth bias to prevent shadow acne", -2.0f); -AutoCVar_Float CVAR_ShadowDepthBiasClamp(CVarCategory::Client | CVarCategory::Rendering, "shadowDepthBiasClamp", "clamp of depth bias to prevent shadow acne", 0.0f); -AutoCVar_Float CVAR_ShadowDepthBiasSlopeFactor(CVarCategory::Client | CVarCategory::Rendering, "shadowDepthBiasSlope", "slope factor of depth bias to prevent shadow acne", -5.0f); +// The derived page/pool geometry every consumer must agree on: the update pass constants, the +// debug overlay and the pool state reset all read this instead of re-deriving it from the cvars +struct SVSMDerivedConfig +{ + u32 pageSize = 0; + u32 pageTableSize = 0; + u32 poolPagesPerRow = 0; + u32 dynamicPoolPagesPerRow = 0; +}; -#define TIMESLICED_CASCADES 0 +static SVSMDerivedConfig DeriveSVSMConfig(u32 maxPageTableSize) +{ + SVSMDerivedConfig config; + config.pageSize = static_cast(glm::max(CVAR_SVSMPageSize.Get(), 16)); + config.pageTableSize = glm::clamp(static_cast(CVAR_SVSMVirtualSize.Get()) / config.pageSize, 16u, maxPageTableSize); + config.poolPagesPerRow = glm::min(static_cast(CVAR_SVSMPoolSize.Get()) / config.pageSize, maxPageTableSize); + config.dynamicPoolPagesPerRow = glm::min(static_cast(CVAR_SVSMDynamicPoolSize.Get()) / config.pageSize, maxPageTableSize); + return config; +} -ShadowRenderer::ShadowRenderer(Renderer::Renderer* renderer, GameRenderer* gameRenderer, DebugRenderer* debugRenderer, TerrainRenderer* terrainRenderer, ModelRenderer* modelRenderer, RenderResources& resources) +ShadowRenderer::ShadowRenderer(Renderer::Renderer* renderer, GameRenderer* gameRenderer, TerrainRenderer* terrainRenderer, ModelRenderer* modelRenderer, RenderResources& resources) : _renderer(renderer) , _gameRenderer(gameRenderer) - , _debugRenderer(debugRenderer) , _terrainRenderer(terrainRenderer) , _modelRenderer(modelRenderer) + , _svsmPrepareDescriptorSet(Renderer::DescriptorSetSlot::PER_PASS) + , _svsmInvalidateAABBsDescriptorSet(Renderer::DescriptorSetSlot::PER_PASS) + , _svsmPageUpdateADescriptorSet(Renderer::DescriptorSetSlot::PER_PASS) + , _svsmPageMarkDescriptorSet(Renderer::DescriptorSetSlot::PER_PASS) + , _svsmPageUpdateBDescriptorSet(Renderer::DescriptorSetSlot::PER_PASS) + , _svsmDynamicMarkDescriptorSet(Renderer::DescriptorSetSlot::PER_PASS) + , _svsmDynamicUpdateDescriptorSet(Renderer::DescriptorSetSlot::PER_PASS) + , _svsmFinalizeDescriptorSet(Renderer::DescriptorSetSlot::PER_PASS) + , _svsmPageClearDescriptorSet(Renderer::DescriptorSetSlot::PER_PASS) + , _svsmDynamicPageClearDescriptorSet(Renderer::DescriptorSetSlot::PER_PASS) + , _svsmPageTableDebugDescriptorSet(Renderer::DescriptorSetSlot::PER_PASS) + , _svsmPoolDebugDescriptorSet(Renderer::DescriptorSetSlot::PER_PASS) { ZoneScoped; CreatePermanentResources(resources); @@ -51,110 +237,1144 @@ void ShadowRenderer::Update(f32 deltaTime, RenderResources& resources) { ZoneScoped; - CVarSystem* cvarSystem = CVarSystem::Get(); - const u32 numCascades = static_cast(*cvarSystem->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "shadowCascadeNum")); + if (SanitizeSVSMConfigCVars(SVSM_MAX_PAGE_TABLE_SIZE)) + { + _svsmForceInvalidateAll = true; // Pages cached under the pre-snap addressing are garbage + } - const bool debugMatrices = CVAR_ShadowDebugMatrices.Get(); - const i32 debugMatrixIndex = CVAR_ShadowDebugMatrixIndex.Get(); - if (debugMatrices && debugMatrixIndex >= 0 && debugMatrixIndex < static_cast(numCascades)) + // Live config edits: a pageSize change reshapes the pool page counts and free lists in place + // (the pool textures keep their dimensions, only the page grid over them changes). The pool + // size cvars are restart-only once the pool images exist — the Engine cannot destroy images, + // so recreating them at new dimensions would leak the old texture — and revert with a warning { - const Camera& debugCascadeCamera = resources.cameras[debugMatrixIndex + 1]; // +1 because the first camera is the main camera + if (_svsmPagePool != Renderer::ImageID::Invalid() && + (static_cast(CVAR_SVSMPoolSize.Get()) != _svsmAppliedPoolSize || static_cast(CVAR_SVSMDynamicPoolSize.Get()) != _svsmAppliedDynamicPoolSize)) + { + NC_LOG_WARNING("SVSM: svsmPoolSize/svsmDynamicPoolSize changes need a restart once the pools exist, reverting to {0}/{1}", _svsmAppliedPoolSize, _svsmAppliedDynamicPoolSize); + CVAR_SVSMPoolSize.Set(static_cast(_svsmAppliedPoolSize)); + CVAR_SVSMDynamicPoolSize.Set(static_cast(_svsmAppliedDynamicPoolSize)); + } + + const bool configChanged = static_cast(glm::max(CVAR_SVSMPageSize.Get(), 16)) != _svsmAppliedPageSize + || static_cast(CVAR_SVSMPoolSize.Get()) != _svsmAppliedPoolSize + || static_cast(CVAR_SVSMDynamicPoolSize.Get()) != _svsmAppliedDynamicPoolSize; + if (configChanged) + { + ResetSVSMPoolState(resources); + _svsmForceInvalidateAll = true; + if (_svsmPagePool != Renderer::ImageID::Invalid()) + { + _svsmPoolNeedsClear = true; // The old page grid's depth is garbage under the new one + } + } + } - Camera& mainCamera = resources.cameras[0]; + // Caster-toggle transitions re-bake the whole static cache: resident pages hold the toggled + // class's baked depth, and marked pages never age out while visible, so without this the old + // shadows persist indefinitely after a toggle + { + CVarSystem* cvarSystem = CVarSystem::Get(); + const bool modelsCastShadow = *cvarSystem->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "shadowModelsCastShadow"_h) == 1; + const bool terrainCastShadow = *cvarSystem->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "shadowTerrainCastShadow"_h) == 1; + if (modelsCastShadow != _svsmModelsCastShadow || terrainCastShadow != _svsmTerrainCastShadow) + { + _svsmModelsCastShadow = modelsCastShadow; + _svsmTerrainCastShadow = terrainCastShadow; + _svsmForceInvalidateAll = true; + } + } - mainCamera = debugCascadeCamera; - resources.cameras.SetDirtyElement(0); + // Night gate: below the horizon the material pass multiplies shadows to nothing + // (shadowStrength 0, and lightInfo.y already stops the sampler), so the whole producer side + // can sleep. Enter after a sustained second so the dusk threshold can't flicker the cache; + // leave immediately with a full re-bake — the sun moved all night, the cache is garbage at + // dawn regardless + { + if (CVAR_SVSMNightGate.Get() != 0 && CVAR_ShadowStrength.GetFloat() <= 0.0f) + { + _svsmNightTimer += deltaTime; + if (_svsmNightTimer > 1.0f) + { + _svsmNightActive = true; + } + } + else + { + if (_svsmNightActive) + { + _svsmForceInvalidateAll = true; + } + _svsmNightActive = false; + _svsmNightTimer = 0.0f; + } } - if (CVAR_ShadowDrawMatrices.Get()) + // SVSM: last frame's page table stats plus this frame's caster bounds. The ModelRenderer's + // per-instance classifier owns the dynamic set (moved or bone-pushed within the grace window) + // and its transition invalidations; this just copies the results and uploads. With the split + // off, the classifier feeds everything through static invalidation like v1 + _svsmDirtyAABBs.clear(); + _svsmDynamicAABBs.clear(); + _svsmDirtyAABBOverflow = false; + _svsmNumDynamicCasters = 0; + _svsmCasterTransitionsIn = 0; + _svsmCasterTransitionsOut = 0; + _svsmDynamicAABBsDropped = 0; + const bool frozen = CVAR_SVSMFreeze.Get() != 0; + if (CVAR_ShadowEnabled.Get() && !frozen && !_svsmNightActive) { - Color colors[] = + // The pools are the SVSM VRAM cost, only allocated once shadows are actually used + if (_svsmPagePool == Renderer::ImageID::Invalid()) + { + Renderer::ImageDesc poolDesc; + poolDesc.debugName = "SVSMPagePool"; + poolDesc.dimensions = vec2(CVAR_SVSMPoolSize.Get(), CVAR_SVSMPoolSize.Get()); + poolDesc.dimensionType = Renderer::ImageDimensionType::DIMENSION_ABSOLUTE; + poolDesc.format = Renderer::ImageFormat::R32_UINT; + poolDesc.sampleCount = Renderer::SampleCount::SAMPLE_COUNT_1; + poolDesc.clearUInts = uvec4(0, 0, 0, 0); + + _svsmPagePool = _renderer->CreateImage(poolDesc); + + poolDesc.debugName = "SVSMDynamicPagePool"; + poolDesc.dimensions = vec2(CVAR_SVSMDynamicPoolSize.Get(), CVAR_SVSMDynamicPoolSize.Get()); + _svsmDynamicPagePool = _renderer->CreateImage(poolDesc); + + _svsmPoolNeedsClear = true; // Fresh VRAM is garbage that must never be sampled, zero both once + } + + // Diagnostics only (perf editor stats), rendering decisions never read this — it is a + // frame old, the GPU gates its own per-view work through Finalize's fill dispatch args. + // Single-buffered across frames in flight: this map can race the in-flight GPU copy, so + // values may tear (house readback pattern, accepted — stats can read inconsistent) + u32* svsmData = static_cast(_renderer->MapBuffer(_svsmDataReadBackBuffer)); + if (svsmData != nullptr) { - Color::Red, - Color::Green, - Color::Blue, - Color::Yellow, - Color::Magenta, - Color::Cyan, - Color::PastelOrange, - Color::PastelGreen - }; + memcpy(_svsmDataReadBack, svsmData, sizeof(u32) * SVSM_DATA_UINT_COUNT); + } + _renderer->UnmapBuffer(_svsmDataReadBackBuffer); + + // svsmValidateDynamic one-shot: check the snapshotted dynamic pool invariants — every + // physical page referenced by at most one resident entry, free-list entries unique and + // unreferenced, resident + free == pool. Aliasing here is the ghost-shadow failure mode + if (_svsmValidatePending) + { + const u32 tableUints = SVSM_MAX_CLIPMAPS * SVSM_MAX_PAGE_TABLE_SIZE * SVSM_MAX_PAGE_TABLE_SIZE; + const u32* validateData = static_cast(_renderer->MapBuffer(_svsmDynamicValidateReadBackBuffer)); + if (validateData != nullptr) + { + const u32* freeListData = validateData + tableUints; + const u32 poolPages = _svsmDynamicPoolPages; + const u32 rawFreeCount = freeListData[0]; + const u32 freeCount = glm::min(rawFreeCount, SVSM_MAX_POOL_PAGES); + + std::vector physRefCounts(poolPages, 0); + u32 numResident = 0; + u32 numAliasedRefs = 0; + u32 numBadPhys = 0; + for (u32 i = 0; i < tableUints; i++) + { + u32 entry = validateData[i]; + if (entry == 0 || (entry & SVSMPageEntry::Resident) == 0) + continue; + + numResident++; + u32 physicalPage = entry & SVSMPageEntry::PhysMask; + if (physicalPage >= poolPages) + { + numBadPhys++; + continue; + } + if (++physRefCounts[physicalPage] > 1) + { + numAliasedRefs++; + } + } + + std::vector freeSeen(poolPages, 0); + u32 numBadFree = 0; + u32 numFreeDuplicates = 0; + u32 numFreeButReferenced = 0; + for (u32 i = 0; i < freeCount; i++) + { + u32 physicalPage = freeListData[4 + i]; + if (physicalPage >= poolPages) + { + numBadFree++; + continue; + } + if (freeSeen[physicalPage]++ != 0) + { + numFreeDuplicates++; + } + else if (physRefCounts[physicalPage] != 0) + { + numFreeButReferenced++; + } + } + + bool leaked = numResident + rawFreeCount != poolPages; + if (numAliasedRefs != 0 || numBadPhys != 0 || numBadFree != 0 || numFreeDuplicates != 0 || numFreeButReferenced != 0 || leaked) + { + NC_LOG_ERROR("SVSM dynamic pool validation FAILED: {0} resident + {1} free of {2} pages | {3} aliased refs, {4} bad phys, {5} free dups, {6} free-but-referenced, {7} bad free entries", numResident, rawFreeCount, poolPages, numAliasedRefs, numBadPhys, numFreeDuplicates, numFreeButReferenced, numBadFree); + } + else + { + NC_LOG_INFO("SVSM dynamic pool validation OK: {0} resident + {1} free = {2} pages", numResident, rawFreeCount, poolPages); + } + } + _renderer->UnmapBuffer(_svsmDynamicValidateReadBackBuffer); + _svsmValidatePending = false; + CVAR_SVSMValidateDynamic.Set(0); + } - for (u32 i = 0; i < numCascades; i++) + // Spawned/despawned/re-modeled instances and the classifier's transition/spill + // invalidations always re-bake cached static pages + if (_modelRenderer->DrainShadowInvalidations(_svsmDirtyAABBs, SVSM_MAX_DIRTY_AABBS) > SVSM_MAX_DIRTY_AABBS) { - const Camera& debugCascadeCamera = resources.cameras[i + 1]; // +1 because the first camera is the main camera - _debugRenderer->DrawFrustum(debugCascadeCamera.worldToClip, colors[i]); + _svsmDirtyAABBOverflow = true; + } + + // The dynamic caster set — moved or bone-pushed within the grace window — is classified + // per instance by the ModelRenderer (one classifier feeds both this list and the GPU + // instance mask, so they can never disagree). Capped and oversize-spilled at the source, + // the copy just has to fit + const std::vector& dynamicAABBs = _modelRenderer->GetDynamicCasterAABBs(); + NC_ASSERT(dynamicAABBs.size() <= SVSM_MAX_DYNAMIC_AABBS * 2, "SVSM: ModelRenderer emitted more dynamic caster AABBs than the buffer holds, the source cap is broken"); + _svsmDynamicAABBs.assign(dynamicAABBs.begin(), dynamicAABBs.end()); + + _svsmNumDynamicCasters = _modelRenderer->GetNumDynamicCasters(); + _svsmDynamicAABBsDropped = _modelRenderer->GetNumDynamicCastersDropped(); + _modelRenderer->GetDynamicCasterTransitions(_svsmCasterTransitionsIn, _svsmCasterTransitionsOut); + + // Upload this frame's AABB lists through the frame-synced staging ring, the render graph + // issues a global upload barrier before any pass executes + if (!_svsmDirtyAABBs.empty()) + { + size_t uploadBytes = sizeof(vec4) * _svsmDirtyAABBs.size(); + auto uploadBuffer = _renderer->CreateUploadBuffer(_svsmDirtyAABBBuffer, 0, uploadBytes); + memcpy(uploadBuffer->mappedMemory, _svsmDirtyAABBs.data(), uploadBytes); + } + if (!_svsmDynamicAABBs.empty()) + { + size_t uploadBytes = sizeof(vec4) * _svsmDynamicAABBs.size(); + auto uploadBuffer = _renderer->CreateUploadBuffer(_svsmDynamicAABBBuffer, 0, uploadBytes); + memcpy(uploadBuffer->mappedMemory, _svsmDynamicAABBs.data(), uploadBytes); + } + } + else + { + // Shadows off, frozen or night-gated (the update pass doesn't run, so nothing would + // consume the queue): spawn/despawn and classifier-transition invalidations would + // accumulate unboundedly. Discard them (maxPairs 0 appends nothing) and re-bake the whole + // cache on resume instead — the classifier keeps ticking meanwhile, so its state stays + // current + if (_modelRenderer->DrainShadowInvalidations(_svsmDirtyAABBs, 0) > 0) + { + _svsmForceInvalidateAll = true; } } } -void ShadowRenderer::AddShadowPass(Renderer::RenderGraph* renderGraph, RenderResources& resources, u8 frameIndex) +bool ShadowRenderer::IsSVSMActive() const { - struct ShadowPassData + return CVAR_ShadowEnabled.Get() != 0 && !_svsmNightActive; +} + +struct ShadowRenderer::SVSMUpdatePassData +{ + Renderer::DepthImageResource depth; + + Renderer::BufferMutableResource cameras; + Renderer::BufferMutableResource svsmDataBuffer; + Renderer::BufferMutableResource pageTableBuffer; + Renderer::BufferMutableResource freeListBuffer; + Renderer::BufferResource dirtyAABBBuffer; + Renderer::BufferMutableResource clearListBuffer; + Renderer::BufferMutableResource dynamicPageTableBuffer; + Renderer::BufferMutableResource dynamicFreeListBuffer; + Renderer::BufferMutableResource dynamicClearListBuffer; + Renderer::BufferResource dynamicAABBBuffer; + Renderer::BufferMutableResource svsmDataReadBackBuffer; + Renderer::BufferMutableResource dynamicValidateReadBackBuffer; + Renderer::ImageMutableResource pagePool; + Renderer::ImageMutableResource dynamicPagePool; + + Renderer::DescriptorSetResource prepareSet; + Renderer::DescriptorSetResource invalidateSet; + Renderer::DescriptorSetResource updateASet; + Renderer::DescriptorSetResource markSet; + Renderer::DescriptorSetResource updateBSet; + Renderer::DescriptorSetResource dynamicMarkSet; + Renderer::DescriptorSetResource dynamicUpdateSet; + Renderer::DescriptorSetResource finalizeSet; + Renderer::DescriptorSetResource clearSet; + Renderer::DescriptorSetResource dynamicClearSet; + + bool DeclareResources(ShadowRenderer& owner, RenderResources& resources, Renderer::RenderGraphBuilder& builder) { - Renderer::DepthImageMutableResource shadowDepthCascades[Renderer::Settings::MAX_SHADOW_CASCADES]; + using BufferUsage = Renderer::BufferPassUsage; - Renderer::DescriptorSetResource lightDescriptorSet; + depth = builder.Read(resources.depth, Renderer::PipelineType::COMPUTE); + cameras = builder.Write(resources.cameras.GetBuffer(), BufferUsage::COMPUTE); + svsmDataBuffer = builder.Write(owner._svsmDataBuffer, BufferUsage::COMPUTE | BufferUsage::TRANSFER); + pageTableBuffer = builder.Write(owner._svsmPageTableBuffer, BufferUsage::COMPUTE); + freeListBuffer = builder.Write(owner._svsmFreeListBuffer, BufferUsage::COMPUTE); + dirtyAABBBuffer = builder.Read(owner._svsmDirtyAABBBuffer, BufferUsage::COMPUTE); + clearListBuffer = builder.Write(owner._svsmClearListBuffer, BufferUsage::COMPUTE); + dynamicPageTableBuffer = builder.Write(owner._svsmDynamicPageTableBuffer, BufferUsage::COMPUTE | BufferUsage::TRANSFER); + dynamicFreeListBuffer = builder.Write(owner._svsmDynamicFreeListBuffer, BufferUsage::COMPUTE | BufferUsage::TRANSFER); + dynamicClearListBuffer = builder.Write(owner._svsmDynamicClearListBuffer, BufferUsage::COMPUTE); + dynamicAABBBuffer = builder.Read(owner._svsmDynamicAABBBuffer, BufferUsage::COMPUTE); + svsmDataReadBackBuffer = builder.Write(owner._svsmDataReadBackBuffer, BufferUsage::TRANSFER); + dynamicValidateReadBackBuffer = builder.Write(owner._svsmDynamicValidateReadBackBuffer, BufferUsage::TRANSFER); + builder.Write(owner._svsmFillArgsBuffer, BufferUsage::COMPUTE); + pagePool = builder.Write(owner._svsmPagePool, Renderer::PipelineType::COMPUTE, Renderer::LoadMode::LOAD); + dynamicPagePool = builder.Write(owner._svsmDynamicPagePool, Renderer::PipelineType::COMPUTE, Renderer::LoadMode::LOAD); + + prepareSet = builder.Use(owner._svsmPrepareDescriptorSet); + invalidateSet = builder.Use(owner._svsmInvalidateAABBsDescriptorSet); + updateASet = builder.Use(owner._svsmPageUpdateADescriptorSet); + markSet = builder.Use(owner._svsmPageMarkDescriptorSet); + updateBSet = builder.Use(owner._svsmPageUpdateBDescriptorSet); + dynamicMarkSet = builder.Use(owner._svsmDynamicMarkDescriptorSet); + dynamicUpdateSet = builder.Use(owner._svsmDynamicUpdateDescriptorSet); + finalizeSet = builder.Use(owner._svsmFinalizeDescriptorSet); + clearSet = builder.Use(owner._svsmPageClearDescriptorSet); + dynamicClearSet = builder.Use(owner._svsmDynamicPageClearDescriptorSet); + + return true; + } +}; + +struct ShadowRenderer::SVSMUpdateRecorder +{ + struct Constants + { + vec4 lightDirection; + u32 numClipmaps; + u32 pageTableSize; + u32 pageSize; + u32 poolPagesPerRow; + f32 clipmap0Extent; + f32 markBorderTexels; + u32 evictAge; + u32 invalidateAll; + u32 numDirtyAABBs; + u32 allocClipmap; + f32 casterMargin; + f32 zHalfRange; + u32 fillDrawCallCount; + f32 padding1; + f32 resolutionScale; + u32 dynamicPoolPagesPerRow; + u32 renderBudget; + u32 dynamicPhase; + u32 fillInstanceCount; + u32 fillCellCount; }; - CVarSystem* cvarSystem = CVarSystem::Get(); - u32 numCascades = static_cast(*cvarSystem->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "shadowCascadeNum")); - numCascades = std::min(numCascades, static_cast(resources.shadowDepthCascades.size())); + ShadowRenderer& owner; + SVSMUpdatePassData& data; + Renderer::RenderGraphResources& graphResources; + Renderer::CommandList& commandList; + u8 frameIndex; + u32 numClipmaps; + u32 numDirtyAABBs; + u32 numDynamicAABBs; + bool dynamicSplit; + SVSMDerivedConfig config; + u32 tableCapacity; + Constants* constants; + + SVSMUpdateRecorder(ShadowRenderer& owner, SVSMUpdatePassData& data, Renderer::RenderGraphResources& graphResources, Renderer::CommandList& commandList, + u8 frameIndex, u32 numClipmaps, const vec3& lightDirection, u32 invalidateCause, u32 numDirtyAABBs, bool dynamicSplit, u32 numDynamicAABBs); + + void Record(); + void ClearPhysicalPoolsIfNeeded(); + void PrepareFrameStateAndClipmapAnchors(); + void InvalidatePagesTouchedByChangedCasters(); + void RecycleStaleStaticPages(); + void MarkPagesNeededByVisibleReceivers(); + void AllocateAndQueueStaticPages(); + void RebuildDynamicPageSet(); + void MarkPagesTouchedByDynamicCasters(); + void BuildClipmapCamerasAndGeometryDispatches(); + void ClearPagesQueuedForRendering(); + void CaptureDiagnostics(); +}; + +ShadowRenderer::SVSMUpdateRecorder::SVSMUpdateRecorder(ShadowRenderer& owner, SVSMUpdatePassData& data, + Renderer::RenderGraphResources& graphResources, Renderer::CommandList& commandList, u8 frameIndex, u32 numClipmaps, + const vec3& lightDirection, u32 invalidateCause, u32 numDirtyAABBs, bool dynamicSplit, u32 numDynamicAABBs) + : owner(owner) + , data(data) + , graphResources(graphResources) + , commandList(commandList) + , frameIndex(frameIndex) + , numClipmaps(numClipmaps) + , numDirtyAABBs(numDirtyAABBs) + , numDynamicAABBs(numDynamicAABBs) + , dynamicSplit(dynamicSplit) + , config(DeriveSVSMConfig(SVSM_MAX_PAGE_TABLE_SIZE)) + , tableCapacity(SVSM_MAX_CLIPMAPS * SVSM_MAX_PAGE_TABLE_SIZE * SVSM_MAX_PAGE_TABLE_SIZE) + , constants(graphResources.FrameNew()) +{ + constants->lightDirection = vec4(lightDirection, 0.0f); + constants->numClipmaps = numClipmaps; + constants->pageTableSize = config.pageTableSize; + constants->pageSize = config.pageSize; + constants->poolPagesPerRow = config.poolPagesPerRow; + constants->clipmap0Extent = CVAR_SVSMClipmap0Extent.GetFloat(); + constants->markBorderTexels = CVAR_SVSMMarkBorderTexels.GetFloat(); + constants->evictAge = static_cast(glm::clamp(CVAR_SVSMPageEvictAge.Get(), 1, 254)); + constants->invalidateAll = invalidateCause; + constants->numDirtyAABBs = numDirtyAABBs; + constants->allocClipmap = 0; + constants->casterMargin = CVAR_ShadowCasterMargin.GetFloat(); + constants->zHalfRange = CVAR_SVSMZHalfRange.GetFloat(); + constants->padding1 = 0.0f; + constants->resolutionScale = CVAR_SVSMResolutionScale.GetFloat(); + constants->dynamicPoolPagesPerRow = dynamicSplit ? config.dynamicPoolPagesPerRow : 0; + constants->renderBudget = static_cast(glm::max(CVAR_SVSMRenderBudget.Get(), 0)); + constants->dynamicPhase = 0; + + // Finalize turns these record-time counts into same-frame per-view indirect dispatch args. + constants->fillInstanceCount = owner._modelRenderer->GetOpaqueCullingResources().GetNumInstances(); + constants->fillCellCount = owner._terrainRenderer->GetNumDrawCalls(); + constants->fillDrawCallCount = owner._modelRenderer->GetOpaqueCullingResources().GetDrawCallCount(); +} + +void ShadowRenderer::SVSMUpdateRecorder::Record() +{ + GPU_SCOPED_PROFILER_ZONE(commandList, SVSMUpdate); + + ClearPhysicalPoolsIfNeeded(); + PrepareFrameStateAndClipmapAnchors(); + InvalidatePagesTouchedByChangedCasters(); + RecycleStaleStaticPages(); + MarkPagesNeededByVisibleReceivers(); + AllocateAndQueueStaticPages(); + + if (dynamicSplit) + { + RebuildDynamicPageSet(); + } + + BuildClipmapCamerasAndGeometryDispatches(); + ClearPagesQueuedForRendering(); + CaptureDiagnostics(); +} + +void ShadowRenderer::SVSMUpdateRecorder::ClearPhysicalPoolsIfNeeded() +{ + if (!owner._svsmPoolNeedsClear) + return; + + // Fresh VRAM must be zero before sampling or reversed-depth atomics can preserve garbage. + commandList.Clear(data.pagePool, uvec4(0, 0, 0, 0)); + commandList.ImageBarrier(data.pagePool); + commandList.Clear(data.dynamicPagePool, uvec4(0, 0, 0, 0)); + commandList.ImageBarrier(data.dynamicPagePool); + owner._svsmPoolNeedsClear = false; +} + +void ShadowRenderer::SVSMUpdateRecorder::PrepareFrameStateAndClipmapAnchors() +{ + // Detect global invalidation, snap clipmap windows and reset frame statistics/list headers. + commandList.BeginPipeline(owner._svsmPreparePipeline); + commandList.PushConstant(constants, 0, sizeof(Constants)); + commandList.BindDescriptorSet(data.prepareSet, frameIndex); + commandList.Dispatch(1, 1, 1); + commandList.EndPipeline(owner._svsmPreparePipeline); + + commandList.BufferBarrier(data.svsmDataBuffer, Renderer::BufferPassUsage::COMPUTE); + // Clear-list header resets must be visible to the later atomic appenders. Barriers are + // per-buffer, so publishing svsmData alone does not cover either list. + commandList.BufferBarrier(data.clearListBuffer, Renderer::BufferPassUsage::COMPUTE); + commandList.BufferBarrier(data.dynamicClearListBuffer, Renderer::BufferPassUsage::COMPUTE); +} + +void ShadowRenderer::SVSMUpdateRecorder::InvalidatePagesTouchedByChangedCasters() +{ + if (numDirtyAABBs == 0) + return; + + commandList.BeginPipeline(owner._svsmInvalidateAABBsPipeline); + commandList.PushConstant(constants, 0, sizeof(Constants)); + commandList.BindDescriptorSet(data.invalidateSet, frameIndex); + commandList.Dispatch((numDirtyAABBs + 63) / 64, numClipmaps, 1); + commandList.EndPipeline(owner._svsmInvalidateAABBsPipeline); + + commandList.BufferBarrier(data.pageTableBuffer, Renderer::BufferPassUsage::COMPUTE); + commandList.BufferBarrier(data.svsmDataBuffer, Renderer::BufferPassUsage::COMPUTE); +} + +void ShadowRenderer::SVSMUpdateRecorder::RecycleStaleStaticPages() +{ + // Apply toroidal/global invalidation, age entries and return stale physical pages. The full + // capacity is visited so entries orphaned by configuration changes also age out. + commandList.BeginPipeline(owner._svsmPageUpdateAPipeline); + commandList.PushConstant(constants, 0, sizeof(Constants)); + commandList.BindDescriptorSet(data.updateASet, frameIndex); + commandList.Dispatch(tableCapacity / 256, 1, 1); + commandList.EndPipeline(owner._svsmPageUpdateAPipeline); + + commandList.BufferBarrier(data.pageTableBuffer, Renderer::BufferPassUsage::COMPUTE); + commandList.BufferBarrier(data.freeListBuffer, Renderer::BufferPassUsage::COMPUTE); +} + +void ShadowRenderer::SVSMUpdateRecorder::MarkPagesNeededByVisibleReceivers() +{ + commandList.BeginPipeline(owner._svsmPageMarkPipeline); + commandList.PushConstant(constants, 0, sizeof(Constants)); + data.markSet.Bind("_depth"_h, data.depth); + commandList.BindDescriptorSet(data.markSet, frameIndex); + + const uvec2 depthDimensions = graphResources.GetImageDimensions(data.depth); + commandList.Dispatch((depthDimensions.x + 15) / 16, (depthDimensions.y + 15) / 16, 1); + commandList.EndPipeline(owner._svsmPageMarkPipeline); + + commandList.BufferBarrier(data.pageTableBuffer, Renderer::BufferPassUsage::COMPUTE); +} + +void ShadowRenderer::SVSMUpdateRecorder::AllocateAndQueueStaticPages() +{ + // Allocate finest-to-coarsest so pool starvation lands on the coarsest fallback rings. This + // also re-dirties invalid pages and builds the dirty rectangles and physical clear list. + commandList.BeginPipeline(owner._svsmPageUpdateBPipeline); + commandList.BindDescriptorSet(data.updateBSet, frameIndex); + + for (u32 clipmapIndex = 0; clipmapIndex < numClipmaps; clipmapIndex++) + { + Constants* allocConstants = graphResources.FrameNew(); + *allocConstants = *constants; + allocConstants->allocClipmap = clipmapIndex; + + commandList.PushConstant(allocConstants, 0, sizeof(Constants)); + commandList.Dispatch((config.pageTableSize * config.pageTableSize) / 256, 1, 1); + + if (clipmapIndex + 1 < numClipmaps) + { + commandList.BufferBarrier(data.freeListBuffer, Renderer::BufferPassUsage::COMPUTE); + } + } + + commandList.EndPipeline(owner._svsmPageUpdateBPipeline); + + commandList.BufferBarrier(data.pageTableBuffer, Renderer::BufferPassUsage::COMPUTE); + commandList.BufferBarrier(data.svsmDataBuffer, Renderer::BufferPassUsage::COMPUTE); + commandList.BufferBarrier(data.clearListBuffer, Renderer::BufferPassUsage::COMPUTE); +} + +void ShadowRenderer::SVSMUpdateRecorder::MarkPagesTouchedByDynamicCasters() +{ + if (numDynamicAABBs == 0) + return; + + Constants* dynamicMarkConstants = graphResources.FrameNew(); + *dynamicMarkConstants = *constants; + dynamicMarkConstants->numDirtyAABBs = numDynamicAABBs; + + commandList.BeginPipeline(owner._svsmDynamicMarkPipeline); + commandList.PushConstant(dynamicMarkConstants, 0, sizeof(Constants)); + commandList.BindDescriptorSet(data.dynamicMarkSet, frameIndex); + + commandList.Dispatch((numDynamicAABBs + 63) / 64, numClipmaps, 1); + commandList.EndPipeline(owner._svsmDynamicMarkPipeline); - if (numCascades == 0) + commandList.BufferBarrier(data.dynamicPageTableBuffer, Renderer::BufferPassUsage::COMPUTE); +} + +void ShadowRenderer::SVSMUpdateRecorder::RebuildDynamicPageSet() +{ + MarkPagesTouchedByDynamicCasters(); + + // Release before acquire. Combining free-list pushes and pops in one dispatch can expose a + // slot before its physical page index is written, aliasing one page under two table entries. + commandList.BeginPipeline(owner._svsmDynamicUpdatePipeline); + commandList.PushConstant(constants, 0, sizeof(Constants)); + commandList.BindDescriptorSet(data.dynamicUpdateSet, frameIndex); + commandList.Dispatch(tableCapacity / 256, 1, 1); + + commandList.BufferBarrier(data.dynamicPageTableBuffer, Renderer::BufferPassUsage::COMPUTE); + commandList.BufferBarrier(data.dynamicFreeListBuffer, Renderer::BufferPassUsage::COMPUTE); + + Constants* dynamicAcquireConstants = graphResources.FrameNew(); + *dynamicAcquireConstants = *constants; + dynamicAcquireConstants->dynamicPhase = 1; + + commandList.PushConstant(dynamicAcquireConstants, 0, sizeof(Constants)); + commandList.Dispatch(tableCapacity / 256, 1, 1); + commandList.EndPipeline(owner._svsmDynamicUpdatePipeline); + + commandList.BufferBarrier(data.dynamicPageTableBuffer, Renderer::BufferPassUsage::COMPUTE); + commandList.BufferBarrier(data.dynamicFreeListBuffer, Renderer::BufferPassUsage::COMPUTE); + commandList.BufferBarrier(data.dynamicClearListBuffer, Renderer::BufferPassUsage::COMPUTE); + commandList.BufferBarrier(data.svsmDataBuffer, Renderer::BufferPassUsage::COMPUTE); +} + +void ShadowRenderer::SVSMUpdateRecorder::BuildClipmapCamerasAndGeometryDispatches() +{ + commandList.BeginPipeline(owner._svsmFinalizePipeline); + commandList.PushConstant(constants, 0, sizeof(Constants)); + commandList.BindDescriptorSet(data.finalizeSet, frameIndex); + + commandList.Dispatch(1, 1, 1); + commandList.EndPipeline(owner._svsmFinalizePipeline); + + commandList.BufferBarrier(data.cameras, Renderer::BufferPassUsage::COMPUTE); +} + +void ShadowRenderer::SVSMUpdateRecorder::ClearPagesQueuedForRendering() +{ + commandList.BeginPipeline(owner._svsmPageClearPipeline); + commandList.PushConstant(constants, 0, sizeof(Constants)); + + data.clearSet.Bind("_pagePool"_h, data.pagePool); + commandList.BindDescriptorSet(data.clearSet, frameIndex); + + commandList.DispatchIndirect(data.clearListBuffer, 0); + commandList.EndPipeline(owner._svsmPageClearPipeline); + + if (!dynamicSplit) + return; + + Constants* dynamicClearConstants = graphResources.FrameNew(); + *dynamicClearConstants = *constants; + dynamicClearConstants->poolPagesPerRow = constants->dynamicPoolPagesPerRow; + + commandList.BeginPipeline(owner._svsmPageClearPipeline); + commandList.PushConstant(dynamicClearConstants, 0, sizeof(Constants)); + + data.dynamicClearSet.Bind("_pagePool"_h, data.dynamicPagePool); + commandList.BindDescriptorSet(data.dynamicClearSet, frameIndex); + + commandList.DispatchIndirect(data.dynamicClearListBuffer, 0); + commandList.EndPipeline(owner._svsmPageClearPipeline); +} + +void ShadowRenderer::SVSMUpdateRecorder::CaptureDiagnostics() +{ + commandList.CopyBuffer(data.svsmDataReadBackBuffer, 0, data.svsmDataBuffer, 0, sizeof(u32) * SVSM_DATA_UINT_COUNT); + + if (!dynamicSplit || owner._svsmValidatePending || CVAR_SVSMValidateDynamic.Get() == 0) + return; + + const u32 tableUints = SVSM_MAX_CLIPMAPS * SVSM_MAX_PAGE_TABLE_SIZE * SVSM_MAX_PAGE_TABLE_SIZE; + commandList.CopyBuffer(data.dynamicValidateReadBackBuffer, 0, data.dynamicPageTableBuffer, 0, sizeof(u32) * tableUints); + commandList.CopyBuffer(data.dynamicValidateReadBackBuffer, sizeof(u32) * tableUints, data.dynamicFreeListBuffer, 0, sizeof(u32) * (4 + SVSM_MAX_POOL_PAGES)); + owner._svsmValidatePending = true; +} + +void ShadowRenderer::AddSVSMUpdatePass(Renderer::RenderGraph* renderGraph, RenderResources& resources, u8 frameIndex) +{ + const u32 numClipmaps = static_cast(glm::clamp(CVAR_SVSMNumClipmaps.Get(), 1, static_cast(SVSM_MAX_CLIPMAPS))); + const bool enabled = CVAR_ShadowEnabled.Get() && !CVAR_SVSMFreeze.Get() && !_svsmNightActive; + if (!enabled || _svsmPagePool == Renderer::ImageID::Invalid()) return; - const bool shadowEnabled = CVAR_ShadowEnabled.Get(); + // Record-time inputs, the shadow sun steps in discrete intervals so cached pages stay valid + // between steps + entt::registry* registry = ServiceLocator::GetEnttRegistries()->gameRegistry; + auto& dayNightCycle = registry->ctx().get(); + const f32 shadowTimeOfDay = ECS::Systems::GetShadowTimeOfDay(dayNightCycle.GetTimeInSecondsF32()); + // GetLightDirection points toward the sun (the shading convention); SVSM works with the + // direction the light travels + const vec3 lightDirection = -ECS::Systems::UpdateAreaLights::GetLightDirection(shadowTimeOfDay); + + u32 invalidateCause = 0; + if (CVAR_SVSMInvalidateAll.Get() != 0 || _svsmForceInvalidateAll) + { + invalidateCause |= SVSMCause::Manual; + CVAR_SVSMInvalidateAll.Set(0); + _svsmForceInvalidateAll = false; + } + if (_svsmDirtyAABBOverflow) + { + invalidateCause |= SVSMCause::AABBOverflow; + } + + const u32 numDirtyAABBs = static_cast(_svsmDirtyAABBs.size() / 2); + const bool dynamicSplit = CVAR_SVSMDynamicSplit.Get() == 1; + const u32 numDynamicAABBs = dynamicSplit ? static_cast(_svsmDynamicAABBs.size() / 2) : 0; + + renderGraph->AddPass("SVSM Update", + [this, &resources](SVSMUpdatePassData& data, Renderer::RenderGraphBuilder& builder) + { + return data.DeclareResources(*this, resources, builder); + }, + [this, frameIndex, numClipmaps, lightDirection, invalidateCause, numDirtyAABBs, dynamicSplit, numDynamicAABBs](SVSMUpdatePassData& data, Renderer::RenderGraphResources& graphResources, Renderer::CommandList& commandList) + { + SVSMUpdateRecorder recorder(*this, data, graphResources, commandList, frameIndex, numClipmaps, + lightDirection, invalidateCause, numDirtyAABBs, dynamicSplit, numDynamicAABBs); + recorder.Record(); + }); +} + +void ShadowRenderer::AddSVSMDebugOverlayPass(Renderer::RenderGraph* renderGraph, RenderResources& resources, u8 frameIndex) +{ + struct Data + { + Renderer::ImageMutableResource target; + Renderer::ImageResource pagePool; + + Renderer::DescriptorSetResource debugSet; + Renderer::DescriptorSetResource poolDebugSet; + }; - renderGraph->AddPass("Shadow Pass", - [=, &resources](ShadowPassData& data, Renderer::RenderGraphBuilder& builder) + const i32 debugClipmap = CVAR_SVSMDebugClipmap.Get(); + const i32 poolMode = CVAR_SVSMDebugShowPool.Get(); // 1 = static pool, 2 = dynamic pool + const bool showPool = poolMode != 0 && _svsmPagePool != Renderer::ImageID::Invalid(); + const u32 numClipmaps = static_cast(glm::clamp(CVAR_SVSMNumClipmaps.Get(), 1, static_cast(SVSM_MAX_CLIPMAPS))); + if (!CVAR_ShadowEnabled.Get() || (debugClipmap < 0 && !showPool)) + return; + + renderGraph->AddPass("SVSM Debug Overlay", + [this, &resources, showPool, poolMode](Data& data, Renderer::RenderGraphBuilder& builder) { - for (u32 i = 0; i < numCascades; i++) + data.target = builder.Write(resources.sceneColor, Renderer::PipelineType::COMPUTE, Renderer::LoadMode::LOAD); + builder.Read(_svsmPageTableBuffer, Renderer::BufferPassUsage::COMPUTE); + if (showPool) { - data.shadowDepthCascades[i] = builder.Write(resources.shadowDepthCascades[i], Renderer::PipelineType::GRAPHICS, Renderer::LoadMode::CLEAR); + data.pagePool = builder.Read(poolMode == 2 ? _svsmDynamicPagePool : _svsmPagePool, Renderer::PipelineType::COMPUTE); } - data.lightDescriptorSet = builder.Use(resources.lightDescriptorSet); + data.debugSet = builder.Use(_svsmPageTableDebugDescriptorSet); + data.poolDebugSet = builder.Use(_svsmPoolDebugDescriptorSet); return true; // Return true from setup to enable this pass, return false to disable it }, - [=](ShadowPassData& data, Renderer::RenderGraphResources& graphResources, Renderer::CommandList& commandList) + [this, frameIndex, debugClipmap, numClipmaps, showPool, poolMode](Data& data, Renderer::RenderGraphResources& graphResources, Renderer::CommandList& commandList) { - Renderer::DepthImageMutableResource cascadeDepthResource; - for (u32 i = 0; i < Renderer::Settings::MAX_SHADOW_CASCADES; i++) + GPU_SCOPED_PROFILER_ZONE(commandList, SVSMDebugOverlay); + + uvec2 targetDimensions = graphResources.GetImageDimensions(data.target); + + if (debugClipmap >= 0) { - if (i < numCascades) + struct DebugConstants { - cascadeDepthResource = data.shadowDepthCascades[i]; - } + u32 clipmapIndex; + u32 pageTableSize; + u32 cellSize; + u32 padding0; + ivec2 screenOffset; + }; + + const u32 pageTableSize = DeriveSVSMConfig(SVSM_MAX_PAGE_TABLE_SIZE).pageTableSize; + const u32 cellSize = 8; + const u32 regionSize = pageTableSize * cellSize; + + DebugConstants* constants = graphResources.FrameNew(); + constants->clipmapIndex = glm::min(static_cast(debugClipmap), numClipmaps - 1); + constants->pageTableSize = pageTableSize; + constants->cellSize = cellSize; + constants->screenOffset = ivec2(glm::max(static_cast(targetDimensions.x) - static_cast(regionSize) - 8, 0), 8); + + commandList.BeginPipeline(_svsmPageTableDebugPipeline); + commandList.PushConstant(constants, 0, sizeof(DebugConstants)); + + data.debugSet.Bind("_target"_h, data.target); + commandList.BindDescriptorSet(data.debugSet, frameIndex); - data.lightDescriptorSet.BindArray("_shadowCascadeRTs", cascadeDepthResource, i); + commandList.Dispatch((regionSize + 15) / 16, (regionSize + 15) / 16, 1); + commandList.EndPipeline(_svsmPageTableDebugPipeline); + } + + if (showPool) + { + struct PoolDebugConstants + { + u32 poolSize; + u32 regionSize; + u32 padding0; + u32 padding1; + ivec2 screenOffset; + }; + + const u32 regionSize = 512; + + PoolDebugConstants* constants = graphResources.FrameNew(); + constants->poolSize = static_cast(poolMode == 2 ? CVAR_SVSMDynamicPoolSize.Get() : CVAR_SVSMPoolSize.Get()); + constants->regionSize = regionSize; + constants->screenOffset = ivec2(glm::max(static_cast(targetDimensions.x) - static_cast(regionSize) - 8, 0), glm::max(static_cast(targetDimensions.y) - static_cast(regionSize) - 8, 0)); + + commandList.BeginPipeline(_svsmPoolDebugPipeline); + commandList.PushConstant(constants, 0, sizeof(PoolDebugConstants)); + + data.poolDebugSet.Bind("_pagePool"_h, data.pagePool); + data.poolDebugSet.Bind("_target"_h, data.target); + commandList.BindDescriptorSet(data.poolDebugSet, frameIndex); + + commandList.Dispatch((regionSize + 15) / 16, (regionSize + 15) / 16, 1); + commandList.EndPipeline(_svsmPoolDebugPipeline); } }); } +bool ShadowRenderer::GetSVSMClipmapStats(u32 clipmapIndex, SVSMClipmapStats& outStats) const +{ + if (clipmapIndex >= SVSM_MAX_CLIPMAPS) + return false; + + outStats.marked = _svsmDataReadBack[SVSMDataOffsets::StatsMarked + clipmapIndex]; + outStats.resident = _svsmDataReadBack[SVSMDataOffsets::StatsResident + clipmapIndex]; + outStats.dirty = _svsmDataReadBack[SVSMDataOffsets::StatsDirty + clipmapIndex]; + outStats.evicted = _svsmDataReadBack[SVSMDataOffsets::StatsEvicted + clipmapIndex]; + outStats.invalidated = _svsmDataReadBack[SVSMDataOffsets::StatsInvalidated + clipmapIndex]; + outStats.dynamicLive = _svsmDataReadBack[SVSMDataOffsets::StatsDynamicLive + clipmapIndex]; + outStats.deferred = _svsmDataReadBack[SVSMDataOffsets::StatsDeferred + clipmapIndex]; + outStats.extent = glm::uintBitsToFloat(_svsmDataReadBack[SVSMDataOffsets::Extent + clipmapIndex]); + return true; +} + +void ShadowRenderer::GetSVSMGlobalStats(u32& outFreePages, u32& outTotalPages, u32& outOverflow, u32& outInvalidationCause) const +{ + outFreePages = _svsmDataReadBack[SVSMDataOffsets::StatsFreeListCount]; + outTotalPages = _svsmPoolPages; + outOverflow = _svsmDataReadBack[SVSMDataOffsets::StatsOverflow]; + outInvalidationCause = _svsmDataReadBack[SVSMDataOffsets::StatsInvalidationCause]; +} + +void ShadowRenderer::GetSVSMDynamicStats(u32& outLivePages, u32& outTotalPages, u32& outOverflow) const +{ + outLivePages = _svsmDataReadBack[SVSMDataOffsets::StatsDynamicTotal]; + outTotalPages = _svsmDynamicPoolPages; + outOverflow = _svsmDataReadBack[SVSMDataOffsets::StatsDynamicOverflow]; +} + +u32 ShadowRenderer::GetSVSMBudgetUsed() const +{ + return _svsmDataReadBack[SVSMDataOffsets::StatsBudgetUsed]; +} + +void ShadowRenderer::AddSVSMBindPass(Renderer::RenderGraph* renderGraph, RenderResources& resources, u8 frameIndex) +{ + struct Data + { + Renderer::ImageResource svsmPagePool; + Renderer::ImageResource svsmDynamicPagePool; + + Renderer::DescriptorSetResource lightDescriptorSet; + }; + + // Runs unconditionally (even under svsmFreeze, when the update pass early-outs): every LIGHT + // set consumer needs the pool bindings valid, real pools or the placeholder before they exist + renderGraph->AddPass("SVSM Bind", + [this, &resources](Data& data, Renderer::RenderGraphBuilder& builder) + { + data.svsmPagePool = builder.Read(GetSVSMPagePoolOrPlaceholder(), Renderer::PipelineType::COMPUTE); + data.svsmDynamicPagePool = builder.Read(GetSVSMDynamicPagePoolOrPlaceholder(), Renderer::PipelineType::COMPUTE); + + data.lightDescriptorSet = builder.Use(resources.lightDescriptorSet); + + return true; // Return true from setup to enable this pass, return false to disable it + }, + [](Data& data, Renderer::RenderGraphResources& graphResources, Renderer::CommandList& commandList) + { + data.lightDescriptorSet.Bind("_svsmPagePool"_h, data.svsmPagePool); + data.lightDescriptorSet.Bind("_svsmDynamicPagePool"_h, data.svsmDynamicPagePool); + }); +} + void ShadowRenderer::CreatePermanentResources(RenderResources& resources) { ZoneScoped; - Renderer::SamplerDesc samplerDesc; - samplerDesc.enabled = true; - samplerDesc.filter = Renderer::SamplerFilter::MIN_MAG_MIP_LINEAR; - samplerDesc.addressU = Renderer::TextureAddressMode::WRAP; - samplerDesc.addressV = Renderer::TextureAddressMode::WRAP; - samplerDesc.addressW = Renderer::TextureAddressMode::CLAMP; - samplerDesc.shaderVisibility = Renderer::ShaderVisibility::PIXEL; - samplerDesc.comparisonEnabled = true; - samplerDesc.comparisonFunc = Renderer::ComparisonFunc::GREATER; - - _shadowCmpSampler = _renderer->CreateSampler(samplerDesc); - resources.lightDescriptorSet.Bind("_shadowCmpSampler"_h, _shadowCmpSampler); - - samplerDesc.filter = Renderer::SamplerFilter::MIN_MAG_MIP_POINT; - samplerDesc.addressU = Renderer::TextureAddressMode::CLAMP; - samplerDesc.addressV = Renderer::TextureAddressMode::CLAMP; - samplerDesc.comparisonEnabled = false; - - _shadowPointClampSampler = _renderer->CreateSampler(samplerDesc); - resources.lightDescriptorSet.Bind("_shadowPointClampSampler"_h, _shadowPointClampSampler); -} \ No newline at end of file + SanitizeSVSMConfigCVars(SVSM_MAX_PAGE_TABLE_SIZE); + + // SVSM page table lifecycle + { + Renderer::ComputePipelineDesc pipelineDesc; + Renderer::ComputeShaderDesc shaderDesc; + + pipelineDesc.debugName = "SVSM Prepare"; + shaderDesc.shaderEntry = _gameRenderer->GetShaderEntry("Shadows/SVSMPrepare.cs"_h, "Shadows/SVSMPrepare.cs"); + pipelineDesc.computeShader = _renderer->LoadShader(shaderDesc); + _svsmPreparePipeline = _renderer->CreatePipeline(pipelineDesc); + _svsmPrepareDescriptorSet.RegisterPipeline(_renderer, _svsmPreparePipeline); + _svsmPrepareDescriptorSet.Init(_renderer); + + pipelineDesc.debugName = "SVSM Invalidate AABBs"; + shaderDesc.shaderEntry = _gameRenderer->GetShaderEntry("Shadows/SVSMInvalidateAABBs.cs"_h, "Shadows/SVSMInvalidateAABBs.cs"); + pipelineDesc.computeShader = _renderer->LoadShader(shaderDesc); + _svsmInvalidateAABBsPipeline = _renderer->CreatePipeline(pipelineDesc); + _svsmInvalidateAABBsDescriptorSet.RegisterPipeline(_renderer, _svsmInvalidateAABBsPipeline); + _svsmInvalidateAABBsDescriptorSet.Init(_renderer); + + pipelineDesc.debugName = "SVSM Page Update A"; + shaderDesc.shaderEntry = _gameRenderer->GetShaderEntry("Shadows/SVSMPageUpdateA.cs"_h, "Shadows/SVSMPageUpdateA.cs"); + pipelineDesc.computeShader = _renderer->LoadShader(shaderDesc); + _svsmPageUpdateAPipeline = _renderer->CreatePipeline(pipelineDesc); + _svsmPageUpdateADescriptorSet.RegisterPipeline(_renderer, _svsmPageUpdateAPipeline); + _svsmPageUpdateADescriptorSet.Init(_renderer); + + pipelineDesc.debugName = "SVSM Page Mark"; + shaderDesc.shaderEntry = _gameRenderer->GetShaderEntry("Shadows/SVSMPageMark.cs"_h, "Shadows/SVSMPageMark.cs"); + pipelineDesc.computeShader = _renderer->LoadShader(shaderDesc); + _svsmPageMarkPipeline = _renderer->CreatePipeline(pipelineDesc); + _svsmPageMarkDescriptorSet.RegisterPipeline(_renderer, _svsmPageMarkPipeline); + _svsmPageMarkDescriptorSet.Init(_renderer); + + pipelineDesc.debugName = "SVSM Page Update B"; + shaderDesc.shaderEntry = _gameRenderer->GetShaderEntry("Shadows/SVSMPageUpdateB.cs"_h, "Shadows/SVSMPageUpdateB.cs"); + pipelineDesc.computeShader = _renderer->LoadShader(shaderDesc); + _svsmPageUpdateBPipeline = _renderer->CreatePipeline(pipelineDesc); + _svsmPageUpdateBDescriptorSet.RegisterPipeline(_renderer, _svsmPageUpdateBPipeline); + _svsmPageUpdateBDescriptorSet.Init(_renderer); + + pipelineDesc.debugName = "SVSM Dynamic Mark"; + shaderDesc.shaderEntry = _gameRenderer->GetShaderEntry("Shadows/SVSMDynamicMark.cs"_h, "Shadows/SVSMDynamicMark.cs"); + pipelineDesc.computeShader = _renderer->LoadShader(shaderDesc); + _svsmDynamicMarkPipeline = _renderer->CreatePipeline(pipelineDesc); + _svsmDynamicMarkDescriptorSet.RegisterPipeline(_renderer, _svsmDynamicMarkPipeline); + _svsmDynamicMarkDescriptorSet.Init(_renderer); + + pipelineDesc.debugName = "SVSM Dynamic Update"; + shaderDesc.shaderEntry = _gameRenderer->GetShaderEntry("Shadows/SVSMDynamicUpdate.cs"_h, "Shadows/SVSMDynamicUpdate.cs"); + pipelineDesc.computeShader = _renderer->LoadShader(shaderDesc); + _svsmDynamicUpdatePipeline = _renderer->CreatePipeline(pipelineDesc); + _svsmDynamicUpdateDescriptorSet.RegisterPipeline(_renderer, _svsmDynamicUpdatePipeline); + _svsmDynamicUpdateDescriptorSet.Init(_renderer); + + pipelineDesc.debugName = "SVSM Finalize"; + shaderDesc.shaderEntry = _gameRenderer->GetShaderEntry("Shadows/SVSMFinalize.cs"_h, "Shadows/SVSMFinalize.cs"); + pipelineDesc.computeShader = _renderer->LoadShader(shaderDesc); + _svsmFinalizePipeline = _renderer->CreatePipeline(pipelineDesc); + _svsmFinalizeDescriptorSet.RegisterPipeline(_renderer, _svsmFinalizePipeline); + _svsmFinalizeDescriptorSet.Init(_renderer); + + pipelineDesc.debugName = "SVSM Page Clear"; + shaderDesc.shaderEntry = _gameRenderer->GetShaderEntry("Shadows/SVSMPageClear.cs"_h, "Shadows/SVSMPageClear.cs"); + pipelineDesc.computeShader = _renderer->LoadShader(shaderDesc); + _svsmPageClearPipeline = _renderer->CreatePipeline(pipelineDesc); + _svsmPageClearDescriptorSet.RegisterPipeline(_renderer, _svsmPageClearPipeline); + _svsmPageClearDescriptorSet.Init(_renderer); + + // Second set instance on the same clear pipeline, bound to the dynamic list + pool + _svsmDynamicPageClearDescriptorSet.RegisterPipeline(_renderer, _svsmPageClearPipeline); + _svsmDynamicPageClearDescriptorSet.Init(_renderer); + + pipelineDesc.debugName = "SVSM Page Table Debug"; + shaderDesc.shaderEntry = _gameRenderer->GetShaderEntry("Shadows/SVSMPageTableDebug.cs"_h, "Shadows/SVSMPageTableDebug.cs"); + pipelineDesc.computeShader = _renderer->LoadShader(shaderDesc); + _svsmPageTableDebugPipeline = _renderer->CreatePipeline(pipelineDesc); + _svsmPageTableDebugDescriptorSet.RegisterPipeline(_renderer, _svsmPageTableDebugPipeline); + _svsmPageTableDebugDescriptorSet.Init(_renderer); + + pipelineDesc.debugName = "SVSM Pool Debug"; + shaderDesc.shaderEntry = _gameRenderer->GetShaderEntry("Shadows/SVSMPoolDebug.cs"_h, "Shadows/SVSMPoolDebug.cs"); + pipelineDesc.computeShader = _renderer->LoadShader(shaderDesc); + _svsmPoolDebugPipeline = _renderer->CreatePipeline(pipelineDesc); + _svsmPoolDebugDescriptorSet.RegisterPipeline(_renderer, _svsmPoolDebugPipeline); + _svsmPoolDebugDescriptorSet.Init(_renderer); + + Renderer::BufferDesc bufferDesc; + bufferDesc.name = "SVSMDirtyAABBs"; + bufferDesc.size = sizeof(vec4) * SVSM_MAX_DIRTY_AABBS * 2; + bufferDesc.usage = Renderer::BufferUsage::STORAGE_BUFFER | Renderer::BufferUsage::TRANSFER_DESTINATION; + _svsmDirtyAABBBuffer = _renderer->CreateBuffer(_svsmDirtyAABBBuffer, bufferDesc); + + bufferDesc.name = "SVSMDynamicAABBs"; + bufferDesc.size = sizeof(vec4) * SVSM_MAX_DYNAMIC_AABBS * 2; + _svsmDynamicAABBBuffer = _renderer->CreateBuffer(_svsmDynamicAABBBuffer, bufferDesc); + + bufferDesc.name = "SVSMDynamicClearList"; + bufferDesc.size = sizeof(u32) * (4 + SVSM_MAX_POOL_PAGES); + bufferDesc.usage = Renderer::BufferUsage::STORAGE_BUFFER | Renderer::BufferUsage::INDIRECT_ARGUMENT_BUFFER; + _svsmDynamicClearListBuffer = _renderer->CreateAndFillBuffer(_svsmDynamicClearListBuffer, bufferDesc, [](void* mappedMemory, size_t size) + { + u32* values = static_cast(mappedMemory); + memset(values, 0, size); + values[1] = 1; // Indirect group counts y and z + values[2] = 1; + }); + + bufferDesc.name = "SVSMClearList"; + bufferDesc.size = sizeof(u32) * (4 + SVSM_MAX_POOL_PAGES); // [0..2] indirect dispatch args, physical pages from [4] + bufferDesc.usage = Renderer::BufferUsage::STORAGE_BUFFER | Renderer::BufferUsage::INDIRECT_ARGUMENT_BUFFER; + _svsmClearListBuffer = _renderer->CreateAndFillBuffer(_svsmClearListBuffer, bufferDesc, [](void* mappedMemory, size_t size) + { + u32* values = static_cast(mappedMemory); + memset(values, 0, size); + values[1] = 1; // Indirect group counts y and z + values[2] = 1; + }); + + // The config-independent buffer binds. The config-shaped buffers (SVSMData, page tables, + // free lists) are created and bound by ResetSVSMPoolState below + _svsmPrepareDescriptorSet.Bind("_clearList"_h, _svsmClearListBuffer); + _svsmPrepareDescriptorSet.Bind("_dynamicClearList"_h, _svsmDynamicClearListBuffer); + _svsmInvalidateAABBsDescriptorSet.Bind("_dirtyAABBs"_h, _svsmDirtyAABBBuffer); + _svsmPageUpdateBDescriptorSet.Bind("_clearList"_h, _svsmClearListBuffer); + _svsmDynamicMarkDescriptorSet.Bind("_dynamicAABBs"_h, _svsmDynamicAABBBuffer); + _svsmDynamicUpdateDescriptorSet.Bind("_dynamicClearList"_h, _svsmDynamicClearListBuffer); + _svsmPageClearDescriptorSet.Bind("_clearList"_h, _svsmClearListBuffer); + _svsmDynamicPageClearDescriptorSet.Bind("_clearList"_h, _svsmDynamicClearListBuffer); + // _srcCameras / _depth / _target / _rwCameras / _pagePool are bound per frame, those resources can be recreated + + // Per-view fill dispatch args written by Finalize, consumed by the geometry passes' + // DispatchIndirect: per clipmap 5 x uvec3 (model static, model dynamic, terrain static, + // model static overhead, model dynamic overhead). Prefilled with zero groups so a + // pre-Finalize consumer launches nothing + Renderer::BufferDesc fillArgsDesc; + fillArgsDesc.name = "SVSMFillDispatchArgs"; + fillArgsDesc.size = SVSM_FILL_ARGS_VIEW_STRIDE * SVSM_MAX_CLIPMAPS; + fillArgsDesc.usage = Renderer::BufferUsage::STORAGE_BUFFER | Renderer::BufferUsage::INDIRECT_ARGUMENT_BUFFER; + _svsmFillArgsBuffer = _renderer->CreateAndFillBuffer(_svsmFillArgsBuffer, fillArgsDesc, [](void* mappedMemory, size_t size) + { + u32* values = static_cast(mappedMemory); + for (u32 i = 0; i < SVSM_MAX_CLIPMAPS * 15; i += 3) + { + values[i + 0] = 0; + values[i + 1] = 1; + values[i + 2] = 1; + } + }); + _svsmFinalizeDescriptorSet.Bind("_fillDispatchArgs"_h, _svsmFillArgsBuffer); + + bufferDesc.name = "SVSMDataReadBack"; + bufferDesc.size = sizeof(u32) * SVSM_DATA_UINT_COUNT; + bufferDesc.usage = Renderer::BufferUsage::STORAGE_BUFFER | Renderer::BufferUsage::TRANSFER_DESTINATION; + bufferDesc.cpuAccess = Renderer::BufferCPUAccess::ReadOnly; + _svsmDataReadBackBuffer = _renderer->CreateBuffer(_svsmDataReadBackBuffer, bufferDesc); + + bufferDesc.name = "SVSMDynamicValidateReadBack"; + bufferDesc.size = sizeof(u32) * (SVSM_MAX_CLIPMAPS * SVSM_MAX_PAGE_TABLE_SIZE * SVSM_MAX_PAGE_TABLE_SIZE + 4 + SVSM_MAX_POOL_PAGES); + _svsmDynamicValidateReadBackBuffer = _renderer->CreateBuffer(_svsmDynamicValidateReadBackBuffer, bufferDesc); + + Renderer::ImageDesc placeholderDesc; + placeholderDesc.debugName = "SVSMPagePoolPlaceholder"; + placeholderDesc.dimensions = vec2(1, 1); + placeholderDesc.dimensionType = Renderer::ImageDimensionType::DIMENSION_ABSOLUTE; + placeholderDesc.format = Renderer::ImageFormat::R32_UINT; + placeholderDesc.sampleCount = Renderer::SampleCount::SAMPLE_COUNT_1; + placeholderDesc.clearUInts = uvec4(0, 0, 0, 0); + + _svsmPagePoolPlaceholder = _renderer->CreateImage(placeholderDesc); + + // Creates the config-shaped buffers and binds them everywhere, here and again on live + // config edits + ResetSVSMPoolState(resources); + } + + BindCameraBuffers(resources); +} + +void ShadowRenderer::ResetSVSMPoolState(RenderResources& resources) +{ + // (Re)shapes everything svsmPageSize touches: pool page counts, free lists, page tables and + // the SVSMData scratch (zeroed SVSMData reads as uninitialized, Prepare re-derives the + // anchors). The refills happen CPU-side so GPU eviction never runs against a half-rebuilt + // free list, and CreateAndFillBuffer swaps in a fresh buffer (deferred-destroying the old one + // past the frames in flight), so every consumer set rebinds below. The pool textures are NOT + // recreated — their dimensions only depend on the pool size cvars, which are restart-only + // once the pools exist (the Engine cannot destroy images) + const SVSMDerivedConfig config = DeriveSVSMConfig(SVSM_MAX_PAGE_TABLE_SIZE); + _svsmAppliedPageSize = config.pageSize; + _svsmAppliedPoolSize = static_cast(CVAR_SVSMPoolSize.Get()); + _svsmAppliedDynamicPoolSize = static_cast(CVAR_SVSMDynamicPoolSize.Get()); + + // The physical page index is 12 bits in the table entry + _svsmPoolPages = glm::min(config.poolPagesPerRow * config.poolPagesPerRow, SVSM_MAX_POOL_PAGES); + _svsmDynamicPoolPages = glm::min(config.dynamicPoolPagesPerRow * config.dynamicPoolPagesPerRow, SVSM_MAX_POOL_PAGES); + + Renderer::BufferDesc bufferDesc; + bufferDesc.name = "SVSMData"; + bufferDesc.size = sizeof(u32) * SVSM_DATA_UINT_COUNT; + bufferDesc.usage = Renderer::BufferUsage::STORAGE_BUFFER | Renderer::BufferUsage::TRANSFER_DESTINATION | Renderer::BufferUsage::TRANSFER_SOURCE; + _svsmDataBuffer = _renderer->CreateAndFillBuffer(_svsmDataBuffer, bufferDesc, [](void* mappedMemory, size_t size) + { + memset(mappedMemory, 0, size); // prevLightDirection.w = 0 marks the state uninitialized + }); + + bufferDesc.name = "SVSMPageTable"; + bufferDesc.size = sizeof(u32) * SVSM_MAX_CLIPMAPS * SVSM_MAX_PAGE_TABLE_SIZE * SVSM_MAX_PAGE_TABLE_SIZE; + bufferDesc.usage = Renderer::BufferUsage::STORAGE_BUFFER; + _svsmPageTableBuffer = _renderer->CreateAndFillBuffer(_svsmPageTableBuffer, bufferDesc, [](void* mappedMemory, size_t size) + { + memset(mappedMemory, 0, size); // A zeroed entry is a free slot + }); + + bufferDesc.name = "SVSMFreeList"; + bufferDesc.size = sizeof(u32) * (4 + SVSM_MAX_POOL_PAGES); + bufferDesc.usage = Renderer::BufferUsage::STORAGE_BUFFER; + const u32 poolPages = _svsmPoolPages; + _svsmFreeListBuffer = _renderer->CreateAndFillBuffer(_svsmFreeListBuffer, bufferDesc, [poolPages](void* mappedMemory, size_t size) + { + u32* values = static_cast(mappedMemory); + values[0] = poolPages; // [0] = count, [1..3] padding, entries from [4] + values[1] = 0; + values[2] = 0; + values[3] = 0; + for (u32 i = 0; i < SVSM_MAX_POOL_PAGES; i++) + { + values[4 + i] = i < poolPages ? i : 0; + } + }); + + bufferDesc.name = "SVSMDynamicPageTable"; + bufferDesc.size = sizeof(u32) * SVSM_MAX_CLIPMAPS * SVSM_MAX_PAGE_TABLE_SIZE * SVSM_MAX_PAGE_TABLE_SIZE; + bufferDesc.usage = Renderer::BufferUsage::STORAGE_BUFFER | Renderer::BufferUsage::TRANSFER_SOURCE; // Source of the svsmValidateDynamic snapshot copy + _svsmDynamicPageTableBuffer = _renderer->CreateAndFillBuffer(_svsmDynamicPageTableBuffer, bufferDesc, [](void* mappedMemory, size_t size) + { + memset(mappedMemory, 0, size); + }); + + bufferDesc.name = "SVSMDynamicFreeList"; + bufferDesc.size = sizeof(u32) * (4 + SVSM_MAX_POOL_PAGES); + bufferDesc.usage = Renderer::BufferUsage::STORAGE_BUFFER | Renderer::BufferUsage::TRANSFER_SOURCE; + const u32 dynamicPoolPages = _svsmDynamicPoolPages; + _svsmDynamicFreeListBuffer = _renderer->CreateAndFillBuffer(_svsmDynamicFreeListBuffer, bufferDesc, [dynamicPoolPages](void* mappedMemory, size_t size) + { + u32* values = static_cast(mappedMemory); + memset(values, 0, size); + values[0] = dynamicPoolPages; + for (u32 i = 0; i < dynamicPoolPages; i++) + { + values[4 + i] = i; + } + }); + + _svsmPrepareDescriptorSet.Bind("_svsmData"_h, _svsmDataBuffer); + _svsmPrepareDescriptorSet.Bind("_freeList"_h, _svsmFreeListBuffer); + _svsmInvalidateAABBsDescriptorSet.Bind("_svsmData"_h, _svsmDataBuffer); + _svsmInvalidateAABBsDescriptorSet.Bind("_pageTable"_h, _svsmPageTableBuffer); + _svsmPageUpdateADescriptorSet.Bind("_svsmData"_h, _svsmDataBuffer); + _svsmPageUpdateADescriptorSet.Bind("_pageTable"_h, _svsmPageTableBuffer); + _svsmPageUpdateADescriptorSet.Bind("_freeList"_h, _svsmFreeListBuffer); + _svsmPageMarkDescriptorSet.Bind("_svsmData"_h, _svsmDataBuffer); + _svsmPageMarkDescriptorSet.Bind("_pageTable"_h, _svsmPageTableBuffer); + _svsmPageUpdateBDescriptorSet.Bind("_svsmData"_h, _svsmDataBuffer); + _svsmPageUpdateBDescriptorSet.Bind("_pageTable"_h, _svsmPageTableBuffer); + _svsmPageUpdateBDescriptorSet.Bind("_freeList"_h, _svsmFreeListBuffer); + _svsmDynamicMarkDescriptorSet.Bind("_svsmData"_h, _svsmDataBuffer); + _svsmDynamicMarkDescriptorSet.Bind("_pageTable"_h, _svsmPageTableBuffer); + _svsmDynamicMarkDescriptorSet.Bind("_dynamicPageTable"_h, _svsmDynamicPageTableBuffer); + _svsmDynamicUpdateDescriptorSet.Bind("_svsmData"_h, _svsmDataBuffer); + _svsmDynamicUpdateDescriptorSet.Bind("_dynamicPageTable"_h, _svsmDynamicPageTableBuffer); + _svsmDynamicUpdateDescriptorSet.Bind("_dynamicFreeList"_h, _svsmDynamicFreeListBuffer); + _svsmFinalizeDescriptorSet.Bind("_svsmData"_h, _svsmDataBuffer); + _svsmPageTableDebugDescriptorSet.Bind("_pageTable"_h, _svsmPageTableBuffer); + + // The material pass samples SVSM through the LIGHT set. The buffers bind here, the pools + // bind per frame in the material pass (real pools once created, a placeholder before) + resources.lightDescriptorSet.Bind("_svsmData"_h, _svsmDataBuffer); + resources.lightDescriptorSet.Bind("_svsmPageTable"_h, _svsmPageTableBuffer); + resources.lightDescriptorSet.Bind("_svsmDynamicPageTable"_h, _svsmDynamicPageTableBuffer); + + // The terrain/model SVSM page-render sets read these permanent buffers, bind them once + // here (the pools stay per-frame, they are created lazily). Same for the cameras buffer + _terrainRenderer->BindSVSMBuffers(_svsmDataBuffer, _svsmPageTableBuffer); + _modelRenderer->BindSVSMBuffers(_svsmDataBuffer, _svsmPageTableBuffer, _svsmDynamicPageTableBuffer); +} + +void ShadowRenderer::BindCameraBuffers(RenderResources& resources) +{ + Renderer::BufferID camerasBuffer = resources.cameras.GetBuffer(); + + _svsmPrepareDescriptorSet.Bind("_srcCameras"_h, camerasBuffer); + _svsmPageMarkDescriptorSet.Bind("_srcCameras"_h, camerasBuffer); + _svsmFinalizeDescriptorSet.Bind("_rwCameras"_h, camerasBuffer); +} diff --git a/Source/Game-Lib/Game-Lib/Rendering/Shadow/ShadowRenderer.h b/Source/Game-Lib/Game-Lib/Rendering/Shadow/ShadowRenderer.h index 76fb006b..4b5852e3 100644 --- a/Source/Game-Lib/Game-Lib/Rendering/Shadow/ShadowRenderer.h +++ b/Source/Game-Lib/Game-Lib/Rendering/Shadow/ShadowRenderer.h @@ -1,9 +1,16 @@ #pragma once +#include "Game-Lib/Rendering/Camera.h" + #include #include #include #include +#include + +#include +#include +#include #include #include @@ -14,7 +21,6 @@ namespace Renderer } struct RenderResources; -class DebugRenderer; class GameRenderer; class ModelRenderer; class TerrainRenderer; @@ -22,44 +28,185 @@ class TerrainRenderer; class ShadowRenderer { public: - ShadowRenderer(Renderer::Renderer* renderer, GameRenderer* gameRenderer, DebugRenderer* debugRenderer, TerrainRenderer* terrainRenderer, ModelRenderer* modelRenderer, RenderResources& resources); + ShadowRenderer(Renderer::Renderer* renderer, GameRenderer* gameRenderer, TerrainRenderer* terrainRenderer, ModelRenderer* modelRenderer, RenderResources& resources); ~ShadowRenderer(); void Update(f32 deltaTime, RenderResources& resources); - void AddShadowPass(Renderer::RenderGraph* renderGraph, RenderResources& resources, u8 frameIndex); + // SVSM: page marking, page table lifecycle and allocation + void AddSVSMUpdatePass(Renderer::RenderGraph* renderGraph, RenderResources& resources, u8 frameIndex); -private: - void CreatePermanentResources(RenderResources& resources); + // Binds the (lazily created) page pools into the LIGHT set every frame, runs even when the + // update pass is disabled (svsmFreeze) + void AddSVSMBindPass(Renderer::RenderGraph* renderGraph, RenderResources& resources, u8 frameIndex); + void AddSVSMDebugOverlayPass(Renderer::RenderGraph* renderGraph, RenderResources& resources, u8 frameIndex); -private: - struct ShadowCascadeDebugInformation + struct SVSMClipmapStats { - public: - vec3 frustumCorners[8]; - vec3 frustumPlanePos; - vec4 frustumPlanes[6]; + u32 marked = 0; + u32 resident = 0; + u32 dirty = 0; + u32 evicted = 0; + u32 invalidated = 0; + u32 dynamicLive = 0; + u32 deferred = 0; + f32 extent = 0.0f; }; + // One frame old readback values + bool GetSVSMClipmapStats(u32 clipmapIndex, SVSMClipmapStats& outStats) const; + void GetSVSMGlobalStats(u32& outFreePages, u32& outTotalPages, u32& outOverflow, u32& outInvalidationCause) const; + void GetSVSMDynamicStats(u32& outLivePages, u32& outTotalPages, u32& outOverflow) const; + u32 GetSVSMBudgetUsed() const; // Defined in the .cpp next to the SVSMData mirror offsets + + // Binds the cameras buffer into every SVSM set that reads or writes it. Called at init + // (buffer binds only reach the canonical descriptor copies at a later FlipFrame, so mid-frame + // binds leave a hole) and again if the cameras buffer is ever recreated + void BindCameraBuffers(RenderResources& resources); + + // Current-frame CPU knowledge: with no dynamic casters at all, the dynamic fills/draws skip + // entirely. Per-view skipping is GPU-driven (Finalize's fill dispatch args) — a readback-based + // per-view skip is a frame late and drops freshly acquired dynamic pages' draws for a frame + bool HasSVSMDynamicCasters() const { return !_svsmDynamicAABBs.empty(); } + + // Shadows enabled and not night-gated: the single condition the clipmap culling and SVSM + // geometry passes gate on (freeze deliberately excluded, frozen frames keep recording draws) + bool IsSVSMActive() const; + + // CPU-side caster classification counts, rebuilt each Update from the ModelRenderer's + // classifier: live set size, this frame's enter/leave transitions, and static-spilled casters + void GetSVSMCasterStats(u32& outDynamicCasters, u32& outTransitionsIn, u32& outTransitionsOut, u32& outDroppedAABBs) const + { + outDynamicCasters = _svsmNumDynamicCasters; + outTransitionsIn = _svsmCasterTransitionsIn; + outTransitionsOut = _svsmCasterTransitionsOut; + outDroppedAABBs = _svsmDynamicAABBsDropped; + } + + // SVSM resources for the terrain/model page render passes, the pools are created lazily on + // the first shadow-enabled frame + Renderer::BufferID GetSVSMDataBuffer() const { return _svsmDataBuffer; } + Renderer::BufferID GetSVSMPageTableBuffer() const { return _svsmPageTableBuffer; } + Renderer::ImageID GetSVSMPagePool() const { return _svsmPagePool; } + Renderer::BufferID GetSVSMDynamicPageTableBuffer() const { return _svsmDynamicPageTableBuffer; } + Renderer::ImageID GetSVSMDynamicPagePool() const { return _svsmDynamicPagePool; } + + // Finalize-written per-view fill dispatch args: per clipmap 5 x uvec3 (model static fill, + // model dynamic fill, terrain static fill, model static overhead, model dynamic overhead — + // the overhead args gate the drawcall-granularity clear + CreateIndirect companions), + // byte stride SVSM_FILL_ARGS_VIEW_STRIDE + Renderer::BufferID GetSVSMFillArgsBuffer() const { return _svsmFillArgsBuffer; } + static constexpr u32 SVSM_FILL_ARGS_VIEW_STRIDE = 5 * 3 * sizeof(u32); + static constexpr u32 SVSM_FILL_ARGS_DYNAMIC_OFFSET = 3 * sizeof(u32); + static constexpr u32 SVSM_FILL_ARGS_TERRAIN_OFFSET = 6 * sizeof(u32); + static constexpr u32 SVSM_FILL_ARGS_STATIC_OVERHEAD_OFFSET = 9 * sizeof(u32); + static constexpr u32 SVSM_FILL_ARGS_DYNAMIC_OVERHEAD_OFFSET = 12 * sizeof(u32); + + // The single clipmap-count cap every clamp site uses. The camera buffer and bitmask slices + // are laid out against the Engine's view cap, the two must stay equal + static constexpr u32 SVSM_MAX_CLIPMAPS = 8; + static_assert(SVSM_MAX_CLIPMAPS == Renderer::Settings::MAX_SHADOW_CASCADES, "SVSM clipmap cap must match the Engine shadow view cap, both size the camera buffer and bitmask slices"); + + // Dynamic AABB buffer capacity; the ModelRenderer's classifier caps its emission against this + // and spills the excess to the static path (never excluded from both pools) + static constexpr u32 SVSM_MAX_DYNAMIC_AABBS = 4096; + + // Scalar layout size of SVSMData in Shadows/SVSM.inc.slang. The .cpp mirrors the struct + // (SVSMDataMirror) and static_asserts against this so layout drift breaks the build instead + // of silently misreporting the readback. Tail: clipRect{MinX,MinY,MaxX,MaxY}[24] at 220..315 + static constexpr u32 SVSM_DATA_UINT_COUNT = 316; + + // For the material pass LIGHT set bindings, which must stay valid before the pools exist. + // Nothing samples the placeholder while the page tables have no resident entries + Renderer::ImageID GetSVSMPagePoolOrPlaceholder() const { return _svsmPagePool != Renderer::ImageID::Invalid() ? _svsmPagePool : _svsmPagePoolPlaceholder; } + Renderer::ImageID GetSVSMDynamicPagePoolOrPlaceholder() const { return _svsmDynamicPagePool != Renderer::ImageID::Invalid() ? _svsmDynamicPagePool : _svsmPagePoolPlaceholder; } + +private: + struct SVSMUpdatePassData; + struct SVSMUpdateRecorder; + + void CreatePermanentResources(RenderResources& resources); + void ResetSVSMPoolState(RenderResources& resources); + private: Renderer::Renderer* _renderer = nullptr; GameRenderer* _gameRenderer = nullptr; - DebugRenderer* _debugRenderer = nullptr; TerrainRenderer* _terrainRenderer = nullptr; ModelRenderer* _modelRenderer = nullptr; - Renderer::SamplerID _shadowCmpSampler; - Renderer::SamplerID _shadowPointClampSampler; + static constexpr u32 SVSM_MAX_PAGE_TABLE_SIZE = 64; // Pages per row, buffers are sized for this cap + static constexpr u32 SVSM_MAX_POOL_PAGES = 4096; // Physical page index is 12 bits in the table entry + static constexpr u32 SVSM_MAX_DIRTY_AABBS = 1024; + + Renderer::ComputePipelineID _svsmPreparePipeline; + Renderer::ComputePipelineID _svsmInvalidateAABBsPipeline; + Renderer::ComputePipelineID _svsmPageUpdateAPipeline; + Renderer::ComputePipelineID _svsmPageMarkPipeline; + Renderer::ComputePipelineID _svsmPageUpdateBPipeline; + Renderer::ComputePipelineID _svsmDynamicMarkPipeline; + Renderer::ComputePipelineID _svsmDynamicUpdatePipeline; + Renderer::ComputePipelineID _svsmFinalizePipeline; + Renderer::ComputePipelineID _svsmPageClearPipeline; + Renderer::ComputePipelineID _svsmPageTableDebugPipeline; + Renderer::ComputePipelineID _svsmPoolDebugPipeline; + Renderer::DescriptorSet _svsmPrepareDescriptorSet; + Renderer::DescriptorSet _svsmInvalidateAABBsDescriptorSet; + Renderer::DescriptorSet _svsmPageUpdateADescriptorSet; + Renderer::DescriptorSet _svsmPageMarkDescriptorSet; + Renderer::DescriptorSet _svsmPageUpdateBDescriptorSet; + Renderer::DescriptorSet _svsmDynamicMarkDescriptorSet; + Renderer::DescriptorSet _svsmDynamicUpdateDescriptorSet; + Renderer::DescriptorSet _svsmFinalizeDescriptorSet; + Renderer::DescriptorSet _svsmPageClearDescriptorSet; + Renderer::DescriptorSet _svsmDynamicPageClearDescriptorSet; + Renderer::DescriptorSet _svsmPageTableDebugDescriptorSet; + Renderer::DescriptorSet _svsmPoolDebugDescriptorSet; - Renderer::TextureArrayID _shadowDepthTextures; - u32 _numInitializedShadowDepthImages = 0; + Renderer::BufferID _svsmDataBuffer; + Renderer::BufferID _svsmPageTableBuffer; + Renderer::BufferID _svsmFreeListBuffer; + Renderer::BufferID _svsmDirtyAABBBuffer; + Renderer::BufferID _svsmClearListBuffer; + Renderer::BufferID _svsmDynamicPageTableBuffer; + Renderer::BufferID _svsmDynamicFreeListBuffer; + Renderer::BufferID _svsmDynamicClearListBuffer; + Renderer::BufferID _svsmDynamicAABBBuffer; + Renderer::BufferID _svsmDataReadBackBuffer; // Diagnostics only (perf editor stats), no rendering decision reads it + Renderer::BufferID _svsmDynamicValidateReadBackBuffer; // svsmValidateDynamic one-shot: dynamic table + free list + Renderer::BufferID _svsmFillArgsBuffer; + Renderer::ImageID _svsmPagePool; + Renderer::ImageID _svsmDynamicPagePool; + Renderer::ImageID _svsmPagePoolPlaceholder; + u32 _svsmDataReadBack[SVSM_DATA_UINT_COUNT] = { 0 }; - mat4x4 _cascadeProjectionMatrices[Renderer::Settings::MAX_SHADOW_CASCADES]; + std::vector _svsmDirtyAABBs; // Static invalidation (min, max) pairs, uploaded by the update pass + std::vector _svsmDynamicAABBs; // This frame's dynamic caster (min, max) pairs + bool _svsmDirtyAABBOverflow = false; + u32 _svsmNumDynamicCasters = 0; // Classifier live set: moved or bone-pushed within the grace window + u32 _svsmCasterTransitionsIn = 0; // Classifier enter/leave running totals, each re-bakes static pages + u32 _svsmCasterTransitionsOut = 0; + u32 _svsmDynamicAABBsDropped = 0; // Spilled to the static path this frame (cap or oversize) + bool _svsmForceInvalidateAll = false; // Set when invalidations were discarded while SVSM was inactive + bool _svsmValidatePending = false; // A validation copy was recorded, Update maps and checks it next frame + bool _svsmPoolNeedsClear = false; + u32 _svsmPoolPages = 0; + u32 _svsmDynamicPoolPages = 0; - ShadowCascadeDebugInformation _cascadeDebugInformation[Renderer::Settings::MAX_SHADOW_CASCADES]; + // The config ResetSVSMPoolState last shaped the free lists and page counts for. pageSize is + // live-editable (a change refills them in place); the pool sizes are restart-only once the + // pool images exist, the Engine cannot destroy images to recreate them at new dimensions + u32 _svsmAppliedPageSize = 0; + u32 _svsmAppliedPoolSize = 0; + u32 _svsmAppliedDynamicPoolSize = 0; - vec3 _boundingBoxMin; - vec3 _boundingBoxMax; + // Caster-toggle cvar states as of last Update: a flip must re-bake the whole static cache, + // resident pages keep the toggled class's baked depth forever otherwise (marked pages never + // age out while visible). Initialized to the cvar defaults so startup does not invalidate + bool _svsmModelsCastShadow = true; + bool _svsmTerrainCastShadow = true; - u32 _currentCascade = 0; -}; \ No newline at end of file + // Night gate: entered after shadowStrength sits at 0 for a sustained second (the dusk + // threshold must not flicker the cache), left immediately with a full re-bake + bool _svsmNightActive = false; + f32 _svsmNightTimer = 0.0f; +}; diff --git a/Source/Game-Lib/Game-Lib/Rendering/Skybox/SkyboxRenderer.cpp b/Source/Game-Lib/Game-Lib/Rendering/Skybox/SkyboxRenderer.cpp index 177a200c..26efeae7 100644 --- a/Source/Game-Lib/Game-Lib/Rendering/Skybox/SkyboxRenderer.cpp +++ b/Source/Game-Lib/Game-Lib/Rendering/Skybox/SkyboxRenderer.cpp @@ -6,11 +6,15 @@ #include "Game-Lib/Util/ServiceLocator.h" #include "Game-Lib/Application/EnttRegistries.h" +#include + #include #include #include +AutoCVar_Int CVAR_SkyboxDrawSun(CVarCategory::Client | CVarCategory::Rendering, "skyboxDrawSun", "draw a sun disc in the skybox at the directional light's position", 1, CVarFlags::EditCheckbox); + SkyboxRenderer::SkyboxRenderer(Renderer::Renderer* renderer, GameRenderer* gameRenderer, DebugRenderer* debugRenderer) : _renderer(renderer) , _gameRenderer(gameRenderer) @@ -74,6 +78,7 @@ void SkyboxRenderer::AddSkyboxPass(Renderer::RenderGraph* renderGraph, RenderRes commandList.BindDescriptorSet(data.globalSet, frameIndex); // Skyband Color Push Constant + _skybandColors.sunDirection.w = static_cast(CVAR_SkyboxDrawSun.Get()); commandList.PushConstant(&_skybandColors, 0, sizeof(SkybandColors)); // NumVertices hardcoded as we use a Fullscreen Triangle (Check FullscreenTriangle.vs for more information) @@ -93,6 +98,12 @@ void SkyboxRenderer::SetSkybandColors(const vec3& skyTopColor, const vec3& skyMi _skybandColors.horizon = vec4(skyHorizonColor, 0.0f); } +void SkyboxRenderer::SetSunDirection(const vec3& directionToSun) +{ + // w is stomped from skyboxDrawSun in AddSkyboxPass every frame + _skybandColors.sunDirection = vec4(glm::normalize(directionToSun), 0.0f); +} + void SkyboxRenderer::CreatePermanentResources() { ZoneScoped; diff --git a/Source/Game-Lib/Game-Lib/Rendering/Skybox/SkyboxRenderer.h b/Source/Game-Lib/Game-Lib/Rendering/Skybox/SkyboxRenderer.h index 453a81ab..31b06113 100644 --- a/Source/Game-Lib/Game-Lib/Rendering/Skybox/SkyboxRenderer.h +++ b/Source/Game-Lib/Game-Lib/Rendering/Skybox/SkyboxRenderer.h @@ -24,6 +24,7 @@ class SkyboxRenderer vec4 bottom = vec4(0.60f, 0.86f, 0.96f, 0.0f); vec4 aboveHorizon = vec4(0.69f, 0.85f, 0.88f, 0.0f); vec4 horizon = vec4(0.71f, 0.71f, 0.71f, 0.0f); + vec4 sunDirection = vec4(0.0f, 1.0f, 0.0f, 0.0f); // xyz = direction to the sun, w = enabled }; public: @@ -35,6 +36,7 @@ class SkyboxRenderer void AddSkyboxPass(Renderer::RenderGraph* renderGraph, RenderResources& resources, u8 frameIndex); void SetSkybandColors(const vec3& skyTopColor, const vec3& skyMiddleColor, const vec3& skyBottomColor, const vec3& skyAboveHorizonColor, const vec3& skyHorizonColor); + void SetSunDirection(const vec3& directionToSun); private: void CreatePermanentResources(); diff --git a/Source/Game-Lib/Game-Lib/Rendering/Terrain/TerrainRenderer.cpp b/Source/Game-Lib/Game-Lib/Rendering/Terrain/TerrainRenderer.cpp index fb0d16b3..4fe4f9b7 100644 --- a/Source/Game-Lib/Game-Lib/Rendering/Terrain/TerrainRenderer.cpp +++ b/Source/Game-Lib/Game-Lib/Rendering/Terrain/TerrainRenderer.cpp @@ -3,6 +3,7 @@ #include "Game-Lib/ECS/Singletons/Database/TextureSingleton.h" #include "Game-Lib/Rendering/RenderUtils.h" #include "Game-Lib/Rendering/GameRenderer.h" +#include "Game-Lib/Rendering/Shadow/ShadowRenderer.h" #include "Game-Lib/Rendering/RenderResources.h" #include "Game-Lib/Rendering/Debug/DebugRenderer.h" #include "Game-Lib/Util/ServiceLocator.h" @@ -42,6 +43,7 @@ TerrainRenderer::TerrainRenderer(Renderer::Renderer* renderer, GameRenderer* gam , _resetIndirectDescriptorSet(Renderer::DescriptorSetSlot::PER_PASS) , _cullingPassDescriptorSet(Renderer::DescriptorSetSlot::PER_PASS) , _geometryFillPassDescriptorSet(Renderer::DescriptorSetSlot::PER_PASS) + , _svsmDrawDescriptorSet(Renderer::DescriptorSetSlot::PER_PASS) { ZoneScoped; @@ -127,18 +129,10 @@ void TerrainRenderer::AddOccluderPass(Renderer::RenderGraph* renderGraph, Render const bool disableTwoStepCulling = CVAR_TerrainDisableTwoStepCulling.Get(); - CVarSystem* cvarSystem = CVarSystem::Get(); - - u32 numCascades = 0; - if (CVAR_TerrainCastShadow.Get() == 1) - { - numCascades = *cvarSystem->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "shadowCascadeNum"); - } - struct Data { Renderer::ImageMutableResource visibilityBuffer; - Renderer::DepthImageMutableResource depth[Renderer::Settings::MAX_VIEWS]; + Renderer::DepthImageMutableResource depth; Renderer::BufferMutableResource culledInstanceBuffer; Renderer::BufferMutableResource culledInstanceBitMaskBuffer; @@ -151,16 +145,12 @@ void TerrainRenderer::AddOccluderPass(Renderer::RenderGraph* renderGraph, Render }; renderGraph->AddPass("Terrain Occluders", - [this, &resources, frameIndex, numCascades](Data& data, Renderer::RenderGraphBuilder& builder) // Setup + [this, &resources, frameIndex](Data& data, Renderer::RenderGraphBuilder& builder) // Setup { using BufferUsage = Renderer::BufferPassUsage; data.visibilityBuffer = builder.Write(resources.visibilityBuffer, Renderer::PipelineType::GRAPHICS, Renderer::LoadMode::LOAD); - data.depth[0] = builder.Write(resources.depth, Renderer::PipelineType::GRAPHICS, Renderer::LoadMode::LOAD); - for (u32 i = 1; i < numCascades + 1; i++) - { - data.depth[i] = builder.Write(resources.shadowDepthCascades[i-1], Renderer::PipelineType::GRAPHICS, Renderer::LoadMode::LOAD); - } + data.depth = builder.Write(resources.depth, Renderer::PipelineType::GRAPHICS, Renderer::LoadMode::LOAD); builder.Read(_instanceDatas.GetBuffer(), BufferUsage::COMPUTE | BufferUsage::GRAPHICS); builder.Read(_vertices.GetBuffer(), BufferUsage::GRAPHICS); @@ -180,7 +170,7 @@ void TerrainRenderer::AddOccluderPass(Renderer::RenderGraph* renderGraph, Render return true; // Return true from setup to enable this pass, return false to disable it }, - [this, &resources, frameIndex, numCascades, disableTwoStepCulling, cullingEnabled, cvarSystem](Data& data, Renderer::RenderGraphResources& graphResources, Renderer::CommandList& commandList) // Execute + [this, &resources, frameIndex, disableTwoStepCulling, cullingEnabled](Data& data, Renderer::RenderGraphResources& graphResources, Renderer::CommandList& commandList) // Execute { GPU_SCOPED_PROFILER_ZONE(commandList, TerrainOccluders); @@ -193,80 +183,49 @@ void TerrainRenderer::AddOccluderPass(Renderer::RenderGraph* renderGraph, Render commandList.BufferBarrier(data.culledInstanceBitMaskBuffer, Renderer::BufferPassUsage::TRANSFER); } - for (u32 i = 0; i < numCascades + 1; i++) + // Reset the counters { - std::string markerName = (i == 0) ? "Main" : "Cascade " + std::to_string(i - 1); - commandList.PushMarker(markerName, Color::White); - - // Reset the counters - { - commandList.BufferBarrier(data.argumentBuffer, Renderer::BufferPassUsage::TRANSFER); - commandList.FillBuffer(data.argumentBuffer, 4, 16, 0); // Reset everything but indexCount to 0 - commandList.BufferBarrier(data.argumentBuffer, Renderer::BufferPassUsage::TRANSFER); - } - - // Fill the occluders to draw - FillDrawCallsParams fillParams; - fillParams.passName = "Occluders"; - fillParams.cellCount = cellCount; - fillParams.viewIndex = i; - fillParams.diffAgainstPrev = false; - fillParams.currentBitmaskIndex = !frameIndex; // Occluders consume last frame's culling output - fillParams.fillSet = data.fillSet; - - FillDrawCalls(frameIndex, graphResources, commandList, fillParams); - commandList.BufferBarrier(data.culledInstanceBuffer, Renderer::BufferPassUsage::COMPUTE); - commandList.BufferBarrier(data.culledInstanceBuffer, Renderer::BufferPassUsage::GRAPHICS); - - // Draw the occluders - if (CVAR_TerrainOccludersEnabled.Get()) - { - commandList.PushMarker("Occlusion Draw", Color::White); - - if (i == 1) - { - uvec2 shadowDepthDimensions = _renderer->GetImageDimensions(resources.shadowDepthCascades[0]); - - commandList.SetViewport(0, 0, static_cast(shadowDepthDimensions.x), static_cast(shadowDepthDimensions.y), 0.0f, 1.0f); - commandList.SetScissorRect(0, shadowDepthDimensions.x, 0, shadowDepthDimensions.y); - - f32 biasConstantFactor = static_cast(*cvarSystem->GetFloatCVar(CVarCategory::Client | CVarCategory::Rendering, "shadowDepthBiasConstant")); - f32 biasClamp = static_cast(*cvarSystem->GetFloatCVar(CVarCategory::Client | CVarCategory::Rendering, "shadowDepthBiasClamp")); - f32 biasSlopeFactor = static_cast(*cvarSystem->GetFloatCVar(CVarCategory::Client | CVarCategory::Rendering, "shadowDepthBiasSlope")); - commandList.SetDepthBias(biasConstantFactor, biasClamp, biasSlopeFactor); - } - - DrawParams drawParams; - drawParams.shadowPass = i != 0; - drawParams.viewIndex = i; - drawParams.cullingEnabled = cullingEnabled; - drawParams.visibilityBuffer = data.visibilityBuffer; - drawParams.depth = data.depth[i]; - drawParams.instanceBuffer = ToBufferResource(data.culledInstanceBuffer); - drawParams.argumentBuffer = ToBufferResource(data.argumentBuffer); + commandList.BufferBarrier(data.argumentBuffer, Renderer::BufferPassUsage::TRANSFER); + commandList.FillBuffer(data.argumentBuffer, 4, 16, 0); // Reset everything but indexCount to 0 + commandList.BufferBarrier(data.argumentBuffer, Renderer::BufferPassUsage::TRANSFER); + } - drawParams.globalDescriptorSet = data.globalSet; - drawParams.drawDescriptorSet = data.drawSet; + // Fill the occluders to draw + FillDrawCallsParams fillParams; + fillParams.markerName = "Occluders Fill"; + fillParams.cellCount = cellCount; + fillParams.viewIndex = 0; + fillParams.diffAgainstPrev = false; + fillParams.currentBitmaskIndex = !frameIndex; // Occluders consume last frame's culling output + fillParams.fillSet = data.fillSet; + + FillDrawCalls(frameIndex, graphResources, commandList, fillParams); + commandList.BufferBarrier(data.culledInstanceBuffer, Renderer::BufferPassUsage::COMPUTE); + commandList.BufferBarrier(data.culledInstanceBuffer, Renderer::BufferPassUsage::GRAPHICS); + + // Draw the occluders + if (CVAR_TerrainOccludersEnabled.Get()) + { + commandList.PushMarker("Occlusion Draw", Color::White); - Draw(resources, frameIndex, graphResources, commandList, drawParams); + DrawParams drawParams; + drawParams.viewIndex = 0; + drawParams.cullingEnabled = cullingEnabled; + drawParams.visibilityBuffer = data.visibilityBuffer; + drawParams.depth = data.depth; + drawParams.instanceBuffer = ToBufferResource(data.culledInstanceBuffer); + drawParams.argumentBuffer = ToBufferResource(data.argumentBuffer); - commandList.PopMarker(); - } + drawParams.globalDescriptorSet = data.globalSet; + drawParams.drawDescriptorSet = data.drawSet; - // Copy drawn count - { - u32 dstOffset = i * sizeof(u32); - commandList.CopyBuffer(data.occluderDrawCountReadBackBuffer, dstOffset, data.argumentBuffer, 4, 4); - } + Draw(resources, frameIndex, graphResources, commandList, drawParams); commandList.PopMarker(); } - // Finish by resetting the viewport, scissor and depth bias - vec2 renderSize = _renderer->GetRenderSize(); - commandList.SetViewport(0, 0, renderSize.x, renderSize.y, 0.0f, 1.0f); - commandList.SetScissorRect(0, static_cast(renderSize.x), 0, static_cast(renderSize.y)); - commandList.SetDepthBias(0, 0, 0); + // Copy drawn count + commandList.CopyBuffer(data.occluderDrawCountReadBackBuffer, 0, data.argumentBuffer, 4, 4); }); } @@ -284,7 +243,7 @@ void TerrainRenderer::AddCullingPass(Renderer::RenderGraph* renderGraph, RenderR if (_instanceDatas.Count() == 0) return; - u32 numCascades = *CVarSystem::Get()->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "shadowCascadeNum"_h); + const u32 numShadowViews = 0; // Main view only, cascades are culled in their own block late in the frame struct Data { @@ -324,7 +283,7 @@ void TerrainRenderer::AddCullingPass(Renderer::RenderGraph* renderGraph, RenderR return true; // Return true from setup to enable this pass, return false to disable it }, - [this, frameIndex, numCascades](Data& data, Renderer::RenderGraphResources& graphResources, Renderer::CommandList& commandList) // Execute + [this, frameIndex, numShadowViews](Data& data, Renderer::RenderGraphResources& graphResources, Renderer::CommandList& commandList) // Execute { GPU_SCOPED_PROFILER_ZONE(commandList, TerrainCulling); @@ -361,43 +320,8 @@ void TerrainRenderer::AddCullingPass(Renderer::RenderGraph* renderGraph, RenderR commandList.BufferBarrier(data.argumentBuffer, Renderer::BufferPassUsage::COMPUTE); // Cull instances on GPU - Renderer::ComputePipelineID pipeline = _cullingPipeline; - commandList.BeginPipeline(pipeline); - - struct CullConstants - { - u32 viewportSizeX; - u32 viewportSizeY; - u32 numCascades; - u32 occlusionEnabled; - u32 bitMaskBufferSizePerView; - u32 currentBitmaskIndex; - }; - - vec2 viewportSize = _renderer->GetRenderSize(); - - CullConstants* cullConstants = graphResources.FrameNew(); - cullConstants->viewportSizeX = u32(viewportSize.x); - cullConstants->viewportSizeY = u32(viewportSize.y); - cullConstants->numCascades = numCascades; - cullConstants->occlusionEnabled = CVAR_OcclusionCullingEnabled.Get(); - const u32 cellCount = static_cast(_cellDatas.Count()); - cullConstants->bitMaskBufferSizePerView = RenderUtils::CalcCullingBitmaskUints(cellCount); - cullConstants->currentBitmaskIndex = frameIndex; - - commandList.PushConstant(cullConstants, 0, sizeof(CullConstants)); - data.cullingSet.Bind("_depthPyramid"_h, data.depthPyramid); - - // Bind descriptorset - commandList.BindDescriptorSet(data.debugSet, frameIndex); - commandList.BindDescriptorSet(data.globalSet, frameIndex); - //commandList.BindDescriptorSet(Renderer::DescriptorSetSlot::SHADOWS, &resources.shadowDescriptorSet, frameIndex); - commandList.BindDescriptorSet(data.cullingSet, frameIndex); - - commandList.Dispatch((cellCount + 31) / 32, 1, 1); - - commandList.EndPipeline(pipeline); + RunCullingDispatch(graphResources, commandList, frameIndex, numShadowViews, CVAR_OcclusionCullingEnabled.Get() != 0, true, data.debugSet, data.globalSet, data.cullingSet); }); } @@ -413,17 +337,10 @@ void TerrainRenderer::AddGeometryPass(Renderer::RenderGraph* renderGraph, Render const bool cullingEnabled = true;//CVAR_TerrainCullingEnabled.Get(); - CVarSystem* cvarSystem = CVarSystem::Get(); - u32 numCascades = 0; - if (CVAR_TerrainCastShadow.Get() == 1) - { - numCascades = *cvarSystem->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "shadowCascadeNum"); - } - struct Data { Renderer::ImageMutableResource visibilityBuffer; - Renderer::DepthImageMutableResource depth[Renderer::Settings::MAX_VIEWS]; + Renderer::DepthImageMutableResource depth; Renderer::BufferMutableResource culledInstanceBuffer; @@ -437,16 +354,12 @@ void TerrainRenderer::AddGeometryPass(Renderer::RenderGraph* renderGraph, Render }; renderGraph->AddPass("Terrain Geometry", - [this, &resources, frameIndex, cullingEnabled, numCascades](Data& data, Renderer::RenderGraphBuilder& builder) + [this, &resources, frameIndex, cullingEnabled](Data& data, Renderer::RenderGraphBuilder& builder) { using BufferUsage = Renderer::BufferPassUsage; data.visibilityBuffer = builder.Write(resources.visibilityBuffer, Renderer::PipelineType::GRAPHICS, Renderer::LoadMode::LOAD); - data.depth[0] = builder.Write(resources.depth, Renderer::PipelineType::GRAPHICS, Renderer::LoadMode::LOAD); - for (u32 i = 1; i < numCascades + 1; i++) - { - data.depth[i] = builder.Write(resources.shadowDepthCascades[i - 1], Renderer::PipelineType::GRAPHICS, Renderer::LoadMode::LOAD); - } + data.depth = builder.Write(resources.depth, Renderer::PipelineType::GRAPHICS, Renderer::LoadMode::LOAD); builder.Write(_culledInstanceBitMaskBuffer.Get(frameIndex), BufferUsage::COMPUTE | BufferUsage::TRANSFER); builder.Write(_culledInstanceBitMaskBuffer.Get(!frameIndex), BufferUsage::COMPUTE | BufferUsage::TRANSFER); @@ -470,71 +383,313 @@ void TerrainRenderer::AddGeometryPass(Renderer::RenderGraph* renderGraph, Render return true; // Return true from setup to enable this pass, return false to disable it }, - [this, &resources, frameIndex, cullingEnabled, numCascades, cvarSystem](Data& data, Renderer::RenderGraphResources& graphResources, Renderer::CommandList& commandList) + [this, &resources, frameIndex, cullingEnabled](Data& data, Renderer::RenderGraphResources& graphResources, Renderer::CommandList& commandList) { GPU_SCOPED_PROFILER_ZONE(commandList, TerrainGeometryPass); - for (u32 i = 0; i < numCascades + 1; i++) + + // Reset the counters + { + commandList.BufferBarrier(data.argumentBuffer, Renderer::BufferPassUsage::TRANSFER); + commandList.FillBuffer(data.argumentBuffer, 4, 16, 0); // Reset everything but indexCount to 0 + commandList.BufferBarrier(data.argumentBuffer, Renderer::BufferPassUsage::TRANSFER); + } + + if (CVAR_TerrainGeometryEnabled.Get()) + { + const u32 cellCount = static_cast(_cellDatas.Count()); + + // Fill the cells to draw + FillDrawCallsParams fillParams; + fillParams.markerName = "Geometry Fill"; + fillParams.cellCount = cellCount; + fillParams.viewIndex = 0; + fillParams.diffAgainstPrev = true; + fillParams.currentBitmaskIndex = frameIndex; + fillParams.fillSet = data.fillSet; + + FillDrawCalls(frameIndex, graphResources, commandList, fillParams); + commandList.BufferBarrier(data.culledInstanceBuffer, Renderer::BufferPassUsage::COMPUTE); + commandList.BufferBarrier(data.culledInstanceBuffer, Renderer::BufferPassUsage::GRAPHICS); + + commandList.PushMarker("Draw", Color::White); + + DrawParams drawParams; + drawParams.viewIndex = 0; + drawParams.cullingEnabled = cullingEnabled; + drawParams.visibilityBuffer = data.visibilityBuffer; + drawParams.depth = data.depth; + drawParams.instanceBuffer = ToBufferResource(data.culledInstanceBuffer); + drawParams.argumentBuffer = ToBufferResource(data.argumentBuffer); + + drawParams.globalDescriptorSet = data.globalSet; + drawParams.drawDescriptorSet = data.geometryPassSet; + + Draw(resources, frameIndex, graphResources, commandList, drawParams); + + commandList.PopMarker(); + } + + if (cullingEnabled) + { + commandList.CopyBuffer(data.drawCountReadBackBuffer, 0, data.argumentBuffer, 4, 4); + } + }); +} + +void TerrainRenderer::AddClipmapCullingPass(Renderer::RenderGraph* renderGraph, RenderResources& resources, u8 frameIndex) +{ + ZoneScoped; + + if (!CVAR_TerrainRendererEnabled.Get()) + return; + + if (_instanceDatas.Count() == 0) + return; + + if (*CVarSystem::Get()->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "shadowEnabled"_h) == 0) + return; + + if (CVAR_TerrainCastShadow.Get() != 1) + return; + + // The same per-view culling the main view uses, run against the clipmap cameras + const u32 numShadowViews = static_cast(glm::clamp(*CVarSystem::Get()->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "svsmNumClipmaps"_h), i32(1), i32(ShadowRenderer::SVSM_MAX_CLIPMAPS))); + + struct Data + { + Renderer::ImageResource depthPyramid; + + Renderer::BufferMutableResource currentInstanceBitMaskBuffer; + + Renderer::DescriptorSetResource debugSet; + Renderer::DescriptorSetResource globalSet; + Renderer::DescriptorSetResource cullingSet; + }; + + renderGraph->AddPass("Terrain Clipmap Culling", + [this, &resources, frameIndex](Data& data, Renderer::RenderGraphBuilder& builder) // Setup + { + using BufferUsage = Renderer::BufferPassUsage; + + data.depthPyramid = builder.Read(resources.depthPyramid, Renderer::PipelineType::COMPUTE); + + builder.Read(resources.cameras.GetBuffer(), BufferUsage::COMPUTE); + builder.Read(_instanceDatas.GetBuffer(), BufferUsage::COMPUTE); + builder.Read(_cellHeightRanges.GetBuffer(), BufferUsage::COMPUTE); + + data.currentInstanceBitMaskBuffer = builder.Write(_culledInstanceBitMaskBuffer.Get(frameIndex), BufferUsage::COMPUTE); + builder.Write(_culledInstanceBitMaskBuffer.Get(!frameIndex), BufferUsage::COMPUTE); // Both bitmask bindings are RW in the shader + builder.Write(_culledInstanceBuffer, BufferUsage::COMPUTE); // Not written by the cascade dispatch, but bound RW in the culling set + + data.debugSet = builder.Use(_debugRenderer->GetDebugDescriptorSet()); + data.globalSet = builder.Use(resources.globalDescriptorSet); + data.cullingSet = builder.Use(_cullingPassDescriptorSet); + + _debugRenderer->RegisterCullingPassBufferUsage(builder); + + return true; // Return true from setup to enable this pass, return false to disable it + }, + [this, frameIndex, numShadowViews](Data& data, Renderer::RenderGraphResources& graphResources, Renderer::CommandList& commandList) // Execute + { + GPU_SCOPED_PROFILER_ZONE(commandList, TerrainClipmapCulling); + + // Frustum-only cull of the clipmap views into their bitmask slices, no occlusion. + // _depthPyramid stays bound from the main culling pass, rebinding here would rewrite + // an already-bound set + RunCullingDispatch(graphResources, commandList, frameIndex, numShadowViews, false, false, data.debugSet, data.globalSet, data.cullingSet); + }); +} + +void TerrainRenderer::RunCullingDispatch(Renderer::RenderGraphResources& graphResources, Renderer::CommandList& commandList, u8 frameIndex, u32 numShadowViews, bool occlusionEnabled, bool cullMainView, Renderer::DescriptorSetResource& debugSet, Renderer::DescriptorSetResource& globalSet, Renderer::DescriptorSetResource& cullingSet) +{ + Renderer::ComputePipelineID pipeline = _cullingPipeline; + commandList.BeginPipeline(pipeline); + + struct CullConstants + { + u32 viewportSizeX; + u32 viewportSizeY; + u32 numShadowViews; + u32 occlusionEnabled; + u32 bitMaskBufferSizePerView; + u32 currentBitmaskIndex; + u32 debugDrawView; + u32 cullMainView; + }; + + vec2 viewportSize = _renderer->GetRenderSize(); + + const u32 cellCount = static_cast(_cellDatas.Count()); + + CullConstants* cullConstants = graphResources.FrameNew(); + cullConstants->viewportSizeX = u32(viewportSize.x); + cullConstants->viewportSizeY = u32(viewportSize.y); + cullConstants->numShadowViews = numShadowViews; + cullConstants->occlusionEnabled = occlusionEnabled; + cullConstants->bitMaskBufferSizePerView = RenderUtils::CalcCullingBitmaskUints(cellCount); + cullConstants->currentBitmaskIndex = frameIndex; + cullConstants->debugDrawView = static_cast(*CVarSystem::Get()->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "shadowDebugCullingView"_h)); + cullConstants->cullMainView = cullMainView; + + commandList.PushConstant(cullConstants, 0, sizeof(CullConstants)); + + commandList.BindDescriptorSet(debugSet, frameIndex); + commandList.BindDescriptorSet(globalSet, frameIndex); + commandList.BindDescriptorSet(cullingSet, frameIndex); + + commandList.Dispatch((cellCount + 31) / 32, 1, 1); + + commandList.EndPipeline(pipeline); +} + +void TerrainRenderer::AddSVSMGeometryPass(Renderer::RenderGraph* renderGraph, RenderResources& resources, u8 frameIndex, ShadowRenderer* shadowRenderer) +{ + ZoneScoped; + + if (!CVAR_TerrainRendererEnabled.Get()) + return; + + if (_instanceDatas.Count() == 0) + return; + + CVarSystem* cvarSystem = CVarSystem::Get(); + + if (*cvarSystem->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "shadowEnabled"_h) == 0) + return; + + if (CVAR_TerrainCastShadow.Get() != 1) + return; + + if (shadowRenderer->GetSVSMPagePool() == Renderer::ImageID::Invalid()) + return; + + const u32 numClipmaps = static_cast(glm::clamp(*cvarSystem->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "svsmNumClipmaps"_h), i32(1), i32(ShadowRenderer::SVSM_MAX_CLIPMAPS))); + const u32 virtualSize = static_cast(*cvarSystem->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "svsmVirtualSize"_h)); + + struct Data + { + Renderer::ImageMutableResource pagePool; + Renderer::BufferResource svsmData; + Renderer::BufferResource pageTable; + Renderer::BufferResource svsmFillArgsBuffer; + + Renderer::BufferMutableResource culledInstanceBuffer; + + Renderer::BufferMutableResource argumentBuffer; + Renderer::BufferMutableResource drawCountReadBackBuffer; + + Renderer::DescriptorSetResource globalSet; + Renderer::DescriptorSetResource fillSet; + Renderer::DescriptorSetResource geometryPassSet; + Renderer::DescriptorSetResource svsmSet; + }; + + renderGraph->AddPass("Terrain SVSM Geometry", + [this, &resources, frameIndex, shadowRenderer](Data& data, Renderer::RenderGraphBuilder& builder) // Setup + { + using BufferUsage = Renderer::BufferPassUsage; + + data.pagePool = builder.Write(shadowRenderer->GetSVSMPagePool(), Renderer::PipelineType::GRAPHICS, Renderer::LoadMode::LOAD); + data.svsmData = builder.Read(shadowRenderer->GetSVSMDataBuffer(), BufferUsage::GRAPHICS); + data.pageTable = builder.Read(shadowRenderer->GetSVSMPageTableBuffer(), BufferUsage::GRAPHICS); + data.svsmFillArgsBuffer = builder.Read(shadowRenderer->GetSVSMFillArgsBuffer(), BufferUsage::COMPUTE); // Consumed read-only by DispatchIndirect in the per-view fills + + builder.Write(_culledInstanceBitMaskBuffer.Get(frameIndex), BufferUsage::COMPUTE | BufferUsage::TRANSFER); + builder.Write(_culledInstanceBitMaskBuffer.Get(!frameIndex), BufferUsage::COMPUTE | BufferUsage::TRANSFER); + data.culledInstanceBuffer = builder.Write(_culledInstanceBuffer, BufferUsage::GRAPHICS | BufferUsage::COMPUTE); + + builder.Read(resources.cameras.GetBuffer(), BufferUsage::GRAPHICS); + builder.Read(_vertices.GetBuffer(), BufferUsage::GRAPHICS); + builder.Read(_cellDatas.GetBuffer(), BufferUsage::GRAPHICS); + builder.Read(_chunkDatas.GetBuffer(), BufferUsage::GRAPHICS); + builder.Read(_instanceDatas.GetBuffer(), BufferUsage::COMPUTE | BufferUsage::GRAPHICS); + + data.argumentBuffer = builder.Write(_argumentBuffer, BufferUsage::TRANSFER | BufferUsage::GRAPHICS | BufferUsage::COMPUTE); + data.drawCountReadBackBuffer = builder.Write(_drawCountReadBackBuffer, BufferUsage::TRANSFER); + + data.globalSet = builder.Use(resources.globalDescriptorSet); + data.fillSet = builder.Use(_geometryFillPassDescriptorSet); + data.geometryPassSet = builder.Use(resources.terrainDescriptorSet); + data.svsmSet = builder.Use(_svsmDrawDescriptorSet); + + return true; // Return true from setup to enable this pass, return false to disable it + }, + [this, &resources, frameIndex, numClipmaps, virtualSize](Data& data, Renderer::RenderGraphResources& graphResources, Renderer::CommandList& commandList) + { + GPU_SCOPED_PROFILER_ZONE(commandList, TerrainSVSMGeometry); + + const u32 cellCount = static_cast(_cellDatas.Count()); + + // svsmProfileGeometry: per-view fill/draw GPU timings for the perf editor's render pass list + const bool profileSVSM = *CVarSystem::Get()->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "svsmProfileGeometry"_h) != 0; + RenderUtils::SVSMGeometryProfiler Profiled(_renderer, commandList, "Terrain SVSM", profileSVSM); + + // _svsmData/_pageTable are bound once at init through BindSVSMBuffers, only the lazily + // created pool binds per frame (image binds write the descriptor immediately) + data.svsmSet.Bind("_pagePool"_h, data.pagePool); + + // View-invariant fill params, cvar reads and marker strings — the loop body runs per clipmap + const bool svsmClipRects = *CVarSystem::Get()->GetIntCVar(CVarCategory::Client | CVarCategory::Rendering, "svsmClipRects"_h) != 0; + + FillDrawCallsParams fillParams; + fillParams.markerName = "SVSM Geometry Fill"; + fillParams.cellCount = cellCount; + fillParams.diffAgainstPrev = false; + fillParams.currentBitmaskIndex = frameIndex; + fillParams.fillSet = data.fillSet; + fillParams.fillArgsBuffer = data.svsmFillArgsBuffer; + + for (u32 i = 1; i < numClipmaps + 1; i++) { - std::string markerName = (i == 0) ? "Main" : "Cascade " + std::to_string(i - 1); + std::string markerName = "Clipmap " + std::to_string(i - 1); commandList.PushMarker(markerName, Color::White); - // Reset the counters + // Reset the counters. The previous view's draw consumed the args at DRAW_INDIRECT + // and the fill wrote them in compute — the reset must wait on both (WAR/WAW) { - commandList.BufferBarrier(data.argumentBuffer, Renderer::BufferPassUsage::TRANSFER); + commandList.BufferBarrier(data.argumentBuffer, Renderer::BufferPassUsage::GRAPHICS | Renderer::BufferPassUsage::COMPUTE | Renderer::BufferPassUsage::TRANSFER); commandList.FillBuffer(data.argumentBuffer, 4, 16, 0); // Reset everything but indexCount to 0 commandList.BufferBarrier(data.argumentBuffer, Renderer::BufferPassUsage::TRANSFER); } if (CVAR_TerrainGeometryEnabled.Get()) { - const u32 cellCount = static_cast(_cellDatas.Count()); - - // Fill the occluders to draw - FillDrawCallsParams fillParams; - fillParams.passName = "Geometry"; - fillParams.cellCount = cellCount; + // Finalize zeroed the group count for rings with no dirty pages this frame fillParams.viewIndex = i; - fillParams.diffAgainstPrev = true; - fillParams.currentBitmaskIndex = frameIndex; - fillParams.fillSet = data.fillSet; + fillParams.fillArgsByteOffset = (i - 1) * ShadowRenderer::SVSM_FILL_ARGS_VIEW_STRIDE + ShadowRenderer::SVSM_FILL_ARGS_TERRAIN_OFFSET; - FillDrawCalls(frameIndex, graphResources, commandList, fillParams); + Profiled("Fill", i, [&] { FillDrawCalls(frameIndex, graphResources, commandList, fillParams); }); commandList.BufferBarrier(data.culledInstanceBuffer, Renderer::BufferPassUsage::COMPUTE); commandList.BufferBarrier(data.culledInstanceBuffer, Renderer::BufferPassUsage::GRAPHICS); if (i == 1) { - uvec2 shadowDepthDimensions = _renderer->GetImageDimensions(resources.shadowDepthCascades[0]); - - commandList.SetViewport(0, 0, static_cast(shadowDepthDimensions.x), static_cast(shadowDepthDimensions.y), 0.0f, 1.0f); - commandList.SetScissorRect(0, shadowDepthDimensions.x, 0, shadowDepthDimensions.y); - - f32 biasConstantFactor = static_cast(*cvarSystem->GetFloatCVar(CVarCategory::Client | CVarCategory::Rendering, "shadowDepthBiasConstant")); - f32 biasClamp = static_cast(*cvarSystem->GetFloatCVar(CVarCategory::Client | CVarCategory::Rendering, "shadowDepthBiasClamp")); - f32 biasSlopeFactor = static_cast(*cvarSystem->GetFloatCVar(CVarCategory::Client | CVarCategory::Rendering, "shadowDepthBiasSlope")); - commandList.SetDepthBias(biasConstantFactor, biasClamp, biasSlopeFactor); + // The viewport spans the virtual texture so SV_Position.xy is the virtual + // texel. No depth bias: there is no depth attachment for it to apply to + commandList.SetViewport(0, 0, static_cast(virtualSize), static_cast(virtualSize), 0.0f, 1.0f); + commandList.SetScissorRect(0, virtualSize, 0, virtualSize); } commandList.PushMarker("Draw", Color::White); DrawParams drawParams; - drawParams.shadowPass = i != 0; + drawParams.svsmPass = true; drawParams.viewIndex = i; - drawParams.cullingEnabled = cullingEnabled; - drawParams.visibilityBuffer = data.visibilityBuffer; - drawParams.depth = data.depth[i]; + drawParams.cullingEnabled = true; + drawParams.svsmExtent = uvec2(virtualSize, virtualSize); drawParams.instanceBuffer = ToBufferResource(data.culledInstanceBuffer); drawParams.argumentBuffer = ToBufferResource(data.argumentBuffer); drawParams.globalDescriptorSet = data.globalSet; drawParams.drawDescriptorSet = data.geometryPassSet; + drawParams.svsmDescriptorSet = data.svsmSet; - Draw(resources, frameIndex, graphResources, commandList, drawParams); + Profiled("Draw", i, [&] { RenderUtils::DrawSVSMClipRects(svsmClipRects, drawParams, [&](DrawParams& rectDrawParams) { Draw(resources, frameIndex, graphResources, commandList, rectDrawParams); }); }); commandList.PopMarker(); } - if (cullingEnabled) + // Copy drawn count { u32 dstOffset = i * sizeof(u32); commandList.CopyBuffer(data.drawCountReadBackBuffer, dstOffset, data.argumentBuffer, 4, 4); @@ -543,14 +698,19 @@ void TerrainRenderer::AddGeometryPass(Renderer::RenderGraph* renderGraph, Render commandList.PopMarker(); } - // Finish by resetting the viewport, scissor and depth bias + // Finish by resetting the viewport and scissor vec2 renderSize = _renderer->GetRenderSize(); commandList.SetViewport(0, 0, renderSize.x, renderSize.y, 0.0f, 1.0f); commandList.SetScissorRect(0, static_cast(renderSize.x), 0, static_cast(renderSize.y)); - commandList.SetDepthBias(0, 0, 0); }); } +void TerrainRenderer::BindSVSMBuffers(Renderer::BufferID svsmDataBuffer, Renderer::BufferID pageTableBuffer) +{ + _svsmDrawDescriptorSet.Bind("_svsmData"_h, svsmDataBuffer); + _svsmDrawDescriptorSet.Bind("_pageTable"_h, pageTableBuffer); +} + void TerrainRenderer::Clear() { ZoneScoped; @@ -1036,7 +1196,7 @@ void TerrainRenderer::CreatePipelines() std::vector permutationFields = { { "EDITOR_PASS", "0" }, - { "SHADOW_PASS", "0" } + { "SVSM_PASS", "0" } }; u32 shaderEntryNameHash = Renderer::GetShaderEntryNameHash("Terrain/Draw.vs", permutationFields); vertexShaderDesc.shaderEntry = _gameRenderer->GetShaderEntry(shaderEntryNameHash, "Terrain/Draw.vs"); @@ -1064,10 +1224,11 @@ void TerrainRenderer::CreatePipelines() _drawPipeline = _renderer->CreatePipeline(pipelineDesc); } - // Draw shadow + // Draw SVSM pages: attachment-less, the pixel shader writes depth into the page pool with + // image atomics. Depth clamp stays on so pancaked casters rasterize instead of clipping { Renderer::GraphicsPipelineDesc pipelineDesc; - pipelineDesc.debugName = "Terrain Draw Shadow"; + pipelineDesc.debugName = "Terrain Draw SVSM"; // Shaders Renderer::VertexShaderDesc vertexShaderDesc; @@ -1075,28 +1236,25 @@ void TerrainRenderer::CreatePipelines() std::vector permutationFields = { { "EDITOR_PASS", "0" }, - { "SHADOW_PASS", "1" } + { "SVSM_PASS", "1" } }; u32 shaderEntryNameHash = Renderer::GetShaderEntryNameHash("Terrain/Draw.vs", permutationFields); vertexShaderDesc.shaderEntry = _gameRenderer->GetShaderEntry(shaderEntryNameHash, "Terrain/Draw.vs"); pipelineDesc.states.vertexShader = _renderer->LoadShader(vertexShaderDesc); } - // Depth state - pipelineDesc.states.depthStencilState.depthEnable = true; - pipelineDesc.states.depthStencilState.depthWriteEnable = true; - pipelineDesc.states.depthStencilState.depthFunc = Renderer::ComparisonFunc::GREATER; + Renderer::PixelShaderDesc pixelShaderDesc; + { + pixelShaderDesc.shaderEntry = _gameRenderer->GetShaderEntry("Terrain/DrawSVSM.ps"_h, "Terrain/DrawSVSM.ps"); + pipelineDesc.states.pixelShader = _renderer->LoadShader(pixelShaderDesc); + } - // Rasterizer state + // Rasterizer state, no depth or color attachments pipelineDesc.states.rasterizerState.cullMode = Renderer::CullMode::NONE; pipelineDesc.states.rasterizerState.frontFaceMode = Renderer::Settings::FRONT_FACE_STATE; - pipelineDesc.states.rasterizerState.depthBiasEnabled = true; pipelineDesc.states.rasterizerState.depthClampEnabled = true; - // Render targets - pipelineDesc.states.depthStencilFormat = Renderer::DepthImageFormat::D32_FLOAT; - - _drawShadowPipeline = _renderer->CreatePipeline(pipelineDesc); + _drawSVSMPipeline = _renderer->CreatePipeline(pipelineDesc); } } @@ -1113,6 +1271,9 @@ void TerrainRenderer::InitDescriptorSets() _geometryFillPassDescriptorSet.RegisterPipeline(_renderer, _fillDrawCallsPipeline); _geometryFillPassDescriptorSet.Init(_renderer); + + _svsmDrawDescriptorSet.RegisterPipeline(_renderer, _drawSVSMPipeline); + _svsmDrawDescriptorSet.Init(_renderer); } void TerrainRenderer::SyncToGPU() @@ -1199,15 +1360,21 @@ void TerrainRenderer::Draw(const RenderResources& resources, u8 frameIndex, Rend graphResources.InitializeRenderPassDesc(renderPassDesc); // Render targets - if (!params.shadowPass) + if (params.svsmPass) + { + // Attachment-less: with no render targets BeginRenderPass has no extent fallback, + // the render area must be explicit or it collapses to zero + renderPassDesc.extent = params.svsmExtent; + } + else { renderPassDesc.renderTargets[0] = params.visibilityBuffer; + renderPassDesc.depthStencil = params.depth; } - renderPassDesc.depthStencil = params.depth; commandList.BeginRenderPass(renderPassDesc); // Set pipeline - Renderer::GraphicsPipelineID pipeline = _drawPipeline; + Renderer::GraphicsPipelineID pipeline = params.svsmPass ? _drawSVSMPipeline : _drawPipeline; commandList.BeginPipeline(pipeline); // Set index buffer @@ -1216,11 +1383,16 @@ void TerrainRenderer::Draw(const RenderResources& resources, u8 frameIndex, Rend struct PushConstants { u32 viewIndex; + u32 svsmRectIndex; }; PushConstants* constants = graphResources.FrameNew(); constants->viewIndex = params.viewIndex; - commandList.PushConstant(constants, 0, sizeof(PushConstants)); + constants->svsmRectIndex = params.svsmRectIndex; + + // Only the SVSM vertex permutation declares the rect index, the other pipelines' push range + // is a single uint + commandList.PushConstant(constants, 0, params.svsmPass ? sizeof(PushConstants) : sizeof(u32)); // Bind descriptors params.drawDescriptorSet.Bind("_culledInstanceDatas"_h, params.instanceBuffer); @@ -1228,6 +1400,10 @@ void TerrainRenderer::Draw(const RenderResources& resources, u8 frameIndex, Rend // Bind descriptorset commandList.BindDescriptorSet(params.globalDescriptorSet, frameIndex); commandList.BindDescriptorSet(params.drawDescriptorSet, frameIndex); + if (params.svsmPass) + { + commandList.BindDescriptorSet(params.svsmDescriptorSet, frameIndex); + } if (params.cullingEnabled) { @@ -1246,7 +1422,7 @@ void TerrainRenderer::Draw(const RenderResources& resources, u8 frameIndex, Rend void TerrainRenderer::FillDrawCalls(u8 frameIndex, Renderer::RenderGraphResources& graphResources, Renderer::CommandList& commandList, FillDrawCallsParams& params) { - commandList.PushMarker(params.passName + " Fill", Color::White); + commandList.PushMarker(params.markerName, Color::White); Renderer::ComputePipelineID pipeline = _fillDrawCallsPipeline; commandList.BeginPipeline(pipeline); @@ -1271,7 +1447,14 @@ void TerrainRenderer::FillDrawCalls(u8 frameIndex, Renderer::RenderGraphResource // Bind descriptorset commandList.BindDescriptorSet(params.fillSet, frameIndex); - commandList.Dispatch((params.cellCount + 31) / 32, 1, 1); + if (params.fillArgsBuffer != Renderer::BufferResource::Invalid()) + { + commandList.DispatchIndirect(params.fillArgsBuffer, params.fillArgsByteOffset); + } + else + { + commandList.Dispatch((params.cellCount + 31) / 32, 1, 1); + } commandList.EndPipeline(pipeline); commandList.PopMarker(); diff --git a/Source/Game-Lib/Game-Lib/Rendering/Terrain/TerrainRenderer.h b/Source/Game-Lib/Game-Lib/Rendering/Terrain/TerrainRenderer.h index e34b11b0..7881cf93 100644 --- a/Source/Game-Lib/Game-Lib/Rendering/Terrain/TerrainRenderer.h +++ b/Source/Game-Lib/Game-Lib/Rendering/Terrain/TerrainRenderer.h @@ -13,6 +13,7 @@ class DebugRenderer; class GameRenderer; +class ShadowRenderer; struct RenderResources; namespace Renderer @@ -48,6 +49,12 @@ class TerrainRenderer void AddOccluderPass(Renderer::RenderGraph* renderGraph, RenderResources& resources, u8 frameIndex); void AddCullingPass(Renderer::RenderGraph* renderGraph, RenderResources& resources, u8 frameIndex); void AddGeometryPass(Renderer::RenderGraph* renderGraph, RenderResources& resources, u8 frameIndex); + void AddClipmapCullingPass(Renderer::RenderGraph* renderGraph, RenderResources& resources, u8 frameIndex); // Per-clipmap-view frustum culling into the bitmask slices + void AddSVSMGeometryPass(Renderer::RenderGraph* renderGraph, RenderResources& resources, u8 frameIndex, ShadowRenderer* shadowRenderer); + + // Called once by ShadowRenderer at init, buffer binds must happen before the first frame + // (they only reach the canonical descriptor copies at a later FlipFrame) + void BindSVSMBuffers(Renderer::BufferID svsmDataBuffer, Renderer::BufferID pageTableBuffer); void Clear(); void Reserve(u32 numChunks); @@ -78,18 +85,21 @@ class TerrainRenderer struct DrawParams { public: - bool shadowPass = false; + bool svsmPass = false; // Attachment-less page render, needs an explicit render area + u32 svsmRectIndex = 0xFFFFFFFFu; // Clip rect this draw renders (0-2), SVSM_CLIP_RECT_DISABLED = no clipping u32 viewIndex = 0; bool cullingEnabled = false; Renderer::ImageMutableResource visibilityBuffer; Renderer::DepthImageMutableResource depth; + uvec2 svsmExtent = uvec2(0, 0); Renderer::BufferResource instanceBuffer; Renderer::BufferResource argumentBuffer; Renderer::DescriptorSetResource globalDescriptorSet; Renderer::DescriptorSetResource drawDescriptorSet; + Renderer::DescriptorSetResource svsmDescriptorSet; u32 argumentsIndex = 0; }; @@ -98,7 +108,7 @@ class TerrainRenderer struct FillDrawCallsParams { public: - std::string passName; + std::string markerName; // Full marker string, prebuilt by the caller (the SVSM pass fills per view) u32 cellCount; u32 viewIndex; @@ -106,9 +116,19 @@ class TerrainRenderer u32 currentBitmaskIndex = 0; Renderer::DescriptorSetResource fillSet; + + // When set the fill dispatches indirectly from these GPU-written args (SVSM per-view + // gating), zero groups for views with no page work this frame + Renderer::BufferResource fillArgsBuffer; // Finalize-written dispatch args, consumed read-only via DispatchIndirect + u32 fillArgsByteOffset = 0; }; void FillDrawCalls(u8 frameIndex, Renderer::RenderGraphResources& graphResources, Renderer::CommandList& commandList, FillDrawCallsParams& params); + // The terrain cull dispatch shared by AddCullingPass and AddClipmapCullingPass: one thread + // per cell against the per-view frustums, writing the bitmask slices. The depth pyramid bind + // stays at the main culling callsite (the clipmap pass reuses the already-bound set) + void RunCullingDispatch(Renderer::RenderGraphResources& graphResources, Renderer::CommandList& commandList, u8 frameIndex, u32 numShadowViews, bool occlusionEnabled, bool cullMainView, Renderer::DescriptorSetResource& debugSet, Renderer::DescriptorSetResource& globalSet, Renderer::DescriptorSetResource& cullingSet); + private: struct TerrainVertex { @@ -155,7 +175,7 @@ class TerrainRenderer Renderer::ComputePipelineID _fillDrawCallsPipeline; Renderer::ComputePipelineID _cullingPipeline; Renderer::GraphicsPipelineID _drawPipeline; - Renderer::GraphicsPipelineID _drawShadowPipeline; + Renderer::GraphicsPipelineID _drawSVSMPipeline; Renderer::GPUVector _cellIndices; Renderer::GPUVector _vertices; @@ -188,6 +208,7 @@ class TerrainRenderer Renderer::DescriptorSet _resetIndirectDescriptorSet; Renderer::DescriptorSet _cullingPassDescriptorSet; Renderer::DescriptorSet _geometryFillPassDescriptorSet; + Renderer::DescriptorSet _svsmDrawDescriptorSet; std::vector _cellBoundingBoxes; std::vector _chunkBoundingBoxes; diff --git a/Source/Resources/Scripts/Editor/ClockEditor.luau b/Source/Resources/Scripts/Editor/ClockEditor.luau index 656e3ffb..da3508ae 100644 --- a/Source/Resources/Scripts/Editor/ClockEditor.luau +++ b/Source/Resources/Scripts/Editor/ClockEditor.luau @@ -5,6 +5,9 @@ Layer 1100 sits above the dev top bar (1000) so the window draws over the bar and open dropdowns. Speed control is +/- step buttons rather than a slider: the Slider widget's inner-panel anchoring doesn't compose with non-default anchors on the slider itself, follow-up once NumericInput exists. + +Speed steps are x10 per click: the day is 86400 real seconds, so additive +steps around 1x are imperceptible. 0x (paused) sits below 1x on the way down. ]] local EditorButton = require("@src/API/UI/EditorButton") @@ -20,7 +23,7 @@ local ROW_H = 20 local SIDE_PAD = 6 local ROW3_PAD = 5 local BTN_TEXT_TPL = "DevToolsText" -local SPEED_STEP = 0.25 +local SPEED_FACTOR = 10.0 local LAYER = 1100 local WINDOW_BG_COLOR = vector.create(0.10, 0.10, 0.12) @@ -53,7 +56,7 @@ local function FormatTime(seconds) end local function FormatSpeed(speed) - return string.format("Speed: %.2fx", speed) + return string.format("Speed: %gx", speed) end local canvas = UI.GetCanvas("ClockEditor", 0, 0, 1920, 1080, false) @@ -121,7 +124,7 @@ speedUpBtn.text:SetPosX(1) -- Row 3: Speed readout (left) and HH:MM:SS readout (right). local row3Y = row2Y - ROW_H - ROW3_PAD -local speedText = windowPanel:NewText("Speed: 1.00x", SIDE_PAD, row3Y, 1, BTN_TEXT_TPL) +local speedText = windowPanel:NewText("Speed: 1x", SIDE_PAD, row3Y, 1, BTN_TEXT_TPL) speedText:SetAnchor(0.0, 1.0) speedText:SetRelativePoint(0.0, 1.0) @@ -142,11 +145,16 @@ resetSpeedBtn:SetOnMouseUp(function() RefreshSpeedText() end) speedDownBtn:SetOnMouseUp(function() - Time.SetSpeedModifier(math.max(0.0, Time.GetSpeedModifier() - SPEED_STEP)) + local speed = Time.GetSpeedModifier() / SPEED_FACTOR + if speed < 1.0 then + speed = 0.0 -- Pause instead of crawling below 1x + end + Time.SetSpeedModifier(speed) RefreshSpeedText() end) speedUpBtn:SetOnMouseUp(function() - Time.SetSpeedModifier(Time.GetSpeedModifier() + SPEED_STEP) + local speed = math.max(1.0, Time.GetSpeedModifier() * SPEED_FACTOR) + Time.SetSpeedModifier(math.min(speed, Time.SecondsPerDay)) -- Cap at a full day per real second RefreshSpeedText() end) diff --git a/Source/Shaders/Shaders/DescriptorSet/Light.inc.slang b/Source/Shaders/Shaders/DescriptorSet/Light.inc.slang index a2e51a31..cfcfc269 100644 --- a/Source/Shaders/Shaders/DescriptorSet/Light.inc.slang +++ b/Source/Shaders/Shaders/DescriptorSet/Light.inc.slang @@ -1,17 +1,19 @@ #ifndef LIGHT_SET_INCLUDED #define LIGHT_SET_INCLUDED -#ifndef MAX_SHADOW_CASCADES -#define MAX_SHADOW_CASCADES 8 // Has to be kept in sync with the one in RenderSettings.h -#endif +#include "Shadows/SVSM.inc.slang" // WARNING: For now, when you change this file, make sure you recompile ALL shaders or you might get descriptor set reflection issues // This will be fixed later when we have better dependency tracking [[vk::binding(0, LIGHT)]] StructuredBuffer _entityTiles; -[[vk::binding(1, LIGHT)]] SamplerComparisonState _shadowCmpSampler; -[[vk::binding(2, LIGHT)]] SamplerState _shadowPointClampSampler; -[[vk::binding(3, LIGHT)]] Texture2D _shadowCascadeRTs[MAX_SHADOW_CASCADES]; -[[vk::binding(4, LIGHT)]] StructuredBuffer _packedDecals; // All decals in the world +[[vk::binding(1, LIGHT)]] StructuredBuffer _packedDecals; // All decals in the world -#endif // LIGHT_SET_INCLUDED \ No newline at end of file +// SVSM sampling resources +[[vk::binding(2, LIGHT)]] StructuredBuffer _svsmData; +[[vk::binding(3, LIGHT)]] StructuredBuffer _svsmPageTable; +[[vk::binding(4, LIGHT)]] Texture2D _svsmPagePool; +[[vk::binding(5, LIGHT)]] StructuredBuffer _svsmDynamicPageTable; +[[vk::binding(6, LIGHT)]] Texture2D _svsmDynamicPagePool; + +#endif // LIGHT_SET_INCLUDED diff --git a/Source/Shaders/Shaders/Include/Culling.inc.slang b/Source/Shaders/Shaders/Include/Culling.inc.slang index 6116e86d..8dbfebf3 100644 --- a/Source/Shaders/Shaders/Include/Culling.inc.slang +++ b/Source/Shaders/Shaders/Include/Culling.inc.slang @@ -10,8 +10,6 @@ struct AABBMinMax float3 max; }; -#define NUM_CULL_VIEWS 1 + MAX_SHADOW_CASCADES // Main view plus max number of cascades - // Based on the work of WickedEngine, thank you! https://github.com/turanszkij/WickedEngine/ struct Plane { diff --git a/Source/Shaders/Shaders/Include/Lighting.inc.slang b/Source/Shaders/Shaders/Include/Lighting.inc.slang index d94de990..5e8eb589 100644 --- a/Source/Shaders/Shaders/Include/Lighting.inc.slang +++ b/Source/Shaders/Shaders/Include/Lighting.inc.slang @@ -25,16 +25,27 @@ DirectionalLight LoadDirectionalLight(uint index) return light; } -float3 ApplyLighting(float2 uv, float3 materialColor, PixelVertexData pixelVertexData, uint4 lightInfo, ShadowSettings shadowSettings) +float3 ApplyLighting(float3 materialColor, PixelVertexData pixelVertexData, uint4 lightInfo, ShadowSettings shadowSettings) { uint numDirectionalLights = lightInfo.x; - uint numCascades = lightInfo.z; float3 ambientColor = float3(0.0f, 0.0f, 0.0f); float3 directionalColor = float3(0.0f, 0.0f, 0.0f); float ambientOcclusion = _ambientOcclusion.Load(int3(pixelVertexData.pixelPos, 0)).r; + // Directional Light Shadows: clipmap selection, page walk and far fade all live in + // GetSVSMShadowFactor. Light-independent (the sun direction comes from the SVSM state, not + // the light), so it samples once outside the loop — it is the most expensive part of the pass + float shadowFactor = 1.0f; + if (shadowSettings.enableShadows && numDirectionalLights > 0) + { + shadowFactor = GetSVSMShadowFactor(pixelVertexData.worldPos, pixelVertexData.worldNormal, shadowSettings); + + // Sun elevation strength, fades shadows out around dawn/dusk + shadowFactor = lerp(1.0f, shadowFactor, shadowSettings.strength); + } + for (uint i = 0; i < numDirectionalLights; i++) { DirectionalLight light = LoadDirectionalLight(i); @@ -50,18 +61,20 @@ float3 ApplyLighting(float2 uv, float3 materialColor, PixelVertexData pixelVerte // Directional Light float nDotL = saturate(dot(pixelVertexData.worldNormal, light.direction.xyz)); // Dot product between normal and light direction - float3 lightColor = light.color.rgb * light.color.a * nDotL; - - // Directional Light Shadows + + // The directional light fades out with the sun's elevation (shadowSettings.strength is + // clamp(sunElevationSin / 0.1), same ramp the shadow fade rides): below the horizon a + // Lambert term would light down-facing surfaces from underneath, ambient/skyband carry + // the night look instead. Like the shadow lerp below, this assumes light 0 is the sun + float3 lightColor = light.color.rgb * light.color.a * nDotL * shadowSettings.strength; + + // Directional Light Shadows: shadowColor is a multiplicative tint on the directional + // term (fully shadowed keeps shadowColor's fraction of it), never a replacement — a + // constant floor exceeds the Lambert term at low sun/grazing angles and inverts the + // shadows. Shadowed can never exceed lit, and ambient carries shadowed areas if (shadowSettings.enableShadows) { - shadowSettings.cascadeIndex = GetShadowCascadeIndexFromDepth(pixelVertexData.viewPos.z, numCascades); - Camera cascadeCamera = _cameras[shadowSettings.cascadeIndex + 1]; // +1 because the first camera is the main camera - - float4 shadowPosition = mul(float4(pixelVertexData.worldPos, 1.0f), cascadeCamera.worldToClip); - - float shadowFactor = GetShadowFactor(uv, shadowPosition, shadowSettings); - lightColor = lerp(light.shadowColor.rgb, lightColor, shadowFactor); + lightColor *= lerp(light.shadowColor.rgb, float3(1.0f, 1.0f, 1.0f), shadowFactor); } directionalColor += lightColor; diff --git a/Source/Shaders/Shaders/Include/Shadows.inc.slang b/Source/Shaders/Shaders/Include/Shadows.inc.slang index 3763f4bd..cdd43a75 100644 --- a/Source/Shaders/Shaders/Include/Shadows.inc.slang +++ b/Source/Shaders/Shaders/Include/Shadows.inc.slang @@ -7,205 +7,396 @@ struct ShadowSettings { bool enableShadows; - float filterSize; - float penumbraFilterSize; + float strength; // Driven by sun elevation, fades shadows out around dawn/dusk + float normalOffset; // Receiver offset along the surface normal in shadow texels - uint cascadeIndex; // Filled in by ApplyLighting + float svsmConstantBias; // Compare bias toward the sun in world meters, no hardware bias in the software depth path }; -float TextureProj(float4 P, float2 offset, uint shadowCascadeIndex) +// One texel from the virtual shadow map through the page table, resident = false when the page +// has no physical backing (the caller falls back to a coarser clipmap) +float SVSMLoadDepth(uint clipmapIndex, int2 virtualTexel, uint pageTableSize, uint pageSize, uint poolPagesPerRow, out bool resident) { - float shadow = 1.0f; + int2 localPage = virtualTexel / int(pageSize); + int2 anchorPageMin = int2(_svsmData[0].anchorPageMinX[clipmapIndex], _svsmData[0].anchorPageMinY[clipmapIndex]); + int2 slot = WrapPageSlot(anchorPageMin + localPage, pageTableSize); - float4 shadowCoord = P / P.w; - shadowCoord.xy = shadowCoord.xy * 0.5f + 0.5f; + uint entry = _svsmPageTable[PageEntryIndex(clipmapIndex, slot, pageTableSize)]; - if (shadowCoord.z > -1.0f && shadowCoord.z < 1.0f) + // INVALID content may predate a Z-range or sun change, i.e. live in a different depth + // mapping — never sample it, the coarser fallback is always in the current mapping + resident = (entry & (SVSM_PAGE_RESIDENT | SVSM_PAGE_INVALID)) == SVSM_PAGE_RESIDENT; + if (!resident) { - float3 sc = float3(float2(shadowCoord.x, 1.0f - shadowCoord.y) + offset, shadowCoord.z); - //shadow = _shadowRT.SampleLevel(_shadowSampler, sc.xy, sc.z); - shadow = _shadowCascadeRTs[shadowCascadeIndex].SampleCmpLevelZero(_shadowCmpSampler, sc.xy, sc.z); + return 0.0f; } - return shadow; + + uint physicalPage = GetPagePhysical(entry); + uint2 physicalBase = uint2(physicalPage % poolPagesPerRow, physicalPage / poolPagesPerRow) * pageSize; + uint2 pageTexel = uint2(virtualTexel - localPage * int(pageSize)); + + return asfloat(_svsmPagePool[physicalBase + pageTexel]); } -// 9 imad (+ 6 iops with final shuffle) -uint3 PCG3DHash(uint3 v) +// Dynamic caster depth for the same texel, 0 when no dynamic page exists there. Non-residency is +// the normal state for almost every page and must never drive the coarser-clipmap fallback +float SVSMLoadDynamicDepth(uint clipmapIndex, int2 virtualTexel, uint pageTableSize, uint pageSize, out bool resident) { - v = v * 1664525u + 1013904223u; + int2 localPage = virtualTexel / int(pageSize); + int2 anchorPageMin = int2(_svsmData[0].anchorPageMinX[clipmapIndex], _svsmData[0].anchorPageMinY[clipmapIndex]); + int2 slot = WrapPageSlot(anchorPageMin + localPage, pageTableSize); - v.x += v.y * v.z; - v.y += v.z * v.x; - v.z += v.x * v.y; - - v ^= v >> 16u; + uint entry = _svsmDynamicPageTable[PageEntryIndex(clipmapIndex, slot, pageTableSize)]; + resident = (entry & SVSM_PAGE_RESIDENT) != 0; + if (!resident) + { + return 0.0f; + } - v.x += v.y * v.z; - v.y += v.z * v.x; - v.z += v.x * v.y; + uint poolPagesPerRow = _svsmData[0].configDynamicPoolPagesPerRow; + uint physicalPage = GetPagePhysical(entry); + uint2 physicalBase = uint2(physicalPage % poolPagesPerRow, physicalPage / poolPagesPerRow) * pageSize; + uint2 pageTexel = uint2(virtualTexel - localPage * int(pageSize)); - return v; + return asfloat(_svsmDynamicPagePool[physicalBase + pageTexel]); } -// Percentage Closer Filtering -float FilterPCF(float2 screenUV, float4 shadowCoord, ShadowSettings shadowSettings) +// Residency alone: one page-table read, no pool load — for probes that only need to know whether +// a dynamic page exists at the texel +bool SVSMIsDynamicPageResident(uint clipmapIndex, int2 virtualTexel, uint pageTableSize, uint pageSize) { - float2 texDim; - _shadowCascadeRTs[0].GetDimensions(texDim.x, texDim.y); + int2 localPage = virtualTexel / int(pageSize); + int2 anchorPageMin = int2(_svsmData[0].anchorPageMinX[clipmapIndex], _svsmData[0].anchorPageMinY[clipmapIndex]); + int2 slot = WrapPageSlot(anchorPageMin + localPage, pageTableSize); - const float scale = 0.5f; - float dx = scale * (1.0f / texDim.x); - float dy = scale * (1.0f / texDim.y); + uint entry = _svsmDynamicPageTable[PageEntryIndex(clipmapIndex, slot, pageTableSize)]; + return (entry & SVSM_PAGE_RESIDENT) != 0; +} - // Calculate directions for the filtering - uint4 u = uint4(screenUV, uint(screenUV.x) ^ uint(screenUV.y), uint(screenUV.x) + uint(screenUV.y)); - float3 rand = normalize(PCG3DHash(u.xyz)); +// One ring of the SVSM PCF filter: the 4x4 texel block shared by all 9 bilinear taps spans at +// most 2x2 pages, so their table entries resolve once and the taps only pay pool loads. One +// implementation for both the static and dynamic halves of GetSVSMShadowFactor, parameterized on +// the (table, pool, pool row count) pair and the missing-page policy: +// - missingPagesReadLit = false (static cache): INVALID content may live in a dead depth mapping +// and counts as missing; any missing touched page aborts the ring (returns false, the caller +// falls back to a coarser clipmap, worst case ends lit). +// - missingPagesReadLit = true (dynamic pool): non-resident pages read depth 0 = lit — correct, +// no dynamic caster covers that texel — and the ring always resolves. +// The callers' containment margin keeps the whole block inside the window +bool SVSMFilterRing(uint clipmapIndex, float2 texelPos, float compareDepth, uint pageTableSize, uint pageSize, uint poolPagesPerRow, bool missingPagesReadLit, StructuredBuffer pageTable, Texture2D pagePool, out float shadowFactor) +{ + shadowFactor = 1.0f; - float2 dirA = normalize(rand.xy); - float2 dirB = normalize(float2(-dirA.y, dirA.x)); + float2 blockBaseF = floor(texelPos - 0.5f); + int2 blockBase = int2(blockBaseF) - 1; + float2 fracPart = texelPos - 0.5f - blockBaseF; - dirA *= dx; - dirB *= dy; + int2 anchorPageMin = int2(_svsmData[0].anchorPageMinX[clipmapIndex], _svsmData[0].anchorPageMinY[clipmapIndex]); + int2 basePage = blockBase / int(pageSize); + int2 baseRel = blockBase - basePage * int(pageSize); - // Add together all the filter taps - float shadowFactor = 0.0f; - const int range = 2; - int count = 0; + bool pageResident[2][2]; + uint2 pagePhysBase[2][2]; [unroll] - for (int x = -range; x <= range; x++) + for (int py = 0; py < 2; py++) { [unroll] - for (int y = -range; y <= range; y++) + for (int px = 0; px < 2; px++) { - shadowFactor += TextureProj(shadowCoord, dirA * x + dirB * y, shadowSettings.cascadeIndex); - count++; + int2 slot = WrapPageSlot(anchorPageMin + basePage + int2(px, py), pageTableSize); + uint entry = pageTable[PageEntryIndex(clipmapIndex, slot, pageTableSize)]; + + pageResident[py][px] = missingPagesReadLit ? (entry & SVSM_PAGE_RESIDENT) != 0 + : (entry & (SVSM_PAGE_RESIDENT | SVSM_PAGE_INVALID)) == SVSM_PAGE_RESIDENT; + uint physicalPage = GetPagePhysical(entry); + pagePhysBase[py][px] = uint2(physicalPage % poolPagesPerRow, physicalPage / poolPagesPerRow) * pageSize; } } - return shadowFactor / float(count); -} + if (!missingPagesReadLit) + { + bool2 crossesPage = bool2(baseRel.x + 3 >= int(pageSize), baseRel.y + 3 >= int(pageSize)); + bool allResident = pageResident[0][0] + && (!crossesPage.x || pageResident[0][1]) + && (!crossesPage.y || pageResident[1][0]) + && (!(crossesPage.x && crossesPage.y) || pageResident[1][1]); + if (!allResident) + { + return false; + } + } + float lit[4][4]; -// Percentage Closer Soft Shadows, credits: https://www.gamedev.net/tutorials/programming/graphics/contact-hardening-soft-shadows-made-fast-r4906/ -float InterleavedGradientNoise(float2 screenPos) -{ - float3 magic = float3(0.06711056f, 0.00583715f, 52.9829189f); - return frac(magic.z * frac(dot(screenPos, magic.xy))); + [unroll] + for (int y = 0; y < 4; y++) + { + [unroll] + for (int x = 0; x < 4; x++) + { + int2 rel = baseRel + int2(x, y); + int2 pageSel = int2(rel.x >= int(pageSize) ? 1 : 0, rel.y >= int(pageSize) ? 1 : 0); + uint2 pageTexel = uint2(rel - pageSel * int(pageSize)); + float storedDepth = pageResident[pageSel.y][pageSel.x] ? asfloat(pagePool[pagePhysBase[pageSel.y][pageSel.x] + pageTexel]) : 0.0f; + lit[y][x] = compareDepth > storedDepth ? 1.0f : 0.0f; + } + } + + float factor = 0.0f; + + [unroll] + for (int ty = 0; ty < 3; ty++) + { + [unroll] + for (int tx = 0; tx < 3; tx++) + { + float litX0 = lerp(lit[ty + 0][tx + 0], lit[ty + 0][tx + 1], fracPart.x); + float litX1 = lerp(lit[ty + 1][tx + 0], lit[ty + 1][tx + 1], fracPart.x); + factor += lerp(litX0, litX1, fracPart.y); + } + } + shadowFactor = factor / 9.0f; + return true; } -float2 VogelDiskSample(int sampleIndex, int samplesCount, float phi) +// Software percentage-closer filtering over the sparse clipmaps: select the finest covering +// clipmap, walk to coarser ones when a touched page is not resident, 3x3 bilinear-weighted taps +// over a shared 4x4 texel block (SVSMFilterRing). Returns 1 (lit) when nothing is resident, the +// marking pass guarantees residency for everything the main view actually sees. Static and +// dynamic resolve their rings independently (see the dynamic block below) and combine via min +float GetSVSMShadowFactor(float3 worldPos, float3 worldNormal, ShadowSettings shadowSettings) { - float goldenAngle = 2.4f; + uint numClipmaps = _svsmData[0].configNumClipmaps; + float4 lightDirection = _svsmData[0].prevLightDirection; + if (numClipmaps == 0 || lightDirection.w < 0.5f) + { + return 1.0f; + } - float r = sqrt(sampleIndex + 0.5f) / sqrt(samplesCount); - float theta = sampleIndex * goldenAngle + phi; + uint pageTableSize = _svsmData[0].configPageTableSize; + uint pageSize = _svsmData[0].configPageSize; + uint poolPagesPerRow = _svsmData[0].configPoolPagesPerRow; - float sine, cosine; - sincos(theta, sine, cosine); + float3x3 lightRotation = BuildLightRotation(lightDirection.xyz); + float3 relativePos = worldPos - _cameras[0].eyePosition.xyz; // Camera-relative, world-magnitude wrap math loses texels + float2 lightSpaceRelCam = mul(relativePos, lightRotation).xy; - return float2(r * cosine, r * sine); -} - -// This should be tuned for the visuals we want -float AvgBlockersDepthToPenumbra(float shadowMapViewZ, float avgBlockersDepth) -{ - float penumbra = (shadowMapViewZ - avgBlockersDepth) / avgBlockersDepth; - penumbra *= penumbra; - return saturate(80.0f * penumbra); -} + // Same margin the marking pass used, so every page the filter can touch was marked. Floor ring + // bias -1: the walk must pass through the ring the marker marked despite float divergence + int selectedClipmap = SelectClipmap(lightSpaceRelCam, length(relativePos), _svsmData, numClipmaps, pageTableSize, pageSize, _svsmData[0].configMarkBorderTexels + 2.0f, _svsmData[0].configResolutionScale, -1); + if (selectedClipmap < 0) + { + return 1.0f; // Outside the coarsest window, beyond all shadow data + } -float Penumbra(float gradientNoise, float2 shadowMapUV, float shadowMapViewZ, float penumbraFilterSize, int samplesCount, uint shadowCascadeIndex) -{ - float avgBlockersDepth = 0.0f; - float blockersCount = 0.0f; + float zRangeMin = _svsmData[0].zRangeMin; + float zRangeMax = _svsmData[0].zRangeMax; + float zRangeLength = max(zRangeMax - zRangeMin, 1.0f); + + // Ring-independent scalars hoisted out of the walks, structured-buffer loads don't reliably CSE + float zHalf = zRangeLength * 0.5f; + float camMinusZAnchor = _svsmData[0].configCamMinusZAnchor; + float compareBias = shadowSettings.svsmConstantBias / zRangeLength; + + // Receivers below the window's far plane have no representable occluder: the raster clamps + // their casters away (pages stay 0 = lit), but an unclamped receiver depth goes negative and + // then even EMPTY texels compare as shadowing — the camera-anchored window climbs with the + // camera, so terrain more than zHalfRange below it (along the light axis) blacks out square. + // No data means lit. The Z window is global, so no ring can do better — bail out entirely + if (zHalf - (mul(relativePos, lightRotation).z + camMinusZAnchor) < 0.0f) + { + return 1.0f; + } - for (int i = 0; i < samplesCount; i++) + float staticFactor = 1.0f; + for (uint k = uint(selectedClipmap); k < numClipmaps; k++) { - float2 sampleUV = VogelDiskSample(i, samplesCount, gradientNoise); - sampleUV = shadowMapUV + penumbraFilterSize * sampleUV; + float extent = _svsmData[0].extent[k]; + float texelWorld = extent / float(pageTableSize * pageSize); + + // Receiver pushed along its surface normal in clipmap texels, same acne control as the CSM path + float3 lightSpacePos = mul(relativePos + worldNormal * (texelWorld * shadowSettings.normalOffset), lightRotation); + float2 windowPos = lightSpacePos.xy + float2(_svsmData[0].camMinusWindowMinX[k], _svsmData[0].camMinusWindowMinY[k]); - // _shadowCascadeRTs[shadowCascadeIndex].SampleCmpLevelZero(_shadowSampler, sc.xy, sc.z); - float sampleDepth = _shadowCascadeRTs[shadowCascadeIndex].SampleLevel(_shadowPointClampSampler, sampleUV, 0).x; + // Reversed Z: 1 = sun side. lightSpacePos is camera-relative, the raster + // maps absolute light-space Z, camMinusZAnchor reconciles the two with small quantities. + // Positive bias moves the receiver toward the sun + float compareDepth = (zHalf - (lightSpacePos.z + camMinusZAnchor)) / zRangeLength + compareBias; - if (sampleDepth < shadowMapViewZ) + float2 texelPos = windowPos / texelWorld; + + float shadowFactor; + if (!SVSMFilterRing(k, texelPos, compareDepth, pageTableSize, pageSize, poolPagesPerRow, false, _svsmPageTable, _svsmPagePool, shadowFactor)) { - avgBlockersDepth += sampleDepth; - blockersCount += 1.0f; + continue; // Fall back to a coarser clipmap, worst case ends lit } - } - if (blockersCount > 0.0f) - { - avgBlockersDepth /= blockersCount; - return AvgBlockersDepthToPenumbra(shadowMapViewZ, avgBlockersDepth); + // Fade to unshadowed over the outer 15% of the coarsest window, beyond it there is no shadow data + if (k == numClipmaps - 1) + { + float halfExtent = extent * 0.5f; + float centerDistance = max(abs(windowPos.x - halfExtent), abs(windowPos.y - halfExtent)); + float fadeStart = 0.85f * halfExtent; + float fade = saturate((centerDistance - fadeStart) / (0.15f * halfExtent)); + shadowFactor = lerp(shadowFactor, 1.0f, fade); + } + + staticFactor = shadowFactor; + break; } - else + + // Dynamic casters resolve independently of the static ring: static residency is a long-lived + // cache, dynamic pages exist only at rings marked THIS frame (and marking has per-frame noise + // from animated-geometry pixels). Coupling the two flickers dynamic shadows whenever the + // static resolve lands on a ring whose dynamic page came and went — so walk the dynamic + // table on its own and combine. Depths share the global Z window, min() keeps the darker result + float dynamicFactor = 1.0f; + if (_svsmData[0].statsDynamicTotal != 0) { - return 0.0f; + for (uint kd = uint(selectedClipmap); kd < numClipmaps; kd++) + { + if (_svsmData[0].statsDynamicLive[kd] == 0) + { + continue; // No dynamic pages anywhere in this ring this frame + } + + float extent = _svsmData[0].extent[kd]; + float texelWorld = extent / float(pageTableSize * pageSize); + + float3 lightSpacePos = mul(relativePos + worldNormal * (texelWorld * shadowSettings.normalOffset), lightRotation); + float2 windowPos = lightSpacePos.xy + float2(_svsmData[0].camMinusWindowMinX[kd], _svsmData[0].camMinusWindowMinY[kd]); + float2 texelPos = windowPos / texelWorld; + + // Center-texel residency probe: DynamicMark pads the caster footprint by a full page, + // far wider than the filter, so a non-resident center means no dynamic caster here + if (!SVSMIsDynamicPageResident(kd, int2(floor(texelPos)), pageTableSize, pageSize)) + { + continue; + } + + float compareDepth = (zHalf - (lightSpacePos.z + camMinusZAnchor)) / zRangeLength + compareBias; + + // Same filter as the static ring against the dynamic table/pool; non-resident pages + // read lit, so this always resolves + SVSMFilterRing(kd, texelPos, compareDepth, pageTableSize, pageSize, _svsmData[0].configDynamicPoolPagesPerRow, true, _svsmDynamicPageTable, _svsmDynamicPagePool, dynamicFactor); + break; + } } + + return min(staticFactor, dynamicFactor); } -float FilterPCSS(float2 screenUV, float4 P, ShadowSettings shadowSettings) +// Diagnostic view of the SVSM compare: single center-texel sample, classification colors. +// Dark purple = no SVSM state, blue = outside coverage (XY window or below the Z window), +// magenta = page never resident, +// red = shadowed (brightness = margin up to 8m, green channel rises toward 512m -> yellow), +// green = lit (brightness = margin up to 8m, blue channel rises toward 512m -> cyan) +float3 GetSVSMDebugColor(float3 worldPos, float3 worldNormal, float normalOffset, float constantBias) { - float2 texDim; - _shadowCascadeRTs[0].GetDimensions(texDim.x, texDim.y); + uint numClipmaps = _svsmData[0].configNumClipmaps; + float4 lightDirection = _svsmData[0].prevLightDirection; + if (numClipmaps == 0 || lightDirection.w < 0.5f) + { + return float3(0.1f, 0.0f, 0.3f); + } - float4 shadowCoord = P / P.w; - shadowCoord.xy = shadowCoord.xy * 0.5f + 0.5f; - shadowCoord.y = -shadowCoord.y; + uint pageTableSize = _svsmData[0].configPageTableSize; + uint pageSize = _svsmData[0].configPageSize; + uint poolPagesPerRow = _svsmData[0].configPoolPagesPerRow; - const float tau = 6.28318; - float gradientNoise = tau * InterleavedGradientNoise(screenUV * texDim); + float3x3 lightRotation = BuildLightRotation(lightDirection.xyz); + float3 relativePos = worldPos - _cameras[0].eyePosition.xyz; + float2 lightSpaceRelCam = mul(relativePos, lightRotation).xy; - float shadow = 0.0f; - //if (shadowCoord.z > -1.0f && shadowCoord.z < 1.0f) + int selectedClipmap = SelectClipmap(lightSpaceRelCam, length(relativePos), _svsmData, numClipmaps, pageTableSize, pageSize, _svsmData[0].configMarkBorderTexels + 2.0f, _svsmData[0].configResolutionScale, -1); + if (selectedClipmap < 0) { - float penumbra = 1.0f - Penumbra(gradientNoise, shadowCoord.xy, shadowCoord.z, shadowSettings.penumbraFilterSize, 16, shadowSettings.cascadeIndex); + return float3(0.0f, 0.3f, 1.0f); + } - for (int i = 0; i < 16; i++) - { - float2 sampleUV = VogelDiskSample(i, 16, gradientNoise); - sampleUV = shadowCoord.xy + sampleUV * penumbra * shadowSettings.filterSize; + float zRangeMin = _svsmData[0].zRangeMin; + float zRangeMax = _svsmData[0].zRangeMax; + float zRangeLength = max(zRangeMax - zRangeMin, 1.0f); - shadow += _shadowCascadeRTs[shadowSettings.cascadeIndex].SampleCmpLevelZero(_shadowCmpSampler, sampleUV, shadowCoord.z); - } + // Below the window's far plane, same bail as the sampler: no representable occluder + if (zRangeLength * 0.5f - (mul(relativePos, lightRotation).z + _svsmData[0].configCamMinusZAnchor) < 0.0f) + { + return float3(0.0f, 0.3f, 1.0f); } - shadow /= 16.0f; - return shadow; -} + // Same independent static/dynamic resolves as GetSVSMShadowFactor, single center tap each. + // The classification compares whichever depth is more occluding against its own receiver + float deltaMeters = 0.0f; + bool anyResident = false; -#define SHADOW_FILTER_MODE_OFF 0 -#define SHADOW_FILTER_MODE_PCF 1 // Percentage Closer Filtering -#define SHADOW_FILTER_MODE_PCSS 2 // Percentage Closer Soft Shadows + for (uint k = uint(selectedClipmap); k < numClipmaps; k++) + { + float extent = _svsmData[0].extent[k]; + float texelWorld = extent / float(pageTableSize * pageSize); -#ifndef SHADOW_FILTER_MODE -#define SHADOW_FILTER_MODE SHADOW_FILTER_MODE_OFF -#endif + float3 lightSpacePos = mul(relativePos + worldNormal * (texelWorld * normalOffset), lightRotation); + float2 windowPos = lightSpacePos.xy + float2(_svsmData[0].camMinusWindowMinX[k], _svsmData[0].camMinusWindowMinY[k]); -float GetShadowFactor(float2 screenUV, float4 shadowCoord, ShadowSettings shadowSettings) -{ -#if SHADOW_FILTER_MODE == SHADOW_FILTER_MODE_PCF - return lerp(0.0f, 1.0f, FilterPCF(screenUV, shadowCoord, shadowSettings)); -#elif SHADOW_FILTER_MODE == SHADOW_FILTER_MODE_PCSS - return saturate(FilterPCSS(screenUV, shadowCoord, shadowSettings)); -#else - return lerp(0.0f, 1.0f, TextureProj(shadowCoord, float2(0, 0), shadowSettings.cascadeIndex)); -#endif -} + float zHalf = zRangeLength * 0.5f; + float receiverDepth = (zHalf - (lightSpacePos.z + _svsmData[0].configCamMinusZAnchor)) / zRangeLength; + float compareDepth = receiverDepth + constantBias / zRangeLength; -uint GetShadowCascadeIndexFromDepth(float depth, uint numCascades) -{ - uint cascadeIndex = 0; - for (int i = numCascades; i > 0; i--) + bool resident; + float storedDepth = SVSMLoadDepth(k, int2(windowPos / texelWorld), pageTableSize, pageSize, poolPagesPerRow, resident); + if (!resident) + { + continue; + } + + deltaMeters = (storedDepth - compareDepth) * zRangeLength; + anyResident = true; + break; + } + + if (_svsmData[0].statsDynamicTotal != 0) { - if (depth > _cameras[i].eyePosition.w) + for (uint kd = uint(selectedClipmap); kd < numClipmaps; kd++) { - cascadeIndex = i; + if (_svsmData[0].statsDynamicLive[kd] == 0) + { + continue; + } + + float extent = _svsmData[0].extent[kd]; + float texelWorld = extent / float(pageTableSize * pageSize); + + float3 lightSpacePos = mul(relativePos + worldNormal * (texelWorld * normalOffset), lightRotation); + float2 windowPos = lightSpacePos.xy + float2(_svsmData[0].camMinusWindowMinX[kd], _svsmData[0].camMinusWindowMinY[kd]); + + bool resident; + float dynamicDepth = SVSMLoadDynamicDepth(kd, int2(windowPos / texelWorld), pageTableSize, pageSize, resident); + if (!resident) + { + continue; + } + + float zHalf = zRangeLength * 0.5f; + float receiverDepth = (zHalf - (lightSpacePos.z + _svsmData[0].configCamMinusZAnchor)) / zRangeLength; + float compareDepth = receiverDepth + constantBias / zRangeLength; + + float dynamicDelta = (dynamicDepth - compareDepth) * zRangeLength; + deltaMeters = anyResident ? max(deltaMeters, dynamicDelta) : dynamicDelta; + anyResident = true; break; } } - return cascadeIndex; + + if (!anyResident) + { + return float3(1.0f, 0.0f, 1.0f); + } + + if (deltaMeters > 0.0f) + { + return float3(saturate(deltaMeters / 8.0f), saturate(deltaMeters / 512.0f), 0.0f); // Shadowed + } + return float3(0.0f, saturate(-deltaMeters / 8.0f), saturate(-deltaMeters / 512.0f)); // Lit } + #endif // SHADOWS_INCLUDED \ No newline at end of file diff --git a/Source/Shaders/Shaders/Include/VisibilityBuffers.inc.slang b/Source/Shaders/Shaders/Include/VisibilityBuffers.inc.slang index 95d7aee6..49b96234 100644 --- a/Source/Shaders/Shaders/Include/VisibilityBuffers.inc.slang +++ b/Source/Shaders/Shaders/Include/VisibilityBuffers.inc.slang @@ -542,21 +542,4 @@ float3 IDToColor3(uint ID) return color; } -float3 CascadeIDToColor(uint cascadeID) -{ - const uint cascadeCount = 8; - static float3 cascadeColors[cascadeCount] = - { - float3(1.0f, 0.0f, 0.0f), // Red - float3(0.0f, 1.0f, 0.0f), // Green - float3(0.0f, 0.0f, 1.0f), // Blue - float3(1.0f, 1.0f, 0.0f), // Yellow - float3(1.0f, 0.0f, 1.0f), // Purple - float3(0.0f, 1.0f, 1.0f), // Cyan - float3(1.0f, 0.5f, 0.0f), // Orange - float3(0.0f, 0.5f, 1.0f) // Light Blue - }; - - return cascadeColors[cascadeID % cascadeCount]; -} #endif // VISIBILITYBUFFERS_INCLUDED \ No newline at end of file diff --git a/Source/Shaders/Shaders/Material/MaterialPass.cs.slang b/Source/Shaders/Shaders/Material/MaterialPass.cs.slang index 68864f19..bdd3d273 100644 --- a/Source/Shaders/Shaders/Material/MaterialPass.cs.slang +++ b/Source/Shaders/Shaders/Material/MaterialPass.cs.slang @@ -1,5 +1,4 @@ -permutation DEBUG_ID = [0, 1, 2, 3, 4]; -permutation SHADOW_FILTER_MODE = [0, 1, 2]; // Off, PCF, PCSS +permutation DEBUG_ID = [0, 1, 2, 3, 4]; // Off, TypeID, ObjectID, TriangleID, SVSM compare margin permutation EDITOR_MODE = [0, 1]; // Off, Terrain #include "DescriptorSet/Debug.inc.slang" @@ -17,7 +16,7 @@ permutation EDITOR_MODE = [0, 1]; // Off, Terrain struct Constants { float4 renderInfo; // x = Render Width, y = Render Height, z = 1/Width, w = 1/Height - uint4 lightInfo; // x = Num Directional Lights, y = Num Point Lights, z = Num Cascades, w = Shadows Enabled + uint4 lightInfo; // x = Num Directional Lights, y = Shadows Ready (enabled, strength > 0, pool created), zw = UNUSED uint4 tileInfo; // xy = Num Tiles float4 fogColor; float4 fogSettings; // x = Enabled, y = Begin Fog Blend Dist, z = End Fog Blend Dist, w = UNUSED @@ -28,6 +27,7 @@ struct Constants float4 patchEdgeColor; float4 vertexColor; float4 brushColor; + float4 shadowSettings; // x = Shadow Strength, y = Normal Offset Bias, z = SVSM Constant Bias (world meters), w = UNUSED }; [[vk::push_constant]] Constants _constants; @@ -136,18 +136,18 @@ float4 ShadeTerrain(const uint2 pixelPos, const float2 screenUV, const Visibilit // Apply vertex color color.rgb *= pixelVertexData.color; - // TODO: Don't hardcode this ShadowSettings shadowSettings; - shadowSettings.enableShadows = _constants.lightInfo.w == 1; - shadowSettings.filterSize = 3.0f; - shadowSettings.penumbraFilterSize = 3.0f; + shadowSettings.enableShadows = _constants.lightInfo.y == 1; + shadowSettings.strength = _constants.shadowSettings.x; + shadowSettings.normalOffset = _constants.shadowSettings.y; + shadowSettings.svsmConstantBias = _constants.shadowSettings.z; // Apply lighting - color.rgb = ApplyLighting(screenUV, color.rgb, pixelVertexData, _constants.lightInfo, shadowSettings); + color.rgb = ApplyLighting(color.rgb, pixelVertexData, _constants.lightInfo, shadowSettings); // Apply decals color.rgb += ApplyDecals(tileIndex, pixelVertexData.worldPos, pixelVertexData.worldNormal); - + /*#if EDITOR_MODE == 1 // Get settings from push constants const float brushHardness = _constants.brushSettings.x; @@ -282,14 +282,14 @@ float4 ShadeModel(const uint2 pixelPos, const float2 screenUV, const VisibilityB color = BlendModel(blendingMode, color, shadedColor); // Or on this } - // TODO: Don't hardcode this ShadowSettings shadowSettings; - shadowSettings.enableShadows = _constants.lightInfo.w == 1; - shadowSettings.filterSize = 3.0f; - shadowSettings.penumbraFilterSize = 3.0f; + shadowSettings.enableShadows = _constants.lightInfo.y == 1; + shadowSettings.strength = _constants.shadowSettings.x; + shadowSettings.normalOffset = _constants.shadowSettings.y; + shadowSettings.svsmConstantBias = _constants.shadowSettings.z; // Apply lighting - color.rgb = ApplyLighting(screenUV, color.rgb, pixelVertexData, _constants.lightInfo, shadowSettings); + color.rgb = ApplyLighting(color.rgb, pixelVertexData, _constants.lightInfo, shadowSettings); // Apply highlight color.rgb *= pixelVertexData.highlightIntensity; @@ -335,11 +335,10 @@ void main(uint3 dispatchThreadId : SV_DispatchThreadID) float3 debugColor = IDToColor3(vBuffer.triangleID); _resolvedColor[pixelPos] = float4(debugColor, 1); return; -#elif DEBUG_ID == 4 // CascadeID - uint numCascades = _constants.lightInfo.z; +#elif DEBUG_ID == 4 // SVSM compare margin PixelVertexData pixelVertexData = GetPixelVertexData(pixelPos, vBuffer, 0, _constants.renderInfo.xy); - uint cascadeIndex = GetShadowCascadeIndexFromDepth(pixelVertexData.viewPos.z, numCascades); - _resolvedColor[pixelPos] = float4(CascadeIDToColor(cascadeIndex), 1); + float3 debugColor = GetSVSMDebugColor(pixelVertexData.worldPos, pixelVertexData.worldNormal, _constants.shadowSettings.y, _constants.shadowSettings.z); + _resolvedColor[pixelPos] = float4(debugColor, 1); return; #endif diff --git a/Source/Shaders/Shaders/Model/Draw.ps.slang b/Source/Shaders/Shaders/Model/Draw.ps.slang index 0d0fcadd..9e2e982f 100644 --- a/Source/Shaders/Shaders/Model/Draw.ps.slang +++ b/Source/Shaders/Shaders/Model/Draw.ps.slang @@ -1,10 +1,10 @@ -permutation SHADOW_PASS = [0, 1]; #define GEOMETRY_PASS 1 #include "DescriptorSet/Global.inc.slang" #include "Include/Common.inc.slang" #include "Include/VisibilityBuffers.inc.slang" +#include "Model/ModelAlphaKey.inc.slang" #include "Model/ModelShared.inc.slang" struct PSInput @@ -12,72 +12,24 @@ struct PSInput uint4 drawIDInstanceIDTextureDataIDInstanceRefID : TEXCOORD0; float4 uv01 : TEXCOORD1; -#if !SHADOW_PASS uint triangleID : SV_PrimitiveID; -#endif }; -#if !SHADOW_PASS struct PSOutput { uint2 visibilityBuffer : SV_Target0; }; -#else -#define PSOutput void -#endif [shader("fragment")] PSOutput main(PSInput input) { - uint drawCallID = input.drawIDInstanceIDTextureDataIDInstanceRefID.x; - uint instanceID = input.drawIDInstanceIDTextureDataIDInstanceRefID.y; uint textureDataID = input.drawIDInstanceIDTextureDataIDInstanceRefID.z; uint instanceRefID = input.drawIDInstanceIDTextureDataIDInstanceRefID.w; - TextureData textureData = LoadModelTextureData(_packedModelTextureDatas, textureDataID); + AlphaKeyDiscard(textureDataID, input.uv01.xy); - for (uint textureUnitIndex = textureData.textureUnitOffset; textureUnitIndex < textureData.textureUnitOffset + textureData.numTextureUnits; textureUnitIndex++) - { - ModelTextureUnit textureUnit = _modelTextureUnits[textureUnitIndex]; - - uint blendingMode = (textureUnit.data1 >> 11) & 0x7; - - if (blendingMode != 1) // ALPHA KEY - continue; - - uint texture0SamplerIndex = (textureUnit.data1 >> 1) & 0x3; - uint texture1SamplerIndex = (textureUnit.data1 >> 3) & 0x3; - uint materialType = (textureUnit.data1 >> 16) & 0xFFFF; - - if (materialType == 0x8000) - continue; - - float4 texture1 = _modelTextures[NonUniformResourceIndex(textureUnit.textureIDs[0])].Sample(_samplers[texture0SamplerIndex], input.uv01.xy); - float4 texture2 = float4(1, 1, 1, 1); - - uint vertexShaderId = materialType & 0xFF; - if (vertexShaderId > 2) - { - // ENV uses generated UVCoords based on camera pos + geometry normal in frame space - texture2 = _modelTextures[NonUniformResourceIndex(textureUnit.textureIDs[1])].Sample(_samplers[texture1SamplerIndex], input.uv01.zw); - } - - // Experimental alphakey discard without shading, if this has issues check github history for cModel.ps.hlsl - float4 diffuseColor = float4(1, 1, 1, 1); - // TODO: per-instance diffuseColor - - // TODO : minAlpha is set based on the Pixel Shader Id - float minAlpha = texture1.a; // min(texture1.a, min(texture2.a, diffuseColor.a)); - if (minAlpha < (128.0 / 255.0)) - { - discard; - } - } - -#if !SHADOW_PASS PSOutput output; output.visibilityBuffer = PackVisibilityBuffer(ObjectType::ModelOpaque, instanceRefID, input.triangleID); return output; -#endif -} \ No newline at end of file +} diff --git a/Source/Shaders/Shaders/Model/Draw.vs.slang b/Source/Shaders/Shaders/Model/Draw.vs.slang index fc780812..0246f650 100644 --- a/Source/Shaders/Shaders/Model/Draw.vs.slang +++ b/Source/Shaders/Shaders/Model/Draw.vs.slang @@ -1,5 +1,5 @@ permutation EDITOR_PASS = [0, 1]; -permutation SHADOW_PASS = [0, 1]; +permutation SVSM_PASS = [0, 1]; // 1 adds per-draw clip-rect distances for the SVSM page render #define GEOMETRY_PASS 1 #include "DescriptorSet/Global.inc.slang" @@ -7,14 +7,24 @@ permutation SHADOW_PASS = [0, 1]; #include "Include/Common.inc.slang" #include "Model/ModelShared.inc.slang" +#if SVSM_PASS +#include "Shadows/SVSM.inc.slang" +#endif struct Constants { uint viewIndex; +#if SVSM_PASS + uint svsmRectIndex; // Which clip rect this draw renders, SVSM_CLIP_RECT_DISABLED = no clipping +#endif }; [[vk::push_constant]] Constants _constants; +#if SVSM_PASS +[[vk::binding(0, PER_PASS)]] StructuredBuffer _svsmData; // Same binding the SVSM pixel shader declares +#endif + struct VSInput { uint vertexID : SV_VulkanVertexID; @@ -28,6 +38,9 @@ struct VSOutput nointerpolation uint4 drawIDInstanceIDTextureDataIDInstanceRefID : TEXCOORD0; float4 uv01 : TEXCOORD1; #endif +#if SVSM_PASS + float4 clipDistances : SV_ClipDistance; // Fixed-function culls fragments outside the draw's clip rect +#endif }; [shader("vertex")] @@ -45,14 +58,16 @@ VSOutput main(VSInput input) float4x4 boneTransformMatrix = CalcBoneTransformMatrix(_instanceBoneMatrices, instanceData, vertex); float4 position = mul(float4(vertex.position, 1.0f), boneTransformMatrix); - // Save the skinned vertex position (in model-space) if this vertex was animated - if (instanceData.boneMatrixOffset != 4294967295) - { - uint localVertexID = input.vertexID - instanceData.modelVertexOffset; // This gets the local vertex ID relative to the model - uint animatedVertexID = localVertexID + instanceData.animatedVertexOffset; // This makes it relative to the animated instance - - StoreAnimatedVertexPosition(_animatedModelVertexPositions, animatedVertexID, position.xyz); - } + // TODO: Separate skinning compute shader — the cache below is currently dead (its consumer + // left with the visibility-buffer barycentrics; the material pass re-skins analytically) + //// Save the skinned vertex position (in model-space) if this vertex was animated + //if (instanceData.boneMatrixOffset != 4294967295) + //{ + // uint localVertexID = input.vertexID - instanceData.modelVertexOffset; // This gets the local vertex ID relative to the model + // uint animatedVertexID = localVertexID + instanceData.animatedVertexOffset; // This makes it relative to the animated instance + // + // StoreAnimatedVertexPosition(_animatedModelVertexPositions, animatedVertexID, position.xyz); + //} float4x4 instanceMatrix = _modelInstanceMatrices[instanceID]; position = mul(position, instanceMatrix); @@ -61,6 +76,10 @@ VSOutput main(VSInput input) VSOutput output; output.position = mul(position, _cameras[_constants.viewIndex].worldToClip); +#if SVSM_PASS + output.clipDistances = SVSMRectClipDistances(position.xyz, _constants.viewIndex - 1, _constants.svsmRectIndex, _cameras[0].eyePosition.xyz, _svsmData); +#endif + #if !EDITOR_PASS output.drawIDInstanceIDTextureDataIDInstanceRefID = int4(drawID, instanceID, instanceRef.extraID, instanceRefID); diff --git a/Source/Shaders/Shaders/Model/DrawSVSM.ps.slang b/Source/Shaders/Shaders/Model/DrawSVSM.ps.slang new file mode 100644 index 00000000..9cc00c45 --- /dev/null +++ b/Source/Shaders/Shaders/Model/DrawSVSM.ps.slang @@ -0,0 +1,66 @@ +permutation SVSM_DYNAMIC = [0, 1]; // 1 = dynamic caster split pass, renders into the dynamic pool + +#define GEOMETRY_PASS 1 + +#include "DescriptorSet/Global.inc.slang" +#include "DescriptorSet/Model.inc.slang" + +#include "Include/Common.inc.slang" +#include "Model/ModelAlphaKey.inc.slang" +#include "Model/ModelShared.inc.slang" +#include "Shadows/SVSM.inc.slang" + +// SVSM page render for models: keeps the alpha-key discard from Model/Draw.ps, then writes +// reversed-Z depth into the physical page pool with image atomics. No attachments, viewport +// spans the full virtual texture so SV_Position.xy is the window-local virtual texel. Paired +// with the SVSM_PASS=1 vertex permutation. +// +// SVSM_DYNAMIC renders the dynamic-instance fill into the dynamic pool: same bindings, the pass +// binds the dynamic table/pool instead, and dynamic pages render whenever RESIDENT (every frame) + +struct Constants +{ + uint viewIndex; // 0 = main camera, clipmap k renders as view k + 1 + uint svsmRectIndex; // Consumed by the vertex shader's clip rect, declared here so the push block matches +}; + +[[vk::push_constant]] Constants _constants; + +[[vk::binding(0, PER_PASS)]] StructuredBuffer _svsmData; +[[vk::binding(1, PER_PASS)]] StructuredBuffer _pageTable; +[[vk::binding(2, PER_PASS)]] RWTexture2D _pagePool; + +struct PSInput +{ + float4 position : SV_Position; + uint4 drawIDInstanceIDTextureDataIDInstanceRefID : TEXCOORD0; + float4 uv01 : TEXCOORD1; +}; + +[shader("fragment")] +void main(PSInput input) +{ +#if SVSM_DYNAMIC + const uint requiredFlags = SVSM_PAGE_RESIDENT; + const uint poolPagesPerRow = _svsmData[0].configDynamicPoolPagesPerRow; +#else + const uint requiredFlags = SVSM_PAGE_RESIDENT | SVSM_PAGE_DIRTY; + const uint poolPagesPerRow = _svsmData[0].configPoolPagesPerRow; +#endif + + // Fragments over pages that don't render this frame skip the alpha-key sampling entirely. + // The skip diverges only along page borders, where a wrong implicit mip for one quad row in a + // depth-only alpha test is invisible — Sample stays Sample + uint pageEntry = SVSMLoadPageEntry(input.position.xy, _constants.viewIndex - 1, _svsmData, _pageTable); + if ((pageEntry & requiredFlags) != requiredFlags) + { + return; + } + + uint textureDataID = input.drawIDInstanceIDTextureDataIDInstanceRefID.z; + + // Same alpha-key discard as Model/Draw.ps, without the visibility-buffer write + AlphaKeyDiscard(textureDataID, input.uv01.xy); + + SVSMWritePageDepthEntry(input.position.xyz, pageEntry, requiredFlags, poolPagesPerRow, _svsmData, _pagePool); +} diff --git a/Source/Shaders/Shaders/Model/DrawSkybox.vs.slang b/Source/Shaders/Shaders/Model/DrawSkybox.vs.slang index 830c8f67..c1bd405b 100644 --- a/Source/Shaders/Shaders/Model/DrawSkybox.vs.slang +++ b/Source/Shaders/Shaders/Model/DrawSkybox.vs.slang @@ -37,14 +37,16 @@ VSOutput main(VSInput input) float4x4 boneTransformMatrix = CalcBoneTransformMatrix(_instanceBoneMatrices, instanceData, vertex); float4 position = mul(float4(vertex.position, 1.0f), boneTransformMatrix); - // Save the skinned vertex position (in model-space) if this vertex was animated - if (instanceData.boneMatrixOffset != 4294967295) - { - uint localVertexID = input.vertexID - instanceData.modelVertexOffset; // This gets the local vertex ID relative to the model - uint animatedVertexID = localVertexID + instanceData.animatedVertexOffset; // This makes it relative to the animated instance - - StoreAnimatedVertexPosition(_animatedModelVertexPositions, animatedVertexID, position.xyz); - } + // TODO: Separate skinning compute shader — the cache below is currently dead (its consumer + // left with the visibility-buffer barycentrics; the material pass re-skins analytically) + //// Save the skinned vertex position (in model-space) if this vertex was animated + //if (instanceData.boneMatrixOffset != 4294967295) + //{ + // uint localVertexID = input.vertexID - instanceData.modelVertexOffset; // This gets the local vertex ID relative to the model + // uint animatedVertexID = localVertexID + instanceData.animatedVertexOffset; // This makes it relative to the animated instance + // + // StoreAnimatedVertexPosition(_animatedModelVertexPositions, animatedVertexID, position.xyz); + //} float4x4 instanceMatrix = _modelInstanceMatrices[instanceID]; position = mul(position, instanceMatrix); diff --git a/Source/Shaders/Shaders/Model/DrawTransparent.vs.slang b/Source/Shaders/Shaders/Model/DrawTransparent.vs.slang index 75fb90dc..b0c08c0f 100644 --- a/Source/Shaders/Shaders/Model/DrawTransparent.vs.slang +++ b/Source/Shaders/Shaders/Model/DrawTransparent.vs.slang @@ -39,14 +39,16 @@ VSOutput main(VSInput input) float4x4 boneTransformMatrix = CalcBoneTransformMatrix(_instanceBoneMatrices, instanceData, vertex); float4 position = mul(float4(vertex.position, 1.0f), boneTransformMatrix); - // Save the skinned vertex position (in model-space) if this vertex was animated - if (instanceData.boneMatrixOffset != 4294967295) - { - uint localVertexID = input.vertexID - instanceData.modelVertexOffset; // This gets the local vertex ID relative to the model - uint animatedVertexID = localVertexID + instanceData.animatedVertexOffset; // This makes it relative to the animated instance - - StoreAnimatedVertexPosition(_animatedModelVertexPositions, animatedVertexID, position.xyz); - } + // TODO: Separate skinning compute shader — the cache below is currently dead (its consumer + // left with the visibility-buffer barycentrics; the material pass re-skins analytically) + //// Save the skinned vertex position (in model-space) if this vertex was animated + //if (instanceData.boneMatrixOffset != 4294967295) + //{ + // uint localVertexID = input.vertexID - instanceData.modelVertexOffset; // This gets the local vertex ID relative to the model + // uint animatedVertexID = localVertexID + instanceData.animatedVertexOffset; // This makes it relative to the animated instance + // + // StoreAnimatedVertexPosition(_animatedModelVertexPositions, animatedVertexID, position.xyz); + //} float4x4 instanceMatrix = _modelInstanceMatrices[instanceID]; position = mul(position, instanceMatrix); diff --git a/Source/Shaders/Shaders/Model/ModelAlphaKey.inc.slang b/Source/Shaders/Shaders/Model/ModelAlphaKey.inc.slang new file mode 100644 index 00000000..fe5726f3 --- /dev/null +++ b/Source/Shaders/Shaders/Model/ModelAlphaKey.inc.slang @@ -0,0 +1,43 @@ +#ifndef MODEL_ALPHA_KEY_INCLUDED +#define MODEL_ALPHA_KEY_INCLUDED + +#include "DescriptorSet/Model.inc.slang" +#include "Model/ModelShared.inc.slang" + +// Alpha-key discard shared by the model fragment passes that need cutouts without shading +// (Model/Draw.ps, Model/DrawSVSM.ps). Fragment-only (Sample + discard), keep it out of the +// includes compute shaders pull in. +// Experimental alphakey discard without shading, if this has issues check github history for +// cModel.ps.hlsl +void AlphaKeyDiscard(uint textureDataID, float2 uv0) +{ + TextureData textureData = LoadModelTextureData(_packedModelTextureDatas, textureDataID); + + for (uint textureUnitIndex = textureData.textureUnitOffset; textureUnitIndex < textureData.textureUnitOffset + textureData.numTextureUnits; textureUnitIndex++) + { + ModelTextureUnit textureUnit = _modelTextureUnits[textureUnitIndex]; + + uint blendingMode = (textureUnit.data1 >> 11) & 0x7; + + if (blendingMode != 1) // ALPHA KEY + continue; + + uint texture0SamplerIndex = (textureUnit.data1 >> 1) & 0x3; + uint materialType = (textureUnit.data1 >> 16) & 0xFFFF; + + if (materialType == 0x8000) + continue; + + float4 texture1 = _modelTextures[NonUniformResourceIndex(textureUnit.textureIDs[0])].Sample(_samplers[texture0SamplerIndex], uv0); + + // TODO : minAlpha is set based on the Pixel Shader Id, and per-instance diffuseColor + // should participate + float minAlpha = texture1.a; + if (minAlpha < (128.0 / 255.0)) + { + discard; + } + } +} + +#endif // MODEL_ALPHA_KEY_INCLUDED diff --git a/Source/Shaders/Shaders/Shadows/LightBasis.inc.slang b/Source/Shaders/Shaders/Shadows/LightBasis.inc.slang new file mode 100644 index 00000000..3f78760c --- /dev/null +++ b/Source/Shaders/Shaders/Shadows/LightBasis.inc.slang @@ -0,0 +1,100 @@ +#ifndef LIGHT_BASIS_INCLUDED +#define LIGHT_BASIS_INCLUDED + +// Light-space basis and camera matrix helpers shared by the SVSM update chain and the sampler. +// +// Matrix layout rule (load-bearing): shaders consume glm column-major uploads with +// mul(rowVector, M), so a Slang float4x4 row i must hold what glm stores in column i. +// glm composition C = A * B therefore becomes C_s = mul(B_s, A_s) here. + +// Constant up for the stable path, falls back to Z when the light is near vertical +float3 StableUp(float3 lightDirection) +{ + return abs(lightDirection.y) > 0.99f ? float3(0.0f, 0.0f, 1.0f) : float3(0.0f, 1.0f, 0.0f); +} + +// Matches glm::lookAtLH memory layout (GLM_FORCE_LEFT_HANDED) +float4x4 BuildLookAt(float3 eye, float3 center, float3 up) +{ + float3 f = normalize(center - eye); + float3 s = normalize(cross(up, f)); + float3 u = cross(f, s); + + float4x4 m; + m[0] = float4(s.x, u.x, f.x, 0.0f); + m[1] = float4(s.y, u.y, f.y, 0.0f); + m[2] = float4(s.z, u.z, f.z, 0.0f); + m[3] = float4(-dot(s, eye), -dot(u, eye), -dot(f, eye), 1.0f); + return m; +} + +// Rotation-only light basis, lightSpace = mul(cameraRelativeWorld, R). The rows mirror +// BuildLookAt(0, lightDirection, StableUp) term for term so window coordinates and the +// clipmap view matrices agree bit for bit +float3x3 BuildLightRotation(float3 lightDirection) +{ + float3 f = normalize(lightDirection); + float3 s = normalize(cross(StableUp(lightDirection), f)); + float3 u = cross(f, s); + + float3x3 m; + m[0] = float3(s.x, u.x, f.x); + m[1] = float3(s.y, u.y, f.y); + m[2] = float3(s.z, u.z, f.z); + return m; +} + +// Analytic inverse of a BuildLookAt matrix (rigid transform) +float4x4 InverseLookAt(float4x4 view, float3 eye) +{ + float3 s = float3(view[0].x, view[1].x, view[2].x); + float3 u = float3(view[0].y, view[1].y, view[2].y); + float3 f = float3(view[0].z, view[1].z, view[2].z); + + float4x4 m; + m[0] = float4(s, 0.0f); + m[1] = float4(u, 0.0f); + m[2] = float4(f, 0.0f); + m[3] = float4(eye, 1.0f); + return m; +} + +// Matches glm::orthoLH_ZO memory layout (GLM_FORCE_DEPTH_ZERO_TO_ONE) +float4x4 BuildOrtho(float left, float right, float bottom, float top, float zNear, float zFar) +{ + float4x4 m; + m[0] = float4(2.0f / (right - left), 0.0f, 0.0f, 0.0f); + m[1] = float4(0.0f, 2.0f / (top - bottom), 0.0f, 0.0f); + m[2] = float4(0.0f, 0.0f, 1.0f / (zFar - zNear), 0.0f); + m[3] = float4(-(right + left) / (right - left), -(top + bottom) / (top - bottom), -zNear / (zFar - zNear), 1.0f); + return m; +} + +// Analytic inverse of a BuildOrtho matrix, uses the post-snap projection values +float4x4 InverseOrtho(float4x4 proj) +{ + float3 scale = float3(proj[0].x, proj[1].y, proj[2].z); + float3 translation = proj[3].xyz; + + float4x4 m; + m[0] = float4(1.0f / scale.x, 0.0f, 0.0f, 0.0f); + m[1] = float4(0.0f, 1.0f / scale.y, 0.0f, 0.0f); + m[2] = float4(0.0f, 0.0f, 1.0f / scale.z, 0.0f); + m[3] = float4(-translation / scale, 1.0f); + return m; +} + +// glm row i of M, planes are extracted from mathematical rows +float4 MatrixRow(float4x4 m, uint i) +{ + return float4(m[0][i], m[1][i], m[2][i], m[3][i]); +} + +// Gribb-Hartmann row (inside if dot(n, p) + d >= 0) to the culling encoding vec4(n, dot(n, pointOnPlane)) +float4 ExtractPlane(float4 row) +{ + float normalLength = length(row.xyz); + return float4(row.xyz / normalLength, -row.w / normalLength); +} + +#endif // LIGHT_BASIS_INCLUDED diff --git a/Source/Shaders/Shaders/Shadows/SVSM.inc.slang b/Source/Shaders/Shaders/Shadows/SVSM.inc.slang new file mode 100644 index 00000000..d2ec4098 --- /dev/null +++ b/Source/Shaders/Shaders/Shadows/SVSM.inc.slang @@ -0,0 +1,314 @@ +#ifndef SVSM_INCLUDED +#define SVSM_INCLUDED + +#include "Shadows/LightBasis.inc.slang" + +// Shared types and helpers for the Sparse Virtual Shadow Map chain: +// SVSMPrepare.cs -> SVSMInvalidateAABBs.cs -> SVSMPageUpdateA.cs -> SVSMPageMark.cs -> SVSMPageUpdateB.cs +// +// Clipmap k is a camera-centered light-space window of clipmap0Extent * 2^k meters, backed by a +// pageTableSize^2 page table over a virtual pageTableSize * pageSize texture. Page content is +// world-stable: a physical page holds a fixed absolute light-space page until its table slot is +// reused (toroidal wrap) or its content is invalidated. +// +// Precision rule: no wrap math at world magnitude. World coordinates reach ~17k and the finest +// texels are millimeters, so all addressing is anchor-relative: Prepare stores per clipmap the +// page-snapped window origin as an integer absolute page index (anchorPageMin) plus the camera +// offset inside the window (camMinusWindowMin), and every pass works with +// windowPos = lightSpace(P - camera) + camMinusWindowMin, which is at most one window extent. + +// Push constants shared by all SVSM dispatches. One layout everywhere, but allocClipmap, +// dynamicPhase and numDirtyAABBs vary per dispatch +struct SVSMConstants +{ + float4 lightDirection; // xyz = direction the light travels, quantized shadow sun + uint numClipmaps; + uint pageTableSize; // pages per virtual texture row + uint pageSize; // texels per page row + uint poolPagesPerRow; // physical pool pages per row, total pages = poolPagesPerRow^2 + float clipmap0Extent; // world extent of the finest clipmap window + float markBorderTexels; // filter footprint margin, marks neighbor pages near page borders + uint evictAge; // frames without a mark before a resident page returns to the pool + uint invalidateAll; // CPU-requested invalidation cause bits (SVSM_CAUSE_*) + uint numDirtyAABBs; // Static dirty AABB count; for DynamicMark this carries the DYNAMIC AABB count instead + uint allocClipmap; // UpdateB runs once per clipmap, finest first, so pool starvation always lands on the coarsest ring + float casterMargin; // Sun-side sweep on the near cull plane, depth clamp pancakes those casters + float zHalfRange; // Half depth range of every clipmap window around the quantized Z anchor + uint fillDrawCallCount; // Model draw calls this frame, Finalize sizes the per-view fill overhead args from it + float padding1; + float resolutionScale; // Eye-distance clipmap floor scale, <= 0 disables + uint dynamicPoolPagesPerRow; // Caster-split dynamic pool pages per row, 0 while the split is off + uint renderBudget; // Static pages rendered per frame, 0 = unlimited; the coarsest two rings are exempt + uint dynamicPhase; // DynamicUpdate: 0 = release pages, 1 = acquire + bookkeeping. Split so free-list pushes and pops never share a dispatch + uint fillInstanceCount; // Model instances this frame, Finalize sizes the per-view fill dispatch args from it + uint fillCellCount; // Terrain cells this frame, same purpose +}; + +// Page table entry bits. A zeroed entry is a free slot +#define SVSM_PAGE_PHYS_MASK 0xFFFu // Physical page index, flat, pool is capped to 4096 pages +#define SVSM_PAGE_AGE_SHIFT 12 +#define SVSM_PAGE_AGE_MASK (0xFFu << SVSM_PAGE_AGE_SHIFT) +#define SVSM_PAGE_INVALID (1u << 24) // Content is stale, re-render before the next sample +#define SVSM_PAGE_RESIDENT (1u << 25) // Backed by a physical page +#define SVSM_PAGE_MARKED (1u << 26) // Touched by a visible sample this frame +#define SVSM_PAGE_DIRTY (1u << 27) // Renders this frame, always consumed the frame it is set + +// Invalidation cause bits, CPU bits come in through SVSMConstants::invalidateAll +#define SVSM_CAUSE_SUN_STEP 1u +#define SVSM_CAUSE_Z_RANGE 2u +#define SVSM_CAUSE_MANUAL 4u +#define SVSM_CAUSE_AABB_OVERFLOW 8u + +// Persistent GPU state + per-frame stats. Scalar arrays only so the std430 layout matches the +// flat u32 readback on the CPU +struct SVSMData +{ + float4 prevLightDirection; // w > 0.5 once valid + + float zRangeMin; // Reserved for the stage 2 Z fit + float zRangeMax; + uint frameInvalidationFlags; // Transient: written by Prepare (or AABB overflow), consumed by UpdateA the same frame + uint padding0; + + int anchorPageMinX[8]; // Absolute page index of the window min corner + int anchorPageMinY[8]; + int prevAnchorPageMinX[8]; + int prevAnchorPageMinY[8]; + float camMinusWindowMinX[8]; // Camera light-space XY minus the window min corner, small + float camMinusWindowMinY[8]; + float extent[8]; + float pageWorld[8]; // extent / pageTableSize + uint clipmapInvalidate[8]; // Transient: clipmap config changed, UpdateA invalidates the whole clipmap + + int dirtyRectMinX[8]; // Window-local page coords of this frame's dirty pages + int dirtyRectMinY[8]; + int dirtyRectMaxX[8]; + int dirtyRectMaxY[8]; + + uint statsMarked[8]; + uint statsResident[8]; + uint statsDirty[8]; + uint statsEvicted[8]; + uint statsInvalidated[8]; + uint statsOverflow; + uint statsInvalidationCause; + uint statsFreeListCount; // Snapshot at the start of the frame + + // Config mirror written by Prepare so draw-time shaders only need this buffer plus viewIndex + uint configPageTableSize; + uint configPageSize; + uint configPoolPagesPerRow; + uint configNumClipmaps; + float configMarkBorderTexels; // The sampler mirrors the marking margin through this + float configPadding0; + float configCamMinusZAnchor; // Camera light-space Z minus the Z anchor: the sampler works + // camera-relative for precision, the raster works absolute, this + // small offset reconciles the two depth mappings + float configResolutionScale; // Eye-distance clipmap floor scale for the sampler, <= 0 disables + uint padding3; + + // Static/dynamic caster split: dynamic pages live in their own small pool and re-render every + // frame, so cached static content is never touched by animated/moving casters + int dynamicRectMinX[8]; // Window-local page coords of this frame's live dynamic pages + int dynamicRectMinY[8]; + int dynamicRectMaxX[8]; + int dynamicRectMaxY[8]; + + uint statsDynamicLive[8]; + uint statsDynamicOverflow; + uint statsDynamicTotal; // Live dynamic pages across all clipmaps, gates the sampler's dynamic lookup + uint configDynamicPoolPagesPerRow; + uint padding4; + + // Render budget: pages past the per-frame cap stay MARKED+INVALID and retry next frame + uint statsDeferred[8]; + uint statsBudgetUsed; + uint padding5; + uint padding6; + uint padding7; + + // Fragment clip rects: this frame's dirty pages classified by UpdateB into three window-local + // page rects per clipmap, indexed [rect * 8 + clipmap]. Rect 0 = newly exposed X columns, + // 1 = newly exposed Y rows, 2 = everything else. The SVSM vertex shaders clip each of the + // three static draws to one rect (SV_ClipDistance) so an L-shaped toroidal update rasterizes + // two thin stripes instead of its huge bounding box + int clipRectMinX[24]; + int clipRectMinY[24]; + int clipRectMaxX[24]; + int clipRectMaxY[24]; +}; + +#define SVSM_CLIP_RECT_DISABLED 0xFFFFFFFFu + +// Clip distances for one static SVSM draw: positive inside the rect, negative outside, so the +// fixed-function clipper culls triangles before rasterization. Same window math as the sampler. +// An empty rect clips everything, a disabled index (dynamic draws) clips nothing +float4 SVSMRectClipDistances(float3 worldPos, uint clipmapIndex, uint rectIndex, float3 cameraPos, StructuredBuffer svsmData) +{ + if (rectIndex == SVSM_CLIP_RECT_DISABLED) + { + return float4(1.0f, 1.0f, 1.0f, 1.0f); + } + + uint rectSlot = rectIndex * 8 + clipmapIndex; + int2 rectMin = int2(svsmData[0].clipRectMinX[rectSlot], svsmData[0].clipRectMinY[rectSlot]); + int2 rectMax = int2(svsmData[0].clipRectMaxX[rectSlot], svsmData[0].clipRectMaxY[rectSlot]); + if (any(rectMax < rectMin)) + { + return float4(-1.0f, -1.0f, -1.0f, -1.0f); + } + + float3x3 lightRotation = BuildLightRotation(svsmData[0].prevLightDirection.xyz); + float2 lightSpaceXY = mul(worldPos - cameraPos, lightRotation).xy; + float2 windowPos = lightSpaceXY + float2(svsmData[0].camMinusWindowMinX[clipmapIndex], svsmData[0].camMinusWindowMinY[clipmapIndex]); + + uint pageSize = svsmData[0].configPageSize; + float texelWorld = svsmData[0].extent[clipmapIndex] / float(svsmData[0].configPageTableSize * pageSize); + float2 texel = windowPos / texelWorld; + + float2 minTexel = float2(rectMin * int(pageSize)); + float2 maxTexel = float2((rectMax + int2(1, 1)) * int(pageSize)); + return float4(texel.x - minTexel.x, maxTexel.x - texel.x, texel.y - minTexel.y, maxTexel.y - texel.y); +} + +uint GetPagePhysical(uint entry) +{ + return entry & SVSM_PAGE_PHYS_MASK; +} + +uint GetPageAge(uint entry) +{ + return (entry & SVSM_PAGE_AGE_MASK) >> SVSM_PAGE_AGE_SHIFT; +} + +uint PackPageEntry(uint physicalPage, uint age, uint flags) +{ + return (physicalPage & SVSM_PAGE_PHYS_MASK) | (age << SVSM_PAGE_AGE_SHIFT) | flags; +} + +// Table slot of an absolute page index, pageTableSize is a power of two +int2 WrapPageSlot(int2 absolutePage, uint pageTableSize) +{ + return absolutePage & int2(pageTableSize - 1, pageTableSize - 1); +} + +uint PageEntryIndex(uint clipmapIndex, int2 slot, uint pageTableSize) +{ + return clipmapIndex * pageTableSize * pageTableSize + uint(slot.y) * pageTableSize + uint(slot.x); +} + +// Window-local page rect of a world AABB in one clipmap: light-space XY rect via the |R| +// extents trick, one page of pad so filter taps just outside the caster footprint are covered +// too, clamped to the window. Returns false when the AABB misses this clipmap's window. +// Page granularity is coarse (quarter meters at the finest clipmap) so plain world-magnitude +// light-space math is precise enough here. windowMin uses the same reconstruction Prepare used +// for camMinusWindowMin, deterministic vs the stored anchors +bool SVSMComputeAABBPageRect(float3 aabbMin, float3 aabbMax, float3 lightDirection, float pageWorld, int2 anchorPageMin, uint pageTableSize, out int2 pageMin, out int2 pageMax) +{ + float3 center = (aabbMin + aabbMax) * 0.5f; + float3 halfExtents = (aabbMax - aabbMin) * 0.5f; + + float3x3 lightRotation = BuildLightRotation(lightDirection); + float2 centerLS = mul(center, lightRotation).xy; + float2 halfLS = mul(halfExtents, float3x3(abs(lightRotation[0]), abs(lightRotation[1]), abs(lightRotation[2]))).xy; + + float2 windowMin = float2(anchorPageMin) * pageWorld; + + pageMin = int2(floor((centerLS - halfLS - windowMin) / pageWorld)) - 1; + pageMax = int2(floor((centerLS + halfLS - windowMin) / pageWorld)) + 1; + pageMin = max(pageMin, int2(0, 0)); + pageMax = min(pageMax, int2(pageTableSize - 1, pageTableSize - 1)); + + return all(pageMax >= pageMin); +} + +// Finest clipmap whose window contains the position with margin to spare, -1 if none. +// One shared function for marking and sampling: the margin guarantees the filter footprint and +// the border page marks never leave the selected window. lightSpaceRelCam = mul(P - cam, R).xy +// +// Eye-distance floor: clipmap containment is light-space-centric but resolution needs are +// eye-centric (at grazing sun, distant relief sits close in light-space XY and would burn fine +// texels on content whose screen pixels are meters wide). Rings whose texels are finer than the +// sample's approximate screen footprint are skipped. floorRingBias: the sampler passes -1 so its +// walk always passes through the ring the marker marked, regardless of float divergence between +// their eye distance reconstructions. resolutionScale <= 0 disables the floor +int SelectClipmap(float2 lightSpaceRelCam, float eyeDistance, StructuredBuffer svsmData, uint numClipmaps, uint pageTableSize, uint pageSize, float marginTexels, float resolutionScale, int floorRingBias) +{ + uint startClipmap = 0; + if (resolutionScale > 0.0f) + { + // ~1000 pixels across the FOV as the fixed reference, exactness is irrelevant + float minTexelWorld = eyeDistance * resolutionScale * (1.0f / 1000.0f); + for (uint k = 0; k + 1 < numClipmaps; k++) + { + float texelWorld = svsmData[0].extent[k] / float(pageTableSize * pageSize); + if (texelWorld >= minTexelWorld) + { + break; + } + startClipmap++; + } + startClipmap = uint(max(int(startClipmap) + floorRingBias, 0)); + } + + for (uint k = startClipmap; k < numClipmaps; k++) + { + float extent = svsmData[0].extent[k]; + float texelWorld = extent / float(pageTableSize * pageSize); + float margin = marginTexels * texelWorld; + + float2 windowPos = lightSpaceRelCam + float2(svsmData[0].camMinusWindowMinX[k], svsmData[0].camMinusWindowMinY[k]); + if (all(windowPos > float2(margin, margin)) && all(windowPos < float2(extent - margin, extent - margin))) + { + return int(k); + } + } + return -1; +} + +// The page-table entry under a page-render fragment. svPosition.xy comes straight from +// SV_Position: window-local virtual texel coordinates (the viewport spans the full virtual +// texture) +uint SVSMLoadPageEntry(float2 svPositionXY, uint clipmapIndex, StructuredBuffer svsmData, StructuredBuffer pageTable) +{ + uint pageTableSize = svsmData[0].configPageTableSize; + uint pageSize = svsmData[0].configPageSize; + + int2 texel = int2(svPositionXY); + int2 localPage = texel / int(pageSize); + int2 anchorPageMin = int2(svsmData[0].anchorPageMinX[clipmapIndex], svsmData[0].anchorPageMinY[clipmapIndex]); + int2 slot = WrapPageSlot(anchorPageMin + localPage, pageTableSize); + + return pageTable[PageEntryIndex(clipmapIndex, slot, pageTableSize)]; +} + +// Software depth write for the SVSM page render passes, taking a pre-loaded page entry (the pixel +// shader may have fetched it for its early alpha-key skip). z is the reversed ortho depth; depth +// clamp disables clipping so pancaked casters saturate here instead. +// requiredFlags: static pages render only while DIRTY, dynamic pages render whenever RESIDENT +void SVSMWritePageDepthEntry(float3 svPosition, uint entry, uint requiredFlags, uint poolPagesPerRow, StructuredBuffer svsmData, RWTexture2D pagePool) +{ + if ((entry & requiredFlags) != requiredFlags) + { + return; + } + + uint pageSize = svsmData[0].configPageSize; + int2 texel = int2(svPosition.xy); + int2 localPage = texel / int(pageSize); + + uint physicalPage = GetPagePhysical(entry); + uint2 physicalBase = uint2(physicalPage % poolPagesPerRow, physicalPage / poolPagesPerRow) * pageSize; + uint2 pageTexel = uint2(texel - localPage * int(pageSize)); + + InterlockedMax(pagePool[physicalBase + pageTexel], asuint(saturate(svPosition.z))); +} + +// Fragments outside pages that render this frame exit after one table load +void SVSMWritePageDepth(float3 svPosition, uint clipmapIndex, uint requiredFlags, uint poolPagesPerRow, StructuredBuffer svsmData, StructuredBuffer pageTable, RWTexture2D pagePool) +{ + uint entry = SVSMLoadPageEntry(svPosition.xy, clipmapIndex, svsmData, pageTable); + SVSMWritePageDepthEntry(svPosition, entry, requiredFlags, poolPagesPerRow, svsmData, pagePool); +} + +#endif // SVSM_INCLUDED diff --git a/Source/Shaders/Shaders/Shadows/SVSMDynamicMark.cs.slang b/Source/Shaders/Shaders/Shadows/SVSMDynamicMark.cs.slang new file mode 100644 index 00000000..3f49752b --- /dev/null +++ b/Source/Shaders/Shaders/Shadows/SVSMDynamicMark.cs.slang @@ -0,0 +1,69 @@ +#include "Shadows/SVSM.inc.slang" + +// Marks dynamic pages: for each dynamic caster AABB and clipmap, pages that the caster overlaps +// AND that are visible this frame (marked in the static table) get MARKED in the dynamic table. +// Runs after SVSMPageMark so the static visibility marks are current. One thread per +// (AABB, clipmap) pair, same light-space rect math as SVSMInvalidateAABBs — for an ortho light a +// caster's light-space XY equals its shadow's XY, so this covers where the shadow lands. + +[[vk::push_constant]] SVSMConstants _constants; + +[[vk::binding(0, PER_PASS)]] StructuredBuffer _svsmData; +[[vk::binding(1, PER_PASS)]] StructuredBuffer _pageTable; // Static table, read for visibility +[[vk::binding(2, PER_PASS)]] RWStructuredBuffer _dynamicPageTable; +[[vk::binding(3, PER_PASS)]] StructuredBuffer _dynamicAABBs; // Pairs of world (min, max) + +struct CSInput +{ + uint3 dispatchThreadID : SV_DispatchThreadID; +}; + +[shader("compute")] +[numthreads(64, 1, 1)] +void main(CSInput input) +{ + uint aabbIndex = input.dispatchThreadID.x; + uint clipmapIndex = input.dispatchThreadID.y; + if (aabbIndex >= _constants.numDirtyAABBs || clipmapIndex >= _constants.numClipmaps) + { + return; + } + + float3 aabbMin = _dynamicAABBs[aabbIndex * 2 + 0].xyz; + float3 aabbMax = _dynamicAABBs[aabbIndex * 2 + 1].xyz; + + uint pageTableSize = _constants.pageTableSize; + float pageWorld = _svsmData[0].pageWorld[clipmapIndex]; + int2 anchorPageMin = int2(_svsmData[0].anchorPageMinX[clipmapIndex], _svsmData[0].anchorPageMinY[clipmapIndex]); + + int2 pageMin; + int2 pageMax; + if (!SVSMComputeAABBPageRect(aabbMin, aabbMax, _constants.lightDirection.xyz, pageWorld, anchorPageMin, pageTableSize, pageMin, pageMax)) + { + return; // Outside this clipmap's window + } + + // A huge dynamic mover marks at most this many pages per ring, coarser clipmaps cover it + // (span quarters per ring). The CPU classifier routes casters that would exceed this in EVERY + // ring to the static path instead, so the cutoff never orphans a caster from both pools + int2 span = pageMax - pageMin + int2(1, 1); + if (span.x * span.y > 1024) + { + return; + } + + for (int y = pageMin.y; y <= pageMax.y; y++) + { + for (int x = pageMin.x; x <= pageMax.x; x++) + { + int2 slot = WrapPageSlot(anchorPageMin + int2(x, y), pageTableSize); + uint entryIndex = PageEntryIndex(clipmapIndex, slot, pageTableSize); + + // Only where something visible actually samples this frame + if ((_pageTable[entryIndex] & SVSM_PAGE_MARKED) != 0) + { + InterlockedOr(_dynamicPageTable[entryIndex], SVSM_PAGE_MARKED); + } + } + } +} diff --git a/Source/Shaders/Shaders/Shadows/SVSMDynamicUpdate.cs.slang b/Source/Shaders/Shaders/Shadows/SVSMDynamicUpdate.cs.slang new file mode 100644 index 00000000..69fc713f --- /dev/null +++ b/Source/Shaders/Shaders/Shadows/SVSMDynamicUpdate.cs.slang @@ -0,0 +1,129 @@ +#include "Shadows/SVSM.inc.slang" + +// Dynamic page lifecycle, one thread per entry, runs after SVSMDynamicMark. Dispatched TWICE: +// dynamicPhase 0 releases pages nobody marked, dynamicPhase 1 allocates marked entries from the +// dynamic free list and queues every live page for the per-frame clear+render. The phases must +// not share a dispatch: a pop racing a push can read the list slot before the pushed physical +// index lands, aliasing one physical page under two table entries (ghost shadows). +// Dynamic pages are transient by design — they re-render every frame, so there is no age, no +// DIRTY bookkeeping and no invalidation. Clears MARKED on write so next frame starts clean. + +[[vk::push_constant]] SVSMConstants _constants; + +[[vk::binding(0, PER_PASS)]] RWStructuredBuffer _svsmData; +[[vk::binding(1, PER_PASS)]] RWStructuredBuffer _dynamicPageTable; +[[vk::binding(2, PER_PASS)]] RWStructuredBuffer _dynamicFreeList; // [0] = count, entries from [4] +[[vk::binding(3, PER_PASS)]] RWStructuredBuffer _dynamicClearList; // [0..2] = indirect dispatch args, physical pages from [4] + +groupshared uint gLiveCount; +groupshared int gRectMinX; +groupshared int gRectMinY; +groupshared int gRectMaxX; +groupshared int gRectMaxY; + +struct CSInput +{ + uint3 dispatchThreadID : SV_DispatchThreadID; + uint groupIndex : SV_GroupIndex; +}; + +[shader("compute")] +[numthreads(256, 1, 1)] +void main(CSInput input) +{ + if (input.groupIndex == 0) + { + gLiveCount = 0; + gRectMinX = 0x7FFFFFFF; + gRectMinY = 0x7FFFFFFF; + gRectMaxX = -1; + gRectMaxY = -1; + } + GroupMemoryBarrierWithGroupSync(); + + uint pageTableSize = _constants.pageTableSize; + uint entriesPerClipmap = pageTableSize * pageTableSize; + uint entryIndex = input.dispatchThreadID.x; + uint clipmapIndex = entryIndex / entriesPerClipmap; // Group-uniform, entriesPerClipmap is a multiple of 256 + + uint entry = _dynamicPageTable[entryIndex]; + // No early returns before the group barrier below. Phase 0 only releases; allocation and + // bookkeeping run in phase 1, after the free-list barrier between the two dispatches + if (entry != 0 && _constants.dynamicPhase == 0) + { + bool marked = (entry & SVSM_PAGE_MARKED) != 0; + bool resident = (entry & SVSM_PAGE_RESIDENT) != 0; + if (!marked && resident) + { + // No dynamic caster overlaps this page anymore, return it + uint writeIndex; + InterlockedAdd(_dynamicFreeList[0], 1, writeIndex); + _dynamicFreeList[4 + writeIndex] = GetPagePhysical(entry); + _dynamicPageTable[entryIndex] = 0; + } + } + else if (entry != 0) + { + bool marked = (entry & SVSM_PAGE_MARKED) != 0; + bool resident = (entry & SVSM_PAGE_RESIDENT) != 0; + + if (marked && !resident) + { + uint previousCount; + InterlockedAdd(_dynamicFreeList[0], uint(-1), previousCount); + if (int(previousCount) <= 0) + { + InterlockedAdd(_dynamicFreeList[0], 1); + InterlockedAdd(_svsmData[0].statsDynamicOverflow, 1); + entry = 0; // Stays non-resident, the sampler simply sees no dynamic caster here + } + else + { + uint physicalPage = _dynamicFreeList[4 + previousCount - 1]; + entry = PackPageEntry(physicalPage, 0, SVSM_PAGE_RESIDENT); + resident = true; + } + } + else if (marked && resident) + { + entry = PackPageEntry(GetPagePhysical(entry), 0, SVSM_PAGE_RESIDENT); // Clear MARKED for next frame + } + + if (resident) + { + // Every live dynamic page clears and re-renders this frame + uint clearIndex; + InterlockedAdd(_dynamicClearList[0], 1, clearIndex); + _dynamicClearList[4 + clearIndex] = GetPagePhysical(entry); + + InterlockedAdd(gLiveCount, 1); + + if (clipmapIndex < _constants.numClipmaps) + { + uint slotLinear = entryIndex - clipmapIndex * entriesPerClipmap; + int2 slot = int2(slotLinear & (pageTableSize - 1), slotLinear / pageTableSize); + int2 anchor = int2(_svsmData[0].anchorPageMinX[clipmapIndex], _svsmData[0].anchorPageMinY[clipmapIndex]); + int2 localPage = WrapPageSlot(slot - anchor, pageTableSize); + + InterlockedMin(gRectMinX, localPage.x); + InterlockedMin(gRectMinY, localPage.y); + InterlockedMax(gRectMaxX, localPage.x); + InterlockedMax(gRectMaxY, localPage.y); + } + } + + _dynamicPageTable[entryIndex] = entry; + } + + GroupMemoryBarrierWithGroupSync(); + + if (input.groupIndex == 0 && clipmapIndex < 8 && gLiveCount != 0) + { + InterlockedAdd(_svsmData[0].statsDynamicLive[clipmapIndex], gLiveCount); + InterlockedAdd(_svsmData[0].statsDynamicTotal, gLiveCount); + InterlockedMin(_svsmData[0].dynamicRectMinX[clipmapIndex], gRectMinX); + InterlockedMin(_svsmData[0].dynamicRectMinY[clipmapIndex], gRectMinY); + InterlockedMax(_svsmData[0].dynamicRectMaxX[clipmapIndex], gRectMaxX); + InterlockedMax(_svsmData[0].dynamicRectMaxY[clipmapIndex], gRectMaxY); + } +} diff --git a/Source/Shaders/Shaders/Shadows/SVSMFinalize.cs.slang b/Source/Shaders/Shaders/Shadows/SVSMFinalize.cs.slang new file mode 100644 index 00000000..01afdbb5 --- /dev/null +++ b/Source/Shaders/Shaders/Shadows/SVSMFinalize.cs.slang @@ -0,0 +1,145 @@ +#include "Include/Camera.inc.slang" +#include "Shadows/SVSM.inc.slang" + +// Last SVSM dispatch, single thread, runs after UpdateB and DynamicUpdate so the dirty rects and +// per-ring page stats are final: writes the clipmap render cameras into the shared cameras buffer +// that culling and the page render passes read, and the per-view fill dispatch args that gate the +// geometry passes' instance fills. Same GPU-writes-the-camera-slots shape the retired SDSM fit +// pass used, minus the texel snap: the window center is aligned to the absolute page grid by +// construction. +// +// The camera MATRICES always map the full clipmap window (the render viewport spans the whole +// virtual texture). The cull PLANES are tightened to this frame's dirty page rect, XY-tight is +// safe for an ortho light (a caster's light-space XY equals its shadow's XY); the near plane is +// swept toward the sun by the caster margin, depth clamp pancakes those casters at render time. +// +// The fill args are the same-frame GPU truth the per-view skip must run on: a ring with no DIRTY +// static pages skips its static fill, a ring with no resident dynamic pages skips its dynamic +// fill. Gating this from a CPU readback (one frame old) drops freshly acquired dynamic pages' +// draws for a frame — resident-but-undrawn pages read lit and flicker dynamic shadows out + +[[vk::push_constant]] SVSMConstants _constants; + +[[vk::binding(0, PER_PASS)]] RWStructuredBuffer _rwCameras; // Same buffer as the global _cameras, element 0 = main camera +[[vk::binding(1, PER_PASS)]] StructuredBuffer _svsmData; +[[vk::binding(2, PER_PASS)]] RWStructuredBuffer _fillDispatchArgs; // Per clipmap: 5 x uvec3 (model static fill, model dynamic fill, terrain static fill, model static overhead, model dynamic overhead) + +[shader("compute")] +[numthreads(1, 1, 1)] +void main() +{ + float3 lightDirection = _constants.lightDirection.xyz; + float3x3 lightRotation = BuildLightRotation(lightDirection); + float3 upDir = StableUp(lightDirection); + + // Per-view fill dispatch args, zero groups for rings with no page work this frame. The + // overhead args gate the fill's drawcall-granularity companions (instance-count clear + + // CreateIndirect) on the same bits — safe because a skipped CreateIndirect means nothing + // reads the stale instance counts, and the tiny per-fill counter reset stays unconditional + uint modelFillGroups = (_constants.fillInstanceCount + 31) / 32; + uint terrainFillGroups = (_constants.fillCellCount + 31) / 32; + uint modelOverheadGroups = (_constants.fillDrawCallCount + 31) / 32; + for (uint k = 0; k < 8; k++) + { + bool inUse = k < _constants.numClipmaps; + bool hasDirty = inUse && _svsmData[0].statsDirty[k] != 0; + bool hasDynamic = inUse && _svsmData[0].statsDynamicLive[k] != 0; + + uint base = k * 15; + _fillDispatchArgs[base + 0] = hasDirty ? modelFillGroups : 0; + _fillDispatchArgs[base + 1] = 1; + _fillDispatchArgs[base + 2] = 1; + _fillDispatchArgs[base + 3] = hasDynamic ? modelFillGroups : 0; + _fillDispatchArgs[base + 4] = 1; + _fillDispatchArgs[base + 5] = 1; + _fillDispatchArgs[base + 6] = hasDirty ? terrainFillGroups : 0; + _fillDispatchArgs[base + 7] = 1; + _fillDispatchArgs[base + 8] = 1; + _fillDispatchArgs[base + 9] = hasDirty ? modelOverheadGroups : 0; + _fillDispatchArgs[base + 10] = 1; + _fillDispatchArgs[base + 11] = 1; + _fillDispatchArgs[base + 12] = hasDynamic ? modelOverheadGroups : 0; + _fillDispatchArgs[base + 13] = 1; + _fillDispatchArgs[base + 14] = 1; + } + + float zRangeMin = _svsmData[0].zRangeMin; + float zRangeMax = _svsmData[0].zRangeMax; + float zCenter = (zRangeMin + zRangeMax) * 0.5f; + float zHalf = max((zRangeMax - zRangeMin) * 0.5f, 1.0f); + + for (uint k = 0; k < _constants.numClipmaps; k++) + { + float extent = _svsmData[0].extent[k]; + float pageWorld = _svsmData[0].pageWorld[k]; + float halfExtent = extent * 0.5f; + + int2 anchorPageMin = int2(_svsmData[0].anchorPageMinX[k], _svsmData[0].anchorPageMinY[k]); + float2 windowMinLS = float2(anchorPageMin) * pageWorld; + float2 centerLS = windowMinLS + float2(halfExtent, halfExtent); + + // Rotate the light-space center back to world, mul(v, transpose(R)) inverts the rotation + float3 worldCenter = mul(float3(centerLS, zCenter), transpose(lightRotation)); + float3 shadowCameraPos = worldCenter - lightDirection * zHalf; // Sun side + + // Bottom/top swapped: the engine's viewport Y-flip would otherwise mirror the page content + // against the marking/sampling texel math, this way SV_Position.xy IS the virtual texel + float4x4 proj = BuildOrtho(-halfExtent, halfExtent, halfExtent, -halfExtent, 2.0f * zHalf, 0.0f); // Reversed Z, sun side = 1 + float4x4 view = BuildLookAt(shadowCameraPos, worldCenter, upDir); + + Camera clipmapCamera = (Camera)0; + clipmapCamera.worldToView = view; + clipmapCamera.viewToClip = proj; + clipmapCamera.viewToWorld = InverseLookAt(view, shadowCameraPos); + clipmapCamera.clipToView = InverseOrtho(proj); + clipmapCamera.worldToClip = mul(view, proj); // glm viewToClip * worldToView + clipmapCamera.clipToWorld = mul(clipmapCamera.clipToView, clipmapCamera.viewToWorld); // glm viewToWorld * clipToView + + clipmapCamera.eyePosition = float4(shadowCameraPos, 0.0f); + clipmapCamera.eyeRotation = float4(0.0f, 0.0f, 0.0f, 0.0f); + clipmapCamera.nearFar = float4(0.0f, 0.0f, 0.0f, 0.0f); + + // Union of the static dirty rect and the dynamic live rect: one cull volume per view serves + // both fill classes, the class filter and the PS page checks absorb the over-inclusion + int2 dirtyMin = min(int2(_svsmData[0].dirtyRectMinX[k], _svsmData[0].dirtyRectMinY[k]), + int2(_svsmData[0].dynamicRectMinX[k], _svsmData[0].dynamicRectMinY[k])); + int2 dirtyMax = max(int2(_svsmData[0].dirtyRectMaxX[k], _svsmData[0].dirtyRectMaxY[k]), + int2(_svsmData[0].dynamicRectMaxX[k], _svsmData[0].dynamicRectMaxY[k])); + if (any(dirtyMax < dirtyMin)) + { + // Nothing renders into this clipmap this frame, cull everything + for (uint p = 0; p < 6; p++) + { + clipmapCamera.frustum[p] = float4(0.0f, 1.0f, 0.0f, 3.0e38f); // dot(n, p) - w is always far below -radius + } + } + else + { + // Cull volume over the dirty pages only, window-local rect back to center-relative extents + float cullMinX = float(dirtyMin.x) * pageWorld - halfExtent; + float cullMaxX = float(dirtyMax.x + 1) * pageWorld - halfExtent; + float cullMinY = float(dirtyMin.y) * pageWorld - halfExtent; + float cullMaxY = float(dirtyMax.y + 1) * pageWorld - halfExtent; + + float4x4 cullProj = BuildOrtho(cullMinX, cullMaxX, cullMaxY, cullMinY, 2.0f * zHalf, 0.0f); // Same Y mirror as proj, plane extraction is sign-agnostic + float4x4 cullMatrix = mul(view, cullProj); + + float4 row0 = MatrixRow(cullMatrix, 0); + float4 row1 = MatrixRow(cullMatrix, 1); + float4 row2 = MatrixRow(cullMatrix, 2); + float4 row3 = MatrixRow(cullMatrix, 3); + + clipmapCamera.frustum[0] = ExtractPlane(row3 + row0); // Left + clipmapCamera.frustum[1] = ExtractPlane(row3 - row0); // Right + clipmapCamera.frustum[2] = ExtractPlane(row3 + row1); // Bottom + clipmapCamera.frustum[3] = ExtractPlane(row3 - row1); // Top + clipmapCamera.frustum[4] = ExtractPlane(row3 - row2); // Near, reversed Z so depth 1 is near + clipmapCamera.frustum[5] = ExtractPlane(row2); // Far + + // Casters beyond the near plane still render correctly (depth clamp pancakes them) + clipmapCamera.frustum[4].w -= _constants.casterMargin; + } + + _rwCameras[1 + k] = clipmapCamera; + } +} diff --git a/Source/Shaders/Shaders/Shadows/SVSMInvalidateAABBs.cs.slang b/Source/Shaders/Shaders/Shadows/SVSMInvalidateAABBs.cs.slang new file mode 100644 index 00000000..8f6a3b3e --- /dev/null +++ b/Source/Shaders/Shaders/Shadows/SVSMInvalidateAABBs.cs.slang @@ -0,0 +1,61 @@ +#include "Shadows/SVSM.inc.slang" + +// Marks pages under moved or animated instances as INVALID. One thread per (AABB, clipmap) pair, +// each ORs the flag over the covered page rect. Page granularity is coarse (quarter meters at the +// finest clipmap) so plain world-magnitude light-space math is precise enough here. + +[[vk::push_constant]] SVSMConstants _constants; + +[[vk::binding(0, PER_PASS)]] RWStructuredBuffer _svsmData; +[[vk::binding(1, PER_PASS)]] RWStructuredBuffer _pageTable; +[[vk::binding(2, PER_PASS)]] StructuredBuffer _dirtyAABBs; // Pairs of world (min, max) + +struct CSInput +{ + uint3 dispatchThreadID : SV_DispatchThreadID; +}; + +[shader("compute")] +[numthreads(64, 1, 1)] +void main(CSInput input) +{ + uint aabbIndex = input.dispatchThreadID.x; + uint clipmapIndex = input.dispatchThreadID.y; + if (aabbIndex >= _constants.numDirtyAABBs || clipmapIndex >= _constants.numClipmaps) + { + return; + } + + float3 aabbMin = _dirtyAABBs[aabbIndex * 2 + 0].xyz; + float3 aabbMax = _dirtyAABBs[aabbIndex * 2 + 1].xyz; + + uint pageTableSize = _constants.pageTableSize; + float pageWorld = _svsmData[0].pageWorld[clipmapIndex]; + int2 anchorPageMin = int2(_svsmData[0].anchorPageMinX[clipmapIndex], _svsmData[0].anchorPageMinY[clipmapIndex]); + + int2 pageMin; + int2 pageMax; + if (!SVSMComputeAABBPageRect(aabbMin, aabbMax, _constants.lightDirection.xyz, pageWorld, anchorPageMin, pageTableSize, pageMin, pageMax)) + { + return; // Outside this clipmap's window + } + + // A huge mover would loop for thousands of pages from one thread, fall back to a full + // invalidation instead. UpdateA consumes frameInvalidationFlags after this pass's barrier + int2 span = pageMax - pageMin + int2(1, 1); + if (span.x * span.y > 1024) + { + InterlockedOr(_svsmData[0].frameInvalidationFlags, SVSM_CAUSE_AABB_OVERFLOW); + InterlockedOr(_svsmData[0].statsInvalidationCause, SVSM_CAUSE_AABB_OVERFLOW); + return; + } + + for (int y = pageMin.y; y <= pageMax.y; y++) + { + for (int x = pageMin.x; x <= pageMax.x; x++) + { + int2 slot = WrapPageSlot(anchorPageMin + int2(x, y), pageTableSize); + InterlockedOr(_pageTable[PageEntryIndex(clipmapIndex, slot, pageTableSize)], SVSM_PAGE_INVALID); + } + } +} diff --git a/Source/Shaders/Shaders/Shadows/SVSMPageClear.cs.slang b/Source/Shaders/Shaders/Shadows/SVSMPageClear.cs.slang new file mode 100644 index 00000000..18472693 --- /dev/null +++ b/Source/Shaders/Shaders/Shadows/SVSMPageClear.cs.slang @@ -0,0 +1,34 @@ +#include "Shadows/SVSM.inc.slang" + +// Clears this frame's dirty pages in the physical pool, dispatched indirectly from the clear +// list UpdateB filled: one workgroup per dirty page. Cleared depth 0 (reversed Z far) reads as +// fully lit, so a dirty page with no casters is still correct. + +[[vk::push_constant]] SVSMConstants _constants; + +[[vk::binding(0, PER_PASS)]] StructuredBuffer _clearList; // [0..2] = indirect dispatch args, physical pages from [4] +[[vk::binding(1, PER_PASS)]] RWTexture2D _pagePool; + +struct CSInput +{ + uint3 groupID : SV_GroupID; + uint groupIndex : SV_GroupIndex; +}; + +[shader("compute")] +[numthreads(256, 1, 1)] +void main(CSInput input) +{ + uint physicalPage = _clearList[4 + input.groupID.x]; + + uint pageSize = _constants.pageSize; + uint poolPagesPerRow = _constants.poolPagesPerRow; + uint2 physicalBase = uint2(physicalPage % poolPagesPerRow, physicalPage / poolPagesPerRow) * pageSize; + + uint texelsPerPage = pageSize * pageSize; + for (uint i = input.groupIndex; i < texelsPerPage; i += 256) + { + uint2 texel = uint2(i % pageSize, i / pageSize); + _pagePool[physicalBase + texel] = 0; + } +} diff --git a/Source/Shaders/Shaders/Shadows/SVSMPageMark.cs.slang b/Source/Shaders/Shaders/Shadows/SVSMPageMark.cs.slang new file mode 100644 index 00000000..224cff0d --- /dev/null +++ b/Source/Shaders/Shaders/Shadows/SVSMPageMark.cs.slang @@ -0,0 +1,131 @@ +#include "Include/Camera.inc.slang" +#include "Shadows/SVSM.inc.slang" + +// Marks the pages touched by visible depth samples: one thread per depth pixel reconstructs the +// world position and marks the page it samples. Positions are reconstructed camera-relative (clipToView + rotate-only +// viewToWorld) and addressed through the anchor-relative window math from SVSM.inc. +// +// Samples within markBorderTexels of a page border also mark the neighbor page so the filter +// footprint is resident at sampling time; the selection margin guarantees those neighbors exist +// inside the selected clipmap's window. +// +// A 16x16 group usually lands in a handful of distinct pages, so marks are deduplicated through a +// small LDS cache and flushed as a few global ORs instead of 256. + +[[vk::push_constant]] SVSMConstants _constants; + +[[vk::binding(0, PER_PASS)]] Texture2D _depth; +[[vk::binding(1, PER_PASS)]] StructuredBuffer _srcCameras; // Element 0 = main camera +[[vk::binding(2, PER_PASS)]] StructuredBuffer _svsmData; +[[vk::binding(3, PER_PASS)]] RWStructuredBuffer _pageTable; + +#define MARK_CACHE_SIZE 16 +#define MARK_CACHE_EMPTY 0xFFFFFFFFu + +groupshared uint gMarkCache[MARK_CACHE_SIZE]; + +void MarkPage(uint entryIndex) +{ + // Dedup through the cache, linear probing. Falls back to a direct OR when full + for (uint i = 0; i < MARK_CACHE_SIZE; i++) + { + uint existing; + InterlockedCompareExchange(gMarkCache[i], MARK_CACHE_EMPTY, entryIndex, existing); + if (existing == MARK_CACHE_EMPTY || existing == entryIndex) + { + return; + } + } + InterlockedOr(_pageTable[entryIndex], SVSM_PAGE_MARKED); +} + +struct CSInput +{ + uint3 dispatchThreadID : SV_DispatchThreadID; + uint groupIndex : SV_GroupIndex; +}; + +[shader("compute")] +[numthreads(16, 16, 1)] +void main(CSInput input) +{ + if (input.groupIndex < MARK_CACHE_SIZE) + { + gMarkCache[input.groupIndex] = MARK_CACHE_EMPTY; + } + GroupMemoryBarrierWithGroupSync(); + + uint2 dim; + _depth.GetDimensions(dim.x, dim.y); + + // Clamp instead of early return: the LDS mark-cache flush below needs every thread at the + // group barrier, and the duplicate edge marks dedup through the cache anyway + uint2 pixel = min(input.dispatchThreadID.xy, dim - 1); + float depth = _depth[pixel]; + + // Cleared depth is exactly 0.0 (reversed Z far plane), sky never needs shadow pages + if (depth != 0.0f) + { + Camera mainCamera = _srcCameras[0]; + + float2 uv = (float2(pixel) + 0.5f) / float2(dim); + float2 ndc = float2(uv.x, 1.0f - uv.y) * 2.0f - 1.0f; + + float4 viewPos = mul(float4(ndc, depth, 1.0f), mainCamera.clipToView); + viewPos.xyz /= viewPos.w; + + // Camera-relative world position, rotation only + float3 relativePos = mul(float4(viewPos.xyz, 0.0f), mainCamera.viewToWorld).xyz; + float2 lightSpaceRelCam = mul(relativePos, BuildLightRotation(_constants.lightDirection.xyz)).xy; + + uint pageTableSize = _constants.pageTableSize; + uint pageSize = _constants.pageSize; + + // +2 texels of slack over the border marks so snap jitter can never select a clipmap whose + // border pages fall outside the window + float eyeDistance = length(viewPos.xyz); + int clipmapIndex = SelectClipmap(lightSpaceRelCam, eyeDistance, _svsmData, _constants.numClipmaps, pageTableSize, pageSize, _constants.markBorderTexels + 2.0f, _constants.resolutionScale, 0); + if (clipmapIndex >= 0) + { + float extent = _svsmData[0].extent[clipmapIndex]; + float texelWorld = extent / float(pageTableSize * pageSize); + float2 windowPos = lightSpaceRelCam + float2(_svsmData[0].camMinusWindowMinX[clipmapIndex], _svsmData[0].camMinusWindowMinY[clipmapIndex]); + + int2 texel = int2(windowPos / texelWorld); + int2 localPage = texel / int(pageSize); + int2 pageTexel = texel - localPage * int(pageSize); + int2 anchorPageMin = int2(_svsmData[0].anchorPageMinX[clipmapIndex], _svsmData[0].anchorPageMinY[clipmapIndex]); + + int border = int(_constants.markBorderTexels); + int2 neighborStep = int2(0, 0); + if (pageTexel.x < border) neighborStep.x = -1; + else if (pageTexel.x >= int(pageSize) - border) neighborStep.x = 1; + if (pageTexel.y < border) neighborStep.y = -1; + else if (pageTexel.y >= int(pageSize) - border) neighborStep.y = 1; + + // The containing page plus the bordering neighbors (corner marks all three) + for (int sy = 0; sy < 2; sy++) + { + for (int sx = 0; sx < 2; sx++) + { + int2 pageOffset = int2(sx * neighborStep.x, sy * neighborStep.y); + if ((sx == 1 && pageOffset.x == 0) || (sy == 1 && pageOffset.y == 0)) + { + continue; // No border on this axis, skip the duplicate + } + + int2 page = clamp(localPage + pageOffset, int2(0, 0), int2(pageTableSize - 1, pageTableSize - 1)); + int2 slot = WrapPageSlot(anchorPageMin + page, pageTableSize); + MarkPage(PageEntryIndex(clipmapIndex, slot, pageTableSize)); + } + } + } + } + + GroupMemoryBarrierWithGroupSync(); + + if (input.groupIndex < MARK_CACHE_SIZE && gMarkCache[input.groupIndex] != MARK_CACHE_EMPTY) + { + InterlockedOr(_pageTable[gMarkCache[input.groupIndex]], SVSM_PAGE_MARKED); + } +} diff --git a/Source/Shaders/Shaders/Shadows/SVSMPageTableDebug.cs.slang b/Source/Shaders/Shaders/Shadows/SVSMPageTableDebug.cs.slang new file mode 100644 index 00000000..35651cf9 --- /dev/null +++ b/Source/Shaders/Shaders/Shadows/SVSMPageTableDebug.cs.slang @@ -0,0 +1,88 @@ +#include "Shadows/SVSM.inc.slang" + +// Draws one clipmap's page table as a colored grid straight into the scene color corner. +// Black = free slot, green = resident (darkens with age), orange = resident but invalidated, +// red = dirty (rendered this frame), yellow border tint = marked this frame, magenta = marked but +// not resident (pool overflow). + +struct DebugConstants +{ + uint clipmapIndex; + uint pageTableSize; + uint cellSize; + uint padding0; + int2 screenOffset; +}; + +[[vk::push_constant]] DebugConstants _constants; + +[[vk::binding(0, PER_PASS)]] StructuredBuffer _pageTable; +[[vk::binding(1, PER_PASS)]] RWTexture2D _target; + +struct CSInput +{ + uint3 dispatchThreadID : SV_DispatchThreadID; +}; + +[shader("compute")] +[numthreads(16, 16, 1)] +void main(CSInput input) +{ + uint pageTableSize = _constants.pageTableSize; + uint cellSize = _constants.cellSize; + uint regionSize = pageTableSize * cellSize; + if (any(input.dispatchThreadID.xy >= uint2(regionSize, regionSize))) + { + return; + } + + uint2 targetDim; + _target.GetDimensions(targetDim.x, targetDim.y); + int2 targetPixel = _constants.screenOffset + int2(input.dispatchThreadID.xy); + if (any(targetPixel < int2(0, 0)) || any(targetPixel >= int2(targetDim))) + { + return; + } + + uint2 slot = input.dispatchThreadID.xy / cellSize; + uint2 cellPixel = input.dispatchThreadID.xy - slot * cellSize; + + // The table is indexed by wrapped slots, displaying slot space directly shows the torus + uint entry = _pageTable[PageEntryIndex(_constants.clipmapIndex, int2(slot), pageTableSize)]; + + float3 color = float3(0.05f, 0.05f, 0.05f); + if (entry != 0) + { + bool resident = (entry & SVSM_PAGE_RESIDENT) != 0; + bool invalid = (entry & SVSM_PAGE_INVALID) != 0; + bool marked = (entry & SVSM_PAGE_MARKED) != 0; + bool dirty = (entry & SVSM_PAGE_DIRTY) != 0; + + if (dirty) + { + color = float3(0.9f, 0.1f, 0.1f); + } + else if (resident && invalid) + { + color = float3(0.9f, 0.5f, 0.1f); + } + else if (resident) + { + float freshness = 1.0f - float(GetPageAge(entry)) / 255.0f; + color = float3(0.1f, 0.25f + 0.5f * freshness, 0.1f); + } + else if (marked) + { + color = float3(0.9f, 0.1f, 0.9f); // Marked but never became resident, pool overflow + } + + // Marked pages get a bright border ring so the working set pops against cached pages + bool onCellBorder = any(cellPixel == uint2(0, 0)) || any(cellPixel == uint2(cellSize - 1, cellSize - 1)); + if (marked && onCellBorder) + { + color = float3(0.9f, 0.9f, 0.2f); + } + } + + _target[uint2(targetPixel)] = float4(color, 1.0f); +} diff --git a/Source/Shaders/Shaders/Shadows/SVSMPageUpdateA.cs.slang b/Source/Shaders/Shaders/Shadows/SVSMPageUpdateA.cs.slang new file mode 100644 index 00000000..f14bd52b --- /dev/null +++ b/Source/Shaders/Shaders/Shadows/SVSMPageUpdateA.cs.slang @@ -0,0 +1,131 @@ +#include "Shadows/SVSM.inc.slang" + +// Page table maintenance, one thread per entry, runs before this frame's marking: +// applies toroidal and global invalidation, ages entries, evicts stale pages back to the free +// list and clears last frame's MARKED/DIRTY bits (safe because dirty pages always rendered the +// frame they were marked). Must run before marking: this is the only pass that read-modify-writes +// entries non-atomically. + +[[vk::push_constant]] SVSMConstants _constants; + +[[vk::binding(0, PER_PASS)]] RWStructuredBuffer _svsmData; +[[vk::binding(1, PER_PASS)]] RWStructuredBuffer _pageTable; +[[vk::binding(2, PER_PASS)]] RWStructuredBuffer _freeList; // [0] = count, entries from [4] + +groupshared uint gResidentCount; +groupshared uint gInvalidatedCount; +groupshared uint gEvictedCount; + +struct CSInput +{ + uint3 dispatchThreadID : SV_DispatchThreadID; + uint groupIndex : SV_GroupIndex; +}; + +[shader("compute")] +[numthreads(256, 1, 1)] +void main(CSInput input) +{ + if (input.groupIndex == 0) + { + gResidentCount = 0; + gInvalidatedCount = 0; + gEvictedCount = 0; + } + GroupMemoryBarrierWithGroupSync(); + + uint pageTableSize = _constants.pageTableSize; + uint entriesPerClipmap = pageTableSize * pageTableSize; + uint entryIndex = input.dispatchThreadID.x; + uint clipmapIndex = entryIndex / entriesPerClipmap; // Group-uniform, entriesPerClipmap is a multiple of 256 + + uint entry = _pageTable[entryIndex]; + if (entry != 0) + { + bool resident = (entry & SVSM_PAGE_RESIDENT) != 0; + bool invalid = (entry & SVSM_PAGE_INVALID) != 0; + bool markedLastFrame = (entry & SVSM_PAGE_MARKED) != 0; + uint age = markedLastFrame ? 0 : min(GetPageAge(entry) + 1, 255u); + + bool clipmapActive = clipmapIndex < _constants.numClipmaps; + if (clipmapActive) + { + // Toroidal reuse: valid only if the slot maps to the same absolute page under the old + // and the new window. Handles any slide distance, teleports included + uint slotLinear = entryIndex - clipmapIndex * entriesPerClipmap; + int2 slot = int2(slotLinear & (pageTableSize - 1), slotLinear / pageTableSize); + + int2 anchor = int2(_svsmData[0].anchorPageMinX[clipmapIndex], _svsmData[0].anchorPageMinY[clipmapIndex]); + int2 prevAnchor = int2(_svsmData[0].prevAnchorPageMinX[clipmapIndex], _svsmData[0].prevAnchorPageMinY[clipmapIndex]); + int2 absNew = anchor + WrapPageSlot(slot - anchor, pageTableSize); + int2 absOld = prevAnchor + WrapPageSlot(slot - prevAnchor, pageTableSize); + + bool stale = any(absNew != absOld); + stale = stale || (_svsmData[0].frameInvalidationFlags != 0); + stale = stale || (_svsmData[0].clipmapInvalidate[clipmapIndex] != 0); + + if (stale && resident && !invalid) + { + InterlockedAdd(gInvalidatedCount, 1); + } + invalid = invalid || stale; + } + + // Pressure eviction: when the pool runs dry the stale cache recycles within frames instead + // of squatting for the full evict age, so the finest clipmaps can always allocate. + // The age field saturates at 255, an evict age at or above that would never trigger + uint effectiveEvictAge = min(_constants.evictAge, 254u); + uint poolPages = _constants.poolPagesPerRow * _constants.poolPagesPerRow; + uint freeCount = _svsmData[0].statsFreeListCount; // Start-of-frame snapshot from Prepare + if (freeCount < poolPages / 8) + { + effectiveEvictAge = min(effectiveEvictAge, 30u); + } + if (freeCount < poolPages / 32) + { + effectiveEvictAge = min(effectiveEvictAge, 2u); + } + + // Invalidated pages that nothing marked are dead weight: their content may live in an old + // depth mapping (Z-range/sun change) which the sampler refuses, so free them immediately + // instead of letting them squat for the full evict age. Marked ones re-render in UpdateB + if (resident && (age > effectiveEvictAge || (invalid && !markedLastFrame))) + { + // Return the physical page and free the slot + uint writeIndex; + InterlockedAdd(_freeList[0], 1, writeIndex); + _freeList[4 + writeIndex] = GetPagePhysical(entry); + InterlockedAdd(gEvictedCount, 1); + entry = 0; + } + else + { + uint flags = (resident ? SVSM_PAGE_RESIDENT : 0u) | ((invalid && resident) ? SVSM_PAGE_INVALID : 0u); + entry = PackPageEntry(GetPagePhysical(entry), age, flags); + if (resident) + { + InterlockedAdd(gResidentCount, 1); + } + } + + _pageTable[entryIndex] = entry; + } + + GroupMemoryBarrierWithGroupSync(); + + if (input.groupIndex == 0 && clipmapIndex < 8) + { + if (gResidentCount != 0) + { + InterlockedAdd(_svsmData[0].statsResident[clipmapIndex], gResidentCount); + } + if (gInvalidatedCount != 0) + { + InterlockedAdd(_svsmData[0].statsInvalidated[clipmapIndex], gInvalidatedCount); + } + if (gEvictedCount != 0) + { + InterlockedAdd(_svsmData[0].statsEvicted[clipmapIndex], gEvictedCount); + } + } +} diff --git a/Source/Shaders/Shaders/Shadows/SVSMPageUpdateB.cs.slang b/Source/Shaders/Shaders/Shadows/SVSMPageUpdateB.cs.slang new file mode 100644 index 00000000..3228b1d5 --- /dev/null +++ b/Source/Shaders/Shaders/Shadows/SVSMPageUpdateB.cs.slang @@ -0,0 +1,191 @@ +#include "Shadows/SVSM.inc.slang" + +// Allocation, one thread per entry, runs after marking: marked non-resident entries pop a +// physical page from the free list and become RESIDENT|DIRTY, marked invalid entries re-dirty in +// place. Dirty pages accumulate the per-clipmap window-local dirty rect that stage 2 uses to +// tighten the render frusta. +// +// Dispatched once per clipmap (allocClipmap), finest first with a barrier between, so under pool +// pressure the starvation always lands on the coarsest ring where the fallback costs the least. +// +// Lifecycle invariant: DIRTY pages are always rendered the same frame they are set, UpdateA +// clears the bits next frame counting on that. Pool exhaustion leaves the entry non-resident and +// counts an overflow, the sampler falls back to a coarser clipmap. + +[[vk::push_constant]] SVSMConstants _constants; + +[[vk::binding(0, PER_PASS)]] RWStructuredBuffer _svsmData; +[[vk::binding(1, PER_PASS)]] RWStructuredBuffer _pageTable; +[[vk::binding(2, PER_PASS)]] RWStructuredBuffer _freeList; // [0] = count, entries from [4] +[[vk::binding(3, PER_PASS)]] RWStructuredBuffer _clearList; // [0..2] = indirect dispatch args, physical pages from [4] + +groupshared uint gMarkedCount; +groupshared uint gDirtyCount; +groupshared uint gDeferredCount; +groupshared int gRectMinX; +groupshared int gRectMinY; +groupshared int gRectMaxX; +groupshared int gRectMaxY; + +// Clip rects: dirty pages classified as newly-exposed X columns (0), Y rows (1) or other (2), +// each of the three static draws clips its fragments to one of these +groupshared int gClipRectMinX[3]; +groupshared int gClipRectMinY[3]; +groupshared int gClipRectMaxX[3]; +groupshared int gClipRectMaxY[3]; + +struct CSInput +{ + uint3 dispatchThreadID : SV_DispatchThreadID; + uint groupIndex : SV_GroupIndex; +}; + +[shader("compute")] +[numthreads(256, 1, 1)] +void main(CSInput input) +{ + if (input.groupIndex == 0) + { + gMarkedCount = 0; + gDirtyCount = 0; + gDeferredCount = 0; + gRectMinX = 0x7FFFFFFF; + gRectMinY = 0x7FFFFFFF; + gRectMaxX = -1; + gRectMaxY = -1; + } + if (input.groupIndex < 3) + { + gClipRectMinX[input.groupIndex] = 0x7FFFFFFF; + gClipRectMinY[input.groupIndex] = 0x7FFFFFFF; + gClipRectMaxX[input.groupIndex] = -1; + gClipRectMaxY[input.groupIndex] = -1; + } + GroupMemoryBarrierWithGroupSync(); + + uint pageTableSize = _constants.pageTableSize; + uint entriesPerClipmap = pageTableSize * pageTableSize; + uint clipmapIndex = _constants.allocClipmap; + uint entryIndex = clipmapIndex * entriesPerClipmap + input.dispatchThreadID.x; + + uint entry = input.dispatchThreadID.x < entriesPerClipmap ? _pageTable[entryIndex] : 0; + if ((entry & SVSM_PAGE_MARKED) != 0) + { + InterlockedAdd(gMarkedCount, 1); + + // Render budget: pages past the per-frame cap defer (entry untouched: MARKED keeps it + // alive, INVALID persists, no alloc) and retry next frame. The coarsest two rings are + // exempt, they are the whole-scene fallback and only hold a handful of pages. The + // finest-first dispatch order means the near-camera pages win the budget. + // No early return, the group barrier below must stay uniform + bool wantsRender = (entry & SVSM_PAGE_RESIDENT) == 0 || (entry & SVSM_PAGE_INVALID) != 0; + bool deferred = false; + if (wantsRender && _constants.renderBudget != 0 && clipmapIndex + 2 < _constants.numClipmaps) + { + uint budgetIndex; + InterlockedAdd(_svsmData[0].statsBudgetUsed, 1, budgetIndex); + if (budgetIndex >= _constants.renderBudget) + { + InterlockedAdd(gDeferredCount, 1); + deferred = true; + + // Give the claim back so the stat reads pages actually rendered, not attempts. + // Deferral decisions are unaffected, each uses its own pre-add value + InterlockedAdd(_svsmData[0].statsBudgetUsed, uint(-1)); + } + } + + if (!deferred) + { + if ((entry & SVSM_PAGE_RESIDENT) == 0) + { + uint previousCount; + InterlockedAdd(_freeList[0], uint(-1), previousCount); + if (int(previousCount) <= 0) + { + // Pool exhausted, restore and leave the entry non-resident + InterlockedAdd(_freeList[0], 1); + InterlockedAdd(_svsmData[0].statsOverflow, 1); + } + else + { + uint physicalPage = _freeList[4 + previousCount - 1]; + entry = PackPageEntry(physicalPage, 0, SVSM_PAGE_RESIDENT | SVSM_PAGE_MARKED | SVSM_PAGE_DIRTY); + } + } + else if ((entry & SVSM_PAGE_INVALID) != 0) + { + entry = (entry & ~SVSM_PAGE_INVALID) | SVSM_PAGE_DIRTY; + } + + if ((entry & SVSM_PAGE_DIRTY) != 0) + { + InterlockedAdd(gDirtyCount, 1); + + // Queue the physical page for the indirect clear, one workgroup per page + uint clearIndex; + InterlockedAdd(_clearList[0], 1, clearIndex); + _clearList[4 + clearIndex] = GetPagePhysical(entry); + + // Window-local page coords for the dirty rect + uint slotLinear = entryIndex - clipmapIndex * entriesPerClipmap; + int2 slot = int2(slotLinear & (pageTableSize - 1), slotLinear / pageTableSize); + int2 anchor = int2(_svsmData[0].anchorPageMinX[clipmapIndex], _svsmData[0].anchorPageMinY[clipmapIndex]); + int2 localPage = WrapPageSlot(slot - anchor, pageTableSize); + + InterlockedMin(gRectMinX, localPage.x); + InterlockedMin(gRectMinY, localPage.y); + InterlockedMax(gRectMaxX, localPage.x); + InterlockedMax(gRectMaxY, localPage.y); + + // Classify for the clip rects: a toroidal slide dirties an L whose bounding box + // covers most of the window, but its X and Y stripes are thin on their own + int2 absPage = anchor + localPage; + int2 prevAnchor = int2(_svsmData[0].prevAnchorPageMinX[clipmapIndex], _svsmData[0].prevAnchorPageMinY[clipmapIndex]); + bool exposedX = absPage.x < prevAnchor.x || absPage.x >= prevAnchor.x + int(pageTableSize); + bool exposedY = absPage.y < prevAnchor.y || absPage.y >= prevAnchor.y + int(pageTableSize); + uint clipRect = exposedX ? 0 : (exposedY ? 1 : 2); + + InterlockedMin(gClipRectMinX[clipRect], localPage.x); + InterlockedMin(gClipRectMinY[clipRect], localPage.y); + InterlockedMax(gClipRectMaxX[clipRect], localPage.x); + InterlockedMax(gClipRectMaxY[clipRect], localPage.y); + } + + _pageTable[entryIndex] = entry; + } + } + + GroupMemoryBarrierWithGroupSync(); + + if (input.groupIndex == 0 && clipmapIndex < 8) + { + if (gMarkedCount != 0) + { + InterlockedAdd(_svsmData[0].statsMarked[clipmapIndex], gMarkedCount); + } + if (gDirtyCount != 0) + { + InterlockedAdd(_svsmData[0].statsDirty[clipmapIndex], gDirtyCount); + InterlockedMin(_svsmData[0].dirtyRectMinX[clipmapIndex], gRectMinX); + InterlockedMin(_svsmData[0].dirtyRectMinY[clipmapIndex], gRectMinY); + InterlockedMax(_svsmData[0].dirtyRectMaxX[clipmapIndex], gRectMaxX); + InterlockedMax(_svsmData[0].dirtyRectMaxY[clipmapIndex], gRectMaxY); + + for (uint r = 0; r < 3; r++) + { + if (gClipRectMaxX[r] >= 0) + { + InterlockedMin(_svsmData[0].clipRectMinX[r * 8 + clipmapIndex], gClipRectMinX[r]); + InterlockedMin(_svsmData[0].clipRectMinY[r * 8 + clipmapIndex], gClipRectMinY[r]); + InterlockedMax(_svsmData[0].clipRectMaxX[r * 8 + clipmapIndex], gClipRectMaxX[r]); + InterlockedMax(_svsmData[0].clipRectMaxY[r * 8 + clipmapIndex], gClipRectMaxY[r]); + } + } + } + if (gDeferredCount != 0) + { + InterlockedAdd(_svsmData[0].statsDeferred[clipmapIndex], gDeferredCount); + } + } +} diff --git a/Source/Shaders/Shaders/Shadows/SVSMPoolDebug.cs.slang b/Source/Shaders/Shaders/Shadows/SVSMPoolDebug.cs.slang new file mode 100644 index 00000000..78cf1aec --- /dev/null +++ b/Source/Shaders/Shaders/Shadows/SVSMPoolDebug.cs.slang @@ -0,0 +1,49 @@ +#include "Shadows/SVSM.inc.slang" + +// Draws a downsampled view of the physical page pool straight into the scene color corner: +// grayscale reversed-Z depth (bright = close to the sun), untouched texels stay black. + +struct DebugConstants +{ + uint poolSize; // Pool texture resolution + uint regionSize; // On-screen size of the debug view + uint padding0; + uint padding1; + int2 screenOffset; +}; + +[[vk::push_constant]] DebugConstants _constants; + +[[vk::binding(0, PER_PASS)]] Texture2D _pagePool; +[[vk::binding(1, PER_PASS)]] RWTexture2D _target; + +struct CSInput +{ + uint3 dispatchThreadID : SV_DispatchThreadID; +}; + +[shader("compute")] +[numthreads(16, 16, 1)] +void main(CSInput input) +{ + uint regionSize = _constants.regionSize; + if (any(input.dispatchThreadID.xy >= uint2(regionSize, regionSize))) + { + return; + } + + uint2 targetDim; + _target.GetDimensions(targetDim.x, targetDim.y); + int2 targetPixel = _constants.screenOffset + int2(input.dispatchThreadID.xy); + if (any(targetPixel < int2(0, 0)) || any(targetPixel >= int2(targetDim))) + { + return; + } + + uint2 poolTexel = input.dispatchThreadID.xy * (_constants.poolSize / regionSize); + float depth = asfloat(_pagePool[poolTexel]); + + // sqrt spreads the reversed-Z range so distant depth stays visible + float value = sqrt(saturate(depth)); + _target[uint2(targetPixel)] = float4(value, value, value, 1.0f); +} diff --git a/Source/Shaders/Shaders/Shadows/SVSMPrepare.cs.slang b/Source/Shaders/Shaders/Shadows/SVSMPrepare.cs.slang new file mode 100644 index 00000000..1a0acb67 --- /dev/null +++ b/Source/Shaders/Shaders/Shadows/SVSMPrepare.cs.slang @@ -0,0 +1,139 @@ +#include "Include/Camera.inc.slang" +#include "Shadows/SVSM.inc.slang" + +// First SVSM dispatch, single thread: detects invalidation events, snaps the per-clipmap window +// anchors to the absolute page grid and resets the per-frame stats. +// +// The anchors are the precision foundation for every later pass: anchorPageMin is exact integer +// math, and camMinusWindowMin is the only place the world-magnitude cancellation happens, so all +// consumers share the same (sub-texel) absolute placement error instead of diverging. + +[[vk::push_constant]] SVSMConstants _constants; + +[[vk::binding(0, PER_PASS)]] RWStructuredBuffer _svsmData; +[[vk::binding(1, PER_PASS)]] StructuredBuffer _srcCameras; // Element 0 = main camera +[[vk::binding(2, PER_PASS)]] StructuredBuffer _freeList; // [0] = count, entries from [4] +[[vk::binding(3, PER_PASS)]] RWStructuredBuffer _clearList; // [0..2] = indirect dispatch args, physical pages from [4] +[[vk::binding(4, PER_PASS)]] RWStructuredBuffer _dynamicClearList; // Same layout, for the caster-split dynamic pool + +[shader("compute")] +[numthreads(1, 1, 1)] +void main() +{ + // Start-of-frame snapshot, the free list only changes in UpdateA/UpdateB later in the frame + _svsmData[0].statsFreeListCount = _freeList[0]; + + // Reset the dirty page clear lists, UpdateB / DynamicUpdate append and bump the indirect group counts + _clearList[0] = 0; + _clearList[1] = 1; + _clearList[2] = 1; + _dynamicClearList[0] = 0; + _dynamicClearList[1] = 1; + _dynamicClearList[2] = 1; + + uint cause = _constants.invalidateAll; + + // The quantized shadow sun stepped, every page's light basis changed + float3 lightDirection = _constants.lightDirection.xyz; + float4 prevLightDirection = _svsmData[0].prevLightDirection; + if (prevLightDirection.w > 0.5f && dot(prevLightDirection.xyz, lightDirection) < 0.999999f) + { + cause |= SVSM_CAUSE_SUN_STEP; + } + + float3x3 lightRotation = BuildLightRotation(lightDirection); + float3 cameraLightSpace = mul(_srcCameras[0].eyePosition.xyz, lightRotation); + + // One global Z window for all clipmaps, anchored on the camera in coarse steps: stored page + // depths change meaning when it moves, so it must move rarely and invalidate when it does + const float zAnchorStep = 512.0f; + float zAnchor = floor(cameraLightSpace.z / zAnchorStep + 0.5f) * zAnchorStep; + float zRangeMin = zAnchor - _constants.zHalfRange; + float zRangeMax = zAnchor + _constants.zHalfRange; + if (prevLightDirection.w > 0.5f && (_svsmData[0].zRangeMin != zRangeMin || _svsmData[0].zRangeMax != zRangeMax)) + { + cause |= SVSM_CAUSE_Z_RANGE; + } + _svsmData[0].zRangeMin = zRangeMin; + _svsmData[0].zRangeMax = zRangeMax; + _svsmData[0].prevLightDirection = float4(lightDirection, 1.0f); + + _svsmData[0].frameInvalidationFlags = cause; + _svsmData[0].statsInvalidationCause = cause; + _svsmData[0].statsOverflow = 0; + + uint pageTableSize = _constants.pageTableSize; + + // Config mirror for the draw-time shaders, they only get viewIndex as a push constant + _svsmData[0].configPageTableSize = pageTableSize; + _svsmData[0].configPageSize = _constants.pageSize; + _svsmData[0].configPoolPagesPerRow = _constants.poolPagesPerRow; + _svsmData[0].configNumClipmaps = _constants.numClipmaps; + _svsmData[0].configMarkBorderTexels = _constants.markBorderTexels; + _svsmData[0].configCamMinusZAnchor = cameraLightSpace.z - zAnchor; + _svsmData[0].configResolutionScale = _constants.resolutionScale; + + _svsmData[0].statsDynamicOverflow = 0; + _svsmData[0].statsDynamicTotal = 0; + _svsmData[0].configDynamicPoolPagesPerRow = _constants.dynamicPoolPagesPerRow; + _svsmData[0].statsBudgetUsed = 0; + + for (uint k = 0; k < 8; k++) + { + // Per-frame stats and dirty rect sentinels + _svsmData[0].statsMarked[k] = 0; + _svsmData[0].statsResident[k] = 0; + _svsmData[0].statsDirty[k] = 0; + _svsmData[0].statsEvicted[k] = 0; + _svsmData[0].statsInvalidated[k] = 0; + _svsmData[0].statsDynamicLive[k] = 0; + _svsmData[0].statsDeferred[k] = 0; + _svsmData[0].dirtyRectMinX[k] = int(pageTableSize); + _svsmData[0].dirtyRectMinY[k] = int(pageTableSize); + _svsmData[0].dirtyRectMaxX[k] = -1; + _svsmData[0].dirtyRectMaxY[k] = -1; + _svsmData[0].dynamicRectMinX[k] = int(pageTableSize); + _svsmData[0].dynamicRectMinY[k] = int(pageTableSize); + _svsmData[0].dynamicRectMaxX[k] = -1; + _svsmData[0].dynamicRectMaxY[k] = -1; + + for (uint r = 0; r < 3; r++) + { + _svsmData[0].clipRectMinX[r * 8 + k] = int(pageTableSize); + _svsmData[0].clipRectMinY[r * 8 + k] = int(pageTableSize); + _svsmData[0].clipRectMaxX[r * 8 + k] = -1; + _svsmData[0].clipRectMaxY[r * 8 + k] = -1; + } + + if (k >= _constants.numClipmaps) + { + _svsmData[0].clipmapInvalidate[k] = 0; + continue; + } + + float extent = _constants.clipmap0Extent * exp2(float(k)); + float pageWorld = extent / float(pageTableSize); + + // A window geometry change makes the whole clipmap's content unaddressable + _svsmData[0].clipmapInvalidate[k] = (_svsmData[0].extent[k] != extent || _svsmData[0].pageWorld[k] != pageWorld) ? 1u : 0u; + + // Window min corner snapped to the absolute page grid, camera sits pageTableSize/2 pages in. + // floor can flip by one page when the camera rests exactly on a boundary, that only costs a + // toroidal row invalidation, never correctness + int2 anchorPageMin = int2(floor(cameraLightSpace.xy / pageWorld)) - int(pageTableSize / 2); + + _svsmData[0].prevAnchorPageMinX[k] = _svsmData[0].anchorPageMinX[k]; + _svsmData[0].prevAnchorPageMinY[k] = _svsmData[0].anchorPageMinY[k]; + _svsmData[0].anchorPageMinX[k] = anchorPageMin.x; + _svsmData[0].anchorPageMinY[k] = anchorPageMin.y; + + // The one world-magnitude subtraction, its rounding shifts the whole clipmap by a fraction + // of a texel and is shared by every consumer through this stored value + float2 windowMin = float2(anchorPageMin) * pageWorld; + _svsmData[0].camMinusWindowMinX[k] = cameraLightSpace.x - windowMin.x; + _svsmData[0].camMinusWindowMinY[k] = cameraLightSpace.y - windowMin.y; + + _svsmData[0].extent[k] = extent; + _svsmData[0].pageWorld[k] = pageWorld; + } +} diff --git a/Source/Shaders/Shaders/Skybox/Skybox.ps.slang b/Source/Shaders/Shaders/Skybox/Skybox.ps.slang index 0b8a9325..e54d0ac9 100644 --- a/Source/Shaders/Shaders/Skybox/Skybox.ps.slang +++ b/Source/Shaders/Shaders/Skybox/Skybox.ps.slang @@ -10,6 +10,7 @@ struct SkybandColors float4 bottom; float4 aboveHorizon; float4 horizon; + float4 sunDirection; // xyz = direction to the sun, w = enabled }; [[vk::push_constant]] SkybandColors _skybandColors; @@ -62,6 +63,32 @@ PSOutput main(VertexOutput input) : SV_Target color = lerp(_skybandColors.middle, _skybandColors.top, blendFactor); } + // Sun disc where the view ray aligns with the directional light + if (_skybandColors.sunDirection.w > 0.0f) + { + // Same NDC convention as ScreenToView in Culling.inc.slang + float2 ndc = float2(input.uv.x, 1.0f - input.uv.y) * 2.0f - 1.0f; + + // Build the ray in view space (camera at origin) and rotate it to world as a direction, + // going through clipToWorld instead subtracts two huge world positions and the + // cancellation noise makes the disc shimmer and facet + float4 viewPos = mul(float4(ndc, 1.0f, 1.0f), _cameras[0].clipToView); // Reversed Z, depth 1 is the near plane + float3 viewRay = normalize(viewPos.xyz / viewPos.w); + float3 viewDir = normalize(mul(float4(viewRay, 0.0f), _cameras[0].viewToWorld).xyz); + + float cosSun = dot(viewDir, _skybandColors.sunDirection.xyz); + + const float discEdge = 0.9993f; // ~2.1 degree angular radius + const float glowEdge = 0.997f; + + float disc = smoothstep(discEdge, discEdge + 0.0003f, cosSun); + float glow = saturate((cosSun - glowEdge) / (1.0f - glowEdge)); + glow = glow * glow * 0.3f; + + const float3 sunColor = float3(1.0f, 0.95f, 0.8f); + color.rgb = lerp(color.rgb, sunColor, saturate(disc + glow)); + } + PSOutput output; output.color = float4(color.rgb, 1.0f); return output; diff --git a/Source/Shaders/Shaders/Terrain/Culling.cs.slang b/Source/Shaders/Shaders/Terrain/Culling.cs.slang index ed902701..b06c2d18 100644 --- a/Source/Shaders/Shaders/Terrain/Culling.cs.slang +++ b/Source/Shaders/Shaders/Terrain/Culling.cs.slang @@ -11,10 +11,12 @@ struct Constants { uint viewportSizeX; uint viewportSizeY; - uint numCascades; + uint numShadowViews; uint occlusionCull; uint bitMaskBufferSizePerView; uint currentBitmaskIndex; + uint debugDrawView; // viewIndex of the view to debug draw culling results for, 0xFFFFFFFF = off + uint cullMainView; // The late clipmap dispatch must not re-cull view 0, it would overwrite the occlusion-culled bitmask slice }; struct HeightRange @@ -100,7 +102,8 @@ struct BitmaskInput void CullForCamera(DrawInput drawInput, Camera camera, - BitmaskInput bitmaskInput) + BitmaskInput bitmaskInput, + bool debugDrawResults) { bool isVisible = true; @@ -131,19 +134,17 @@ void CullForCamera(DrawInput drawInput, } } - // Debug draw AABB boxes - /*float3 cameraPos = _cameras[0].eyePosition.xyz; - float3 aabbPos = (drawInput.aabb.min + drawInput.aabb.max) / 2.0f; - float distanceToCamera = distance(cameraPos, aabbPos); - - if (isVisible) + if (debugDrawResults) { - DrawAABB3D(drawInput.aabb, DebugColor::GREEN); + if (isVisible) + { + DrawAABB3D(drawInput.aabb, DebugColor::GREEN); + } + else + { + DrawAABB3D(drawInput.aabb, DebugColor::RED); + } } - else - { - DrawAABB3D(drawInput.aabb, DebugColor::RED); - }*/ } [shader("compute")] @@ -176,19 +177,21 @@ void main(CSInput input) bitmaskInput.bitmaskOffset = 0; // Main camera view + if (_constants.cullMainView) { - CullForCamera(drawInput, _cameras[0], bitmaskInput); + CullForCamera(drawInput, _cameras[0], bitmaskInput, _constants.debugDrawView == 0); } - // Shadow cascades - for (uint i = 1; i < _constants.numCascades + 1; i++) + // Shadow clipmap views + for (uint i = 1; i < _constants.numShadowViews + 1; i++) { - drawInput.shouldOcclusionCull = false; // No occlusion culling for shadow cascades... yet? + drawInput.shouldOcclusionCull = false; // No occlusion culling for shadow views... yet? bitmaskInput.bitmaskOffset = _constants.bitMaskBufferSizePerView * i; CullForCamera(drawInput, _cameras[i], - bitmaskInput); + bitmaskInput, + _constants.debugDrawView == i); } } \ No newline at end of file diff --git a/Source/Shaders/Shaders/Terrain/Draw.vs.slang b/Source/Shaders/Shaders/Terrain/Draw.vs.slang index 10f506f7..ff1978d1 100644 --- a/Source/Shaders/Shaders/Terrain/Draw.vs.slang +++ b/Source/Shaders/Shaders/Terrain/Draw.vs.slang @@ -1,18 +1,28 @@ permutation EDITOR_PASS = [0, 1]; -permutation SHADOW_PASS = [0, 1]; +permutation SVSM_PASS = [0, 1]; // 1 adds per-draw clip-rect distances for the SVSM page render #define GEOMETRY_PASS 1 #include "DescriptorSet/Global.inc.slang" #include "Terrain/TerrainShared.inc.slang" +#if SVSM_PASS +#include "Shadows/SVSM.inc.slang" +#endif struct Constants { uint viewIndex; +#if SVSM_PASS + uint svsmRectIndex; // Which clip rect this draw renders, SVSM_CLIP_RECT_DISABLED = no clipping +#endif }; [[vk::push_constant]] Constants _constants; +#if SVSM_PASS +[[vk::binding(0, PER_PASS)]] StructuredBuffer _svsmData; // Same binding the SVSM pixel shader declares +#endif + struct VSInput { uint vertexID : SV_VulkanVertexID; @@ -22,9 +32,12 @@ struct VSInput struct VSOutput { float4 position : SV_Position; -#if !EDITOR_PASS && !SHADOW_PASS +#if !EDITOR_PASS && !SVSM_PASS uint instanceID : TEXCOORD0; #endif +#if SVSM_PASS + float4 clipDistances : SV_ClipDistance; // Fixed-function culls fragments outside the draw's clip rect +#endif }; [shader("vertex")] @@ -39,6 +52,9 @@ VSOutput main(VSInput input) { const float NaN = asfloat(0b01111111100000000000000000000000); output.position = float4(NaN, NaN, NaN, NaN); +#if SVSM_PASS + output.clipDistances = float4(-1.0f, -1.0f, -1.0f, -1.0f); +#endif return output; } @@ -50,9 +66,12 @@ VSOutput main(VSInput input) output.position = mul(float4(vertex.position, 1.0f), _cameras[_constants.viewIndex].worldToClip); -#if !EDITOR_PASS && !SHADOW_PASS +#if !EDITOR_PASS && !SVSM_PASS output.instanceID = instanceData.globalCellID; #endif +#if SVSM_PASS + output.clipDistances = SVSMRectClipDistances(vertex.position, _constants.viewIndex - 1, _constants.svsmRectIndex, _cameras[0].eyePosition.xyz, _svsmData); +#endif return output; -} \ No newline at end of file +} diff --git a/Source/Shaders/Shaders/Terrain/DrawSVSM.ps.slang b/Source/Shaders/Shaders/Terrain/DrawSVSM.ps.slang new file mode 100644 index 00000000..4b19f223 --- /dev/null +++ b/Source/Shaders/Shaders/Terrain/DrawSVSM.ps.slang @@ -0,0 +1,32 @@ +#define GEOMETRY_PASS 1 + +#include "DescriptorSet/Global.inc.slang" + +#include "Shadows/SVSM.inc.slang" + +// SVSM page render: no attachments, the fragment writes reversed-Z depth into the physical page +// pool with image atomics. Paired with the SVSM_PASS=1 vertex permutation, viewport spans the +// full virtual texture so SV_Position.xy is the window-local virtual texel. + +struct Constants +{ + uint viewIndex; // 0 = main camera, clipmap k renders as view k + 1 + uint svsmRectIndex; // Consumed by the vertex shader's clip rect, declared here so the push block matches +}; + +[[vk::push_constant]] Constants _constants; + +[[vk::binding(0, PER_PASS)]] StructuredBuffer _svsmData; +[[vk::binding(1, PER_PASS)]] StructuredBuffer _pageTable; +[[vk::binding(2, PER_PASS)]] RWTexture2D _pagePool; + +struct PSInput +{ + float4 position : SV_Position; +}; + +[shader("fragment")] +void main(PSInput input) +{ + SVSMWritePageDepth(input.position.xyz, _constants.viewIndex - 1, SVSM_PAGE_RESIDENT | SVSM_PAGE_DIRTY, _svsmData[0].configPoolPagesPerRow, _svsmData, _pageTable, _pagePool); +} diff --git a/Source/Shaders/Shaders/Utils/CreateIndirectAfterCulling.cs.slang b/Source/Shaders/Shaders/Utils/CreateIndirectAfterCulling.cs.slang index a16d1a13..cad5966a 100644 --- a/Source/Shaders/Shaders/Utils/CreateIndirectAfterCulling.cs.slang +++ b/Source/Shaders/Shaders/Utils/CreateIndirectAfterCulling.cs.slang @@ -33,7 +33,7 @@ typedef Draw DrawType; [[vk::push_constant]] Constants _constants; [[vk::binding(0, PER_PASS)]] StructuredBuffer _drawCalls; [[vk::binding(1, PER_PASS)]] ByteAddressBuffer _drawCallDatas; -[[vk::binding(2, PER_PASS)]] StructuredBuffer _culledInstanceCounts; // One uint per draw call +[[vk::binding(2, PER_PASS)]] RWStructuredBuffer _culledInstanceCounts; // One uint per draw call, cleared after consumption [[vk::binding(3, PER_PASS)]] RWStructuredBuffer _culledDrawCalls; [[vk::binding(4, PER_PASS)]] RWByteAddressBuffer _culledDrawCallCount; @@ -75,7 +75,9 @@ void main(CSInput input) DrawType drawCall = _drawCalls[drawCallID]; - // Load instanceCount and baseInstanceLookup + // Load instanceCount and baseInstanceLookup. Clearing after consumption keeps the buffer + // always-zero between uses, so the per-fill FillBuffer reset can be skipped + // (svsmGatedFillSetup) — a fill only ever adds onto zeros uint instanceCount = _culledInstanceCounts[drawCallID]; if (instanceCount == 0) { @@ -85,6 +87,7 @@ void main(CSInput input) return; #endif } + _culledInstanceCounts[drawCallID] = 0; uint baseInstanceLookup = GetBaseInstanceLookup(drawCallID); diff --git a/Source/Shaders/Shaders/Utils/Culling.cs.slang b/Source/Shaders/Shaders/Utils/Culling.cs.slang index c74009e7..c94e69ee 100644 --- a/Source/Shaders/Shaders/Utils/Culling.cs.slang +++ b/Source/Shaders/Shaders/Utils/Culling.cs.slang @@ -12,7 +12,7 @@ struct Constants uint viewportSizeX; uint viewportSizeY; uint maxDrawCount; - uint numCascades; + uint numShadowViews; uint occlusionCull; uint instanceIDOffset; uint modelIDOffset; @@ -45,7 +45,7 @@ struct CullingData [[vk::binding(2, PER_PASS)]] StructuredBuffer _cullingDatas; [[vk::binding(3, PER_PASS)]] StructuredBuffer _instanceMatrices; [[vk::binding(4, PER_PASS)]] SamplerState _depthSampler; -[[vk::binding(5, PER_PASS)]] Texture2D _depthPyramid; // TODO: Occlusion culling for shadow cascades? +[[vk::binding(5, PER_PASS)]] Texture2D _depthPyramid; // TODO: Occlusion culling for shadow views? #if USE_BITMASKS // Both bitmask generations stay bound; currentBitmaskIndex selects which one is written this frame @@ -322,10 +322,10 @@ void main(CSInput input) // Shadow is only supported if we USE_BITMASKS #if USE_BITMASKS - // Shadow cascades - for (uint i = 1; i < _constants.numCascades + 1; i++) + // Shadow clipmap views + for (uint i = 1; i < _constants.numShadowViews + 1; i++) { - drawInput.shouldOcclusionCull = false; // No occlusion culling for shadow cascades... yet? + drawInput.shouldOcclusionCull = false; // No occlusion culling for shadow views... yet? bitmaskInput.bitmaskOffset = _constants.bitMaskBufferUintsPerView * i; diff --git a/Source/Shaders/Shaders/Utils/CullingInstanced.cs.slang b/Source/Shaders/Shaders/Utils/CullingInstanced.cs.slang index 1bb72dc2..cf02eee9 100644 --- a/Source/Shaders/Shaders/Utils/CullingInstanced.cs.slang +++ b/Source/Shaders/Shaders/Utils/CullingInstanced.cs.slang @@ -21,6 +21,10 @@ struct Constants uint cullingDataIsWorldspace; // TODO: This controls two things, are both needed? I feel like one counters the other but I'm not sure... uint debugDrawColliders; uint currentBitmaskIndex; + uint numShadowViews; + uint bitMaskBufferUintsPerView; + uint debugDrawView; // viewIndex of the view to debug draw culling results for, 0xFFFFFFFF = off + uint cullMainView; // The late clipmap dispatch must not re-cull view 0, it would overwrite the occlusion-culled bitmask slice and the instance lookup }; struct InstanceRef @@ -53,7 +57,7 @@ struct CullingData [[vk::binding(3, PER_PASS)]] StructuredBuffer _cullingDatas; [[vk::binding(4, PER_PASS)]] StructuredBuffer _instanceMatrices; [[vk::binding(5, PER_PASS)]] SamplerState _depthSampler; -[[vk::binding(6, PER_PASS)]] Texture2D _depthPyramid; // TODO: Occlusion culling for shadow cascades? +[[vk::binding(6, PER_PASS)]] Texture2D _depthPyramid; // TODO: Occlusion culling for shadow views? #if USE_BITMASKS // TODO: When instanceCounts resizes we need to invalidate (zero out) the prev bitmask since a single index in current and prev would point to different actual drawcalls @@ -193,6 +197,34 @@ struct CullOutput RWStructuredBuffer instanceLookupTable; }; +#if USE_BITMASKS +// Shadow clipmap views only need the visibility bitmask, the per-view draw sets are filled from it later in the geometry pass +void CullForClipmap(DrawInput drawInput, Camera camera, uint bitmaskOffset, bool debugDrawResults) +{ + bool isVisible = drawInput.instanceCount > 0 && IsSphereInsideFrustum(camera, drawInput.sphere); + + uint bitMask = WaveActiveBallot(isVisible).x; + + // The first thread writes the bitmask + if (drawInput.csInput.groupThreadID.x == 0) + { + WriteCurrentBitmask(drawInput.csInput.groupID.x + bitmaskOffset, bitMask); + } + + if (debugDrawResults) + { + if (isVisible) + { + DrawAABB3D(drawInput.aabb, DebugColor::GREEN); + } + else + { + DrawAABB3D(drawInput.aabb, DebugColor::RED); + } + } +} +#endif + void CullForCamera(DrawInput drawInput, Camera camera, #if USE_BITMASKS @@ -375,6 +407,7 @@ void main(CSInput input) cullOutput.instanceLookupTable = _culledInstanceLookupTable; // Cull Main Camera + if (_constants.cullMainView) { CullForCamera(drawInput, _cameras[0], @@ -383,4 +416,13 @@ void main(CSInput input) #endif cullOutput); } + +#if USE_BITMASKS + // Shadow clipmap views + for (uint i = 1; i < _constants.numShadowViews + 1; i++) + { + bool debugDrawResults = (_constants.debugDrawView == i); + CullForClipmap(drawInput, _cameras[i], _constants.bitMaskBufferUintsPerView * i, debugDrawResults); + } +#endif } \ No newline at end of file diff --git a/Source/Shaders/Shaders/Utils/FillInstancedDrawCallsFromBitmask.cs.slang b/Source/Shaders/Shaders/Utils/FillInstancedDrawCallsFromBitmask.cs.slang index 5dffa65e..3ce897f2 100644 --- a/Source/Shaders/Shaders/Utils/FillInstancedDrawCallsFromBitmask.cs.slang +++ b/Source/Shaders/Shaders/Utils/FillInstancedDrawCallsFromBitmask.cs.slang @@ -1,4 +1,5 @@ permutation IS_INDEXED = [0, 1]; +permutation DYNAMIC_FILTER = [0, 1]; // 1 adds the per-instance dynamic mask filter for the SVSM caster split #include "DescriptorSet/Global.inc.slang" @@ -12,6 +13,8 @@ struct Constants uint baseInstanceLookupOffset; // Byte offset into drawCallDatas where the baseInstanceLookup is stored uint drawCallDataSize; uint currentBitmaskIndex; + uint bitmaskOffset; // Selects which view's bitmask slice to fill from + uint keepDynamic; // DYNAMIC_FILTER only: 0 = keep static instances, 1 = keep dynamic instances }; struct InstanceRef @@ -39,6 +42,10 @@ typedef Draw DrawType; [[vk::binding(3, PER_PASS)]] RWByteAddressBuffer _culledInstanceCounts; // One uint per draw call [[vk::binding(4, PER_PASS)]] RWStructuredBuffer _culledInstanceLookupTable; // One uint per instance, contains instanceRefID of what survives culling, and thus can get reordered +#if DYNAMIC_FILTER +[[vk::binding(6, PER_PASS)]] StructuredBuffer _dynamicInstanceMask; // One bit per instanceID, rebuilt every frame +#endif + struct CSInput { uint3 dispatchThreadId : SV_DispatchThreadID; @@ -64,11 +71,11 @@ void main(CSInput input) uint bitMask; if (_constants.currentBitmaskIndex == 0) { - bitMask = _culledDrawCallsBitMask0[input.groupID.x]; + bitMask = _culledDrawCallsBitMask0[input.groupID.x + _constants.bitmaskOffset]; } else { - bitMask = _culledDrawCallsBitMask1[input.groupID.x]; + bitMask = _culledDrawCallsBitMask1[input.groupID.x + _constants.bitmaskOffset]; } uint bitIndex = input.groupThreadID.x; @@ -76,6 +83,13 @@ void main(CSInput input) { InstanceRef instanceRef = _instanceRefTable[instanceRefID]; +#if DYNAMIC_FILTER + // SVSM caster split: keep only the requested class + bool isDynamic = (_dynamicInstanceMask[instanceRef.instanceID / 32] & (1u << (instanceRef.instanceID & 31))) != 0; + if (isDynamic != (_constants.keepDynamic != 0)) + return; +#endif + uint countByteOffset = instanceRef.drawID * sizeof(uint); uint instanceIndex; diff --git a/Submodules/Engine b/Submodules/Engine index d05d0aa7..e84f13ed 160000 --- a/Submodules/Engine +++ b/Submodules/Engine @@ -1 +1 @@ -Subproject commit d05d0aa7cb5e544c17093396206d7478692c61af +Subproject commit e84f13ed4a77ebe2c9290d14287db5f33ecc6457