From e57224300c3b501c3304b02b1e6ebf8eb0378bea Mon Sep 17 00:00:00 2001 From: Andrew Jong Date: Thu, 13 Aug 2026 19:13:22 -0700 Subject: [PATCH] unreal: add lumen-gi-enabled / lumen-reflections-enabled capture settings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit UE scene captures don't render with the global illumination / reflection method the viewport uses: a capture view seeds its method from the project CVar but never blends level PostProcessVolumes, and on hardware-ray-tracing projects its rays have no acceleration-structure data unless the component opts in — so a Lumen-lit level comes out flat and washed out in captured images, with all indirect light missing. Since UE 5.5 the engine supports Lumen in scene captures when the capture's post-process settings resolve to Lumen (the renderer then maintains a dedicated Lumen scene per capture's persistent view state, which UnrealCamera already enables via bAlwaysPersistRenderingState). Add two per-capture tri-state settings, keyed off whether the jsonc key is present: - absent (default): match the viewport. Scene (RGB) captures mirror the project's r.DynamicGlobalIlluminationMethod / r.ReflectionMethod, so Lumen projects get Lumen captures out of the box; every other image type resolves to None — their output is a replacement material, so dynamic GI underneath is pure GPU/VRAM waste (and previously they silently inherited the project CVar). - lumen-gi-enabled / lumen-reflections-enabled = true: force Lumen for this capture regardless of the project default. - false: force the method to None. This is an active override — an un-overridden capture inherits the project CVar, so omitting the override is not a way to turn Lumen off. The resolved methods are pinned as explicit post-process overrides, and bUseRayTracingIfEnabled is set whenever either method resolves to Lumen so hardware-ray-traced Lumen works when the project uses it (software Lumen works without). On engines older than 5.5 the overrides are ignored for captures. Co-Authored-By: Claude Fable 5 --- core_sim/include/core_sim/sensors/camera.hpp | 17 ++++ core_sim/src/constant.hpp | 3 + core_sim/src/sensors/camera.cpp | 16 ++++ docs/sensors/camera_capture_settings.md | 2 + .../Private/Sensors/UnrealCamera.cpp | 80 +++++++++++++++++++ 5 files changed, 118 insertions(+) diff --git a/core_sim/include/core_sim/sensors/camera.hpp b/core_sim/include/core_sim/sensors/camera.hpp index 4d660f0c4..8839b6e56 100644 --- a/core_sim/include/core_sim/sensors/camera.hpp +++ b/core_sim/include/core_sim/sensors/camera.hpp @@ -101,6 +101,23 @@ struct CaptureSettings { float auto_exposure_histogram_log_max = std::numeric_limits::quiet_NaN(); // 4; float motion_blur_amount = std::numeric_limits::quiet_NaN(); + // Lumen dynamic GI / reflections for this capture's scene render (UE 5.5+ + // supports Lumen in scene captures). Tri-state, kLumenAuto when the key is + // absent from the config: + // auto: Scene (RGB) captures mirror the project's methods + // (r.DynamicGlobalIlluminationMethod / r.ReflectionMethod) so the + // captured image matches the viewport; every other image type + // resolves to None (their output is a replacement material — GI + // underneath is pure GPU/VRAM waste). + // 1: force Lumen for this capture. + // 0: force the method to None. This must be an active override — an + // un-overridden capture inherits the project CVar, so "no override" + // does not mean "no Lumen". + // Each Lumen-enabled capture maintains its own Lumen scene (GPU time + + // VRAM cost). + static constexpr int kLumenAuto = -1; + int lumen_gi_enabled = kLumenAuto; + int lumen_reflections_enabled = kLumenAuto; float target_gamma = 2.5f; // This would be reset to kSceneTargetGamma for scene as default int projection_mode = 0; // ECameraProjectionMode::Perspective diff --git a/core_sim/src/constant.hpp b/core_sim/src/constant.hpp index aa51e31ac..649ff65b0 100644 --- a/core_sim/src/constant.hpp +++ b/core_sim/src/constant.hpp @@ -202,6 +202,9 @@ class Constant { static constexpr const char* auto_exposure_histogram_log_max = "auto-exposure-histogram-log-max"; static constexpr const char* motion_blur_amount = "motion-blur-amount"; + static constexpr const char* lumen_gi_enabled = "lumen-gi-enabled"; + static constexpr const char* lumen_reflections_enabled = + "lumen-reflections-enabled"; static constexpr const char* target_gamma = "target-gamma"; static constexpr const char* max_depth_meters = "max-depth-meters"; static constexpr const char* chromatic_aberration_intensity = "chromatic-aberration-intensity"; diff --git a/core_sim/src/sensors/camera.cpp b/core_sim/src/sensors/camera.cpp index 1b6140e81..eca4bf6fe 100644 --- a/core_sim/src/sensors/camera.cpp +++ b/core_sim/src/sensors/camera.cpp @@ -1345,6 +1345,22 @@ void Camera::Loader::LoadCaptureSetting(const json& json) { setting.motion_blur_amount = JsonUtils::GetNumber( json, Constant::Config::motion_blur_amount, setting.motion_blur_amount); + // Tri-state: absent key = kLumenAuto (match the project/viewport method), + // explicit true/false forces the method on/off for this capture. + if (JsonUtils::HasKey(json, Constant::Config::lumen_gi_enabled)) { + setting.lumen_gi_enabled = + JsonUtils::GetBoolean(json, Constant::Config::lumen_gi_enabled, false) + ? 1 + : 0; + } + if (JsonUtils::HasKey(json, Constant::Config::lumen_reflections_enabled)) { + setting.lumen_reflections_enabled = + JsonUtils::GetBoolean(json, Constant::Config::lumen_reflections_enabled, + false) + ? 1 + : 0; + } + setting.target_gamma = JsonUtils::GetNumber( json, Constant::Config::target_gamma, setting.target_gamma); diff --git a/docs/sensors/camera_capture_settings.md b/docs/sensors/camera_capture_settings.md index 0ed736ee7..46f86fa3d 100644 --- a/docs/sensors/camera_capture_settings.md +++ b/docs/sensors/camera_capture_settings.md @@ -96,6 +96,8 @@ A single camera can output multiple capture types. The following parameters desc | auto-exposure-histogram-log-min | | Defines the lower bounds for the brightness range of the generated histogram when using the HDR (Eye Adaptation) visualization mode;log-min value for histogram of scene colors; E.g.:0.8 ==> 80% of the screen pixels are darker than the luminance value A | | auto-exposure-histogram-log-max | | log-max value for histogram of scene colors; E.g.:0.95 ==> 95% of the screen pixels are darker than the luminance value B. Current luminance value (C) = avg(A, B) | | motion-blur-amount | [0, 1] | Blurs objects based on it's motion determined using velocity maps. A value of 0.25 - 0.5 looks reasonable. Avoid 1.0 | +| lumen-gi-enabled | true/false (default: match viewport) | Controls Lumen dynamic global illumination for this capture. **When omitted**, RGB Scene captures mirror the project's global illumination method (`r.DynamicGlobalIlluminationMethod`) so the captured image matches the viewport — i.e. Lumen projects get Lumen captures by default — while non-Scene image types (depth, segmentation, ...) run with no dynamic GI, since their output is a replacement material and GI underneath only costs GPU/VRAM. Set `false` to force a capture back to no dynamic GI, or `true` to force Lumen regardless of the project default. Requires UE 5.5+ (older engines ignore this for captures); each Lumen-enabled capture maintains its own Lumen scene (GPU time + VRAM cost). | +| lumen-reflections-enabled | true/false (default: match viewport) | Same tri-state semantics as `lumen-gi-enabled`, but for the reflection method (`r.ReflectionMethod`). | | target-gamma | | Target gamma to be used by the render target (UTextureRenderTarget2D) | | max-depth-meters | > 0.0 | Maximum depth for Depth Visualization. Objects beyond the maximum depth will be true white, while close objects will be completely black. | diff --git a/unreal/Blocks/Plugins/ProjectAirSim/Source/ProjectAirSim/Private/Sensors/UnrealCamera.cpp b/unreal/Blocks/Plugins/ProjectAirSim/Source/ProjectAirSim/Private/Sensors/UnrealCamera.cpp index ff4cb5b4d..816d0bca6 100644 --- a/unreal/Blocks/Plugins/ProjectAirSim/Source/ProjectAirSim/Private/Sensors/UnrealCamera.cpp +++ b/unreal/Blocks/Plugins/ProjectAirSim/Source/ProjectAirSim/Private/Sensors/UnrealCamera.cpp @@ -18,6 +18,7 @@ #include "Engine/TextureRenderTarget2D.h" #include "Engine/World.h" #include "GameFramework/PlayerController.h" +#include "HAL/IConsoleManager.h" #include "ImagePackingAsyncTask.h" #include "ImageUtils.h" #include "Materials/MaterialInstanceDynamic.h" @@ -168,6 +169,58 @@ void HideDebugDrawComponent(USceneCaptureComponent2D* CaptureComponent, #endif } +// Resolves the tri-state lumen-gi-enabled / lumen-reflections-enabled +// capture settings into the concrete methods to pin on this capture's +// post-process overrides: +// auto (key absent): Scene (RGB) captures mirror the project's methods +// (r.DynamicGlobalIlluminationMethod / r.ReflectionMethod) so the +// captured image matches what the viewport renders; every other image +// type resolves to None — their output comes from a replacement +// material, so dynamic GI underneath is pure GPU/VRAM waste (each +// Lumen-enabled capture maintains its own Lumen scene). +// true: force Lumen for this capture regardless of the project default. +// false: force None. This must be an active override — a capture's view +// inherits the project CVar when un-overridden, so doing nothing does +// not mean "no Lumen". +static void ResolveCaptureLumenMethods( + const projectairsim::CaptureSettings& CaptureSettings, + EDynamicGlobalIlluminationMethod::Type& OutGIMethod, + EReflectionMethod::Type& OutReflectionMethod) { + const bool bIsSceneCapture = + CaptureSettings.image_type == + projectairsim::MathUtils::ToNumeric(projectairsim::ImageType::kScene); + + if (CaptureSettings.lumen_gi_enabled == + projectairsim::CaptureSettings::kLumenAuto) { + static const auto* GIMethodCVar = + IConsoleManager::Get().FindTConsoleVariableDataInt( + TEXT("r.DynamicGlobalIlluminationMethod")); + OutGIMethod = (bIsSceneCapture && GIMethodCVar) + ? static_cast( + GIMethodCVar->GetValueOnGameThread()) + : EDynamicGlobalIlluminationMethod::None; + } else { + OutGIMethod = CaptureSettings.lumen_gi_enabled + ? EDynamicGlobalIlluminationMethod::Lumen + : EDynamicGlobalIlluminationMethod::None; + } + + if (CaptureSettings.lumen_reflections_enabled == + projectairsim::CaptureSettings::kLumenAuto) { + static const auto* ReflectionMethodCVar = + IConsoleManager::Get().FindTConsoleVariableDataInt( + TEXT("r.ReflectionMethod")); + OutReflectionMethod = (bIsSceneCapture && ReflectionMethodCVar) + ? static_cast( + ReflectionMethodCVar->GetValueOnGameThread()) + : EReflectionMethod::None; + } else { + OutReflectionMethod = CaptureSettings.lumen_reflections_enabled + ? EReflectionMethod::Lumen + : EReflectionMethod::None; + } +} + void UUnrealCamera::CreateComponents() { Captures.Init(nullptr, ImageTypeCount()); RenderTargets.Init(nullptr, ImageTypeCount()); @@ -431,6 +484,17 @@ void UUnrealCamera::UpdateCaptureComponentSetting( if (!std::isnan(CaptureSettings.target_gamma)) RenderTarget->TargetGamma = CaptureSettings.target_gamma; + // Lumen needs ray tracing data in the capture's scene render when the + // project uses hardware ray tracing; without this flag the capture's rays + // miss everything even though the resolved GI method says Lumen. Harmless + // otherwise (software Lumen works without it). + EDynamicGlobalIlluminationMethod::Type GIMethod; + EReflectionMethod::Type ReflectionMethod; + ResolveCaptureLumenMethods(CaptureSettings, GIMethod, ReflectionMethod); + Capture->bUseRayTracingIfEnabled = + GIMethod == EDynamicGlobalIlluminationMethod::Lumen || + ReflectionMethod == EReflectionMethod::Lumen; + Capture->ProjectionType = static_cast(CaptureSettings.projection_mode); @@ -449,6 +513,22 @@ void UUnrealCamera::UpdateCaptureComponentSetting( void UUnrealCamera::UpdateCameraPostProcessingSetting( FPostProcessSettings& PostProcessSettings, const projectairsim::CaptureSettings& CaptureSettings) { + // Pin this capture's global illumination / reflection method explicitly + // (see ResolveCaptureLumenMethods). When the method resolves to Lumen the + // engine maintains a dedicated Lumen scene for the capture's persistent + // view state (UE 5.5+; earlier engines ignore this for captures) — this is + // what makes the captured image match the Lumen-lit editor/game viewport + // instead of losing all indirect light. + { + EDynamicGlobalIlluminationMethod::Type GIMethod; + EReflectionMethod::Type ReflectionMethod; + ResolveCaptureLumenMethods(CaptureSettings, GIMethod, ReflectionMethod); + PostProcessSettings.bOverride_DynamicGlobalIlluminationMethod = true; + PostProcessSettings.DynamicGlobalIlluminationMethod = GIMethod; + PostProcessSettings.bOverride_ReflectionMethod = true; + PostProcessSettings.ReflectionMethod = ReflectionMethod; + } + if (CaptureSettings.auto_exposure_method >= 0) { PostProcessSettings.bOverride_AutoExposureMethod = true; PostProcessSettings.AutoExposureMethod =