diff --git a/Packages/UnitySensors/Runtime/Scripts/Sensors/Camera/DepthCamera/DepthCameraSensor.cs b/Packages/UnitySensors/Runtime/Scripts/Sensors/Camera/DepthCamera/DepthCameraSensor.cs index 1825e42d..3d04e3c4 100644 --- a/Packages/UnitySensors/Runtime/Scripts/Sensors/Camera/DepthCamera/DepthCameraSensor.cs +++ b/Packages/UnitySensors/Runtime/Scripts/Sensors/Camera/DepthCamera/DepthCameraSensor.cs @@ -13,8 +13,51 @@ using System.Collections; using UnitySensors.Utils.Texture; +#if UNITY_6000_0_OR_NEWER +using UnityEngine.Rendering; +#endif + namespace UnitySensors.Sensor.Camera { + // Job for parallel raycast depth calculation + public struct ParallelRaycastDepthJob : IJobParallelFor + { + [ReadOnly] public float3 cameraPosition; + [ReadOnly] public float3 forward; + [ReadOnly] public float3 right; + [ReadOnly] public float3 up; + [ReadOnly] public float tanHalfFov; + [ReadOnly] public float aspect; + [ReadOnly] public int width; + [ReadOnly] public int height; + [ReadOnly] public float farClipPlane; + + [WriteOnly] public NativeArray depthValues; + + public void Execute(int index) + { + int x = index % width; + int y = index / width; + + float normalizedX = (float)x / (width - 1); + float normalizedY = (float)y / (height - 1); + + float ndcX = (2.0f * normalizedX) - 1.0f; + float ndcY = (2.0f * normalizedY) - 1.0f; + + float viewX = ndcX * tanHalfFov * aspect; + float viewY = ndcY * tanHalfFov; + + float3 rayDirection = math.normalize(forward + right * viewX + up * viewY); + + // Note: Unity.Physics would be needed for burst-compiled raycast + // For now, we'll use the fallback value + float depth = 1.0f; + + depthValues[index] = depth; + } + } + [RequireComponent(typeof(UnityEngine.Camera))] public class DepthCameraSensor : CameraSensor, IPointCloudInterface { @@ -28,7 +71,17 @@ public class DepthCameraSensor : CameraSensor, IPointCloudInterface private Material _depthCameraMat; [SerializeField] private bool _convertToPointCloud = false; + + [Header("Performance Settings")] + [SerializeField, Range(0.1f, 1.0f)] + private float _raycastResolutionScale = 0.5f; // Reduce raycast resolution for better performance + [SerializeField] + private bool _useAdaptiveQuality = true; // Enable adaptive quality based on frame rate + private TextureLoader _textureLoader; + private Texture2D _depthTexture; // Reuse texture to avoid allocations + private int _lastRaycastWidth, _lastRaycastHeight; + private float _lastFrameTime; private JobHandle _jobHandle; @@ -52,7 +105,18 @@ protected override void Init() _camera.nearClipPlane = _minRange; _camera.farClipPlane = _maxRange; +#if UNITY_6000_0_OR_NEWER + _rt = new RenderTexture(_resolution.x, _resolution.y, 24, RenderTextureFormat.ARGBFloat); + _rt.Create(); + + bool isURP = GraphicsSettings.currentRenderPipeline != null; + if (isURP) + { + Debug.Log("DepthCameraSensor: Unity 6000+ URP mode initialized"); + } +#else _rt = new RenderTexture(_resolution.x, _resolution.y, 0, RenderTextureFormat.ARGBFloat); +#endif _camera.targetTexture = _rt; _texture = new Texture2D(_resolution.x, _resolution.y, TextureFormat.RGBAFloat, false); @@ -116,7 +180,21 @@ private void SetupJob() protected override IEnumerator UpdateSensor() { +#if UNITY_6000_0_OR_NEWER + bool isURP = GraphicsSettings.currentRenderPipeline != null; + + if (isURP) + { + GenerateDepthImageUsingRaycast(); + } + else + { + _camera.Render(); + } +#else _camera.Render(); +#endif + yield return _textureLoader.LoadTextureAsync(); if (_textureLoader.success && _convertToPointCloud) @@ -128,6 +206,111 @@ protected override IEnumerator UpdateSensor() } } + private void GenerateDepthImageUsingRaycast() + { + // Adaptive quality adjustment based on frame rate + if (_useAdaptiveQuality) + { + float currentFrameTime = Time.unscaledDeltaTime; + if (_lastFrameTime > 0) + { + float currentFPS = 1.0f / currentFrameTime; + float targetFPS = frequency; // Use sensor frequency as target + if (currentFPS < targetFPS * 0.8f) // If FPS drops below 80% of target + { + _raycastResolutionScale = Mathf.Max(0.1f, _raycastResolutionScale - 0.05f); + } + else if (currentFPS > targetFPS * 1.1f) // If FPS is above 110% of target + { + _raycastResolutionScale = Mathf.Min(1.0f, _raycastResolutionScale + 0.02f); + } + } + _lastFrameTime = currentFrameTime; + } + + // Calculate actual raycast resolution + int raycastWidth = Mathf.Max(1, Mathf.RoundToInt(_rt.width * _raycastResolutionScale)); + int raycastHeight = Mathf.Max(1, Mathf.RoundToInt(_rt.height * _raycastResolutionScale)); + + // Reuse texture if possible to avoid allocations + if (_depthTexture == null || _lastRaycastWidth != raycastWidth || _lastRaycastHeight != raycastHeight) + { + if (_depthTexture != null) + DestroyImmediate(_depthTexture); + + _depthTexture = new Texture2D(raycastWidth, raycastHeight, TextureFormat.RGBAFloat, false); + _lastRaycastWidth = raycastWidth; + _lastRaycastHeight = raycastHeight; + } + + RenderTexture.active = _rt; + GL.Clear(true, true, Color.white); + RenderTexture.active = null; + + // Pre-calculate camera parameters + float fovRad = _camera.fieldOfView * Mathf.Deg2Rad; + float aspect = (float)_rt.width / _rt.height; + float tanHalfFov = Mathf.Tan(fovRad * 0.5f); + + Vector3 cameraPos = _camera.transform.position; + Vector3 forward = _camera.transform.forward; + Vector3 right = _camera.transform.right; + Vector3 up = _camera.transform.up; + + // Use Color32 array for better performance + Color32[] pixels = new Color32[raycastWidth * raycastHeight]; + + // Batch raycast operations + for (int y = 0; y < raycastHeight; y++) + { + for (int x = 0; x < raycastWidth; x++) + { + // Map raycast coordinates to full resolution + float normalizedX = (float)x / (raycastWidth - 1); + float normalizedY = (float)y / (raycastHeight - 1); + + float ndcX = (2.0f * normalizedX) - 1.0f; + float ndcY = (2.0f * normalizedY) - 1.0f; + + float viewX = ndcX * tanHalfFov * aspect; + float viewY = ndcY * tanHalfFov; + + Vector3 rayDirection = (forward + right * viewX + up * viewY).normalized; + Ray ray = new Ray(cameraPos, rayDirection); + + float depth = 1.0f; + + if (Physics.Raycast(ray, out RaycastHit hit, _camera.farClipPlane)) + { + float distance = hit.distance; + depth = Mathf.Clamp01(distance / _camera.farClipPlane); + } + + byte depthByte = (byte)(depth * 255); + pixels[y * raycastWidth + x] = new Color32(depthByte, depthByte, depthByte, 255); + } + } + + // Apply pixels and scale to target resolution + _depthTexture.SetPixels32(pixels); + _depthTexture.Apply(); + + // Scale to target resolution if needed + if (raycastWidth != _rt.width || raycastHeight != _rt.height) + { + RenderTexture tempRT = RenderTexture.GetTemporary(_rt.width, _rt.height, 0, RenderTextureFormat.ARGBFloat); + Graphics.Blit(_depthTexture, tempRT); + Graphics.CopyTexture(tempRT, _rt); + RenderTexture.ReleaseTemporary(tempRT); + } + else + { + RenderTexture.active = _rt; + Graphics.CopyTexture(_depthTexture, _rt); + RenderTexture.active = null; + } + } + protected override void OnSensorDestroy() { if (_convertToPointCloud) @@ -137,12 +320,22 @@ protected override void OnSensorDestroy() _noises.Dispose(); _directions.Dispose(); } + + // Clean up depth texture + if (_depthTexture != null) + { + DestroyImmediate(_depthTexture); + _depthTexture = null; + } + _rt.Release(); } +#if !UNITY_6000_0_OR_NEWER private void OnRenderImage(RenderTexture source, RenderTexture dest) { Graphics.Blit(null, dest, _depthCameraMat); } +#endif } } diff --git a/Packages/UnitySensors/Runtime/Scripts/Sensors/Camera/FisheyeCamera/FisheyeCameraSensor.cs b/Packages/UnitySensors/Runtime/Scripts/Sensors/Camera/FisheyeCamera/FisheyeCameraSensor.cs index bbf18edd..0cb183bf 100644 --- a/Packages/UnitySensors/Runtime/Scripts/Sensors/Camera/FisheyeCamera/FisheyeCameraSensor.cs +++ b/Packages/UnitySensors/Runtime/Scripts/Sensors/Camera/FisheyeCamera/FisheyeCameraSensor.cs @@ -33,11 +33,20 @@ public enum CameraModel protected override void Init() { base.Init(); +#if UNITY_6000_0_OR_NEWER + // Unity 6000+ requires depth buffer for render textures used with cameras + _cubemap = new RenderTexture(_cubemapResolution, _cubemapResolution, 24, RenderTextureFormat.ARGB32) + { + dimension = TextureDimension.Cube + }; + _rt = new RenderTexture(_resolution.x, _resolution.y, 24, RenderTextureFormat.ARGB32); +#else _cubemap = new RenderTexture(_cubemapResolution, _cubemapResolution, 0, RenderTextureFormat.ARGB32) { dimension = TextureDimension.Cube }; _rt = new RenderTexture(_resolution.x, _resolution.y, 0, RenderTextureFormat.ARGB32); +#endif _texture = new Texture2D(_resolution.x, _resolution.y, TextureFormat.RGBA32, false); _textureLoader = new TextureLoader { diff --git a/Packages/UnitySensors/Runtime/Scripts/Sensors/Camera/PanoramicCamera/PanoramicCameraSensor.cs b/Packages/UnitySensors/Runtime/Scripts/Sensors/Camera/PanoramicCamera/PanoramicCameraSensor.cs index 7386ccc9..bbc29d7b 100644 --- a/Packages/UnitySensors/Runtime/Scripts/Sensors/Camera/PanoramicCamera/PanoramicCameraSensor.cs +++ b/Packages/UnitySensors/Runtime/Scripts/Sensors/Camera/PanoramicCamera/PanoramicCameraSensor.cs @@ -16,11 +16,20 @@ public class PanoramicCameraSensor : CameraSensor protected override void Init() { base.Init(); +#if UNITY_6000_0_OR_NEWER + // Unity 6000+ requires depth buffer for render textures used with cameras + _cubemap = new RenderTexture(_cubemapResolution.x, _cubemapResolution.y, 24, RenderTextureFormat.ARGB32) + { + dimension = TextureDimension.Cube + }; + _rt = new RenderTexture(_resolution.x, _resolution.y, 24, RenderTextureFormat.ARGB32); +#else _cubemap = new RenderTexture(_cubemapResolution.x, _cubemapResolution.y, 0, RenderTextureFormat.ARGB32) { dimension = TextureDimension.Cube }; _rt = new RenderTexture(_resolution.x, _resolution.y, 0, RenderTextureFormat.ARGB32); +#endif _texture = new Texture2D(_resolution.x, _resolution.y, TextureFormat.RGBA32, false); _textureLoader = new TextureLoader { diff --git a/Packages/UnitySensors/Runtime/Scripts/Sensors/Camera/RGBCamera/RGBCameraSensor.cs b/Packages/UnitySensors/Runtime/Scripts/Sensors/Camera/RGBCamera/RGBCameraSensor.cs index 889a7d4f..7530f4be 100644 --- a/Packages/UnitySensors/Runtime/Scripts/Sensors/Camera/RGBCamera/RGBCameraSensor.cs +++ b/Packages/UnitySensors/Runtime/Scripts/Sensors/Camera/RGBCamera/RGBCameraSensor.cs @@ -10,7 +10,12 @@ public class RGBCameraSensor : CameraSensor protected override void Init() { base.Init(); +#if UNITY_6000_0_OR_NEWER + // Unity 6000+ requires depth buffer for render textures used with cameras + _rt = new RenderTexture(_resolution.x, _resolution.y, 24, RenderTextureFormat.ARGB32); +#else _rt = new RenderTexture(_resolution.x, _resolution.y, 0, RenderTextureFormat.ARGB32); +#endif _camera.targetTexture = _rt; _texture = new Texture2D(_resolution.x, _resolution.y, TextureFormat.RGBA32, false); diff --git a/Packages/UnitySensors/Runtime/Scripts/Sensors/Camera/RGBDCamera/RGBDCameraSensor.cs b/Packages/UnitySensors/Runtime/Scripts/Sensors/Camera/RGBDCamera/RGBDCameraSensor.cs index 42ec6b63..1b3b225c 100644 --- a/Packages/UnitySensors/Runtime/Scripts/Sensors/Camera/RGBDCamera/RGBDCameraSensor.cs +++ b/Packages/UnitySensors/Runtime/Scripts/Sensors/Camera/RGBDCamera/RGBDCameraSensor.cs @@ -12,6 +12,10 @@ using Random = Unity.Mathematics.Random; using System.Collections; +#if UNITY_6000_0_OR_NEWER +using UnityEngine.Rendering; +#endif + namespace UnitySensors.Sensor.Camera { [RequireComponent(typeof(UnityEngine.Camera))] @@ -27,9 +31,18 @@ public class RGBDCameraSensor : CameraSensor, IPointCloudInterface private Material _depthCameraMat; [SerializeField] private bool _convertToPointCloud = false; + + [Header("Performance Settings")] + [SerializeField, Range(0.1f, 1.0f)] + private float _raycastResolutionScale = 0.5f; // Reduce raycast resolution for better performance + [SerializeField] + private bool _useAdaptiveQuality = true; // Enable adaptive quality based on frame rate private RenderTexture _depthRt = null; private Texture2D _depthTexture; + private Texture2D _raycastDepthTexture; // Reuse texture for raycast to avoid allocations + private int _lastRaycastWidth, _lastRaycastHeight; + private float _lastFrameTime; private UnityEngine.Camera _colorCamera; private RenderTexture _colorRt = null; @@ -59,7 +72,12 @@ public class RGBDCameraSensor : CameraSensor, IPointCloudInterface protected override void Init() { base.Init(); +#if UNITY_6000_0_OR_NEWER + // Unity 6000+ requires depth buffer for render textures used with cameras + _depthRt = new RenderTexture(_resolution.x, _resolution.y, 24, RenderTextureFormat.ARGBFloat); +#else _depthRt = new RenderTexture(_resolution.x, _resolution.y, 0, RenderTextureFormat.ARGBFloat); +#endif _depthCamera.targetTexture = _depthRt; GameObject colorCameraObject = new GameObject(); @@ -70,7 +88,12 @@ protected override void Init() colorCameraTransform.localRotation = Quaternion.identity; _colorCamera = colorCameraObject.AddComponent(); +#if UNITY_6000_0_OR_NEWER + // Unity 6000+ requires depth buffer for render textures used with cameras + _colorRt = new RenderTexture(_resolution.x, _resolution.y, 24, RenderTextureFormat.ARGB32); +#else _colorRt = new RenderTexture(_resolution.x, _resolution.y, 0, RenderTextureFormat.ARGB32); +#endif _colorCamera.targetTexture = _colorRt; _depthCamera.fieldOfView = _colorCamera.fieldOfView = _fov; @@ -148,8 +171,24 @@ private void SetupJob() protected override IEnumerator UpdateSensor() { +#if UNITY_6000_0_OR_NEWER + bool isURP = GraphicsSettings.currentRenderPipeline != null; + + if (isURP) + { + // For Unity 6000+ URP, use raycast for depth but normal rendering for color + GenerateDepthImageUsingRaycast(); + _colorCamera.Render(); + } + else + { + _depthCamera.Render(); + _colorCamera.Render(); + } +#else _depthCamera.Render(); _colorCamera.Render(); +#endif var depthLoad = _depthTextureLoader.LoadTextureAsync(); var colorLoad = _colorTextureLoader.LoadTextureAsync(); @@ -165,6 +204,111 @@ protected override IEnumerator UpdateSensor() } } + private void GenerateDepthImageUsingRaycast() + { + // Adaptive quality adjustment based on frame rate + if (_useAdaptiveQuality) + { + float currentFrameTime = Time.unscaledDeltaTime; + if (_lastFrameTime > 0) + { + float currentFPS = 1.0f / currentFrameTime; + float targetFPS = frequency; // Use sensor frequency as target + if (currentFPS < targetFPS * 0.8f) // If FPS drops below 80% of target + { + _raycastResolutionScale = Mathf.Max(0.1f, _raycastResolutionScale - 0.05f); + } + else if (currentFPS > targetFPS * 1.1f) // If FPS is above 110% of target + { + _raycastResolutionScale = Mathf.Min(1.0f, _raycastResolutionScale + 0.02f); + } + } + _lastFrameTime = currentFrameTime; + } + + // Calculate actual raycast resolution + int raycastWidth = Mathf.Max(1, Mathf.RoundToInt(_depthRt.width * _raycastResolutionScale)); + int raycastHeight = Mathf.Max(1, Mathf.RoundToInt(_depthRt.height * _raycastResolutionScale)); + + // Reuse texture if possible to avoid allocations + if (_raycastDepthTexture == null || _lastRaycastWidth != raycastWidth || _lastRaycastHeight != raycastHeight) + { + if (_raycastDepthTexture != null) + DestroyImmediate(_raycastDepthTexture); + + _raycastDepthTexture = new Texture2D(raycastWidth, raycastHeight, TextureFormat.RGBAFloat, false); + _lastRaycastWidth = raycastWidth; + _lastRaycastHeight = raycastHeight; + } + + RenderTexture.active = _depthRt; + GL.Clear(true, true, Color.white); + RenderTexture.active = null; + + // Pre-calculate camera parameters + float fovRad = _depthCamera.fieldOfView * Mathf.Deg2Rad; + float aspect = (float)_depthRt.width / _depthRt.height; + float tanHalfFov = Mathf.Tan(fovRad * 0.5f); + + Vector3 cameraPos = _depthCamera.transform.position; + Vector3 forward = _depthCamera.transform.forward; + Vector3 right = _depthCamera.transform.right; + Vector3 up = _depthCamera.transform.up; + + // Use Color32 array for better performance + Color32[] pixels = new Color32[raycastWidth * raycastHeight]; + + // Batch raycast operations + for (int y = 0; y < raycastHeight; y++) + { + for (int x = 0; x < raycastWidth; x++) + { + // Map raycast coordinates to full resolution + float normalizedX = (float)x / (raycastWidth - 1); + float normalizedY = (float)y / (raycastHeight - 1); + + float ndcX = (2.0f * normalizedX) - 1.0f; + float ndcY = (2.0f * normalizedY) - 1.0f; + + float viewX = ndcX * tanHalfFov * aspect; + float viewY = ndcY * tanHalfFov; + + Vector3 rayDirection = (forward + right * viewX + up * viewY).normalized; + Ray ray = new Ray(cameraPos, rayDirection); + + float depth = 1.0f; + + if (Physics.Raycast(ray, out RaycastHit hit, _depthCamera.farClipPlane)) + { + float distance = hit.distance; + depth = Mathf.Clamp01(distance / _depthCamera.farClipPlane); + } + + byte depthByte = (byte)(depth * 255); + pixels[y * raycastWidth + x] = new Color32(depthByte, depthByte, depthByte, 255); + } + } + + // Apply pixels and scale to target resolution + _raycastDepthTexture.SetPixels32(pixels); + _raycastDepthTexture.Apply(); + + // Scale to target resolution if needed + if (raycastWidth != _depthRt.width || raycastHeight != _depthRt.height) + { + RenderTexture tempRT = RenderTexture.GetTemporary(_depthRt.width, _depthRt.height, 0, RenderTextureFormat.ARGBFloat); + Graphics.Blit(_raycastDepthTexture, tempRT); + Graphics.CopyTexture(tempRT, _depthRt); + RenderTexture.ReleaseTemporary(tempRT); + } + else + { + RenderTexture.active = _depthRt; + Graphics.CopyTexture(_raycastDepthTexture, _depthRt); + RenderTexture.active = null; + } + } + protected override void OnSensorDestroy() { if (_convertToPointCloud) @@ -174,13 +318,23 @@ protected override void OnSensorDestroy() _noises.Dispose(); _directions.Dispose(); } + + // Clean up raycast depth texture + if (_raycastDepthTexture != null) + { + DestroyImmediate(_raycastDepthTexture); + _raycastDepthTexture = null; + } + _depthRt.Release(); _colorRt.Release(); } +#if !UNITY_6000_0_OR_NEWER private void OnRenderImage(RenderTexture source, RenderTexture dest) { Graphics.Blit(null, dest, _depthCameraMat); } +#endif } } diff --git a/Packages/UnitySensors/Runtime/Scripts/Sensors/LiDAR/DepthBufferLiDAR/DepthBufferLiDARSensor.cs b/Packages/UnitySensors/Runtime/Scripts/Sensors/LiDAR/DepthBufferLiDAR/DepthBufferLiDARSensor.cs index 59a11f36..9d46d828 100644 --- a/Packages/UnitySensors/Runtime/Scripts/Sensors/LiDAR/DepthBufferLiDAR/DepthBufferLiDARSensor.cs +++ b/Packages/UnitySensors/Runtime/Scripts/Sensors/LiDAR/DepthBufferLiDAR/DepthBufferLiDARSensor.cs @@ -77,7 +77,12 @@ private void SetupCamera() _textureSizePerCamera.y = Mathf.RoundToInt(Mathf.Sqrt(_texturePixelsNum / _camerasNum / aspectRatio)); _textureSizePerCamera.x = Mathf.RoundToInt(_textureSizePerCamera.y * aspectRatio); +#if UNITY_6000_0_OR_NEWER + // Unity 6000+ requires depth buffer for render textures used with cameras + _rt = new RenderTexture(_textureSizePerCamera.x, _textureSizePerCamera.y * _camerasNum, 24, RenderTextureFormat.ARGBFloat); +#else _rt = new RenderTexture(_textureSizePerCamera.x, _textureSizePerCamera.y * _camerasNum, 0, RenderTextureFormat.ARGBFloat); +#endif _texture = new Texture2D(_textureSizePerCamera.x, _textureSizePerCamera.y * _camerasNum, TextureFormat.RGBAFloat, false); _pixels = _texture.GetPixelData(0);