From 48a1c3df5c34ccd8b3d51bfc137f1ee16ffa3f9d Mon Sep 17 00:00:00 2001 From: Andrew Jong Date: Thu, 6 Aug 2026 19:55:11 -0700 Subject: [PATCH 1/2] =?UTF-8?q?cpp=20client:=20initialize=20fis=5Fcanceled?= =?UTF-8?q?=5F=20=E2=80=94=20uninitialized=20flag=20wedged=20all=20queued?= =?UTF-8?q?=20requests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TAsyncResultProviderBase's constructor initializer list skipped fis_canceled_, leaving it uninitialized heap memory. Both client worker threads consult FIsCanceled(): when the garbage read true, the sending thread silently skipped sending the request and the receiving thread popped the response entry without ever calling SetDone — so the caller's Wait() blocked forever and, with it, every later request (the ROS2 C++ bridge's clock/services wedged permanently, nondeterministically by heap state). Found via gdb thread dump against a live sim; the Python client was unaffected, which localized the fault. Co-Authored-By: Claude Fable 5 --- client/cpp/ProjectAirsimClientLib/src/AsyncResultInternal.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/client/cpp/ProjectAirsimClientLib/src/AsyncResultInternal.h b/client/cpp/ProjectAirsimClientLib/src/AsyncResultInternal.h index 5a710f83..4f0cf6c5 100644 --- a/client/cpp/ProjectAirsimClientLib/src/AsyncResultInternal.h +++ b/client/cpp/ProjectAirsimClientLib/src/AsyncResultInternal.h @@ -34,7 +34,11 @@ template class TAsyncResultProviderBase : public TRefCounted { public: TAsyncResultProviderBase(void) - : cv_done_(), fis_done_(false), mutex_(), status_(Status::InProgress) {} + : cv_done_(), + fis_canceled_(false), + fis_done_(false), + mutex_(), + status_(Status::InProgress) {} // Returns whether the task has been requested to cancel the operation bool FIsCanceled(void) const { return (fis_canceled_); } From acd7972479fa018e6fdbd94a8a0778f2d52a6104 Mon Sep 17 00:00:00 2001 From: Andrew Jong Date: Wed, 26 Aug 2026 12:49:01 -0700 Subject: [PATCH 2/2] =?UTF-8?q?depth:=20fix=20packing=20=E2=80=94=20transm?= =?UTF-8?q?it=20raw=20float16=20meters=20bit-exactly=20(encoding=2016FC1)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The depth materials write METERS to the render target's fp16 R channel, but the packing did static_cast(meters) while declaring the wire as 16UC1 millimeters: consumers received depth quantized to WHOLE meters (a camera 3.24 m from a wall read raw 3), and sky/no-hit pixels (huge or inf) wrapped around in the bare cast into phantom finite depths. The wire now carries each pixel's FFloat16 bit pattern verbatim (little-endian, new encoding label 16FC1): the packing loop is a pure bit copy — no value conversion, no added quantization beyond the fp16 render target itself, no 65 m range cap — and sky/no-hit arrives as +inf for clients to map to their own invalid-depth convention. Downstream decoders updated to match (the encoding label change is deliberate so older clients fail loudly rather than misread fp16 bits as integers; 16UC1 branches kept for older sims): - ROS2 C++ bridge: 16FC1 -> standard ROS 32FC1 float meters, non-finite -> NaN. - Legacy Python rosbridge: same, via convert_image_16fc1_to_ros. - Python client unpack_image: float16 numpy view; example scripts accept the new encoding for depth. Co-Authored-By: Claude Fable 5 --- .../computer_vision/cv_capture.py | 2 +- .../computer_vision/cv_mode.py | 2 +- .../multirotor/hello_drone.py | 2 +- .../multirotor/multi_agent_drone.py | 2 +- .../camera_image_types.py | 2 +- .../projectairsim/src/projectairsim/utils.py | 10 +++- .../projectairsim_rosbridge/msg_converter.py | 42 ++++++++++++++- .../ros2_conversion_utils.hpp | 53 +++++++++++++++++++ .../Private/Sensors/ImagePackingAsyncTask.cpp | 28 ++++++---- 9 files changed, 124 insertions(+), 19 deletions(-) diff --git a/client/python/airsimv1_scripts_migrated/computer_vision/cv_capture.py b/client/python/airsimv1_scripts_migrated/computer_vision/cv_capture.py index a06b05b5..d5d6bc62 100644 --- a/client/python/airsimv1_scripts_migrated/computer_vision/cv_capture.py +++ b/client/python/airsimv1_scripts_migrated/computer_vision/cv_capture.py @@ -57,7 +57,7 @@ def camera_callback(camera_info, camera_name): responses.update(drone.get_images(camera_id="front_left", image_type_ids=[ImageType.SCENE])) for i, response in enumerate(responses.values()): - if response["encoding"] == "16UC1": + if response["encoding"] in ("16UC1", "16FC1"): projectairsim_log().info("Type %s, size %d, pos %s" % (response["encoding"], len(response["data"]), pprint.pformat([response["pos_x"],response["pos_y"],response["pos_z"]]))) filename = os.path.normpath(os.path.join(tmp_dir, str(x) + "_" + str(i) + '.pfm')) else: diff --git a/client/python/airsimv1_scripts_migrated/computer_vision/cv_mode.py b/client/python/airsimv1_scripts_migrated/computer_vision/cv_mode.py index 2e00cd8a..5b785466 100644 --- a/client/python/airsimv1_scripts_migrated/computer_vision/cv_mode.py +++ b/client/python/airsimv1_scripts_migrated/computer_vision/cv_mode.py @@ -66,7 +66,7 @@ def to_quaternion(pitch, roll, yaw): #responses.update(drone.GetImages("back_center", [ImageType.DISPARITY_NORMALIZED, ImageType.SURFACE_NORMALS])) for idx, response in enumerate(responses.values()): - if response["encoding"] == "16UC1": + if response["encoding"] in ("16UC1", "16FC1"): filename = os.path.join(tmp_dir, str(x) + "_" + str(idx) + ".pfm") else: filename = os.path.join(tmp_dir, str(x) + "_" + str(idx) + ".png") diff --git a/client/python/airsimv1_scripts_migrated/multirotor/hello_drone.py b/client/python/airsimv1_scripts_migrated/multirotor/hello_drone.py index b9610ac5..ae0aa360 100644 --- a/client/python/airsimv1_scripts_migrated/multirotor/hello_drone.py +++ b/client/python/airsimv1_scripts_migrated/multirotor/hello_drone.py @@ -92,7 +92,7 @@ async def main(): for idx, image in enumerate(images.values()): img_np = unpack_image(image) - if image["encoding"] == "16UC1": + if image["encoding"] in ("16UC1", "16FC1"): file_save_path = os.path.join(tmp_dir, str(idx) + ".pfm") else: file_save_path = os.path.join(tmp_dir, str(idx) + ".png") diff --git a/client/python/airsimv1_scripts_migrated/multirotor/multi_agent_drone.py b/client/python/airsimv1_scripts_migrated/multirotor/multi_agent_drone.py index 676fe9db..fc248609 100644 --- a/client/python/airsimv1_scripts_migrated/multirotor/multi_agent_drone.py +++ b/client/python/airsimv1_scripts_migrated/multirotor/multi_agent_drone.py @@ -66,7 +66,7 @@ async def main(): responses = [v for d in (responses1, responses2) for v in d.values()] for idx, response in enumerate(responses): - if response["encoding"] == "16UC1": + if response["encoding"] in ("16UC1", "16FC1"): filename = os.path.join(tmp_dir, str(idx) + ".pfm") else: filename = os.path.join(tmp_dir, str(idx) + ".png") diff --git a/client/python/example_user_scripts/camera_image_types.py b/client/python/example_user_scripts/camera_image_types.py index b565f12c..286d1f43 100644 --- a/client/python/example_user_scripts/camera_image_types.py +++ b/client/python/example_user_scripts/camera_image_types.py @@ -67,7 +67,7 @@ async def main(): for index, image in enumerate(images.values()): img_np = unpack_image(image) - if image["encoding"] == "16UC1": + if image["encoding"] in ("16UC1", "16FC1"): file_save_path = os.path.join(save_path, str(index) + ".pfm") else: file_save_path = os.path.join(save_path, str(index) + ".png") diff --git a/client/python/projectairsim/src/projectairsim/utils.py b/client/python/projectairsim/src/projectairsim/utils.py index 298a037c..02af25b0 100644 --- a/client/python/projectairsim/src/projectairsim/utils.py +++ b/client/python/projectairsim/src/projectairsim/utils.py @@ -417,8 +417,14 @@ def unpack_image(image): Returns: The image in openCV decoded form """ - # 16UC1 is used for serializing depth images - if image["encoding"] == "16UC1": + # 16FC1 is used for serializing depth images: raw IEEE 754 half-precision + # (binary16) METERS, little-endian, bit-exact with the sim's fp16 render + # target. Sky / no-hit pixels arrive as +inf. + if image["encoding"] == "16FC1": + img_dtype = "float16" + img_shape = [image["height"], image["width"]] + # 16UC1: legacy uint16 depth from older sims + elif image["encoding"] == "16UC1": img_dtype = "uint16" img_shape = [image["height"], image["width"]] elif image["encoding"] == "AVX": diff --git a/ros/node/projectairsim-rosbridge/src/projectairsim_rosbridge/msg_converter.py b/ros/node/projectairsim-rosbridge/src/projectairsim_rosbridge/msg_converter.py index d7156cda..50fd3dce 100644 --- a/ros/node/projectairsim-rosbridge/src/projectairsim_rosbridge/msg_converter.py +++ b/ros/node/projectairsim-rosbridge/src/projectairsim_rosbridge/msg_converter.py @@ -160,9 +160,13 @@ def convert_image_to_ros(self, projectairsim_topic_name, projectairsim_image): return self.convert_image_16uc1_to_ros( projectairsim_topic_name, projectairsim_image ) + elif projectairsim_image["encoding"] == "16FC1": + return self.convert_image_16fc1_to_ros( + projectairsim_topic_name, projectairsim_image + ) else: raise ValueError( - f"Can only handle image encoding BGR or 16UC1, not \"{projectairsim_image['encoding']}\"" + f"Can only handle image encoding BGR, 16UC1 or 16FC1, not \"{projectairsim_image['encoding']}\"" ) def convert_image_bgr8_to_ros( @@ -229,6 +233,42 @@ def convert_image_16uc1_to_ros( return image + def convert_image_16fc1_to_ros( + self, projectairsim_topic_name, projectairsim_image_16fc1 + ): + """ + Convert a Project AirSim 16FC1 image message into a ROS image message. + + 16FC1 is raw IEEE 754 half-precision (binary16) depth in METERS, + little-endian, bit-exact with the sim's fp16 render target. Decoded to + the standard ROS 32FC1 float-meters depth image; non-finite pixels + (sky / no hit arrive as +inf) become NaN per the ROS depth convention. + + Arguments: + projectairsim_topic_name - The Project AirSim topic name + projectairsim_image_16fc1 - The 16FC1 image message received from the Project AirSim topic + + Return: + (return) - Corresponding ROS Image message + """ + image = rossensmsg.Image() + image.header.stamp = self.ros_node.get_time_now_msg() + # image.header.frame_id must be set by caller + + image.height = projectairsim_image_16fc1["height"] + image.width = projectairsim_image_16fc1["width"] + image.encoding = "32FC1" + image.is_bigendian = projectairsim_image_16fc1["big_endian"] + + nparray = np.frombuffer( + projectairsim_image_16fc1["data"], dtype=np.float16 + ).astype(np.float32) + nparray[~np.isfinite(nparray)] = np.nan + image.data = nparray.tobytes() + image.step = image.width * 4 + + return image + def convert_imu_to_ros(self, projectairsim_topic_name, projectairsim_msg): """ Convert a Project AirSim IMU sensor message into a ROS Imu message. diff --git a/ros/projectairsim_ros2_cpp/include/projectairsim_ros2_cpp/ros2_conversion_utils.hpp b/ros/projectairsim_ros2_cpp/include/projectairsim_ros2_cpp/ros2_conversion_utils.hpp index 07efa0ce..85273a30 100644 --- a/ros/projectairsim_ros2_cpp/include/projectairsim_ros2_cpp/ros2_conversion_utils.hpp +++ b/ros/projectairsim_ros2_cpp/include/projectairsim_ros2_cpp/ros2_conversion_utils.hpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -342,6 +343,35 @@ inline void PopulateCameraInfoFromJson(const json& msg, FillFixedArray(msg.value("projection_matrix", json::array()), &camera_info->p); } +// Convert an IEEE 754 half-precision (binary16) bit pattern to float32. +inline float HalfBitsToFloat(uint16_t half_bits) { + const uint32_t sign = static_cast(half_bits & 0x8000u) << 16; + uint32_t exponent = (half_bits >> 10) & 0x1Fu; + uint32_t mantissa = half_bits & 0x3FFu; + uint32_t float_bits; + if (exponent == 0) { + if (mantissa == 0) { + float_bits = sign; // +/- zero + } else { + // Subnormal half: normalize into a float32 exponent/mantissa. + exponent = 127 - 15 + 1; + while ((mantissa & 0x400u) == 0) { + mantissa <<= 1; + --exponent; + } + mantissa &= 0x3FFu; + float_bits = sign | (exponent << 23) | (mantissa << 13); + } + } else if (exponent == 31) { + float_bits = sign | 0x7F800000u | (mantissa << 13); // inf / NaN + } else { + float_bits = sign | ((exponent - 15 + 127) << 23) | (mantissa << 13); + } + float result; + std::memcpy(&result, &float_bits, sizeof(result)); + return result; +} + inline bool PopulateImagePayloadFromJson(const json& msg, sensor_msgs::msg::Image* image) { image->height = static_cast(NumberOr(msg, "height")); @@ -355,11 +385,34 @@ inline bool PopulateImagePayloadFromJson(const json& msg, return true; } if (encoding == "16UC1") { + // Legacy sims: uint16 depth. Kept for compatibility with older servers. image->encoding = "mono16"; image->data = BytesFromJsonString(msg.value("data", "")); image->step = 2 * image->width; return true; } + if (encoding == "16FC1") { + // Depth as raw IEEE 754 half-precision (binary16) METERS, little-endian, + // bit-exact with the sim's fp16 render target. Decode to the standard + // ROS 32FC1 float-meters depth image; non-finite pixels (sky / no hit + // arrive as +inf) become NaN per the ROS depth convention. + const auto payload = BytesFromJsonString(msg.value("data", "")); + const size_t pixel_count = payload.size() / 2; + image->encoding = "32FC1"; + image->step = 4 * image->width; + image->data.resize(pixel_count * 4); + const uint8_t* src = payload.data(); + float* dst = reinterpret_cast(image->data.data()); + for (size_t i = 0; i < pixel_count; ++i, src += 2) { + const uint16_t bits = static_cast(src[0]) | + (static_cast(src[1]) << 8); + const float meters = HalfBitsToFloat(bits); + dst[i] = std::isfinite(meters) + ? meters + : std::numeric_limits::quiet_NaN(); + } + return true; + } return false; } diff --git a/unreal/Blocks/Plugins/ProjectAirSim/Source/ProjectAirSim/Private/Sensors/ImagePackingAsyncTask.cpp b/unreal/Blocks/Plugins/ProjectAirSim/Source/ProjectAirSim/Private/Sensors/ImagePackingAsyncTask.cpp index 04d12596..b08f8935 100644 --- a/unreal/Blocks/Plugins/ProjectAirSim/Source/ProjectAirSim/Private/Sensors/ImagePackingAsyncTask.cpp +++ b/unreal/Blocks/Plugins/ProjectAirSim/Source/ProjectAirSim/Private/Sensors/ImagePackingAsyncTask.cpp @@ -28,21 +28,26 @@ void FImagePackingAsyncTask::DoWork() { // Handle Depth image requests here. // 1. Currently, we do not support Compression for depth images - // 2. We also do not support sending back Floats since our ImageResponse - // Message is limited to uint8 for now. Instead, we convert Float16 from - // Unreal to uint16 and then pack it in two uint8s that have to be unpacked - // properly on client side. NOTE: This case also handles PixelsAsFloat - // implicitly i.e. it ignores it and always sends back uint16 for depth + // 2. Depth is transmitted as the render target's IEEE 754 half-precision + // (binary16) METERS, bit-exact: each pixel's FFloat16 bit pattern is + // packed little-endian into two uint8s (encoding "16FC1" below) and + // reinterpreted as float16 on the client side. No value conversion + // happens here, so no precision is lost beyond the fp16 render target + // itself, and there is no range cap: sky / no-hit pixels arrive as +inf + // for clients to map to their own invalid-depth convention. NOTE: This + // case also handles PixelsAsFloat implicitly i.e. it ignores it and + // always sends back float16 for depth if (bIsDepthImage && !ImageRequest.bCompress) { ImgResponse.ImageDataUInt8.resize( RenderResult.Width * RenderResult.Height * 2 * sizeof(uint8)); uint8* DstPtr = ImgResponse.ImageDataUInt8.data(); for (const auto& SrcPixel : RenderResult.UnrealImageFloat) { - float DepthMilli = SrcPixel.R.GetFloat(); - uint16 DepthUint16 = static_cast(DepthMilli); - *DstPtr++ = static_cast(DepthUint16 & 0xFF); // least significant byte - *DstPtr++ = static_cast((DepthUint16 >> 8) & 0xFF); // most significant byte + // The depth materials write METERS to R; transmit the fp16 bit + // pattern as-is (see the encoding comment above). + const uint16 DepthHalfBits = SrcPixel.R.Encoded; + *DstPtr++ = static_cast(DepthHalfBits & 0xFF); // least significant byte + *DstPtr++ = static_cast((DepthHalfBits >> 8) & 0xFF); // most significant byte } } // Normal RGB images without compression or PixelsAsFloat requested @@ -128,8 +133,9 @@ void FImagePackingAsyncTask::DoWork() { ImgEncoding = "BGR"; } } else { // bIsDepthImage - // 16-bit unsigned, 1 channel for depth in mm - ImgEncoding = "16UC1"; + // IEEE 754 half-precision (binary16), 1 channel, depth in METERS, + // little-endian — bit-exact with the fp16 render target + ImgEncoding = "16FC1"; } ImageMessages.emplace(