From 30c457d795dcae75a64aa65fd5eea5cd29b50455 Mon Sep 17 00:00:00 2001 From: Thomas Tanon Date: Wed, 7 Oct 2026 11:45:17 +0200 Subject: [PATCH 1/3] DiskManager: include the description in file names Ease debugging: the file names are now `.tmp.{description}.{randomness}` instead of `.tmp{randomness}` --- datafusion/execution/src/disk_manager.rs | 43 ++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/datafusion/execution/src/disk_manager.rs b/datafusion/execution/src/disk_manager.rs index 313379f01291f..8095f88d7d985 100644 --- a/datafusion/execution/src/disk_manager.rs +++ b/datafusion/execution/src/disk_manager.rs @@ -373,6 +373,7 @@ impl DiskManager { parent_temp_dir: Arc::clone(&local_dirs[dir_index]), tempfile: Arc::new( Builder::new() + .prefix(&escape_for_temp_file_name_prefix(request_description)) .tempfile_in(local_dirs[dir_index].as_ref()) .map_err(DataFusionError::IoError)?, ), @@ -585,6 +586,32 @@ impl SpillFile for RefCountedTempFile { })) } } + +/// Escape a string to be safe to use in a temp file name prefix. +/// +/// All non-alphanumeric characters are replaced by `_` and characters are put in lowercase. +/// A `.tmp.` is added at the beginning and a last `.` is set at the end. +/// +/// For example "This kind.of /input" becomes ".tmp.this_kind_of_input." +fn escape_for_temp_file_name_prefix(text: &str) -> String { + let mut output = String::with_capacity(text.len() + 6); + output.push_str(".tmp."); + let mut add_separator = false; + for c in text.chars() { + if c.is_alphanumeric() { + if add_separator && !output.is_empty() { + output.push('_'); + add_separator = false; + } + output.extend(c.to_lowercase()); + } else { + add_separator = true; + } + } + output.push('.'); + output +} + #[cfg(test)] mod tests { use super::*; @@ -622,7 +649,7 @@ mod tests { } #[test] - fn file_in_right_dir() -> Result<()> { + fn file_in_right_dir_and_with_description() -> Result<()> { let local_dir1 = TempDir::new()?; let local_dir2 = TempDir::new()?; let local_dir3 = TempDir::new()?; @@ -636,11 +663,23 @@ mod tests { ); assert!(dm.tmp_files_enabled()); - let actual = dm.create_tmp_file("Testing")?; + let actual = dm.create_tmp_file("Testing with./spécial")?; // the file should be in one of the specified local directories assert_path_in_dirs(actual.path().unwrap(), local_dirs.into_iter()); + let file_name = actual + .path() + .unwrap() + .file_name() + .unwrap() + .to_str() + .unwrap(); + assert!( + file_name.starts_with(".tmp.testing_with_spécial."), + "{file_name}" + ); + Ok(()) } From f3033c91a13716aaebad86e50b8f946bbe42ecb7 Mon Sep 17 00:00:00 2001 From: Thomas Tanon Date: Thu, 8 Oct 2026 14:39:24 +0200 Subject: [PATCH 2/3] Cut too large descriptions --- datafusion/execution/src/disk_manager.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/datafusion/execution/src/disk_manager.rs b/datafusion/execution/src/disk_manager.rs index 8095f88d7d985..c66960b9c190a 100644 --- a/datafusion/execution/src/disk_manager.rs +++ b/datafusion/execution/src/disk_manager.rs @@ -593,6 +593,8 @@ impl SpillFile for RefCountedTempFile { /// A `.tmp.` is added at the beginning and a last `.` is set at the end. /// /// For example "This kind.of /input" becomes ".tmp.this_kind_of_input." +/// +/// The output is also truncated to fit in 128 bytes. fn escape_for_temp_file_name_prefix(text: &str) -> String { let mut output = String::with_capacity(text.len() + 6); output.push_str(".tmp."); @@ -607,6 +609,9 @@ fn escape_for_temp_file_name_prefix(text: &str) -> String { } else { add_separator = true; } + if output.len() >= 123 { + break; // We limit the output at 126 bytes and keep the space for an extra UTF-8 char and the final dot + } } output.push('.'); output @@ -1214,4 +1219,14 @@ mod tests { Ok(()) } + + #[test] + fn test_escape_for_temp_file_name_prefix() { + assert_eq!(escape_for_temp_file_name_prefix("foo"), ".tmp.foo."); + assert_eq!(escape_for_temp_file_name_prefix("fooé \nt"), ".tmp.fooé_t."); + assert_eq!( + escape_for_temp_file_name_prefix(&"a".repeat(200)), + format!(".tmp.{}.", "a".repeat(118)) + ); + } } From 02ee6daeb0cfb328eb642061f9a3ee0bef08fc09 Mon Sep 17 00:00:00 2001 From: Thomas Tanon Date: Thu, 8 Oct 2026 14:40:04 +0200 Subject: [PATCH 3/3] Fix typo --- datafusion/execution/src/disk_manager.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/execution/src/disk_manager.rs b/datafusion/execution/src/disk_manager.rs index c66960b9c190a..5a883e8884613 100644 --- a/datafusion/execution/src/disk_manager.rs +++ b/datafusion/execution/src/disk_manager.rs @@ -610,7 +610,7 @@ fn escape_for_temp_file_name_prefix(text: &str) -> String { add_separator = true; } if output.len() >= 123 { - break; // We limit the output at 126 bytes and keep the space for an extra UTF-8 char and the final dot + break; // We limit the output at 128 bytes and keep the space for an extra UTF-8 char and the final dot } } output.push('.');