From 9657d20d0888f295c86c37656fac3e4e3f0bedba Mon Sep 17 00:00:00 2001 From: xyuzh Date: Wed, 26 Aug 2026 01:12:43 -0700 Subject: [PATCH 1/3] [core][sandbox] Preserve archived mtimes when extracting image layers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit extract_tar_layer wrote files with copyfileobj and never called os.utime, so every file in the sandbox rootfs carried the extraction time as its mtime. That breaks tools that trust timestamps — most visibly apt, which revalidates its package lists with If-Modified-Since derived from the file mtime: with a reset-to-now mtime, mirrors answer 304 and apt keeps stale image-baked lists whose Release files have long expired, failing every 'apt-get install' with 404s. Found running Terminal-Bench 2.1 under Harbor, where every verifier bootstraps with apt-get and ~30% of tasks failed on this. Directory mtimes are applied after the extraction loop (extracting children would bump them) and are best-effort. Signed-off-by: xyuzh --- .../sandbox/_internal/image_utils.py | 14 ++++++++ .../sandbox/tests/test_image_manager.py | 32 +++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/python/ray/experimental/sandbox/_internal/image_utils.py b/python/ray/experimental/sandbox/_internal/image_utils.py index 8fa53bcb1618..e338d00b1424 100644 --- a/python/ray/experimental/sandbox/_internal/image_utils.py +++ b/python/ray/experimental/sandbox/_internal/image_utils.py @@ -180,6 +180,7 @@ def extract_tar_layer( else: tar_fileobj = tar_input + dir_mtimes = [] with tarfile.open(fileobj=tar_fileobj, mode="r:*") as tar: for member in tar.getmembers(): name = member.name.lstrip("/") @@ -261,8 +262,15 @@ def extract_tar_layer( shutil.copyfileobj(f_in, f_out) if member.mode: os.chmod(target_path, member.mode) + # Preserve the archived mtime: tools inside the sandbox rely + # on it (apt revalidates its package lists with + # If-Modified-Since from the file mtime, and a reset-to-now + # mtime makes mirrors answer 304 for stale baked lists). + os.utime(target_path, (member.mtime, member.mtime)) elif member.isdir(): os.makedirs(target_path, exist_ok=True) + # Applied after the loop: extracting children would bump it. + dir_mtimes.append((target_path, member.mtime)) elif member.issym(): os.makedirs(parent_dir, exist_ok=True) try: @@ -280,6 +288,12 @@ def extract_tar_layer( except OSError: pass + for dir_path, mtime in reversed(dir_mtimes): + try: + os.utime(dir_path, (mtime, mtime)) + except OSError: + pass + def pull_and_extract_container_image( image: str, diff --git a/python/ray/experimental/sandbox/tests/test_image_manager.py b/python/ray/experimental/sandbox/tests/test_image_manager.py index b88280e41b8f..9b68c8cf4c4b 100644 --- a/python/ray/experimental/sandbox/tests/test_image_manager.py +++ b/python/ray/experimental/sandbox/tests/test_image_manager.py @@ -510,5 +510,37 @@ def test_prepare_oci_bundle_no_resolv_without_host_side_networking(tmp_path): assert _prepare(mgr, tmp_path, network=network) is None +def test_extract_tar_layer_preserves_mtimes(tmp_path): + """Archived mtimes survive extraction: apt inside the sandbox validates + its package lists with If-Modified-Since from the file mtime, so a + reset-to-extraction-time mtime makes mirrors answer 304 for stale + image-baked lists.""" + import io + import os + import tarfile + + from ray.experimental.sandbox._internal.image_utils import extract_tar_layer + + archived_mtime = 1_600_000_000 # 2020-09-13, clearly not "now" + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w") as tar: + dir_info = tarfile.TarInfo("etc") + dir_info.type = tarfile.DIRTYPE + dir_info.mtime = archived_mtime + tar.addfile(dir_info) + file_info = tarfile.TarInfo("etc/os-release") + data = b"ID=debian\n" + file_info.size = len(data) + file_info.mtime = archived_mtime + tar.addfile(file_info, io.BytesIO(data)) + + dest = tmp_path / "rootfs" + dest.mkdir() + extract_tar_layer(buf.getvalue(), str(dest)) + + assert int(os.path.getmtime(dest / "etc" / "os-release")) == archived_mtime + assert int(os.path.getmtime(dest / "etc")) == archived_mtime + + if __name__ == "__main__": sys.exit(pytest.main(["-v", __file__])) From 788d21fdc0732bf3a5b4cddbc471736f6514bca1 Mon Sep 17 00:00:00 2001 From: xyuzh Date: Thu, 27 Aug 2026 17:28:38 -0700 Subject: [PATCH 2/3] [core][sandbox] Make mtime preservation best-effort and symlink-safe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review feedback: wrap the per-file utime in try/except OSError like the directory pass (preserving mtimes is best-effort), and skip preserved symlinks in the directory pass — under UsrMerge images (/bin -> usr/bin) utime would follow the link and stamp the target with the wrong time. Signed-off-by: xyuzh --- .../ray/experimental/sandbox/_internal/image_utils.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/python/ray/experimental/sandbox/_internal/image_utils.py b/python/ray/experimental/sandbox/_internal/image_utils.py index e338d00b1424..e8df0aadef45 100644 --- a/python/ray/experimental/sandbox/_internal/image_utils.py +++ b/python/ray/experimental/sandbox/_internal/image_utils.py @@ -266,11 +266,18 @@ def extract_tar_layer( # on it (apt revalidates its package lists with # If-Modified-Since from the file mtime, and a reset-to-now # mtime makes mirrors answer 304 for stale baked lists). - os.utime(target_path, (member.mtime, member.mtime)) + # Best-effort, like the directory pass below. + try: + os.utime(target_path, (member.mtime, member.mtime)) + except OSError: + pass elif member.isdir(): os.makedirs(target_path, exist_ok=True) # Applied after the loop: extracting children would bump it. - dir_mtimes.append((target_path, member.mtime)) + # Skip preserved symlinks (UsrMerge /bin -> usr/bin): utime + # would follow them and stamp the target with the wrong time. + if not os.path.islink(target_path): + dir_mtimes.append((target_path, member.mtime)) elif member.issym(): os.makedirs(parent_dir, exist_ok=True) try: From 5d142ad359a25d7c64229a0d7859febffe9a998f Mon Sep 17 00:00:00 2001 From: Philipp Moritz Date: Thu, 27 Aug 2026 18:03:13 -0700 Subject: [PATCH 3/3] Update image_utils.py Fix another small nit pointed out by Andrew Signed-off-by: Philipp Moritz --- python/ray/experimental/sandbox/_internal/image_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ray/experimental/sandbox/_internal/image_utils.py b/python/ray/experimental/sandbox/_internal/image_utils.py index e8df0aadef45..de2a803a4776 100644 --- a/python/ray/experimental/sandbox/_internal/image_utils.py +++ b/python/ray/experimental/sandbox/_internal/image_utils.py @@ -295,7 +295,7 @@ def extract_tar_layer( except OSError: pass - for dir_path, mtime in reversed(dir_mtimes): + for dir_path, mtime in dir_mtimes: try: os.utime(dir_path, (mtime, mtime)) except OSError: