From c07bfb12ecdf6030bd932e63aa8338979e3f376b Mon Sep 17 00:00:00 2001 From: Mike German Date: Wed, 8 Jul 2026 09:01:40 -0400 Subject: [PATCH] fix(fileutils): drain pipe buffers after process exits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When run() breaks out of the read loop (p.poll() returns non-None), bytes that the subprocess wrote just before exiting may still be sitting in the kernel pipe buffer. With O_NONBLOCK a single read() call returns only what fits in one syscall, so a large burst of final output (common in nuclear simulation codes) could be silently dropped. Add a post-exit drain loop for both stdout and stderr. Once the write end of the pipe is closed (process exited), read_async() returns b'' (EOF) rather than raising EAGAIN, so the drain loop terminates cleanly. Add a regression test that emits 160 000 bytes of output—well past a typical 64 KB pipe buffer—and asserts every byte is captured. Closes #49 Signed-off-by: Mike German --- src/watts/fileutils.py | 12 ++++++++++++ tests/test_fileutils.py | 22 ++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/src/watts/fileutils.py b/src/watts/fileutils.py index 196599b..0dd3804 100644 --- a/src/watts/fileutils.py +++ b/src/watts/fileutils.py @@ -140,3 +140,15 @@ def read_async(fd): if p.poll() is not None: break + + # Drain any data remaining in the pipe buffers after the process exits. + # With O_NONBLOCK, a single read() per iteration may not consume all buffered + # bytes (the kernel returns what fits in one syscall). Once the write end of + # the pipe is closed (process exited) read_async returns b'' on EOF rather + # than raising EAGAIN, so this loop terminates cleanly. + for fd, stream in ((p.stdout, sys.stdout), (p.stderr, sys.stderr)): + while True: + chunk = read_async(fd) + if not chunk: + break + stream.write(chunk.decode()) diff --git a/tests/test_fileutils.py b/tests/test_fileutils.py index d418989..e6ae66a 100644 --- a/tests/test_fileutils.py +++ b/tests/test_fileutils.py @@ -7,6 +7,8 @@ import subprocess import sys +import pytest + from watts.fileutils import tee_stdout, tee_stderr, run @@ -55,3 +57,23 @@ def test_run(run_in_tmpdir): run(['env']) assert f.getvalue() == file_output + +@pytest.mark.skipif(sys.platform == 'win32', reason="O_NONBLOCK not available on Windows") +def test_run_captures_all_output(run_in_tmpdir): + # Emit enough output to fill multiple pipe-buffer reads (typically 64 KB), + # then verify every byte is captured. This is a regression test for the + # post-exit drain: without draining after p.poll() returns, the last chunk + # of buffered data could be silently dropped. + line = "x" * 79 + "\n" # 80 bytes per line + n_lines = 2000 # 160 000 bytes — well past one pipe-buffer read + script = f"import sys; [sys.stdout.write({line!r}) for _ in range({n_lines})]" + + with redirect_stdout(io.StringIO()) as f: + run([sys.executable, '-c', script]) + + captured = f.getvalue() + assert captured == line * n_lines, ( + f"Expected {n_lines * 80} bytes but got {len(captured)}; " + "output was likely truncated due to missing post-exit drain" + ) +