From 073aee01d96dbfbf29b828f0fd07edb925b2858f Mon Sep 17 00:00:00 2001 From: Zhaoqi Xu Date: Tue, 13 Oct 2026 09:20:25 +0800 Subject: [PATCH 1/3] expand, unexpand: flush output after each line When reading line-at-a-time from stdin, BufWriter only flushed on exit, so an interactive session showed no converted output until the user quit. Match cut by skipping BufWriter when stdout is a terminal, and on the buffered path flush once per input chunk that completed a line, so piped line-at-a-time producers also see prompt output without a write per line on large inputs. Fixes #11091 --- src/uu/expand/src/expand.rs | 30 +++++++++++++++++++++--------- src/uu/unexpand/src/unexpand.rs | 24 +++++++++++++++++++----- tests/by-util/test_expand.rs | 22 ++++++++++++++++++++++ tests/by-util/test_unexpand.rs | 20 ++++++++++++++++++++ 4 files changed, 82 insertions(+), 14 deletions(-) diff --git a/src/uu/expand/src/expand.rs b/src/uu/expand/src/expand.rs index 21f3cc36f4d..f5e2ff68a7b 100644 --- a/src/uu/expand/src/expand.rs +++ b/src/uu/expand/src/expand.rs @@ -8,7 +8,7 @@ use clap::{Arg, ArgAction, ArgMatches, Command}; use std::ffi::OsString; use std::fs::File; -use std::io::{BufRead, BufReader, BufWriter, Read, Write, stdin, stdout}; +use std::io::{BufRead, BufReader, BufWriter, IsTerminal, Read, Write, stdin, stdout}; use std::num::IntErrorKind; use std::path::Path; use std::str::from_utf8; @@ -363,7 +363,7 @@ fn write_spaces(output: &mut impl Write, mut n: usize) -> std::io::Result<()> { fn expand_buf( buf: &[u8], - output: &mut BufWriter, + output: &mut impl Write, tabstops: &[usize], options: &Options, col: &mut usize, @@ -439,11 +439,7 @@ fn expand_buf( Ok(()) } -fn expand_file( - file: &OsString, - output: &mut BufWriter, - options: &Options, -) -> UResult<()> { +fn expand_file(file: &OsString, output: &mut impl Write, options: &Options) -> UResult<()> { let mut input = open(file)?; let ts = options.tabstops.as_ref(); let mut col = 0; @@ -451,10 +447,19 @@ fn expand_file( match input.fill_buf() { Ok([]) => break, Ok(buf) => { + let had_newline = buf.contains(&b'\n'); expand_buf(buf, output, ts, options, &mut col) .map_err_context(|| translate!("expand-error-failed-to-write-output"))?; let len = buf.len(); input.consume(len); + // Line-buffer when a newline was in this chunk so interactive stdin + // (and any producer that pauses between lines) shows output promptly. + // Fixes #11091. + if had_newline { + output + .flush() + .map_err_context(|| translate!("expand-error-failed-to-write-output"))?; + } } Err(e) => return Err(e.map_err_context(|| file.maybe_quote().to_string())), } @@ -463,7 +468,14 @@ fn expand_file( } fn expand(options: &Options) -> UResult<()> { - let mut output = BufWriter::new(stdout()); + // Match `cut`: when stdout is a terminal, skip BufWriter so each line appears + // immediately (GNU expand is line-buffered on a tty). Otherwise keep the + // buffered path for throughput on pipes/files. Fixes #11091. + let mut output: Box = if stdout().is_terminal() { + Box::new(stdout()) + } else { + Box::new(BufWriter::new(stdout())) + }; for file in &options.files { if let Err(e) = expand_file(file, &mut output, options) { @@ -471,7 +483,7 @@ fn expand(options: &Options) -> UResult<()> { set_exit_code(1); } } - // Flush once at the end + // Flush once at the end (no-op for the unbuffered tty path) output .flush() .map_err_context(|| translate!("expand-error-failed-to-write-output"))?; diff --git a/src/uu/unexpand/src/unexpand.rs b/src/uu/unexpand/src/unexpand.rs index 1353585a584..9512ccb8115 100644 --- a/src/uu/unexpand/src/unexpand.rs +++ b/src/uu/unexpand/src/unexpand.rs @@ -8,7 +8,7 @@ use clap::{Arg, ArgAction, Command}; use std::ffi::OsString; use std::fs::File; -use std::io::{self, BufRead, BufReader, BufWriter, Read, Stdin, Stdout, Write, stdin, stdout}; +use std::io::{self, BufRead, BufReader, BufWriter, IsTerminal, Read, Stdin, Write, stdin, stdout}; use std::num::IntErrorKind; use uucore::char_width::char_info_at; use uucore::display::Quotable; @@ -365,7 +365,7 @@ fn next_tabstop(tab_config: &TabConfig, col: usize) -> Option { } fn write_tabs( - output: &mut BufWriter, + output: &mut impl Write, tab_config: &TabConfig, print_state: &mut PrintState, amode: bool, @@ -502,7 +502,7 @@ impl PrintState { #[allow(clippy::cognitive_complexity)] fn unexpand_buf( buf: &[u8], - output: &mut BufWriter, + output: &mut impl Write, options: &Options, lastcol: usize, tab_config: &TabConfig, @@ -639,7 +639,7 @@ fn unexpand_buf( fn unexpand_file( file: &OsString, - output: &mut BufWriter, + output: &mut impl Write, options: &Options, lastcol: usize, tab_config: &TabConfig, @@ -657,21 +657,35 @@ fn unexpand_file( .fill_buf() .map_err(|e| e.map_err_context(|| file.maybe_quote().to_string()) as Box)? { + let mut had_newline = false; for line in buf.split_inclusive(|b| *b == b'\n') { unexpand_buf(line, output, options, lastcol, tab_config, &mut print_state)?; if let Some(b'\n') = line.last() { print_state.new_line(); + had_newline = true; } } let len = buf.len(); input.consume(len); + // Flush once per chunk that completed a line, before the next (possibly + // blocking) read, so interactive stdin shows output promptly without a + // write per line on large inputs. Fixes #11091. + if had_newline { + output.flush()?; + } } // write out anything remaining write_tabs(output, tab_config, &mut print_state, options.aflag) } fn unexpand(options: &Options) -> UResult<()> { - let mut output = BufWriter::new(stdout()); + // Match `cut` / `expand`: unbuffered on a tty so interactive stdin shows + // each converted line immediately. Fixes #11091. + let mut output: Box = if stdout().is_terminal() { + Box::new(stdout()) + } else { + Box::new(BufWriter::new(stdout())) + }; let tab_config = &options.tab_config; let lastcol = if tab_config.tabstops.len() > 1 && tab_config.increment_size.is_none() diff --git a/tests/by-util/test_expand.rs b/tests/by-util/test_expand.rs index 49d19bbe90e..f78504d3cb8 100644 --- a/tests/by-util/test_expand.rs +++ b/tests/by-util/test_expand.rs @@ -5,6 +5,8 @@ // spell-checker:ignore (ToDO) taaaa tbbbb tcccc +#[cfg(unix)] +use std::process::Stdio; use uucore::display::Quotable; use uutests::new_ucmd; @@ -496,3 +498,23 @@ fn test_buffered_reads_new_line_no_tabs_in_first_chunk() { .succeeds() .stdout_is_fixture("new_line_in_chunk_expected.txt"); } + +/// Interactive / line-at-a-time stdin must show each expanded line before the +/// next input arrives. BufWriter alone only flushes on exit. Regression for #11091. +#[cfg(unix)] +#[test] +fn test_stdin_tty_flushes_each_line() { + let mut child = new_ucmd!() + .args(&["-t3"]) + .set_stdin(Stdio::piped()) + .set_stdout(Stdio::piped()) + .run_no_wait(); + + child.write_in(b"\ta\n"); + assert_eq!(&child.stdout_exact_bytes(5), b" a\n"); + + child.write_in(b"b\tc\n"); + assert_eq!(&child.stdout_exact_bytes(5), b"b c\n"); + + child.wait().unwrap().success(); +} diff --git a/tests/by-util/test_unexpand.rs b/tests/by-util/test_unexpand.rs index 675b83f3556..78aaa203725 100644 --- a/tests/by-util/test_unexpand.rs +++ b/tests/by-util/test_unexpand.rs @@ -5,6 +5,8 @@ // spell-checker:ignore contenta edgecase behaviour tcaf +#[cfg(unix)] +use std::process::Stdio; use uutests::{at_and_ucmd, new_ucmd}; #[test] @@ -544,3 +546,21 @@ fn test_buffered_read_edgecase_behaviour() { .stdout_only(String::from_utf8(expected).unwrap()); } } + +/// Interactive / line-at-a-time stdin must show each unexpanded line before the +/// next input arrives. Regression for #11091. +#[cfg(unix)] +#[test] +fn test_stdin_tty_flushes_each_line() { + let mut child = new_ucmd!() + .args(&["-t3"]) + .set_stdin(Stdio::piped()) + .set_stdout(Stdio::piped()) + .run_no_wait(); + + // three leading spaces -> one tab at tabstop 3 + child.write_in(b" a\n"); + assert_eq!(&child.stdout_exact_bytes(3), b"\ta\n"); + + child.wait().unwrap().success(); +} From 0a022524e33fb3384ea4fbaf8e5b55cede10052e Mon Sep 17 00:00:00 2001 From: Zhaoqi Xu Date: Tue, 13 Oct 2026 14:12:45 +0800 Subject: [PATCH 2/3] expand, unexpand: only flush per read when stdin can block Keep the plain BufWriter path and flush before the next read only when the input is stdin and stdin is not a regular file. File arguments and redirected regular files are fully buffered again, avoiding the slowdown. --- src/uu/expand/src/expand.rs | 49 +++++++++++++++++++++------------ src/uu/unexpand/src/unexpand.rs | 43 +++++++++++++++++------------ 2 files changed, 58 insertions(+), 34 deletions(-) diff --git a/src/uu/expand/src/expand.rs b/src/uu/expand/src/expand.rs index f5e2ff68a7b..059de242345 100644 --- a/src/uu/expand/src/expand.rs +++ b/src/uu/expand/src/expand.rs @@ -8,7 +8,7 @@ use clap::{Arg, ArgAction, ArgMatches, Command}; use std::ffi::OsString; use std::fs::File; -use std::io::{BufRead, BufReader, BufWriter, IsTerminal, Read, Write, stdin, stdout}; +use std::io::{BufRead, BufReader, BufWriter, Read, Write, stdin, stdout}; use std::num::IntErrorKind; use std::path::Path; use std::str::from_utf8; @@ -363,7 +363,7 @@ fn write_spaces(output: &mut impl Write, mut n: usize) -> std::io::Result<()> { fn expand_buf( buf: &[u8], - output: &mut impl Write, + output: &mut BufWriter, tabstops: &[usize], options: &Options, col: &mut usize, @@ -439,23 +439,45 @@ fn expand_buf( Ok(()) } -fn expand_file(file: &OsString, output: &mut impl Write, options: &Options) -> UResult<()> { +/// Whether stdin is a regular file, i.e. reads from it never wait for a writer. +fn stdin_is_regular_file() -> bool { + #[cfg(unix)] + { + use std::os::fd::AsFd; + stdin() + .as_fd() + .try_clone_to_owned() + .map(File::from) + .and_then(|f| f.metadata()) + .is_ok_and(|m| m.is_file()) + } + #[cfg(not(unix))] + { + false + } +} + +fn expand_file( + file: &OsString, + output: &mut BufWriter, + options: &Options, +) -> UResult<()> { let mut input = open(file)?; + // Only input that can block between lines (a pipe or terminal on stdin) + // needs output flushed before the next read. Regular files keep the fully + // buffered path. Fixes #11091. + let flush_lines = file == "-" && !stdin_is_regular_file(); let ts = options.tabstops.as_ref(); let mut col = 0; loop { match input.fill_buf() { Ok([]) => break, Ok(buf) => { - let had_newline = buf.contains(&b'\n'); expand_buf(buf, output, ts, options, &mut col) .map_err_context(|| translate!("expand-error-failed-to-write-output"))?; let len = buf.len(); input.consume(len); - // Line-buffer when a newline was in this chunk so interactive stdin - // (and any producer that pauses between lines) shows output promptly. - // Fixes #11091. - if had_newline { + if flush_lines { output .flush() .map_err_context(|| translate!("expand-error-failed-to-write-output"))?; @@ -468,14 +490,7 @@ fn expand_file(file: &OsString, output: &mut impl Write, options: &Options) -> U } fn expand(options: &Options) -> UResult<()> { - // Match `cut`: when stdout is a terminal, skip BufWriter so each line appears - // immediately (GNU expand is line-buffered on a tty). Otherwise keep the - // buffered path for throughput on pipes/files. Fixes #11091. - let mut output: Box = if stdout().is_terminal() { - Box::new(stdout()) - } else { - Box::new(BufWriter::new(stdout())) - }; + let mut output = BufWriter::new(stdout()); for file in &options.files { if let Err(e) = expand_file(file, &mut output, options) { @@ -483,7 +498,7 @@ fn expand(options: &Options) -> UResult<()> { set_exit_code(1); } } - // Flush once at the end (no-op for the unbuffered tty path) + // Flush once at the end output .flush() .map_err_context(|| translate!("expand-error-failed-to-write-output"))?; diff --git a/src/uu/unexpand/src/unexpand.rs b/src/uu/unexpand/src/unexpand.rs index 9512ccb8115..7c70fef39e0 100644 --- a/src/uu/unexpand/src/unexpand.rs +++ b/src/uu/unexpand/src/unexpand.rs @@ -8,7 +8,7 @@ use clap::{Arg, ArgAction, Command}; use std::ffi::OsString; use std::fs::File; -use std::io::{self, BufRead, BufReader, BufWriter, IsTerminal, Read, Stdin, Write, stdin, stdout}; +use std::io::{self, BufRead, BufReader, BufWriter, Read, Stdin, Stdout, Write, stdin, stdout}; use std::num::IntErrorKind; use uucore::char_width::char_info_at; use uucore::display::Quotable; @@ -365,7 +365,7 @@ fn next_tabstop(tab_config: &TabConfig, col: usize) -> Option { } fn write_tabs( - output: &mut impl Write, + output: &mut BufWriter, tab_config: &TabConfig, print_state: &mut PrintState, amode: bool, @@ -502,7 +502,7 @@ impl PrintState { #[allow(clippy::cognitive_complexity)] fn unexpand_buf( buf: &[u8], - output: &mut impl Write, + output: &mut BufWriter, options: &Options, lastcol: usize, tab_config: &TabConfig, @@ -637,14 +637,34 @@ fn unexpand_buf( Ok(()) } +/// Whether stdin is a regular file, i.e. reads from it never wait for a writer. +fn stdin_is_regular_file() -> bool { + #[cfg(unix)] + { + use std::os::fd::AsFd; + stdin() + .as_fd() + .try_clone_to_owned() + .map(File::from) + .and_then(|f| f.metadata()) + .is_ok_and(|m| m.is_file()) + } + #[cfg(not(unix))] + { + false + } +} + fn unexpand_file( file: &OsString, - output: &mut impl Write, + output: &mut BufWriter, options: &Options, lastcol: usize, tab_config: &TabConfig, ) -> UResult<()> { let mut input = open(file)?; + // See expand: flush before the next read only when stdin can block. #11091 + let flush_lines = file == "-" && !stdin_is_regular_file(); let mut print_state = PrintState { col: 0, scol: 0, @@ -657,20 +677,15 @@ fn unexpand_file( .fill_buf() .map_err(|e| e.map_err_context(|| file.maybe_quote().to_string()) as Box)? { - let mut had_newline = false; for line in buf.split_inclusive(|b| *b == b'\n') { unexpand_buf(line, output, options, lastcol, tab_config, &mut print_state)?; if let Some(b'\n') = line.last() { print_state.new_line(); - had_newline = true; } } let len = buf.len(); input.consume(len); - // Flush once per chunk that completed a line, before the next (possibly - // blocking) read, so interactive stdin shows output promptly without a - // write per line on large inputs. Fixes #11091. - if had_newline { + if flush_lines { output.flush()?; } } @@ -679,13 +694,7 @@ fn unexpand_file( } fn unexpand(options: &Options) -> UResult<()> { - // Match `cut` / `expand`: unbuffered on a tty so interactive stdin shows - // each converted line immediately. Fixes #11091. - let mut output: Box = if stdout().is_terminal() { - Box::new(stdout()) - } else { - Box::new(BufWriter::new(stdout())) - }; + let mut output = BufWriter::new(stdout()); let tab_config = &options.tab_config; let lastcol = if tab_config.tabstops.len() > 1 && tab_config.increment_size.is_none() From a56ecf7d5dcfc5ced5e73f9f7164ff85f03a28df Mon Sep 17 00:00:00 2001 From: Zhaoqi Xu Date: Tue, 13 Oct 2026 15:20:09 +0800 Subject: [PATCH 3/3] expand, unexpand: fstat stdin directly instead of cloning the fd --- src/uu/expand/Cargo.toml | 2 +- src/uu/expand/src/expand.rs | 11 +++-------- src/uu/unexpand/Cargo.toml | 2 +- src/uu/unexpand/src/unexpand.rs | 11 +++-------- tests/by-util/test_expand.rs | 2 +- tests/by-util/test_unexpand.rs | 2 +- 6 files changed, 10 insertions(+), 20 deletions(-) diff --git a/src/uu/expand/Cargo.toml b/src/uu/expand/Cargo.toml index 9d639b12ed0..12059def5fb 100644 --- a/src/uu/expand/Cargo.toml +++ b/src/uu/expand/Cargo.toml @@ -21,7 +21,7 @@ fluent = { workspace = true } thiserror = { workspace = true } uucore = { workspace = true } -[target.'cfg(any(target_os = "linux", target_os = "android", target_os = "freebsd"))'.dependencies] +[target.'cfg(unix)'.dependencies] rustix = { workspace = true, features = ["fs"] } [dev-dependencies] diff --git a/src/uu/expand/src/expand.rs b/src/uu/expand/src/expand.rs index 059de242345..6e48415325a 100644 --- a/src/uu/expand/src/expand.rs +++ b/src/uu/expand/src/expand.rs @@ -443,13 +443,8 @@ fn expand_buf( fn stdin_is_regular_file() -> bool { #[cfg(unix)] { - use std::os::fd::AsFd; - stdin() - .as_fd() - .try_clone_to_owned() - .map(File::from) - .and_then(|f| f.metadata()) - .is_ok_and(|m| m.is_file()) + rustix::fs::fstat(stdin()) + .is_ok_and(|st| rustix::fs::FileType::from_raw_mode(st.st_mode).is_file()) } #[cfg(not(unix))] { @@ -465,7 +460,7 @@ fn expand_file( let mut input = open(file)?; // Only input that can block between lines (a pipe or terminal on stdin) // needs output flushed before the next read. Regular files keep the fully - // buffered path. Fixes #11091. + // buffered path. let flush_lines = file == "-" && !stdin_is_regular_file(); let ts = options.tabstops.as_ref(); let mut col = 0; diff --git a/src/uu/unexpand/Cargo.toml b/src/uu/unexpand/Cargo.toml index 2b107526b64..0f73d1879cd 100644 --- a/src/uu/unexpand/Cargo.toml +++ b/src/uu/unexpand/Cargo.toml @@ -21,7 +21,7 @@ fluent = { workspace = true } thiserror = { workspace = true } uucore = { workspace = true } -[target.'cfg(any(target_os = "linux", target_os = "android", target_os = "freebsd"))'.dependencies] +[target.'cfg(unix)'.dependencies] rustix = { workspace = true, features = ["fs"] } [dev-dependencies] diff --git a/src/uu/unexpand/src/unexpand.rs b/src/uu/unexpand/src/unexpand.rs index 7c70fef39e0..a64cba68a84 100644 --- a/src/uu/unexpand/src/unexpand.rs +++ b/src/uu/unexpand/src/unexpand.rs @@ -641,13 +641,8 @@ fn unexpand_buf( fn stdin_is_regular_file() -> bool { #[cfg(unix)] { - use std::os::fd::AsFd; - stdin() - .as_fd() - .try_clone_to_owned() - .map(File::from) - .and_then(|f| f.metadata()) - .is_ok_and(|m| m.is_file()) + rustix::fs::fstat(stdin()) + .is_ok_and(|st| rustix::fs::FileType::from_raw_mode(st.st_mode).is_file()) } #[cfg(not(unix))] { @@ -663,7 +658,7 @@ fn unexpand_file( tab_config: &TabConfig, ) -> UResult<()> { let mut input = open(file)?; - // See expand: flush before the next read only when stdin can block. #11091 + // See expand: flush before the next read only when stdin can block. let flush_lines = file == "-" && !stdin_is_regular_file(); let mut print_state = PrintState { col: 0, diff --git a/tests/by-util/test_expand.rs b/tests/by-util/test_expand.rs index f78504d3cb8..5c2e74b6e19 100644 --- a/tests/by-util/test_expand.rs +++ b/tests/by-util/test_expand.rs @@ -500,7 +500,7 @@ fn test_buffered_reads_new_line_no_tabs_in_first_chunk() { } /// Interactive / line-at-a-time stdin must show each expanded line before the -/// next input arrives. BufWriter alone only flushes on exit. Regression for #11091. +/// next input arrives. BufWriter alone only flushes on exit. #[cfg(unix)] #[test] fn test_stdin_tty_flushes_each_line() { diff --git a/tests/by-util/test_unexpand.rs b/tests/by-util/test_unexpand.rs index 78aaa203725..c108fbbc3d2 100644 --- a/tests/by-util/test_unexpand.rs +++ b/tests/by-util/test_unexpand.rs @@ -548,7 +548,7 @@ fn test_buffered_read_edgecase_behaviour() { } /// Interactive / line-at-a-time stdin must show each unexpanded line before the -/// next input arrives. Regression for #11091. +/// next input arrives. #[cfg(unix)] #[test] fn test_stdin_tty_flushes_each_line() {