diff --git a/NEWS.md b/NEWS.md index 5d2c35aa9..199464db1 100644 --- a/NEWS.md +++ b/NEWS.md @@ -48,6 +48,8 @@ 13. `setnafill()` now accepts a logical vector for the `cols` argument, which must be the same length as the number of columns in `x`, [#4113](https://github.com/Rdatatable/data.table/issues/4113). Thanks to @MichaelChirico for the suggestion and @venom1204 for the PR. +14. `nafill()` and `setnafill()` gain a `limit` argument to restrict the maximum number of consecutive `NA` values filled during `locf` or `nocb` operations, [#7677](https://github.com/Rdatatable/data.table/issues/7677). Thanks to @jaynewton for the suggestion and @venom1204 for the PR. + ### BUG FIXES 1. `fread()` with `skip=0` and `(header=TRUE|FALSE)` no longer skips the first row when it has fewer fields than subsequent rows, [#7463](https://github.com/Rdatatable/data.table/issues/7463). Thanks @emayerhofer for the report and @ben-schwen for the fix. diff --git a/R/shift.R b/R/shift.R index a1ece5f8e..870311b25 100644 --- a/R/shift.R +++ b/R/shift.R @@ -26,17 +26,21 @@ shift = function(x, n=1L, fill, type=c("lag", "lead", "shift", "cyclic"), give.n ans } -nafill = function(x, type=c("const","locf","nocb"), fill=NA, nan=NA) { +nafill = function(x, type=c("const", "locf", "nocb"), fill=NA, nan=NA, limit=Inf) { type = match.arg(type) - .Call(CnafillR, x, type, fill, nan_is_na(nan), FALSE, NULL) + if (!is.numeric(limit) || length(limit) != 1L || is.na(limit) || limit < 0) + stopf("limit must be a non-negative scalar numeric") + .Call(CnafillR, x, type, fill, nan_is_na(nan), FALSE, NULL, as.double(floor(limit))) } -setnafill = function(x, type=c("const","locf","nocb"), fill=NA, nan=NA, cols=seq_along(x)) { +setnafill = function(x, type=c("const", "locf", "nocb"), fill=NA, nan=NA, cols=seq_along(x), limit=Inf) { type = match.arg(type) if (is.logical(cols)) { if (length(cols) != length(x)) stopf("'cols' is a logical vector of length %d but there are %d columns", length(cols), length(x)) if (anyNA(cols)) stopf("'cols' contains NA at position %d", which(is.na(cols))[1L]) cols = which(cols) } - invisible(.Call(CnafillR, x, type, fill, nan_is_na(nan), TRUE, cols)) + if (!is.numeric(limit) || length(limit) != 1L || is.na(limit) || limit < 0) + stopf("limit must be a non-negative scalar numeric") + invisible(.Call(CnafillR, x, type, fill, nan_is_na(nan), TRUE, cols, as.double(floor(limit)))) } diff --git a/inst/tests/tests.Rraw b/inst/tests/tests.Rraw index 7cb84b7ec..f0324054e 100644 --- a/inst/tests/tests.Rraw +++ b/inst/tests/tests.Rraw @@ -21985,3 +21985,19 @@ DT3 = data.table(a=c(1,NA), b=c("x",NA), c=c(3,NA)) test(2387.04, setnafill(copy(DT3), type="locf", cols=sapply(DT3, is.numeric)), data.table(a=c(1,1), b=c("x",NA), c=c(3,3))) test(2387.05, setnafill(copy(DT3), type="locf", cols=c(TRUE,NA,FALSE)), error="'cols' contains NA at position 2") test(2387.06, setnafill(copy(DT3), type="locf", cols=c(TRUE,FALSE)), error="'cols' is a logical vector of length 2 but there are 3 columns") + +# #7677: add limit argument to nafill() +test(2388.01, list(nafill(c(1, NA, NA, NA, 5), type="locf", limit=1), nafill(c(1, NA, NA, 5, NA, NA, NA, 9), type="locf", limit=2), nafill(c(1, NA, NA, 4, NA), type="locf", limit=Inf)), list(c(1, 1, NA, NA, 5), c(1, 1, 1, 5, 5, 5, NA, 9), c(1, 1, 1, 4, 4))) +test(2388.02, list(nafill(c(1L, NA, NA), type="locf", limit=1), nafill(c("a", NA, NA), type="locf", limit=1)), list(c(1L, 1L, NA), c("a", "a", NA))) +test(2388.03, nafill(c(NA, NA, 3, NA, NA), type="locf", limit=1), c(NA, NA, 3, 3, NA)) +test(2388.04, nafill(c(1, NA, NA, 5), type="nocb", limit=1), c(1, NA, 5, 5)) +test(2388.05, setnafill(data.table(a=c(1, NA, NA)), type="locf", limit=1), data.table(a=c(1, 1, NA))) +test(2388.06, nafill(c(1, NA, NA), type="locf", limit=0), c(1, NA, NA)) +test(2388.07, nafill(c(NA, 1), "locf", fill=0, limit=0), c(0, 1)) +test(2388.08, nafill(c(1, NA, NA), type="locf", limit=1.5), c(1, 1, NA)) +test(2388.09, nafill(c(1, NA, NA), type="const", fill=0, limit=1), c(1, 0, 0)) +lims = list(-1, NA, NaN, NULL, c(1, 2), 1+2i); for (i in seq_along(lims)) { test(2388.09 + i / 100, nafill(1:5, limit=lims[[i]]), error="limit must be a non-negative scalar numeric") } +test(2388.16, setnafill(data.table(a=1:5), limit=-1), error="limit must be a non-negative scalar numeric") +test(2388.17, nafill(c(1.1, NA, NA, 4.4), type="nocb", limit=1), c(1.1, NA, 4.4, 4.4)) +test(2388.18, nafill(c(1.1, NA, NA), type="locf", limit=1, nan=NaN), c(1.1, 1.1, NA)) +test(2388.19, nafill(c(1.1, NA, NA), type="nocb", limit=1, nan=NaN), c(NA, 1.1, 1.1)) diff --git a/man/nafill.Rd b/man/nafill.Rd index af04ff4f0..c132a3a03 100644 --- a/man/nafill.Rd +++ b/man/nafill.Rd @@ -10,8 +10,8 @@ Fast fill missing values using constant value, \emph{last observation carried forward} or \emph{next observation carried backward}. } \usage{ -nafill(x, type=c("const", "locf", "nocb"), fill=NA, nan=NA) -setnafill(x, type=c("const", "locf", "nocb"), fill=NA, nan=NA, cols=seq_along(x)) +nafill(x, type=c("const", "locf", "nocb"), fill=NA, nan=NA, limit=Inf) +setnafill(x, type=c("const", "locf", "nocb"), fill=NA, nan=NA, cols=seq_along(x), limit=Inf) } \arguments{ \item{x}{ Vector, list, data.frame or data.table of logical, numeric or character columns. } @@ -19,6 +19,7 @@ setnafill(x, type=c("const", "locf", "nocb"), fill=NA, nan=NA, cols=seq_along(x) \item{fill}{ Value to be used to replace missing observations. See examples. } \item{nan}{ Either \code{NaN} or \code{NA}; if the former, \code{NaN} is treated as distinct from \code{NA}, otherwise, they are treated the same during replacement. See Examples. } \item{cols}{ Numeric, character or logical vector specifying columns to be updated. A logical vector must be the same length as the number of columns in \code{x}. } + \item{limit}{The maximum number of consecutive \code{NA} values to fill. Must be a non-negative scalar numeric. Default is \code{Inf}. Fractional values are truncated via \code{floor}. This argument is ignored when \code{type="const"}.} } \details{ Supported types are \emph{logical}, \emph{integer}, \emph{double}, \emph{character}, and \emph{factor}, as well as classes built on top of these such as \code{Date}, \code{IDate}, and \code{POSIXct}. @@ -51,6 +52,13 @@ nafill(dt, "nocb") setnafill(dt, "locf", cols=c("v2","v3")) dt + +# limit= restricts the number of consecutive fills +y = c(1, NA, NA, NA, 5) +nafill(c(NA, 1), "locf", fill=0, limit=0) # Result: 0, 1 (Boundary case) +nafill(y, "locf", limit=1) # Only fills the first NA +nafill(y, "locf", limit=Inf) # Fills all NAs (default) + } \seealso{ \code{\link{shift}}, \code{\link{data.table}}, \code{\link{fcoalesce}} diff --git a/src/data.table.h b/src/data.table.h index df46c4a33..4c4d112df 100644 --- a/src/data.table.h +++ b/src/data.table.h @@ -311,9 +311,11 @@ SEXP copyAsGrowable(SEXP x); SEXP resizeVector(SEXP x, SEXP size); // nafill.c -void nafillDouble(double *x, uint_fast64_t nx, unsigned int type, double fill, bool nan_is_na, ans_t *ans, bool verbose); -void nafillInteger(int32_t *x, uint_fast64_t nx, unsigned int type, int32_t fill, ans_t *ans, bool verbose); -SEXP nafillR(SEXP obj, SEXP type, SEXP fill, SEXP nan_is_na_arg, SEXP inplace, SEXP cols); +void nafillDouble(double *x, uint_fast64_t nx, unsigned int type, double fill, bool nan_is_na, ans_t *ans, bool verbose, uint_fast64_t limit); +void nafillInteger(int32_t *x, uint_fast64_t nx, unsigned int type, int32_t fill, ans_t *ans, bool verbose, uint_fast64_t limit); +void nafillInteger64(int64_t *x, uint_fast64_t nx, unsigned int type, int64_t fill, ans_t *ans, bool verbose, uint_fast64_t limit); +void nafillString(const SEXP *x, uint_fast64_t nx, unsigned int type, SEXP fill, ans_t *ans, bool verbose, uint_fast64_t limit); +SEXP nafillR(SEXP obj, SEXP type, SEXP fill, SEXP nan_is_na_arg, SEXP inplace, SEXP cols, SEXP limit); // between.c SEXP between(SEXP x, SEXP lower, SEXP upper, SEXP incbounds, SEXP NAbounds, SEXP check); diff --git a/src/nafill.c b/src/nafill.c index 8a8e8eab5..f8afc8ba3 100644 --- a/src/nafill.c +++ b/src/nafill.c @@ -1,48 +1,60 @@ #include "data.table.h" -void nafillDouble(double *x, uint_fast64_t nx, unsigned int type, double fill, bool nan_is_na, ans_t *ans, bool verbose) { +void nafillDouble(double *x, uint_fast64_t nx, unsigned int type, double fill, bool nan_is_na, ans_t *ans, bool verbose, uint_fast64_t limit) { double tic=0.0; - if (verbose) - tic = omp_get_wtime(); + if (verbose) tic = omp_get_wtime(); if (type==0) { // const if (nan_is_na) { - for (uint_fast64_t i=0; idbl_v[i] = ISNAN(x[i]) ? fill : x[i]; - } + for (uint_fast64_t i=0; idbl_v[i] = ISNAN(x[i]) ? fill : x[i]; } else { - for (uint_fast64_t i=0; idbl_v[i] = ISNA(x[i]) ? fill : x[i]; - } + for (uint_fast64_t i=0; idbl_v[i] = ISNA(x[i]) ? fill : x[i]; } } else if (type==1) { // locf + uint_fast64_t fills = 0; if (nan_is_na) { ans->dbl_v[0] = ISNAN(x[0]) ? fill : x[0]; + if (ISNAN(x[0])) fills = 1; for (uint_fast64_t i=1; idbl_v[i] = ISNAN(x[i]) ? ans->dbl_v[i-1] : x[i]; + if (ISNAN(x[i])) { + if (fills < limit) { ans->dbl_v[i] = ans->dbl_v[i-1]; fills++; } + else ans->dbl_v[i] = x[i]; + } else { ans->dbl_v[i] = x[i]; fills = 0; } } } else { ans->dbl_v[0] = ISNA(x[0]) ? fill : x[0]; + if (ISNA(x[0])) fills = 1; for (uint_fast64_t i=1; idbl_v[i] = ISNA(x[i]) ? ans->dbl_v[i-1] : x[i]; + if (ISNA(x[i])) { + if (fills < limit) { ans->dbl_v[i] = ans->dbl_v[i-1]; fills++; } + else ans->dbl_v[i] = x[i]; + } else { ans->dbl_v[i] = x[i]; fills = 0; } } } } else if (type==2) { // nocb + uint_fast64_t fills = 0; if (nan_is_na) { ans->dbl_v[nx-1] = ISNAN(x[nx-1]) ? fill : x[nx-1]; + if (ISNAN(x[nx-1])) fills = 1; for (int_fast64_t i=nx-2; i>=0; i--) { - ans->dbl_v[i] = ISNAN(x[i]) ? ans->dbl_v[i+1] : x[i]; + if (ISNAN(x[i])) { + if (fills < limit) { ans->dbl_v[i] = ans->dbl_v[i+1]; fills++; } + else ans->dbl_v[i] = x[i]; + } else { ans->dbl_v[i] = x[i]; fills = 0; } } } else { ans->dbl_v[nx-1] = ISNA(x[nx-1]) ? fill : x[nx-1]; + if (ISNA(x[nx-1])) fills = 1; for (int_fast64_t i=nx-2; i>=0; i--) { - ans->dbl_v[i] = ISNA(x[i]) ? ans->dbl_v[i+1] : x[i]; + if (ISNA(x[i])) { + if (fills < limit) { ans->dbl_v[i] = ans->dbl_v[i+1]; fills++; } + else ans->dbl_v[i] = x[i]; + } else { ans->dbl_v[i] = x[i]; fills = 0; } } } } - if (verbose) - snprintf(ans->message[0], 500, _("%s: took %.3fs\n"), __func__, omp_get_wtime()-tic); + if (verbose) snprintf(ans->message[0], 500, _("%s: took %.3fs\n"), __func__, omp_get_wtime()-tic); } -void nafillInteger(int32_t *x, uint_fast64_t nx, unsigned int type, int32_t fill, ans_t *ans, bool verbose) { +void nafillInteger(int32_t *x, uint_fast64_t nx, unsigned int type, int32_t fill, ans_t *ans, bool verbose, uint_fast64_t limit) { double tic=0.0; if (verbose) tic = omp_get_wtime(); @@ -51,20 +63,30 @@ void nafillInteger(int32_t *x, uint_fast64_t nx, unsigned int type, int32_t fill ans->int_v[i] = x[i]==NA_INTEGER ? fill : x[i]; } } else if (type==1) { // locf + uint_fast64_t fills = 0; ans->int_v[0] = x[0]==NA_INTEGER ? fill : x[0]; + if (x[0]==NA_INTEGER) fills = 1; for (uint_fast64_t i=1; iint_v[i] = x[i]==NA_INTEGER ? ans->int_v[i-1] : x[i]; + if (x[i]==NA_INTEGER) { + if (fills < limit) { ans->int_v[i] = ans->int_v[i-1]; fills++; } + else ans->int_v[i] = x[i]; + } else { ans->int_v[i] = x[i]; fills = 0; } } } else if (type==2) { // nocb + uint_fast64_t fills = 0; ans->int_v[nx-1] = x[nx-1]==NA_INTEGER ? fill : x[nx-1]; + if (x[nx-1]==NA_INTEGER) fills = 1; for (int_fast64_t i=nx-2; i>=0; i--) { - ans->int_v[i] = x[i]==NA_INTEGER ? ans->int_v[i+1] : x[i]; + if (x[i]==NA_INTEGER) { + if (fills < limit) { ans->int_v[i] = ans->int_v[i+1]; fills++; } + else ans->int_v[i] = x[i]; + } else { ans->int_v[i] = x[i]; fills = 0; } } } if (verbose) snprintf(ans->message[0], 500, _("%s: took %.3fs\n"), __func__, omp_get_wtime()-tic); } -void nafillInteger64(int64_t *x, uint_fast64_t nx, unsigned int type, int64_t fill, ans_t *ans, bool verbose) { +void nafillInteger64(int64_t *x, uint_fast64_t nx, unsigned int type, int64_t fill, ans_t *ans, bool verbose, uint_fast64_t limit) { double tic=0.0; if (verbose) tic = omp_get_wtime(); @@ -73,21 +95,31 @@ void nafillInteger64(int64_t *x, uint_fast64_t nx, unsigned int type, int64_t fi ans->int64_v[i] = x[i]==NA_INTEGER64 ? fill : x[i]; } } else if (type==1) { // locf + uint_fast64_t fills = 0; ans->int64_v[0] = x[0]==NA_INTEGER64 ? fill : x[0]; + if (x[0]==NA_INTEGER64) fills = 1; for (uint_fast64_t i=1; iint64_v[i] = x[i]==NA_INTEGER64 ? ans->int64_v[i-1] : x[i]; + if (x[i]==NA_INTEGER64) { + if (fills < limit) { ans->int64_v[i] = ans->int64_v[i-1]; fills++; } + else ans->int64_v[i] = x[i]; + } else { ans->int64_v[i] = x[i]; fills = 0; } } } else if (type==2) { // nocb + uint_fast64_t fills = 0; ans->int64_v[nx-1] = x[nx-1]==NA_INTEGER64 ? fill : x[nx-1]; + if (x[nx-1]==NA_INTEGER64) fills = 1; for (int_fast64_t i=nx-2; i>=0; i--) { - ans->int64_v[i] = x[i]==NA_INTEGER64 ? ans->int64_v[i+1] : x[i]; + if (x[i]==NA_INTEGER64) { + if (fills < limit) { ans->int64_v[i] = ans->int64_v[i+1]; fills++; } + else ans->int64_v[i] = x[i]; + } else { ans->int64_v[i] = x[i]; fills = 0; } } } if (verbose) snprintf(ans->message[0], 500, _("%s: took %.3fs\n"), __func__, omp_get_wtime()-tic); } -void nafillString(const SEXP *x, uint_fast64_t nx, unsigned int type, SEXP fill, ans_t *ans, bool verbose) { +void nafillString(const SEXP *x, uint_fast64_t nx, unsigned int type, SEXP fill, ans_t *ans, bool verbose, uint_fast64_t limit) { double tic=0.0; if (verbose) tic = omp_get_wtime(); @@ -96,16 +128,26 @@ void nafillString(const SEXP *x, uint_fast64_t nx, unsigned int type, SEXP fill, SET_STRING_ELT(ans->char_v, i, x[i]==NA_STRING ? fill : x[i]); } } else if (type==1) { // locf + uint_fast64_t fills = 0; SET_STRING_ELT(ans->char_v, 0, x[0]==NA_STRING ? fill : x[0]); + if (x[0]==NA_STRING) fills = 1; const SEXP* thisans = SEXPPTR_RO(ans->char_v); // takes out STRING_ELT from loop for (uint_fast64_t i=1; ichar_v, i, x[i]==NA_STRING ? thisans[i-1] : x[i]); + if (x[i]==NA_STRING) { + if (fills < limit) { SET_STRING_ELT(ans->char_v, i, thisans[i-1]); fills++; } + else SET_STRING_ELT(ans->char_v, i, x[i]); + } else { SET_STRING_ELT(ans->char_v, i, x[i]); fills = 0; } } } else if (type==2) { // nocb + uint_fast64_t fills = 0; SET_STRING_ELT(ans->char_v, nx-1, x[nx-1]==NA_STRING ? fill : x[nx-1]); + if (x[nx-1]==NA_STRING) fills = 1; const SEXP* thisans = SEXPPTR_RO(ans->char_v); // takes out STRING_ELT from loop for (int_fast64_t i=nx-2; i>=0; i--) { - SET_STRING_ELT(ans->char_v, i, x[i]==NA_STRING ? thisans[i+1] : x[i]); + if (x[i]==NA_STRING) { + if (fills < limit) { SET_STRING_ELT(ans->char_v, i, thisans[i+1]); fills++; } + else SET_STRING_ELT(ans->char_v, i, x[i]); + } else { SET_STRING_ELT(ans->char_v, i, x[i]); fills = 0; } } } if (verbose) @@ -117,20 +159,18 @@ void nafillString(const SEXP *x, uint_fast64_t nx, unsigned int type, SEXP fill, over columns of the input data. This includes handling different data types and applying the designated filling method to each column in parallel. */ -SEXP nafillR(SEXP obj, SEXP type, SEXP fill, SEXP nan_is_na_arg, SEXP inplace, SEXP cols) { +SEXP nafillR(SEXP obj, SEXP type, SEXP fill, SEXP nan_is_na_arg, SEXP inplace, SEXP cols, SEXP limit) { int protecti=0; const bool verbose = GetVerbose(); - - if (!xlength(obj)) - return(obj); - + if (!xlength(obj)) return(obj); double tic=0.0; - if (verbose) - tic = omp_get_wtime(); + if (verbose) tic = omp_get_wtime(); + + const double limit_d = REAL(limit)[0]; + const uint_fast64_t limit_n = !R_FINITE(limit_d) || limit_d >= (double)UINT_FAST64_MAX ? UINT_FAST64_MAX : (uint_fast64_t)limit_d; bool copy = !LOGICAL(inplace)[0]; - if (!IS_TRUE_OR_FALSE(nan_is_na_arg)) - error(_("'%s' must be TRUE or FALSE"), "nan_is_na"); // # nocov + if (!IS_TRUE_OR_FALSE(nan_is_na_arg)) error(_("'%s' must be TRUE or FALSE"), "nan_is_na"); bool nan_is_na = LOGICAL(nan_is_na_arg)[0]; SEXP x = R_NilValue; @@ -186,21 +226,7 @@ SEXP nafillR(SEXP obj, SEXP type, SEXP fill, SEXP nan_is_na_arg, SEXP inplace, S for (R_len_t i=0; i