Append rows to data.table (passed as argument to a function)

Question

I am passing some data.tables to a function and want to collect the growing results in the passed data.tables over multiple function calls. The rows are added (appended) within the function.

Is there a way to append rows to a data.table "by reference/inplace"?

Any workaround if this is not possible?

Edit: My goal is adding multiple rows at once within the function and the number of rows can be very big (that's why I am using a 'data.table').

library(data.table)

validate <- function(data, rule, valid.result, checked.rules) {
  # ... find errors

  # How to append "rule" to "checked.rules"?

  findings <- data.table(err.code = rule$rule.id, msg = "some blah blah")  # just an stupid example
  # How to append all "finding"s to "valid.results"?
}

data          <- data.table(a=1:10, b=21:30)
valid.result  <- data.table(err.code = integer(0), msg       = character(0))  # empty validation results table
checked.rules <- data.table(rule.id  = integer(0), rule.name = character(0))  # empty table
rules         <- data.table(rule.id  = 1:4,        rule.name = c("too big", "too small", "too late", "empty"))

validate(data, rules[3, ], valid.result, checked.rules)
validate(data, rules[1, ], valid.result, checked.rules)
validate(data, rules[4, ], valid.result, checked.rules)

Expected results:

checked.rules
# rule.id  rule.name
# 1:       3  too late
# 2:       1   too big
# 3:       4     empty

valid.results
# err.code  msg
# 1:        3 some blah blah
# 2:        1 some blah blah
# 3:        4 some blah blah

Answer 1

As already mentioned in the link provided by @Henrik currently data.tables can't add rows by reference. Accordingly I'd go with rbindlist (which also works just fine to add multiple rows):

library(data.table)

validate <- function(data, rule, valid.result, checked.rules) {
  # ... find errors

  # How to append "rule" to "checked.rules"?
  checked.rules <<- rbindlist(list(checked.rules, rule))

  findings <- data.table(err.code = rule$rule.id, msg = "some blah blah")  # just an stupid example
  # How to append all "finding"s to "valid.results"?
  valid.result <<- rbindlist(list(valid.result, findings))
}

data          <- data.table(a=1:10, b=21:30)
valid.result  <- data.table(err.code = integer(0), msg       = character(0))  # empty validation results table
checked.rules <- data.table(rule.id  = integer(0), rule.name = character(0))  # empty table
rules         <- data.table(rule.id  = 1:4,        rule.name = c("too big", "too small", "too late", "empty"))

validate(data, rules[3, ], valid.result, checked.rules)
validate(data, rules[1, ], valid.result, checked.rules)
validate(data, rules[4, ], valid.result, checked.rules)

print(checked.rules)
print(valid.result)

Answer 2

After reading the links in the comments and the proposal of @ismirsehregal to use a list I ended up with using an environment so that I can collect multiple results "by reference".

I have done a benchmark for two variants:

1. rbind the intermediate result at the end of each function call into the "cumulative" result ("append within the function").

2. collect the intermediate results for each function call and rbindlist only once at the end ("append outside of the function").

The code is simplified resulting in abt. 9 mio rows after 20 function calls:

library(data.table)
library(microbenchmark)

validate.rbind <- function(data, results) {
  findings <- data.table(err.code = 100, msg = rep("some blah blah", sample(1E6, 1) + 1))  # just an stupid example
  results$valid.result <- rbind(results$valid.result, findings) # same as: rbindlist(list(results$valid.result, findings))
}

validate.rbindlist <- function(data, results) {
  findings <- data.table(err.code = 100, msg = rep("some blah blah", sample(1E6, 1) + 1))  # just an stupid example
  assign(paste0("res", sprintf("%02d", results$counter)), findings, envir = results)
  results$counter = results$counter + 1
}

microbenchmark(
  rbind.per.call = {
    set.seed(0815)   # make random numbers reproducible
    data                 <- data.table(a=1:100, b=21:30)
    results              <- new.env()   # use an environment to pass arguments by reference
    results$valid.result <- data.table(err.code = integer(0), msg = character(0))  # empty validation results table
    for (i in 1:20) {
      validate.rbind(data, results)
    }
  },
  rbindlist.once = {
    set.seed(0815)   # make random numbers reproducible
    data                 <- data.table(a=1:100, b=21:30)
    results              <- new.env()   # use an environment to pass arguments by reference
    results$counter      <- 1
    for (i in 1:20) {
      validate.rbindlist(data, results)
    }
    result.vars <- ls(envir = results, pattern = "^res.*")  # identify the result tables via the used naming pattern
    results$valid.result <- rbindlist(mget(result.vars, envir = results))
    rm(list = result.vars, envir = results)  # remove the intermediate result tables (keep only the total result)
  },
  times = 10)

Solution 2 is four times faster

Unit: milliseconds
           expr       min        lq      mean    median        uq       max neval
 rbind.per.call 1021.2956 1114.8187 1198.7033 1153.7775 1324.6672 1477.5669    10
 rbindlist.once  231.0477  249.7195  305.0974  260.2499  275.3446  713.1155    10

and the memory footprint (observed with gc()) is even better:

# Memory consumption for rbind.per.call:
#            used (Mb)  gc trigger  (Mb) max used  (Mb)
# Ncells   510152  27.3     940480  50.3   847768  45.3
# Vcells 19636460 149.9   55027624 419.9 52254173 398.7

# Memory consumption for rbindlist.once:
#            used (Mb)  gc trigger  (Mb) max used  (Mb)
# Ncells   604335  32.3    1168576  62.5   940480  50.3
# Vcells 19859703 151.6   55503896 423.5 39082073 298.2

PS: I did not test the linked set variation since I don't expect a better performance and because it is more complicated to use