How to be concise in R code: 1) Reading lists of *.csv files -> 2) rename lists -> 3) merge -> 4) get follow up rates

Question

Below is a working code, where

1. I read lists of csv files from 9 different folders

2. Make each folder a list.

3. Rename elements of a list (drop '.csv' and others).

4. Join within each folder to make 1 big dataframe.

5. Merge all 9 dataframes to make 1 dataframe.

6. Get follow up RIDs and loss-to-follow-up RIDs and their rates.

• From as1 to as3, there are AS{n}_AREA columns and from as4 to as9 there are AS{n}_DATA_CLASS columns For MRE,

I have used ideas given here mostly, but when I add them up, the code somehow looks redundant and more could be done to look tidy to others.

Any ideas please? Thank you!

library(dplyr); library(plyr)
library(magrittr); library(stringr) 
library(ExclusionTable)
library(lubridate)
library(tidyverse); library(tidyr)
library(janitor)
library(survival)
library(ggsurvfit); library(gtsummary)
library(zoo)
library(tidycmprsk)

# AA cohort (2 of 3)
## as

i=1
num_fu = c(1,2,3,4,5,6,7,8,9)
as <- data.frame()
df <- data.frame()
dfs <- data.frame()
data_dir <- 'C:/Users/thepr/Documents/data/as'

assign(paste0("flnames", i), list.files(path = paste0(data_dir, i), pattern = "\\.csv", full.names = TRUE))
assign(paste0("as", i, "_list"), lapply(get(paste0("flnames", i)),
                                        function(x){base::as.data.frame(read.csv(x))}))
nm <- gsub(".csv", "", basename(eval(parse(text = paste0("flnames", i))))) %>% str_sub(., 1,6)
assign(paste0("as", i, "_list"), setNames(get(paste0("as", i, "_list")), nm))
df <- Reduce(full_join, get(paste0("as", i, "_list")))
assign(paste0("as",i), df[!duplicated(base::as.list(df))])
dfs <- df


for (i in 2:length(num_fu)){
RID_common <- as1$RID %in% get(paste0("as", i))$RID

      assign(paste0("flnames", i), list.files(path = paste0(data_dir, i), pattern = "\\.csv", full.names = TRUE))
      assign(paste0("as", i, "_list"), lapply(get(paste0("flnames", i)),
                                              function(x){base::as.data.frame(read.csv(x))}))
      nm <- gsub(".csv", "", basename(eval(parse(text = paste0("flnames", i))))) %>% str_sub(., 1,6)
      assign(paste0("as", i, "_list"), setNames(get(paste0("as", i, "_list")), nm))
      df <- Reduce(full_join, get(paste0("as", i, "_list")))
      assign(paste0("as",i), df[!duplicated(base::as.list(df))])
      
      dfs <- merge(dfs, df, by = "RID", all.x = TRUE)
      dfs <- dfs[!duplicated(base::as.list(dfs))]
            if(paste0("AS", i, "_AREA") %in% colnames(get(paste0("as", i)))){
              assign(paste0("fu_",i-1), get(paste0("as", i))[RID_common, c("RID", paste0("AS", i, "_AREA"))])
              assign(paste0("fu_loss_",i-1), get(paste0("as", i))[!RID_common, c("RID", paste0("AS", i, "_AREA"))])
            # FU rate
              assign(paste0("fu_rate_", i-1), nrow(get(paste0("as", i)))/nrow(as1))
            }
            else if(paste0("AS", i, "_DATA_CLASS") %in% colnames(get(paste0("as", i)))){
              assign(paste0("fu_",i-1), get(paste0("as", i))[RID_common, c("RID", paste0("AS", i, "_DATA_CLASS"))])
              assign(paste0("fu_loss_",i-1), get(paste0("as", i))[!RID_common, c("RID", paste0("AS", i, "_DATA_CLASS"))])
            # FU rate
              assign(paste0("fu_rate_", i-1), nrow(get(paste0("as", i)))/nrow(as1))
            }
            else{}
}

Answer 1

Use Sys.glob and abbreviate. This gives a flat list rather than a list of lists. No packages are used.

data_dir <- "C:/Users/thepr/Documents/data"
pat <- file.path(data_dir, "as[1-9]", "*.csv")
files <- Sys.glob(pat)

L <- Map(read.csv, files)
names(L) <- abbreviate(basename(names(L)), 6)

or maybe use these names instead of the last line:

names(L) <- paste(basename(dirname(files)), basename(files), sep = ".") |>
  abbreviate(6)

Answer 2

Thanks to @Gregor Thomas, library could be tidied as follows:

library(tidyverse) #Includes: dplyr, stringr, tidyr, purrr
library(magrittr)
library(lubridate)
library(ExclusionTable)
library(zoo) #as.Date function
library(janitor) #For Regression analysis
library(survival) #For Regression analysis
library(ggsurvfit) #For Regression analysis
library(gtsummary) #For Regression analysis 
library(tidycmprsk) #For Regression analysis

Thanks to @joran and @moodymudskipper, instead of using for loops and eval(parse(text = ...))) or get(paste0( ...)), I have used vectors and lapply.

The thing here is the use of nested lists. I have a list of folders where each folder contains 8+ csv files.

data_dir <- "C:/Users/thepr/Documents/data/as"
num_fu <- 1 : 9
dirs <- paste0(data_dir, num_fu)


as_list <- lapply(dirs, function(x) {
  files <- list.files(x, pattern = "\\.csv$", full.names = TRUE)
  names(files) <- str_sub(basename(files), 1, 6)
  Reduce(full_join, lapply(files, read.csv))
})

As per follow-up and loss rate, I will post concise answers soon!