How to find the outliers for two columns in data frame

Question

I need to get the total number of outliers for variable1 for each type of variable2 and variable 3 and then show it in a table. It also needs to show only cases where variable4 is larger than 1.5. I have it working but I think there's something wrong with my code because the output for each is just 0, which isn't correct.

When I do boxplot.stats(df$variable1)$out I get a huge list of outliers. But when I use the code below, it just says 0 for each one.

high <- mean(df$variable1) + sd(df$variable1) * 3
low <- mean(df$variable1) - sd(df$variable1) * 3

df%>%
  filter(varaible4>1.5)%>%
     group_by(variable2, variable3) %>% 
       tally(variable1 < low ||variable1 > high)

A table shows up with each type of variable2 and variable3...but the tally just says 0 for each one.

Please take a look at [this guide](https://stackoverflow.com/questions/5963269/how-to-make-a-great-r-reproducible-example) and provide some raw data. — Benjamin Ye, Oct 19 '19 at 18:46

score 0 · Answer 1 · answered Oct 19 '19 at 21:32

Perhaps you can use scale instead of defining high and low thresholds and using tally.

Here is an implementation based on some random data:

library(dplyr)

df = data.frame(variable1 = runif(100,1,10),
                variable2 = round(runif(100,1,3)),
                variable3 = round(runif(100,1,3)),
                variable4 = runif(100,1,5))
df$variable1[c(5,13,95)] = 1000

df1 <- df %>% 
  filter(variable4>1.5)%>%
  group_by(variable2, variable3) %>% 
  mutate(individual_outliers = abs(scale(variable1) > 3),
         total_outliers = sum(individual_outliers))

> df1
# A tibble: 91 x 6
# Groups:   variable2, variable3 [9]
   variable1 variable2 variable3 variable4 individual_outliers total_outliers
       <dbl>     <dbl>     <dbl>     <dbl>               <int>          <int>
 1      6.86         2         3      2.82                   0              0
 2      4.89         1         2      3.27                   0              0
 3      4.19         2         3      3.03                   0              0
 4      2.05         2         3      2.31                   0              0
 5   1000            3         2      2.08                   1              1
 6      9.36         2         2      3.85                   0              0
 7      8.40         3         3      3.81                   0              0
 8      8.33         3         2      2.32                   0              1
 9      7.92         2         1      4.58                   0              0
10      8.13         3         1      2.48                   0              0
# ... with 81 more rows

score 0 · Answer 2 · answered Oct 20 '19 at 11:02

Data:

df <- data.frame(variable1 = runif(1000,1,10),
                 variable2 = round(runif(1000,1,3)),
                 variable3 = round(runif(1000,1,3)),
                 variable4 = runif(1000,1,5),
                 variable5 = rep(LETTERS[1:4], 250),
                 variable6 = rep(LETTERS[5:9], 200), stringsAsFactors = F)

df$variable1[c(5,13,95)] = 1000

Multivariate outlier detection:

# Create a grouping vector: 

grouping_vars <- c("variable5", "variable6")

# Split apply combine function: 

tmp_df <- do.call(rbind, lapply(split(df[,sapply(df, is.numeric)], df[,grouping_vars]), function(x){

    # Calculate mahalanobis distance:

    md <- mahalanobis(x, colMeans(x), cov(x), inverted = FALSE)

    # Calculate the iqr of the md: 

    iqr <- quantile(md, .75) - quantile(md, .25)

    # Classify the lower threshold outliers:

    lwr <- ifelse(md > (quantile(md, .75) + (1.5 * iqr)) | (md < (quantile(md, .25) - (1.5 * iqr))),

                  "outlier",

                  "not outlier")

    # Classify the upper threshold outliers:

    upr <- ifelse(md > (quantile(md, .75) + (3 * iqr)) | (md < (quantile(md, .25) - (3 * iqr))),

                  "outlier",

                  "not outlier")

    # Bind all of the vecs together: 

    cbind(x, md, lwr, upr)

    }

   )

  )


# Extract the group from the row names:

tmp_df <- data.frame(cbind(df[,!(sapply(df, is.numeric))], 

                     grouping_vars = row.names(tmp_df), tmp_df), row.names = NULL)

df <- tmp_df[,c(names(df), setdiff(names(tmp_df), names(df)))]

Univariate outlier detection:

# Use boxplot stats mean(x) +- 1.5 * IQR: 

outliers_classified <- do.call("rbind", lapply(split(df, df[,grouping_vars]), function(x){

      if(is.numeric(x)){

        ifelse(x %in% boxplot.stats(x)$out, NA, x)

      }else{

        x

      }

    }

  )

)

How to find the outliers for two columns in data frame

2 Answers2

Data:

Multivariate outlier detection:

Univariate outlier detection: