Running 'testthat.R' [126s]
Running the tests in 'tests/testthat.R' failed.
Complete output:
  > library(testthat)
  > library(cutpointr)
  > 
  > test_check("cutpointr")
  -- 1. Failure: Plotting with bootstrapping is silent (@test-cutpointr.R#109)  --
  `plot(opt_cut)` produced warnings.
  
  -- 2. Failure: AUC calculation is correct and works with Inf and -Inf (@test-cut
  `auc_cutpointr` not equal to 0.428.
  1/1 mismatches
  [1] 0.42 - 0.428 == -0.00811
  
  -- 3. Failure: AUC calculation is correct and works with Inf and -Inf (@test-cut
  cp$AUC not equal to 0.428.
  1/1 mismatches
  [1] 0.42 - 0.428 == -0.00811
  
  -- 4. Error: Metric colnames that are already in cutpointr are modified (@test-c
  task 4 failed - "missing value where TRUE/FALSE needed"
  1: cutpointr(suicide, dsi, suicide, gender, method = oc_youden_normal, metric = metricfunc, 
         boot_runs = 5) at testthat/test-cutpointr.R:303
  2: cutpointr.default(suicide, dsi, suicide, gender, method = oc_youden_normal, metric = metricfunc, 
         boot_runs = 5)
  3: cutpointr_internal(x, class, subgroup, method, metric, pos_class, neg_class, direction, 
         boot_runs, use_midpoints, break_ties, na.rm, allowParallel, predictor, outcome, 
         mod_name, subgroup_var, tol_metric, ...)
  4: dat %>% dplyr::transmute_(boot = ~purrr::map2(dat$data, dat$pos_class, function(g, 
         pc) {
         boot_g <- foreach::foreach(rep = 1:boot_runs, .combine = rbind, .export = c("method", 
             "direction", "metric", "break_ties", "neg_class", "mn", "use_midpoints", 
             "predictor", "outcome", "tol_metric")) %seq_or_par% {
             b_ind <- simple_boot(g, outcome)
             optcut_b <- method(data = g[b_ind, ], x = predictor, metric_func = metric, 
                 class = outcome, direction = direction, pos_class = pc, neg_class = neg_class, 
                 tol_metric = tol_metric, use_midpoints = use_midpoints, ...)
             optcut_b <- check_method_cols(optcut_b)
             optcut_b <- tibble::as_tibble(optcut_b)
             optcut_b <- apply_break_ties(optcut_b, break_ties)
             if (!(has_column(optcut_b, "roc_curve"))) {
                 roc_curve_b <- roc(data = g[b_ind, ], x = predictor, class = outcome, 
                     pos_class = pc, neg_class = neg_class, direction = direction, silent = TRUE)
                 roc_curve_b <- tidyr::nest_(roc_curve_b, key_col = "roc_curve")
                 optcut_b <- dplyr::bind_cols(optcut_b, roc_curve_b)
             }
             opt_ind_b <- get_opt_ind(roc_curve = optcut_b$roc_curve[[1]], oc = unlist(optcut_b$optimal_cutpoint), 
                 direction = direction)
             auc_b <- auc(tpr = optcut_b$roc_curve[[1]]$tpr, fpr = optcut_b$roc_curve[[1]]$fpr)
             Sens_Spec_b <- sesp_from_oc(optcut_b$roc_curve[[1]], oc = unlist(optcut_b$optimal_cutpoint), 
                 direction = direction, opt_ind = opt_ind_b)
             Acc_b <- accuracy_from_oc(optcut_b$roc_curve[[1]], oc = unlist(optcut_b$optimal_cutpoint), 
                 direction = direction, opt_ind = opt_ind_b)[, "accuracy"]
             kap_b <- kappa_from_oc(optcut_b$roc_curve[[1]], oc = unlist(optcut_b$optimal_cutpoint), 
                 direction = direction, opt_ind = opt_ind_b)
             metric_b <- metric(tp = optcut_b$roc_curve[[1]]$tp[opt_ind_b], fp = optcut_b$roc_curve[[1]]$fp[opt_ind_b], 
                 tn = optcut_b$roc_curve[[1]]$tn[opt_ind_b], fn = optcut_b$roc_curve[[1]]$fn[opt_ind_b])
             metric_b <- check_metric_name(metric_b)
             roc_curve_oob <- roc(data = g[-b_ind, ], x = predictor, class = outcome, 
                 pos_class = pc, neg_class = neg_class, direction = direction, silent = TRUE)
             opt_ind_oob <- get_opt_ind(roc_curve = roc_curve_oob, oc = unlist(optcut_b$optimal_cutpoint), 
                 direction = direction)
             auc_oob <- auc(tpr = roc_curve_oob$tpr, fpr = roc_curve_oob$fpr)
             Sens_Spec_oob <- sesp_from_oc(roc_curve_oob, oc = unlist(optcut_b$optimal_cutpoint), 
                 direction = direction, opt_ind = opt_ind_oob)
             Acc_oob <- accuracy_from_oc(roc_curve_oob, oc = unlist(optcut_b$optimal_cutpoint), 
                 direction = direction, opt_ind = opt_ind_oob)[, "accuracy"]
             kap_oob <- kappa_from_oc(roc_curve_oob, oc = unlist(optcut_b$optimal_cutpoint), 
                 direction = direction, opt_ind = opt_ind_oob)
             metric_oob <- metric(tp = roc_curve_oob$tp[opt_ind_oob], fp = roc_curve_oob$fp[opt_ind_oob], 
                 tn = roc_curve_oob$tn[opt_ind_oob], fn = roc_curve_oob$fn[opt_ind_oob])
             metric_oob <- check_metric_name(metric_oob)
             mn <- make.names(colnames(metric_oob))
             bootstrap <- tibble::tibble(optimal_cutpoint = optcut_b$optimal_cutpoint, 
                 AUC_b = auc_b, AUC_oob = auc_oob)
             bootstrap <- add_list(bootstrap, metric_b[, mn], paste0(mn, "_b"))
             bootstrap <- add_list(bootstrap, metric_oob[, mn], paste0(mn, "_oob"))
             bootstrap <- add_list(bootstrap, Acc_b, "acc_b")
             bootstrap <- add_list(bootstrap, Acc_oob, "acc_oob")
             bootstrap <- add_list(bootstrap, Sens_Spec_b[, "sensitivity"], "sensitivity_b")
             bootstrap <- add_list(bootstrap, Sens_Spec_oob[, "sensitivity"], "sensitivity_oob")
             bootstrap <- add_list(bootstrap, Sens_Spec_b[, "specificity"], "specificity_b")
             bootstrap <- add_list(bootstrap, Sens_Spec_oob[, "specificity"], "specificity_oob")
             bootstrap <- add_list(bootstrap, kap_b[, "cohens_kappa"], "kappa_b")
             bootstrap <- add_list(bootstrap, kap_oob[, "cohens_kappa"], "kappa_oob")
             bootstrap <- add_list(bootstrap, optcut_b$roc_curve[[1]]$tp[opt_ind_b], "TP_b")
             bootstrap <- add_list(bootstrap, optcut_b$roc_curve[[1]]$fp[opt_ind_b], "FP_b")
             bootstrap <- add_list(bootstrap, optcut_b$roc_curve[[1]]$tn[opt_ind_b], "TN_b")
             bootstrap <- add_list(bootstrap, optcut_b$roc_curve[[1]]$fn[opt_ind_b], "FN_b")
             bootstrap <- add_list(bootstrap, roc_curve_oob$tp[opt_ind_oob], "TP_oob")
             bootstrap <- add_list(bootstrap, roc_curve_oob$fp[opt_ind_oob], "FP_oob")
             bootstrap <- add_list(bootstrap, roc_curve_oob$tn[opt_ind_oob], "TN_oob")
             bootstrap <- add_list(bootstrap, roc_curve_oob$fn[opt_ind_oob], "FN_oob")
             bootstrap$roc_curve_b = optcut_b$roc_curve
             roc_curve_oob <- tidyr::nest_(roc_curve_oob, key_col = "roc_curve_oob")
             bootstrap <- dplyr::bind_cols(bootstrap, roc_curve_oob)
             return(bootstrap)
         }
         return(boot_g)
     }))
  5: withVisible(eval(quote(`_fseq`(`_lhs`)), env, env))
  6: eval(quote(`_fseq`(`_lhs`)), env, env)
  7: eval(quote(`_fseq`(`_lhs`)), env, env)
  8: `_fseq`(`_lhs`)
  9: freduce(value, `_function_list`)
  10: withVisible(function_list[[k]](value))
  ...
  13: transmute_.default(., boot = ~purrr::map2(dat$data, dat$pos_class, function(g, pc) {
         boot_g <- foreach::foreach(rep = 1:boot_runs, .combine = rbind, .export = c("method", 
             "direction", "metric", "break_ties", "neg_class", "mn", "use_midpoints", 
             "predictor", "outcome", "tol_metric")) %seq_or_par% {
             b_ind <- simple_boot(g, outcome)
             optcut_b <- method(data = g[b_ind, ], x = predictor, metric_func = metric, 
                 class = outcome, direction = direction, pos_class = pc, neg_class = neg_class, 
                 tol_metric = tol_metric, use_midpoints = use_midpoints, ...)
             optcut_b <- check_method_cols(optcut_b)
             optcut_b <- tibble::as_tibble(optcut_b)
             optcut_b <- apply_break_ties(optcut_b, break_ties)
             if (!(has_column(optcut_b, "roc_curve"))) {
                 roc_curve_b <- roc(data = g[b_ind, ], x = predictor, class = outcome, 
                     pos_class = pc, neg_class = neg_class, direction = direction, silent = TRUE)
                 roc_curve_b <- tidyr::nest_(roc_curve_b, key_col = "roc_curve")
                 optcut_b <- dplyr::bind_cols(optcut_b, roc_curve_b)
             }
             opt_ind_b <- get_opt_ind(roc_curve = optcut_b$roc_curve[[1]], oc = unlist(optcut_b$optimal_cutpoint), 
                 direction = direction)
             auc_b <- auc(tpr = optcut_b$roc_curve[[1]]$tpr, fpr = optcut_b$roc_curve[[1]]$fpr)
             Sens_Spec_b <- sesp_from_oc(optcut_b$roc_curve[[1]], oc = unlist(optcut_b$optimal_cutpoint), 
                 direction = direction, opt_ind = opt_ind_b)
             Acc_b <- accuracy_from_oc(optcut_b$roc_curve[[1]], oc = unlist(optcut_b$optimal_cutpoint), 
                 direction = direction, opt_ind = opt_ind_b)[, "accuracy"]
             kap_b <- kappa_from_oc(optcut_b$roc_curve[[1]], oc = unlist(optcut_b$optimal_cutpoint), 
                 direction = direction, opt_ind = opt_ind_b)
             metric_b <- metric(tp = optcut_b$roc_curve[[1]]$tp[opt_ind_b], fp = optcut_b$roc_curve[[1]]$fp[opt_ind_b], 
                 tn = optcut_b$roc_curve[[1]]$tn[opt_ind_b], fn = optcut_b$roc_curve[[1]]$fn[opt_ind_b])
             metric_b <- check_metric_name(metric_b)
             roc_curve_oob <- roc(data = g[-b_ind, ], x = predictor, class = outcome, 
                 pos_class = pc, neg_class = neg_class, direction = direction, silent = TRUE)
             opt_ind_oob <- get_opt_ind(roc_curve = roc_curve_oob, oc = unlist(optcut_b$optimal_cutpoint), 
                 direction = direction)
             auc_oob <- auc(tpr = roc_curve_oob$tpr, fpr = roc_curve_oob$fpr)
             Sens_Spec_oob <- sesp_from_oc(roc_curve_oob, oc = unlist(optcut_b$optimal_cutpoint), 
                 direction = direction, opt_ind = opt_ind_oob)
             Acc_oob <- accuracy_from_oc(roc_curve_oob, oc = unlist(optcut_b$optimal_cutpoint), 
                 direction = direction, opt_ind = opt_ind_oob)[, "accuracy"]
             kap_oob <- kappa_from_oc(roc_curve_oob, oc = unlist(optcut_b$optimal_cutpoint), 
                 direction = direction, opt_ind = opt_ind_oob)
             metric_oob <- metric(tp = roc_curve_oob$tp[opt_ind_oob], fp = roc_curve_oob$fp[opt_ind_oob], 
                 tn = roc_curve_oob$tn[opt_ind_oob], fn = roc_curve_oob$fn[opt_ind_oob])
             metric_oob <- check_metric_name(metric_oob)
             mn <- make.names(colnames(metric_oob))
             bootstrap <- tibble::tibble(optimal_cutpoint = optcut_b$optimal_cutpoint, 
                 AUC_b = auc_b, AUC_oob = auc_oob)
             bootstrap <- add_list(bootstrap, metric_b[, mn], paste0(mn, "_b"))
             bootstrap <- add_list(bootstrap, metric_oob[, mn], paste0(mn, "_oob"))
             bootstrap <- add_list(bootstrap, Acc_b, "acc_b")
             bootstrap <- add_list(bootstrap, Acc_oob, "acc_oob")
             bootstrap <- add_list(bootstrap, Sens_Spec_b[, "sensitivity"], "sensitivity_b")
             bootstrap <- add_list(bootstrap, Sens_Spec_oob[, "sensitivity"], "sensitivity_oob")
             bootstrap <- add_list(bootstrap, Sens_Spec_b[, "specificity"], "specificity_b")
             bootstrap <- add_list(bootstrap, Sens_Spec_oob[, "specificity"], "specificity_oob")
             bootstrap <- add_list(bootstrap, kap_b[, "cohens_kappa"], "kappa_b")
             bootstrap <- add_list(bootstrap, kap_oob[, "cohens_kappa"], "kappa_oob")
             bootstrap <- add_list(bootstrap, optcut_b$roc_curve[[1]]$tp[opt_ind_b], "TP_b")
             bootstrap <- add_list(bootstrap, optcut_b$roc_curve[[1]]$fp[opt_ind_b], "FP_b")
             bootstrap <- add_list(bootstrap, optcut_b$roc_curve[[1]]$tn[opt_ind_b], "TN_b")
             bootstrap <- add_list(bootstrap, optcut_b$roc_curve[[1]]$fn[opt_ind_b], "FN_b")
             bootstrap <- add_list(bootstrap, roc_curve_oob$tp[opt_ind_oob], "TP_oob")
             bootstrap <- add_list(bootstrap, roc_curve_oob$fp[opt_ind_oob], "FP_oob")
             bootstrap <- add_list(bootstrap, roc_curve_oob$tn[opt_ind_oob], "TN_oob")
             bootstrap <- add_list(bootstrap, roc_curve_oob$fn[opt_ind_oob], "FN_oob")
             bootstrap$roc_curve_b = optcut_b$roc_curve
             roc_curve_oob <- tidyr::nest_(roc_curve_oob, key_col = "roc_curve_oob")
             bootstrap <- dplyr::bind_cols(bootstrap, roc_curve_oob)
             return(bootstrap)
         }
         return(boot_g)
     }))
  14: transmute(.data, !!!dots)
  15: transmute.default(.data, !!!dots)
  16: mutate(.data, !!!dots)
  17: mutate.tbl_df(.data, !!!dots)
  18: mutate_impl(.data, dots, caller_env())
  19: purrr::map2(dat$data, dat$pos_class, function(g, pc) {
         boot_g <- foreach::foreach(rep = 1:boot_runs, .combine = rbind, .export = c("method", 
             "direction", "metric", "break_ties", "neg_class", "mn", "use_midpoints", 
             "predictor", "outcome", "tol_metric")) %seq_or_par% {
             b_ind <- simple_boot(g, outcome)
             optcut_b <- method(data = g[b_ind, ], x = predictor, metric_func = metric, 
                 class = outcome, direction = direction, pos_class = pc, neg_class = neg_class, 
                 tol_metric = tol_metric, use_midpoints = use_midpoints, ...)
             optcut_b <- check_method_cols(optcut_b)
             optcut_b <- tibble::as_tibble(optcut_b)
             optcut_b <- apply_break_ties(optcut_b, break_ties)
             if (!(has_column(optcut_b, "roc_curve"))) {
                 roc_curve_b <- roc(data = g[b_ind, ], x = predictor, class = outcome, 
                     pos_class = pc, neg_class = neg_class, direction = direction, silent = TRUE)
                 roc_curve_b <- tidyr::nest_(roc_curve_b, key_col = "roc_curve")
                 optcut_b <- dplyr::bind_cols(optcut_b, roc_curve_b)
             }
             opt_ind_b <- get_opt_ind(roc_curve = optcut_b$roc_curve[[1]], oc = unlist(optcut_b$optimal_cutpoint), 
                 direction = direction)
             auc_b <- auc(tpr = optcut_b$roc_curve[[1]]$tpr, fpr = optcut_b$roc_curve[[1]]$fpr)
             Sens_Spec_b <- sesp_from_oc(optcut_b$roc_curve[[1]], oc = unlist(optcut_b$optimal_cutpoint), 
                 direction = direction, opt_ind = opt_ind_b)
             Acc_b <- accuracy_from_oc(optcut_b$roc_curve[[1]], oc = unlist(optcut_b$optimal_cutpoint), 
                 direction = direction, opt_ind = opt_ind_b)[, "accuracy"]
             kap_b <- kappa_from_oc(optcut_b$roc_curve[[1]], oc = unlist(optcut_b$optimal_cutpoint), 
                 direction = direction, opt_ind = opt_ind_b)
             metric_b <- metric(tp = optcut_b$roc_curve[[1]]$tp[opt_ind_b], fp = optcut_b$roc_curve[[1]]$fp[opt_ind_b], 
                 tn = optcut_b$roc_curve[[1]]$tn[opt_ind_b], fn = optcut_b$roc_curve[[1]]$fn[opt_ind_b])
             metric_b <- check_metric_name(metric_b)
             roc_curve_oob <- roc(data = g[-b_ind, ], x = predictor, class = outcome, 
                 pos_class = pc, neg_class = neg_class, direction = direction, silent = TRUE)
             opt_ind_oob <- get_opt_ind(roc_curve = roc_curve_oob, oc = unlist(optcut_b$optimal_cutpoint), 
                 direction = direction)
             auc_oob <- auc(tpr = roc_curve_oob$tpr, fpr = roc_curve_oob$fpr)
             Sens_Spec_oob <- sesp_from_oc(roc_curve_oob, oc = unlist(optcut_b$optimal_cutpoint), 
                 direction = direction, opt_ind = opt_ind_oob)
             Acc_oob <- accuracy_from_oc(roc_curve_oob, oc = unlist(optcut_b$optimal_cutpoint), 
                 direction = direction, opt_ind = opt_ind_oob)[, "accuracy"]
             kap_oob <- kappa_from_oc(roc_curve_oob, oc = unlist(optcut_b$optimal_cutpoint), 
                 direction = direction, opt_ind = opt_ind_oob)
             metric_oob <- metric(tp = roc_curve_oob$tp[opt_ind_oob], fp = roc_curve_oob$fp[opt_ind_oob], 
                 tn = roc_curve_oob$tn[opt_ind_oob], fn = roc_curve_oob$fn[opt_ind_oob])
             metric_oob <- check_metric_name(metric_oob)
             mn <- make.names(colnames(metric_oob))
             bootstrap <- tibble::tibble(optimal_cutpoint = optcut_b$optimal_cutpoint, 
                 AUC_b = auc_b, AUC_oob = auc_oob)
             bootstrap <- add_list(bootstrap, metric_b[, mn], paste0(mn, "_b"))
             bootstrap <- add_list(bootstrap, metric_oob[, mn], paste0(mn, "_oob"))
             bootstrap <- add_list(bootstrap, Acc_b, "acc_b")
             bootstrap <- add_list(bootstrap, Acc_oob, "acc_oob")
             bootstrap <- add_list(bootstrap, Sens_Spec_b[, "sensitivity"], "sensitivity_b")
             bootstrap <- add_list(bootstrap, Sens_Spec_oob[, "sensitivity"], "sensitivity_oob")
             bootstrap <- add_list(bootstrap, Sens_Spec_b[, "specificity"], "specificity_b")
             bootstrap <- add_list(bootstrap, Sens_Spec_oob[, "specificity"], "specificity_oob")
             bootstrap <- add_list(bootstrap, kap_b[, "cohens_kappa"], "kappa_b")
             bootstrap <- add_list(bootstrap, kap_oob[, "cohens_kappa"], "kappa_oob")
             bootstrap <- add_list(bootstrap, optcut_b$roc_curve[[1]]$tp[opt_ind_b], "TP_b")
             bootstrap <- add_list(bootstrap, optcut_b$roc_curve[[1]]$fp[opt_ind_b], "FP_b")
             bootstrap <- add_list(bootstrap, optcut_b$roc_curve[[1]]$tn[opt_ind_b], "TN_b")
             bootstrap <- add_list(bootstrap, optcut_b$roc_curve[[1]]$fn[opt_ind_b], "FN_b")
             bootstrap <- add_list(bootstrap, roc_curve_oob$tp[opt_ind_oob], "TP_oob")
             bootstrap <- add_list(bootstrap, roc_curve_oob$fp[opt_ind_oob], "FP_oob")
             bootstrap <- add_list(bootstrap, roc_curve_oob$tn[opt_ind_oob], "TN_oob")
             bootstrap <- add_list(bootstrap, roc_curve_oob$fn[opt_ind_oob], "FN_oob")
             bootstrap$roc_curve_b = optcut_b$roc_curve
             roc_curve_oob <- tidyr::nest_(roc_curve_oob, key_col = "roc_curve_oob")
             bootstrap <- dplyr::bind_cols(bootstrap, roc_curve_oob)
             return(bootstrap)
         }
         return(boot_g)
     })
  20: .f(.x[[i]], .y[[i]], ...)
  21: foreach::foreach(rep = 1:boot_runs, .combine = rbind, .export = c("method", "direction", 
         "metric", "break_ties", "neg_class", "mn", "use_midpoints", "predictor", "outcome", 
         "tol_metric")) %seq_or_par% {
         b_ind <- simple_boot(g, outcome)
         optcut_b <- method(data = g[b_ind, ], x = predictor, metric_func = metric, class = outcome, 
             direction = direction, pos_class = pc, neg_class = neg_class, tol_metric = tol_metric, 
             use_midpoints = use_midpoints, ...)
         optcut_b <- check_method_cols(optcut_b)
         optcut_b <- tibble::as_tibble(optcut_b)
         optcut_b <- apply_break_ties(optcut_b, break_ties)
         if (!(has_column(optcut_b, "roc_curve"))) {
             roc_curve_b <- roc(data = g[b_ind, ], x = predictor, class = outcome, pos_class = pc, 
                 neg_class = neg_class, direction = direction, silent = TRUE)
             roc_curve_b <- tidyr::nest_(roc_curve_b, key_col = "roc_curve")
             optcut_b <- dplyr::bind_cols(optcut_b, roc_curve_b)
         }
         opt_ind_b <- get_opt_ind(roc_curve = optcut_b$roc_curve[[1]], oc = unlist(optcut_b$optimal_cutpoint), 
             direction = direction)
         auc_b <- auc(tpr = optcut_b$roc_curve[[1]]$tpr, fpr = optcut_b$roc_curve[[1]]$fpr)
         Sens_Spec_b <- sesp_from_oc(optcut_b$roc_curve[[1]], oc = unlist(optcut_b$optimal_cutpoint), 
             direction = direction, opt_ind = opt_ind_b)
         Acc_b <- accuracy_from_oc(optcut_b$roc_curve[[1]], oc = unlist(optcut_b$optimal_cutpoint), 
             direction = direction, opt_ind = opt_ind_b)[, "accuracy"]
         kap_b <- kappa_from_oc(optcut_b$roc_curve[[1]], oc = unlist(optcut_b$optimal_cutpoint), 
             direction = direction, opt_ind = opt_ind_b)
         metric_b <- metric(tp = optcut_b$roc_curve[[1]]$tp[opt_ind_b], fp = optcut_b$roc_curve[[1]]$fp[opt_ind_b], 
             tn = optcut_b$roc_curve[[1]]$tn[opt_ind_b], fn = optcut_b$roc_curve[[1]]$fn[opt_ind_b])
         metric_b <- check_metric_name(metric_b)
         roc_curve_oob <- roc(data = g[-b_ind, ], x = predictor, class = outcome, pos_class = pc, 
             neg_class = neg_class, direction = direction, silent = TRUE)
         opt_ind_oob <- get_opt_ind(roc_curve = roc_curve_oob, oc = unlist(optcut_b$optimal_cutpoint), 
             direction = direction)
         auc_oob <- auc(tpr = roc_curve_oob$tpr, fpr = roc_curve_oob$fpr)
         Sens_Spec_oob <- sesp_from_oc(roc_curve_oob, oc = unlist(optcut_b$optimal_cutpoint), 
             direction = direction, opt_ind = opt_ind_oob)
         Acc_oob <- accuracy_from_oc(roc_curve_oob, oc = unlist(optcut_b$optimal_cutpoint), 
             direction = direction, opt_ind = opt_ind_oob)[, "accuracy"]
         kap_oob <- kappa_from_oc(roc_curve_oob, oc = unlist(optcut_b$optimal_cutpoint), 
             direction = direction, opt_ind = opt_ind_oob)
         metric_oob <- metric(tp = roc_curve_oob$tp[opt_ind_oob], fp = roc_curve_oob$fp[opt_ind_oob], 
             tn = roc_curve_oob$tn[opt_ind_oob], fn = roc_curve_oob$fn[opt_ind_oob])
         metric_oob <- check_metric_name(metric_oob)
         mn <- make.names(colnames(metric_oob))
         bootstrap <- tibble::tibble(optimal_cutpoint = optcut_b$optimal_cutpoint, AUC_b = auc_b, 
             AUC_oob = auc_oob)
         bootstrap <- add_list(bootstrap, metric_b[, mn], paste0(mn, "_b"))
         bootstrap <- add_list(bootstrap, metric_oob[, mn], paste0(mn, "_oob"))
         bootstrap <- add_list(bootstrap, Acc_b, "acc_b")
         bootstrap <- add_list(bootstrap, Acc_oob, "acc_oob")
         bootstrap <- add_list(bootstrap, Sens_Spec_b[, "sensitivity"], "sensitivity_b")
         bootstrap <- add_list(bootstrap, Sens_Spec_oob[, "sensitivity"], "sensitivity_oob")
         bootstrap <- add_list(bootstrap, Sens_Spec_b[, "specificity"], "specificity_b")
         bootstrap <- add_list(bootstrap, Sens_Spec_oob[, "specificity"], "specificity_oob")
         bootstrap <- add_list(bootstrap, kap_b[, "cohens_kappa"], "kappa_b")
         bootstrap <- add_list(bootstrap, kap_oob[, "cohens_kappa"], "kappa_oob")
         bootstrap <- add_list(bootstrap, optcut_b$roc_curve[[1]]$tp[opt_ind_b], "TP_b")
         bootstrap <- add_list(bootstrap, optcut_b$roc_curve[[1]]$fp[opt_ind_b], "FP_b")
         bootstrap <- add_list(bootstrap, optcut_b$roc_curve[[1]]$tn[opt_ind_b], "TN_b")
         bootstrap <- add_list(bootstrap, optcut_b$roc_curve[[1]]$fn[opt_ind_b], "FN_b")
         bootstrap <- add_list(bootstrap, roc_curve_oob$tp[opt_ind_oob], "TP_oob")
         bootstrap <- add_list(bootstrap, roc_curve_oob$fp[opt_ind_oob], "FP_oob")
         bootstrap <- add_list(bootstrap, roc_curve_oob$tn[opt_ind_oob], "TN_oob")
         bootstrap <- add_list(bootstrap, roc_curve_oob$fn[opt_ind_oob], "FN_oob")
         bootstrap$roc_curve_b = optcut_b$roc_curve
         roc_curve_oob <- tidyr::nest_(roc_curve_oob, key_col = "roc_curve_oob")
         bootstrap <- dplyr::bind_cols(bootstrap, roc_curve_oob)
         return(bootstrap)
     }
  22: e$fun(obj, substitute(ex), parent.frame(), e$data)
  
  -- 5. Failure: Results for p_chisquared are equal to results by OptimalCutpoints
  round(opt_cut_cp$optimal_cutpoint, 4) not equal to c(0.9335, 0.5676).
  2/2 mismatches (average diff: 0.44)
  [1] 0.431 - 0.933 == -0.502
  [2] 0.946 - 0.568 ==  0.379
  
  -- 6. Failure: Results for prod_sens_spec are equal to results by OptimalCutpoin
  round(opt_cut_cp$optimal_cutpoint, 4) not equal to c(0.7998, 0.6371).
  2/2 mismatches (average diff: 0.126)
  [1] 0.778 - 0.800 == -0.0215
  [2] 0.406 - 0.637 == -0.2314
  
  -- 7. Error: Results for abs_d_ppv_npv are equal to results by OptimalCutpoints 
  non-numeric argument to mathematical function
  1: expect_equal(round(opt_cut_cp$optimal_cutpoint, 4), c(0.2501, 0.6781)) at testthat/test-cutpointr.R:532
  2: quasi_label(enquo(object), label)
  3: eval_bare(get_expr(quo), get_env(quo))
  
  -- 8. Failure: Results for sum_ppv_npv are equal to results by OptimalCutpoints 
  round(opt_cut_cp$optimal_cutpoint, 4) not equal to c(-0.7339, 1.7835).
  2/2 mismatches (average diff: 0.932)
  [1] 1.10 - -0.734 == 1.8326
  [2] 1.81 -  1.784 == 0.0315
  
  -- 9. Failure: Results for prod_ppv_npv are equal to results by OptimalCutpoints
  round(opt_cut_cp$optimal_cutpoint, 4) not equal to c(-0.7339, 1.7835).
  2/2 mismatches (average diff: 1.47)
  [1] 1.099 - -0.734 ==  1.83
  [2] 0.678 -  1.784 == -1.11
  
  -- 10. Error: Results for accuracy are equal to results by OptimalCutpoints (@te
  non-numeric argument to mathematical function
  1: expect_equal(round(opt_cut_cp$optimal_cutpoint, 4), c(0.9771, 0.0744)) at testthat/test-cutpointr.R:583
  2: quasi_label(enquo(object), label)
  3: eval_bare(get_expr(quo), get_env(quo))
  
  -- 11. Failure: Results for misclassification_cost are equal to results by Optim
  round(opt_cut_cp$optimal_cutpoint, 4) not equal to c(-0.7498, -0.0824).
  2/2 mismatches (average diff: 0.342)
  [1] -0.297 - -0.7498 == 0.453
  [2]  0.148 - -0.0824 == 0.231
  
  -- 12. Failure: LOESS smoothing does not return warnings or errors (@test-cutpoi
  `suppressMessages(...)` produced warnings.
  
  -- 13. Failure: LOESS smoothing does not return warnings or errors (@test-cutpoi
  round(cp$optimal_cutpoint, 2) not equal to c(-1.29, 1.01).
  2/2 mismatches (average diff: 2.71)
  [1]  0.87 - -1.29 ==  2.16
  [2] -2.25 -  1.01 == -3.26
  
  -- 14. Failure: bootstrapped cutpoints lead to plausible results (@test-cutpoint
  cp$optimal_cutpoint not equal to 1.9.
  1/1 mismatches
  [1] 2.1 - 1.9 == 0.2
  
  -- 15. Failure: bootstrapped cutpoints lead to plausible results (@test-cutpoint
  cp$optimal_cutpoint not equal to c(2.2, 3.2).
  2/2 mismatches (average diff: 0.25)
  [1] 2.1 - 2.2 == -0.1
  [2] 3.6 - 3.2 ==  0.4
  
  -- 16. Failure: bootstrapped cutpoints lead to plausible results (@test-cutpoint
  cp$optimal_cutpoint not equal to c(2.1, 2.2).
  2/2 mismatches (average diff: 0.332)
  [1] 2.60 - 2.1 == 0.500
  [2] 2.36 - 2.2 == 0.164
  
  -- 17. Failure: summary is printed correctly (@test-cutpointr.R#1179)  ---------
  `print\(scp\)` does not match "accuracy_oob 0.8342".
  Actual value: "Method: oc_youden_kernel \\nPredictor: dsi \\nOutcome: suicide \\nDirection: >= \\nNr\. of bootstraps: 10 \\n\\n optimal_cutpoint accuracy    acc sensitivity specificity    AUC n_pos n_neg\\n           1\.1742   0\.8647 0\.8647      0\.8889      0\.8629 0\.9238    36   496\\n\\nCutpoint 1\.17422055119795:\\n          observation\\nprediction yes  no\\n       yes  32  68\\n       no    4 428\\n\\n\\nPredictor summary: \\n Min\. 5% 1st Qu\. Median   Mean 3rd Qu\. 95% Max\.     SD\\n    0  0       0      0 0\.9211       1   5   11 1\.8527\\n\\nPredictor summary per class: \\n    Min\.   5% 1st Qu\. Median   Mean 3rd Qu\.  95% Max     SD\\nno     0 0\.00       0      0 0\.6331       0 4\.00  10 1\.4122\\nyes    0 0\.75       4      5 4\.8889       6 9\.25  11 2\.5498\\n\\nBootstrap summary: \\n# A tibble: 13 x 10\\n   Variable       Min\.  `5%` `1st Qu\.` Median  Mean `3rd Qu\.` `95%`  Max\.     SD\\n   <chr>         <dbl> <dbl>     <dbl>  <dbl> <dbl>     <dbl> <dbl> <dbl>  <dbl>\\n 1 optimal_cutp~ 1\.16  1\.16      1\.16   1\.16  1\.26      1\.19  1\.70  2\.10  0\.294 \\n 2 AUC_b         0\.832 0\.866     0\.916  0\.927 0\.920     0\.936 0\.951 0\.959 0\.0341\\n 3 AUC_oob       0\.835 0\.856     0\.887  0\.921 0\.917     0\.952 0\.963 0\.965 0\.0422\\n 4 accuracy_b    0\.838 0\.84      0\.848  0\.862 0\.862     0\.874 0\.886 0\.887 0\.0175\\n 5 accuracy_oob  0\.852 0\.852     0\.864  0\.872 0\.874     0\.879 0\.899 0\.908 0\.0169\\n 6 acc_b         0\.838 0\.84      0\.848  0\.862 0\.862     0\.874 0\.886 0\.887 0\.0175\\n 7 acc_oob       0\.852 0\.852     0\.864  0\.872 0\.874     0\.879 0\.899 0\.908 0\.0169\\n 8 sensitivity_b 0\.788 0\.811     0\.842  0\.882 0\.884     0\.918 0\.965 0\.971 0\.0575\\n 9 sensitivity_~ 0\.6   0\.63      0\.791  0\.903 0\.851     0\.935 0\.974 1     0\.130 \\n10 specificity_b 0\.835 0\.837     0\.844  0\.858 0\.860     0\.875 0\.885 0\.886 0\.0189\\n11 specificity_~ 0\.844 0\.845     0\.857  0\.877 0\.874     0\.890 0\.899 0\.903 0\.0206\\n12 kappa_b       0\.328 0\.340     0\.367  0\.400 0\.403     0\.435 0\.471 0\.480 0\.0488\\n13 kappa_oob     0\.280 0\.284     0\.374  0\.412 0\.403     0\.461 0\.483 0\.488 0\.073 "
  
  -- 18. Failure: summary is printed correctly (@test-cutpointr.R#1195)  ---------
  `print\(scp\)` does not match "accuracy_oob 0.8201".
  Actual value: "Method: oc_youden_normal \\nPredictor: dsi \\nOutcome: suicide \\nDirection: >= \\nSubgroups: female, male \\nNr\. of bootstraps: 10 \\n\\nSubgroup: female \\n-------------------------------------------------------------------------------- \\n optimal_cutpoint accuracy    acc sensitivity specificity    AUC n_pos n_neg\\n           2\.4778   0\.8954 0\.8954      0\.8148      0\.9014 0\.9446    27   365\\n\\nCutpoint 2\.47775393352595:\\n          observation\\nprediction yes  no\\n       yes  22  36\\n       no    5 329\\n\\n\\nPredictor summary: \\n Min\. 5% 1st Qu\. Median   Mean 3rd Qu\. 95% Max\.     SD\\n    0  0       0      0 0\.8393       1   5   10 1\.7452\\n\\nPredictor summary per class: \\n    Min\.  5% 1st Qu\. Median   Mean 3rd Qu\. 95% Max     SD\\nno     0 0\.0       0      0 0\.5479       0   4  10 1\.3181\\nyes    0 1\.3       4      5 4\.7778       6   7   9 2\.0444\\n\\nBootstrap summary: \\n# A tibble: 13 x 10\\n   Variable       Min\.  `5%` `1st Qu\.` Median  Mean `3rd Qu\.` `95%`  Max\.     SD\\n   <chr>         <dbl> <dbl>     <dbl>  <dbl> <dbl>     <dbl> <dbl> <dbl>  <dbl>\\n 1 optimal_cutp~ 2\.16  2\.24      2\.34   2\.46  2\.46      2\.56  2\.69  2\.76  0\.173 \\n 2 AUC_b         0\.890 0\.900     0\.928  0\.955 0\.946     0\.966 0\.979 0\.979 0\.0294\\n 3 AUC_oob       0\.905 0\.906     0\.909  0\.955 0\.944     0\.974 0\.980 0\.980 0\.0334\\n 4 accuracy_b    0\.867 0\.870     0\.887  0\.904 0\.898     0\.913 0\.913 0\.913 0\.0176\\n 5 accuracy_oob  0\.867 0\.868     0\.875  0\.893 0\.891     0\.904 0\.910 0\.913 0\.0167\\n 6 acc_b         0\.867 0\.870     0\.887  0\.904 0\.898     0\.913 0\.913 0\.913 0\.0176\\n 7 acc_oob       0\.867 0\.868     0\.875  0\.893 0\.891     0\.904 0\.910 0\.913 0\.0167\\n 8 sensitivity_b 0\.714 0\.730     0\.766  0\.807 0\.811     0\.824 0\.926 0\.952 0\.0699\\n 9 sensitivity_~ 0\.692 0\.702     0\.778  0\.840 0\.833     0\.894 0\.962 1     0\.095 \\n10 specificity_b 0\.874 0\.879     0\.893  0\.912 0\.904     0\.914 0\.921 0\.921 0\.0162\\n11 specificity_~ 0\.877 0\.880     0\.886  0\.899 0\.896     0\.903 0\.913 0\.913 0\.0122\\n12 kappa_b       0\.313 0\.337     0\.395  0\.470 0\.450     0\.501 0\.534 0\.544 0\.075 \\n13 kappa_oob     0\.375 0\.377     0\.388  0\.438 0\.456     0\.502 0\.585 0\.602 0\.0803\\n\\nSubgroup: male \\n-------------------------------------------------------------------------------- \\n optimal_cutpoint accuracy    acc sensitivity specificity    AUC n_pos n_neg\\n           3\.1723   0\.8643 0\.8643      0\.6667      0\.8779 0\.8617     9   131\\n\\nCutpoint 3\.17225507835137:\\n          observation\\nprediction yes  no\\n       yes   6  16\\n       no    3 115\\n\\n\\nPredictor summary: \\n Min\. 5% 1st Qu\. Median Mean 3rd Qu\. 95% Max\.     SD\\n    0  0       0      0 1\.15       1   6   11 2\.1151\\n\\nPredictor summary per class: \\n    Min\.  5% 1st Qu\. Median   Mean 3rd Qu\.  95% Max     SD\\nno     0 0\.0       0      0 0\.8702       1  5\.0   6 1\.6286\\nyes    0 0\.4       3      4 5\.2222       8 10\.6  11 3\.8333\\n\\nBootstrap summary: \\n# A tibble: 13 x 10\\n   Variable       Min\.  `5%` `1st Qu\.` Median  Mean `3rd Qu\.` `95%`  Max\.     SD\\n   <chr>         <dbl> <dbl>     <dbl>  <dbl> <dbl>     <dbl> <dbl> <dbl>  <dbl>\\n 1 optimal_cutp~ 2\.63  2\.73      3\.13   3\.25  3\.32      3\.61  3\.88  3\.94  0\.411 \\n 2 AUC_b         0\.718 0\.763     0\.849  0\.885 0\.883     0\.938 0\.980 0\.986 0\.0799\\n 3 AUC_oob       0\.628 0\.640     0\.795  0\.864 0\.846     0\.949 0\.985 0\.997 0\.129 \\n 4 accuracy_b    0\.8   0\.803     0\.818  0\.843 0\.842     0\.861 0\.886 0\.893 0\.0306\\n 5 accuracy_oob  0\.787 0\.807     0\.837  0\.875 0\.869     0\.906 0\.916 0\.918 0\.0436\\n 6 acc_b         0\.8   0\.803     0\.818  0\.843 0\.842     0\.861 0\.886 0\.893 0\.0306\\n 7 acc_oob       0\.787 0\.807     0\.837  0\.875 0\.869     0\.906 0\.916 0\.918 0\.0436\\n 8 sensitivity_b 0\.538 0\.577     0\.691  0\.739 0\.779     0\.885 1     1     0\.156 \\n 9 sensitivity_~ 0\.333 0\.363     0\.5    0\.5   0\.623     0\.875 1     1     0\.266 \\n10 specificity_b 0\.814 0\.815     0\.825  0\.854 0\.851     0\.871 0\.884 0\.889 0\.027 \\n11 specificity_~ 0\.8   0\.818     0\.865  0\.902 0\.888     0\.915 0\.937 0\.938 0\.0439\\n12 kappa_b       0\.238 0\.243     0\.273  0\.343 0\.327     0\.363 0\.404 0\.432 0\.0621\\n13 kappa_oob     0\.103 0\.106     0\.218  0\.293 0\.28      0\.356 0\.432 0\.465 0\.117 "
  
  -- 19. Failure: summary is printed correctly (@test-cutpointr.R#1211)  ---------
  `print\(scp\)` does not match "accuracy_oob 0.8163".
  Actual value: "Method: oc_youden_normal \\nPredictor: x \\nOutcome: class \\nDirection: >= \\nSubgroups: female, male \\nNr\. of bootstraps: 10 \\n\\nSubgroup: female \\n-------------------------------------------------------------------------------- \\n optimal_cutpoint accuracy    acc sensitivity specificity    AUC n_pos n_neg\\n           2\.4778   0\.8954 0\.8954      0\.8148      0\.9014 0\.9446    27   365\\n\\nCutpoint 2\.47775393352595:\\n          observation\\nprediction yes  no\\n       yes  22  36\\n       no    5 329\\n\\n\\nPredictor summary: \\n Min\. 5% 1st Qu\. Median   Mean 3rd Qu\. 95% Max\.     SD\\n    0  0       0      0 0\.8393       1   5   10 1\.7452\\n\\nPredictor summary per class: \\n    Min\.  5% 1st Qu\. Median   Mean 3rd Qu\. 95% Max     SD\\nno     0 0\.0       0      0 0\.5479       0   4  10 1\.3181\\nyes    0 1\.3       4      5 4\.7778       6   7   9 2\.0444\\n\\nBootstrap summary: \\n# A tibble: 13 x 10\\n   Variable       Min\.  `5%` `1st Qu\.` Median  Mean `3rd Qu\.` `95%`  Max\.     SD\\n   <chr>         <dbl> <dbl>     <dbl>  <dbl> <dbl>     <dbl> <dbl> <dbl>  <dbl>\\n 1 optimal_cutp~ 2\.29  2\.30      2\.39   2\.46  2\.48      2\.61  2\.65  2\.66  0\.137 \\n 2 AUC_b         0\.910 0\.911     0\.930  0\.948 0\.942     0\.954 0\.966 0\.968 0\.0202\\n 3 AUC_oob       0\.870 0\.894     0\.928  0\.945 0\.941     0\.959 0\.977 0\.978 0\.0314\\n 4 accuracy_b    0\.847 0\.856     0\.882  0\.898 0\.890     0\.902 0\.908 0\.911 0\.0201\\n 5 accuracy_oob  0\.878 0\.878     0\.883  0\.884 0\.894     0\.901 0\.925 0\.931 0\.0184\\n 6 acc_b         0\.847 0\.856     0\.882  0\.898 0\.890     0\.902 0\.908 0\.911 0\.0201\\n 7 acc_oob       0\.878 0\.878     0\.883  0\.884 0\.894     0\.901 0\.925 0\.931 0\.0184\\n 8 sensitivity_b 0\.667 0\.674     0\.744  0\.810 0\.787     0\.842 0\.864 0\.875 0\.0725\\n 9 sensitivity_~ 0\.625 0\.644     0\.778  0\.826 0\.807     0\.878 0\.917 0\.917 0\.0987\\n10 specificity_b 0\.857 0\.866     0\.886  0\.904 0\.897     0\.908 0\.917 0\.918 0\.0196\\n11 specificity_~ 0\.874 0\.877     0\.888  0\.897 0\.899     0\.904 0\.931 0\.941 0\.0194\\n12 kappa_b       0\.258 0\.31      0\.404  0\.448 0\.436     0\.493 0\.522 0\.523 0\.0802\\n13 kappa_oob     0\.287 0\.313     0\.439  0\.486 0\.467     0\.504 0\.590 0\.624 0\.0963\\n\\nSubgroup: male \\n-------------------------------------------------------------------------------- \\n optimal_cutpoint accuracy    acc sensitivity specificity    AUC n_pos n_neg\\n           3\.1723   0\.8643 0\.8643      0\.6667      0\.8779 0\.8617     9   131\\n\\nCutpoint 3\.17225507835137:\\n          observation\\nprediction yes  no\\n       yes   6  16\\n       no    3 115\\n\\n\\nPredictor summary: \\n Min\. 5% 1st Qu\. Median Mean 3rd Qu\. 95% Max\.     SD\\n    0  0       0      0 1\.15       1   6   11 2\.1151\\n\\nPredictor summary per class: \\n    Min\.  5% 1st Qu\. Median   Mean 3rd Qu\.  95% Max     SD\\nno     0 0\.0       0      0 0\.8702       1  5\.0   6 1\.6286\\nyes    0 0\.4       3      4 5\.2222       8 10\.6  11 3\.8333\\n\\nBootstrap summary: \\n# A tibble: 13 x 10\\n   Variable      Min\.   `5%` `1st Qu\.` Median  Mean `3rd Qu\.` `95%`  Max\.     SD\\n   <chr>        <dbl>  <dbl>     <dbl>  <dbl> <dbl>     <dbl> <dbl> <dbl>  <dbl>\\n 1 optimal_cu~  2\.31  2\.60       2\.99   3\.16  3\.40      3\.45  4\.86  5\.92  0\.956 \\n 2 AUC_b        0\.748 0\.778      0\.874  0\.893 0\.893     0\.944 0\.975 1     0\.0723\\n 3 AUC_oob      0\.333 0\.494      0\.739  0\.891 0\.818     0\.949 0\.990 0\.990 0\.201 \\n 4 accuracy_b   0\.829 0\.841      0\.859  0\.875 0\.883     0\.9   0\.939 0\.964 0\.0377\\n 5 accuracy_o~  0\.78  0\.789      0\.807  0\.841 0\.844     0\.875 0\.914 0\.922 0\.0466\\n 6 acc_b        0\.829 0\.841      0\.859  0\.875 0\.883     0\.9   0\.939 0\.964 0\.0377\\n 7 acc_oob      0\.78  0\.789      0\.807  0\.841 0\.844     0\.875 0\.914 0\.922 0\.0466\\n 8 sensitivit~  0\.583 0\.591      0\.673  0\.739 0\.734     0\.776 0\.9   1     0\.117 \\n 9 sensitivit~  0     0\.075      0\.375  0\.775 0\.655     1     1     1     0\.380 \\n10 specificit~  0\.852 0\.858      0\.868  0\.891 0\.895     0\.916 0\.944 0\.963 0\.0341\\n11 specificit~  0\.788 0\.796      0\.821  0\.848 0\.860     0\.888 0\.959 0\.978 0\.06  \\n12 kappa_b      0\.184 0\.229      0\.371  0\.450 0\.418     0\.480 0\.558 0\.600 0\.119 \\n13 kappa_oob   -0\.037 0\.0219     0\.170  0\.247 0\.260     0\.337 0\.527 0\.624 0\.183 "
  
  == testthat results  ===========================================================
  OK: 352 SKIPPED: 0 FAILED: 19
  1. Failure: Plotting with bootstrapping is silent (@test-cutpointr.R#109) 
  2. Failure: AUC calculation is correct and works with Inf and -Inf (@test-cutpointr.R#142) 
  3. Failure: AUC calculation is correct and works with Inf and -Inf (@test-cutpointr.R#144) 
  4. Error: Metric colnames that are already in cutpointr are modified (@test-cutpointr.R#303) 
  5. Failure: Results for p_chisquared are equal to results by OptimalCutpoints (@test-cutpointr.R#498) 
  6. Failure: Results for prod_sens_spec are equal to results by OptimalCutpoints (@test-cutpointr.R#515) 
  7. Error: Results for abs_d_ppv_npv are equal to results by OptimalCutpoints (@test-cutpointr.R#532) 
  8. Failure: Results for sum_ppv_npv are equal to results by OptimalCutpoints (@test-cutpointr.R#549) 
  9. Failure: Results for prod_ppv_npv are equal to results by OptimalCutpoints (@test-cutpointr.R#566) 
  1. ...
  
  Error: testthat unit tests failed
  Execution halted