d <- readRDS(file='KDD2009preds.RDS') # from KDD2009Cross.Rmd
set.seed(25325)
treatedTrainP <- d$treatedTrainP
treatedTestP <- d$treatedTestP
WVPlots::ROCPlotPair(treatedTrainP,'gbm','randomForest','churn',TRUE,
                     'AUC comparison on train',
                     nrep=1000,
                     parallelCluster = cl)

testD <- WVPlots::ROCPlotPair(treatedTestP,'gbm','randomForest','churn',TRUE,
                     'AUC comparison on test',
                     nrep=1000,
                     parallelCluster = cl, returnScores = TRUE)
print(testD$plot)

statsPaired <- testD$aucsig
print(statsPaired$eFreq)  # fraction of time we saw reversal
## [1] 0.113
print(statsPaired$eValue) # Student probability of reversal
## [1] 0.1014348
statsPaired$observedScore1-statsPaired$observedScore2
## [1] 0.01208274
WVPlots::ShadedDensity(statsPaired$resampledScores, "diff", 0, 
                       title="paired differences in resampled test AUCs") +
   geom_vline(xintercept = statsPaired$observedScore1-statsPaired$observedScore2)