See http://www.win-vector.com/blog/2014/02/bad-bayes-an-example-of-why-you-need-hold-out-testing/

library('WVPlots')
## Loading required package: ggplot2
## Loading required package: grid
## Loading required package: gridExtra
## Loading required package: reshape2
## Loading required package: ROCR
## Loading required package: gplots
## 
## Attaching package: 'gplots'
## 
## The following object is masked from 'package:stats':
## 
##     lowess
## 
## Loading required package: plyr
## Loading required package: stringr
# make an example data set with no meaningfull signal
mkData <- function(rows,features,ndup) {
  yValues <- factor(c('A','B'))
  xValues <- factor(c('a','b'))
  yData = sample(yValues,replace=TRUE,size=rows)
  d <- data.frame(y=yData,stringsAsFactors=FALSE)
  mkRandVar <- function(v) {
    sample(xValues,replace=TRUE,size=rows)
  }
  varNames <- paste('v',seq_len(features),sep='_')
  # don't use replicate as it turns factors back to strings 
  varValues <- data.frame(lapply(varNames,mkRandVar),
                          stringsAsFactors = FALSE)
  colnames(varValues) <- varNames
  d <- cbind(d,varValues)
  if(ndup>0) {
    d <- d[rep(seq_len(rows),ndup+1),,drop=FALSE]
  }
  list(d=d,varNames=varNames,xValues=xValues,yValues=yValues)
}

runExample <- function(rows,features,trainer,predictor,ndup=0,title) {
  print('********************')
  print(title)
  print(sys.call(0)) # print call and arguments
  defs <- mkData(rows,features,ndup)
  dTrain <- defs$d
  xValues <- defs$xValues
  yValues <- defs$yValues
  varNames <- defs$varNames
  dTest <- mkData(rows,features,ndup)$d
  model <- trainer(yName='y',varNames=varNames,yValues=yValues,
                   data=dTrain)
  dTrain$predScore <- predictor(model,newdata=dTrain,yValues=yValues)
  scoreThreshold <- median(dTrain$predScore)
  dTrain$pred <- ifelse(dTrain$predScore>=scoreThreshold,
                        as.character(yValues[[2]]),
                        as.character(yValues[[1]]))
  tabTrain <- table(truth=dTrain$y,
                    predict=dTrain$pred)
  print('train set results')
  print(tabTrain)
  if(length(unique(dTrain$pred))>1) {
     print(fisher.test(tabTrain))
     print(ROCPlot(dTrain,'predScore','y',title=paste('Train',title,'ROC plot')))
  }
  dTest$predScore <- predictor(model,newdata=dTest,yValues=yValues)
  dTest$pred <- ifelse(dTest$predScore>=scoreThreshold,
                       as.character(yValues[[2]]),
                       as.character(yValues[[1]]))
  tabTest <- table(truth=dTest$y,
                   predict=dTest$pred)
  print('hold-out test set results')
  print(tabTest)
  if(length(unique(dTest$pred))>1) {
     print(fisher.test(tabTest))
     print(ROCPlot(dTest,'predScore','y',title=paste('Test',title,'ROC plot')))
  }
  print('********************')
  list(tabTrain=tabTrain,tabTest=tabTest)
}
library(e1071)
set.seed(123525)   # make result more repeatable
res <- runExample(rows=800,features=400,title='Naive Bayes',
                  trainer=function(yName,varNames,yValues,data) {
                    formula <- as.formula(paste(yName,paste(varNames,collapse=' + '),
                                                sep=' ~ '))
                    naiveBayes(formula,data) 
                  },
                  predictor=function(model,newdata,yValues) { 
                    predict(model,newdata,type='raw')[,yValues[[2]],drop=TRUE]
                  }
)
## [1] "********************"
## [1] "Naive Bayes"
## runExample(rows = 800, features = 400, title = "Naive Bayes", 
##     trainer = function(yName, varNames, yValues, data) {
##         formula <- as.formula(paste(yName, paste(varNames, collapse = " + "), 
##             sep = " ~ "))
##         naiveBayes(formula, data)
##     }, predictor = function(model, newdata, yValues) {
##         predict(model, newdata, type = "raw")[, yValues[[2]], 
##             drop = TRUE]
##     })
## [1] "train set results"
##      predict
## truth   A   B
##     A 303  89
##     B  97 311
## 
##  Fisher's Exact Test for Count Data
## 
## data:  tabTrain
## p-value < 2.2e-16
## alternative hypothesis: true odds ratio is not equal to 1
## 95 percent confidence interval:
##   7.758333 15.368197
## sample estimates:
## odds ratio 
##   10.87203

## [1] "hold-out test set results"
##      predict
## truth   A   B
##     A 206 190
##     B 213 191
## 
##  Fisher's Exact Test for Count Data
## 
## data:  tabTest
## p-value = 0.8874
## alternative hypothesis: true odds ratio is not equal to 1
## 95 percent confidence interval:
##  0.7295008 1.2957197
## sample estimates:
## odds ratio 
##  0.9722492

## [1] "********************"
library(rpart)
res <- runExample(rows=800,features=400,title='Decision Tree',
   trainer=function(yName,varNames,yValues,data) {
     formula <- as.formula(paste(yName,paste(varNames,collapse=' + '),
        sep=' ~ '))
     rpart(formula,data) 
   },
   predictor=function(model,newdata,yValues) { 
      predict(model,newdata,type='prob')[,yValues[[2]],drop=TRUE]
   }
)
## [1] "********************"
## [1] "Decision Tree"
## runExample(rows = 800, features = 400, title = "Decision Tree", 
##     trainer = function(yName, varNames, yValues, data) {
##         formula <- as.formula(paste(yName, paste(varNames, collapse = " + "), 
##             sep = " ~ "))
##         rpart(formula, data)
##     }, predictor = function(model, newdata, yValues) {
##         predict(model, newdata, type = "prob")[, yValues[[2]], 
##             drop = TRUE]
##     })
## [1] "train set results"
##      predict
## truth   A   B
##     A 291 102
##     B  96 311
## 
##  Fisher's Exact Test for Count Data
## 
## data:  tabTrain
## p-value < 2.2e-16
## alternative hypothesis: true odds ratio is not equal to 1
## 95 percent confidence interval:
##   6.618877 12.913771
## sample estimates:
## odds ratio 
##   9.209628

## [1] "hold-out test set results"
##      predict
## truth   A   B
##     A 196 192
##     B 199 213
## 
##  Fisher's Exact Test for Count Data
## 
## data:  tabTest
## p-value = 0.5715
## alternative hypothesis: true odds ratio is not equal to 1
## 95 percent confidence interval:
##  0.8199783 1.4560350
## sample estimates:
## odds ratio 
##   1.092529