Boston data
library(MASS)
library(DT)
datatable(Boston, rownames = FALSE)
Split into train and test
set.seed(3416)
library(caret)
Loading required package: ggplot2
Loading required package: lattice
TRAIN <- createDataPartition(Boston$medv,
p = 0.75,
list = FALSE,
times = 1)
BostonTrain <- Boston[TRAIN, ]
BostonTest <- Boston[-TRAIN, ]
Pre-process the data
pp_BostonTrain <- preProcess(BostonTrain[, -14],
method = c("center", "scale", "BoxCox"))
pp_BostonTrain
Created from 381 samples and 13 variables
Pre-processing:
- Box-Cox transformation (11)
- centered (13)
- ignored (0)
- scaled (13)
Lambda estimates for Box-Cox transformation:
Min. 1st Qu. Median Mean 3rd Qu. Max.
-0.9000 -0.1500 0.3000 0.4455 0.9500 2.0000
BostonTrain_pp <- predict(pp_BostonTrain, newdata = BostonTrain)
datatable(BostonTrain_pp, rownames = FALSE)
#
pp_BostonTest <- preProcess(BostonTest[, -14],
method = c("center", "scale", "BoxCox"))
pp_BostonTest
Created from 125 samples and 13 variables
Pre-processing:
- Box-Cox transformation (11)
- centered (13)
- ignored (0)
- scaled (13)
Lambda estimates for Box-Cox transformation:
Min. 1st Qu. Median Mean 3rd Qu. Max.
-0.9000 -0.1500 0.1000 0.3636 0.7500 2.0000
BostonTest_pp <- predict(pp_BostonTest, newdata = BostonTest)
datatable(BostonTest_pp, rownames = FALSE)
Linear model
set.seed(123)
library(caret)
myControl <- trainControl(method = "cv", number = 5)
mod_lm <- train(medv ~ .,
data = BostonTrain_pp,
trControl = myControl,
method = "lm")
mod_lm$results$RMSE # Training RMSE
[1] 4.532099
Test RMSE
p <- predict(mod_lm, newdata = BostonTest_pp)
RMSE(BostonTest_pp$medv, p) # Test RMSE
[1] 4.525501
Linear model-Forward Selection
set.seed(123)
library(caret)
myControl <- trainControl(method = "cv", number = 5)
mod_fs <- train(medv ~ .,
data = BostonTrain_pp,
trControl = myControl,
method = "leapForward")
mod_fs$results$RMSE # Training RMSE
[1] 5.504753 5.253257 5.037208
mod_fs
Linear Regression with Forward Selection
381 samples
13 predictor
No pre-processing
Resampling: Cross-Validated (5 fold)
Summary of sample sizes: 306, 304, 304, 305, 305
Resampling results across tuning parameters:
nvmax RMSE Rsquared MAE
2 5.504753 0.6664465 4.156683
3 5.253257 0.6941767 3.878030
4 5.037208 0.7174721 3.759551
RMSE was used to select the optimal model using the smallest value.
The final value used for the model was nvmax = 4.
Test RMSE
p <- predict(mod_fs, newdata = BostonTest_pp)
RMSE(BostonTest_pp$medv, p) # Test RMSE
[1] 4.570235
Linear model-Backward Elimination
set.seed(123)
library(caret)
myControl <- trainControl(method = "cv", number = 5)
mod_be <- train(medv ~ .,
data = BostonTrain_pp,
trControl = myControl,
method = "leapBackward")
mod_be$results$RMSE # Training RMSE
[1] 5.419077 5.183117 4.923204
mod_be
Linear Regression with Backwards Selection
381 samples
13 predictor
No pre-processing
Resampling: Cross-Validated (5 fold)
Summary of sample sizes: 306, 304, 304, 305, 305
Resampling results across tuning parameters:
nvmax RMSE Rsquared MAE
2 5.419077 0.6765806 4.051136
3 5.183117 0.7025103 3.802048
4 4.923204 0.7306559 3.663932
RMSE was used to select the optimal model using the smallest value.
The final value used for the model was nvmax = 4.
Test RMSE
p <- predict(mod_be, newdata = BostonTest_pp)
RMSE(BostonTest_pp$medv, p) # Test RMSE
[1] 4.570235
Elasticnet
set.seed(123)
myControl <- trainControl(method = "cv", number = 5)
mod_glmnet <- train(medv ~ .,
data = BostonTrain_pp,
trControl = myControl,
method = "glmnet",
tuneLength = 12)
Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
There were missing values in resampled performance measures.
mod_glmnet
glmnet
381 samples
13 predictor
No pre-processing
Resampling: Cross-Validated (5 fold)
Summary of sample sizes: 306, 304, 304, 305, 305
Resampling results across tuning parameters:
alpha lambda RMSE Rsquared MAE
0.1000000 0.006268536 4.532439 0.7687279 3.366651
0.1000000 0.012730881 4.532439 0.7687279 3.366651
0.1000000 0.025855371 4.532434 0.7687289 3.366623
0.1000000 0.052510131 4.532340 0.7688297 3.359722
0.1000000 0.106643752 4.535395 0.7687633 3.346245
0.1000000 0.216584681 4.547595 0.7681490 3.325282
0.1000000 0.439865657 4.589139 0.7654739 3.307271
0.1000000 0.893330938 4.694705 0.7581140 3.320672
0.1000000 1.814281591 4.914227 0.7423998 3.396554
0.1000000 3.684656549 5.273332 0.7200617 3.587066
0.1000000 7.483234110 5.829543 0.6977248 3.974399
0.1818182 0.006268536 4.532654 0.7687016 3.368688
0.1818182 0.012730881 4.532654 0.7687016 3.368688
0.1818182 0.025855371 4.532432 0.7687406 3.367360
0.1818182 0.052510131 4.533602 0.7687487 3.358871
0.1818182 0.106643752 4.538685 0.7685477 3.344706
0.1818182 0.216584681 4.556259 0.7675645 3.326035
0.1818182 0.439865657 4.613467 0.7636242 3.318787
0.1818182 0.893330938 4.752037 0.7533679 3.346220
0.1818182 1.814281591 5.011448 0.7346916 3.470463
0.1818182 3.684656549 5.429657 0.7095588 3.722030
0.1818182 7.483234110 6.083102 0.7001301 4.189152
0.2636364 0.006268536 4.532905 0.7686922 3.368850
0.2636364 0.012730881 4.532905 0.7686922 3.368850
0.2636364 0.025855371 4.532862 0.7687195 3.366632
0.2636364 0.052510131 4.534693 0.7686882 3.357596
0.2636364 0.106643752 4.541546 0.7683807 3.343626
0.2636364 0.216584681 4.566515 0.7668266 3.328434
0.2636364 0.439865657 4.637342 0.7618277 3.329551
0.2636364 0.893330938 4.818059 0.7475144 3.387929
0.2636364 1.814281591 5.100450 0.7274659 3.542196
0.2636364 3.684656549 5.532147 0.7076851 3.821693
0.2636364 7.483234110 6.344984 0.7034900 4.394777
0.3454545 0.006268536 4.532998 0.7686950 3.368913
0.3454545 0.012730881 4.532998 0.7686950 3.368913
0.3454545 0.025855371 4.533374 0.7686898 3.366273
0.3454545 0.052510131 4.536039 0.7686047 3.356808
0.3454545 0.106643752 4.545000 0.7681604 3.343759
0.3454545 0.216584681 4.577897 0.7659637 3.331930
0.3454545 0.439865657 4.666327 0.7593644 3.341508
0.3454545 0.893330938 4.880764 0.7419235 3.430270
0.3454545 1.814281591 5.198336 0.7185241 3.619982
0.3454545 3.684656549 5.638643 0.7059721 3.919628
0.3454545 7.483234110 6.637428 0.6991054 4.628281
0.4272727 0.006268536 4.533413 0.7686297 3.369439
0.4272727 0.012730881 4.533413 0.7686297 3.369439
0.4272727 0.025855371 4.533958 0.7686509 3.365646
0.4272727 0.052510131 4.537495 0.7685109 3.355996
0.4272727 0.106643752 4.549024 0.7678880 3.344393
0.4272727 0.216584681 4.588161 0.7651822 3.333949
0.4272727 0.439865657 4.701477 0.7562118 3.357889
0.4272727 0.893330938 4.924674 0.7381924 3.465292
0.4272727 1.814281591 5.282171 0.7107997 3.694325
0.4272727 3.684656549 5.742911 0.7049116 4.009762
0.4272727 7.483234110 6.948048 0.6873032 4.877146
0.5090909 0.006268536 4.533625 0.7686237 3.369891
0.5090909 0.012730881 4.533625 0.7686237 3.369891
0.5090909 0.025855371 4.534373 0.7686321 3.365226
0.5090909 0.052510131 4.538761 0.7684363 3.355401
0.5090909 0.106643752 4.553503 0.7675701 3.345300
0.5090909 0.216584681 4.598849 0.7643755 3.337297
0.5090909 0.439865657 4.743445 0.7522809 3.379458
0.5090909 0.893330938 4.964161 0.7348584 3.499293
0.5090909 1.814281591 5.327524 0.7084565 3.744212
0.5090909 3.684656549 5.859680 0.7022780 4.097255
0.5090909 7.483234110 7.236792 0.6807820 5.094855
0.5909091 0.006268536 4.533709 0.7686471 3.369558
0.5909091 0.012730881 4.533703 0.7686489 3.369458
0.5909091 0.025855371 4.534981 0.7685927 3.364541
0.5909091 0.052510131 4.540154 0.7683524 3.354782
0.5909091 0.106643752 4.558549 0.7671995 3.346364
0.5909091 0.216584681 4.611191 0.7634038 3.341505
0.5909091 0.439865657 4.778706 0.7490872 3.399497
0.5909091 0.893330938 5.008661 0.7308346 3.541779
0.5909091 1.814281591 5.368626 0.7069940 3.783998
0.5909091 3.684656549 5.991761 0.6973993 4.201063
0.5909091 7.483234110 7.552228 0.6710425 5.325813
0.6727273 0.006268536 4.533560 0.7686403 3.370216
0.6727273 0.012730881 4.533676 0.7686343 3.369678
0.6727273 0.025855371 4.535553 0.7685605 3.363875
0.6727273 0.052510131 4.541567 0.7682674 3.354559
0.6727273 0.106643752 4.563678 0.7668108 3.347271
0.6727273 0.216584681 4.625375 0.7621938 3.345673
0.6727273 0.439865657 4.816237 0.7456108 3.423569
0.6727273 0.893330938 5.057995 0.7261480 3.585362
0.6727273 1.814281591 5.411921 0.7052613 3.820470
0.6727273 3.684656549 6.139067 0.6892940 4.316349
0.6727273 7.483234110 7.868986 0.6582170 5.568722
0.7545455 0.006268536 4.533341 0.7686565 3.370423
0.7545455 0.012730881 4.533745 0.7686322 3.369765
0.7545455 0.025855371 4.536082 0.7685275 3.363364
0.7545455 0.052510131 4.543215 0.7681604 3.354695
0.7545455 0.106643752 4.568998 0.7663909 3.348084
0.7545455 0.216584681 4.641303 0.7607902 3.351204
0.7545455 0.439865657 4.849905 0.7424856 3.446269
0.7545455 0.893330938 5.110714 0.7209550 3.628710
0.7545455 1.814281591 5.456445 0.7033818 3.854451
0.7545455 3.684656549 6.278055 0.6807482 4.422164
0.7545455 7.483234110 8.168997 0.6551503 5.800493
0.8363636 0.006268536 4.533531 0.7686379 3.370395
0.8363636 0.012730881 4.534076 0.7686039 3.369372
0.8363636 0.025855371 4.536829 0.7684784 3.362967
0.8363636 0.052510131 4.544980 0.7680436 3.354808
0.8363636 0.106643752 4.573321 0.7660719 3.347789
0.8363636 0.216584681 4.659257 0.7591660 3.359311
0.8363636 0.439865657 4.871195 0.7405887 3.464868
0.8363636 0.893330938 5.159761 0.7161013 3.667872
0.8363636 1.814281591 5.505150 0.7009574 3.889771
0.8363636 3.684656549 6.405747 0.6751334 4.506194
0.8363636 7.483234110 8.502518 0.6551503 6.061084
0.9181818 0.006268536 4.533313 0.7686425 3.370200
0.9181818 0.012730881 4.534173 0.7686003 3.369165
0.9181818 0.025855371 4.537409 0.7684461 3.362507
0.9181818 0.052510131 4.546884 0.7679146 3.354943
0.9181818 0.106643752 4.577870 0.7657321 3.347478
0.9181818 0.216584681 4.679381 0.7573027 3.368504
0.9181818 0.439865657 4.888749 0.7390579 3.483665
0.9181818 0.893330938 5.200906 0.7121487 3.698759
0.9181818 1.814281591 5.559250 0.6978243 3.928229
0.9181818 3.684656549 6.541536 0.6678327 4.594709
0.9181818 7.483234110 8.888861 0.6551503 6.370009
1.0000000 0.006268536 4.533643 0.7686382 3.371293
1.0000000 0.012730881 4.534478 0.7685793 3.368978
1.0000000 0.025855371 4.538183 0.7683940 3.362028
1.0000000 0.052510131 4.548905 0.7677757 3.355140
1.0000000 0.106643752 4.582861 0.7653509 3.347769
1.0000000 0.216584681 4.702243 0.7551334 3.380174
1.0000000 0.439865657 4.907375 0.7373982 3.503123
1.0000000 0.893330938 5.233937 0.7091310 3.722923
1.0000000 1.814281591 5.619076 0.6937479 3.972824
1.0000000 3.684656549 6.674007 0.6602592 4.687782
1.0000000 7.483234110 9.331357 0.6367301 6.725990
RMSE was used to select the optimal model using the smallest value.
The final values used for the model were alpha = 0.1 and lambda = 0.05251013.
min(mod_glmnet$results$RMSE) # Training RMSE
[1] 4.53234
plot(mod_glmnet)
Test RMSE
p <- predict(mod_glmnet, newdata = BostonTest_pp)
RMSE(BostonTest_pp$medv, p) # Test RMSE
[1] 4.477451
LASSO
set.seed(123)
myControl <- trainControl(method = "cv", number = 5)
mod_lasso <- train(medv ~ .,
data = BostonTrain_pp,
trControl = myControl,
method = "glmnet",
tuneGrid = expand.grid(alpha = 1, lambda = seq(.01, 2, length = 10))
)
mod_lasso
glmnet
381 samples
13 predictor
No pre-processing
Resampling: Cross-Validated (5 fold)
Summary of sample sizes: 306, 304, 304, 305, 305
Resampling results across tuning parameters:
lambda RMSE Rsquared MAE
0.0100000 4.533857 0.7686211 3.370604
0.2311111 4.723225 0.7532066 3.389616
0.4522222 4.915221 0.7367576 3.509206
0.6733333 5.080905 0.7223991 3.628188
0.8944444 5.234474 0.7090926 3.723285
1.1155556 5.321999 0.7042096 3.786443
1.3366667 5.403792 0.7014806 3.838754
1.5577778 5.496363 0.6983004 3.895305
1.7788889 5.601179 0.6944543 3.960799
2.0000000 5.717715 0.6895725 4.046256
Tuning parameter 'alpha' was held constant at a value of 1
RMSE was used to select the optimal model using the smallest value.
The final values used for the model were alpha = 1 and lambda = 0.01.
min(mod_lasso$results$RMSE) # Training RMSE
[1] 4.533857
plot(mod_lasso)
Test RMSE
p <- predict(mod_lasso, newdata = BostonTest_pp)
RMSE(BostonTest_pp$medv, p) # Test RMSE
[1] 4.50215
Ridge
set.seed(123)
myControl <- trainControl(method = "cv", number = 5)
mod_ridge <- train(medv ~ .,
data = BostonTrain_pp,
trControl = myControl,
method = "glmnet",
tuneGrid = expand.grid(alpha = 0, lambda = seq(.01, 2, length = 10))
)
mod_ridge
glmnet
381 samples
13 predictor
No pre-processing
Resampling: Cross-Validated (5 fold)
Summary of sample sizes: 306, 304, 304, 305, 305
Resampling results across tuning parameters:
lambda RMSE Rsquared MAE
0.0100000 4.614966 0.7636989 3.294877
0.2311111 4.614966 0.7636989 3.294877
0.4522222 4.614966 0.7636989 3.294877
0.6733333 4.614966 0.7636989 3.294877
0.8944444 4.638327 0.7621104 3.297334
1.1155556 4.676050 0.7595030 3.302091
1.3366667 4.713953 0.7568811 3.308025
1.5577778 4.751331 0.7543100 3.314761
1.7788889 4.787935 0.7518072 3.322928
2.0000000 4.823847 0.7493672 3.332524
Tuning parameter 'alpha' was held constant at a value of 0
RMSE was used to select the optimal model using the smallest value.
The final values used for the model were alpha = 0 and lambda = 0.6733333.
min(mod_ridge$results$RMSE) # Training RMSE
[1] 4.614966
plot(mod_ridge)
Test RMSE
p <- predict(mod_ridge, newdata = BostonTest_pp)
RMSE(BostonTest_pp$medv, p) # Test RMSE
[1] 4.203053
Recursive Partitioning (Trees)
library(rpart)
mod_tree <- rpart(medv ~., data = BostonTrain_pp)
mod_tree
n= 381
node), split, n, deviance, yval
* denotes terminal node
1) root 381 33992.1300 22.71365
2) rm< 0.9537213 330 14327.9400 20.14727
4) lstat>=0.4346317 134 2458.8170 15.11567
8) nox>=0.6116656 83 995.7316 13.03614
16) lstat>=0.9553247 45 324.6720 10.91333 *
17) lstat< 0.9553247 38 228.1350 15.55000 *
9) nox< 0.6116656 51 520.0200 18.50000 *
5) lstat< 0.4346317 196 6157.2980 23.58724
10) lstat>=-1.192108 178 3600.5860 22.67584
20) lstat>=-0.2367871 85 435.8631 20.72588 *
21) lstat< -0.2367871 93 2546.1260 24.45806
42) age< 0.7414282 86 1086.6980 23.69535 *
43) age>=0.7414282 7 794.7543 33.82857 *
11) lstat< -1.192108 18 946.7200 32.60000 *
3) rm>=0.9537213 51 3427.0600 39.31961
6) rm< 1.538942 24 736.3200 33.30000 *
7) rm>=1.538942 27 1048.0560 44.67037 *
library(partykit)
Loading required package: grid
Loading required package: libcoin
Loading required package: mvtnorm
plot(as.party(mod_tree))
rpart.plot::rpart.plot(mod_tree)
set.seed(123)
mod_TR <- train(medv ~ .,
data = BostonTrain_pp,
trControl = myControl,
method = "rpart",
tuneLength = 10)
Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, :
There were missing values in resampled performance measures.
mod_TR$bestTune
cp
1 0.003136858
mod_TR2 <- rpart(medv ~.,
data = BostonTrain_pp,
cp = mod_TR$bestTune)
rpart.plot::rpart.plot(mod_TR2)
mod_TR2
n= 381
node), split, n, deviance, yval
* denotes terminal node
1) root 381 33992.1300 22.71365
2) rm< 0.9537213 330 14327.9400 20.14727
4) lstat>=0.4346317 134 2458.8170 15.11567
8) nox>=0.6116656 83 995.7316 13.03614
16) lstat>=0.9553247 45 324.6720 10.91333 *
17) lstat< 0.9553247 38 228.1350 15.55000 *
9) nox< 0.6116656 51 520.0200 18.50000 *
5) lstat< 0.4346317 196 6157.2980 23.58724
10) lstat>=-1.192108 178 3600.5860 22.67584
20) lstat>=-0.2367871 85 435.8631 20.72588 *
21) lstat< -0.2367871 93 2546.1260 24.45806
42) age< 0.7414282 86 1086.6980 23.69535
84) rm< -0.1881725 25 168.9616 20.64400 *
85) rm>=-0.1881725 61 589.5715 24.94590 *
43) age>=0.7414282 7 794.7543 33.82857 *
11) lstat< -1.192108 18 946.7200 32.60000 *
3) rm>=0.9537213 51 3427.0600 39.31961
6) rm< 1.538942 24 736.3200 33.30000 *
7) rm>=1.538942 27 1048.0560 44.67037
14) ptratio>=-0.4488431 7 465.9686 38.88571 *
15) ptratio< -0.4488431 20 265.8695 46.69500 *
Test RMSE
p <- predict(mod_TR2, newdata = BostonTest_pp)
RMSE(BostonTest_pp$medv, p) # Test RMSE
[1] 4.352689
Bagging
set.seed(123)
myControl <- trainControl(method = "cv", number = 5)
mod_tb <- train(medv ~ .,
data = BostonTrain_pp,
trControl = myControl,
method = "treebag"
)
mod_tb
Bagged CART
381 samples
13 predictor
No pre-processing
Resampling: Cross-Validated (5 fold)
Summary of sample sizes: 306, 304, 304, 305, 305
Resampling results:
RMSE Rsquared MAE
4.276616 0.8018699 2.85511
min(mod_tb$results$RMSE) # Training RMSE
[1] 4.276616
Test RMSE
p <- predict(mod_tb, newdata = BostonTest_pp)
RMSE(BostonTest_pp$medv, p) # Test RMSE
[1] 3.557732
Random Forest
set.seed(123)
myControl <- trainControl(method = "cv", number = 5)
mod_rf <- train(medv ~ .,
data = BostonTrain_pp,
trControl = myControl,
method = "ranger",
tuneLength = 12)
mod_rf
Random Forest
381 samples
13 predictor
No pre-processing
Resampling: Cross-Validated (5 fold)
Summary of sample sizes: 306, 304, 304, 305, 305
Resampling results across tuning parameters:
mtry splitrule RMSE Rsquared MAE
2 variance 3.832880 0.8587448 2.518141
2 extratrees 4.377025 0.8255921 2.874423
3 variance 3.566681 0.8729650 2.347728
3 extratrees 3.916764 0.8552296 2.587714
4 variance 3.522161 0.8710633 2.338540
4 extratrees 3.754917 0.8644915 2.482833
5 variance 3.436735 0.8758546 2.274182
5 extratrees 3.640429 0.8698941 2.419017
6 variance 3.430548 0.8745581 2.278716
6 extratrees 3.562520 0.8728870 2.381780
7 variance 3.417798 0.8739443 2.275479
7 extratrees 3.510473 0.8747282 2.350010
8 variance 3.428135 0.8732117 2.282252
8 extratrees 3.487387 0.8758874 2.336196
9 variance 3.400881 0.8737789 2.291316
9 extratrees 3.491114 0.8740669 2.352886
10 variance 3.402442 0.8735321 2.286201
10 extratrees 3.450158 0.8759239 2.324320
11 variance 3.429615 0.8708211 2.309345
11 extratrees 3.452930 0.8752728 2.331822
12 variance 3.410061 0.8719447 2.321055
12 extratrees 3.496607 0.8715495 2.362804
13 variance 3.407644 0.8721424 2.313051
13 extratrees 3.425564 0.8760149 2.306613
Tuning parameter 'min.node.size' was held constant at a value of 5
RMSE was used to select the optimal model using the smallest value.
The final values used for the model were mtry = 9, splitrule = variance
and min.node.size = 5.
min(mod_rf$results$RMSE) # Training RMSE
[1] 3.400881
plot(mod_rf)
Test RMSE
p <- predict(mod_rf, newdata = BostonTest_pp)
RMSE(BostonTest_pp$medv, p) # Test RMSE
[1] 3.102119
Gradient Boosting
set.seed(123)
myControl <- trainControl(method = "cv", number = 5)
mod_gbm <- train(medv ~ .,
data = BostonTrain_pp,
trControl = myControl,
method = "gbm",
tuneLength = 20)
Iter TrainDeviance ValidDeviance StepSize Improve
1 75.6884 -nan 0.1000 14.2046
2 64.4558 -nan 0.1000 9.3659
3 55.6707 -nan 0.1000 8.2594
4 47.9617 -nan 0.1000 7.3983
5 42.5233 -nan 0.1000 5.2637
6 36.7843 -nan 0.1000 4.4107
7 32.2402 -nan 0.1000 3.8943
8 28.4577 -nan 0.1000 3.9754
9 25.6642 -nan 0.1000 2.2407
10 23.3743 -nan 0.1000 2.2380
20 11.2762 -nan 0.1000 0.5320
40 6.6221 -nan 0.1000 -0.0161
60 5.0413 -nan 0.1000 -0.0364
80 3.9552 -nan 0.1000 -0.0087
100 3.2304 -nan 0.1000 -0.0508
120 2.6122 -nan 0.1000 -0.0267
140 2.1447 -nan 0.1000 -0.0429
160 1.7561 -nan 0.1000 -0.0187
180 1.5514 -nan 0.1000 -0.0289
200 1.3145 -nan 0.1000 -0.0384
mod_gbm
Stochastic Gradient Boosting
381 samples
13 predictor
No pre-processing
Resampling: Cross-Validated (5 fold)
Summary of sample sizes: 306, 304, 304, 305, 305
Resampling results across tuning parameters:
interaction.depth n.trees RMSE Rsquared MAE
1 50 4.451152 0.7864359 3.011257
1 100 4.109568 0.8138500 2.775736
1 150 3.929309 0.8298694 2.680151
1 200 3.832390 0.8367381 2.627192
1 250 3.773408 0.8424542 2.608625
1 300 3.713891 0.8462146 2.587394
1 350 3.694013 0.8478988 2.584969
1 400 3.691708 0.8486147 2.596579
1 450 3.677633 0.8493879 2.582316
1 500 3.695428 0.8477856 2.596395
1 550 3.706509 0.8467550 2.588017
1 600 3.706668 0.8468436 2.604690
1 650 3.695580 0.8472985 2.602236
1 700 3.692024 0.8478357 2.600929
1 750 3.691078 0.8478392 2.600378
1 800 3.694967 0.8477300 2.618990
1 850 3.703207 0.8471694 2.618328
1 900 3.707360 0.8460017 2.632847
1 950 3.700380 0.8469821 2.638779
1 1000 3.716516 0.8453866 2.650945
2 50 3.968495 0.8268408 2.657663
2 100 3.709946 0.8469872 2.508794
2 150 3.580199 0.8576971 2.453141
2 200 3.517004 0.8621985 2.437679
2 250 3.509548 0.8626281 2.411141
2 300 3.459232 0.8666147 2.385462
2 350 3.461048 0.8669143 2.373748
2 400 3.454198 0.8675886 2.390997
2 450 3.473485 0.8664676 2.411977
2 500 3.479975 0.8663723 2.420820
2 550 3.500562 0.8648682 2.448401
2 600 3.488901 0.8657505 2.426532
2 650 3.492164 0.8653689 2.433405
2 700 3.485283 0.8658658 2.430141
2 750 3.494319 0.8653589 2.443872
2 800 3.506938 0.8643381 2.452926
2 850 3.513508 0.8637381 2.466779
2 900 3.501116 0.8643771 2.464208
2 950 3.517297 0.8633424 2.461583
2 1000 3.503417 0.8641769 2.464591
3 50 3.700042 0.8491692 2.461813
3 100 3.479851 0.8646017 2.347374
3 150 3.372917 0.8736233 2.285919
3 200 3.352719 0.8743431 2.287300
3 250 3.323706 0.8769912 2.259408
3 300 3.312354 0.8775965 2.262779
3 350 3.351715 0.8749933 2.299495
3 400 3.308819 0.8781178 2.291817
3 450 3.317245 0.8777984 2.286113
3 500 3.319482 0.8774039 2.288868
3 550 3.324284 0.8770846 2.286412
3 600 3.321331 0.8771627 2.294854
3 650 3.320443 0.8773447 2.286381
3 700 3.323092 0.8773616 2.286139
3 750 3.338928 0.8762207 2.295631
3 800 3.338174 0.8760089 2.291177
3 850 3.333413 0.8763985 2.293207
3 900 3.333496 0.8764428 2.290923
3 950 3.340453 0.8758718 2.298565
3 1000 3.344721 0.8754742 2.305501
4 50 3.635411 0.8529060 2.379303
4 100 3.410958 0.8712131 2.288640
4 150 3.349862 0.8747448 2.273745
4 200 3.359076 0.8745910 2.304713
4 250 3.326679 0.8771685 2.270224
4 300 3.340439 0.8767908 2.290605
4 350 3.371555 0.8743083 2.318847
4 400 3.368932 0.8743994 2.319685
4 450 3.390784 0.8729003 2.328889
4 500 3.392564 0.8726008 2.321965
4 550 3.404524 0.8716178 2.326176
4 600 3.415021 0.8709230 2.338148
4 650 3.422479 0.8702395 2.350211
4 700 3.418773 0.8704117 2.347827
4 750 3.427829 0.8695599 2.354638
4 800 3.435631 0.8688879 2.357233
4 850 3.442011 0.8684789 2.359286
4 900 3.439029 0.8687072 2.361057
4 950 3.438445 0.8687354 2.363285
4 1000 3.441250 0.8685503 2.366076
5 50 3.687599 0.8497471 2.424253
5 100 3.461143 0.8666927 2.350173
5 150 3.419019 0.8697179 2.343277
5 200 3.421563 0.8691593 2.349777
5 250 3.425595 0.8690723 2.341879
5 300 3.446546 0.8680009 2.359538
5 350 3.454453 0.8676852 2.360944
5 400 3.463285 0.8672009 2.369296
5 450 3.472229 0.8667449 2.383037
5 500 3.470724 0.8665423 2.378817
5 550 3.487875 0.8653935 2.397105
5 600 3.496351 0.8645940 2.408957
5 650 3.495991 0.8646606 2.413306
5 700 3.501778 0.8643184 2.419154
5 750 3.506894 0.8640021 2.426643
5 800 3.507234 0.8638896 2.429748
5 850 3.512331 0.8634990 2.430840
5 900 3.518025 0.8631704 2.434838
5 950 3.521682 0.8629261 2.436378
5 1000 3.520685 0.8629195 2.437084
6 50 3.562727 0.8621188 2.373177
6 100 3.414576 0.8725981 2.286023
6 150 3.354404 0.8761831 2.246834
6 200 3.358960 0.8755248 2.243536
6 250 3.362844 0.8748492 2.263744
6 300 3.334375 0.8766991 2.251438
6 350 3.335976 0.8767340 2.252848
6 400 3.346424 0.8761219 2.258466
6 450 3.348191 0.8759715 2.264076
6 500 3.359655 0.8750895 2.273217
6 550 3.353388 0.8755587 2.277752
6 600 3.361009 0.8751118 2.280557
6 650 3.364715 0.8748335 2.286891
6 700 3.370369 0.8744029 2.292485
6 750 3.373019 0.8742177 2.295420
6 800 3.365999 0.8746529 2.291531
6 850 3.371152 0.8742445 2.296489
6 900 3.369039 0.8744371 2.297784
6 950 3.373315 0.8740674 2.301897
6 1000 3.374367 0.8740068 2.304299
7 50 3.600198 0.8576736 2.343472
7 100 3.349264 0.8737095 2.219034
7 150 3.299229 0.8774078 2.215100
7 200 3.319528 0.8766071 2.242905
7 250 3.339707 0.8750852 2.276962
7 300 3.339280 0.8754424 2.282841
7 350 3.352456 0.8748033 2.299925
7 400 3.372470 0.8730936 2.316286
7 450 3.399131 0.8712555 2.332561
7 500 3.393273 0.8719018 2.340039
7 550 3.399609 0.8713487 2.342887
7 600 3.410639 0.8706000 2.350960
7 650 3.419303 0.8699738 2.359089
7 700 3.420250 0.8697987 2.358479
7 750 3.423921 0.8696274 2.360539
7 800 3.424717 0.8695570 2.359914
7 850 3.422903 0.8697490 2.358896
7 900 3.426555 0.8694682 2.363363
7 950 3.425600 0.8694990 2.362359
7 1000 3.424421 0.8695982 2.362550
8 50 3.534142 0.8629654 2.313172
8 100 3.358545 0.8741616 2.227429
8 150 3.305382 0.8788410 2.210188
8 200 3.266058 0.8817258 2.197039
8 250 3.276128 0.8807114 2.209759
8 300 3.291441 0.8794441 2.227556
8 350 3.314448 0.8779046 2.236977
8 400 3.326401 0.8773032 2.245626
8 450 3.324268 0.8774341 2.246092
8 500 3.326766 0.8772570 2.256470
8 550 3.326936 0.8772732 2.258736
8 600 3.332522 0.8768278 2.262946
8 650 3.334025 0.8766931 2.265762
8 700 3.342805 0.8760700 2.270930
8 750 3.346872 0.8757576 2.273352
8 800 3.346667 0.8757818 2.271745
8 850 3.347435 0.8756806 2.275168
8 900 3.349427 0.8755582 2.276192
8 950 3.348645 0.8755895 2.275491
8 1000 3.350099 0.8754895 2.277826
9 50 3.600223 0.8566141 2.335488
9 100 3.397071 0.8711413 2.263157
9 150 3.326430 0.8757905 2.234868
9 200 3.335975 0.8752120 2.256410
9 250 3.341405 0.8746543 2.255798
9 300 3.340177 0.8750335 2.273833
9 350 3.361233 0.8734328 2.288459
9 400 3.375981 0.8728186 2.297401
9 450 3.380010 0.8726227 2.304728
9 500 3.386854 0.8722360 2.311340
9 550 3.394522 0.8716256 2.321909
9 600 3.400188 0.8712078 2.327830
9 650 3.394535 0.8717505 2.327926
9 700 3.400401 0.8712486 2.329972
9 750 3.404834 0.8709282 2.333258
9 800 3.403585 0.8710867 2.332798
9 850 3.410977 0.8705410 2.338042
9 900 3.412692 0.8704537 2.339550
9 950 3.423312 0.8696768 2.344603
9 1000 3.423576 0.8696229 2.345321
10 50 3.551660 0.8594353 2.344162
10 100 3.377726 0.8723291 2.261835
10 150 3.350703 0.8739261 2.256344
10 200 3.364239 0.8733467 2.271046
10 250 3.353075 0.8744479 2.277147
10 300 3.369293 0.8736188 2.291680
10 350 3.362223 0.8740732 2.285120
10 400 3.370423 0.8735461 2.292665
10 450 3.381252 0.8728576 2.302221
10 500 3.376267 0.8733703 2.296071
10 550 3.387489 0.8725058 2.305201
10 600 3.383516 0.8727469 2.308350
10 650 3.379627 0.8731621 2.305435
10 700 3.379353 0.8731049 2.306646
10 750 3.380698 0.8730749 2.307528
10 800 3.381078 0.8730169 2.306320
10 850 3.379267 0.8731464 2.306739
10 900 3.379993 0.8730474 2.307428
10 950 3.378785 0.8731574 2.309110
10 1000 3.380822 0.8729767 2.309700
11 50 3.476049 0.8656434 2.270222
11 100 3.298553 0.8780055 2.217835
11 150 3.284762 0.8790490 2.208018
11 200 3.284178 0.8795091 2.218634
11 250 3.304236 0.8782755 2.240085
11 300 3.321624 0.8768016 2.245713
11 350 3.332550 0.8764151 2.258416
11 400 3.321305 0.8771209 2.253855
11 450 3.328842 0.8767080 2.261085
11 500 3.337383 0.8761716 2.266228
11 550 3.336043 0.8763889 2.267463
11 600 3.333611 0.8765204 2.265961
11 650 3.337522 0.8761347 2.271666
11 700 3.331421 0.8765714 2.269293
11 750 3.337736 0.8761556 2.271913
11 800 3.338369 0.8761320 2.270479
11 850 3.339400 0.8760438 2.270568
11 900 3.341383 0.8758189 2.271146
11 950 3.342022 0.8757744 2.271929
11 1000 3.343100 0.8756254 2.272402
12 50 3.500355 0.8656684 2.370335
12 100 3.377388 0.8721419 2.281756
12 150 3.373376 0.8722776 2.266652
12 200 3.349516 0.8735682 2.232636
12 250 3.375206 0.8721383 2.245746
12 300 3.399947 0.8702676 2.268464
12 350 3.396986 0.8705661 2.269501
12 400 3.409390 0.8697219 2.273315
12 450 3.416081 0.8692231 2.279094
12 500 3.421686 0.8689740 2.282378
12 550 3.424757 0.8687100 2.289025
12 600 3.428155 0.8685507 2.289822
12 650 3.428401 0.8684944 2.297986
12 700 3.430481 0.8683887 2.298142
12 750 3.435349 0.8680165 2.306155
12 800 3.434624 0.8680937 2.306660
12 850 3.436419 0.8679792 2.307227
12 900 3.440406 0.8676355 2.311556
12 950 3.438874 0.8677666 2.310788
12 1000 3.438040 0.8678436 2.309864
13 50 3.573538 0.8575720 2.372747
13 100 3.398453 0.8694819 2.285241
13 150 3.396545 0.8697495 2.301357
13 200 3.394866 0.8699206 2.313971
13 250 3.381338 0.8709699 2.304600
13 300 3.414767 0.8684344 2.332100
13 350 3.422011 0.8681369 2.333000
13 400 3.423114 0.8680437 2.329125
13 450 3.423142 0.8680437 2.331525
13 500 3.431193 0.8675941 2.340223
13 550 3.435995 0.8673378 2.337968
13 600 3.444249 0.8667302 2.341254
13 650 3.449477 0.8663039 2.343634
13 700 3.453883 0.8659716 2.349377
13 750 3.454523 0.8658908 2.349951
13 800 3.454474 0.8658835 2.352056
13 850 3.455746 0.8658216 2.353103
13 900 3.458408 0.8655870 2.355712
13 950 3.457647 0.8656278 2.356097
13 1000 3.458746 0.8655097 2.357813
14 50 3.509508 0.8645793 2.326039
14 100 3.400867 0.8717357 2.278107
14 150 3.357513 0.8751319 2.248063
14 200 3.354450 0.8752147 2.246719
14 250 3.358252 0.8753362 2.241946
14 300 3.357438 0.8755660 2.240811
14 350 3.358342 0.8756358 2.238202
14 400 3.366128 0.8751656 2.245917
14 450 3.377133 0.8745649 2.250316
14 500 3.370458 0.8750783 2.258345
14 550 3.377885 0.8745672 2.259154
14 600 3.385521 0.8741311 2.262224
14 650 3.384801 0.8742464 2.265606
14 700 3.381897 0.8745283 2.269435
14 750 3.383198 0.8744353 2.268528
14 800 3.383080 0.8744069 2.266507
14 850 3.383820 0.8743805 2.268870
14 900 3.385582 0.8741716 2.271275
14 950 3.385360 0.8741634 2.272109
14 1000 3.383724 0.8742791 2.271739
15 50 3.524443 0.8625236 2.344654
15 100 3.478066 0.8653876 2.330499
15 150 3.440958 0.8682866 2.317318
15 200 3.436551 0.8687245 2.338543
15 250 3.454775 0.8676263 2.348942
15 300 3.463345 0.8670830 2.358125
15 350 3.465113 0.8671021 2.371358
15 400 3.471281 0.8667889 2.365160
15 450 3.481736 0.8661798 2.382107
15 500 3.489374 0.8657879 2.394309
15 550 3.486224 0.8659965 2.392553
15 600 3.489722 0.8657916 2.398088
15 650 3.498798 0.8652350 2.406346
15 700 3.500283 0.8650770 2.411243
15 750 3.501835 0.8649572 2.411949
15 800 3.502010 0.8649565 2.415881
15 850 3.504662 0.8647291 2.419911
15 900 3.504603 0.8647080 2.420116
15 950 3.508592 0.8644492 2.421693
15 1000 3.510185 0.8643706 2.424943
16 50 3.480292 0.8661379 2.364637
16 100 3.351235 0.8743604 2.329051
16 150 3.316853 0.8776898 2.305955
16 200 3.285044 0.8792062 2.293189
16 250 3.268554 0.8806654 2.277981
16 300 3.277325 0.8801980 2.300204
16 350 3.296195 0.8793428 2.313751
16 400 3.279332 0.8803660 2.297134
16 450 3.289713 0.8797768 2.300444
16 500 3.295215 0.8794980 2.306330
16 550 3.306169 0.8786862 2.310016
16 600 3.304311 0.8787948 2.308780
16 650 3.309007 0.8785331 2.312359
16 700 3.310203 0.8783484 2.315911
16 750 3.313151 0.8781876 2.321599
16 800 3.309144 0.8784136 2.315027
16 850 3.313548 0.8781647 2.318762
16 900 3.313304 0.8782184 2.317921
16 950 3.313595 0.8781890 2.316081
16 1000 3.311923 0.8783148 2.316118
17 50 3.521837 0.8635578 2.312638
17 100 3.396253 0.8705684 2.235455
17 150 3.389813 0.8708138 2.241884
17 200 3.400687 0.8703217 2.253133
17 250 3.409662 0.8703477 2.260618
17 300 3.411331 0.8704195 2.267147
17 350 3.424817 0.8697822 2.271564
17 400 3.436516 0.8690917 2.281009
17 450 3.432944 0.8694749 2.282822
17 500 3.447470 0.8686245 2.291879
17 550 3.443714 0.8689127 2.293932
17 600 3.445267 0.8687809 2.291964
17 650 3.450222 0.8683127 2.297226
17 700 3.453113 0.8681731 2.297534
17 750 3.457776 0.8678932 2.302505
17 800 3.461964 0.8676046 2.306283
17 850 3.459864 0.8678212 2.303313
17 900 3.459815 0.8677994 2.305933
17 950 3.459692 0.8678309 2.306816
17 1000 3.461647 0.8676895 2.307294
18 50 3.475133 0.8652138 2.279452
18 100 3.318117 0.8777758 2.233119
18 150 3.343652 0.8749120 2.244967
18 200 3.326673 0.8762044 2.222860
18 250 3.355968 0.8741937 2.242153
18 300 3.351920 0.8748511 2.259469
18 350 3.367729 0.8736944 2.263664
18 400 3.375179 0.8731673 2.267943
18 450 3.367465 0.8737756 2.269143
18 500 3.387731 0.8722778 2.279568
18 550 3.390345 0.8721510 2.284280
18 600 3.388465 0.8722423 2.286331
18 650 3.393304 0.8718478 2.289949
18 700 3.396631 0.8716770 2.291170
18 750 3.399299 0.8714660 2.293965
18 800 3.404287 0.8711195 2.299957
18 850 3.403829 0.8711300 2.303157
18 900 3.406843 0.8709117 2.304503
18 950 3.408037 0.8709299 2.307370
18 1000 3.408288 0.8708678 2.307114
19 50 3.507833 0.8626627 2.330554
19 100 3.325693 0.8759116 2.233233
19 150 3.275151 0.8792075 2.206423
19 200 3.298094 0.8777064 2.215961
19 250 3.307466 0.8776320 2.206130
19 300 3.323675 0.8766334 2.210116
19 350 3.333471 0.8759270 2.212928
19 400 3.337530 0.8756816 2.222182
19 450 3.349797 0.8747741 2.228137
19 500 3.344613 0.8751162 2.233170
19 550 3.350803 0.8747210 2.242486
19 600 3.358832 0.8740604 2.243485
19 650 3.365364 0.8735821 2.251019
19 700 3.366706 0.8734248 2.253833
19 750 3.368545 0.8733049 2.255918
19 800 3.369470 0.8732248 2.256726
19 850 3.368225 0.8733395 2.255602
19 900 3.367437 0.8734098 2.256305
19 950 3.371530 0.8731676 2.257948
19 1000 3.372622 0.8730043 2.259411
20 50 3.568770 0.8583578 2.354901
20 100 3.424282 0.8682108 2.275533
20 150 3.348994 0.8737787 2.242433
20 200 3.352860 0.8739379 2.237980
20 250 3.351720 0.8741891 2.243985
20 300 3.361999 0.8734664 2.251741
20 350 3.354823 0.8743319 2.247359
20 400 3.356180 0.8745392 2.254295
20 450 3.358151 0.8743574 2.258660
20 500 3.362153 0.8741687 2.262248
20 550 3.369574 0.8736024 2.266376
20 600 3.374862 0.8731966 2.272978
20 650 3.378412 0.8729433 2.275386
20 700 3.379784 0.8727832 2.278675
20 750 3.379932 0.8728234 2.280194
20 800 3.379132 0.8729065 2.282742
20 850 3.380461 0.8727394 2.282683
20 900 3.379012 0.8728288 2.284638
20 950 3.379408 0.8727467 2.284248
20 1000 3.380292 0.8726943 2.284023
Tuning parameter 'shrinkage' was held constant at a value of 0.1
Tuning parameter 'n.minobsinnode' was held constant at a value of 10
RMSE was used to select the optimal model using the smallest value.
The final values used for the model were n.trees = 200, interaction.depth =
8, shrinkage = 0.1 and n.minobsinnode = 10.
min(mod_gbm$results$RMSE) # Training RMSE
[1] 3.266058
plot(mod_gbm)
Test RMSE
p <- predict(mod_gbm, newdata = BostonTest_pp)
RMSE(BostonTest_pp$medv, p) # Test RMSE
[1] 3.007369
Support Vector Machines
set.seed(123)
myControl <- trainControl(method = "cv", number = 5)
mod_svm <- train(medv ~ .,
data = BostonTrain_pp,
trControl = myControl,
method = "svmRadial",
tuneLength = 12)
mod_svm
Support Vector Machines with Radial Basis Function Kernel
381 samples
13 predictor
No pre-processing
Resampling: Cross-Validated (5 fold)
Summary of sample sizes: 306, 304, 304, 305, 305
Resampling results across tuning parameters:
C RMSE Rsquared MAE
0.25 5.130914 0.7486183 3.002551
0.50 4.610427 0.7876803 2.736412
1.00 4.126431 0.8235969 2.558077
2.00 3.714840 0.8531095 2.374209
4.00 3.614707 0.8558090 2.310476
8.00 3.574915 0.8586898 2.306565
16.00 3.553400 0.8590206 2.338155
32.00 3.597014 0.8534230 2.414552
64.00 3.736243 0.8411513 2.522262
128.00 3.863299 0.8308018 2.588757
256.00 4.041082 0.8171449 2.668202
512.00 4.113218 0.8108560 2.693553
Tuning parameter 'sigma' was held constant at a value of 0.0884089
RMSE was used to select the optimal model using the smallest value.
The final values used for the model were sigma = 0.0884089 and C = 16.
min(mod_svm$results$RMSE) # Training RMSE
[1] 3.5534
plot(mod_svm)
Test RMSE
p <- predict(mod_svm, newdata = BostonTest_pp)
RMSE(BostonTest_pp$medv, p) # Test RMSE
[1] 2.721747
k-nearest neighbors
set.seed(123)
myControl <- trainControl(method = "cv", number = 5)
mod_knn <- train(medv ~ .,
data = BostonTrain_pp,
trControl = myControl,
method = "knn",
tuneLength = 12)
mod_knn
k-Nearest Neighbors
381 samples
13 predictor
No pre-processing
Resampling: Cross-Validated (5 fold)
Summary of sample sizes: 306, 304, 304, 305, 305
Resampling results across tuning parameters:
k RMSE Rsquared MAE
5 5.127388 0.7090687 3.178419
7 4.956041 0.7354359 3.147579
9 4.993595 0.7411511 3.164132
11 5.145678 0.7311013 3.236609
13 5.231936 0.7262360 3.248706
15 5.229716 0.7291167 3.243414
17 5.367630 0.7169652 3.337101
19 5.471290 0.7066006 3.395800
21 5.558299 0.7008257 3.464050
23 5.655375 0.6945242 3.536193
25 5.748517 0.6870365 3.585939
27 5.797428 0.6847486 3.619724
RMSE was used to select the optimal model using the smallest value.
The final value used for the model was k = 7.
min(mod_knn$results$RMSE) # Training RMSE
[1] 4.956041
plot(mod_knn)
Test RMSE
p <- predict(mod_knn, newdata = BostonTest_pp)
RMSE(BostonTest_pp$medv, p) # Test RMSE
[1] 3.48389