HOMEWORK 5

#importing the libraries
library(tidyverse) #filter etc.
library(ggplot2) 
library(reshape) #melt
library(directlabels)
library(dplyr)
library(data.table) #data.table
library(caret)
library(TunePareto) #generateCVRuns
library(glmnet)
library(kknn)
library(penalized)

Introduction:

The first task is on uWaveGestureLibrary. It consists over 4000 three-axis accelerometer instances from 8 people with 8 different gestures, our aim is to perform classification task by Nearest Neighbor Approach. Then, the second task is to use a penalized regression approach to classify given EcG data.

Task 1

#reading the data from repository
x_train<-read.table("https://github.com/BU-IE-582/fall20-ilaydacelenk/blob/master/files/HW2_data/uWaveGestureLibrary_X_TRAIN.txt?raw=true") %>% as.data.frame()
y_train<-read.table("https://github.com/BU-IE-582/fall20-ilaydacelenk/blob/master/files/HW2_data/uWaveGestureLibrary_Y_TRAIN.txt?raw=true") %>% as.data.frame()
z_train<-read.table("https://github.com/BU-IE-582/fall20-ilaydacelenk/blob/master/files/HW2_data/uWaveGestureLibrary_Z_TRAIN.txt?raw=true") %>% as.data.frame()

x_test<-read.table("https://github.com/BU-IE-582/fall20-ilaydacelenk/blob/master/files/HW2_data/uWaveGestureLibrary_X_TEST.txt?raw=true") %>% as.data.frame()
y_test<-read.table("https://github.com/BU-IE-582/fall20-ilaydacelenk/blob/master/files/HW2_data/uWaveGestureLibrary_Y_TEST.txt?raw=true") %>% as.data.frame()
z_test<-read.table("https://github.com/BU-IE-582/fall20-ilaydacelenk/blob/master/files/HW2_data/uWaveGestureLibrary_Z_TEST.txt?raw=true") %>% as.data.frame()

x_name <- c("x")
x_names <- c()
for(i in 1:ncol(x_train)){
  x_names <- x_names %>% append(x_name)
}
colnames(x_train) <- make.names(x_names, unique=TRUE)
colnames(x_test) <- make.names(x_names, unique=TRUE)

y_name <- c("y")
y_names <- c()
for(i in 1:ncol(y_train)){
  y_names <- y_names %>% append(y_name)
}
colnames(y_train) <- make.names(y_names, unique=TRUE)
colnames(y_test) <- make.names(y_names, unique=TRUE)

z_name <- c("z")
z_names <- c()
for(i in 1:ncol(z_train)){
  z_names <- z_names %>% append(z_name)
}
colnames(z_train) <- make.names(z_names, unique=TRUE)
colnames(z_test) <- make.names(z_names, unique=TRUE)

train <- cbind(x_train[,-1], y_train[,-1], z_train[,-1]) %>% scale()
test <- cbind(x_test[,-1], y_test[,-1], z_test[,-1]) %>% scale()

class <- x_train[,1]
train_class <- class
trainn <- cbind(class, train) %>% as.data.frame()
trainn$class <- sub("^", "g", trainn$class) %>% as.factor()

class <- x_test[,1]
test_class <- class
testt <- cbind(class, test) %>% as.data.frame()
testt$class <- sub("^", "g", testt$class) %>% as.factor()

Task1a

set.seed(3)
k_levels <- c(1, 3, 5, 7, 9, 11, 13, 15)
nofReplications <- 3
nFolds <- 10
#for balanced folds: stratified =TRUE
indices <- generateCVRuns(trainn$class, ntimes=nofReplications, nfold=nFolds, stratified=TRUE)
cv_results <- data.table()

for(i in 1:nofReplications) {
  rep<-indices[[i]]
  for(j in 1:nFolds){
   test_ind<-rep[[j]] 
   cv_train<-trainn[-test_ind,] 
   cv_test<-trainn[test_ind,]
   
   for(m in 1:length(k_levels)){
      k_tune <- k_levels[m]
      model_knn_euc <- kknn(class~., cv_train, cv_test, k = k_tune, kernel="rectangular", distance=2)
      pred <- model_knn_euc[["fitted.values"]]
      cm_test_man <- confusionMatrix(pred, cv_test$class)
      cv_results <- rbind(cv_results, data.table(Replication=i, Fold=j, K=k_tune, TestId=test_ind,
                                         Predictions=pred, Actual=cv_test$class))
    }   
   
  }
  
}

Euclidean_result <- cv_results[,list(Accuracy=mean(Predictions==Actual)),by=list(K)] %>% arrange(desc(Accuracy))
Euclidean_result

FALSE     K  Accuracy
FALSE 1:  3 0.9494048
FALSE 2:  1 0.9441964
FALSE 3:  5 0.9412202
FALSE 4:  7 0.9378720
FALSE 5:  9 0.9308036
FALSE 6: 11 0.9293155
FALSE 7: 13 0.9281994
FALSE 8: 15 0.9244792

besk_k <- as.matrix(Euclidean_result)

euc_time <- system.time(
  model_knn_euc <- kknn(class~., trainn, testt, k = besk_k[1,1], kernel="rectangular", distance=2)
)

model_knn_euc2 <- kknn(class~., trainn, trainn, k = besk_k[1,1], kernel="rectangular", distance=2)



## REMARK
# Alternatively, we could also use the train function from caret, it will give the same result, for the best tune it gives the same accuracy (I checked)
#k_levels <- c(1, 3, 5, 7, 9, 11, 13, 15)
#nofReplications <- 1
#nFolds <- 10
#fitControl <- trainControl(method = "repeatedcv", number = nFolds, repeats = nofReplications, savePred = TRUE, classProbs = TRUE)
#grid <- expand.grid(k=k_levels)
#model_knn_euc <- train(class ~ ., data = trainn, method = "knn", trControl = fitControl, tuneGrid = grid)
#pred_euc <- predict(model_knn_euc, newdata=test)

Using Euclidean distance, I applied a 10-fold cross-validation nearest-neighbor classifier. The number of neighbors (k) that minimizes the error is found as 3.

set.seed(1)
k_levels <- c(1, 3, 5, 7, 9, 11, 13, 15)
nofReplications <- 3
nFolds <- 10
#for balanced folds: stratified =TRUE
indices <- generateCVRuns(trainn$class, ntimes=nofReplications, nfold=nFolds, stratified=TRUE)
cv_results <- data.table()

for(i in 1:nofReplications) {
  rep<-indices[[i]]
  for(j in 1:nFolds){
   test_ind<-rep[[j]] 
   cv_train<-trainn[-test_ind,] 
   cv_test<-trainn[test_ind,]
   
   for(m in 1:length(k_levels)){
      k_tune <- k_levels[m]
      model_knn_man <- kknn(class~., cv_train, cv_test, k = k_tune, kernel="rectangular", distance=1)
      pred <- model_knn_man[["fitted.values"]]
      cm_test_man <- confusionMatrix(pred, cv_test$class)
      cv_results <- rbind(cv_results, data.table(Replication=i, Fold=j, K=k_tune, TestId=test_ind,
                                         Predictions=pred, Actual=cv_test$class))
    }   
   
  }
  
}

Manhattan_result <- cv_results[,list(Accuracy=mean(Predictions==Actual)),by=list(K)] %>% arrange(desc(Accuracy))
Manhattan_result

FALSE     K  Accuracy
FALSE 1:  3 0.9594494
FALSE 2:  1 0.9568452
FALSE 3:  5 0.9531250
FALSE 4:  7 0.9475446
FALSE 5:  9 0.9453125
FALSE 6: 11 0.9408482
FALSE 7: 13 0.9367560
FALSE 8: 15 0.9345238

besk_k <- as.matrix(Manhattan_result)

man_time <- system.time(
  model_knn_man <- kknn(class~., trainn, testt, k = besk_k[1,1], kernel="rectangular", distance=1)
)

model_knn_man2 <- kknn(class~., trainn, trainn, k = besk_k[1,1], kernel="rectangular", distance=1)

I used Manhattan distance in a similar manner. The number of neighbors (k) that minimizes the error is found as 3.

Task1b

cm_test_euc <- confusionMatrix(model_knn_euc[["fitted.values"]], testt$class)
cm_train_euc <- confusionMatrix(model_knn_euc2[["fitted.values"]], trainn$class)
print(cm_test_euc[["table"]])

FALSE           Reference
FALSE Prediction  g1  g2  g3  g4  g5  g6  g7  g8
FALSE         g1 429   1   1   5   3   5   0   0
FALSE         g2   0 449   0   0   0   0   0   0
FALSE         g3   0   0 414   0   6   5   2   0
FALSE         g4   3   0   0 372   1  14   0   1
FALSE         g5   0   0  14  59 421  29   0   1
FALSE         g6   4   0  21   8   2 395   0   0
FALSE         g7   1   2   4   0   0   1 445   0
FALSE         g8   0   0   0   6   0   0   0 458

print(cm_train_euc[["table"]])

FALSE           Reference
FALSE Prediction  g1  g2  g3  g4  g5  g6  g7  g8
FALSE         g1 120   0   0   1   0   1   0   0
FALSE         g2   0 108   0   0   0   0   0   0
FALSE         g3   0   0 104   0   0   1   0   0
FALSE         g4   0   0   0  99   0   3   0   0
FALSE         g5   0   0   0   9 127   3   0   0
FALSE         g6   2   0   1   0   0 103   0   0
FALSE         g7   0   0   1   0   0   0 112   0
FALSE         g8   0   0   0   1   0   0   0 100

cm_test_euc[["overall"]][["Accuracy"]]

FALSE [1] 0.9444444

cm_train_euc[["overall"]][["Accuracy"]]

FALSE [1] 0.9743304

euc_time

FALSE    user  system elapsed 
FALSE   3.631   0.061   3.698

Using best k value for Euclidean distance, I evaluated the performance on both test and train set as 94.44% and 97.43% respectively. Confusion matrices are shown above, first for the test data then for the train data. The runtime for euclidean distance to predict is 3.631, 0.061, 3.698, 0, 0.

cm_test_man <- confusionMatrix(model_knn_man[["fitted.values"]], testt$class)
cm_train_man <- confusionMatrix(model_knn_man2[["fitted.values"]], trainn$class)
print(cm_test_man[["table"]])

FALSE           Reference
FALSE Prediction  g1  g2  g3  g4  g5  g6  g7  g8
FALSE         g1 432   1   2   3   3   6   1   0
FALSE         g2   0 451   0   0   0   0   0   0
FALSE         g3   0   0 415   0   4   4   2   0
FALSE         g4   2   0   0 384   2  12   0   1
FALSE         g5   0   0  12  48 422  27   0   1
FALSE         g6   3   0  20   8   2 400   0   0
FALSE         g7   0   0   5   0   0   0 444   0
FALSE         g8   0   0   0   7   0   0   0 458

print(cm_train_man[["table"]])

FALSE           Reference
FALSE Prediction  g1  g2  g3  g4  g5  g6  g7  g8
FALSE         g1 121   0   0   1   0   0   0   0
FALSE         g2   0 108   0   0   0   0   0   0
FALSE         g3   0   0 104   0   0   1   0   0
FALSE         g4   0   0   0 102   1   3   0   0
FALSE         g5   0   0   0   6 126   2   0   0
FALSE         g6   1   0   1   0   0 105   0   0
FALSE         g7   0   0   1   0   0   0 112   0
FALSE         g8   0   0   0   1   0   0   0 100

cm_test_man[["overall"]][["Accuracy"]]

FALSE [1] 0.9508654

cm_train_man[["overall"]][["Accuracy"]]

FALSE [1] 0.9799107

man_time

FALSE    user  system elapsed 
FALSE  54.586   0.135  54.789

Using best k value for Manhattan distance, I evaluated the performance on both test and train set as 95.09% and 97.99%, respectively. Confusion matrices are shown above, first for the test data then for the train data. The runtime for euclidean distance to predict is 54.586, 0.135, 54.789, 0, 0.

Task1c

If we compute the distance as final_dist = w1dist_x, + w2dist_y + w3*dist_z, the nearest neighbors may change. This would affect the classification result. I would expect the standard deviation of x-axis to be high, due to the nature of gestures and I expect this axis to be more effective. So, giving it more weight could be reasonable.

Task 2

#reading the data from repository
ecg_train<-read.table("https://github.com/BU-IE-582/fall20-ilaydacelenk/blob/master/files/ecgTRAIN.txt?raw=true")
ecg_test<-read.table("https://github.com/BU-IE-582/fall20-ilaydacelenk/blob/master/files/ecgTEST.txt?raw=true")
# map -1-1 to 0-1
ecg_train[,1][(ecg_train[,1]==-1)] <- 0
ecg_test[,1][(ecg_test[,1]==-1)] <- 0

In this task, given ECG data, the aim is to train different logistic regression models using fused lasso penalties. Above, first I read the datasets and since the target columns consists of -1 and 1’s, I map them into 0-1. I do this by mapping -1 to 0 and keeping 1’s as they are.

Task 2a

set.seed(1)
logreg1 <- optL1(response=ecg_train[,1], penalized=ecg_train[,-1],fusedl = TRUE, fold = 10, model = "logistic")

FALSE lambda= 8.548129  12345678910cvl= -47.91835 
FALSE lambda= 13.83116  12345678910cvl= -55.22797 
FALSE lambda= 5.283034  12345678910cvl= -41.40059 
FALSE lambda= 3.265095  12345678910cvl= -37.5193 
FALSE lambda= 2.017939  12345678910cvl= -36.5004 
FALSE lambda= 1.436027  12345678910cvl= -37.02119 
FALSE lambda= 2.205082  12345678910cvl= -36.47056 
FALSE lambda= 2.194867  12345678910cvl= -36.47008 
FALSE lambda= 2.179852  12345678910cvl= -36.46972 
FALSE lambda= 2.118007  12345678910cvl= -36.47313 
FALSE lambda= 2.174073  12345678910cvl= -36.46969 
FALSE lambda= 2.174176  12345678910cvl= -36.46975 
FALSE lambda= 2.152658  12345678910cvl= -36.47012 
FALSE lambda= 2.165893  12345678910cvl= -36.46975 
FALSE lambda= 2.170949  12345678910cvl= -36.4697 
FALSE lambda= 2.17288   12345678910cvl= -36.46969 
FALSE lambda= 2.173617  12345678910cvl= -36.46969

optlambda1 <- logreg1$lambda
logreg2<-optL2(response=ecg_train[,1], penalized=ecg_train[,-1], fusedl = TRUE, fold = 10, model = "logistic", lambda1=optlambda1, minlambda2 = 0.5)

FALSE lambda= Inf   cvl= -62.61648 
FALSE lambda= 0.5   12345678910cvl= -37.39543 
FALSE lambda= 5     12345678910cvl= -40.29091 
FALSE lambda= 2.218847  12345678910cvl= -37.87516 
FALSE lambda= 3.281153  12345678910cvl= -39.2098 
FALSE lambda= 1.562306  12345678910cvl= -37.44149 
FALSE lambda= 0.9378134     12345678910cvl= -37.79447 
FALSE lambda= 1.545419  12345678910cvl= -37.38885 
FALSE lambda= 1.296695  12345678910cvl= -37.40119 
FALSE lambda= 1.450415  12345678910cvl= -37.33382 
FALSE lambda= 1.427117  12345678910cvl= -37.32608 
FALSE lambda= 1.377301  12345678910cvl= -37.33529 
FALSE lambda= 1.41528   12345678910cvl= -37.32223 
FALSE lambda= 1.400773  12345678910cvl= -37.32322 
FALSE lambda= 1.410314  12345678910cvl= -37.32063 
FALSE lambda= 1.408863  12345678910cvl= -37.32082 
FALSE lambda= 1.41051   12345678910cvl= -37.32069 
FALSE lambda= 1.409829  12345678910cvl= -37.3206 
FALSE lambda= 1.40946   12345678910cvl= -37.32068 
FALSE lambda= 1.409887  12345678910cvl= -37.32059 
FALSE lambda= 1.410071  12345678910cvl= -37.32055

optlambda2 <- logreg2$lambda
model<-penalized(response=ecg_train[,1], penalized=ecg_train[,-1], fusedl = TRUE, model = "logistic", data=ecg_train, lambda1 = optlambda1, lambda2 = optlambda2)

pred_train <- predict(model, penalized=ecg_train[,-1], data=ecg_train)
pred_train[pred_train>0.5] <- 1
pred_train[pred_train<=0.5] <- 0
cm_train <- confusionMatrix(as.factor(pred_train), as.factor(ecg_train$V1))

pred_test <- predict(model,penalized=ecg_test[,-1],data=ecg_test)
pred_test[pred_test>0.5] <- 1
pred_test[pred_test<=0.5] <- 0
cm_test <- confusionMatrix(as.factor(pred_test), as.factor(ecg_test$V1))

print(cm_test[["table"]])

FALSE           Reference
FALSE Prediction  0  1
FALSE          0 24  7
FALSE          1 12 57

print(cm_train[["table"]])

FALSE           Reference
FALSE Prediction  0  1
FALSE          0 23  6
FALSE          1  8 63

Above, I trained a logistic regression model using fused lasso penalties on ECG data. I used the learned model to predict the class for test data. I evaluated the performance on both test and train set as 81% and 86%, respectively. Even though the accuracy for train data is high, it is not 100%, therefore there is no big overfitting problem. Confusion matrices are shown above, first for the test data then for the train data.

Task 2b

model

FALSE Penalized logistic regression object
FALSE 97 regression coefficients of which 21 are non-zero
FALSE 
FALSE Loglikelihood =    -29.58561 
FALSE L1 penalty =   8.885764   at lambda1 =  2.174073 
FALSE L2 penalty =   2.506205   at lambda2 =  1.410071

coefs <- coefficients(model,"all")
plot(coefficients(model,"all"), type="l")

The coefficient are nearly zero when the plot is flat. There are 21 nonzero coefficients in the model.

Task 2c

train2<-matrix(0,nrow = 100,ncol=96) # empty matrix
train2[,1]<-ecg_train[,1] # target column
for(i in (3:96)){
  train2[,(i-1)]=ecg_train[,i]-ecg_train[,(i-1)]
}

test2<-matrix(0,nrow = 100,ncol=96) # empty matrix
test2[,1]<-ecg_test[,1] # target column
for(i in (3:96)){
  test2[,(i-1)]=ecg_test[,i]-ecg_test[,(i-1)]
}

train2 <- train2 %>% data.frame()
test2 <- test2 %>% data.frame()

Above, I create a matrix for the train data which consists of the consecutive differences of instances. This makes sense and can be done since it is a time series data. I do the same for the test data. Below I will train the model on the consecutive differences data.

set.seed(1)
logreg3 <- optL1(response=train2[,1], penalized=train2[,-1], fusedl = TRUE, fold = 10, data=train2, model = "logistic")

FALSE lambda= 5.59004   12345678910cvl= -51.7503 
FALSE lambda= 9.044875  12345678910cvl= -55.79921 
FALSE lambda= 3.454835  12345678910cvl= -50.31286 
FALSE lambda= 0.7497166     12345678910cvl= -45.00509 
FALSE lambda= 1.78298   12345678910cvl= -44.36937 
FALSE lambda= 1.465895  12345678910cvl= -43.2478 
FALSE lambda= 1.319406  12345678910cvl= -42.49278 
FALSE lambda= 1.101804  12345678910cvl= -41.95785 
FALSE lambda= 1.044601  12345678910cvl= -42.13642 
FALSE lambda= 1.150072  12345678910cvl= -41.88783 
FALSE lambda= 1.171714  12345678910cvl= -41.92534 
FALSE lambda= 1.141865  12345678910cvl= -41.87757 
FALSE lambda= 1.126563  12345678910cvl= -41.90369 
FALSE lambda= 1.140999  12345678910cvl= -41.8782 
FALSE lambda= 1.143111  12345678910cvl= -41.87818 
FALSE lambda= 1.142064  12345678910cvl= -41.87742 
FALSE lambda= 1.142276  12345678910cvl= -41.87727 
FALSE lambda= 1.142595  12345678910cvl= -41.8775 
FALSE lambda= 1.14233   12345678910cvl= -41.87723

optlambda3 <- logreg3$lambda

logreg4 <- optL2(response=train2[,1], penalized=train2[,-1], fusedl = TRUE, fold = 10, data=train2, model = "logistic", lambda1=optlambda3, minlambda2 = 5)

FALSE lambda= Inf   cvl= -62.61648 
FALSE lambda= 5     12345678910cvl= -52.0904 
FALSE lambda= 50    12345678910cvl= -60.38104 
FALSE lambda= 22.18847  12345678910cvl= -62.07367 
FALSE lambda= 32.81153  12345678910cvl= -62.80064 
FALSE lambda= 15.62306  12345678910cvl= -61.03053 
FALSE lambda= 11.56541  12345678910cvl= -57.87603 
FALSE lambda= 9.057647  12345678910cvl= -56.75188 
FALSE lambda= 5.840841  12345678910cvl= -54.04102 
FALSE lambda= 7.069552  12345678910cvl= -55.95597 
FALSE lambda= 6.310167  12345678910cvl= -55.02203 
FALSE lambda= 5.519668  12345678910cvl= -53.34443 
FALSE lambda= 5.321173  12345678910cvl= -52.87409 
FALSE lambda= 5.198496  12345678910cvl= -52.58291 
FALSE lambda= 5.122677  12345678910cvl= -52.39623 
FALSE lambda= 5.075819  12345678910cvl= -52.27909 
FALSE lambda= 5.046858  12345678910cvl= -52.20689 
FALSE lambda= 5.02896   12345678910cvl= -52.16234 
FALSE lambda= 5.017898  12345678910cvl= -52.13485 
FALSE lambda= 5.011062  12345678910cvl= -52.11786 
FALSE lambda= 5.006837  12345678910cvl= -52.10737 
FALSE lambda= 5.004225  12345678910cvl= -52.10089 
FALSE lambda= 5.002611  12345678910cvl= -52.09688 
FALSE lambda= 5.001614  12345678910cvl= -52.09441 
FALSE lambda= 5.000997  12345678910cvl= -52.09288

optlambda4 <- logreg4$lambda

model2<-penalized(response=train2[,1], penalized=train2[,-1], fusedl = TRUE, model = "logistic", data=train2, lambda1 = optlambda3, lambda2 = optlambda4)

pred_train <- predict(model2,penalized=train2[,-1], data=train2)
pred_train[pred_train>0.5] <- 1
pred_train[pred_train<=0.5] <- 0
cm_train <- confusionMatrix(as.factor(pred_train), as.factor(train2$X1))


pred_test <- predict(model2,penalized=train2[,-1], data=train2)
pred_test[pred_test>0.5] <- 1
pred_test[pred_test<=0.5] <- 0
cm_test <- confusionMatrix(as.factor(pred_test), as.factor(test2$X1))


print(cm_test[["table"]])

FALSE           Reference
FALSE Prediction  0  1
FALSE          0  6 12
FALSE          1 30 52

print(cm_train[["table"]])

FALSE           Reference
FALSE Prediction  0  1
FALSE          0 15  3
FALSE          1 16 66

Above, I trained a logistic regression model using fused lasso penalties on train data with consecutive differences. I used the learned model to predict the class for test data. I evaluated the performance on both test and train set as 58% and 81%, respectively. The accuracy for test set is very low. The model is not learning very well. Confusion matrices are shown above, first for the test data then for the train data.

Task 2d

model2

FALSE Penalized logistic regression object
FALSE 96 regression coefficients of which 18 are non-zero
FALSE 
FALSE Loglikelihood =    -44.25033 
FALSE L1 penalty =   4.271354   at lambda1 =  1.14233 
FALSE L2 penalty =   6.874049   at lambda2 =  5.000997

coefs <- coefficients(model2,"all")
plot(coefficients(model2,"all"), type="l")

The coefficient are nearly zero when the plot is flat. There are 18 nonzero coefficients in the model.

Task 2e

train3 <- cbind(ecg_train, train2[,-1])
test3 <-  cbind(ecg_test, test2[,-1])

set.seed(1)
logreg5 <- optL1(response=train3[,1], penalized=train3[,-1], fusedl = TRUE, fold = 10, data=train3, model = "logistic")

FALSE lambda= 8.548129  12345678910cvl= -47.91835 
FALSE lambda= 13.83116  12345678910cvl= -55.22797 
FALSE lambda= 5.283034  12345678910cvl= -41.44588 
FALSE lambda= 3.265095  12345678910cvl= -38.31241 
FALSE lambda= 2.017939  12345678910cvl= -38.05055 
FALSE lambda= 2.38626   12345678910cvl= -37.76554 
FALSE lambda= 2.547736  12345678910cvl= -37.80611 
FALSE lambda= 2.402071  12345678910cvl= -37.76263 
FALSE lambda= 2.424931  12345678910cvl= -37.76015 
FALSE lambda= 2.471838  12345678910cvl= -37.77457 
FALSE lambda= 2.441513  12345678910cvl= -37.7649 
FALSE lambda= 2.418925  12345678910cvl= -37.75929 
FALSE lambda= 2.417133  12345678910cvl= -37.75961 
FALSE lambda= 2.420209  12345678910cvl= -37.75906 
FALSE lambda= 2.422012  12345678910cvl= -37.75933 
FALSE lambda= 2.420403  12345678910cvl= -37.75902 
FALSE lambda= 2.421017  12345678910cvl= -37.75906 
FALSE lambda= 2.420613  12345678910cvl= -37.75899

optlambda5 <- logreg5$lambda

logreg6 <- optL2(response=train3[,1], penalized=train3[,-1], fusedl = TRUE, fold = 10, model = "logistic", data=train3, lambda1=optlambda5, minlambda2 = 0.5)

FALSE lambda= Inf   cvl= -62.61648 
FALSE lambda= 0.5   12345678910cvl= -38.24758 
FALSE lambda= 5     12345678910cvl= -42.56478 
FALSE lambda= 2.218847  12345678910cvl= -40.30405 
FALSE lambda= 3.281153  12345678910cvl= -41.76461 
FALSE lambda= 1.562306  12345678910cvl= -40.00708 
FALSE lambda= 1.46923   12345678910cvl= -38.90168 
FALSE lambda= 1.099017  12345678910cvl= -38.55604 
FALSE lambda= 1.26436   12345678910cvl= -38.70848 
FALSE lambda= 0.8702128     12345678910cvl= -38.47092 
FALSE lambda= 0.8512825     12345678910cvl= -38.44985 
FALSE lambda= 0.7171046     12345678910cvl= -38.40192 
FALSE lambda= 0.7480027     12345678910cvl= -38.38349 
FALSE lambda= 0.7648487     12345678910cvl= -38.37535 
FALSE lambda= 0.7978635     12345678910cvl= -38.38093 
FALSE lambda= 0.7748947     12345678910cvl= -38.3712 
FALSE lambda= 0.778025  12345678910cvl= -38.37105 
FALSE lambda= 0.7772847     12345678910cvl= -38.37099 
FALSE lambda= 0.7768678     12345678910cvl= -38.37102 
FALSE lambda= 0.7773595     12345678910cvl= -38.37098 
FALSE lambda= 0.7776137     12345678910cvl= -38.371 
FALSE lambda= 0.7774492     12345678910cvl= -38.37098

optlambda6 <- logreg6$lambda

model3<-penalized(response=train3[,1], penalized=train3[,-1], fusedl = TRUE, model = "logistic", data=train3, lambda1 = optlambda5, lambda2 = optlambda6)

pred_train <- predict(model3, penalized=train3[,-1], data=train3)
pred_train[pred_train>0.5] <- 1
pred_train[pred_train<=0.5] <- 0
cm_train <- confusionMatrix(as.factor(pred_train), as.factor(train3$V1))

pred_test <- predict(model3, penalized=train3[,-1], data=train3)
pred_test[pred_test>0.5] <- 1
pred_test[pred_test<=0.5] <- 0
cm_test <- confusionMatrix(as.factor(pred_test), as.factor(test3$V1))

print(cm_test[["table"]])

FALSE           Reference
FALSE Prediction  0  1
FALSE          0  8 21
FALSE          1 28 43

print(cm_train[["table"]])

FALSE           Reference
FALSE Prediction  0  1
FALSE          0 23  6
FALSE          1  8 63

In this part, I combined train data and consecutive train data by binding the columns. Then, I trained a logistic regression model using fused lasso penalties on combined train data with consecutive differences. I used the learned model to predict the class for test data. I evaluated the performance on both test and train set as 51% and 86%, respectively. The accuracy for test set is very low. The model is not learning very well. Confusion matrices are shown above, first for the test data then for the train data.

Task 2f

model3

FALSE Penalized logistic regression object
FALSE 192 regression coefficients of which 18 are non-zero
FALSE 
FALSE Loglikelihood =    -29.44766 
FALSE L1 penalty =   9.654549   at lambda1 =  2.420613 
FALSE L2 penalty =   2.174234   at lambda2 =  0.7773595

coefs <- coefficients(model3,"all")
plot(coefficients(model3,"all"), type="l")

The coefficient are nearly zero when the plot is flat. There are 18 nonzero coefficients in the model.

Reference

[1] J. Liu, Z. Wang, L. Zhong, J. Wickramasuriya, and V. Vasudevan. uWave: Accelerometer-based personalized gesture recognition and its applications. Pervasive Computing and Communications, IEEE International Conference on, 0:1-9, 2009. (link: https://www.recg.org/publications/liu09percom.pdf)

[2] https://cran.r-project.org/web/packages/kknn/kknn.pdf