# Default of Credit Card Clients Data Set

 23 features are used to predict target column.Some features are catogerical and some are also numeric. Categorical features are converted into dummy variables for linear regression. Original data set is used by other learners such as decision tree, random forest.


**Features**



X1: Amount of the given credit (NT dollar)

X2: Gender (1 = male; 2 = female)

X3: Education (1 = graduate school; 2 = university; 3 = high school; 4 = others)

X4: Marital status (1 = married; 2 = single; 3 = others)

X5: Age (year)

X6 - X11: History of past payment

X12-X17: Amount of bill statement (NT dollar)

X18-X23: Amount of previous payment (NT dollar)

**Target**

Default Payment (Yes = 1, No = 0)

```{r,warning=FALSE,message=FALSE }

library("caret")
library("dplyr")
library("glmnet")
library(Metrics)
require(rpart)
library(rattle)
library(RColorBrewer)
require(data.table,quietly = TRUE)
library(data.table)
library(randomForest)
library(gbm)
library(xgboost)
library(e1071)

```



## Data Manipulation Part 

```{r }

credit = read.table(file = "clipboard", sep = "\t",header = T)
credit = credit[,-1]
colnames(credit) <- credit[1,]
names(credit)[ncol(credit)] = "default_payment"
credit <- credit[-1,]
credit$default_payment = as.factor(credit$default_payment)


# convert categorical value into factor and numerical

for (i in 1:23) {
  if(i < 2 || i > 11 ){
    
    credit[,i] = as.numeric(credit[,i])
  }else if( i == 5){
    credit[,i] = as.double(as.numeric(credit[,i]))
  
  }else{
    credit[,i] = as.factor(credit[,i])}}
  
  
  
```

```{r }
# convert categorical variables to dummy
Dummies =  dummyVars("~.", data = credit[,-ncol(credit)],fullRank = T,drop2nd = T)

credit_dummy = data.frame(predict(Dummies, newdata = credit[,-ncol(credit)]))
credit_dummy$default_payment <- credit$default_payment

# split data as training and test

set.seed(121)

idx <- sample(seq(1, 2), size = nrow(credit), replace = TRUE, prob = c(.75,.25))

train_credit <- credit[idx == 1,]
test_credit <- credit[idx == 2,]

dummy_train <- credit_dummy[idx == 1,]
dummy_test <- credit_dummy[idx == 2,]

# to solve imbalance problem

only_0 <- as.data.frame(credit[credit$default_payment == "0",])
only_1 <- as.data.frame(credit[credit$default_payment == "1",])

only_0_dummy <- as.data.frame(credit_dummy[credit_dummy$default_payment == "0",])
only_1_dummy <- as.data.frame(credit_dummy[credit_dummy$default_payment == "1",])


balanced_train <- rbind( sample_n(only_1,5000) , sample_n(only_0,5000))
train_balanced_dummy <- rbind(sample_n(only_1_dummy,5000),sample_n(only_0_dummy,5000))





```




## Penalized Linear Regression

```{r }
#  unbalanced training set
glmmod = cv.glmnet(as.matrix(dummy_train[,1:(ncol(dummy_train)-1)]),dummy_train$default_payment, family = "binomial",type.measure = "class")


# balanced training set
glmmod_balanced = cv.glmnet(as.matrix(train_balanced_dummy[,1:(ncol(train_balanced_dummy)-1)]),train_balanced_dummy$default_payment, family = "binomial",type.measure = "class")

# evaluate test error performance on model trained with unbalanced data set
linear_regression <- function(x){
prediction_test = predict(glmmod,as.matrix(dummy_test[,1:(ncol(dummy_test)-1)]),type = "class",s = x)
error_unbalanced= confusionMatrix(as.factor(prediction_test), dummy_test$default_payment)

# evaluate test error performance on model trained with balanced data set
prediction_balanced = predict(glmmod_balanced,as.matrix(dummy_test[,1:(ncol(dummy_test)-1)]),type = "class",s = x)
error_balanced= confusionMatrix(as.factor(prediction_balanced), dummy_test$default_payment) 

print("Confusion matrix for model trained unbalanced data set")
print(error_unbalanced$table)      
print(paste("Accuracy for model trained unbalanced data set", error_unbalanced$overall[1]))

print("Confusion matrix for the model trained balanced data set")
print(error_balanced$table)
print(paste("Accuracy for model trained balanced data set", error_balanced$overall[1]))


}

```



```{r }
linear_regression(0.009)
```



```{r }
linear_regression(0.01)
```




```{r }
linear_regression(0.05)
```

 The data set does not have equally distributed target values.This causes imbalance problem.Therefore, two data set were examined. One is original data set other is created from original data set as equally distributed values. When looking at model trained with unbalanced data, different lambda values did not make a difference in terms of accuracy.Thus, model trained with equally distributed data were examined. The best results are obtaned with lambda = 0.05. Test accuracy is achieved as 0.79

## Decision Tree


```{r }
# Classification Tree



Decision_tree <- function(cp,minsplit){
class_tree = rpart(default_payment~.,train_credit, method = "class",  cp = cp, minsplit = minsplit )
class_tree_balanced = rpart(default_payment~.,balanced_train, method = "class",cp = cp, minsplit = minsplit  )



test_prediction_tree= predict(class_tree,test_credit, type = "class")
error_test_tree <- confusionMatrix(test_prediction_tree,test_credit$default_payment)

test_tree_balanced= predict(class_tree_balanced,test_credit, type = "class")
error_tree_balanced <- confusionMatrix(test_tree_balanced,test_credit$default_payment)

train_prediction<- predict(class_tree_balanced,balanced_train,type = "class")
error_train <- confusionMatrix(train_prediction,balanced_train$default_payment)



print("Confusion matrix for D.T.  trained with unbalanced data set")
print(error_test_tree$table)      
print(paste("Accuracy for D.T. trained with unbalanced data set", error_test_tree$overall[1]))

print("Confusion matrix for D.T. test data set")
print(error_tree_balanced$table)
print(paste("Accuracy for test data set", error_tree_balanced$overall[1]))

print(paste("Accuracy for balanced training data", error_train$overall[1]))


}
```




```{r }
Decision_tree(0.0009,30)
```





```{r }
Decision_tree(0.0001,30)

```




```{r }
Decision_tree(0.0001,10)

```

 Highest training accuracy (0.89) was obtained when cp and minsplit were equal to 0.0001 and 10. Although model had high training accuracy, it had lower prediction accuracy (0.71) than other models using different hyperparameter values.This means that this model tends to be overfitting. Optimum results was achieved when cp and minsplit were taken 0.0009 and 30 (training accuracy =0.744 and test accuracy = 0.743).

## Random Forest

```{r }

Random_forest <- function(numfeat){
RF = randomForest(train_credit[ ,1:(ncol(train_credit)-1)],train_credit$default_payment,type = "class" ,mtry = numfeat)

RF_balanced = randomForest(balanced_train[ ,1:(ncol(balanced_train)-1)],balanced_train$default_payment,type = "class", mtry = numfeat )


# Test data
prediction_test_forest = predict(RF,test_credit, type = "class")

error_test_forest <- confusionMatrix(prediction_test_forest,test_credit$default_payment)


prediction_balanced_forest = predict(RF_balanced,test_credit, type = "class")
error_balanced_forest <- confusionMatrix(prediction_balanced_forest,test_credit$default_payment)

train_prediction<- predict(RF_balanced,balanced_train,type = "class")
error_train <- confusionMatrix(train_prediction,balanced_train$default_payment)


print("Confusion matrix for R.F.  trained with unbalanced  data set")
print(error_test_forest$table)      
print(paste("Accuracy for R.F. trained with unbalanced data set ", error_test_forest$overall[1]))

print("Confusion matrix for test data set")
print(error_balanced_forest$table)
print(paste("Accuracy for test data set", error_balanced_forest$overall[1]))

print(paste("Accuracy for balanced training data", error_train$overall[1]))



}

```




```{r }
Random_forest(5)
```




```{r }
Random_forest(10)
```



```{r }
Random_forest(15)

```
 This simulation showed that random forest algorithm tended to be overfitting on this data. Training accuracy was about 1 but test accuracy was equal to 0.83. Nevertheless, the best test accuracy was obtained by random forest algorithm. When number of features which are randomly selected to evaluate which features are used for the aim of splitting was increased, it was observed that model mush more tended to be overfitting.


## Stochastic Gradient Boosting


```{r,warning = F }
# stochastic gradient boosting

control <- trainControl(method = "repeatedcv",number = 5, repeats = 2, allowParallel = T)
tunning  <- expand.grid(n.trees = c(100,150,200), interaction.depth=c(1:3), shrinkage=c(0.01,0.05,0.1), n.minobsinnode=c(20))
                                 
unwantedoutput =  capture.output(stochastic <- train(default_payment~., data = balanced_train,trControl = control,method = "gbm",tuneGrid = tunning))
print(stochastic)  

unwantedoutput =  capture.output(stochastic_unbalanced <- train(default_payment~., data = train_credit,trControl = control,method = "gbm",tuneGrid = tunning))
print(stochastic)  



```





```{r }
prediction_unbalanced <- predict(stochastic_unbalanced,test_credit)
error_unbalanced <- confusionMatrix(prediction_unbalanced,test_credit$default_payment)

error_unbalanced$table
error_unbalanced$overall[1]
```


```{r }
prediction_balanced <- predict(stochastic,test_credit)
error_balanced <- confusionMatrix(prediction_balanced,test_credit$default_payment)

error_balanced$table
print(paste("Accuracy for test set",error_balanced$overall[1]))
```




```{r }
train_prediction<- predict(stochastic,balanced_train)
error_train <- confusionMatrix(train_prediction,balanced_train$default_payment)

error_train$table
print(paste("Accuracy for training data",error_train$overall[1]))
```

 Number of trees, depth and alpha values are choosen 200,3 and 0.05 to acquire the best prediction result. However,training accuracy was inconsistent with test accuracy. Training error was higher than test error. I did not characterize model to be overfitting or underfitting. This might result from my coding mistake. To summarize, random forest algorithm gave best accuracy among other learners


```{r }

```



```{r }

```




```{r }

```



```{r }

```


```{r }

```



```{r }

```


```{r }

```



```{r }

```


```{r }

```


```{r }

```


