Homework 5.Rmd

---
title: "Homework 5"
author: "Jiawen Qi (jiq10)"
date: "November 13, 2016"
output: word_document
---

```{r setup, include=FALSE}
knitr::opts_chunk$set(echo = TRUE)
```

## 1. Import the iris data

Iris data can be accessed in RStudio directly:

```{r}
iris <- iris # read in the data
summary(iris) # give a summary of iris data
head(iris) # view the first six rows of iris data
```

## 2. Data preparation for 10 fold

```{r}
set.seed(66666) # my favorite seed
iris$random.number <- runif(150, min = 0, max = 1) # add a random column
iris <- iris[order(iris$random.number),] # order the data by random number
for (i in 1:nrow(iris)) {
  iris$group[i] <- (i%%10) + 1 # make 10 groups 
}
iris <- iris[,-6]
```

By now, the dataset is ordered randomly and we can do 10 fold.

## 3. Create Naive Bayes Classification Cost and Benefit Function with 10 fold

```{r}
library(e1071) # Probability Theory Group
naiveBayes.CostBenefit <- function(dataset) {
  cost.benefit <- c(0,0,0,0,0,0,0,0,0,0) # initial cost and benefit
  for (i in 1:10) {
    training <- dataset[dataset$group != i, -c(ncol(dataset))] # get training
    testing <- dataset[dataset$group == i, -c(ncol(dataset))] # get testing
    testing.data <- testing[,-c(ncol(testing))] # get testing data
    testing.original <- as.data.frame(testing[,c(ncol(testing))]) # get testing original class
    colnames(testing.original) <- "OriginalSpecies"
    model <- naiveBayes(Species ~., data = training) # navie bayes model
    pred <- as.data.frame(predict(model, testing.data)) # predict
    colnames(pred) <- "PredictedSpecies"
    predTesting <- cbind.data.frame(pred, testing.original)
    confusion <- table(predTesting$PredictedSpecies, predTesting$OriginalSpecies) # get confusion matrix
    print(paste("naive bayes fold", i, "confusion matrix:"))
    print(confusion)
    ##### calculate the cost and benefit #####
    benefit <- confusion[1,1] * 1 + confusion[2,2] * 3 + confusion[3,3] * 1
    cost <- confusion[1,2] * 10 + confusion[3,2] * 10
    totalbenefit <- benefit - cost
    print(paste("naive bayes fold", i, ": benefit is", benefit, "; cost is", cost))
    cost.benefit[i] <- totalbenefit
  }
  print(paste("naive bayes total (benefit-cost) is", sum(cost.benefit)))
}
naiveBayes.CostBenefit.withoutPrint <- function(dataset) {
  cost.benefit <- c(0,0,0,0,0,0,0,0,0,0) # initial cost and benefit
  for (i in 1:10) {
    training <- dataset[dataset$group != i, -c(ncol(dataset))] # get training
    testing <- dataset[dataset$group == i, -c(ncol(dataset))] # get testing
    testing.data <- testing[,-c(ncol(testing))] # get testing data
    testing.original <- as.data.frame(testing[,c(ncol(testing))]) # get testing original class
    colnames(testing.original) <- "OriginalSpecies"
    model <- naiveBayes(Species ~., data = training) # navie bayes model
    pred <- as.data.frame(predict(model, testing.data)) # predict
    colnames(pred) <- "PredictedSpecies"
    predTesting <- cbind.data.frame(pred, testing.original)
    confusion <- table(predTesting$PredictedSpecies, predTesting$OriginalSpecies) # get confusion matrix
    ##### calculate the cost and benefit #####
    benefit <- confusion[1,1] * 1 + confusion[2,2] * 3 + confusion[3,3] * 1
    cost <- confusion[1,2] * 10 + confusion[3,2] * 10
    totalbenefit <- benefit - cost
    cost.benefit[i] <- totalbenefit
  }
  print(paste("naive bayes 10 fold total (benefit-cost) is", sum(cost.benefit)))
}
```

## 4. Create KNN Classification Cost and Benefit Function (k = 10) with 10 fold

```{r}
library(class) # functions for classification
knn10.CostBenefit <- function(dataset) {
  cost.benefit <- c(0,0,0,0,0,0,0,0,0,0) # initial cost and benefit
  for (i in 1:10) {
    training <- dataset[dataset$group != i, -c(ncol(dataset))] # get training
    testing <- dataset[dataset$group == i, -c(ncol(dataset))] # get testing
    training.data <- as.matrix(training[,-c(ncol(training))]) # get training data
    training.class <- as.matrix(training[,c(ncol(training))]) # get training class
    testing.data <- as.matrix(testing[,-c(ncol(testing))]) # get testing data
    testing.original <- as.data.frame(testing[,c(ncol(testing))]) # get testing class
    colnames(testing.original) <- "OriginalSpecies"
    model <- knn(train = training.data, test = testing.data, cl = training.class, k = 10)# knn model, k = 10
    pred <- as.data.frame(model)
    colnames(pred) <- "PredictedSpecies"
    result <- cbind.data.frame(pred, testing.original)
    confusion <- table(result$PredictedSpecies, result$OriginalSpecies) # get confusion matrix
    print(paste("knn k = 10 fold", i, "confusion matrix:"))
    print(confusion)
    ##### calculate the cost and benefit #####
    benefit <- confusion[1,1] * 1 + confusion[2,2] * 3 + confusion[3,3] * 1
    cost <- confusion[1,2] * 10 + confusion[3,2] * 10
    totalbenefit <- benefit - cost
    print(paste("knn k = 10 fold", i, ": benefit is", benefit, "; cost is", cost))
    cost.benefit[i] <- totalbenefit
  }
  print(paste("knn k = 10 10 fold total (benefit-cost) is", sum(cost.benefit)))
}
knn10.CostBenefit.withoutPrint <- function(dataset) {
  cost.benefit <- c(0,0,0,0,0,0,0,0,0,0) # initial cost and benefit
  for (i in 1:10) {
    training <- dataset[dataset$group != i, -c(ncol(dataset))] # get training
    testing <- dataset[dataset$group == i, -c(ncol(dataset))] # get testing
    training.data <- as.matrix(training[,-c(ncol(training))]) # get training data
    training.class <- as.matrix(training[,c(ncol(training))]) # get training class
    testing.data <- as.matrix(testing[,-c(ncol(testing))]) # get testing data
    testing.original <- as.data.frame(testing[,c(ncol(testing))]) # get testing class
    colnames(testing.original) <- "OriginalSpecies"
    model <- knn(train = training.data, test = testing.data, cl = training.class, k = 10)# knn model, k = 10
    pred <- as.data.frame(model)
    colnames(pred) <- "PredictedSpecies"
    result <- cbind.data.frame(pred, testing.original)
    confusion <- table(result$PredictedSpecies, result$OriginalSpecies) # get confusion matrix
    ##### calculate the cost and benefit #####
    benefit <- confusion[1,1] * 1 + confusion[2,2] * 3 + confusion[3,3] * 1
    cost <- confusion[1,2] * 10 + confusion[3,2] * 10
    totalbenefit <- benefit - cost
    cost.benefit[i] <- totalbenefit
  }
  average.cost.benefit <- sum(cost.benefit)/10
  print(paste("knn k = 10 10 fold total (benefit-cost) is", sum(cost.benefit)))
}
```

## 5. Create Decision Tree Classification Cost and Benefit Function (k = 10) with 10 fold

```{r}
library(rpart)
decisionTree.CostBenefit <- function(dataset) {
  cost.benefit <- c(0,0,0,0,0,0,0,0,0,0) # initial cost and benefit
  for (i in 1:10) {
    training <- dataset[dataset$group != i, -c(ncol(dataset))] # get training
    testing <- dataset[dataset$group == i, -c(ncol(dataset))] # get testing
    testing.data <- testing[,-c(ncol(testing))] # get testing data
    testing.original <- as.data.frame(testing[,c(ncol(testing))]) # get testing class
    colnames(testing.original) <- "OriginalSpecies"
    model <- rpart(Species ~., data = training) ## decision tree model
    pred <- predict(model, testing.data, type = "class") ## predict
    confusion <- table(pred, testing.original$OriginalSpecies) # get confusion matrix
    print(paste("decision tree fold", i, "confusion matrix:"))
    print(confusion)
    ##### calculate the cost and benefit #####
    benefit <- confusion[1,1] * 1 + confusion[2,2] * 3 + confusion[3,3] * 1
    cost <- confusion[1,2] * 10 + confusion[3,2] * 10
    totalbenefit <- benefit - cost
    print(paste("decision tree fold", i, ": benefit is", benefit, "; cost is", cost))
    cost.benefit[i] <- totalbenefit
  }
  print(paste("decision tree 10 fold total (benefit-cost) is", sum(cost.benefit)))
}
decisionTree.CostBenefit.withoutPrint <- function(dataset) {
  cost.benefit <- c(0,0,0,0,0,0,0,0,0,0) # initial cost and benefit
  for (i in 1:10) {
    training <- dataset[dataset$group != i, -c(ncol(dataset))] # get training
    testing <- dataset[dataset$group == i, -c(ncol(dataset))] # get testing
    testing.data <- testing[,-c(ncol(testing))] # get testing data
    testing.original <- as.data.frame(testing[,c(ncol(testing))]) # get testing class
    colnames(testing.original) <- "OriginalSpecies"
    model <- rpart(Species ~., data = training) ## decision tree model
    pred <- predict(model, testing.data, type = "class") ## predict
    confusion <- table(pred, testing.original$OriginalSpecies) # get confusion matrix
    ##### calculate the cost and benefit #####
    benefit <- confusion[1,1] * 1 + confusion[2,2] * 3 + confusion[3,3] * 1
    cost <- confusion[1,2] * 10 + confusion[3,2] * 10
    totalbenefit <- benefit - cost
    cost.benefit[i] <- totalbenefit
  }
  print(paste("decision tree 10 fold total (benefit-cost) is", sum(cost.benefit)))
}
```

## 5. All Features (Sepal Length, Sepal Width, Petal Length, Petal Width)

```{r}
dataset <- iris
naiveBayes.CostBenefit(dataset)
knn10.CostBenefit(dataset)
decisionTree.CostBenefit(dataset)
```

## 6. Three Features

Note: To make results more clear, confusion matrix will not be printed out.

### 6.1 Sepal Length, Sepal Width, Petal Length

```{r}
dataset <- iris[,-c(4)]
naiveBayes.CostBenefit.withoutPrint(dataset)
knn10.CostBenefit.withoutPrint(dataset)
decisionTree.CostBenefit.withoutPrint(dataset)
```

### 6.2 Sepal Length, Sepal Width, Petal Width

```{r}
dataset <- iris[,-c(3)]
naiveBayes.CostBenefit.withoutPrint(dataset)
knn10.CostBenefit.withoutPrint(dataset)
decisionTree.CostBenefit.withoutPrint(dataset)
```

### 6.3 Sepal Width, Petal Length, Petal Width

```{r}
dataset <- iris[,-c(1)]
naiveBayes.CostBenefit.withoutPrint(dataset)
knn10.CostBenefit.withoutPrint(dataset)
decisionTree.CostBenefit.withoutPrint(dataset)
```

### 6.4 Sepal Length, Petal Length, Petal Width

```{r}
dataset <- iris[,-c(2)]
naiveBayes.CostBenefit.withoutPrint(dataset)
knn10.CostBenefit.withoutPrint(dataset)
decisionTree.CostBenefit.withoutPrint(dataset)
```

## 7. Two Features

Note: To make results more clear, confusion matrix will not be printed out.

### 7.1 Sepal Length, Sepal Width

```{r}
dataset <- iris[,-c(3,4)]
naiveBayes.CostBenefit.withoutPrint(dataset)
knn10.CostBenefit.withoutPrint(dataset)
decisionTree.CostBenefit.withoutPrint(dataset)
```

### 7.2 Sepal Length, Petal Length

```{r}
dataset <- iris[,-c(2,4)]
naiveBayes.CostBenefit.withoutPrint(dataset)
knn10.CostBenefit.withoutPrint(dataset)
decisionTree.CostBenefit.withoutPrint(dataset)
```

### 7.3 Sepal Length, Petal Width

```{r}
dataset <- iris[,-c(2,3)]
naiveBayes.CostBenefit.withoutPrint(dataset)
knn10.CostBenefit.withoutPrint(dataset)
decisionTree.CostBenefit.withoutPrint(dataset)
```

### 7.4 Sepal Width, Petal Length

```{r}
dataset <- iris[,-c(1,4)]
naiveBayes.CostBenefit.withoutPrint(dataset)
knn10.CostBenefit.withoutPrint(dataset)
decisionTree.CostBenefit.withoutPrint(dataset)
```

### 7.5 Sepal Width, Petal Width

```{r}
dataset <- iris[,-c(1,3)]
naiveBayes.CostBenefit.withoutPrint(dataset)
knn10.CostBenefit.withoutPrint(dataset)
decisionTree.CostBenefit.withoutPrint(dataset)
```

### 7.6 Petal Length, Petal Width

```{r}
dataset <- iris[,-c(1,2)]
naiveBayes.CostBenefit.withoutPrint(dataset)
knn10.CostBenefit.withoutPrint(dataset)
decisionTree.CostBenefit.withoutPrint(dataset)
```

## 8. One Feature

Note: To make results more clear, confusion matrix will not be printed out.

### 8.1 Sepal Length 

```{r}
dataset <- cbind.data.frame(dum = 0,iris[,-c(2,3,4)])
naiveBayes.CostBenefit.withoutPrint(dataset)
knn10.CostBenefit.withoutPrint(dataset)
decisionTree.CostBenefit.withoutPrint(dataset)
```

### 8.2 Sepal Width 

```{r}
dataset <- cbind.data.frame(dum = 0,iris[,-c(1,3,4)])
naiveBayes.CostBenefit.withoutPrint(dataset)
knn10.CostBenefit.withoutPrint(dataset)
decisionTree.CostBenefit.withoutPrint(dataset)
```

### 8.3 Petal Length

```{r}
dataset <- cbind.data.frame(dum = 0,iris[,-c(1,2,4)])
naiveBayes.CostBenefit.withoutPrint(dataset)
knn10.CostBenefit.withoutPrint(dataset)
decisionTree.CostBenefit.withoutPrint(dataset)
```

### 8.4 Petal Width 

```{r}
dataset <- cbind.data.frame(dum = 0,iris[,-c(1,2,3)])
naiveBayes.CostBenefit.withoutPrint(dataset)
knn10.CostBenefit.withoutPrint(dataset)
decisionTree.CostBenefit.withoutPrint(dataset)
```

## 9. Conclusion

Note: The results shown in tables are the total benefit-cost of 10 fold cv for each classification methods.

### 9.1 All Features

### 9.2 Three Features

### 9.3 Two Features

### 9.4 One Features

### 9.5 The Best Model