Code.py

# -*- coding: utf-8 -*-
"""Graduation Project.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1nDOStvzX5XY5pxOBZWRj3bLH7W-kBTCj

importing a library that is not in Colaboratory
"""

!python -m pip install sidetable

"""**Upload of dataset**"""

import pandas as pd

url = 'https://raw.githubusercontent.com/sa190rah/AndroidMalwareDetection/main/drebin-215-dataset-5560malware-9476-benign.csv'
dataset = pd.read_csv(url)

dataset.head()

"""**Histogram chart**"""

import matplotlib.pyplot as plt
colunms = (dataset["transact"],dataset["onServiceConnected"],dataset["bindService"],dataset["attachInterface"],dataset["ServiceConnection"],
       dataset["android.os.Binder"],dataset["SEND_SMS"], dataset["Ljava.lang.Class.getCanonicalName"],dataset["Ljava.lang.Class.getMethods"],
       dataset["Ljava.lang.Class.cast"],dataset["Ljava.net.URLDecoder"],dataset["android.content.pm.Signature"],dataset["android.telephony.SmsManager"]
       )
plt.hist(colunms, range = (0,2))
plt.title("Histogram of Dataset", size=16)

"""**Check data balance**"""

import matplotlib.pyplot as plt
import seaborn as sns

plt.figure(figsize = (10, 5))
plot= dataset["class"].value_counts().reset_index()
graph = sns.barplot(x = "class", y = "class", data=plot)

plt.title("Plot of benign and malicious applications ", size = 18)
plt.xticks(size = 15)
plt.yticks(size = 15)

plt.show()

"""**Frequencies and Percentages**"""

import sidetable
dataset.stb.freq(['class'])

"""

**Feature correlation**"""

import seaborn as sns
import matplotlib.pyplot as plt

plt.figure(figsize=(150,150))

dataset_s = dataset.iloc[:,:215]
correlation = dataset_s.corr()
sns.heatmap(correlation, annot= True)

plt.title("Correlation matrix")
plt.xlabel("cell nucleus features")
plt.ylabel("cell nucleus features")

plt.show()

"""**Change last class Bening/malicious to 0/1**

B = bening = 0 /
S = Malicious = 1
"""

from sklearn.preprocessing import LabelEncoder 
  
le = LabelEncoder() 
  
dataset['class']= le.fit_transform(dataset['class'])

dataset = dataset.apply(pd.to_numeric, errors='coerce', downcast='float')

import numpy as np
dataset.replace([np.inf, -np.inf], np.nan, inplace=True)

dataset.fillna(999, inplace=True)

print("Number of Benign data",len(dataset[dataset["class"]==0]))
print("Number of Malware  data",len(dataset[dataset["class"]==1]))

"""**Train test split**

"""

from sklearn.model_selection import train_test_split

X = dataset.iloc[:,0:215].values
y = dataset.iloc[:,215].values

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=70)

print(X_train.shape, X_test.shape, y_train.shape, y_test.shape)

"""**Handle Missing data and percentage**"""

pd.set_option('display.max_rows', None)
pd.set_option('display.max_columns', None)

"""check missing data"""

dataset.isnull().sum()

"""percentage of all columns"""

#Percentage for each column
dataset.isna().mean()

#non-missing value for each column 
dataset.count()

#check class have no missing values by Return a boolean (false)
dataset['class'].isna()

"""**Resampling Technique**

Random under sampling
"""

import imblearn
from imblearn.under_sampling import RandomUnderSampler
from collections import Counter

rus = RandomUnderSampler(random_state=42, replacement=True)
# fit predictor and target varialbe
x_rus, y_rus = rus.fit_resample(X_train,y_train)


print('original dataset shape:', Counter(y_train))
print('Resample dataset shape', Counter(y_rus))

"""**Algorithms**

Multi-layer Perceptron(MLP)
"""

from sklearn.neural_network import MLPClassifier
MlpClassifier =MLPClassifier()
MlpClassifier.fit(x_rus , y_rus)
name='Multi-layer Perceptron'

"""Support Vector Machine(SVM)"""

from sklearn.svm import SVC
svclassifier = SVC(kernel='linear')
svclassifier.fit(x_rus , y_rus)
name='Support Vector Machine'

"""Stochastic Gradient Descent (SGD)"""

from sklearn.linear_model import SGDClassifier
SGDmodel =SGDClassifier()
SGDmodel.fit(x_rus , y_rus)
nameSGD ='SGDClassifier'

"""GaussianNB"""

from sklearn.naive_bayes import GaussianNB
GnbClassifier =GaussianNB()
GnbClassifier.fit(x_rus , y_rus)
name='GaussianNB'

"""K-Nearest Neighbours (Knn)"""

from sklearn.neighbors import KNeighborsClassifier
Knn = KNeighborsClassifier(n_neighbors=5)
Knn.fit(x_rus , y_rus)
name='K-Nearest Neighbours'

"""Adaptive Boost"""

from sklearn.ensemble import AdaBoostClassifier
AdaBoost= AdaBoostClassifier()
AdaBoost.fit(x_rus , y_rus)
name='AdaBoostClassifier'

"""

**Ensamble method**"""

from sklearn.ensemble import VotingClassifier
VotingClassifierModelone = VotingClassifier(
   estimators=[('Support Vector Machine', svclassifier),
                ('Multi-layer Perceptron', MlpClassifier),
                ('K-Nearest Neighbours', Knn),
                ('SGDClassifier', SGDmodel),
                ('AdaBoostClassifier', AdaBoost),
                ('GaussianNB', GnbClassifier)], 
                voting='hard')

VotingClassifierModelone.fit(x_rus,y_rus)

"""**Cross Validation Testing**

> Upload of new dataset


"""

import pandas as pd

url = 'https://raw.githubusercontent.com/nourasd/dataset/main/malgenome215dataset1260malware2539benign.csv'
dataset2 = pd.read_csv(url)

dataset2.head()

"""

> Encoding target colunm on new dataset

"""

def target(tipo):
    if tipo == 'B':
        return 0
    else:
        return 1

dataset2['class'] = dataset2['class'].apply(target)
dataset2.head()

"""**Cross validation**"""

from sklearn.model_selection import train_test_split

X = dataset2.iloc[:,0:215].values
y = dataset2.iloc[:,215].values

CVX_train, CVX_test, CVy_train, CVy_test = train_test_split(X, y, test_size=0.4, random_state=70)

print(CVX_train.shape, CVX_test.shape, CVy_train.shape, CVy_test.shape)

from sklearn.model_selection import cross_val_score
print(cross_val_score(svclassifier, CVX_test, CVy_test, cv=10, scoring ='accuracy').mean())
print(cross_val_score(MlpClassifier, CVX_test, CVy_test, cv=10, scoring ='accuracy').mean())
print(cross_val_score(Knn, CVX_test, CVy_test, cv=10, scoring ='accuracy').mean())
print(cross_val_score(SGDmodel, CVX_test, CVy_test, cv=10, scoring ='accuracy').mean())
print(cross_val_score(AdaBoost, CVX_test, CVy_test, cv=10, scoring ='accuracy').mean())
print(cross_val_score(GnbClassifier, CVX_test, CVy_test, cv=10, scoring ='accuracy').mean())

"""**Handling Duplicate**"""

#Identify duplicates records in the data
dupes=dataset.duplicated()
sum(dupes)

#Identify number of row before delete duplicate
length1 = len(dataset) 
print(length1)

# Select duplicate rows except last occurrence based on all columns
duplicateRowsDF = dataset[dataset.duplicated(keep='last')]
print("Duplicate Rows except last occurrence based on all columns are :")
print(duplicateRowsDF)

"""**Remove Duplicate**"""

#Remove Duplicate fro the dataset
dataset.drop_duplicates(keep=False,inplace=True)

#Identify duplicates records in the data After delete it
dupes=dataset.duplicated()
sum(dupes)

#Identify number of row after delete duplicate
length1 = len(dataset) 
print(length1)

import matplotlib.pyplot as plt
colunms = (dataset["class"])
plt.hist(colunms, range = (0,2))
plt.title("Histogram of Dataset", size=16)

import sidetable
dataset.stb.freq(['class'])

"""**Train test split**"""

from sklearn.model_selection import train_test_split

X = dataset.iloc[:,0:215].values
y = dataset.iloc[:,215].values

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=70)

print(X_train.shape, X_test.shape, y_train.shape, y_test.shape)

"""**Random under sampling**"""

import imblearn
from imblearn.under_sampling import RandomUnderSampler
from collections import Counter

rus = RandomUnderSampler(random_state=42, replacement=True)
# fit predictor and target varialbe
x_rus, y_rus = rus.fit_resample(X_train,y_train)


print('original dataset shape:', Counter(y_train))
print('Resample dataset shape', Counter(y_rus))

"""**Algorithms**

Multi-layer Perceptron(MLP)
"""

from sklearn.neural_network import MLPClassifier
MlpClassifier =MLPClassifier()
MlpClassifier.fit(x_rus , y_rus)
name='Multi-layer Perceptron'

"""Support Vector Machine(SVM)"""

from sklearn.svm import SVC
svclassifier = SVC(kernel='linear')
svclassifier.fit(x_rus , y_rus)
name='Support Vector Machine'

"""Stochastic Gradient Descent (SGD)"""

from sklearn.linear_model import SGDClassifier
SGDmodel =SGDClassifier()
SGDmodel.fit(x_rus , y_rus)
nameSGD ='SGDClassifier'

"""GaussianNB

"""

from sklearn.naive_bayes import GaussianNB
GnbClassifier =GaussianNB()
GnbClassifier.fit(x_rus , y_rus)
name='GaussianNB'

"""K-Nearest Neighbours (Knn)"""

from sklearn.neighbors import KNeighborsClassifier
Knn = KNeighborsClassifier(n_neighbors=5)
Knn.fit(x_rus , y_rus)
name='K-Nearest Neighbours'

"""Adaptive Boost"""

from sklearn.ensemble import AdaBoostClassifier
AdaBoost= AdaBoostClassifier()
AdaBoost.fit(x_rus , y_rus)
name='AdaBoostClassifier'

"""**Ensamble method**"""

from sklearn.ensemble import VotingClassifier
VotingClassifierModel = VotingClassifier(
   estimators=[('Support Vector Machine', svclassifier),
                ('Multi-layer Perceptron', MlpClassifier),
                ('K-Nearest Neighbours', Knn),
                ('SGDClassifier', SGDmodel),
                ('AdaBoostClassifier', AdaBoost),
                ('GaussianNB', GnbClassifier)], 
                voting='hard')

VotingClassifierModel.fit(x_rus,y_rus)

"""**Cross Validation Testing**


"""

from sklearn.model_selection import cross_val_score
print(cross_val_score(svclassifier, CVX_test, CVy_test, cv=10, scoring ='accuracy').mean())
print(cross_val_score(MlpClassifier, CVX_test, CVy_test, cv=10, scoring ='accuracy').mean())
print(cross_val_score(Knn, CVX_test, CVy_test, cv=10, scoring ='accuracy').mean())
print(cross_val_score(SGDmodel, CVX_test, CVy_test, cv=10, scoring ='accuracy').mean())
print(cross_val_score(AdaBoost, CVX_test, CVy_test, cv=10, scoring ='accuracy').mean())
print(cross_val_score(GnbClassifier, CVX_test, CVy_test, cv=10, scoring ='accuracy').mean())