random_forest.py

import pandas as pd
import numpy
import pickle
import sklearn
import math
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from keras.layers import Input, Dense, Activation, LeakyReLU, BatchNormalization
from keras.layers.core import Layer, Dense, Dropout, Activation, Flatten, Reshape,Permute
from keras.layers import Input, merge, Convolution2D, MaxPooling2D, UpSampling2D, Reshape, core, Dropout,Lambda
from keras.layers.normalization import BatchNormalization
from keras.layers.convolutional import Convolution3D, MaxPooling3D, ZeroPadding3D , ZeroPadding3D , UpSampling3D
from keras.layers.convolutional import Convolution2D, MaxPooling2D, UpSampling2D, ZeroPadding2D,Conv2DTranspose
from keras.layers.convolutional import Convolution1D, MaxPooling1D
from keras.layers import Dense, Conv2D, MaxPooling2D, Dropout, Flatten,Conv2DTranspose,concatenate
from keras.layers.core import Dense, Dropout, Activation, Flatten
from keras.layers.convolutional import Convolution2D, MaxPooling2D,UpSampling2D
from keras.models import Sequential
from keras.optimizers import Adam,Adamax
import keras
from keras.utils import np_utils
from sklearn.ensemble import RandomForestClassifier
import h5py
from sklearn import metrics
import csv

chunksize = 12687
features = ['MachineIdentifier', 'ProductName', 'EngineVersion', 'AppVersion', 'AvSigVersion',
            'IsBeta', 'RtpStateBitfield', 'IsSxsPassiveMode', 'DefaultBrowsersIdentifier', 'AVProductStatesIdentifier',
            'AVProductsInstalled', 'AVProductsEnabled', 'HasTpm', 'CountryIdentifier', 'CityIdentifier', 'OrganizationIdentifier',
            'GeoNameIdentifier', 'LocaleEnglishNameIdentifier', 'Platform', 'Processor', 'OsVer', 'OsBuild', 'OsSuite', 'OsPlatformSubRelease',
            'OsBuildLab', 'SkuEdition', 'IsProtected', 'AutoSampleOptIn', 'PuaMode', 'SMode', 'IeVerIdentifier', 'SmartScreen', 'Firewall',
            'UacLuaenable', 'Census_MDC2FormFactor', 'Census_DeviceFamily', 'Census_OEMNameIdentifier', 'Census_OEMModelIdentifier', 'Census_ProcessorCoreCount',
            'Census_ProcessorManufacturerIdentifier', 'Census_ProcessorModelIdentifier', 'Census_ProcessorClass', 'Census_PrimaryDiskTotalCapacity',
            'Census_PrimaryDiskTypeName', 'Census_SystemVolumeTotalCapacity', 'Census_HasOpticalDiskDrive', 'Census_TotalPhysicalRAM', 'Census_ChassisTypeName',
            'Census_InternalPrimaryDiagonalDisplaySizeInInches', 'Census_InternalPrimaryDisplayResolutionHorizontal', 'Census_InternalPrimaryDisplayResolutionVertical',
            'Census_PowerPlatformRoleName', 'Census_InternalBatteryType', 'Census_InternalBatteryNumberOfCharges', 'Census_OSVersion', 'Census_OSArchitecture',
            'Census_OSBranch', 'Census_OSBuildNumber', 'Census_OSBuildRevision', 'Census_OSEdition', 'Census_OSSkuName',
            'Census_OSInstallTypeName', 'Census_OSInstallLanguageIdentifier', 'Census_OSUILocaleIdentifier', 'Census_OSWUAutoUpdateOptionsName',
            'Census_IsPortableOperatingSystem', 'Census_GenuineStateName', 'Census_ActivationChannel', 'Census_IsFlightingInternal',
            'Census_IsFlightsDisabled', 'Census_FlightRing', 'Census_ThresholdOptIn', 'Census_FirmwareManufacturerIdentifier',
            'Census_FirmwareVersionIdentifier', 'Census_IsSecureBootEnabled', 'Census_IsWIMBootEnabled', 'Census_IsVirtualDevice',
            'Census_IsTouchEnabled', 'Census_IsPenCapable', 'Census_IsAlwaysOnAlwaysConnectedCapable', 'Wdft_IsGamer',
            'Wdft_RegionIdentifier']
c = 0
x = []
y = []
def to_int(n) :
    n = list(n)
    sum = 0
    for i in n:
        sum += ord(i)
    return sum

"""with open('x1','rb') as f:
    x = pickle.load(f)
with open('y1','rb') as f:
    y = pickle.load(f)
print(y)
pca = PCA(n_components=20)
principal_components = pca.fit_transform(x)
principaldf = pd.DataFrame(data=principal_components)
print(principaldf)
print(pca.singular_values_)
x = pca.transform(x)
print(x)
m,n = x.shape
print(x.shape)
#x = x.reshape(1,x.shape[0],x.shape[1],1)
y = y.reshape(y.shape[0] * y.shape[1] * 1)
print(x.shape)
#y = numpy.array(y,dtype=numpy.float64)
X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.1, random_state=42)
"""
clf=RandomForestClassifier(n_estimators=200,verbose=1,n_jobs=7)

#Train the model using the training sets y_pred=clf.predict(X_test)
x = []
y = []
xmean = []
xstd = []
for ab in range(6) :
    with open('x_train'+ str(ab + 1), 'rb') as f:
        x = pickle.load(f)
    with open('y_train' + str(ab + 1), 'rb') as f:
        y = pickle.load(f)
    y = y.reshape(y.shape[0] * y.shape[1] * 1)
    x = numpy.float32(x)
    print(x.shape)
    clf.fit(x,y)

    # prediction on test set
    #y_pred=clf.predict(X_test)

    #Import scikit-learn metrics module for accuracy calculation
    # Model Accuracy, how often is the classifier correct?
    #print("Accuracy:",metrics.accuracy_score(y_test, y_pred))
    #predictions = classifier.predict_proba(X_test)[:,1]
    #print(predictions)

with open('result/random forest/result.csv', 'wb') as csvfile:
    spamwriter = csv.writer(csvfile, delimiter=' ')
    spamwriter.writerow(['MachineIdentifier'] +[',']+ ['HasDetections'])
    for ab in range(6):
        r = []
        with open('label'+ str(ab + 1),'rb') as f:
            r = pickle.load(f)
        r = r[:,0]
        print(r)
        test = []
        with open('x_test' + str(ab + 1),'rb') as f:
            test = pickle.load(f)
        """pca = PCA(n_components=20)
        principal_components = pca.fit_transform(test)
        principaldf = pd.DataFrame(data=principal_components)
        print(principaldf)
        print(pca.singular_values_)
        test = pca.transform(test)"""
        print (test.shape)
        print(numpy.isnan(test).any())
        print(numpy.isfinite(test).all())
        predictions = clf.predict_proba(test)[:,1]
        print(predictions)
        w = numpy.full(r.shape[0], ',', dtype=object)
        z = numpy.empty([r.shape[0], 3], dtype=object)
        z[:,0] = r
        z[:,2] = predictions
        z[:,1] = w
        spamwriter.writerows(z)

reader = []
with open('result/random forest/result.csv', 'r') as f:
    reader = csv.reader(f, delimiter=',', quoting=csv.QUOTE_NONE)
    reader = [[x.strip() for x in row] for row in reader]

#print(reader)

with open('result/random forest/result.csv', 'wb') as csvfile:
    spamwriter = csv.writer(csvfile, delimiter=',')
    spamwriter.writerows(reader)