Show how neural network can be used for feature extraction.

CNN-Feature extraction

import tensorflow as tf 
from tensorflow.keras import datasets, layers, models
import matplotlib.pyplot as plt
import numpy as np

## Load in data
(X_train, y_train), (X_test, y_test) = datasets.cifar10.load_data()

## preproccess data
X_train = X_train / 255
X_test = X_test / 255

def reduce_data(X_train, X_test, y_train, y_test, sample_size):
    return X_train[:sample_size], X_test[:sample_size], y_train[:sample_size], y_test[:sample_size]

X_train, X_test, y_train, y_test = reduce_data(X_train, X_test, y_train, y_test, 10000) 

## import tensorflow objects for modeling
from tensorflow.keras.models import Model, Sequential
from tensorflow.keras.layers import Dense, Flatten, Conv2D, MaxPooling2D

## create CNN for feature extraction
feature_extractor = models.Sequential([
    # cnn
    layers.Conv2D(filters=32, kernel_size=(3,3), activation='relu', input_shape=(32,32,3)),
    layers.MaxPool2D((2,2)),
    layers.Conv2D(filters=32, kernel_size=(3,3), activation='relu', input_shape=(32,32,3)),
    layers.MaxPool2D((2,2)),
    layers.Conv2D(filters=32, kernel_size=(3,3), activation='relu', input_shape=(32,32,3)),
    layers.MaxPool2D((2,2)),
    #dense
    layers.Flatten(input_shape=(32,32,3)),
    layers.Dense(1000, activation='relu'),
])

## create final cnn model by adding softmax layer to feature extractor
## check its summary and then train it
#Add layers for deep learning prediction
x = feature_extractor.output  
x = Dense(128, activation = 'sigmoid', kernel_initializer = 'he_uniform')(x)
prediction_layer = Dense(10, activation = 'softmax')(x)

cnn_model = Model(inputs=feature_extractor.input, outputs=prediction_layer)
cnn_model.compile(optimizer='adam',loss = 'sparse_categorical_crossentropy', metrics = ['accuracy'])
print(cnn_model.summary()) 

history = cnn_model.fit(X_train, y_train, epochs=10, validation_data = (X_test, y_test))

## import randomforrestclassifier form sklearn, and instaniate it
from sklearn.ensemble import RandomForestClassifier
RF_model = RandomForestClassifier(n_estimators = 50, random_state = 42)

## Use feature extractor to get inputs for randomforesttclassifier
X_for_RF = feature_extractor.predict(X_train) #This is out X input to RF
#Send test data through same feature extractor process
X_test_feature = feature_extractor.predict(X_test)

## Train the model 
RF_model.fit(X_for_RF, y_train)

## use cnn model to predict on test set
## use random forest model to predict on test set 
prediction_RF = RF_model.predict(X_test_feature)

prediction_cnn = cnn_model.predict(X_test)
predicted_cnn = prediction_cnn.argmax(axis=1)

## print the accuracy difference between a cnn model alone
## or a cnn model fed into a random forest classifier

from sklearn import metrics

print ("Accuracy CNN = ", metrics.accuracy_score(y_test, predicted_cnn))
print ("Accuracy CNN + RF = ", metrics.accuracy_score(y_test, prediction_RF))