End to End learning of self-driving car in Udacity simulator

End to End learning of self-driving car in Udacity simulator

import os
import h5py
import keras
import numpy as np
from keras.preprocessing.image import ImageDataGenerator, array_to_img, img_to_array, load_img
from keras.models import Sequential
from keras.layers import Convolution2D, MaxPooling2D, ZeroPadding2D,Cropping2D 
from keras.layers import Activation, Dropout, Flatten, Dense, Lambda
from keras.utils.np_utils import to_categorical
from keras.regularizers import l2, activity_l2
import tensorflow as tf
import time

import json
import csv
import cv2
import matplotlib
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
%matplotlib inline


#  read csv file
lines =[]
with open('/data/driving_log.csv') as csvfile:
    reader = csv.reader(csvfile)
    for line in reader:
         lines.append(line)
            
            
# extract steering angle value
lines= np.array(lines)
lines=np.delete(lines, 0, 0)
str=lines[:,3]
str = str.astype('float32')
np.shape(str)  


#omit the images with 0 angle
msk2 = (str!=0)
lines2=lines[msk2]
str2=lines2[:,3]
str2 = str2.astype('float32')
np.shape(str2)


#read images and correct value of angle from left/right side images
images= []
measurements = []

for line in lines2:   
    for i in range(3):
        source_path=line[i]
        tokens =source_path.split('/')
        filename = tokens[-1]
        local_path = ("/data/IMG/"+filename)
        image= cv2.imread(local_path)
        images.append(image)
    correction = 0.2
    measurement = float (line[3])
    measurements.append(measurement)
    measurements.append(measurement+correction )
    measurements.append(measurement-correction )
    
print(len(images))
print(len(measurements))

X_train = np.array(images)
y_train = np.array(measurements
                   

#set the number of epoch and early stopping
nb_epoch = 10
ES=0.0001
data_augmentation = False
filepath='weights.{epoch:02d}-{loss:.2f}.hdf5'

train_data=X_train
train_labels=y_train 
train_data = train_data.astype('float32')
train_labels = train_labels.astype('float32')

print(np.shape(train_data))
print(np.shape(train_labels))
tf.python.control_flow_ops = tf


# create the model architecture and train and save it
model = Sequential()
model.add(Lambda(lambda train_data: train_data/127.5 - 1.,  input_shape=train_data.shape[1:]))
model.add(Cropping2D(cropping=((70,25),(0,0))))

model.add(Convolution2D(64, 3, 3, activation='relu'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))
model.add(Convolution2D(64, 3, 3, activation='relu'))
model.add(MaxPooling2D((2, 2), strides=(None)))

model.add(Convolution2D(128, 3, 3, activation='relu'))
model.add(MaxPooling2D((2, 2), strides=(None)))
model.add(Convolution2D(128, 3, 3, activation='relu'))

model.add(Flatten())
model.add(Dense(200, activation='relu', W_regularizer=l2(0.00001)))
model.add(Dropout(0.5))
model.add(Dense(200, activation='relu', W_regularizer=l2(0.00001)))
model.add(Dropout(0.5))
model.add(Dense(200, activation='relu', W_regularizer=l2(0.00001)))
model.add(Dropout(0.5))
model.add(Dense(1))

model.compile(optimizer="adam", loss="mse")
mc_cb = keras.callbacks.ModelCheckpoint(filepath, monitor='loss', verbose=1, save_best_only=False, save_weights_only=False, mode='auto')
es_cb = keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=ES, patience=0, verbose=1, mode='auto')


if not data_augmentation:
    print('Not using data augmentation.')
    history=model.fit(train_data, train_labels,nb_epoch=nb_epoch, validation_split=0.2, batch_size=32, shuffle=True, callbacks=[mc_cb, es_cb])
    
else:
    print('Using real-time data augmentation.')

    # this will do preprocessing and realtime data augmentation
    datagen = ImageDataGenerator(
        #rescale=1./127.5 - 1,
        featurewise_center=False,  # set input mean to 0 over the dataset
        samplewise_center=False,  # set each sample mean to 0
        featurewise_std_normalization=False,  # divide inputs by std of the dataset
        samplewise_std_normalization=False,  # divide each input by its std
        # rotation_range=10,
        #width_shift_range=0.05,
        #height_shift_range=0.5,
        #shear_range=0.3,
        #channel_shift_range=0.1,
        #zoom_range=0.1,
        #fill_mode='nearest'
        )
  
    history=model.fit_generator(datagen.flow(train_data, train_labels,
                        batch_size=32),
                        samples_per_epoch=train_data.shape[0]/2,
                        nb_epoch=nb_epoch,
                        callbacks=[mc_cb, es_cb]
                        )
    
model.save('model.h5')

plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])

plt.title('model loss')
plt.xlabel('epoch')
plt.ylabel('loss')
plt.legend(['loss', 'val_loss'], loc='upper right')
plt.show()
 
                   
# The idea of these code is inspired by two awesome works
      # "Building powerful image classification models using very little data" (by Mr.François Chollet)            
          # https://blog.keras.io/building-powerful-image-classification-models-using-very-little-data.html    
      # Q&A session of Behavior cloning in Udacity (by Mr.David Silver)            
                   


TOSHI STATS SDN. BHD. and I do not accept any responsibility or liability for loss or damage occasioned to any person or property through using materials, instructions, methods, algorithm or ideas contained herein, or acting or refraining from acting as a result of such use. TOSHI STATS SDN. BHD. and I expressly disclaim all implied warranties, including merchantability or fitness for any particular purpose. There will be no duty on TOSHI STATS SDN. BHD. and me to correct any errors or defects in the codes and the software.                    
  

The goals / steps of this project are the following:
　- Build, a convolution neural network in Keras that predicts steering angles from images
　- Train and validate the model with a training and validation set
　- Test that the model successfully drives around track one without leaving the road

The video of self-driving simulation by deep learning above
https://youtu.be/x5iMEuNCf-0

If you are interested in the simulator, you can access it here.
Udacity's Self-Driving Car Simulator
https://github.com/udacity/self-driving-car-sim