test_iioe.py

import numpy as np
from scipy.special import softmax

def get_results(confidence_threshold):
    print('confidence threshold:',confidence_threshold)
    
    predictions = []
    confidence = []
    exit_level = []
    
    #loop over predictions and determine inference exit device, prediction, and confidence, base upon threshold
    for i in range(len(x_test)):
        if device_confidence['end'][i] > confidence_threshold:
            predictions.append(device_predictions['end'][i])
            confidence.append(device_confidence['end'][i])
            exit_level.append(0)
        elif device_confidence['edge'][i] > confidence_threshold:
            predictions.append(device_predictions['edge'][i])
            confidence.append(device_confidence['edge'][i])
            exit_level.append(1)
        else:
            predictions.append(device_predictions['cloud'][i])
            confidence.append(device_confidence['cloud'][i])
            exit_level.append(2)

    predictions = np.array(predictions)
    confidence = np.array(confidence)
    exit_level =  np.array(exit_level)
    accuracy = (predictions == y_test).sum() / len(predictions)
    
    #percentage of inference on each device
    n_end = (exit_level == 0).sum()
    n_edge = (exit_level == 1).sum()
    n_cloud = (exit_level == 2).sum()
    
    #print stats
    print('acc:',accuracy)
    print('confidence:',confidence.mean())
    print('percent end:',n_end/len(y_test))
    print('percent edge:',n_edge/len(y_test))
    print('percent cloud:',n_cloud/len(y_test))
    print('\n\n')
    

#store test data predictions from each device
device_predictions = {}
#store prediction confidence from each device
device_confidence = {}

#end device predictions
outputs, offload_to_edge = device_models['end'].predict(x_test)
device_predictions['end'] = outputs.argmax(1)
device_confidence['end'] = softmax(outputs,axis=1).max(1)

#edge device predictions
outputs, offload_to_cloud = device_models['edge'].predict(offload_to_edge)
device_predictions['edge'] = outputs.argmax(1)
device_confidence['edge'] = softmax(outputs,axis=1).max(1)

#cloud device predictions
outputs = device_models['cloud'].predict(offload_to_cloud)
device_predictions['cloud'] = outputs.argmax(1)
device_confidence['cloud'] = softmax(outputs,axis=1).max(1)
test_thresholds = [0.,.8,.9,.95,1.]

#test the model at various IIoE confidence thresholds
for threshold in test_thresholds:
    get_results(threshold)