Evaluate model

evaluate.py

from sklearn.metrics import confusion_matrix
import matplotlib.pylab as plt

# Train/Test split was done earlier hence I just need to use Xtest and Ytest
result = []
for feature in Xtest:
   p = get_prediction(feature, model_name)
   result.append(p.payload[0].display_name)
  
labels = ['Optional', 'Trivial', 'Minor', 'Major', 'Critical', 'Blocker']

label_map = {"Optional": 0,
             "Trivial": 1,
             "Minor": 2,
             "Major": 3,
             "Critical": 4,
             "Blocker": 5}

reverse_label_map = {0: "Optional",
                     1: "Trivial",
                     2: "Minor",
                     3: "Major",
                     4: "Critical",
                     5: "Blocker"}

# create confustion matrix
cm = np.zeros((6,6), dtype=int)

i = 0
for l in Ytest:
    actual = l
    predicted = result[i]
    act_idx = label_map[actual]
    pre_idx = label_map[predicted]
    cm[pre_idx, act_idx] += 1
    i = i + 1
    
print(cm)

print("label precision recall")
for label in range(6):
    print(f"{label:5d} {precision(label, cm):9.3f} {recall(label, cm):6.3f}")

print("\n")
print("precision total:", precision_macro_average(cm))
print("recall total:", recall_macro_average(cm))
print("accuracy:", accuracy(cm))

# Taken from https://stackoverflow.com/questions/19233771/sklearn-plot-confusion-matrix-with-labels/48018785
fig = plt.figure()
ax = fig.add_subplot(111)
cax = ax.matshow(cm)
plt.title('Confusion matrix of the classifier')
fig.colorbar(cax)
ax.set_xticklabels([''] + labels)
ax.set_yticklabels([''] + labels)
plt.xlabel('Predicted')
plt.ylabel('True')
plt.show()