D2KB_eval.ipynb

Code to get Confidence weighted Metrics

true_pos = []
false_pos = []
false_neg = []
mctr = 1
for sample_sent, gold_sent in zip(sample, gold):
    pred_id = [str(instance['uri']) for instance in sample_sent['mentions']]
    gold_id = [str(instance['uri']) for instance in gold_sent['mentions']]
    intersect = []
    false_preds = []
    # Check for intersecting URIs
    for idx, uri in enumerate(pred_id):
        if uri in gold_id:
            intersect.append([idx, gold_id.index(uri)])
        else:
            # False Predictions from system
            false_preds.append(idx)
    # Missing preds from system
    missing_preds = len(gold_id) - len(intersect)

    print(len(sample_sent['mentions']), len(gold_sent['mentions']))
    
    tp = 0
    # Start with false preds from system for False Positives
    fp = sum([float(sample_sent['mentions'][fal_pred]['confidence']) for fal_pred in false_preds])
    fn = missing_preds
    for sample_mention_id, gold_mention_id in intersect:
        print(sample_mention_id, gold_mention_id)
        sample_mention = sample_sent['mentions'][sample_mention_id]
        gold_mention = gold_sent['mentions'][gold_mention_id]

        print('begin : ',sample_mention['begin'], gold_mention['begin'])
        print('begin : ',sample_mention['end'], gold_mention['end'])
        print('anchor : ', repr(sample_mention['anchor'].toPython()), repr(gold_mention['anchor'].toPython()))
        print('identRef sample : ',sample_mention['identRef'])
        print('identRef gold : ', gold_mention['identRef'])
        # strong match
        print('confidence : ', sample_mention['confidence'])
#         print('match score (me): ', me_strong_entity_matching(sample_mention, gold_mention))
        print('match score (ma): ', ma_strong_annotation_matching(sample_mention, gold_mention))
#         yt.append(mctr)
        if ma_strong_annotation_matching(sample_mention, gold_mention)==1:
            tp += float(sample_mention['confidence'])
        else:
#             if 'notInWiki' in gold_mention['identRef'][0]:
#                 fp += float(confidence)
            fp += float(sample_mention['confidence'])
        print()
    true_pos.append(tp)
    false_pos.append(fp)
    false_neg.append(fn)
    print('--')

F1_scores = []
pres_scores = []
recall_scores = []

for tp, fp, fn in zip(true_pos, false_pos, false_neg):
    pres_scores.append(tp / (tp+fp))
    recall_scores.append(tp/(tp + fn))
    F1_scores.append((2 * pres_scores[-1] * recall_scores[-1]) / (pres_scores[-1] + recall_scores[-1]))
    
print('Macro-F1', sum(F1_scores)/len(F1_scores))
print('Macro-Recall', sum(recall_scores)/len(recall_scores))
print('Macro-Precision', sum(pres_scores)/len(pres_scores))


precision = sum(true_pos) / (sum(true_pos)+sum(false_pos))
recall = sum(true_pos) / (sum(true_pos)+sum(false_neg))
F1 = (2 * precision * recall) / (precision + recall)
print('Micro-F1', F1)
print('Macro-Recall', recall)
print('Macro-Precision', precision)