Summary evaluation app using Gradio

summary_evaluation.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import json

import gradio as gr


def read_jsonl(fname):
    with open(fname) as f:
        return [json.loads(line) for line in f]


# contains fields:
# id         (int) sample id
# order      (list of ["prediction", "label"] or other way around,
#             used for random display order of summaries)
# text       (str) text to summarize
# label      (str) true summary
# prediction (str) generated summary
data = read_jsonl("data/summary_example.jsonl")

# append-only log file
log_file = "./log_file.txt"

with gr.Blocks() as demo:
    # Introductory text
    gr.Markdown(
        "Which summary is better? "
        "Click previous/next to move between examples. "
        "Use the checkboxes below to rate the better summary."
    )

    # user row
    with gr.Row():
        user = gr.Textbox(label="Enter your username here", placeholder="Username")

    # Navigation row
    with gr.Row():
        # set to interactive=False at the start, this will update once sample_id changes
        previous_sample = gr.Button("previous", interactive=False)
        sample_id = gr.Number(
            0, label="Sample ID", minimum=0, maximum=len(data) - 1, precision=0
        )
        next_sample = gr.Button("next")

    # Full text row
    with gr.Row():
        text_box = gr.Textbox(
            data[0]["text"], label="Text to be summarized", interactive=False
        )

    # Two summaries row
    with gr.Row():
        summary_box_a = gr.Textbox(
            data[0][data[0]["order"][0]], label="Summary A", interactive=False
        )
        summary_box_b = gr.Textbox(
            data[0][data[0]["order"][1]], label="Summary B", interactive=False
        )

    # User response row
    with gr.Row():
        better = gr.Radio(
            ["A", "B", "equal"],
            label="Which summary is better?",
        )
        feedback = gr.Textbox(
            label="Optional feedback?", placeholder="Optional feedback..."
        )
        submit = gr.Button("Submit")

    # Navigation control functions
    previous_sample.click(lambda x: max(x - 1, 0), inputs=sample_id, outputs=sample_id)
    next_sample.click(
        lambda x: min(x + 1, len(data) - 1), inputs=sample_id, outputs=sample_id
    )

    # update summary texts
    sample_id.change(
        lambda i: (
            data[i]["text"],
            data[i][data[i]["order"][0]],
            data[i][data[i]["order"][1]],
        ),
        inputs=sample_id,
        outputs=[text_box, summary_box_a, summary_box_b],
    )

    def update_nav_buttons(value):
        # sets interactivity for previous_sample, next_sample
        if value <= 0:
            return (
                gr.update(interactive=False),
                gr.update(interactive=True),
            )
        elif value >= len(data) - 1:
            return (
                gr.update(interactive=True),
                gr.update(interactive=False),
            )
        else:
            return (
                gr.update(interactive=True),
                gr.update(interactive=True),
            )

    # disable prev/next based on index
    sample_id.change(
        update_nav_buttons,
        inputs=sample_id,
        outputs=[previous_sample, next_sample],
    )

    # re-enable submit after nav click
    sample_id.change(lambda: gr.update(interactive=True), inputs=[], outputs=submit)

    # deselect radio, clear feedback
    sample_id.change(lambda: gr.update(value=None), inputs=[], outputs=better)
    sample_id.change(lambda: gr.update(value=None), inputs=[], outputs=feedback)

    def process_submit(sample_id, username, choice, feedback):
        if choice is None:
            raise gr.Error("No selection made!")
        with open(log_file, "a") as f:
            print(
                json.dumps(
                    {
                        "id": sample_id,
                        "username": username,
                        "choice": choice,
                        "feedback": feedback,
                    }
                ),
                file=f,
            )
        return gr.update(interactive=False)

    better.change(lambda: gr.update(interactive=True), inputs=[], outputs=submit)
    feedback.change(lambda: gr.update(interactive=True), inputs=[], outputs=submit)

    # process submit click
    submit.click(
        process_submit, inputs=[sample_id, user, better, feedback], outputs=submit
    )


demo.launch()

summary_example.jsonl

{"id": 0, "order": ["label", "prediction"], "text": "This is a simple test text.", "label": "Test.", "prediction": "Test prediction."}
{"id": 1, "order": ["prediction", "label"], "text": "This is a simple test text with some extra info.", "label": "Test with info.", "prediction": "Test prediction."}
{"id": 2, "order": ["prediction", "label"], "text": "This is a simple test text like all others.", "label": "Test like all others.", "prediction": "Test prediction."}