app.py

import streamlit as st
import numpy as np
import pandas as pd
import json
import requests

@st.cache
def get_auth_token(host, user, pw):
    st.write("cache miss token!")
    url = f"{host}/api/auth"
    payload = {"username": user, "password": pw}
    response = requests.post(url, json=payload)
    try:
        token = response.json()["access_token"]
        return token
    except:
        return None

@st.cache   
def get_logs(host, token, limit=1000):
    st.write("cache miss logs!")
    url = f"{host}/api/projects/default/logs?exclude_training_data=false&limit={limit}"
    headers = {"Authorization": f"Bearer {token}"}
    try:
        data = requests.get(url, headers=headers).json()
        return data
    except:
        return None


@st.cache   
def get_nlu_data(host, token, limit=100):
    st.write("cache miss data!")    
    url = f"{host}/api/projects/default/data"
    headers = {"Authorization": f"Bearer {token}"}
    try:
        data = requests.get(url, headers=headers).json()
        return data
    except:
        return None
        

@st.cache
def combine_predictions_with_gold_labels(predictions, nlu_data):
    st.write("cache miss combine!")
    messages = {}
    for p in predictions:
        text = p["user_input"]["text"]
        predicted_intent = p["user_input"]["intent"]["name"]
        if text not in messages:
            messages[text] = {
                "model": p["model"],
                "predicted_intent": predicted_intent,
                "gold_intent": None,
                "class": "UNK"
            }
    for d in nlu_data:
        text = d["text"]
        gold_intent = d["intent"]
        if text in messages:
            predicted = messages[text]["predicted_intent"]
            messages[text]["gold_intent"] = gold_intent
            messages[text]["class"] = "true_pos" if gold_intent == predicted else "false_pos"

    intents = list(set([m["predicted_intent"] for m in messages.values()]))
    classes = ["true_pos", "false_pos", "UNK"]
    shape = (len(intents),len(classes))
    counts = np.zeros(shape)
    for t, m in messages.items():
        i = intents.index(m["predicted_intent"])
        j = classes.index(m["class"])
        counts[i, j] += 1

    data = pd.DataFrame(
        np.array(counts),
        columns=classes, index=intents)

    return data

##############################################################
#                                                            #
#                      STREAMLIT WADDUP                      #
#                                                            #
##############################################################

st.title("NLU prediction performance")

host = st.text_input('hostname', 'http://')
user = st.text_input('user', 'me')
pw = st.text_input('password', '13583754de25e3')


token = get_auth_token(host, user, pw)

st.subheader("Training data")
nlu_data = get_nlu_data(host, token)

st.subheader("NLU Predictions")
limit = st.number_input("Number of predictions to fetch", value=1000)
predictions = get_logs(host, token, limit=limit)


st.subheader("Intent classification")

predictions_with_gold_labels = combine_predictions_with_gold_labels(predictions, nlu_data)

n_intents = st.number_input("Number of intents to show", value=10)

order = st.selectbox("Order by:", ["UNK", "false_pos", "true_pos"], index=0)
data = predictions_with_gold_labels.nlargest(n_intents, order)

models = ["20200429-133947",
          "20200429-133720",
          "20200429-114937",
          "20200429-103348",
          "20200422-115149"]

options = st.multiselect(
    'Models',
    models,
    models)

st.write(data)
annotations = data.drop(columns="UNK")
st.bar_chart(annotations)