trec_eval_viz.R

library(scales)
library(magrittr)
library(tidyverse)

theme_set(
  ggthemes::theme_fivethirtyeight(base_size = 10) +
    theme(legend.position = "right",
          legend.direction = "vertical",
          axis.title = element_text())
)

global <- read_csv("~/Downloads/trec2020newstrackbackgroundlinkingresults/_evaluations_trec_trec29_tables_news-bg.csv")

us <- dir("~/Downloads/trec2020newstrackbackgroundlinkingresults/", "trec_eval", full.names = T) %>% 
  set_names(gsub(".*results\\//(.*)\\..*", "\\1", .)) %>% 
  map_df(~read_tsv(., col_names = F), .id = "run") %>% 
  filter(!is.na(X2)) %>% # overall stats
  filter(X1 == "ndcg_cut_5") %>% 
  select(-X1) %>% 
  mutate(X3 = as.numeric(X3),
         run = gsub("tune_ners_embed", "mlt_tune_ners_sbert", run)) %>% 
  rename(
    topic = X2,
    val = X3
  )

us %>% 
  group_by(run) %>% 
  summarise(avg = mean(val)) %>% 
  filter(run != "mlt_tune_ners") %>% 
  bind_rows(
    data.frame(
      run = "TREC median",
      "avg" = global$median %>% mean()
    )
  )

dat <- us %>% 
  filter(run != "mlt_tune_ners") %>% 
  inner_join(global) %>% 
  rowwise() %>% 
  mutate(y_min = min(median, val),
         y_max = max(median, val),
         diff = val - median) %>% 
  ungroup()

dat %>% 
  ggplot(aes(topic, y= val, ymin = y_min, ymax = y_max)) +
  geom_line(aes(y = median, color = "TREC median"), size = 1) +
  geom_line(aes(y = max, color = "TREC max"), size = .2) +
  geom_line(aes(y = min, color = "TREC min"), size = .2) +
  geom_line(aes(color = "run")) +
  scale_y_continuous(breaks = c(0, .5, 1)) +
  scale_x_continuous(breaks = seq(890, 930, by = 5)) +
  scale_color_manual(values = c(
    "TREC median" = "grey",
    "TREC min" = muted("blue"),
    "TREC max" = muted("red"),
    "run" = "black"
    )) +
  facet_wrap(~run, ncol = 1) +
  labs(
    title = "OSC TREC News 2020 runs",
    y = "nDCG@5",
    x = "Topic",
    color = NULL
  ) +
  theme(legend.position = "right")

ggsave("trec2020_performance.png",
       width = 6, height = 4)