Created
September 24, 2020 19:55
-
-
Save nathancday/89ba1578aa3e4f8eb39ca442e3a686c6 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
library(scales) | |
library(magrittr) | |
library(tidyverse) | |
theme_set( | |
ggthemes::theme_fivethirtyeight(base_size = 10) + | |
theme(legend.position = "right", | |
legend.direction = "vertical", | |
axis.title = element_text()) | |
) | |
global <- read_csv("~/Downloads/trec2020newstrackbackgroundlinkingresults/_evaluations_trec_trec29_tables_news-bg.csv") | |
us <- dir("~/Downloads/trec2020newstrackbackgroundlinkingresults/", "trec_eval", full.names = T) %>% | |
set_names(gsub(".*results\\//(.*)\\..*", "\\1", .)) %>% | |
map_df(~read_tsv(., col_names = F), .id = "run") %>% | |
filter(!is.na(X2)) %>% # overall stats | |
filter(X1 == "ndcg_cut_5") %>% | |
select(-X1) %>% | |
mutate(X3 = as.numeric(X3), | |
run = gsub("tune_ners_embed", "mlt_tune_ners_sbert", run)) %>% | |
rename( | |
topic = X2, | |
val = X3 | |
) | |
us %>% | |
group_by(run) %>% | |
summarise(avg = mean(val)) %>% | |
filter(run != "mlt_tune_ners") %>% | |
bind_rows( | |
data.frame( | |
run = "TREC median", | |
"avg" = global$median %>% mean() | |
) | |
) | |
dat <- us %>% | |
filter(run != "mlt_tune_ners") %>% | |
inner_join(global) %>% | |
rowwise() %>% | |
mutate(y_min = min(median, val), | |
y_max = max(median, val), | |
diff = val - median) %>% | |
ungroup() | |
dat %>% | |
ggplot(aes(topic, y= val, ymin = y_min, ymax = y_max)) + | |
geom_line(aes(y = median, color = "TREC median"), size = 1) + | |
geom_line(aes(y = max, color = "TREC max"), size = .2) + | |
geom_line(aes(y = min, color = "TREC min"), size = .2) + | |
geom_line(aes(color = "run")) + | |
scale_y_continuous(breaks = c(0, .5, 1)) + | |
scale_x_continuous(breaks = seq(890, 930, by = 5)) + | |
scale_color_manual(values = c( | |
"TREC median" = "grey", | |
"TREC min" = muted("blue"), | |
"TREC max" = muted("red"), | |
"run" = "black" | |
)) + | |
facet_wrap(~run, ncol = 1) + | |
labs( | |
title = "OSC TREC News 2020 runs", | |
y = "nDCG@5", | |
x = "Topic", | |
color = NULL | |
) + | |
theme(legend.position = "right") | |
ggsave("trec2020_performance.png", | |
width = 6, height = 4) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment