Metaculus_value_estimate.R

# An estimate of the value of information of Metaculus questions.

## Libraries
library(ggplot2)
library(viridis)
library(ggsci)
library(ggthemes)

## Data
data <- NULL
## data <- read.table(file = 'results.tsv', sep = '\t', header = TRUE)
data <- read.csv("results3.csv", header=TRUE)
### View(data)
colnames(data)
getwd()
dim(data)

## Value
### Must be before any manipulations
data$value = data$scope * data$forecasting_fit * data$decision_relevance

## Helpers
save=TRUE## FALSE
saveImage = function(fileName){
  height = 4
  width = floor(height*(1+sqrt(5))/2)
  if(save){
    ggsave(paste(fileName, ".png", sep=""), width=width, height=height)
  } 
}

## scope

## Data processing
data$scope = ifelse(data$scope > 0.5, data$scope, 0) ## ifelse(data$scope > 0.5, data$scope+1, data$scope*2) ## 
scopeLabels = c("Hyperlocal", "Local", "Provincial", "National", "Regional", "Global")

### Simple plot
ggplot(data, aes(scope)) +
  geom_bar()+
  scale_x_continuous(breaks = c(0:5), labels = scopeLabels)
  
## Complex plot
title_text = "scope of a sample of Metaculus questions"
subtitle_text = ""
label_x_axis = "scope level"
label_y_axis = "Count"
ggplot(data, aes(scope)) +
  geom_bar(fill="darkblue")+
  scale_x_continuous(breaks = c(0:5), labels = scopeLabels)+
  theme_tufte() + 
  labs(
    title=title_text,
    subtitle=subtitle_text,
    x=label_x_axis, 
    y=label_y_axis 
  ) +
  theme(
    legend.title = element_blank(),
    plot.title = element_text(hjust = 0.5),
    legend.position="bottom",
    legend.box="vertical",
    axis.text.x=element_text(angle=60, hjust=1)
  )
saveImage("metaculus-scope")

## Decision-relevance

### Processing
data$decision_relevance = ifelse(data$decision_relevance < 5, data$decision_relevance, 4)
data$decision_relevance = ifelse(data$decision_relevance > 0.5, data$decision_relevance, 0) ## ifelse(data$scope > 0.5, data$scope+1, data$scope*2) ## 
decisionRelevanceLabels = c("Remote", "Very indirect", "Indirect", "Nearly direct", "Direct")

### Simple plot
ggplot(data, aes(decision_relevance)) +
  geom_bar()+
  scale_x_continuous(breaks = c(0:4), labels = decisionRelevanceLabels)

## Complex plot
title_text = "Decision-relevance of a sample of Metaculus questions"
subtitle_text = ""
label_x_axis = "Decision-relevance level"
label_y_axis = "Count"
ggplot(data, aes(decision_relevance)) +
  geom_bar(fill="darkblue")+
  scale_x_continuous(breaks = c(0:4), labels = decisionRelevanceLabels)+
  theme_tufte() + 
  labs(
    title=title_text,
    subtitle=subtitle_text,
    x=label_x_axis, 
    y=label_y_axis 
  ) +
  theme(
    legend.title = element_blank(),
    plot.title = element_text(hjust = 0.5),
    legend.position="bottom",
    legend.box="vertical",
    axis.text.x=element_text(angle=60, hjust=1)
  )
saveImage("metaculus-decision-relevance")

## Forecasting fit

### Processing
data$forecasting_fit = ifelse(data$forecasting_fit > 0.5, data$forecasting_fit, 1) ## ifelse(data$scope > 0.5, data$scope+1, data$scope*2) ## 
forecastingFitLabels = c("Terrible", "Not very good", "Good", "Very good")

### Simple plot
ggplot(data, aes(forecasting_fit)) +
  geom_bar()

### Complex plot  
title_text = "Forecasting fit of a sample of Metaculus questions"
subtitle_text = ""
label_x_axis = "Forecasting fit level"
label_y_axis = "Count"
ggplot(data, aes(forecasting_fit)) +
  geom_bar(fill="darkblue")+
  scale_x_continuous(breaks = c(1:4), labels = forecastingFitLabels)+
  theme_tufte() + 
  labs(
    title=title_text,
    subtitle=subtitle_text,
    x=label_x_axis, 
    y=label_y_axis 
  ) +
  theme(
    legend.title = element_blank(),
    plot.title = element_text(hjust = 0.5),
    legend.position="bottom",
    legend.box="vertical",
    axis.text.x=element_text(angle=60, hjust=1)
  )
saveImage("metaculus-forecasting-fit")

## Total importance

### Processing
colnames(data)

View(na.omit(data[data$value>70,]))
View(na.omit(data[data$value<5  & data$scope == 3,]))
View(na.omit(data[data$value==0,]))

### Simple plot
ggplot(data, aes(value)) +
  geom_bar()

## Complex plot
title_text = "Estimated value of Metaculus questions"
subtitle_text = "(in arbitrary units)"
label_x_axis = "value"
label_y_axis = "Count"
ggplot(data, aes(value)) +
  geom_bar(fill="darkblue")+
  theme_tufte() + 
  labs(
    title=title_text,
    subtitle=subtitle_text,
    x=label_x_axis, 
    y=label_y_axis 
  ) +
  theme(
    legend.title = element_blank(),
    plot.title = element_text(hjust = 0.5),
    plot.subtitle = element_text(hjust = 0.5),
    legend.position="bottom",
    legend.box="vertical",
    ## axis.text.x=element_text(angle=60, hjust=1)
  )

saveImage("metaculus-value")

### Complex plot, binned
title_text = "Estimated value of Metaculus questions"
subtitle_text = "(in arbitrary units)"
label_x_axis = "value"
label_y_axis = "Count"
ggplot(data, aes(value)) +
  geom_bar(fill="darkblue")+
  scale_x_binned()+
  theme_tufte() + 
  labs(
    title=title_text,
    subtitle=subtitle_text,
    x=label_x_axis, 
    y=label_y_axis 
  ) +
  theme(
    legend.title = element_blank(),
    plot.title = element_text(hjust = 0.5),
    plot.subtitle = element_text(hjust = 0.5),
    legend.position="bottom",
    legend.box="vertical",
    ## axis.text.x=element_text(angle=60, hjust=1)
  )

saveImage("metaculus-value-binned")