Plot cryptocurrency difficulty correlations (quick and dirty)

calccorr.kt

package diffchecker

import org.jsoup.Jsoup
import java.net.URL
import java.util.*

private val currencies = arrayOf("bitcoin", "xmr", "ltc", "eth", "etc", "dash")
private fun urlFor(currency: String) = "https://bitinfocharts.com/comparison/$currency-difficulty.html"
private val dataRegex = Regex("\\[new Date\\(\"([0-9/]*)\"\\),([0-9.E+]*)]")

private data class DiffAtDate(val date: Calendar, val diff: Double, val currency: String)

fun main(args: Array<String>) {
  val dataByCurr = currencies.associate { it to currencyData(it) }

  // Header
  println(currencies.joinToString(separator = " "))

  // Rows
  for (curr in currencies) {
    print("$curr ")
    for (other in currencies) {
      val currData = dataByCurr[curr]!!
      val otherData = dataByCurr[other]!!

      val matched = (currData + otherData)
          .groupBy { Triple(it.date.get(Calendar.YEAR), it.date.get(Calendar.MONTH), it.date.get(Calendar.DAY_OF_MONTH)) }
          .filterValues { it.size == 2 } // Only where matching dates was successful
          .toList()
          .sortedWith(compareBy({ it.first.first }, { it.first.second }, { it.first.third })) // Sort by year, month, DoM
          .map { it.second.first { it.currency == curr }.diff to it.second.first { it.currency == other }.diff } // ~ Pair<Diff, Diff>

      val pearson = matched.pearson()
      print("$pearson ")
    }
    println()
  }
}

private fun currencyData(currency: String): List<DiffAtDate> {
  val url = urlFor(currency)
  val html = Jsoup.parse(URL(url), 5000)
  val scripts = html.body().getElementsByTag("script").first { "new Dygraph(" in it.html() }.html()
  return dataRegex.findAll(scripts)
      .map { match ->
        val (dateStr, diff) = match.destructured
        val (year, month, day) = dateStr.split('/').map(String::toInt)
        val cal = Calendar.getInstance().apply { set(year, month - 1, day) }
        DiffAtDate(cal, diff.toDouble(), currency)
      }
      .filter { it.date.get(Calendar.YEAR) == 2017 }
      .distinctBy { it.date }
      .sortedBy { it.date }
      .toList()
}

private fun List<Pair<Double, Double>>.pearson(): Double {
  val avgFirst = map { it.first }.average()
  val avgSecond = map { it.second }.average()
  val multipliedDevs = indices.map { i -> (this[i].first - avgFirst) * (this[i].second - avgSecond) }.sum()
  val multipliedSqDevsSqrt = Math.sqrt(
      indices.map { i -> Math.pow(this[i].first - avgFirst, 2.0) }.sum() *
          indices.map { i -> Math.pow(this[i].second - avgSecond, 2.0) }.sum()
  )
  return multipliedDevs / multipliedSqDevsSqrt
}

plotcorr.r

corrplot(as.matrix(read.table("diff_corr.txt", header=TRUE)), method="color")