Skip to content

Instantly share code, notes, and snippets.

@cbruegg
Created September 2, 2017 11:06
Show Gist options
  • Save cbruegg/0b3148c8caa29bc363fc06a47782bbf6 to your computer and use it in GitHub Desktop.
Save cbruegg/0b3148c8caa29bc363fc06a47782bbf6 to your computer and use it in GitHub Desktop.
Plot cryptocurrency difficulty correlations (quick and dirty)
package diffchecker
import org.jsoup.Jsoup
import java.net.URL
import java.util.*
private val currencies = arrayOf("bitcoin", "xmr", "ltc", "eth", "etc", "dash")
private fun urlFor(currency: String) = "https://bitinfocharts.com/comparison/$currency-difficulty.html"
private val dataRegex = Regex("\\[new Date\\(\"([0-9/]*)\"\\),([0-9.E+]*)]")
private data class DiffAtDate(val date: Calendar, val diff: Double, val currency: String)
fun main(args: Array<String>) {
val dataByCurr = currencies.associate { it to currencyData(it) }
// Header
println(currencies.joinToString(separator = " "))
// Rows
for (curr in currencies) {
print("$curr ")
for (other in currencies) {
val currData = dataByCurr[curr]!!
val otherData = dataByCurr[other]!!
val matched = (currData + otherData)
.groupBy { Triple(it.date.get(Calendar.YEAR), it.date.get(Calendar.MONTH), it.date.get(Calendar.DAY_OF_MONTH)) }
.filterValues { it.size == 2 } // Only where matching dates was successful
.toList()
.sortedWith(compareBy({ it.first.first }, { it.first.second }, { it.first.third })) // Sort by year, month, DoM
.map { it.second.first { it.currency == curr }.diff to it.second.first { it.currency == other }.diff } // ~ Pair<Diff, Diff>
val pearson = matched.pearson()
print("$pearson ")
}
println()
}
}
private fun currencyData(currency: String): List<DiffAtDate> {
val url = urlFor(currency)
val html = Jsoup.parse(URL(url), 5000)
val scripts = html.body().getElementsByTag("script").first { "new Dygraph(" in it.html() }.html()
return dataRegex.findAll(scripts)
.map { match ->
val (dateStr, diff) = match.destructured
val (year, month, day) = dateStr.split('/').map(String::toInt)
val cal = Calendar.getInstance().apply { set(year, month - 1, day) }
DiffAtDate(cal, diff.toDouble(), currency)
}
.filter { it.date.get(Calendar.YEAR) == 2017 }
.distinctBy { it.date }
.sortedBy { it.date }
.toList()
}
private fun List<Pair<Double, Double>>.pearson(): Double {
val avgFirst = map { it.first }.average()
val avgSecond = map { it.second }.average()
val multipliedDevs = indices.map { i -> (this[i].first - avgFirst) * (this[i].second - avgSecond) }.sum()
val multipliedSqDevsSqrt = Math.sqrt(
indices.map { i -> Math.pow(this[i].first - avgFirst, 2.0) }.sum() *
indices.map { i -> Math.pow(this[i].second - avgSecond, 2.0) }.sum()
)
return multipliedDevs / multipliedSqDevsSqrt
}
corrplot(as.matrix(read.table("diff_corr.txt", header=TRUE)), method="color")
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment