Mastodon activity stats crawler

README.md

Scripts to crawl activity statistics from the Mastodon network and calculate stats. List of instances to crawl is fetched from instances.social API.

Usage:

ruby fetch.rb
ruby stats.rb

Intermediary results are saved into a JSON file called instances-stats.json

fetch.rb

require 'http'
require 'oj'
require 'ruby-progressbar'
require 'parallelize'

res         = HTTP.get('https://instances.social/api/1.0/instances/list', headers: { 'Authorization' => 'Bearer 9fZs3S6yPW6lvhW6YsrhAIHaefPhe2YLsM5c52cBwRfiPDAZbulOU437PnEEfzsbrBol0SppuZODcPClsPXvclNzFTPXfQfsQpbivawxXTLMCWiLka5tvQWhwAjVRsFW' }, params: { count: 0 }).to_s
res         = Oj.load(res)
list        = res['instances']
instances   = {}
progressbar = ProgressBar.create(total: list.size, format: '%c/%C |%w>%i| %e')

begin
  list.pmap(25) do |item|
    begin
      stats = Oj.load(HTTP.timeout(connect: 10, read: 10, write: 5).get("https://#{item['name']}/api/v1/instance/activity").to_s)
      instances[item['name']] = stats[1]
    rescue
      next
    ensure
      progressbar.increment
    end
  end
ensure
  File.write('instances-stats.json', Oj.dump(instances))
end

Gemfile

source :rubygems

gem 'ruby-progressbar'
gem 'parallelize'
gem 'oj'
gem 'http'

stats.rb

require 'oj'

instances = Oj.load(File.read('instances-stats.json'))

c = 0    # Number of processed instances
l = 0    # Number of weekly logins
r = 0    # Number of weekly registrations
s = 0    # Number of weekly statuses
t = 1724 # Hardcoded number of total known instances

instances.each_value do |item|
  next if item.nil?

  c += 1
  l += item['logins'].to_i
  r += item['registrations'].to_i
  s += item['statuses'].to_i
end

puts "Logins:\t\t#{l}\nRegistrations:\t#{r}\nStatuses:\t#{s}\nData from #{c} instances"
puts
puts "Average logins: #{l / c}\nExtrapolated to #{t} instances: #{(l / c) * t}"