Ruby script to backup the raw data written to Emoncms

.gitignore

*.csv
*.gz
*.bz2

backup-emoncms.rb

#!/usr/bin/env ruby
#
# Ruby script to backup the raw data written to Emoncms
# Fetches the data year-by-year for each of your feeds.
#
# Files are written in the format:
#   emoncms-1287-bedroom-temperature-2023.csv
#
# You must first set the EMONCMS_API_KEY evironment variable:
#   export EMONCMS_API_KEY="c5347069da574350b088ba547ffe7dbc"
#
# Your read-only API key can be found here:
#  https://emoncms.org/user/view
#
# You might then want to compress the CSV files that were downloaded:
#   bzip2 *.csv
#
# Documentation about the Emoncms API is available here:
#  https://emoncms.org/site/api#feed
#

require 'net/http'
require 'json'

START_YEAR=2012
HOSTNAME='emoncms.org'

raise "Please set the EMONCMS_API_KEY environment variable" unless ENV['EMONCMS_API_KEY']


def get_feed(name, args={})
  
  uri = URI.parse("https://#{HOSTNAME}/feed/#{name}.json")
  uri.query = URI.encode_www_form(args.to_a)
  
  res = Net::HTTP.start(uri.host, uri.port, :use_ssl => true) do |http|
    req = Net::HTTP::Get.new(uri)
    req['Authorization'] = "Bearer " + ENV['EMONCMS_API_KEY'].to_s
    req['Accept'] = "application/json"

    http.request(req)
  end
  
  if res.code != '200'
    raise "Failed to fetch feed: #{res.message}"
  end
  
  return res
end

def get_feed_list()
  res = get_feed(:list)
  if res.content_type != 'application/json'
    raise "Feed response was not JSON: #{res.content_type}"
  end

  JSON.parse(res.body, :symbolize_names => true)
end

def get_feed_data(id, year)
  now = Time.now
  start_time = Time.new(year, 1, 1, 0, 0, 0, 0)
  end_time = Time.new(year, 12, 31, 0, 0, 0, 0)
  end_time = now if end_time > now

  res = get_feed(:data,
    :ids => id,
    :start => start_time.to_i,
    :end => end_time.to_i,
    :interval => 'original',
    :average => 0,
    :timeformat => 'unix',
    :csv => 1,
    :skipmissing => 1,
    :limitinterval => 0
  )
  
  return res.body
end


puts "Getting list of feeds"
feeds = get_feed_list
puts " => found #{feeds.length} feeds"

end_year = Time.now.year
feeds.each do |feed|
  puts
  puts "#{feed[:id]}: #{feed[:name]}"
  label = feed[:name].downcase.gsub(/\W+/,'-')

  (START_YEAR...end_year).each do |year|
    filename = "emoncms-#{feed[:id]}-#{label}-#{year}.csv"

    puts " Fetching data for #{year}"
    data = get_feed_data(feed[:id], year)
    if data.nil? or data.empty?
      puts " => no data returned"
    else
      puts " => writing #{data.length} bytes to #{filename}"
      File.open(filename, 'wb') { |file| file.write(data) }
    end
  end
end