retrieve_catalog_metadata.rb

#!/usr/bin/env ruby
#

require 'httparty'
require 'json'
require 'pry'

by_categories = Hash.new { |h,k| h[k] = Array.new }
page = 1
total = 0
collected = 0
begin
  # Grab our current page of results and accumulate
  results = HTTParty.get("http://data.cityofchicago.org/api/search/views.json?limitTo=TABLES&limit=100&page=#{page}")
  total = results["count"]

  results["results"].each do |dataset|
    dataset = dataset["view"]
    puts "yes" if dataset.has_key? 'flags'
    puts "no" if dataset.has_key? 'filteredView'

    begin
      size = dataset["columns"].collect { |c| c["cachedContents"]["null"] + c["cachedContents"]["non_null"] }.max
    rescue
      binding.pry
    end
    by_categories[dataset["category"]] << {
      "url" => "https://data.cityofchicago.org/d/#{dataset["id"]}",
      "name" => dataset["name"],
      "size" => size,
      "value" => 42 # Not sure how this is calculated
    }

    collected += 1
  end

  # To the next page and beyond!
  page += 1
end while collected < total

# Formulate the final hash
output = {
  "name" => "Chicago Data Portal",
  "children" => by_categories.collect { |cat,ds| { "name" => cat, "children" => ds } }
}

puts JSON.pretty_generate(output)