Ruby script to download archived petition data

download-petitions.rb

require 'csv'
require 'faraday'
require 'json'
require 'uri'

url = URI.parse('https://petition.parliament.uk/archived/petitions.json')

response = Faraday.get(url)

unless response.success?
  raise RuntimeError, "Error fetching initial JSON"
end

json = JSON.parse(response.body)
last_page = URI.parse(json['links']['last'])
page_count = last_page.query.scan(/\d+/).first.to_i

CSV.open('petitions.csv', 'wb') do |csv|
  csv << [
    'ID',
    'Title',
    'Description',
    'URL',
    'State',
    'Signature Count',
    'Rejection',
    'Government Response',
    'Opened At',
    'Closed At',
    'Created At',
    'Updated At'
  ]

  $stdout.write("Fetching #{page_count} pages:\n")

  start = Time.now

  page_count.times do |page|
    $stdout.write('.')

    url.query = "page=#{page}"
    response = Faraday.get(url)

    unless response.success?
      raise RuntimeError, "Error fetching page #{page}"
    end

    json = JSON.parse(response.body)

    json['data'].each do |petition|
      attributes = petition['attributes']

      csv << [
        petition['id'],
        attributes['title'],
        attributes['description'],
        petition['links']['self'],
        attributes['state'],
        attributes['signature_count'],
        attributes['rejection'] ? attributes['rejection']['details'] : nil,
        attributes['government_response'] ? attributes['government_response']['details'] : nil,
        attributes['opened_at'],
        attributes['closed_at'],
        attributes['created_at'],
        attributes['updated_at']
      ]
    end
  end

  finish = Time.now
  duration = finish - start

  $stdout.write("\nFinished downloading in #{duration} seconds\n")
end