ORA response extraction

map_responses.rb

#!/usr/bin/env ruby
# This script is being used to parse the JSON data dump from a self-hosted edX server,
# transforming the relevant data into a spreadsheet listing all open-ended responses
# submitted through the system.

require 'set'
require 'json'
require 'nokogiri'
require 'csv'

def course_name(x)
  x.split('/combinedopenended').first.split('i4x://').last
end

def get_prompt(modulekey)
  key = modulekey.split('/').last
  org = course_name(modulekey).gsub('/','_')
  components = Dir.glob("courses/#{org}/**/#{key}.xml")
  file = components[0]
  xml = file && File.read(file)
  Nokogiri::HTML(xml).css('combinedopenended prompt').text.tap { |x| x.delete!("\n") }.strip
end

data = JSON.parse(File.read('dumpdata.json'))

users = []
data.select { |x| x['model'] == 'auth.user' }.each do |user|
  users[user['pk']] = {
    :username => user['fields']['username'],
    :email => user['fields']['email'],
    :id => user['pk']
  }
end


ora = data.select { |x| x['model'] == 'courseware.studentmodule' && x['fields']['module_type'] == 'combinedopenended' }


modules = {}
ora.each do |item|
  fields = item['fields']
  student = fields['student']
  module_key = fields['module_state_key']
  modules[module_key] ||= get_prompt(module_key)
  state = JSON.parse(fields['state'])
  task_states = state['task_states'].map { |task| JSON.parse(task) }
  task_states.map do |task|
    answer = task['child_state'] == 'done' ? task['child_history'][0]['answer'] : task['stored_answer']
    answer.delete!("\n") if answer
    answer = answer.strip if answer
    users[student] ||= {}
    users[student][module_key] = answer if answer
  end
end
users.delete_if { |u| u.nil? }

module_keys = Set.new(modules.keys.map{|x| course_name(x)})

csv_string = ''
module_keys.each do |mod|
  csv_string << "#{mod}\n"
  csv_string << CSV.generate do |csv|
    module_filter = modules.select { |k, v| course_name(k) == mod }
    csv << %w(ID username email).concat(module_filter.values)

    users.each do |user|
      responses = module_filter.map { |k, v| user[k] }
      csv << [user[:id], user[:username], user[:email]].concat(responses) if responses.any?{|x| !x.nil?}
    end
  end
  csv_string << "\n"

end

puts "#{csv_string}"