Comparing options for question set data model

form_document.json

{
  "id": 11,
  "name": "Example form with address history",
  "submission_email": "submissions@example.com",
  "privacy_policy_url": "https://www.gov.uk/help/privacy-notice",
  "form_slug": "example-form-with-address-history",
  "support_email": "example@example.com",
  "support_phone": null,
  "support_url": null,
  "support_url_text": null,
  "declaration_text": "",
  "question_section_completed": true,
  "declaration_section_completed": true,
  "created_at": "2024-08-20T08:07:51.809Z",
  "updated_at": "2024-08-20T10:16:11.630Z",
  "creator_id": 1,
  "organisation_id": 1,
  "what_happens_next_markdown": "This is an example",
  "payment_url": null,
  "start_page": 29,
  "pages": [
    {
      "id": 29,
      "question_text": "What’s your name?",
      "hint_text": "",
      "answer_type": "name",
      "next_page": 30,
      "is_optional": false,
      "answer_settings": {
        "input_type": "full_name",
        "title_needed": "false"
      },
      "created_at": "2024-08-20T08:08:07.977Z",
      "updated_at": "2024-08-20T08:08:07.977Z",
      "form_id": 11,
      "position": 1,
      "page_heading": null,
      "guidance_markdown": null,
      "is_repeatable": false,
      "routing_conditions": []
    },
    {
      "id": 30,
      "question_text": "What was your address?",
      "hint_text": "",
      "answer_type": "address",
      "next_page": 31,
      "is_optional": false,
      "answer_settings": {
        "input_type": {
          "uk_address": "true",
          "international_address": "true"
        }
      },
      "created_at": "2024-08-20T08:08:39.781Z",
      "updated_at": "2024-08-20T08:08:39.781Z",
      "form_id": 11,
      "position": 2,
      "page_heading": null,
      "guidance_markdown": null,
      "is_repeatable": false,
      "routing_conditions": []
    },
    {
      "id": 31,
      "question_text": "What date did you start living at this address?",
      "hint_text": "",
      "answer_type": "date",
      "next_page": 32,
      "is_optional": false,
      "answer_settings": {
        "input_type": "other_date"
      },
      "created_at": "2024-08-20T08:09:00.159Z",
      "updated_at": "2024-08-20T08:09:00.159Z",
      "form_id": 11,
      "position": 3,
      "page_heading": null,
      "guidance_markdown": null,
      "is_repeatable": false,
      "routing_conditions": []
    },
    {
      "id": 32,
      "question_text": "What date did you stop living at this address?",
      "hint_text": "",
      "answer_type": "date",
      "next_page": 33,
      "is_optional": true,
      "answer_settings": {
        "input_type": "other_date"
      },
      "created_at": "2024-08-20T08:09:16.744Z",
      "updated_at": "2024-08-20T08:09:16.744Z",
      "form_id": 11,
      "position": 4,
      "page_heading": null,
      "guidance_markdown": null,
      "is_repeatable": false,
      "routing_conditions": []
    },
    {
      "id": 33,
      "question_text": "Which of these countries have you ever lived in?",
      "hint_text": "",
      "answer_type": "selection",
      "next_page": null,
      "is_optional": true,
      "answer_settings": {
        "only_one_option": "0",
        "selection_options": [
          {
            "name": "England"
          },
          {
            "name": "Northern Ireland"
          },
          {
            "name": "Scotland"
          },
          {
            "name": "Wales"
          },
          {
            "name": "Another country"
          }
        ]
      },
      "created_at": "2024-08-20T10:15:14.052Z",
      "updated_at": "2024-08-20T10:16:05.979Z",
      "form_id": 11,
      "position": 5,
      "page_heading": null,
      "guidance_markdown": null,
      "is_repeatable": false,
      "routing_conditions": []
    }
  ]
}

form_question_sets.json

[{
  "id": 1,
  "name": "What addresses have you lived at in the past 3 years?",
  "question_ids": [30, 31, 32]
}]

option1.rb

# frozen_string_literal: true

require './shared'

################################################################
# create a form document in the new format
################################################################
def to_option1(form, question_sets)
  form = form.deep_dup
  question_sets = question_sets.deep_dup

  pages = form.delete('pages')

  # Turn pages into steps with questions
  steps = pages.map do |page|
    step = page.slice(*STEP_ATTRIBUTES)
    step['type'] = 'Question'
    step['data'] = page.slice(*QUESTION_ATTRIBUTES)
    step
  end

  # Put question sets into steps with question sets
  steps_question_set_id = question_sets.flat_map do |question_set|
    question_set['question_ids'].map do |question_id|
      [question_id, question_set['id']]
    end
  end.to_h
  steps_ids = steps.pluck('id')
  steps = steps
          .chunk { |step| steps_question_set_id[step['id']] || :_alone }
          .map do |key, steps_|
    if key == :_alone
      steps_.sole
    else
      question_set = question_sets.find { |question_set_| question_set_['id'] == key }

      {
        'id' => (steps_ids << steps_ids.max.next).last,
        'type' => 'QuestionSet',
        'data' => {
          'name' => question_set['name'],
          'steps' => steps_
        }
      }
    end
  end

  form['steps'] = steps
  option1__update_position_and_next_step_ids!(form)

  form
end

################################################################
# make sure the object linkage is consistent
################################################################
def option1__update_position_and_next_step_ids!(form)
  steps = form['steps']

  steps.each.with_index(1) do |step, position|
    step['position'] = position

    if step['type'] == 'QuestionSet'
      question_set = step['data']
      step['next_step_id'] = question_set['steps'].first['id']

      question_set['steps'].each.with_index(1) do |question_set_step, question_set_position|
        question_set_step['position'] = question_set_position
        question_set_step['next_step_id'] = question_set['steps'].dig(question_set_position, 'id')
      end

      question_set['steps'].last['next_step_id'] = nil
    else
      step['next_step_id'] = steps.dig(position, 'id')
    end
  end

  form
end

################################################################
# get a question or set
################################################################
def option1_step(form, step_id)
  steps = Enumerator.new do |yielder|
    form['steps'].each do |step|
      yielder << step

      next unless step['type'] == 'QuestionSet'

      step['steps'].each do |step_|
        yielder << step_
      end
    end
  end

  steps
    .find { |step| step['id'] == step_id }
end

################################################################
# get all questions in a set
################################################################
def option1_question_set(form, step_id)
  form['steps']
    .find { |step| step['id'] == step_id }
    .fetch('steps')
end

################################################################
# swap a question or set with the question or set before it
################################################################
def option1_move_up(form, step_id, across_set_boundaries: false)
  form = form.deep_dup
  steps = form['steps']

  all_identifiers = Enumerator.new do |yielder|
    steps.each_with_index do |step, index|
      yielder << [index]

      next unless step['type'] == 'QuestionSet'

      step['data']['steps'].each_index do |set_index|
        yielder << [index, 'data', 'steps', set_index]
      end
    end
  end.to_a

  step_identifiers = all_identifiers.find do |identifiers|
    step = steps.dig(*identifiers)
    step['id'] == step_id
  end

  cursor = all_identifiers.index(step_identifiers)

  return form if cursor.zero?

  higher_step_identifiers = all_identifiers[cursor.pred]

  case
  when step_identifiers.slice(..-2) == higher_step_identifiers.slice(..-2)
    steps_ = step_identifiers.many? ? steps.dig(*step_identifiers.slice(..-2)) : steps
    step = steps_.delete_at(step_identifiers.last)
    steps_.insert(higher_step_identifiers.last, step)

  when step_identifiers.one? && higher_step_identifiers.many? && !across_set_boundaries
    step = steps.delete_at(step_identifiers.last)
    steps.insert(higher_step_identifiers.first, step)
  when step_identifiers.many? && !across_set_boundaries
    return form

  when step_identifiers.one? && higher_step_identifiers.many? && across_set_boundaries
    question_set_steps = steps.dig(*higher_step_identifiers.slice(..2))
    step = steps.delete_at(step_identifiers.first)
    question_set_steps.insert(-1, step)
  when step_identifiers.many? && across_set_boundaries
    question_set_steps = steps.dig(*step_identifiers.slice(..-2))
    step = question_set_steps.delete_at(step_identifiers.last)
    steps.insert(step_identifiers.first, step)
  end

  option1__update_position_and_next_step_ids!(form)

  form
end

option2.rb

# frozen_string_literal: true

require './shared'

################################################################
# create a form document in the new format
################################################################
def to_option2(form, question_sets)
  form = form.deep_dup

  pages_question_set_id = question_sets.flat_map do |question_set|
    question_set['question_ids'].map do |question_id|
      [question_id, question_set['id']]
    end
  end.to_h
  form['pages'].each do |page|
    page['question_set_id'] = pages_question_set_id[page['id']] if pages_question_set_id[page['id']]
  end

  form['question_sets'] = question_sets.deep_dup

  # Add position_in_form, position_in_set, and next_page_id
  form['pages'].each { |page| page.transform_keys!('position' => 'position_in_form', 'next_page' => 'next_page_id') }
  option2__update_position_and_next_page_ids!(form)

  form
end

################################################################
# make sure the object linkage is consistent
################################################################
def option2__update_position_and_next_page_ids!(form)
  pages = form['pages']

  pages
    .chunk { |page| page['question_set_id'] || :_alone }
    .each.with_index(1) do |chunk, position_in_form|
      question_set_id, pages = chunk

      if question_set_id == :_alone
        page = pages.sole
        page['position_in_form'] = position_in_form
      else
        pages.each.with_index(1) do |page, position_in_set|
          page['position_in_set'] = position_in_set

          if position_in_set == 1
            question_set = form['question_sets'].find { |question_set_| question_set_['id'] == question_set_id }
            question_set['first_page'] = page['id']
            page['position_in_form'] = position_in_form
          else
            page['position_in_form'] = nil
          end
        end
      end
    end

  pages.each_cons(2) do |page, next_page|
    page['next_page_id'] = next_page['id']
  end
  pages.last['next_page_id'] = nil

  form
end

################################################################
# get a question
################################################################
def option2_page(form, page_id)
  form['pages']
    .find { |page| page['id'] == page_id }
end

################################################################
# get all questions in a set
################################################################
def option2_question_set(form, question_set_id)
  form['pages']
    .select { |page| page['question_set_id'] == question_set_id }
end

################################################################
# swap a question or set with the question or set before it
################################################################

# if we want to be able to move whole question sets we need two different
# methods for option 2, because there isn't a uniform representation
def option2_move_up(form, page_or_question_set_id, type: :question, across_set_boundaries: false)
  form = form.deep_dup
  pages = form['pages']

  if type == :question_set
    # moving the whole question set

    question_set = form['question_sets'].find { |question_set_| question_set_['id'] == page_or_question_set_id }
    first_page_index = pages.index { |page| page['id'] == question_set['first_page'] }

    return if first_page_index.zero?

    # rather than moving all the question set pages up lets move the higher page down
    higher_page_index = first_page_index.pred
    higher_page = pages.delete_at(higher_page_index)
    last_page_index = pages.rindex { |page| page['question_set_id'] == page_or_question_set_id }
    pages.insert(last_page_index - pages.length, higher_page)
  else
    page_index = pages.index { |page| page['id'] == page_or_question_set_id }

    return form if page_index.zero?

    page = pages[page_index]

    higher_page_index = page_index.pred
    higher_page = pages[higher_page_index]

    case
    when page['question_set_id'] == higher_page['question_set_id']
      pages.delete_at(page_index)
      pages.insert(higher_page_index, page)

    when !page['question_set_id'] && higher_page['question_set_id'] && !across_set_boundaries
      question_set = form['question_sets'].find do |question_set_|
        question_set_['id'] == higher_page['question_set_id']
      end
      higher_page_index = form['pages'].find { |page_| page_['id'] == question_set['first_page'] }
      pages.delete_at(page_index)
      pages.insert(higher_page_index, page)
    when page['question_set_id'] && !across_set_boundaries
      return form

    when !page['question_set_id'] && higher_page['question_set_id'] && across_set_boundaries
      question_set = form['question_sets'].find do |question_set_|
        question_set_['id'] == higher_page['question_set_id']
      end
      question_set['question_ids'] << page['id']
      page['question_set_id'] = higher_page['question_set_id']
    when page['question_set_id'] && across_set_boundaries
      question_set = form['question_sets'].find { |question_set_| question_set_['id'] == page['question_set_id'] }
      question_set['question_ids'].shift
      question_set['first_page'] = page['next_page_id']
      page.delete('question_set_id')
      page.delete('position_in_set')
    end
  end

  option2__update_position_and_next_page_ids!(form)

  form
end

option3a.rb

# frozen_string_literal: true

require './shared'

################################################################
# create a form document in the new format
################################################################
def to_option3a(form, question_sets)
  form = form.deep_dup
  question_sets = question_sets.deep_dup

  pages = form.delete('pages')

  # Turn pages into steps with questions
  steps = pages.map do |page|
    step = page.slice(*STEP_ATTRIBUTES)
    step['type'] = 'Question'
    step['data'] = page.slice(*QUESTION_ATTRIBUTES)
    step
  end

  # Add question sets
  question_sets.each do |question_set|
    index = steps.find_index { |step| step['id'] == question_set['question_ids'].first }
    steps.insert(
      index,
      {
        'id' => steps.pluck('id').max.next,
        'type' => 'QuestionSet',
        'data' => {
          'name' => question_set['name'],
          'step_ids' => question_set['question_ids']
        }
      }
    )
  end

  # Link steps to question sets
  steps_question_set_id = steps
                          .select { |step| step['type'] == 'QuestionSet' }
                          .flat_map do |question_set|
    question_set['data']['step_ids'].map do |question_id|
      [question_id, question_set['id']]
    end
  end
                          .to_h
  steps.each do |step|
    step['question_set_id'] = steps_question_set_id[step['id']] if steps_question_set_id[step['id']]
  end

  form['steps'] = steps
  option3a__update_position_and_next_step_ids!(form)

  form
end

################################################################
# make sure the object linkage is consistent
################################################################
def option3a__update_position_and_next_step_ids!(form)
  steps = form['steps']

  # Add position
  steps.each.with_index(1) do |step, position|
    step['position'] = position
  end

  # Add next_step_id
  steps.each_cons(2) do |step, next_step|
    step['next_step_id'] = next_step['id']
  end
  steps.last['next_step_id'] = nil

  form
end

################################################################
# get a question or set
################################################################
def option3a_step(form, step_id)
  form['steps']
    .find { |step| step['id'] == step_id }
end

################################################################
# get all questions in a set
################################################################
def option3a_question_set(form, step_id)
  form['steps']
    .select { |step| step['question_set_id'] == step_id }
end

################################################################
# swap a question or set with the question or set before it
################################################################
def option3a_move_up(form, step_id, across_set_boundaries: false)
  form = form.deep_dup
  steps = form['steps']

  step_index = steps.index { |step| step['id'] == step_id }

  return form if step_index.zero?

  step = steps[step_index]

  higher_step_index = step_index.pred
  higher_step = steps[higher_step_index]

  case
  when step['type'] == 'QuestionSet'
    higher_step = steps.delete_at(higher_step_index)
    last_step_index = steps.rindex { |step_| step_['question_set_id'] == step['id'] }
    steps.insert(last_step_index - steps.length, higher_step)

  when step['question_set_id'] == higher_step['question_set_id']
    steps.delete_at(step_index)
    steps.insert(higher_step_index, step)

  when !step['question_set_id'] && higher_step['question_set_id'] && !across_set_boundaries
    question_set = steps.find { |question_set_| question_set_['id'] == higher_step['question_set_id'] }
    higher_step_index = form['steps'].find { |step| step['id'] == question_set['first_step'] }

    steps.delete_at(step_index)
    steps.insert(higher_step_index, step)
  when step['question_set_id'] && !across_set_boundaries
    return form

  when !step['question_set_id'] && higher_step['question_set_id'] && across_set_boundaries
    question_set = steps.find { |question_set_| question_set_['id'] == higher_step['question_set_id'] }
    question_set['data']['step_ids'] << step['id']
    step['question_set_id'] = higher_step['question_set_id']
  when step['question_set_id'] && higher_step['type'] == 'QuestionSet' && across_set_boundaries
    question_set = steps.find { |question_set_| question_set_['id'] == step['question_set_id'] }
    question_set['data']['step_ids'].shift
    step.delete('question_set_id')

    steps.delete_at(step_index)
    steps.insert(higher_step_index, step)
  end

  option3a__update_position_and_next_step_ids!(form)

  form
end

option3b.rb

# frozen_string_literal: true

require './shared'

################################################################
# create a form document in the new format
################################################################
def to_option3b(form, question_sets)
  form = form.deep_dup
  question_sets = question_sets.deep_dup

  pages = form.delete('pages')

  # Turn pages into steps with questions
  steps = pages.map do |page|
    step = page.slice(*STEP_ATTRIBUTES)
    step['type'] = 'Question'
    step['data'] = page.slice(*QUESTION_ATTRIBUTES)
    step
  end

  # Add question sets
  question_sets.each do |question_set|
    question_set_start_id = steps.pluck('id').max.next

    first_step_index = steps.find_index { |step| step['id'] == question_set['question_ids'].first }
    steps.insert(
      first_step_index,
      {
        'id' => question_set_start_id,
        'type' => 'QuestionSetStart',
        'data' => {
          'name' => question_set['name'],
          'step_ids' => question_set['question_ids']
        }
      }
    )

    last_step_index = steps.find_index { |step| step['id'] == question_set['question_ids'].last }
    steps.insert(
      last_step_index + 1,
      {
        'id' => steps.pluck('id').max.next,
        'type' => 'QuestionSetEnd',
        'question_set_id' => question_set_start_id
      }
    )
  end

  # Link steps to question sets
  steps_question_set_id = steps
                          .select { |step| step['type'] == 'QuestionSetStart' }
                          .flat_map do |question_set|
    question_set['data']['step_ids'].map do |question_id|
      [question_id, question_set['id']]
    end
  end
                          .to_h
  steps.each do |step|
    step['question_set_id'] = steps_question_set_id[step['id']] if steps_question_set_id[step['id']]
  end

  form['steps'] = steps
  option3b__update_position_and_next_step_ids!(form)

  form
end

################################################################
# make sure the object linkage is consistent
################################################################
def option3b__update_position_and_next_step_ids!(form)
  steps = form['steps']

  # Add position
  steps.each.with_index(1) do |step, position|
    step['position'] = position
  end

  # Add next_step_id
  steps.each_cons(2) do |step, next_step|
    step['next_step_id'] = next_step['id']
  end
  steps.last['next_step_id'] = nil

  form
end

################################################################
# get a question or set
################################################################
def option3b_step(form, step_id)
  form['steps']
    .select { |step| step['id'] == step_id }
end

################################################################
# get all questions in a set
################################################################
def option3b_question_set(form, step_id)
  form['steps']
    .select { |step| step['question_set_id'] == step_id }
end

################################################################
# swap a question or set with the question or set before it
################################################################
def option3b_move_up(form, step_id, across_set_boundaries: false)
  form = form.deep_dup
  steps = form['steps']

  step_index = steps.index { |step| step['id'] == step_id }

  return form if step_index.zero?

  step = steps[step_index]

  higher_step_index = step_index.pred
  higher_step = steps[higher_step_index]

  case
  when step['type'] == 'QuestionSetStart'
    # rather than moving all the question set steps up lets move the higher step down
    higher_step = steps.delete_at(higher_step_index)
    last_step_index = steps.rindex { |step_| step_['question_set_id'] == step['id'] }
    steps.insert(last_step_index - steps.length, higher_step)
  when step['type'] == 'QuestionSetEnd'
    return form

  when step['question_set_id'] == higher_step['question_set_id']
    steps.delete_at(step_index)
    steps.insert(higher_step_index, step)

  when !step['question_set_id'] && higher_step['type'] == 'QuestionSetEnd' && !across_set_boundaries
    question_set = steps.find { |question_set_| question_set_['id'] == higher_step['question_set_id'] }
    higher_step_index = form['steps'].find { |step| step['id'] == question_set['first_step'] }

    steps.delete_at(step_index)
    steps.insert(higher_step_index, step)
  when step['question_set_id'] && higher_step['type'] == 'QuestionSetStart' && !across_set_boundaries
    return form

  when !step['question_set_id'] && higher_step['type'] == 'QuestionSetEnd' && across_set_boundaries
    question_set = steps.find { |question_set_| question_set_['id'] == higher_step['question_set_id'] }
    question_set['data']['step_ids'] << step['id']
    step['question_set_id'] = higher_step['question_set_id']

    steps.delete_at(step_index)
    steps.insert(higher_step_index, step)
  when step['question_set_id'] && higher_step['type'] == 'QuestionSetStart' && across_set_boundaries
    question_set = steps.find { |question_set_| question_set_['id'] == step['question_set_id'] }
    question_set['data']['step_ids'].shift
    step.delete('question_set_id')

    steps.delete_at(step_index)
    steps.insert(higher_step_index, step)
  end

  option3b__update_position_and_next_step_ids!(form)

  form
end

shared.rb

# frozen_string_literal: true

require 'active_support/core_ext/enumerable'
require 'active_support/core_ext/object/deep_dup'
require 'active_support/core_ext/string/inflections'

STEP_ATTRIBUTES = %w[id created_at updated_at form_id is_repeatable routing_conditions].freeze
QUESTION_ATTRIBUTES = %w[question_text hint_text answer_type is_optional answer_settings page_heading
                         guidance_markdown].freeze

test.rb

# frozen_string_literal: true

require 'debug'
require 'JSON'

require './option1'
require './option2'
require './option3a'
require './option3b'

FORM = JSON.load_file!('form_document.json')
QUESTION_SETS = JSON.load_file('form_question_sets.json')

FORM_OPTION1 = to_option1(FORM, QUESTION_SETS)
FORM_OPTION2 = to_option2(FORM, QUESTION_SETS)
FORM_OPTION3A = to_option3a(FORM, QUESTION_SETS)
FORM_OPTION3B = to_option3b(FORM, QUESTION_SETS)

%i[option1 option2 option3a option3b].each do |option|
  form = "form_#{option}".upcase.constantize
  move_up = method("#{option}_move_up")
  to_option = method("to_#{option}")

  raise "#{option}: move_up should not change form if step is already first" unless move_up.call(form, 29) == form
  raise "#{option}: move_up should not change form if step is first in set" unless move_up.call(form, 30) == form
  raise "#{option}: move_up twice should result in same form" unless move_up.call(move_up.call(form, 31), 30) == form

  question_sets1 = QUESTION_SETS.deep_dup
  question_sets1.first['question_ids'].shift

  actual = move_up.call(form, 30, across_set_boundaries: true)
  expected = to_option.call(FORM, question_sets1)

  raise "#{option}: move_up should remove step from set if crossing set boundaries is allowed" unless expected == actual

  question_sets2 = QUESTION_SETS.deep_dup
  question_sets2.first['question_ids'] << 33

  actual = move_up.call(form, 33, across_set_boundaries: true)
  expected = to_option.call(FORM, question_sets2)

  raise "#{option}: move_up should add step to set if crossing set boundaries is allowed" unless expected == actual

  form1 = FORM.deep_dup
  form1['pages'].insert(-2, form1['pages'].shift)

  actual = option == :option2 ? move_up.call(form, 1, type: :question_set) : move_up.call(form, 34)
  expected = to_option.call(form1, QUESTION_SETS)

  raise "#{option}: move_up should be able to move all questions in a set" unless expected == actual
end