application.rb

require_relative 'boot'

require 'rails/all'
require 'csv'
require 'find'

# Enables the garbage collection profiler
GC::Profiler.enable

# Require the gems listed in Gemfile, including any gems
# you've limited to :test, :development, or :production.
Bundler.require(*Rails.groups)

module Rollbook
  class Application < Rails::Application
    # Settings in config/environments/* take precedence over those specified here.
    # Application configuration should go into files in config/initializers
    # -- all .rb files in that directory are automatically loaded.
    # config.threadsafe!

    # Custom directories with classes and modules you want to be autoloadable.
    config.enable_dependency_loading = true
    config.autoload_paths += Dir["#{Rails.root}/app/models/**/"].reject { |path| path.include?('doorkeeper') }
    config.autoload_paths += Dir["#{Rails.root}/app/presenters/**/**/"]
    config.autoload_paths += Dir["#{Rails.root}/lib"]
    config.autoload_paths += Dir["#{Rails.root}/lib/topic-locales-data-migrations"]
    config.autoload_paths += Dir["#{Rails.root}/lib/admin-permissions-data-migrations"]
    config.autoload_paths += Dir["#{Rails.root}/lib/account-and-org-cleanup-data-migrations"]
    config.autoload_paths += Dir["#{Rails.root}/client_lib/**"]

    # Add node_modules as asset source
    config.assets.paths << Rails.root.join("node_modules")

    # This is effectively always set to `true`, but logs instead of throwing
    # See config/initializers/0_rails_5_upgrade.rb
    config.action_controller.permit_all_parameters = false

    config.active_job.queue_adapter = :sidekiq

    # Only load the plugins named here, in the order given (default is alphabetical).
    # :all can be used as a placeholder for all plugins not explicitly named.
    # config.plugins = [:all]
    # Activate observers that should always be running.
    # config.active_record.observers = :cacher, :garbage_collector, :forum_observer

    # Set Time.zone default to the specified zone and make Active Record auto-convert to this zone.
    # Run "rake -D time" for a list of tasks for finding time zone names. Default is UTC.
    # config.time_zone = 'Central Time (US & Canada)'
    config.time_zone = 'UTC'
    # The default locale is :en and all translations from config/locales/*.rb,yml are auto loaded.

    config.i18n.default_locale = :en
    config.i18n.fallbacks = {"pt-PT" => "pt-BR"} # specifically needed for number formatting, since rails-i18n does not have pt-PT
    #
    # if you add another language here, be sure to add it to lib/locale.rb and add a key called text_<new_language> to the student.en.yaml file
    config.i18n.available_locales = [:en, :'en-GB', :ar, :cs, :es, :'es-419', :fr,
      :'fr-CA', :hi, :ko, :'pt-BR', :'pt-PT',:sk , :'zh-CN', :'zh-TW', :de, :nl, :id, :ja,
      :th, :ru, :it, :pl, :ro, :tr, :vi, :da, :el, :fi, :ms, :nb, :sv ]
    # Arabic is still experimental for all except Facebook
    if Rails.env.production? && ENV["CLUSTER"] != "facebook"
      config.i18n.available_locales -= %i(ar,hi)
    end

    # Temoprarily disable new locales in production
    if Rails.env.production?
      config.i18n.available_locales -= %i(da, el, fi, ms, nb, sv)
    end

    # Rails adds all .rb and .yml files from the config/locales directory to the translations load path, automatically.
    config.i18n.load_path += Dir[Rails.root.join('config', 'locales', 'pluralizations', '*.rb')]
    config.i18n.load_path += Dir[Rails.root.join('config', 'locales', 'models', '*.yml')]
    config.i18n.load_path += Dir[Rails.root.join('config', 'locales', '{countries,datetime_formats,timezones}', '*.{yml,rb}')].sort
    config.i18n.load_path += Dir[Rails.root.join('config', 'locales', 'admin', '*.yml')].sort
    config.i18n.load_path += Dir[Rails.root.join('config', 'locales', 'student', '*.yml')].sort
    config.i18n.load_path += Dir[Rails.root.join('config', 'locales', 'client_managed', '**', '*.yml')].sort
    # Google-overridden translation strings
    if ENV["CLUSTER"] == "google_external" || ENV["CLUSTER"] == "google_internal"
      config.i18n.load_path += Dir[Rails.root.join('config', 'locales', 'client_overridden', 'google', '*.yml')].sort
    end
    # Amazon-overridden translation strings
    if ENV["CLUSTER"] == "amazon"
      config.i18n.load_path += Dir[Rails.root.join('config', 'locales', 'client_overridden', 'amazon', '*.yml')].sort
    end
    # Facebook-overridden translation strings
    if ENV["CLUSTER"] == "facebook"
      config.i18n.load_path += Dir[Rails.root.join('config', 'locales', 'client_overridden', 'facebook', '*.yml')].sort
    end

    #
    # Configure the default encoding used in templates for Ruby 1.9.
    config.encoding = "utf-8"

    # Configure sensitive parameters which will be filtered from the log file.
    config.stripe_event_sensitive_params = %w(brand last4 exp_year exp_month)
    config.filter_parameters += [:password, :password_hash, :password_salt, :api_key, :secret_string, :oauth_secret,
                                 :google_auth] + config.stripe_event_sensitive_params

    # Use SQL instead of Active Record's schema dumper when creating the database.
    # This is necessary if your schema can't be completely dumped by the schema dumper,
    # like if you have constraints or database-specific column types
    config.active_record.schema_format = :sql

    config.middleware.use Rack::Attack

    config.connection_timeout_in_milliseconds = 300_000  # 5 minutes
    config.read_only_connection_timeout_in_milliseconds = 900_000  # 15 minutes
    config.reporting_connection_timeout_in_milliseconds = 3_600_000  # 1 hour
    config.reporting_connection_short_timeout_in_milliseconds = 50_000  # 50 seconds

    cache_store_options = {}
    if Rails.env.test?
      cache_store_options[:namespace] = "#{ENV['TEST_ENV_NUMBER'].to_i.to_s}"
    end
    config.cache_store = :dalli_store, ENV['MEMCACHE_URL'], cache_store_options

    # Define custom console prompt
    # From http://code.nevercraft.net/blog/2015/customizing-the-rails-console.html
    console do
      ARGV.push "-r", root.join("lib/console_prompt.rb")
    end

    # Add a catch-all route to render 404
    # This is in an after_initialize to avoid catching gems' routes
    config.after_initialize do |app|
      unless config.consider_all_requests_local
        # weird syntax: `match "*some_param"` will take any path
        # and pass it to controllers as params[:some_param]
        app.routes.append { match '*unmatched_route', to: 'application#raise_not_found!', via: :all }
      end
    end
  end
end

elwyn_ds4_report.rb

# This whole file is one big 🤦 and I'm sorry
module Widget
  module ElwynDs4Report
    def write_records(records)
      @account = @report.account
      if @account.is_elwyn? && @format == 'html' && @report.id == 283499
        content_type = "text/html"
        return records.to_a.to_paginated_html_for_elwyn(@tmp_results_file, only: @record_columns, header: @header, body_only: true, report: @report, paginate_on: :users_id)
      else
        super
      end
    end
  end
end 


class Array
  include ReportHelper
  def to_paginated_html_for_elwyn(io, options = {})
    # only do if the the first object doesn't respond to row
    raise ArgumentError, "First argument must be an IO-type object with a 'write' method (IO, ZipOutputStream, IOStream, etc). Use STDOUT if you want to see the output directly." if not io.respond_to? :write
    raise ArgumentError, "First element of array doesn't respond to method to_row" if not self.empty? and not self.first.respond_to?(:to_row)
    if options[:header_only] or (not options[:body_only] and not options[:footer_only])
      io << "<html><body><table>"

      # write the header into the file
      if options.has_key?(:header)
        header_row = options[:header]
      else
        header_row = self.first.export_columns(options)
      end
      io.write header_row.inject("<tr>") {|row, col| row + "<th>#{col}</th>"} + "</tr>"
    end
    # write each AR object into the file
    if options[:body_only] or (not options[:footer_only] and not options[:header_only])
      prev_page_val = ""
      self.each do |row_obj|
        io.write show_summary(options[:report], row_obj) if options[:report]
        if prev_page_val != row_obj[options[:paginate_on]] 
          paginate_style = 'style="page-break-before: always;"'
          prev_page_val = row_obj[options[:paginate_on]]
          #io.write header_row&.inject("<tr>") {|row, col| row + "<th>#{col}</th>"} + "</tr>"
        elsif prev_page_val == ""
          prev_page_val = row_obj[options[:paginate_on]]
        end
        io.write row_obj.to_row(options).inject("<tr #{paginate_style}>") {|row, col| row + "<td>#{col}</td>"} + "</tr>"
      end
    end
    if options[:footer_only] or (not options[:body_only] and not options[:header_only])
      io.write "</table></body></html>"
    end
  end
end

report_generator.rb

class Report::ReportGenerator
  prepend Widget::ElwynDs4Report

  def initialize(report, args = {})
    @report = report
    @format = args[:download_format].presence || 'csv'
    @is_recurring = args[:recurring]
    @uncompressed = args[:uncompressed]
    @tmpdir = Dir.mktmpdir
    set_record_columns
    set_file_name
    set_header
    @uses_utf8 = args[:uses_utf8] ? args[:uses_utf8] : @report.account.account_preference.uses_utf8_for_export
    @current_user_id = nil # used when splitting into multiple files (Elwyn only)
    set_tmp_results_file
  end

  def generate
    write_header
    process_records
    write_footer

    @tmp_results_file.flush if !@tmp_results_file.closed?
    @tmp_results_file.close if !@tmp_results_file.closed?

    zip_file_if_neccessary

    # return either the zipfile or the raw file if it didn't need zipping
    @zipped_filepath.presence || @filepath
  end

  def to_sql
    @report.find(:all, report_options, quick_filter&.to_sql).to_sql
  rescue
    nil
  end

  private

  def report_options
    { use_reporting_replica:  true,
      other_tables_to_join:   quick_filter&.other_tables_to_join }
  end

  def quick_filter
    return @quick_filter if defined?(@quick_filter)

    if @report.user.present?
      @quick_filter = ReportQuickFilter.build(@report, @report.user, @report.quick_filter)
    else
      @quick_filter = nil
    end
  end

  def set_tmp_results_file(user_id = nil)
    if user_id.present?
      @filepath = "#{@tmpdir}/#{@filename}_#{user_id}.#{@format}"
    else
      @filepath = "#{@tmpdir}/#{@filename}.#{@format}"
    end

    @tmp_results_file = File.new(@filepath, "w+")
  end

  def rename_tmp_file(affix:)
    File.rename(@tmp_results_file, "#{@tmpdir}/#{@filename}_#{affix}.#{@format}")
  end

  def set_record_columns
    if @format == 'html'
      @record_columns = @report.select_column_names.collect {|c| c.to_sym}
    else
      @record_columns = @report.select_column_names(:show_aggregate_cols => true).collect {|c| c.to_sym}.uniq
    end
  end

  def set_file_name
    # make file name consistent by not including timestamp
    if @is_recurring
      @filename = "#{@report.name.strip.gsub(/[<>*|:?"'\/\\]/, '_')}"
    else
      # not recuring, so add timestamp to file
      @filename = "#{@report.name.strip.gsub(/[<>*|:?"'\/\\]/, '_')}_#{Time.now.strftime("%m%d%Y%H%M")}"
    end
    @filename.force_encoding("ascii")
  end

  def set_header
    # the html export displays it like it does in the UI, otherwise display tabular- with all windowed data on every line.
    if @format == 'html'
      @header = @report.column_header_names.map { |h| h[1] }
    else
      @header = @report.column_header_names(:show_aggregate_cols => true).map { |h| h[1] }.uniq
    end

    # force UTF-8 on header values
    @header.map! {|i| i.force_encoding("utf-8")}
  end

  def write_header
    case @format
    when "xml"
      ::Builder::XmlMarkup.new(:target => @tmp_results_file).instruct!
      @tmp_results_file.write "\n<report-rows type='array'>"
    when "html"
      Array.new.to_html(@tmp_results_file, header_only: true, header: @header, only: @record_columns)
    when "xls"
      @book = Spreadsheet::Workbook.new
      sheet = @book.create_worksheet
      Array.new.to_xls(sheet, header_only: true, header: @header, only: @record_columns)
    when "csv"
      Array.new.to_csv(@tmp_results_file, header_only: true, header: @header, only: @record_columns, faster_csv_options: {force_quotes: true}, uses_utf8: @uses_utf8)
    when "pdf"
      @pdf = PdfReportGenerator.new(@tmp_results_file, {:report_name => @report.name, header_only: true, header: @header, only: @record_columns})
    end
  end

  def write_records(records)
    case @format
    when "xml"
      content_type = "text/xml"
      records.each do |record|
        @tmp_results_file.write record.to_xml(skip_instruct: true, only: @record_columns, root: 'report-row')
      end
    when "html"
      content_type = "text/html"
      records.to_a.to_html(@tmp_results_file, only: @record_columns, header: @header, body_only: true, report: @report)
    when "xls"
      sheet = @book.worksheets[0]
      records.to_a.to_xls(sheet, only: @record_columns, header: @header, body_only: true, keep_original_data_type: true)
    when "csv"
      content_type = "text/csv"
      records.to_a.to_csv(@tmp_results_file, body_only: true, only: @record_columns, header: @header, faster_csv_options: {force_quotes: true}, uses_utf8: @uses_utf8)
    when "pdf"
      content_type = "application/pdf"
      @tmp_results_file = @pdf.render(records)
    end
  end

  def process_records
    # IMPORTANT: grouped reports (i.e. collapsed_view = t) do not work with find_in_batches,
    # because the id of the primary table will not be available to order on.
    if @report.collapsed_view
      records = @report.find(:all, report_options, quick_filter&.to_sql)
      if split_into_multiple_files?
        write_multiple_files(records: records)
      else
        write_records(records)
      end
    # for reports that do not use groupings, we can use find_in_batches
    else
      # for pdfs, we limit it to 10,000 records
      if @format == 'pdf' && @report.type != 'ReportCustom'
        opts = report_options.merge({ limit: 10_000 })
      else
        opts = report_options
      end

      @report.find_in_batches(opts, quick_filter&.to_sql) do |recs|
        if split_into_multiple_files?
          write_multiple_files(records: recs)
        else
          write_records(recs)
        end
      end
    end
  end

  # currently only used by Elwyn.
  def write_multiple_files(records:)
    records.group_by { |record| record["users_id"] }.each_with_index do |records, index|
      user_id = records[0]
      user_records = records[1]
      @current_user_id = user_id if @current_user_id.nil?
      full_name = user_records.first["users_full_name"]
      affix = full_name.present? ? "#{user_id}_#{full_name.gsub(" ", "_")}" : user_id

      rename_tmp_file(affix: affix) if index == 0

      if @current_user_id == user_id
        write_records(user_records)
      else
        @current_user_id = user_id
        close_existing_csv if index > 0
        set_tmp_results_file(affix)
        write_header
        write_records(user_records)
      end
    end
  end

  def write_footer
    # write the footer for the report
    case @format
    when "xml"
      @tmp_results_file.write "</report-rows>"
    when "xls"
      # writes the Excel
      @book.write @tmp_results_file.path
    when "html"
      Array.new.to_html(@tmp_results_file, footer_only: true,  only: @record_columns)
    end
  end

  def zip_file_if_neccessary
    if split_into_multiple_files?
      add_tmp_folder_to_zip_file
    # if very large or `is_recurring` then ZIP it up
    # unless explicit `uncompressed` is set to true
    elsif !@uncompressed && (@is_recurring.present? || results_file_big_enough_to_require_zip?)
      add_tmp_file_to_zip_file
      @tmp_results_file.close if !@tmp_results_file.closed?
    end
  end

  def results_file_big_enough_to_require_zip?
    File.size?(@tmp_results_file.path).to_i > 100.megabytes
  end

  def add_tmp_file_to_zip_file
    # if exists open up zip file
    if @zipped_filepath
      tmp_zip_file = Zip::File.open(@zipped_filepath)
    else
      @zipped_filepath = "#{@tmpdir}/#{@filename}.zip"
      FileUtils.rm(@zipped_filepath) if File.exist?(@zipped_filepath)
      tmp_zip_file = Zip::File.new(@zipped_filepath, 'w+')
    end

    # add temp file to zip
    tmp_zip_file.add(File.basename(@tmp_results_file), @tmp_results_file.path)
    tmp_zip_file.close
  end

  def add_tmp_folder_to_zip_file
    # if exists open up zip file
    if @zipped_filepath
      tmp_zip_file = Zip::File.open(@zipped_filepath)
    else
      @zipped_filepath = "#{@tmpdir}/#{@filename}.zip"
      FileUtils.rm(@zipped_filepath) if File.exist?(@zipped_filepath)
      tmp_zip_file = Zip::File.new(@zipped_filepath, 'w+')
    end

    Dir.foreach(@tmpdir) do |item|
      tmp_zip_file.add(File.basename(item), "#{@tmpdir}/#{item}")
    end

    tmp_zip_file.close
  end

  def split_into_multiple_files?
    return @split_into_multiple_files if defined?(@split_into_multiple_files)
    @split_into_multiple_files = @report.account.is_elwyn? &&
                                 @format == "csv" &&
                                 @report.type == "ReportCourseEnrollment" &&
                                 @report.selections.include?("users.id")
  end

  def close_existing_csv
    write_footer
    @tmp_results_file.flush if !@tmp_results_file.closed?
    @tmp_results_file.close if !@tmp_results_file.closed?
  end

end

report_generator_test.rb

require 'test_helper'

class Report::ReportGeneratorTest < ActiveSupport::TestCase
  # these are mostly covered in Report Test
  # Todo migrate them

  setup do
    @account = FactoryBot.create(:account,  storage_limit: 5000000)
    @user = FactoryBot.create(:user, account: @account, authority: "superadmin", email: "next@admin.com")
    @report = ::ReportEnrollmentTask.new({"account_id" => @account.id, "selections"=>ReportEnrollmentTask::DEFAULT_SELECTIONS, "name"=>"1", "is_temporary"=>false, "view_min_authority"=>"report_creator", "groupings"=>"courses.id", "query"=>"enrollments.completed_on:btd"})
  end

  should 'initialize generator' do
    rg = Report::ReportGenerator.new(@report)
    assert rg
  end

  should 'generate csv report' do
    rg = Report::ReportGenerator.new(@report).generate
    assert File.exist?(rg)
    assert rg.include?('.csv')
  end

  should 'generate an html report' do
    rg = Report::ReportGenerator.new(@report, download_format: 'html').generate
    assert rg.include?('.html')
  end

  should 'generate an html report for elwyn' do
    @account.stubs(:is_elwyn?).returns(true)
    Account.any_instance.expects(:is_elwyn?).returns(true)
    course = FactoryBot.create(:course_self_post)
    FactoryBot.create(:enrollment, 
                      account_id: @account.id,
                      user_id: @user.id,
                      course_id: course.id,
                      status: 'passed', 
                      completed_on: Date.today)
    report = ReportCourseEnrollment.create!(id: 283499, 
                                           account_id: @account.id,
                                           user_id: @user.id,
                                           is_temporary: false,
                                           view_min_authority: "report_creator",
                                           name: "DS4 - Transcript Report",
                                           groupings: "users.full_name", 
                                           selections:"courses.duration:sum,users.last_name,users.first_name,\
                                                    users.custom_b,users.hired_on,enrollments.completed_on,\
                                                    courses.name,courses.duration,courses.type,users.id") 
    rg = Report::ReportGenerator.new(report, download_format: 'html').generate
    assert_operator File.read(rg).scan(/style="page-break-before: always;/).size, :>, 0
  end

  should 'generate a xml report' do
    rg = Report::ReportGenerator.new(@report, download_format: 'xml').generate
    assert rg.include?('.xml')
  end

  should 'generate a pdf report' do
    rg = Report::ReportGenerator.new(@report, download_format: 'pdf').generate
    assert rg.include?('.pdf')
  end

  should 'generate a xls report' do
    rg = Report::ReportGenerator.new(@report, download_format: 'xls').generate
    assert rg.include?('.xls')
  end

  should 'generate a report for reports with collapsed view' do
    @report.update_attributes!(collapsed_view: true)
    rg = Report::ReportGenerator.new(@report, download_format: 'csv').generate
    assert rg.include?('.csv')
  end

  context "splitting the files into multiple csvs for elwyn" do
    setup do
      @course_enrollment_report = ReportCourseEnrollment.new({"account_id" => @account.id, "selections"=>"users.full_name,courses.name,users.id", "name"=>"1", "is_temporary"=>false, "view_min_authority"=>"report_creator", "query"=>"courses.is_page_component:false"})
      @course_enrollment_report.account.stubs(:is_elwyn?).returns(true)
    end

    should 'put multiple files within zip file for ReportCourseEnrollment reports' do
      FactoryBot.create(:enrollment, :passed, account: @account)
      rg = Report::ReportGenerator.new(@course_enrollment_report, download_format: 'csv').generate
      assert rg.include?('.zip')
    end

    should 'generate a single csv if the user id column is not part of the selection ReportCourseEnrollment reports' do
      FactoryBot.create(:enrollment, :passed, account: @account)
      @course_enrollment_report.selections = "users.full_name,courses.name"
      rg = Report::ReportGenerator.new(@course_enrollment_report, download_format: 'csv').generate
      assert rg.include?('.csv')
    end

    should 'generate a single csv if the report is not of type ReportCourseEnrollment' do
      FactoryBot.create(:enrollment, :passed, account: @account)
      rg = Report::ReportGenerator.new(@report, download_format: 'csv').generate
      assert rg.include?('.csv')
    end
  end

  context "not splitting the files into multiple csvs for all other accounts" do
    setup do
      @account.stubs(:is_elwyn?).returns(false)
      @course_enrollment_report = ReportCourseEnrollment.new({"account_id" => @account.id, "selections"=>"users.full_name,courses.name,users.id", "name"=>"1", "is_temporary"=>false, "view_min_authority"=>"report_creator", "query"=>"courses.is_page_component:false"})
    end

    should 'generate a single csv for ReportCourseEnrollment reports' do
      FactoryBot.create(:enrollment, :passed, account: @account)
      rg = Report::ReportGenerator.new(@course_enrollment_report, download_format: 'csv').generate
      assert rg.include?('.csv')
    end

    should 'generate a single csv if the user id column is not part of the selection ReportCourseEnrollment reports' do
      FactoryBot.create(:enrollment, :passed, account: @account)
      @course_enrollment_report.selections = "users.full_name,courses.name"
      rg = Report::ReportGenerator.new(@course_enrollment_report, download_format: 'csv').generate
      assert rg.include?('.csv')
    end
  end
end