Downloading S3 files locally to be used with Rails' Active Storage

instructions.md

Download the S3 bucket to your local ./storage folder

  cd <your_app>
  AWS_ACCESS_KEY_ID=<key> AWS_SECRET_ACCESS_KEY=<secret> aws s3 sync s3://<bucket_name> storage

And then run a script to add the missing subfolders expected by Active Storage

  #script.rb 

  storage_folder = Rails.root.join('storage') 
  files = storage_folder.children.select { |file| file.file? && !file.empty? }

  files.each do |path_name|
    dir, basename = path_name.split
    file_name = basename.to_s
    sub_folders = dir.join(file_name[0..1], file_name[2..3])
    sub_folders.mkpath # Create the subfolder used by active_record
    path_name.rename(dir + sub_folders + basename) # Renames file to be moved into subfolder
  end

In my case created a rake task to do everything at once:

  # development_s3.rake
  
  task :s3_to_local do
    
    access_key_id = Rails.application.credentials.dig(:aws, :access_key_id)
    secret_access_key = Rails.application.credentials.dig(:aws, :secret_access_key)
    bucket = "bucket_name"
    
    storage_folder = Rails.root.join('storage') 
    system("AWS_ACCESS_KEY_ID=#{access_key_id} AWS_SECRET_ACCESS_KEY=#{secret_access_key} aws s3 sync s3://#{bucket} #{storage_folder}")

    # Ignores sub_folders and .keep file
    images = storage_folder.children.select { |file| file.file? && !file.empty? }

    images.each do |path_name|
      dir, basename = path_name.split
      file_name = basename.to_s
      sub_folders = dir.join(file_name[0..1], file_name[2..3])
      sub_folders.mkpath # Create the subfolder used by active_record
      path_name.rename(dir + sub_folders + basename) # Renames file to be moved into subfolder
    end

  end