Remove Stray S3 Files

remove_stray_s3_files.rb

# Ever had an app/script that created a bunch of files in S3 without a matching database entry?
# This will find and remove any stray s3 files

# Requirements:
# Designed to be run in a rails console, with the `fog` gem installed

# Usage:
# Update `is_stray_file?` method to return the correct value for your use case
# Set ENV['AWS_ACCESS_KEY'] and ENV['AWS_SECRET_KEY']
# Optionally set ENV['AWS_BUCKET']

# Call `remove_stray_s3_files(ENV["AWS_BUCKET"], "aws_folder", true)` to find any stray items with AWS keys starting with `aws_folder`
# In test mode, it will output the list if files it will delete and counts of the total files matching the path, how many would be deleted, and how many would be kept. 
# Use this output data to verify everything is correct.
# Call `remove_stray_s3_files(ENV["AWS_BUCKET"], "aws_folder", false)` to perform the actual delete operations.

def remove_stray_s3_files(bucket = ENV["AWS_BUCKET"], path = "/", test_mode = true)
  aws = Fog::Storage::AWS.new( { aws_access_key_id: ENV["AWS_ACCESS_KEY"], aws_secret_access_key: ENV["AWS_SECRET_KEY"], path_style: true } )
  directory = aws.directories.get(bucket, prefix: path)
  
  total_count = 0
  delete_count = 0
  
  files = directory.files.all
  files.each do |file|
    total_count += 1
    file.key
    if is_stray_file?(bucket, file.key)
      if test_mode
        puts "Would delete #{bucket}  #{file.key}"
      else
        aws.delete_object(bucket, file.key)
      end
      delete_count += 1
    end
  end
  
  puts "Total count: #{total_count}"
  puts "Delete count: #{delete_count}"
  puts "Not delete count: #{total_count - delete_count}"
end

def is_stray_file?(bucket, key)
  # You'll want to update this to check that this file belongs.
  # Example: Image.where(image_file_path: key).size == 0
  return false
end