Skip to content

Instantly share code, notes, and snippets.

@nicksellen nicksellen/local.rb Secret
Last active Oct 5, 2018

Embed
What would you like to do?
# Run this with:
#
# docker exec -i <idofcontainer> rails runner - < local.rb >/tmp/localfiles
#
# The first line will be a warning message, discard this, then you have a nice list of local files!
# It only takes the original attachments, not the small size.
MediaAttachment.local.find_in_batches(batch_size: 200) do |items|
items.each do |item|
puts item.file.url(:original).gsub(/.*\/social-coop-live\//, '')
end
end
Account.local.find_in_batches(batch_size: 200) do |accounts|
accounts.each do |account|
avatar = account.avatar_original_url.gsub(/.*\/social-coop-live\//, '')
puts avatar if avatar != '/avatars/original/missing.png'
header = account.header_original_url.gsub(/.*\/social-coop-live\//, '')
puts header if header != '/headers/original/missing.png'
end
end
from multiprocessing import Process, Queue
import boto3
opts = {
'service_name':'s3',
'aws_access_key_id':'FOO',
'aws_secret_access_key':'BAR',
'endpoint_url':'https://ams3.digitaloceanspaces.com',
}
bucket_name = 'social-coop-media'
process_count = 10
session = boto3.session.Session()
s3_client = session.client(**opts)
s3 = session.resource(**opts)
bucket = s3.Bucket(bucket_name)
DONE = '___DONE___'
processed = dict()
for item in open('aclprocessed').read().splitlines():
processed[item] = True
def process(q):
while True:
key = q.get()
if key == DONE:
break
s3_client.put_object_acl(Bucket=bucket_name, Key=key, ACL='public-read')
print('processed', key)
if __name__ == '__main__':
q = Queue(process_count)
processes = []
for n in range(process_count):
p = Process(target=process, args=(q,))
p.start()
processes.append(p)
for obj in bucket.objects.all():
if obj.key in processed:
print('skipped', obj.key)
else:
q.put(obj.key)
for n in range(process_count):
q.put(END_KEY)
for p in processes:
p.join()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.
You signed in with another tab or window. Reload to refresh your session. You signed out in another tab or window. Reload to refresh your session.