GLACIERクラスのファイルをのんびりRestoreさせます

construct_queue.py

#!/usr/bin/env python
import csv
import sys
import json
import argparse
import copy


def get_list(path):
    targets = []

    reader = csv.reader(path)
    for row in reader:
        targets.append((row[0], int(row[1])))

    targets.sort(key=lambda t: t[1])
    return targets


def dig(limit, targets_ref):
    queue = []
    targets = copy.deepcopy(targets_ref)

    while targets:
        trial = []
        capacity = limit

        while targets:
            t = targets.pop()

            if t[1] > limit:
                trial.append(t[0])
            elif t[1] > capacity:
                targets.append(t)
            else:
                trial.append(t[0])
                capacity -= t[1]
                continue

            break

        queue.append(trial)

    return queue


def main():
    parser = argparse.ArgumentParser(description='Create queue.')
    parser.add_argument('limit', metavar='LIMIT', type=int,
                        help='size limit per queue (bytes).')
    args = parser.parse_args()

    targets = get_list(sys.stdin)
    queue = dig(args.limit, targets)
    json.dump(queue, sys.stdout)


if __name__ == '__main__':
    main()

dig_bucket.py

#!/usr/bin/env python
import csv
import sys
import boto3
import argparse


def dig_bucket(bucket_name, prefix):
    s3 = boto3.resource('s3')

    def isFrozenFile(obj):
        s3obj = s3.Object(obj.bucket_name, obj.key)
        return obj.size > 0 and \
                obj.storage_class == 'GLACIER' and \
                s3obj.restore is None

    bkt = s3.Bucket(bucket_name)
    writer = csv.writer(sys.stdout)

    for obj in filter(isFrozenFile, bkt.objects.filter(Prefix=prefix)):
        writer.writerow([obj.key, obj.size])


def main():
    parser = argparse.ArgumentParser(description='Create glacier class objects list.')
    parser.add_argument('bucket', metavar='BUCKET', type=str,
                        help='bucket name')
    parser.add_argument('prefix', metavar='PREFIX', type=str,
                        help='key prefix')
    args = parser.parse_args()
    dig_bucket(args.bucket, args.prefix)


if __name__ == "__main__":
    main()

exec.sh

#!/bin/sh
set -ue
umask 0027
export PATH='/bin:/usr/bin'
export LANG='C'
IFS=$' \t\n'

bucket=''
prefix=''
limit=67108864 # (3GB / 48) bytes
delay=1 # hours

list_tmp=$(mktemp)
queue_tmp=$(mktemp)

# bucketの走査
./dig_bucket.py $bucket $prefix > "${list_tmp}"

# 復元キューの生成
./construct_queue.py $limit < "${list_tmp}" > "${queue_tmp}"
rm -f "${list_tmp}"

# 復元
./penetrate_slowly.py $delay < "${queue_tmp}"
rm -f "${queue_tmp}"

penetrate_slowly.py

#!/usr/bin/env python
import sys
import json
import argparse
import boto3


def defrost(restore_days, bucket_name, queue_fd):
    s3 = boto3.resource('s3')
    queue = json.load(queue_fd)

    def takeOnlyNeeded(keys):
        objs = map(lambda k: s3.Object(bucket_name, k), keys)
        filtered = filter(lambda obj: obj.restore is None, objs)
        return list(filtered)

    targets = takeOnlyNeeded(queue.pop())
    while targets == []:
        targets = takeOnlyNeeded(queue.pop())

    for target in targets:
        print(target.key)
        #target.restore_object(RestoreRequest={'Days': days})

    return queue

def main():
    parser = argparse.ArgumentParser(
            description='Defrost glacier class file.',
            epilog='stdin is queues, stdout is consumed queues.')
    parser.add_argument('days', metavar='DAYS', type=int, default=30,
                        help='RestoreRequest[\'Days\']')
    parser.add_argument('bucket', metavar='BUCKET', type=str,
                        help='bucket name')
    args = parser.parse_args()

    queue = defrost(args.days, args.bucket, sys.stdin)
    json.dump(queue, sys.stdout)


if __name__ == "__main__":
    main()