Last active
February 4, 2022 18:27
Star
You must be signed in to star a gist
Create GZ from ZIP - Object Storage to Object Storage
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Create a GZIP file from a ZIP file (with one file in zip contents) | |
Usage: | |
echo '{ | |
"source_file":"<source_object_name>", | |
"source_bucket":"<source_bucket_name>", | |
"target_bucket":"<target_bucket_name>", | |
"target_file":"<target_object_name>" | |
}' | fn invoke distools disfunctionsgz | |
Examples: | |
echo '{ | |
"source_file":"demo.zip", | |
"source_bucket":"a_delta_changes", | |
"target_file":"demo.gz", | |
"target_bucket":"a_delta_changes" | |
}' | fn invoke distools disfunctionsgz | |
You can also use folder names; | |
echo '{ | |
"source_file":"demo.zip", | |
"source_bucket":"a_delta_changes", | |
"target_bucket":"a_delta_changes", | |
"target_file":"cool/newdemo.gz" | |
}' | fn invoke distools disfunctionsgz |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import uuid | |
import io | |
import json | |
import oci | |
from fdk import response | |
from oci.object_storage import UploadManager | |
from oci.object_storage.transfer.constants import MEBIBYTE | |
import gzip | |
import zipfile | |
def handler(ctx, data: io.BytesIO = None): | |
signer = oci.auth.signers.get_resource_principals_signer() | |
body = json.loads(data.getvalue()) | |
source_file = body.get("source_file") | |
source_bucket = body.get("source_bucket") | |
target_file = body.get("target_file") | |
target_bucket = body.get("target_bucket") | |
tmp_dir = '/tmp/' | |
tmp_fname = str(uuid.uuid4()) | |
object_storage_client = oci.object_storage.ObjectStorageClient(config={}, signer=signer) | |
namespace = object_storage_client.get_namespace().data | |
rslt = object_storage_client.get_object(namespace, | |
source_bucket, | |
source_file) | |
with io.BytesIO() as data, gzip.open(tmp_dir + tmp_fname, 'wb') as gz: | |
for chunk in rslt.data.raw.stream(1024 * 1024, decode_content=False): | |
gz.write(chunk) | |
with open(tmp_dir + tmp_fname, 'r') as instream: | |
part_size = 10 * MEBIBYTE # part size (in bytes) | |
upload_manager = UploadManager(object_storage_client, allow_parallel_uploads=True, parallel_process_count=3) | |
stat = upload_manager.upload_stream( | |
namespace, target_bucket, target_file, stream_ref=instream, part_size=part_size) #, progress_callback=progress_callback) | |
resp_data = {"status":"200"} | |
return response.Response( | |
ctx, response_data=resp_data, headers={"Content-Type": "application/json"} | |
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
schema_version: 20180708 | |
name: disfunctionsgz | |
version: 0.0.7 | |
runtime: python | |
entrypoint: /python/bin/fdk /function/func.py handler | |
memory: 1024 | |
timeout: 300 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
fdk | |
oci>=2.55.1 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment