Simple script to convert Parquet files to JSON using AWS Glue

parquet_to_json_in_aws_glue.py

import sys

from awsglue.context import GlueContext
from pyspark.context import SparkContext
from awsglue.utils import getResolvedOptions


def load(context, bucket, prefix=""):

    dynamic_frame = context.create_dynamic_frame_from_options(
        "s3",
        {
            'paths': ['s3://{}/{}'.format(bucket, prefix)],
            'recurse': True,
            'groupFiles': 'inPartition',
            'groupSize': 134217728
        },
        format='parquet'
    )
    return dynamic_frame


def main(context, args):

    input_bucket_name = args['INPUT_BUCKET_NAME']
    input_bucket_prefix = args['INPUT_BUCKET_PREFIX']

    output_bucket_name = args['OUTPUT_BUCKET_NAME']
    output_bucket_prefix = args['OUTPUT_BUCKET_PREFIX']

    df = load(context, input_bucket_name, input_bucket_prefix) \

    df.toDF().write.json(f's3://{output_bucket_name}/{output_bucket_prefix}')


context = GlueContext(SparkContext.getOrCreate())
job_arguments = getResolvedOptions(
        sys.argv,
        [
            'INPUT_BUCKET_NAME',
            'INPUT_BUCKET_PREFIX',
            'OUTPUT_BUCKET_NAME',
            'OUTPUT_BUCKET_PREFIX'
        ]
    )

main(context, job_arguments)