Build ETL from device to cloud: https://h3xagn.com

build-etl-part4-process.py

# upload data to azure
has_json_uploaded, has_csvgz_uploaded = upload_data_to_azure(json_filename, json_data, transformed_filename, transformed_data)

if not has_json_uploaded:
    # generate json path for failed uploads
    json_file_path_failed = f"{base_dir}\\data\\failed_upload\\raw\\{json_filename}"
    with open(json_file_path_failed, "w") as jsonfile:
        json.dump(json_data, jsonfile)

if not has_csvgz_uploaded:
    # generate csv path for failed uploads
    csv_file_path_failed = f"{base_dir}\\data\\failed_upload\\processed\\{transformed_filename}"
    transform_data.to_csv(
        csv_file_path_failed,
        compression="gzip",
        index=False,
    )