Build ETL from device to cloud: https://h3xagn.com

build-etl-part4-blob.py

def upload_data_to_azure(json_filename, json_data, transformed_filename, transformed_data):
    """Upload to Azure Blob Storage"""

    # set upload flags to False
    has_json_uploaded = False
    has_csvgz_uploaded = False

    # get connection string from .env file
    connect_str = os.getenv("AZURE_STORAGE_CONNECTION_STRING")

    # create a blob client using the local file name as the name for the blob
    blob_service_client = BlobServiceClient.from_connection_string(connect_str)

    # define blob container and paths
    blob_client_json = blob_service_client.get_blob_client(container="device-data", blob=f"raw/{json_filename}")
    blob_client_csvgz = blob_service_client.get_blob_client(container="device-data", blob=f"processed/{transformed_filename}")

    # upload raw JSON file
    try:
        json_content_setting = ContentSettings(content_type="application/json")
        blob_client_json.upload_blob(json.dumps(json_data, ensure_ascii=False).encode("utf-8"), overwrite=True, content_settings=json_content_setting)
        has_json_uploaded = True
        log.info(f"--- JSON file uploaded to Azure: '{json_filename}'.")
    except:
        log.error(f"*** JSON file NOT uploaded to Azure: '{json_filename}'.")

    # upload processed compressed CSV file
    try:
        csvgz_content_setting = ContentSettings(content_type="application/x-gzip")

        # TODO: Investigate directly uploading transformed_data data frame
        # read compressed CSV file from disk
        with open(f"{base_dir}/data/processed/{transformed_filename}", "rb") as data:
            blob_client_csvgz.upload_blob(data, overwrite=True, content_settings=csvgz_content_setting)

        has_csvgz_uploaded = True
        log.info(f"--- Transformed file uploaded to Azure: '{transformed_filename}'.")
    except:
        log.error(f"*** Transformed file NOT uploaded to Azure: '{transformed_filename}'.")

    return has_json_uploaded, has_csvgz_uploaded