Build ETL from device to cloud: https://h3xagn.com

built-elt-bonus-server.py

# 01 Web Server/utils/process_json.py

# Compress JSON file
# ----------------------------------------------------------------------

# OLD: save json file to disk
with open(json_file_path, "w") as jsonfile:
     json.dump(json_data, jsonfile)
    
# NEW: save compressed json file to disk
import gzip

with gzip.open(f"{json_file_path}/{json_filename}", "w") as file_zip:
    file_zip.write(json.dumps(json_data).encode("utf-8"))

# Add partitioning
# ----------------------------------------------------------------------

# Original code
json_file_path = f"{base_dir}\\data\\raw\\{json_filename}"

# With the added partitioning, changes to 
json_file_path = f'{base_dir}\\data\\raw\\{json_data["sn"]}\\{file_time.year}\\{file_time.month}\\{file_time.day}'

# create 'json_file_path' if it does not exist
if not os.path.exists(json_file_path):
    os.makedirs(json_file_path)

# The above is implemented for our 'transform_data' and 'upload_data_to_azure' methods as well