json-blob-conversion.py

import csv
import json
import logging

# Set up logging
logging.basicConfig(filename='processing.log', level=logging.INFO)

# Define the JSON file path
json_file = 'input.json'

# Define the CSV file path
csv_file = 'output.csv'

# Define the fieldnames for the CSV
fieldnames = [
    'userId',
    'externalId',
    'fromPhone',
    'toPhone',
    'message',
    'sentDate',
    'sentTime',
    'isRead',
    'isDelivered',
    'isStar',
    'isOut',
    'thread',
    'threadNum',
    'campaignId',
    'dialogId',
    'errorId',
    'contactGroupId',
    'campaignTitle',
    'contactGroupName',
    'replyIntent',
    'replyKeyword',
    'responderTitle',
    'nodeId',
    'name',
    'note',
    'failover'
]

# Check for the last processed index
try:
    with open('checkpoint.txt', 'r') as f:
        checkpoint = int(f.read())
except FileNotFoundError:
    checkpoint = 0
    with open('checkpoint.txt', 'w') as f:
        f.write(str(checkpoint))

try:
    # Open the JSON file and read the JSON strings
    with open(json_file, 'r') as file:
        json_strings = file.readlines()

    # Open the CSV file in append mode
    with open(csv_file, 'a', newline='') as file:
        # Create a CSV writer object
        writer = csv.DictWriter(file, fieldnames=fieldnames)

        # Write the header row if the CSV file is empty
        if file.tell() == 0:
            writer.writeheader()

        # Iterate over the JSON strings starting from the checkpoint
        for i in range(checkpoint, len(json_strings)):
            json_string = json_strings[i].strip()
            
            # Parse the JSON string
            data = json.loads(json_string)

            # Write the data to the CSV file
            writer.writerow({
                'userId': data['userId'],
                'externalId': data['externalId'],
                'fromPhone': data['fromPhone'],
                'toPhone': data['toPhone'],
                'message': data['message'],
                'sentDate': data['sentDate'],
                'sentTime': data['sentTime'],
                'isRead': data['isRead'],
                'isDelivered': data['isDelivered'],
                'isStar': data['isStar'],
                'isOut': data['isOut'],
                'thread': data['thread'],
                'threadNum': data['threadNum'],
                'campaignId': data['campaignId'],
                'dialogId': data['dialogId'],
                'errorId': data['errorId'],
                'contactGroupId': data['contactGroupId'],
                'campaignTitle': json.loads(data['extraInfo']).get('campaignTitle'),
                'contactGroupName': json.loads(data['extraInfo']).get('contactGroupName'),
                'replyIntent': json.loads(data['extraInfo']).get('replyIntent'),
                'replyKeyword': json.loads(data['extraInfo']).get('replyKeyword'),
                'responderTitle': json.loads(data['extraInfo']).get('responderTitle'),
                'nodeId': json.loads(data['extraInfo']).get('nodeId'),
                'name': json.loads(data['extraInfo']).get('name'),
                'note': json.loads(data['extraInfo']).get('note'),
                'failover': json.loads(data['extraInfo']).get('failover')
            })

            # Update the checkpoint after successful processing
            checkpoint = i + 1
            with open('checkpoint.txt', 'w') as f:
                f.write(str(checkpoint))

    # Log successful completion
    logging.info("JSON to CSV conversion completed successfully.")

except Exception as e:
    # Log the exception
    logging.exception("Exception occurred during JSON to CSV conversion: %s", str(e))