Redis Backup Scripts

redis_generate_dump.py

import redis
import csv

csv_file_path = 'metrics_output.csv'

r = redis.Redis(host='localhost', port=6380, decode_responses=True)


def insert_into_sorted_set(sorted_set_name, timestamp, metric):
    try:
        r.zadd(sorted_set_name, {metric: timestamp})
        print(
            f"Inserted {sorted_set_name} with metric {metric} at timestamp {timestamp}")
    except Exception as e:
        print(f"Failed to insert into Redis sorted set: {e}")


def read_csv_and_insert():
    with open(csv_file_path, 'r', newline='') as csvfile:
        csv_reader = csv.reader(csvfile)
        for row in csv_reader:
            if len(row) != 3:
                print(f"Invalid CSV row: {row}")
                continue
            sorted_set_name, timestamp, metric = row
            insert_into_sorted_set(sorted_set_name, timestamp, metric)


read_csv_and_insert()

redis_load_dump.py

import redis
import json
import csv

# Timestamps for the first metrics to be registered
start_timestamp_score = 1685675113

# Timestamp on 01-01-2030, used a very distant time to get all existing values
end_timestamp_score = 1893462733

r = redis.Redis(host='localhost', port=6380, decode_responses=True)


def list_sorted_sets(redis_connection):
    cursor = '0'
    sorted_sets = []

    while True:
        cursor, keys = redis_connection.scan(cursor=cursor, count=100)
        for key in keys:
            key_type = redis_connection.type(key)
            if key_type == 'zset':
                if key != 'text-davinci-002':
                    sorted_sets.append(key)

        if cursor == 0:
            break

    return sorted_sets


def fetch_and_write(sorted_sets_names):
    for sorted_set_name in sorted_sets_names:
        try:
            members = r.zrange(
                sorted_set_name, start_timestamp_score, end_timestamp_score, byscore=True, withscores=True)
            print(f"{len(members)} entries for {sorted_set_name} model")
        except Exception as e:
            print(f"Failed in Redis step: {e}")
            pass
        for member in members:
            try:
                metric, timestamp = member
            except json.JSONDecodeError as e:
                print(f"Failed to decode JSON string: {e}")
                pass
            except ValueError as e:
                print(f"Failed to decode JSON string: {e}")
                pass
            try:
                with open('metrics_output.csv', mode="a", newline='') as csvfile:
                    csv_writer = csv.writer(csvfile)
                    csv_writer.writerow([sorted_set_name, timestamp, metric])
            except Exception as e:
                print(f"Failed to append row to CSV file: {e}")
                pass


sorted_sets = list_sorted_sets(r)
fetch_and_write(sorted_sets)

requirements.txt

redis==4.5.5
hiredis==2.2.3