Redis remove keys

redis_ttl_cleaner.py

import redis
from itertools import zip_longest
import click
from tqdm import tqdm
from time import sleep


REDIS_HOST = "HOST"
BATCH_SIZE = 500
MAX_TTL = 90 * 24 * 60 * 60


def connect(host, port, db, password):
    if password:
        raise Exception("No password is supported")
    r = redis.StrictRedis(host=host, port=port, db=db)
    return r


# iterate a list in batches of size n
def batcher(iterable, n):
    args = [iter(iterable)] * n
    return zip_longest(*args)


def db_size(redis_cli, db_number):
    keyspace = redis_cli.info("keyspace")

    if not keyspace:
        print("Redis is empty")
        return 0

    return keyspace[f"db{db_number}"]["keys"]

@click.command()
@click.option("--host", "host", help="Redis host", default=REDIS_HOST)
@click.option("--db", "db", help="Redis DB", default=8)
@click.option("--port", "port", help="Redis Port", default=6379)
@click.option("--password", "password", help="Redis source password", default=None)
@click.option("--pattern", help="Pattern can be tda-* or the prefix of the keys you want to check", default="search-*")
@click.option("--ttl", help="TTL in days to clean old redis keys - Not Used", default=30)
@click.option("--delete", help="Delete old keys", is_flag=True)
def cleaner(host, db, port, password, pattern, ttl, delete):
    r = connect(host, port, db, password)
    total_keys = db_size(r, db)
    print(f"We have {total_keys} in DB {db}")
    pbar = tqdm()
    total_deletes = 0
    for keybatch in batcher(r.scan_iter(pattern), BATCH_SIZE):
        to_delete = []
        for key in keybatch:
            key_ttl = r.ttl(key)
            if key_ttl > MAX_TTL:
                to_delete.append(key)
        if delete and to_delete:
            tqdm.write(f"Keys to delete {len(to_delete)}")
            r.delete(*to_delete)
            total_deletes += len(to_delete)
        pbar.update(BATCH_SIZE)
        tqdm.write(f"Total keys deleted {total_deletes}")
        sleep(1)

    total_keys = db_size(r, db)
    print(f"We have {total_keys} in DB {db}")
    print(f"We have deleted {total_deletes} keys in DB {db}")


if __name__ == "__main__":
    # To run the script first:
    # > unset PIP_INDEX_URL
    # then > pip install click tqdm redis (depend on which machine you are running the script, redis may be installed)
    # once that is done just execute the command
    # python redis_cleaner.py --host --db --port --password --pattern --delete
    # Remember if you add a pattern you need the *, search* or details*,
    # otherwise it will try to search for just the word and the script won't copy anything
    cleaner()