cluster_setup.py

def setup_cluster(
    cluster_config: Dict,
) -> Union[LocalCluster, FargateCluster]:
    if cluster_config is None:
        n_workers = 3
        threads_per_worker = 2
        return LocalCluster(
            n_workers=n_workers,
            threads_per_worker=threads_per_worker,
            processes=False,
            memory_limit="4GB",
        )
    else:
        return FargateCluster(**cluster_config["cluster_kwargs"])