michalc

# Deletes objects in bulk using boto3's delete_objects, but using multiple threads to achieve some
# parallelism - in spite of the GIL multiple HTTP requests to S3 should happen at the same time.
# Instead of looping over all keys under the root prefix, it walks the tree of keys of delimiter-
# defined "folders" in a depth-first way, which allows each page to be processed by a separate
# thread as its discovered. Depth-first is done because the depth is limited by the maximum key
# size of 1024 in S3, and so means that there is a limit to the memory used by the algorithm to
# store the next requests to make. This would not be the case with breadth-first because there is
# no limit to how many keys are in any folder.
#
# To do the search in parallel, each bit of work (i.e. an HTTP request to fetch a page of keys

michalc

import json
import os
from urllib.parse import urlparse, parse_qsl

import requests
from rich import box
from rich.console import Console
from rich.table import Table

with open(f'{os.environ["HOME"]}/.cf/config.json') as f:

michalc

# docker run --rm -it -p 5432:5432 -e POSTGRES_PASSWORD=password postgres:14

import pprint
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from contextlib import contextmanager

import psycopg2
import psycopg2.extras