Parallel lambda invocations with Python concurrent futures

parallel_lambda_test.py

#!/usr/bin/env python3
# parallel_lambda_test.py

import random
import pendulum
import time
from concurrent import futures


# The problem is that the AWS boto3 lambda invoke function is not truly asynchronous
# even with InvocationType='Event', so to achieve concurrency, we rely on the
# ThreadPoolExecutor, which depending on the vCPU size can invoke ~60 lambdas/sec

def fakeLambda(batch, rank, invoke_time):
    sleep_time = random.randint(3, 8)
    time.sleep(sleep_time)
    curr_time = pendulum.now(tz="UTC")
    print(f"    ..finished. {batch} - #{rank}, took {sleep_time} sec - invoke {invoke_time.strftime('%H:%M:%S')}, now: {curr_time.strftime('%H:%M:%S')}")


def loopLoop():
    with futures.ThreadPoolExecutor() as executor:
        batch = 0

        while True:
            curr_time = pendulum.now(tz="UTC")
            print(f"Batch {batch} - {curr_time.strftime('%H:%M:%S')}")

            for i in range(5):
                executor.submit(fakeLambda, batch, i, curr_time)

            time.sleep(5)
            batch += 1


def main():
    loopLoop()


if __name__ == "__main__":
    main()