Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Parallel lambda invocations with Python concurrent futures
#!/usr/bin/env python3
# parallel_lambda_test.py
import random
import pendulum
import time
from concurrent import futures
# The problem is that the AWS boto3 lambda invoke function is not truly asynchronous
# even with InvocationType='Event', so to achieve concurrency, we rely on the
# ThreadPoolExecutor, which depending on the vCPU size can invoke ~60 lambdas/sec
def fakeLambda(batch, rank, invoke_time):
sleep_time = random.randint(3, 8)
time.sleep(sleep_time)
curr_time = pendulum.now(tz="UTC")
print(f" ..finished. {batch} - #{rank}, took {sleep_time} sec - invoke {invoke_time.strftime('%H:%M:%S')}, now: {curr_time.strftime('%H:%M:%S')}")
def loopLoop():
with futures.ThreadPoolExecutor() as executor:
batch = 0
while True:
curr_time = pendulum.now(tz="UTC")
print(f"Batch {batch} - {curr_time.strftime('%H:%M:%S')}")
for i in range(5):
executor.submit(fakeLambda, batch, i, curr_time)
time.sleep(5)
batch += 1
def main():
loopLoop()
if __name__ == "__main__":
main()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment