Python Parallel

io_bound.py

from concurrent.futures import ThreadPoolExecutor  # Or ProcessPoolExecutor for CPU-bound tasks

with ThreadPoolExecutor() as executor:
    future1 = executor.submit(slow_func1, a, b, c)
    future2 = executor.submit(slow_func2, x, a, z)
    
    x = future1.result()
    y = future2.result()

return x + y

LazyFrame.py

from concurrent.futures import ThreadPoolExecutor
from concurrent.futures import Future
import pandas as pd
import random

class LazyDataFrame:
    def __init__(self, query):
        self.executor = ThreadPoolExecutor()
        self.future = self.executor.submit(read_sql_query, query)
    
    def __getattr__(self, name):
        if hasattr(pd.DataFrame, name):
            # Block until the future is resolved
            df = self.future.result()
            return getattr(df, name)
        else:
            raise AttributeError(f"'LazyDataFrame' object has no attribute '{name}'")
    
    def result(self):
        return self.future.result()

# Example usage:
from concurrent.futures import ThreadPoolExecutor

def read_sql_query(query):
    import time
    duration = random.randint(5, 10)
    time.sleep(duration)
    print(f"Finished query: {query} (duration: {duration}s)")
    return pd.DataFrame({"data": [query]})

lazy_df1 = LazyDataFrame("query1")
lazy_df2 = LazyDataFrame("query2")

print("I can do stuff without waiting for the queries to finish")
print(lazy_df1.head())
print(lazy_df2.head())