Fugue Example

fugue.py

# Import statements
from fugue import FugueWorkflow, SparkExecutionEngine
from typing import List, Any, Dict, Iterable

# Area code to state
area_to_state = {"217": "IL", "312": "IL", "415": "CA", "352": "FL"}

# schema: *, inferred_state:str
def fill_location(df:Iterable[Dict[str,Any]]) -> Iterable[Dict[str,Any]]:
    for row in df:
        potential_values = [row["home_state"], row["work_state"], 
                            area_to_state[row["phone"][0:3]]]
        # Return first non-null value. Returns None if all are None.
        row["inferred_state"] = next((val for val in potential_values 
                                      if val is not None), None)
        yield row

with FugueWorkflow() as dag:
    df = dag.df(df).transform(fill_location)