Returns a new PySpark DataFrame containing the union of two DataFrames. This more advanced version works even when the two DataFrames have different columns and a different order of columns. If a column does not exist in either DataFrame, its fields will be empty.

advanced-dataframe-union.py

def advanced_dataframe_union(df1, df2):
    df1_fields = set((f.name, f.dataType) for f in df1.schema)
    df2_fields = set((f.name, f.dataType) for f in df2.schema)

    df2 = df2.select(
        df2.columns
        + [
            F.lit(None).cast(datatype).alias(name)
            for name, datatype in df1_fields.difference(df2_fields)
        ]
    )

    df1 = df1.select(
        df1.columns
        + [
            F.lit(None).cast(datatype).alias(name)
            for name, datatype in df2_fields.difference(df1_fields)
        ]
    )
    
    return df1.select(df2.columns).union(df2)