Skip to content

Instantly share code, notes, and snippets.

View bjornjorgensen's full-sized avatar

Bjørn Jørgensen bjornjorgensen

View GitHub Profile
from pyspark.sql.types import *
from pyspark.sql.functions import *
def flatten_test(df, sep="_"):
"""Returns a flattened dataframe.
.. versionadded:: x.X.X
Parameters
----------
@bjornjorgensen
bjornjorgensen / flatten_all_spark_schema.py
Last active March 21, 2022 21:25 — forked from AxREki/flatten_all_spark_schema.py
Flatten a Spark DataFrame schema (include struct and array type)
import typing as T
import cytoolz.curried as tz
import pyspark
from pyspark.sql.functions import explode_outer
from pyspark.sql.types import BooleanType
def schema_to_columns(schema: pyspark.sql.types.StructType) -> T.List[T.List[str]]: