lucidyan

from __future__ import print_function

import numpy as np
import scipy.linalg
from scipy.odr import *
import matplotlib as mpl
from mpl_toolkits.mplot3d import Axes3D
from matplotlib import pyplot as plt
import sys
import time

lucidyan

import pandas as pd
from pyspark.sql import DataFrame

# Wrapper for seamless Spark's serialisation
def spark_to_pandas(spark_df: DataFrame) -> pd.DataFrame:
    """
    PySpark toPandas realisation using mapPartitions
    much faster than vanilla version
    fork: https://gist.github.com/lucidyan/1e5d9e490a101cdc1c2ed901568e082b
    origin: https://gist.github.com/joshlk/871d58e01417478176e7