sparkify_training_method.py

import pandas as pd 

def show_results_and_save(model_pipeline, predictions):
    
    df_results = predictions.select(['canceled', 'prediction']).toPandas()
    df_results['prediction'] = df_results.prediction.apply(int)

    # Display the results
    _metrics = plot_metrics(df_results['canceled'], df_results['prediction'])
    
    # Save the results
    _model = model_pipeline.bestModel.stages[-1]
    _model_name = _model.__class__.__name__

    _model_info = {
        'model_name' : _model_name,
        'hyperparameters' : list(zip(list(map(lambda _param: _param.name, list(_model.extractParamMap()))), list(_model.extractParamMap().values())))
    }

    _metrics.update(_model_info)

    _df_metrics = pd.DataFrame([_metrics])
    _df_metrics.to_csv(f'spark_{_model_name}.csv', index = False)