IvanNardini

#! bin/bash

#Pass CLUSTER_NAME, REGION AND BUCKET parameters (or use default parameters)
CLUSTER_NAME=${1:-cluster-00000}
REGION=${2:-europe-west6}
BUCKET=${3:-cloud-demo-databrick-gcp}

#Run job
gcloud dataproc jobs submit pyspark \ 
--cluster ${CLUSTER_NAME} \

IvanNardini

# update local packages
sudo apt-get update -y
# install dependencies
sudo apt-get install -y python3-pip python3-dev python3-venv
# create the python enviroment
python3 -m venv pyenv
# activate a virtual environment¶
source ./pyenv/bin/activate
#install packages
pip install -r ./src/score_interactive_endpoint/requirements.txt

IvanNardini

from sklearn.base import BaseEstimator, TransformerMixin

class Transformer(BaseEstimator, TransformerMixin):
  
  def fit(self, X, y=None):
    return self
  
  def transform(self, X):
    return X
  

IvanNardini

def encoder(data, var, mapping):
    '''
    Encode all variables for training
    :params: data, var, mapping
    :return: DataFrame
    '''
    if var not in data.columns.values.tolist():
        pass
    return data[var].map(mapping)

IvanNardini

class Encoder(BaseEstimator, TransformerMixin):

    """ A transformer that returns DataFrame 
    with variable encoded.

    Parameters
    ----------
    encoding_meta : list, default=None

    """

IvanNardini

'''
Compile pipeline contains the pipeline object
'''
import data_preprocessing as Data_Prep
import feature_engineering as Feat_Eng
from imblearn.pipeline import Pipeline
from imblearn.over_sampling import SMOTE
from sklearn.ensemble import RandomForestClassifier

#Utils

IvanNardini

def train():
    # Read Data
    data = pd.read_csv(DATA_INGESTION['data_path'])
    target = DATA_INGESTION['data_map']['target']
    variables = DATA_INGESTION['data_map']['variables']

    #Preprocessing
    flt = data['umbrella_limit']>=0
    data = data[flt]
    data[target] = data[target].map(FEATURES_ENGINEERING['target_encoding'])