modeling_pipeline

def construct_pipeline(numeric_features, categorical_features, model):
    """
    Constructs a scikit-learn pipeline to be used for modeling. A scikit-learn ColumnTransformer is used to apply
    different transformers to the data. For the numeric data, nulls are imputed with the mean, and then features are
    scaled. For categorical data, nulls are imputed with the constant "unknown", and then the DictVectorizer is applied
    to "dummy code" the features. All features are run through a function to drop specified features and to potentially
    drop irrelevant features based on univariate statistical tests. The last step in the pipeline is a model with a
    predict method.
    :param numeric_features: list of numeric features
    :param categorical_features: list of categorical features
    :param model: instantiated model
    :return: scikit-learn pipeline that can be used for fitting and predicting
    """
    numeric_transformer = Pipeline(steps=[
        ('imputer', SimpleImputer(strategy='mean')),
        ('scaler', StandardScaler())
    ])

    categorical_transformer = Pipeline(steps=[
        ('imputer', SimpleImputer(strategy='constant', fill_value='unknown', add_indicator=True)),
        ('dict_creator', FeaturesToDict()),
        ('dict_vectorizer', DictVectorizer())
    ])

    preprocessor = ColumnTransformer(
        transformers=[
            ('numeric_transformer', numeric_transformer, numeric_features),
            ('categorical_transformer', categorical_transformer, categorical_features)
        ])

    pipeline = Pipeline(steps=
                        [
                            ('feature_dropper', FunctionTransformer(drop_features, validate=False,
                                                                    kw_args={'features_drop_list':
                                                                             FEATURES_TO_DROP})),
                            ('preprocessor', preprocessor),
                            ('feature_selector', SelectPercentile(f_classif)),
                            ('model', model)
                        ])

    return pipeline