dmiekoar

#!/bin/bash

set -e

sudo -H -i -u ec2-user bash << 'EOF'

echo "Install jupyter nbextension"
source /home/ec2-user/anaconda3/bin/activate JupyterSystemEnv
pip install jupyter_contrib_nbextensions
jupyter contrib nbextensions install --user

dmiekoar

Status = IF(
	DATEDIFF(TrustStockVisibility[Reference Date].[Date];TrustStockVisibility[SSL];DAY)>0;
	"Available";
	IF(DATEDIFF(TrustStockVisibility[Expiry Date];TrustStockVisibility[Reference Date].[Date];DAY)>=0; "SSL";"Expired")
)


Details = IF(
	DATEDIFF(TrustStockVisibility[Reference Date].[Date];TrustStockVisibility[SSL];DAY)>0;
	IF(

dmiekoar

Expiry Dates = 
VAR BaseCalendar = 
    CALENDARAUTO() 
RETURN 
    GENERATE( 
        BaseCalendar;
        VAR BaseDate = [Date]
        VAR YearDate = YEAR(BaseDate)
        VAR MonthNumber = MONTH(BaseDate)
        RETURN ROW (

dmiekoar

# user define objective function, given prediction, return gradient and second
# order gradient this is log likelihood loss
def logregobj(preds, dtrain):
    labels = dtrain.get_label()
    preds = 1.0 / (1.0 + np.exp(-preds))
    grad = preds - labels
    hess = preds * (1.0 - preds)
    return grad, hess
##########################################################

dmiekoar

https://github.com/Microsoft/LightGBM/blob/master/examples/python-guide/advanced_example.py
  
  
import math

#A function to calculate Root Mean Squared Logarithmic Error (RMSLE)
def rmsle(y, y_pred):
	assert len(y) == len(y_pred)
	terms_to_sum = [(math.log(y_pred[i] + 1) - math.log(y[i] + 1)) ** 2.0 for i,pred in enumerate(y_pred)]
	return (sum(terms_to_sum) * (1.0/len(y))) ** 0.5

dmiekoar

for i in train['v40_int']:
    result = i.is_integer()
    if result == False:
        print(i)
        
        
#print(np.isposinf(X_scaled).sum().sum()); print(np.isneginf(X_scaled).sum().sum())
#np.isnan(X_scaled).sum()
#X_scaled.isnull().sum().sum()
#X_scaled.isna().sum()

dmiekoar

# Compute the number of occurrences of a zero value 
def check_zeros(dataset):
    temp_df = dataset.copy(); n_total = temp_df.shape[0]; idcol = []; counter = []; zeros_data = []
    features = [c for c in dataset.columns if c not in ['Outcome']]
    for col in features:
        zeros_count = n_total - np.count_nonzero(temp_df[col])
        idcol.append(col)
        counter.append(zeros_count)
        zeros_data.append(zeros_count / n_total * 100)
    zeros_data = pd.DataFrame({'Zero amount': counter,'% Zero' : zeros_data})

dmiekoar

import sys, os
path = os.path.abspath('../../Feature Selector')
sys.path.append(path)
from feature_selector import FeatureSelector

fs = FeatureSelector(data = X, labels = X.Yards)
fs.identify_single_unique()
single_unique = fs.ops['single_unique']
fs.identify_collinear(correlation_threshold=0.9)
correlated_features = fs.ops['collinear']

dmiekoar

from lofo import LOFOImportance, Dataset, plot_importance
from sklearn.model_selection import KFold

nn = X
nn['Yards'] = X['Yards']

# import data
train_df = nn

# extract a sample of the data

dmiekoar

import numpy as np
import pandas as pd
from tqdm import tqdm_notebook
import multiprocessing
import warnings
from sklearn.metrics import check_scoring


class FLOFOImportance:
    """