This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# from sklearn.metrics import f1_score, accuracy_score | |
import numpy as np | |
from collections import defaultdict | |
def get_metrics_cm_for_one_index(cm, index): | |
n_samples = cm.ravel().sum() | |
total_obs = cm[index].sum() | |
tp = cm[index, index] | |
fp = cm[:, index].sum() - cm[index, index] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def treatoutliers(self, df=None, columns=None, factor=1.5, method='IQR', treament='cap'): | |
""" | |
Removes the rows from self.df whose value does not lies in the specified standard deviation | |
:param columns: | |
:param in_stddev: | |
:return: | |
""" | |
# if not columns: | |
# columns = self.mandatory_cols_ + self.optional_cols_ + [self.target_col] | |
if not columns: |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# tunnel | |
gcloud compute ssh --zone=us-east1-b --ssh-flag="-D" --ssh-flag="10001" --ssh-flag="-N" --ssh-flag="-n" "meow-2024-m" | |
# run the cmd in another terminal session | |
google-chrome "http://meow-2024-m:8123" \ | |
--proxy-server="socks5://localhost:10001" \ | |
--host-resolver-rules="MAP * 0.0.0.0 , EXCLUDE localhost" \ | |
--user-data-dir=/tmp/ & |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import xlrd | |
wb=xlrd.open_workbook("/home/wolfram/Downloads/17750_01-05-2018_10619 COC VSWR Fault.xlsx") | |
ws=wb.sheet_by_index(0) | |
data = [[col.value for col in ws.row(n_row)] for n_row in range(ws.nrows)] | |
print(data) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import pandas as pd | |
chunksize = 1000 | |
ip_filename = '/home/wolfram/Downloads/DataParserData.csv' | |
op_filename = 'datafile.csv' | |
prefix = 'cleansedquery_' | |
all_keys = set() | |
first_write = True |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
public override string Finalize() | |
{ | |
//Calculating the summary statistics | |
sum = rowvalue.Sum(); | |
min = rowvalue.Min(); | |
max = rowvalue.Max(); | |
count = rowvalue.Count(); | |
mean=rowvalue.Average(); | |
range=max-min; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
columns = ['store_id', | |
'sale_type', | |
'is_online_sale', | |
'is_pick_up', | |
'pick_up_store_id', | |
'sku_id', | |
'tot_unit_sold', | |
'tot_promotion_price', | |
'tot_reg_price', | |
'tot_otd_price', |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from functools import reduce | |
rename = { | |
'is_online_order': 'is_online_order', | |
'pos_disc_code': 'sale_type', | |
'brand': 'cat_attribute_desc_1', | |
'style_code': 'style_code', | |
'category_code_lvl_1': 'cat_lvl_code_1', | |
'category_code_lvl_2': 'cat_lvl_code_2', | |
'category_code_lvl_3': 'cat_lvl_code_3', | |
'category_code_lvl_4': 'cat_lvl_code_4', |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
VM=$1 | |
OPS=$2 | |
NODES=$4 | |
ZONE=$3 | |
vms=($VM-m) | |
for (( i=0 ; i<$NODES-1; i++ )) | |
do |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from pyspark import SparkConf,SparkContext | |
from pyspark.sql.functions import * | |
from pyspark.sql import * | |
from pyspark.sql.types import * | |
configs = [('spark.eventLog.enabled', 'true'), | |
('spark.dynamicAllocation.minExecutors', '8'), | |
('spark.executor.instances', '1000'), | |
('spark.driver.host', '10.142.0.3'), | |
('spark.yarn.am.memory', '640m'), |