This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
fout = open('classifier_votes.txt','w') | |
counter={} | |
for ll in range(100): | |
print(ll) | |
X_train, X_test, y_train, y_test, ix_train, ix_test = train_test_split(X, y, indices, test_size=0.2, random_state=ll) | |
classifiers=[] | |
classifiers.append(LogisticRegression(class_weight='balanced')) | |
classifiers.append(RandomForestClassifier(n_estimators=10, max_depth=4, random_state=0, | |
max_features=None,criterion="entropy", class_weight='balanced')) |
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#! /usr/bin/env python | |
# Original author Nicholas Kim, modified by Yan Pashkovsky | |
# New license - GPL v3 | |
import sys | |
import time | |
from PyPDF2 import utils, PdfFileReader, PdfFileWriter | |
def get_cmdline_arguments(): | |
"""Retrieve command line arguments.""" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
spark-submit --packages org.apache.spark:spark-streaming-kinesis-asl-assembly_2.10:1.6.1 --master yarn-cluster \ | |
--deploy-mode cluster --executor-memory 16g --num-executors 2 --driver-memory 6g --executor-cores 4 \ | |
--conf spark.yarn.executor.memoryOverhead=1000 --py-files /mnt/app.egg /mnt/KinesisReceiver.py |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Check java version | |
JAVA_VER=$(java -version 2>&1 | sed 's/java version "\(.*\)\.\(.*\)\..*"/\1\2/; 1q') | |
if [ "$JAVA_VER" -lt 18 ] | |
then | |
# Figure out how many versions of Java and javac we currently have | |
NR_OF_JRE_OPTIONS=$(echo 0 | alternatives --config java 2>/dev/null | grep 'There ' | awk '{print $3}' | tail -1) | |
NR_OF_SDK_OPTIONS=$(echo 0 | alternatives --config javac 2>/dev/null | grep 'There ' | awk '{print $3}' | tail -1) | |
# Silent install javac (includes jre) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
aws emr create-cluster --termination-protected --applications Name=Hadoop Name=Hive Name=Pig Name=Hue Name=Ganglia Name=Spark \ | |
--bootstrap-actions '[{"Path":"s3://config-test/utils/boot_script.sh","Name":"Java and Tensorflow Install boot script"}]' \ | |
--ec2-attributes '{"KeyName":"abcdefgh","InstanceProfile":"EMR_EC2_DefaultRole","ServiceAccessSecurityGroup":"sg-a70038df","SubnetId":"subnet-f993ccd1","EmrManagedSlaveSecurityGroup":"sg-000000","EmrManagedMasterSecurityGroup":"sg-000000"}' \ | |
--service-role EMR_DefaultRole --enable-debugging --release-label emr-4.4.0 \ | |
--log-uri 's3n://aws-logs-00000000-us-east-1/elasticmapreduce/' --name 'Agent' \ | |
--instance-groups '[{"InstanceCount":2,"InstanceGroupType":"CORE","InstanceType":"m4.2xlarge","Name":"Core instance group - 2"}\ | |
,{"InstanceCount":1,"InstanceGroupType":"MASTER","InstanceType":"m4.2xlarge","Name":"Master instance group - 1"}]' \ | |
--region us-east-1 |