Skip to content

Instantly share code, notes, and snippets.

View weizhou2273's full-sized avatar
🎯
Focusing

William Zhou weizhou2273

🎯
Focusing
5 followers · 3 following
  • NYC Data Science Academy
View GitHub Profile
@weizhou2273
weizhou2273 / run.sh
Last active August 20, 2017 16:32
~/spark-2.1.1-bin-hadoop2.7/bin/spark-submit --packages org.apache.hadoop:hadoop-aws:2.7.1 --driver-class-path ~/spark-2.1.1-bin-hadoop2.7/mysql-connector-java-5.1.42/mysql-connector-java-5.1.42-bin.jar --jars ~/spark-2.1.1-bin-hadoop2.7/mysql-connector-java-5.1.42/mysql-connector-java-5.1.42-bin.jar ~/capstone/spark_analysis.py
@weizhou2273
weizhou2273 / crontab_spark.sh
Created August 13, 2017 14:22
#install cron
sudo apt-get install cron
#Schedule task
sudo crontab -e
#Schedule runnning task every 2 minutes
*/2 * * * * source ./capstone/run.sh
#Save cron file
# Check your crontab task
sudo crontab -l
@weizhou2273
weizhou2273 / spark_analysis.py
Created August 13, 2017 00:57
from __future__ import print_function
import sys
import re
from operator import add
import pandas as pd
from pyspark.sql.types import StructField, StructType, StringType
from pyspark.sql import Row
from pyspark.sql.types import *
from pyspark.sql import SQLContext
import json
@weizhou2273
weizhou2273 / run_pyspark.py
Created August 12, 2017 20:04
# navigate to spark folder
cd spark-2.1.1-bin-hadoop2.7/
# Initiate spark console
./bin/pyspark
@weizhou2273
weizhou2273 / download_unzip_mysql_driver.sh
Last active August 12, 2017 19:51
# download mysql_driver from https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-5.1.42.tar.gz
wget https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-5.1.42.tar.gz
# unzip mysql driver
tar -xvzf mysql-connector-java-5.1.42.tar.gz
# remove tar.gz
rm mysql-connector-java-5.1.42.tar.gz
@weizhou2273
weizhou2273 / download_unzip_spark.sh
Last active August 12, 2017 19:55
# Download spark from https://d3kbcqa49mib13.cloudfront.net/spark-2.2.0-bin-hadoop2.7.tgz
wget https://d3kbcqa49mib13.cloudfront.net/spark-2.2.0-bin-hadoop2.7.tgz
# Unzip spark
tar -xvzf spark-2.1.1-bin-hadoop2.7.tgz
# remove tgz file
rm spark-2.1.1-bin-hadoop2.7.tgz
@weizhou2273
weizhou2273 / delete_kinesis.py
Created August 9, 2017 03:49
import boto3
from moto import mock_kinesis
import credentials
import sys
from botocore.exceptions import ClientError
aws_key_id = credentials.aws['key_id']
aws_key = credentials.aws['key']
@weizhou2273
weizhou2273 / install_python_package.sh
Created August 6, 2017 19:57
sudo apt-get install pip
sudo pip install boto3
sudo pip install tweepy
sudo pip install time
sudo pip install json
sudo pip install os
sudp pip install uuid
@weizhou2273
weizhou2273 / install_python.sh
Created August 6, 2017 19:45
sudo apt-get install python
@weizhou2273
weizhou2273 / setup_EC2_environment.sh
Created August 6, 2017 15:37
# install java
sudo add-apt-repository ppa:webupd8team/java
sudo apt-get update
sudo apt-get install oracle-java8-installer
sudo update-alternatives --config java
readlink -f $(which java)
# /usr/lib/jvm/java-8-oracle
sudo vi /etc/profile
# {vi environment}
source JAVA_HOME="/usr/lib/jvm/java-8-oracle/jre"