| #!/public/spark-0.9.1/bin/pyspark | |
| import os | |
| import sys | |
| # Set the path for spark installation | |
| # this is the path where you have built spark using sbt/sbt assembly | |
| os.environ['SPARK_HOME'] = "/public/spark-0.9.1" | |
| # os.environ['SPARK_HOME'] = "/home/jie/d2/spark-0.9.1" | |
| # Append to PYTHONPATH so that pyspark could be found |
| -- based on http://stackoverflow.com/questions/21767780/how-to-find-size-of-database-schema-table-in-redshift | |
| SELECT name AS table_name, ROUND((COUNT(*) / 1024.0),2) as "Size in Gigabytes" | |
| FROM stv_blocklist | |
| INNER JOIN | |
| (SELECT DISTINCT id, name FROM stv_tbl_perm) names | |
| ON names.id = stv_blocklist.tbl | |
| GROUP BY name | |
| ORDER BY "Size in Gigabytes" DESC |
| alias a='ack' | |
| alias a?='alias | grep -i' | |
| alias adx='rake db:drop && rake db:create && heroku pg:transfer --from black --to postgres://postgres@localhost/staq_development --confirm staqweb --app staqweb && rails r "User.all.each { |u| u.update_attribute(:password,%q(password)) }" && rake db:test:prepare' | |
| alias b='bundle' | |
| alias bb='bundle install --binstubs=.bundle/bin --path=.bundle/gems && bundle package --all && reload ; sd' | |
| alias bc='bin/console' | |
| alias be='bundle exec' | |
| alias bea='bundle exec annotate' | |
| alias bu='bundle update' | |
| alias bus='bundle update staq_extraction' |
| # http://stackoverflow.com/questions/1960799/using-gitdropbox-together-effectively/1961515#1961515 | |
| export REPONAME=???? | |
| take ~/Dropbox/git/$REPONAME.git | |
| git init --bare | |
| cd ~/code/$REPONAME | |
| git remote rm origin | |
| git remote add origin ~/Dropbox/git/$REPONAME.git | |
| git push -u origin master |
| #!/usr/bin/python | |
| # -*- coding: utf-8 -*- | |
| # | |
| # Copyright 2012 Google Inc. All Rights Reserved. | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| #on cluster | |
| thrift /spark/sbin/start-thriftserver.sh --master yarn-client | |
| #ssh tunnel, direct 10000 to unused 8157 | |
| ssh -i ~/caserta-1.pem -N -L 8157:ec2-54-221-27-21.compute-1.amazonaws.com:10000 hadoop@ec2-54-221-27-21.compute-1.amazonaws.com | |
| #see this for JDBC config on client http://blogs.aws.amazon.com/bigdata/post/TxT7CJ0E7CRX88/Using-Amazon-EMR-with-SQL-Workbench-and-other-BI-Tools |
virtualenvwrapper has broken dependencies right now on macOS, so install like this:
sudo pip install pbr
sudo pip install --no-deps stevedore
sudo pip install --no-deps virtualenvwrapper
So that the commands mkvirtualenv and workon stay in your shell, you're going to want to do this (if you use zsh or another shell, change the filename here):