Skip to content

Instantly share code, notes, and snippets.

@PseudoSky
Last active August 19, 2016 18:09
Show Gist options
  • Save PseudoSky/dd03fd6a4fb548de95ec to your computer and use it in GitHub Desktop.
Save PseudoSky/dd03fd6a4fb548de95ec to your computer and use it in GitHub Desktop.
OLD, look for .bash_profile
#!/bin/bash
md() {
mkdir -p "$*"
cd "$*"
}
alias gl="git log --graph --pretty=format:'%Cred%h%Creset -%C(yellow)%d%Creset %s %C(yellow):%an%Creset %Cgreen(%cr)%Creset' --abbrev-commit --date=relative";
alias gb='git branch'
alias simps='python -m SimpleHTTPServer'
alias kl='wget --limit-rate=200k --no-clobber --convert-links --random-wait -r -p -E -e robots=off -U mozilla'
alias flush_cache='dscacheutil -flushcache'
alias ports='netstat -tulp'
alias bi='bower install --save'
alias subl='/Applications/Sublime\ Text.app/Contents/SharedSupport/bin/subl'
alias clean_me='rm -rf ~/.Trash/* & rm ~/.DS_Store*'
alias op='open .'
alias ls="gls --color=auto"
alias ll="gls --color=auto -lh"
alias la="gls --color=auto -a"
alias l="gls --color=auto"
alias ni="npm install --save"
alias nig="npm install -g"
alias nl='npm link'
alias helpe='echo "wget --limit-rate=200k --no-clobber --convert-links --random-wait -r -p -E -e robots=off -U mozilla"'
alias bup='source ~/dot/path & source ~/dot/alias & source ~/dot/func & source ~/dot/bashrc'
alias cds='cd ~/Desktop/Duality'
alias ..='cd ../'
alias ....='cd ../../'
alias ......='cd ../../../'
alias ls='ls -a'
alias compr='tar -zcvf'
alias pack='dpkg --get-selections| less'
alias gay='git add -A'
alias hmm='cat ~/.info | less'
alias ka='killall'
alias bi='bower install --save'
alias rs='rails s'
alias mcomp='mvn compile -Dskiptests'
alias mpack='mvn package -Pdist,native -DskipTests -Dtar'
alias ports='netstat -tulp'
alias rs='rails server'
alias ga='git add -A' # add a file to staging
alias gc='git commit -m'
alias gs='git status'
alias inst="cd ~/Documents/Install"
alias apps="cd ~/Documents/Application"
alias lp='dpkg --get-selections'
alias lu='cat /etc/passwd | cut -d: -f1 | sort'
alias lg='cat /etc/group |cut -d: -f1 | sort'
alias report='hdfs dfsadmin -report'
# Linux only
function lserv(){
sudo service --status-all | grep -E "\[ \+ \]|\[ - \] + \w{0,100} + ((\-+\w{0,100})?){0,10}";
}
function dumpy(){
if [ $2 ]
then
mahout vectordump -i $1 -p true -vs 15 -dt sequencefile -sort $1 -d $2 -n true -o .dump
else
mahout vectordump -i $1 -p true -vs 15 -sort $1 -n true -o .dump
fi
# mahout vectordump -i $1 -p true -vs 15 -dt sequencefile -sort $1 -d $2 -n true -o .dump
if [ $3 ]
then
tail -n $3 .dump
else
tail -n 20 .dump
fi
}
#function ports(){
# echo $1
# echo <password>|sudo -S lsof -w -n -i tcp:80
#}
alias cd='pushd'
alias bd='popd'
export GREP_OPTIONS='--color=auto'
export SLIMERJSLAUNCHER=/Applications/FirefoxNightly.app/Contents/MacOS/firefox
export NVM_SYMLINK_CURRENT="true"
export NVM_DIR="/usr/local/nvm"
export JAVA_HOME=/Library/Java/JavaVirtualMachines/jdk1.8.0_45.jdk/Contents/Home
export PATH="$PATH:/usr/local/nvm/versions/node/v0.12.1/bin"
export NODE_ENV="development"
export PATH=$JAVA_HOME/bin:$PATH
export PS1="\[$(tput bold)\]\[$(tput setaf 1)\][\[$(tput setaf 3)\]\u\[$(tput setaf 2)\]@\[$(tput setaf 4)\]\h \[$(tput setaf 5)\]\W\[$(tput setaf 1)\]]\[$(tput setaf 7)\]\\$ \[$(tput sgr0)\]"
export PATH="/usr/local/sbin:$PATH"
. ~/dot/alias
. ~/dot/functions/bashmarks.bash
. "$NVM_DIR/nvm.sh"
export MAVEN_OPTS="-Xmx1024M -Xss128M -XX:MaxPermSize=1024M -XX:+CMSClassUnloadingEnabled"
#export JAVA_HOME=/usr/lib/jvm/java-7-oracle
#export APACHE=/Users/winston/APACHE
export PHANTOMJS_BIN=/Applications/phantomjs/bin
export M2_REPO=/Users/winston/.m2
export R_HOME=/Users/winston/.rbenv/versions/2.1.1/lib/ruby/gems/2.1.0
#export INSTALL=/Users/winston/hadoop
export HD=/hadoop
export HADOOP_INSTALL=$HD/hadoop
export HADOOP_PREFIX=$HADOOP_INSTALL
export HADOOP_MAPRED_HOME=$HADOOP_INSTALL
export HADOOP_COMMON_HOME=$HADOOP_INSTALL
export HADOOP_HDFS_HOME=$HADOOP_INSTALL
export YARN_HOME=$HADOOP_INSTALL
#export FINDBUGS_HOME=$HADOOP_INSTALL/findbugs-2.0.3
#export JETTY_HOME=$HADOOP_INSTALL/jetty-distribution
export HADOOP_HOME=$HADOOP_INSTALL
# export MAHOUT_LOCAL=$HADOOP/mahout
export MAHOUT_HOME=$HD/mahout
export HBASE_HOME=$HD/hbase
export MAHOUT_DIR=$HD/mahout
export PIG_HOME=$HD/pig
#export HADOOP_PREFIX=$HADOOP_INSTALL/
export VAR=$HD/var
###################################### PIDS
export YARN_PID_DIR=$VAR/pid
export HADOOP_PID_DIR=$VAR/pid
export HADOOP_MAPRED_PID_DIR=$VAR/pid
###################################### CONFIGS
export HADOOP_CONF_DIR=$HADOOP_INSTALL/etc/hadoop
export YARN_CONF_DIR=$HADOOP_CONF_DIR
export HADOOP_MAPRED_CONF_DIR=$HADOOP_CONF_DIR
export PIG_CONF_DIR=$PIG_HOME/conf
export MAHOUT_CONF_DIR=$MAHOUT_HOME/conf
export HBASE_CONF_DIR=$HBASE_HOME/conf
###################################### Spark
export SPARK_HOME=$HADOOP_INSTALL/spark
export SPARK_BIN=$SPARK_HOME/bin
###################################### LOGS
export YARN_LOG_DIR=$VAR/log/yarn
export HADOOP_LOG_DIR=$VAR/log/hadoop
export HADOOP_MAPRED_LOG_DIR=$VAR/log/mapred
export HCAT_LOG_DIR=$VAR/log/hcat
export HIVE_LOG_DIR=$VAR/log/hive
#for j in $HADOOP_HOME/lib/extra*.jar; do
# if [ "$HADOOP_CLASSPATH" ]; then
# export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:$j
# else
# export HADOOP_CLASSPATH=$j
# fi
#done
## Old additions
export HIVE_PREFIX=$HD/hive
export HIVE_HOME=$HIVE_PREFIX
export HIVE_DIR=$HIVE_PREFIX
export HCAT_PREFIX=$HIVE_PREFIX/hcatalog
export HCAT_HOME=$HIVE_PREFIX/hcatalog
export HCAT_DIR=$HIVE_PREFIX/hcatalog
HADOOP_BIN=$HADOOP_HOME/bin
HIVE_BIN=$HIVE_HOME/bin
HCAT_BIN=$HCAT_HOME/bin
HCAT_SBIN=$HCAT_HOME/sbin
MAHOUT_BIN=$MAHOUT_HOME/bin
PIG_BIN=$PIG_HOME/bin
ZOOKEEPER_BIN=$ZOOKEEPER_HOME/bin
HBASE_BIN=$HBASE_HOME/bin
export TITAN_HOME=$HD/titan
export TITAN_BIN=$HD/titan/bin
export CASSANDRA_HOME=$HD/cassandra
export CASSANDRA_BIN=$CASSANDRA_HOME/bin
export ELASTICSEARCH_HOME=$HD/elasticsearch
export ELASTICSEARCH_BIN=$ELASTICSEARCH_HOME/bin
#export PYTHONHOME=/usr/local/bin/python2
export CXX=clang++
export JAVA_HOME=/Library/Java/JavaVirtualMachines/jdk1.8.0_45.jdk/Contents/Home
export PATH=$HADOOP_BIN:$JAVA_HOME/bin:$ELASTICSEARCH_BIN:$CASSANDRA_BIN:$TITAN_BIN:$HIVE_BIN:$HCAT_SBIN:$HCAT_BIN:$MAHOUT_BIN:$PIG_BIN:$ZOOKEEPER_BIN:$HBASE_BIN:$HOME/.rbenv/bin:$HOME/.bin:$HOME/bin:$PATH
export NVM_DIR="/Users/winston/.nvm"
[ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" # This loads nvm
export PATH=/usr/local:$PATH:/usr/local/lib/node_modules
export PATH=$SPARK_BIN:$PHANTOMJS_BIN:$PATH
export PATH="/usr/local/opt/coreutils/libexec/gnubin:$PATH"
#export PYTHONPATH=/usr/local/lib/python2.7/site-packages
export NODE_VERSION="v0.10.34"
export NODE_PATH="$NVM_DIR/$NODE_VERSION"
echo $NODE_PATH;
export PATH=/usr/local/bin:$NODE_PATH/bin:$PYTHONPATH:$PATH
export ARCHFLAGS="-arch x86_64"
source ~/.nvm/nvm.sh
if [ -f $(brew --prefix)/etc/bash_completion ]; then
source $(brew --prefix)/etc/bash_completion
fi
scrape () {
#wget -r -A .jpg http://site.with.images/url/
#wget -mk --header="User-Agent: Mozilla/5.0 (Windows NT 6.0) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.97 Safari/537.11" --header="Referer: http://xmodulo.com/" $1;
wget --header="User-Agent: Mozilla/5.0 (Windows NT 6.0) AppleWebKit/537.11 (KHTML,like Gecko) Chrome/23.0.1271.97 Safari/537.11" --header="Referer: http://xmodulo.com/" --recursive --html-extension --page-requisites --convert-links $1;
}
# Extract most compressed file formats
extract () {
if [ -f $1 ] ; then
case $1 in
*.tar.bz2) tar xvjf $1 ;;
*.tar.gz) tar xvzf $1 ;;
*.bz2) bunzip2 $1 ;;
*.rar) rar x $1 ;;
*.gz) gunzip $1 ;;
*.tar) tar xvf $1 ;;
*.tbz2) tar xvjf $1 ;;
*.tgz) tar xvzf $1 ;;
*.zip) unzip $1 ;;
*.Z) uncompress $1 ;;
*.7z) 7z x $1 ;;
*) echo "don't know how to extract '$1'..." ;;
esac
else
echo "'$1' is not a valid file!"
fi
}
# Linux only, needs lynx (checkip.dyndns.org might change some day)
myip (){
lynx -dump -hiddenlinks=ignore -nolist http://checkip.dyndns.org:8245/ | grep "Current IP Address" | cut -d":" -f2 | cut -d" " -f2
}
__ansi = {
csi: String.fromCharCode(0x1B) + '[',
reset: '0',
text_prop: 'm',
foreground: '3',
bright: '1',
underline: '4',
colors: {
red: '1',
green: '2',
yellow: '3',
blue: '4',
magenta: '5',
cyan: '6'
}
}
if (_isWindows()) {
print("\nSorry! MongoDB Shell Enhancements for Hackers isn't compatible with Windows.\n");
}
var ver = db.version().split(".");
if ( ver[0] <= parseInt("2") && ver[1] < parseInt("2") ) {
print(colorize("\nSorry! Mongo version 2.2.x and above is required! Please upgrade.\n", "red", true));
}
setVerboseShell(true);
setIndexParanoia(true);
setAutoMulti(true);
__indent = " "
function setIndexParanoia( value ) {
if( value == undefined ) value = true;
_indexParanoia = value;
}
function setAutoMulti( value ) {
if( value == undefined ) value = true;
_autoMulti = value;
}
function controlCode( parameters ) {
if ( parameters == undefined ) {
parameters = "";
}
else if (typeof(parameters) == 'object' && (parameters instanceof Array)) {
parameters = parameters.join(';');
}
return __ansi.csi + String(parameters) + String(__ansi.text_prop);
}
function applyColorCode( string, properties ) {
return controlCode(properties) + String(string) + controlCode();
}
function colorize( string, color, bright, underline ) {
var params = [];
var code = __ansi.foreground + __ansi.colors[color];
params.push(code);
if ( bright == true ) params.push(__ansi.bright);
if ( underline == true ) params.push(__ansi.underline);
return applyColorCode( string, params );
}
ObjectId.prototype.toString = function() {
return this.str;
}
ObjectId.prototype.tojson = function(indent, nolint) {
return tojson(this);
}
Date.prototype.tojson = function() {
var UTC = Date.printAsUTC ? 'UTC' : '';
var year = this['get'+UTC+'FullYear']().zeroPad(4);
var month = (this['get'+UTC+'Month']() + 1).zeroPad(2);
var date = this['get'+UTC+'Date']().zeroPad(2);
var hour = this['get'+UTC+'Hours']().zeroPad(2);
var minute = this['get'+UTC+'Minutes']().zeroPad(2);
var sec = this['get'+UTC+'Seconds']().zeroPad(2)
if (this['get'+UTC+'Milliseconds']())
sec += '.' + this['get'+UTC+'Milliseconds']().zeroPad(3)
var ofs = 'Z';
if (!Date.printAsUTC) {
var ofsmin = this.getTimezoneOffset();
if (ofsmin != 0){
ofs = ofsmin > 0 ? '-' : '+'; // This is correct
ofs += (ofsmin/60).zeroPad(2)
ofs += (ofsmin%60).zeroPad(2)
}
}
var date = colorize('"' + year + month + date + 'T' + hour +':' + minute + ':' + sec + ofs + '"', "cyan");
return 'ISODate(' + date + ')';
}
Array.tojson = function( a , indent , nolint ){
var lineEnding = nolint ? " " : "\n";
if (!indent)
indent = "";
if ( nolint )
indent = "";
if (a.length == 0) {
return "[ ]";
}
var s = "[" + lineEnding;
indent += __indent;
for ( var i=0; i<a.length; i++){
s += indent + tojson( a[i], indent , nolint );
if ( i < a.length - 1 ){
s += "," + lineEnding;
}
}
if ( a.length == 0 ) {
s += indent;
}
indent = indent.substring(__indent.length);
s += lineEnding+indent+"]";
return s;
}
tojson = function( x, indent , nolint ) {
if ( x === null )
return colorize("null", "red", true);
if ( x === undefined )
return colorize("undefined", "magenta", true);
if ( x.isObjectId ) {
return 'ObjectId(' + colorize('"' + x.str + '"', "green", false, true) + ')';
}
if (!indent)
indent = "";
switch ( typeof x ) {
case "string": {
var s = "\"";
for ( var i=0; i<x.length; i++ ){
switch (x[i]){
case '"': s += '\\"'; break;
case '\\': s += '\\\\'; break;
case '\b': s += '\\b'; break;
case '\f': s += '\\f'; break;
case '\n': s += '\\n'; break;
case '\r': s += '\\r'; break;
case '\t': s += '\\t'; break;
default: {
var code = x.charCodeAt(i);
if (code < 0x20){
s += (code < 0x10 ? '\\u000' : '\\u00') + code.toString(16);
} else {
s += x[i];
}
}
}
}
s += "\""
return colorize(s, "green", true);
}
case "number":
return colorize(x, "red")
case "boolean":
return colorize("" + x, "blue");
case "object": {
var s = tojsonObject( x, indent , nolint );
if ( ( nolint == null || nolint == true ) && s.length < 80 && ( indent == null || indent.length == 0 ) ){
s = s.replace( /[\s\r\n ]+/gm , " " );
}
return s;
}
case "function":
return colorize(x.toString(), "magenta")
default:
throw "tojson can't handle type " + ( typeof x );
}
}
tojsonObject = function( x, indent , nolint ) {
var lineEnding = nolint ? " " : "\n";
var tabSpace = nolint ? "" : __indent;
assert.eq( ( typeof x ) , "object" , "tojsonObject needs object, not [" + ( typeof x ) + "]" );
if (!indent)
indent = "";
if ( typeof( x.tojson ) == "function" && x.tojson != tojson ) {
return x.tojson(indent,nolint);
}
if ( x.constructor && typeof( x.constructor.tojson ) == "function" && x.constructor.tojson != tojson ) {
return x.constructor.tojson( x, indent , nolint );
}
if ( x.toString() == "[object MaxKey]" )
return "{ $maxKey : 1 }";
if ( x.toString() == "[object MinKey]" )
return "{ $minKey : 1 }";
var s = "{" + lineEnding;
// push one level of indent
indent += tabSpace;
var total = 0;
for ( var k in x ) total++;
if ( total == 0 ) {
s += indent + lineEnding;
}
var keys = x;
if ( typeof( x._simpleKeys ) == "function" )
keys = x._simpleKeys();
var num = 1;
for ( var k in keys ){
var val = x[k];
if ( val == DB.prototype || val == DBCollection.prototype )
continue;
s += indent + colorize("\"" + k + "\"", "yellow") + ": " + tojson( val, indent , nolint );
if (num != total) {
s += ",";
num++;
}
s += lineEnding;
}
// pop one level of indent
indent = indent.substring(__indent.length);
return s + indent + "}";
}
// Hardcode multi update -- now you only need to remember upsert
DBCollection.prototype.update = function( query , obj , upsert, multi ) {
assert( query , "need a query" );
assert( obj , "need an object" );
var firstKey = null;
for (var k in obj) { firstKey = k; break; }
if (firstKey != null && firstKey[0] == '$') {
// for mods we only validate partially, for example keys may have dots
this._validateObject( obj );
} else {
// we're basically inserting a brand new object, do full validation
this._validateForStorage( obj );
}
// can pass options via object for improved readability
if ( typeof(upsert) === 'object' ) {
assert( multi === undefined, "Fourth argument must be empty when specifying upsert and multi with an object." );
opts = upsert;
multi = opts.multi;
upsert = opts.upsert;
}
this._mongo.update( this._fullName , query , obj , upsert ? true : false , _autoMulti ? true : multi );
}
// Override group because map/reduce style is deprecated
DBCollection.prototype.agg_group = function( name, group_field, operation, op_value, filter ) {
var ops = [];
var group_op = { $group: { _id: '$' + group_field } };
if (filter != undefined) {
ops.push({ '$match': filter })
}
group_op['$group'][name] = { };
group_op['$group'][name]['$' + operation] = op_value
ops.push(group_op);
return this.aggregate(ops);
}
// Function that groups and counts by group after applying filter
DBCollection.prototype.gcount = function( group_field, filter ) {
return this.agg_group('count', group_field, 'sum', 1, filter);
}
// Function that groups and sums sum_field after applying filter
DBCollection.prototype.gsum = function( group_field, sum_field, filter ) {
return this.agg_group('sum', group_field, 'sum', '$' + sum_field, filter);
}
// Function that groups and averages avg_feld after applying filter
DBCollection.prototype.gavg = function( group_field, avg_field, filter ) {
return this.agg_group('avg', group_field, 'avg', '$' + avg_field, filter);
}
// Improve the default prompt with process and db name
prompt = function() {
var serverstatus = db.serverStatus();
var host = serverstatus.host.split('.')[0];
var process = serverstatus.process;
var version = db.serverBuildInfo().version;
return process + '~ db:' + db + ' >> ';
}
DBQuery.prototype.shellPrint = function(){
try {
var start = new Date().getTime();
var n = 0;
while ( this.hasNext() && n < DBQuery.shellBatchSize ){
var s = this._prettyShell ? tojson( this.next() ) : tojson( this.next() , "" , true );
print( s );
n++;
}
var output = []
if (typeof _verboseShell !== 'undefined' && _verboseShell) {
var time = new Date().getTime() - start;
var slowms = this._db.setProfilingLevel().slowms;
var fetched = "Fetched " + n + " record(s) in ";
if (time > slowms) {
fetched += colorize(time + "ms", "red", true);
} else {
fetched += colorize(time + "ms", "green", true);
}
output.push(fetched);
}
if (typeof _indexParanoia !== 'undefined' && _indexParanoia) {
var explain = this.clone();
explain._ensureSpecial();
explain._query.$explain = true;
explain._limit = Math.abs(n._limit) * -1;
var result = explain.next();
var type = result.cursor;
var index_use = "Index["
if (type == "BasicCursor") {
index_use += colorize( "none", "red", true);
} else {
index_use += colorize( result.cursor.substring(12), "green", true );
}
index_use += "]";
output.push(index_use);
}
if ( this.hasNext() ) {
___it___ = this;
output.push("More[" + colorize("true", "green", true) + "]");
}
else {
___it___ = null;
output.push("More[" + colorize("false", "red", true) + "]");
}
print(output.join(" -- "));
}
catch ( e ){
print( e );
}
}
// Set print to .pretty() by default
DBQuery.prototype._prettyShell = true
build (){
ext=""; # Default value
reg_ext="^.+\.(([pP][dD][fF])|([jJ][pP][gG])|([pP][nN][gG])))$"
reg_ext="^.+\.([a-zA-Z]){0,3}$"
rex="^ext\=${reg_ext}" #[^a-zA-Z0-9\ ]
rex='^ext\=(\.[a-z]{1,3})$'
echo "$rex";
for var in "$@"
do
echo "";
# echo "$var";
# Check if it is setting or changing the extension
if [[ $var =~ $rex ]];
then
echo "Setting extension to ${BASH_REMATCH[1]}";
ext=${BASH_REMATCH[1]} ;
else
# Contains an extension already?
if [[ $var =~ $reg_ext ]]
then
# echo "Already has ext $BASH_REMATCH";
cur_ext="" ;
else
# eval echo "No extension setting to $ext";
eval "cur_ext=$ext" ;
fi
echo "$var -> $var$cur_ext"
# touch "$var$cur_ext"
fi
done
}
visitor () {
if [ -d $1 ]; then
echo "$2 better be a valid command...";
folder=`ls --quote-name --quoting-style=escape -1 $1`
for line in $folder ; do
`$2 $1/"$line"`;
done
else
echo "The file must be a directory";
fi
# for f in `ls -Al --author /usr/local/lib/node_modules | cut -d ' ' -f3
# ls -lGua /usr/local/lib/node_modules | cut -c 16- | egrep '(\w)+\s{3}'
# ls -lGua /usr/local/lib/node_modules | cut -c 16- | grep -o -E '\s(([a-z])+([a-zA-Z0-9]|\.|\-)*)+$'
}
sni () {
if [ -n "$1" ]; then
echo 'installing '$1;
sudo npm install -g $1;
if [ -d `pwd`/node_modules ]; then
echo 'linking '$1;
npm link $1;
touch .dependencies;
echo $1 >> .dependencies;
else
echo 'no node_modules in current dir';
fi
else
echo $1 'not defined';
fi
}
ni () {
if [ -n "$1" ]; then
echo 'Installing $1';
echo $1;
npm install --save $1;
echo 'Install complete';
if [ -d './node_modules/'$1 ]; then
echo 'Successful: ./node_modules/'$1;
else
echo "Failed: Something Wrong";
fi;
else
echo "You have to put in a module or two";
fi;
# echo 'Fixing directory permissions';
# sudo chown -R winston '/usr/local/lib/node_modules/$(1)';
# else
# ehco 'If the file installed correctly, the permissions may need to be set';
# echo 'sudo chown -R $USER /usr/local/lib/node_modules/$1';
# echo '\n';
# echo 'If it didnt';
# echo 'sudo npm install -g $1';
# fi
# if [ -d `pwd`'/node_modules' ]; then
# echo 'Linking '$1;
# npm link $1;
# touch .dependencies;
# echo $1 >> .dependencies;
# else
# echo 'no node_modules in current dir';
# fi
# else
# if [ -f `pwd`'/package.json' ]; then
# npm install;
# else
# echo '$1 not defined';
# fi
# fi
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment