|
export IDE_DBG="-Xdebug -Xrunjdwp:transport=dt_socket,server=n,address=5005,suspend=y" |
|
|
|
## Spark |
|
# alias to start a local Spark cluster with a single worker node |
|
alias spark-start="mkdir -p /tmp/spark && $SPARK_HOME/sbin/start-master.sh >/tmp/spark/master-start.out 2>/tmp/spark/master-start.err && $SPARK_HOME/bin/spark-class org.apache.spark.deploy.worker.Worker spark://`hostname`:7077 -c 6 -m 2048M >/tmp/spark/worker.out 2>/tmp/spark/worker.err &" |
|
# alias to stop aforementioned single-worker local cluster |
|
alias spark-stop="$SPARK_HOME/sbin/stop-master.sh && pkill -f org.apache.spark.deploy.worker.Worker" |
|
# to potentially enable event logging, create default events dir (can be useful for debugging) |
|
mkdir -p /tmp/spark-events |
|
|
|
|
|
## SDC |
|
alias run-sdc='echo "Running command $HOME/src/streamsets/datacollector/dist/target/streamsets-datacollector-*-SNAPSHOT/streamsets-datacollector-*-SNAPSHOT/bin/streamsets dc" ; $HOME/src/streamsets/datacollector/dist/target/streamsets-datacollector-*-SNAPSHOT/streamsets-datacollector-*-SNAPSHOT/bin/streamsets dc' |
|
alias compile-sdc='echo "Running command: mvn clean install -DskipTests -Pdist,ui" ; mvn clean install -DskipTests -Pdist,ui' |
|
alias mvn-compile-profiles-dist-ui='echo "Running command: mvn clean install -Pdist,ui" ; mvn clean install -Pdist,ui' |
|
alias mvn-compile-profile-dist='echo “Running command: mvn clean install -Pdist" ; mvn clean install -Pdist' |
|
alias sdcdebug='export SDC_JAVA_OPTS=$IDE_DBG' |
|
alias sdcclear='export SDC_JAVA_OPTS=' # -n -> remove the export property from each NAME |
|
|
|
# sdc container stuff (you will need to ask someone from the team for the docker file to build the image) |
|
# alias for running sdc locally. In order for it to correctly run I need to be in $HOME/src/streamsets/datacollector/dist/target/streamsets-datacollector-X.Y.Z-SNAPSHOT/streamsets-datacollector-X.Y.Z-SNAPSHOT |
|
# also note sdc-container image must be in the "docker images" list. otherwise I have to create it running "docker build $HOME/src/docker-images/sdc-container/ -t sdc-container" |
|
alias sdc-container-start-simple='docker run --net=cluster --name=sdc-container -it -p 18630:18630 -v ${SDC_DIST_DIRECTORY}:/sdc sdc-container' |
|
# alias for running sdc locally without further checks needed |
|
alias sdc-container-start='docker stop $(docker ps -a -q --filter ancestor=sdc-container --format={{.ID}}); docker rm $(docker ps -a -q --filter ancestor=sdc-container --format={{.ID}}); docker network create cluster || true; docker run --net=cluster --name=sdc-container -it -p 5005:5005 -p 18630:18630 -v ${SDC_DIST_DIRECTORY}:/sdc sdc-container' |
|
# alias to create sdc-container aka (also known as) java-callback. Dockerfile must be located at the provided path (currently: $HOME/src/docker-images/sdc-container/) |
|
alias sdc-container-build='docker build $HOME/src/docker-images/sdc-container/ -t sdc-container' |
|
|
|
# Function to run sdc docker image for version $1 exposed into port 18630 |
|
function run-sdc-docker-image() { |
|
docker run --restart on-failure -p 18630:18630 -d --name streamsets-dc streamsets/datacollector:$1 dc |
|
} |
|
|
|
run-sdc-with-stf () { |
|
if [[ $# -lt 1 ]] |
|
then |
|
echo "Usage: run-sdc-with-stf '<SDC-IMAGE-VERSION> [<SCH-SERVER-URL>] [<SCH-USERNAME> <SCH-PASSWORD>]'" |
|
echo "Example 1: run-sdc-with-stf 3.16.0 => runs sdc without registering it in sch" |
|
echo "Example 2: run-sdc-with-stf 3.16.0 http://192.168.1.33:18631 => runs sdc registering it in sch using admin@admin both as sch username and sch user password" |
|
echo "Example 3: run-sdc-with-stf 3.16.0 http://192.168.1.33:18631 user@org examplePass => runs sdc registering it in sch using user@org as sch username and examplePass as sch password" |
|
echo "Example 4: run-sdc-with-stf 3.16.0 http://192.168.1.36:18631 user@org examplePass stage-lib-1 stage-lib-2 stage-lib-n" |
|
else |
|
if [[ $# -lt 2 ]] |
|
then |
|
commandToRun="stf -v start sdc --version $1 --enable-base-http-url 'private'" |
|
elif [[ $# -lt 3 ]] |
|
then |
|
commandToRun="stf -v start sdc --version $1 --sch-server-url $2 --sch-username 'admin@admin' --sch-password 'admin@admin' --enable-base-http-url 'private'" |
|
elif [[ $# -eq 4 ]] |
|
then |
|
commandToRun="stf -v start sdc --version $1 --sch-server-url $2 --sch-username $3 --sch-password $4 --enable-base-http-url 'private'" |
|
else |
|
commandToRun="stf -v start sdc --version $1 --sch-server-url $2 --sch-username $3 --sch-password $4 --enable-base-http-url 'private' --stage-lib ${@:5}" |
|
fi |
|
echo "Running command: $commandToRun" |
|
eval "$commandToRun" |
|
fi |
|
} |
|
|
|
|
|
## Docker |
|
alias docker-stop-all-containers='docker stop $(docker ps -a -q)' |
|
alias docker-remove-all-stopped-containers='docker rm $(docker ps -a -q)' |
|
|
|
# Function to open bash shell into container named (or container id) $1 |
|
function docker-term() { |
|
docker exec -it $1 /bin/bash |
|
} |
|
|
|
|
|
## SCH/DPM |
|
alias start-influxdb='docker run --name influxdb -d -p 8086:8086 -v $PWD:/var/lib/influxdb influxdb' |
|
alias start-mysql='docker run --name mysql-5.7 -d -p 3306:3306 -e MYSQL_ROOT_PASSWORD=root mysql:5.7' |
|
alias compile-dpm='pushd $HOME/src/streamsets/domainserver/; echo "running command ./gradlew clean installDist -PfilterFile=dev-mysql"; ./gradlew clean installDist -PfilterFile=dev-mysql; popd' |
|
alias run-dpm='$DPM_DIST/bin/streamsets dpm' |
|
alias dpm-debug='export DPM_JAVA_OPTS=$IDE_DBG' |
|
alias dpm-debug-ui='export DPM_JAVA_OPTS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=1055 -Ddpm.static-web.dir=$HOME/src/streamsets/domainserver/server-ui/dist"' |
|
alias dpm-clear='export DPM_JAVA_OPTS=' |
|
alias set-dpm-dist='pushd $HOME/src/streamsets/domainserver/build/install/streamsets-dpm-*/ && export DPM_DIST=$(pwd) && popd' |
