Beam run java worker pipeline

commands.sh

# build all binaries skipping docker images and running tests
./gradlew build -x rat -x javadoc -x test -x website -x docs -x sdist -x lintPy27 -x lintPy27_3 -x lintPy3 -x testWebsite -x spotlessJava -x goTest -x checkstyleMain -x buildDockerImage -x spotbugsMain -x checkstyleTest -x buildLocalWebsite -x goVet -x gofmt

# skip compiling java tests
-x compileTestJava -x shadowJar -x compileJava

# Build java worker jav (DF runner harness)
./gradlew :beam-runners-google-cloud-dataflow-java-fn-api-worker:build -x spotlessJava -x rat -x test
./gradlew :beam-runners-google-cloud-dataflow-java-fn-api-worker:shadowJar

# Build Python SDK Harness docker image
./gradlew -Pdocker-repository-root=gcr.io/dataflow-build/migryz/beam -p sdks/python/container docker
gcloud docker -- push gcr.io/dataflow-build/migryz/beam/python:latest

# run pipeline
virtualenv env
. ./env/bin/activate

clear && clear && java -cp beam-examples-java-2.12.0-SNAPSHOT.jar org.apache.beam.examples.MinimalWordCount   --runner DataflowRunner   --num_workers 1   --project google.com:clouddfe   --input gs://dataflow-samples/shakespeare/kinglear.txt   --output gs://clouddfe-test/staging-$USER/output   --temp_location gs://clouddfe-test/staging-$USER/   --worker_harness_container_image gcr.io/dataflow-build/$USER/beam/python:latest   --sdk_location build/apache-beam-2.12.0.dev0.tar.gz   --dataflow_worker_jar '/usr/local/google/home/migryz/work/osbeam/myfork/runners/google-cloud-dataflow-java/worker/build/libs/beam-runners-google-cloud-dataflow-java-fn-api-worker-2.12.0-SNAPSHOT.jar'
clear && clear && python -m apache_beam.examples.wordcount   --runner DataflowRunner   --num_workers 1   --project google.com:clouddfe   --output gs://clouddfe-test/staging-$USER/output   --temp_location gs://clouddfe-test/staging-$USER/   --worker_harness_container_image gcr.io/dataflow-build/$USER/beam/python:latest   --experiment beam_fn_api   --sdk_location build/apache-beam-2.12.0.dev0.tar.gz   --dataflow_worker_jar '/usr/local/google/home/migryz/work/osbeam/myfork/runners/google-cloud-dataflow-java/worker/build/libs/beam-runners-google-cloud-dataflow-java-fn-api-worker-2.12.0-SNAPSHOT.jar'

# run unittests for python
python setup.py nosetests -s --tests apache_beam.runners.portability.fn_api_runner_test:FnApiRunnerTest.test_element_count_metrics

# run unittest for java
clear && clear && ./gradlew :beam-runners-google-cloud-dataflow-java-fn-api-worker:test --tests org.apache.beam.runners.dataflow.worker.SystemIdToDfePCollectionNameMappingBuilderTest -s --info

############################################
# run HelloWorld examples integration test
./gradlew :beam-runners-google-cloud-dataflow-java-fn-api-worker:build -x spotlessJava -x rat -x test
./gradlew :beam-runners-google-cloud-dataflow-java-fn-api-worker:shadowJar

# Disabling undesired test
sed -i 's/^  \/\/ include \"\*\*\/WindowedWordCountIT\.class\"$/  include \"\*\*\/WindowedWordCountIT\.class\"/' ./runners/google-cloud-dataflow-java/examples/build.gradle
# Enabling test back
sed -i 's/^  include \"\*\*\/WindowedWordCountIT\.class\"$/  \/\/ include \"\*\*\/WindowedWordCountIT\.class\"/' ./runners/google-cloud-dataflow-java/examples/build.gradle

clear && clear && ./gradlew :beam-runners-google-cloud-dataflow-java-examples:preCommitFnApiWorker -PdataflowProject=google.com:clouddfe -Pproject=google.com:clouddfe -PgcpProject=google.com:clouddfe -PgcsTempRoot=gs://clouddfe-test/stating-migryz -PdataflowTempRoot=gs://clouddfe-test/staging-migryz -PdockerImageRoot=gcr.io/dataflow-build/migryz/beam
clear && clear && ./gradlew :beam-runners-google-cloud-dataflow-java-examples:preCommitLegacyWorker -PdataflowProject=google.com:clouddfe -Pproject=google.com:clouddfe -PgcpProject=google.com:clouddfe -PgcsTempRoot=gs://clouddfe-test/stating-migryz -PdataflowTempRoot=gs://clouddfe-test/staging-migryz -PdockerImageRoot=gcr.io/dataflow-build/migryz/beam