Skip to content

Instantly share code, notes, and snippets.

@stain
Last active July 2, 2019 10:55
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save stain/87fc34a13eed60179447729cdf7f9ac7 to your computer and use it in GitHub Desktop.
Save stain/87fc34a13eed60179447729cdf7f9ac7 to your computer and use it in GitHub Desktop.
### Editing this command? Also update https://ci.commonwl.org/job/cwl-spec-pr/configure
venv() {
if ! test -d $1 ; then
virtualenv $1
fi
. $1/bin/activate
}
venv venv
(. venv/bin/activate && pip install -U setuptools wheel pip)
(. venv/bin/activate && pip install --ignore-installed --verbose -U cwltest cwltool)
shutdownGalaxy() {
bash scripts/end_cwl_conformance.bash
}
trap "shutdownGalaxy" EXIT
START_DIR=$(pwd)
GALAXY_SKIP_CLIENT_BUILD=1 sh scripts/common_startup.sh
(. .venv/bin/activate && pip install psycopg2)
bash scripts/run_for_cwl_conformance.bash
### Editing this command? Also update https://ci.commonwl.org/job/cwl-spec-pr/configure
(. venv/bin/activate && cd common-workflow-language && ./run_test.sh \
--junit-xml=${START_DIR}/result.xml \
RUNNER=${START_DIR}/scripts/cwl-runner) || /bin/true
set -e
shutdownGalaxy || /bin/true
#!/bin/bash
export LC_ALL=en_US.UTF-8
cloneorpull() {
if test -d $1 ; then
(cd $1 && git pull)
else
git clone $2
fi
}
venv() {
if ! test -d $1 ; then
virtualenv $1
fi
. $1/bin/activate
}
if [[ "${implementation}" == *arvados* ]]
then
cloneorpull arvados https://github.com/curoverse/arvados.git
docker volume ls -qf dangling=true | grep -v metadata.db | xargs -r docker volume rm || true
docker pull arvados/arvbox-demo:latest
export PATH=$PATH:$PWD/tools/arvbox/bin
export ARVBOX_CONTAINER=cwltest
arvados/sdk/cwl/test_with_arvbox.sh --config localdemo --leave-running --junit-xml=/tmp/junit.xml \
RUNNER=/tmp/cwltest/arv-cwl-containers
CODE=$?
docker exec $ARVBOX_CONTAINER cat /tmp/junit.xml > junit.xml
arvbox reset -f
docker volume ls -qf dangling=true | grep -v metadata.db | xargs -r docker volume rm || true
docker rmi $(docker images -a --no-trunc | grep "^arvados/arvbox-demo" | grep -v "latest" | sed 's/ */ /g' \
| cut -d' ' -f3) || true
exit $CODE
elif [[ "${implementation}" == *cwltool* ]]
then
cloneorpull cwltool https://github.com/common-workflow-language/cwltool.git
venv cwltool-venv
(. cwltool-venv/bin/activate && PIP_DOWNLOAD_CACHE=/var/lib/jenkins/pypi-cache/ pip install -U setuptools wheel pip)
(. cwltool-venv/bin/activate && PIP_DOWNLOAD_CACHE=/var/lib/jenkins/pypi-cache/ pip install ./cwltool/)
(. cwltool-venv/bin/activate && PIP_DOWNLOAD_CACHE=/var/lib/jenkins/pypi-cache/ pip install "cwltest>=1.0.20180518074130")
(. cwltool-venv/bin/activate && cd common-workflow-language && \
./run_test.sh --junit-xml=result.xml RUNNER=cwltool DRAFT=$version EXTRA=--enable-dev -j2)
elif [[ "${implementation}" == *rabix* ]]
then
cloneorpull bunny https://github.com/rabix/bunny.git
venv cwltest-venv
(. cwltest-venv/bin/activate && PIP_DOWNLOAD_CACHE=/var/lib/jenkins/pypi-cache/ pip install -U setuptools wheel pip)
(. cwltest-venv/bin/activate && PIP_DOWNLOAD_CACHE=/var/lib/jenkins/pypi-cache/ pip install "cwltest>=1.0.20180518074130")
cd bunny
mvn clean package -P cwl
tar xzf rabix-cli/target/rabix-cli-*-release.tar.gz
set +e
LC_ALL=C
LANG=C
(. ../cwltest-venv/bin/activate && export DOCKER_HOST="unix:///var/run/docker.sock" \
&& cd ../common-workflow-language && ./run_test.sh --junit-xml=result.xml \
RUNNER=../bunny/rabix-cli-*/rabix DRAFT=$version -j$(($(nproc) / 2)))
set -e
elif [[ "${implementation}" == *cromwell* ]]
then
CWLDIR=$PWD
if [ -d "${CWLDIR}/v1.0" ];
then
docker run --mount type=bind,source=${CWLDIR},destination=/cwl debian:latest rm -Rf /cwl/v1.0
fi
cloneorpull cromwell https://github.com/broadinstitute/cromwell.git
venv cromwell_venv
(. cromwell_venv/bin/activate && pip install -U setuptools wheel pip)
(. cromwell_venv/bin/activate && pip install -U "cwltest>=1.0.20180518074130" cwltool)
cd cromwell
sbt clean assembly
CROMWELL_JAR=$(find "$(pwd)/server/target/scala-2.12" -name "cromwell-*.jar")
CENTAUR_CWL_JAR=$(find "$(pwd)/centaurCwlRunner/target/scala-2.12" -name "centaur-cwl-runner-*.jar")
shutdownCromwell() {
if [ -z "${CROMWELL_PID}" ]; then
if [ -d "${CWLDIR}/v1.0" ]; then
docker run --mount type=bind,source=${CWLDIR},destination=/cwl debian:latest rm -Rf /cwl/v1.0
fi
if [ -z "${CROMWELL_PID}" ]; then
kill -9 ${CROMWELL_PID}
fi
fi
}
trap "shutdownCromwell" EXIT
START_DIR=$(pwd)
cd ../common-workflow-language/v1.0/v1.0
java \
-Xmx2g \
-Dsystem.new-workflow-poll-rate=1 \
-Dbackend.providers.Local.config.script-epilogue="sleep 5 && sync" \
-Dsystem.job-shell=/bin/sh \
-jar ${CROMWELL_JAR} server &
CROMWELL_PID=$$
sleep 10
cd ${START_DIR}
cat > ${START_DIR}/centaur-cwl-runner << EOF
#!/bin/sh
exec java -Xmx1g -jar ${CENTAUR_CWL_JAR} \$@
EOF
chmod a+x ${START_DIR}/centaur-cwl-runner
(. ../cromwell_venv/bin/activate && cd ../common-workflow-language && \
./run_test.sh --junit-xml=${START_DIR}/result.xml -j$(($(nproc) /2 ))\
RUNNER=${START_DIR}/centaur-cwl-runner) || /bin/true
set -e
shutdownCromwell
fi
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment