Created
April 5, 2019 11:21
-
-
Save imoutsatsos/256239cb2eb8a9a5932520e77601656b to your computer and use it in GitHub Desktop.
Example Jenkins pipeline for launching Tensorboard
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
pipeline | |
{ | |
agent | |
{ | |
node | |
{ | |
label params.NodeLabel | |
// this whole construction is meant to select the node using its label 7g ry04 ry03 | |
} | |
} | |
stages { //this is a jenkins stuff telling it stages of the build and such | |
stage('run') { | |
steps { | |
script { | |
def command_do = command; | |
//comment the jobs build: | |
currentBuild.displayName = "#${BUILD_NUMBER} ${checkout} ${env.NODE_NAME}" // ${env_name} | |
currentBuild.description = "${desc} \n ${command_do}" | |
echo currentBuild.description; // to see it in the log also | |
echo " " | |
echo " " | |
git( | |
url: 'ssh://git@we/repo.git', | |
credentialsId: 'jenkins_node_key', | |
branch: params.checkout | |
) | |
def notforceinstall = run_setup_py == false; | |
virtualenvs_local = true // the package from setup.py links the code with the virtualenv | |
def env_install_sh = "" | |
if (virtualenvs_local) | |
{ | |
/* | |
todo add | |
i=0 | |
tput sc | |
while fuser /var/lib/dpkg/lock >/dev/null 2>&1 ; do | |
case \$((\$i % 4)) in | |
0 ) j="-" ;; | |
1 ) j="\\" ;; | |
2 ) j="|" ;; | |
3 ) j="/" ;; | |
esac | |
tput rc | |
echo -en "\r[\$j] Waiting for other software managers to finish..." | |
sleep 0.5 | |
((i=i+1)) | |
*/ | |
env_install_sh = """ | |
while fuser /var/lib/dpkg/lock >/dev/null 2>&1 ; do | |
echo wait for dpkg | |
sleep 1 | |
done | |
if [ -f ${env.WORKSPACE}/${env_name}/bin/activate ] && ${notforceinstall}; then | |
. ${env.WORKSPACE}/${env_name}/bin/activate | |
else | |
if [ -f ${env.WORKSPACE}/${env_name}/bin/activate ]; then | |
lsof ${env.WORKSPACE}/${env_name}/bin/python || echo 'unused' | |
echo 'bin activate exists' | |
else | |
virtualenv ${env.WORKSPACE}/${env_name} | |
fi | |
. ${env.WORKSPACE}/${env_name}/bin/activate | |
which pip | |
if [ -f /etc/pip.conf ]; then | |
echo '/etc/pip.conf exists' | |
cat /etc/pip.conf | |
else | |
echo '/etc/pip.conf does not exist, copying pip.conf into user settings' | |
mkdir -p ~/.pip | |
cp pip.conf ~/.pip/ | |
fi | |
if [ -f requirements-tf-gpu.txt ]; then | |
echo Installing tensorflow | |
pip install -r requirements-tf-gpu.txt | |
echo Installing python dependencies | |
./install_python_deps.sh | |
echo Installing | |
pip install -e . | |
else | |
pip install -e .[tf_gpu] | |
fi | |
fi | |
""" | |
/* | |
updated from: | |
${env.WORKSPACE}/${env_name}/bin/pip install -e .[tf_gpu] | |
${env.WORKSPACE}/${env_name}/bin/pip install nvgpu | |
*/ | |
} | |
else | |
{ | |
env_install_sh = """ | |
if workon ${env_name} ; then | |
echo "workon succeeded" | |
else | |
if [ -f /_envs/${env_name}/bin/activate ] && ${notforceinstall}; then | |
. /_envs/${env_name}/bin/activate | |
elif [ -f ./${env_name}/bin/activate ] && ${notforceinstall}; then | |
. ./${env_name}/bin/activate | |
elif [ -f /home/jenkins/${env_name}/bin/activate ] && ${notforceinstall}; then | |
. /home/jenkins/${env_name}/bin/activate | |
else | |
pip install virtualenv || echo 'not found pip' | |
virtualenv /home/jenkins/${env_name} | |
. /home/jenkins/${env_name}/bin/activate | |
mkdir -p ~/.pip | |
cp pip.conf ~/.pip/ | |
pip install -e .[tf_gpu] | |
pip install nvgpu | |
fi | |
fi | |
""" | |
} | |
// runs a shell script using jenkins groovy command 'sh' | |
sh """ | |
nvidia-smi || true | |
aws iam list-access-keys || echo 'aws iam failed' | |
pwd | |
whoami | |
which pip | |
echo \$VIRTUAL_ENV | |
echo ----------------------------------------------------------------------------------- | |
${env_install_sh} | |
echo ----------------------------------------------------------------------------------- | |
echo \$VIRTUAL_ENV | |
echo ----------------------------------------------------------------------------------- | |
echo setup complete | |
if [ "${gpu}" = "auto" ]; then | |
export CUDA_VISIBLE_DEVICES=\$(nvgpu available -l 1) | |
else | |
export CUDA_VISIBLE_DEVICES=${gpu} | |
fi | |
${command_do} | |
""" | |
/* | |
todo: | |
run tensorboard and get his pid and then terminate it | |
if [ "${run_tensorboard}" = "True" ]; then | |
tensorboard --logdir ${env.WORKSPACE}/Graph & | |
fi | |
*/ | |
currentBuild.rawBuild.keepLog(true) | |
// if we run another sh, the virtualenv is forgotten! | |
} | |
} | |
} | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment