Example Jenkins pipeline for launching Tensorboard
pipeline | |
{ | |
agent | |
{ | |
node | |
{ | |
label params.NodeLabel | |
// this whole construction is meant to select the node using its label 7g ry04 ry03 | |
} | |
} | |
stages { //this is a jenkins stuff telling it stages of the build and such | |
stage('run') { | |
steps { | |
script { | |
def command_do = command; | |
//comment the jobs build: | |
currentBuild.displayName = "#${BUILD_NUMBER} ${checkout} ${env.NODE_NAME}" // ${env_name} | |
currentBuild.description = "${desc} \n ${command_do}" | |
echo currentBuild.description; // to see it in the log also | |
echo " " | |
echo " " | |
git( | |
url: 'ssh://git@we/repo.git', | |
credentialsId: 'jenkins_node_key', | |
branch: params.checkout | |
) | |
def notforceinstall = run_setup_py == false; | |
virtualenvs_local = true // the package from setup.py links the code with the virtualenv | |
def env_install_sh = "" | |
if (virtualenvs_local) | |
{ | |
/* | |
todo add | |
i=0 | |
tput sc | |
while fuser /var/lib/dpkg/lock >/dev/null 2>&1 ; do | |
case \$((\$i % 4)) in | |
0 ) j="-" ;; | |
1 ) j="\\" ;; | |
2 ) j="|" ;; | |
3 ) j="/" ;; | |
esac | |
tput rc | |
echo -en "\r[\$j] Waiting for other software managers to finish..." | |
sleep 0.5 | |
((i=i+1)) | |
*/ | |
env_install_sh = """ | |
while fuser /var/lib/dpkg/lock >/dev/null 2>&1 ; do | |
echo wait for dpkg | |
sleep 1 | |
done | |
if [ -f ${env.WORKSPACE}/${env_name}/bin/activate ] && ${notforceinstall}; then | |
. ${env.WORKSPACE}/${env_name}/bin/activate | |
else | |
if [ -f ${env.WORKSPACE}/${env_name}/bin/activate ]; then | |
lsof ${env.WORKSPACE}/${env_name}/bin/python || echo 'unused' | |
echo 'bin activate exists' | |
else | |
virtualenv ${env.WORKSPACE}/${env_name} | |
fi | |
. ${env.WORKSPACE}/${env_name}/bin/activate | |
which pip | |
if [ -f /etc/pip.conf ]; then | |
echo '/etc/pip.conf exists' | |
cat /etc/pip.conf | |
else | |
echo '/etc/pip.conf does not exist, copying pip.conf into user settings' | |
mkdir -p ~/.pip | |
cp pip.conf ~/.pip/ | |
fi | |
if [ -f requirements-tf-gpu.txt ]; then | |
echo Installing tensorflow | |
pip install -r requirements-tf-gpu.txt | |
echo Installing python dependencies | |
./install_python_deps.sh | |
echo Installing | |
pip install -e . | |
else | |
pip install -e .[tf_gpu] | |
fi | |
fi | |
""" | |
/* | |
updated from: | |
${env.WORKSPACE}/${env_name}/bin/pip install -e .[tf_gpu] | |
${env.WORKSPACE}/${env_name}/bin/pip install nvgpu | |
*/ | |
} | |
else | |
{ | |
env_install_sh = """ | |
if workon ${env_name} ; then | |
echo "workon succeeded" | |
else | |
if [ -f /_envs/${env_name}/bin/activate ] && ${notforceinstall}; then | |
. /_envs/${env_name}/bin/activate | |
elif [ -f ./${env_name}/bin/activate ] && ${notforceinstall}; then | |
. ./${env_name}/bin/activate | |
elif [ -f /home/jenkins/${env_name}/bin/activate ] && ${notforceinstall}; then | |
. /home/jenkins/${env_name}/bin/activate | |
else | |
pip install virtualenv || echo 'not found pip' | |
virtualenv /home/jenkins/${env_name} | |
. /home/jenkins/${env_name}/bin/activate | |
mkdir -p ~/.pip | |
cp pip.conf ~/.pip/ | |
pip install -e .[tf_gpu] | |
pip install nvgpu | |
fi | |
fi | |
""" | |
} | |
// runs a shell script using jenkins groovy command 'sh' | |
sh """ | |
nvidia-smi || true | |
aws iam list-access-keys || echo 'aws iam failed' | |
pwd | |
whoami | |
which pip | |
echo \$VIRTUAL_ENV | |
echo ----------------------------------------------------------------------------------- | |
${env_install_sh} | |
echo ----------------------------------------------------------------------------------- | |
echo \$VIRTUAL_ENV | |
echo ----------------------------------------------------------------------------------- | |
echo setup complete | |
if [ "${gpu}" = "auto" ]; then | |
export CUDA_VISIBLE_DEVICES=\$(nvgpu available -l 1) | |
else | |
export CUDA_VISIBLE_DEVICES=${gpu} | |
fi | |
${command_do} | |
""" | |
/* | |
todo: | |
run tensorboard and get his pid and then terminate it | |
if [ "${run_tensorboard}" = "True" ]; then | |
tensorboard --logdir ${env.WORKSPACE}/Graph & | |
fi | |
*/ | |
currentBuild.rawBuild.keepLog(true) | |
// if we run another sh, the virtualenv is forgotten! | |
} | |
} | |
} | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment