Example Jenkins pipeline for launching Tensorboard

tensorboard.pipeline

pipeline
{
    agent
    {
        node
        {
            label params.NodeLabel
            // this whole construction is meant to select the node using its label 7g ry04 ry03
        }
    }
    stages { //this is a jenkins stuff telling it stages of the build and such
        stage('run') {
            steps {
                 script {
                    def command_do = command;
    
                    //comment the jobs build:
                    currentBuild.displayName = "#${BUILD_NUMBER} ${checkout} ${env.NODE_NAME}" // ${env_name}
                    currentBuild.description = "${desc} \n ${command_do}"
    
                    echo currentBuild.description; // to see it in the log also
                    echo " "
                    echo " "
                     
                    git(
                       url: 'ssh://git@we/repo.git',
                       credentialsId: 'jenkins_node_key',
                       branch: params.checkout
                    )
    
                    def notforceinstall = run_setup_py == false;
    
                    virtualenvs_local = true // the package from setup.py links the code with the virtualenv
    
                    def env_install_sh = ""
                    if (virtualenvs_local)
                    {
                        /*
                        todo add
                        i=0 
                        tput sc 
                        while fuser /var/lib/dpkg/lock >/dev/null 2>&1 ; do
                             case \$((\$i % 4)) in
                                 0 ) j="-" ;;
                                 1 ) j="\\" ;;
                                 2 ) j="|" ;;
                                 3 ) j="/" ;;
                             esac
                             tput rc
                             echo -en "\r[\$j] Waiting for other software managers to finish..." 
                             sleep 0.5
                             ((i=i+1)) 
                        */
                        env_install_sh = """
                        
                        while fuser /var/lib/dpkg/lock >/dev/null 2>&1 ; do
                            echo wait for dpkg
                            sleep 1
                        done
                       
                        if [ -f ${env.WORKSPACE}/${env_name}/bin/activate ] && ${notforceinstall}; then
                           . ${env.WORKSPACE}/${env_name}/bin/activate
                        else   
                            if [ -f ${env.WORKSPACE}/${env_name}/bin/activate ]; then
                                lsof ${env.WORKSPACE}/${env_name}/bin/python || echo 'unused'
                                echo 'bin activate exists'
                            else
                                virtualenv ${env.WORKSPACE}/${env_name}
                            fi
                            . ${env.WORKSPACE}/${env_name}/bin/activate
                            which pip
                            if [ -f /etc/pip.conf ]; then
                                echo '/etc/pip.conf exists'
                                cat /etc/pip.conf
                            else
                                echo '/etc/pip.conf does not exist, copying pip.conf into user settings'
                                mkdir -p ~/.pip
                                cp pip.conf ~/.pip/
                            fi
                            
                            if [ -f requirements-tf-gpu.txt ]; then
                                echo Installing tensorflow
                                pip install -r requirements-tf-gpu.txt
                                echo Installing python dependencies
                                ./install_python_deps.sh
                                echo Installing 
                                pip install -e .
                            else
                                pip install -e .[tf_gpu]
                            fi    
                            
                        fi
                        """
                        /*
                        updated from:
                        ${env.WORKSPACE}/${env_name}/bin/pip install -e .[tf_gpu]
                        ${env.WORKSPACE}/${env_name}/bin/pip install nvgpu
                        */
                    }
                    else
                    {
                        env_install_sh = """
                        if workon ${env_name} ; then
                            echo "workon succeeded"
                        else
                            if [ -f /_envs/${env_name}/bin/activate ] && ${notforceinstall}; then
                               . /_envs/${env_name}/bin/activate
                            elif [ -f ./${env_name}/bin/activate ] && ${notforceinstall}; then
                               . ./${env_name}/bin/activate
                            elif [ -f /home/jenkins/${env_name}/bin/activate ] && ${notforceinstall}; then
                               . /home/jenkins/${env_name}/bin/activate
                            else   
                                pip install virtualenv || echo 'not found pip'
                                virtualenv /home/jenkins/${env_name}
                                . /home/jenkins/${env_name}/bin/activate
                                mkdir -p ~/.pip
                                cp pip.conf ~/.pip/
                                pip install -e .[tf_gpu]
                                pip install nvgpu
                            fi
                        fi
                        """
                    }
    
                    // runs a shell script using jenkins groovy command 'sh'
                    sh """
                    nvidia-smi || true
                    aws iam list-access-keys  || echo 'aws iam failed'
                    pwd
                    whoami
                    which pip
                    echo \$VIRTUAL_ENV
                    echo -----------------------------------------------------------------------------------
                    ${env_install_sh}
                    echo -----------------------------------------------------------------------------------
                    echo \$VIRTUAL_ENV
                    echo -----------------------------------------------------------------------------------
                    echo setup complete
                    
                    if [ "${gpu}" = "auto" ]; then
                        export CUDA_VISIBLE_DEVICES=\$(nvgpu available -l 1)
                    else
                        export CUDA_VISIBLE_DEVICES=${gpu}
                    fi
                    
                    ${command_do}
                    """
                    /*
                    todo:
                    run tensorboard and get his pid and then terminate it
                    if [ "${run_tensorboard}" = "True" ]; then
                        tensorboard --logdir ${env.WORKSPACE}/Graph &
                    fi
                    */
                    
                    currentBuild.rawBuild.keepLog(true)
                    // if we run another sh, the virtualenv is forgotten!
                 }
            }
        }
    }
}