Provision for training

Jenkinsfile.groovy

pipeline {
  
  // use only nodes marked as 'tensorflow'
  agent { node { label 'tensorflow' } }
  
  // build parameters - these are prompted interactively
  parameters {
    string(defaultValue: '', description: 'Problem Name', name: 'problem')
  }
  
  environment {
    // convenience: define params as env variables
    PROBLEM_NAME = "${params.problem}"
    BUCKET       = "gs://training.innodatalabs.com/${params.problem}"
  }
  
  stages {
      
    // make sure our private PyPI is accessible from this node
    stage('Provision Private PyPI') {
      steps {
        withCredentials([file(credentialsId: "pip-conf-secret-file", variable: 'PIP_CONF')]) {
          sh "mkdir -p ~/.config/pip; cp -f $PIP_CONF ~/.config/pip/pip.conf"
        }
      }
    }
        
    // apt install all required packages
    // EC2 comes up with apt update processes already running. Therefore have to wait up to 10 minutes
    // before our apt install can succeed
    stage('Provision virtualenv') {
      steps {
        retry(20) {
          sleep(30)
          sh 'sudo apt-get install virtualenv -y'
        }
      }
    }

    // check out project and prepare Python3 virtual environment
    stage('Prepare') {
      steps {
        git credentialsId: "mikes-github-username-password", url: 'https://github.com/mkroutikov/my-cool-private-repo.git', branch: 'master'
        sh 'rm -rf .venv; virtualenv .venv -p python3'
        sh '''
        . .venv/bin/activate
        pip install -r requirements.txt
        pip install tensorflow
        pip install -e .
        '''
      }
    }
        
    // do the real thing. Since tensorflow trainer writes to Google Storage, need
    // GOOGLE_APLICATION_CREDENTIALS. For completeness, add timeout
    stage('Training') {
      steps {
        echo "Training problem $PROBLEM_NAME"
        withCredentials([file(credentialsId: "gcloud-storage-secret", variable: 'GOOGLE_APPLICATION_CREDENTIALS')]) {
          timeout(time: 5, unit: 'HOURS') {
            sh """
            . .venv/bin/activate
            my-cool-trainer --data_dir $BUCKET/data --problem $PROBLEM_NAME
            """
          }
        }
        echo "All done"
      }
    }
  }
}