Instantly share code, notes, and snippets.

Embed
What would you like to do?
Provision for training
pipeline {
// use only nodes marked as 'tensorflow'
agent { node { label 'tensorflow' } }
// build parameters - these are prompted interactively
parameters {
string(defaultValue: '', description: 'Problem Name', name: 'problem')
}
environment {
// convenience: define params as env variables
PROBLEM_NAME = "${params.problem}"
BUCKET = "gs://training.innodatalabs.com/${params.problem}"
}
stages {
// make sure our private PyPI is accessible from this node
stage('Provision Private PyPI') {
steps {
withCredentials([file(credentialsId: "pip-conf-secret-file", variable: 'PIP_CONF')]) {
sh "mkdir -p ~/.config/pip; cp -f $PIP_CONF ~/.config/pip/pip.conf"
}
}
}
// apt install all required packages
// EC2 comes up with apt update processes already running. Therefore have to wait up to 10 minutes
// before our apt install can succeed
stage('Provision virtualenv') {
steps {
retry(20) {
sleep(30)
sh 'sudo apt-get install virtualenv -y'
}
}
}
// check out project and prepare Python3 virtual environment
stage('Prepare') {
steps {
git credentialsId: "mikes-github-username-password", url: 'https://github.com/mkroutikov/my-cool-private-repo.git', branch: 'master'
sh 'rm -rf .venv; virtualenv .venv -p python3'
sh '''
. .venv/bin/activate
pip install -r requirements.txt
pip install tensorflow
pip install -e .
'''
}
}
// do the real thing. Since tensorflow trainer writes to Google Storage, need
// GOOGLE_APLICATION_CREDENTIALS. For completeness, add timeout
stage('Training') {
steps {
echo "Training problem $PROBLEM_NAME"
withCredentials([file(credentialsId: "gcloud-storage-secret", variable: 'GOOGLE_APPLICATION_CREDENTIALS')]) {
timeout(time: 5, unit: 'HOURS') {
sh """
. .venv/bin/activate
my-cool-trainer --data_dir $BUCKET/data --problem $PROBLEM_NAME
"""
}
}
echo "All done"
}
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment