Provision for training
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
pipeline { | |
// use only nodes marked as 'tensorflow' | |
agent { node { label 'tensorflow' } } | |
// build parameters - these are prompted interactively | |
parameters { | |
string(defaultValue: '', description: 'Problem Name', name: 'problem') | |
} | |
environment { | |
// convenience: define params as env variables | |
PROBLEM_NAME = "${params.problem}" | |
BUCKET = "gs://training.innodatalabs.com/${params.problem}" | |
} | |
stages { | |
// make sure our private PyPI is accessible from this node | |
stage('Provision Private PyPI') { | |
steps { | |
withCredentials([file(credentialsId: "pip-conf-secret-file", variable: 'PIP_CONF')]) { | |
sh "mkdir -p ~/.config/pip; cp -f $PIP_CONF ~/.config/pip/pip.conf" | |
} | |
} | |
} | |
// apt install all required packages | |
// EC2 comes up with apt update processes already running. Therefore have to wait up to 10 minutes | |
// before our apt install can succeed | |
stage('Provision virtualenv') { | |
steps { | |
retry(20) { | |
sleep(30) | |
sh 'sudo apt-get install virtualenv -y' | |
} | |
} | |
} | |
// check out project and prepare Python3 virtual environment | |
stage('Prepare') { | |
steps { | |
git credentialsId: "mikes-github-username-password", url: 'https://github.com/mkroutikov/my-cool-private-repo.git', branch: 'master' | |
sh 'rm -rf .venv; virtualenv .venv -p python3' | |
sh ''' | |
. .venv/bin/activate | |
pip install -r requirements.txt | |
pip install tensorflow | |
pip install -e . | |
''' | |
} | |
} | |
// do the real thing. Since tensorflow trainer writes to Google Storage, need | |
// GOOGLE_APLICATION_CREDENTIALS. For completeness, add timeout | |
stage('Training') { | |
steps { | |
echo "Training problem $PROBLEM_NAME" | |
withCredentials([file(credentialsId: "gcloud-storage-secret", variable: 'GOOGLE_APPLICATION_CREDENTIALS')]) { | |
timeout(time: 5, unit: 'HOURS') { | |
sh """ | |
. .venv/bin/activate | |
my-cool-trainer --data_dir $BUCKET/data --problem $PROBLEM_NAME | |
""" | |
} | |
} | |
echo "All done" | |
} | |
} | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment