Provision for training
pipeline { | |
// use only nodes marked as 'tensorflow' | |
agent { node { label 'tensorflow' } } | |
// build parameters - these are prompted interactively | |
parameters { | |
string(defaultValue: '', description: 'Problem Name', name: 'problem') | |
} | |
environment { | |
// convenience: define params as env variables | |
PROBLEM_NAME = "${params.problem}" | |
BUCKET = "gs://training.innodatalabs.com/${params.problem}" | |
} | |
stages { | |
// make sure our private PyPI is accessible from this node | |
stage('Provision Private PyPI') { | |
steps { | |
withCredentials([file(credentialsId: "pip-conf-secret-file", variable: 'PIP_CONF')]) { | |
sh "mkdir -p ~/.config/pip; cp -f $PIP_CONF ~/.config/pip/pip.conf" | |
} | |
} | |
} | |
// apt install all required packages | |
// EC2 comes up with apt update processes already running. Therefore have to wait up to 10 minutes | |
// before our apt install can succeed | |
stage('Provision virtualenv') { | |
steps { | |
retry(20) { | |
sleep(30) | |
sh 'sudo apt-get install virtualenv -y' | |
} | |
} | |
} | |
// check out project and prepare Python3 virtual environment | |
stage('Prepare') { | |
steps { | |
git credentialsId: "mikes-github-username-password", url: 'https://github.com/mkroutikov/my-cool-private-repo.git', branch: 'master' | |
sh 'rm -rf .venv; virtualenv .venv -p python3' | |
sh ''' | |
. .venv/bin/activate | |
pip install -r requirements.txt | |
pip install tensorflow | |
pip install -e . | |
''' | |
} | |
} | |
// do the real thing. Since tensorflow trainer writes to Google Storage, need | |
// GOOGLE_APLICATION_CREDENTIALS. For completeness, add timeout | |
stage('Training') { | |
steps { | |
echo "Training problem $PROBLEM_NAME" | |
withCredentials([file(credentialsId: "gcloud-storage-secret", variable: 'GOOGLE_APPLICATION_CREDENTIALS')]) { | |
timeout(time: 5, unit: 'HOURS') { | |
sh """ | |
. .venv/bin/activate | |
my-cool-trainer --data_dir $BUCKET/data --problem $PROBLEM_NAME | |
""" | |
} | |
} | |
echo "All done" | |
} | |
} | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment