Last active
April 22, 2021 16:32
-
-
Save priyanlc/49f508ddf5890a63ab3be70d36cf819a to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
stages: | |
- test | |
- build | |
- deploy | |
test: | |
stage: test | |
image: python:3.7-stretch | |
script: | |
- echo "Testing" | |
- apt-get update && | |
apt-get install default-jdk -y | |
- pip3 install --upgrade pip && | |
pip3 install pip-tools && | |
make install-test | |
- make run-tests | |
build:stage_one: | |
stage: build | |
image: python:3.7-stretch | |
script: | |
- echo "Building" | |
- make src_package | |
artifacts: | |
paths: | |
- build/ | |
- dist/ | |
- poc_pyspark_docker.egg-info/ | |
image: | |
name: gcr.io/kaniko-project/executor:debug-0fe0a64113fb9755f6896a805f67dbfdb02acf0d | |
entrypoint: [""] | |
variables: | |
IMAGE_LABELS: > | |
--label vcs-url=$CI_PROJECT_URL | |
--label com.gitlab.ci.user=$GITLAB_USER_NAME | |
--label com.gitlab.ci.email=$GITLAB_USER_EMAIL | |
--label com.gitlab.ci.tagorbranch=$CI_COMMIT_REF_NAME | |
--label com.gitlab.ci.pipelineurl=$CI_PIPELINE_URL | |
--label com.gitlab.ci.commiturl=$CI_PROJECT_URL/commit/$CI_COMMIT_SHA | |
--label com.gitlab.ci.cijoburl=$CI_JOB_URL | |
--label com.gitlab.ci.mrurl=$CI_PROJECT_URL/-/merge_requests/$CI_MERGE_REQUEST_ID | |
build-for-docker-hub-registry: | |
stage: build | |
# This causes CI CD Variables to be taken from GitLab Project definition for variable scope "push-to-docker-hub", | |
# which overrides CI_REGISTRY, CI_REGISTRY_IMAGE, CI_REGISTRY_USER and CI_REGISTRY_PASSWORD | |
# these variables with this scope would need to be created if you copy this project to another group or instance | |
environment: | |
name: push-to-docker-hub | |
script: | |
- | | |
echo "Building and shipping image to $CI_REGISTRY_IMAGE" | |
if [[ "$CI_COMMIT_REF_NAME" == "master" ]]; then ADDLATESTTAG="--destination $CI_REGISTRY_IMAGE:latest"; fi | |
if [[ -n "$ADDITIONALTAGLIST" ]]; then | |
for TAG in $ADDITIONALTAGLIST; do | |
FORMATTEDTAGLIST=" ${FORMATTEDTAGLIST} --destination $CI_REGISTRY_IMAGE:$TAG" | |
done | |
fi | |
mkdir -p /kaniko/.docker | |
echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$(echo -n $CI_REGISTRY_USER:$CI_REGISTRY_PASSWORD | base64)\"}}}" > /kaniko/.docker/config.json | |
/kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --destination $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME --destination $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA $ADDLATESTTAG $FORMATTEDTAGLIST --label build-date=$(date -Iseconds) $IMAGE_LABELS | |
production: | |
stage: deploy | |
image: python:3.7-stretch | |
script: | |
- apt-get update && | |
apt-get install curl | |
- | | |
curl -X POST -H "Authorization: Bearer $DATABRICKS_PASSWORD" $DATABRICKS_CLUSTER/api/2.0/clusters/create -d '{ "cluster_name": "IL-PySpark-POC", "num_workers": 1,"node_type_id": "m4.large","docker_image": {"url": "plc9/spark-poc-il:latest","basic_auth": {"username": "plc9","password": "!@£"}},"spark_version":"6.4.x-scala2.11", "spark_python_task": { "python_file": "file:/src_package/poc/ny_taxi/new_york_taxi_feature_eng.py" }}' |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment