gitlab-ci.yml

stages:
  - test
  - build
  - deploy

test:
    stage: test
    image: python:3.7-stretch
    script:
      - echo "Testing"
      - apt-get update &&
        apt-get install default-jdk -y
      - pip3 install --upgrade pip &&
        pip3 install pip-tools &&
        make install-test
      - make run-tests


build:stage_one:
  stage: build
  image: python:3.7-stretch
  script:
    - echo "Building"
    - make src_package
  artifacts:
      paths:
        - build/
        - dist/
        - poc_pyspark_docker.egg-info/

image:
  name: gcr.io/kaniko-project/executor:debug-0fe0a64113fb9755f6896a805f67dbfdb02acf0d
  entrypoint: [""]

variables:
  IMAGE_LABELS: >
    --label vcs-url=$CI_PROJECT_URL
    --label com.gitlab.ci.user=$GITLAB_USER_NAME
    --label com.gitlab.ci.email=$GITLAB_USER_EMAIL
    --label com.gitlab.ci.tagorbranch=$CI_COMMIT_REF_NAME
    --label com.gitlab.ci.pipelineurl=$CI_PIPELINE_URL
    --label com.gitlab.ci.commiturl=$CI_PROJECT_URL/commit/$CI_COMMIT_SHA
    --label com.gitlab.ci.cijoburl=$CI_JOB_URL
    --label com.gitlab.ci.mrurl=$CI_PROJECT_URL/-/merge_requests/$CI_MERGE_REQUEST_ID

build-for-docker-hub-registry:
  stage: build
  # This causes CI CD Variables to be taken from GitLab Project definition for variable scope "push-to-docker-hub",
  # which overrides CI_REGISTRY, CI_REGISTRY_IMAGE, CI_REGISTRY_USER and CI_REGISTRY_PASSWORD
  # these variables with this scope would need to be created if you copy this project to another group or instance
  environment:
    name: push-to-docker-hub
  script:
    - |
      echo "Building and shipping image to $CI_REGISTRY_IMAGE"
      if [[ "$CI_COMMIT_REF_NAME" == "master" ]]; then ADDLATESTTAG="--destination $CI_REGISTRY_IMAGE:latest"; fi
      if [[ -n "$ADDITIONALTAGLIST" ]]; then
        for TAG in $ADDITIONALTAGLIST; do
          FORMATTEDTAGLIST=" ${FORMATTEDTAGLIST} --destination $CI_REGISTRY_IMAGE:$TAG"
        done
      fi
      mkdir -p /kaniko/.docker
      echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$(echo -n $CI_REGISTRY_USER:$CI_REGISTRY_PASSWORD | base64)\"}}}" > /kaniko/.docker/config.json
      /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --destination $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME --destination $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA $ADDLATESTTAG $FORMATTEDTAGLIST --label build-date=$(date -Iseconds) $IMAGE_LABELS

production:
  stage: deploy
  image: python:3.7-stretch
  script:
    - apt-get update &&
        apt-get install curl
    - |
      curl -X POST -H "Authorization: Bearer $DATABRICKS_PASSWORD" $DATABRICKS_CLUSTER/api/2.0/clusters/create -d '{ "cluster_name": "IL-PySpark-POC", "num_workers": 1,"node_type_id": "m4.large","docker_image": {"url": "plc9/spark-poc-il:latest","basic_auth": {"username": "plc9","password": "!@£"}},"spark_version":"6.4.x-scala2.11", "spark_python_task": { "python_file": "file:/src_package/poc/ny_taxi/new_york_taxi_feature_eng.py" }}'