config.yml

version: 2.1

orbs:
  win: circleci/windows@2.2.0

# No windows executor is listed here since windows builds use win/default and modify
# the Python version through the conda environment.
executors:
  docker:
    parameters:
      python_version:
        type: string
    docker:
      - image: 350138855857.dkr.ecr.eu-west-2.amazonaws.com/kedro-builder:<<parameters.python_version>>
    resource_class: medium+

commands:
  setup_conda:
    steps:
      - run:
          name: Run conda.sh
          command: echo ". /home/circleci/miniconda/etc/profile.d/conda.sh" >> $BASH_ENV
      - run:
          name: Activate conda environment
          command: echo "conda deactivate; conda activate kedro_builder" >> $BASH_ENV
      - run:
          # pytables does not work properly with python 3.9 to handle our HDFDataSet
          # if pip-installed, so we install this dependency via conda
          name: Install conda packages
          command: echo "conda install -c conda-forge pytables -y" >> $BASH_ENV

  setup_requirements:
    steps:
      - run:
          name: Install pip setuptools
          command: make install-pip-setuptools
      - run:
          # Virtualenv 20.0.20 broke pre-commit, capped for now
          name: Install venv for some pre-commit hooks
          command: conda install -y "virtualenv<20.0"
      - run:
          name: Install requirements and test requirements
          command: pip install --upgrade -r test_requirements.txt
      - run:
          # this is needed to fix java cacerts so
          # spark can automatically download packages from mvn
          # https://stackoverflow.com/a/50103533/1684058
          name: Fix cacerts
          command: |
            sudo rm /etc/ssl/certs/java/cacerts
            sudo update-ca-certificates -f
      - run:
          # Since recently Spark installation for some reason does not have enough permissions to execute
          # /home/circleci/miniconda/envs/kedro_builder/lib/python3.X/site-packages/pyspark/bin/spark-class.
          # So fixing it manually here.
          name: Fix Spark permissions
          command: sudo chmod -R u+x /home/circleci/miniconda/envs/kedro_builder/lib/
      - run:
          name: Print Python environment
          command: make print-python-env
      - run:
          name: Pip freeze
          command: pip freeze

  setup:
    steps:
      - checkout
      - setup_conda
      - setup_requirements

  # Windows specific commands
  win_setup_conda:
    parameters:
      python_version:
        type: string
    steps:
      - run:
          name: Initialize conda
          command: conda init powershell
      - run:
          name: Create 'kedro_builder' conda environment
          command: conda create -n kedro_builder python=<<parameters.python_version>> -y

  win_setup_env:
    steps:
      - run:
          # Required for Tensorflow tests
          name: Install Microsoft Visual C++ Redistributable
          command: |
            $ProgressPreference = "SilentlyContinue"
            Invoke-WebRequest https://aka.ms/vs/16/release/vc_redist.x64.exe -OutFile vc_redist.x64.exe
            .\vc_redist.x64.exe /S /v/qn
      - run:
          name: Install Java 8
          command: |
            $ProgressPreference = "SilentlyContinue"
            Invoke-WebRequest https://github.com/AdoptOpenJDK/openjdk8-upstream-binaries/releases/download/jdk8u252-b09/OpenJDK8U-jdk_x64_windows_8u252b09.zip -OutFile OpenJDK8U.zip
            Expand-Archive .\OpenJDK8U.zip -DestinationPath C:\OpenJDK8U
      - run:
          name: Create Inbound rules for Java
          command: |
            New-NetFirewallRule -DisplayName "Allow JDK UDP" -Profile "Public" -Protocol "UDP" -Direction Inbound -Program "C:\OpenJDK8U\openjdk-8u252-b09\bin\java.exe" -Action Allow
            New-NetFirewallRule -DisplayName "Allow JDK TCP" -Profile "Public" -Protocol "TCP" -Direction Inbound -Program "C:\OpenJDK8U\openjdk-8u252-b09\bin\java.exe" -Action Allow
      - run:
          name: Set Java environment variables
          command: |
            [Environment]::SetEnvironmentVariable("Path", [Environment]::GetEnvironmentVariable('Path', 'Machine') + ";C:\OpenJDK8U\openjdk-8u252-b09\bin", "Machine")
            setx /m JAVA_HOME "C:\OpenJDK8U\openjdk-8u252-b09"
      - run:
          name: Setup Hadoop binary
          command: |
            $ProgressPreference = "SilentlyContinue"
            Invoke-WebRequest https://github.com/steveloughran/winutils/raw/master/hadoop-2.6.3/bin/winutils.exe -OutFile winutils.exe
            New-Item -ItemType directory -Path C:\hadoop\bin
            mv .\winutils.exe C:\hadoop\bin
            setx /m HADOOP_HOME "C:\hadoop\"
      - run:
          name: Install 'make' command
          command: choco install make

  win_setup_requirements:
    steps:
      # pytables and Fiona have a series of binary dependencies under Windows that
      # are best handled by conda-installing instead of pip-installing them.
      - run:
          name: Install pytables
          command: conda activate kedro_builder; conda install -c conda-forge pytables -y
      - run:
          name: Install GDAL
          command: conda activate kedro_builder; conda install -c conda-forge gdal -y
      - run:
          name: Install Fiona
          command: conda activate kedro_builder; conda install -c conda-forge fiona -y
      - run:
          name: Install all requirements
          command: conda activate kedro_builder; pip install -r test_requirements.txt -U
      - run:
          name: Print Python environment
          command: conda activate kedro_builder; make print-python-env
      - run:
          name: Pip freeze
          command: conda activate kedro_builder; pip freeze

  win_setup:
    parameters:
      python_version:
        type: string
    steps:
      - checkout
      - win_setup_conda:
          python_version: <<parameters.python_version>>
      - win_setup_env
      - restore_cache:
          key: kedro-deps-v1-win-{{ checksum "requirements.txt" }}-{{ checksum "test_requirements.txt" }}
      - win_setup_requirements

jobs:
  e2e_tests:
    parameters:
      python_version:
        type: string
    executor:
      name: docker
      python_version: <<parameters.python_version>>
    steps:
      - setup
      - run:
          name: Run e2e tests
          command: make e2e-tests

  win_e2e_tests:
    parameters:
      python_version:
        type: string
    executor: win/default
    steps:
      - checkout
      - win_setup_conda:
          python_version: <<parameters.python_version>>
      - run:
          name: Install 'make' command
          command: choco install make
      - run:
          name: Install dependencies
          command: conda activate kedro_builder; pip install -r features/windows_reqs.txt
      - run:
          name: Run e2e tests
          command: conda activate kedro_builder; make e2e-tests

  unit_tests:
    parameters:
      python_version:
        type: string
    executor:
      name: docker
      python_version: <<parameters.python_version>>
    steps:
      - setup
      - run:
          name: Run unit tests
          command: make test

  win_unit_tests:
    parameters:
      python_version:
        type: string
    executor: win/default
    steps:
      - win_setup:
          python_version: <<parameters.python_version>>
      - run:
          # geopandas and tensorflow conflicts when imported simultaneously.
          # The HDF5 header files used to compile this application do not match
          # the version used by the HDF5 library to which this application is linked.
          # Data corruption or segmentation faults may occur if the application continues.
          # This can happen when an application was compiled by one version of HDF5 but
          # linked with a different version of static or shared HDF5 library.
          # You should recompile the application or check your shared library related
          # settings such as 'LD_LIBRARY_PATH'.
          # You can, at your own risk, disable this warning by setting the environment
          # variable 'HDF5_DISABLE_VERSION_CHECK' to a value of '1'.
          # Setting it to 2 or higher will suppress the warning messages totally.
          name: Set HDF5_DISABLE_VERSION_CHECK environment variable
          command: setx /m HDF5_DISABLE_VERSION_CHECK 1
      - run:
          name: Run unit tests without spark
          command: conda activate kedro_builder; make test-no-spark

  lint:
    parameters:
      python_version:
        type: string
    executor:
      name: docker
      python_version: <<parameters.python_version>>
    steps:
      - setup
      - run:
          name: Install pre-commit hooks
          command: pre-commit install --install-hooks
      - run:
          name: Run linters
          command: make lint

  pip_compile:
    parameters:
      python_version:
        type: string
    executor:
      name: docker
      python_version: <<parameters.python_version>>
    steps:
      - setup
      - run:
          name: Pip-compile requirements file
          command: make pip-compile

  win_pip_compile:
    parameters:
      python_version:
        type: string
    executor: win/default
    steps:
      - win_setup:
          python_version: <<parameters.python_version>>
      - when:
          # Save cache only for Python 3.7. There is no need to save it for each Python.
          condition:
            equal: ["3.7", <<parameters.python_version>>]
          steps:
            - save_cache:
                key: kedro-deps-v1-win-{{ checksum "requirements.txt" }}-{{ checksum "test_requirements.txt" }}
                paths:
                  # Cache pip cache and conda packages directories
                  - c:\tools\miniconda3\pkgs
                  - c:\users\circleci\appdata\local\pip\cache
      - run:
          name: Pip-compile requirements file
          command: conda activate kedro_builder; make pip-compile

  build_docs:
    executor:
      name: docker
      python_version: "3.7"
    steps:
      - setup
      - run:
          name: Build docs
          command: make build-docs

  docs_linkcheck:
    executor:
      name: docker
      python_version: "3.7"
    steps:
      - setup
      - run:
          name: Check for broken links
          command: make linkcheck

  # Trigger kedro-viz build to ensure tests in that project pass
  viz_build:
    docker:
      - image: spotify/alpine # for bash and curl
    steps:
      - run:
          name: Trigger kedro-viz build
          command: |
            curl --location --request POST \
              --url https://circleci.com/api/v2/project/github/quantumblacklabs/kedro-viz/pipeline \
              --header "Circle-Token: $CIRCLE_TOKEN" \
              --header 'content-type: application/json' \
              --data '{"branch":"main"}'

  all_circleci_checks_succeeded:
    docker:
      - image: circleci/python # any light-weight image
    steps:
      - run:
          name: Success!
          command: echo "All checks passed"

workflows:
  version: 2.1
  regular:
    jobs:
      - e2e_tests:
          matrix:
            parameters:
              python_version: ["3.7", "3.8", "3.9"]
      - win_e2e_tests:
          matrix:
            parameters:
              python_version: ["3.7", "3.8", "3.9"]
      - unit_tests:
          matrix:
            parameters:
              python_version: ["3.7", "3.8", "3.9"]
      - win_unit_tests:
          matrix:
            parameters:
              python_version: ["3.7", "3.8", "3.9"]
      - lint:
          matrix:
            parameters:
              python_version: ["3.7", "3.8", "3.9"]
      - pip_compile:
          matrix:
            parameters:
              python_version: ["3.7", "3.8", "3.9"]
      - win_pip_compile:
          matrix:
            parameters:
              python_version: ["3.7", "3.8", "3.9"]
      - build_docs
      - docs_linkcheck
      - viz_build:
          filters:
            branches:
              only: main

      - all_circleci_checks_succeeded:
          requires:
            - e2e_tests
            - win_e2e_tests
            - unit_tests
            - win_unit_tests
            - lint
            - pip_compile
            - win_pip_compile
            - build_docs
            - docs_linkcheck