config.yml

version: 2.1

orbs:
  win: circleci/windows@2.2.0

executors:
  py37:
    docker:
      - image: 350138855857.dkr.ecr.eu-west-2.amazonaws.com/kedro-builder:3.7
    resource_class: medium+
  py38:
    docker:
      - image: 350138855857.dkr.ecr.eu-west-2.amazonaws.com/kedro-builder:3.8
    resource_class: medium+
  py39:
    docker:
      - image: 350138855857.dkr.ecr.eu-west-2.amazonaws.com/kedro-builder:3.9
    resource_class: medium+

commands:
  setup_conda:
    description: Activate conda environment
    steps:
      - run:
          name: Run conda.sh
          command: echo ". /home/circleci/miniconda/etc/profile.d/conda.sh" >> $BASH_ENV
      - run:
          name: Activate conda environment
          command: echo "conda deactivate; conda activate kedro_builder" >> $BASH_ENV
      - run:
          # pytables does not work properly with python 3.9 to handle our HDFDataSet
          # if pip-installed, so we install this dependency via conda
          name: Install conda packages
          command: echo "conda install -c conda-forge pytables -y" >> $BASH_ENV

  setup_requirements:
    description: Install PIP dependencies
    steps:
      - run:
          name: Install pip setuptools
          command: make install-pip-setuptools
      - run:
          # Virtualenv 20.0.20 broke pre-commit, capped for now
          name: Install venv for some pre-commit hooks
          command: conda install -y "virtualenv<20.0"
      - run:
          name: Install requirements and test requirements
          command: pip install --upgrade -r test_requirements.txt
      - run:
          # this is needed to fix java cacerts so
          # spark can automatically download packages from mvn
          # https://stackoverflow.com/a/50103533/1684058
          name: Fix cacerts
          command: |
            sudo rm /etc/ssl/certs/java/cacerts
            sudo update-ca-certificates -f
      - run:
          # Since recently Spark installation for some reason does not have enough permissions to execute
          # /home/circleci/miniconda/envs/kedro_builder/lib/python3.X/site-packages/pyspark/bin/spark-class.
          # So fixing it manually here.
          name: Fix Spark permissions
          command: sudo chmod -R u+x /home/circleci/miniconda/envs/kedro_builder/lib/
      - run:
          name: Print Python environment
          command: make print-python-env
      - run:
          name: Pip freeze
          command: pip freeze

  setup_pre_commit:
    description: Install pre-commit hooks
    steps:
      - run:
          name: Install pre-commit hooks
          command: pre-commit install --install-hooks
      - run:
          name: Run pre-commit hooks
          command: pre-commit install --hook-type pre-push

  unit_tests:
    description: Run unit tests
    steps:
      - checkout
      - setup_conda
      - setup_requirements
      - run:
          name: Run unit tests
          command: make test

  lint:
    description: Run linters
    steps:
      - checkout
      - setup_conda
      - setup_requirements
      - setup_pre_commit
      - run:
          name: Run linters
          command: make lint

  e2e_tests:
    description: Run all end to end tests
    steps:
      - checkout
      - setup_conda
      - setup_requirements
      - run:
          name: Run e2e tests
          command: make e2e-tests

  build_docs:
    description: Build docs
    steps:
      - checkout
      - setup_conda
      - setup_requirements
      - run:
          name: Build docs
          command: make build-docs

  docs_linkcheck:
    description: Build docs and check for broken links
    steps:
      - checkout
      - setup_conda
      - setup_requirements
      - run:
          name: Check for broken links
          command: make linkcheck

  pip_compile:
    description: Pip-compile requirements file
    steps:
      - checkout
      - setup_conda
      - setup_requirements
      - run:
          name: Pip-compile requirements file
          command: make pip-compile

  run_viz_build:
    description: Deploy kedro-viz to ensure tests in that project pass
    steps:
      - run:
          name: Trigger a kedro-viz build
          command: |
            curl --location --request POST \
              --url https://circleci.com/api/v2/project/github/quantumblacklabs/kedro-viz/pipeline \
              --header "Circle-Token: $CIRCLE_TOKEN" \
              --header 'content-type: application/json' \
              --data '{"branch":"main"}'

  # Windows-related commands
  win_setup_conda:
    description: Setup conda
    parameters:
      python_version:
        type: string
    steps:
      - run:
          name: Initialize conda
          command: conda init powershell
      - run:
          name: Create 'kedro_builder' conda environment
          command: |
            conda create --name kedro_builder python=<< parameters.python_version >> -y

  win_setup_env:
    description: Setup environment
    steps:
      - run:
          # Required for Tensorflow tests
          name: Install Microsoft Visual C++ Redistributable
          command: |
            $ProgressPreference = "SilentlyContinue"
            Invoke-WebRequest https://aka.ms/vs/16/release/vc_redist.x64.exe -OutFile vc_redist.x64.exe
            .\vc_redist.x64.exe /S /v/qn
      - run:
          name: Install Java 8
          command: |
            $ProgressPreference = "SilentlyContinue"
            Invoke-WebRequest https://github.com/AdoptOpenJDK/openjdk8-upstream-binaries/releases/download/jdk8u252-b09/OpenJDK8U-jdk_x64_windows_8u252b09.zip -OutFile OpenJDK8U.zip
            Expand-Archive .\OpenJDK8U.zip -DestinationPath C:\OpenJDK8U
      - run:
          name: Create Inbound rules for Java
          command: |
            New-NetFirewallRule -DisplayName "Allow JDK UDP" -Profile "Public" -Protocol "UDP" -Direction Inbound -Program "C:\OpenJDK8U\openjdk-8u252-b09\bin\java.exe" -Action Allow
            New-NetFirewallRule -DisplayName "Allow JDK TCP" -Profile "Public" -Protocol "TCP" -Direction Inbound -Program "C:\OpenJDK8U\openjdk-8u252-b09\bin\java.exe" -Action Allow
      - run:
          name: Set Java environment variables
          command: |
            [Environment]::SetEnvironmentVariable("Path", [Environment]::GetEnvironmentVariable('Path', 'Machine') + ";C:\OpenJDK8U\openjdk-8u252-b09\bin", "Machine")
            setx /m JAVA_HOME "C:\OpenJDK8U\openjdk-8u252-b09"
      - run:
          name: Setup Hadoop binary
          command: |
            $ProgressPreference = "SilentlyContinue"
            Invoke-WebRequest https://github.com/steveloughran/winutils/raw/master/hadoop-2.6.3/bin/winutils.exe -OutFile winutils.exe
            New-Item -ItemType directory -Path C:\hadoop\bin
            mv .\winutils.exe C:\hadoop\bin
            setx /m HADOOP_HOME "C:\hadoop\"
      - run:
          name: Install 'make' command
          command: choco install make

  win_setup_requirements:
    description: Install Kedro dependencies
    steps:
      # pytables and Fiona have a series of binary dependencies under Windows that
      # are best handled by conda-installing instead of pip-installing them.
      - run:
          name: Install pytables
          command: conda activate kedro_builder; conda install -c conda-forge pytables -y
      - run:
          name: Install GDAL
          command: conda activate kedro_builder; conda install -c conda-forge gdal -y
      - run:
          name: Install Fiona
          command: conda activate kedro_builder; conda install -c conda-forge fiona -y
      - run:
          name: Install all requirements
          command: conda activate kedro_builder; pip install -r test_requirements.txt -U
      - run:
          name: Print Python environment
          command: conda activate kedro_builder; make print-python-env
      - run:
          name: Pip freeze
          command: conda activate kedro_builder; pip freeze

  win_unit_tests:
    description: Run unit tests
    steps:
      - checkout
      - win_setup_env
      - restore_cache:
          key: kedro-deps-v1-win-{{ checksum "requirements.txt" }}-{{ checksum "test_requirements.txt" }}
      - win_setup_requirements
      - run:
          # geopandas and tensorflow conflicts when imported simultaneously.
          # The HDF5 header files used to compile this application do not match
          # the version used by the HDF5 library to which this application is linked.
          # Data corruption or segmentation faults may occur if the application continues.
          # This can happen when an application was compiled by one version of HDF5 but
          # linked with a different version of static or shared HDF5 library.
          # You should recompile the application or check your shared library related
          # settings such as 'LD_LIBRARY_PATH'.
          # You can, at your own risk, disable this warning by setting the environment
          # variable 'HDF5_DISABLE_VERSION_CHECK' to a value of '1'.
          # Setting it to 2 or higher will suppress the warning messages totally.
          name: Set HDF5_DISABLE_VERSION_CHECK environment variable
          command: setx /m HDF5_DISABLE_VERSION_CHECK 1
      - run:
          name: Run unit tests
          command: |
            conda activate kedro_builder
            pytest .\tests --ignore .\tests\extras\datasets\spark --ignore .\tests\extras\datasets\tensorflow --ignore tests\framework\session\test_session_hooks.py --no-cov

  win_e2e_tests:
    description: Run all end to end tests
    steps:
      - checkout
      - run:
          name: Install dependencies
          command: |
            conda activate kedro_builder
            pip install -r features/windows_reqs.txt
            choco install make
      - run:
          name: Run e2e tests
          command: conda activate kedro_builder; make e2e-tests

  win_pip_compile:
    description: Pip-compile requirements file
    parameters:
      cache_save:
        type: boolean
        default: false
    steps:
      - checkout
      - win_setup_env
      - restore_cache:
          key: kedro-deps-v1-win-{{ checksum "requirements.txt" }}-{{ checksum "test_requirements.txt" }}
      - win_setup_requirements
      - when:
          # Cache when `parameters.cache_save` is True
          condition: << parameters.cache_save >>
          steps:
            - save_cache:
                key: kedro-deps-v1-win-{{ checksum "requirements.txt" }}-{{ checksum "test_requirements.txt" }}
                paths:
                  # Cache pip cache and conda packages directories
                  - c:\tools\miniconda3\pkgs
                  - c:\users\circleci\appdata\local\pip\cache
      - run:
          name: Pip-compile requirements file
          command: conda activate kedro_builder; make pip-compile

jobs:
  unit_tests_37:
    executor: py37
    steps: [unit_tests]

  linters_37:
    executor: py37
    steps: [lint]

  e2e_tests_37:
    executor: py37
    steps: [e2e_tests]

  docs_37:
    executor: py37
    steps: [build_docs]

  docs_linkcheck_37:
    executor: py37
    steps: [docs_linkcheck]

  unit_tests_38:
    executor: py38
    steps: [unit_tests]

  linters_38:
    executor: py38
    steps: [lint]

  e2e_tests_38:
    executor: py38
    steps: [e2e_tests]

  unit_tests_39:
    executor: py39
    steps: [unit_tests]

  linters_39:
    executor: py39
    steps: [lint]

  e2e_tests_39:
    executor: py39
    steps: [e2e_tests]

  pip_compile_37:
    executor: py37
    steps: [pip_compile]

  pip_compile_38:
    executor: py38
    steps: [pip_compile]

  pip_compile_39:
    executor: py39
    steps: [pip_compile]

  run_kedro_viz:
    docker:
      - image: spotify/alpine # for bash and curl
    steps: [run_viz_build]

  all_circleci_checks_succeeded:
    docker:
      - image: circleci/python # any light-weight image

    steps:
      - run:
          name: Success!
          command: echo "All checks passed"

  # Windows-related jobs
  win_unit_tests_37:
    executor:
      name: win/default
    steps:
      - win_setup_conda:
          python_version: "3.7"
      - win_unit_tests

  win_unit_tests_38:
    executor:
      name: win/default
    steps:
      - win_setup_conda:
          python_version: "3.8"
      - checkout
      - win_setup_env
      - restore_cache:
          key: kedro-deps-v1-win-{{ checksum "requirements.txt" }}-{{ checksum "test_requirements.txt" }}
      - win_setup_requirements
      - run:
          name: Set HDF5_DISABLE_VERSION_CHECK environment variable
          command: setx /m HDF5_DISABLE_VERSION_CHECK 1
      - run:
          name: Run unit tests without Spark and TensorFlow
          # Run `test_parallel_runner.py` separately because of `Windows fatal exception: stack overflow`
          command: |
            conda activate kedro_builder
            pytest .\tests --ignore .\tests\extras\datasets\spark --ignore .\tests\extras\datasets\tensorflow --ignore tests\framework\session\test_session_hooks.py --ignore .\tests\runner\test_parallel_runner.py --no-cov
            if ($?) { pytest .\tests\runner\test_parallel_runner.py --no-cov }

  win_unit_tests_39:
    executor:
      name: win/default
    steps:
      - win_setup_conda:
          python_version: "3.9"
      - checkout
      - win_setup_env
      - restore_cache:
          key: kedro-deps-v1-win-{{ checksum "requirements.txt" }}-{{ checksum "test_requirements.txt" }}
      - win_setup_requirements
      - run:
          name: Set HDF5_DISABLE_VERSION_CHECK environment variable
          command: setx /m HDF5_DISABLE_VERSION_CHECK 1
      - run:
          name: Run unit tests without Spark and TensorFlow
          # Run `test_parallel_runner.py` separately because of `Windows fatal exception: stack overflow`
          command: |
            conda activate kedro_builder
            pytest .\tests --ignore .\tests\extras\datasets\spark --ignore .\tests\extras\datasets\tensorflow --ignore tests\framework\session\test_session_hooks.py --ignore .\tests\runner\test_parallel_runner.py --no-cov
            if ($?) { pytest .\tests\runner\test_parallel_runner.py --no-cov }

  win_e2e_tests_37:
    executor:
      name: win/default
    steps:
      - win_setup_conda:
          python_version: "3.7"
      - win_e2e_tests

  win_e2e_tests_38:
    executor:
      name: win/default
    steps:
      - win_setup_conda:
          python_version: "3.8"
      - win_e2e_tests

  win_e2e_tests_39:
    executor:
      name: win/default
    steps:
      - win_setup_conda:
          python_version: "3.9"
      - win_e2e_tests

  win_pip_compile_37:
    executor:
      name: win/default
    steps:
      - win_setup_conda:
          python_version: "3.7"
      - win_pip_compile:
          # Save cache only for Python 3.7. There is no need to save it for each Python.
          cache_save: true

  win_pip_compile_38:
    executor:
      name: win/default
    steps:
      - win_setup_conda:
          python_version: "3.8"
      - win_pip_compile

  win_pip_compile_39:
    executor:
      name: win/default
    steps:
      - win_setup_conda:
          python_version: "3.9"
      - win_pip_compile

workflows:
  version: 2
  regular:
    jobs:
      - docs_linkcheck_37
      - unit_tests_37
      - linters_37
      - e2e_tests_37
      - docs_37
      - unit_tests_38
      - linters_38
      - e2e_tests_38
      - unit_tests_39
      - linters_39
      - e2e_tests_39
      - pip_compile_37
      - pip_compile_38
      - pip_compile_39
      - win_unit_tests_37
      - win_unit_tests_38
      - win_unit_tests_39
      - win_pip_compile_37
      - win_pip_compile_38
      - win_pip_compile_39
      - win_e2e_tests_37
      - win_e2e_tests_38
      - win_e2e_tests_39
      - run_kedro_viz:
          filters:
            branches:
              only:
                - master
      - all_circleci_checks_succeeded:
          requires:
            - unit_tests_37
            - linters_37
            - e2e_tests_37
            - docs_37
            - docs_linkcheck_37
            - unit_tests_38
            - linters_38
            - e2e_tests_38
            - unit_tests_39
            - linters_39
            - e2e_tests_39
            - pip_compile_37
            - pip_compile_38
            - pip_compile_39
            - win_pip_compile_37
            - win_pip_compile_38
            - win_pip_compile_39
            - win_unit_tests_37
            # Skipped due to Windows fatal exception: stack overflow
            # - win_unit_tests_38
            # - win_unit_tests_39
            - win_e2e_tests_37
            - win_e2e_tests_38
            - win_e2e_tests_39