Example ansible for creating shippable PySpark environments

conda_environment.yml

name: {{ item.env_name }}

dependencies:
  - pip:
    - --trusted-host {{ item.trusted_host }}
    - --extra-index-url {{ item.pypi_url }}

    - my_internal_package
    - another_internal_pkg

conda_pyspark_env.yml

---
- vars:
    WORKSPACE: "/tmp"
    conda_path: "{{ WORKSPACE }}/miniconda2/bin/conda" 
    dest: "/some/dir"
    env_name: "conda_spark_env"
    py_version: "2"
    py_packages:
        - requests
        - six
    base_packages:
        - conda-build
        - anaconda-client
    include_packages:
        - pyspark==2.1.1
        - boto3==1.4.4
        - pandas
  tasks:

    - name: Clean - miniconda
      file:
        state: absent
        path: "{{ WORKSPACE }}/miniconda2"

    - name: Get miniconda
      get_url:
          url: https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh
          dest: "{{ WORKSPACE }}/miniconda.sh"
          owner: "{{ ansible_user_id }}"
          group: "{{ ansible_user_id }}"
          mode: 0754

    - name: Install miniconda
      shell: >
          {{ WORKSPACE }}/miniconda.sh -f -b -p {{ WORKSPACE }}/miniconda2
      args:
        creates: "{{ WORKSPACE }}/miniconda2"
        
    - name: Delete miniconda installer
      file:
        path: "{{ WORKSPACE }}/minconda.sh"
        state: absent

    - name: Point miniconda to binstar repo for public packages
      copy:
        src: condarc.yaml
        dest: "~/.condarc"
        owner: "{{ ansible_user_id }}"
        group: "{{ ansible_user_id }}"
        mode: 0644

    - name: Install miniconda base_packages
      command: "{{ conda_path }} install -f -y {{ base_packages |join(' ') }}"

    - name: Update miniconda
      command: "{{ conda_path }} update -y --all"

    - name: Install miniconda include_packages
      command: "{{ conda_path }} install -f -y {{ include_packages |join(' ') }}"
      
    - name: Create py_packages dirs locally
      shell: >
          mkdir "{{ WORKSPACE }}/{{ item }}"
      args:
        creates: "{{ WORKSPACE }}/{{ item }}"
      with_items: "{{ py_packages }}"

    - name: Create meta.yaml
      copy:
        src: "{{ item }}-meta.yaml"
        dest: "{{ WORKSPACE }}/{{ item }}/meta.yaml"
        owner: "{{ ansible_user_id }}"
        group: "{{ ansible_user_id }}"
        mode: 0644
      with_items: "{{ py_packages }}"

    - name: Create build.sh
      copy:
        src: "py-build.sh"
        dest: "{{ WORKSPACE }}/{{ item }}/build.sh"
        owner: "{{ ansible_user_id }}"
        group: "{{ ansible_user_id }}"
        mode: 0644
      with_items: "{{ py_packages }}"

    - name: Build py_packages
      command: "{{ conda_path }} build {{ WORKSPACE }}/{{ item }}"
      with_items: "{{ py_packages }}"
      
    - name: Purge builds
      command: "{{ conda_path }} build purge"

    - name: Create conda pyspark env
      command: "{{ conda_path }} create -p {{ env_name }} --use-local --copy -y python={{ py_version }} {{ include_packages |join(' ') }} {{ py_packages |join(' ') }}"
      args:
        chdir: "{{ WORKSPACE }}"
        creates: "{{ WORKSPACE }}/{{env_name}}"

    - shell: source ~/.bash_profile && echo "${PYPI_URL}"
      register: PYPI_URL

    - shell: source ~/.bash_profile && echo "${PYPI_HOST}"
      register: PYPI_HOST

    - name: create environment.yml
      template:
        src: "{{ item.template_name }}.j2"
        dest: "/data/environments/{{ item.template_name }}"
      with_items:
        - { template_name: 'conda_environment.yml',
            env_name: '{{ env_name }}',
            pypi_url: "{{ PYPI_URL.stdout }}",
            trusted_host: "{{ PYPI_HOST.stdout }}" }
      
    - name: Update with environment.yml
      command: "{{ conda_path }} env update -p {{ env_name }} --file {{ dest }}/conda_environment.yml"
      args:
        chdir: "{{ WORKSPACE }}"

    - name: Zip conda env
      command: zip -r "{{ env_name }}.zip" "{{ env_name }}"
      args:
        chdir: "{{ WORKSPACE }}"
        creates: "{{ WORKSPACE }}/{{ env_name }}.zip"

    - name: Move conda env
      command: mv "{{ env_name }}.zip" "{{ dest }}"
      args:
          chdir: "{{ WORKSPACE }}"

    - name: Cleanup - unzipped env
      file:
        state: absent
        path: "{{ WORKSPACE }}/{{ env_name }}/"

    - name: Cleanup - py_packages
      file:
        state: absent
        path: "{{ WORKSPACE }}/{{ item }}"
      with_items: "{{ py_packages }}"

    - name: Cleanup - miniconda
      file:
        state: absent
        path: "{{ WORKSPACE }}/miniconda2"

pip_pyspark_env.yml

---
- vars:
    WORKSPACE: "/tmp"
    dest: "/some/dir"
    env_name: "spark_env"
    py_version: "python2.7"
    py_packages:
        - requests
        - six
  tasks:
    - name: Make sure env doesn't exist
      file:
        state: absent
        path: "{{ WORKSPACE }}/{{ env_name }}/"

    - name: Create base env
      command: "virtualenv {{ env_name }} --python={{ py_version }}"
      args:
        chdir: "{{ WORKSPACE }}"
        creates: "{{ WORKSPACE }}/{{ env_name }}"

    - name: Install ds packages
      shell: "{{ WORKSPACE }}/{{ env_name }}/bin/pip install {{ item }}"
      args:
        chdir: "{{ WORKSPACE }}"
      with_items: "{{ py_packages }}"

    - name: Make base env relocatable
      shell: "virtualenv --relocatable {{ env_name }}"
      args:
        chdir: "{{ WORKSPACE }}"

    - name: Zip env
      command: zip -r "{{ env_name }}.zip" "{{ env_name }}"
      args:
        chdir: "{{ WORKSPACE }}"
        creates: "{{ WORKSPACE }}/{{ env_name }}.zip"

    - name: Move env
      command: mv "{{ env_name }}.zip" "{{ dest }}"
      args:
          chdir: "{{ WORKSPACE }}"

    - name: Cleanup - unzipped env
      file:
        state: absent
        path: "{{ WORKSPACE }}/{{ env_name }}/"