Dataflow Flex Templates

Dockerfile

FROM gcr.io/dataflow-templates-base/python3-template-launcher-base


RUN mkdir -p /dataflow/template
WORKDIR /dataflow/template

COPY pipeline ${WORKDIR}/pipeline

COPY spec/python_command_spec.json ${WORKDIR}/python_command_spec.json

ENV DATAFLOW_PYTHON_COMMAND_SPEC ${WORKDIR}/python_command_spec.json

RUN pip install avro-python3 pyarrow==0.11.1 apache-beam[gcp]==2.24.0

COPY setup.py ${WORKDIR}/setup.py 
COPY main.py ${WORKDIR}/main.py

# Super important to add these lines.
ENV FLEX_TEMPLATE_PYTHON_SETUP_FILE="${WORKDIR}/setup.py"
ENV FLEX_TEMPLATE_PYTHON_PY_FILE="${WORKDIR}/main.py"

image_spec.json

{
  "docker_template_spec": {
    "docker_image": "gcr.io/my-awesome-project/my_logic:latest"
  }
}

python_command_spec.json

{
    "pyFile":"/dataflow/template/main.py"
}

setup.py

import setuptools

setuptools.setup(
    packages=setuptools.find_packages(),
    install_requires=[
        'apache-beam[gcp]==2.24.0',
        'google-cloud-storage'
    ],
 )