Prefect CI Build Example

build.py

import uuid
from os import environ, path

import docker
from prefect.environments import KubernetesJobEnvironment
from prefect.environments.storage import Docker

from sable_batch.flows import (
    flow1,
    flow2,
    flow3,
)

FLOWS = [
    flow1,
    flow2,
    flow3,
]


def main():
    here = path.dirname(path.realpath(__file__))
    registry_url = "gcr.io/xxx/yyy"
    image_tag = uuid.uuid4().hex
    tls_config = None
    base_url = None
    dockerfile = path.join(here, "..", "Dockerfile")

    # Workaround for CircleCI Docker in Docker
    if environ.get("CI"):
        tls_config = docker.tls.TLSConfig(
            client_cert=(
                path.join(environ.get("DOCKER_CERT_PATH", ""), "cert.pem"),
                path.join(environ.get("DOCKER_CERT_PATH", ""), "key.pem"),
            ),
            verify=False,
        )
        base_url = environ.get("DOCKER_HOST")

    storage = Docker(
        image_name="xxx",
        registry_url=registry_url,
        image_tag=image_tag,
        base_url=base_url,  # required for CircleCI
        tls_config=tls_config,  # required for CircleCI
        dockerfile=dockerfile,
    )
    for flow_file in FLOWS:
        flow = flow_file.flow
        print(f"Adding {flow.name}")
        flow.environment = KubernetesJobEnvironment(
            job_spec_file=path.join(here, "job_spec.yaml"), unique_job_name=True
        )
        flow.storage = storage
        storage.add_flow(flow)

    storage.build(push=True)

    # Only register flows once the Docker image has been uploaded
    for flow_file in FLOWS:
        flow = flow_file.flow
        flow.register(project_name="default", build=False)


if __name__ == "__main__":
    main()