Skip to content

Instantly share code, notes, and snippets.

@denzuko
Last active October 29, 2023 22:28
Show Gist options
  • Save denzuko/6f99c173903c7c54199bd0bff2c790d8 to your computer and use it in GitHub Desktop.
Save denzuko/6f99c173903c7c54199bd0bff2c790d8 to your computer and use it in GitHub Desktop.
Docker based tasks for Airflow, k8s, and nomad.
from __future__ import annotations
import logging, sys, time
from airflow.decorators import task
from airflow.models.dag import DAG
from airflow.hooks.base_hook import BaseHook
from airflow.utils.dates import days_ago
from airflow.operators.python import PythonVirtualenvOperator, is_venv_installed
default_args = {
'start_date': days_ago(2),
'catchup': False,
'tags': ["cicd","yourname/app"]
}
with DAG(dag_id="cicd_docker_build", schedule_interval='@once, default_args=default_args) as dag:
if not is_venv_installed():
log.error("The virtualenv_python example task requires virtualenv, please install it.")
sys.exit(1)
@task.virtualenv(task_id="build_docker_image",
requirements=["dockerpy"],
system_site_packages=False)
def build_image(**op_kwargs):
from docker import DockerClient
from docker.errors import APIError, TLSParameterError
try:
docker_host = BaseHook.get_connection('docker_host')
client = DockerClient(base_url=docker_host.get_uri())
client.images.build(tag='yourname/app:latest', path ='./')
except (APIError, TLSParameterError) as err:
log.error(err)
sys.exit(1)
@task.virtualenv(task_id="publish_docker_image",
requirements=["dockerpy"],
system_site_packages=False)
def push_image(**op_kwargs):
from docker import DockerClient
from docker.errors import APIError, TLSParameterError
try:
docker_repo = BaseHook.get_connection('docker.io')
docker_host = BaseHook.get_connection('docker_host')
client = DockerClient(base_url=docker_host.get_uri())
client.login(
username=docker_repo.user,
password=docker_repo.password,
registry=docker_repo.hostname)
client.api.push('yourname/app:latest')
except (APIError, TLSParameterError) as err:
log.error(err)
sys.exit(1)
build_task = build_image()
release_task = push_image()
build_task >> release_task
---
version: '3.9'
services:
runner:
image: 'yourname/app:latest'
build:
context: .
args: []
command: build
...
FROM python:3.9.18-alpine3.18
RUN pip install pipx invoke[invocations] scrapy scapy beautifulsoup ansible[dockerpy,pywinrm]
WORKDIR /src
COPY tasks.py /src
ENTRYPOINT invoke
CMD --list
#!/usr/bin/env python3
import task from invoke
@task(default=True)
def build(ctx, clean=False):
if clean:
print("Cleaning!")
print("Building!")
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment