dockerfile configuration automation / interpolation with Jinja2

.env

DEBIAN_RELEASE=bookworm

directory structure layout

❯ pwd
/home/xps/netcrave-docker-driver/setup
❯ tree
.
├── docker
│   ├── haproxy
│   │   ├── Dockerfile
│   │   └── templates
│   │       └── etc
│   │           └── haproxy
│   │               └── haproxy.cfg.jinja2
│   ├── squid
│   │   └── templates
│   │       └── etc
│   │           └── squid.conf.jinja2
│   └── vyos
│       └── templates
│           └── etc
│               └── boot.config.jinja2
└── docker-compose.yml

12 directories, 5 files

# docker-compose build haproxy from the CWD

docker-compose.yml

  haproxy:
    image: debian
    networks:
      netcrave-docker-driver-haproxy:
        ipv4_address: 192.0.0.26
        ipv6_address: 2001:db8:aaaa:aaba:192:0:0:26
    logging:
      driver: "fluentd"
      options:
        fluentd-address: 192.0.0.38
    build:
      context: .
      args:
        DEBIAN_RELEASE: ${DEBIAN_RELEASE}
        TEMPLATE_ROOT: ./docker/haproxy/templates
      dockerfile: ./docker/haproxy/Dockerfile

Dockerfile

ARG DEBIAN_RELEASE

FROM debian:$DEBIAN_RELEASE

ENV DEBIAN_FRONTEND noninteractive

RUN apt update

RUN apt -y --no-install-recommends install python-is-python3 python3-pip python3-dotenv python3-jinja2 curl

ADD .env /opt/conf/.env

ARG TEMPLATE_ROOT

ADD $TEMPLATE_ROOT /opt/conf/templates

RUN <<EOT python
from dotenv import load_dotenv;
import os, sys;
from pathlib import Path
import jinja2
from itertools import islice
from itertools import chain

for index in chain.from_iterable([["{}/{}".format(
    x, a) for a in z if a.endswith(".jinja2")] for x, y, z in os.walk(
    "/opt", topdown=True, onerror=None, followlinks=False) if len(z) > 0]):

    cur_path = Path(next(islice(Path(index).parents, 1)))
    out_path = Path("/") / Path("/".join(islice(cur_path.parts, 4, None)))

    print("current path: {}".format(cur_path))
    print("output path: {}".format(out_path))

    out_path.mkdir(parents = True, exist_ok = True)

    templateLoader = jinja2.FileSystemLoader(searchpath = cur_path)
    templateEnv = jinja2.Environment(loader = templateLoader)

    template_file = Path(list(Path(index).parts).pop())
    out_file = Path(template_file.name.replace(".jinja2", ""))

    print("template file name: {}".format(template_file))

    template = templateEnv.get_template(str(template_file))

    load_dotenv("/opt/conf/.env")
    outputText = template.render(os.environ)

    print("output file name: {}".format(out_file))
    print("full output path: {}".format(out_path / out_file))

    open(out_path / out_file, 'w+').write(outputText)

    print("saved")
EOT

RUN apt -y remove python-is-python3 python3-pip python3-dotenv python3-jinja2 curl

RUN apt -y autoremove

RUN rm -rf /opt/conf


############################################ OK, do whatever you want now

RUN apt --option Dpkg::Options::="--force-confold" --no-install-recommends -y install haproxy 

dockerfile_templating.py

from dotenv import load_dotenv;
import os, sys;
from pathlib import Path
import jinja2
from itertools import islice
from itertools import chain

for index in chain.from_iterable([["{}/{}".format(
    x, a) for a in z if a.endswith(".jinja2")] for x, y, z in os.walk(
    "/opt", topdown=True, onerror=None, followlinks=False) if len(z) > 0]):

    cur_path = Path(next(islice(Path(index).parents, 1)))
    out_path = Path("/") / Path("/".join(islice(cur_path.parts, 4, None)))

    print("current path: {}".format(cur_path))
    print("output path: {}".format(out_path))

    out_path.mkdir(parents = True, exist_ok = True)

    templateLoader = jinja2.FileSystemLoader(searchpath = cur_path)
    templateEnv = jinja2.Environment(loader = templateLoader)

    template_file = Path(list(Path(index).parts).pop())
    out_file = Path(template_file.name.replace(".jinja2", ""))

    print("template file name: {}".format(template_file))

    template = templateEnv.get_template(str(template_file))

    load_dotenv("/opt/conf/.env")
    outputText = template.render(os.environ)

    print("output file name: {}".format(out_file))
    print("full output path: {}".format(out_path / out_file))

    open(out_path / out_file, 'w+').write(outputText)

    print("saved")

haproxy.cfg

      global # bookworm
      stats socket  ipv6@[2001:db8:aaaa:aaba:192:0:0:26]:23          level admin
      stats socket  /run/haproxy/sock                       mode 666 level admin
      stats timeout 86400s

haproxy.cfg.jinja2

      global # {{ DEBIAN_RELEASE }}
      stats socket  ipv6@[2001:db8:aaaa:aaba:192:0:0:26]:23          level admin
      stats socket  /run/haproxy/sock                       mode 666 level admin
      stats timeout 86400s

This might come in handy later .. courtesy stack overflow https://stackoverflow.com/questions/6027558/flatten-nested-dictionaries-compressing-keys

def flatten(dictionary, parent_key='', separator='_'):
    items = []
    for key, value in dictionary.items():
        new_key = parent_key + separator + key if parent_key else key
        if isinstance(value, MutableMapping):
            items.extend(flatten(value, new_key, separator=separator).items())
        else:
            items.append((new_key, value))
    return dict(items)
flatten(compose)

{'version': '3.8',
 'services_ipam_image': 'debian',
 'services_ipam_networks_netcrave-docker-driver-ipam_ipv4_address': '192.0.0.2',
 'services_ipam_networks_netcrave-docker-driver-ipam_ipv6_address': '2001:db8:aaaa:aaaa:192:0:0:2',
 'services_ipam_logging_driver': 'fluentd',
 'services_ipam_logging_options_fluentd-address': '192.0.0.38',
 'services_ipam_volumes': ['/run/netcrave/docker/ifconfig'],
 'services_ifconfig_image': 'debian',
 'services_ifconfig_networks_netcrave-docker-driver-ifconfig_ipv4_address': '192.0.0.6',
 'services_ifconfig_networks_netcrave-docker-driver-ifconfig_ipv6_address': '2001:db8:aaaa:aaab:192:0:0:6',
 'services_ifconfig_logging_driver': 'fluentd',
 'services_ifconfig_logging_options_fluentd-address': '192.0.0.38',
 'services_ifconfig_volumes': ['/run/netcrave/docker/ifconfig'],
 'services_haproxycfg_image': 'debian',
 'services_haproxycfg_networks_netcrave-docker-driver-haproxycfg_ipv4_address': '192.0.0.10',
 'services_haproxycfg_networks_netcrave-docker-driver-haproxycfg_ipv6_address': '2001:db8:aaaa:aaac:192:0:0:10',

This is a pretty boss example, was kinda hoping for something non-recursive but what do you expect coming from stack overflow, https://stackoverflow.com/questions/4151320/efficient-circular-buffer this is probably a better tool for the job.

https://stackoverflow.com/questions/33923/what-is-tail-recursion this talks a little about the advantages of tail recursive optimization and reusing stack frames for subsequent calls, I used to wonder a lot about this actually, and whether it would ever become more common place

https://stackoverflow.com/questions/13591970/does-python-optimize-tail-recursion

I reckon this is close enough to a tail call, with two caveats:

• the lambda function logic has to be more complex to handle a multi-dimensional parameter and return one that is structurally identical, but it can be done.
• This relies on a fixed number of turns

list(itertools.accumulate([lambda x: x + 1] * 32, func = lambda tail, head: head(tail), initial = 1))
-> [1,
 2,
 3,
 4,
 5,
 6,
 7,
 8,
 9,
 10,
...

well, maybe try a mutable map instead of deque

In [29]: calls = collections.deque([lambda x: (x[0] + 1, x[1], x[0] > 2 and calls.append(lambda x: (x[0], None, None)) or None)] * 32)

In [30]: list(itertools.accumulate(calls, func = lambda v, f: f(v), initial = (1, None)))
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
Cell In [30], line 1
----> 1 list(itertools.accumulate(calls, func = lambda v, f: f(v), initial = (1, None)))

RuntimeError: deque mutated during iteration

In [31]: 

I guess stacking parameters is an option but it only stacks after every call

calls = collections.deque([lambda x: (x[0] + 1, x[1]((x[0] * 2, lambda x: True)), True)] * 32)

list(itertools.accumulate(calls, func = lambda tail, head: head((tail[0], head)), initial = (1, lambda x: (x[0], x[1]), None)))

[(1, <function __main__.<lambda>(x)>, None),
 (2, (3, True, True), True),
 (3, (5, True, True), True),
 (4, (7, True, True), True),
 (5, (9, True, True), True),
 (6, (11, True, True), True),
 (7, (13, True, True), True),
 (8, (15, True, True), True),
 (9, (17, True, True), True),
 (10, (19, True, True), True),
 (11, (21, True, True), True),
 (12, (23, True, True), True),
 (13, (25, True, True), True),
 (14, (27, True, True), True),
 (15, (29, True, True), True),
 (16, (31, True, True), True),
 (17, (33, True, True), True),