Use jinja2 templates to generate multiple terraform providers in a loop

.gitignore

# Doit database files
.doit.db.*

# Terraform files created from templates
_*.tf

dodo.py

from jinja2 import Environment, FileSystemLoader
from pathlib import Path


DEFAULT_REGION = "us-east-1"
REGIONS = [
    "ap-northeast-1",
    "ap-northeast-2",
    "ap-northeast-3",
    "ap-south-1",
    "ap-southeast-1",
    "ap-southeast-2",
    "ca-central-1",
    "eu-central-1",
    "eu-north-1",
    "eu-west-1",
    "eu-west-2",
    "eu-west-3",
    "sa-east-1",
    "us-east-1",
    "us-east-2",
    "us-west-1",
    "us-west-2",
]


def task_render():
    """ "Render jinja2 templates.

    Yields:
        Render task for each jinja2 template file.

    """
    cwd = Path(".")
    environment = Environment(loader=FileSystemLoader(cwd))
    j2_templates = cwd.glob("*.j2")

    def render_template(template_path):
        """Render template.

        Args:
            template_path: Path to jinja2 template file

        """
        template = environment.get_template(str(template_path))
        with open(f"_{template_path.stem}", "w") as output_file:
            output_file.write(template.render(default_region=DEFAULT_REGION, regions=REGIONS))

    for j2_template in j2_templates:
        target = f"_{j2_template.stem}"
        yield {
            "name": target,
            "actions": [
                (
                    render_template,
                    [j2_template],
                    {},
                ),
            ],
            "file_dep": [j2_template],
            "targets": [target],
            "clean": True,
        }

provider.tf.j2

{% for region in regions %}
provider "aws" {
{%- if region != default_region %}
  alias = "{{ region | replace("-", "_") }}"
{%- endif %}
  default_tags {
    tags = local.tags
  }
  region = "{{ region }}"
}
{% endfor %}

@nimblenitin It might be possible to do some magic with a null resource and a local-exec provisioner. However, I believe that wouldn't be a cleaner solution because it probably requires running terraform twice (one to render the template and one to generate the plan with all the resources defined) and that would be more confusing that just running doit when the template is updated and then terraform.

Thanks appreciate your response.

Hey @jcollado I want to take input in the form of kind of dictionary with values- Profile, region and populate that in template, was trying but could not get anywhere with doit documentation. Below is what should populate. Can you please help with the modified dodo.py code for the same if possible? Also I did not understand why you have put this condition- if region != default_region. You could just mention the value of default_region in region list right?

{% for region in regions %}
provider "aws" {
{%- if region != default_region %}
alias = "{{ region | replace("-", "_") }}"
{%- endif %}
default_tags {
tags = local.tags
}
region = "{{ region }}"
profile = "{{ profiles }}"
}
{% endfor %}

hey @jcollado never mind I got it right. Appreciate you sharing the workaround :)

@jcollado so there is one more thing which I cannot get. I am going to be using a file as a data source for account_details below. Can you please tell me how I can make it work?

from jinja2 import Environment, FileSystemLoader
from pathlib import Path

account_details = [
    {'alias': 'MEMBER1', 'region': 'us-east-1', 'member_account_id': '0000000000', 'ccs_mem_account_id': '0000000000'},
    {'alias': 'MEMBER2', 'region': 'us-east-2', 'member_account_id': '0000000000', 'ccs_mem_account_id': '0000000000'}
]

def task_render():
    """ "Render jinja2 templates.
    Yields:
        Render task for each jinja2 template file.
    """
    cwd = Path(".")
    environment = Environment(loader=FileSystemLoader(cwd))
    j2_templates = cwd.glob("*.j2")

    def render_template(template_path):
        """Render template.
        Args:
            template_path: Path to jinja2 template file
        """
        template = environment.get_template(str(template_path))
        with open(f"{template_path.stem}", "w") as output_file:
            output_file.write(template.render(data=account_details))

    for j2_template in j2_templates:
        target = f"_{j2_template.stem}"
        yield {
            "name": target,
            "actions": [
                (
                    render_template,
                    [j2_template],
                    {},
                ),
            ],
            "file_dep": [j2_template],
            "targets": [target],
            "clean": True,
        }               render_template,
                    [j2_template],
                    {},
                ),
            ],
            "file_dep": [j2_template],
            "targets": [target],
            "clean": True,
        }

@nimblenitin It depends on the content of your template, but if you expect account_details to be available in the template file, you need to pass it to the template.render method. One way to do that would be as follows:

template.render(account_details=account_details)

Thanks. Will try it out