Use jinja2 templates to generate multiple terraform providers in a loop

.gitignore

# Doit database files
.doit.db.*

# Terraform files created from templates
_*.tf

dodo.py

from jinja2 import Environment, FileSystemLoader
from pathlib import Path


DEFAULT_REGION = "us-east-1"
REGIONS = [
    "ap-northeast-1",
    "ap-northeast-2",
    "ap-northeast-3",
    "ap-south-1",
    "ap-southeast-1",
    "ap-southeast-2",
    "ca-central-1",
    "eu-central-1",
    "eu-north-1",
    "eu-west-1",
    "eu-west-2",
    "eu-west-3",
    "sa-east-1",
    "us-east-1",
    "us-east-2",
    "us-west-1",
    "us-west-2",
]


def task_render():
    """ "Render jinja2 templates.

    Yields:
        Render task for each jinja2 template file.

    """
    cwd = Path(".")
    environment = Environment(loader=FileSystemLoader(cwd))
    j2_templates = cwd.glob("*.j2")

    def render_template(template_path):
        """Render template.

        Args:
            template_path: Path to jinja2 template file

        """
        template = environment.get_template(str(template_path))
        with open(f"_{template_path.stem}", "w") as output_file:
            output_file.write(template.render(default_region=DEFAULT_REGION, regions=REGIONS))

    for j2_template in j2_templates:
        target = f"_{j2_template.stem}"
        yield {
            "name": target,
            "actions": [
                (
                    render_template,
                    [j2_template],
                    {},
                ),
            ],
            "file_dep": [j2_template],
            "targets": [target],
            "clean": True,
        }

provider.tf.j2

{% for region in regions %}
provider "aws" {
{%- if region != default_region %}
  alias = "{{ region | replace("-", "_") }}"
{%- endif %}
  default_tags {
    tags = local.tags
  }
  region = "{{ region }}"
}
{% endfor %}

Note that an underscore is used as prefix for terraform files generated from jinja2 templates. This is used in the .gitignore file to avoid committing those files to the repository by mistake.

Hey Javier appreciate this solution. Can you please tell me how I can run this example. Please share the terraform structured file repo if you have and commands to run this if I need to run anything before running terraform plan. Also can you tell me if there is any dependency for this like Python installation etc?

@nimblenitin In a python virtual environment, the dependencies could be installed with:
$ pip install doit jinja2

and the template could be rendered with:
$ doit

The _provider.tf output file would look as follows:

provider "aws" {
  alias = "ap_northeast_1"
  default_tags {
    tags = local.tags
  }
  region = "ap-northeast-1"
}

provider "aws" {
  alias = "ap_northeast_2"
  default_tags {
    tags = local.tags
  }
  region = "ap-northeast-2"
}

provider "aws" {
  alias = "ap_northeast_3"
  default_tags {
    tags = local.tags
  }
  region = "ap-northeast-3"
}

provider "aws" {
  alias = "ap_south_1"
  default_tags {
    tags = local.tags
  }
  region = "ap-south-1"
}

provider "aws" {
  alias = "ap_southeast_1"
  default_tags {
    tags = local.tags
  }
  region = "ap-southeast-1"
}

provider "aws" {
  alias = "ap_southeast_2"
  default_tags {
    tags = local.tags
  }
  region = "ap-southeast-2"
}

provider "aws" {
  alias = "ca_central_1"
  default_tags {
    tags = local.tags
  }
  region = "ca-central-1"
}

provider "aws" {
  alias = "eu_central_1"
  default_tags {
    tags = local.tags
  }
  region = "eu-central-1"
}

provider "aws" {
  alias = "eu_north_1"
  default_tags {
    tags = local.tags
  }
  region = "eu-north-1"
}

provider "aws" {
  alias = "eu_west_1"
  default_tags {
    tags = local.tags
  }
  region = "eu-west-1"
}

provider "aws" {
  alias = "eu_west_2"
  default_tags {
    tags = local.tags
  }
  region = "eu-west-2"
}

provider "aws" {
  alias = "eu_west_3"
  default_tags {
    tags = local.tags
  }
  region = "eu-west-3"
}

provider "aws" {
  alias = "sa_east_1"
  default_tags {
    tags = local.tags
  }
  region = "sa-east-1"
}

provider "aws" {
  default_tags {
    tags = local.tags
  }
  region = "us-east-1"
}

provider "aws" {
  alias = "us_east_2"
  default_tags {
    tags = local.tags
  }
  region = "us-east-2"
}

provider "aws" {
  alias = "us_west_1"
  default_tags {
    tags = local.tags
  }
  region = "us-west-1"
}

provider "aws" {
  alias = "us_west_2"
  default_tags {
    tags = local.tags
  }
  region = "us-west-2"
}

After that, you would be able to run terraform plan as usual.

Note that local.tags value is missing from the example code, but you can add your own or remove that from the jinja2 template if you prefer not to use any tag.

thanks @jcollado one more thing- I would be sharing this module as terraform registry to user, I would ideally be taking input like region from user and expect user to use to do terraform plan directly without running $ doit. Is that possible to just take input from user using the module and populate this template without user having to run the $ doit command and installing dependency?

@nimblenitin It might be possible to do some magic with a null resource and a local-exec provisioner. However, I believe that wouldn't be a cleaner solution because it probably requires running terraform twice (one to render the template and one to generate the plan with all the resources defined) and that would be more confusing that just running doit when the template is updated and then terraform.

Thanks appreciate your response.

Hey @jcollado I want to take input in the form of kind of dictionary with values- Profile, region and populate that in template, was trying but could not get anywhere with doit documentation. Below is what should populate. Can you please help with the modified dodo.py code for the same if possible? Also I did not understand why you have put this condition- if region != default_region. You could just mention the value of default_region in region list right?

{% for region in regions %}
provider "aws" {
{%- if region != default_region %}
alias = "{{ region | replace("-", "_") }}"
{%- endif %}
default_tags {
tags = local.tags
}
region = "{{ region }}"
profile = "{{ profiles }}"
}
{% endfor %}

hey @jcollado never mind I got it right. Appreciate you sharing the workaround :)

@jcollado so there is one more thing which I cannot get. I am going to be using a file as a data source for account_details below. Can you please tell me how I can make it work?

from jinja2 import Environment, FileSystemLoader
from pathlib import Path

account_details = [
    {'alias': 'MEMBER1', 'region': 'us-east-1', 'member_account_id': '0000000000', 'ccs_mem_account_id': '0000000000'},
    {'alias': 'MEMBER2', 'region': 'us-east-2', 'member_account_id': '0000000000', 'ccs_mem_account_id': '0000000000'}
]

def task_render():
    """ "Render jinja2 templates.
    Yields:
        Render task for each jinja2 template file.
    """
    cwd = Path(".")
    environment = Environment(loader=FileSystemLoader(cwd))
    j2_templates = cwd.glob("*.j2")

    def render_template(template_path):
        """Render template.
        Args:
            template_path: Path to jinja2 template file
        """
        template = environment.get_template(str(template_path))
        with open(f"{template_path.stem}", "w") as output_file:
            output_file.write(template.render(data=account_details))

    for j2_template in j2_templates:
        target = f"_{j2_template.stem}"
        yield {
            "name": target,
            "actions": [
                (
                    render_template,
                    [j2_template],
                    {},
                ),
            ],
            "file_dep": [j2_template],
            "targets": [target],
            "clean": True,
        }               render_template,
                    [j2_template],
                    {},
                ),
            ],
            "file_dep": [j2_template],
            "targets": [target],
            "clean": True,
        }

@nimblenitin It depends on the content of your template, but if you expect account_details to be available in the template file, you need to pass it to the template.render method. One way to do that would be as follows:

template.render(account_details=account_details)

Thanks. Will try it out