[BUG] files needed for packerlicious bug: https://github.com/elreydetoda/packerlicious/issues/1

kali-template.json

{
  "builders": [
    {
      "boot_command": [
        "<esc><wait>",
        "install <wait>",
        " preseed/url=http://{{ .HTTPIP }}:{{ .HTTPPort }}/{{user `preseed_path`}} <wait>",
        "debian-installer=en_US.UTF-8 <wait>",
        "auto <wait>",
        "locale=en_US.UTF-8 <wait>",
        "kbd-chooser/method=us <wait>",
        "keyboard-configuration/xkb-keymap=us <wait>",
        "netcfg/get_hostname={{ .Name }} <wait>",
        "netcfg/get_domain=vagrantup.com <wait>",
        "fb=false <wait>",
        "debconf/frontend=noninteractive <wait>",
        "console-setup/ask_detect=false <wait>",
        "console-keymaps-at/keymap=us <wait>",
        "grub-installer/bootdev=/dev/sda <wait>",
        "<enter><wait>"
      ],
      "boot_wait": "10s",
      "cpus": "{{ user `cpus` }}",
      "disk_size": "{{user `disk_size`}}",
      "guest_additions_path": "VBoxGuestAdditions_{{.Version}}.iso",
      "guest_os_type": "Debian_64",
      "hard_drive_interface": "sata",
      "headless": "{{ user `headless` }}",
      "http_directory": "{{user `http_directory`}}",
      "iso_checksum": "{{user `iso_checksum`}}",
      "iso_url": "{{ user `iso_url` }}",
      "memory": "{{ user `memory` }}",
      "output_directory": "{{ user `build_directory` }}/packer-{{user `template`}}-virtualbox",
      "shutdown_command": "echo 'vagrant' | sudo -S /sbin/shutdown -hP now",
      "ssh_password": "vagrant",
      "ssh_port": 22,
      "ssh_timeout": "10000s",
      "ssh_username": "vagrant",
      "type": "virtualbox-iso",
      "virtualbox_version_file": ".vbox_version",
      "vm_name": "{{ user `template` }}"
    },
    {
      "boot_command": [
        "<esc><wait>",
        "install <wait>",
        " preseed/url=http://{{ .HTTPIP }}:{{ .HTTPPort }}/{{user `preseed_path`}} <wait>",
        "debian-installer=en_US.UTF-8 <wait>",
        "auto <wait>",
        "locale=en_US.UTF-8 <wait>",
        "kbd-chooser/method=us <wait>",
        "keyboard-configuration/xkb-keymap=us <wait>",
        "netcfg/get_hostname={{ .Name }} <wait>",
        "netcfg/get_domain=vagrantup.com <wait>",
        "fb=false <wait>",
        "debconf/frontend=noninteractive <wait>",
        "console-setup/ask_detect=false <wait>",
        "console-keymaps-at/keymap=us <wait>",
        "grub-installer/bootdev=/dev/sda <wait>",
        "<enter><wait>"
      ],
      "boot_wait": "10s",
      "cpus": "{{ user `cpus` }}",
      "disk_size": "{{user `disk_size`}}",
      "guest_os_type": "debian8-64",
      "headless": "{{ user `headless` }}",
      "http_directory": "{{user `http_directory`}}",
      "iso_checksum": "{{user `iso_checksum`}}",
      "iso_url": "{{ user `iso_url` }}",
      "memory": "{{ user `memory` }}",
      "output_directory": "{{ user `build_directory` }}/packer-{{user `template`}}-vmware",
      "shutdown_command": "echo 'vagrant' | sudo -S /sbin/shutdown -hP now",
      "ssh_password": "vagrant",
      "ssh_port": 22,
      "ssh_timeout": "10000s",
      "ssh_username": "vagrant",
      "tools_upload_flavor": "linux",
      "type": "vmware-iso",
      "vm_name": "{{ user `template` }}",
      "vmx_data": {
        "cpuid.coresPerSocket": "1",
        "ethernet0.pciSlotNumber": "32"
      },
      "vmx_remove_ethernet_interfaces": true
    }
  ],
  "post-processors": [
    [
      {
        "output": "{{ user `build_directory` }}/{{user `box_basename`}}.{{.Provider}}.box",
        "type": "vagrant",
        "compression_level": 9,
        "vagrantfile_template": "{{ user `vagrantfile` }}"
      },
      {
        "type": "vagrant-cloud",
        "box_tag": "{{user `vm_name`}}",
        "access_token": "{{user `vagrant_cloud_token`}}",
        "version": "{{user `vm_version`}}",
        "keep_input_artifact": "true"
      }
    ]
  ],
  "provisioners": [
    {
      "environment_vars": [
        "HOME_DIR=/home/vagrant",
        "http_proxy={{user `http_proxy`}}",
        "https_proxy={{user `https_proxy`}}",
        "no_proxy={{user `no_proxy`}}"
      ],
      "execute_command": "echo 'vagrant' | {{.Vars}} sudo -S -E sh -eux '{{.Path}}'",
      "expect_disconnect": true,
      "scripts": [
        "{{user `bento_debian_dir`}}/scripts/update.sh",
        "{{user `bento_debian_dir`}}/../_common/motd.sh",
        "{{user `bento_debian_dir`}}/../_common/sshd.sh",
        "{{user `bento_debian_dir`}}/scripts/networking.sh",
        "{{user `bento_debian_dir`}}/scripts/sudoers.sh",
        "{{user `bento_debian_dir`}}/../_common/vagrant.sh",
        "{{user `bento_debian_dir`}}/scripts/systemd.sh",
        "{{user `bento_debian_dir`}}/../_common/virtualbox.sh",
        "{{user `bento_debian_dir`}}/../ubuntu/scripts/vmware.sh",
        "{{user `bento_debian_dir`}}/../_common/parallels.sh"
      ],
      "type": "shell"
    },
    {
      "type": "shell",
      "environment_vars": [
        "HOME_DIR=/home/vagrant",
        "http_proxy={{user `http_proxy`}}",
        "https_proxy={{user `https_proxy`}}",
        "no_proxy={{user `no_proxy`}}"
      ],
      "execute_command": "echo 'vagrant' | {{.Vars}} sudo -S -E bash -eux '{{.Path}}'",
      "expect_disconnect": "true",
      "scripts": [
        "prov_packer/customization.sh",
        "prov_packer/docker.sh",
        "prov_packer/full-update.sh",
        "prov_packer/networking.sh",
        "prov_packer/virtualbox.sh"
      ]
    },
    {
      "type": "shell",
      "environment_vars": [
        "HOME_DIR=/home/vagrant",
        "http_proxy={{user `http_proxy`}}",
        "https_proxy={{user `https_proxy`}}",
        "no_proxy={{user `no_proxy`}}"
      ],
      "execute_command": "echo 'vagrant' | {{.Vars}} sudo -S -E sh -eux '{{.Path}}'",
      "expect_disconnect": "true",
      "scripts": [
        "prov_packer/cleanup.sh",
        "{{user `bento_debian_dir`}}/../_common/minimize.sh"
      ]
    }
  ],
  "variables": {
    "box_basename": "red-automated_kali",
    "build_directory": ".",
    "cpus": "2",
    "disk_size": "65536",
    "headless": "",
    "http_directory": "install/http",
    "http_proxy": "{{env `http_proxy`}}",
    "https_proxy": "{{env `https_proxy`}}",
    "iso_checksum": "",
    "memory": "4096",
    "no_proxy": "{{env `no_proxy`}}",
    "preseed_path": "kali-linux-rolling-preseed.cfg",
    "bento_debian_dir": "prov_packer/bento/packer_templates/debian",
    "build_script_dir": "prov_packer",
    "iso_checksum_type": "",
    "iso_url": "",
    "template": "packerAutoKali",
    "vagrant_cloud_token": "",
    "vagrantfile": "install/vagrantfile-kali_linux.template",
    "vm_version": "",
    "vm_name": ""
  },
  "sensitive-variables": [
    "vagrant_cloud_token"
  ]
}

Pipfile

[[source]]
name = "pypi"
url = "https://pypi.org/simple"
verify_ssl = true

[dev-packages]
# for python dev
pylint = "*"
bandit = "*"
black = "==20.8b1"
# for ansible dev
ansible-lint = "*"
yamllint = "*"
molecule = "*"

[packages]
# for python prompt
bullet = "*"
# for python aws mfa (sts)
boto3 = "*"
# for infra prov
ansible = "*"
packerlicious = "*"
packet-python = "*"

[requires]
python_version = "3"

template-gen.py

#!/usr/bin/env python3
import json
import pathlib
from inspect import getframeinfo, currentframe
from pprint import pprint
from typing import NoReturn
from copy import deepcopy
from packerlicious import Template as packer_template
from packerlicious import provisioner as packer_provisioner
from packerlicious import builder as packer_builder
from packerlicious import post_processor as packer_post_processor

# TODO: add more + better logging, w/cli arg optional

# start each section with a pre-defined message and it's name
def section_meta(current_status: str, current_func: str) -> NoReturn:
    # adding extra spacing
    print('')

    # using logging function to print
    logging('{} {} function'.format(current_status, current_func))

    # adding extra spacing
    print('')

# logging func
def logging(log_str: str) -> NoReturn:
    pprint(log_str)

# getting pre-existing packer template from bento project
def get_packer_template(packer_template_path: pathlib) -> json:

    # replacing all instances of template_dir with a user var
    #   of bento_debian_path, because that is the location it
    #   is expecting for relative file traversal
    json_string = packer_template_path.read_text().replace('template_dir', 'user `bento_debian_dir`')
    return json.loads(json_string)

def variable_alterations(packer_template_data: dict, new_vars: dict) -> dict:
    section_meta('starting', getframeinfo(currentframe()).function)

    # only selecting the vars section from the template
    variables_section = packer_template_data['variables']


    # list of items to remove from the template
    remove_list = [
        'build_timestamp',
        'git_revision',
        'guest_additions_url',
        'iso_name',
        'mirror',
        'mirror_directory',
        'name',
        'template',
        'version'
        ]

        # removing all items in list above
    for var in remove_list:
        logging('removed: {}'.format(var))
        del variables_section[var]

    # either adding or updating values in template
    sub_dict = {
        'bento_debian_dir': str(new_vars['bento_debian_dir']),
        'box_basename': str(new_vars['build_vm_base_output_name']),
        'build_directory': str(new_vars['project_root']),
        'build_script_dir': str(new_vars['packer_script_dir']),
        'cpus': new_vars['build_cpus'],
        'headless': '',
        'http_directory': str(new_vars['http_dir']),
        'iso_checksum': '',
        'iso_checksum_type': '',
        'iso_url': '',
        'memory': new_vars['build_memory'],
        'preseed_path': str(new_vars['preseed_file']),
        'template': 'packerAutoKali',
        'vagrant_cloud_token': '',
        'vagrantfile': str(new_vars['vagrant_template_file']),
        'vm_version': '',
        'vm_name': ''
    }

    logging('updating properties for variables, section')
    variables_section.update(sub_dict)

    # logging(variables_section)
    section_meta('exiting', getframeinfo(currentframe()).function)
    return packer_template_data

def sensitive_variables(packer_template_data: dict, sensitive_vars: list) -> dict:
    section_meta('starting', getframeinfo(currentframe()).function)

    logging(
        'adding variables to sensative vars secions: {}'.format(
            ', '.join(sensitive_vars)
            )
    )
    packer_template_data['sensitive-variables'] = sensitive_vars

    section_meta('exiting', getframeinfo(currentframe()).function)

    return packer_template_data

def builder_alterations(packer_template_data: dict, new_builder_data: dict) -> dict:
    section_meta('starting', getframeinfo(currentframe()).function)

    packer_builder_list = packer_template_data['builders']

    removing_builders_list = []

    for builder in packer_builder_list:
        if builder['type'] not in new_builder_data['supported_builder_list']:
            removing_builders_list.append(builder)

    for removed_builder in removing_builders_list:
        logging('removing: {}'.format(removed_builder['type']))
        packer_builder_list.remove(removed_builder)

    # defining what properties have to be removed from builder
    prop_removal = [ 'guest_additions_url' ]

    # removing properties and logging
    for prop_rm in prop_removal:
        for builder_dict in packer_builder_list:
            if prop_rm in builder_dict:
                logging('removed: {} from: {}'.format(prop_rm, builder_dict['type']))
                del builder_dict[prop_rm]

    prop_update = {
        'iso_url': '{{ user `iso_url` }}'
    }

    for builder_dict in packer_builder_list:
        logging('updated property: {} in: {}'.format(prop_update, builder_dict['type']))
        builder_dict.update(prop_update)

    # logging(packer_builder_list)

    section_meta('exiting', getframeinfo(currentframe()).function)

    return packer_template_data

def provisioner_alterations(packer_template_data: dict, new_prov_data: dict) -> dict:
    section_meta('starting', getframeinfo(currentframe()).function)

    packer_prov_list = packer_template_data['provisioners']
    bento_prov = packer_prov_list[0]
    # creating deepcopy of env to use later, so we can alter it and it not
    #   mess up the original:
    #   https://stackoverflow.com/questions/2612802/how-to-clone-or-copy-a-list

    bento_env_vars = bento_prov['environment_vars']
    bento_exec_cmd = bento_prov['execute_command']
    bash_exec_cmd = bento_prov['execute_command'].replace(' sh ', ' bash ')
    bento_copy_prov = deepcopy(bento_prov)

    cleanup_scripts = []

    # minimize.sh
    cleanup_scripts.append(bento_prov['scripts'].pop())
    # cleanup.sh
    cleanup_scripts.append(bento_prov['scripts'].pop())

    personal_script_dict = {
        'environment_vars': bento_env_vars,
        'execute_command': bash_exec_cmd,
        'expect_disconnect': 'true',
        'scripts': new_prov_data['scripts_custom_list']
    }

    packerlicious_prov = packer_provisioner.Shell().from_dict(title='CustomSystemScripts', d=personal_script_dict)
    # adding my custom scripts
    packer_prov_list.append(packerlicious_prov.to_dict())

    ## SHELL: move last 2 scripts (cleanup) to bottom
    # clearing scripts section of all previous scripts
    bento_copy_prov['scripts'].clear()

    for script in reversed(cleanup_scripts):
        bento_copy_prov['scripts'].append(script)

    # removing for packerlicious usage
    del bento_copy_prov['type']
    # altering the the path for the cleanup.sh, so it
    # doesn't try to uninstall X11 packages
    bento_copy_prov['scripts'][0] = '{}/cleanup.sh'.format(new_prov_data['prov_packer_dir'])

    packerlicious_prov = packer_provisioner.Shell().from_dict(title='CleanupBentoScripts', d=bento_copy_prov)

    packer_prov_list.append(packerlicious_prov.to_dict())

    # print(json.dumps(packer_prov_list, indent=2))
    section_meta('exiting', getframeinfo(currentframe()).function)
    return packer_template_data

def post_processor_alterations(packer_template_data: dict, new_post_data: dict) -> dict:
    section_meta('starting', getframeinfo(currentframe()).function)
    post_processor_list = packer_template_data['post-processors']

    # updating vagrant processor
    post_processor_list[0].update(
        {
            'compression_level': 9 ,
            'vagrantfile_template': '{{ user `vagrantfile` }}'
        }
    )

    print(new_post_data)
    vagrant_cloud_post_processor = packer_post_processor.VagrantCloud().from_dict(title='VagrantCloudPP', d=new_post_data['vagrant-cloud'])
    print(vagrant_cloud_post_processor.to_dict())

    post_processor_list[0] = [
        post_processor_list[0],
        vagrant_cloud_post_processor.to_dict()
    ]
    # print(json.dumps(post_processor_list, indent=2))
    section_meta('exiting', getframeinfo(currentframe()).function)
    return packer_template_data

def write_packer_template(packer_template_path: pathlib, packer_template_data: dict) -> NoReturn:
    section_meta('starting', getframeinfo(currentframe()).function)

    # logging(packer_template_data)
    packer_template_path.write_text(json.dumps(packer_template_data, indent=2), encoding='utf-8')

    section_meta('exiting', getframeinfo(currentframe()).function)

# TODO: adding aspirations
# def add_builder_hyperv():

def main():

    ### section with lots of variables to get used throughout the script

    ## General

    # directory where the script is currently at, which should be
    #   the scripts folder
    script_dir = pathlib.Path(__file__).parent

    # project root directory
    project_root = script_dir.parent

    # packer provisioning scripts dir
    prov_packer_dir = project_root / 'prov_packer'

    new_packer_template = project_root / 'kali-template.json'

    http_preseed_dir = project_root / 'install' / 'http'
    http_preseed_file = 'kali-linux-rolling-preseed.cfg'
    vagrant_template_file = project_root / 'install' / 'vagrantfile-kali_linux.template'

    build_cpus = '2'
    build_memory = '4096'
    build_vm_base_output_name = 'red-automated_kali'

    ## builders section of variables
    supported_builder_list = [ 'virtualbox-iso', 'vmware-iso' ]
    ## provisioner section of variables
    scripts_removal_list = [
        'virtualbox.sh'
    ]
    prov_packer_dir_str = str(prov_packer_dir)
    scripts_custom_list = [
        '{}/customization.sh'.format(prov_packer_dir_str),
        '{}/docker.sh'.format(prov_packer_dir_str),
        '{}/full-update.sh'.format(prov_packer_dir_str),
        '{}/networking.sh'.format(prov_packer_dir_str),
        '{}/virtualbox.sh'.format(prov_packer_dir_str)
    ]

    ## Bento

    # bento dir
    bento_base_dir = prov_packer_dir / 'bento'

    # bento packer_templates dir
    bento_packer_template = bento_base_dir / 'packer_templates'

    # necessary folder in packer templates dir
    bento_common_scripts = bento_packer_template / '_common'
    bento_debian_dir = bento_packer_template / 'debian'

    bento_current_packer_template = bento_debian_dir / 'debian-10.5-amd64.json'

    ###

    # read in bento template
    old_packer_data = get_packer_template(bento_current_packer_template)

    ### variable alterations section
    # variables to update
    new_variables = {
        'bento_debian_dir': bento_debian_dir,
        'packer_script_dir': prov_packer_dir,
        'http_dir': http_preseed_dir,
        'preseed_file': http_preseed_file,
        'project_root': project_root,
        'build_cpus': build_cpus,
        'build_memory': build_memory,
        'vagrant_template_file': vagrant_template_file,
        'build_vm_base_output_name': build_vm_base_output_name
    }

    # updating variables
    updated_packer_data = variable_alterations(old_packer_data, new_variables)

    ### sensitive variables section
    # list of sensitive variables
    sensitive_var_list = [
        'vagrant_cloud_token'
    ]

    # adding list of sensative variables
    updated_packer_data = sensitive_variables(updated_packer_data, sensitive_var_list)

    ### builder alterations section
    builder_info_dict = {
        'supported_builder_list': supported_builder_list
    }
    updated_packer_data = builder_alterations(updated_packer_data, builder_info_dict)

    ### provisioner alterations section
    prov_info_dict = {
        'prov_packer_dir': prov_packer_dir,
        'scripts_removal_list': scripts_removal_list,
        'scripts_custom_list': scripts_custom_list
    }
    updated_packer_data = provisioner_alterations(updated_packer_data, prov_info_dict)

    ### post provisioner alterations section
    post_processor_dict = {
        'vagrant-cloud': {
            'box_tag': '{{user `vm_name`}}',
            'access_token': '{{user `vagrant_cloud_token`}}',
            'version': '{{user `vm_version`}}',
            'keep_input_artifact': True
            }
    }
    updated_packer_data = post_processor_alterations(updated_packer_data, post_processor_dict)

    # logging(updated_packer_data)

    # writing out to file
    write_packer_template(new_packer_template, updated_packer_data)

if __name__ == "__main__":
    main()