Last active
September 12, 2023 11:55
-
-
Save MasaSip/21001b4c86a979468d1b56b8bc461c9e to your computer and use it in GitHub Desktop.
Dbx deployment with Jinja templating
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# conf/deployment.j2 | |
build: | |
no_build: true | |
environments: | |
{% for env, conf in envs.items() %} | |
{{ env }}: | |
workflows: | |
# DLT pipeline | |
- name: "dbx-dlt-demo-pipeline" | |
access_control_list: | |
- user_name: "{{ conf.get('workflow_owner') }}" | |
permission_level: "IS_OWNER" | |
target: "demo-db" | |
workflow_type: "pipeline" | |
clusters: | |
- policy_id: "{{ conf.get('policy_id') }}" | |
libraries: | |
- notebook: | |
path: "/Some/path/notebooks/sample_dlt_notebook" | |
# Workflow to orchestrate the whole data pipeline | |
- name: "dbx-demo-job" | |
workflow_type: "job-v2.1" | |
access_control_list: | |
- user_name: "{{ conf.get('workflow_owner') }}" | |
permission_level: "IS_OWNER" | |
{% if conf.get('failure_alerts') != None %} | |
email_notifications: | |
on_failure: | |
- "{{ conf.get('failure_alerts') }}" | |
{% endif %} | |
tasks: | |
- task_key: "pipeline-task" | |
pipeline_task: | |
pipeline_id: "pipeline://dbx-dlt-demo-pipeline" | |
{% endfor %} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# conf/deployment_variables.py | |
envs = { | |
'dev': { | |
'policy_id': "0123456789012345", | |
'workflow_owner': "service-principal://some-service-principal-dev", | |
}, | |
'test': { | |
'policy_id': "3456789012345678", | |
'workflow_owner': "service-principal://some-service-principal-test", | |
'failure_alerts': "failures-in-test@company.com", | |
}, | |
'prod': { | |
'policy_id': "6789012345678901", | |
'workflow_owner': "service-principal://some-service-principal-prod", | |
'failure_alerts': "failures-in-prod@company.com", | |
} | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# generate_deployment_file.py | |
import jinja2 | |
from conf.deployment_variables import envs | |
templateLoader = jinja2.FileSystemLoader(searchpath="conf") | |
templateEnv = jinja2.Environment(loader=templateLoader, trim_blocks=True, lstrip_blocks=True) | |
template = templateEnv.get_template("deployment.j2") | |
template_out = template.render(envs=envs) | |
with open(f"conf/deployment.yaml", "w") as file: | |
file.write(template_out) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment