Digdag secrets

acl.yml

# System secret ACL policy
---
acl:
  operators:
    # TD
    td:
      secrets:
        - td.*
    td_load: 
      secrets:
        - td.*
    td_run: 
      secrets:
        - td.*
    td_for_each: 
      secrets:
        - td.*
    td_ddl: 
      secrets:
        - td.*
    td_table_export: 
      secrets:
        - td.*
    td_wait: 
      secrets:
        - td.*
    td_wait_table: 
      secrets:
        - td.*
    # MySQL
    mysql:
      - mysql.*

secrets.yml

# Site/project secrets stored separately (not in workflow file)
# Digdag accesses these kv's through a plugin interface.
# Storage could be Vault etc.
---
secret_storage:
  td.apikey: # ...
  mysql.user: # ...
  mysql.password: # ...
  a.mysql.user: # ...
  a.mysql.password: # ...
  b.mysql.user: # ...
  b.mysql.password: # ...

workflow.yml

# Digdag workflow file
---
+foobar:
  # td operator can access `td.*` by default (declared by annotations or system acl policy?)
  td>: query.sql

+mysql:
  # mysql operator can access `mysql.*` by default? (declared by annotations or system acl policy?)
  mysql>: query.sql

+mysql1:
  _secrets:
    mysql:
      user: a.mysql.user          # overrides `mysql.user`
      password: a.mysql.password  # overrides `mysql.password`
  mysql>: query.sql

+mysql2:
  _secrets:
    mysql: b.mysql                # overrides `mysql.*` ?
  mysql>: query.sql

# _secrets has the same scoping rules as _export
+scoped:
  _secrets:
    mysql: b.mysql                # overrides `mysql.*` for both +a and +b
  +a:
    mysql>: query.sql
  +b:
    mysql>: query.sql

+script:
  _secrets:
    td:
      apikey: td.apikey  # rb operator cannot access any secrets by default but the user 
                         # can explicitly specify secrets that should be accessible                         
  rb>: foobar

+shell:
  _secrets: # Tell digdag to pass in td.apikey to sh operator
    td:
      apikey: td.apikey
  _env:
    ENGINE: 'pig'
    TD_APIKEY: # And tell operator to assign td.apikey to TD_APIKEY environment variable when executing shell command
      secret: td.apikey
  sh>: td -k $TD_APIKEY query -t $ENGINE 'SELECT ...'

+parameterization:
  for_each>:
    customer:
      - a
      - b
  _do:
    +analyze:
      _secrets: # Secret access can be parameterized
        mysql: ${customer}.mysql
      mysql>: query.sql