template-pipeline-config.yaml

gistfile1.txt

default:
  pipeline:
    name: '{%!=name}'
    ppid: 2
    meta:
      origin:
        ssnam: '' # source system name
        ssurl: '' # populate if api url
      destn: 
        dbms: "postgres" # one of postgres, mssql, mysql, mongod
        dbname: "cabbage_patch"
        schema: "public"
        hostname: "{PROD_HOST}"
        username: "{PROD_DB_USR}"
        password: "{PROD_DB_PWD}"
      author: 'HCA'
      license: 
        file: 'LICENSE'
    envir:
    
      # source env is persisted while run env is created & destroyed
      #
      source:
      
        # system variables are expected to be set prior to this
        #
        input:
          sysvars:
            - 'PROD_HOST'   # pando
            - 'PROD_DB_USR' # cabbage
            - 'PROD_DB_PWD' # v*
         
         output:
         
            # do following db writes on completion of this pipeline
            #
            dout-append: 
              - {rvarname} $=> @tablename ???
            dout-create:
              -
            dout-replace:
        
         eval.on.exit:
          
      
      # on init run the following before execution of this pipeline process
      run:
        # load
        library: ['data.table', 'stringr', 'lubridate']
        
        # globals to create when initialized (and destroyed when finished)
        globvars:
          var1: ''