README.md

The included apply-template.rb provides a way generate Logstash config fragments from a shared template.

This can be useful for shared verbose configuration that is shared across multiple pipelines.

For example, if we are using multiple pipelines with pipelines.yml

 - pipeline.id: one
   path.config: "${LOGSTASH_HOME}/pipelines/one/*.conf"
 - pipeline.id: two
   path.config: "${LOGSTASH_HOME}/pipelines/two/*.conf"
 - pipeline.id: three
   path.config: "${LOGSTASH_HOME}/pipelines/three/*.conf"

And the following directory structure:

${LOGSTASH_HOME}/pipelines/
  input-beats.erb
  one/
    input-beats-a.yml
    input-beats-b.yml
    filters.conf
    outputs.conf
  two/
    input-beats.yml
    inputs.conf
    filters.conf
    outputs.conf
  three/
    input-beats.yml
    inputs.conf
    filters.conf
    outputs.conf

WHERE pipelines/input-beats.erb contained:

input {
  beats {
    port => <%= port || 5044 %>
    ssl => true
    ssl_certificate => "/path/to/logstash.crt"
    ssl_key => "/path/to/logstash.crt"
    ssl_certificate_authorities => "/path/to/org.ca"
    ssl_verify_mode => force_peer
    tags => <%= Array(tags).inspect %>
  }
}

AND pipelines/one/input-beats-a.yml contained:

---
port: 5045
tags: source-a

AND pipelines/one/input-beats-b.yml contained:

---
port: 5046
tags: source-b

AND pipelines/two/input-beats.yml contained:

---
port: 5047
tags: two

AND pipelines/three/input-beats.yml contained:

---
port: 5048
tags:
 - three
 - third

Then running the command:

./apply-template.rb pipelines/input-beats.erb pipelines/*/input-beats*.yml

Would produce four output files, resulting in a directory structure like:

${LOGSTASH_HOME}/pipelines/
  input-beats.erb
  one/
    input-beats-a.conf
    input-beats-a.yml
    input-beats-b.conf
    input-beats-b.yml
    filters.conf
    outputs.conf
  two/
    input-beats.conf
    input-beats.yml
    inputs.conf
    filters.conf
    outputs.conf
  three/
    input-beats.conf
    inputs.conf
    filters.conf
    outputs.conf

WHERE pipelines/one/input-beats-a.conf would contain:

## GENERATED: 2022-04-22T00:14:56+00:00
## TEMPLATE:  REDACTED/pipelines/input-beats.erb
## CONFIG:    REDACTED/pipelines/one/input-beats-a.yml
input {
  beats {
    port => 5045
    ssl => true
    ssl_certificate => "/path/to/logstash.crt"
    ssl_key => "/path/to/logstash.crt"
    ssl_certificate_authorities => "/path/to/org.ca"
    ssl_verify_mode => force_peer
    tags => ["source-a"]
  }
}

AND pipelines/one/input-beats-b.conf would contain:

## GENERATED: 2022-04-22T00:14:56+00:00
## TEMPLATE:  REDACTED/pipelines/input-beats.erb
## CONFIG:    REDACTED/pipelines/one/input-beats-b.yml
input {
  beats {
    port => 5046
    ssl => true
    ssl_certificate => "/path/to/logstash.crt"
    ssl_key => "/path/to/logstash.crt"
    ssl_certificate_authorities => "/path/to/org.ca"
    ssl_verify_mode => force_peer
    tags => ["source-b"]
  }
}

AND pipelines/three/input-beats.conf would contain:

## GENERATED: 2022-04-22T00:14:56+00:00
## TEMPLATE:  REDACTED/pipelines/input-beats.erb
## CONFIG:    REDACTED/pipelines/three/input-beats.yml
input {
  beats {
    port => 5048
    ssl => true
    ssl_certificate => "/path/to/logstash.crt"
    ssl_key => "/path/to/logstash.crt"
    ssl_certificate_authorities => "/path/to/org.ca"
    ssl_verify_mode => force_peer
    tags => ["three", "third"]
  }
}

AND pipelines/two/input-beats.conf would contain:

## GENERATED: 2022-04-22T00:14:56+00:00
## TEMPLATE:  REDACTED/pipelines/input-beats.erb
## CONFIG:    REDACTED/pipelines/two/input-beats.yml
input {
  beats {
    port => 5047
    ssl => true
    ssl_certificate => "/path/to/logstash.crt"
    ssl_key => "/path/to/logstash.crt"
    ssl_certificate_authorities => "/path/to/org.ca"
    ssl_verify_mode => force_peer
    tags => ["two"]
  }
}

apply-template.rb

#!/usr/bin/env ruby

# Copyright 2022 Ry Biesemeyer
# Made freely available under the MIT License

def usage!(reason=nil)
  puts reason unless reason.nil?
  puts <<~USAGE
    Combine an ERB template with one or more YAML config files.
    Produces one output file in the same directory as each
    discovered config.

    usage: #{$PROGRAM_NAME} [FLAGS] ERB_TEMPLATE YAML_CONFIG...

    ERB_TEMPLATE: the path to an ERB-formatted template

    YAML_CONFIG:  one or more YAML-formatted configs.
                  glob wildcards are expanded.

    FLAGS:

      --help:             show this help and exit

      --extension=EXT:    save output files with the provided
                          extension (default: `conf`)

      --header-prefix=XX: use the provided header prefix
                          (default: '## ')

      --header-skip:      do not add a header to the generated
                          output files

      --dry-run:          log actions to STDERR instead of
                          writing the generated files to disk

  USAGE
  exit 1
end

usage! if ARGV.include?('--help')

flags, args = ARGV.partition { |arg| arg.start_with?('--') }

# process flags
opts = flags.each_with_object({}) do |flag, memo|
  k,v = flag.split('=',2)
  memo[k]=v
end

extension =     opts.delete('--extension') { 'conf' }
header_prefix = opts.delete('--header-prefix') { '## ' }
header_skip =  (opts.delete('--header-skip') { false }).nil?
dry_run =      (opts.delete('--dry-run') { false }).nil?

usage!("Unhandled flags: #{opts.keys}") unless opts.empty?

# validate remaining args
erb_template = args.shift || usage!("missing required ERB_TEMPLATE")
yaml_config = args.any? ? args.flat_map(&Dir.method(:glob)) : usage!("missing required YAML_CONFIG")

# define flag-controlled behaviour
if dry_run
  def write_file(file_path, &block)
    $stderr.puts("DRY-RUN: #{file_path}")
    yield($stderr)
  end
else
  def write_file(file_path, &block)
    File.open(file_path, 'w+', &block)
    $stderr.puts("GENERATED: #{file_path}")
  end
end

# begin execution
require 'erb'
require 'yaml'
require 'ostruct'
require 'time'

template = ERB.new(File.read(erb_template))
yaml_config.each do |config_path|
  
  config = YAML.safe_load(File.read(config_path))
  namespace = OpenStruct.new(config)

  contents = template.result(namespace.instance_eval { binding })

  out_name = File.join(File.dirname(config_path), "#{File.basename(config_path, '.*')}.#{extension}")
  write_file(out_name) do |handle|
    unless header_skip
      handle.puts("#{header_prefix}GENERATED: #{Time.now.iso8601}")
      handle.puts("#{header_prefix}TEMPLATE:  #{File.absolute_path erb_template}")
      handle.puts("#{header_prefix}CONFIG:    #{File.absolute_path config_path}")
    end
    handle.puts(contents)
  end
end