Skip to content

Instantly share code, notes, and snippets.

@ekohl

ekohl/README.md

Last active August 11, 2020 00:31
Show Gist options
  • Save ekohl/6b136d1dab4313edd5bd8eab5a7aacc0 to your computer and use it in GitHub Desktop.
Save ekohl/6b136d1dab4313edd5bd8eab5a7aacc0 to your computer and use it in GitHub Desktop.
Download ZIP
Explode Hiera
Raw
README.md

This script explodes Hiera. That means it reads a hiera config inside a module and converts all possible paths into the various interpolated parts.

The goal is to provide a possible solution to puppetlabs/puppet-strings#250.

Example output from puppet-openvmtools:

# Files

data/Debian.Debian.7.yaml
  facts.os.family => Debian
  facts.os.name => Debian
  facts.os.release.major => 7
data/Debian.Debian.8.yaml
  facts.os.family => Debian
  facts.os.name => Debian
  facts.os.release.major => 8
data/Debian.Debian.9.yaml
  facts.os.family => Debian
  facts.os.name => Debian
  facts.os.release.major => 9
data/Debian.Ubuntu.14.yaml
  facts.os.family => Debian
  facts.os.name => Ubuntu
  facts.os.release.major => 14
data/Debian.Ubuntu.16.yaml
  facts.os.family => Debian
  facts.os.name => Ubuntu
  facts.os.release.major => 16
data/Debian.Ubuntu.18.yaml
  facts.os.family => Debian
  facts.os.name => Ubuntu
  facts.os.release.major => 18
data/FreeBSD.FreeBSD.10.yaml
  facts.os.family => FreeBSD
  facts.os.name => FreeBSD
  facts.os.release.major => 10
data/FreeBSD.FreeBSD.11.yaml
  facts.os.family => FreeBSD
  facts.os.name => FreeBSD
  facts.os.release.major => 11
data/FreeBSD.FreeBSD.12.yaml
  facts.os.family => FreeBSD
  facts.os.name => FreeBSD
  facts.os.release.major => 12
data/RedHat.CentOS.6.yaml
  facts.os.family => RedHat
  facts.os.name => CentOS
  facts.os.release.major => 6
data/RedHat.CentOS.7.yaml
  facts.os.family => RedHat
  facts.os.name => CentOS
  facts.os.release.major => 7
data/RedHat.CentOS.8.yaml
  facts.os.family => RedHat
  facts.os.name => CentOS
  facts.os.release.major => 8
data/RedHat.Fedora.19.yaml
  facts.os.family => RedHat
  facts.os.name => Fedora
  facts.os.release.major => 19
data/RedHat.Fedora.20.yaml
  facts.os.family => RedHat
  facts.os.name => Fedora
  facts.os.release.major => 20
data/RedHat.Fedora.21.yaml
  facts.os.family => RedHat
  facts.os.name => Fedora
  facts.os.release.major => 21
data/RedHat.Fedora.22.yaml
  facts.os.family => RedHat
  facts.os.name => Fedora
  facts.os.release.major => 22
data/RedHat.Fedora.23.yaml
  facts.os.family => RedHat
  facts.os.name => Fedora
  facts.os.release.major => 23
data/RedHat.Fedora.24.yaml
  facts.os.family => RedHat
  facts.os.name => Fedora
  facts.os.release.major => 24
data/RedHat.Fedora.25.yaml
  facts.os.family => RedHat
  facts.os.name => Fedora
  facts.os.release.major => 25
data/RedHat.OracleLinux.6.yaml
  facts.os.family => RedHat
  facts.os.name => OracleLinux
  facts.os.release.major => 6
data/RedHat.OracleLinux.7.yaml
  facts.os.family => RedHat
  facts.os.name => OracleLinux
  facts.os.release.major => 7
data/RedHat.OracleLinux.8.yaml
  facts.os.family => RedHat
  facts.os.name => OracleLinux
  facts.os.release.major => 8
data/RedHat.RedHat.6.yaml
  facts.os.family => RedHat
  facts.os.name => RedHat
  facts.os.release.major => 6
data/RedHat.RedHat.7.yaml
  facts.os.family => RedHat
  facts.os.name => RedHat
  facts.os.release.major => 7
data/RedHat.RedHat.8.yaml
  facts.os.family => RedHat
  facts.os.name => RedHat
  facts.os.release.major => 8
data/Suse.OpenSUSE.11.yaml
  facts.os.family => Suse
  facts.os.name => OpenSUSE
  facts.os.release.major => 11
data/Suse.OpenSUSE.12.yaml
  facts.os.family => Suse
  facts.os.name => OpenSUSE
  facts.os.release.major => 12
data/Suse.OpenSUSE.13.yaml
  facts.os.family => Suse
  facts.os.name => OpenSUSE
  facts.os.release.major => 13
data/Suse.OpenSUSE.15.yaml
  facts.os.family => Suse
  facts.os.name => OpenSUSE
  facts.os.release.major => 15
data/Suse.OpenSUSE.42.yaml
  facts.os.family => Suse
  facts.os.name => OpenSUSE
  facts.os.release.major => 42
data/Suse.SLES.12.yaml
  facts.os.family => Suse
  facts.os.name => SLES
  facts.os.release.major => 12
data/Suse.SLES.13.yaml
  facts.os.family => Suse
  facts.os.name => SLES
  facts.os.release.major => 13
data/Suse.SLES.14.yaml
  facts.os.family => Suse
  facts.os.name => SLES
  facts.os.release.major => 14
data/Suse.SLES.15.yaml
  facts.os.family => Suse
  facts.os.name => SLES
  facts.os.release.major => 15
data/Debian.yaml
  facts.os.family => Debian
data/FreeBSD.yaml
  facts.os.family => FreeBSD

# Overrides (short)

openvmtools::supported
  data/Debian.Debian.7.yaml => true
  data/Debian.Debian.8.yaml => true
  data/Debian.Debian.9.yaml => true
  data/Debian.Ubuntu.14.yaml => true
  data/Debian.Ubuntu.16.yaml => true
  data/Debian.Ubuntu.18.yaml => true
  data/FreeBSD.FreeBSD.10.yaml => true
  data/FreeBSD.FreeBSD.11.yaml => true
  data/FreeBSD.FreeBSD.12.yaml => true
  data/RedHat.CentOS.6.yaml => true
  data/RedHat.CentOS.7.yaml => true
  data/RedHat.CentOS.8.yaml => true
  data/RedHat.Fedora.19.yaml => true
  data/RedHat.Fedora.20.yaml => true
  data/RedHat.Fedora.21.yaml => true
  data/RedHat.Fedora.22.yaml => true
  data/RedHat.Fedora.23.yaml => true
  data/RedHat.Fedora.24.yaml => true
  data/RedHat.Fedora.25.yaml => true
  data/RedHat.OracleLinux.6.yaml => true
  data/RedHat.OracleLinux.7.yaml => true
  data/RedHat.OracleLinux.8.yaml => true
  data/RedHat.RedHat.6.yaml => true
  data/RedHat.RedHat.7.yaml => true
  data/RedHat.RedHat.8.yaml => true
  data/Suse.OpenSUSE.11.yaml => true
  data/Suse.OpenSUSE.12.yaml => true
  data/Suse.OpenSUSE.13.yaml => true
  data/Suse.OpenSUSE.15.yaml => true
  data/Suse.OpenSUSE.42.yaml => true
  data/Suse.SLES.12.yaml => true
  data/Suse.SLES.13.yaml => true
  data/Suse.SLES.14.yaml => true
  data/Suse.SLES.15.yaml => true
openvmtools::manage_epel
  data/RedHat.CentOS.6.yaml => true
  data/RedHat.OracleLinux.6.yaml => true
  data/RedHat.RedHat.6.yaml => true
openvmtools::desktop_package_name
  data/Debian.yaml => open-vm-toolbox
  data/FreeBSD.yaml => open-vm-tools
openvmtools::service_name
  data/Debian.yaml => open-vm-tools
  data/FreeBSD.yaml => vmware_guestd
openvmtools::service_hasstatus
  data/Debian.yaml => false
openvmtools::desktop_package_conflicts
  data/FreeBSD.yaml => true
openvmtools::package_name
  data/FreeBSD.yaml => open-vm-tools-nox11

# Overrides (long)

openvmtools::supported
  data/Debian.Debian.7.yaml => true
    facts.os.family => Debian
    facts.os.name => Debian
    facts.os.release.major => 7
  data/Debian.Debian.8.yaml => true
    facts.os.family => Debian
    facts.os.name => Debian
    facts.os.release.major => 8
  data/Debian.Debian.9.yaml => true
    facts.os.family => Debian
    facts.os.name => Debian
    facts.os.release.major => 9
  data/Debian.Ubuntu.14.yaml => true
    facts.os.family => Debian
    facts.os.name => Ubuntu
    facts.os.release.major => 14
  data/Debian.Ubuntu.16.yaml => true
    facts.os.family => Debian
    facts.os.name => Ubuntu
    facts.os.release.major => 16
  data/Debian.Ubuntu.18.yaml => true
    facts.os.family => Debian
    facts.os.name => Ubuntu
    facts.os.release.major => 18
  data/FreeBSD.FreeBSD.10.yaml => true
    facts.os.family => FreeBSD
    facts.os.name => FreeBSD
    facts.os.release.major => 10
  data/FreeBSD.FreeBSD.11.yaml => true
    facts.os.family => FreeBSD
    facts.os.name => FreeBSD
    facts.os.release.major => 11
  data/FreeBSD.FreeBSD.12.yaml => true
    facts.os.family => FreeBSD
    facts.os.name => FreeBSD
    facts.os.release.major => 12
  data/RedHat.CentOS.6.yaml => true
    facts.os.family => RedHat
    facts.os.name => CentOS
    facts.os.release.major => 6
  data/RedHat.CentOS.7.yaml => true
    facts.os.family => RedHat
    facts.os.name => CentOS
    facts.os.release.major => 7
  data/RedHat.CentOS.8.yaml => true
    facts.os.family => RedHat
    facts.os.name => CentOS
    facts.os.release.major => 8
  data/RedHat.Fedora.19.yaml => true
    facts.os.family => RedHat
    facts.os.name => Fedora
    facts.os.release.major => 19
  data/RedHat.Fedora.20.yaml => true
    facts.os.family => RedHat
    facts.os.name => Fedora
    facts.os.release.major => 20
  data/RedHat.Fedora.21.yaml => true
    facts.os.family => RedHat
    facts.os.name => Fedora
    facts.os.release.major => 21
  data/RedHat.Fedora.22.yaml => true
    facts.os.family => RedHat
    facts.os.name => Fedora
    facts.os.release.major => 22
  data/RedHat.Fedora.23.yaml => true
    facts.os.family => RedHat
    facts.os.name => Fedora
    facts.os.release.major => 23
  data/RedHat.Fedora.24.yaml => true
    facts.os.family => RedHat
    facts.os.name => Fedora
    facts.os.release.major => 24
  data/RedHat.Fedora.25.yaml => true
    facts.os.family => RedHat
    facts.os.name => Fedora
    facts.os.release.major => 25
  data/RedHat.OracleLinux.6.yaml => true
    facts.os.family => RedHat
    facts.os.name => OracleLinux
    facts.os.release.major => 6
  data/RedHat.OracleLinux.7.yaml => true
    facts.os.family => RedHat
    facts.os.name => OracleLinux
    facts.os.release.major => 7
  data/RedHat.OracleLinux.8.yaml => true
    facts.os.family => RedHat
    facts.os.name => OracleLinux
    facts.os.release.major => 8
  data/RedHat.RedHat.6.yaml => true
    facts.os.family => RedHat
    facts.os.name => RedHat
    facts.os.release.major => 6
  data/RedHat.RedHat.7.yaml => true
    facts.os.family => RedHat
    facts.os.name => RedHat
    facts.os.release.major => 7
  data/RedHat.RedHat.8.yaml => true
    facts.os.family => RedHat
    facts.os.name => RedHat
    facts.os.release.major => 8
  data/Suse.OpenSUSE.11.yaml => true
    facts.os.family => Suse
    facts.os.name => OpenSUSE
    facts.os.release.major => 11
  data/Suse.OpenSUSE.12.yaml => true
    facts.os.family => Suse
    facts.os.name => OpenSUSE
    facts.os.release.major => 12
  data/Suse.OpenSUSE.13.yaml => true
    facts.os.family => Suse
    facts.os.name => OpenSUSE
    facts.os.release.major => 13
  data/Suse.OpenSUSE.15.yaml => true
    facts.os.family => Suse
    facts.os.name => OpenSUSE
    facts.os.release.major => 15
  data/Suse.OpenSUSE.42.yaml => true
    facts.os.family => Suse
    facts.os.name => OpenSUSE
    facts.os.release.major => 42
  data/Suse.SLES.12.yaml => true
    facts.os.family => Suse
    facts.os.name => SLES
    facts.os.release.major => 12
  data/Suse.SLES.13.yaml => true
    facts.os.family => Suse
    facts.os.name => SLES
    facts.os.release.major => 13
  data/Suse.SLES.14.yaml => true
    facts.os.family => Suse
    facts.os.name => SLES
    facts.os.release.major => 14
  data/Suse.SLES.15.yaml => true
    facts.os.family => Suse
    facts.os.name => SLES
    facts.os.release.major => 15
openvmtools::manage_epel
  data/RedHat.CentOS.6.yaml => true
    facts.os.family => RedHat
    facts.os.name => CentOS
    facts.os.release.major => 6
  data/RedHat.OracleLinux.6.yaml => true
    facts.os.family => RedHat
    facts.os.name => OracleLinux
    facts.os.release.major => 6
  data/RedHat.RedHat.6.yaml => true
    facts.os.family => RedHat
    facts.os.name => RedHat
    facts.os.release.major => 6
openvmtools::desktop_package_name
  data/Debian.yaml => open-vm-toolbox
    facts.os.family => Debian
  data/FreeBSD.yaml => open-vm-tools
    facts.os.family => FreeBSD
openvmtools::service_name
  data/Debian.yaml => open-vm-tools
    facts.os.family => Debian
  data/FreeBSD.yaml => vmware_guestd
    facts.os.family => FreeBSD
openvmtools::service_hasstatus
  data/Debian.yaml => false
    facts.os.family => Debian
openvmtools::desktop_package_conflicts
  data/FreeBSD.yaml => true
    facts.os.family => FreeBSD
openvmtools::package_name
  data/FreeBSD.yaml => open-vm-tools-nox11
    facts.os.family => FreeBSD
Raw
explode_hiera.rb
require 'yaml'
class HierarchyDataPath
attr_reader :datadir, :path, :regex, :mapping
def initialize(datadir, path)
@datadir = datadir
@path = path
@regex, @mapping = HierarchyDataPath.path2regex(path)
end
def matches
result = {}
Dir.chdir(datadir) do
Dir['**'].each do |entry|
next unless File.file?(entry)
regex.match(entry) do |match|
full_path = File.join(datadir, entry)
interpolations = {}
mapping.each do |name, interpolation|
interpolations[interpolation] = match.named_captures[name]
end
result[full_path] = interpolations
end
end
end
result
end
def self.path2regex(path)
mapping = {}
intermediate_result = path
# First pass - intermediate replacements
path.scan(/%{[^}]+}/).each_with_index do |interpolation, i|
replacement = "X_INTERPOLATION_#{i}_X"
mapping[replacement] = interpolation[2..-2]
intermediate_result = intermediate_result.sub(interpolation, replacement)
end
# Second pass - escape any special chars
escaped = Regexp.escape(intermediate_result)
# Third pass - replacement intermediates with regex
mapping.each_key do |replacement|
escaped = escaped.sub(replacement, "(?<#{replacement}>.+)")
end
[Regexp.new(escaped), mapping]
end
def to_s
File.join(datadir, path)
end
end
class HieraData
attr_reader :config_path, :data_paths
def initialize(config_path)
@config_path = config_path
@data_paths = []
load_config
end
def files
@files ||= begin
result = {}
data_paths.each do |dp|
dp.matches.each do |file, interpolations|
unless result.key?(file)
result[file] = interpolations
end
end
end
result
end
end
def to_s
config_path
end
private
def load_config
config = YAML.load(File.read(config_path))
unless config['version'] == 5
raise "Unsupported version '#{config['version']}'"
end
hierarchy = config['hierarchy']
return unless hierarchy
hierarchy.each do |level|
data_hash = level['data_hash'] || config['defaults']['data_hash']
next unless data_hash == 'yaml_data'
datadir = level['datadir'] || config['defaults']['datadir']
if level['path']
data_paths << HierarchyDataPath.new(datadir, level['path'])
elsif level['paths']
level['paths'].each do |path|
data_paths << HierarchyDataPath.new(datadir, path)
end
end
end
end
end
hiera_data = HieraData.new('hiera.yaml')
overrides = {}
hiera_data.files.each_key do |file|
data = YAML.load(File.read(file))
data.each do |key, value|
overrides[key] ||= {}
overrides[key][file] = value
end
end
puts "# Files"
puts
hiera_data.files.each do |file, interpolations|
puts file
interpolations.each do |interpolation, value|
puts " #{interpolation} => #{value}"
end
end
puts
puts "# Overrides (short)"
puts
overrides.each do |key, files|
puts key
files.each do |file, hiera_value|
puts " #{file} => #{hiera_value}"
end
end
puts
puts "# Overrides (long)"
puts
overrides.each do |key, files|
puts key
files.each do |file, hiera_value|
puts " #{file} => #{hiera_value}"
hiera_data.files[file].each do |interpolation, interpolation_value|
puts " #{interpolation} => #{interpolation_value}"
end
end
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment