preprocess.cf -- Run external preprocessing commands from CFEngine.

example-generated-script.sh

#!/bin/sh
exec /bin/echo \
  < '/tmp/TEST.in' > '/tmp/TEST.out' \
  -D'ipv4_172' \
  -D'have_aptitude' \
  -D'agent' \
  -D'GMT_Yr2015' \
  -D'net_iface_docker0' \
  -D'x86_64' \
  -D'Min25_30' \
  -D'ubuntu_14' \
  -D'linux_3_16_0_31_generic' \
  -D'GMT_Hr15' \
  -D'linux_x86_64' \
  -D'ipv4_172_17_42_1' \
  -D'fe80__226_2dff_fef6_a3c' \
  -D'ubuntu' \
  -D'xenia' \
  -D'cfengine_3_6_0' \
  -D'GMT_Day3' \
  -D'172_17_42_1' \
  -D'GMT_March' \
  -D'inform_mode' \
  -D'localhost' \
  -D'debian_jessie' \
  -D'4_cpus' \
  -D'PK_MD5_e8ba8d5aae1943b817fe0cc204b7f8d6' \
  -D'Hr16_Q2' \
  -D'ipv4_130' \
  -D'GMT_Lcycle_2' \
  -D'GMT_Q2' \
  -D'GMT_Hr15_Q2' \
  -D'Min29' \
  -D'compiled_on_linux_gnu' \
  -D'Hr16' \
  -D'linux' \
  -D'Afternoon' \
  -D'Day3' \
  -D'64_bit' \
  -D'mac_56_84_7a_fe_97_99' \
  -D'GMT_Min25_30' \
  -D'Q2' \
  -D'cfengine_3_6_0_55355d1' \
  -D'GMT_Tuesday' \
  -D'cfengine' \
  -D'cfengine_3' \
  -D'ipv4_127_0' \
  -D'linux_x86_64_3_16_0_31_generic' \
  -D'ipv4_172_17' \
  -D'cfengine_3_6' \
  -D'ipv4_127' \
  -D'127_0_0_1' \
  -D'ubuntu_14_10' \
  -D'Lcycle_2' \
  -D'March' \
  -D'ipv4_172_17_42' \
  -D'GMT_Min29' \
  -D'debian' \
  -D'130_60_144_175' \
  -D'ipv4_130_60' \
  -D'GMT_Afternoon' \
  -D'ipv4_130_60_144_175' \
  -D'net_iface_eth0' \
  -D'any' \
  -D'Tuesday' \
  -D'verbose_mode' \
  -D'ipv4_130_60_144' \
  -D'community_edition' \
  -D'linux_x86_64_3_16_0_31_generic__41_Ubuntu_SMP_Tue_Feb_10_15_24_04_UTC_2015' \
  -D'net_iface_lo' \
  -D'ipv4_127_0_0_1' \
  -D'ipv4_127_0_0' \
  -D'Yr2015' \
  -D'mac_00_26_2d_f6_0a_3c' \
  -D'first_pass' \
  -D'sys.systime=1425396556' \
  -D'sys.flavour=ubuntu_14' \
  -D'sys.hardware_addresses=00:26:2d:f6:0a:3c' \
  -D'sys.fqhost=xenia' \
  -D'sys.ipv4_1[docker0]=172' \
  -D'sys.uptime=7588' \
  -D'sys.bindir=/home/rmurri/.cfagent/bin' \
  -D'sys.arch=x86_64' \
  -D'sys.ipv4=130.60.144.175' \
  -D'sys.ipv4_2[eth0]=130.60' \
  -D'sys.maildir=/var/spool/mail' \
  -D'sys.logdir=/home/rmurri/.cfagent' \
  -D'sys.ipv4_1[lo]=127' \
  -D'sys.crontab=/var/spool/cron/crontabs/rmurri' \
  -D'sys.interface=docker0' \
  -D'sys.cf_monitord="/home/rmurri/.cfagent/bin/cf-monitord"' \
  -D'sys.key_digest=MD5=e8ba8d5aae1943b817fe0cc204b7f8d6' \
  -D'sys.local_libdir=lib/3.6' \
  -D'sys.cpus=4' \
  -D'sys.cf_version_minor=6' \
  -D'sys.ipv4[docker0]=172.17.42.1' \
  -D'sys.cf_report="/home/rmurri/.cfagent/bin/cf-report"' \
  -D'sys.class=linux' \
  -D'sys.ip_addresses=127.0.0.1' \
  -D'sys.cf_serverd="/home/rmurri/.cfagent/bin/cf-serverd"' \
  -D'sys.date=Tue Mar  3 16:29:16 2015' \
  -D'sys.interface_flags[lo]=up loopback running' \
  -D'sys.interfaces=eth0' \
  -D'sys.cf_runagent="/home/rmurri/.cfagent/bin/cf-runagent"' \
  -D'sys.flavor=ubuntu_14' \
  -D'sys.ostype=linux_x86_64' \
  -D'sys.uqhost=xenia' \
  -D'sys.os=linux' \
  -D'sys.interface_flags[eth0]=up broadcast running multicast' \
  -D'sys.piddir=/home/rmurri/.cfagent' \
  -D'sys.failsafe_policy_path=/home/rmurri/.cfagent/inputs/failsafe.cf' \
  -D'sys.exports=/etc/exports' \
  -D'sys.ipv4[eth0]=130.60.144.175' \
  -D'sys.release=3.16.0-31-generic' \
  -D'sys.cf_version=3.6.0.55355d1' \
  -D'sys.domain=' \
  -D'sys.ipv4_1[eth0]=130' \
  -D'sys.fstab=/etc/fstab' \
  -D'sys.hardware_flags=up loopback running' \
  -D'sys.cf_promises="/home/rmurri/.cfagent/bin/cf-promises"' \
  -D'sys.cf_version_patch=0' \
  -D'sys.version=#41-Ubuntu SMP Tue Feb 10 15:24:04 UTC 2015' \
  -D'sys.hardware_mac[eth0]=00:26:2d:f6:0a:3c' \
  -D'sys.cf_version_major=3' \
  -D'sys.doc_root=/var/www' \
  -D'sys.cf_key="/home/rmurri/.cfagent/bin/cf-key"' \
  -D'sys.ipv4_3[lo]=127.0.0' \
  -D'sys.masterdir=/home/rmurri/.cfagent/masterfiles' \
  -D'sys.cf_twin="/home/rmurri/.cfagent/bin/cf-agent"' \
  -D'sys.cf_execd="/home/rmurri/.cfagent/bin/cf-execd"' \
  -D'sys.ipv4_3[eth0]=130.60.144' \
  -D'sys.ipv4_2[lo]=127.0' \
  -D'sys.update_policy_path=/home/rmurri/.cfagent/inputs/update.cf' \
  -D'sys.hardware_mac[docker0]=56:84:7a:fe:97:99' \
  -D'sys.interface_flags[docker0]=up broadcast multicast' \
  -D'sys.ipv4_3[docker0]=172.17.42' \
  -D'sys.cdate=Tue_Mar__3_16_29_16_2015' \
  -D'sys.libdir=/home/rmurri/.cfagent/inputs/lib/3.6' \
  -D'sys.workdir=/home/rmurri/.cfagent' \
  -D'sys.long_arch=linux_x86_64_3_16_0_31_generic__41_Ubuntu_SMP_Tue_Feb_10_15_24_04_UTC_2015' \
  -D'sys.ipv4[lo]=127.0.0.1' \
  -D'sys.host=xenia' \
  -D'sys.ipv4_2[docker0]=172.17' \
  -D'sys.inputdir=/home/rmurri/.cfagent/inputs' \
  -D'sys.cf_agent="/home/rmurri/.cfagent/bin/cf-agent"' \
  -D'sys.resolv=/etc/resolv.conf' \
  -D'sys.sysday=16497' \
  -D'sys.hardware_addresses=56:84:7a:fe:97:99' \
  -D'sys.ip_addresses=130.60.144.175' \
  -D'sys.ip_addresses=172.17.42.1' \
  -D'sys.interfaces=docker0' \
  -D'sys.hardware_flags=up broadcast running multicast' \
  -D'sys.hardware_flags=up broadcast multicast' \
  -D'bin.rmdir=/bin/rmdir' \
  -D'bin.uniq=/usr/bin/uniq' \
  -D'bin.awk=/usr/bin/awk' \
  -D'bin.kill=/bin/kill' \
  -D'bin.gpasswd=/usr/bin/gpasswd' \
  -D'bin.userdel=/usr/sbin/userdel' \
  -D'bin.cut=/usr/bin/cut' \
  -D'bin.umount=/bin/umount' \
  -D'bin.xargs=/usr/bin/xargs' \
  -D'bin.pkill=/usr/bin/pkill' \
  -D'bin.python=/usr/bin/python' \
  -D'bin.cat=/bin/cat' \
  -D'bin.mount=/bin/mount' \
  -D'bin.rm=/bin/rm' \
  -D'bin.paste=/usr/bin/paste' \
  -D'bin.getent=/usr/bin/getent' \
  -D'bin.tar=/bin/tar' \
  -D'bin.sort=/usr/bin/sort' \
  -D'bin.grep=/bin/grep' \
  -D'bin.mail=/usr/bin/mail' \
  -D'bin.test=/usr/bin/test' \
  -D'bin.env=/usr/bin/env' \
  -D'bin.cp=/bin/cp' \
  -D'bin.perl=/usr/bin/perl' \
  -D'bin.tr=/usr/bin/tr' \
  -D'bin.echo=/bin/echo' \
  -D'bin.fgrep=/bin/fgrep' \
  -D'bin.egrep=/bin/egrep' \
  -D'bin.mktemp=/bin/mktemp' \
  -D'bin.mountpoint=/bin/mountpoint' \
  -D'bin.sed=/bin/sed' \
  -D'bin.id=/usr/bin/id' \
  -D'bin.useradd=/usr/sbin/useradd' \
  -D'bin.join=/usr/bin/join' \
  -D'bin.ip=/sbin/ip' \
  -D'bin.ifconfig=/sbin/ifconfig' \
  -D'bin.pgrep=/usr/bin/pgrep' \
  -D'bin.ln=/bin/ln' \
  -D'control_common.inputs=lib/cfengine_stdlib.cf' \
  -D'control_common.bundlesequence=inventory_facts' \
  -D'control_common.inputs=lib/gc3_stdlib.cf' \
  -D'control_common.inputs=lib/sh.cf' \
  -D'control_common.inputs=preprocess.cf' \
  -D'control_common.bundlesequence=test' \
  -D'send_email_message.tmpfile=/tmp/cfengine-email.xenia.1425396556.377400.txt' \
  -D'send_email_message.now=1425396556' \
  -D'send_email_message.c_tmpfile=_tmp_cfengine_email_xenia_1425396556_377400_txt' \
  -D'send_email_message.random=377400' \
  -D'preprocess.tmpfile=/tmp/pp.QlVRfEEF' \
  -D'preprocess.input=/tmp/TEST.in' \
  -D'preprocess.output=/tmp/TEST.out' \
  -D'preprocess.pp=/bin/echo' \
  -D'append_user_field.val=@(allusers)' \
  -D'cronjob.crontab=/var/spool/cron/crontabs' \

preprocess.cf

bundle agent preprocess(pp, input, output)
{
  vars:
    first_pass.!later_pass::
      "tmpfile"   string  => execresult("/usr/bin/env mktemp -t pp.XXXXXXXX", 
                                        "noshell");

      "state"     data  => datastate();
      "classes"   slist => getindices("state[classes]");
      
      # `state[vars]` lists *all* variable bundles, including things
      # like `const` and all bundles in the stdlib.  But it is a
      # problem as it includes any `edit_line` bundle and thus
      # generates infinite recursion down in the call chain, when we
      # are going to write the variables into a temp file... So filter
      # out problematic bundles, as well as bundles whose name starts
      # with `_` (sort of "non-public" by our convention)
      "__grps" slist => getindices("state[vars]");
      "_grps"  slist => grep("[a-zA-Z0-9]+[a-zA-Z0-9_(){}\[\]]*$", "__grps");
      "grps"   slist => filter("(const|append_if_no_lines?|insert_lines?)", 
                               "_grps", "true", "true", 99999);
      

  classes:
      # This apparently circular definition is needed to tell apart
      # the first pass over the bundle from the other ones.  We rely
      # on the fact that class definition statements are executed in
      # the order they are listed.
      #
      # - on 1st pass, `first_pass` is initially undefined, so it
      #   expands to `!any` in the definition of `later_pass`, hence
      #   `later_pass` is undefined too.
      # - but the second line *defines* `first_pass`,
      # - so on the 2nd and later passes, `first_pass` expands to
      #   `any` and thus `later_pass` will be defined.
      #
      "later_pass"  expression => "first_pass";
      "first_pass"  expression => "!later_pass";


  files:
    first_pass.!later_pass::
      "$(tmpfile)"
      perms => m("0700"),
      edit_defaults => empty,
      edit_line     => insert_lines("#!/bin/sh$(const.n)exec $(pp) \\");

      "$(tmpfile)"
      edit_defaults => std_defs,
      edit_line     => append_if_no_line("  < '$(input)' > '$(output)' \\");

      "$(tmpfile)"
      edit_defaults => std_defs,
      edit_line     => append_if_no_line("  -D'$(classes)' \\");

    later_pass::
      "$(tmpfile)"
      delete     => tidy,
      depends_on => { "run_preprocessor" };


  methods:
    first_pass.!later_pass::
      "Add `-Dvar=value` lines"
      usebundle => _append_vars_to_file("$(tmpfile)", "$(this.bundle).state", "$(grps)"),
      handle    => "add_vars";


  commands:
    first_pass.!later_pass::
      "$(tmpfile)"
      comment    => "Run preprocessor $(pp) with input file $(input) and output file $(output)",
      depends_on => { "add_vars" },
      handle     => "run_preprocessor";
}

bundle agent _append_vars_to_file(filename, statevar, grpname)
{
  vars:
      # List of all variable names, includes local/private variables
      # of the form `sys#uqhost` which must be filtered out.  Let's
      # also filter out variables whose name begins with an underscore
      # character `_`.
      "_vars" slist => getindices("$(statevar)[vars][$(grpname)]");
      "var"   slist => grep("[a-zA-Z0-9]+[a-zA-Z0-9_(){}\[\]-]*$", "_vars");

      # THIS DOES NOT WORK, although it should according to the documentation of `variablesmatching`
      #"var" slist => variablesmatching("$(grpname):[a-zA-Z0-9]+[a-zA-Z0-9_(){}\[\]]*$");

  files:
      "$(filename)"
      edit_defaults => std_defs,
      edit_line     => append_if_no_line("  -D'$(grpname).$(var)=$($(grpname).$(var))' \\");
}


bundle agent cpp(input, output)
# Warning: `cpp` discards blank lines!
{
  methods:
      "cpp" usebundle => preprocess("cpp -P", "$(input)", "$(output)");
}


bundle agent m4(input, output)
{
  methods:
      "m4" usebundle => preprocess("m4", "$(input)", "$(output)");
}

TEST.in

Example processing using the `j2pp` preprocessor;
see: http://github.com/uzh/j2pp

Domain name: {{sys.domain}}

L2 addresses (list):
{% for hwaddr in sys.hardware_addresses -%}
* {{ hwaddr }}
{% endfor %}

IP network interfaces:
{% for ifname, ipv4addr in sys.ipv4.iteritems() -%}
* {{ ifname }}: {{ ipv4addr }}
{% endfor %}

TEST.out

Example processing using the `j2pp` preprocessor;
see: http://github.com/uzh/j2pp

Domain name: 

L2 addresses (list):
* 00:26:2d:f6:0a:3c
* 56:84:7a:fe:97:99


IP network interfaces:
* lo: 127.0.0.1
* docker0: 172.17.42.1
* eth0: 130.60.144.175