design exploration of using bolt for system-level testing

bolt-tester.md

bolt as acceptance tester

• How to separate nodes, platforms and drivers when writing a test plan? See https://groups.google.com/a/puppet.com/d/msg/dept-quality-engineering-archive/r7q179wG-s0/q51jJmeqEwAJ for background

• declarativesystems/puppet_vagrant is a good starting point for vagrant support

  • how to extract connection info from resource?

• See this example and this function to check out how more nodes can be added to the inventory dynamically.

  • The core get_targets call traces through to inventory.rb#L85 and creates bolt's target.rb.
  • The example only contains the plan, so I'm left wondering if there is anything needed to be done to make get_targets($r.public_ip_address) work.
  • this elsif branch suggests that any string not matching anything else will eventually be converted to a new empty Target.

• The bolt provision example shows how to add EC2 nodes.

minimal example plan

This plan runs puppet device locally to talk to a vagrant VM running the PANOS appliance.

plan test_plan() {
  apply(local) {
    vagrant_vm {
      'panos':
        ensure => running,
        box    => 'palo_alto/panos71', # TODO: locate local pre-configured box
    }
  }

  # retrieve the vagrant_vm's properties
  # TOD: add ip_address read-only attribute
  $panos_vm = run_task('resource', 'vagrant_vm').filter |$r| { $r.name == 'panos' }

  apply(local) {
    file {
      'spec/fixtures/test-password.conf':
        # render a file with test credentials and the discovered IP address
        content => epp_template('panos/fixtures/password.conf.epp', { host => $panos_vm.ip_address} ),
    }
  }

  test_plan::run_puppet_device('examples/resources.pp', 'pavm')
}


# Runs `puppet device` and fails if an error is encountered.
function test_plan::run_puppet_device(String $manifest_file, String $target) {
  $r = run_task(
    device_manager::run_puppet_device,
    local,
    'Executing the test manifest',
    { target => 'panos', params => "--modulepath spec/fixtures/modules/ --deviceconfig spec/fixtures/device.conf --target ${target} --strict=error --detailed-exitcodes --verbose --trace --apply ${manifest_file}" })

  # TODO: add $params param to device_manager::run_puppet_device task
  # TODO: add change/error detection to device_manager::run_puppet_device task

  unless $r.ok {
    $r.each |$result| {
      warning("${node} errored with a message: ${result.error.message}")
    }
    fail('Test run failed')
  }

  # return results, pending better error handling through the task above
  $r
}

single-node example plan

plan test_plan() {
  apply() {
    vagrant_vm {
      'panos':
        ensure => running,
        box    => 'palo_alto/panos71', # TODO: locate local pre-configured box
    }
  }

  apply(local) {
    device_manager { 'panos.example.com':
      type        => 'panos',
      credentials => {
        host     => 'panos.example.com', # TODO: retrieve IP info from vagrant resource
        user     => admin,
        password => admin,
      }
    }
  }

  $r = run_task(
    device_manager::run_puppet_device,
    local,
    'Executing the test manifest',
    { target => 'panos', params => '--apply /tmp/example.pp' })

  unless $r.ok {
    $r.each |$result| {
      warning("${node} errored with a message: ${result.error.message}")
    }
    fail('Test run failed')
  }
}

example plan with an agent

plan test_plan() {

  apply() {
    vagrant_vm {
      'agent':
        ensure => running,
        box    => 'debian/stretch64',
      'panos':
        ensure => running,
        box    => 'palo_alto/panos71',
    }
  }

  $agent = test_plan::vagrant_targets('agent')

  # from https://github.com/puppetlabs/puppetlabs-puppet_agent/blob/fa0c9e2ecb0cfde1d916fedf97559e2e91daf763/tasks/install.json
  run_task(puppet_agent::install, $agent)

  # TBD:
  # * choose which node to upload the module to
  #   * requires to distinguish between agent nodes and device nodes
  # * where to get the module from -> use local `pdk build`
  # * Use [`file_upload`](https://puppet.com/docs/bolt/0.x/plan_functions.html#file-upload) to push the package to the target node
  # * run `puppet module install` on the target node
  # * do we actually need to deploy an agent, if we want to test a device module?
  run_plan(deploy_module, $agent)

  apply($agent) {
    device_manager { 'panos.example.com':
      type        => 'panos',
      credentials => {
        host     => 'panos.example.com', # TBD: this needs additional vagrant setup to work; problem might go away when connecting directly, instead of via agent
        user     => admin,
        password => admin,
      }
    }
  }

  file_upload('examples/resource.pp', '/tmp/example.pp', $agent)

  $r = run_task(
    device_manager::run_puppet_device,
    $agent,
    'Executing the test manifest',
    { target => 'panos', params => '--apply /tmp/example.pp' })

  unless $r.ok {
    $r.each |$result| {
      warning("${node} errored with a message: ${result.error.message}")
    }
    fail('Test run failed')
  }
}