puppetserver-at-cern.md

gistfile1.txt

haproxy in front of CERN's puppetserves uses [puppetlabs-haproxy](https://forge.puppet.com/puppetlabs/haproxy)


## HaProxy Service

We use `rh-haproxy18-haproxy`


```puppet
exec{'/usr/bin/yum -y erase haproxy':
    onlyif => '/usr/bin/rpm -q haproxy',
}
-> file{'/usr/lib/systemd/system/haproxy.service':
  ensure => link,
  target => '/usr/lib/systemd/system/rh-haproxy18-haproxy.service',
  notify => Class['systemd::systemctl::daemon_reload'],
}
-> class{'::haproxy':
  package_name        => 'rh-haproxy18',
  config_dir          => $haproxy_config_dir,
  config_file         => "${haproxy_config_dir}/haproxy.cfg.withoutteigi",
  config_validate_cmd => '/opt/rh/rh-haproxy18/root/usr/sbin/haproxy -f % -c',
  global_options      => {
    'user'                      => 'haproxy',
    'group'                     => 'haproxy',
    'log'                       => '127.0.0.1 local2',
    'tune.ssl.default-dh-param' => '2048',
    'stats'                     => ['socket /var/lib/haproxy/stats level admin','timeout 2m'],
  },
  defaults_options    => {
    'mode'       => 'http',
    'log'        => 'global',
    'log-format' => '%f\ %ci\ %b\ %s\ [%t]\ %ST\ %B\ %U\ %Tr\ %Tt\ %r',
    'timeout'    =>  ['client 1m', 'server 4m', 'connect 10s'],
  },
  restart_command     => '/usr/bin/systemctl reload haproxy',
}
```

# HAProxy Peer Nodes
Multiple HAProxy nodes are loaded from the PuppetDB query of nodes


```puppet
$_pool_members = query_facts("hostgroup=\"punch/puppet/hap/${pool_name}\"",['ipaddress','fqdn'])
$_pool_members_ip = split(inline_template('<%= arr = [] ;
                     @_pool_members.keys.sort.each{|k| arr.push(@_pool_members[k]["ipaddress"])} ; 
                     arr.join(",") %>'),',')
$_pool_members_name = split(inline_template('<%= arr = [] ;
                     @_pool_members.keys.sort.each{|k| arr.push(@_pool_members[k]["fqdn"])} ;
                     arr.join(",") %>'),',')

haproxy::peer{'mypeermembers':
  peers_name   => 'mypeers',
  port         => 5000,
  ipaddresses  => $_pool_members_ip,
  server_names => $_pool_members_name,
}


```

# Frontends and Backends
Because we have serveral frontends (qa, production, devel) we use a defined type for this.

So the basic one:

```puppet
::hg_punch::private::hap_puppet_frontend_backend{'puppet-inter':
    port           => '8140',
    query          => "hostgroup~\"punch/puppet/ps/inter/${be_pool_name}\"",
    exclusive_envs => undef,
  }
```

with a type `hg_punch::private:::hap_puppet_frontend_backend`  as below. A lot of crazy
stuff in their to send nodes in the QA puppet environment to advance puppet servers
for instance.



```puppet
# === punch::private::hap_puppet_frontend_backend
#
# Allows configuring frontend and backend options for HA Proxy within Punch
#
# === Parameters
#
# [*port*]
#  The port to which the HA Proxy frontend should bind. Required.
#
# [*bk_port*]
#  The port on which the balanced backend memeber listen. Default to *port*
#
# [*query*]
# Arbitrary PuppetDB query to select the balanced backend members for HA Proxy.
# Required.
#
# [*frontend_bind_options*]
# An array containing bind options for the HA Proxy frontend. By default it
# configures an SSL connection.
#
# [*frontend_options*]
# An array containing options for the HA Proxy frontend.
#
# [*backend_options*]
# An array containing options for the HA Proxy backend.
#
# [*manage_frontend_firewall*]
# A boolean to set whether you want this resource to open the ports on the
# firewall for the HAProxy frontend ports. If you set it to false, you have to
# manage the firewall yourself. Defaults to true.
#
# [*exclusive_envs*]
# An array of environments. If specified then all queries will be blocked
# unless one of these environments is being used as a parameter in the URL.
#
# [*qasplit*]
# A Boolean defaulting to false. If true then two backends will be created,
# production servers in one backend and non-production backends on the other.
# Puppet traffic from from qa agents will be directed to the non-production
# backend.
#
# [*qaenvs*]
# An array of environments that will be considered to be `qa` environments
# with respect to `qasplit` being enabled. Defaults to just `['qa']`.

define hg_punch::private::hap_puppet_frontend_backend (
  $port,
  $query,
  $bk_port = $port,
  $frontend_bind_options = [
    'ssl',
    'crt', "/etc/opt/rh/rh-haproxy18/haproxy/${::fqdn}.pem",
    'ca-file', '/etc/pki/tls/certs/CERN-bundle.pem',
    'verify', 'required', 'no-sslv3',
    'v4v6',
  ],
  $frontend_options = {
    'http-request'    => [
      'set-header X-Client-Verify-Real %[ssl_c_verify]',
      'set-header X-Client-Verify NONE if !{ hdr_val(X-Client-Verify-Real) eq 0 }',
      'set-header X-Client-Verify SUCCESS if { hdr_val(X-Client-Verify-Real) eq 0 }',
      'set-header X-Client-DN CN=%{+Q}[ssl_c_s_dn(cn)]',
      'deny if { method PUT POST } { hdr_val(Content-length) ge 62914560 }',
      ],
  },
  $backend_options = {
      'balance'     => 'leastconn',
      'stick-table' => 'type ip size 20k peers mypeers',
  },
  $manage_frontend_firewall = true,
  $frontend_name = "frontend-${title}",
  $default_backend = "backend-${title}",
  Optional[Array] $exclusive_envs = undef,
  Boolean $qasplit = false,
  Array[String[1],1] $qaenvs = ['qa'],
) {


  if $manage_frontend_firewall {
    ensure_resource('firewall', "100 allow haproxy to handle IPv4 Puppet requests (${frontend_name})", {
      proto  => 'tcp',
      dport  => $port,
      action => 'accept',
    })

    ensure_resource('firewall', "100 allow haproxy to handle IPv6 Puppet requests (${frontend_name})", {
      provider => 'ip6tables',
      proto    => 'tcp',
      dport    => $port,
      action   => 'accept',
    })
  }

  if $exclusive_envs {
    $_acl = join($exclusive_envs.map |$_env| { "{ urlp(environment) ${_env} }" },' or ')
    if has_key($frontend_options, 'http-request') {
      $_extra_http_request_options = {
        'http-request' => concat($frontend_options['http-request'],["deny unless ${_acl}"]),
      }
    } else {
      $_extra_http_request_options = {
        'http-request' => ["deny unless ${_acl}"],
      }
    }
    $_frontend_options = merge($frontend_options, $_extra_http_request_options)
  } else {
    $_frontend_options = $frontend_options
  }

  if $qasplit {
    $_acl = join($qaenvs.map |$_env| { "urlp(environment) ${_env}" },' or ')
    $_qamatch = { 'acl'         => ["qaacl ${_acl}"],
                  'use_backend' => ["backend-${title}-qa if qaacl"],
                }
  } else {
    $_qamatch = {}
  }

  ensure_resource('haproxy::frontend', $frontend_name, {
    collect_exported => false,
    bind             => { ":::${port}" => $frontend_bind_options },
    options          => merge($_frontend_options, $_qamatch,
        {
          'default_backend' => $default_backend,
        }),
    require          => Package['CERN-CA-certs'],
  })

  haproxy::backend {"backend-${title}":
    collect_exported => false,
    options          => $backend_options,
  }

  if $qasplit {
    haproxy::backend {"backend-${title}-qa":
      collect_exported => false,
      options          => $backend_options,
    }
    $_node_facts = ['ipaddress','agent_specified_environment']
  } else {
    $_node_facts = ['ipaddress']
  }

  $_all_members = query_facts($query,$_node_facts)
  $_all_members_name = sort(keys($_all_members))
  $_all_members_ip   = $_all_members_name.map | $_m | { $_all_members[$_m]['ipaddress'] }

  # When filtering a hash the lambda function recieves each key, value pair in the form
  # of an array [key, value]
  # https://puppet.com/docs/puppet/4.9/function.html#filter
  if $qasplit {
    $_main_members_name = sort(keys($_all_members.filter | $_m | { $_m[1]['agent_specified_environment'] == 'production' }))
  } else {
    $_main_members_name = $_all_members_name
  }
  $_main_members_ip   = $_main_members_name.map | $_m | { $_all_members[$_m]['ipaddress'] }

  # The main set of balancer members is all hosts normally or non-qa hosts
  # if qasplit is enabled.
  haproxy::balancermember{"backend-${title}-members":
    listening_service => "backend-${title}",
    ports             => [$bk_port],
    ipaddresses       => $_main_members_ip,
    server_names      => $_main_members_name,
    options           => ['check'],
  }

  if $qasplit {
    $_qa_members_name = sort(difference($_all_members_name, $_main_members_name))
    $_qa_members_ip   = $_qa_members_name.map | $_m | { $_all_members[$_m]['ipaddress'] }

    haproxy::balancermember{"backend-${title}-qa-members":
      listening_service => "backend-${title}-qa",
      ports             => [$bk_port],
      ipaddresses       => $_qa_members_ip,
      server_names      => $_qa_members_name,
      options           => ['check'],
    }
  }

  $_all_members_ip_port = suffix($_all_members_ip,":${port}-${bk_port}")

  cernfw::allow_host{$_all_members_ip_port:
    proto => 'tcp',
    dport => '1-65535',
    sport => $bk_port,
  }

}
```