logsearch-cf.yml

compilation:
  cloud_properties:
    cpu: 2
    disk: 8192
    ram: 4096
  network: default
  reuse_compilation_vms: true
  workers: 4
director_uuid: <%= `bosh status --uuid`.strip %>
disk_pools:
- cloud_properties: {}
  disk_size: 10240
  name: elasticsearch_master
- cloud_properties: {}
  disk_size: 10240
  name: elasticsearch_data
- cloud_properties: {}
  disk_size: 10240
  name: queue
- cloud_properties: {}
  disk_size: 10240
  name: cluster_monitor
jobs:
- instances: 1
  name: elasticsearch_master
  networks:
  - name: default
    static_ips:
    - 172.16.0.171
  persistent_disk_pool: elasticsearch_master
  properties:
    elasticsearch:
      node:
        allow_data: false
        allow_master: true
    syslog_forwarder:
      config:
      - file: /var/vcap/sys/log/elasticsearch/elasticsearch.stdout.log
        service: elasticsearch
      - file: /var/vcap/sys/log/elasticsearch/elasticsearch.stderr.log
        service: elasticsearch
  resource_pool: elasticsearch_master
  templates:
  - name: elasticsearch
    release: logsearch
  - name: syslog_forwarder
    release: logsearch
  update:
    max_in_flight: 1
- instances: 1
  name: cluster_monitor
  networks:
  - name: default
    static_ips:
    - 172.16.0.173
  persistent_disk_pool: cluster_monitor
  properties:
    curator:
      elasticsearch:
        host: 127.0.0.1
        port: 9200
      purge_logs:
        retention_period: 7
    elasticsearch:
      cluster_name: monitor
      master_hosts:
      - 127.0.0.1
      node:
        allow_data: true
        allow_master: true
    elasticsearch_config:
      elasticsearch:
        host: 127.0.0.1
        port: 9200
      index_prefix: logstash-
      templates:
      - shards-and-replicas: '{ "template" : "logstash-*", "order" : 100, "settings"
          : { "number_of_shards" : 1, "number_of_replicas" : 0 } }'
      - index-settings: /var/vcap/jobs/elasticsearch_config/index-templates/index-settings.json
      - index-mappings: /var/vcap/jobs/elasticsearch_config/index-templates/index-mappings.json
    kibana:
      elasticsearch:
        host: 127.0.0.1
        port: 9200
      memory_limit: 30
      port: 5601
      wait_for_templates:
      - shards-and-replicas
    logstash_ingestor:
      syslog:
        port: 514
    logstash_parser:
      filters:
      - monitor: /var/vcap/packages/logsearch-config/logstash-filters-monitor.conf
      logstash_parser.elasticsearch.index: logstash-%{+YYYY.MM.dd}
    nats_to_syslog:
      debug: true
    redis:
      host: 127.0.0.1
      maxmemory: 10
  resource_pool: cluster_monitor
  templates:
  - name: queue
    release: logsearch
  - name: parser
    release: logsearch
  - name: ingestor_syslog
    release: logsearch
  - name: elasticsearch
    release: logsearch
  - name: elasticsearch_config
    release: logsearch
  - name: curator
    release: logsearch
  - name: kibana
    release: logsearch
  - name: nats_to_syslog
    release: logsearch
- instances: 1
  name: queue
  networks:
  - name: default
    static_ips:
    - 172.16.0.174
  persistent_disk_pool: queue
  properties:
    syslog_forwarder:
      config:
      - file: /var/vcap/sys/log/queue/queue.log
        service: queue_redis
  resource_pool: queue
  templates:
  - name: queue
    release: logsearch
  - name: syslog_forwarder
    release: logsearch
- instances: 1
  name: maintenance
  networks:
  - name: default
  properties:
    elasticsearch_config:
      index_prefix: logs-
      templates:
      - shards-and-replicas: /var/vcap/jobs/elasticsearch_config/index-templates/shards-and-replicas.json
      - index-settings: /var/vcap/jobs/elasticsearch_config/index-templates/index-settings.json
      - index-mappings-lfc: /var/vcap/jobs/elasticsearch-config-lfc/index-mappings.json
    syslog_forwarder:
      config:
      - file: /var/vcap/sys/log/curator/curator.log
        service: curator
      - file: /var/vcap/sys/log/elasticsearch_config/job.control.log
        service: elasticsearch_config
  resource_pool: maintenance
  templates:
  - name: elasticsearch_config
    release: logsearch
  - name: curator
    release: logsearch
  - name: syslog_forwarder
    release: logsearch
  - name: elasticsearch-config-lfc
    release: logsearch-for-cloudfoundry
  update:
    serial: true
- instances: 1
  name: elasticsearch_data
  networks:
  - name: default
  persistent_disk_pool: elasticsearch_data
  properties:
    elasticsearch:
      node:
        allow_data: true
        allow_master: false
    syslog_forwarder:
      config:
      - file: /var/vcap/sys/log/elasticsearch/elasticsearch.stdout.log
        service: elasticsearch
      - file: /var/vcap/sys/log/elasticsearch/elasticsearch.stderr.log
        service: elasticsearch
  resource_pool: elasticsearch_data
  templates:
  - name: elasticsearch
    release: logsearch
  - name: syslog_forwarder
    release: logsearch
  update:
    max_in_flight: 1
- instances: 1
  name: kibana
  networks:
  - name: default
    static_ips:
    - 172.16.0.176
  properties:
    kibana:
      env:
      - KIBANA_DOMAIN: elktesting.footoo.org
      - KIBANA_OAUTH2_CLIENT_ID: my_kibana_client_id
      - KIBANA_OAUTH2_CLIENT_SECRET: my_kibana_client_password
      - SKIP_SSL_VALIDATION: true
      - CF_API_URI: https://api.cftesting.footoo.org
      - CF_SYSTEM_ORG: admin
      - REDIS_HOST: 172.16.0.174
      - SESSION_EXPIRATION_MS: 43200000
      - NODE_ENV: production
      - USE_HTTPS: false
      plugins:
      - auth: /var/vcap/packages/kibana-auth-plugin/kibana-auth-plugin.tar.gz
    syslog_forwarder:
      config:
      - file: /var/vcap/sys/log/kibana/kibana.stdout.log
        service: kibana
      - file: /var/vcap/sys/log/kibana/kibana.stderr.log
        service: kibana
  resource_pool: kibana
  templates:
  - name: kibana
    release: logsearch
  - name: syslog_forwarder
    release: logsearch
  - name: kibana-auth-plugin
    release: logsearch-for-cloudfoundry
- instances: 1
  name: ingestor
  networks:
  - name: default
    static_ips:
    - 172.16.0.172
  properties:
    cloudfoundry:
      api_endpoint: https://api.cftesting.footoo.org
      doppler_endpoint: wss://doppler.cftesting.footoo.org
      firehose_events: LogMessage
      firehose_password: admin-password
      firehose_user: admin
      skip_ssl_validation: true
    logstash_ingestor:
      debug: false
      relp:
        port: null
    syslog:
      host: 172.16.0.172
      port: 5514
    syslog_forwarder:
      config:
      - file: /var/vcap/sys/log/ingestor_syslog/ingestor_syslog.stdout.log
        service: ingestor_syslog
      - file: /var/vcap/sys/log/ingestor_syslog/ingestor_syslog.stderr.log
        service: ingestor_syslog
  resource_pool: ingestor
  templates:
  - name: ingestor_syslog
    release: logsearch
  - name: syslog_forwarder
    release: logsearch
  - name: ingestor_cloudfoundry-firehose
    release: logsearch-for-cloudfoundry
- instances: 0
  name: ingestor-bosh-nats
  networks:
  - name: default
    static_ips: []
  properties:
    logstash_ingestor:
      syslog:
        port: 514
    syslog_forwarder:
      config:
      - file: /var/vcap/sys/log/nats_to_syslog/nats_to_syslog.stdout.log
        service: nats_to_syslog
      - file: /var/vcap/sys/log/nats_to_syslog/nats_to_syslog.stderr.log
        service: nats_to_syslog
  resource_pool: ingestor
  templates:
  - name: ingestor_syslog
    release: logsearch
  - name: nats_to_syslog
    release: logsearch
  - name: syslog_forwarder
    release: logsearch
- instances: 1
  name: parser
  networks:
  - name: default
  properties:
    logstash_parser:
      debug: false
      deployment_dictionary:
      - /var/vcap/packages/logsearch-config/deployment_lookup.yml
      - /var/vcap/jobs/parser-config-lfc/config/deployment_lookup.yml
      deployment_name:
        cf: fcf-hitnic-dev
        diego: fcf-hitnic-dev-diego
      elasticsearch:
        index: logs-%{[@metadata][index]}-%{+YYYY.MM.dd}
        index_type: '%{@type}'
      filters:
      - logsearch-for-cf: /var/vcap/packages/logsearch-config-logstash-filters/logstash-filters-default.conf
    syslog_forwarder:
      config:
      - file: /var/vcap/sys/log/elasticsearch/elasticsearch.stdout.log
        service: elasticsearch
      - file: /var/vcap/sys/log/elasticsearch/elasticsearch.stderr.log
        service: elasticsearch
      - file: /var/vcap/sys/log/parser/parser.stdout.log
        service: parser
      - file: /var/vcap/sys/log/parser/parser.stderr.log
        service: parser
  resource_pool: parser
  templates:
  - name: parser
    release: logsearch
  - name: elasticsearch
    release: logsearch
  - name: syslog_forwarder
    release: logsearch
  - name: parser-config-lfc
    release: logsearch-for-cloudfoundry
  update:
    max_in_flight: 4
    serial: false
- instances: 1
  name: ls-router
  networks:
  - default:
    - gateway
    - dns
    name: default
    static_ips:
    - 172.16.0.180
  properties:
    haproxy:
      cluster_monitor:
        backend_servers:
        - 172.16.0.173
      ingestor:
        backend_servers:
        - 172.16.0.172
      kibana:
        backend_servers:
        - 172.16.0.176
      syslog_server: 172.16.0.173
  resource_pool: haproxy
  templates:
  - name: haproxy
    release: logsearch
- instances: 1
  lifecycle: errand
  name: enable_shard_allocation
  networks:
  - name: default
  properties:
    enable_shard_allocation:
      elasticsearch:
        master_node: 172.16.0.171
  resource_pool: errand
  templates:
  - name: enable_shard_allocation
    release: logsearch
- instances: 1
  lifecycle: errand
  name: smoke-tests
  networks:
  - name: default
  properties:
    smoke_tests:
      elasticsearch_master:
        host: 172.16.0.171
      syslog_ingestor:
        host: 172.16.0.180
        port: 5514
  release: logsearch
  resource_pool: errand
  templates:
  - name: smoke-tests
    release: logsearch
- instances: 1
  lifecycle: errand
  name: create-uaa-client
  networks:
  - name: default
  properties:
    create-uaa-client:
      cloudfoundry:
        system_domain: cftesting.footoo.org
        uaa_admin_client_id: admin
        uaa_admin_client_secret: admin-secret
      kibana_domain: elktesting.footoo.org
      oauth2_client_id: my_kibana_client_id
      oauth2_client_secret: my_kibana_client_password
  release: logsearch-for-cloudfoundry
  resource_pool: errand
  templates:
  - name: create-uaa-client
    release: logsearch-for-cloudfoundry
- instances: 1
  lifecycle: errand
  name: upload-kibana-objects
  networks:
  - name: default
  properties:
    cloudfoundry:
      firehose_events: LogMessage
    elasticsearch:
      host: 172.16.0.171
      port: 9200
    kibana_objects:
      upload_data_files: []
      upload_predefined_kibana_objects: true
  release: logsearch-for-cloudfoundry
  resource_pool: errand
  templates:
  - name: upload-kibana-objects
    release: logsearch-for-cloudfoundry
name: logsearch
networks:
- name: default
  subnets:
  - cloud_properties:
      name: fcf-testing
    dns:
    - 202.118.224.100
    gateway: 172.16.0.1
    range: 172.16.0.0/16
    reserved:
    - 172.16.0.0 - 172.16.0.170
    static:
    - 172.16.0.171 - 172.16.0.180
  type: manual
properties:
  curator:
    elasticsearch:
      host: 172.16.0.171
      port: 9200
  elasticsearch:
    cluster_name: logsearch
    exec: null
    master_hosts:
    - 172.16.0.171
  elasticsearch_config:
    elasticsearch:
      host: 172.16.0.171
    templates:
    - shards-and-replicas: /var/vcap/jobs/elasticsearch_config/index-templates/shards-and-replicas.json
    - index-settings: /var/vcap/jobs/elasticsearch_config/index-templates/index-settings.json
    - index-mappings: /var/vcap/jobs/elasticsearch_config/index-templates/index-mappings.json
  kibana:
    elasticsearch:
      host: 172.16.0.171
      port: 9200
  logstash_ingestor:
    debug: false
  logstash_parser:
    debug: false
  nats_to_syslog:
    nats:
      machines:
      - 172.16.0.12
      password: nats-password
      port: 4222
      subject: '>'
      user: nats-user
    syslog:
      host: 127.0.0.1
      port: 514
  redis:
    host: 172.16.0.174
  syslog_forwarder:
    host: 172.16.0.173
    port: 514
releases:
- name: logsearch
  version: latest
- name: logsearch-for-cloudfoundry
  version: latest
resource_pools:
- cloud_properties:
    cpu: 2
    disk: 8192
    ram: 4096
  env:
    bosh:
      password: $6$4gDD3aV0rdqlrKC$2axHCxGKIObs6tAmMTqYCspcdvQXh3JJcvWOY2WGb4SrdXtnCyNaWlrf3WEqvYR2MYizEGp3kMmbpwBC6jsHt0
  name: elasticsearch_master
  network: default
  stemcell:
    name: bosh-vsphere-esxi-ubuntu-trusty-go_agent
    version: latest
- cloud_properties:
    cpu: 2
    disk: 8192
    ram: 4096
  env:
    bosh:
      password: $6$4gDD3aV0rdqlrKC$2axHCxGKIObs6tAmMTqYCspcdvQXh3JJcvWOY2WGb4SrdXtnCyNaWlrf3WEqvYR2MYizEGp3kMmbpwBC6jsHt0
  name: elasticsearch_data
  network: default
  stemcell:
    name: bosh-vsphere-esxi-ubuntu-trusty-go_agent
    version: latest
- cloud_properties:
    cpu: 2
    disk: 8192
    ram: 4096
  env:
    bosh:
      password: $6$4gDD3aV0rdqlrKC$2axHCxGKIObs6tAmMTqYCspcdvQXh3JJcvWOY2WGb4SrdXtnCyNaWlrf3WEqvYR2MYizEGp3kMmbpwBC6jsHt0
  name: ingestor
  network: default
  stemcell:
    name: bosh-vsphere-esxi-ubuntu-trusty-go_agent
    version: latest
- cloud_properties:
    cpu: 2
    disk: 4096
    ram: 2048
  env:
    bosh:
      password: $6$4gDD3aV0rdqlrKC$2axHCxGKIObs6tAmMTqYCspcdvQXh3JJcvWOY2WGb4SrdXtnCyNaWlrf3WEqvYR2MYizEGp3kMmbpwBC6jsHt0
  name: queue
  network: default
  stemcell:
    name: bosh-vsphere-esxi-ubuntu-trusty-go_agent
    version: latest
- cloud_properties:
    cpu: 2
    disk: 8192
    ram: 4096
  env:
    bosh:
      password: $6$4gDD3aV0rdqlrKC$2axHCxGKIObs6tAmMTqYCspcdvQXh3JJcvWOY2WGb4SrdXtnCyNaWlrf3WEqvYR2MYizEGp3kMmbpwBC6jsHt0
  name: parser
  network: default
  stemcell:
    name: bosh-vsphere-esxi-ubuntu-trusty-go_agent
    version: latest
- cloud_properties:
    cpu: 2
    disk: 8192
    ram: 4096
  env:
    bosh:
      password: $6$4gDD3aV0rdqlrKC$2axHCxGKIObs6tAmMTqYCspcdvQXh3JJcvWOY2WGb4SrdXtnCyNaWlrf3WEqvYR2MYizEGp3kMmbpwBC6jsHt0
  name: kibana
  network: default
  stemcell:
    name: bosh-vsphere-esxi-ubuntu-trusty-go_agent
    version: latest
- cloud_properties:
    cpu: 2
    disk: 8192
    ram: 4096
  env:
    bosh:
      password: $6$4gDD3aV0rdqlrKC$2axHCxGKIObs6tAmMTqYCspcdvQXh3JJcvWOY2WGb4SrdXtnCyNaWlrf3WEqvYR2MYizEGp3kMmbpwBC6jsHt0
  name: maintenance
  network: default
  stemcell:
    name: bosh-vsphere-esxi-ubuntu-trusty-go_agent
    version: latest
- cloud_properties:
    cpu: 2
    disk: 8192
    ram: 4096
  env:
    bosh:
      password: $6$4gDD3aV0rdqlrKC$2axHCxGKIObs6tAmMTqYCspcdvQXh3JJcvWOY2WGb4SrdXtnCyNaWlrf3WEqvYR2MYizEGp3kMmbpwBC6jsHt0
  name: cluster_monitor
  network: default
  stemcell:
    name: bosh-vsphere-esxi-ubuntu-trusty-go_agent
    version: latest
- cloud_properties:
    cpu: 2
    disk: 8192
    ram: 4096
  env:
    bosh:
      password: $6$4gDD3aV0rdqlrKC$2axHCxGKIObs6tAmMTqYCspcdvQXh3JJcvWOY2WGb4SrdXtnCyNaWlrf3WEqvYR2MYizEGp3kMmbpwBC6jsHt0
  name: haproxy
  network: default
  stemcell:
    name: bosh-vsphere-esxi-ubuntu-trusty-go_agent
    version: latest
- cloud_properties:
    cpu: 2
    disk: 8192
    ram: 1024
  env:
    bosh:
      password: $6$4gDD3aV0rdqlrKC$2axHCxGKIObs6tAmMTqYCspcdvQXh3JJcvWOY2WGb4SrdXtnCyNaWlrf3WEqvYR2MYizEGp3kMmbpwBC6jsHt0
  name: errand
  network: default
  stemcell:
    name: bosh-vsphere-esxi-ubuntu-trusty-go_agent
    version: latest
update:
  canaries: 1
  canary_watch_time: 30000-600000
  max_errors: 1
  max_in_flight: 1
  serial: false
  update_watch_time: 5000-600000