jakubhajek

# Maintainer: Jakub Hajek, jakub.hajek@cometari.com
#
# docker stack deploy -c stack-elastic.yml elastic
#
# The stack creates Elasticsearch cluster consiting of
# - 3 dedicated master nodes in order to keep quorum
# - 4 dedicated data nodes to manage CRUD,
# - 2 coordination node acting like kind of load balancer in multi instance environments
#
#

jakubhajek

# docker stack deploy -c stack-nfs.yml nfs
#
# This is an example compose file with data volume mounted from remote server via NFS protocol


version: "3.7"

services:
  nginx:
    image: nginx:1.17

jakubhajek

We use Logstash to transform specific business logs that don't contain any unique value which could be used to calculate fingerprint in order to avoid duplicates. That's why we calculate fingerprints with the entire document/log. 

That approach works correctly for the same document which has been sent once again to Logstash we received the same fingerprints which are expecting behavior. 

Please see the below steps to replicate that issue.
I attached logstash.conf which is a simple pipeline calculating fingerprint and saving the output in a file. 

1. Run Logstash instance:

docker run -p 41311:41311 -v ${PWD}/pipeline/:/usr/share/logstash/pipeline  logstash:6.8.4

jakubhajek

# Maintainer: Jakub Hajek, jakub.hajek@cometari.com
#
# docker stack deploy -c stack-elastic.yml elastic
#
# The stack creates Elasticsearch cluster consiting of
# - 3 dedicated master nodes in order to keep quorum
# - 4 dedicated data nodes to manage CRUD,
#
# Docker compose file to easily deploy Elasticsearch cluster 7.x on Docker Swarm cluster.

jakubhajek

# docker stack deploy -c stack-tr-main.yml traefik --prune

version: "3.7"
services:
  main:
    image: traefik:v2.1.2
    healthcheck:
      test: wget --quiet --tries=1 --spider https://traefik.labs.cometari.eu/ping || exit 1
      interval: 3s
      timeout: 1s

jakubhajek

[http]
    [http.middlewares]
         [http.middlewares.https-redirect.redirectscheme]
              scheme = "https"
              permanent = true
         [http.middlewares.security-headers.headers]
              # CORS
              AccessControlAllowMethods = ["GET", "OPTIONS", "PUT"]
              AccessControlAllowOrigin = "origin-list-or-null"

jakubhajek

# docker stack deploy -c stack-app.yml app --with-registry-auth --prune
version: "3.7"

services:
  backend:
    image: jakubhajek/nodejs-backend:latest
    healthcheck:
      test: wget --quiet --tries=1 --spider http://localhost:3000/ || exit 1
      interval: 3s
      timeout: 1s

jakubhajek

# docker stack deploy -c stack-canary.yml canary --with-registry-auth --prune
version: "3.7"

services:
  app1:
    image: jakubhajek/app1-node:v1
    healthcheck:
      test: wget --quiet --tries=1 --spider http://localhost:3000/ || exit 1
      interval: 3s
      timeout: 1s

jakubhajek

http:
  services:
    canary:
      weighted:
        services:
          # Load balancing between Traefik services
          - name: app1_svc@docker
            weight: 1
          - name: app2_svc@docker
            weight: 5

jakubhajek

log:
  level: info
  format: json

accessLog:
  format: json
  bufferingSize: 5

providers:
  docker: