compose-minio-nas.yml

# https://docs.min.io/docs/minio-gateway-for-nas.html
# https://gist.github.com/pascalandy/1771c666aab221a9be5ea4bb6dbce70b

version: '3.7'

networks:
  ntw_minio:
    external: true

secrets:
  secret_key:
    external: true
  access_key:
    external: true

services:

  minio1:
    image: minio/minio:RELEASE.2020-01-25T02-50-51Z
    hostname: minio1
    volumes:
      - /mnt/clusterdata/minio/container/vol:/container/vol
    ports:
      - "9001:9000"
    networks:
      - ntw_minio
    deploy:
      restart_policy:
        delay: 10s
        max_attempts: 10
        window: 60s
      placement:  
        constraints:
          - node.labels.minio1==true
    command: gateway nas /container/vol
    secrets:
      - secret_key
      - access_key
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
      interval: 30s
      timeout: 20s
      retries: 3

  minio2:
    image: minio/minio:RELEASE.2020-01-25T02-50-51Z
    hostname: minio2
    volumes:
      - /mnt/clusterdata/minio/container/vol:/container/vol
    ports:
      - "9002:9000"
    networks:
      - ntw_minio
    deploy:
      restart_policy:
        delay: 10s
        max_attempts: 10
        window: 60s
      placement:
        constraints:
          - node.labels.minio2==true
    command: gateway nas /container/vol
    secrets:
      - secret_key
      - access_key
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
      interval: 30s
      timeout: 20s
      retries: 3

  minio3:
    image: minio/minio:RELEASE.2020-01-25T02-50-51Z
    hostname: minio3
    volumes:
      - /mnt/clusterdata/minio/container/vol:/container/vol
    ports:
      - "9003:9000"
    networks:
      - ntw_minio
    deploy:
      restart_policy:
        delay: 10s
        max_attempts: 10
        window: 60s
      placement:
        constraints:
          - node.labels.minio3==true
    command: gateway nas /container/vol
    secrets:
      - secret_key
      - access_key
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
      interval: 30s
      timeout: 20s
      retries: 3

compose_minio.yaml

# https://gist.github.com/pascalandy/1771c666aab221a9be5ea4bb6dbce70b

version: '3.7'

networks:
  ntw_minio:
    external: true

secrets:
  secret_key:
    external: true
  access_key:
    external: true

services:

  minio1:
    image: minio/minio:RELEASE.2020-01-25T02-50-51Z
    hostname: minio1
    volumes:
      - /mnt/clusterdata/minio/export1:/export1
      - /mnt/clusterdata/minio/export2:/export2
    ports:
      - "9001:9000"
    networks:
      - ntw_minio
    deploy:
      restart_policy:
        delay: 10s
        max_attempts: 10
        window: 60s
      placement:
        constraints:
          - node.labels.minio1==true
    command: server http://minio{1...3}/export{1...2}
    secrets:
      - secret_key
      - access_key
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
      interval: 30s
      timeout: 20s
      retries: 3

  minio2:
    image: minio/minio:RELEASE.2020-01-25T02-50-51Z
    hostname: minio2
    volumes:
      - /mnt/clusterdata/minio/export1:/export1
      - /mnt/clusterdata/minio/export2:/export2
    ports:
      - "9002:9000"
    networks:
      - ntw_minio
    deploy:
      restart_policy:
        delay: 10s
        max_attempts: 10
        window: 60s
      placement:
        constraints:
          - node.labels.minio2==true
    command: server http://minio{1...3}/export{1...2}
    secrets:
      - secret_key
      - access_key
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
      interval: 30s
      timeout: 20s
      retries: 3

  minio3:
    image: minio/minio:RELEASE.2020-01-25T02-50-51Z
    hostname: minio3
    volumes:
      - /mnt/clusterdata/minio/export1:/export1
      - /mnt/clusterdata/minio/export2:/export2
    ports:
      - "9003:9000"
    networks:
      - ntw_minio
    deploy:
      restart_policy:
        delay: 10s
        max_attempts: 10
        window: 60s
      placement:
        constraints:
          - node.labels.minio3==true
    command: server http://minio{1...3}/export{1...2}
    secrets:
      - secret_key
      - access_key
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
      interval: 30s
      timeout: 20s
      retries: 3

Context

Minio (the solution) must sync data of a directory /mnt/clusterdata/minio/container/vol of my 3 nodes.

Requirements / use case

• The solution must sync the data in /mnt/clusterdata/minio/container/vol
• I have a 3 nodes set up on docker swarm
• Our docker containers mount there volume to /mnt/clusterdata/minio/container/vol
• My stateful apps will mount their data at
  • /mnt/clusterdata/minio/container/vol/app1/bin
  • /mnt/clusterdata/minio/container/vol/app2/bin
  • /mnt/clusterdata/minio/container/vol/appN/bin
• Everything needs to happen via the CLI (no GUI operation)
• No need external sync to the cloud (like AWS s3)
• I expect the data to flow via the swarm ingress (not via public traffic)
• The orchestrator (Docker Swarm in my case) does not need to care about "volume" or a plug-in.
• The orchestrator will run Minio services and Minio will sync stuff on our nodes

Gist

https://gist.github.com/pascalandy/1771c666aab221a9be5ea4bb6dbce70b

Cluster context

Minio must have a minimum of 4 drives. In our case, we run 6 drives on 3 nodes.

1) on all nodes:

### install utilities
apk update && apk upgrade && apk add --no-cache               \
    nano bash git curl wget unzip openssl tar ca-certificates && \

### create stateful directory
mkdir -pv /mnt/clusterdata/minio/container/vol

2) on node: manager1

### copy compose file

curl -L https://gist.githubusercontent.com/pascalandy/1771c666aab221a9be5ea4bb6dbce70b/raw/4302059848a887288ea1488b1ad1b86590e8afb5/compose-minio-nas.yml \
-o compose-minio-nas.yml && sleep 1 && \

### network

NTW_MINIO="ntw_minio"

if [ ! "$(docker network ls --filter name=${NTW_MINIO} -q)" ]; then
   docker network create --driver overlay --attachable ${NTW_MINIO}
else
   echo "Network: ${NTW_MINIO} already exist!"
   fct_count1
fi

docker network ls | grep "ntw_"; echo;

### secrets

echo "user_eW9afmXBOp_br8T" | docker secret create access_key -
echo "pass_C1mvqM5w4iq4RjgeJsLvT5xId_qUCX_qVKi" | docker secret create secret_key -
   #Access Key: min 3 chars, max 20 chars
   #Secret Key: min 8 chars, max 40 chars

### constraints

docker node update --label-add minio1=true manager1
docker node update --label-add minio2=true manager2
docker node update --label-add minio3=true manager3

### deploy

docker stack deploy stk_minio --compose-file=compose-minio-nas.yml

### see stack

docker service ls

3) validate that files are synching

WIP: 2020-01-30_12h20

on node1:

touch /mnt/clusterdata/minio/container/vol/testfile

on node2

ls /mnt/clusterdata/minio/container/vol

on node3:

ls /mnt/clusterdata/minio/container/vol

Official docs

• https://docs.min.io/docs/minio-gateway-for-nas.html
• https://docs.min.io/docs/deploy-minio-on-docker-swarm.html
• https://docs.min.io/docs/minio-client-complete-guide.html#mirror