docker-compose-site1.yml

version: '3.7'

# Settings and configurations that are common for all containers
x-minio-common: &minio-common
  image: quay.io/minio/minio:${RELEASE}
  command: server http://site1-minio{1...4}/data{1...2}
  environment:
    - MINIO_PROMETHEUS_AUTH_TYPE=public
    - CI=true

# starts 4 docker containers running minio server instances.
# using nginx reverse proxy, load balancing, you can access
# it through port 9000.
services:
  site1-minio1:
    <<: *minio-common
    hostname: site1-minio1
    volumes:
      - site1-data1-1:/data1
      - site1-data1-2:/data2

  site1-minio2:
    <<: *minio-common
    hostname: site1-minio2
    volumes:
      - site1-data2-1:/data1
      - site1-data2-2:/data2

  site1-minio3:
    <<: *minio-common
    hostname: site1-minio3
    volumes:
      - site1-data3-1:/data1
      - site1-data3-2:/data2

  site1-minio4:
    <<: *minio-common
    hostname: site1-minio4
    volumes:
      - site1-data4-1:/data1
      - site1-data4-2:/data2

  site1-nginx:
    image: nginx:1.19.2-alpine
    hostname: site1-nginx
    volumes:
      - ./nginx-site1.conf:/etc/nginx/nginx.conf:ro
    ports:
      - "9001:9001"
    depends_on:
      - site1-minio1
      - site1-minio2
      - site1-minio3
      - site1-minio4

## By default this config uses default local driver,
## For custom volumes replace with volume driver configuration.
volumes:
  site1-data1-1:
  site1-data1-2:
  site1-data2-1:
  site1-data2-2:
  site1-data3-1:
  site1-data3-2:
  site1-data4-1:
  site1-data4-2:

docker-compose-site2.yml

version: '3.7'

# Settings and configurations that are common for all containers
x-minio-common: &minio-common
  image: quay.io/minio/minio:${RELEASE}
  command: server http://site2-minio{1...4}/data{1...2}
  environment:
    - MINIO_PROMETHEUS_AUTH_TYPE=public
    - CI=true

# starts 4 docker containers running minio server instances.
# using nginx reverse proxy, load balancing, you can access
# it through port 9000.
services:
  site2-minio1:
    <<: *minio-common
    hostname: site2-minio1
    volumes:
      - site2-data1-1:/data1
      - site2-data1-2:/data2

  site2-minio2:
    <<: *minio-common
    hostname: site2-minio2
    volumes:
      - site2-data2-1:/data1
      - site2-data2-2:/data2

  site2-minio3:
    <<: *minio-common
    hostname: site2-minio3
    volumes:
      - site2-data3-1:/data1
      - site2-data3-2:/data2

  site2-minio4:
    <<: *minio-common
    hostname: site2-minio4
    volumes:
      - site2-data4-1:/data1
      - site2-data4-2:/data2

  site2-nginx:
    image: nginx:1.19.2-alpine
    hostname: site2-nginx
    volumes:
      - ./nginx-site2.conf:/etc/nginx/nginx.conf:ro
    ports:
      - "9002:9002"
    depends_on:
      - site2-minio1
      - site2-minio2
      - site2-minio3
      - site2-minio4

## By default this config uses default local driver,
## For custom volumes replace with volume driver configuration.
volumes:
  site2-data1-1:
  site2-data1-2:
  site2-data2-1:
  site2-data2-2:
  site2-data3-1:
  site2-data3-2:
  site2-data4-1:
  site2-data4-2:

nginx-site1.conf

user  nginx;
worker_processes  auto;

error_log  /var/log/nginx/error.log warn;
pid        /var/run/nginx.pid;

events {
    worker_connections  4096;
}

http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;

    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    access_log  /var/log/nginx/access.log  main;
    sendfile        on;
    keepalive_timeout  65;

    # include /etc/nginx/conf.d/*.conf;

    upstream minio {
        server site1-minio1:9000;
        server site1-minio2:9000;
        server site1-minio3:9000;
        server site1-minio4:9000;
    }

    server {
        listen       9001;
        listen  [::]:9001;
        server_name  localhost;

        # To allow special characters in headers
        ignore_invalid_headers off;
        # Allow any size file to be uploaded.
        # Set to a value such as 1000m; to restrict file size to a specific value
        client_max_body_size 0;
        # To disable buffering
        proxy_buffering off;
        proxy_request_buffering off;

        location / {
            proxy_set_header Host $http_host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;

            proxy_connect_timeout 300;
            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
            proxy_http_version 1.1;
            proxy_set_header Connection "";
            chunked_transfer_encoding off;

            proxy_pass http://minio;
        }
    }
}

nginx-site2.conf

user  nginx;
worker_processes  auto;

error_log  /var/log/nginx/error.log warn;
pid        /var/run/nginx.pid;

events {
    worker_connections  4096;
}

http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;

    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    access_log  /var/log/nginx/access.log  main;
    sendfile        on;
    keepalive_timeout  65;

    # include /etc/nginx/conf.d/*.conf;

    upstream minio {
        server site2-minio1:9000;
        server site2-minio2:9000;
        server site2-minio3:9000;
        server site2-minio4:9000;
    }

    server {
        listen       9002;
        listen  [::]:9002;
        server_name  localhost;

        # To allow special characters in headers
        ignore_invalid_headers off;
        # Allow any size file to be uploaded.
        # Set to a value such as 1000m; to restrict file size to a specific value
        client_max_body_size 0;
        # To disable buffering
        proxy_buffering off;
        proxy_request_buffering off;

        location / {
            proxy_set_header Host $http_host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;

            proxy_connect_timeout 300;
            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
            proxy_http_version 1.1;
            proxy_set_header Connection "";
            chunked_transfer_encoding off;

            proxy_pass http://minio;
        }
    }
}

reproducer.sh

#!/bin/bash

set -x

docker-compose -f docker-compose-site1.yaml rm -s -f
docker-compose -f docker-compose-site2.yaml rm -s -f
for volume in $(docker volume ls -q | grep minio); do
	docker volume rm ${volume}
done

cat >.env <<EOF
RELEASE=RELEASE.2023-08-29T23-07-35Z
EOF

docker-compose -f docker-compose-site1.yaml up -d
docker-compose -f docker-compose-site2.yaml up -d

sleep 30

mc alias set site1 http://site1-nginx:9001 minioadmin minioadmin --api s3v4
mc alias set site2 http://site2-nginx:9002 minioadmin minioadmin --api s3v4

mc ready site1/
mc ready site2/

mc admin replicate add site1 site2
mc mb site1/testbucket/
mc cp -r /usr/bin site1/testbucket/

sleep 1

failed_count_site1=$(./s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site1-nginx:9001 -bucket testbucket 2>&1 | grep FAILED | wc -l)
failed_count_site2=$(./s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l)

echo $failed_count_site1
echo $failed_count_site2

mc cp -r /usr/bin site1/testbucket/

sleep 1

failed_count_site1=$(./s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site1-nginx:9001 -bucket testbucket 2>&1 | grep FAILED | wc -l)
failed_count_site2=$(./s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l)

echo $failed_count_site1
echo $failed_count_site2

cat >.env <<EOF
RELEASE=dev
EOF

docker-compose -f docker-compose-site1.yaml up -d
docker-compose -f docker-compose-site2.yaml up -d

mc ready site1/
mc ready site2/

failed_count_site1=$(./s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site1-nginx:9001 -bucket testbucket 2>&1 | grep FAILED | wc -l)
failed_count_site2=$(./s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l)

echo $failed_count_site1
echo $failed_count_site2

failed_count_site1=$(./s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site1-nginx:9001 -bucket testbucket 2>&1 | grep FAILED | wc -l)
failed_count_site2=$(./s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l)

echo $failed_count_site1
echo $failed_count_site2