Repro of bug in sidekick - minio caching server interaction.

docker-compose.yml

version: "3.3"

services:
  server:
    image: quay.io/minio/minio
    ports:
      - "9005:9005"
    command: server /objects --address :9005
    environment:
      - MINIO_ROOT_USER=user
      - MINIO_ROOT_PASSWORD=password
  cache:
    image: quay.io/minio/minio
    ports:
      - "9002:9002"
    command: server /objects --address :9002
    environment:
      - MINIO_ROOT_USER=cacheuser
      - MINIO_ROOT_PASSWORD=cachepassword
  sidekick:
    image: minio/sidekick
    ports:
      - "9092:9092"
    environment:
      SIDEKICK_CACHE_ENDPOINT: "http://cache:9002"
      SIDEKICK_CACHE_ACCESS_KEY: "cacheuser"
      SIDEKICK_CACHE_SECRET_KEY: "cachepassword"
      SIDEKICK_CACHE_BUCKET: "cachebucket"
      SIDEKICK_CACHE_MIN_SIZE: 1B
      SIDEKICK_CACHE_HEALTH_DURATION: "120"
    command: --health-path=/minio/health/ready --health-duration=120 --address :9092 --log http://server:9005
    depends_on:
      - server
      - cache
  dockerapp:
    image: ubuntu
    tty:  true
    stdin_open: true
    command: bash -c "
        apt update;
        apt install wget -y;

        wget https://dl.min.io/client/mc/release/linux-amd64/mc -P /usr/bin;
        chmod +x /usr/bin/mc;

        echo 'Create buckets on server and caching server';
        mc mb cache/cachebucket;
        mc mb server/data;

        echo 'Create a downloadable file. This file is bigger than SIDEKICK_CACHE_MIN_SIZE';
        echo 'testfile' > testfile.txt;
        mc cp testfile.txt server/data/testfile.txt;

        echo 'Now try downloading it via the sidekick that uses cache';

        while true; do
          mc cp sidekick/data/testfile.txt downloaded.content ;
          sleep 60;
        done;
        "
    environment:
      MC_HOST_cache: "http://cacheuser:cachepassword@cache:9002"
      MC_HOST_sidekick: "http://user:password@sidekick:9092"
      MC_HOST_server: "http://user:password@server:9005"
    depends_on:
      - sidekick

output.txt

dockerapp_1  | Create buckets on server and caching server
dockerapp_1  | mc: Configuration written to `/root/.mc/config.json`. Please update your access credentials.
dockerapp_1  | mc: Successfully created `/root/.mc/share`.
dockerapp_1  | mc: Initialized share uploads `/root/.mc/share/uploads.json` file.
dockerapp_1  | mc: Initialized share downloads `/root/.mc/share/downloads.json` file.
dockerapp_1  | Bucket created successfully `cache/cachebucket`.
dockerapp_1  | Bucket created successfully `server/data`.
dockerapp_1  | Create a downloadable file. This file is bigger than SIDEKICK_CACHE_MIN_SIZE
testfile.txt:  9 B / 9 B  841 B/s 0sNow try downloading it via the sidekick that uses cache
sidekick_1   |  TRACE: 07:09:11.173 [200 OK]  http://server:9005 GET /data/?location= 172.19.0.5        3.775ms      ↑ 77 B ↓ 496 B
sidekick_1   |  TRACE: 07:09:11.177 [200 OK]  http://server:9005 HEAD /data/testfile.txt 172.19.0.5        2.604ms      ↑ 77 B ↓ 434 B
sidekick_1   |  TRACE: 07:09:11.180 [200 OK]  http://server:9005 HEAD /data/testfile.txt 172.19.0.5        2.493ms      ↑ 77 B ↓ 434 B
sidekick_1   |  TRACE: 07:09:11.183 [200 OK]  http://server:9005 HEAD /data/testfile.txt 172.19.0.5        2.521ms      ↑ 77 B ↓ 434 B
sidekick_1   |  TRACE: 07:09:11.189 [200 OK]  http://server:9005 HEAD /data/testfile.txt 172.19.0.5        2.889ms      ↑ 77 B ↓ 434 B
sidekick_1   |  TRACE: 07:09:11.192 [200 OK]  http://server:9005 HEAD /data/testfile.txt 172.19.0.5        2.253ms      ↑ 77 B ↓ 434 B
sidekick_1   |  TRACE: 07:09:11.196 [200 OK]  http://server:9005 HEAD /data/testfile.txt 172.19.0.5        2.255ms      ↑ 77 B ↓ 434 B
sidekick_1   |  TRACE: 07:09:11.199 [200 OK]  http://server:9005 GET /data/testfile.txt 172.19.0.5        2.735ms      ↑ 86 B ↓ 440 B
cache_1      |
cache_1      | API: PutObject(bucket=cachebucket, object=70/f9/70f9875b2bb57d222b0206ff467fab6932ecc89796543cb2587820a9ed236a8d)
cache_1      | Time: 07:09:11 UTC 09/11/2021
cache_1      | DeploymentID: 2c1e5877-7492-4ce0-89df-fef9fa1c53ee
cache_1      | RequestID: 16A3B2DCBEB28543
cache_1      | RemoteHost: 172.19.0.4
cache_1      | Host: cache:9002
cache_1      | UserAgent: MinIO (linux; amd64) minio-go/v7.0.11 /sidekick/1.0.0
cache_1      | Error: unexpected EOF (*errors.errorString)
cache_1      |        5: cmd/fs-v1-helpers.go:339:cmd.fsCreateFile()
cache_1      |        4: cmd/fs-v1.go:1123:cmd.(*FSObjects).putObject()
cache_1      |        3: cmd/fs-v1.go:1048:cmd.(*FSObjects).PutObject()
cache_1      |        2: cmd/object-handlers.go:1718:cmd.objectAPIHandlers.PutObjectHandler()
cache_1      |        1: net/http/server.go:2049:http.HandlerFunc.ServeHTTP()