docker-compose.yml

version: '3'

services:
  fluentd:
    build:
        context: .
    environment:
      FLUENTD_OPT: -v
    ports:
      - '24224:24224'
    depends_on:
      - localstack

  localstack:
    image: localstack/localstack:0.8.7
    ports:
      - "4572:4572"
    environment:
      - SERVICES=s3
    volumes:
      - /tmp/localstack:/tmp/localstack
      - /var/run/docker.sock:/var/run/docker.sock

Dockerfile

# or v1.0-onbuild
FROM fluent/fluentd:v1.2.6-onbuild

# below RUN includes plugin as examples elasticsearch is not required
# you may customize including plugins as you wish

RUN apk add --update --virtual .build-deps \
        sudo build-base ruby-dev \
 && sudo gem install \
        fluent-plugin-s3 \
 && sudo gem sources --clear-all \
 && apk del .build-deps \
 && rm -rf /var/cache/apk/* \
           /home/fluent/.gem/ruby/2.4.0/cache/*.gem

fluent.conf

## built-in TCP input
## $ echo <json> | fluent-cat <tag>
<source>
  @type forward
  @id forward_input
</source>

<match debug.test>
  @type s3

  s3_endpoint http://localstack:4572
  force_path_style true
  aws_key_id dummy
  aws_sec_key dummy

  s3_bucket sample-bucket
  s3_region ap-northeast-1

  path ${tag}/dt=%Y%m%d/
  s3_object_key_format %{path}%{time_slice}_%{index}.%{file_extension}

  <buffer tag,time>
    @type file
    path /fluentd/log

    timekey 3600 # 1 hour partition
    timekey_wait 10m
    timekey_use_utc true # use utc

    flush_mode interval
    flush_interval 1m
  </buffer>

  <format>
    @type json
  </format>
</match>

$ mkdir plugins # on-buildの処理で必要になる
$ docker-compose up localstack fluentd

# 適当にデータを投げる
$ echo '{"json":"message"}' | fluent-cat debug.test

# 強制flushするか、fluentdのdebug logでflushされることを確認する

# fileを確認
$ aws --endpoint-url=http://localhost:4572 s3 ls s3://sample-bucket/debug.test/