Read compressed docker logs without decompressing them all to disks simultaneously. (see https://github.com/moby/moby/issues/41678)

read_logs.go

package main

import (
	"compress/gzip"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"os"
	"strconv"

	"github.com/docker/docker/client"
)

type dockerLine struct {
	Log    string `json:"log"`
	Stream string `json:"stream"`
	time   string
}

func (l dockerLine) print() error {
	stream := os.Stderr
	if l.Stream == "stdout" {
		stream = os.Stdout
	}
	_, err := stream.WriteString(l.Log)
	return err
}

type parsedDockerLine struct {
	line dockerLine
	err  error
}

func readDockerLogLines(rawContents io.Reader) chan parsedDockerLine {
	ch := make(chan parsedDockerLine)
	go func() {
		dec := json.NewDecoder(rawContents)
		for {
			var line dockerLine
			err := dec.Decode(&line)
			if err != nil {
				ch <- parsedDockerLine{dockerLine{}, err}
				break
			}
			ch <- parsedDockerLine{line, nil}
		}
		close(ch)
	}()
	return ch
}

func printLines(lines chan parsedDockerLine) error {
	exitError := error(nil)
	for line := range lines {
		if line.err != nil {
			if line.err != io.EOF {
				exitError = line.err
			}
		} else {
			if printErr := line.line.print(); printErr != nil {
				exitError = printErr
				break
			}
		}
	}
	return exitError
}

type logOpts struct {
	MaxFile string `json:"max-file"`
}
type dockerConf struct {
	DataRoot string  `json:"data-root"`
	LogOpts  logOpts `json:"log-opts"`
}

type logConf struct {
	logFilePathBase string
	maxFile         uint64
}

func readDockerConf(containerID string) (conf logConf, err error) {
	cli, err := client.NewClientWithOpts(client.WithVersion("1.40"), client.FromEnv)
	if err != nil {
		return
	}

	containerInfo, err := cli.ContainerInspect(context.Background(), containerID)
	if err != nil {
		return
	}

	conf.logFilePathBase = containerInfo.LogPath
	conf.maxFile = 1

	logConfig := containerInfo.HostConfig.LogConfig
	if logConfig.Type != "json-file" {
		err = fmt.Errorf(
			"Container '%s' uses the '%s' logging backend. Only 'json-file' is supported",
			containerInfo.Name, logConfig.Type)
		return
	}
	maxFileStr := logConfig.Config["max-file"]
	if maxFileStr != "" {
		conf.maxFile, err = strconv.ParseUint(maxFileStr, 10, 32)
	}
	return
}

func readAllLogFiles(conf logConf) chan parsedDockerLine {
	ch := make(chan parsedDockerLine)
	go func() {
		for i := conf.maxFile - 1; i > 0; i-- {
			logFilePath := fmt.Sprintf("%s.%d.gz", conf.logFilePathBase, i)
			fileObj, err := os.Open(logFilePath)
			if err != nil {
				ch <- parsedDockerLine{dockerLine{}, err}
				continue
			}
			rawContents, err := gzip.NewReader(fileObj)
			if err != nil {
				ch <- parsedDockerLine{dockerLine{}, err}
				continue
			}
			fileLines := readDockerLogLines(rawContents)
			for line := range fileLines {
				ch <- line
			}
		}
		fileObj, err := os.Open(conf.logFilePathBase)
		if err != nil {
			ch <- parsedDockerLine{dockerLine{}, err}
		} else {
			fileLines := readDockerLogLines(fileObj)
			for line := range fileLines {
				ch <- line
			}
		}
		close(ch)
	}()
	return ch
}

func main() {
	if len(os.Args) < 2 {
		log.Fatal("Usage: read-logs CONTAINER_ID")
	}
	containerID := os.Args[1]
	conf, err := readDockerConf(containerID)
	if err != nil {
		log.Fatal(err)
	}
	lines := readAllLogFiles(conf)
	exitError := printLines(lines)
	if exitError != nil {
		log.Fatal(exitError)
	}
}

This is a workaround for moby/moby#41678

How is this implemented @lovasoa , do you have a readme file?
It's been a while touching Docker logs but I'm guessing this overrides json-file option to use this logging driver plugin?

If I do :

docker plugin install <org/image> // what would be the image url for this plugin ?

Thank you so much, great work (been to Paris couple times - cool place and cool people)

The code here is not a docker plugin. It's a normal executable. Just install go and run this with go run.

Oh ok, so it still works in conjunction with json-file driver.
Thank you sir, appreciated your speedy reply.