Libbeat diskqueue cannot handle multi-byte message correctly.

serialize_test.go

package diskqueue

import (
	"encoding/json"
	"testing"

	"github.com/elastic/beats/v7/libbeat/beat"
	"github.com/elastic/beats/v7/libbeat/common"
	"github.com/elastic/beats/v7/libbeat/publisher"
	"github.com/stretchr/testify/assert"
)

func TestSerialize(t *testing.T) {

	asciiOnly := "{\"name\": \"Momotaro\"}"
	multiBytes := "{\"name\": \"桃太郎\"}"

	encoder := newEventEncoder()
	event := publisher.Event{
		Content: beat.Event{
			Fields: common.MapStr{
				"ascii_only":  asciiOnly,
				"multi_bytes": multiBytes,
			},
		},
	}
	serialized, err := encoder.encode(&event)
	if err != nil {
		panic(err)
	}

	/*
		Use json.Unmarshal to decode the serialized bytes.
		The multibyte part gets parsed correctly.
	*/
	decodedEntry := entry{}
	json.Unmarshal(serialized, &decodedEntry)
	decodedAsciiOnly, err := decodedEntry.Fields.GetValue("ascii_only")
	assert.Equal(t, asciiOnly, decodedAsciiOnly)

	decodedMultiBytes, err := decodedEntry.Fields.GetValue("multi_bytes")
	assert.Equal(t, multiBytes, decodedMultiBytes)

	/*
		Use decoder to decode the serialized bytes.
		The multibyte part gets garbled.
	*/
	decoder := newEventDecoder()
	buf := decoder.Buffer(len(serialized))
	copy(buf, serialized)
	decoded, err := decoder.Decode()
	if err != nil {
		panic(err)
	}

	decodedAsciiOnly, err = decoded.Content.Fields.GetValue("ascii_only")
	assert.Equal(t, asciiOnly, decodedAsciiOnly)

	decodedMultiBytes, err = decoded.Content.Fields.GetValue("multi_bytes")
	assert.Equal(t, multiBytes, decodedMultiBytes)

}

test_result.txt

Running tool: /usr/local/go/bin/go test -timeout 30m -tags integration -run ^TestSerialize$ github.com/elastic/beats/v7/libbeat/publisher/queue/diskqueue

--- FAIL: TestSerialize (0.00s)
    /Users/koji/dev/elastic/beats/libbeat/publisher/queue/diskqueue/serialize_test.go:64: 
        	Error Trace:	serialize_test.go:64
        	Error:      	Not equal: 
        	            	expected: "{\"name\": \"桃太郎\"}"
        	            	actual  : "{\"name\": \"太郎\\\"}\"}"
        	            	
        	            	Diff:
        	            	--- Expected
        	            	+++ Actual
        	            	@@ -1 +1 @@
        	            	-{"name": "桃太郎"}
        	            	+{"name": "太郎\"}"}
        	Test:       	TestSerialize
FAIL
FAIL	github.com/elastic/beats/v7/libbeat/publisher/queue/diskqueue	0.686s
FAIL