Skip to content

Instantly share code, notes, and snippets.

@nilium
Created April 7, 2017 15:18
Show Gist options
  • Save nilium/bc8a391979978e4b7b7b35cdc5ad9c79 to your computer and use it in GitHub Desktop.
Save nilium/bc8a391979978e4b7b7b35cdc5ad9c79 to your computer and use it in GitHub Desktop.
Zap protobuf marshaler -- maybe eventually document it, test it, put it in a repo.. 'til then, snippet.
package pbzap
import (
"encoding/base64"
"fmt"
"reflect"
"sort"
"strings"
"sync"
"github.com/golang/protobuf/proto"
"go.uber.org/zap/zapcore"
)
// Proto returns a zap Field for a protobuf message. The field is an object log marshaler, so is
// marshaled lazily.
func Proto(name string, value proto.Message) zapcore.Field {
return zapcore.Field{
Interface: protobufZapMarshaler{val: value, encType: true},
Key: name,
Type: zapcore.ObjectMarshalerType,
}
}
type protobufZapMarshaler struct {
val proto.Message
encType bool
}
var (
protoFieldBlacklist = []string{
// Insert blacklist fields here or something -- should really be controllable by
// something else. Probably something better than a table that's walked per-field.
// Could generate a regexp, but then there's a regexp in this code.
"password",
"private",
"secret",
"auth",
}
)
func isProtoBlacklistName(name string) bool {
name = strings.ToLower(name)
for _, forbidden := range protoFieldBlacklist {
if forbidden == name {
return true
} else if idx := strings.Index(name, forbidden); idx == -1 {
// No match
} else if idx == 0 || name[idx-1] == '_' || name[idx-1] == ':' {
return true
}
}
return false
}
func (m protobufZapMarshaler) MarshalLogObject(enc zapcore.ObjectEncoder) error {
v := reflect.ValueOf(m.val)
if m.val == nil || !v.IsValid() || v.IsNil() {
return nil
}
if v.Kind() == reflect.Interface {
v = v.Elem()
}
if v.Kind() == reflect.Ptr {
v = v.Elem()
}
typ := v.Type()
props := proto.GetProperties(typ)
if m.encType {
enc.AddString("@type", proto.MessageName(m.val))
}
for fid, prop := range props.Prop {
if prop == nil || isProtoBlacklistName(prop.OrigName) {
continue
}
fv := v.Field(fid)
if fv == reflect.Zero(fv.Type()) {
continue
}
// handle oneof fields -- logic borrowed from jsonpb
if typ.Field(fid).Tag.Get("protobuf_oneof") != "" {
container := fv.Elem().Elem()
fv = container.Field(0)
ft := container.Type().Field(0)
var oneofProp proto.Properties
oneofProp.Init(ft.Type, ft.Name, ft.Tag.Get("protobuf"), &ft)
prop = &oneofProp
}
if fv == reflect.Zero(fv.Type()) {
continue
}
key := prop.JSONName
if key == "" {
key = prop.OrigName
}
// Discard error
encodeZapKeyValue(key, prop, typ, fv, enc)
}
return nil
}
type protoMapZapMarshaler struct {
val reflect.Value
prop *proto.Properties
}
func (m protoMapZapMarshaler) MarshalLogObject(enc zapcore.ObjectEncoder) error {
if m.val.IsNil() {
return nil
}
type keyDesc struct {
key reflect.Value
name string
}
keys := make([]keyDesc, m.val.Len())
for i, key := range m.val.MapKeys() {
keys[i] = keyDesc{key: key, name: fmt.Sprint(key.Interface())}
}
sort.Slice(keys, func(i, j int) bool { return keys[i].name < keys[j].name })
for _, kp := range keys {
fv := m.val.MapIndex(kp.key)
if fv.Kind() == reflect.Interface {
fv = fv.Elem()
}
if isProtoBlacklistName(kp.name) || !fv.IsValid() || fv == reflect.Zero(fv.Type()) {
continue
}
// Discard error
encodeZapKeyValue(kp.name, m.prop, m.val.Type(), fv, enc)
}
return nil
}
type protoSliceZapMarshaler struct {
slice reflect.Value
prop *proto.Properties
}
func (m protoSliceZapMarshaler) MarshalLogArray(enc zapcore.ArrayEncoder) error {
len := m.slice.Len()
switch m.slice.Type().Elem().Kind() {
case reflect.Slice:
if m.slice.Type().Elem().Elem().Kind() != reflect.Uint8 {
return nil
}
// Can only have repeated slices of []byte in protobuf
for i := 0; i < len; i++ {
v := m.slice.Index(i)
enc.AppendString(base64.StdEncoding.EncodeToString(v.Bytes()))
}
case reflect.Uint32, reflect.Uint64:
for i := 0; i < len; i++ {
enc.AppendUint64(m.slice.Index(i).Uint())
}
case reflect.Int32, reflect.Int64:
if m.prop.Enum != "" {
mapping := getEnumValueMapping(m.prop.Enum)
for i := 0; i < len; i++ {
v := int32(m.slice.Index(i).Int())
if name := mapping[v]; name != "" {
enc.AppendString(name)
} else {
enc.AppendInt32(v) // unidentified enum value
}
}
break
}
for i := 0; i < len; i++ {
enc.AppendInt64(m.slice.Index(i).Int())
}
case reflect.String:
for i := 0; i < len; i++ {
enc.AppendString(m.slice.Index(i).String())
}
case reflect.Ptr:
for i := 0; i < len; i++ {
v := m.slice.Index(i)
if !v.IsValid() || v.IsNil() {
enc.AppendString("")
continue
} else if pb, _ := v.Interface().(proto.Message); pb != nil {
enc.AppendObject(protobufZapMarshaler{val: pb})
} else {
enc.AppendString("")
}
}
}
return nil
}
func encodeZapKeyValue(key string, prop *proto.Properties, container reflect.Type, fv reflect.Value, enc zapcore.ObjectEncoder) error {
switch fv.Kind() {
case reflect.Float32:
enc.AddFloat32(key, float32(fv.Float()))
case reflect.Float64:
enc.AddFloat64(key, fv.Float())
case reflect.Int32:
if prop.Enum == "" {
enc.AddInt32(key, int32(fv.Int()))
break
}
v := int32(fv.Int())
if v == 0 {
break
}
if name := getEnumValueMapping(prop.Enum)[v]; name != "" {
enc.AddString(key, name)
} else {
enc.AddInt32(key, v)
}
case reflect.Int64:
enc.AddInt64(key, fv.Int())
case reflect.Uint32, reflect.Uint64:
enc.AddUint64(key, fv.Uint())
case reflect.String:
enc.AddString(key, fv.String())
case reflect.Ptr: // proto.Message
if fv.IsNil() {
break
}
enc.AddObject(key, protobufZapMarshaler{val: fv.Interface().(proto.Message)})
case reflect.Map:
if fv.IsNil() || fv.Len() == 0 {
break
}
enc.AddObject(key, protoMapZapMarshaler{val: fv, prop: prop})
case reflect.Slice:
if fv.IsNil() || fv.Len() == 0 {
break
}
switch fv.Type().Elem().Kind() {
case reflect.Uint8:
enc.AddBinary(key, fv.Bytes())
default:
enc.AddArray(key, protoSliceZapMarshaler{slice: fv, prop: prop})
}
}
return nil
}
var (
enumValueMappings = map[string]map[int32]string{}
enumValueMappingsLock sync.Mutex
)
func getEnumValueMapping(enum string) map[int32]string {
enumValueMappingsLock.Lock()
defer enumValueMappingsLock.Unlock()
m := enumValueMappings[enum]
if m != nil {
return m
}
kv := proto.EnumValueMap(enum)
m = make(map[int32]string, len(kv))
for k, v := range kv {
m[v] = k
}
enumValueMappings[enum] = m
return m
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment