A forward compatible reworking of Heka Message

compatible_message.proto

package message;

import "gogo.proto";

option (gogoproto.sizer_all) = true;
option (gogoproto.marshaler_all) = true;
option (gogoproto.unmarshaler_all) = true;
option java_package = "org.mozilla.heka";
option java_outer_classname = "HekaMessage";

message Header {
  enum MACFunction {
    MD5      = 0;
    SHA1     = 1;
    POLY1305 = 2;
  }
  required uint32           message_length      = 1; // length in bytes

  optional MACFunction      mac_function        = 3 [default = MD5];
  optional string           mac_source          = 4;
  optional uint32           mac_key_version     = 5;
  optional bytes            mac                 = 6;
}

//probably have this in a separate file for easy extension
enum RepresentationType {
  COUNT             = 0;
  KB                = 1;
  MB                = 2;
  GB                = 3;
  MM                = 4;
  //So many different ways to represent time... https://xkcd.com/927/
  EPOCH_MILLI       = 5; //can we use this as the default way to represent time
  EPOCH_MICRO       = 6;
  EPOCH_NANO        = 7;
  DATETIME          = 8;  // RFC 3339, sec 5.6
  DATETIME_NANO     = 9;  // RFC 3339, sec 5.6
  RFC822_DATETIME   = 10; // RFC 822
  RFC822_DT_WITH_TZ = 11; // RFC 822
  RFC850_DATETIME   = 12; // RFC 850
  ANSI_C_DATETIME   = 13; // ANSI C
  UNIX_DATE         = 14; // IEEE Std 1003.1-2001, Section 7.3.5
  RUBY_DATETIME     = 15; // Because Ruby just had to be different
  TIME_KITCHEN      = 16; // ditto Chef
  TIMESTAMP         = 17;
  TIMESTAMP_MILLIS  = 18;
  TIMESTAMP_MICRO   = 19;
  TIMESTAMP_NANO    = 20;
  EMAIL_ADDRESS     = 21; // RFC 5322, sec 3.4.1
  HOSTNAME          = 22; // RFC 1034, sec 3.1
  IPV4              = 22; // RFC 2673, sec 3.2
  IPV6              = 23; // RFC 2373, sec 2.2
  URI               = 24; // RFC 3986
}

enum ValueType {
  STRING           = 0;
  BYTES            = 1;
  INTEGER          = 2; //use when negative value are an option, but unlikely
  DOUBLE           = 3;
  BOOL             = 4;
  SIGNED_INTEGER   = 5; //use when negative values are likely
  UNSIGNED_INTEGER = 6; //use when negative values are impossible
}

message DeprecatedField {
  required string       name            = 1;
  optional ValueType    value_type      = 2 [default = STRING];
  optional string       representation  = 3;
  repeated string       value_string    = 4;
  repeated bytes        value_bytes     = 5;
  repeated int64        value_integer   = 6 [packed=true];
  repeated double       value_double    = 7 [packed=true];
  repeated bool         value_bool      = 8 [packed=true];
}

message Metric {
  message FieldMetaData {
    optional string             name            = 1;
    optional ValueType          value_type      = 2 [default = INTEGER];
    optional RepresentationType representation  = 3;
    required uint32             value_offset    = 4; //offset in to the field type's value array
  }

  message FieldData {
    repeated string value_string           = 4;
    repeated bytes  value_bytes            = 5;
    repeated int64  value_integer          = 6  [packed=true];
    repeated double value_double           = 7  [packed=true];
    repeated bool   value_bool             = 8  [packed=true];
    repeated sint64 value_signed_integer   = 9  [packed=true];
    repeated uint64 value_unsigned_integer = 10 [packed=true];
  }

  repeated FieldMetaData metadata       = 1; //only gets sent periodically
  optional FieldData     fields         = 2;
}

message StructuredLog {
  message DataField {
    required string name   = 1;
    optional bytes  object = 2;
  }
  repeated DataField      structured_data = 1; //event map in slf4j

  repeated string         marker          = 2; //for slf4j

  enum SyslogFacility {
    option allow_alias = true;
    LOG_KERN     = 0;
    LOG_USER     = 1;
    LOG_MAIL     = 2;
    LOG_DAEMON   = 3;
    LOG_AUTH     = 4;
    LOG_SYSLOG   = 5;
    LOG_LPR      = 6;
    LOG_NEWS     = 7;
    LOG_UUCP     = 8;
    LOG_CRON     = 9;
    LOG_AUTHPRIV = 10;
    LOG_FTP      = 11;
    LOG_NTP      = 12;
    LOG_AUDIT    = 13;
    LOG_SECURITY = 13;
    LOG_ALERT    = 14;
    LOG_CONSOLE  = 14;
    LOG_CRONPRIV = 15;
    LOG_LOCAL0   = 16;
    LOG_LOCAL1   = 17;
    LOG_LOCAL2   = 18;
    LOG_LOCAL3   = 19;
    LOG_LOCAL4   = 20;
    LOG_LOCAL5   = 21;
    LOG_LOCAL6   = 22;
    LOG_LOCAL7   = 23;
  }
  optional SyslogFacility facility        = 3; // for syslog style messages
  optional bytes          raw_payload     = 4; // used when the payload field has something that isn't UTF-8
}
  
message Message {
  //Identifier
  optional string         hostname        = 9;
  optional string         source          = 11;
  optional int32          pid             = 8;
  optional uint64         tid             = 12; // thread_id
  optional uint64         sequence_number = 13;
  
  required bytes          uuid            = 1;
  required int64          timestamp       = 2; // nanoseconds since UNIX epoch
  optional string         type            = 3;
  optional string         logger          = 4;
  enum Severity {
    option allow_alias = true;
    EMERGENCY     = 0;
    ALERT         = 1;
    CRITICAL      = 2;
    ERROR         = 3;
    WARNING       = 4;
    NOTICE        = 5;
    INFORMATIONAL = 6;
    INFO          = 6;
    DEBUG         = 7;
    TRACE         = 8;
  }
  optional Severity        severity       = 5 [default = DEBUG]; //not populated for metrics
  optional string          payload        = 6; // slf4j format strings & syslog messages go here
  optional string          env_version    = 7;
  repeated DeprecatedField fields         = 10 [deprecated = true];
  oneof event_type {
    StructuredLog          log            = 14;
    Metric                 metric         = 15;
  }
}