Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
A forward compatible reworking of Heka Message
package message;
import "gogo.proto";
option (gogoproto.sizer_all) = true;
option (gogoproto.marshaler_all) = true;
option (gogoproto.unmarshaler_all) = true;
option java_package = "org.mozilla.heka";
option java_outer_classname = "HekaMessage";
message Header {
enum MACFunction {
MD5 = 0;
SHA1 = 1;
POLY1305 = 2;
}
required uint32 message_length = 1; // length in bytes
optional MACFunction mac_function = 3 [default = MD5];
optional string mac_source = 4;
optional uint32 mac_key_version = 5;
optional bytes mac = 6;
}
//probably have this in a separate file for easy extension
enum RepresentationType {
COUNT = 0;
KB = 1;
MB = 2;
GB = 3;
MM = 4;
//So many different ways to represent time... https://xkcd.com/927/
EPOCH_MILLI = 5; //can we use this as the default way to represent time
EPOCH_MICRO = 6;
EPOCH_NANO = 7;
DATETIME = 8; // RFC 3339, sec 5.6
DATETIME_NANO = 9; // RFC 3339, sec 5.6
RFC822_DATETIME = 10; // RFC 822
RFC822_DT_WITH_TZ = 11; // RFC 822
RFC850_DATETIME = 12; // RFC 850
ANSI_C_DATETIME = 13; // ANSI C
UNIX_DATE = 14; // IEEE Std 1003.1-2001, Section 7.3.5
RUBY_DATETIME = 15; // Because Ruby just had to be different
TIME_KITCHEN = 16; // ditto Chef
TIMESTAMP = 17;
TIMESTAMP_MILLIS = 18;
TIMESTAMP_MICRO = 19;
TIMESTAMP_NANO = 20;
EMAIL_ADDRESS = 21; // RFC 5322, sec 3.4.1
HOSTNAME = 22; // RFC 1034, sec 3.1
IPV4 = 22; // RFC 2673, sec 3.2
IPV6 = 23; // RFC 2373, sec 2.2
URI = 24; // RFC 3986
}
enum ValueType {
STRING = 0;
BYTES = 1;
INTEGER = 2; //use when negative value are an option, but unlikely
DOUBLE = 3;
BOOL = 4;
SIGNED_INTEGER = 5; //use when negative values are likely
UNSIGNED_INTEGER = 6; //use when negative values are impossible
}
message DeprecatedField {
required string name = 1;
optional ValueType value_type = 2 [default = STRING];
optional string representation = 3;
repeated string value_string = 4;
repeated bytes value_bytes = 5;
repeated int64 value_integer = 6 [packed=true];
repeated double value_double = 7 [packed=true];
repeated bool value_bool = 8 [packed=true];
}
message Metric {
message FieldMetaData {
optional string name = 1;
optional ValueType value_type = 2 [default = INTEGER];
optional RepresentationType representation = 3;
required uint32 value_offset = 4; //offset in to the field type's value array
}
message FieldData {
repeated string value_string = 4;
repeated bytes value_bytes = 5;
repeated int64 value_integer = 6 [packed=true];
repeated double value_double = 7 [packed=true];
repeated bool value_bool = 8 [packed=true];
repeated sint64 value_signed_integer = 9 [packed=true];
repeated uint64 value_unsigned_integer = 10 [packed=true];
}
repeated FieldMetaData metadata = 1; //only gets sent periodically
optional FieldData fields = 2;
}
message StructuredLog {
message DataField {
required string name = 1;
optional bytes object = 2;
}
repeated DataField structured_data = 1; //event map in slf4j
repeated string marker = 2; //for slf4j
enum SyslogFacility {
option allow_alias = true;
LOG_KERN = 0;
LOG_USER = 1;
LOG_MAIL = 2;
LOG_DAEMON = 3;
LOG_AUTH = 4;
LOG_SYSLOG = 5;
LOG_LPR = 6;
LOG_NEWS = 7;
LOG_UUCP = 8;
LOG_CRON = 9;
LOG_AUTHPRIV = 10;
LOG_FTP = 11;
LOG_NTP = 12;
LOG_AUDIT = 13;
LOG_SECURITY = 13;
LOG_ALERT = 14;
LOG_CONSOLE = 14;
LOG_CRONPRIV = 15;
LOG_LOCAL0 = 16;
LOG_LOCAL1 = 17;
LOG_LOCAL2 = 18;
LOG_LOCAL3 = 19;
LOG_LOCAL4 = 20;
LOG_LOCAL5 = 21;
LOG_LOCAL6 = 22;
LOG_LOCAL7 = 23;
}
optional SyslogFacility facility = 3; // for syslog style messages
optional bytes raw_payload = 4; // used when the payload field has something that isn't UTF-8
}
message Message {
//Identifier
optional string hostname = 9;
optional string source = 11;
optional int32 pid = 8;
optional uint64 tid = 12; // thread_id
optional uint64 sequence_number = 13;
required bytes uuid = 1;
required int64 timestamp = 2; // nanoseconds since UNIX epoch
optional string type = 3;
optional string logger = 4;
enum Severity {
option allow_alias = true;
EMERGENCY = 0;
ALERT = 1;
CRITICAL = 2;
ERROR = 3;
WARNING = 4;
NOTICE = 5;
INFORMATIONAL = 6;
INFO = 6;
DEBUG = 7;
TRACE = 8;
}
optional Severity severity = 5 [default = DEBUG]; //not populated for metrics
optional string payload = 6; // slf4j format strings & syslog messages go here
optional string env_version = 7;
repeated DeprecatedField fields = 10 [deprecated = true];
oneof event_type {
StructuredLog log = 14;
Metric metric = 15;
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.