Probuf.txt

%%% File    : gpb_eqc.erl
%%% Author  : Thomas Arts <thomas.arts@quviq.com>
%%% Further developed by: Tomas Abrahamsson <tab@lysator.liu.se>
%%% Description : Testing protocol buffer implemented by Tomas Abrahamsson
%%% Created : 12 May 2010 by Thomas Arts

-module(gpb_eqc).

-include_lib("eqc/include/eqc.hrl").
-include("gpb.hrl").

-compile(export_all).

%% Eunit integration
qc_prop_test_() ->
    AllProps = [Fn || {Fn,Arity} <- ?MODULE:module_info(exports),
                      is_property(atom_to_list(Fn), Arity)],
    {Descr, PropsToTest} =
        case find_protoc() of
            false ->
                {"QuickCheck tests"
                 " (note: 'protoc' not in $PATH and no PROTOC env var defined,"
                 " so excluding some properties)",
                 AllProps -- [prop_encode_decode_via_protoc]};
            P when is_list(P) ->
                {"QuickCheck tests", AllProps}
        end,
    {Descr,
     {timeout, 600,  %% timeout for all tests
      [{timeout, 300, %% timeout for each test
        [{atom_to_list(Prop),
          fun() -> true = eqc:quickcheck(?MODULE:Prop()) end}]}
       || Prop <- PropsToTest]}}.

is_property("prop_"++_, 0) -> true;
is_property(_, _)          -> false.


message_defs() ->
    message_defs([]).

message_defs(Opts) ->
    %% Can we have messages that refer to themselves?
    %% Actually not if field is required, since then we cannot generate
    %% a message of that kind.
    %% left_of/1 guarantees that the messages only refer to earlier definitions
    %% Enums are globally unique. Hence, we generate them globally
    DoAny = not lists:member(no_any, Opts),
    ?LET({MsgNames,Any}, {eqc_gen:non_empty(ulist("m")),
                          elements([[], ['google.protobuf.Any' || DoAny]])},
         ?LET(EnumDefs,enums(),
              begin
                  AnyDef = [mk_anymsg_def() || Any /= []],
                  EnumNames = [EName || {{enum,EName},_}<-EnumDefs],
                  Enum0Names = [EName || {{enum,EName},[{_,0}|_]}<-EnumDefs],
                 shuffle(EnumDefs
                          ++ AnyDef
                          ++ [{{msg,Msg},message_fields(
                                           left_of(Msg,Any++MsgNames),
                                           EnumNames, Enum0Names,
                                           Opts)}
                              || Msg<-MsgNames])
              end)).

mk_anymsg_def() ->
    {{msg,'google.protobuf.Any'},
     [#?gpb_field{name=type_url, type=string,
                  fnum=1, rnum=2, occurrence=required, opts=[]},
      #?gpb_field{name=value, type=bytes,
                  fnum=2, rnum=3, occurrence=required, opts=[]}]}.

%% Take all values left of a certain value
left_of(X,Xs) ->
    lists:takewhile(fun(Y) ->
                            Y/=X
                    end,Xs).

message_fields(MsgNames, EnumNames, Enum0Names, Opts) ->
    %% can we have definitions without any field?
    DoMapFields = not lists:member(no_maps, Opts),
    ?LET({FieldDefs, FNumBase0},
         {eqc_gen:non_empty(
            list({elements([{required, msg_field_type(MsgNames, EnumNames)},
                            {optional, msg_field_type(MsgNames, EnumNames)},
                            {repeated, msg_field_type(MsgNames, EnumNames)},
                            {oneof,    oneof_fields(MsgNames, EnumNames)}]
                           ++ [{repeated, map_field(MsgNames, Enum0Names)}
                               || DoMapFields]),
                  field_name()})),
          uint(10)},
         mk_fields(FieldDefs, FNumBase0+1)).

message_name() ->
    elements([m1,m2,m3,m4,m5,m6]).

field_name() ->
    elements([a,b,c,field1,f]).

oneof_fields(MsgNames, EnumNames) ->
    ?LET({FieldDefs, FNumBase0},
         {eqc_gen:non_empty(
            list({elements([{optional, msg_field_type(MsgNames, EnumNames)}]),
                  field_name()})),
          uint(10)},
         mk_fields(FieldDefs, FNumBase0+1)).

map_field(MsgNames, EnumNames) ->
    ?LET({KeyType,ValueType}, {elements(map_key_types()),
                               msg_field_type(MsgNames, EnumNames)},
         {map,KeyType,ValueType}).

msg_field_type([], []) ->
    elements(basic_msg_field_types());
msg_field_type([], EnumNames) ->
    ?LET(EnumName,elements(EnumNames),
         elements(basic_msg_field_types() ++ [{enum, EnumName}]));
msg_field_type(MsgNames, []) ->
    ?LET(MsgName,elements(MsgNames),
         elements(basic_msg_field_types() ++ [{'msg',MsgName}]));
msg_field_type(MsgNames, EnumNames) ->
    ?LET({MsgName, EnumName}, {elements(MsgNames), elements(EnumNames)},
         elements(basic_msg_field_types() ++
                  [{enum, EnumName}, {'msg',MsgName}])).

basic_msg_field_types() ->
    [bool,sint32,sint64,int32,int64,uint32,
     uint64,
     fixed64,sfixed64,double,
     fixed32,
     sfixed32,
     float,
     bytes,
     string
    ].

map_key_types() ->
    [bool,sint32,sint64,int32,int64,uint32,
     uint64,
     fixed64,sfixed64,
     fixed32,
     sfixed32,
     string
    ].

mk_fields(FieldDefs, FNumBase) ->
    UFieldDefs = keyunique(2, FieldDefs),
    {Fields, _NextFNum} =
        lists:mapfoldl(
          fun({{{required, Type}, FieldName}, RNum}, FNum) ->
                  {#?gpb_field{name=FieldName, fnum=FNum, rnum=RNum,
                               type=Type, occurrence=required, opts=[]},
                   FNum+1};
             ({{{optional, Type}, FieldName}, RNum}, FNum) ->
                  {#?gpb_field{name=FieldName, fnum=FNum, rnum=RNum,
                               type=Type, occurrence=optional, opts=[]},
                   FNum+1};
             ({{{repeated, Type}, FieldName}, RNum}, FNum) ->
                  Opts = case Type of
                             {map,_,_} -> [];
                             {msg,_} -> []; %% FIXME: why not packed?
                             string  -> []; %% FIXME: why not packed?
                             bytes   -> []; %% FIXME: why not packed?
                             _       -> elements([[], [packed]])
                         end,
                  {#?gpb_field{name=FieldName, fnum=FNum, rnum=RNum,
                               type=Type, occurrence=repeated, opts=Opts},
                   FNum+1};
             ({{{oneof, OFields}, FieldName}, RNum}, FNum) ->
                  %% Oneof fields, must have unique names and field numbers
                  %% (within the message)
                  {OFields2, NewFNum} =
                      lists:mapfoldl(
                        fun(#?gpb_field{name=ONm}=F, OFNum) ->
                                OFieldName = combine_name(FieldName, ONm),
                                {F#?gpb_field{name=OFieldName,
                                              rnum=RNum, fnum=OFNum},
                                 OFNum+1}
                        end,
                        FNum,
                        OFields),
                  {#gpb_oneof{name=FieldName, rnum=RNum, fields=OFields2},
                   NewFNum}
          end,
          FNumBase,
          seq_index(UFieldDefs, 2)),
    Fields.

keyunique(_N, []) ->
    [];
keyunique(N, [Tuple|Rest]) ->
    [Tuple| keyunique(N, [ T2 || T2<-Rest, element(N,T2)/=element(N,Tuple)])].

seq_index(L, Start) ->
    lists:zip(L, lists:seq(1+(Start-1),length(L)+(Start-1))).

combine_name(NameA, NameB) ->
    list_to_atom(lists:concat([NameA, "_", NameB])).

%% In fact, we should improve this to have different enums containing same value
%% e.g. [ {{enum,e1},[{x1,10}]}, {{enum,x2},[{x2,10}]} ]
enums() ->
    ?LET({N,Values,Names},{int(),ulist("x"),ulist("e")},
         ?LET(Constants,unique_values(Values,N),
              enums(Names,Constants))).

ulist(String) ->
    ?LET(N,nat(),
         [ list_to_atom(String++integer_to_list(K)) || K<-lists:seq(1,N) ]).

%% Unique names and unqiue values
%% Example
%% enum file_open_return_values { enoent=1, eacess=2 }
unique_values([],_N) ->
    [];
unique_values([Cons|Conss],N) ->
    ?LET(Next,nat(),
         [{Cons,N} | unique_values(Conss,Next+N+1)]).

enums([],_Conss) ->
    [];
enums(_Enames,[]) ->
    [];
enums([Ename|Enames],Conss) ->
    ?LET(Element,elements(Conss),
         begin
             Prefix = left_of(Element,Conss)++[Element],
             [{{enum,Ename},Prefix}|enums(Enames,Conss--Prefix)]
         end).


%% generator for messages that respect message definitions

message(MessageDefs,Opts) ->
    MsgDefs = [MD || {{msg,_MsgName},_}=MD <- MessageDefs], % filter out enums
    CandidateMsgDefs = case proplists:get_value(any_translate, Opts) of
                           undefined ->
                               MsgDefs;
                           _ ->
                               [MD || {{msg,Name},_}=MD <- MsgDefs,
                                      Name /= 'google.protobuf.Any']
                       end,
    ?LET({{msg,Msg},_Fields},oneof(CandidateMsgDefs),
         message(Msg,MessageDefs,Opts)).

message(Msg,MessageDefs,Opts) ->
    Fields = proplists:get_value({msg,Msg},MessageDefs),
    FieldValues = [case Field of
                       #?gpb_field{} -> field_value(Field, MessageDefs, Opts);
                       #gpb_oneof{}  -> oneof_value(Field, MessageDefs, Opts)
                   end
                   || Field <- Fields],
    list_to_tuple([Msg|FieldValues]).

oneof_value(#gpb_oneof{fields=OFields}, MessageDefs, Opts) ->
    ?LET(OField, oneof(OFields),
         begin
             #?gpb_field{name=Name} = OField,
             oneof([undefined,
                    {Name, field_value(OField#?gpb_field{occurrence=required},
                                       MessageDefs,
                                       Opts)}])
         end).

field_value(#?gpb_field{type=Type, occurrence=Occurrence}, MsgDefs, Opts) ->
    field_val2(Type, Occurrence, MsgDefs, Opts).

field_val2(Type, optional, MsgDefs, Opts) ->
    default(undefined, value(Type,MsgDefs,Opts));
field_val2({map,KeyType,ValueType}, repeated, MsgDefs, Opts) ->
    ?LET(Keys, list(value(KeyType, MsgDefs, Opts)),
         [{Key, value(ValueType, MsgDefs, Opts)} || Key <- lists:usort(Keys)]);
field_val2(Type, repeated, MsgDefs, Opts) ->
    list(value(Type, MsgDefs, Opts));
field_val2(Type, required, MsgDefs, Opts) ->
    value(Type,MsgDefs, Opts).

value({msg,M},MessageDefs,Opts) ->
    if M == 'google.protobuf.Any' ->
            case proplists:get_value(any_translate,Opts) of
                undefined ->
                    message(M,MessageDefs,Opts);
                _ ->
                    uint(32)
            end;
       M /= 'google.protobuf.Any' ->
            message(M,MessageDefs,Opts)
    end;
value({enum,E},MessageDefs,_Opts) ->
    {value, {{enum,E},EnumValues}} = lists:keysearch({enum,E}, 1, MessageDefs),
    ?LET({Symbolic, _ActualValue}, elements(EnumValues),
         Symbolic);
value({map,KeyType,ValueType}, MessageDefs, Opts) ->
    {value(KeyType, MessageDefs, Opts), value(ValueType, MessageDefs, Opts)};
value(bool,_,_) ->
    bool();
value(sint32,_,_) ->
    sint(32);
value(sint64,_,_) ->
    sint(64);
value(int32,_,_) ->
    int(32);
value(int64,_,_) ->
    int(64);
value(uint32,_,_) ->
    uint(32);
value(uint64,_,_) ->
    uint(64);
value(fixed64,_,_) ->
    uint(64);
value(sfixed64,_,_) ->
    sint(64);
value(fixed32,_,_) ->
    uint(32);
value(sfixed32,_,_) ->
    sint(32);
value(double, _,_) ->
    frequency([{70,real()}, {10,'infinity'}, {10,'-infinity'}, {10,nan}]);
value(float, _,_) ->
    frequency([{70,real()}, {10,'infinity'}, {10,'-infinity'}, {10,nan}]);
value(bytes, _,_) ->
    binary();
value(string, _,_) ->
    list(encodable_unicode_code_point()).

encodable_unicode_code_point() ->
    %% http://www.unicode.org/versions/Unicode6.0.0/ch03.pdf
    %% * Section 3.3, D9: Code points: integers in the range 0..10ffff
    %% * Section 3.9: The Unicode Standard supports three character
    %%   encoding forms: UTF-32, UTF-16, and UTF-8. Each encoding form
    %%   maps the Unicode code points U+0000..U+D7FF and
    %%   U+E000..U+10FFFF to unique code unit sequences
    ?SUCHTHAT(CP, oneof([uint(16), choose(16#10000, 16#10FFFF)]),
              (0 =< CP andalso CP =< 16#d7ff)
              orelse
                (16#e000 =< CP andalso CP =< 16#10ffff)).

sint(Base) ->
    int(Base).

int(Base) ->
    ?LET(I,uint(Base),
         begin
             << N:Base/signed >> = <<I:Base>>, N
         end).

uint(Base) ->
    oneof([ choose(0,pow2(B)-1) || B<-lists:seq(1,Base)]).

pow2(0)            -> 1;
pow2(N) when N > 0 -> 2*pow2(N-1);
pow2(N) when N < 0 -> 1/pow2(-N).

%%% properties

prop_encode_decode() ->
    Mod = gpb_eqc_m,
    ?FORALL(
       MsgDefs,message_defs(),
       ?FORALL(
          {Encoder, Decoder, COpts}, encoder_decoder(Mod),
          ?FORALL(
             Msg, message(MsgDefs, COpts),
             begin
                 MsgName = element(1, Msg),
                 install_msg_defs(Mod, MsgDefs, Encoder, Decoder, COpts),
                 Bin = encode_msg(Msg, MsgDefs, Encoder, COpts),
                 DecodedMsg = decode_msg(Bin, MsgName, MsgDefs, Decoder, COpts),
                 ?WHENFAIL(io:format("~p /= ~p\n",[Msg, DecodedMsg]),
                           msg_approximately_equals(Msg, DecodedMsg,
                                                    MsgDefs, COpts))
             end))).

%% add a round-trip via the `protoc' program in the protobuf package.
%% The `protoc' is the compiler generates code from a .proto file, but
%% it can also decode and encode messages on the fly, given a .proto
%% file, so we can use it as an sort of interop test.
prop_encode_decode_via_protoc() ->
    MDOpts1 = case check_protoc_can_do_map_fields() of
                  true  -> [];
                  false -> [no_maps]
              end,
    MDOpts = MDOpts1 ++ [no_any],
    Mod = gpb_eqc_m,
    ?FORALL(
       MsgDefs,message_defs(MDOpts),
       ?FORALL(
          {Encoder, Decoder, COpts}, encoder_decoder(Mod, [no_any]),
          ?FORALL(
             Msg, message(MsgDefs, COpts),
             begin
                 MsgName = element(1, Msg),
                 install_msg_defs(Mod, MsgDefs, Encoder, Decoder, COpts),
                 TmpDir = get_create_tmpdir(),
                 install_msg_defs_as_proto(MsgDefs, TmpDir),
                 GpbBin = encode_msg(Msg, MsgDefs, Encoder, COpts),
                 ProtoBin = decode_then_reencode_via_protoc(
                              GpbBin, Msg, TmpDir),
                 DecodedMsg = decode_msg(ProtoBin, MsgName, MsgDefs,
                                         Decoder, COpts),
                 ?WHENFAIL(io:format("~p /= ~p\n",[Msg,DecodedMsg]),
                           msg_approximately_equals(Msg, DecodedMsg,
                                                    MsgDefs, COpts))
             end))).

%% test that we can ignore unknown fields
prop_encode_decode_with_skip() ->
    Mod1 = gpb_eqc_m1,
    Mod2 = gpb_eqc_m2,
    ?FORALL(
       MsgDefs, message_defs([no_any]),
       ?FORALL(
          InitialMsg, message(MsgDefs, []),
          ?FORALL(
             {{SubMsg, SubDefs},
              {Encoder1, Decoder1, COpts1},
              {Encoder2, Decoder2, COpts2}},
             {message_subset_defs(InitialMsg, MsgDefs),
              encoder_decoder(Mod1, [no_any]),
              encoder_decoder(Mod2, [no_any])},
             begin
                 MsgName = element(1, InitialMsg),
                 install_msg_defs(Mod1, MsgDefs, Encoder1, Decoder1, COpts1),
                 install_msg_defs(Mod2, SubDefs, Encoder2, Decoder2, COpts2),
                 Encoded = encode_msg(InitialMsg, MsgDefs, Encoder1, COpts1),
                 %% now decode the byte sequence with a decoder that knows
                 %% only a subset of the fields for each message.
                 Decoded = decode_msg(Encoded, MsgName,  SubDefs, Decoder2,
                                      COpts2),
                 ?WHENFAIL(io:format("~p /= ~p\n",[SubMsg, Decoded]),
                           msg_approximately_equals(SubMsg, Decoded,
                                                    SubDefs, []))
             end))).

%% test merging of messages
prop_merge() ->
    Mod = gpb_eqc_m,
    ?FORALL(
       MsgDefs, message_defs(),
       ?FORALL(
          MsgName, oneof([M || {{msg,M},_} <- MsgDefs]),
          ?FORALL(
             {Encoder, Decoder, COpts}, encoder_decoder(Mod),
             ?FORALL(
                {Msg1, Msg2}, {message(MsgName, MsgDefs, COpts),
                               message(MsgName, MsgDefs, COpts)},
                begin
                    install_msg_defs(Mod, MsgDefs, Encoder, Decoder, COpts),
                    MergedMsg = merge_msgs(Msg1, Msg2, MsgDefs,
                                           Encoder, Decoder, COpts),
                    Bin1 = encode_msg(Msg1, MsgDefs, Encoder, COpts),
                    Bin2 = encode_msg(Msg2, MsgDefs, Encoder, COpts),
                    MergedBin = <<Bin1/binary,Bin2/binary>>,
                    DecodedMerge = decode_msg(MergedBin, MsgName, MsgDefs,
                                              Decoder, COpts),
                    msg_equals(MergedMsg, DecodedMerge, MsgDefs, COpts)
                end)))).

%% compute a subset of the fields, and also a subset of the msg,
%% corresponding to the subset of the fields.
%% Return {SubsetMsg, SubsetDefs}
message_subset_defs(Msg, MsgDefs) ->
    ?LET(DefsWithSkips,
         [case Elem of
              {{enum,_}, _}=Enum ->
                  Enum;
              {{msg,'google.protobuf.Any'}, _Fields}=MsgDef -> % keep intact
                  MsgDef;
              {{msg,MsgName}, MsgFields} ->
                  {{msg, MsgName}, msg_fields_subset_skips(MsgFields)}
          end
          || Elem <- MsgDefs],
         begin
             SubsetMsg  = remove_fields_by_skips(Msg, DefsWithSkips),
             SubsetDefs = remove_skips_from_defs(DefsWithSkips),
             {SubsetMsg, SubsetDefs}
         end).

msg_fields_subset_skips(Fields) when length(Fields) == 1 ->
    %% can't remove anything if there's only one field
    ?LET(_, int(),
         Fields);
msg_fields_subset_skips(Fields) ->
    ?LET(Fields2, [elements([Field, skip]) || Field <- Fields],
         %% compensate for removed fields
         Fields2).

remove_fields_by_skips(Msg, DefsWithSkips) ->
    MsgName = element(1, Msg),
    {{msg,MsgName}, MsgDef} = lists:keyfind({msg,MsgName}, 1, DefsWithSkips),
    Fields = [case Field of
                  #?gpb_field{type={msg, _SubMsgName}, occurrence=repeated} ->
                      [remove_fields_by_skips(Elem, DefsWithSkips)
                       || Elem <- Value];
                  #?gpb_field{type={map,_,{msg, _SubMsgName}}} ->
                      [{K,remove_fields_by_skips(V, DefsWithSkips)}
                       || {K, V} <- Value];
                  #?gpb_field{type={msg, _SubMsgName}, occurrence=Occurrence} ->
                      if Occurrence == optional, Value == undefined ->
                              Value;
                         true ->
                              remove_fields_by_skips(Value, DefsWithSkips)
                      end;
                  #gpb_oneof{fields=OFields} ->
                      case Value of
                          undefined ->
                              Value;
                          {OFName, Value2} ->
                              Pos = #?gpb_field.name,
                              case lists:keyfind(OFName, Pos, OFields) of
                                  #?gpb_field{type={msg, _SubMsgName}} ->
                                      {OFName, remove_fields_by_skips(
                                                 Value2, DefsWithSkips)};
                                  _ ->
                                      Value
                              end
                      end;
                  _ ->
                      Value
              end
              || {Value, Field} <- lists:zip(tl(tuple_to_list(Msg)), MsgDef),
                 Field /= skip],
    list_to_tuple([MsgName | Fields]).

remove_skips_from_defs(DefsWithSkips) ->
    [case Elem of
         {{enum,_}, _}=Enum ->
             Enum;
         {{msg,MsgName}, FieldsAndSkips} ->
             {{msg, MsgName}, remove_skips_recalculate_rnums(FieldsAndSkips)}
     end
     || Elem <- DefsWithSkips].

remove_skips_recalculate_rnums(FieldsAndSkips) ->
    {RecalculatedFieldsReversed, _TotalNumSkipped} =
        lists:foldl(
          fun(skip, {Fs, NumSkipped}) ->
                  {Fs, NumSkipped+1};
             (#?gpb_field{rnum=RNum}=F, {Fs, NumSkipped}) ->
                  {[F#?gpb_field{rnum=RNum-NumSkipped} | Fs], NumSkipped};
             (#gpb_oneof{rnum=RNum, fields=OFs}=F, {Fs, NumSkipped}) ->
                  OFs2 = [O#?gpb_field{rnum=RNum-NumSkipped} || O <- OFs],
                  F2 = F#gpb_oneof{rnum=RNum-NumSkipped,
                                   fields=OFs2},
                  {[F2 | Fs], NumSkipped}
          end,
          {[], 0},
          FieldsAndSkips),
    lists:reverse(RecalculatedFieldsReversed).

encoder_decoder(Mod) -> encoder_decoder(Mod, []).
encoder_decoder(Mod, Opts) ->
    DoAny = not lists:member(no_any, Opts),
    ?LET(
       {Encoder, Decoder}, {oneof([gpb, Mod]), oneof([gpb, Mod])},
       ?LET(
          COpts1,
          [{copy_bytes,        oneof([false, true, auto, choose(2,4)])},
           {field_pass_method, oneof([pass_as_record, pass_as_params])}
           | map_opts()],
          ?LET(
             COpts2, any_translation_opts(COpts1),
             if DoAny, Encoder /= gpb, Decoder /= gpb ->
                     {Encoder, Decoder, COpts1 ++ COpts2};
                true ->
                     {Encoder, Decoder, COpts1}
             end))).

map_opts() ->
    HaveMaps = case get(cache_have_maps) of
                   undefined ->
                       V = have_maps(),
                       put(cache_have_maps, V),
                       V;
                   V ->
                       V
               end,
    if HaveMaps ->
            [{maps, oneof([false, true])},
             {maps_unset_optional, oneof([present_undefined, omitted])}];
       not HaveMaps ->
            []
    end.

have_maps() ->
    try maps:size(maps:new()) of
        0 ->
            true
    catch error:undef ->
            false
    end.

any_translation_opts(Opts0) ->
    DoMaps = proplists:get_bool(maps, Opts0),
    ?LET(DoTranslate, oneof([false, {true, bool(), bool()}]),
         case DoTranslate of
             {true, TranslateMerge, TranslateVerify} ->
                 Encode = if DoMaps ->
                                  [{encode,{?MODULE,any_tr_pack_m,['$1']}}];
                             not DoMaps ->
                                  [{encode,{?MODULE,any_tr_pack_r,['$1']}}]
                          end,
                 Decode = if DoMaps ->
                                  [{decode,{?MODULE,any_tr_unpack_m,['$1']}}];
                             not DoMaps ->
                                  [{decode,{?MODULE,any_tr_unpack_r,['$1']}}]
                          end,
                 Merge  = [{merge,{?MODULE,any_tr_merge,['$1','$2']}}
                           || TranslateMerge],
                 Verify = [{verify,{?MODULE,any_tr_verify,['$1','$errorf']}}
                           || TranslateVerify],
                 [{any_translate, Encode ++ Decode ++ Merge ++ Verify}];
             false ->
                 []
         end).


any_tr_pack_m(N) ->
    NStr = integer_to_list(N),
    maps:from_list([{type_url,NStr},
                    {value,list_to_binary(NStr)}]).

any_tr_unpack_m(M) ->
    ML = maps:to_list(M),
    NStr = proplists:get_value(type_url,ML),
    NBin = proplists:get_value(value,ML),
    N = list_to_integer(NStr),
    N = list_to_integer(binary_to_list(NBin)),
    N.


any_tr_pack_r(N) ->
    NStr = integer_to_list(N),
    {'google.protobuf.Any',NStr,list_to_binary(NStr)}.

any_tr_unpack_r({'google.protobuf.Any',NStr,NBin}) ->
    N = list_to_integer(NStr),
    N = list_to_integer(binary_to_list(NBin)),
    N.

any_tr_merge(_,N2) ->
    N2.

any_tr_verify(V, _ErrorF) when is_integer(V) -> ok;
any_tr_verify(_, ErrorF) -> ErrorF(not_an_integer).


encode_msg(Msg, MsgDefs, Encoder, COpts) ->
    case Encoder of
        gpb ->
            gpb:encode_msg(Msg, MsgDefs);
        _ ->
            case proplists:get_value(maps, COpts) of
                false ->
                    Encoder:encode_msg(Msg);
                true ->
                    map_encode_msg(Msg, MsgDefs, Encoder, COpts)
            end
    end.

map_encode_msg(Msg, MsgDefs, Encoder, COpts) ->
    MsgAsMap = msg_to_map(Msg, MsgDefs, COpts),
    MsgName = element(1, Msg),
    Encoder:encode_msg(MsgAsMap, MsgName).

decode_msg(Bin, MsgName, MsgDefs, Decoder, COpts) ->
    case Decoder of
        gpb ->
            gpb:decode_msg(Bin, MsgName, MsgDefs);
        _ ->
            case proplists:get_value(maps, COpts) of
                false ->
                    Decoder:decode_msg(Bin, MsgName);
                true ->
                    map_decode_msg(Bin, MsgName, MsgDefs, Decoder, COpts)
            end
    end.

map_decode_msg(Bin, MsgName, MsgDefs, Decoder, COpts) ->
    MsgAsMap = Decoder:decode_msg(Bin, MsgName),
    map_to_msg(MsgAsMap, MsgName, MsgDefs, COpts).

merge_msgs(Msg1, Msg2, MsgDefs, Encoder, Decoder, COpts) ->
    if Encoder == gpb, Decoder == gpb ->
            gpb:merge_msgs(Msg1, Msg2, MsgDefs);
       Encoder /= gpb ->
            case proplists:get_value(maps, COpts) of
                false ->
                    Encoder:merge_msgs(Msg1, Msg2);
                true ->
                    map_merge_msgs(Msg1, Msg2, MsgDefs, Encoder, COpts)
            end;
       Decoder /= gpb ->
            case proplists:get_value(maps, COpts) of
                false ->
                    Decoder:merge_msgs(Msg1, Msg2);
                true ->
                    map_merge_msgs(Msg1, Msg2, MsgDefs, Decoder, COpts)
            end
    end.

map_merge_msgs(Msg1, Msg2, MsgDefs, Mod, COpts) ->
    Msg1AsMap = msg_to_map(Msg1, MsgDefs, COpts),
    Msg2AsMap = msg_to_map(Msg2, MsgDefs, COpts),
    MsgName = element(1, Msg1),
    ResultAsMap = Mod:merge_msgs(Msg1AsMap, Msg2AsMap, MsgName),
    map_to_msg(ResultAsMap, MsgName, MsgDefs, COpts).

install_msg_defs(Mod, MsgDefs, Encoder, Decoder, COpts) ->
    if Encoder == gpb, Decoder == gpb ->
            ok; %% nothing needs to be done
       true ->
            install_msg_defs_aux(Mod, MsgDefs, COpts)
    end.

install_msg_defs(Mod, MsgDefs) ->
    install_msg_defs_aux(Mod, MsgDefs, [{copy_bytes, auto}]).

install_msg_defs_aux(Mod, MsgDefs, Opts) when is_list(Opts) ->
    Opts2 = [binary, {verify, always}, return_warnings | Opts],
    {{ok, Mod, Code, _},_} = {gpb_compile:msg_defs(Mod, MsgDefs, Opts2),
                              compile},
    ok = delete_old_versions_of_code(Mod),
    {{module, Mod},_} = {code:load_binary(Mod, "<nofile>", Code), load_code},
    ok.

delete_old_versions_of_code(Mod) ->
    code:purge(Mod),
    code:delete(Mod),
    code:purge(Mod),
    code:delete(Mod),
    ok.

msg_equals(Msg1, Msg2, MsgDefs, Opts) ->
    case msg_approximately_equals(Msg1, Msg2, MsgDefs, Opts) of
        true  ->
            true;
        false ->
            %% Run equals, even though we know it'll return
            %% false, because it'll show the messages
            %% appropritately -- e.g. not when shrinking.
            equals(Msg1,Msg2)
    end.

msg_approximately_equals(M1, M2, MsgDefs, Opts)
  when is_tuple(M1), is_tuple(M2),
       element(1,M1) == element(1,M2),
       tuple_size(M1) == tuple_size(M2) ->
    MsgName = element(1,M1),
    {{msg,MsgName},Fields} = lists:keyfind({msg,MsgName},1,MsgDefs),
    lists:all(fun({F1, F2, Field}) ->
                      field_approximately_equals(F1, F2, Field, MsgDefs, Opts)
              end,
              lists:zip3(tl(tuple_to_list(M1)),
                         tl(tuple_to_list(M2)),
                         Fields));
msg_approximately_equals(_X, _Y, _MsgDefs, _Opts) ->
    false.

field_approximately_equals(F1, F2, #?gpb_field{type={map,_,VT}},
                           MsgDefs, Opts) ->
    DoTranslateAny = proplists:get_value(any_translate,Opts) /= undefined,
    lists:all(fun({{K1,V1}, {K2,V2}}) ->
                      case VT of
                          {msg,'google.protobuf.Any'} when DoTranslateAny ->
                              is_value_approx_eq(K1,K2)
                                  andalso
                                  is_value_approx_eq(V1,V2);
                          {msg,_} ->
                              is_value_approx_eq(K1,K2)
                                  andalso
                                  msg_approximately_equals(V1,V2,MsgDefs,Opts);
                          _ ->
                              is_value_approx_eq(K1,K2)
                                  andalso
                                  is_value_approx_eq(V1,V2)
                      end
              end,
              lists:zip(lists:sort(F1),lists:sort(F2)));
field_approximately_equals(F1, F2, #?gpb_field{type={msg,MsgName},
                                               occurrence=Occ},
                           MsgDefs, Opts) ->
    DoTranslateAny = proplists:get_value(any_translate,Opts) /= undefined,
    case {Occ, MsgName} of
        {_, 'google.protobuf.Any'} when DoTranslateAny ->
            is_value_approx_eq(F1, F2);
        {repeated,_} ->
            lists:all(fun({E1,E2}) ->
                              msg_approximately_equals(E1, E2, MsgDefs, Opts)
                      end,
                      lists:zip(F1,F2));
        {optional,_} ->
            if F1 == undefined, F2 == undefined ->
                    true;
               true ->
                    msg_approximately_equals(F1, F2, MsgDefs, Opts)
            end;
        {required,_} ->
            msg_approximately_equals(F1, F2, MsgDefs, Opts)
    end;
field_approximately_equals({T,F1}, {T,F2}, #gpb_oneof{fields=OFs},
                           MsgDefs, Opts) ->
    DoTranslateAny = proplists:get_value(any_translate,Opts) /= undefined,
    case lists:keyfind(T,#?gpb_field.name,OFs) of
        #?gpb_field{type={msg,'google.protobuf.Any'}} when DoTranslateAny ->
            is_value_approx_eq(F1, F2);
        #?gpb_field{type={msg,_}} ->
            msg_approximately_equals(F1, F2, MsgDefs, Opts);
        _ ->
            is_value_approx_eq(F1, F2)
    end;
field_approximately_equals(F1, F2, _Field, _MsgDefs, _Opts) ->
    is_value_approx_eq(F1, F2).

is_value_approx_eq(F1, F2) when is_float(F1), is_float(F2) ->
    is_float_equivalent(F1, F2);
is_value_approx_eq(L1, L2) when is_list(L1), is_list(L2) ->
    lists:all(fun({E1,E2}) -> is_value_approx_eq(E1,E2) end, lists:zip(L1,L2));
is_value_approx_eq(X, X) ->
    true;
is_value_approx_eq(_X, _Y) ->
    io:format("Not equal: ~p <--> ~p~n", [_X, _Y]),
    false.

-define(ABS_ERROR, 1.0e-10). %% was: 1.0e-16
-define(REL_ERROR, 1.0e-6).  %% was: 1.0e-10

is_float_equivalent(F, F) -> true;
is_float_equivalent(F1,F2) ->
 if (abs(F1-F2) < ?ABS_ERROR) -> true;
    (abs(F1) > abs(F2)) -> abs( (F1-F2)/F1 ) < ?REL_ERROR;
    (abs(F1) < abs(F2)) -> abs( (F1-F2)/F2 ) < ?REL_ERROR
end.

is_within_percent(F1, F2, PercentsAllowedDeviation) ->
    AllowedDeviation = PercentsAllowedDeviation / 100,
    abs(F1 - F2) < (AllowedDeviation * F1).

% Recursively translate a record to a map
msg_to_map(Msg, MsgDefs, COpts) ->
    MsgName = element(1, Msg),
    {{msg,MsgName},Fields} = lists:keyfind({msg,MsgName}, 1, MsgDefs),
    FVs = [case F of
               #?gpb_field{name=FName}=Field ->
                   V2 = field_to_map(V, Field, MsgDefs, COpts),
                   {FName, V2};
               #gpb_oneof{name=FName, fields=OFields} ->
                   V2 = case V of
                            undefined ->
                                undefined;
                            {Tag, TV} ->
                                Field = lists:keyfind(Tag, #?gpb_field.name,
                                                      OFields),
                                {Tag, field_to_map(TV, Field, MsgDefs, COpts)}
                        end,
                   {FName, V2}
           end
           || {F,V} <- lists:zip(Fields, tl(tuple_to_list(Msg)))],
    case proplists:get_value(maps_unset_optional, COpts) of
        present_undefined ->
            maps:from_list(FVs);
        omitted ->
            maps:from_list([FV || {_,Value}=FV <- FVs,
                                  Value /= undefined])
    end.

field_to_map(V, #?gpb_field{type={msg,MsgName},occurrence=Occurrence},
             MsgDefs,COpts) ->
    submsg_to_map1(Occurrence, MsgName, V, MsgDefs, COpts);
field_to_map(V, #?gpb_field{type={map,_,ValueType}}, MsgDefs, COpts) ->
    maps:from_list(
      [case ValueType of
           {msg, MsgName} ->
               {K, submsg_to_map1(required, MsgName, V2, MsgDefs, COpts)};
           _        -> {K, V2}
       end
       || {K,V2} <- V]);
field_to_map(V, _FieldDef, _MsgDefs, _COpts) ->
    V.

submsg_to_map1(Occurrence, MsgName, V, MsgDefs, COpts) ->
    if MsgName == 'google.protobuf.Any' ->
            case proplists:get_value(any_translate, COpts) of
                undefined ->
                    submsg_to_map2(Occurrence, V, MsgDefs, COpts);
                _ ->
                    %% it is really an integer since we've added translations
                    %% for it
                    V
            end;
       true ->
            submsg_to_map2(Occurrence, V, MsgDefs, COpts)
    end.

submsg_to_map2(required, V, MsgDefs, COpts) ->
    msg_to_map(V, MsgDefs, COpts);
submsg_to_map2(repeated, Seq, MsgDefs, COpts) ->
    [msg_to_map(Elem, MsgDefs, COpts) || Elem <- Seq];
submsg_to_map2(optional, V, MsgDefs, COpts) ->
    if V == undefined -> undefined;
       V /= undefined -> msg_to_map(V, MsgDefs, COpts)
    end.

map_to_msg(Map, MsgName, MsgDefs, COpts) ->
    {{msg,MsgName},Fields} = lists:keyfind({msg,MsgName}, 1, MsgDefs),
    list_to_tuple(
      [MsgName |
       [case F of
            #?gpb_field{name=FName}=Field ->
                case maps:find(FName, Map) of
                    error ->
                        undefined;
                    {ok, V} ->
                        field_from_map(V, Field, MsgDefs, COpts)
                end;
            #gpb_oneof{name=FName, fields=OFields} ->
                case maps:find(FName, Map) of
                    error ->
                        undefined;
                    {ok, undefined} ->
                        undefined;
                    {ok, {Tag, TV}} ->
                        Field = lists:keyfind(Tag, #?gpb_field.name, OFields),
                        {Tag, field_from_map(TV, Field, MsgDefs, COpts)}
                end
        end
        || F <- Fields]]).

field_from_map(V, #?gpb_field{type={msg,SubMsgName}, occurrence=Occurrence},
               MsgDefs, COpts) ->
    submsg_from_map1(Occurrence, V, SubMsgName, MsgDefs, COpts);
field_from_map(V, #?gpb_field{type={map,_,ValueType}}, MsgDefs, COpts) ->
    [case ValueType of
         {msg, Name2} -> {K, submsg_from_map1(required, V2, Name2, MsgDefs,
                                              COpts)};
         _ -> {K,V2}
     end
     || {K,V2} <- maps:to_list(V)];
field_from_map(V, _FieldDef, _MsgDefs, _COpts) ->
    V.

submsg_from_map1(Occurrence, Map, MsgName, MsgDefs, COpts) ->
    if MsgName == 'google.protobuf.Any' ->
            case proplists:get_value(any_translate, COpts) of
                undefined ->
                    submsg_from_map2(Occurrence, Map, MsgName, MsgDefs, COpts);
                _ ->
                    %% it is really an integer since we've added translations
                    %% for it
                    Map
            end;
       true ->
            submsg_from_map2(Occurrence, Map, MsgName, MsgDefs, COpts)
    end.

submsg_from_map2(required, Map, MsgName, MsgDefs, COpts) ->
    map_to_msg(Map, MsgName, MsgDefs, COpts);
submsg_from_map2(repeated, Seq, MsgName, MsgDefs, COpts) ->
    [map_to_msg(Elem, MsgName, MsgDefs, COpts) || Elem <- Seq];
submsg_from_map2(optional, V, MsgName, MsgDefs, COpts) ->
    if V == undefined -> undefined;
       V /= undefined -> map_to_msg(V, MsgName, MsgDefs, COpts)
    end.

get_create_tmpdir() ->
    D = filename:join("/tmp", f("~s-~s", [?MODULE, os:getpid()])),
    filelib:ensure_dir(filename:join(D, "dummy-file-name")),
    [file:delete(X) || X <- filelib:wildcard(filename:join(D,"*"))],
    D.

delete_tmpdir(TmpDir) ->
    [file:delete(X) || X <- filelib:wildcard(filename:join(TmpDir,"*"))],
    file:del_dir(TmpDir).

install_msg_defs_as_proto(MsgDefs, TmpDir) ->
    ProtoFile = filename:join(TmpDir, "x.proto"),
    ok = file:write_file(ProtoFile, msg_defs_to_proto(MsgDefs)).

msg_defs_to_proto(MsgDefs) ->
    iolist_to_binary(
      ["syntax=\"proto2\";\n",
       lists:map(fun msg_def_to_proto/1, MsgDefs)]).

contains_mapfield([{{msg,_},Fields} | Rest]) ->
    HasMapField = lists:any(fun(#?gpb_field{type={map,_,_}}) -> true;
                               (_) -> false
                            end,
                            Fields),
    if HasMapField -> true;
       not HasMapField -> contains_mapfield(Rest)
    end;
contains_mapfield([_ | Rest]) ->
    contains_mapfield(Rest);
contains_mapfield([]) ->
    false.

msg_def_to_proto({{enum, Name}, EnumValues}) ->
    f("enum ~s {~n"
      "~s"
      "}~n~n",
      [Name, lists:map(fun({N,V}) -> f("  ~s = ~w;~n", [N, V]) end,
                       EnumValues)]);
msg_def_to_proto({{msg, Name}, Fields}) ->
    f("message ~s {~n"
      "~s"
      "}~n~n",
      [Name, lists:map(fun(#?gpb_field{}=F) -> field_to_proto(F, unless_map);
                          (#gpb_oneof{}=F) ->  oneof_to_proto(F)
                       end,
                       Fields)]).

field_to_proto(#?gpb_field{name=FName, fnum=FNum, type=Type, opts=Opts,
                           occurrence=Occurrence}, ShowOccurrence) ->
    Packed = lists:member(packed, Opts),
    f("  ~s ~s ~s = ~w~s;~n",
      [case {ShowOccurrence,Type} of
           {unless_map,{map,_,_}} -> "  ";
           {unless_map,_}         -> Occurrence;
           {false,_}              -> "  "
       end,
       fmt_type(Type),
       FName,
       FNum,
       if Packed     -> " [packed=true]";
          not Packed -> ""
       end]).

fmt_type({msg,Name2})  -> Name2;
fmt_type({enum,Name2}) -> Name2;
fmt_type({map,KT,VT})  -> f("map<~s,~s>", [fmt_type(KT),fmt_type(VT)]);
fmt_type(Type)         -> Type.

oneof_to_proto(#gpb_oneof{name=FName, fields=OFields}) ->
    f("  oneof ~s {~n"
      "~s"
      "  };~n",
      [FName, [field_to_proto(OField, false) || OField <- OFields]]).

decode_then_reencode_via_protoc(GpbBin, Msg, TmpDir) ->
    ProtoFile = filename:join(TmpDir, "x.proto"),
    ETxtFile = filename:join(TmpDir, "x.etxt"),
    EMsgFile = filename:join(TmpDir, "x.emsg"),
    PMsgFile = filename:join(TmpDir, "x.pmsg"),
    TxtFile = filename:join(TmpDir, "x.txt"),
    MsgName = element(1, Msg),
    ok = file:write_file(ETxtFile, iolist_to_binary(f("~p~n", [Msg]))),
    ok = file:write_file(EMsgFile, GpbBin),
    Protoc = find_protoc(),
    DRStr = os:cmd(f("'~s' --proto_path '~s'"
                     " --decode=~s '~s'"
                     " < '~s' > '~s'; echo $?~n",
                     [Protoc, TmpDir, MsgName, ProtoFile, EMsgFile, TxtFile])),
    0 = list_to_integer(lib:nonl(DRStr)),
    ERStr = os:cmd(f("'~s' --proto_path '~s'"
                     " --encode=~s '~s'"
                     " < '~s' > '~s'; echo $?~n",
                     [Protoc, TmpDir, MsgName, ProtoFile, TxtFile, PMsgFile])),
    0 = list_to_integer(lib:nonl(ERStr)),
    {ok, ProtoBin} = file:read_file(PMsgFile),
    ProtoBin.


f(F,A) -> io_lib:format(F,A).

check_protoc_can_do_map_fields() ->
    case find_protoc_version() of
        {ok, Vsn} when Vsn >= [3,0] ->
            true;
        {ok, _} ->
            false;
        {error,Reason} ->
            error(Reason)
    end.

find_protoc_version() ->
    Output = os:cmd(find_protoc() ++ " --version"),
    case find_protoc_version_aux(string:tokens(Output, " \t\r\n"), Output) of
        {ok, _}=Res -> Res;
        {error, X}=Res ->
            io:format(user,"Trouble finding protoc version in ~s~n", [X]),
            Res
    end.

find_protoc() ->
    case os:getenv("PROTOC") of
        false  -> os:find_executable("protoc");
        Protoc -> Protoc
    end.

find_protoc_version_aux(["libprotoc", VersionStr | _], All) ->
    try {ok, [list_to_integer(X) || X <- string:tokens(VersionStr, ".")]}
    catch error:badarg -> {error, {failed_to_interpret, VersionStr, All}}
    end;
find_protoc_version_aux([_ | Rest], All) ->
    find_protoc_version_aux(Rest, All);
find_protoc_version_aux([], All) ->
    {error, {no_version_string_found, All}}.