Pack repeated fields with Nanopb library

main.c

#include <pb_decode.h>
#include <pb_encode.h>
#include <stdbool.h>
#include <stddef.h>
#include <stdio.h>

#include "Test.pb.h"

bool str_encode(pb_ostream_t *stream, const pb_field_t *field,
                void *const *arg) {
    const char *str = *arg;

    if (!pb_encode_tag_for_field(stream, field)) {
        return false;
    }

    return pb_encode_string(stream, (unsigned char *)str, strlen(str));
}

struct repeated {
    void **repeated_f;
    int index;
    size_t max_size;
};

bool req_encode(pb_ostream_t *stream, const pb_field_t *field,
                void *const *arg) {
    struct repeated *req = *arg;
    while (req->index < req->max_size) {
        SearchRequest *sreq = ((SearchRequest **)req->repeated_f)[req->index];
        printf("%s: Try to encode: %s\n", __func__, sreq->query.arg);
        ++req->index;

        if (!pb_encode_tag(stream, PB_WT_STRING, field->tag)) {
            return false;
        }

        if (!pb_encode_submessage(stream, SearchRequest_fields, sreq)) {
            return false;
        }
    }

    return true;
}

int main(void) {
    const char *str1 = "Hello";
    const char *str2 = "World";
    unsigned char buffer[128];
    size_t message_length;
    bool status;
    Result result_msg = Result_init_default;
    pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof buffer);
    SearchRequest first = {
        .query = {.funcs = {.encode = str_encode}, .arg = str1}};
    SearchRequest second = {
        .query = {.funcs = {.encode = str_encode}, .arg = str2}};
    SearchRequest *requests[] = {&first, &second};
    struct repeated r1 = {.repeated_f = requests, .index = 0, .max_size = 2};

    result_msg.requests.funcs.encode = req_encode;
    result_msg.requests.arg = &r1;
    status = pb_encode(&stream, Result_fields, &result_msg);
    message_length = stream.bytes_written;

    if (!status) {
        printf("Encoding failed: %s\n", PB_GET_ERROR(&stream));
        return -1;
    }

    printf("Success: %zu written\n", message_length);
    for (size_t i = 0; i < message_length; ++i) {
        printf("%02hhx", buffer[i]);
    }
    puts("");

    return 0;
}

Test.proto

syntax = "proto3";

message SearchRequest {
  string query = 1;
}

message Result {
  repeated SearchRequest requests = 1;
}

Hello, thank you for posting this example, It is very similar to what I was doing.
I am hunting for decode repeated field example, do you have it by any chance?

Hello, thank you for posting this example, It is very similar to what I was doing.
I am hunting for decode repeated field example, do you have it by any chance?

Hey! Nothing special with decoding. Just specify a decode function and it will be called as many time as required to decode repeated elements one by one.

There is the example:

syntax = "proto3";                                                                                                                                                          
                                                                                                                                                                             
 message Information {                                                                                                                                                       
     repeated int32 info = 1;                                                                                                                                                
 }

And the code:

#include "Temp.pb.h"

#include <pb_decode.h>
#include <pb_encode.h>

#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>

#define ARRAY_SIZE(x) (sizeof(x) / sizeof(x[0]))

static bool info_encode(pb_ostream_t *stream, const pb_field_t *field, void *const *arg);
static bool info_decode(pb_istream_t *stream, const pb_field_t *field, void **arg);
void clean_up(unsigned char **ptr) {
    puts("clean_up() call");
    free(*ptr);
}

struct array {
    unsigned int *d;
    size_t size;
};

int main(void) {
    unsigned char buffer[1024];
    Information info = Information_init_zero;
    unsigned int buf1[] = {0xCADEADBE, 0xFE, 0xFE, 0xED};
    struct array arr = {
            .d = buf1,
            .size = ARRAY_SIZE(buf1)
    };
    info.info.funcs.encode = info_encode;
    info.info.arg = &arr;
    pb_ostream_t ostream = pb_ostream_from_buffer(buffer, sizeof buffer);

    if (!pb_encode_delimited(&ostream, Information_fields, &info)) {
        fprintf(stderr, "Unable to encode Information\n");
        return 1;
    }

    for (size_t i = 0; i < ostream.bytes_written; ++i) {
        printf("%02hhx", buffer[i]);
    }

    puts("");

    {
        __attribute__((cleanup(clean_up))) unsigned char *ptr = malloc(1024);
    }

    Information info_enc = Information_init_zero;
    info_enc.info.funcs.decode = info_decode;
    pb_istream_t istream = pb_istream_from_buffer(buffer, ostream.bytes_written);

    pb_decode_delimited(&istream, Information_fields, &info_enc);

    return 0;
}

static bool info_encode(pb_ostream_t *stream, const pb_field_t *field, void *const *arg) {
    struct array *p_arr = *arg;

    for (size_t i = 0; i < p_arr->size; ++i)
    {
        // if (!pb_encode_tag(stream, PB_WT_64BIT, field->tag)) {
        //     return false;
        // }
        if (!pb_encode_tag_for_field(stream, field))
        {
            return false;
        }

        if (!pb_encode_varint(stream, p_arr->d[i]))
        {
            return false;
        }
    }

    return true;
}

static bool info_decode(pb_istream_t *stream, const pb_field_t *field, void **arg) {
    puts("info_decode()");
    printf("Bytes left: %zu\n", stream->bytes_left);

    // uint64_t len = 0;
    //
    // printf("%s: length: %lu\n", __func__, len);
    uint64_t num;

    if (!pb_decode_varint(stream, &num))
    {
        puts("Error fixed32 decode");
        return false;
    }

    printf("%u\n", (uint32_t)num);

    return true;
}

Thank you, my proto has repeated submessage and this was giving me hard time - but moved to protobuf-c and that works much better and easier to work with.

I am trying to use this as a template for decoding a repeated sub message.

syntax = "proto3";
message SubInfo {                                                                                                                                                       
     int32 data = 1;                                                                                                                                                              
 }                                                                                                                                               
 message Information {                                                                                                                                                       
     repeated SubInfo info = 1;                                                                                                                                                
 }

And then i am attempting to decode as follows

    ...
    Information info_dec = Information_init_zero;
    info_dec.info.funcs.decode = info_decode;
    pb_istream_t istream = pb_istream_from_buffer(buffer, ostream.bytes_written);

    if (!pb_decode(&istream, Information_fields, &info_dec))
    {
          (1)return false;
     }
     ...

static bool info_decode(pb_istream_t *stream, const pb_field_t *field, void **arg) {
    SubInfo sInfo = SubInfo_init_zero;
    if (!pb_decode(stream, SubInfo_fields, &sInfo))
    {
       (2) return false;
    }

    printf("sInfo.data)
    return true;
}

The first callback works, and returns true, but then it falls back to the (1) return false. I had multiple messages added to the repeated field. I tried the pb_decode_delimited, but that would immediately drop into its failed state. Any assistance would be greatly appreciated.

Good morning, Thanks for the response. What if the sub message was more complex, consisting of multiple fields and or other messages? A better setup for my question is here https://groups.google.com/g/nanopb/c/d1kiB0jFNVY. Your setup has gotten me the closest, i was able to successfully unpack the first repeated message, but the second one fails. Any insight would be appreciated. Skyler

…

On Mon, Oct 4, 2021 at 6:16 AM Vladimir Petrigo ***@***.***> wrote: ***@***.**** commented on this gist. ------------------------------ Hi @ssevans87 <https://github.com/ssevans87>. You should not call pb_decode from the callback. Instead you should call pb_decode_varing since int32 is encoded as varint with zig-zag encoding. — You are receiving this because you were mentioned. Reply to this email directly, view it on GitHub <https://gist.github.com/df85bbb8197e3be76f644bb8290444c4#gistcomment-3915862>, or unsubscribe <https://github.com/notifications/unsubscribe-auth/AGC6NEDM2X2I6IXG5MA7VMDUFGSJ5ANCNFSM43TELXBQ> . Triage notifications on the go with GitHub Mobile for iOS <https://apps.apple.com/app/apple-store/id1477376905?ct=notification-email&mt=8&pt=524675> or Android <https://play.google.com/store/apps/details?id=com.github.android&referrer=utm_campaign%3Dnotification-email%26utm_medium%3Demail%26utm_source%3Dgithub>.

@ssevans87, sorry misunderstood your question at first. Here is the snippet:
• Repeated.proto

syntax = "proto3";

message Sub {
	int32 num = 1;
	int32 num3 = 2;
}

message Top {
	repeated Sub fields = 1;
}

• dummy app:

// repeated_proto.cpp : Defines the entry point for the application.
//

#include "Repeated.pb.h"

#include <pb_encode.h>
#include <pb_decode.h>

#include <stdlib.h>
#include <stdio.h>

static int value = 10;

static bool fields_encode(pb_ostream_t* stream, const pb_field_t* field, void* const* arg)
{
	Sub message = Sub_init_default;

	for (size_t i = 0; i < 15; ++i) {
		message.num = value;
		message.num2 = value / 10;
		++value;

	    if (!pb_encode_tag_for_field(stream, field)) {
			return false;
		}

		if (!pb_encode_submessage(stream, Sub_fields, &message)) {
			return false;
		}
	}

	return true;
}

bool fields_decode(pb_istream_t* stream, const pb_field_t* field, void** arg)
{
	Sub sub = Sub_init_default;

	if (!pb_decode(stream, Sub_fields, &sub)) {
		return false;
	}

	printf("Sub - num: %d, num2: %d\r\n", sub.num, sub.num2);

	return true;
}

int main(void) {
	unsigned char encode_buf[256];

	Top message = Top_init_default;
	message.fields.funcs.encode = fields_encode;
	pb_ostream_t ostream = pb_ostream_from_buffer(encode_buf, sizeof encode_buf);
	bool status_en = pb_encode(&ostream, Top_fields, &message);

	printf("Encode status: %s, bytes written: %llu\r\n", status_en ? "OK" : "ERR", ostream.bytes_written);

	for (size_t i = 0; i < ostream.bytes_written; ++i) {
		printf("%02hhx", encode_buf[i]);
	}

	puts("");

	pb_istream_t istream = pb_istream_from_buffer(encode_buf, ostream.bytes_written);
	Top message_dec = Top_init_default;
	message_dec.fields.funcs.decode = fields_decode;

	bool status_dec = pb_decode(&istream, Top_fields, &message_dec);

	printf("Decode status: %s, bytes left: %llu\r\n", status_dec ? "OK" : "ERR", istream.bytes_left);

	return 0;
}

So, for submessages you have the following steps:
• encode tag with pb_encode_tag_for_field()
• encode payload with pb_encode_submessage()

For decoding you just decode the submessage one by one. No need to iterate manually, since Protobuf knows the field type (by tag encoded) and length that was put by submessage encoding call. That should be it.

@ssevans87 would you tell whether snippet above was helpful?

@vpetrigo Hello. I can tell, the snippet above was very helpful for me.
due to your example I was able to pack and unpack something more complex. Thanks!

Thanks a lot for the snippet!!! it was very helpful for me.