a benchmark on model serving: i/o format, data processing and beyond

detection.md

• 400 images
• DB network for detection
• single V100 GPU

serialization	image encoding	total processing time (s)	data size
msgpack	jpeg bytes	22, 21, 22	332,830
bson	jpeg bytes	22, 22, 22	332,840
json	jpeg bytes	bytes not JSON serializable
json	base64	23, 23, 23	443,774
bson	base64	22, 22, 22	443, 777
msgpack	base64	22, 23, 23	443,767

Note: total processing time includes deserialization, decoding and network inference

Conclusion: use msgpack + raw jpeg bytes for communication

req.md

file name	disk size
req_data_1.json	3.8M
req_data_2.json	263K
req_data_3.bin	2.9M

(test image size: 1000, 234K)

req.py

import json
import cv2
import base64

img = cv2.imread('test.jpg')

# type 1
im_byte = img.tobytes()
im_shape = list(img.shape)
data = {
    'img_bytes': base64.b64encode(im_byte).decode('ascii'),
    'img_shape': im_shape
}
json.dump(data, open('req_data_1.json', 'w'))

# type 2
im_str = cv2.imencode('.jpg', img)[1].tostring()
b64 = base64.b64encode(im_str)
data = {
    'img_b64str': b64.decode('ascii')
}
json.dump(data, open('req_data_2.json', 'w'))

# type 3
open('req_data_3.bin', 'wb').write(im_byte)

resp.md

file name	disk size
res_data_1.json	1.6K
res_data_2.json	715B
res_data_3.bin	528B

resp.py

import json
import numpy as np
import base64

det_box = [[[694, 909], [761, 910], [761, 922], [694, 921]], [[700, 890], [762, 890], [762, 903], [700, 903]],
           [[593, 885], [652, 885], [652, 897], [593, 897]], [[360, 887], [403, 879], [406, 892], [363, 901]],
           [[590, 871], [657, 871], [657, 884], [590, 884]], [[369, 813], [394, 813], [394, 827], [369, 827]],
           [[690, 758], [770, 758], [770, 773], [690, 773]], [[594, 752], [653, 750], [653, 763], [594, 765]],
           [[281, 744], [381, 745], [381, 762], [281, 761]], [[285, 731], [380, 733], [380, 748], [285, 746]],
           [[696, 723], [763, 723], [763, 740], [696, 740]], [[588, 722], [659, 722], [659, 742], [588, 742]],
           [[485, 700], [515, 697], [521, 782], [491, 784]], [[282, 685], [380, 684], [380, 697], [282, 698]],
           [[495, 675], [516, 675], [516, 713], [495, 713]], [[287, 664], [377, 664], [377, 683], [287, 683]],
           [[281, 644], [382, 645], [382, 662], [281, 661]], [[302, 630], [361, 630], [361, 642], [302, 642]],
           [[325, 589], [381, 587], [381, 601], [325, 603]], [[303, 564], [406, 561], [407, 587], [304, 590]],
           [[497, 487], [777, 489], [777, 511], [497, 509]], [[498, 452], [550, 447], [553, 483], [502, 487]],
           [[546, 415], [773, 413], [775, 488], [547, 490]], [[500, 420], [548, 420], [548, 454], [500, 454]],
           [[629, 385], [773, 387], [773, 413], [629, 411]], [[498, 383], [626, 388], [625, 417], [497, 412]],
           [[417, 287], [683, 287], [683, 326], [417, 326]], [[673, 285], [786, 285], [786, 328], [673, 328]],
           [[421, 231], [613, 233], [613, 281], [421, 279]], [[197, 169], [417, 169], [417, 342], [197, 342]],
           [[589, 107], [754, 108], [754, 123], [589, 122]], [[588, 71], [738, 73], [738, 106], [588, 104]],
           [[239, 69], [431, 69], [431, 111], [239, 111]]]

print(len(det_box))
print(type(det_box[0][0][0]))

# type 1
json.dump({'res': det_box}, open('res_data_1.json', 'w'))

# type 2
npBytes = np.array(det_box, dtype=np.int16).tobytes()
json.dump({'res': base64.b64encode(npBytes).decode('ascii')},
          open('res_data_2.json', 'w'))

# type 3
open('res_data_3.bin', 'wb').write(npBytes)