TF Model Optimization 5

inference_tfserving.py

def inference_tfserving(eval_data, batch=BATCH_SIZE, 
                        repeat=1000, signature='predict'):
  url = 'http://localhost:8501/v1/models/mnist_classifier:predict'

  instances = [[float(i) for i in list(eval_data[img])] for img in range(batch)]

  request_data = {'signature_name': signature, 
                  'instances': instances}
  time_start = datetime.utcnow() 
  for i in range(repeat):
    response = requests.post(url, data=json.dumps(request_data))
  time_end = datetime.utcnow() 
  time_elapsed_sec = (time_end - time_start).total_seconds()

  print('Total elapsed time: {} seconds'.format(time_elapsed_sec))
  print('Time for batch size {} repeated {} times'.format(BATCH_SIZE, repeat))
  print('Average latency per batch: {} seconds'.format(time_elapsed_sec/repeat))