check_inference.py

import numpy as np
import cv2
import os
import tritonclient.http as httpclient
import time
import random

NUM_TRIES = 10
MODEL_NAME = 'XXXX'

triton_client = httpclient.InferenceServerClient(url='localhost:8000')

try:
    server_status = triton_client.is_server_live()
except Exception as e:
    print(f'Could not get inference server status from localhost:8000')

# triton_client.load_model('200713_105933_Inception_V3_TF2_Test2_0k')

image_dir_list = ['./tests/camera/images/' + f for f in os.listdir('./tests/camera/images/')]

times = []
old_results = None
results = []
overall_start = time.time()
for _ in range(NUM_TRIES):
    image_0 = cv2.imread(random.sample(image_dir_list, 1)[0]).astype(np.float32)[0:299, 0:299]
    image_1 = cv2.imread(random.sample(image_dir_list, 1)[0]).astype(np.float32)[0:299, 0:299]
    images = np.array([image_0, image_1])
    inputs = [httpclient.InferInput('input', images.shape, 'FP32')]
    inputs[0].set_data_from_numpy(images, binary_data=False)
    start = time.time()
    try:
        # results.append(triton_client.async_infer(model_name=MODEL_NAME, inputs=inputs))
        print(triton_client.infer(model_name=MODEL_NAME, inputs=inputs).as_numpy('out'))
        end = time.time()
        times.append(end - start)
    except Exception as e:
        print(f'Could not run inference for: {MODEL_NAME} due to: {str(e)}')

# for result in results:
#     print(result.get_result(block=True).as_numpy('out'))

overall_end = time.time()
print(f'total time is {overall_end - overall_start}')

avg_time = sum(times)/len(times)
print(f'average time is {avg_time}')
print(times)