examples/research_projects/quantization-qdqbert/evaluate-hf-trt-qa.py/model_infer
def model_infer(inputs, context, d_inputs, h_output0, h_output1, d_output0, d_output1, stream):
    input_ids = np.asarray(inputs["input_ids"], dtype=np.int32)
    attention_mask = np.asarray(inputs["attention_mask"], dtype=np.int32)
    token_type_ids = np.asarray(inputs["token_type_ids"], dtype=np.int32)

    # Copy inputs
    cuda.memcpy_htod_async(d_inputs[0], input_ids.ravel(), stream)
    cuda.memcpy_htod_async(d_inputs[1], attention_mask.ravel(), stream)
    cuda.memcpy_htod_async(d_inputs[2], token_type_ids.ravel(), stream)
    # start time
    start_time = time.time()
    # Run inference
    context.execute_async(
        bindings=[int(d_inp) for d_inp in d_inputs] + [int(d_output0), int(d_output1)], stream_handle=stream.handle
    )
    # Transfer predictions back from GPU
    cuda.memcpy_dtoh_async(h_output0, d_output0, stream)
    cuda.memcpy_dtoh_async(h_output1, d_output1, stream)
    # Synchronize the stream and take time
    stream.synchronize()
    # end time
    end_time = time.time()
    infer_time = end_time - start_time
    outputs = (h_output0, h_output1)
    # print(outputs)
    return outputs, infer_time