public Map<String, Tensor> evaluate(String modelName, Map<String, Tensor> inputs, Set<String> outputNames) {
        var requestBuilder = GrpcService.ModelInferRequest.newBuilder()
                .setModelName(modelName);

        // Get model metadata to convert vespa tensor types to onnx types
        var metadata = invokeGrpc(grpcInferenceStub, s -> s.modelMetadata(
                GrpcService.ModelMetadataRequest.newBuilder()
                        .setName(modelName)
                        .build()));

        inputs.forEach((name, tensor) -> addInputToBuilder(metadata.getInputsList(), requestBuilder, tensor, name));

        // Returns all output if none is specified
        outputNames.forEach(name -> requestBuilder.addOutputs(
                GrpcService.ModelInferRequest.InferRequestedOutputTensor.newBuilder()
                        .setName(name)
                        .build()));

        var response = invokeGrpc(grpcInferenceStub, s -> s.modelInfer(requestBuilder.build()));

        Map<String, Tensor> outputs = new HashMap<>();
        for (int i = 0; i < response.getOutputsCount(); i++) {
            var tritonTensor = response.getOutputs(i);
            var name = OnnxImporter.asValidIdentifier(tritonTensor.getName());
            var outputBuffer =
                    ByteBuffer.wrap(response.getRawOutputContents(i).toByteArray())
                            .order(ByteOrder.LITTLE_ENDIAN);
            var tensor = createTensorFromRawOutput(outputBuffer, tritonTensor.getDatatype(), tritonTensor.getShapeList());
            outputs.put(name, tensor);
        }

        return outputs;
    }
Experimental model inference using Nvidia Triton as ONNX backend.

@author bjorncs
model-integration/src/main/java/ai/vespa/triton/TritonOnnxClient.java