private de.kherud.llama.InferenceParameters setInferenceParameters(Prompt prompt, InferenceParameters options) {
        var inferParams = new de.kherud.llama.InferenceParameters(prompt.asString().stripLeading());

        // We always set this to some value to avoid infinite token generation
        inferParams.setNPredict(maxTokens);

        options.ifPresent(InferenceParameters.OPTION_TEMPERATURE, (v) -> inferParams.setTemperature(Float.parseFloat(v)));
        options.ifPresent(InferenceParameters.OPTION_TOP_K, (v) -> inferParams.setTopK(Integer.parseInt(v)));
        options.ifPresent(InferenceParameters.OPTION_TOP_P, (v) -> inferParams.setTopP(Integer.parseInt(v)));
        options.ifPresent(InferenceParameters.OPTION_N_PREDICT, (v) -> inferParams.setNPredict(Integer.parseInt(v)));
        options.ifPresent(InferenceParameters.OPTION_REPEAT_PENALTY, (v) -> inferParams.setRepeatPenalty(Float.parseFloat(v)));
        options.ifPresent(InferenceParameters.OPTION_SEED, (v) -> inferParams.setSeed(Integer.parseInt(v)));
        options.ifPresent(InferenceParameters.OPTION_JSON_SCHEMA, (v) -> {
            var grammar = JsonSchemaToGrammar.convert(v);
            inferParams.setGrammar(grammar);
        });
        
        inferParams.setUseChatTemplate(true);
        return inferParams;
    }
A language model running locally on the container node.

@author lesters
@author glebashnik
model-integration/src/main/java/ai/vespa/llm/clients/LocalLLM.java