public boolean annotate(StringFieldValue text) {
        if (text.getSpanTree(SpanTrees.LINGUISTICS) != null) return true;  // Already annotated with LINGUISTICS.

        Tokenizer tokenizer = factory.getTokenizer();
        String input = (text.getString().length() <= config.getMaxTokenizeLength())
                       ? text.getString()
                       : Text.substringByCodepoints(text.getString(), 0, config.getMaxTokenizeLength());
        Iterable<Token> tokens = tokenizer.tokenize(input, config.asLinguisticsParameters());
        TermOccurrences termOccurrences = new TermOccurrences(config.getMaxTermOccurrences());
        SpanTree tree = new SpanTree(SpanTrees.LINGUISTICS);
        for (Token token : tokens)
            addAnnotationSpan(text.getString(), tree.spanList(), token, config.getStemMode(), config.getLowercase(),
                              termOccurrences, config.getMaxTokenLength());

        if (tree.numAnnotations() == 0) return false;
        text.setSpanTree(tree);
        return true;
    }
Annotates the given string with the appropriate linguistics annotations.

@param text the text to annotate
@return whether anything was annotated
indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java