private String processToken(String token, LinguisticsParameters parameters) {
        String original = token;
        log.log(Level.FINEST, () -> "processToken '" + original + "'");
        token = normalizer.normalize(token);
        if (parameters.lowercase())
            token = LinguisticsCase.toLowerCase(token);
        if (parameters.removeAccents())
            token = transformer.accentDrop(token, parameters.language());
        if (parameters.stemMode() != StemMode.NONE) {
            String oldToken = token;
            token = stemmer.stem(token);
            String newToken = token;
            log.log(Level.FINEST, () -> "stem '" + oldToken + "' to '" + newToken + "'");
        }
        String result = token;
        log.log(Level.FINEST, () -> "processed token is: " + result);
        return result;
    }
Tokenize the input, and apply the given transform to each token string.
linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java