private String processToken(String token, Stemmer stemmer, LinguisticsParameters linguisticsParameters) {
        token = normalizer.normalize(token);
        if (linguisticsParameters.lowercase())
            token = LinguisticsCase.toLowerCase(token);
        if (linguisticsParameters.removeAccents())
            token = transformer.accentDrop(token, linguisticsParameters.language());
        if (linguisticsParameters.stemMode() != StemMode.NONE) {
            String tokenOrEmpty = stemmer.stem(token).toString();
            if (!tokenOrEmpty.isEmpty())
                token = tokenOrEmpty;
        }
        return token;
    }
Chinese segmenter if we're segmenting Chinese
opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java