@Override
    public Iterable<Token> tokenize(String input, LinguisticsParameters parameters) {
        if (chineseSegmenter.isPresent() && ( parameters.language() == Language.CHINESE_SIMPLIFIED ||
                                              parameters.language() == Language.CHINESE_TRADITIONAL))
            return segmentChinese(input);

        Stemmer stemmer = stemmerFor(parameters.language(), parameters.stemMode());
        if (stemmer == null)
            return simpleTokenizer.tokenize(input, parameters);
        else
            return simpleTokenizer.tokenize(input, token -> processToken(token, stemmer, parameters));
    }
Chinese segmenter if we're segmenting Chinese
opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java