private Iterable<Token> segmentChinese(String input) {
        if (input.isEmpty()) return List.of();

        List<Token> tokens = new ArrayList<>();
        // In "search" mode Jieba will index grams of tokens longer than 2 characters
        var jiebaMode = ( mode == Mode.index && createCjkGrams ) ? JiebaSegmenter.SegMode.INDEX : JiebaSegmenter.SegMode.SEARCH;
        for (SegToken token : chineseSegmenter.get().process(input, jiebaMode)) {
            int nextCode = token.word.codePointAt(0);
            TokenType tokenType = SimpleTokenType.valueOf(nextCode);
            String originToken = input.substring(token.startOffset, token.startOffset + token.word.length());
            SimpleToken simpleToken = new SimpleToken(originToken)
                                              .setOffset(token.startOffset)
                                              .setType(tokenType)
                                              .setTokenString(token.word);
            tokens.add(simpleToken);
        }
        return tokens;
    }
Chinese segmenter if we're segmenting Chinese
opennlp-linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java