public Iterable<Token> tokenize(String input, Function<String, String> tokenProcessor) {
        if (input.isEmpty()) return List.of();

        List<Token> tokens = new ArrayList<>();
        int nextCode = input.codePointAt(0);
        TokenType prevType = SimpleTokenType.valueOf(nextCode);
        TokenType tokenType = prevType;
        TokenScript prevScript  = SimpleTokenScript.valueOf(nextCode);
        TokenScript tokenScript = prevScript;
        for (int prev = 0, next = Character.charCount(nextCode); next <= input.length(); ) {
            nextCode = next < input.length() ? input.codePointAt(next) : SPACE_CODE;
            TokenType nextType = SimpleTokenType.valueOf(nextCode);
            TokenScript nextScript = SimpleTokenScript.valueOf(nextCode);

            if (isAtTokenBoundary(prevType, nextType)) {
                String original = input.substring(prev, next);
                tokens.add(new SimpleToken(original).setOffset(prev)
                                                    .setType(tokenType)
                                                    .setTokenString(tokenProcessor.apply(original))
                                                    .setScript(tokenScript));
                prev = next;
                prevType = nextType;
                prevScript = nextScript;
                tokenType = prevType;
                tokenScript = prevScript;
            }
            else {
                tokenType = determineType(tokenType, nextType);
                tokenScript = determineScript(tokenScript, nextScript);
            }
            next += Character.charCount(nextCode);
        }
        return tokens;
    }
Tokenize the input, and apply the given transform to each token string.
linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java