private Gram findNext() {
            // Skip to next indexable character
            while (i < input.length() && !isIndexable(input.codePointAt(i))) {
                i = input.nextIndex(i);
                isFirstAfterSeparator = true;
            }
            if (i >= input.length()) return null; // no indexable characters

            int tokenStart = i;
            UnicodeString gram = input.substring(tokenStart, n);
            int tokenEnd = tokenEnd(gram);
            gram = new UnicodeString(gram.toString().substring(0, tokenEnd));
            if (gram.codePointCount() == n) { // normal case: got a full length gram
                Gram g = new Gram(i, gram.codePointCount());
                i = input.nextIndex(i);
                isFirstAfterSeparator = false;
                return g;
            }
            else { // gram is too short due either to being a symbol, being followed by a non-word separator, or end of string
                if (isFirstAfterSeparator || ( gram.codePointCount() == 1 && characterClasses.isSymbol(gram.codePointAt(0)))) { // make a gram anyway
                    Gram g = new Gram(i, gram.codePointCount());
                    i = input.nextIndex(i);
                    isFirstAfterSeparator = false;
                    return g;
                } else { // skip to next
                    i = input.skip(gram.codePointCount(), i);
                    isFirstAfterSeparator = true;
                    return findNext();
                }
            }
        }
The next gram or null if not determined yet
linguistics/src/main/java/com/yahoo/language/process/GramSplitter.java