public List<Token> tokenize(String string, String defaultIndexName, IndexFacts.Session indexFacts) {
        this.source = string;
        tokens.clear();
        parensToEat = 0;
        Index topLevelIndex = Index.nullIndex;
        Index defaultIndex = indexFacts.getIndex(defaultIndexName);
        if (defaultIndexName != null) {
            topLevelIndex = defaultIndex;
        }
        Index currentIndex = topLevelIndex;
        for (int i = 0; i < source.length(); i++) {
            if (currentIndex.isExact()) {
                i = consumeExact(i, currentIndex); // currentIndex may change after seeing a colon below
                currentIndex = topLevelIndex;
            }
            else {
                i = consumeSpecialToken(i);
            }

            if (i >= source.length()) break;

            int c = source.codePointAt(i);
            if (characterClasses.isSymbol(c)) { // treat each symbol is a separate word
                addToken(WORD, Character.toString(c), i, i + 1);
            } else if (characterClasses.isLetterOrDigit(c) || (c == '\'' && acceptApostropheAsWordCharacter(currentIndex))) {
                i = consumeWordOrNumber(i, currentIndex);
            } else if (Character.isWhitespace(c)) {
                addToken(SPACE, " ", i, i + 1);
            } else if ((c == '\u3001' || c == '\u3002') && ! parserSettings.keepIdeographicPunctuation()) {
                addToken(SPACE, " ", i, i + 1);
            } else if (c == '"' || c == '\u201C' || c == '\u201D'
                    || c == '\u201E' || c == '\u201F' || c == '\u2039'
                    || c == '\u203A' || c == '\u00AB' || c == '\u00BB'
                    || c == '\u301D' || c == '\u301E' || c == '\u301F'
                    || c == '\uFF02') {
                addToken(QUOTE, "\"", i, i + 1);
            } else if (c == '-' || c == '\uFF0D') {
                addToken(MINUS, "-", i, i + 1);
            } else if (c == '+' || c == '\uFF0B') {
                addToken(PLUS, "+", i, i + 1);
            } else if (c == '.' || c == '\uFF0E') {
                addToken(DOT, ".", i, i + 1);
            } else if (c == ',' || c == '\uFF0C') {
                addToken(COMMA, ",", i, i + 1);
            } else if (c == ':' || c == '\uFF1A') {
                currentIndex = determineCurrentIndex(defaultIndex, indexFacts);
                addToken(COLON, ":", i, i + 1);
            } else if (c == '(' || c == '\uFF08') {
                addToken(LBRACE, "(", i, i + 1);
                parensToEat++;
            } else if (c == ')' || c == '\uFF09') {
                addToken(RBRACE, ")", i, i + 1);
                parensToEat--;
                if (parensToEat < 0) parensToEat = 0;
            } else if (c == '[' || c == '\uFF3B') {
                addToken(LSQUAREBRACKET, "[", i, i + 1);
            } else if (c == ']' || c == '\uFF3D') {
                addToken(RSQUAREBRACKET, "]", i, i + 1);
            } else if (c == '{') {
                addToken(LCURLYBRACKET, "{", i, i + 1);
            } else if (c == '}') {
                addToken(RCURLYBRACKET, "}", i, i + 1);
            } else if (c == ';' || c == '\uFF1B') {
                addToken(SEMICOLON, ";", i, i + 1);
            } else if (c == '>' || c == '\uFF1E') {
                addToken(GREATER, ">", i, i + 1);
            } else if (c == '<' || c == '\uFF1C') {
                addToken(SMALLER, "<", i, i + 1);
            } else if (c == '!' || c == '\uFF01') {
                addToken(EXCLAMATION, "!", i, i + 1);
            } else if (c == '_' || c == '\uFF3F') {
                addToken(UNDERSCORE, "_", i, i + 1);
            } else if (c == '^' || c == '\uFF3E') {
                addToken(HAT, "^", i, i + 1);
            } else if (c == '*' || c == '\uFF0A') {
                addToken(STAR, "*", i, i + 1);
            } else if (c == '$' || c == '\uFF04') {
                addToken(DOLLAR, "$", i, i + 1);
            } else {
                addToken(NOISE, "<NOISE>", i, i + 1);
            }
        }
        addToken(EOF, "<EOF>", source.length(), source.length());
        source = null;
        return tokens;
    }
Resets this tokenizer and create tokens from the given string.

@param string the string to tokenize
@param defaultIndexName the name of the index to use as default
@param indexFacts information about the indexes we will search
@return a read-only list of tokens. This list can only be used by this thread
container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java