private int consumeWordOrNumber(int start, Index currentIndex) {
        int tokenEnd = start;
        SpecialTokens.Token substringToken = null;
        boolean digitsOnly = true;
        // int underscores = 0;
        // boolean underscoresOnly = true;
        boolean quotesOnly = true;

        while (tokenEnd < source.length()) {
            if (substringSpecialTokens) {
                substringToken = getSpecialToken(tokenEnd);
                if (substringToken != null) break;
            }

            int c = source.codePointAt(tokenEnd);

            if (characterClasses.isLetter(c)) {
                digitsOnly = false;
                // if (c != '_') {
                //     if (underscores > 3) {
                //         break;
                //     } else {
                //         underscores = 0;
                //     }
                //     underscoresOnly = false;
                // } else {
                //     underscores += 1;
                // }
                quotesOnly = false;
            } else if (characterClasses.isLatinDigit(c)) {
                // Yes, do nothing as long as the underscore logic
                // is deactivated.
                // underscoresOnly = false;
                quotesOnly = false;
            } else if (c == '\'') {
                if ( ! acceptApostropheAsWordCharacter(currentIndex)) {
                    break;
                }
                // Otherwise consume apostrophes...
                digitsOnly = false;
            } else {
                break;
            }
            tokenEnd += Character.charCount(c);
        }
        // if (underscores > 3 && !underscoresOnly) {
        //     tokenEnd -= underscores;
        // }
        if (tokenEnd>start) {
            // if (underscoresOnly) {
            //     addToken(NOISE, source.substring(start, tokenEnd), start, tokenEnd);
            // } else
            if (quotesOnly) {
                addToken(NOISE, source.substring(start, tokenEnd), start, tokenEnd);
            } else {
                addToken(digitsOnly ? NUMBER : WORD, source.substring(start, tokenEnd), start, tokenEnd);
            }
        }

        if (substringToken == null)
            return --tokenEnd;
        // TODO: test the logic around tokenEnd with friends
        addToken(toToken(substringToken, tokenEnd, source));
        return --tokenEnd + substringToken.token().length();
    }
Consumes a word or number <i>and/or possibly</i> a special token starting within this word or number
container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java