protected Item word(String indexName, boolean quoted) {
        int position = tokens.getPosition();
        Item item = null;

        try {
            if ( ! tokens.currentIs(WORD)
                && ((!tokens.currentIs(NUMBER) && !tokens.currentIs(MINUS)
                && !tokens.currentIs(UNDERSCORE)) || (!submodes.url && !submodes.site))) {
                return null;
            }
            Token word = tokens.next();

            if (submodes.url) {
                item = new WordItem(word, true);
            } else {
                item = segment(indexName, word, quoted);
            }

            if (submodes.url || submodes.site) {
                StringBuilder buffer = null;
                Token token = tokens.currentNoIgnore();

                while (token.kind == WORD || token.kind == NUMBER || token.kind == MINUS || token.kind == UNDERSCORE) {
                    if (buffer == null) {
                        buffer = getStringContents(item);
                    }
                    buffer.append(token);
                    tokens.skipNoIgnore();
                    token = tokens.currentNoIgnore();
                }
                if (buffer != null) {
                    Substring termSubstring = ((BlockItem) item).getOrigin();
                    Substring substring = new Substring(termSubstring.start, token.substring.start, termSubstring.getSuperstring()); // XXX: Unsafe end?
                    String str = buffer.toString();
                    item = new WordItem(str, "", true, substring);
                }
            }
            return item;
        } finally {
            if (item == null) {
                tokens.setPosition(position);
            }
        }
    }
Returns a WordItem if this is a non CJK query,
a WordItem or SegmentItem if this is a CJK query,
null if the current item is not a word

@param quoted whether this token is inside quotes
container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java