protected Item segment(String indexName, Token token, boolean quoted) {
        String normalizedToken = normalize(token.toString());

        if (token.isSpecial()) {
            WordItem w = new WordItem(token.toString(), true, token.substring);
            w.setWords(false);
            w.setFromSpecialToken(true);
            return w;
        }

        if (language == Language.UNKNOWN) {
            return new WordItem(normalizedToken, true, token.substring);
        }


        Segmenter segmenter = environment.getLinguistics().getSegmenter();
        List<String> segments = segmenter.segment(normalizedToken, language);
        if (segments.isEmpty()) {
            return null;
        }

        if (segments.size() == 1) {
            return new WordItem(segments.get(0), "", true, token.substring);
        }

        CompositeItem composite;
        if (indexFacts.getIndex(indexName).getPhraseSegmenting() || quoted) {
            composite = new PhraseSegmentItem(token.toString(), normalizedToken, true, false, token.substring);
        }
        else {
            composite = new AndSegmentItem(token.toString(), true, false);
        }
        int n = 0;
        WordItem previous = null;
        for (String segment : segments) {
            WordItem w = new WordItem(segment, "", true, token.substring);
            w.setFromSegmented(true);
            w.setSegmentIndex(n++);
            w.setStemmed(false);
            if (previous != null)
                previous.setConnectivity(w, 1.0);
            previous = w;
            composite.addItem(w);
        }
        composite.lock();
        return composite;
    }
Segments a token

@param indexName the index name which preceded this token, or null if none
@param token the token to segment
@param quoted whether this segment is within quoted text
@return the resulting item
container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java