private List<FieldPart> tokenizeUnknown() {
        List<FieldPart> pre = new ArrayList<>();
        if (rawContent.length() == 0)
            return pre;
        int i = 0;
        int j = 0;
        i = rawContent.indexOf('\u001E');
        if (i == 0) {
            pre.add(new SeparatorFieldPart(rawContent.substring(0,1)));
            j = 1;
            i = rawContent.indexOf('\u001E', j);
        }
        while(i != -1) {
            tokenizeSnippet(pre, rawContent.substring(j, i));
            pre.add(new SeparatorFieldPart(rawContent.substring(i,i+1)));
            i++;
            j = i;
            i = rawContent.indexOf('\u001E', j);
        }
        if (j < rawContent.length()) {
            tokenizeSnippet(pre, rawContent.substring(j));
        }
        return pre;
    }
Creates a hit field

@param fieldname The field name
@param content The field content
@param cjk true if the content is CJK text
@param xmlProperty true if this should not quote XML syntax
container-search/src/main/java/com/yahoo/prelude/hitfield/HitField.java