private void tokenizeSnippet(List<FieldPart> resultParts, String content) {
        int head = 0;
        int tail = 0;
        boolean justFinishedIncompleteAnnotation = false;
        int numRawHighLightChars = 0;
        List<FieldPart> localParts = new ArrayList<>();
        if (content.length() == 0) {
            return;
        }

        boolean prevHeadLetterOrDigital = Character.isLetterOrDigit(content.charAt(0));

        for ( ;head < content.length(); head++) {
            char headChar = content.charAt(head);
            if (isAnnotationChar(headChar)) {
                if (headChar == AnnotateStringFieldPart.RAW_ANNOTATE_BEGIN_CHAR) {
                    int nextHead = content.indexOf(AnnotateStringFieldPart.RAW_ANNOTATE_END_CHAR, head);
                    boolean incompleteAnnotation = (nextHead == -1);
                    boolean skippedInvalidHighlightChar = false;
                    if (head > tail) {
                        int currHead = head;
                        if (incompleteAnnotation &&
                            content.charAt(head-1) == JuniperSearcher.RAW_HIGHLIGHT_CHAR &&
                            numRawHighLightChars % 2 == 1)
                        {
                            currHead--; // skip invalid highlight char
                            skippedInvalidHighlightChar = true;
                        }
                        localParts.add(createToken(content.substring(tail, currHead), prevHeadLetterOrDigital));
                    }
                    if (!skippedInvalidHighlightChar) {
                        localParts.add(new AnnotateStringFieldPart(content, head));
                    }
                    head = nextHead;
                } else if (headChar == AnnotateStringFieldPart.RAW_ANNOTATE_SEPARATOR_CHAR) {
                    localParts.clear();
                    head = content.indexOf(AnnotateStringFieldPart.RAW_ANNOTATE_END_CHAR, head);
                    justFinishedIncompleteAnnotation = true;
                } else if (headChar == AnnotateStringFieldPart.RAW_ANNOTATE_END_CHAR) {
                    localParts.clear();
                    justFinishedIncompleteAnnotation = true;
                }
                if (head == -1) {
                    head = content.length();
                } else {
                    if (head + 1 < content.length()) {
                        prevHeadLetterOrDigital = Character.isLetterOrDigit(content.charAt(head + 1));
                    }
                }
                tail = head + 1;
            } else {
                if (headChar == JuniperSearcher.RAW_HIGHLIGHT_CHAR) {
                    if (justFinishedIncompleteAnnotation) {
                        tail = head + 1; // skip invalid highlight char
                    } else {
                        ++numRawHighLightChars;
                    }
                }
                boolean currHeadLetterOrDigital = Character.isLetterOrDigit(headChar);
                if (currHeadLetterOrDigital != prevHeadLetterOrDigital & head > tail) {
                    localParts.add(createToken(content.substring(tail, head), prevHeadLetterOrDigital));
                    tail = head;
                    prevHeadLetterOrDigital = currHeadLetterOrDigital;
                }
                justFinishedIncompleteAnnotation = false;
            }
        }
        if (head > tail) {
            localParts.add(createToken(content.substring(tail), prevHeadLetterOrDigital));
        }
        resultParts.addAll(localParts);
    }
Creates a hit field

@param fieldname The field name
@param content The field content
@param cjk true if the content is CJK text
@param xmlProperty true if this should not quote XML syntax
container-search/src/main/java/com/yahoo/prelude/hitfield/HitField.java