@Test
    void testCjk() {
        HitField hf = new HitField("boo", "hmm\u001fgr");
        assertEquals(2, hf.getTokenizedContent().size());
        assertEquals("hmmgr", hf.getContent());
        List l = new ArrayList();
        l.add(new StringFieldPart("foo", true));
        l.add(new StringFieldPart("bar", true));
        hf.setTokenizedContent(l);
        assertEquals("foobar", hf.getContent());
    }
Tests the HitField class

@author Lars Chr Jensen
container-search/src/test/java/com/yahoo/prelude/hitfield/test/HitFieldTestCase.java