@Test
    void testSpecialTokenConfigurationOther() {
        Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());

        tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("other"));
        List<?> tokens = tokenizer.tokenize(
                "with space,!!!*** [huh] or ------ " + "know, &&&%%% b.s.d.");

        assertEquals(new Token(WORD, "with"), tokens.get(0));
        assertEquals(new Token(SPACE, " "), tokens.get(1));
        assertEquals(new Token(WORD, "space"), tokens.get(2));
        assertEquals(new Token(COMMA, ","), tokens.get(3));
        assertEquals(new Token(WORD, "!!!***"), tokens.get(4));
        assertEquals(new Token(SPACE, " "), tokens.get(5));
        assertEquals(new Token(WORD, "[huh]"), tokens.get(6));
        assertEquals(new Token(SPACE, " "), tokens.get(7));
        assertEquals(new Token(WORD, "or"), tokens.get(8));
        assertEquals(new Token(SPACE, " "), tokens.get(9));
        assertEquals(new Token(WORD, "------"), tokens.get(10));
        assertEquals(new Token(SPACE, " "), tokens.get(11));
        assertEquals(new Token(WORD, "know"), tokens.get(12));
        assertEquals(new Token(COMMA, ","), tokens.get(13));
        assertEquals(new Token(SPACE, " "), tokens.get(14));
        assertEquals(new Token(WORD, "&&&%%%"), tokens.get(15));
        assertEquals(new Token(SPACE, " "), tokens.get(16));
        assertEquals(new Token(WORD, "b"), tokens.get(17));
        assertEquals(new Token(DOT, "."), tokens.get(18));
        assertEquals(new Token(WORD, "s"), tokens.get(19));
        assertEquals(new Token(DOT, "."), tokens.get(20));
        assertEquals(new Token(WORD, "d"), tokens.get(21));
        assertEquals(new Token(DOT, "."), tokens.get(22));

        assertTrue(((Token) tokens.get(10)).isSpecial());
    }
In cjk languages, special tokens must be recognized as substrings of strings not
separated by space, as special token recognition happens before tokenization
container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java