@Test
    void testTokenReplacing() {
        Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());
        tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("replacing"));

        List<?> tokens = tokenizer.tokenize("with space, c++ or .... know, not b.s.d.");
        assertEquals(new Token(WORD, "with-space"), tokens.get(0));
        assertEquals(new Token(COMMA, ","), tokens.get(1));
        assertEquals(new Token(SPACE, " "), tokens.get(2));
        assertEquals(new Token(WORD, "cpp"), tokens.get(3));
        assertEquals(new Token(SPACE, " "), tokens.get(4));
        assertEquals(new Token(WORD, "or"), tokens.get(5));
        assertEquals(new Token(SPACE, " "), tokens.get(6));
        assertEquals(new Token(WORD, "...."), tokens.get(7));
        assertEquals(new Token(SPACE, " "), tokens.get(8));
        assertEquals(new Token(WORD, "knuwww"), tokens.get(9));
        assertEquals(new Token(COMMA, ","), tokens.get(10));
        assertEquals(new Token(SPACE, " "), tokens.get(11));
        assertEquals(new Token(WORD, "not"), tokens.get(12));
        assertEquals(new Token(SPACE, " "), tokens.get(13));
        assertEquals(new Token(WORD, "b.s.d."), tokens.get(14));

        assertTrue(((Token) tokens.get(9)).isSpecial());
        assertFalse(((Token) tokens.get(12)).isSpecial());
    }
In cjk languages, special tokens must be recognized as substrings of strings not
separated by space, as special token recognition happens before tokenization
container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java