@Test
    void testSpecialTokenNonMatch() {
        Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());

        tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("default"));
        List<?> tokens = tokenizer.tokenize("c++ c+ aS/400 i/o .net i/ooo ap.net");

        assertEquals(new Token(WORD, "c++"), tokens.get(0));
        assertEquals(new Token(SPACE, " "), tokens.get(1));
        assertEquals(new Token(WORD, "c+"), tokens.get(2));
        assertEquals(new Token(SPACE, " "), tokens.get(3));
        assertEquals(new Token(WORD, "as/400"), tokens.get(4));
        assertEquals(new Token(SPACE, " "), tokens.get(5));
        assertEquals(new Token(WORD, "i/o"), tokens.get(6));
        assertEquals(new Token(SPACE, " "), tokens.get(7));
        assertEquals(new Token(WORD, ".net"), tokens.get(8));
        assertEquals(new Token(SPACE, " "), tokens.get(9));
        assertEquals(new Token(WORD, "i"), tokens.get(10));
        assertEquals(new Token(NOISE, "<NOISE>"), tokens.get(11));
        assertEquals(new Token(WORD, "ooo"), tokens.get(12));
        assertEquals(new Token(SPACE, " "), tokens.get(13));
        assertEquals(new Token(WORD, "ap"), tokens.get(14));
        assertEquals(new Token(WORD, ".net"), tokens.get(15));
    }
In cjk languages, special tokens must be recognized as substrings of strings not
separated by space, as special token recognition happens before tokenization
container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java