@Test
    void testExactMatchTokenizationEndsByColon() {
        SearchDefinition sd = new SearchDefinition("testsd");

        Index index1 = new Index("testexact1");
        index1.setExact(true, null);
        sd.addIndex(index1);

        Index index2 = new Index("testexact2");
        index2.setExact(true, "()/aa*::*&");
        sd.addIndex(index2);

        IndexFacts facts = new IndexFacts(new IndexModel(sd));
        Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());
        IndexFacts.Session session = facts.newSession(Set.of(), Set.of());
        List<?> tokens = tokenizer.tokenize("normal a:b (normal testexact1:!/%#%&+-+ ) testexact2:ho_/&%&/()/aa*::*&b:", session);
        assertEquals(new Token(WORD, "normal"), tokens.get(0));
        assertEquals(new Token(SPACE, " "), tokens.get(1));
        assertEquals(new Token(WORD, "a"), tokens.get(2));
        assertEquals(new Token(COLON, ":"), tokens.get(3));
        assertEquals(new Token(WORD, "b"), tokens.get(4));
        assertEquals(new Token(SPACE, " "), tokens.get(5));
        assertEquals(new Token(LBRACE, "("), tokens.get(6));
        assertEquals(new Token(WORD, "normal"), tokens.get(7));
        assertEquals(new Token(SPACE, " "), tokens.get(8));
        assertEquals(new Token(WORD, "testexact1"), tokens.get(9));
        assertEquals(new Token(COLON, ":"), tokens.get(10));
        assertEquals(new Token(WORD, "!/%#%&+-+"), tokens.get(11));
        assertEquals(new Token(SPACE, " "), tokens.get(12));
        assertEquals(new Token(RBRACE, ")"), tokens.get(13));
        assertEquals(new Token(SPACE, " "), tokens.get(14));
        assertEquals(new Token(WORD, "testexact2"), tokens.get(15));
        assertEquals(new Token(COLON, ":"), tokens.get(16));
        assertEquals(new Token(WORD, "ho_/&%&/"), tokens.get(17));
        assertEquals(new Token(WORD, "b"), tokens.get(18));
        assertEquals(new Token(COLON, ":"), tokens.get(19));
    }
In cjk languages, special tokens must be recognized as substrings of strings not
separated by space, as special token recognition happens before tokenization
container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java