@Test
    void testExactMatchHeuristics() {
        SearchDefinition sd = new SearchDefinition("testsd");

        Index index1 = new Index("testexact1");
        index1.setExact(true, null);
        sd.addIndex(index1);

        Index index2 = new Index("testexact2");
        index2.setExact(true, "()/aa*::*&");
        sd.addIndex(index2);

        IndexFacts indexFacts = new IndexFacts(new IndexModel(sd));
        IndexFacts.Session facts = indexFacts.newSession(Set.of(), Set.of());

        Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());
        List<?> tokens = tokenizer.tokenize("normal a:b (normal testexact1:foo) testexact2:bar", facts);
        assertEquals(new Token(WORD, "normal"), tokens.get(0));
        assertEquals(new Token(SPACE, " "), tokens.get(1));
        assertEquals(new Token(WORD, "a"), tokens.get(2));
        assertEquals(new Token(COLON, ":"), tokens.get(3));
        assertEquals(new Token(WORD, "b"), tokens.get(4));
        assertEquals(new Token(SPACE, " "), tokens.get(5));
        assertEquals(new Token(LBRACE, "("), tokens.get(6));
        assertEquals(new Token(WORD, "normal"), tokens.get(7));
        assertEquals(new Token(SPACE, " "), tokens.get(8));
        assertEquals(new Token(WORD, "testexact1"), tokens.get(9));
        assertEquals(new Token(COLON, ":"), tokens.get(10));
        assertEquals(new Token(WORD, "foo"), tokens.get(11));
        assertEquals(new Token(RBRACE, ")"), tokens.get(12));
        assertEquals(new Token(SPACE, " "), tokens.get(13));
        assertEquals(new Token(WORD, "testexact2"), tokens.get(14));
        assertEquals(new Token(COLON, ":"), tokens.get(15));
        assertEquals(new Token(WORD, "bar"), tokens.get(16));

        tokens = tokenizer.tokenize("testexact1:a*teens", facts);
        assertEquals(new Token(WORD, "testexact1"), tokens.get(0));
        assertEquals(new Token(COLON, ":"),         tokens.get(1));
        assertEquals(new Token(WORD, "a*teens"),    tokens.get(2));

        tokens = tokenizer.tokenize("testexact1:foo\"bar", facts);
        assertEquals(new Token(WORD, "testexact1"), tokens.get(0));
        assertEquals(new Token(COLON, ":"),         tokens.get(1));
        assertEquals(new Token(WORD, "foo\"bar"),   tokens.get(2));

        tokens = tokenizer.tokenize("testexact1:foo!bar", facts);
        assertEquals(new Token(WORD, "testexact1"), tokens.get(0));
        assertEquals(new Token(COLON, ":"),         tokens.get(1));
        assertEquals(new Token(WORD, "foo!bar"),    tokens.get(2));

        tokens = tokenizer.tokenize("testexact1:foo! ", facts);
        assertEquals(new Token(WORD, "testexact1"), tokens.get(0));
        assertEquals(new Token(COLON, ":"),         tokens.get(1));
        assertEquals(new Token(WORD, "foo"),        tokens.get(2));
        assertEquals(new Token(EXCLAMATION, "!"),   tokens.get(3));
        assertEquals(new Token(SPACE, " "),         tokens.get(4));

        tokens = tokenizer.tokenize("testexact1:foo!! ", facts);
        assertEquals(new Token(WORD, "testexact1"), tokens.get(0));
        assertEquals(new Token(COLON, ":"),         tokens.get(1));
        assertEquals(new Token(WORD, "foo"),        tokens.get(2));
        assertEquals(new Token(EXCLAMATION, "!"),   tokens.get(3));
        assertEquals(new Token(EXCLAMATION, "!"),   tokens.get(4));
        assertEquals(new Token(SPACE, " "),         tokens.get(5));

        tokens = tokenizer.tokenize("testexact1:foo!100 ", facts);
        assertEquals(new Token(WORD, "testexact1"), tokens.get(0));
        assertEquals(new Token(COLON, ":"),         tokens.get(1));
        assertEquals(new Token(WORD, "foo"),        tokens.get(2));
        assertEquals(new Token(EXCLAMATION, "!"),   tokens.get(3));
        assertEquals(new Token(NUMBER, "100"),      tokens.get(4));
        assertEquals(new Token(SPACE, " "),         tokens.get(5));

        tokens = tokenizer.tokenize("testexact1:foo*!100 ", facts);
        assertEquals(new Token(WORD, "testexact1"), tokens.get(0));
        assertEquals(new Token(COLON, ":"),         tokens.get(1));
        assertEquals(new Token(WORD, "foo"),        tokens.get(2));
        assertEquals(new Token(STAR, "*"),          tokens.get(3));
        assertEquals(new Token(EXCLAMATION, "!"),   tokens.get(4));
        assertEquals(new Token(NUMBER, "100"),      tokens.get(5));
        assertEquals(new Token(SPACE, " "),         tokens.get(6));

        tokens = tokenizer.tokenize("testexact1: *\"foo bar\"*!100 ", facts);
        assertEquals(new Token(WORD, "testexact1"), tokens.get(0));
        assertEquals(new Token(COLON, ":"),         tokens.get(1));
        assertEquals(new Token(STAR, "*"),          tokens.get(2));
        assertEquals(new Token(WORD, "foo bar"),        tokens.get(3));
        assertEquals(new Token(STAR, "*"),          tokens.get(4));
        assertEquals(new Token(EXCLAMATION, "!"),   tokens.get(5));
        assertEquals(new Token(NUMBER, "100"),      tokens.get(6));
        assertEquals(new Token(SPACE, " "),         tokens.get(7));

        tokens = tokenizer.tokenize("testexact1: *\"foo bar\"*!100", facts);
        assertEquals(new Token(WORD, "testexact1"), tokens.get(0));
        assertEquals(new Token(COLON, ":"),         tokens.get(1));
        assertEquals(new Token(STAR, "*"),          tokens.get(2));
        assertEquals(new Token(WORD, "foo bar"),        tokens.get(3));
        assertEquals(new Token(STAR, "*"),          tokens.get(4));
        assertEquals(new Token(EXCLAMATION, "!"),   tokens.get(5));
        assertEquals(new Token(NUMBER, "100"),      tokens.get(6));

        tokens = tokenizer.tokenize("testexact1: *foobar*!100", facts);
        assertEquals(new Token(WORD, "testexact1"), tokens.get(0));
        assertEquals(new Token(COLON, ":"),         tokens.get(1));
        assertEquals(new Token(STAR, "*"),          tokens.get(2));
        assertEquals(new Token(WORD, "foobar"),     tokens.get(3));
        assertEquals(new Token(STAR, "*"),          tokens.get(4));
        assertEquals(new Token(EXCLAMATION, "!"),   tokens.get(5));
        assertEquals(new Token(NUMBER, "100"),      tokens.get(6));

        tokens = tokenizer.tokenize("testexact1: *foobar*!100!", facts);
        assertEquals(new Token(WORD, "testexact1"), tokens.get(0));
        assertEquals(new Token(COLON, ":"),         tokens.get(1));
        assertEquals(new Token(STAR, "*"),          tokens.get(2));
        assertEquals(new Token(WORD, "foobar*!100"), tokens.get(3));
        assertEquals(new Token(EXCLAMATION, "!"),   tokens.get(4));

        tokens = tokenizer.tokenize("testexact1:foo(bar)", facts);
        assertEquals(new Token(WORD, "testexact1"), tokens.get(0));
        assertEquals(new Token(COLON, ":"),         tokens.get(1));
        assertEquals(new Token(WORD, "foo(bar)"),    tokens.get(2));

        tokens = tokenizer.tokenize("testexact1:\"foo\"", facts);
        assertEquals(new Token(WORD, "testexact1"), tokens.get(0));
        assertEquals(new Token(COLON, ":"),         tokens.get(1));
        assertEquals(new Token(WORD, "foo"),        tokens.get(2));

        tokens = tokenizer.tokenize("testexact1: foo", facts);
        assertEquals(new Token(WORD, "testexact1"), tokens.get(0));
        assertEquals(new Token(COLON, ":"),         tokens.get(1));
        assertEquals(new Token(WORD, "foo"),        tokens.get(2));

        tokens = tokenizer.tokenize("testexact1: \"foo\"", facts);
        assertEquals(new Token(WORD, "testexact1"), tokens.get(0));
        assertEquals(new Token(COLON, ":"),         tokens.get(1));
        assertEquals(new Token(WORD, "foo"),        tokens.get(2));

        tokens = tokenizer.tokenize("testexact1: \"foo\"", facts);
        assertEquals(new Token(WORD, "testexact1"), tokens.get(0));
        assertEquals(new Token(COLON, ":"),         tokens.get(1));
        assertEquals(new Token(WORD, "foo"),    tokens.get(2));

        tokens = tokenizer.tokenize("testexact1:vespa testexact2:resolved", facts);
        assertEquals(new Token(WORD, "testexact1"), tokens.get(0));
        assertEquals(new Token(COLON, ":"),         tokens.get(1));
        assertEquals(new Token(WORD, "vespa"),      tokens.get(2));
        assertEquals(new Token(SPACE, " "),         tokens.get(3));
        assertEquals(new Token(WORD, "testexact2"), tokens.get(4));
        assertEquals(new Token(COLON, ":"),         tokens.get(5));
        assertEquals(new Token(WORD, "resolved"),   tokens.get(6));

        tokens = tokenizer.tokenize("testexact1:\"news search\" testexact2:resolved", facts);
        assertEquals(new Token(WORD, "testexact1"), tokens.get(0));
        assertEquals(new Token(COLON, ":"),         tokens.get(1));
        assertEquals(new Token(WORD, "news search"), tokens.get(2));
        assertEquals(new Token(SPACE, " "),         tokens.get(3));
        assertEquals(new Token(WORD, "testexact2"), tokens.get(4));
        assertEquals(new Token(COLON, ":"),         tokens.get(5));
        assertEquals(new Token(WORD, "resolved"),   tokens.get(6));

        tokens = tokenizer.tokenize("(testexact1:\"news search\" testexact1:vespa)", facts);
        assertEquals(new Token(LBRACE, "("),        tokens.get(0));
        assertEquals(new Token(WORD, "testexact1"), tokens.get(1));
        assertEquals(new Token(COLON, ":"),         tokens.get(2));
        assertEquals(new Token(WORD, "news search"), tokens.get(3));
        assertEquals(new Token(SPACE, " "),         tokens.get(4));
        assertEquals(new Token(WORD, "testexact1"), tokens.get(5));
        assertEquals(new Token(COLON, ":"),         tokens.get(6));
        assertEquals(new Token(WORD, "vespa"),      tokens.get(7));
        assertEquals(new Token(RBRACE, ")"),        tokens.get(8));

        tokens = tokenizer.tokenize("testexact1:news*", facts);
        assertEquals(new Token(WORD, "testexact1"), tokens.get(0));
        assertEquals(new Token(COLON, ":"),         tokens.get(1));
        assertEquals(new Token(WORD, "news"),       tokens.get(2));
        assertEquals(new Token(STAR, "*"),          tokens.get(3));

        tokens = tokenizer.tokenize("testexact1:\"news\"*", facts);
        assertEquals(new Token(WORD, "testexact1"), tokens.get(0));
        assertEquals(new Token(COLON, ":"),         tokens.get(1));
        assertEquals(new Token(WORD, "news"),       tokens.get(2));
        assertEquals(new Token(STAR, "*"),          tokens.get(3));

        tokens = tokenizer.tokenize("testexact1:\"news search\"!200", facts);
        assertEquals(new Token(WORD, "testexact1"), tokens.get(0));
        assertEquals(new Token(COLON, ":"),         tokens.get(1));
        assertEquals(new Token(WORD, "news search"), tokens.get(2));
        assertEquals(new Token(EXCLAMATION, "!"),   tokens.get(3));
        assertEquals(new Token(NUMBER, "200"),      tokens.get(4));

        tokens = tokenizer.tokenize("testexact1:vespa!200", facts);
        assertEquals(new Token(WORD, "testexact1"), tokens.get(0));
        assertEquals(new Token(COLON, ":"),         tokens.get(1));
        assertEquals(new Token(WORD, "vespa"),      tokens.get(2));
        assertEquals(new Token(EXCLAMATION, "!"),   tokens.get(3));
        assertEquals(new Token(NUMBER, "200"),      tokens.get(4));

        tokens = tokenizer.tokenize("testexact1:*\"news\"*", facts);
        assertEquals(new Token(WORD, "testexact1"), tokens.get(0));
        assertEquals(new Token(COLON, ":"),         tokens.get(1));
        assertEquals(new Token(STAR, "*"),          tokens.get(2));
        assertEquals(new Token(WORD, "news"),       tokens.get(3));
        assertEquals(new Token(STAR, "*"),          tokens.get(4));

        tokens = tokenizer.tokenize("normal(testexact1:foo) testexact2:bar", facts);
        assertEquals(new Token(WORD, "normal"),     tokens.get(0));
        assertEquals(new Token(LBRACE, "("),        tokens.get(1));
        assertEquals(new Token(WORD, "testexact1"), tokens.get(2));
        assertEquals(new Token(COLON, ":"),         tokens.get(3));
        assertEquals(new Token(WORD, "foo"),        tokens.get(4));
        assertEquals(new Token(RBRACE, ")"),        tokens.get(5));
        assertEquals(new Token(SPACE, " "),         tokens.get(6));
        assertEquals(new Token(WORD, "testexact2"), tokens.get(7));
        assertEquals(new Token(COLON, ":"),         tokens.get(8));
        assertEquals(new Token(WORD, "bar"),        tokens.get(9));

        tokens = tokenizer.tokenize("normal testexact1:(foo testexact2:bar", facts);
        assertEquals(new Token(WORD, "normal"),     tokens.get(0));
        assertEquals(new Token(SPACE, " "),         tokens.get(1));
        assertEquals(new Token(WORD, "testexact1"), tokens.get(2));
        assertEquals(new Token(COLON, ":"),         tokens.get(3));
        assertEquals(new Token(WORD, "(foo"),       tokens.get(4));
        assertEquals(new Token(SPACE, " "),         tokens.get(5));
        assertEquals(new Token(WORD, "testexact2"), tokens.get(6));
        assertEquals(new Token(COLON, ":"),         tokens.get(7));
        assertEquals(new Token(WORD, "bar"),        tokens.get(8));

        tokens = tokenizer.tokenize("normal testexact1:foo! testexact2:bar", facts);
        assertEquals(new Token(WORD, "normal"),     tokens.get(0));
        assertEquals(new Token(SPACE, " "),         tokens.get(1));
        assertEquals(new Token(WORD, "testexact1"), tokens.get(2));
        assertEquals(new Token(COLON, ":"),         tokens.get(3));
        assertEquals(new Token(WORD, "foo"),        tokens.get(4));
        assertEquals(new Token(EXCLAMATION, "!"),   tokens.get(5));
        assertEquals(new Token(SPACE, " "),         tokens.get(6));
        assertEquals(new Token(WORD, "testexact2"), tokens.get(7));
        assertEquals(new Token(COLON, ":"),         tokens.get(8));
        assertEquals(new Token(WORD, "bar"),        tokens.get(9));

        tokens = tokenizer.tokenize("normal testexact1:foo* testexact2:bar", facts);
        assertEquals(new Token(WORD, "normal"),     tokens.get(0));
        assertEquals(new Token(SPACE, " "),         tokens.get(1));
        assertEquals(new Token(WORD, "testexact1"), tokens.get(2));
        assertEquals(new Token(COLON, ":"),         tokens.get(3));
        assertEquals(new Token(WORD, "foo"),        tokens.get(4));
        assertEquals(new Token(STAR, "*"),          tokens.get(5));
        assertEquals(new Token(SPACE, " "),         tokens.get(6));
        assertEquals(new Token(WORD, "testexact2"), tokens.get(7));
        assertEquals(new Token(COLON, ":"),         tokens.get(8));
        assertEquals(new Token(WORD, "bar"),        tokens.get(9));

        tokens = tokenizer.tokenize("normal testexact1: foo* testexact2:bar", facts);
        assertEquals(new Token(WORD, "normal"),     tokens.get(0));
        assertEquals(new Token(SPACE, " "),         tokens.get(1));
        assertEquals(new Token(WORD, "testexact1"), tokens.get(2));
        assertEquals(new Token(COLON, ":"),         tokens.get(3));
        assertEquals(new Token(WORD, "foo"),        tokens.get(4));
        assertEquals(new Token(STAR, "*"),          tokens.get(5));
        assertEquals(new Token(SPACE, " "),         tokens.get(6));
        assertEquals(new Token(WORD, "testexact2"), tokens.get(7));
        assertEquals(new Token(COLON, ":"),         tokens.get(8));
        assertEquals(new Token(WORD, "bar"),        tokens.get(9));

        tokens = tokenizer.tokenize("normal testexact1:\" foo\"* testexact2:bar", facts);
        assertEquals(new Token(WORD, "normal"),     tokens.get(0));
        assertEquals(new Token(SPACE, " "),         tokens.get(1));
        assertEquals(new Token(WORD, "testexact1"), tokens.get(2));
        assertEquals(new Token(COLON, ":"),         tokens.get(3));
        assertEquals(new Token(WORD, " foo"),        tokens.get(4));
        assertEquals(new Token(STAR, "*"),          tokens.get(5));
        assertEquals(new Token(SPACE, " "),         tokens.get(6));
        assertEquals(new Token(WORD, "testexact2"), tokens.get(7));
        assertEquals(new Token(COLON, ":"),         tokens.get(8));
        assertEquals(new Token(WORD, "bar"),        tokens.get(9));
    }
In cjk languages, special tokens must be recognized as substrings of strings not
separated by space, as special token recognition happens before tokenization
container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java