@Test
    void testSingleQuoteAsWordCharacter() {
        Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());

        tokenizer.setSpecialTokens(createSpecialTokens().getSpecialTokens("default"));
        List<?> tokens = tokenizer.tokenize("drive (to hwy88, 88) +or language:en nalle:a'a ugcapi_1 'a' 'a a'");

        assertEquals(new Token(WORD, "drive"), tokens.get(0));
        assertEquals(new Token(SPACE, " "), tokens.get(1));
        assertEquals(new Token(LBRACE, "("), tokens.get(2));
        assertEquals(new Token(WORD, "to"), tokens.get(3));
        assertEquals(new Token(SPACE, " "), tokens.get(4));
        assertEquals(new Token(WORD, "hwy88"), tokens.get(5));
        assertEquals(new Token(COMMA, ","), tokens.get(6));
        assertEquals(new Token(SPACE, " "), tokens.get(7));
        assertEquals(new Token(NUMBER, "88"), tokens.get(8));
        assertEquals(new Token(RBRACE, ")"), tokens.get(9));
        assertEquals(new Token(SPACE, " "), tokens.get(10));
        assertEquals(new Token(PLUS, "+"), tokens.get(11));
        assertEquals(new Token(WORD, "or"), tokens.get(12));
        assertEquals(new Token(SPACE, " "), tokens.get(13));
        assertEquals(new Token(WORD, "language"), tokens.get(14));
        assertEquals(new Token(COLON, ":"), tokens.get(15));
        assertEquals(new Token(WORD, "en"), tokens.get(16));
        assertEquals(new Token(SPACE, " "), tokens.get(17));
        assertEquals(new Token(WORD, "nalle"), tokens.get(18));
        assertEquals(new Token(COLON, ":"), tokens.get(19));
        assertEquals(new Token(WORD, "a'a"), tokens.get(20));
        assertEquals(new Token(SPACE, " "), tokens.get(21));
        assertEquals(new Token(WORD, "ugcapi"), tokens.get(22));
        assertEquals(new Token(UNDERSCORE, "_"), tokens.get(23));
        assertEquals(new Token(NUMBER, "1"), tokens.get(24));
        assertEquals(new Token(SPACE, " "), tokens.get(25));
        assertEquals(new Token(WORD, "'a'"), tokens.get(26));
        assertEquals(new Token(SPACE, " "), tokens.get(27));
        assertEquals(new Token(WORD, "'a"), tokens.get(28));
        assertEquals(new Token(SPACE, " "), tokens.get(29));
        assertEquals(new Token(WORD, "a'"), tokens.get(30));
    }
In cjk languages, special tokens must be recognized as substrings of strings not
separated by space, as special token recognition happens before tokenization
container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java