private SpecialTokenRegistry createSpecialTokens() {
        List<SpecialTokens.Token> tokens = new ArrayList<>();
        tokens.add(new SpecialTokens.Token("c+"));
        tokens.add(new SpecialTokens.Token("c++"));
        tokens.add(new SpecialTokens.Token(".net"));
        tokens.add(new SpecialTokens.Token("tcp/ip"));
        tokens.add(new SpecialTokens.Token("i/o"));
        tokens.add(new SpecialTokens.Token("c#"));
        tokens.add(new SpecialTokens.Token("AS/400"));
        tokens.add(new SpecialTokens.Token("...."));
        tokens.add(new SpecialTokens.Token("b.s.d."));
        tokens.add(new SpecialTokens.Token("with space"));
        tokens.add(new SpecialTokens.Token("dvd\\xB1r"));
        SpecialTokens defaultTokens = new SpecialTokens("default", tokens);

        tokens = new ArrayList<>();
        tokens.add(new SpecialTokens.Token("[huh]"));
        tokens.add(new SpecialTokens.Token("&&&%%%"));
        tokens.add(new SpecialTokens.Token("------"));
        tokens.add(new SpecialTokens.Token("!!!***"));
        SpecialTokens otherTokens = new SpecialTokens("other", tokens);

        tokens = new ArrayList<>();
        tokens.add(new SpecialTokens.Token("...."));
        tokens.add(new SpecialTokens.Token("c++", "cpp"));
        tokens.add(new SpecialTokens.Token("b.s.d."));
        tokens.add(new SpecialTokens.Token("with space", "with-space"));
        tokens.add(new SpecialTokens.Token("c#"));
        tokens.add(new SpecialTokens.Token("know", "knuwww"));
        SpecialTokens replacingTokens = new SpecialTokens("replacing", tokens);

        return new SpecialTokenRegistry(List.of(defaultTokens, otherTokens, replacingTokens));
    }
In cjk languages, special tokens must be recognized as substrings of strings not
separated by space, as special token recognition happens before tokenization
container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java