@Test
    public final void test() {
        assertEquals(TokenType.ALPHABETIC, tokenType('a'));
        assertEquals(TokenType.ALPHABETIC, tokenType('\u02c1'));
        assertEquals(TokenType.ALPHABETIC, tokenType('\u02c1'));
        assertEquals(TokenType.ALPHABETIC, tokenType('\u01c0'));
        assertEquals(TokenType.SYMBOL, tokenType('\u20dd'));
        assertEquals(TokenType.ALPHABETIC, tokenType('\u0912'));
        assertEquals(TokenType.NUMERIC, tokenType('1'));
        assertEquals(TokenType.PUNCTUATION, tokenType('.'));
        assertEquals(TokenType.PUNCTUATION, tokenType('\u0f3b'));
        assertEquals(TokenType.PUNCTUATION, tokenType('\u0f3c'));
        assertEquals(TokenType.PUNCTUATION, tokenType('\u203f'));
        assertEquals(TokenType.SYMBOL, tokenType('\u2044'));
        assertEquals(TokenType.SYMBOL, tokenType('$'));
        assertEquals(TokenType.ALPHABETIC, tokenType('\u2132'));
        assertEquals(TokenType.ALPHABETIC, tokenType('\uD800', '\uDFC8'));
    }
Check simple token types.

@author Steinar Knutsen
linguistics/src/test/java/com/yahoo/language/simple/SimpleTokenTypeTestCase.java