public void testTokenTypes(Language language) {
        var tester = new OpenNlpLinguisticsTester();
        assertEquals(TokenType.ALPHABETIC, tester.tokenize("word", language).iterator().next().getType());
        assertEquals(TokenType.NUMERIC, tester.tokenize("123", language).iterator().next().getType());
        assertEquals(TokenType.SPACE, tester.tokenize(" ", language).iterator().next().getType());
        assertEquals(TokenType.PUNCTUATION, tester.tokenize(".", language).iterator().next().getType());
        assertEquals(TokenType.ALPHABETIC, tester.tokenize("123word", language).iterator().next().getType());

        var tokens = tester.tokenize("123 123word word123", language).iterator();
        assertEquals(TokenType.NUMERIC, tokens.next().getType());
        assertEquals(TokenType.SPACE, tokens.next().getType());
        assertEquals(TokenType.ALPHABETIC, tokens.next().getType());
        assertEquals(TokenType.SPACE, tokens.next().getType());
        assertEquals(TokenType.ALPHABETIC, tokens.next().getType());
    }
Test of tokenization, with stemming and accent removal

@author matskin
@author bratseth
opennlp-linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpTokenizationTestCase.java