@Test
    public void testTokenIterator() {
        var parameters = new LinguisticsParameters(Language.ENGLISH, StemMode.NONE, false, true);
        var tester = new OpenNlpLinguisticsTester();
        Iterator<Token> it = tester.tokenizer().tokenize("", parameters).iterator();
        assertFalse(it.hasNext());
        try {
            it.next();
            fail();
        } catch (NoSuchElementException e) {
            // success
        }

        it = tester.tokenizer().tokenize("", parameters).iterator();
        assertFalse(it.hasNext());

        it = tester.tokenizer().tokenize("one two three", parameters).iterator();
        assertNotNull(it.next());
        assertNotNull(it.next());
        assertNotNull(it.next());
        assertNotNull(it.next());
        assertNotNull(it.next());
        assertFalse(it.hasNext());
    }
Test of tokenization, with stemming and accent removal

@author matskin
@author bratseth
opennlp-linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpTokenizationTestCase.java