@Test
    public void testLargeTokenGuard() {
        var parameters = new LinguisticsParameters(Language.ENGLISH, StemMode.NONE, false, true);
        var tester = new OpenNlpLinguisticsTester();
        String input = "ab".repeat(128 * 256);
        Iterator<Token> it = tester.tokenizer().tokenize(input, parameters).iterator();
        assertTrue(it.hasNext());
        assertNotNull(it.next().getTokenString());
        assertFalse(it.hasNext());
    }
Test of tokenization, with stemming and accent removal

@author matskin
@author bratseth
opennlp-linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpTokenizationTestCase.java