@Test
    public void testIndexability() {
        var tester = new OpenNlpLinguisticsTester();
        String input = "tafsirnya\u0648\u0643\u064F\u0646\u0652";
        for (StemMode stemMode : new StemMode[] { StemMode.NONE, StemMode.SHORTEST }) {
            for (Language l : List.of(Language.INDONESIAN, Language.ENGLISH, Language.ARABIC)) {
                for (boolean accentDrop : new boolean[] { true, false }) {
                    var parameters = new LinguisticsParameters(l, stemMode, accentDrop, true);
                    for (Token token : tester.tokenizer().tokenize(input, parameters)) {
                        if (token.getTokenString().isEmpty()) {
                            assertFalse(token.isIndexable());
                        }
                    }
                }
            }
        }
    }
Test of tokenization, with stemming and accent removal

@author matskin
@author bratseth
opennlp-linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpTokenizationTestCase.java