public Segments segment(String[] tokens) {
    Segments segments = new Segments(tokens);
    LinkedList<Detector> detectors = new LinkedList<>();

    int i=0;


    while(i<tokens.length){
      detectors.add(new Detector(fsa.getState(), i));

      ListIterator<Detector> det_it = detectors.listIterator();
      while(det_it.hasNext()){
        Detector d = det_it.next();
        d.state().deltaWord(tokens[i]);
        if(d.state().isFinal()){
          segments.add(new Segment(d.index(),i+1,d.state().data().getInt(0)));
        }

        if(!d.state().isValid()){
          det_it.remove();
        }
      }
      i++;
    }

    return segments;
  }
API for accessing the Segmenter automata.

@author Peter Boros
fsa/src/main/java/com/yahoo/fsa/segmenter/Segmenter.java