@Override
    public IdlToken next() {
        IdlToken token = super.next();

        // Buffer documentation comments as they're encountered.
        if (token == IdlToken.DOC_COMMENT) {
            int start = getCurrentTokenStart();
            int span = getCurrentTokenSpan();
            int docStart;

            if (span <= 3) {
                // An empty doc comment.
                docStart = start;
            } else if (model.charAt(start + 3) == ' ') {
                // Skip single space.
                docStart = start + 4;
            } else {
                docStart = start + 3;
            }

            // Strip the \n and \r\n from the end.
            int end = getCurrentTokenEnd();
            if (model.charAt(end - 1) == '\n') {
                end--;
                if (end >= 0 && model.charAt(end - 1) == '\r') {
                    end--;
                }
            }

            docCommentLines.add(getModel(docStart, end));
        }

        return token;
    }
A specialized Tokenizer that adds skipping and documentation buffering.
smithy-model/src/main/java/software/amazon/smithy/model/loader/IdlInternalTokenizer.java