@Test
    public void testValidSlicing() throws ParseException {
        int distBits = 4;
        long buckets = 1 << distBits;
        BucketIdFactory idFactory = new BucketIdFactory();
        for (int slices = 1; slices <= 2 * buckets; slices++) {
            long bucketsTotal = 0;
            for (int sliceId = 0; sliceId < slices; sliceId++) {
                ProgressToken progress = new ProgressToken();

                // docsel will be unknown --> entire bucket range will be covered
                VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
                        "id.group != \"yahoo.com\"", idFactory, distBits, progress, slices, sliceId);

                String context = "slices: " + slices + ", sliceId: " + sliceId;
                assertEquals(context, progress.getDistributionBitCount(), distBits);
                assertTrue(context, iter.getBucketSource() instanceof VisitorIterator.DistributionRangeBucketSource);

                assertEquals(context, progress.getFinishedBucketCount(), Math.min(buckets, sliceId));
                assertEquals(context, progress.getTotalBucketCount(), buckets);

                // First, get+update half of the buckets, marking them as done
                long bucketCount = 0;

                // Do buckets in the first half.
                while (iter.hasNext() && progress.getFinishedBucketCount() < buckets / 2) {
                    VisitorIterator.BucketProgress ids = iter.getNext();
                    iter.update(ids.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
                    ++bucketCount;
                    ++bucketsTotal;
                }

                if (slices + sliceId < buckets) { // Otherwise, we're already done ...
                    assertEquals(context, ((buckets / 2) + slices - sliceId - 1) / slices, bucketCount);
                    // Should be no buckets in limbo at this point
                    assertFalse(context, progress.hasActive());
                    assertFalse(context, progress.hasPending());
                    assertFalse(context, iter.isDone());
                    assertTrue(context, iter.hasNext());
                    assertEquals(context, progress.getFinishedBucketCount(), bucketCount * slices + sliceId);
                    assertFalse(context, progress.isFinished());
                }

                while (iter.hasNext()) {
                    VisitorIterator.BucketProgress ids = iter.getNext();
                    iter.update(ids.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
                    ++bucketCount;
                    ++bucketsTotal;
                }

                assertEquals(context, (buckets + slices - sliceId - 1) / slices, bucketCount);
                // Should be no buckets in limbo at this point
                assertFalse(context, progress.hasActive());
                assertFalse(context, progress.hasPending());
                assertTrue(context, iter.isDone());
                assertFalse(context, iter.hasNext());
                assertEquals(context, progress.getFinishedBucketCount(), buckets);
                assertTrue(context, progress.isFinished());
            }
            assertEquals("slices: " + slices, buckets, bucketsTotal);
        }
    }
Tests for VisitorIterator and ProgressToken (kept in one test case because their
interactions are so tightly coupled)

@author vekterli
documentapi/src/test/java/com/yahoo/documentapi/VisitorIteratorTestCase.java