@Test
    public void testProgressSerializationRange() throws ParseException {
        int distBits = 4;
        int buckets = 1 << distBits;

        BucketIdFactory idFactory = new BucketIdFactory();
        ProgressToken progress = new ProgressToken();

        // docsel will be unknown --> entire bucket range will be covered
        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
                "id.group != \"yahoo.com\"", idFactory, distBits, progress);

        assertEquals(progress.getDistributionBitCount(), distBits);
        assertTrue(iter.getBucketSource() instanceof VisitorIterator.DistributionRangeBucketSource);

        assertEquals(progress.getFinishedBucketCount(), 0);
        assertEquals(progress.getTotalBucketCount(), buckets);

        // First, get+update half of the buckets, marking them as done
        long bucketCount = 0;
        long bucketStop = buckets / 2;

        while (iter.hasNext() && bucketCount != bucketStop) {
            VisitorIterator.BucketProgress ids = iter.getNext();
            iter.update(ids.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
            ++bucketCount;
        }
        assertEquals(bucketCount, bucketStop);
        // Should be no buckets in limbo at this point
        assertFalse(progress.hasActive());
        assertFalse(progress.hasPending());
        assertFalse(iter.isDone());
        assertTrue(iter.hasNext());
        assertEquals(progress.getFinishedBucketCount(), bucketCount);
        assertFalse(progress.isFinished());

        StringBuilder desired = new StringBuilder();
        desired.append("VDS bucket progress file (50.0% completed)\n");
        desired.append(distBits);
        desired.append('\n');
        desired.append(bucketCount); // Finished == cursor for this
        desired.append('\n');
        desired.append(bucketCount);
        desired.append('\n');
        desired.append(buckets);
        desired.append('\n');

        assertEquals(desired.toString(), progress.toString());

        // Test import, in which case distribution bits are 1
        BucketIdFactory idFactory2 = new BucketIdFactory();

        // De-serialization with no pending buckets
        {
            ProgressToken progDs = new ProgressToken(progress.toString());

            assertEquals(progDs.getDistributionBitCount(), distBits);
            assertEquals(progDs.getTotalBucketCount(), buckets);
            assertEquals(progDs.getFinishedBucketCount(), bucketCount);

            VisitorIterator iterDs = VisitorIterator.createFromDocumentSelection(
                "id.group != \"yahoo.com\"", idFactory2, 1, progDs);

            assertFalse(progDs.hasPending());
            assertFalse(progDs.hasActive());
            assertTrue(iterDs.hasNext());
            assertFalse(iterDs.isDone());
            assertEquals(distBits, iterDs.getDistributionBitCount());
            assertEquals(distBits, progDs.getDistributionBitCount());

            // Iterator must start up on next bucket in range
            VisitorIterator.BucketProgress idDs = iterDs.getNext();
            long resumeKey = ProgressToken.makeNthBucketKey(bucketCount, distBits);
            assertEquals(idDs.getSuperbucket(), new BucketId(ProgressToken.keyToBucketId(resumeKey)));
            assertEquals(idDs.getProgress(), new BucketId());
        }

        // Now fetch a subset of the remaining buckets without finishing them,
        // keeping some in the active set and some in pending
        int pendingTotal = buckets / 8;
        int activeTotal = buckets / 8;
        List<VisitorIterator.BucketProgress> trackedBuckets = new ArrayList<>();

        // Pre-fetch, since otherwise we'd reuse pending buckets
        for (int i = 0; i < pendingTotal + activeTotal; ++i) {
            trackedBuckets.add(iter.getNext());
        }

        for (int i = 0; i < pendingTotal + activeTotal; ++i) {
            VisitorIterator.BucketProgress idTemp = trackedBuckets.get(i);
            if (i < activeTotal) {
                // Make them 50% done
                iter.update(idTemp.getSuperbucket(),
                        new BucketId(distBits + 2, idTemp.getSuperbucket().getId() | (2 * buckets)));
            }
            // else: leave hanging as active
        }

        assertEquals(progress.getActiveBucketCount(), activeTotal);
        assertEquals(progress.getPendingBucketCount(), pendingTotal);

        // we can't reuse the existing string builder, since the bucket cursor
        // has changed
        desired = new StringBuilder();
        desired.append("VDS bucket progress file (").append(progress.percentFinished()).append("% completed)\n");
        desired.append(distBits);
        desired.append('\n');
        desired.append(bucketCount + pendingTotal + activeTotal);
        desired.append('\n');
        desired.append(bucketCount);
        desired.append('\n');
        desired.append(buckets);
        desired.append('\n');

        assertEquals(progress.getBuckets().entrySet().size(), pendingTotal + activeTotal);

        for (Map.Entry<ProgressToken.BucketKeyWrapper, ProgressToken.BucketEntry> entry
                : progress.getBuckets().entrySet()) {
            desired.append(Long.toHexString(ProgressToken.keyToBucketId(entry.getKey().key())));
            desired.append(':');
            desired.append(Long.toHexString(entry.getValue().getProgress().getRawId()));
            desired.append('\n');
        }

        assertEquals(progress.toString(), desired.toString());

        {
            // Deserialization with pending buckets
            ProgressToken progDs = new ProgressToken(progress.toString());

            assertEquals(progDs.getDistributionBitCount(), distBits);
            assertEquals(progDs.getTotalBucketCount(), buckets);
            assertEquals(progDs.getFinishedBucketCount(), bucketCount);

            VisitorIterator iterDs = VisitorIterator.createFromDocumentSelection(
                "id.group != \"yahoo.com\"", idFactory2, 1, progDs);

            // All started but nonfinished buckets get placed in pending upon
            // deserialization
            assertEquals(progDs.getPendingBucketCount(), pendingTotal + activeTotal);
            assertEquals(distBits, progDs.getDistributionBitCount());
            assertEquals(distBits, iterDs.getDistributionBitCount());
            assertFalse(progDs.hasActive());
            assertTrue(iterDs.hasNext());
            assertFalse(iterDs.isDone());
            assertEquals(progDs.getBucketCursor(), bucketCount + pendingTotal + activeTotal);
        }

        // Finish all the active buckets
        for (int i = activeTotal; i < activeTotal + pendingTotal; ++i) {
            iter.update(trackedBuckets.get(i).getSuperbucket(), ProgressToken.FINISHED_BUCKET);
            ++bucketCount;
        }

        assertEquals(progress.getActiveBucketCount(), 0);
        boolean consistentNext = true;
        // Get all pending/remaining sourced and finish them all
        while (!iter.isDone()) {
            if (!iter.hasNext()) {
                consistentNext = false;
                break;
            }
            VisitorIterator.BucketProgress bp = iter.getNext();
            iter.update(bp.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
            ++bucketCount;
        }

        assertTrue(consistentNext);
        assertFalse(iter.hasNext());
        assertTrue(progress.isFinished());
        // Cumulative number of finished buckets must match 2^distbits
        assertEquals(bucketCount, buckets);
        StringBuilder finished = new StringBuilder();
        finished.append("VDS bucket progress file (100.0% completed)\n");
        finished.append(distBits);
        finished.append('\n');
        finished.append(buckets); // Cursor
        finished.append('\n');
        finished.append(buckets); // Finished
        finished.append('\n');
        finished.append(buckets); // Total
        finished.append('\n');

        assertEquals(progress.toString(), finished.toString());
    }
Tests for VisitorIterator and ProgressToken (kept in one test case because their
interactions are so tightly coupled)

@author vekterli
documentapi/src/test/java/com/yahoo/documentapi/VisitorIteratorTestCase.java