@Test
    public void testProgressSerializationExplicit() throws ParseException {
        int distBits = 16;

        BucketIdFactory idFactory = new BucketIdFactory();
        ProgressToken progress = new ProgressToken();

        VisitorIterator iter = VisitorIterator.createFromDocumentSelection(
                "id.user == 1234 or id.user == 6789 or id.user == 8009", idFactory, distBits, progress);

        assertEquals(progress.getDistributionBitCount(), distBits);
        assertTrue(iter.getBucketSource() instanceof VisitorIterator.ExplicitBucketSource);

        assertEquals(progress.getFinishedBucketCount(), 0);
        assertEquals(progress.getTotalBucketCount(), 3);
        assertEquals(progress.getPendingBucketCount(), 3);

        VisitorIterator.BucketProgress bp1 = iter.getNext();
        VisitorIterator.BucketProgress bp2 = iter.getNext();
        assertEquals(progress.getPendingBucketCount(), 1);
        assertEquals(progress.getActiveBucketCount(), 2);
        // Buckets are ordered by their reverse bucket id key
        assertEquals(bp1.getSuperbucket(), new BucketId(32, 1234));
        assertEquals(bp1.getProgress(), new BucketId());
        // Put bucket 1234 back into pending
        iter.update(bp1.getSuperbucket(), new BucketId(36, 1234));
        assertEquals(progress.getPendingBucketCount(), 2);

        assertEquals(bp2.getSuperbucket(), new BucketId(32, 8009));
        assertEquals(bp2.getProgress(), new BucketId());

        {
            StringBuilder desired = new StringBuilder();
            desired.append("VDS bucket progress file (").append(progress.percentFinished()).append("% completed)\n");
            desired.append(distBits);
            desired.append('\n');
            desired.append(0);
            desired.append('\n');
            desired.append(0);
            desired.append('\n');
            desired.append(3);
            desired.append('\n');
            // Pending/active buckets are written in an increasing (key, not
            // bucket-id!) order
            desired.append(Long.toHexString(new BucketId(32, 1234).getRawId()));
            desired.append(':');
            desired.append(Long.toHexString(new BucketId(36, 1234).getRawId()));
            desired.append('\n');
            desired.append(Long.toHexString(new BucketId(32, 8009).getRawId()));
            desired.append(":0\n");
            desired.append(Long.toHexString(new BucketId(32, 6789).getRawId()));
            desired.append(":0\n");

            assertEquals(desired.toString(), progress.toString());

            ProgressToken prog2 = new ProgressToken(progress.toString());
            assertEquals(prog2.getDistributionBitCount(), distBits);
            assertEquals(prog2.getTotalBucketCount(), 3);
            assertEquals(prog2.getFinishedBucketCount(), 0);

            VisitorIterator iter2 = VisitorIterator.createFromDocumentSelection(
                "id.user == 1234 or id.user == 6789 or id.user == 8009", idFactory, distBits, prog2);

            assertEquals(prog2.getPendingBucketCount(), 3);
            assertFalse(prog2.hasActive());
            assertTrue(iter2.hasNext());
            assertFalse(iter2.isDone());

            assertTrue(iter2.getBucketSource() instanceof VisitorIterator.ExplicitBucketSource);
            assertFalse(iter2.getBucketSource().hasNext());

            VisitorIterator.BucketProgress bp = iter2.getNext();
            assertEquals(bp.getSuperbucket(), new BucketId(32, 1234));
            assertEquals(bp.getProgress(), new BucketId(36, 1234));
            assertEquals(prog2.getPendingBucketCount(), 2);

            assertTrue(iter2.hasNext());
            assertFalse(iter2.isDone());
            bp = iter2.getNext();
            assertEquals(bp.getSuperbucket(), new BucketId(32, 8009));
            assertEquals(bp.getProgress(), new BucketId());
            assertEquals(prog2.getPendingBucketCount(), 1);

            assertTrue(iter2.hasNext());
            assertFalse(iter2.isDone());
            bp = iter2.getNext();
            assertEquals(prog2.getPendingBucketCount(), 0);
            assertEquals(bp.getSuperbucket(), new BucketId(32, 6789));
            assertEquals(bp.getProgress(), new BucketId());
            assertFalse(iter2.hasNext());
            assertFalse(iter2.isDone()); // Active buckets
            assertEquals(prog2.getActiveBucketCount(), 3);
        }

        // Finish off all active buckets
        assertTrue(iter.hasNext());
        assertFalse(iter.isDone());
        bp1 = iter.getNext();
        assertEquals(bp1.getSuperbucket(), new BucketId(32, 1234));
        assertEquals(bp1.getProgress(), new BucketId(36, 1234));

        iter.update(bp1.getSuperbucket(), ProgressToken.FINISHED_BUCKET);

        assertTrue(iter.hasNext());
        assertFalse(iter.isDone());
        bp1 = iter.getNext();
        assertEquals(bp1.getSuperbucket(), new BucketId(32, 6789));
        assertEquals(bp1.getProgress(), new BucketId());

        // Just to make sure Java serializes the long properly
        assertEquals(
                progress.toString(),
                "VDS bucket progress file (" + progress.percentFinished() + "% completed)\n" +
                "16\n" +
                "0\n" +
                "1\n" +
                "3\n" +
                "8000000000001f49:0\n" +
                "8000000000001a85:0\n");

        iter.update(bp1.getSuperbucket(), ProgressToken.FINISHED_BUCKET);

        // At this point, we've got one active but no pending, so hasNext == false,
        // but isDone is also == false
        assertFalse(iter.hasNext());
        assertFalse(iter.isDone());
        assertEquals(progress.getPendingBucketCount(), 0);
        assertEquals(progress.getActiveBucketCount(), 1);

        iter.update(bp2.getSuperbucket(), ProgressToken.FINISHED_BUCKET);
        assertFalse(iter.hasNext());
        assertTrue(iter.isDone());
        assertTrue(progress.isFinished());
        assertEquals(progress.getActiveBucketCount(), 0);

        {
            StringBuilder finished = new StringBuilder();
            finished.append("VDS bucket progress file (100.0% completed)\n");
            finished.append(distBits);
            finished.append('\n');
            finished.append(0); // Cursor (not used by explicit)
            finished.append('\n');
            finished.append(3); // Finished
            finished.append('\n');
            finished.append(3); // Total
            finished.append('\n');

            assertEquals(finished.toString(), progress.toString());
        }
    }
Tests for VisitorIterator and ProgressToken (kept in one test case because their
interactions are so tightly coupled)

@author vekterli
documentapi/src/test/java/com/yahoo/documentapi/VisitorIteratorTestCase.java