@Test
    public void node_failing() {
        NodeFailTester tester = NodeFailTester.withTwoApplications(6);
        String downHost1 = tester.nodeRepository.nodes().list(Node.State.active).owner(NodeFailTester.app1).asList().get(1).hostname();
        String downHost2 = tester.nodeRepository.nodes().list(Node.State.active).owner(NodeFailTester.app2).asList().get(3).hostname();
        // No liveness evidence yet:
        assertFalse(tester.nodeRepository.nodes().node(downHost1).get().history().isDown());
        assertFalse(tester.nodeRepository.nodes().node(downHost1).get().history().isUp());
        assertFalse(tester.nodeRepository.nodes().node(downHost1).get().history().isSuspended());
        assertFalse(tester.nodeRepository.nodes().node(downHost1).get().history().isResumed());

        // For a day all nodes work so nothing happens
        for (int minutes = 0; minutes < 24 * 60; minutes +=5 ) {
            tester.runMaintainers();
            tester.clock.advance(Duration.ofMinutes(5));

            assertEquals(0, tester.deployer.activations);
            assertEquals(8, tester.nodeRepository.nodes().list(Node.State.active).nodeType(NodeType.tenant).size());
            assertEquals(0, tester.nodeRepository.nodes().list(Node.State.failed).nodeType(NodeType.tenant).size());
            assertFalse(tester.nodeRepository.nodes().node(downHost1).get().history().isDown());
            assertTrue(tester.nodeRepository.nodes().node(downHost1).get().history().isUp());
            assertFalse(tester.nodeRepository.nodes().node(downHost1).get().history().isSuspended());
            assertTrue(tester.nodeRepository.nodes().node(downHost1).get().history().isResumed());
        }

        tester.serviceMonitor.setHostDown(downHost1);
        tester.serviceMonitor.setHostDown(downHost2);
        // nothing happens the first 45 minutes
        for (int minutes = 0; minutes < 45; minutes +=5 ) {
            tester.runMaintainers();
            tester.clock.advance(Duration.ofMinutes(5));
            assertEquals(0, tester.deployer.activations);
            assertEquals(8, tester.nodeRepository.nodes().list(Node.State.active).nodeType(NodeType.tenant).size());
            assertEquals(0, tester.nodeRepository.nodes().list(Node.State.failed).nodeType(NodeType.tenant).size());
            assertTrue(tester.nodeRepository.nodes().node(downHost1).get().history().isDown());
            assertFalse(tester.nodeRepository.nodes().node(downHost1).get().history().isUp());
        }
        tester.serviceMonitor.setHostUp(downHost1);

        // downHost2 should now be failed and replaced, but not downHost1
        tester.clock.advance(Duration.ofDays(1));
        tester.runMaintainers();
        assertFalse(tester.nodeRepository.nodes().node(downHost1).get().history().isDown());
        assertTrue(tester.nodeRepository.nodes().node(downHost1).get().history().isUp());
        assertEquals(1, tester.deployer.activations);
        assertEquals(8, tester.nodeRepository.nodes().list(Node.State.active).nodeType(NodeType.tenant).size());
        assertEquals(1, tester.nodeRepository.nodes().list(Node.State.failed).nodeType(NodeType.tenant).size());
        assertEquals(downHost2, tester.nodeRepository.nodes().list(Node.State.failed).nodeType(NodeType.tenant).asList().get(0).hostname());

        // downHost1 fails again
        tester.serviceMonitor.setHostDown(downHost1);
        tester.runMaintainers();
        tester.clock.advance(Duration.ofMinutes(5));
        // the system goes down
        tester.clock.advance(Duration.ofMinutes(120));
        tester.failer = tester.createFailer();
        tester.runMaintainers();
        // the host is still down and fails
        tester.clock.advance(Duration.ofMinutes(5));
        tester.runMaintainers();
        assertEquals(2, tester.deployer.activations);
        assertEquals(8, tester.nodeRepository.nodes().list(Node.State.active).nodeType(NodeType.tenant).size());
        assertEquals(2, tester.nodeRepository.nodes().list(Node.State.failed).nodeType(NodeType.tenant).size());

        // the last host goes down
        Node lastNode = tester.highestIndex(tester.nodeRepository.nodes().list(Node.State.active).owner(NodeFailTester.app1));
        tester.serviceMonitor.setHostDown(lastNode.hostname());
        // it is not failed because there are no ready nodes to replace it
        for (int minutes = 0; minutes < 75; minutes +=5 ) {
            tester.runMaintainers();
            tester.clock.advance(Duration.ofMinutes(5));
            assertEquals(2, tester.deployer.activations);
            assertEquals(8, tester.nodeRepository.nodes().list(Node.State.active).nodeType(NodeType.tenant).size());
            assertEquals(2, tester.nodeRepository.nodes().list(Node.State.failed).nodeType(NodeType.tenant).size());
        }

        // A new node is available
        tester.createReadyNodes(1, 16, NodeFailTester.nodeResources);
        tester.clock.advance(Duration.ofDays(1));
        tester.runMaintainers();
        // The node is now failed
        assertEquals(3, tester.deployer.activations);
        assertEquals(8, tester.nodeRepository.nodes().list(Node.State.active).nodeType(NodeType.tenant).size());
        assertEquals(3, tester.nodeRepository.nodes().list(Node.State.failed).nodeType(NodeType.tenant).size());
        assertTrue("The index of the last failed node is not reused",
                   tester.highestIndex(tester.nodeRepository.nodes().list(Node.State.active).owner(NodeFailTester.app1)).allocation().get().membership().index()
                   >
                   lastNode.allocation().get().membership().index());

        assertEquals("Node failing does not cause recording of scaling events",
                     1,
                     tester.nodeRepository.applications().get(NodeFailTester.app1).get().cluster(NodeFailTester.testCluster).get().scalingEvents().size());
    }
Tests automatic failing of nodes.

@author bratseth
@author mpolden
node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java