@Test
    public void node_failing_throttle() {
        // Throttles based on an absolute number in small zone
        {
            // 10 hosts with 7 container and 7 content nodes, total 24 nodes
            NodeFailTester tester = NodeFailTester.withTwoApplications(10, 7, 7);

            List<String> failedHostHostnames = tester.nodeRepository.nodes().list().stream()
                    .flatMap(node -> node.parentHostname().stream())
                    .collect(Collectors.groupingBy(h -> h, Collectors.counting()))
                    .entrySet().stream()
                    .sorted(Comparator.comparingLong((Map.Entry<String, Long> e) -> e.getValue()).reversed())
                    .limit(3)
                    .map(Map.Entry::getKey)
                    .toList();

            // 3 hosts fail. 2 of them and all of their children are allowed to fail
            failedHostHostnames.forEach(hostname -> tester.serviceMonitor.setHostDown(hostname));
            tester.runMaintainers();
            tester.clock.advance(Duration.ofMinutes(61));

            tester.runMaintainers();
            tester.runMaintainers();  // hosts are typically failed in the 2. maintain()
            assertEquals(2 + /* hosts */
                         (2 * 2) /* containers per host */,
                         tester.nodeRepository.nodes().list(Node.State.failed).size());
            assertEquals("Throttling is indicated by the metric", 1, tester.metric.values.get(NodeFailer.throttlingActiveMetric));
            assertEquals("Throttled host failures", 1, tester.metric.values.get(NodeFailer.throttledHostFailuresMetric));

            // 24 more hours pass without any other nodes being failed out
            for (int minutes = 0, interval = 30; minutes <= 23 * 60; minutes += interval) {
                tester.clock.advance(Duration.ofMinutes(interval));
            }
            tester.runMaintainers();
            assertEquals(6, tester.nodeRepository.nodes().list(Node.State.failed).size());
            assertEquals("Throttling is indicated by the metric", 1, tester.metric.values.get(NodeFailer.throttlingActiveMetric));
            assertEquals("Throttled host failures", 1, tester.metric.values.get(NodeFailer.throttledHostFailuresMetric));

            // The final host and its containers are failed out
            tester.clock.advance(Duration.ofMinutes(30));
            tester.runMaintainers();
            tester.runMaintainers();  // hosts are failed in the 2. maintain()
            assertEquals(9, tester.nodeRepository.nodes().list(Node.State.failed).size());
            assertEquals("Throttling is not indicated by the metric, as no throttled attempt is made", 0, tester.metric.values.get(NodeFailer.throttlingActiveMetric));
            assertEquals("No throttled node failures", 0, tester.metric.values.get(NodeFailer.throttledNodeFailuresMetric));

            // Nothing else to fail
            tester.clock.advance(Duration.ofHours(25));
            tester.runMaintainers();
            assertEquals(9, tester.nodeRepository.nodes().list(Node.State.failed).size());
            assertEquals("Throttling is not indicated by the metric", 0, tester.metric.values.get(NodeFailer.throttlingActiveMetric));
            assertEquals("No throttled node failures", 0, tester.metric.values.get(NodeFailer.throttledNodeFailuresMetric));
        }

        // Throttles based on percentage in large zone
        {
            NodeFailTester tester = NodeFailTester.withTwoApplications(300, 100, 100);
            NodeList allNodes = tester.nodeRepository.nodes().list();
            assertEquals(500, allNodes.size());

            // 2 hours pass, many nodes fail
            tester.runMaintainers();
            int downNodes = 25;     // 5%
            int allowedToFail = 20; // 4%
            allNodes.state(Node.State.active)
                    .nodeType(NodeType.tenant)
                    .stream()
                    .limit(downNodes)
                    .forEach(host -> tester.serviceMonitor.setHostDown(host.hostname()));
            tester.runMaintainers();
            tester.clock.advance(Duration.ofHours(2));
            tester.runMaintainers();

            // Fails nodes up to throttle limit
            assertEquals(allowedToFail, tester.nodeRepository.nodes().list(Node.State.failed).size());
            assertEquals("Throttling is indicated by the metric.", 1, tester.metric.values.get(NodeFailer.throttlingActiveMetric));
            assertEquals("Throttled node failures", downNodes - allowedToFail, tester.metric.values.get(NodeFailer.throttledNodeFailuresMetric));

            // 6 more hours pass, no more nodes are failed
            tester.clock.advance(Duration.ofHours(6));
            tester.runMaintainers();
            assertEquals(allowedToFail, tester.nodeRepository.nodes().list(Node.State.failed).size());
            assertEquals("Throttling is indicated by the metric.", 1, tester.metric.values.get(NodeFailer.throttlingActiveMetric));
            assertEquals("Throttled node failures", downNodes - allowedToFail, tester.metric.values.get(NodeFailer.throttledNodeFailuresMetric));

            // 18 more hours pass, 24 hours since the first batch of nodes were failed. The remaining nodes are failed
            tester.clock.advance(Duration.ofHours(18));
            tester.runMaintainers();
            assertEquals(downNodes, tester.nodeRepository.nodes().list(Node.State.failed).size());
            assertEquals("Throttling is not indicated by the metric, as no throttled attempt is made.", 0, tester.metric.values.get(NodeFailer.throttlingActiveMetric));
            assertEquals("No throttled node failures", 0, tester.metric.values.get(NodeFailer.throttledNodeFailuresMetric));
        }
    }
Tests automatic failing of nodes.

@author bratseth
@author mpolden
node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java