Upgrade Slurm to version 22.05.6 (#473)

Revert "Implement workaround for dynamic nodes stuck in bad Slurm state"
(commit dedda89)

This is because SchedMD merged a fix in Slurm 22.05.6.

Signed-off-by: Jacopo De Amicis <jdamicis@amazon.it>

Author: jdeamicis

CHANGELOG.md

@@ -9,6 +9,10 @@ This file is used to list changes made in each version of the aws-parallelcluste
 **ENHANCEMENTS**
 - Add support for launching nodes across multiple availability zones to increase capacity availability.
 
+**CHANGES**
+- Do not consider dynamic nodes in IDLE+CLOUD+COMPLETING+POWER_DOWN+NOT_RESPONDING as unhealthy anymore.
+  - The root cause has been fixed in Slurm 22.05.6.
+
 3.3.0
 ------
 

src/slurm_plugin/slurm_resources.py

@@ -98,7 +98,6 @@ class SlurmNode(metaclass=ABCMeta):
     SLURM_SCONTROL_POWER_STATES = [{"IDLE", "CLOUD", "POWERED_DOWN"}, {"IDLE", "CLOUD", "POWERED_DOWN", "POWER_DOWN"}]
     SLURM_SCONTROL_REBOOT_REQUESTED_STATE = "REBOOT_REQUESTED"
     SLURM_SCONTROL_REBOOT_ISSUED_STATE = "REBOOT_ISSUED"
-    SLURM_SCONTROL_BUG_STUCK_DYNAMIC_NODES_STATE = {"IDLE", "CLOUD", "COMPLETING", "POWER_DOWN", "NOT_RESPONDING"}
 
     EC2_ICE_ERROR_CODES = {
         "InsufficientInstanceCapacity",
@@ -421,9 +420,6 @@ def is_state_healthy(self, terminate_drain_nodes, terminate_down_nodes, log_warn
                 if log_warn_if_unhealthy:
                     logger.warning("Node state check: node %s in DOWN, node state: %s", self, self.state_string)
             return False
-        # Workaround for IDLE+CLOUD+COMPLETING+POWER_DOWN+NOT_RESPONDING bug
-        if self.is_stuck_in_bug_state():
-            return False
         return True
 
     def is_healthy(self, terminate_drain_nodes, terminate_down_nodes, log_warn_if_unhealthy=True):
@@ -465,10 +461,6 @@ def needs_reset_when_inactive(self):
         """Check if the node need to be reset if node is inactive."""
         return self.is_nodeaddr_set() or (not (self.is_power() or self.is_powering_down() or self.is_down()))
 
-    def is_stuck_in_bug_state(self):
-        """Check if the node is stuck in a bug state IDLE+CLOUD+COMPLETING+POWER_DOWN+NOT_RESPONDING."""
-        return self.states == self.SLURM_SCONTROL_BUG_STUCK_DYNAMIC_NODES_STATE
-
 
 class EC2InstanceHealthState:
     def __init__(self, id, state, instance_status, system_status, scheduled_events):

tests/slurm_plugin/slurm_resources/test_slurm_resources.py

@@ -859,18 +859,6 @@ def test_slurm_node_is_bootstrap_failure(
             None,
             True,
         ),
-        # Workaround for nodes stuck in IDLE+CLOUD+COMPLETING+POWER_DOWN+NOT_RESPONDING
-        (
-            DynamicNode(
-                "queue-dy-c5xlarge-1",
-                "queue-dy-c5xlarge-1",
-                "queue-dy-c5xlarge-1",
-                "IDLE+CLOUD+COMPLETING+POWER_DOWN+NOT_RESPONDING",
-                "queue",
-            ),
-            None,
-            False,
-        ),
     ],
     ids=[
         "basic",
@@ -883,7 +871,6 @@ def test_slurm_node_is_bootstrap_failure(
         "power_unhealthy1",
         "power_unhealthy2",
         "power_healthy",
-        "dynamic_node_stuck_in_bug_state",
     ],
 )
 def test_slurm_node_is_healthy(node, instance, expected_result):