From 2affeaf8b89a9e3c5cfe0fde3dfce214e7291e8c Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Thu, 18 Dec 2025 19:09:30 -0500 Subject: [PATCH 1/2] CKS: Fix issue with scaling down CKS Nodes when deployed in HA mode --- .../KubernetesClusterScaleWorker.java | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index dba858ed8099..5bd8da463703 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -441,11 +441,19 @@ private void scaleDownKubernetesClusterSize() throws CloudRuntimeException { if (this.nodeIds != null) { vmList = getKubernetesClusterVMMapsForNodes(this.nodeIds).stream().filter(vm -> !vm.isExternalNode()).collect(Collectors.toList()); } else { - vmList = getKubernetesClusterVMMaps(); - vmList = vmList.stream() + List workerVMsMap = getKubernetesClusterVMMaps().stream() .filter(vm -> !vm.isExternalNode() && !vm.isControlNode() && !vm.isEtcdNode()) .collect(Collectors.toList()); - vmList = vmList.subList((int) (kubernetesCluster.getControlNodeCount() + clusterSize - 1), vmList.size()); + int totalWorkerNodes = workerVMsMap.size(); + int desiredWorkerNodes = clusterSize == null ? (int) kubernetesCluster.getNodeCount() : clusterSize.intValue(); + int toRemoveCount = Math.max(0, totalWorkerNodes - desiredWorkerNodes); + if (toRemoveCount == 0) { + logger.info("No nodes to remove from Kubernetes cluster: {}", kubernetesCluster); + return; + } + + int startIndex = Math.max(0, totalWorkerNodes - toRemoveCount); + vmList = new ArrayList<>(workerVMsMap.subList(startIndex, totalWorkerNodes)); } Collections.reverse(vmList); removeNodesFromCluster(vmList); From dd07d7d525a63eb4973ec576fea254be6876626a Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Fri, 19 Dec 2025 09:15:06 -0500 Subject: [PATCH 2/2] extract code and add unit tests --- .../KubernetesClusterScaleWorker.java | 26 +++++--- .../KubernetesClusterScaleWorkerTest.java | 66 ++++++++++++++++++- 2 files changed, 80 insertions(+), 12 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index 5bd8da463703..6d22a1a3a03d 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -441,24 +441,30 @@ private void scaleDownKubernetesClusterSize() throws CloudRuntimeException { if (this.nodeIds != null) { vmList = getKubernetesClusterVMMapsForNodes(this.nodeIds).stream().filter(vm -> !vm.isExternalNode()).collect(Collectors.toList()); } else { - List workerVMsMap = getKubernetesClusterVMMaps().stream() - .filter(vm -> !vm.isExternalNode() && !vm.isControlNode() && !vm.isEtcdNode()) - .collect(Collectors.toList()); - int totalWorkerNodes = workerVMsMap.size(); - int desiredWorkerNodes = clusterSize == null ? (int) kubernetesCluster.getNodeCount() : clusterSize.intValue(); - int toRemoveCount = Math.max(0, totalWorkerNodes - desiredWorkerNodes); - if (toRemoveCount == 0) { + vmList = getWorkerNodesToRemove(); + if (vmList.isEmpty()) { logger.info("No nodes to remove from Kubernetes cluster: {}", kubernetesCluster); return; } - - int startIndex = Math.max(0, totalWorkerNodes - toRemoveCount); - vmList = new ArrayList<>(workerVMsMap.subList(startIndex, totalWorkerNodes)); } Collections.reverse(vmList); removeNodesFromCluster(vmList); } + public List getWorkerNodesToRemove() { + List workerVMsMap = getKubernetesClusterVMMaps().stream() + .filter(vm -> !vm.isExternalNode() && !vm.isControlNode() && !vm.isEtcdNode()) + .collect(Collectors.toList()); + int totalWorkerNodes = workerVMsMap.size(); + int desiredWorkerNodes = clusterSize == null ? (int) kubernetesCluster.getNodeCount() : clusterSize.intValue(); + int toRemoveCount = Math.max(0, totalWorkerNodes - desiredWorkerNodes); + if (toRemoveCount == 0) { + return new ArrayList<>(); + } + int startIndex = Math.max(0, totalWorkerNodes - toRemoveCount); + return new ArrayList<>(workerVMsMap.subList(startIndex, totalWorkerNodes)); + } + private void scaleUpKubernetesClusterSize(final long newVmCount) throws CloudRuntimeException { if (!kubernetesCluster.getState().equals(KubernetesCluster.State.Scaling)) { stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.ScaleUpRequested); diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java index 847c8bd6d291..c9299bdbaa67 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java @@ -17,8 +17,8 @@ package com.cloud.kubernetes.cluster.actionworkers; import com.cloud.kubernetes.cluster.KubernetesCluster; -import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; +import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; import com.cloud.offering.ServiceOffering; import com.cloud.service.ServiceOfferingVO; @@ -29,15 +29,17 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; + import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; +import java.util.Arrays; import java.util.List; -import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.DEFAULT; import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.CONTROL; +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.DEFAULT; @RunWith(MockitoJUnitRunner.class) public class KubernetesClusterScaleWorkerTest { @@ -125,4 +127,64 @@ public void testCalculateNewClusterCountAndCapacityNodeTypeScaleControlOffering( Assert.assertEquals(expectedCores, newClusterCapacity.first().longValue()); Assert.assertEquals(expectedMemory, newClusterCapacity.second().longValue()); } + + + @Test + public void testGetWorkerNodesToRemoveForDownsize_singleRemoval() { + KubernetesCluster kubernetesCluster = Mockito.mock(KubernetesCluster.class); + KubernetesClusterManagerImpl clusterManager = Mockito.mock(KubernetesClusterManagerImpl.class); + KubernetesClusterScaleWorker worker = new KubernetesClusterScaleWorker(kubernetesCluster, new java.util.HashMap<>(), 2L, null, false, null, null, clusterManager); + KubernetesClusterScaleWorker spyWorker = Mockito.spy(worker); + + KubernetesClusterVmMapVO vm1 = Mockito.mock(KubernetesClusterVmMapVO.class); + Mockito.when(vm1.isExternalNode()).thenReturn(false); + Mockito.when(vm1.isControlNode()).thenReturn(false); + Mockito.when(vm1.isEtcdNode()).thenReturn(false); + + KubernetesClusterVmMapVO vm2 = Mockito.mock(KubernetesClusterVmMapVO.class); + Mockito.when(vm2.isExternalNode()).thenReturn(false); + Mockito.when(vm2.isControlNode()).thenReturn(false); + Mockito.when(vm2.isEtcdNode()).thenReturn(false); + + KubernetesClusterVmMapVO vm3 = Mockito.mock(KubernetesClusterVmMapVO.class); + Mockito.when(vm3.isExternalNode()).thenReturn(false); + Mockito.when(vm3.isControlNode()).thenReturn(false); + Mockito.when(vm3.isEtcdNode()).thenReturn(false); + + Mockito.doReturn(Arrays.asList(vm1, vm2, vm3)).when(spyWorker).getKubernetesClusterVMMaps(); + + List toRemove = spyWorker.getWorkerNodesToRemove(); + + Assert.assertEquals(1, toRemove.size()); + Assert.assertSame(vm3, toRemove.get(0)); + } + + @Test + public void testGetWorkerNodesToRemoveForDownsize_noRemoval() { + KubernetesCluster kubernetesCluster = Mockito.mock(KubernetesCluster.class); + + KubernetesClusterScaleWorker worker = new KubernetesClusterScaleWorker(kubernetesCluster, new java.util.HashMap<>(), 3L, null, false, null, null, clusterManager); + KubernetesClusterScaleWorker spyWorker = Mockito.spy(worker); + + KubernetesClusterVmMapVO vm1 = Mockito.mock(KubernetesClusterVmMapVO.class); + Mockito.when(vm1.isExternalNode()).thenReturn(false); + Mockito.when(vm1.isControlNode()).thenReturn(false); + Mockito.when(vm1.isEtcdNode()).thenReturn(false); + + KubernetesClusterVmMapVO vm2 = Mockito.mock(KubernetesClusterVmMapVO.class); + Mockito.when(vm2.isExternalNode()).thenReturn(false); + Mockito.when(vm2.isControlNode()).thenReturn(false); + Mockito.when(vm2.isEtcdNode()).thenReturn(false); + + KubernetesClusterVmMapVO vm3 = Mockito.mock(KubernetesClusterVmMapVO.class); + Mockito.when(vm3.isExternalNode()).thenReturn(false); + Mockito.when(vm3.isControlNode()).thenReturn(false); + Mockito.when(vm3.isEtcdNode()).thenReturn(false); + + Mockito.doReturn(Arrays.asList(vm1, vm2, vm3)).when(spyWorker).getKubernetesClusterVMMaps(); + + List toRemove = spyWorker.getWorkerNodesToRemove(); + + Assert.assertTrue(toRemove.isEmpty()); + } }