add support for gke node group (and update waiter)

CHANGELOG.md

@@ -14,6 +14,8 @@ and **Merged pull requests**. Critical items to know are:
 The versions coincide with releases on pip. Only major versions will be released as tags on Github.
 
 ## [0.0.x](https://github.com/converged-computing/kubescaler/tree/main) (0.0.x)
+ - do not use the waiter for nodegroup_active it does not work! (0.0.18)
+   - support for Google Cloud instance group creation, etc.
  - support adding one-off node groups to a cluster (0.0.17)
  - allow manual customization and timing of nodegroup (e.g., for spot) (0.0.16)
  - extensive changes to aws client (thanks to @rajibhossen!) (0.0.15)

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2022-2023 LLNS, LLC and other HPCIC DevTools Developers.
+Copyright (c) 2022-2024 LLNS, LLC and other HPCIC DevTools Developers.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

kubescaler/cluster.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2023-2024 Lawrence Livermore National Security, LLC and other
 # HPCIC DevTools Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (MIT)

kubescaler/decorators.py

@@ -1,4 +1,4 @@
-# Copyright 2022-2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2023-2024 Lawrence Livermore National Security, LLC and other
 # HPCIC DevTools Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (MIT)

kubescaler/defaults.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2023-2024 Lawrence Livermore National Security, LLC and other
 # HPCIC DevTools Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (MIT)

kubescaler/logger.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2023-2024 Lawrence Livermore National Security, LLC and other
 # HPCIC DevTools Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (MIT)

kubescaler/scaler/aws/ami.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2023-2024 Lawrence Livermore National Security, LLC and other
 # HPCIC DevTools Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (MIT)

kubescaler/scaler/aws/cluster.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2023-2024 Lawrence Livermore National Security, LLC and other
 # HPCIC DevTools Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (MIT)
@@ -306,13 +306,33 @@ def _generate_configuration(self):
             token["status"]["expirationTimestamp"], "%Y-%m-%dT%H:%M:%SZ"
         )
 
+    def waiter_wait_for_nodes(self, nodegroup_name):
+        """
+        Use the "waiter" provided by eks to wait for nodes.
+
+        It is not recommended to use this function as it is flaky.
+        We are keeping it here to preserve the code to try again,
+        as perhaps the flakiness might improve!
+        """
+        try:
+            print(f"Waiting for {nodegroup_name} nodegroup...")
+            waiter = self.eks.get_waiter("nodegroup_active")
+            # MaxAttempts defaults to 120, and Delay 30 seconds
+            waiter.wait(clusterName=self.cluster_name, nodegroupName=nodegroup_name)
+        except Exception as e:
+            # Allow waiting 3 more minutes
+            print(f"Waiting for nodegroup creation exceeded wait time: {e}")
+            time.sleep(180)
+
     @timed
     def wait_for_nodes(self):
         """
         Wait for the nodes to be ready.
 
-        We do this separately to allow timing. This function would be improved if
-        we didn't need subprocess, but the waiter doesn't seem to work.
+        We do this separately to allow timing. This function
+        can't get a perfectly accurate timing given the sleep, but the
+        waiter doesn't work. But I suspect the waiter has a sleep too, so
+        maybe not so bad.
         """
         start = time.time()
         kubectl = self.get_k8s_client()
@@ -329,10 +349,9 @@ def wait_for_nodes(self):
             if ready_count >= self.node_count:
                 break
         print(f"Time for kubernetes to get nodes - {time.time()-start}")
-        return ready_count
         # The waiter doesn't seem to work - so we call kubectl until it's ready
-        # waiter = self.eks.get_waiter("nodegroup_active")
-        # waiter.wait(clusterName=self.cluster_name, nodegroupName=self.node_autoscaling_group_name)
+        # self.waiter_wait_for_nodes(self.node_autoscaling_group_name)
+        return ready_count
 
     @timed
     def watch_for_nodes_in_k8s(self, count):
@@ -778,15 +797,9 @@ def _create_nodegroup(self, node_group, nodegroup_name):
         if node_group is None:
             raise ValueError("Could not create nodegroup")
 
-        try:
-            print(f"Waiting for {nodegroup_name} nodegroup...")
-            waiter = self.eks.get_waiter("nodegroup_active")
-            # MaxAttempts defaults to 120, and Delay 30 seconds
-            waiter.wait(clusterName=self.cluster_name, nodegroupName=nodegroup_name)
-        except Exception as e:
-            # Allow waiting 3 more minutes
-            print(f"Waiting for nodegroup creation exceeded wait time: {e}")
-            time.sleep(180)
+        # DO NOT USE THE WAITER, it is buggy and does not work.
+        # self.waiter_wait_for_nodes(nodegroup_name)
+        self.wait_for_nodes()
 
         # Retrieve the same metadata if we had retrieved it
         return self.eks.describe_nodegroup(

kubescaler/scaler/aws/template.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2023-2024 Lawrence Livermore National Security, LLC and other
 # HPCIC DevTools Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (MIT)

kubescaler/scaler/aws/token.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Copyright 2023-2024 Lawrence Livermore National Security, LLC and other
 # HPCIC DevTools Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (MIT)