formatted using 'black' + rescan vms if there is some vm which is pending scheduling due to resource deficiency

main.py

@@ -12,7 +12,7 @@ import logging
 from decouple import config
 from collections import Counter
 from functools import reduce
-from etcd3_wrapper import Etcd3Wrapper
+from etcd3_wrapper import Etcd3Wrapper, EtcdEntry, PseudoEtcdMeta
 from datetime import datetime
 
 logging.basicConfig(
@@ -141,13 +141,27 @@ def dead_host_mitigation(client: Etcd3Wrapper, dead_hosts_keys):
             client.put(vm.key, vm.value, value_in_json=True)
 
 
+def assign_host(client, vm_prefix, host_prefix, e):
+    host_name = get_suitable_host(
+        client, vm_prefix, host_prefix, e.value["specs"]
+    )
+    if host_name:
+        e.value["status"] = "SCHEDULED_DEPLOY"
+        e.value["hostname"] = host_name
+        client.put(e.key, json.dumps(e.value))
+        return host_name
+    return None
+
+
 def main(vm_prefix, host_prefix):
     client = Etcd3Wrapper(
         host=config("ETCD_HOST"), port=int(config("ETCD_PORT"))
     )
-
-    for events_iterator in [client.get_prefix(vm_prefix),
-                            client.watch_prefix(vm_prefix, timeout=10)]:
+    RESCAN_VMS = False
+    for events_iterator in [
+        client.get_prefix(vm_prefix),
+        client.watch_prefix(vm_prefix, timeout=10),
+    ]:
         for e in events_iterator:
             try:
                 e.value = json.loads(e.value)
@@ -165,17 +179,24 @@ def main(vm_prefix, host_prefix):
                 dead_hosts = dead_host_detection(hosts)
                 dead_host_mitigation(client, dead_hosts)
 
+                if RESCAN_VMS:
+                    RESCAN_VMS = False  # Assume we won't need it after this
+                    vms = client.get_prefix(vm_prefix, value_in_json=True)
+
+                    for vm in vms:
+                        fake_e = EtcdEntry(
+                            PseudoEtcdMeta(key=vm.key), value=vm.value
+                        )
+                        if (assign_host(client, vm_prefix, host_prefix,
+                                        fake_e) is None):
+                            # We need it because we still have vm left
+                            # to schedule
+                            RESCAN_VMS = True
+
             elif e_status == "REQUESTED_NEW":
-                host_name = get_suitable_host(
-                    client, vm_prefix, host_prefix, e.value["specs"]
-                )
-                if host_name:
-                    e.value["status"] = "SCHEDULED_DEPLOY"
-                    e.value["hostname"] = host_name
-                    client.put(e.key, json.dumps(e.value))
-                else:
-                    # email admin
+                if assign_host(client, vm_prefix, host_prefix, e) is None:
                     print("No Resource Left. Emailing admin....")
+                    RESCAN_VMS = True
 
 
 if __name__ == "__main__":