order of statement changed

etcd3_wrapper

@@ -1 +1 @@
-Subproject commit d079acadf29e6df55c329574604148631d4ad4bc
+Subproject commit 9abc74e387bf94832a0ad95644d667abb87e5529

main.py

@@ -13,6 +13,7 @@ from collections import Counter
 from functools import reduce
 from etcd3_wrapper import Etcd3Wrapper, EtcdEntry, PseudoEtcdMeta
 from datetime import datetime
+from ucloud_common.enums import VMStatus, RUNNING_VM_STATUSES
 
 logging.basicConfig(
     level=logging.DEBUG,
@@ -33,6 +34,7 @@ class VmPool(object):
 
     @staticmethod
     def by_host(vms, host):
+        print(vms)
         return list(filter(lambda x: x[1]["hostname"] == host, vms))
 
     @staticmethod
@@ -126,7 +128,6 @@ def dead_host_mitigation(client: Etcd3Wrapper, dead_hosts_keys):
         host = client.get(host_key, value_in_json=True)
         host.value["status"] = "DEAD"
         host.value["last_heartbeat"] = datetime.utcnow().isoformat()
-        client.put(host.key, host.value, value_in_json=True)
 
         # Find all vms that were hosted on this dead host
         all_vms = client.get_prefix(config("VM_PREFIX"), value_in_json=True)
@@ -135,9 +136,11 @@ def dead_host_mitigation(client: Etcd3Wrapper, dead_hosts_keys):
         )
         for vm in vms_hosted_on_dead_host:
             vm.value["hostname"] = ""
-            if vm.value["status"] != "STOPPED":
-                vm.value["status"] = "REQUESTED_NEW"
+            if vm.value["status"] in RUNNING_VM_STATUSES:
+                vm.value["status"] = VMStatus.requested_start
             client.put(vm.key, vm.value, value_in_json=True)
+        
+        client.put(host.key, host.value, value_in_json=True)
 
 
 def assign_host(client, vm_prefix, host_prefix, e):
@@ -177,7 +180,6 @@ def main(vm_prefix, host_prefix):
             e_status = e.value["status"]
 
             if e_status == "TIMEOUT":
-                client.client.delete(e.key)
                 logging.info("Timeout")
                 hosts = client.get_prefix(host_prefix, value_in_json=True)
                 dead_hosts = dead_host_detection(hosts)

ucloud_common/enums.py

@@ -0,0 +1,34 @@
+from enum import Enum
+
+class VMStatus(Enum):
+    # Must be only assigned to brand new VM
+    requested_new = "REQUESTED_NEW"
+
+    # Only Assigned to already created vm
+    requested_start = "REQUESTED_START"
+
+    # These all are for running vms
+    requested_shutdown = "REQUESTED_SHUTDOWN"
+    requested_suspend = "REQUESTED_SUSPEND"
+    requested_resume = "REQUESTED_RESUME"
+    requested_migrate = "REQUESTED_MIGRATE"
+
+    # either its image is not found or user requested
+    # to delete it
+    deleted = "DELETED"
+    
+    stopped = "STOPPED" # After requested_shutdown
+    killed = "KILLED"  # either host died or vm died itself
+
+    running = "RUNNING"
+    suspended = "SUSPENDED"
+
+
+class HostStatus(Enum):
+    alive = "ALIVE"
+    dead = "DEAD"
+
+
+RUNNING_VM_STATUSES = [VMStatus.requested_shutdown, VMStatus.requested_suspend,
+                       VMStatus.requested_resume, VMStatus.requested_migrate,
+                       VMStatus.running, VMStatus.suspended]