diff --git a/etcd3_wrapper b/etcd3_wrapper index d079aca..9abc74e 160000 --- a/etcd3_wrapper +++ b/etcd3_wrapper @@ -1 +1 @@ -Subproject commit d079acadf29e6df55c329574604148631d4ad4bc +Subproject commit 9abc74e387bf94832a0ad95644d667abb87e5529 diff --git a/main.py b/main.py index 99633cc..74be3e4 100644 --- a/main.py +++ b/main.py @@ -13,6 +13,7 @@ from collections import Counter from functools import reduce from etcd3_wrapper import Etcd3Wrapper, EtcdEntry, PseudoEtcdMeta from datetime import datetime +from ucloud_common.enums import VMStatus, RUNNING_VM_STATUSES logging.basicConfig( level=logging.DEBUG, @@ -33,6 +34,7 @@ class VmPool(object): @staticmethod def by_host(vms, host): + print(vms) return list(filter(lambda x: x[1]["hostname"] == host, vms)) @staticmethod @@ -126,7 +128,6 @@ def dead_host_mitigation(client: Etcd3Wrapper, dead_hosts_keys): host = client.get(host_key, value_in_json=True) host.value["status"] = "DEAD" host.value["last_heartbeat"] = datetime.utcnow().isoformat() - client.put(host.key, host.value, value_in_json=True) # Find all vms that were hosted on this dead host all_vms = client.get_prefix(config("VM_PREFIX"), value_in_json=True) @@ -135,9 +136,11 @@ def dead_host_mitigation(client: Etcd3Wrapper, dead_hosts_keys): ) for vm in vms_hosted_on_dead_host: vm.value["hostname"] = "" - if vm.value["status"] != "STOPPED": - vm.value["status"] = "REQUESTED_NEW" + if vm.value["status"] in RUNNING_VM_STATUSES: + vm.value["status"] = VMStatus.requested_start client.put(vm.key, vm.value, value_in_json=True) + + client.put(host.key, host.value, value_in_json=True) def assign_host(client, vm_prefix, host_prefix, e): @@ -177,7 +180,6 @@ def main(vm_prefix, host_prefix): e_status = e.value["status"] if e_status == "TIMEOUT": - client.client.delete(e.key) logging.info("Timeout") hosts = client.get_prefix(host_prefix, value_in_json=True) dead_hosts = dead_host_detection(hosts) diff --git a/ucloud_common/__init__.py b/ucloud_common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ucloud_common/enums.py b/ucloud_common/enums.py new file mode 100644 index 0000000..6d5f403 --- /dev/null +++ b/ucloud_common/enums.py @@ -0,0 +1,34 @@ +from enum import Enum + +class VMStatus(Enum): + # Must be only assigned to brand new VM + requested_new = "REQUESTED_NEW" + + # Only Assigned to already created vm + requested_start = "REQUESTED_START" + + # These all are for running vms + requested_shutdown = "REQUESTED_SHUTDOWN" + requested_suspend = "REQUESTED_SUSPEND" + requested_resume = "REQUESTED_RESUME" + requested_migrate = "REQUESTED_MIGRATE" + + # either its image is not found or user requested + # to delete it + deleted = "DELETED" + + stopped = "STOPPED" # After requested_shutdown + killed = "KILLED" # either host died or vm died itself + + running = "RUNNING" + suspended = "SUSPENDED" + + +class HostStatus(Enum): + alive = "ALIVE" + dead = "DEAD" + + +RUNNING_VM_STATUSES = [VMStatus.requested_shutdown, VMStatus.requested_suspend, + VMStatus.requested_resume, VMStatus.requested_migrate, + VMStatus.running, VMStatus.suspended] \ No newline at end of file