From 01296e998e787ffb424ffba88176ca553a805c7d Mon Sep 17 00:00:00 2001 From: meow Date: Tue, 30 Jul 2019 18:13:05 +0500 Subject: [PATCH] code added --- .gitignore | 3 + Pipfile | 13 ++ Pipfile.lock | 179 ++++++++++++++++++ __init__.py | 0 Makefile => docs/Makefile | 0 .../ucloud-networking.dot | 0 .../ucloud-vm-states.dot | 0 docs/vm_states.dot | 39 ++++ helpers.py | 25 +++ host.py | 56 ++++++ rbd.py | 15 ++ test.py | 13 ++ vm.py | 87 +++++++++ 13 files changed, 430 insertions(+) create mode 100644 Pipfile create mode 100644 Pipfile.lock create mode 100644 __init__.py rename Makefile => docs/Makefile (100%) rename ucloud-networking.dot => docs/ucloud-networking.dot (100%) rename ucloud-vm-states.dot => docs/ucloud-vm-states.dot (100%) create mode 100644 docs/vm_states.dot create mode 100644 helpers.py create mode 100644 host.py create mode 100644 rbd.py create mode 100644 test.py create mode 100644 vm.py diff --git a/.gitignore b/.gitignore index e33609d..93056d5 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,4 @@ *.png +__pycache__/ +etcd3_wrapper + diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..78ddc12 --- /dev/null +++ b/Pipfile @@ -0,0 +1,13 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] + +[packages] +python-etcd3 = {editable = true,git = "https://github.com/kragniz/python-etcd3"} +pylint = "*" + +[requires] +python_version = "3.7" diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 0000000..20edd9a --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,179 @@ +{ + "_meta": { + "hash": { + "sha256": "ffcb074068a4397c180d89ef54fa8dfe23f2a7625c422bb69e932fa2718da782" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.7" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "astroid": { + "hashes": [ + "sha256:6560e1e1749f68c64a4b5dee4e091fce798d2f0d84ebe638cf0e0585a343acf4", + "sha256:b65db1bbaac9f9f4d190199bb8680af6f6f84fd3769a5ea883df8a91fe68b4c4" + ], + "version": "==2.2.5" + }, + "grpcio": { + "hashes": [ + "sha256:03b78b4e7dcdfe3e257bb528cc93923f9cbbab6d5babf15a60d21e9a4a70b1a2", + "sha256:1ce0ccfbdfe84387dbcbf44adb4ae16ec7ae70e166ffab478993eb1ea1cba3ce", + "sha256:22e167a9406d73dd19ffe8ed6a485f17e6eac82505be8c108897f15e68badcbb", + "sha256:31d0aeca8d8ee2301c62c5c340e0889d653b1280d68f9fa203982cb6337b050e", + "sha256:44c7f99ca17ebbcc96fc54ed00b454d8313f1eac28c563098d8b901025aff941", + "sha256:5471444f53f9db6a1f1f11f5dbc173228881df8446380b6b98f90afb8fd8348e", + "sha256:561bca3b1bde6d6564306eb05848fd155136e9c3a25d2961129b1e2edba22fce", + "sha256:5bf58e1d2c2f55365c06e8cb5abe067b88ca2e5550fb62009c41df4b54505acf", + "sha256:6b7163d1e85d76b0815df63fcc310daec02b44532bb433f743142d4febcb181f", + "sha256:766d79cddad95f5f6020037fe60ea8b98578afdf0c59d5a60c106c1bdd886303", + "sha256:770b7372d5ca68308ff66d7baee53369fa5ce985f84bcb6aa1948c1f2f7b02f2", + "sha256:7ab178da777fc0f55b6aef5a755f99726e8e4b75e3903954df07b27059b54fcf", + "sha256:8078305e77c2f6649d36b24d8778096413e474d9d7892c6f92cfb589c9d71b2e", + "sha256:85600b63a386d860eeaa955e9335e18dd0d7e5477e9214825abf2c2884488369", + "sha256:857d9b939ae128be1c0c792eb885c7ff6a386b9dea899ac4b06f4d90a31f9d87", + "sha256:87a41630c90c179fa5c593400f30a467c498972c702f348d41e19dafeb1d319e", + "sha256:8805d486c6128cc0fcc8ecf16c4095d99a8693a541ef851429ab334e028a4a97", + "sha256:8d71b7a89c306a41ccc7741fc9409b14f5b86727455c2a1c0c7cfcb0f784e1f2", + "sha256:9e1b80bd65f8f160880cb4dad7f55697f6d37b2d7f251fc0c2128e811928f369", + "sha256:9e290c84a145ae2411ee0ec9913c41cd7500e2e7485fe93632434d84ef4fda67", + "sha256:9ec9f88b5bc94bd99372f27cdd53af1c92ba06717380b127733b953cfb181174", + "sha256:a0a02a8b4ba6deadf706d5f849539b3685b72b186a3c9ef5d43e8972ed60fb6f", + "sha256:a4059c59519f5940e01a071f74ae2a60ea8f6185b03d22a09d40c7959a36b16b", + "sha256:a6e028c2a6da2ebfa2365a5b32531d311fbfec0e3600fc27e901b64f0ff7e54e", + "sha256:adcdebf9f8463df4120c427cf6c9aed39258bccd03ed37b6939e7a145d64d6e0", + "sha256:bdec982610259d07156a58f80b8c3e69be7751a9208bc577b059c5193d087fad", + "sha256:cefc4d4251ffb73feb303d4b7e9d6c367cb60f2db16d259ea28b114045f965aa", + "sha256:d4145c8aa6afbac10ad27e408f7ce15992fe89ba5d0b4abca31c0c2729864c03", + "sha256:da76dc5ad719ee99de5ea28a5629ff92172cbb4a70d8a6ae3a5b7a53c7382ce1", + "sha256:dde2452c08ef8b6426ccab6b5b6de9f06d836d9937d6870e68153cbf8cb49348", + "sha256:e3d88091d2539a4868750914a6fe7b9ec50e42b913851fc1b77423b5bd918530", + "sha256:f9c67cfe6278499d7f83559dc6322a8bbb108e307817a3d7acbfea807b3603cc" + ], + "version": "==1.22.0" + }, + "isort": { + "hashes": [ + "sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1", + "sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd" + ], + "version": "==4.3.21" + }, + "lazy-object-proxy": { + "hashes": [ + "sha256:159a745e61422217881c4de71f9eafd9d703b93af95618635849fe469a283661", + "sha256:23f63c0821cc96a23332e45dfaa83266feff8adc72b9bcaef86c202af765244f", + "sha256:3b11be575475db2e8a6e11215f5aa95b9ec14de658628776e10d96fa0b4dac13", + "sha256:3f447aff8bc61ca8b42b73304f6a44fa0d915487de144652816f950a3f1ab821", + "sha256:4ba73f6089cd9b9478bc0a4fa807b47dbdb8fad1d8f31a0f0a5dbf26a4527a71", + "sha256:4f53eadd9932055eac465bd3ca1bd610e4d7141e1278012bd1f28646aebc1d0e", + "sha256:64483bd7154580158ea90de5b8e5e6fc29a16a9b4db24f10193f0c1ae3f9d1ea", + "sha256:6f72d42b0d04bfee2397aa1862262654b56922c20a9bb66bb76b6f0e5e4f9229", + "sha256:7c7f1ec07b227bdc561299fa2328e85000f90179a2f44ea30579d38e037cb3d4", + "sha256:7c8b1ba1e15c10b13cad4171cfa77f5bb5ec2580abc5a353907780805ebe158e", + "sha256:8559b94b823f85342e10d3d9ca4ba5478168e1ac5658a8a2f18c991ba9c52c20", + "sha256:a262c7dfb046f00e12a2bdd1bafaed2408114a89ac414b0af8755c696eb3fc16", + "sha256:acce4e3267610c4fdb6632b3886fe3f2f7dd641158a843cf6b6a68e4ce81477b", + "sha256:be089bb6b83fac7f29d357b2dc4cf2b8eb8d98fe9d9ff89f9ea6012970a853c7", + "sha256:bfab710d859c779f273cc48fb86af38d6e9210f38287df0069a63e40b45a2f5c", + "sha256:c10d29019927301d524a22ced72706380de7cfc50f767217485a912b4c8bd82a", + "sha256:dd6e2b598849b3d7aee2295ac765a578879830fb8966f70be8cd472e6069932e", + "sha256:e408f1eacc0a68fed0c08da45f31d0ebb38079f043328dce69ff133b95c29dc1" + ], + "version": "==1.4.1" + }, + "mccabe": { + "hashes": [ + "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", + "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" + ], + "version": "==0.6.1" + }, + "protobuf": { + "hashes": [ + "sha256:05c36022fef3c7d3562ac22402965c0c2b9fe8421f459bb377323598996e407f", + "sha256:139b7eadcca0a861d60b523cb37d9475505e0dfb07972436b15407c2b968d87e", + "sha256:15f683006cb77fb849b1f561e509b03dd2b7dcc749086b8dd1831090d0ba4740", + "sha256:2ad566b7b7cdd8717c7af1825e19f09e8fef2787b77fcb979588944657679604", + "sha256:35cfcf97642ef62108e10a9431c77733ec7eaab8e32fe4653de20403429907cb", + "sha256:387822859ecdd012fdc25ec879f7f487da6e1d5b1ae6115e227e6be208836f71", + "sha256:4df14cbe1e7134afcfdbb9f058949e31c466de27d9b2f7fb4da9e0b67231b538", + "sha256:586c4ca37a7146d4822c700059f150ac3445ce0aef6f3ea258640838bb892dc2", + "sha256:58b11e530e954d29ab3180c48dc558a409f705bf16739fd4e0d3e07924ad7add", + "sha256:63c8c98ccb8c95f41c18fb829aeeab21c6249adee4ed75354125bdc44488f30e", + "sha256:72edcbacd0c73eef507d2ff1af99a6c27df18e66a3ff4351e401182e4de62b03", + "sha256:83dc8a561b3b954fd7002c690bb83278b8d1742a1e28abba9aaef28b0c8b437d", + "sha256:913171ecc84c2726b86574e40549a0ea619d569657c5a5ff782a3be7d81401a5", + "sha256:aabb7c741d3416671c3e6fe7c52970a226e6a8274417a97d7d795f953fadef36", + "sha256:b3452bbda12b1cbe2187d416779de07b2ab4c497d83a050e43c344778763721d", + "sha256:c5d5b8d4a9212338297fa1fa44589f69b470c0ba1d38168b432d577176b386a8", + "sha256:d86ee389c2c4fc3cebabb8ce83a8e97b6b3b5dc727b7419c1ccdc7b6e545a233", + "sha256:f2db8c754de788ab8be5e108e1e967c774c0942342b4f8aaaf14063889a6cfdc" + ], + "version": "==3.9.0" + }, + "pylint": { + "hashes": [ + "sha256:5d77031694a5fb97ea95e828c8d10fc770a1df6eb3906067aaed42201a8a6a09", + "sha256:723e3db49555abaf9bf79dc474c6b9e2935ad82230b10c1138a71ea41ac0fff1" + ], + "index": "pypi", + "version": "==2.3.1" + }, + "python-etcd3": { + "editable": true, + "git": "https://github.com/kragniz/python-etcd3", + "ref": "cdc4c48bde88a795230a02aa574df84ed9ccfa52" + }, + "six": { + "hashes": [ + "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", + "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" + ], + "version": "==1.12.0" + }, + "tenacity": { + "hashes": [ + "sha256:a0c3c5f7ae0c33f5556c775ca059c12d6fd8ab7121613a713e8b7d649908571b", + "sha256:b87c1934daa0b2ccc7db153c37b8bf91d12f165936ade8628e7b962b92dc7705" + ], + "version": "==5.0.4" + }, + "typed-ast": { + "hashes": [ + "sha256:18511a0b3e7922276346bcb47e2ef9f38fb90fd31cb9223eed42c85d1312344e", + "sha256:262c247a82d005e43b5b7f69aff746370538e176131c32dda9cb0f324d27141e", + "sha256:2b907eb046d049bcd9892e3076c7a6456c93a25bebfe554e931620c90e6a25b0", + "sha256:354c16e5babd09f5cb0ee000d54cfa38401d8b8891eefa878ac772f827181a3c", + "sha256:4e0b70c6fc4d010f8107726af5fd37921b666f5b31d9331f0bd24ad9a088e631", + "sha256:630968c5cdee51a11c05a30453f8cd65e0cc1d2ad0d9192819df9978984529f4", + "sha256:66480f95b8167c9c5c5c87f32cf437d585937970f3fc24386f313a4c97b44e34", + "sha256:71211d26ffd12d63a83e079ff258ac9d56a1376a25bc80b1cdcdf601b855b90b", + "sha256:95bd11af7eafc16e829af2d3df510cecfd4387f6453355188342c3e79a2ec87a", + "sha256:bc6c7d3fa1325a0c6613512a093bc2a2a15aeec350451cbdf9e1d4bffe3e3233", + "sha256:cc34a6f5b426748a507dd5d1de4c1978f2eb5626d51326e43280941206c209e1", + "sha256:d755f03c1e4a51e9b24d899561fec4ccaf51f210d52abdf8c07ee2849b212a36", + "sha256:d7c45933b1bdfaf9f36c579671fec15d25b06c8398f113dab64c18ed1adda01d", + "sha256:d896919306dd0aa22d0132f62a1b78d11aaf4c9fc5b3410d3c666b818191630a", + "sha256:ffde2fbfad571af120fcbfbbc61c72469e72f550d676c3342492a9dfdefb8f12" + ], + "markers": "implementation_name == 'cpython'", + "version": "==1.4.0" + }, + "wrapt": { + "hashes": [ + "sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1" + ], + "version": "==1.11.2" + } + }, + "develop": {} +} diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Makefile b/docs/Makefile similarity index 100% rename from Makefile rename to docs/Makefile diff --git a/ucloud-networking.dot b/docs/ucloud-networking.dot similarity index 100% rename from ucloud-networking.dot rename to docs/ucloud-networking.dot diff --git a/ucloud-vm-states.dot b/docs/ucloud-vm-states.dot similarity index 100% rename from ucloud-vm-states.dot rename to docs/ucloud-vm-states.dot diff --git a/docs/vm_states.dot b/docs/vm_states.dot new file mode 100644 index 0000000..a7a87d0 --- /dev/null +++ b/docs/vm_states.dot @@ -0,0 +1,39 @@ +digraph G{ + + subgraph cluster_1 { + style=filled; + color=lightgrey; + node [style=filled,color=white]; + REQUESTED_NEW -> SCHEDULED_DEPLOY [label="assign_host()"]; + SCHEDULED_DEPLOY -> REQUESTED_START [label="create_vm()"]; + label="ucloud-scheduler"; + } + + subgraph cluster_2 { + style=filled; + color=lightgrey; + node [style=filled,color=white]; + + REQUESTED_START -> RUNNING [label="start_vm()"] + RUNNING -> SUSPENDED [label="REQUESTED_SUSPEND"] + RUNNING -> RUNNING [label="REQUESTED_RESUME"] + RUNNING -> SHUTDOWN [label="REQUESTED_SHUTDOWN"] + SUSPENDED -> RUNNING [label="REQUESTED_RESUME"] + + } + + subgraph cluster_3 { + style=filled; + color=lightgrey; + + SCHEDULED_DEPLOY -> REQUESTED_NEW [label="Host Died"] + REQUESTED_START -> KILLED [label="Host Died"] + RUNNING -> KILLED [label="Host Died OR VM Died"] + SUSPENDED -> KILLED [label="VM Died"] + SHUTDOWN -> SHUTDOWN [label="VM Died"] + + label="ucloud-vm & ucloud-scheduler"; + + } + +} \ No newline at end of file diff --git a/helpers.py b/helpers.py new file mode 100644 index 0000000..951bf18 --- /dev/null +++ b/helpers.py @@ -0,0 +1,25 @@ +from .etcd3_wrapper import EtcdEntry + + +class SpecificEtcdEntryBase(object): + + def __init__(self, e: EtcdEntry): + self.key = e.key + + for k in e.value.keys(): + self.__setattr__(k, e.value[k]) + + def original_keys(self): + r = dict(self.__dict__) + del r["key"] + return r + + @property + def value(self): + return self.original_keys() + + def __repr__(self): + _return = f"""{self.key}""" + for key in self.original_keys(): + _return += f",{key} = {self.__getattribute__(key)}" + return _return diff --git a/host.py b/host.py new file mode 100644 index 0000000..03ee057 --- /dev/null +++ b/host.py @@ -0,0 +1,56 @@ +from .helpers import SpecificEtcdEntryBase +from .etcd3_wrapper import EtcdEntry +from datetime import datetime + + +class HostStatus(object): + alive = "ALIVE" + dead = "DEAD" + + +class HostEntry(SpecificEtcdEntryBase): + def __init__(self, e: EtcdEntry): + self.specs = dict() + self.hostname = "" + self.status = "" + self.last_heartbeat = "" + + super().__init__(e) + + def update_heartbeat(self): + self.status = HostStatus.alive + self.last_heartbeat = datetime.utcnow().isoformat() + + def is_alive(self): + last_heartbeat = datetime.fromisoformat(self.last_heartbeat) + delta = datetime.utcnow() - last_heartbeat + if delta.total_seconds() > 60: + return False + return True + + def declare_dead(self): + self.status = HostStatus.dead + self.last_heartbeat = datetime.utcnow().isoformat() + + +class HostPool(object): + def __init__(self, etcd_client, host_prefix): + self.client = etcd_client + self.prefix = host_prefix + + @property + def hosts(self): + _hosts = self.client.get_prefix(self.prefix, value_in_json=True) + return [HostEntry(host) for host in _hosts] + + def get(self, key): + v = self.client.get(key, value_in_json=True) + return HostEntry(v) + + def put(self, obj: HostEntry): + self.client.put(obj.key, obj.value, value_in_json=True) + + def by_status(self, status, _hosts=None): + if _hosts is None: + _hosts = self.hosts + return list(filter(lambda x: x.status == status, _hosts)) \ No newline at end of file diff --git a/rbd.py b/rbd.py new file mode 100644 index 0000000..b87a54c --- /dev/null +++ b/rbd.py @@ -0,0 +1,15 @@ +import subprocess + + +class RBD(object): + @staticmethod + def ls(pool): + output = "" + try: + output = subprocess.check_output( + ["rbd", "ls", pool], stderr=subprocess.PIPE + ).decode("utf-8").strip() + except subprocess.CalledProcessError as e: + raise Exception(e.stderr) + return output.split("\n") + diff --git a/test.py b/test.py new file mode 100644 index 0000000..5549645 --- /dev/null +++ b/test.py @@ -0,0 +1,13 @@ +from vm import VmPool +from etcd3_wrapper import Etcd3Wrapper + +client = Etcd3Wrapper() +vm_pool = VmPool(client, "/v1/vm") +# vms = vm_pool.by_status("REQUESTED_NEW") +# for vm in vms: +# print(vm) +# print() + +v = vm_pool.get("/v1/vm/8402926c7a164966982eae48b2d6d1a9") +# print(dir(v)) +print(v) \ No newline at end of file diff --git a/vm.py b/vm.py new file mode 100644 index 0000000..b544166 --- /dev/null +++ b/vm.py @@ -0,0 +1,87 @@ +from etcd3_wrapper import EtcdEntry +from .helpers import SpecificEtcdEntryBase + + +class VMStatus(object): + # Must be only assigned to brand new VM + requested_new = "REQUESTED_NEW" + + # Host assigned to VM but not created yet. + scheduled_deploy = "SCHEDULED_DEPLOY" + + # Only Assigned to already created vm + requested_start = "REQUESTED_START" + + # These all are for running vms + requested_shutdown = "REQUESTED_SHUTDOWN" + requested_suspend = "REQUESTED_SUSPEND" + requested_resume = "REQUESTED_RESUME" + requested_migrate = "REQUESTED_MIGRATE" + + # either its image is not found or user requested + # to delete it + deleted = "DELETED" + + stopped = "STOPPED" # After requested_shutdown + killed = "KILLED" # either host died or vm died itself + + running = "RUNNING" + suspended = "SUSPENDED" + + +RUNNING_VM_STATUSES = [VMStatus.requested_shutdown, VMStatus.requested_suspend, + VMStatus.requested_resume, VMStatus.requested_migrate, + VMStatus.running, VMStatus.suspended] + + +class VMEntry(SpecificEtcdEntryBase): + def __init__(self, e: EtcdEntry): + self.owner = "" + self.specs = dict() + self.hostname = "" + self.status = "" + self.image_uuid = "" + + super().__init__(e) + + @property + def uuid(self): + return self.key.split("/")[-1] + + def declare_killed(self): + self.hostname = "" + if self.status in RUNNING_VM_STATUSES: + self.status = VMStatus.killed + + +class VmPool(object): + def __init__(self, etcd_client, vm_prefix): + self.client = etcd_client + self.prefix = vm_prefix + + @property + def vms(self): + _vms = self.client.get_prefix(self.prefix, value_in_json=True) + return [VMEntry(vm) for vm in _vms] + + def by_host(self, host, _vms=None): + if _vms is None: + _vms = self.vms + return list(filter(lambda x: x.hostname == host, _vms)) + + def by_status(self, status, _vms=None): + if _vms is None: + _vms = self.vms + return list(filter(lambda x: x.status == status, _vms)) + + def except_status(self, status, _vms=None): + if _vms is None: + _vms = self.vms + return list(filter(lambda x: x.status != status, _vms)) + + def get(self, key): + v = self.client.get(key, value_in_json=True) + return VMEntry(v) + + def put(self, obj: VMEntry): + self.client.put(obj.key, obj.value, value_in_json=True)