From b28fe5d78bf576cfaf9135c073fa7aac9b189b94 Mon Sep 17 00:00:00 2001 From: Ahmed Bilal Khalid Date: Fri, 13 Sep 2019 16:26:26 +0500 Subject: [PATCH] a --- Pipfile | 9 +-- Pipfile.lock | 197 +++++++++++----------------------------------- config.py | 11 ++- main.py | 29 ++++--- virtualmachine.py | 151 +++++++++++++++++++++-------------- 5 files changed, 165 insertions(+), 232 deletions(-) mode change 100755 => 100644 Pipfile.lock diff --git a/Pipfile b/Pipfile index 736a116..e86ef9e 100755 --- a/Pipfile +++ b/Pipfile @@ -7,15 +7,14 @@ verify_ssl = true bandit = "*" pylama = "*" prospector = "*" +pylint = "*" [packages] python-decouple = "*" -cython = "*" -pylint = "*" -grpcio = "*" -python-etcd3 = {editable = true,git = "https://github.com/kragniz/python-etcd3"} sshtunnel = "*" bitmath = "*" +etcd3-wrapper = "*" +ucloud-common = "*" [requires] -python_version = "3.7" +python_version = "3.5" diff --git a/Pipfile.lock b/Pipfile.lock old mode 100755 new mode 100644 index 98c3a89..9f29807 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,11 +1,11 @@ { "_meta": { "hash": { - "sha256": "05c0e0fbcad89f0740bfca9356e4493d95eb898c35017258f750c8c4674034fa" + "sha256": "0219283481dc052f9ca13f16133fc9a622bfb63371dc633103aa365b98268a41" }, "pipfile-spec": 6, "requires": { - "python_version": "3.7" + "python_version": "3.5" }, "sources": [ { @@ -23,13 +23,6 @@ ], "version": "==0.24.0" }, - "astroid": { - "hashes": [ - "sha256:6560e1e1749f68c64a4b5dee4e091fce798d2f0d84ebe638cf0e0585a343acf4", - "sha256:b65db1bbaac9f9f4d190199bb8680af6f6f84fd3769a5ea883df8a91fe68b4c4" - ], - "version": "==2.2.5" - }, "bcrypt": { "hashes": [ "sha256:0258f143f3de96b7c14f762c770f5fc56ccd72f8a1857a451c1cd9a655d9ac89", @@ -112,39 +105,19 @@ ], "version": "==2.7" }, - "cython": { + "etcd3": { "hashes": [ - "sha256:07efba7b32c082c519b75e3b03821c2f32848e2b3e9986c784bbd8ffaf0666d7", - "sha256:08db41daf18fabf7b7a85e39aa26954f6246994540043194af026c0df65a4942", - "sha256:19bbe3caf885a1d2e2c30eacc10d1e45dbbefb156493fe1d5d1adc1668cc1269", - "sha256:1c574f2f2ba760b82b2bcf6262e77e75589247dc5ef796a3ff1b2213e50ee452", - "sha256:1dfe672c686e34598bdbaa93c3b30acb3720ae9258232a4f68ba04ee9969063d", - "sha256:283faea84e6c4e54c3f5c8ff89aa2b6c1c3a813aad4f6d48ed3b9cc9043ef9f9", - "sha256:2a145888d0942e7c36e86a7b7c7e2923cb9f7055805a3b72dcb137e3efdb0979", - "sha256:3f75065936e16569d6e13dfd76de988f5eabeae460aa54770c9b961ab6f747fc", - "sha256:4d78124f5f281f1d5d5b7919cbbc65a7073ff93562def81ee78a8307e6e72494", - "sha256:5ba4d088b8e5d59b8a5911ca9c72952acf3c83296b57daf75af92fb2af1e8423", - "sha256:6b19daeda1d5d1dfc973b291246f6a63a663b20c33980724d6d073c562719536", - "sha256:790c7dc80fd1c3e38acefe06027e2f5a8466c128c7e47c6e140fd5316132574d", - "sha256:7f8c4e648881454ba3ba0bcf3b21a9e1878a67d20ea2b8d9ec1c4c628592ab6b", - "sha256:8bcd3f597290f9902548d6355898d7e376e7f3762f89db9cd50b2b58429df9e8", - "sha256:8ffb18f71972a5c718a8600d9f52e3507f0d6fb72a978e03270d34a7035c98fb", - "sha256:92f025df1cb391e09f65775598c7dfb7efad72d74713775db54e267f62ca94a1", - "sha256:93cf1c72472a2fd0ef4c52f6074dab08fc28d475b9c824ba73a52701f7a48ae1", - "sha256:9a7fa692cdc967fdbf6a053c1975137d01f6935dede2ef222c71840b290caf79", - "sha256:a68eb0c1375f2401de881692b30370a51e550052b8e346b2f71bbdbdc74a214f", - "sha256:ac3b7a12ddd52ea910ee3a041e6bc65df7a52f0ba7bd10fb7123502af482c152", - "sha256:b402b700edaf571a0bae18ec35d5b71c266873a6616412b672435c10b6d8f041", - "sha256:c29d069a4a30f472482343c866f7486731ad638ef9af92bfe5fca9c7323d638e", - "sha256:d822311498f185db449b687336b4e5db7638c8d8b03bdf10ae91d74e23c7cc0c", - "sha256:dccc8df9e1ac158b06777bbaaeb4516f245f9b147701ae25e6023960e4a0c2a3", - "sha256:e31f4b946c2765b2f35440fdb4b00c496dfc5babc53c7ae61966b41171d1d59f", - "sha256:eb43f9e582cc221ee2832e25ea6fe5c06f2acc9da6353c562e922f107db12af8", - "sha256:f07822248110fd6213db8bc2745fdbbccef6f2b3d18ac91a7fba29c6bc575da5", - "sha256:ff69854f123b959d4ae14bd5330714bb9ee4360052992dc0fbd0a3dee4261f95" + "sha256:25a524b9f032c6631ff0097532907dea81243eaa63c3744510fd1598cc4e0e87" + ], + "version": "==0.10.0" + }, + "etcd3-wrapper": { + "hashes": [ + "sha256:0296a4cc7c75c6c432f54e95699271894716e99048c9987df55b6885ed9d5d07", + "sha256:8c4e90593ea6586978f0fbd484e46fd7d8554e06cb9804d34805a1f15a046b63" ], "index": "pypi", - "version": "==0.29.13" + "version": "==0.5.2" }, "grpcio": { "hashes": [ @@ -183,43 +156,6 @@ ], "version": "==1.23.0" }, - "isort": { - "hashes": [ - "sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1", - "sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd" - ], - "version": "==4.3.21" - }, - "lazy-object-proxy": { - "hashes": [ - "sha256:159a745e61422217881c4de71f9eafd9d703b93af95618635849fe469a283661", - "sha256:23f63c0821cc96a23332e45dfaa83266feff8adc72b9bcaef86c202af765244f", - "sha256:3b11be575475db2e8a6e11215f5aa95b9ec14de658628776e10d96fa0b4dac13", - "sha256:3f447aff8bc61ca8b42b73304f6a44fa0d915487de144652816f950a3f1ab821", - "sha256:4ba73f6089cd9b9478bc0a4fa807b47dbdb8fad1d8f31a0f0a5dbf26a4527a71", - "sha256:4f53eadd9932055eac465bd3ca1bd610e4d7141e1278012bd1f28646aebc1d0e", - "sha256:64483bd7154580158ea90de5b8e5e6fc29a16a9b4db24f10193f0c1ae3f9d1ea", - "sha256:6f72d42b0d04bfee2397aa1862262654b56922c20a9bb66bb76b6f0e5e4f9229", - "sha256:7c7f1ec07b227bdc561299fa2328e85000f90179a2f44ea30579d38e037cb3d4", - "sha256:7c8b1ba1e15c10b13cad4171cfa77f5bb5ec2580abc5a353907780805ebe158e", - "sha256:8559b94b823f85342e10d3d9ca4ba5478168e1ac5658a8a2f18c991ba9c52c20", - "sha256:a262c7dfb046f00e12a2bdd1bafaed2408114a89ac414b0af8755c696eb3fc16", - "sha256:acce4e3267610c4fdb6632b3886fe3f2f7dd641158a843cf6b6a68e4ce81477b", - "sha256:be089bb6b83fac7f29d357b2dc4cf2b8eb8d98fe9d9ff89f9ea6012970a853c7", - "sha256:bfab710d859c779f273cc48fb86af38d6e9210f38287df0069a63e40b45a2f5c", - "sha256:c10d29019927301d524a22ced72706380de7cfc50f767217485a912b4c8bd82a", - "sha256:dd6e2b598849b3d7aee2295ac765a578879830fb8966f70be8cd472e6069932e", - "sha256:e408f1eacc0a68fed0c08da45f31d0ebb38079f043328dce69ff133b95c29dc1" - ], - "version": "==1.4.1" - }, - "mccabe": { - "hashes": [ - "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", - "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" - ], - "version": "==0.6.1" - }, "paramiko": { "hashes": [ "sha256:99f0179bdc176281d21961a003ffdb2ec369daac1a1007241f53374e376576cf", @@ -254,14 +190,6 @@ ], "version": "==2.19" }, - "pylint": { - "hashes": [ - "sha256:5d77031694a5fb97ea95e828c8d10fc770a1df6eb3906067aaed42201a8a6a09", - "sha256:723e3db49555abaf9bf79dc474c6b9e2935ad82230b10c1138a71ea41ac0fff1" - ], - "index": "pypi", - "version": "==2.3.1" - }, "pynacl": { "hashes": [ "sha256:05c26f93964373fc0abe332676cb6735f0ecad27711035b9472751faa8521255", @@ -293,11 +221,6 @@ "index": "pypi", "version": "==3.1" }, - "python-etcd3": { - "editable": true, - "git": "https://github.com/kragniz/python-etcd3", - "ref": "cdc4c48bde88a795230a02aa574df84ed9ccfa52" - }, "six": { "hashes": [ "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", @@ -319,32 +242,13 @@ ], "version": "==5.1.1" }, - "typed-ast": { + "ucloud-common": { "hashes": [ - "sha256:18511a0b3e7922276346bcb47e2ef9f38fb90fd31cb9223eed42c85d1312344e", - "sha256:262c247a82d005e43b5b7f69aff746370538e176131c32dda9cb0f324d27141e", - "sha256:2b907eb046d049bcd9892e3076c7a6456c93a25bebfe554e931620c90e6a25b0", - "sha256:354c16e5babd09f5cb0ee000d54cfa38401d8b8891eefa878ac772f827181a3c", - "sha256:4e0b70c6fc4d010f8107726af5fd37921b666f5b31d9331f0bd24ad9a088e631", - "sha256:630968c5cdee51a11c05a30453f8cd65e0cc1d2ad0d9192819df9978984529f4", - "sha256:66480f95b8167c9c5c5c87f32cf437d585937970f3fc24386f313a4c97b44e34", - "sha256:71211d26ffd12d63a83e079ff258ac9d56a1376a25bc80b1cdcdf601b855b90b", - "sha256:95bd11af7eafc16e829af2d3df510cecfd4387f6453355188342c3e79a2ec87a", - "sha256:bc6c7d3fa1325a0c6613512a093bc2a2a15aeec350451cbdf9e1d4bffe3e3233", - "sha256:cc34a6f5b426748a507dd5d1de4c1978f2eb5626d51326e43280941206c209e1", - "sha256:d755f03c1e4a51e9b24d899561fec4ccaf51f210d52abdf8c07ee2849b212a36", - "sha256:d7c45933b1bdfaf9f36c579671fec15d25b06c8398f113dab64c18ed1adda01d", - "sha256:d896919306dd0aa22d0132f62a1b78d11aaf4c9fc5b3410d3c666b818191630a", - "sha256:ffde2fbfad571af120fcbfbbc61c72469e72f550d676c3342492a9dfdefb8f12" + "sha256:bb0ff7526368a80a9ac70cfb4bc71a9fe6df463f329d53baf24c7320e4b2ba00", + "sha256:db76b07c39ed057dc83a91092dddc72f3df8102f4977c430c28ed915b38c9667" ], - "markers": "implementation_name == 'cpython'", - "version": "==1.4.0" - }, - "wrapt": { - "hashes": [ - "sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1" - ], - "version": "==1.11.2" + "index": "pypi", + "version": "==0.5.3" } }, "develop": { @@ -363,13 +267,6 @@ "index": "pypi", "version": "==1.6.2" }, - "ddt": { - "hashes": [ - "sha256:474546b4020ce8a2f9550ba8899c28aa2c284c7bbf175bddede98be949d1ca7c", - "sha256:d13e6af8f36238e89d00f4ebccf2bda4f6d1878be560a6600689e42077e164e3" - ], - "version": "==1.2.1" - }, "dodgy": { "hashes": [ "sha256:65e13cf878d7aff129f1461c13cb5fd1bb6dfe66bb5327e09379c3877763280c" @@ -385,10 +282,10 @@ }, "gitpython": { "hashes": [ - "sha256:259a8b6d6a4a118738c4a65fa990f8c8c91525bb43970aed2868952ebb86ceb8", - "sha256:73aa7b59e58dd3435121421c33c284e5ef51bc7b2f4373e1a1e4cc06e9c928ec" + "sha256:947cc75913e7b6da108458136607e2ee0e40c20be1e12d4284e7c6c12956c276", + "sha256:d2f4945f8260f6981d724f5957bc076398ada55cb5d25aaee10108bcdc894100" ], - "version": "==3.0.1" + "version": "==3.0.2" }, "isort": { "hashes": [ @@ -399,26 +296,26 @@ }, "lazy-object-proxy": { "hashes": [ - "sha256:159a745e61422217881c4de71f9eafd9d703b93af95618635849fe469a283661", - "sha256:23f63c0821cc96a23332e45dfaa83266feff8adc72b9bcaef86c202af765244f", - "sha256:3b11be575475db2e8a6e11215f5aa95b9ec14de658628776e10d96fa0b4dac13", - "sha256:3f447aff8bc61ca8b42b73304f6a44fa0d915487de144652816f950a3f1ab821", - "sha256:4ba73f6089cd9b9478bc0a4fa807b47dbdb8fad1d8f31a0f0a5dbf26a4527a71", - "sha256:4f53eadd9932055eac465bd3ca1bd610e4d7141e1278012bd1f28646aebc1d0e", - "sha256:64483bd7154580158ea90de5b8e5e6fc29a16a9b4db24f10193f0c1ae3f9d1ea", - "sha256:6f72d42b0d04bfee2397aa1862262654b56922c20a9bb66bb76b6f0e5e4f9229", - "sha256:7c7f1ec07b227bdc561299fa2328e85000f90179a2f44ea30579d38e037cb3d4", - "sha256:7c8b1ba1e15c10b13cad4171cfa77f5bb5ec2580abc5a353907780805ebe158e", - "sha256:8559b94b823f85342e10d3d9ca4ba5478168e1ac5658a8a2f18c991ba9c52c20", - "sha256:a262c7dfb046f00e12a2bdd1bafaed2408114a89ac414b0af8755c696eb3fc16", - "sha256:acce4e3267610c4fdb6632b3886fe3f2f7dd641158a843cf6b6a68e4ce81477b", - "sha256:be089bb6b83fac7f29d357b2dc4cf2b8eb8d98fe9d9ff89f9ea6012970a853c7", - "sha256:bfab710d859c779f273cc48fb86af38d6e9210f38287df0069a63e40b45a2f5c", - "sha256:c10d29019927301d524a22ced72706380de7cfc50f767217485a912b4c8bd82a", - "sha256:dd6e2b598849b3d7aee2295ac765a578879830fb8966f70be8cd472e6069932e", - "sha256:e408f1eacc0a68fed0c08da45f31d0ebb38079f043328dce69ff133b95c29dc1" + "sha256:02b260c8deb80db09325b99edf62ae344ce9bc64d68b7a634410b8e9a568edbf", + "sha256:18f9c401083a4ba6e162355873f906315332ea7035803d0fd8166051e3d402e3", + "sha256:1f2c6209a8917c525c1e2b55a716135ca4658a3042b5122d4e3413a4030c26ce", + "sha256:2f06d97f0ca0f414f6b707c974aaf8829c2292c1c497642f63824119d770226f", + "sha256:616c94f8176808f4018b39f9638080ed86f96b55370b5a9463b2ee5c926f6c5f", + "sha256:63b91e30ef47ef68a30f0c3c278fbfe9822319c15f34b7538a829515b84ca2a0", + "sha256:77b454f03860b844f758c5d5c6e5f18d27de899a3db367f4af06bec2e6013a8e", + "sha256:83fe27ba321e4cfac466178606147d3c0aa18e8087507caec78ed5a966a64905", + "sha256:84742532d39f72df959d237912344d8a1764c2d03fe58beba96a87bfa11a76d8", + "sha256:874ebf3caaf55a020aeb08acead813baf5a305927a71ce88c9377970fe7ad3c2", + "sha256:9f5caf2c7436d44f3cec97c2fa7791f8a675170badbfa86e1992ca1b84c37009", + "sha256:a0c8758d01fcdfe7ae8e4b4017b13552efa7f1197dd7358dc9da0576f9d0328a", + "sha256:a4def978d9d28cda2d960c279318d46b327632686d82b4917516c36d4c274512", + "sha256:ad4f4be843dace866af5fc142509e9b9817ca0c59342fdb176ab6ad552c927f5", + "sha256:ae33dd198f772f714420c5ab698ff05ff900150486c648d29951e9c70694338e", + "sha256:b4a2b782b8a8c5522ad35c93e04d60e2ba7f7dcb9271ec8e8c3e08239be6c7b4", + "sha256:c462eb33f6abca3b34cdedbe84d761f31a60b814e173b98ede3c81bb48967c4f", + "sha256:fd135b8d35dfdcdb984828c84d695937e58cc5f49e1c854eb311c4d6aa03f4f1" ], - "version": "==1.4.1" + "version": "==1.4.2" }, "mccabe": { "hashes": [ @@ -429,10 +326,10 @@ }, "pbr": { "hashes": [ - "sha256:56e52299170b9492513c64be44736d27a512fa7e606f21942160b68ce510b4bc", - "sha256:9b321c204a88d8ab5082699469f52cc94c5da45c51f114113d01b3d993c24cdf" + "sha256:2c8e420cd4ed4cec4e7999ee47409e876af575d4c35a45840d59e8b5f3155ab8", + "sha256:b32c8ccaac7b1a20c0ce00ce317642e6cf231cf038f9875e0280e28af5bf7ac9" ], - "version": "==5.4.2" + "version": "==5.4.3" }, "pep8-naming": { "hashes": [ @@ -556,16 +453,16 @@ }, "snowballstemmer": { "hashes": [ - "sha256:9f3b9ffe0809d174f7047e121431acf99c89a7040f0ca84f94ba53a498e6d0c9" + "sha256:713e53b79cbcf97bc5245a06080a33d54a77e7cce2f789c835a143bcdb5c033e" ], - "version": "==1.9.0" + "version": "==1.9.1" }, "stevedore": { "hashes": [ - "sha256:7be098ff53d87f23d798a7ce7ae5c31f094f3deb92ba18059b1aeb1ca9fec0a0", - "sha256:7d1ce610a87d26f53c087da61f06f9b7f7e552efad2a7f6d2322632b5f932ea2" + "sha256:01d9f4beecf0fbd070ddb18e5efb10567801ba7ef3ddab0074f54e3cd4e91730", + "sha256:e0739f9739a681c7a1fda76a102b65295e96a144ccdb552f2ae03c5f0abe8a14" ], - "version": "==1.30.1" + "version": "==1.31.0" }, "typed-ast": { "hashes": [ diff --git a/config.py b/config.py index ce0e204..3a5b583 100755 --- a/config.py +++ b/config.py @@ -18,8 +18,13 @@ logging.basicConfig( etcd_client = Etcd3Wrapper(host=config("ETCD_URL")) -host_pool = HostPool(etcd_client, "/v1/host") -vm_pool = VmPool(etcd_client, "/v1/vm") -request_pool = RequestPool(etcd_client, "/v1/request") +HOST_PREFIX = config("HOST_PREFIX") +VM_PREFIX = config("VM_PREFIX") +REQUEST_PREFIX = config("REQUEST_PREFIX") + + +host_pool = HostPool(etcd_client, HOST_PREFIX) +vm_pool = VmPool(etcd_client, VM_PREFIX) +request_pool = RequestPool(etcd_client, REQUEST_PREFIX) running_vms = [] diff --git a/main.py b/main.py index 87e237b..a4a67ef 100755 --- a/main.py +++ b/main.py @@ -7,7 +7,8 @@ from ucloud_common.host import HostEntry from ucloud_common.request import RequestEntry, RequestType from config import (vm_pool, host_pool, request_pool, - etcd_client, logging, running_vms) + etcd_client, logging, running_vms, + REQUEST_PREFIX) def update_heartbeat(host: HostEntry): @@ -16,7 +17,7 @@ def update_heartbeat(host: HostEntry): host_pool.put(host) time.sleep(10) - logging.info(f"Updated last heartbeat time {host.last_heartbeat}") + logging.info("Updated last heartbeat time %s", host.last_heartbeat) def maintenance(host): @@ -49,10 +50,9 @@ def maintenance(host): # This is to capture poweroff/shutdown of a VM # initiated by user inside VM. OR crash of VM by some # user running process - if (_vm and not _vm.handle.is_running())\ - or not _vm: - vm_entry.add_log(f"{vm_entry.key} is not running but is said to be running." - "So, shutting it down and declare it killed") + if (_vm and not _vm.handle.is_running()) or not _vm: + vm_entry.add_log("""{} is not running but is said to be running. + So, shutting it down and declare it killed""".format(vm_entry.key)) vm_entry.declare_killed() vm_pool.put(vm_entry) if _vm: @@ -69,7 +69,7 @@ def main(): print("No Such Host") exit(1) - logging.info(f"{'*' * 5} Session Started {'*' * 5}") + logging.info("%s Session Started %s", '*' * 5, '*' * 5) # It is seen that under heavy load, timeout event doesn't come # in a predictive manner (which is intentional because we give @@ -88,8 +88,8 @@ def main(): exit(-1) for events_iterator in [ - etcd_client.get_prefix("/v1/request/", value_in_json=True), - etcd_client.watch_prefix("/v1/request/", timeout=10, value_in_json=True), + etcd_client.get_prefix(REQUEST_PREFIX, value_in_json=True), + etcd_client.watch_prefix(REQUEST_PREFIX, timeout=10, value_in_json=True), ]: for request_event in events_iterator: request_event = RequestEntry(request_event) @@ -100,12 +100,15 @@ def main(): continue # If the event is directed toward me OR I am destination of a InitVMMigration - if hasattr(request_event, "hostname") and request_event.hostname == host.key or\ - hasattr(request_event, "destination") and request_event.destination == host.key: + if ((hasattr(request_event, "hostname") and + request_event.hostname == host.key) or + (hasattr(request_event, "destination") and + request_event.destination == host.key)): + request_pool.client.client.delete(request_event.key) vm_entry = vm_pool.get(request_event.uuid) - logging.debug(f"EVENT: {request_event}") + logging.debug("EVENT: %s", request_event) if request_event.type == RequestType.StartVM: virtualmachine.start(vm_entry) @@ -122,7 +125,7 @@ def main(): elif request_event.type == RequestType.TransferVM: virtualmachine.transfer(request_event) - logging.info(f"Running VMs {running_vms}") + logging.info("Running VMs %s", running_vms) main() diff --git a/virtualmachine.py b/virtualmachine.py index 84497bc..634055d 100755 --- a/virtualmachine.py +++ b/virtualmachine.py @@ -7,53 +7,69 @@ import subprocess import traceback import errno -import qmp import tempfile -import bitmath import time import os -from config import (vm_pool, request_pool, etcd_client, - logging, running_vms, WITHOUT_CEPH) -from ucloud_common.vm import VMStatus, VMEntry +from os.path import join from typing import Union from functools import wraps -from dataclasses import dataclass -from ucloud_common.request import RequestEntry, RequestType +import bitmath import sshtunnel from decouple import config -from os.path import join from ucloud_common.helpers import get_ipv4_address +from ucloud_common.vm import VMStatus, VMEntry +from ucloud_common.request import RequestEntry, RequestType + +import qmp + +from config import ( + vm_pool, + request_pool, + etcd_client, + logging, + running_vms, + WITHOUT_CEPH, +) -@dataclass class VM: - key: str - handle: qmp.QEMUMachine - vnc_socket_file: tempfile.NamedTemporaryFile + def __init__(self, key, handle, vnc_socket_file): + self.key = key # type: str + self.handle = handle # type: qmp.QEMUMachine + self.vnc_socket_file = vnc_socket_file # type: tempfile.NamedTemporaryFile def __repr__(self): - return f"VM({self.key})" + return "VM({})".format(self.key) -def get_start_command_args(vm_entry, vnc_sock_filename: str, migration=False, migration_port=4444): +def get_start_command_args( + vm_entry, vnc_sock_filename: str, migration=False, migration_port=4444 +): threads_per_core = 1 vm_memory = int(bitmath.Byte(int(vm_entry.specs["ram"])).to_MB()) vm_cpus = int(vm_entry.specs["cpu"]) vm_uuid = vm_entry.uuid if WITHOUT_CEPH: - command = f"-drive file={os.path.join('/var/vm', vm_uuid)},format=raw,if=virtio,cache=none" + command = "-drive file={},format=raw,if=virtio,cache=none".format( + os.path.join("/var/vm", vm_uuid) + ) else: - command = f"-drive file=rbd:uservms/{vm_uuid},format=raw,if=virtio,cache=none" + command = "-drive file=rbd:uservms/{},format=raw,if=virtio,cache=none".format( + vm_uuid + ) + + command += " -device virtio-rng-pci -vnc unix:{}".format(vnc_sock_filename) + command += " -m {} -smp cores={},threads={}".format( + vm_memory, vm_cpus, threads_per_core + ) + command += " -name {}".format(vm_uuid) - command += (f" -device virtio-rng-pci -vnc unix:{vnc_sock_filename}" - f" -m {vm_memory} -smp cores={vm_cpus},threads={threads_per_core}" - f" -name {vm_uuid}") if migration: - command += f" -incoming tcp:0:{migration_port}" + command += " -incoming tcp:0:{}".format(migration_port) return command.split(" ") @@ -66,12 +82,13 @@ def create_vm_object(vm_entry, migration=False, migration_port=4444): # REQUIREMENT: Use Unix Socket instead of TCP Port for VNC vnc_sock_file = tempfile.NamedTemporaryFile() - qemu_args = get_start_command_args(vm_entry=vm_entry, - vnc_sock_filename=vnc_sock_file.name, - migration=migration, - migration_port=migration_port) - qemu_machine = qmp.QEMUMachine("/usr/bin/qemu-system-x86_64", - args=qemu_args) + qemu_args = get_start_command_args( + vm_entry=vm_entry, + vnc_sock_filename=vnc_sock_file.name, + migration=migration, + migration_port=migration_port, + ) + qemu_machine = qmp.QEMUMachine("/usr/bin/qemu-system-x86_64", args=qemu_args) return VM(vm_entry.key, qemu_machine, vnc_sock_file) @@ -86,17 +103,13 @@ def need_running_vm(func): if vm: try: status = vm.handle.command("query-status") - logging.debug(f"VM Status Check - {status}") + logging.debug("VM Status Check - %s", status) except Exception as exception: - logging.info( - f"{func.__name__} failed - VM {e} {exception} - Unknown Error" - ) + logging.info("%s failed - VM %s %s", func.__name__, e, exception) else: return func(e) else: - logging.info( - f"{func.__name__} failed because VM {e.key} is not running" - ) + logging.info("%s failed because VM %s is not running", func.__name__, e.key) return return wrapper @@ -106,23 +119,39 @@ def create(vm_entry: VMEntry): vm_hdd = int(bitmath.Byte(int(vm_entry.specs["hdd"])).to_MB()) if WITHOUT_CEPH: - _command_to_create = ["cp", - os.path.join("/var/image", vm_entry.image_uuid), - os.path.join("/var/vm", vm_entry.uuid)] + _command_to_create = [ + "cp", + os.path.join("/var/image", vm_entry.image_uuid), + os.path.join("/var/vm", vm_entry.uuid), + ] - _command_to_extend = ["qemu-img", "resize", os.path.join("/var/vm", vm_entry.uuid), vm_entry.specs["hdd"]] + _command_to_extend = [ + "qemu-img", + "resize", + os.path.join("/var/vm", vm_entry.uuid), + vm_entry.specs["hdd"], + ] else: - _command_to_create = ["rbd", "clone", - f"images/{vm_entry.image_uuid}@protected", - f"uservms/{vm_entry.uuid}"] + _command_to_create = [ + "rbd", + "clone", + "images/{}@protected".format(vm_entry.image_uuid), + "uservms/{}".format(vm_entry.uuid), + ] - _command_to_extend = ["rbd", "resize", f"uservms/{vm_entry.uuid}", "--size", vm_hdd] + _command_to_extend = [ + "rbd", + "resize", + "uservms/{}".format(vm_entry.uuid), + "--size", + vm_hdd, + ] try: subprocess.check_output(_command_to_create) except subprocess.CalledProcessError as e: if e.returncode == errno.EEXIST: - logging.debug(f"Image for vm {vm_entry.uuid} exists") + logging.debug("Image for vm %s exists", vm_entry.uuid) # File Already exists. No Problem Continue return @@ -145,19 +174,19 @@ def start(vm_entry: VMEntry): # VM already running. No need to proceed further. if _vm: - logging.info(f"VM {vm_entry.uuid} already running") + logging.info("VM %s already running", vm_entry.uuid) return else: create(vm_entry) - logging.info(f"Starting {vm_entry.key}") + logging.info("Starting %s", vm_entry.key) vm = create_vm_object(vm_entry) try: vm.handle.launch() except (qmp.QEMUMachineError, TypeError, Exception): vm_entry.declare_killed() - vm_entry.add_log(f"Machine Error occurred - {traceback.format_exc()}") + vm_entry.add_log("Machine Error occurred | %s", traceback.format_exc()) vm_pool.put(vm_entry) else: running_vms.append(vm) @@ -178,9 +207,9 @@ def stop(vm_entry): def delete(vm_entry): - logging.info(f"Deleting VM {vm_entry}") + logging.info("Deleting VM | %s", vm_entry) stop(vm_entry) - path_without_protocol = vm_entry.path[vm_entry.path.find(":") + 1:] + path_without_protocol = vm_entry.path[vm_entry.path.find(":") + 1 :] if WITHOUT_CEPH: vm_deletion_command = ["rm", os.path.join("/var/vm", vm_entry.uuid)] @@ -211,14 +240,16 @@ def transfer(request_event): ssh_username=config("ssh_username"), ssh_pkey=config("ssh_pkey"), ssh_private_key_password=config("ssh_private_key_password"), - remote_bind_address=('127.0.0.1', _port), + remote_bind_address=("127.0.0.1", _port), ) try: tunnel.start() except sshtunnel.BaseSSHTunnelForwarderError: - logging.exception(f"Couldn't establish connection to ({_host}, 22)") + logging.exception("Couldn't establish connection to (%s, 22)", _host) else: - vm.handle.command("migrate", uri=f"tcp:{_host}:{tunnel.local_bind_port}") + vm.handle.command( + "migrate", uri="tcp:{}:{}".format(_host, tunnel.local_bind_port) + ) status = vm.handle.command("query-migrate")["status"] while status not in ["failed", "completed"]: @@ -250,10 +281,10 @@ def init_migration(vm_entry, destination_host_key): if _vm: # VM already running. No need to proceed further. - logging.info(f"{_vm.key} Already running") + logging.info("%s Already running", _vm.key) return - logging.info(f"Starting {vm_entry.key}") + logging.info("Starting %s", vm_entry.key) vm = create_vm_object(vm_entry, migration=True, migration_port=4444) @@ -269,13 +300,11 @@ def init_migration(vm_entry, destination_host_key): running_vms.append(vm) - r = RequestEntry.from_scratch(type=RequestType.TransferVM, - hostname=vm_entry.hostname, - parameters={ - "host": get_ipv4_address(), - "port": 4444, - }, - uuid=vm_entry.uuid, - destination_host_key=destination_host_key - ) + r = RequestEntry.from_scratch( + type=RequestType.TransferVM, + hostname=vm_entry.hostname, + parameters={"host": get_ipv4_address(), "port": 4444}, + uuid=vm_entry.uuid, + destination_host_key=destination_host_key, + ) request_pool.put(r)