uncloud
/
ucloud-vm
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Request mechanism rolled out, migration mechanism incorporated, Code style now inclined toward EAFP, DeleteVM now working, resume/suspend functionality removed

This commit is contained in:
ahmadbilalkhalid 2019-08-12 17:49:28 +05:00
parent 70ec3f832b
commit 945b34d76c
4 changed files with 407 additions and 10840 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Pipfile
View File

@ -12,6 +12,7 @@ python-decouple = "*"
cython = "*" cython = "*"
pylint = "*" pylint = "*"
python-etcd3 = {editable = true,git = "https://github.com/kragniz/python-etcd3"} python-etcd3 = {editable = true,git = "https://github.com/kragniz/python-etcd3"}
sshtunnel = "*"
[requires] [requires]
python_version = "3.7" python_version = "3.7"

179
Pipfile.lock generated
View File

@ -1,7 +1,7 @@
{ {
"_meta": { "_meta": {
"hash": { "hash": {
"sha256": "6f0726e9d014ad330ab36d11b44a15c59316234e343697e46ea2a10ab0997a90" "sha256": "8f820ace244a315a831e3330707a6ec24895ed73f8fa1646ce6889915aec0db4"
}, },
"pipfile-spec": 6, "pipfile-spec": 6,
"requires": { "requires": {
@ -16,6 +16,13 @@
] ]
}, },
"default": { "default": {
"asn1crypto": {
"hashes": [
"sha256:2f1adbb7546ed199e3c90ef23ec95c5cf3585bac7d11fb7eb562a3fe89c64e87",
"sha256:9d5c20441baf0cb60a4ac34cc447c6c189024b6b4c6cd7877034f4965c464e49"
],
"version": "==0.24.0"
},
"astroid": { "astroid": {
"hashes": [ "hashes": [
"sha256:6560e1e1749f68c64a4b5dee4e091fce798d2f0d84ebe638cf0e0585a343acf4", "sha256:6560e1e1749f68c64a4b5dee4e091fce798d2f0d84ebe638cf0e0585a343acf4",
@ -23,6 +30,81 @@
], ],
"version": "==2.2.5" "version": "==2.2.5"
}, },
"bcrypt": {
"hashes": [
"sha256:0258f143f3de96b7c14f762c770f5fc56ccd72f8a1857a451c1cd9a655d9ac89",
"sha256:0b0069c752ec14172c5f78208f1863d7ad6755a6fae6fe76ec2c80d13be41e42",
"sha256:19a4b72a6ae5bb467fea018b825f0a7d917789bcfe893e53f15c92805d187294",
"sha256:5432dd7b34107ae8ed6c10a71b4397f1c853bd39a4d6ffa7e35f40584cffd161",
"sha256:69361315039878c0680be456640f8705d76cb4a3a3fe1e057e0f261b74be4b31",
"sha256:6fe49a60b25b584e2f4ef175b29d3a83ba63b3a4df1b4c0605b826668d1b6be5",
"sha256:74a015102e877d0ccd02cdeaa18b32aa7273746914a6c5d0456dd442cb65b99c",
"sha256:763669a367869786bb4c8fcf731f4175775a5b43f070f50f46f0b59da45375d0",
"sha256:8b10acde4e1919d6015e1df86d4c217d3b5b01bb7744c36113ea43d529e1c3de",
"sha256:9fe92406c857409b70a38729dbdf6578caf9228de0aef5bc44f859ffe971a39e",
"sha256:a190f2a5dbbdbff4b74e3103cef44344bc30e61255beb27310e2aec407766052",
"sha256:a595c12c618119255c90deb4b046e1ca3bcfad64667c43d1166f2b04bc72db09",
"sha256:c9457fa5c121e94a58d6505cadca8bed1c64444b83b3204928a866ca2e599105",
"sha256:cb93f6b2ab0f6853550b74e051d297c27a638719753eb9ff66d1e4072be67133",
"sha256:d7bdc26475679dd073ba0ed2766445bb5b20ca4793ca0db32b399dccc6bc84b7",
"sha256:ff032765bb8716d9387fd5376d987a937254b0619eff0972779515b5c98820bc"
],
"version": "==3.1.7"
},
"cffi": {
"hashes": [
"sha256:041c81822e9f84b1d9c401182e174996f0bae9991f33725d059b771744290774",
"sha256:046ef9a22f5d3eed06334d01b1e836977eeef500d9b78e9ef693f9380ad0b83d",
"sha256:066bc4c7895c91812eff46f4b1c285220947d4aa46fa0a2651ff85f2afae9c90",
"sha256:066c7ff148ae33040c01058662d6752fd73fbc8e64787229ea8498c7d7f4041b",
"sha256:2444d0c61f03dcd26dbf7600cf64354376ee579acad77aef459e34efcb438c63",
"sha256:300832850b8f7967e278870c5d51e3819b9aad8f0a2c8dbe39ab11f119237f45",
"sha256:34c77afe85b6b9e967bd8154e3855e847b70ca42043db6ad17f26899a3df1b25",
"sha256:46de5fa00f7ac09f020729148ff632819649b3e05a007d286242c4882f7b1dc3",
"sha256:4aa8ee7ba27c472d429b980c51e714a24f47ca296d53f4d7868075b175866f4b",
"sha256:4d0004eb4351e35ed950c14c11e734182591465a33e960a4ab5e8d4f04d72647",
"sha256:4e3d3f31a1e202b0f5a35ba3bc4eb41e2fc2b11c1eff38b362de710bcffb5016",
"sha256:50bec6d35e6b1aaeb17f7c4e2b9374ebf95a8975d57863546fa83e8d31bdb8c4",
"sha256:55cad9a6df1e2a1d62063f79d0881a414a906a6962bc160ac968cc03ed3efcfb",
"sha256:5662ad4e4e84f1eaa8efce5da695c5d2e229c563f9d5ce5b0113f71321bcf753",
"sha256:59b4dc008f98fc6ee2bb4fd7fc786a8d70000d058c2bbe2698275bc53a8d3fa7",
"sha256:73e1ffefe05e4ccd7bcea61af76f36077b914f92b76f95ccf00b0c1b9186f3f9",
"sha256:a1f0fd46eba2d71ce1589f7e50a9e2ffaeb739fb2c11e8192aa2b45d5f6cc41f",
"sha256:a2e85dc204556657661051ff4bab75a84e968669765c8a2cd425918699c3d0e8",
"sha256:a5457d47dfff24882a21492e5815f891c0ca35fefae8aa742c6c263dac16ef1f",
"sha256:a8dccd61d52a8dae4a825cdbb7735da530179fea472903eb871a5513b5abbfdc",
"sha256:ae61af521ed676cf16ae94f30fe202781a38d7178b6b4ab622e4eec8cefaff42",
"sha256:b012a5edb48288f77a63dba0840c92d0504aa215612da4541b7b42d849bc83a3",
"sha256:d2c5cfa536227f57f97c92ac30c8109688ace8fa4ac086d19d0af47d134e2909",
"sha256:d42b5796e20aacc9d15e66befb7a345454eef794fdb0737d1af593447c6c8f45",
"sha256:dee54f5d30d775f525894d67b1495625dd9322945e7fee00731952e0368ff42d",
"sha256:e070535507bd6aa07124258171be2ee8dfc19119c28ca94c9dfb7efd23564512",
"sha256:e1ff2748c84d97b065cc95429814cdba39bcbd77c9c85c89344b317dc0d9cbff",
"sha256:ed851c75d1e0e043cbf5ca9a8e1b13c4c90f3fbd863dacb01c0808e2b5204201"
],
"version": "==1.12.3"
},
"cryptography": {
"hashes": [
"sha256:24b61e5fcb506424d3ec4e18bca995833839bf13c59fc43e530e488f28d46b8c",
"sha256:25dd1581a183e9e7a806fe0543f485103232f940fcfc301db65e630512cce643",
"sha256:3452bba7c21c69f2df772762be0066c7ed5dc65df494a1d53a58b683a83e1216",
"sha256:41a0be220dd1ed9e998f5891948306eb8c812b512dc398e5a01846d855050799",
"sha256:5751d8a11b956fbfa314f6553d186b94aa70fdb03d8a4d4f1c82dcacf0cbe28a",
"sha256:5f61c7d749048fa6e3322258b4263463bfccefecb0dd731b6561cb617a1d9bb9",
"sha256:72e24c521fa2106f19623a3851e9f89ddfdeb9ac63871c7643790f872a305dfc",
"sha256:7b97ae6ef5cba2e3bb14256625423413d5ce8d1abb91d4f29b6d1a081da765f8",
"sha256:961e886d8a3590fd2c723cf07be14e2a91cf53c25f02435c04d39e90780e3b53",
"sha256:96d8473848e984184b6728e2c9d391482008646276c3ff084a1bd89e15ff53a1",
"sha256:ae536da50c7ad1e002c3eee101871d93abdc90d9c5f651818450a0d3af718609",
"sha256:b0db0cecf396033abb4a93c95d1602f268b3a68bb0a9cc06a7cff587bb9a7292",
"sha256:cfee9164954c186b191b91d4193989ca994703b2fff406f71cf454a2d3c7327e",
"sha256:e6347742ac8f35ded4a46ff835c60e68c22a536a8ae5c4422966d06946b6d4c6",
"sha256:f27d93f0139a3c056172ebb5d4f9056e770fdf0206c2f422ff2ebbad142e09ed",
"sha256:f57b76e46a58b63d1c6375017f4564a28f19a5ca912691fd2e4261b3414b618d"
],
"version": "==2.7"
},
"cython": { "cython": {
"hashes": [ "hashes": [
"sha256:07efba7b32c082c519b75e3b03821c2f32848e2b3e9986c784bbd8ffaf0666d7", "sha256:07efba7b32c082c519b75e3b03821c2f32848e2b3e9986c784bbd8ffaf0666d7",
@ -131,6 +213,13 @@
], ],
"version": "==0.6.1" "version": "==0.6.1"
}, },
"paramiko": {
"hashes": [
"sha256:99f0179bdc176281d21961a003ffdb2ec369daac1a1007241f53374e376576cf",
"sha256:f4b2edfa0d226b70bd4ca31ea7e389325990283da23465d572ed1f70a7583041"
],
"version": "==2.6.0"
},
"protobuf": { "protobuf": {
"hashes": [ "hashes": [
"sha256:05c36022fef3c7d3562ac22402965c0c2b9fe8421f459bb377323598996e407f", "sha256:05c36022fef3c7d3562ac22402965c0c2b9fe8421f459bb377323598996e407f",
@ -154,6 +243,12 @@
], ],
"version": "==3.9.0" "version": "==3.9.0"
}, },
"pycparser": {
"hashes": [
"sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3"
],
"version": "==2.19"
},
"pylint": { "pylint": {
"hashes": [ "hashes": [
"sha256:5d77031694a5fb97ea95e828c8d10fc770a1df6eb3906067aaed42201a8a6a09", "sha256:5d77031694a5fb97ea95e828c8d10fc770a1df6eb3906067aaed42201a8a6a09",
@ -162,6 +257,30 @@
"index": "pypi", "index": "pypi",
"version": "==2.3.1" "version": "==2.3.1"
}, },
"pynacl": {
"hashes": [
"sha256:05c26f93964373fc0abe332676cb6735f0ecad27711035b9472751faa8521255",
"sha256:0c6100edd16fefd1557da078c7a31e7b7d7a52ce39fdca2bec29d4f7b6e7600c",
"sha256:0d0a8171a68edf51add1e73d2159c4bc19fc0718e79dec51166e940856c2f28e",
"sha256:1c780712b206317a746ace34c209b8c29dbfd841dfbc02aa27f2084dd3db77ae",
"sha256:2424c8b9f41aa65bbdbd7a64e73a7450ebb4aa9ddedc6a081e7afcc4c97f7621",
"sha256:2d23c04e8d709444220557ae48ed01f3f1086439f12dbf11976e849a4926db56",
"sha256:30f36a9c70450c7878053fa1344aca0145fd47d845270b43a7ee9192a051bf39",
"sha256:37aa336a317209f1bb099ad177fef0da45be36a2aa664507c5d72015f956c310",
"sha256:4943decfc5b905748f0756fdd99d4f9498d7064815c4cf3643820c9028b711d1",
"sha256:57ef38a65056e7800859e5ba9e6091053cd06e1038983016effaffe0efcd594a",
"sha256:5bd61e9b44c543016ce1f6aef48606280e45f892a928ca7068fba30021e9b786",
"sha256:6482d3017a0c0327a49dddc8bd1074cc730d45db2ccb09c3bac1f8f32d1eb61b",
"sha256:7d3ce02c0784b7cbcc771a2da6ea51f87e8716004512493a2b69016326301c3b",
"sha256:a14e499c0f5955dcc3991f785f3f8e2130ed504fa3a7f44009ff458ad6bdd17f",
"sha256:a39f54ccbcd2757d1d63b0ec00a00980c0b382c62865b61a505163943624ab20",
"sha256:aabb0c5232910a20eec8563503c153a8e78bbf5459490c49ab31f6adf3f3a415",
"sha256:bd4ecb473a96ad0f90c20acba4f0bf0df91a4e03a1f4dd6a4bdc9ca75aa3a715",
"sha256:e2da3c13307eac601f3de04887624939aca8ee3c9488a0bb0eca4fb9401fc6b1",
"sha256:f67814c38162f4deb31f68d590771a29d5ae3b1bd64b75cf232308e5c74777e0"
],
"version": "==1.3.0"
},
"python-decouple": { "python-decouple": {
"hashes": [ "hashes": [
"sha256:1317df14b43efee4337a4aa02914bf004f010cd56d6c4bd894e6474ec8c4fe2d" "sha256:1317df14b43efee4337a4aa02914bf004f010cd56d6c4bd894e6474ec8c4fe2d"
@ -181,6 +300,13 @@
], ],
"version": "==1.12.0" "version": "==1.12.0"
}, },
"sshtunnel": {
"hashes": [
"sha256:c813fdcda8e81c3936ffeac47cb69cfb2d1f5e77ad0de656c6dab56aeebd9249"
],
"index": "pypi",
"version": "==0.1.5"
},
"tenacity": { "tenacity": {
"hashes": [ "hashes": [
"sha256:a0c3c5f7ae0c33f5556c775ca059c12d6fd8ab7121613a713e8b7d649908571b", "sha256:a0c3c5f7ae0c33f5556c775ca059c12d6fd8ab7121613a713e8b7d649908571b",
@ -225,19 +351,6 @@
"index": "pypi", "index": "pypi",
"version": "==1.6.2" "version": "==1.6.2"
}, },
"ddt": {
"hashes": [
"sha256:474546b4020ce8a2f9550ba8899c28aa2c284c7bbf175bddede98be949d1ca7c",
"sha256:d13e6af8f36238e89d00f4ebccf2bda4f6d1878be560a6600689e42077e164e3"
],
"version": "==1.2.1"
},
"gitdb": {
"hashes": [
"sha256:a3ebbc27be035a2e874ed904df516e35f4a29a778a764385de09de9e0f139658"
],
"version": "==0.6.4"
},
"gitdb2": { "gitdb2": {
"hashes": [ "hashes": [
"sha256:83361131a1836661a155172932a13c08bda2db3674e4caa32368aa6eb02f38c2", "sha256:83361131a1836661a155172932a13c08bda2db3674e4caa32368aa6eb02f38c2",
@ -247,10 +360,10 @@
}, },
"gitpython": { "gitpython": {
"hashes": [ "hashes": [
"sha256:7428f1cc5e72d53e65c3259d5cebc22fb2b07f973c49d95b3c3d26c64890a3c3", "sha256:c15c55ff890cd3a6a8330059e80885410a328f645551b55a91d858bfb3eb2573",
"sha256:a0f744a4941eac5d99033eb0adcbec83bf443ee173fda4292d92a906aedce952" "sha256:df752b6b6f06f11213e91c4925aea7eaf9e37e88fb71c8a7a1aa0a5c10852120"
], ],
"version": "==2.1.12" "version": "==2.1.13"
}, },
"mccabe": { "mccabe": {
"hashes": [ "hashes": [
@ -296,19 +409,21 @@
}, },
"pyyaml": { "pyyaml": {
"hashes": [ "hashes": [
"sha256:57acc1d8533cbe51f6662a55434f0dbecfa2b9eaf115bede8f6fd00115a0c0d3", "sha256:0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9",
"sha256:588c94b3d16b76cfed8e0be54932e5729cc185caffaa5a451e7ad2f7ed8b4043", "sha256:01adf0b6c6f61bd11af6e10ca52b7d4057dd0be0343eb9283c878cf3af56aee4",
"sha256:68c8dd247f29f9a0d09375c9c6b8fdc64b60810ebf07ba4cdd64ceee3a58c7b7", "sha256:5124373960b0b3f4aa7df1707e63e9f109b5263eca5976c66e08b1c552d4eaf8",
"sha256:70d9818f1c9cd5c48bb87804f2efc8692f1023dac7f1a1a5c61d454043c1d265", "sha256:5ca4f10adbddae56d824b2c09668e91219bb178a1eee1faa56af6f99f11bf696",
"sha256:86a93cccd50f8c125286e637328ff4eef108400dd7089b46a7be3445eecfa391", "sha256:7907be34ffa3c5a32b60b95f4d95ea25361c951383a894fec31be7252b2b6f34",
"sha256:a0f329125a926876f647c9fa0ef32801587a12328b4a3c741270464e3e4fa778", "sha256:7ec9b2a4ed5cad025c2278a1e6a19c011c80a3caaac804fd2d329e9cc2c287c9",
"sha256:a3c252ab0fa1bb0d5a3f6449a4826732f3eb6c0270925548cac342bc9b22c225", "sha256:87ae4c829bb25b9fe99cf71fbb2140c448f534e24c998cc60f39ae4f94396a73",
"sha256:b4bb4d3f5e232425e25dda21c070ce05168a786ac9eda43768ab7f3ac2770955", "sha256:9de9919becc9cc2ff03637872a440195ac4241c80536632fffeb6a1e25a74299",
"sha256:cd0618c5ba5bda5f4039b9398bb7fb6a317bb8298218c3de25c47c4740e4b95e", "sha256:a5a85b10e450c66b49f98846937e8cfca1db3127a9d5d1e31ca45c3d0bef4c5b",
"sha256:ceacb9e5f8474dcf45b940578591c7f3d960e82f926c707788a570b51ba59190", "sha256:b0997827b4f6a7c286c01c5f60384d218dca4ed7d9efa945c3e1aa623d5709ae",
"sha256:fe6a88094b64132c4bb3b631412e90032e8cfe9745a58370462240b8cb7553cd" "sha256:b631ef96d3222e62861443cc89d6563ba3eeb816eeb96b2629345ab795e53681",
"sha256:bf47c0607522fdbca6c9e817a6e81b08491de50f3766a7a0e6a5be7905961b41",
"sha256:f81025eddd0327c7d4cfe9b62cf33190e1e736cc6e97502b3ec425f574b3e7a8"
], ],
"version": "==5.1.1" "version": "==5.1.2"
}, },
"six": { "six": {
"hashes": [ "hashes": [
@ -317,12 +432,6 @@
], ],
"version": "==1.12.0" "version": "==1.12.0"
}, },
"smmap": {
"hashes": [
"sha256:0e2b62b497bd5f0afebc002eda4d90df9d209c30ef257e8673c90a6b5c119d62"
],
"version": "==0.9.0"
},
"smmap2": { "smmap2": {
"hashes": [ "hashes": [
"sha256:0555a7bf4df71d1ef4218e4807bbf9b201f910174e6e08af2e138d4e517b4dde", "sha256:0555a7bf4df71d1ef4218e4807bbf9b201f910174e6e08af2e138d4e517b4dde",

10644
log.txt
View File

File diff suppressed because it is too large Load Diff

423
main.py
View File

@ -12,21 +12,29 @@ import logging
import subprocess import subprocess
import threading import threading
import time import time
import traceback
import sshtunnel
import errno
from etcd3_wrapper import Etcd3Wrapper
from dataclasses import dataclass from dataclasses import dataclass
from typing import Union from typing import Union
from functools import wraps from functools import wraps
from string import Template
from os.path import join
from ucloud_common.rbd import RBD from etcd3_wrapper import Etcd3Wrapper
from ucloud_common.vm import VmPool, VMEntry, VMStatus from ucloud_common.vm import VmPool, VMStatus, VMEntry
from ucloud_common.host import HostPool, HostEntry from ucloud_common.host import HostPool, HostEntry
from ucloud_common.request import RequestEntry, RequestPool, RequestType
from ucloud_common.helpers import get_ipv4_address
running_vms = [] running_vms = []
vnc_port_pool = list(range(0, 100)) vnc_port_pool = list(range(0, 100))
client = Etcd3Wrapper() client = Etcd3Wrapper()
VM_POOL = None vm_pool = None
HOST_POOL = None host_pool = None
request_pool = None
logging.basicConfig( logging.basicConfig(
level=logging.DEBUG, level=logging.DEBUG,
@ -37,16 +45,37 @@ logging.basicConfig(
) )
def get_start_command_args(image_path, vnc_port, migration=False, migration_port=4444):
_args = ("-drive file=$image_path,format=raw,if=virtio,cache=none"
" -m 1024 -device virtio-rng-pci -enable-kvm -vnc :$vnc_port")
if migration:
_args = _args + " -incoming tcp:0:$migration_port"
args_template = Template(_args)
if migration:
args = args_template.substitute(image_path=image_path, vnc_port=vnc_port,
migration_port=migration_port)
else:
args = args_template.substitute(image_path=image_path, vnc_port=vnc_port)
return args.split(" ")
@dataclass @dataclass
class VM: class VM:
key: str key: str
vm: qmp.QEMUMachine handle: qmp.QEMUMachine
def __repr__(self):
return f"VM({self.key})"
def update_heartbeat(host: HostEntry): def update_heartbeat(host: HostEntry):
while True: while True:
host.update_heartbeat() host.update_heartbeat()
HOST_POOL.put(host) host_pool.put(host)
time.sleep(10) time.sleep(10)
logging.info(f"Updated last heartbeat time {host.last_heartbeat}") logging.info(f"Updated last heartbeat time {host.last_heartbeat}")
@ -58,7 +87,7 @@ def need_running_vm(func):
vm = get_vm(running_vms, e.key) vm = get_vm(running_vms, e.key)
if vm: if vm:
try: try:
status = vm.vm.command("query-status") status = vm.handle.command("query-status")
logging.debug(f"VM Status Check - {status}") logging.debug(f"VM Status Check - {status}")
except OSError: except OSError:
logging.info( logging.info(
@ -75,119 +104,79 @@ def need_running_vm(func):
return wrapper return wrapper
def create_vm(vm): def create_vm(vm_entry: VMEntry):
image = client.get(f"/v1/image/{vm.value['image_uuid']}", value_in_json=True) _command_to_create = f"rbd clone images/{vm_entry.image_uuid}@protected uservms/{vm_entry.uuid}"
if image:
logging.debug(image)
logging.info("Creating New VM...")
_command_to_create = f"rbd clone images/{vm.image_uuid}@protected uservms/{vm.uuid}"
try:
subprocess.call(_command_to_create.split(" "))
vm.status = VMStatus.requested_start
VM_POOL.put(vm)
except:
logging.exception("Can't clone image")
else:
logging.info(f"Image not found for {vm.image_uuid}")
def start_vm(e):
vm_path = f"rbd:uservms/{e.uuid}"
try: try:
user_vms = RBD.ls("uservms") subprocess.check_call(_command_to_create.split(" "))
except: except subprocess.CalledProcessError as e:
logging.info("Can't access uservms pool") if e.returncode == errno.EEXIST:
return logging.debug(f"Image for vm {vm_entry.uuid} exists")
# File Already exists. No Problem Continue
if e.uuid not in user_vms:
logging.info(f"Image file of vm {e.key} does not exists")
logging.info(f"Deleting vm {e.key}")
client.client.delete(e.key)
return
_vm = get_vm(running_vms, e.key)
if _vm:
logging.info(f"{e.key} already running")
e.status = VMStatus.running
VM_POOL.put(e)
return
# FIXME: There should be better vnc port allocation scheme
vm = qmp.QEMUMachine(
"/usr/bin/qemu-system-x86_64",
test_dir="vm_socklog",
args=[
vm_path,
"-boot",
"c", # First Boot Hard drive
"-m",
"1024", # RAM limit
# Ever growing port number
"-vnc",
f":{vnc_port_pool.pop(0)}", # Allow VNC
],
)
try:
logging.info(f"Starting {e.key}")
vm.launch()
if vm.is_running():
running_vms.append(VM(e.key, vm))
e.status = VMStatus.running
VM_POOL.put(e)
else:
e.declare_killed()
VM_POOL.put(e)
return return
except (qmp.QEMUMachineError, TypeError): else:
logging.exception(f"Machine Error Occurred on {e.key}") # This exception catches all other exceptions
e.declare_killed() # i.e FileNotFound (BaseImage), pool Does Not Exists etc.
VM_POOL.put(e) logging.exception(f"Can't clone image - {e}")
else: else:
logging.info(f"Started Successfully {e.key}") logging.info("New VM Created")
def start_vm(vm_entry: VMEntry):
_vm = get_vm(running_vms, vm_entry.key)
# VM already running. No need to proceed further.
if _vm:
logging.info(f"VM {vm_entry.uuid} already running")
return
else:
create_vm(vm_entry)
logging.info(f"Starting {vm_entry.key}")
# FIXME: There should be better vnc port allocation scheme
vm = qmp.QEMUMachine(
"/usr/bin/qemu-system-x86_64",
args=get_start_command_args(vm_entry.path, vnc_port_pool.pop(0)),
)
try:
vm.launch()
except (qmp.QEMUMachineError, TypeError, Exception):
vm_entry.declare_killed()
vm_entry.add_log(f"Machine Error occurred - {traceback.format_exc()}")
vm_pool.put(vm_entry)
else:
running_vms.append(VM(vm_entry.key, vm))
vm_entry.status = VMStatus.running
vm_entry.add_log("Started successfully")
vm_pool.put(vm_entry)
@need_running_vm @need_running_vm
def suspend_vm(e): def stop_vm(vm_entry):
vm = get_vm(running_vms, e.key) vm = get_vm(running_vms, vm_entry.key)
vm.vm.command("stop") vm.handle.shutdown()
if vm.vm.command("query-status")["status"] == "paused": if not vm.handle.is_running():
e.status = VMStatus.suspended vm_entry.add_log("Shutdown successfully")
VM_POOL.put(e) vm_entry.declare_stopped()
logging.info(f"Successfully suspended VM {e.key}") vm_pool.put(vm_entry)
else:
logging.info(f"Suspending VM {e.key} failed")
@need_running_vm
def resume_vm(e):
vm = get_vm(running_vms, e.key)
vm.vm.command("cont")
if vm.vm.command("query-status")["status"] == "running":
e.status = VMStatus.running
VM_POOL.put(e)
logging.info(f"Successfully resumed VM {e.key}")
else:
logging.info(f"Resuming VM {e.key} failed")
@need_running_vm
def shutdown_vm(e):
vm = get_vm(running_vms, e.key)
vm.vm.shutdown()
if not vm.vm.is_running():
logging.info(f"VM {e.key} shutdown successfully")
e.status = VMStatus.stopped
VM_POOL.put(e)
running_vms.remove(vm) running_vms.remove(vm)
def delete_vm(e): def delete_vm(vm_entry):
# TODO: Delete VM Image From CEPH logging.info(f"Deleting VM {vm_entry}")
logging.info(f"Deleting VM {e.key}") stop_vm(vm_entry)
shutdown_vm(e) path_without_protocol = vm_entry.path[vm_entry.path.find(":") + 1:]
client.client.delete(e.key) try:
rc = subprocess.call(f"rbd rm {path_without_protocol}".split(" "))
except FileNotFoundError as e:
logging.exception(e)
except Exception as e:
logging.exception(f"Unknown error occurred - {e}")
else:
if rc == 0:
client.client.delete(vm_entry.key)
else:
logging.info("Some unknown problem occur while deleting vm file")
def get_vm(vm_list: list, vm_key) -> Union[VM, None]: def get_vm(vm_list: list, vm_key) -> Union[VM, None]:
@ -195,87 +184,199 @@ def get_vm(vm_list: list, vm_key) -> Union[VM, None]:
def maintenance(host): def maintenance(host):
_vms = VM_POOL.by_host(host.key) # To capture vm running according to running_vms list
alleged_running_vms = VM_POOL.by_status("RUNNING", _vms)
for vm in alleged_running_vms: # This is to capture successful migration of a VM.
_vm = get_vm(running_vms, vm.key) # Suppose, this host is running "vm1" and user initiated
if (_vm and not _vm.vm.is_running()) or _vm is None: # request to migrate this "vm1" to some other host. On,
logging.debug(f"{_vm} {vm.key}") # successful migration the destination host would set
logging.info( # the vm hostname to itself. Thus, we are checking
f"{vm.key} is not running but is said to be running" # whether this host vm is successfully migrated. If yes
) # then we shutdown "vm1" on this host.
logging.info(f"Updating {vm.key} status to KILLED")
vm.declare_killed() for vm in running_vms:
VM_POOL.put(vm) with vm_pool.get_put(vm.key) as vm_entry:
running_vms.remove(vm) if vm_entry.hostname != host.key and not vm_entry.in_migration:
vm.handle.shutdown()
vm_entry.add_log("VM on source host shutdown.")
# To check vm running according to etcd entries
alleged_running_vms = vm_pool.by_status("RUNNING", vm_pool.by_host(host.key))
for vm_entry in alleged_running_vms:
_vm = get_vm(running_vms, vm_entry.key)
# Whether, the allegedly running vm is in our
# running_vms list or not if it is said to be
# running on this host but it is not then we
# need to shut it down
# This is to capture poweroff/shutdown of a VM
# initiated by user inside VM. OR crash of VM by some
# user running process
if (_vm and not _vm.handle.is_running())\
or not _vm:
vm_entry.add_log(f"{vm_entry.key} is not running but is said to be running."
"So, shutting it down and declare it killed")
vm_entry.declare_killed()
vm_pool.put(vm_entry)
if _vm:
running_vms.remove(_vm)
def transfer_vm(request_event):
# This function would run on source host i.e host on which the vm
# is running initially. This host would be responsible for transferring
# vm state to destination host.
_host, _port = request_event.parameters["host"], request_event.parameters["port"]
_uuid = request_event.uuid
_destination = request_event.destination_host_key
vm = get_vm(running_vms, join("/v1/vm", _uuid))
if vm:
tunnel = sshtunnel.SSHTunnelForwarder(
(_host, 22),
ssh_username="meow",
ssh_pkey="~/.ssh/id_rsa",
ssh_private_key_password="***REMOVED***",
remote_bind_address=('127.0.0.1', _port),
)
try:
tunnel.start()
except sshtunnel.BaseSSHTunnelForwarderError:
logging.exception(f"Couldn't establish connection to ({_host}, 22)")
else:
vm.handle.command("migrate", uri=f"tcp:{_host}:{tunnel.local_bind_port}")
status = vm.handle.command("query-migrate")["status"]
while status not in ["failed", "completed"]:
time.sleep(2)
status = vm.handle.command("query-migrate")["status"]
with vm_pool.get_put(request_event.uuid) as source_vm:
if status == "failed":
source_vm.add_log("Migration Failed")
elif status == "completed":
# If VM is successfully migrated then shutdown the VM
# on this host and update hostname to destination host key
source_vm.add_log("Successfully migrated")
source_vm.hostname = _destination
running_vms.remove(vm)
vm.handle.shutdown()
source_vm.in_migration = False # VM transfer finished
finally:
tunnel.close()
def init_vm_migration(vm_entry, destination_host_key):
# This function would run on destination host i.e host on which the vm
# would be transferred after migration. This host would be responsible
# for starting VM that would receive state of VM running on source host.
_vm = get_vm(running_vms, vm_entry.key)
if _vm:
# VM already running. No need to proceed further.
logging.log(f"{_vm.key} Already running")
return
logging.info(f"Starting {vm_entry.key}")
# FIXME: There should be better vnc port allocation scheme
actual_vm = qmp.QEMUMachine(
"/usr/bin/qemu-system-x86_64",
args=get_start_command_args(vm_entry.path, 100, migration=True, migration_port=4444),
)
try:
actual_vm.launch()
except Exception as e:
# We don't care whether MachineError or any other error occurred
logging.exception(e)
actual_vm.shutdown()
else:
vm_entry.in_migration = True
vm_pool.put(vm_entry)
running_vms.append(VM(vm_entry.key, actual_vm))
r = RequestEntry.from_scratch(type=RequestType.TransferVM,
hostname=vm_entry.hostname,
parameters={
"host": get_ipv4_address(),
"port": 4444,
},
uuid=vm_entry.uuid,
destination_host_key=destination_host_key
)
request_pool.put(r)
def main(): def main():
argparser = argparse.ArgumentParser() argparser = argparse.ArgumentParser()
argparser.add_argument( argparser.add_argument("hostname", help="Name of this host. e.g /v1/host/1")
"hostname", help="Name of this host. e.g /v1/host/1"
)
args = argparser.parse_args() args = argparser.parse_args()
global HOST_POOL, VM_POOL global host_pool, vm_pool, request_pool
HOST_POOL = HostPool(client, "/v1/host") host_pool = HostPool(client, "/v1/host")
VM_POOL = VmPool(client, "/v1/vm") vm_pool = VmPool(client, "/v1/vm")
request_pool = RequestPool(client, "/v1/request")
host = HOST_POOL.get(args.hostname) host = host_pool.get(args.hostname)
if not host: if not host:
print("No Such Host") print("No Such Host")
exit(1) exit(1)
logging.info(f"{'*' * 5} Session Started {'*' * 5}")
# It is seen that under heavy load, timeout event doesn't come # It is seen that under heavy load, timeout event doesn't come
# in a predictive manner which delays heart beat update which # in a predictive manner which delays heart beat update which
# in turn misunderstood by scheduler that the host is dead # in turn misunderstood by scheduler that the host is dead
# when it is actually alive. So, to ensure that we update the # when it is actually alive. So, to ensure that we update the
# heart beat in a predictive manner we start Heart beat updating # heart beat in a predictive manner we start Heart beat updating
# mechanism in separated thread # mechanism in separated thread
heartbeat_updating_thread = threading.Thread(target=update_heartbeat, args=(host,)) heartbeat_updating_thread = threading.Thread(target=update_heartbeat, args=(host,))
heartbeat_updating_thread.start() try:
heartbeat_updating_thread.start()
except Exception as e:
print("No Need To Go Further. Our heartbeat updating mechanism is not working")
logging.exception(e)
exit(-1)
for events_iterator in [ for events_iterator in [
client.get_prefix("/v1/vm/", value_in_json=True), client.get_prefix("/v1/request/", value_in_json=True),
client.watch_prefix("/v1/vm/", timeout=10, value_in_json=True), client.watch_prefix("/v1/request/", timeout=10, value_in_json=True),
]: ]:
for e in events_iterator: for request_event in events_iterator:
e = VMEntry(e) request_event = RequestEntry(request_event)
if e.status == "TIMEOUT": if request_event.type == "TIMEOUT":
logging.info("Timeout") logging.info("Timeout Event")
maintenance(host) maintenance(host)
continue continue
# TODO: Re-evaluate Migration Design # If the event is directed toward me OR I am destination of a InitVMMigration
if hasattr(e, "migration_destination"): if hasattr(request_event, "hostname") and request_event.hostname == host.key or\
e_migration_destination = e.value["migration_destination"] hasattr(request_event, "destination") and request_event.destination == host.key:
else: request_pool.client.client.delete(request_event.key)
e_migration_destination = "" vm_entry = vm_pool.get(request_event.uuid)
# If the event is directed toward me or logging.debug(f"EVENT: {request_event}")
# I am destination of a REQUESTED_MIGRATION
if e.hostname == host.key or e_migration_destination == host.key:
logging.debug(f"EVENT: {e}")
if e.status == "SCHEDULED_DEPLOY": if request_event.type == RequestType.StartVM:
create_vm(e) start_vm(vm_entry)
elif e.status == "REQUESTED_SUSPEND": elif request_event.type == RequestType.StopVM:
suspend_vm(e) stop_vm(vm_entry)
elif e.status == "REQUESTED_RESUME": elif request_event.type == RequestType.DeleteVM:
resume_vm(e) delete_vm(vm_entry)
elif e.status == "REQUESTED_START": elif request_event.type == RequestType.InitVMMigration:
start_vm(e) init_vm_migration(vm_entry, host.key)
elif e.status == "REQUESTED_SHUTDOWN": elif request_event.type == RequestType.TransferVM:
shutdown_vm(e) transfer_vm(request_event)
elif e.status == "DELETED":
delete_vm(e)
logging.info(f"Running VMs {running_vms}") logging.info(f"Running VMs {running_vms}")