Request mechanism rolled out, migration mechanism incorporated, Code style now inclined toward EAFP, DeleteVM now working, resume/suspend functionality removed
This commit is contained in:
parent
70ec3f832b
commit
945b34d76c
4 changed files with 407 additions and 10840 deletions
1
Pipfile
1
Pipfile
|
|
@ -12,6 +12,7 @@ python-decouple = "*"
|
|||
cython = "*"
|
||||
pylint = "*"
|
||||
python-etcd3 = {editable = true,git = "https://github.com/kragniz/python-etcd3"}
|
||||
sshtunnel = "*"
|
||||
|
||||
[requires]
|
||||
python_version = "3.7"
|
||||
|
|
|
|||
179
Pipfile.lock
generated
179
Pipfile.lock
generated
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"_meta": {
|
||||
"hash": {
|
||||
"sha256": "6f0726e9d014ad330ab36d11b44a15c59316234e343697e46ea2a10ab0997a90"
|
||||
"sha256": "8f820ace244a315a831e3330707a6ec24895ed73f8fa1646ce6889915aec0db4"
|
||||
},
|
||||
"pipfile-spec": 6,
|
||||
"requires": {
|
||||
|
|
@ -16,6 +16,13 @@
|
|||
]
|
||||
},
|
||||
"default": {
|
||||
"asn1crypto": {
|
||||
"hashes": [
|
||||
"sha256:2f1adbb7546ed199e3c90ef23ec95c5cf3585bac7d11fb7eb562a3fe89c64e87",
|
||||
"sha256:9d5c20441baf0cb60a4ac34cc447c6c189024b6b4c6cd7877034f4965c464e49"
|
||||
],
|
||||
"version": "==0.24.0"
|
||||
},
|
||||
"astroid": {
|
||||
"hashes": [
|
||||
"sha256:6560e1e1749f68c64a4b5dee4e091fce798d2f0d84ebe638cf0e0585a343acf4",
|
||||
|
|
@ -23,6 +30,81 @@
|
|||
],
|
||||
"version": "==2.2.5"
|
||||
},
|
||||
"bcrypt": {
|
||||
"hashes": [
|
||||
"sha256:0258f143f3de96b7c14f762c770f5fc56ccd72f8a1857a451c1cd9a655d9ac89",
|
||||
"sha256:0b0069c752ec14172c5f78208f1863d7ad6755a6fae6fe76ec2c80d13be41e42",
|
||||
"sha256:19a4b72a6ae5bb467fea018b825f0a7d917789bcfe893e53f15c92805d187294",
|
||||
"sha256:5432dd7b34107ae8ed6c10a71b4397f1c853bd39a4d6ffa7e35f40584cffd161",
|
||||
"sha256:69361315039878c0680be456640f8705d76cb4a3a3fe1e057e0f261b74be4b31",
|
||||
"sha256:6fe49a60b25b584e2f4ef175b29d3a83ba63b3a4df1b4c0605b826668d1b6be5",
|
||||
"sha256:74a015102e877d0ccd02cdeaa18b32aa7273746914a6c5d0456dd442cb65b99c",
|
||||
"sha256:763669a367869786bb4c8fcf731f4175775a5b43f070f50f46f0b59da45375d0",
|
||||
"sha256:8b10acde4e1919d6015e1df86d4c217d3b5b01bb7744c36113ea43d529e1c3de",
|
||||
"sha256:9fe92406c857409b70a38729dbdf6578caf9228de0aef5bc44f859ffe971a39e",
|
||||
"sha256:a190f2a5dbbdbff4b74e3103cef44344bc30e61255beb27310e2aec407766052",
|
||||
"sha256:a595c12c618119255c90deb4b046e1ca3bcfad64667c43d1166f2b04bc72db09",
|
||||
"sha256:c9457fa5c121e94a58d6505cadca8bed1c64444b83b3204928a866ca2e599105",
|
||||
"sha256:cb93f6b2ab0f6853550b74e051d297c27a638719753eb9ff66d1e4072be67133",
|
||||
"sha256:d7bdc26475679dd073ba0ed2766445bb5b20ca4793ca0db32b399dccc6bc84b7",
|
||||
"sha256:ff032765bb8716d9387fd5376d987a937254b0619eff0972779515b5c98820bc"
|
||||
],
|
||||
"version": "==3.1.7"
|
||||
},
|
||||
"cffi": {
|
||||
"hashes": [
|
||||
"sha256:041c81822e9f84b1d9c401182e174996f0bae9991f33725d059b771744290774",
|
||||
"sha256:046ef9a22f5d3eed06334d01b1e836977eeef500d9b78e9ef693f9380ad0b83d",
|
||||
"sha256:066bc4c7895c91812eff46f4b1c285220947d4aa46fa0a2651ff85f2afae9c90",
|
||||
"sha256:066c7ff148ae33040c01058662d6752fd73fbc8e64787229ea8498c7d7f4041b",
|
||||
"sha256:2444d0c61f03dcd26dbf7600cf64354376ee579acad77aef459e34efcb438c63",
|
||||
"sha256:300832850b8f7967e278870c5d51e3819b9aad8f0a2c8dbe39ab11f119237f45",
|
||||
"sha256:34c77afe85b6b9e967bd8154e3855e847b70ca42043db6ad17f26899a3df1b25",
|
||||
"sha256:46de5fa00f7ac09f020729148ff632819649b3e05a007d286242c4882f7b1dc3",
|
||||
"sha256:4aa8ee7ba27c472d429b980c51e714a24f47ca296d53f4d7868075b175866f4b",
|
||||
"sha256:4d0004eb4351e35ed950c14c11e734182591465a33e960a4ab5e8d4f04d72647",
|
||||
"sha256:4e3d3f31a1e202b0f5a35ba3bc4eb41e2fc2b11c1eff38b362de710bcffb5016",
|
||||
"sha256:50bec6d35e6b1aaeb17f7c4e2b9374ebf95a8975d57863546fa83e8d31bdb8c4",
|
||||
"sha256:55cad9a6df1e2a1d62063f79d0881a414a906a6962bc160ac968cc03ed3efcfb",
|
||||
"sha256:5662ad4e4e84f1eaa8efce5da695c5d2e229c563f9d5ce5b0113f71321bcf753",
|
||||
"sha256:59b4dc008f98fc6ee2bb4fd7fc786a8d70000d058c2bbe2698275bc53a8d3fa7",
|
||||
"sha256:73e1ffefe05e4ccd7bcea61af76f36077b914f92b76f95ccf00b0c1b9186f3f9",
|
||||
"sha256:a1f0fd46eba2d71ce1589f7e50a9e2ffaeb739fb2c11e8192aa2b45d5f6cc41f",
|
||||
"sha256:a2e85dc204556657661051ff4bab75a84e968669765c8a2cd425918699c3d0e8",
|
||||
"sha256:a5457d47dfff24882a21492e5815f891c0ca35fefae8aa742c6c263dac16ef1f",
|
||||
"sha256:a8dccd61d52a8dae4a825cdbb7735da530179fea472903eb871a5513b5abbfdc",
|
||||
"sha256:ae61af521ed676cf16ae94f30fe202781a38d7178b6b4ab622e4eec8cefaff42",
|
||||
"sha256:b012a5edb48288f77a63dba0840c92d0504aa215612da4541b7b42d849bc83a3",
|
||||
"sha256:d2c5cfa536227f57f97c92ac30c8109688ace8fa4ac086d19d0af47d134e2909",
|
||||
"sha256:d42b5796e20aacc9d15e66befb7a345454eef794fdb0737d1af593447c6c8f45",
|
||||
"sha256:dee54f5d30d775f525894d67b1495625dd9322945e7fee00731952e0368ff42d",
|
||||
"sha256:e070535507bd6aa07124258171be2ee8dfc19119c28ca94c9dfb7efd23564512",
|
||||
"sha256:e1ff2748c84d97b065cc95429814cdba39bcbd77c9c85c89344b317dc0d9cbff",
|
||||
"sha256:ed851c75d1e0e043cbf5ca9a8e1b13c4c90f3fbd863dacb01c0808e2b5204201"
|
||||
],
|
||||
"version": "==1.12.3"
|
||||
},
|
||||
"cryptography": {
|
||||
"hashes": [
|
||||
"sha256:24b61e5fcb506424d3ec4e18bca995833839bf13c59fc43e530e488f28d46b8c",
|
||||
"sha256:25dd1581a183e9e7a806fe0543f485103232f940fcfc301db65e630512cce643",
|
||||
"sha256:3452bba7c21c69f2df772762be0066c7ed5dc65df494a1d53a58b683a83e1216",
|
||||
"sha256:41a0be220dd1ed9e998f5891948306eb8c812b512dc398e5a01846d855050799",
|
||||
"sha256:5751d8a11b956fbfa314f6553d186b94aa70fdb03d8a4d4f1c82dcacf0cbe28a",
|
||||
"sha256:5f61c7d749048fa6e3322258b4263463bfccefecb0dd731b6561cb617a1d9bb9",
|
||||
"sha256:72e24c521fa2106f19623a3851e9f89ddfdeb9ac63871c7643790f872a305dfc",
|
||||
"sha256:7b97ae6ef5cba2e3bb14256625423413d5ce8d1abb91d4f29b6d1a081da765f8",
|
||||
"sha256:961e886d8a3590fd2c723cf07be14e2a91cf53c25f02435c04d39e90780e3b53",
|
||||
"sha256:96d8473848e984184b6728e2c9d391482008646276c3ff084a1bd89e15ff53a1",
|
||||
"sha256:ae536da50c7ad1e002c3eee101871d93abdc90d9c5f651818450a0d3af718609",
|
||||
"sha256:b0db0cecf396033abb4a93c95d1602f268b3a68bb0a9cc06a7cff587bb9a7292",
|
||||
"sha256:cfee9164954c186b191b91d4193989ca994703b2fff406f71cf454a2d3c7327e",
|
||||
"sha256:e6347742ac8f35ded4a46ff835c60e68c22a536a8ae5c4422966d06946b6d4c6",
|
||||
"sha256:f27d93f0139a3c056172ebb5d4f9056e770fdf0206c2f422ff2ebbad142e09ed",
|
||||
"sha256:f57b76e46a58b63d1c6375017f4564a28f19a5ca912691fd2e4261b3414b618d"
|
||||
],
|
||||
"version": "==2.7"
|
||||
},
|
||||
"cython": {
|
||||
"hashes": [
|
||||
"sha256:07efba7b32c082c519b75e3b03821c2f32848e2b3e9986c784bbd8ffaf0666d7",
|
||||
|
|
@ -131,6 +213,13 @@
|
|||
],
|
||||
"version": "==0.6.1"
|
||||
},
|
||||
"paramiko": {
|
||||
"hashes": [
|
||||
"sha256:99f0179bdc176281d21961a003ffdb2ec369daac1a1007241f53374e376576cf",
|
||||
"sha256:f4b2edfa0d226b70bd4ca31ea7e389325990283da23465d572ed1f70a7583041"
|
||||
],
|
||||
"version": "==2.6.0"
|
||||
},
|
||||
"protobuf": {
|
||||
"hashes": [
|
||||
"sha256:05c36022fef3c7d3562ac22402965c0c2b9fe8421f459bb377323598996e407f",
|
||||
|
|
@ -154,6 +243,12 @@
|
|||
],
|
||||
"version": "==3.9.0"
|
||||
},
|
||||
"pycparser": {
|
||||
"hashes": [
|
||||
"sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3"
|
||||
],
|
||||
"version": "==2.19"
|
||||
},
|
||||
"pylint": {
|
||||
"hashes": [
|
||||
"sha256:5d77031694a5fb97ea95e828c8d10fc770a1df6eb3906067aaed42201a8a6a09",
|
||||
|
|
@ -162,6 +257,30 @@
|
|||
"index": "pypi",
|
||||
"version": "==2.3.1"
|
||||
},
|
||||
"pynacl": {
|
||||
"hashes": [
|
||||
"sha256:05c26f93964373fc0abe332676cb6735f0ecad27711035b9472751faa8521255",
|
||||
"sha256:0c6100edd16fefd1557da078c7a31e7b7d7a52ce39fdca2bec29d4f7b6e7600c",
|
||||
"sha256:0d0a8171a68edf51add1e73d2159c4bc19fc0718e79dec51166e940856c2f28e",
|
||||
"sha256:1c780712b206317a746ace34c209b8c29dbfd841dfbc02aa27f2084dd3db77ae",
|
||||
"sha256:2424c8b9f41aa65bbdbd7a64e73a7450ebb4aa9ddedc6a081e7afcc4c97f7621",
|
||||
"sha256:2d23c04e8d709444220557ae48ed01f3f1086439f12dbf11976e849a4926db56",
|
||||
"sha256:30f36a9c70450c7878053fa1344aca0145fd47d845270b43a7ee9192a051bf39",
|
||||
"sha256:37aa336a317209f1bb099ad177fef0da45be36a2aa664507c5d72015f956c310",
|
||||
"sha256:4943decfc5b905748f0756fdd99d4f9498d7064815c4cf3643820c9028b711d1",
|
||||
"sha256:57ef38a65056e7800859e5ba9e6091053cd06e1038983016effaffe0efcd594a",
|
||||
"sha256:5bd61e9b44c543016ce1f6aef48606280e45f892a928ca7068fba30021e9b786",
|
||||
"sha256:6482d3017a0c0327a49dddc8bd1074cc730d45db2ccb09c3bac1f8f32d1eb61b",
|
||||
"sha256:7d3ce02c0784b7cbcc771a2da6ea51f87e8716004512493a2b69016326301c3b",
|
||||
"sha256:a14e499c0f5955dcc3991f785f3f8e2130ed504fa3a7f44009ff458ad6bdd17f",
|
||||
"sha256:a39f54ccbcd2757d1d63b0ec00a00980c0b382c62865b61a505163943624ab20",
|
||||
"sha256:aabb0c5232910a20eec8563503c153a8e78bbf5459490c49ab31f6adf3f3a415",
|
||||
"sha256:bd4ecb473a96ad0f90c20acba4f0bf0df91a4e03a1f4dd6a4bdc9ca75aa3a715",
|
||||
"sha256:e2da3c13307eac601f3de04887624939aca8ee3c9488a0bb0eca4fb9401fc6b1",
|
||||
"sha256:f67814c38162f4deb31f68d590771a29d5ae3b1bd64b75cf232308e5c74777e0"
|
||||
],
|
||||
"version": "==1.3.0"
|
||||
},
|
||||
"python-decouple": {
|
||||
"hashes": [
|
||||
"sha256:1317df14b43efee4337a4aa02914bf004f010cd56d6c4bd894e6474ec8c4fe2d"
|
||||
|
|
@ -181,6 +300,13 @@
|
|||
],
|
||||
"version": "==1.12.0"
|
||||
},
|
||||
"sshtunnel": {
|
||||
"hashes": [
|
||||
"sha256:c813fdcda8e81c3936ffeac47cb69cfb2d1f5e77ad0de656c6dab56aeebd9249"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==0.1.5"
|
||||
},
|
||||
"tenacity": {
|
||||
"hashes": [
|
||||
"sha256:a0c3c5f7ae0c33f5556c775ca059c12d6fd8ab7121613a713e8b7d649908571b",
|
||||
|
|
@ -225,19 +351,6 @@
|
|||
"index": "pypi",
|
||||
"version": "==1.6.2"
|
||||
},
|
||||
"ddt": {
|
||||
"hashes": [
|
||||
"sha256:474546b4020ce8a2f9550ba8899c28aa2c284c7bbf175bddede98be949d1ca7c",
|
||||
"sha256:d13e6af8f36238e89d00f4ebccf2bda4f6d1878be560a6600689e42077e164e3"
|
||||
],
|
||||
"version": "==1.2.1"
|
||||
},
|
||||
"gitdb": {
|
||||
"hashes": [
|
||||
"sha256:a3ebbc27be035a2e874ed904df516e35f4a29a778a764385de09de9e0f139658"
|
||||
],
|
||||
"version": "==0.6.4"
|
||||
},
|
||||
"gitdb2": {
|
||||
"hashes": [
|
||||
"sha256:83361131a1836661a155172932a13c08bda2db3674e4caa32368aa6eb02f38c2",
|
||||
|
|
@ -247,10 +360,10 @@
|
|||
},
|
||||
"gitpython": {
|
||||
"hashes": [
|
||||
"sha256:7428f1cc5e72d53e65c3259d5cebc22fb2b07f973c49d95b3c3d26c64890a3c3",
|
||||
"sha256:a0f744a4941eac5d99033eb0adcbec83bf443ee173fda4292d92a906aedce952"
|
||||
"sha256:c15c55ff890cd3a6a8330059e80885410a328f645551b55a91d858bfb3eb2573",
|
||||
"sha256:df752b6b6f06f11213e91c4925aea7eaf9e37e88fb71c8a7a1aa0a5c10852120"
|
||||
],
|
||||
"version": "==2.1.12"
|
||||
"version": "==2.1.13"
|
||||
},
|
||||
"mccabe": {
|
||||
"hashes": [
|
||||
|
|
@ -296,19 +409,21 @@
|
|||
},
|
||||
"pyyaml": {
|
||||
"hashes": [
|
||||
"sha256:57acc1d8533cbe51f6662a55434f0dbecfa2b9eaf115bede8f6fd00115a0c0d3",
|
||||
"sha256:588c94b3d16b76cfed8e0be54932e5729cc185caffaa5a451e7ad2f7ed8b4043",
|
||||
"sha256:68c8dd247f29f9a0d09375c9c6b8fdc64b60810ebf07ba4cdd64ceee3a58c7b7",
|
||||
"sha256:70d9818f1c9cd5c48bb87804f2efc8692f1023dac7f1a1a5c61d454043c1d265",
|
||||
"sha256:86a93cccd50f8c125286e637328ff4eef108400dd7089b46a7be3445eecfa391",
|
||||
"sha256:a0f329125a926876f647c9fa0ef32801587a12328b4a3c741270464e3e4fa778",
|
||||
"sha256:a3c252ab0fa1bb0d5a3f6449a4826732f3eb6c0270925548cac342bc9b22c225",
|
||||
"sha256:b4bb4d3f5e232425e25dda21c070ce05168a786ac9eda43768ab7f3ac2770955",
|
||||
"sha256:cd0618c5ba5bda5f4039b9398bb7fb6a317bb8298218c3de25c47c4740e4b95e",
|
||||
"sha256:ceacb9e5f8474dcf45b940578591c7f3d960e82f926c707788a570b51ba59190",
|
||||
"sha256:fe6a88094b64132c4bb3b631412e90032e8cfe9745a58370462240b8cb7553cd"
|
||||
"sha256:0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9",
|
||||
"sha256:01adf0b6c6f61bd11af6e10ca52b7d4057dd0be0343eb9283c878cf3af56aee4",
|
||||
"sha256:5124373960b0b3f4aa7df1707e63e9f109b5263eca5976c66e08b1c552d4eaf8",
|
||||
"sha256:5ca4f10adbddae56d824b2c09668e91219bb178a1eee1faa56af6f99f11bf696",
|
||||
"sha256:7907be34ffa3c5a32b60b95f4d95ea25361c951383a894fec31be7252b2b6f34",
|
||||
"sha256:7ec9b2a4ed5cad025c2278a1e6a19c011c80a3caaac804fd2d329e9cc2c287c9",
|
||||
"sha256:87ae4c829bb25b9fe99cf71fbb2140c448f534e24c998cc60f39ae4f94396a73",
|
||||
"sha256:9de9919becc9cc2ff03637872a440195ac4241c80536632fffeb6a1e25a74299",
|
||||
"sha256:a5a85b10e450c66b49f98846937e8cfca1db3127a9d5d1e31ca45c3d0bef4c5b",
|
||||
"sha256:b0997827b4f6a7c286c01c5f60384d218dca4ed7d9efa945c3e1aa623d5709ae",
|
||||
"sha256:b631ef96d3222e62861443cc89d6563ba3eeb816eeb96b2629345ab795e53681",
|
||||
"sha256:bf47c0607522fdbca6c9e817a6e81b08491de50f3766a7a0e6a5be7905961b41",
|
||||
"sha256:f81025eddd0327c7d4cfe9b62cf33190e1e736cc6e97502b3ec425f574b3e7a8"
|
||||
],
|
||||
"version": "==5.1.1"
|
||||
"version": "==5.1.2"
|
||||
},
|
||||
"six": {
|
||||
"hashes": [
|
||||
|
|
@ -317,12 +432,6 @@
|
|||
],
|
||||
"version": "==1.12.0"
|
||||
},
|
||||
"smmap": {
|
||||
"hashes": [
|
||||
"sha256:0e2b62b497bd5f0afebc002eda4d90df9d209c30ef257e8673c90a6b5c119d62"
|
||||
],
|
||||
"version": "==0.9.0"
|
||||
},
|
||||
"smmap2": {
|
||||
"hashes": [
|
||||
"sha256:0555a7bf4df71d1ef4218e4807bbf9b201f910174e6e08af2e138d4e517b4dde",
|
||||
|
|
|
|||
393
main.py
393
main.py
|
|
@ -12,21 +12,29 @@ import logging
|
|||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import traceback
|
||||
import sshtunnel
|
||||
import errno
|
||||
|
||||
from etcd3_wrapper import Etcd3Wrapper
|
||||
from dataclasses import dataclass
|
||||
from typing import Union
|
||||
from functools import wraps
|
||||
from string import Template
|
||||
from os.path import join
|
||||
|
||||
from ucloud_common.rbd import RBD
|
||||
from ucloud_common.vm import VmPool, VMEntry, VMStatus
|
||||
from etcd3_wrapper import Etcd3Wrapper
|
||||
from ucloud_common.vm import VmPool, VMStatus, VMEntry
|
||||
from ucloud_common.host import HostPool, HostEntry
|
||||
from ucloud_common.request import RequestEntry, RequestPool, RequestType
|
||||
from ucloud_common.helpers import get_ipv4_address
|
||||
|
||||
running_vms = []
|
||||
vnc_port_pool = list(range(0, 100))
|
||||
|
||||
client = Etcd3Wrapper()
|
||||
VM_POOL = None
|
||||
HOST_POOL = None
|
||||
vm_pool = None
|
||||
host_pool = None
|
||||
request_pool = None
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.DEBUG,
|
||||
|
|
@ -37,16 +45,37 @@ logging.basicConfig(
|
|||
)
|
||||
|
||||
|
||||
def get_start_command_args(image_path, vnc_port, migration=False, migration_port=4444):
|
||||
_args = ("-drive file=$image_path,format=raw,if=virtio,cache=none"
|
||||
" -m 1024 -device virtio-rng-pci -enable-kvm -vnc :$vnc_port")
|
||||
|
||||
if migration:
|
||||
_args = _args + " -incoming tcp:0:$migration_port"
|
||||
|
||||
args_template = Template(_args)
|
||||
|
||||
if migration:
|
||||
args = args_template.substitute(image_path=image_path, vnc_port=vnc_port,
|
||||
migration_port=migration_port)
|
||||
else:
|
||||
args = args_template.substitute(image_path=image_path, vnc_port=vnc_port)
|
||||
|
||||
return args.split(" ")
|
||||
|
||||
|
||||
@dataclass
|
||||
class VM:
|
||||
key: str
|
||||
vm: qmp.QEMUMachine
|
||||
handle: qmp.QEMUMachine
|
||||
|
||||
def __repr__(self):
|
||||
return f"VM({self.key})"
|
||||
|
||||
|
||||
def update_heartbeat(host: HostEntry):
|
||||
while True:
|
||||
host.update_heartbeat()
|
||||
HOST_POOL.put(host)
|
||||
host_pool.put(host)
|
||||
time.sleep(10)
|
||||
|
||||
logging.info(f"Updated last heartbeat time {host.last_heartbeat}")
|
||||
|
|
@ -58,7 +87,7 @@ def need_running_vm(func):
|
|||
vm = get_vm(running_vms, e.key)
|
||||
if vm:
|
||||
try:
|
||||
status = vm.vm.command("query-status")
|
||||
status = vm.handle.command("query-status")
|
||||
logging.debug(f"VM Status Check - {status}")
|
||||
except OSError:
|
||||
logging.info(
|
||||
|
|
@ -75,119 +104,79 @@ def need_running_vm(func):
|
|||
return wrapper
|
||||
|
||||
|
||||
def create_vm(vm):
|
||||
image = client.get(f"/v1/image/{vm.value['image_uuid']}", value_in_json=True)
|
||||
if image:
|
||||
logging.debug(image)
|
||||
logging.info("Creating New VM...")
|
||||
|
||||
_command_to_create = f"rbd clone images/{vm.image_uuid}@protected uservms/{vm.uuid}"
|
||||
def create_vm(vm_entry: VMEntry):
|
||||
_command_to_create = f"rbd clone images/{vm_entry.image_uuid}@protected uservms/{vm_entry.uuid}"
|
||||
try:
|
||||
subprocess.call(_command_to_create.split(" "))
|
||||
vm.status = VMStatus.requested_start
|
||||
VM_POOL.put(vm)
|
||||
except:
|
||||
logging.exception("Can't clone image")
|
||||
subprocess.check_call(_command_to_create.split(" "))
|
||||
except subprocess.CalledProcessError as e:
|
||||
if e.returncode == errno.EEXIST:
|
||||
logging.debug(f"Image for vm {vm_entry.uuid} exists")
|
||||
# File Already exists. No Problem Continue
|
||||
return
|
||||
else:
|
||||
logging.info(f"Image not found for {vm.image_uuid}")
|
||||
# This exception catches all other exceptions
|
||||
# i.e FileNotFound (BaseImage), pool Does Not Exists etc.
|
||||
logging.exception(f"Can't clone image - {e}")
|
||||
else:
|
||||
logging.info("New VM Created")
|
||||
|
||||
|
||||
def start_vm(e):
|
||||
vm_path = f"rbd:uservms/{e.uuid}"
|
||||
def start_vm(vm_entry: VMEntry):
|
||||
_vm = get_vm(running_vms, vm_entry.key)
|
||||
|
||||
try:
|
||||
user_vms = RBD.ls("uservms")
|
||||
except:
|
||||
logging.info("Can't access uservms pool")
|
||||
return
|
||||
|
||||
if e.uuid not in user_vms:
|
||||
logging.info(f"Image file of vm {e.key} does not exists")
|
||||
logging.info(f"Deleting vm {e.key}")
|
||||
client.client.delete(e.key)
|
||||
return
|
||||
|
||||
_vm = get_vm(running_vms, e.key)
|
||||
# VM already running. No need to proceed further.
|
||||
if _vm:
|
||||
logging.info(f"{e.key} already running")
|
||||
e.status = VMStatus.running
|
||||
VM_POOL.put(e)
|
||||
logging.info(f"VM {vm_entry.uuid} already running")
|
||||
return
|
||||
else:
|
||||
create_vm(vm_entry)
|
||||
|
||||
logging.info(f"Starting {vm_entry.key}")
|
||||
|
||||
# FIXME: There should be better vnc port allocation scheme
|
||||
vm = qmp.QEMUMachine(
|
||||
"/usr/bin/qemu-system-x86_64",
|
||||
test_dir="vm_socklog",
|
||||
args=[
|
||||
vm_path,
|
||||
"-boot",
|
||||
"c", # First Boot Hard drive
|
||||
"-m",
|
||||
"1024", # RAM limit
|
||||
# Ever growing port number
|
||||
"-vnc",
|
||||
f":{vnc_port_pool.pop(0)}", # Allow VNC
|
||||
],
|
||||
args=get_start_command_args(vm_entry.path, vnc_port_pool.pop(0)),
|
||||
)
|
||||
try:
|
||||
logging.info(f"Starting {e.key}")
|
||||
vm.launch()
|
||||
if vm.is_running():
|
||||
running_vms.append(VM(e.key, vm))
|
||||
e.status = VMStatus.running
|
||||
VM_POOL.put(e)
|
||||
except (qmp.QEMUMachineError, TypeError, Exception):
|
||||
vm_entry.declare_killed()
|
||||
vm_entry.add_log(f"Machine Error occurred - {traceback.format_exc()}")
|
||||
vm_pool.put(vm_entry)
|
||||
else:
|
||||
e.declare_killed()
|
||||
VM_POOL.put(e)
|
||||
return
|
||||
except (qmp.QEMUMachineError, TypeError):
|
||||
logging.exception(f"Machine Error Occurred on {e.key}")
|
||||
e.declare_killed()
|
||||
VM_POOL.put(e)
|
||||
else:
|
||||
logging.info(f"Started Successfully {e.key}")
|
||||
running_vms.append(VM(vm_entry.key, vm))
|
||||
vm_entry.status = VMStatus.running
|
||||
vm_entry.add_log("Started successfully")
|
||||
vm_pool.put(vm_entry)
|
||||
|
||||
|
||||
@need_running_vm
|
||||
def suspend_vm(e):
|
||||
vm = get_vm(running_vms, e.key)
|
||||
vm.vm.command("stop")
|
||||
if vm.vm.command("query-status")["status"] == "paused":
|
||||
e.status = VMStatus.suspended
|
||||
VM_POOL.put(e)
|
||||
logging.info(f"Successfully suspended VM {e.key}")
|
||||
else:
|
||||
logging.info(f"Suspending VM {e.key} failed")
|
||||
|
||||
|
||||
@need_running_vm
|
||||
def resume_vm(e):
|
||||
vm = get_vm(running_vms, e.key)
|
||||
vm.vm.command("cont")
|
||||
if vm.vm.command("query-status")["status"] == "running":
|
||||
e.status = VMStatus.running
|
||||
VM_POOL.put(e)
|
||||
logging.info(f"Successfully resumed VM {e.key}")
|
||||
else:
|
||||
logging.info(f"Resuming VM {e.key} failed")
|
||||
|
||||
|
||||
@need_running_vm
|
||||
def shutdown_vm(e):
|
||||
vm = get_vm(running_vms, e.key)
|
||||
vm.vm.shutdown()
|
||||
if not vm.vm.is_running():
|
||||
logging.info(f"VM {e.key} shutdown successfully")
|
||||
e.status = VMStatus.stopped
|
||||
VM_POOL.put(e)
|
||||
def stop_vm(vm_entry):
|
||||
vm = get_vm(running_vms, vm_entry.key)
|
||||
vm.handle.shutdown()
|
||||
if not vm.handle.is_running():
|
||||
vm_entry.add_log("Shutdown successfully")
|
||||
vm_entry.declare_stopped()
|
||||
vm_pool.put(vm_entry)
|
||||
running_vms.remove(vm)
|
||||
|
||||
|
||||
def delete_vm(e):
|
||||
# TODO: Delete VM Image From CEPH
|
||||
logging.info(f"Deleting VM {e.key}")
|
||||
shutdown_vm(e)
|
||||
client.client.delete(e.key)
|
||||
def delete_vm(vm_entry):
|
||||
logging.info(f"Deleting VM {vm_entry}")
|
||||
stop_vm(vm_entry)
|
||||
path_without_protocol = vm_entry.path[vm_entry.path.find(":") + 1:]
|
||||
try:
|
||||
rc = subprocess.call(f"rbd rm {path_without_protocol}".split(" "))
|
||||
except FileNotFoundError as e:
|
||||
logging.exception(e)
|
||||
except Exception as e:
|
||||
logging.exception(f"Unknown error occurred - {e}")
|
||||
else:
|
||||
if rc == 0:
|
||||
client.client.delete(vm_entry.key)
|
||||
else:
|
||||
logging.info("Some unknown problem occur while deleting vm file")
|
||||
|
||||
|
||||
def get_vm(vm_list: list, vm_key) -> Union[VM, None]:
|
||||
|
|
@ -195,87 +184,199 @@ def get_vm(vm_list: list, vm_key) -> Union[VM, None]:
|
|||
|
||||
|
||||
def maintenance(host):
|
||||
_vms = VM_POOL.by_host(host.key)
|
||||
alleged_running_vms = VM_POOL.by_status("RUNNING", _vms)
|
||||
# To capture vm running according to running_vms list
|
||||
|
||||
for vm in alleged_running_vms:
|
||||
_vm = get_vm(running_vms, vm.key)
|
||||
if (_vm and not _vm.vm.is_running()) or _vm is None:
|
||||
logging.debug(f"{_vm} {vm.key}")
|
||||
logging.info(
|
||||
f"{vm.key} is not running but is said to be running"
|
||||
# This is to capture successful migration of a VM.
|
||||
# Suppose, this host is running "vm1" and user initiated
|
||||
# request to migrate this "vm1" to some other host. On,
|
||||
# successful migration the destination host would set
|
||||
# the vm hostname to itself. Thus, we are checking
|
||||
# whether this host vm is successfully migrated. If yes
|
||||
# then we shutdown "vm1" on this host.
|
||||
|
||||
for vm in running_vms:
|
||||
with vm_pool.get_put(vm.key) as vm_entry:
|
||||
if vm_entry.hostname != host.key and not vm_entry.in_migration:
|
||||
vm.handle.shutdown()
|
||||
vm_entry.add_log("VM on source host shutdown.")
|
||||
# To check vm running according to etcd entries
|
||||
alleged_running_vms = vm_pool.by_status("RUNNING", vm_pool.by_host(host.key))
|
||||
|
||||
for vm_entry in alleged_running_vms:
|
||||
_vm = get_vm(running_vms, vm_entry.key)
|
||||
|
||||
# Whether, the allegedly running vm is in our
|
||||
# running_vms list or not if it is said to be
|
||||
# running on this host but it is not then we
|
||||
# need to shut it down
|
||||
|
||||
# This is to capture poweroff/shutdown of a VM
|
||||
# initiated by user inside VM. OR crash of VM by some
|
||||
# user running process
|
||||
if (_vm and not _vm.handle.is_running())\
|
||||
or not _vm:
|
||||
vm_entry.add_log(f"{vm_entry.key} is not running but is said to be running."
|
||||
"So, shutting it down and declare it killed")
|
||||
vm_entry.declare_killed()
|
||||
vm_pool.put(vm_entry)
|
||||
if _vm:
|
||||
running_vms.remove(_vm)
|
||||
|
||||
|
||||
def transfer_vm(request_event):
|
||||
# This function would run on source host i.e host on which the vm
|
||||
# is running initially. This host would be responsible for transferring
|
||||
# vm state to destination host.
|
||||
|
||||
_host, _port = request_event.parameters["host"], request_event.parameters["port"]
|
||||
_uuid = request_event.uuid
|
||||
_destination = request_event.destination_host_key
|
||||
vm = get_vm(running_vms, join("/v1/vm", _uuid))
|
||||
|
||||
if vm:
|
||||
tunnel = sshtunnel.SSHTunnelForwarder(
|
||||
(_host, 22),
|
||||
ssh_username="meow",
|
||||
ssh_pkey="~/.ssh/id_rsa",
|
||||
ssh_private_key_password="***REMOVED***",
|
||||
remote_bind_address=('127.0.0.1', _port),
|
||||
)
|
||||
logging.info(f"Updating {vm.key} status to KILLED")
|
||||
vm.declare_killed()
|
||||
VM_POOL.put(vm)
|
||||
try:
|
||||
tunnel.start()
|
||||
except sshtunnel.BaseSSHTunnelForwarderError:
|
||||
logging.exception(f"Couldn't establish connection to ({_host}, 22)")
|
||||
else:
|
||||
vm.handle.command("migrate", uri=f"tcp:{_host}:{tunnel.local_bind_port}")
|
||||
|
||||
status = vm.handle.command("query-migrate")["status"]
|
||||
while status not in ["failed", "completed"]:
|
||||
time.sleep(2)
|
||||
status = vm.handle.command("query-migrate")["status"]
|
||||
|
||||
with vm_pool.get_put(request_event.uuid) as source_vm:
|
||||
if status == "failed":
|
||||
source_vm.add_log("Migration Failed")
|
||||
elif status == "completed":
|
||||
# If VM is successfully migrated then shutdown the VM
|
||||
# on this host and update hostname to destination host key
|
||||
source_vm.add_log("Successfully migrated")
|
||||
source_vm.hostname = _destination
|
||||
running_vms.remove(vm)
|
||||
vm.handle.shutdown()
|
||||
source_vm.in_migration = False # VM transfer finished
|
||||
finally:
|
||||
tunnel.close()
|
||||
|
||||
|
||||
def init_vm_migration(vm_entry, destination_host_key):
|
||||
# This function would run on destination host i.e host on which the vm
|
||||
# would be transferred after migration. This host would be responsible
|
||||
# for starting VM that would receive state of VM running on source host.
|
||||
|
||||
_vm = get_vm(running_vms, vm_entry.key)
|
||||
|
||||
if _vm:
|
||||
# VM already running. No need to proceed further.
|
||||
logging.log(f"{_vm.key} Already running")
|
||||
return
|
||||
|
||||
logging.info(f"Starting {vm_entry.key}")
|
||||
|
||||
# FIXME: There should be better vnc port allocation scheme
|
||||
actual_vm = qmp.QEMUMachine(
|
||||
"/usr/bin/qemu-system-x86_64",
|
||||
args=get_start_command_args(vm_entry.path, 100, migration=True, migration_port=4444),
|
||||
)
|
||||
|
||||
try:
|
||||
actual_vm.launch()
|
||||
except Exception as e:
|
||||
# We don't care whether MachineError or any other error occurred
|
||||
logging.exception(e)
|
||||
actual_vm.shutdown()
|
||||
else:
|
||||
vm_entry.in_migration = True
|
||||
vm_pool.put(vm_entry)
|
||||
|
||||
running_vms.append(VM(vm_entry.key, actual_vm))
|
||||
r = RequestEntry.from_scratch(type=RequestType.TransferVM,
|
||||
hostname=vm_entry.hostname,
|
||||
parameters={
|
||||
"host": get_ipv4_address(),
|
||||
"port": 4444,
|
||||
},
|
||||
uuid=vm_entry.uuid,
|
||||
destination_host_key=destination_host_key
|
||||
)
|
||||
request_pool.put(r)
|
||||
|
||||
|
||||
def main():
|
||||
argparser = argparse.ArgumentParser()
|
||||
argparser.add_argument(
|
||||
"hostname", help="Name of this host. e.g /v1/host/1"
|
||||
)
|
||||
argparser.add_argument("hostname", help="Name of this host. e.g /v1/host/1")
|
||||
args = argparser.parse_args()
|
||||
|
||||
global HOST_POOL, VM_POOL
|
||||
HOST_POOL = HostPool(client, "/v1/host")
|
||||
VM_POOL = VmPool(client, "/v1/vm")
|
||||
global host_pool, vm_pool, request_pool
|
||||
host_pool = HostPool(client, "/v1/host")
|
||||
vm_pool = VmPool(client, "/v1/vm")
|
||||
request_pool = RequestPool(client, "/v1/request")
|
||||
|
||||
host = HOST_POOL.get(args.hostname)
|
||||
host = host_pool.get(args.hostname)
|
||||
if not host:
|
||||
print("No Such Host")
|
||||
exit(1)
|
||||
|
||||
logging.info(f"{'*' * 5} Session Started {'*' * 5}")
|
||||
|
||||
# It is seen that under heavy load, timeout event doesn't come
|
||||
# in a predictive manner which delays heart beat update which
|
||||
# in turn misunderstood by scheduler that the host is dead
|
||||
# when it is actually alive. So, to ensure that we update the
|
||||
# heart beat in a predictive manner we start Heart beat updating
|
||||
# mechanism in separated thread
|
||||
|
||||
heartbeat_updating_thread = threading.Thread(target=update_heartbeat, args=(host,))
|
||||
try:
|
||||
heartbeat_updating_thread.start()
|
||||
except Exception as e:
|
||||
print("No Need To Go Further. Our heartbeat updating mechanism is not working")
|
||||
logging.exception(e)
|
||||
exit(-1)
|
||||
|
||||
for events_iterator in [
|
||||
client.get_prefix("/v1/vm/", value_in_json=True),
|
||||
client.watch_prefix("/v1/vm/", timeout=10, value_in_json=True),
|
||||
client.get_prefix("/v1/request/", value_in_json=True),
|
||||
client.watch_prefix("/v1/request/", timeout=10, value_in_json=True),
|
||||
]:
|
||||
for e in events_iterator:
|
||||
e = VMEntry(e)
|
||||
for request_event in events_iterator:
|
||||
request_event = RequestEntry(request_event)
|
||||
|
||||
if e.status == "TIMEOUT":
|
||||
logging.info("Timeout")
|
||||
if request_event.type == "TIMEOUT":
|
||||
logging.info("Timeout Event")
|
||||
maintenance(host)
|
||||
continue
|
||||
|
||||
# TODO: Re-evaluate Migration Design
|
||||
if hasattr(e, "migration_destination"):
|
||||
e_migration_destination = e.value["migration_destination"]
|
||||
else:
|
||||
e_migration_destination = ""
|
||||
# If the event is directed toward me OR I am destination of a InitVMMigration
|
||||
if hasattr(request_event, "hostname") and request_event.hostname == host.key or\
|
||||
hasattr(request_event, "destination") and request_event.destination == host.key:
|
||||
request_pool.client.client.delete(request_event.key)
|
||||
vm_entry = vm_pool.get(request_event.uuid)
|
||||
|
||||
# If the event is directed toward me or
|
||||
# I am destination of a REQUESTED_MIGRATION
|
||||
if e.hostname == host.key or e_migration_destination == host.key:
|
||||
logging.debug(f"EVENT: {e}")
|
||||
logging.debug(f"EVENT: {request_event}")
|
||||
|
||||
if e.status == "SCHEDULED_DEPLOY":
|
||||
create_vm(e)
|
||||
if request_event.type == RequestType.StartVM:
|
||||
start_vm(vm_entry)
|
||||
|
||||
elif e.status == "REQUESTED_SUSPEND":
|
||||
suspend_vm(e)
|
||||
elif request_event.type == RequestType.StopVM:
|
||||
stop_vm(vm_entry)
|
||||
|
||||
elif e.status == "REQUESTED_RESUME":
|
||||
resume_vm(e)
|
||||
elif request_event.type == RequestType.DeleteVM:
|
||||
delete_vm(vm_entry)
|
||||
|
||||
elif e.status == "REQUESTED_START":
|
||||
start_vm(e)
|
||||
elif request_event.type == RequestType.InitVMMigration:
|
||||
init_vm_migration(vm_entry, host.key)
|
||||
|
||||
elif e.status == "REQUESTED_SHUTDOWN":
|
||||
shutdown_vm(e)
|
||||
|
||||
elif e.status == "DELETED":
|
||||
delete_vm(e)
|
||||
elif request_event.type == RequestType.TransferVM:
|
||||
transfer_vm(request_event)
|
||||
|
||||
logging.info(f"Running VMs {running_vms}")
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue