simplifies logic + add unit tests
This commit is contained in:
parent
17b40b13cc
commit
7354af7e07
5 changed files with 463 additions and 27 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -1,4 +1,5 @@
|
||||||
.env
|
.env
|
||||||
.idea/
|
.idea/
|
||||||
|
.vscode
|
||||||
__pycache__/
|
__pycache__/
|
||||||
venv/
|
venv/
|
||||||
3
Pipfile
3
Pipfile
|
|
@ -4,6 +4,9 @@ url = "https://pypi.org/simple"
|
||||||
verify_ssl = true
|
verify_ssl = true
|
||||||
|
|
||||||
[dev-packages]
|
[dev-packages]
|
||||||
|
bandit = "*"
|
||||||
|
flake8 = "*"
|
||||||
|
black = "==19.3b0"
|
||||||
|
|
||||||
[packages]
|
[packages]
|
||||||
etcd3 = "*"
|
etcd3 = "*"
|
||||||
|
|
|
||||||
248
Pipfile.lock
generated
Normal file
248
Pipfile.lock
generated
Normal file
|
|
@ -0,0 +1,248 @@
|
||||||
|
{
|
||||||
|
"_meta": {
|
||||||
|
"hash": {
|
||||||
|
"sha256": "38090ff76faaefba3892389e2d49d44cdb57752738163a27c99193e960d550e1"
|
||||||
|
},
|
||||||
|
"pipfile-spec": 6,
|
||||||
|
"requires": {
|
||||||
|
"python_version": "3.7"
|
||||||
|
},
|
||||||
|
"sources": [
|
||||||
|
{
|
||||||
|
"name": "pypi",
|
||||||
|
"url": "https://pypi.org/simple",
|
||||||
|
"verify_ssl": true
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"default": {
|
||||||
|
"etcd3": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:25a524b9f032c6631ff0097532907dea81243eaa63c3744510fd1598cc4e0e87"
|
||||||
|
],
|
||||||
|
"index": "pypi",
|
||||||
|
"version": "==0.10.0"
|
||||||
|
},
|
||||||
|
"grpcio": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:0232add03144dd3cf9b660e2718244cb8e175370dca4d3855cb4e489a7811b53",
|
||||||
|
"sha256:0f20e6dcb1b8662cdca033bb97c0a8116a5343e3ebc7f71c5fe7f89039978350",
|
||||||
|
"sha256:10b07a623d33d4966f45c85d410bc6a79c5ac6341f06c3beda6c22be12cbfe07",
|
||||||
|
"sha256:10c0476d5a52d21f402fc073745dc43b87cc8e080a1f49bbff4e1059019310fb",
|
||||||
|
"sha256:289dae0b35c59d191c524e976dd0a6f8c995d2062e72621eb866ad0f4472a635",
|
||||||
|
"sha256:2be726f16142d358a0df1e81d583d6820ee561a7856a79cca2fbe49989308be7",
|
||||||
|
"sha256:4338d2a81f5b4ca022e085040b3cfce19419a5ce44aa7e6810ac1df05365bed7",
|
||||||
|
"sha256:4c535b46f20e66bee3097583231977e721acdfcb1671d1490c99b7be8902ce18",
|
||||||
|
"sha256:557154aef70a0e979700cc9528bc8b606b668084a29a0d57dbc4b06b078a2f1c",
|
||||||
|
"sha256:5bfdd7e6647498f979dc46583723c852d97b25afe995d55aa1c76a5f9816bc1f",
|
||||||
|
"sha256:87d8943ae7aa6ca5bbad732867d7f17d2550e4966a0c15b52088e8b579422e47",
|
||||||
|
"sha256:89d8719d8de4d137678f7caa979e1b0a6fd4026f8096ceef8c2d164bbabefaf2",
|
||||||
|
"sha256:9c3f4af989ce860710ac1864dc2e867dd87e6cee51a2368df1b253596868e52f",
|
||||||
|
"sha256:9da52c3c728883aee429bb7c315049f50b2139f680cd86bb1165418e4f93a982",
|
||||||
|
"sha256:9e9736659987beab42d18525ed10d21f80a1ba8389eac03425fbfd5684e6bbf0",
|
||||||
|
"sha256:9ebcbb1a054cab362d29d3be571d43d6b9b23302d9fc4b43e5327000da1680a9",
|
||||||
|
"sha256:a93e08636623e24c939851e2e0c0140b14f524b2980c9cdc4ea52b70a871c7e0",
|
||||||
|
"sha256:ac322d86d1a079e0a118d544443ee16f320af0062c191b4754c0c6ec2fc79310",
|
||||||
|
"sha256:b1fb101459868f52df6b61e7bb13375e50badf17a160e39fe1d51ae19e53f461",
|
||||||
|
"sha256:b39aac96cceac624a23d540473835086a3ffa77c91030189988c073488434493",
|
||||||
|
"sha256:b65507bc273c6dbf539175a786a344cc0ac78d50e5584f72c6599733f8a3301f",
|
||||||
|
"sha256:be5bb6e47417e537c884a2e2ff2e1a8b2c064a998fcfdfcc67528d4e63e7ebaf",
|
||||||
|
"sha256:c92de6a28a909c4f460dc1bbbcb50d676cf0b1f40224b222761f73fdd851b522",
|
||||||
|
"sha256:c9f5962eb7fa7607b20eb0e4f59ed35829bd600fc0eacb626a6db83229a3e445",
|
||||||
|
"sha256:d00bdf9c546ed6e649f785c55b05288e8b2dbb6bf2eb74b6c579fa0d591d35bd",
|
||||||
|
"sha256:da804b1dd8293bd9d61b1e6ea989c887ba042a808a4fbdd80001cfa059aafed2",
|
||||||
|
"sha256:ead6c5aa3e807345913649c3be395aaca2bbb2d225f18b8f31f37eab225508f6",
|
||||||
|
"sha256:eb4d81550ce6f826af4ec6e8d98be347fe96291d718bf115c3f254621ae8d98d",
|
||||||
|
"sha256:ef6a18ec8fd32ec81748fe720544ea2fb2d2dc50fd6d06739d5e2eb8f0626a1c",
|
||||||
|
"sha256:fad42835656e0b6d3b7ffc900598e776722e30f43b7234a48f2576ca30f31a47",
|
||||||
|
"sha256:fb98dbfee0d963b49ae5754554028cf62e6bd695f22de16d242ba9d2f0b7339b",
|
||||||
|
"sha256:fb9cd9bb8d26dc17c2dd715a46bca3a879ec8283879b164e85863110dc6e3b2a"
|
||||||
|
],
|
||||||
|
"version": "==1.21.1"
|
||||||
|
},
|
||||||
|
"protobuf": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:03f43eac9d5b651f976e91cf46a25b75e5779d98f0f4114b0abfed83376d75f8",
|
||||||
|
"sha256:0c94b21e6de01362f91a86b372555d22a60b59708599ca9d5032ae9fdf8e3538",
|
||||||
|
"sha256:2d2a9f30f61f4063fadd7fb68a2510a6939b43c0d6ceeec5c4704f22225da28e",
|
||||||
|
"sha256:34a0b05fca061e4abb77dd180209f68d8637115ff319f51e28a6a9382d69853a",
|
||||||
|
"sha256:358710fd0db25372edcf1150fa691f48376a134a6c69ce29f38f185eea7699e6",
|
||||||
|
"sha256:41e47198b94c27ba05a08b4a95160656105745c462af574e4bcb0807164065c0",
|
||||||
|
"sha256:8c61cc8a76e9d381c665aecc5105fa0f1878cf7db8b5cd17202603bcb386d0fc",
|
||||||
|
"sha256:a6eebc4db759e58fdac02efcd3028b811effac881d8a5bad1996e4e8ee6acb47",
|
||||||
|
"sha256:a9c12f7c98093da0a46ba76ec40ace725daa1ac4038c41e4b1466afb5c45bb01",
|
||||||
|
"sha256:cb95068492ba0859b8c9e61fa8ba206a83c64e5d0916fb4543700b2e2b214115",
|
||||||
|
"sha256:cd98476ce7bb4dcd6a7b101f5eecdc073dafea19f311e36eb8fba1a349346277",
|
||||||
|
"sha256:ce64cfbea18c535176bdaa10ba740c0fc4c6d998a3f511c17bedb0ae4b3b167c",
|
||||||
|
"sha256:dcbb59eac73fd454e8f2c5fba9e3d3320fd4707ed6a9d3ea3717924a6f0903ea",
|
||||||
|
"sha256:dd67f34458ae716029e2a71ede998e9092493b62a519236ca52e3c5202096c87",
|
||||||
|
"sha256:e3c96056eb5b7284a20e256cb0bf783c8f36ad82a4ae5434a7b7cd02384144a7",
|
||||||
|
"sha256:f612d584d7a27e2f39e7b17878430a959c1bc09a74ba09db096b468558e5e126",
|
||||||
|
"sha256:f6de8a7d6122297b81566e5bd4df37fd5d62bec14f8f90ebff8ede1c9726cd0a",
|
||||||
|
"sha256:fa529d9261682b24c2aaa683667253175c9acebe0a31105394b221090da75832"
|
||||||
|
],
|
||||||
|
"version": "==3.8.0"
|
||||||
|
},
|
||||||
|
"python-decouple": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:1317df14b43efee4337a4aa02914bf004f010cd56d6c4bd894e6474ec8c4fe2d"
|
||||||
|
],
|
||||||
|
"index": "pypi",
|
||||||
|
"version": "==3.1"
|
||||||
|
},
|
||||||
|
"six": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
|
||||||
|
"sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"
|
||||||
|
],
|
||||||
|
"version": "==1.12.0"
|
||||||
|
},
|
||||||
|
"tenacity": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:a0c3c5f7ae0c33f5556c775ca059c12d6fd8ab7121613a713e8b7d649908571b",
|
||||||
|
"sha256:b87c1934daa0b2ccc7db153c37b8bf91d12f165936ade8628e7b962b92dc7705"
|
||||||
|
],
|
||||||
|
"version": "==5.0.4"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"develop": {
|
||||||
|
"appdirs": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92",
|
||||||
|
"sha256:d8b24664561d0d34ddfaec54636d502d7cea6e29c3eaf68f3df6180863e2166e"
|
||||||
|
],
|
||||||
|
"version": "==1.4.3"
|
||||||
|
},
|
||||||
|
"attrs": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79",
|
||||||
|
"sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399"
|
||||||
|
],
|
||||||
|
"version": "==19.1.0"
|
||||||
|
},
|
||||||
|
"bandit": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:f89adaff792d1f9b72859784c5f7964c6b5a5f32ca0ca458c9643e02d4fdceac",
|
||||||
|
"sha256:fa1fee3cb60a3dca89b7a86c0be82af0e830def961728aba9290854fe18c1f90"
|
||||||
|
],
|
||||||
|
"index": "pypi",
|
||||||
|
"version": "==1.6.1"
|
||||||
|
},
|
||||||
|
"black": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:09a9dcb7c46ed496a9850b76e4e825d6049ecd38b611f1224857a79bd985a8cf",
|
||||||
|
"sha256:68950ffd4d9169716bcb8719a56c07a2f4485354fec061cdd5910aa07369731c"
|
||||||
|
],
|
||||||
|
"index": "pypi",
|
||||||
|
"version": "==19.3b0"
|
||||||
|
},
|
||||||
|
"click": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13",
|
||||||
|
"sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7"
|
||||||
|
],
|
||||||
|
"version": "==7.0"
|
||||||
|
},
|
||||||
|
"entrypoints": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19",
|
||||||
|
"sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451"
|
||||||
|
],
|
||||||
|
"version": "==0.3"
|
||||||
|
},
|
||||||
|
"flake8": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:859996073f341f2670741b51ec1e67a01da142831aa1fdc6242dbf88dffbe661",
|
||||||
|
"sha256:a796a115208f5c03b18f332f7c11729812c8c3ded6c46319c59b53efd3819da8"
|
||||||
|
],
|
||||||
|
"index": "pypi",
|
||||||
|
"version": "==3.7.7"
|
||||||
|
},
|
||||||
|
"gitdb2": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:83361131a1836661a155172932a13c08bda2db3674e4caa32368aa6eb02f38c2",
|
||||||
|
"sha256:e3a0141c5f2a3f635c7209d56c496ebe1ad35da82fe4d3ec4aaa36278d70648a"
|
||||||
|
],
|
||||||
|
"version": "==2.0.5"
|
||||||
|
},
|
||||||
|
"gitpython": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:563221e5a44369c6b79172f455584c9ebbb122a13368cc82cb4b5addff788f82",
|
||||||
|
"sha256:8237dc5bfd6f1366abeee5624111b9d6879393d84745a507de0fda86043b65a8"
|
||||||
|
],
|
||||||
|
"version": "==2.1.11"
|
||||||
|
},
|
||||||
|
"mccabe": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42",
|
||||||
|
"sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"
|
||||||
|
],
|
||||||
|
"version": "==0.6.1"
|
||||||
|
},
|
||||||
|
"pbr": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:9181e2a34d80f07a359ff1d0504fad3a47e00e1cf2c475b0aa7dcb030af54c40",
|
||||||
|
"sha256:94bdc84da376b3dd5061aa0c3b6faffe943ee2e56fa4ff9bd63e1643932f34fc"
|
||||||
|
],
|
||||||
|
"version": "==5.3.1"
|
||||||
|
},
|
||||||
|
"pycodestyle": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:95a2219d12372f05704562a14ec30bc76b05a5b297b21a5dfe3f6fac3491ae56",
|
||||||
|
"sha256:e40a936c9a450ad81df37f549d676d127b1b66000a6c500caa2b085bc0ca976c"
|
||||||
|
],
|
||||||
|
"version": "==2.5.0"
|
||||||
|
},
|
||||||
|
"pyflakes": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:17dbeb2e3f4d772725c777fabc446d5634d1038f234e77343108ce445ea69ce0",
|
||||||
|
"sha256:d976835886f8c5b31d47970ed689944a0262b5f3afa00a5a7b4dc81e5449f8a2"
|
||||||
|
],
|
||||||
|
"version": "==2.1.1"
|
||||||
|
},
|
||||||
|
"pyyaml": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:57acc1d8533cbe51f6662a55434f0dbecfa2b9eaf115bede8f6fd00115a0c0d3",
|
||||||
|
"sha256:588c94b3d16b76cfed8e0be54932e5729cc185caffaa5a451e7ad2f7ed8b4043",
|
||||||
|
"sha256:68c8dd247f29f9a0d09375c9c6b8fdc64b60810ebf07ba4cdd64ceee3a58c7b7",
|
||||||
|
"sha256:70d9818f1c9cd5c48bb87804f2efc8692f1023dac7f1a1a5c61d454043c1d265",
|
||||||
|
"sha256:86a93cccd50f8c125286e637328ff4eef108400dd7089b46a7be3445eecfa391",
|
||||||
|
"sha256:a0f329125a926876f647c9fa0ef32801587a12328b4a3c741270464e3e4fa778",
|
||||||
|
"sha256:a3c252ab0fa1bb0d5a3f6449a4826732f3eb6c0270925548cac342bc9b22c225",
|
||||||
|
"sha256:b4bb4d3f5e232425e25dda21c070ce05168a786ac9eda43768ab7f3ac2770955",
|
||||||
|
"sha256:cd0618c5ba5bda5f4039b9398bb7fb6a317bb8298218c3de25c47c4740e4b95e",
|
||||||
|
"sha256:ceacb9e5f8474dcf45b940578591c7f3d960e82f926c707788a570b51ba59190",
|
||||||
|
"sha256:fe6a88094b64132c4bb3b631412e90032e8cfe9745a58370462240b8cb7553cd"
|
||||||
|
],
|
||||||
|
"version": "==5.1.1"
|
||||||
|
},
|
||||||
|
"six": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
|
||||||
|
"sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"
|
||||||
|
],
|
||||||
|
"version": "==1.12.0"
|
||||||
|
},
|
||||||
|
"smmap2": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:0555a7bf4df71d1ef4218e4807bbf9b201f910174e6e08af2e138d4e517b4dde",
|
||||||
|
"sha256:29a9ffa0497e7f2be94ca0ed1ca1aa3cd4cf25a1f6b4f5f87f74b46ed91d609a"
|
||||||
|
],
|
||||||
|
"version": "==2.0.5"
|
||||||
|
},
|
||||||
|
"stevedore": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:7be098ff53d87f23d798a7ce7ae5c31f094f3deb92ba18059b1aeb1ca9fec0a0",
|
||||||
|
"sha256:7d1ce610a87d26f53c087da61f06f9b7f7e552efad2a7f6d2322632b5f932ea2"
|
||||||
|
],
|
||||||
|
"version": "==1.30.1"
|
||||||
|
},
|
||||||
|
"toml": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c",
|
||||||
|
"sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e"
|
||||||
|
],
|
||||||
|
"version": "==0.10.0"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
82
main.py
82
main.py
|
|
@ -1,12 +1,17 @@
|
||||||
# TODO
|
# TODO
|
||||||
# send an email to an email address defined by env['admin-email'] if resources are finished
|
# 1. on startup check if there is any VM with status REQUESTED_NEW already
|
||||||
# v3) Introduce a status endpoint of the scheduler - maybe expose a prometheus compatible output
|
# 2. send an email to an email address defined by env['admin-email']
|
||||||
|
# if resources are finished
|
||||||
|
# 3. v3) Introduce a status endpoint of the scheduler -
|
||||||
|
# maybe expose a prometheus compatible output
|
||||||
|
|
||||||
import etcd3
|
import etcd3
|
||||||
import json
|
import json
|
||||||
|
import argparse
|
||||||
|
|
||||||
from decouple import config
|
from decouple import config
|
||||||
from collections import Counter
|
from collections import Counter
|
||||||
|
from functools import reduce
|
||||||
|
|
||||||
|
|
||||||
class VmPool(object):
|
class VmPool(object):
|
||||||
|
|
@ -15,7 +20,9 @@ class VmPool(object):
|
||||||
self.vms = []
|
self.vms = []
|
||||||
|
|
||||||
_vms = self.client.get_prefix(vm_prefix)
|
_vms = self.client.get_prefix(vm_prefix)
|
||||||
self.vms = [(vm[1].key.decode("utf-8"), json.loads(vm[0])) for vm in _vms]
|
self.vms = [
|
||||||
|
(vm[1].key.decode("utf-8"), json.loads(vm[0])) for vm in _vms
|
||||||
|
]
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def by_host(vms, host):
|
def by_host(vms, host):
|
||||||
|
|
@ -30,19 +37,15 @@ class VmPool(object):
|
||||||
return list(filter(lambda x: x[1]["status"] != status, vms))
|
return list(filter(lambda x: x[1]["status"] != status, vms))
|
||||||
|
|
||||||
|
|
||||||
def accumulated_specs(vms):
|
def accumulated_specs(vms_specs):
|
||||||
"""Accumulate specs of all :param vms:"""
|
if vms_specs == []:
|
||||||
specs = Counter()
|
return {}
|
||||||
for vm in vms:
|
return reduce((lambda x, y: Counter(x) + Counter(y)), vms_specs)
|
||||||
_, _value = vm
|
|
||||||
vm_specs = _value["specs"]
|
|
||||||
specs += Counter(vm_specs)
|
|
||||||
return specs
|
|
||||||
|
|
||||||
|
|
||||||
def remaining_resources(host_specs, vms):
|
def remaining_resources(host_specs, vms_specs):
|
||||||
"""Return remaining resources host_specs - vms"""
|
"""Return remaining resources host_specs - vms"""
|
||||||
vms_specs = Counter(vms)
|
vms_specs = Counter(vms_specs)
|
||||||
remaining = Counter(host_specs)
|
remaining = Counter(host_specs)
|
||||||
remaining.subtract(vms_specs)
|
remaining.subtract(vms_specs)
|
||||||
|
|
||||||
|
|
@ -51,10 +54,13 @@ def remaining_resources(host_specs, vms):
|
||||||
|
|
||||||
def get_suitable_host(etcd_client, vm_prefix, host_prefix, vm_specs):
|
def get_suitable_host(etcd_client, vm_prefix, host_prefix, vm_specs):
|
||||||
vm_pool = VmPool(etcd_client, vm_prefix)
|
vm_pool = VmPool(etcd_client, vm_prefix)
|
||||||
hosts = client.get_prefix(host_prefix)
|
hosts = etcd_client.get_prefix(host_prefix)
|
||||||
|
|
||||||
for host in hosts:
|
for host in hosts:
|
||||||
_host_name, host_specs = host[1].key.decode("utf-8"), json.loads(host[0])
|
_host_name, host_specs = (
|
||||||
|
host[1].key.decode("utf-8"),
|
||||||
|
json.loads(host[0]),
|
||||||
|
)
|
||||||
|
|
||||||
# Get All Virtual Machines
|
# Get All Virtual Machines
|
||||||
vms = vm_pool.vms
|
vms = vm_pool.vms
|
||||||
|
|
@ -65,29 +71,37 @@ def get_suitable_host(etcd_client, vm_prefix, host_prefix, vm_specs):
|
||||||
# Filter them by status
|
# Filter them by status
|
||||||
vms = VmPool.except_status(vms, "REQUESTED_NEW")
|
vms = VmPool.except_status(vms, "REQUESTED_NEW")
|
||||||
|
|
||||||
# Accumulate all of their combined specs
|
running_vms_specs = [vm[1]["specs"] for vm in vms]
|
||||||
vms_accumulated_specs = accumulated_specs(vms)
|
|
||||||
|
|
||||||
# Find out remaining resources after host_specs - already running vm_specs
|
# Accumulate all of their combined specs
|
||||||
remaining = remaining_resources(host_specs, vms_accumulated_specs)
|
running_vms_accumulated_specs = accumulated_specs(running_vms_specs)
|
||||||
|
|
||||||
|
# Find out remaining resources after
|
||||||
|
# host_specs - already running vm_specs
|
||||||
|
remaining = remaining_resources(
|
||||||
|
host_specs, running_vms_accumulated_specs
|
||||||
|
)
|
||||||
|
|
||||||
# Find out remaining - new_vm_specs
|
# Find out remaining - new_vm_specs
|
||||||
remaining = remaining_resources(remaining, vm_specs)
|
remaining = remaining_resources(remaining, vm_specs)
|
||||||
|
|
||||||
# if remaining resources >= 0 return this host_name
|
# if remaining resources >= 0 return this host_name
|
||||||
if all(map(lambda x: True if remaining[x] >= 0 else False, remaining)):
|
if all(
|
||||||
|
map(lambda x: True if remaining[x] >= 0 else False, remaining)
|
||||||
|
):
|
||||||
return _host_name
|
return _host_name
|
||||||
|
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
VM_PREFIX = "/v1/vm/"
|
def main(vm_prefix, host_prefix):
|
||||||
HOST_PREFIX = "/v1/host/"
|
|
||||||
|
|
||||||
client = etcd3.client(host=config("ETCD_HOST"), port=int(config("ETCD_PORT")))
|
client = etcd3.client(
|
||||||
|
host=config("ETCD_HOST"), port=int(config("ETCD_PORT"))
|
||||||
|
)
|
||||||
|
|
||||||
|
events_iterator, _ = client.watch_prefix(vm_prefix)
|
||||||
|
|
||||||
while True:
|
|
||||||
events_iterator, cancel = client.watch_prefix(VM_PREFIX)
|
|
||||||
for event in events_iterator:
|
for event in events_iterator:
|
||||||
key = event.key
|
key = event.key
|
||||||
value = event.value
|
value = event.value
|
||||||
|
|
@ -98,8 +112,9 @@ while True:
|
||||||
print(key, value)
|
print(key, value)
|
||||||
|
|
||||||
if value["status"] == "REQUESTED_NEW":
|
if value["status"] == "REQUESTED_NEW":
|
||||||
host_name = get_suitable_host(client, VM_PREFIX, HOST_PREFIX, value["specs"])
|
host_name = get_suitable_host(
|
||||||
print("hostname", host_name)
|
client, vm_prefix, host_prefix, value["specs"]
|
||||||
|
)
|
||||||
if host_name:
|
if host_name:
|
||||||
value["status"] = "SCHEDULED_DEPLOY"
|
value["status"] = "SCHEDULED_DEPLOY"
|
||||||
value["hostname"] = host_name
|
value["hostname"] = host_name
|
||||||
|
|
@ -107,3 +122,16 @@ while True:
|
||||||
else:
|
else:
|
||||||
# email admin
|
# email admin
|
||||||
print("No Resource Left. Emailing admin....")
|
print("No Resource Left. Emailing admin....")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
argparser = argparse.ArgumentParser()
|
||||||
|
argparser.add_argument(
|
||||||
|
"--vm_prefix", required=False, default=config("VM_PREFIX")
|
||||||
|
)
|
||||||
|
argparser.add_argument(
|
||||||
|
"--host_prefix", required=False, default=config("HOST_PREFIX")
|
||||||
|
)
|
||||||
|
args = argparser.parse_args()
|
||||||
|
|
||||||
|
main(args.vm_prefix, args.host_prefix)
|
||||||
|
|
|
||||||
156
tests/test_basics.py
Normal file
156
tests/test_basics.py
Normal file
|
|
@ -0,0 +1,156 @@
|
||||||
|
import unittest
|
||||||
|
import sys
|
||||||
|
import etcd3
|
||||||
|
import json
|
||||||
|
|
||||||
|
sys.path.insert(0, "../")
|
||||||
|
from main import accumulated_specs, remaining_resources, VmPool
|
||||||
|
|
||||||
|
|
||||||
|
class TestFunctions(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.client = etcd3.client()
|
||||||
|
self.host_prefix = "/v1/host"
|
||||||
|
self.vm_prefix = "/v1/vm"
|
||||||
|
|
||||||
|
# These deletion could also be in
|
||||||
|
# tearDown() but it is more appropriate here
|
||||||
|
# as it enable us to check the ETCD store
|
||||||
|
# even after test is run
|
||||||
|
self.client.delete_prefix(self.host_prefix)
|
||||||
|
self.client.delete_prefix(self.vm_prefix)
|
||||||
|
|
||||||
|
self.create_hosts()
|
||||||
|
self.create_vms()
|
||||||
|
|
||||||
|
def create_hosts(self):
|
||||||
|
host1 = """{
|
||||||
|
"cpu": 32,
|
||||||
|
"ram": 128,
|
||||||
|
"hdd": 1024,
|
||||||
|
"sdd": 0
|
||||||
|
}"""
|
||||||
|
|
||||||
|
host2 = """{
|
||||||
|
"cpu": 16,
|
||||||
|
"ram": 64,
|
||||||
|
"hdd": 512,
|
||||||
|
"sdd": 0
|
||||||
|
}"""
|
||||||
|
|
||||||
|
host3 = """{
|
||||||
|
"cpu": 16,
|
||||||
|
"ram": 32,
|
||||||
|
"hdd": 256,
|
||||||
|
"sdd": 256
|
||||||
|
}"""
|
||||||
|
with self.client.lock("lock"):
|
||||||
|
self.client.put(f"{self.host_prefix}/1", host1)
|
||||||
|
self.client.put(f"{self.host_prefix}/2", host2)
|
||||||
|
self.client.put(f"{self.host_prefix}/3", host3)
|
||||||
|
|
||||||
|
def create_vms(self):
|
||||||
|
vm1 = json.dumps(
|
||||||
|
{
|
||||||
|
"owner": "meow",
|
||||||
|
"specs": {"cpu": 4, "ram": 8, "hdd": 100, "sdd": 256},
|
||||||
|
"hostname": "",
|
||||||
|
"status": "REQUESTED_NEW",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
vm2 = json.dumps(
|
||||||
|
{
|
||||||
|
"owner": "meow",
|
||||||
|
"specs": {"cpu": 16, "ram": 64, "hdd": 512, "sdd": 0},
|
||||||
|
"hostname": "",
|
||||||
|
"status": "REQUESTED_NEW",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
vm3 = json.dumps(
|
||||||
|
{
|
||||||
|
"owner": "meow",
|
||||||
|
"specs": {"cpu": 16, "ram": 32, "hdd": 128, "sdd": 0},
|
||||||
|
"hostname": "",
|
||||||
|
"status": "REQUESTED_NEW",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
vm4 = json.dumps(
|
||||||
|
{
|
||||||
|
"owner": "meow",
|
||||||
|
"specs": {"cpu": 16, "ram": 64, "hdd": 512, "sdd": 0},
|
||||||
|
"hostname": "",
|
||||||
|
"status": "REQUESTED_NEW",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
vm5 = json.dumps(
|
||||||
|
{
|
||||||
|
"owner": "meow",
|
||||||
|
"specs": {"cpu": 2, "ram": 2, "hdd": 10, "sdd": 0},
|
||||||
|
"hostname": "",
|
||||||
|
"status": "REQUESTED_NEW",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
vm6 = json.dumps(
|
||||||
|
{
|
||||||
|
"owner": "meow",
|
||||||
|
"specs": {"cpu": 10, "ram": 22, "hdd": 146, "sdd": 0},
|
||||||
|
"hostname": "",
|
||||||
|
"status": "REQUESTED_NEW",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
vm7 = json.dumps(
|
||||||
|
{
|
||||||
|
"owner": "meow",
|
||||||
|
"specs": {"cpu": 10, "ram": 22, "hdd": 146, "sdd": 0},
|
||||||
|
"hostname": "",
|
||||||
|
"status": "REQUESTED_NEW",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
self.client.put(f"{self.vm_prefix}/1", vm1)
|
||||||
|
self.client.put(f"{self.vm_prefix}/2", vm2)
|
||||||
|
self.client.put(f"{self.vm_prefix}/3", vm3)
|
||||||
|
self.client.put(f"{self.vm_prefix}/4", vm4)
|
||||||
|
self.client.put(f"{self.vm_prefix}/5", vm5)
|
||||||
|
self.client.put(f"{self.vm_prefix}/6", vm6)
|
||||||
|
self.client.put(f"{self.vm_prefix}/7", vm7)
|
||||||
|
|
||||||
|
def test_accumulated_specs(self):
|
||||||
|
vms = [
|
||||||
|
{"ssd": 10, "cpu": 4, "ram": 8},
|
||||||
|
{"hdd": 10, "cpu": 4, "ram": 8},
|
||||||
|
{"cpu": 8, "ram": 32},
|
||||||
|
]
|
||||||
|
self.assertEqual(
|
||||||
|
accumulated_specs(vms),
|
||||||
|
{"ssd": 10, "cpu": 16, "ram": 48, "hdd": 10},
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_remaining_resources(self):
|
||||||
|
host_specs = {"ssd": 10, "cpu": 16, "ram": 48, "hdd": 10}
|
||||||
|
vms_specs = {"ssd": 10, "cpu": 32, "ram": 12, "hdd": 0}
|
||||||
|
resultant_specs = {"ssd": 0, "cpu": -16, "ram": 36, "hdd": 10}
|
||||||
|
self.assertEqual(
|
||||||
|
remaining_resources(host_specs, vms_specs), resultant_specs
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_vmpool(self):
|
||||||
|
vm_pool = VmPool(self.client, self.vm_prefix)
|
||||||
|
print(self.client.get(f"{self.vm_prefix}/1")[1].key)
|
||||||
|
self.assertEqual(
|
||||||
|
vm_pool.by_host(vm_pool.vms, f"{self.host_prefix}/1"),
|
||||||
|
[
|
||||||
|
(
|
||||||
|
f"{self.vm_prefix}/1",
|
||||||
|
{
|
||||||
|
"owner": "meow",
|
||||||
|
"specs": {"cpu": 4, "ram": 8, "hdd": 100, "sdd": 256},
|
||||||
|
"hostname": f"{self.host_prefix}/1",
|
||||||
|
"status": "SCHEDULED_DEPLOY",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
Loading…
Add table
Reference in a new issue