new tests added, dead host detection/mitigation, fix bug in specs difference computing code
This commit is contained in:
parent
5f4b7088ff
commit
f078b9b245
8 changed files with 450 additions and 138 deletions
3
.gitignore
vendored
3
.gitignore
vendored
|
|
@ -2,4 +2,5 @@
|
|||
.idea/
|
||||
.vscode
|
||||
__pycache__/
|
||||
venv/
|
||||
venv/
|
||||
log.txt
|
||||
|
|
|
|||
2
Pipfile
2
Pipfile
|
|
@ -11,6 +11,8 @@ black = "==19.3b0"
|
|||
[packages]
|
||||
etcd3 = "*"
|
||||
python-decouple = "*"
|
||||
pytest = "*"
|
||||
coverage = "*"
|
||||
|
||||
[requires]
|
||||
python_version = "3.7"
|
||||
|
|
|
|||
239
Pipfile.lock
generated
239
Pipfile.lock
generated
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"_meta": {
|
||||
"hash": {
|
||||
"sha256": "38090ff76faaefba3892389e2d49d44cdb57752738163a27c99193e960d550e1"
|
||||
"sha256": "8a70e21524bd2bfc041ea7520f90d4a0e981a6c888623f62b76c9df05e9028e1"
|
||||
},
|
||||
"pipfile-spec": 6,
|
||||
"requires": {
|
||||
|
|
@ -16,6 +16,57 @@
|
|||
]
|
||||
},
|
||||
"default": {
|
||||
"atomicwrites": {
|
||||
"hashes": [
|
||||
"sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4",
|
||||
"sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6"
|
||||
],
|
||||
"version": "==1.3.0"
|
||||
},
|
||||
"attrs": {
|
||||
"hashes": [
|
||||
"sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79",
|
||||
"sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399"
|
||||
],
|
||||
"version": "==19.1.0"
|
||||
},
|
||||
"coverage": {
|
||||
"hashes": [
|
||||
"sha256:3684fabf6b87a369017756b551cef29e505cb155ddb892a7a29277b978da88b9",
|
||||
"sha256:39e088da9b284f1bd17c750ac672103779f7954ce6125fd4382134ac8d152d74",
|
||||
"sha256:3c205bc11cc4fcc57b761c2da73b9b72a59f8d5ca89979afb0c1c6f9e53c7390",
|
||||
"sha256:465ce53a8c0f3a7950dfb836438442f833cf6663d407f37d8c52fe7b6e56d7e8",
|
||||
"sha256:48020e343fc40f72a442c8a1334284620f81295256a6b6ca6d8aa1350c763bbe",
|
||||
"sha256:5296fc86ab612ec12394565c500b412a43b328b3907c0d14358950d06fd83baf",
|
||||
"sha256:5f61bed2f7d9b6a9ab935150a6b23d7f84b8055524e7be7715b6513f3328138e",
|
||||
"sha256:68a43a9f9f83693ce0414d17e019daee7ab3f7113a70c79a3dd4c2f704e4d741",
|
||||
"sha256:6b8033d47fe22506856fe450470ccb1d8ba1ffb8463494a15cfc96392a288c09",
|
||||
"sha256:7ad7536066b28863e5835e8cfeaa794b7fe352d99a8cded9f43d1161be8e9fbd",
|
||||
"sha256:7bacb89ccf4bedb30b277e96e4cc68cd1369ca6841bde7b005191b54d3dd1034",
|
||||
"sha256:839dc7c36501254e14331bcb98b27002aa415e4af7ea039d9009409b9d2d5420",
|
||||
"sha256:8f9a95b66969cdea53ec992ecea5406c5bd99c9221f539bca1e8406b200ae98c",
|
||||
"sha256:932c03d2d565f75961ba1d3cec41ddde00e162c5b46d03f7423edcb807734eab",
|
||||
"sha256:988529edadc49039d205e0aa6ce049c5ccda4acb2d6c3c5c550c17e8c02c05ba",
|
||||
"sha256:998d7e73548fe395eeb294495a04d38942edb66d1fa61eb70418871bc621227e",
|
||||
"sha256:9de60893fb447d1e797f6bf08fdf0dbcda0c1e34c1b06c92bd3a363c0ea8c609",
|
||||
"sha256:9e80d45d0c7fcee54e22771db7f1b0b126fb4a6c0a2e5afa72f66827207ff2f2",
|
||||
"sha256:a545a3dfe5082dc8e8c3eb7f8a2cf4f2870902ff1860bd99b6198cfd1f9d1f49",
|
||||
"sha256:a5d8f29e5ec661143621a8f4de51adfb300d7a476224156a39a392254f70687b",
|
||||
"sha256:aca06bfba4759bbdb09bf52ebb15ae20268ee1f6747417837926fae990ebc41d",
|
||||
"sha256:bb23b7a6fd666e551a3094ab896a57809e010059540ad20acbeec03a154224ce",
|
||||
"sha256:bfd1d0ae7e292105f29d7deaa9d8f2916ed8553ab9d5f39ec65bcf5deadff3f9",
|
||||
"sha256:c62ca0a38958f541a73cf86acdab020c2091631c137bd359c4f5bddde7b75fd4",
|
||||
"sha256:c709d8bda72cf4cd348ccec2a4881f2c5848fd72903c185f363d361b2737f773",
|
||||
"sha256:c968a6aa7e0b56ecbd28531ddf439c2ec103610d3e2bf3b75b813304f8cb7723",
|
||||
"sha256:df785d8cb80539d0b55fd47183264b7002077859028dfe3070cf6359bf8b2d9c",
|
||||
"sha256:f406628ca51e0ae90ae76ea8398677a921b36f0bd71aab2099dfed08abd0322f",
|
||||
"sha256:f46087bbd95ebae244a0eda01a618aff11ec7a069b15a3ef8f6b520db523dcf1",
|
||||
"sha256:f8019c5279eb32360ca03e9fac40a12667715546eed5c5eb59eb381f2f501260",
|
||||
"sha256:fc5f4d209733750afd2714e9109816a29500718b32dd9a5db01c0cb3a019b96a"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==4.5.3"
|
||||
},
|
||||
"etcd3": {
|
||||
"hashes": [
|
||||
"sha256:25a524b9f032c6631ff0097532907dea81243eaa63c3744510fd1598cc4e0e87"
|
||||
|
|
@ -25,63 +76,113 @@
|
|||
},
|
||||
"grpcio": {
|
||||
"hashes": [
|
||||
"sha256:0232add03144dd3cf9b660e2718244cb8e175370dca4d3855cb4e489a7811b53",
|
||||
"sha256:0f20e6dcb1b8662cdca033bb97c0a8116a5343e3ebc7f71c5fe7f89039978350",
|
||||
"sha256:10b07a623d33d4966f45c85d410bc6a79c5ac6341f06c3beda6c22be12cbfe07",
|
||||
"sha256:10c0476d5a52d21f402fc073745dc43b87cc8e080a1f49bbff4e1059019310fb",
|
||||
"sha256:289dae0b35c59d191c524e976dd0a6f8c995d2062e72621eb866ad0f4472a635",
|
||||
"sha256:2be726f16142d358a0df1e81d583d6820ee561a7856a79cca2fbe49989308be7",
|
||||
"sha256:4338d2a81f5b4ca022e085040b3cfce19419a5ce44aa7e6810ac1df05365bed7",
|
||||
"sha256:4c535b46f20e66bee3097583231977e721acdfcb1671d1490c99b7be8902ce18",
|
||||
"sha256:557154aef70a0e979700cc9528bc8b606b668084a29a0d57dbc4b06b078a2f1c",
|
||||
"sha256:5bfdd7e6647498f979dc46583723c852d97b25afe995d55aa1c76a5f9816bc1f",
|
||||
"sha256:87d8943ae7aa6ca5bbad732867d7f17d2550e4966a0c15b52088e8b579422e47",
|
||||
"sha256:89d8719d8de4d137678f7caa979e1b0a6fd4026f8096ceef8c2d164bbabefaf2",
|
||||
"sha256:9c3f4af989ce860710ac1864dc2e867dd87e6cee51a2368df1b253596868e52f",
|
||||
"sha256:9da52c3c728883aee429bb7c315049f50b2139f680cd86bb1165418e4f93a982",
|
||||
"sha256:9e9736659987beab42d18525ed10d21f80a1ba8389eac03425fbfd5684e6bbf0",
|
||||
"sha256:9ebcbb1a054cab362d29d3be571d43d6b9b23302d9fc4b43e5327000da1680a9",
|
||||
"sha256:a93e08636623e24c939851e2e0c0140b14f524b2980c9cdc4ea52b70a871c7e0",
|
||||
"sha256:ac322d86d1a079e0a118d544443ee16f320af0062c191b4754c0c6ec2fc79310",
|
||||
"sha256:b1fb101459868f52df6b61e7bb13375e50badf17a160e39fe1d51ae19e53f461",
|
||||
"sha256:b39aac96cceac624a23d540473835086a3ffa77c91030189988c073488434493",
|
||||
"sha256:b65507bc273c6dbf539175a786a344cc0ac78d50e5584f72c6599733f8a3301f",
|
||||
"sha256:be5bb6e47417e537c884a2e2ff2e1a8b2c064a998fcfdfcc67528d4e63e7ebaf",
|
||||
"sha256:c92de6a28a909c4f460dc1bbbcb50d676cf0b1f40224b222761f73fdd851b522",
|
||||
"sha256:c9f5962eb7fa7607b20eb0e4f59ed35829bd600fc0eacb626a6db83229a3e445",
|
||||
"sha256:d00bdf9c546ed6e649f785c55b05288e8b2dbb6bf2eb74b6c579fa0d591d35bd",
|
||||
"sha256:da804b1dd8293bd9d61b1e6ea989c887ba042a808a4fbdd80001cfa059aafed2",
|
||||
"sha256:ead6c5aa3e807345913649c3be395aaca2bbb2d225f18b8f31f37eab225508f6",
|
||||
"sha256:eb4d81550ce6f826af4ec6e8d98be347fe96291d718bf115c3f254621ae8d98d",
|
||||
"sha256:ef6a18ec8fd32ec81748fe720544ea2fb2d2dc50fd6d06739d5e2eb8f0626a1c",
|
||||
"sha256:fad42835656e0b6d3b7ffc900598e776722e30f43b7234a48f2576ca30f31a47",
|
||||
"sha256:fb98dbfee0d963b49ae5754554028cf62e6bd695f22de16d242ba9d2f0b7339b",
|
||||
"sha256:fb9cd9bb8d26dc17c2dd715a46bca3a879ec8283879b164e85863110dc6e3b2a"
|
||||
"sha256:03b78b4e7dcdfe3e257bb528cc93923f9cbbab6d5babf15a60d21e9a4a70b1a2",
|
||||
"sha256:1ce0ccfbdfe84387dbcbf44adb4ae16ec7ae70e166ffab478993eb1ea1cba3ce",
|
||||
"sha256:22e167a9406d73dd19ffe8ed6a485f17e6eac82505be8c108897f15e68badcbb",
|
||||
"sha256:31d0aeca8d8ee2301c62c5c340e0889d653b1280d68f9fa203982cb6337b050e",
|
||||
"sha256:44c7f99ca17ebbcc96fc54ed00b454d8313f1eac28c563098d8b901025aff941",
|
||||
"sha256:5471444f53f9db6a1f1f11f5dbc173228881df8446380b6b98f90afb8fd8348e",
|
||||
"sha256:561bca3b1bde6d6564306eb05848fd155136e9c3a25d2961129b1e2edba22fce",
|
||||
"sha256:5bf58e1d2c2f55365c06e8cb5abe067b88ca2e5550fb62009c41df4b54505acf",
|
||||
"sha256:6b7163d1e85d76b0815df63fcc310daec02b44532bb433f743142d4febcb181f",
|
||||
"sha256:766d79cddad95f5f6020037fe60ea8b98578afdf0c59d5a60c106c1bdd886303",
|
||||
"sha256:770b7372d5ca68308ff66d7baee53369fa5ce985f84bcb6aa1948c1f2f7b02f2",
|
||||
"sha256:7ab178da777fc0f55b6aef5a755f99726e8e4b75e3903954df07b27059b54fcf",
|
||||
"sha256:8078305e77c2f6649d36b24d8778096413e474d9d7892c6f92cfb589c9d71b2e",
|
||||
"sha256:85600b63a386d860eeaa955e9335e18dd0d7e5477e9214825abf2c2884488369",
|
||||
"sha256:857d9b939ae128be1c0c792eb885c7ff6a386b9dea899ac4b06f4d90a31f9d87",
|
||||
"sha256:87a41630c90c179fa5c593400f30a467c498972c702f348d41e19dafeb1d319e",
|
||||
"sha256:8805d486c6128cc0fcc8ecf16c4095d99a8693a541ef851429ab334e028a4a97",
|
||||
"sha256:8d71b7a89c306a41ccc7741fc9409b14f5b86727455c2a1c0c7cfcb0f784e1f2",
|
||||
"sha256:9e1b80bd65f8f160880cb4dad7f55697f6d37b2d7f251fc0c2128e811928f369",
|
||||
"sha256:9e290c84a145ae2411ee0ec9913c41cd7500e2e7485fe93632434d84ef4fda67",
|
||||
"sha256:9ec9f88b5bc94bd99372f27cdd53af1c92ba06717380b127733b953cfb181174",
|
||||
"sha256:a0a02a8b4ba6deadf706d5f849539b3685b72b186a3c9ef5d43e8972ed60fb6f",
|
||||
"sha256:a4059c59519f5940e01a071f74ae2a60ea8f6185b03d22a09d40c7959a36b16b",
|
||||
"sha256:a6e028c2a6da2ebfa2365a5b32531d311fbfec0e3600fc27e901b64f0ff7e54e",
|
||||
"sha256:adcdebf9f8463df4120c427cf6c9aed39258bccd03ed37b6939e7a145d64d6e0",
|
||||
"sha256:bdec982610259d07156a58f80b8c3e69be7751a9208bc577b059c5193d087fad",
|
||||
"sha256:cefc4d4251ffb73feb303d4b7e9d6c367cb60f2db16d259ea28b114045f965aa",
|
||||
"sha256:d4145c8aa6afbac10ad27e408f7ce15992fe89ba5d0b4abca31c0c2729864c03",
|
||||
"sha256:da76dc5ad719ee99de5ea28a5629ff92172cbb4a70d8a6ae3a5b7a53c7382ce1",
|
||||
"sha256:dde2452c08ef8b6426ccab6b5b6de9f06d836d9937d6870e68153cbf8cb49348",
|
||||
"sha256:e3d88091d2539a4868750914a6fe7b9ec50e42b913851fc1b77423b5bd918530",
|
||||
"sha256:f9c67cfe6278499d7f83559dc6322a8bbb108e307817a3d7acbfea807b3603cc"
|
||||
],
|
||||
"version": "==1.21.1"
|
||||
"version": "==1.22.0"
|
||||
},
|
||||
"importlib-metadata": {
|
||||
"hashes": [
|
||||
"sha256:6dfd58dfe281e8d240937776065dd3624ad5469c835248219bd16cf2e12dbeb7",
|
||||
"sha256:cb6ee23b46173539939964df59d3d72c3e0c1b5d54b84f1d8a7e912fe43612db"
|
||||
],
|
||||
"version": "==0.18"
|
||||
},
|
||||
"more-itertools": {
|
||||
"hashes": [
|
||||
"sha256:3ad685ff8512bf6dc5a8b82ebf73543999b657eded8c11803d9ba6b648986f4d",
|
||||
"sha256:8bb43d1f51ecef60d81854af61a3a880555a14643691cc4b64a6ee269c78f09a"
|
||||
],
|
||||
"version": "==7.1.0"
|
||||
},
|
||||
"packaging": {
|
||||
"hashes": [
|
||||
"sha256:0c98a5d0be38ed775798ece1b9727178c4469d9c3b4ada66e8e6b7849f8732af",
|
||||
"sha256:9e1cbf8c12b1f1ce0bb5344b8d7ecf66a6f8a6e91bcb0c84593ed6d3ab5c4ab3"
|
||||
],
|
||||
"version": "==19.0"
|
||||
},
|
||||
"pluggy": {
|
||||
"hashes": [
|
||||
"sha256:0825a152ac059776623854c1543d65a4ad408eb3d33ee114dff91e57ec6ae6fc",
|
||||
"sha256:b9817417e95936bf75d85d3f8767f7df6cdde751fc40aed3bb3074cbcb77757c"
|
||||
],
|
||||
"version": "==0.12.0"
|
||||
},
|
||||
"protobuf": {
|
||||
"hashes": [
|
||||
"sha256:03f43eac9d5b651f976e91cf46a25b75e5779d98f0f4114b0abfed83376d75f8",
|
||||
"sha256:0c94b21e6de01362f91a86b372555d22a60b59708599ca9d5032ae9fdf8e3538",
|
||||
"sha256:2d2a9f30f61f4063fadd7fb68a2510a6939b43c0d6ceeec5c4704f22225da28e",
|
||||
"sha256:34a0b05fca061e4abb77dd180209f68d8637115ff319f51e28a6a9382d69853a",
|
||||
"sha256:358710fd0db25372edcf1150fa691f48376a134a6c69ce29f38f185eea7699e6",
|
||||
"sha256:41e47198b94c27ba05a08b4a95160656105745c462af574e4bcb0807164065c0",
|
||||
"sha256:8c61cc8a76e9d381c665aecc5105fa0f1878cf7db8b5cd17202603bcb386d0fc",
|
||||
"sha256:a6eebc4db759e58fdac02efcd3028b811effac881d8a5bad1996e4e8ee6acb47",
|
||||
"sha256:a9c12f7c98093da0a46ba76ec40ace725daa1ac4038c41e4b1466afb5c45bb01",
|
||||
"sha256:cb95068492ba0859b8c9e61fa8ba206a83c64e5d0916fb4543700b2e2b214115",
|
||||
"sha256:cd98476ce7bb4dcd6a7b101f5eecdc073dafea19f311e36eb8fba1a349346277",
|
||||
"sha256:ce64cfbea18c535176bdaa10ba740c0fc4c6d998a3f511c17bedb0ae4b3b167c",
|
||||
"sha256:dcbb59eac73fd454e8f2c5fba9e3d3320fd4707ed6a9d3ea3717924a6f0903ea",
|
||||
"sha256:dd67f34458ae716029e2a71ede998e9092493b62a519236ca52e3c5202096c87",
|
||||
"sha256:e3c96056eb5b7284a20e256cb0bf783c8f36ad82a4ae5434a7b7cd02384144a7",
|
||||
"sha256:f612d584d7a27e2f39e7b17878430a959c1bc09a74ba09db096b468558e5e126",
|
||||
"sha256:f6de8a7d6122297b81566e5bd4df37fd5d62bec14f8f90ebff8ede1c9726cd0a",
|
||||
"sha256:fa529d9261682b24c2aaa683667253175c9acebe0a31105394b221090da75832"
|
||||
"sha256:05c36022fef3c7d3562ac22402965c0c2b9fe8421f459bb377323598996e407f",
|
||||
"sha256:139b7eadcca0a861d60b523cb37d9475505e0dfb07972436b15407c2b968d87e",
|
||||
"sha256:15f683006cb77fb849b1f561e509b03dd2b7dcc749086b8dd1831090d0ba4740",
|
||||
"sha256:2ad566b7b7cdd8717c7af1825e19f09e8fef2787b77fcb979588944657679604",
|
||||
"sha256:35cfcf97642ef62108e10a9431c77733ec7eaab8e32fe4653de20403429907cb",
|
||||
"sha256:387822859ecdd012fdc25ec879f7f487da6e1d5b1ae6115e227e6be208836f71",
|
||||
"sha256:4df14cbe1e7134afcfdbb9f058949e31c466de27d9b2f7fb4da9e0b67231b538",
|
||||
"sha256:586c4ca37a7146d4822c700059f150ac3445ce0aef6f3ea258640838bb892dc2",
|
||||
"sha256:58b11e530e954d29ab3180c48dc558a409f705bf16739fd4e0d3e07924ad7add",
|
||||
"sha256:63c8c98ccb8c95f41c18fb829aeeab21c6249adee4ed75354125bdc44488f30e",
|
||||
"sha256:72edcbacd0c73eef507d2ff1af99a6c27df18e66a3ff4351e401182e4de62b03",
|
||||
"sha256:83dc8a561b3b954fd7002c690bb83278b8d1742a1e28abba9aaef28b0c8b437d",
|
||||
"sha256:913171ecc84c2726b86574e40549a0ea619d569657c5a5ff782a3be7d81401a5",
|
||||
"sha256:aabb7c741d3416671c3e6fe7c52970a226e6a8274417a97d7d795f953fadef36",
|
||||
"sha256:b3452bbda12b1cbe2187d416779de07b2ab4c497d83a050e43c344778763721d",
|
||||
"sha256:c5d5b8d4a9212338297fa1fa44589f69b470c0ba1d38168b432d577176b386a8",
|
||||
"sha256:d86ee389c2c4fc3cebabb8ce83a8e97b6b3b5dc727b7419c1ccdc7b6e545a233",
|
||||
"sha256:f2db8c754de788ab8be5e108e1e967c774c0942342b4f8aaaf14063889a6cfdc"
|
||||
],
|
||||
"version": "==3.8.0"
|
||||
"version": "==3.9.0"
|
||||
},
|
||||
"py": {
|
||||
"hashes": [
|
||||
"sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa",
|
||||
"sha256:dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53"
|
||||
],
|
||||
"version": "==1.8.0"
|
||||
},
|
||||
"pyparsing": {
|
||||
"hashes": [
|
||||
"sha256:1873c03321fc118f4e9746baf201ff990ceb915f433f23b395f5580d1840cb2a",
|
||||
"sha256:9b6323ef4ab914af344ba97510e966d64ba91055d6b9afa6b30799340e89cc03"
|
||||
],
|
||||
"version": "==2.4.0"
|
||||
},
|
||||
"pytest": {
|
||||
"hashes": [
|
||||
"sha256:6ef6d06de77ce2961156013e9dff62f1b2688aa04d0dc244299fe7d67e09370d",
|
||||
"sha256:a736fed91c12681a7b34617c8fcefe39ea04599ca72c608751c31d89579a3f77"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==5.0.1"
|
||||
},
|
||||
"python-decouple": {
|
||||
"hashes": [
|
||||
|
|
@ -103,6 +204,20 @@
|
|||
"sha256:b87c1934daa0b2ccc7db153c37b8bf91d12f165936ade8628e7b962b92dc7705"
|
||||
],
|
||||
"version": "==5.0.4"
|
||||
},
|
||||
"wcwidth": {
|
||||
"hashes": [
|
||||
"sha256:3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e",
|
||||
"sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c"
|
||||
],
|
||||
"version": "==0.1.7"
|
||||
},
|
||||
"zipp": {
|
||||
"hashes": [
|
||||
"sha256:4970c3758f4e89a7857a973b1e2a5d75bcdc47794442f2e2dd4fe8e0466e809a",
|
||||
"sha256:8a5712cfd3bb4248015eb3b0b3c54a5f6ee3f2425963ef2a0125b8bc40aafaec"
|
||||
],
|
||||
"version": "==0.5.2"
|
||||
}
|
||||
},
|
||||
"develop": {
|
||||
|
|
@ -122,11 +237,11 @@
|
|||
},
|
||||
"bandit": {
|
||||
"hashes": [
|
||||
"sha256:f89adaff792d1f9b72859784c5f7964c6b5a5f32ca0ca458c9643e02d4fdceac",
|
||||
"sha256:fa1fee3cb60a3dca89b7a86c0be82af0e830def961728aba9290854fe18c1f90"
|
||||
"sha256:336620e220cf2d3115877685e264477ff9d9abaeb0afe3dc7264f55fa17a3952",
|
||||
"sha256:41e75315853507aa145d62a78a2a6c5e3240fe14ee7c601459d0df9418196065"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==1.6.1"
|
||||
"version": "==1.6.2"
|
||||
},
|
||||
"black": {
|
||||
"hashes": [
|
||||
|
|
@ -152,11 +267,11 @@
|
|||
},
|
||||
"flake8": {
|
||||
"hashes": [
|
||||
"sha256:859996073f341f2670741b51ec1e67a01da142831aa1fdc6242dbf88dffbe661",
|
||||
"sha256:a796a115208f5c03b18f332f7c11729812c8c3ded6c46319c59b53efd3819da8"
|
||||
"sha256:19241c1cbc971b9962473e4438a2ca19749a7dd002dd1a946eaba171b4114548",
|
||||
"sha256:8e9dfa3cecb2400b3738a42c54c3043e821682b9c840b0448c0503f781130696"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==3.7.7"
|
||||
"version": "==3.7.8"
|
||||
},
|
||||
"gitdb2": {
|
||||
"hashes": [
|
||||
|
|
@ -181,10 +296,10 @@
|
|||
},
|
||||
"pbr": {
|
||||
"hashes": [
|
||||
"sha256:9181e2a34d80f07a359ff1d0504fad3a47e00e1cf2c475b0aa7dcb030af54c40",
|
||||
"sha256:94bdc84da376b3dd5061aa0c3b6faffe943ee2e56fa4ff9bd63e1643932f34fc"
|
||||
"sha256:36ebd78196e8c9588c972f5571230a059ff83783fabbbbedecc07be263ccd7e6",
|
||||
"sha256:5a03f59455ad54f01a94c15829b8b70065462b7bd8d5d7e983306b59127fc841"
|
||||
],
|
||||
"version": "==5.3.1"
|
||||
"version": "==5.4.0"
|
||||
},
|
||||
"pycodestyle": {
|
||||
"hashes": [
|
||||
|
|
|
|||
0
__init__
Normal file
0
__init__
Normal file
|
|
@ -1 +1 @@
|
|||
Subproject commit cb2a416a17d6789e613ba3b9957917770f4211e1
|
||||
Subproject commit 615a0709186e74ee5c9ae78f385fd0c4c4b3834d
|
||||
110
main.py
110
main.py
|
|
@ -5,13 +5,23 @@
|
|||
# 3. v3) Introduce a status endpoint of the scheduler -
|
||||
# maybe expose a prometheus compatible output
|
||||
|
||||
import etcd3
|
||||
import json
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
from decouple import config
|
||||
from collections import Counter
|
||||
from functools import reduce
|
||||
from etcd3_wrapper import Etcd3Wrapper
|
||||
from datetime import datetime
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.DEBUG,
|
||||
filename="log.txt",
|
||||
filemode="a",
|
||||
format="%(asctime)s: %(levelname)s - %(message)s",
|
||||
datefmt="%d-%b-%y %H:%M:%S",
|
||||
)
|
||||
|
||||
|
||||
class VmPool(object):
|
||||
|
|
@ -20,9 +30,7 @@ class VmPool(object):
|
|||
self.vms = []
|
||||
|
||||
_vms = self.client.get_prefix(vm_prefix)
|
||||
self.vms = [
|
||||
(vm[1].key.decode("utf-8"), json.loads(vm[0])) for vm in _vms
|
||||
]
|
||||
self.vms = [(vm.key, json.loads(vm.value)) for vm in _vms]
|
||||
|
||||
@staticmethod
|
||||
def by_host(vms, host):
|
||||
|
|
@ -38,7 +46,7 @@ class VmPool(object):
|
|||
|
||||
|
||||
def accumulated_specs(vms_specs):
|
||||
if vms_specs == []:
|
||||
if not vms_specs:
|
||||
return {}
|
||||
return reduce((lambda x, y: Counter(x) + Counter(y)), vms_specs)
|
||||
|
||||
|
|
@ -51,16 +59,13 @@ def remaining_resources(host_specs, vms_specs):
|
|||
|
||||
return remaining
|
||||
|
||||
|
||||
def get_suitable_host(etcd_client, vm_prefix, host_prefix, vm_specs):
|
||||
vm_pool = VmPool(etcd_client, vm_prefix)
|
||||
hosts = etcd_client.get_prefix(host_prefix)
|
||||
hosts = etcd_client.get_prefix(host_prefix, value_in_json=True)
|
||||
hosts = filter(lambda h: h.value["status"] == "ALIVE", hosts)
|
||||
|
||||
for host in hosts:
|
||||
_host_name, host_specs = (
|
||||
host[1].key.decode("utf-8"),
|
||||
json.loads(host[0]),
|
||||
)
|
||||
_host_name, host_value = (host.key, host.value)
|
||||
|
||||
# Get All Virtual Machines
|
||||
vms = vm_pool.vms
|
||||
|
|
@ -72,19 +77,19 @@ def get_suitable_host(etcd_client, vm_prefix, host_prefix, vm_specs):
|
|||
vms = VmPool.except_status(vms, "REQUESTED_NEW")
|
||||
|
||||
running_vms_specs = [vm[1]["specs"] for vm in vms]
|
||||
|
||||
# Accumulate all of their combined specs
|
||||
running_vms_accumulated_specs = accumulated_specs(running_vms_specs)
|
||||
print(running_vms_accumulated_specs)
|
||||
|
||||
# Find out remaining resources after
|
||||
# host_specs - already running vm_specs
|
||||
print(host_value)
|
||||
remaining = remaining_resources(
|
||||
host_specs, running_vms_accumulated_specs
|
||||
host_value["specs"], running_vms_accumulated_specs
|
||||
)
|
||||
|
||||
print(remaining)
|
||||
# Find out remaining - new_vm_specs
|
||||
remaining = remaining_resources(remaining, vm_specs)
|
||||
|
||||
# if remaining resources >= 0 return this host_name
|
||||
if all(
|
||||
map(lambda x: True if remaining[x] >= 0 else False, remaining)
|
||||
|
|
@ -94,31 +99,78 @@ def get_suitable_host(etcd_client, vm_prefix, host_prefix, vm_specs):
|
|||
return None
|
||||
|
||||
|
||||
def main(vm_prefix, host_prefix):
|
||||
def dead_host_detection(hosts):
|
||||
dead_hosts_keys = []
|
||||
for host in hosts:
|
||||
# Bring out your dead! - Monty Python and the Holy Grail
|
||||
|
||||
client = etcd3.client(
|
||||
if "status" in host.value and "last_heartbeat" in host.value:
|
||||
# Don't count that is already buried
|
||||
if host.value["status"] == "DEAD":
|
||||
continue
|
||||
|
||||
last_heartbeat = datetime.fromisoformat(
|
||||
host.value["last_heartbeat"]
|
||||
)
|
||||
delta = datetime.utcnow() - last_heartbeat
|
||||
if delta.total_seconds() > 60:
|
||||
dead_hosts_keys.append(host.key)
|
||||
else:
|
||||
dead_hosts_keys.append(host.key)
|
||||
|
||||
return dead_hosts_keys
|
||||
|
||||
|
||||
def dead_host_mitigation(client: Etcd3Wrapper, dead_hosts_keys):
|
||||
for host_key in dead_hosts_keys:
|
||||
host = client.get(host_key, value_in_json=True)
|
||||
host.value["status"] = "DEAD"
|
||||
host.value["last_heartbeat"] = datetime.utcnow().isoformat()
|
||||
client.put(host.key, host.value, value_in_json=True)
|
||||
|
||||
# Find all vms that were hosted on this dead host
|
||||
all_vms = client.get_prefix(config("VM_PREFIX"), value_in_json=True)
|
||||
vms_hosted_on_dead_host = filter(
|
||||
lambda _vm: _vm.value["hostname"] == host_key, all_vms
|
||||
)
|
||||
for vm in vms_hosted_on_dead_host:
|
||||
vm.value["host"] = ""
|
||||
vm.value["status"] = "REQUESTED_NEW"
|
||||
client.put(vm.key, vm.value, value_in_json=True)
|
||||
|
||||
|
||||
def main(vm_prefix, host_prefix):
|
||||
client = Etcd3Wrapper(
|
||||
host=config("ETCD_HOST"), port=int(config("ETCD_PORT"))
|
||||
)
|
||||
|
||||
events_iterator, _ = client.watch_prefix(vm_prefix)
|
||||
events_iterator = client.watch_prefix(vm_prefix, timeout=10)
|
||||
|
||||
for event in events_iterator:
|
||||
key = event.key
|
||||
value = event.value
|
||||
if not value:
|
||||
for e in events_iterator:
|
||||
try:
|
||||
e.value = json.loads(e.value)
|
||||
except json.JSONDecodeError:
|
||||
logging.error(f"Invalid JSON {e.value}")
|
||||
continue
|
||||
value = json.loads(event.value)
|
||||
|
||||
print(key, value)
|
||||
logging.debug(e.key, e.value)
|
||||
|
||||
if value["status"] == "REQUESTED_NEW":
|
||||
e_status = e.value["status"]
|
||||
|
||||
if e_status == "TIMEOUT":
|
||||
logging.info("Timeout")
|
||||
hosts = client.get_prefix(host_prefix, value_in_json=True)
|
||||
dead_hosts = dead_host_detection(hosts)
|
||||
dead_host_mitigation(client, dead_hosts)
|
||||
|
||||
elif e_status == "REQUESTED_NEW":
|
||||
host_name = get_suitable_host(
|
||||
client, vm_prefix, host_prefix, value["specs"]
|
||||
client, vm_prefix, host_prefix, e.value["specs"]
|
||||
)
|
||||
if host_name:
|
||||
value["status"] = "SCHEDULED_DEPLOY"
|
||||
value["hostname"] = host_name
|
||||
client.put(key, json.dumps(value))
|
||||
e.value["status"] = "SCHEDULED_DEPLOY"
|
||||
e.value["hostname"] = host_name
|
||||
client.put(e.key, json.dumps(e.value))
|
||||
else:
|
||||
# email admin
|
||||
print("No Resource Left. Emailing admin....")
|
||||
|
|
|
|||
|
|
@ -1,53 +1,83 @@
|
|||
import unittest
|
||||
import sys
|
||||
import etcd3
|
||||
import json
|
||||
import multiprocessing
|
||||
import time
|
||||
|
||||
sys.path.insert(0, "../")
|
||||
from main import accumulated_specs, remaining_resources, VmPool
|
||||
from datetime import datetime
|
||||
from os.path import dirname
|
||||
|
||||
|
||||
BASE_DIR = dirname(dirname(__file__))
|
||||
sys.path.insert(0, BASE_DIR)
|
||||
|
||||
from main import (
|
||||
accumulated_specs,
|
||||
remaining_resources,
|
||||
VmPool,
|
||||
dead_host_detection,
|
||||
dead_host_mitigation,
|
||||
main,
|
||||
)
|
||||
|
||||
from etcd3_wrapper import Etcd3Wrapper
|
||||
|
||||
|
||||
class TestFunctions(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.client = etcd3.client()
|
||||
self.host_prefix = "/v1/host"
|
||||
self.vm_prefix = "/v1/vm"
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.client = Etcd3Wrapper()
|
||||
cls.host_prefix = "/test/host"
|
||||
cls.vm_prefix = "/test/vm"
|
||||
|
||||
# These deletion could also be in
|
||||
# tearDown() but it is more appropriate here
|
||||
# as it enable us to check the ETCD store
|
||||
# even after test is run
|
||||
self.client.delete_prefix(self.host_prefix)
|
||||
self.client.delete_prefix(self.vm_prefix)
|
||||
cls.client.client.delete_prefix(cls.host_prefix)
|
||||
cls.client.client.delete_prefix(cls.vm_prefix)
|
||||
cls.create_hosts(cls)
|
||||
cls.create_vms(cls)
|
||||
|
||||
self.create_hosts()
|
||||
self.create_vms()
|
||||
cls.p = multiprocessing.Process(
|
||||
target=main, args=[cls.vm_prefix, cls.host_prefix]
|
||||
)
|
||||
cls.p.start()
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
cls.p.terminate()
|
||||
|
||||
def create_hosts(self):
|
||||
host1 = """{
|
||||
host1 = {
|
||||
"cpu": 32,
|
||||
"ram": 128,
|
||||
"hdd": 1024,
|
||||
"sdd": 0
|
||||
}"""
|
||||
|
||||
host2 = """{
|
||||
"sdd": 0,
|
||||
"status": "ALIVE",
|
||||
"last_heartbeat": datetime.utcnow().isoformat(),
|
||||
}
|
||||
host2 = {
|
||||
"cpu": 16,
|
||||
"ram": 64,
|
||||
"hdd": 512,
|
||||
"sdd": 0
|
||||
}"""
|
||||
"sdd": 0,
|
||||
"status": "ALIVE",
|
||||
"last_heartbeat": datetime.utcnow().isoformat(),
|
||||
}
|
||||
|
||||
host3 = """{
|
||||
host3 = {
|
||||
"cpu": 16,
|
||||
"ram": 32,
|
||||
"hdd": 256,
|
||||
"sdd": 256
|
||||
}"""
|
||||
with self.client.lock("lock"):
|
||||
self.client.put(f"{self.host_prefix}/1", host1)
|
||||
self.client.put(f"{self.host_prefix}/2", host2)
|
||||
self.client.put(f"{self.host_prefix}/3", host3)
|
||||
"sdd": 256,
|
||||
"status": "ALIVE",
|
||||
"last_heartbeat": datetime.utcnow().isoformat(),
|
||||
}
|
||||
with self.client.client.lock("lock"):
|
||||
self.client.put(f"{self.host_prefix}/1", host1, value_in_json=True)
|
||||
self.client.put(f"{self.host_prefix}/2", host2, value_in_json=True)
|
||||
self.client.put(f"{self.host_prefix}/3", host3, value_in_json=True)
|
||||
|
||||
def create_vms(self):
|
||||
vm1 = json.dumps(
|
||||
|
|
@ -121,35 +151,64 @@ class TestFunctions(unittest.TestCase):
|
|||
{"cpu": 8, "ram": 32},
|
||||
]
|
||||
self.assertEqual(
|
||||
accumulated_specs(vms),
|
||||
{"ssd": 10, "cpu": 16, "ram": 48, "hdd": 10},
|
||||
accumulated_specs(vms), {"ssd": 10, "cpu": 16, "ram": 48, "hdd": 10}
|
||||
)
|
||||
|
||||
def test_remaining_resources(self):
|
||||
host_specs = {"ssd": 10, "cpu": 16, "ram": 48, "hdd": 10}
|
||||
vms_specs = {"ssd": 10, "cpu": 32, "ram": 12, "hdd": 0}
|
||||
resultant_specs = {"ssd": 0, "cpu": -16, "ram": 36, "hdd": 10}
|
||||
self.assertEqual(
|
||||
remaining_resources(host_specs, vms_specs), resultant_specs
|
||||
)
|
||||
self.assertEqual(remaining_resources(host_specs, vms_specs),
|
||||
resultant_specs)
|
||||
|
||||
def test_vmpool(self):
|
||||
self.p.join(1)
|
||||
vm_pool = VmPool(self.client, self.vm_prefix)
|
||||
print(self.client.get(f"{self.vm_prefix}/1")[1].key)
|
||||
self.assertEqual(
|
||||
vm_pool.by_host(vm_pool.vms, f"{self.host_prefix}/1"),
|
||||
[
|
||||
(
|
||||
f"{self.vm_prefix}/1",
|
||||
{
|
||||
"owner": "meow",
|
||||
"specs": {"cpu": 4, "ram": 8, "hdd": 100, "sdd": 256},
|
||||
"hostname": f"{self.host_prefix}/1",
|
||||
"status": "SCHEDULED_DEPLOY",
|
||||
},
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
# vm_pool by host
|
||||
actual = vm_pool.by_host(vm_pool.vms, f"{self.host_prefix}/3")
|
||||
ground_truth = [
|
||||
(
|
||||
f"{self.vm_prefix}/1",
|
||||
{
|
||||
"owner": "meow",
|
||||
"specs": {"cpu": 4, "ram": 8, "hdd": 100, "sdd": 256},
|
||||
"hostname": f"{self.host_prefix}/3",
|
||||
"status": "SCHEDULED_DEPLOY",
|
||||
},
|
||||
)
|
||||
]
|
||||
self.assertEqual(actual[0], ground_truth[0])
|
||||
|
||||
# vm_pool by status
|
||||
actual = vm_pool.by_status(vm_pool.vms, "REQUESTED_NEW")
|
||||
ground_truth = [
|
||||
(
|
||||
f"{self.vm_prefix}/7",
|
||||
{
|
||||
"owner": "meow",
|
||||
"specs": {"cpu": 10, "ram": 22, "hdd": 146, "sdd": 0},
|
||||
"hostname": "",
|
||||
"status": "REQUESTED_NEW",
|
||||
},
|
||||
)
|
||||
]
|
||||
self.assertEqual(actual[0], ground_truth[0])
|
||||
|
||||
# vm_pool by except status
|
||||
actual = vm_pool.except_status(vm_pool.vms, "SCHEDULED_DEPLOY")
|
||||
ground_truth = [
|
||||
(
|
||||
f"{self.vm_prefix}/7",
|
||||
{
|
||||
"owner": "meow",
|
||||
"specs": {"cpu": 10, "ram": 22, "hdd": 146, "sdd": 0},
|
||||
"hostname": "",
|
||||
"status": "REQUESTED_NEW",
|
||||
},
|
||||
)
|
||||
]
|
||||
self.assertEqual(actual[0], ground_truth[0])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
83
tests/test_dead_host_mechanism.py
Normal file
83
tests/test_dead_host_mechanism.py
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
import unittest
|
||||
import sys
|
||||
import json
|
||||
import multiprocessing
|
||||
import time
|
||||
|
||||
from datetime import datetime
|
||||
from os.path import dirname
|
||||
BASE_DIR = dirname(dirname(__file__))
|
||||
sys.path.insert(0, BASE_DIR)
|
||||
|
||||
from main import (
|
||||
accumulated_specs,
|
||||
remaining_resources,
|
||||
VmPool,
|
||||
dead_host_detection,
|
||||
dead_host_mitigation,
|
||||
main,
|
||||
)
|
||||
|
||||
from etcd3_wrapper import Etcd3Wrapper
|
||||
|
||||
|
||||
class TestDeadHostMechanism(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.client = Etcd3Wrapper()
|
||||
self.host_prefix = "/test/host"
|
||||
self.vm_prefix = "/test/vm"
|
||||
|
||||
self.client.client.delete_prefix(self.host_prefix)
|
||||
self.client.client.delete_prefix(self.vm_prefix)
|
||||
|
||||
self.create_hosts()
|
||||
|
||||
def create_hosts(self):
|
||||
host1 = {
|
||||
"cpu": 32,
|
||||
"ram": 128,
|
||||
"hdd": 1024,
|
||||
"sdd": 0,
|
||||
"status": "ALIVE",
|
||||
"last_heartbeat": datetime.utcnow().isoformat(),
|
||||
}
|
||||
host2 = {
|
||||
"cpu": 16,
|
||||
"ram": 64,
|
||||
"hdd": 512,
|
||||
"sdd": 0,
|
||||
"status": "ALIVE",
|
||||
"last_heartbeat": datetime(2011, 1, 1).isoformat(),
|
||||
}
|
||||
|
||||
host3 = {"cpu": 16, "ram": 32, "hdd": 256, "sdd": 256}
|
||||
host4 = {
|
||||
"cpu": 16,
|
||||
"ram": 32,
|
||||
"hdd": 256,
|
||||
"sdd": 256,
|
||||
"status": "DEAD",
|
||||
"last_heartbeat": datetime(2011, 1, 1).isoformat(),
|
||||
}
|
||||
with self.client.client.lock("lock"):
|
||||
self.client.put(f"{self.host_prefix}/1", host1, value_in_json=True)
|
||||
self.client.put(f"{self.host_prefix}/2", host2, value_in_json=True)
|
||||
self.client.put(f"{self.host_prefix}/3", host3, value_in_json=True)
|
||||
self.client.put(f"{self.host_prefix}/4", host4, value_in_json=True)
|
||||
|
||||
def test_dead_host_detection(self):
|
||||
hosts = self.client.get_prefix(self.host_prefix, value_in_json=True)
|
||||
deads = dead_host_detection(hosts)
|
||||
self.assertEqual(deads, ["/test/host/2", "/test/host/3"])
|
||||
return deads
|
||||
|
||||
def test_dead_host_mitigation(self):
|
||||
deads = self.test_dead_host_detection()
|
||||
dead_host_mitigation(self.client, deads)
|
||||
hosts = self.client.get_prefix(self.host_prefix, value_in_json=True)
|
||||
deads = dead_host_detection(hosts)
|
||||
self.assertEqual(deads, [])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Loading…
Reference in a new issue