new tests added, dead host detection/mitigation, fix bug in specs difference computing code

.gitignore

@@ -3,3 +3,4 @@
 .vscode
 __pycache__/
 venv/
+log.txt

Pipfile

@@ -11,6 +11,8 @@ black = "==19.3b0"
 [packages]
 etcd3 = "*"
 python-decouple = "*"
+pytest = "*"
+coverage = "*"
 
 [requires]
 python_version = "3.7"

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "38090ff76faaefba3892389e2d49d44cdb57752738163a27c99193e960d550e1"
+            "sha256": "8a70e21524bd2bfc041ea7520f90d4a0e981a6c888623f62b76c9df05e9028e1"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -16,6 +16,57 @@
         ]
     },
     "default": {
+        "atomicwrites": {
+            "hashes": [
+                "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4",
+                "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6"
+            ],
+            "version": "==1.3.0"
+        },
+        "attrs": {
+            "hashes": [
+                "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79",
+                "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399"
+            ],
+            "version": "==19.1.0"
+        },
+        "coverage": {
+            "hashes": [
+                "sha256:3684fabf6b87a369017756b551cef29e505cb155ddb892a7a29277b978da88b9",
+                "sha256:39e088da9b284f1bd17c750ac672103779f7954ce6125fd4382134ac8d152d74",
+                "sha256:3c205bc11cc4fcc57b761c2da73b9b72a59f8d5ca89979afb0c1c6f9e53c7390",
+                "sha256:465ce53a8c0f3a7950dfb836438442f833cf6663d407f37d8c52fe7b6e56d7e8",
+                "sha256:48020e343fc40f72a442c8a1334284620f81295256a6b6ca6d8aa1350c763bbe",
+                "sha256:5296fc86ab612ec12394565c500b412a43b328b3907c0d14358950d06fd83baf",
+                "sha256:5f61bed2f7d9b6a9ab935150a6b23d7f84b8055524e7be7715b6513f3328138e",
+                "sha256:68a43a9f9f83693ce0414d17e019daee7ab3f7113a70c79a3dd4c2f704e4d741",
+                "sha256:6b8033d47fe22506856fe450470ccb1d8ba1ffb8463494a15cfc96392a288c09",
+                "sha256:7ad7536066b28863e5835e8cfeaa794b7fe352d99a8cded9f43d1161be8e9fbd",
+                "sha256:7bacb89ccf4bedb30b277e96e4cc68cd1369ca6841bde7b005191b54d3dd1034",
+                "sha256:839dc7c36501254e14331bcb98b27002aa415e4af7ea039d9009409b9d2d5420",
+                "sha256:8f9a95b66969cdea53ec992ecea5406c5bd99c9221f539bca1e8406b200ae98c",
+                "sha256:932c03d2d565f75961ba1d3cec41ddde00e162c5b46d03f7423edcb807734eab",
+                "sha256:988529edadc49039d205e0aa6ce049c5ccda4acb2d6c3c5c550c17e8c02c05ba",
+                "sha256:998d7e73548fe395eeb294495a04d38942edb66d1fa61eb70418871bc621227e",
+                "sha256:9de60893fb447d1e797f6bf08fdf0dbcda0c1e34c1b06c92bd3a363c0ea8c609",
+                "sha256:9e80d45d0c7fcee54e22771db7f1b0b126fb4a6c0a2e5afa72f66827207ff2f2",
+                "sha256:a545a3dfe5082dc8e8c3eb7f8a2cf4f2870902ff1860bd99b6198cfd1f9d1f49",
+                "sha256:a5d8f29e5ec661143621a8f4de51adfb300d7a476224156a39a392254f70687b",
+                "sha256:aca06bfba4759bbdb09bf52ebb15ae20268ee1f6747417837926fae990ebc41d",
+                "sha256:bb23b7a6fd666e551a3094ab896a57809e010059540ad20acbeec03a154224ce",
+                "sha256:bfd1d0ae7e292105f29d7deaa9d8f2916ed8553ab9d5f39ec65bcf5deadff3f9",
+                "sha256:c62ca0a38958f541a73cf86acdab020c2091631c137bd359c4f5bddde7b75fd4",
+                "sha256:c709d8bda72cf4cd348ccec2a4881f2c5848fd72903c185f363d361b2737f773",
+                "sha256:c968a6aa7e0b56ecbd28531ddf439c2ec103610d3e2bf3b75b813304f8cb7723",
+                "sha256:df785d8cb80539d0b55fd47183264b7002077859028dfe3070cf6359bf8b2d9c",
+                "sha256:f406628ca51e0ae90ae76ea8398677a921b36f0bd71aab2099dfed08abd0322f",
+                "sha256:f46087bbd95ebae244a0eda01a618aff11ec7a069b15a3ef8f6b520db523dcf1",
+                "sha256:f8019c5279eb32360ca03e9fac40a12667715546eed5c5eb59eb381f2f501260",
+                "sha256:fc5f4d209733750afd2714e9109816a29500718b32dd9a5db01c0cb3a019b96a"
+            ],
+            "index": "pypi",
+            "version": "==4.5.3"
+        },
         "etcd3": {
             "hashes": [
                 "sha256:25a524b9f032c6631ff0097532907dea81243eaa63c3744510fd1598cc4e0e87"
@@ -25,63 +76,113 @@
         },
         "grpcio": {
             "hashes": [
-                "sha256:0232add03144dd3cf9b660e2718244cb8e175370dca4d3855cb4e489a7811b53",
+                "sha256:03b78b4e7dcdfe3e257bb528cc93923f9cbbab6d5babf15a60d21e9a4a70b1a2",
-                "sha256:0f20e6dcb1b8662cdca033bb97c0a8116a5343e3ebc7f71c5fe7f89039978350",
+                "sha256:1ce0ccfbdfe84387dbcbf44adb4ae16ec7ae70e166ffab478993eb1ea1cba3ce",
-                "sha256:10b07a623d33d4966f45c85d410bc6a79c5ac6341f06c3beda6c22be12cbfe07",
+                "sha256:22e167a9406d73dd19ffe8ed6a485f17e6eac82505be8c108897f15e68badcbb",
-                "sha256:10c0476d5a52d21f402fc073745dc43b87cc8e080a1f49bbff4e1059019310fb",
+                "sha256:31d0aeca8d8ee2301c62c5c340e0889d653b1280d68f9fa203982cb6337b050e",
-                "sha256:289dae0b35c59d191c524e976dd0a6f8c995d2062e72621eb866ad0f4472a635",
+                "sha256:44c7f99ca17ebbcc96fc54ed00b454d8313f1eac28c563098d8b901025aff941",
-                "sha256:2be726f16142d358a0df1e81d583d6820ee561a7856a79cca2fbe49989308be7",
+                "sha256:5471444f53f9db6a1f1f11f5dbc173228881df8446380b6b98f90afb8fd8348e",
-                "sha256:4338d2a81f5b4ca022e085040b3cfce19419a5ce44aa7e6810ac1df05365bed7",
+                "sha256:561bca3b1bde6d6564306eb05848fd155136e9c3a25d2961129b1e2edba22fce",
-                "sha256:4c535b46f20e66bee3097583231977e721acdfcb1671d1490c99b7be8902ce18",
+                "sha256:5bf58e1d2c2f55365c06e8cb5abe067b88ca2e5550fb62009c41df4b54505acf",
-                "sha256:557154aef70a0e979700cc9528bc8b606b668084a29a0d57dbc4b06b078a2f1c",
+                "sha256:6b7163d1e85d76b0815df63fcc310daec02b44532bb433f743142d4febcb181f",
-                "sha256:5bfdd7e6647498f979dc46583723c852d97b25afe995d55aa1c76a5f9816bc1f",
+                "sha256:766d79cddad95f5f6020037fe60ea8b98578afdf0c59d5a60c106c1bdd886303",
-                "sha256:87d8943ae7aa6ca5bbad732867d7f17d2550e4966a0c15b52088e8b579422e47",
+                "sha256:770b7372d5ca68308ff66d7baee53369fa5ce985f84bcb6aa1948c1f2f7b02f2",
-                "sha256:89d8719d8de4d137678f7caa979e1b0a6fd4026f8096ceef8c2d164bbabefaf2",
+                "sha256:7ab178da777fc0f55b6aef5a755f99726e8e4b75e3903954df07b27059b54fcf",
-                "sha256:9c3f4af989ce860710ac1864dc2e867dd87e6cee51a2368df1b253596868e52f",
+                "sha256:8078305e77c2f6649d36b24d8778096413e474d9d7892c6f92cfb589c9d71b2e",
-                "sha256:9da52c3c728883aee429bb7c315049f50b2139f680cd86bb1165418e4f93a982",
+                "sha256:85600b63a386d860eeaa955e9335e18dd0d7e5477e9214825abf2c2884488369",
-                "sha256:9e9736659987beab42d18525ed10d21f80a1ba8389eac03425fbfd5684e6bbf0",
+                "sha256:857d9b939ae128be1c0c792eb885c7ff6a386b9dea899ac4b06f4d90a31f9d87",
-                "sha256:9ebcbb1a054cab362d29d3be571d43d6b9b23302d9fc4b43e5327000da1680a9",
+                "sha256:87a41630c90c179fa5c593400f30a467c498972c702f348d41e19dafeb1d319e",
-                "sha256:a93e08636623e24c939851e2e0c0140b14f524b2980c9cdc4ea52b70a871c7e0",
+                "sha256:8805d486c6128cc0fcc8ecf16c4095d99a8693a541ef851429ab334e028a4a97",
-                "sha256:ac322d86d1a079e0a118d544443ee16f320af0062c191b4754c0c6ec2fc79310",
+                "sha256:8d71b7a89c306a41ccc7741fc9409b14f5b86727455c2a1c0c7cfcb0f784e1f2",
-                "sha256:b1fb101459868f52df6b61e7bb13375e50badf17a160e39fe1d51ae19e53f461",
+                "sha256:9e1b80bd65f8f160880cb4dad7f55697f6d37b2d7f251fc0c2128e811928f369",
-                "sha256:b39aac96cceac624a23d540473835086a3ffa77c91030189988c073488434493",
+                "sha256:9e290c84a145ae2411ee0ec9913c41cd7500e2e7485fe93632434d84ef4fda67",
-                "sha256:b65507bc273c6dbf539175a786a344cc0ac78d50e5584f72c6599733f8a3301f",
+                "sha256:9ec9f88b5bc94bd99372f27cdd53af1c92ba06717380b127733b953cfb181174",
-                "sha256:be5bb6e47417e537c884a2e2ff2e1a8b2c064a998fcfdfcc67528d4e63e7ebaf",
+                "sha256:a0a02a8b4ba6deadf706d5f849539b3685b72b186a3c9ef5d43e8972ed60fb6f",
-                "sha256:c92de6a28a909c4f460dc1bbbcb50d676cf0b1f40224b222761f73fdd851b522",
+                "sha256:a4059c59519f5940e01a071f74ae2a60ea8f6185b03d22a09d40c7959a36b16b",
-                "sha256:c9f5962eb7fa7607b20eb0e4f59ed35829bd600fc0eacb626a6db83229a3e445",
+                "sha256:a6e028c2a6da2ebfa2365a5b32531d311fbfec0e3600fc27e901b64f0ff7e54e",
-                "sha256:d00bdf9c546ed6e649f785c55b05288e8b2dbb6bf2eb74b6c579fa0d591d35bd",
+                "sha256:adcdebf9f8463df4120c427cf6c9aed39258bccd03ed37b6939e7a145d64d6e0",
-                "sha256:da804b1dd8293bd9d61b1e6ea989c887ba042a808a4fbdd80001cfa059aafed2",
+                "sha256:bdec982610259d07156a58f80b8c3e69be7751a9208bc577b059c5193d087fad",
-                "sha256:ead6c5aa3e807345913649c3be395aaca2bbb2d225f18b8f31f37eab225508f6",
+                "sha256:cefc4d4251ffb73feb303d4b7e9d6c367cb60f2db16d259ea28b114045f965aa",
-                "sha256:eb4d81550ce6f826af4ec6e8d98be347fe96291d718bf115c3f254621ae8d98d",
+                "sha256:d4145c8aa6afbac10ad27e408f7ce15992fe89ba5d0b4abca31c0c2729864c03",
-                "sha256:ef6a18ec8fd32ec81748fe720544ea2fb2d2dc50fd6d06739d5e2eb8f0626a1c",
+                "sha256:da76dc5ad719ee99de5ea28a5629ff92172cbb4a70d8a6ae3a5b7a53c7382ce1",
-                "sha256:fad42835656e0b6d3b7ffc900598e776722e30f43b7234a48f2576ca30f31a47",
+                "sha256:dde2452c08ef8b6426ccab6b5b6de9f06d836d9937d6870e68153cbf8cb49348",
-                "sha256:fb98dbfee0d963b49ae5754554028cf62e6bd695f22de16d242ba9d2f0b7339b",
+                "sha256:e3d88091d2539a4868750914a6fe7b9ec50e42b913851fc1b77423b5bd918530",
-                "sha256:fb9cd9bb8d26dc17c2dd715a46bca3a879ec8283879b164e85863110dc6e3b2a"
+                "sha256:f9c67cfe6278499d7f83559dc6322a8bbb108e307817a3d7acbfea807b3603cc"
             ],
-            "version": "==1.21.1"
+            "version": "==1.22.0"
+        },
+        "importlib-metadata": {
+            "hashes": [
+                "sha256:6dfd58dfe281e8d240937776065dd3624ad5469c835248219bd16cf2e12dbeb7",
+                "sha256:cb6ee23b46173539939964df59d3d72c3e0c1b5d54b84f1d8a7e912fe43612db"
+            ],
+            "version": "==0.18"
+        },
+        "more-itertools": {
+            "hashes": [
+                "sha256:3ad685ff8512bf6dc5a8b82ebf73543999b657eded8c11803d9ba6b648986f4d",
+                "sha256:8bb43d1f51ecef60d81854af61a3a880555a14643691cc4b64a6ee269c78f09a"
+            ],
+            "version": "==7.1.0"
+        },
+        "packaging": {
+            "hashes": [
+                "sha256:0c98a5d0be38ed775798ece1b9727178c4469d9c3b4ada66e8e6b7849f8732af",
+                "sha256:9e1cbf8c12b1f1ce0bb5344b8d7ecf66a6f8a6e91bcb0c84593ed6d3ab5c4ab3"
+            ],
+            "version": "==19.0"
+        },
+        "pluggy": {
+            "hashes": [
+                "sha256:0825a152ac059776623854c1543d65a4ad408eb3d33ee114dff91e57ec6ae6fc",
+                "sha256:b9817417e95936bf75d85d3f8767f7df6cdde751fc40aed3bb3074cbcb77757c"
+            ],
+            "version": "==0.12.0"
         },
         "protobuf": {
             "hashes": [
-                "sha256:03f43eac9d5b651f976e91cf46a25b75e5779d98f0f4114b0abfed83376d75f8",
+                "sha256:05c36022fef3c7d3562ac22402965c0c2b9fe8421f459bb377323598996e407f",
-                "sha256:0c94b21e6de01362f91a86b372555d22a60b59708599ca9d5032ae9fdf8e3538",
+                "sha256:139b7eadcca0a861d60b523cb37d9475505e0dfb07972436b15407c2b968d87e",
-                "sha256:2d2a9f30f61f4063fadd7fb68a2510a6939b43c0d6ceeec5c4704f22225da28e",
+                "sha256:15f683006cb77fb849b1f561e509b03dd2b7dcc749086b8dd1831090d0ba4740",
-                "sha256:34a0b05fca061e4abb77dd180209f68d8637115ff319f51e28a6a9382d69853a",
+                "sha256:2ad566b7b7cdd8717c7af1825e19f09e8fef2787b77fcb979588944657679604",
-                "sha256:358710fd0db25372edcf1150fa691f48376a134a6c69ce29f38f185eea7699e6",
+                "sha256:35cfcf97642ef62108e10a9431c77733ec7eaab8e32fe4653de20403429907cb",
-                "sha256:41e47198b94c27ba05a08b4a95160656105745c462af574e4bcb0807164065c0",
+                "sha256:387822859ecdd012fdc25ec879f7f487da6e1d5b1ae6115e227e6be208836f71",
-                "sha256:8c61cc8a76e9d381c665aecc5105fa0f1878cf7db8b5cd17202603bcb386d0fc",
+                "sha256:4df14cbe1e7134afcfdbb9f058949e31c466de27d9b2f7fb4da9e0b67231b538",
-                "sha256:a6eebc4db759e58fdac02efcd3028b811effac881d8a5bad1996e4e8ee6acb47",
+                "sha256:586c4ca37a7146d4822c700059f150ac3445ce0aef6f3ea258640838bb892dc2",
-                "sha256:a9c12f7c98093da0a46ba76ec40ace725daa1ac4038c41e4b1466afb5c45bb01",
+                "sha256:58b11e530e954d29ab3180c48dc558a409f705bf16739fd4e0d3e07924ad7add",
-                "sha256:cb95068492ba0859b8c9e61fa8ba206a83c64e5d0916fb4543700b2e2b214115",
+                "sha256:63c8c98ccb8c95f41c18fb829aeeab21c6249adee4ed75354125bdc44488f30e",
-                "sha256:cd98476ce7bb4dcd6a7b101f5eecdc073dafea19f311e36eb8fba1a349346277",
+                "sha256:72edcbacd0c73eef507d2ff1af99a6c27df18e66a3ff4351e401182e4de62b03",
-                "sha256:ce64cfbea18c535176bdaa10ba740c0fc4c6d998a3f511c17bedb0ae4b3b167c",
+                "sha256:83dc8a561b3b954fd7002c690bb83278b8d1742a1e28abba9aaef28b0c8b437d",
-                "sha256:dcbb59eac73fd454e8f2c5fba9e3d3320fd4707ed6a9d3ea3717924a6f0903ea",
+                "sha256:913171ecc84c2726b86574e40549a0ea619d569657c5a5ff782a3be7d81401a5",
-                "sha256:dd67f34458ae716029e2a71ede998e9092493b62a519236ca52e3c5202096c87",
+                "sha256:aabb7c741d3416671c3e6fe7c52970a226e6a8274417a97d7d795f953fadef36",
-                "sha256:e3c96056eb5b7284a20e256cb0bf783c8f36ad82a4ae5434a7b7cd02384144a7",
+                "sha256:b3452bbda12b1cbe2187d416779de07b2ab4c497d83a050e43c344778763721d",
-                "sha256:f612d584d7a27e2f39e7b17878430a959c1bc09a74ba09db096b468558e5e126",
+                "sha256:c5d5b8d4a9212338297fa1fa44589f69b470c0ba1d38168b432d577176b386a8",
-                "sha256:f6de8a7d6122297b81566e5bd4df37fd5d62bec14f8f90ebff8ede1c9726cd0a",
+                "sha256:d86ee389c2c4fc3cebabb8ce83a8e97b6b3b5dc727b7419c1ccdc7b6e545a233",
-                "sha256:fa529d9261682b24c2aaa683667253175c9acebe0a31105394b221090da75832"
+                "sha256:f2db8c754de788ab8be5e108e1e967c774c0942342b4f8aaaf14063889a6cfdc"
             ],
-            "version": "==3.8.0"
+            "version": "==3.9.0"
+        },
+        "py": {
+            "hashes": [
+                "sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa",
+                "sha256:dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53"
+            ],
+            "version": "==1.8.0"
+        },
+        "pyparsing": {
+            "hashes": [
+                "sha256:1873c03321fc118f4e9746baf201ff990ceb915f433f23b395f5580d1840cb2a",
+                "sha256:9b6323ef4ab914af344ba97510e966d64ba91055d6b9afa6b30799340e89cc03"
+            ],
+            "version": "==2.4.0"
+        },
+        "pytest": {
+            "hashes": [
+                "sha256:6ef6d06de77ce2961156013e9dff62f1b2688aa04d0dc244299fe7d67e09370d",
+                "sha256:a736fed91c12681a7b34617c8fcefe39ea04599ca72c608751c31d89579a3f77"
+            ],
+            "index": "pypi",
+            "version": "==5.0.1"
         },
         "python-decouple": {
             "hashes": [
@@ -103,6 +204,20 @@
                 "sha256:b87c1934daa0b2ccc7db153c37b8bf91d12f165936ade8628e7b962b92dc7705"
             ],
             "version": "==5.0.4"
+        },
+        "wcwidth": {
+            "hashes": [
+                "sha256:3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e",
+                "sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c"
+            ],
+            "version": "==0.1.7"
+        },
+        "zipp": {
+            "hashes": [
+                "sha256:4970c3758f4e89a7857a973b1e2a5d75bcdc47794442f2e2dd4fe8e0466e809a",
+                "sha256:8a5712cfd3bb4248015eb3b0b3c54a5f6ee3f2425963ef2a0125b8bc40aafaec"
+            ],
+            "version": "==0.5.2"
         }
     },
     "develop": {
@@ -122,11 +237,11 @@
         },
         "bandit": {
             "hashes": [
-                "sha256:f89adaff792d1f9b72859784c5f7964c6b5a5f32ca0ca458c9643e02d4fdceac",
+                "sha256:336620e220cf2d3115877685e264477ff9d9abaeb0afe3dc7264f55fa17a3952",
-                "sha256:fa1fee3cb60a3dca89b7a86c0be82af0e830def961728aba9290854fe18c1f90"
+                "sha256:41e75315853507aa145d62a78a2a6c5e3240fe14ee7c601459d0df9418196065"
             ],
             "index": "pypi",
-            "version": "==1.6.1"
+            "version": "==1.6.2"
         },
         "black": {
             "hashes": [
@@ -152,11 +267,11 @@
         },
         "flake8": {
             "hashes": [
-                "sha256:859996073f341f2670741b51ec1e67a01da142831aa1fdc6242dbf88dffbe661",
+                "sha256:19241c1cbc971b9962473e4438a2ca19749a7dd002dd1a946eaba171b4114548",
-                "sha256:a796a115208f5c03b18f332f7c11729812c8c3ded6c46319c59b53efd3819da8"
+                "sha256:8e9dfa3cecb2400b3738a42c54c3043e821682b9c840b0448c0503f781130696"
             ],
             "index": "pypi",
-            "version": "==3.7.7"
+            "version": "==3.7.8"
         },
         "gitdb2": {
             "hashes": [
@@ -181,10 +296,10 @@
         },
         "pbr": {
             "hashes": [
-                "sha256:9181e2a34d80f07a359ff1d0504fad3a47e00e1cf2c475b0aa7dcb030af54c40",
+                "sha256:36ebd78196e8c9588c972f5571230a059ff83783fabbbbedecc07be263ccd7e6",
-                "sha256:94bdc84da376b3dd5061aa0c3b6faffe943ee2e56fa4ff9bd63e1643932f34fc"
+                "sha256:5a03f59455ad54f01a94c15829b8b70065462b7bd8d5d7e983306b59127fc841"
             ],
-            "version": "==5.3.1"
+            "version": "==5.4.0"
         },
         "pycodestyle": {
             "hashes": [

etcd3_wrapper

@@ -1 +1 @@
-Subproject commit cb2a416a17d6789e613ba3b9957917770f4211e1
+Subproject commit 615a0709186e74ee5c9ae78f385fd0c4c4b3834d

main.py

@@ -5,13 +5,23 @@
 #  3. v3) Introduce a status endpoint of the scheduler -
 #     maybe expose a prometheus compatible output
 
-import etcd3
 import json
 import argparse
+import logging
 
 from decouple import config
 from collections import Counter
 from functools import reduce
+from etcd3_wrapper import Etcd3Wrapper
+from datetime import datetime
+
+logging.basicConfig(
+    level=logging.DEBUG,
+    filename="log.txt",
+    filemode="a",
+    format="%(asctime)s: %(levelname)s - %(message)s",
+    datefmt="%d-%b-%y %H:%M:%S",
+)
 
 
 class VmPool(object):
@@ -20,9 +30,7 @@ class VmPool(object):
         self.vms = []
 
         _vms = self.client.get_prefix(vm_prefix)
-        self.vms = [
+        self.vms = [(vm.key, json.loads(vm.value)) for vm in _vms]
-            (vm[1].key.decode("utf-8"), json.loads(vm[0])) for vm in _vms
-        ]
 
     @staticmethod
     def by_host(vms, host):
@@ -38,7 +46,7 @@ class VmPool(object):
 
 
 def accumulated_specs(vms_specs):
-    if vms_specs == []:
+    if not vms_specs:
         return {}
     return reduce((lambda x, y: Counter(x) + Counter(y)), vms_specs)
 
@@ -51,16 +59,13 @@ def remaining_resources(host_specs, vms_specs):
 
     return remaining
 
-
 def get_suitable_host(etcd_client, vm_prefix, host_prefix, vm_specs):
     vm_pool = VmPool(etcd_client, vm_prefix)
-    hosts = etcd_client.get_prefix(host_prefix)
+    hosts = etcd_client.get_prefix(host_prefix, value_in_json=True)
+    hosts = filter(lambda h: h.value["status"] == "ALIVE", hosts)
 
     for host in hosts:
-        _host_name, host_specs = (
+        _host_name, host_value = (host.key, host.value)
-            host[1].key.decode("utf-8"),
-            json.loads(host[0]),
-        )
 
         # Get All Virtual Machines
         vms = vm_pool.vms
@@ -72,19 +77,19 @@ def get_suitable_host(etcd_client, vm_prefix, host_prefix, vm_specs):
         vms = VmPool.except_status(vms, "REQUESTED_NEW")
 
         running_vms_specs = [vm[1]["specs"] for vm in vms]
-
         # Accumulate all of their combined specs
         running_vms_accumulated_specs = accumulated_specs(running_vms_specs)
+        print(running_vms_accumulated_specs)
 
         # Find out remaining resources after
         # host_specs - already running vm_specs
+        print(host_value)
         remaining = remaining_resources(
-            host_specs, running_vms_accumulated_specs
+            host_value["specs"], running_vms_accumulated_specs
         )
-
+        print(remaining)
         # Find out remaining - new_vm_specs
         remaining = remaining_resources(remaining, vm_specs)
-
         # if remaining resources >= 0 return this host_name
         if all(
             map(lambda x: True if remaining[x] >= 0 else False, remaining)
@@ -94,31 +99,78 @@ def get_suitable_host(etcd_client, vm_prefix, host_prefix, vm_specs):
     return None
 
 
-def main(vm_prefix, host_prefix):
+def dead_host_detection(hosts):
+    dead_hosts_keys = []
+    for host in hosts:
+        # Bring out your dead! - Monty Python and the Holy Grail
 
-    client = etcd3.client(
+        if "status" in host.value and "last_heartbeat" in host.value:
+            # Don't count that is already buried
+            if host.value["status"] == "DEAD":
+                continue
+
+            last_heartbeat = datetime.fromisoformat(
+                host.value["last_heartbeat"]
+            )
+            delta = datetime.utcnow() - last_heartbeat
+            if delta.total_seconds() > 60:
+                dead_hosts_keys.append(host.key)
+        else:
+            dead_hosts_keys.append(host.key)
+
+    return dead_hosts_keys
+
+
+def dead_host_mitigation(client: Etcd3Wrapper, dead_hosts_keys):
+    for host_key in dead_hosts_keys:
+        host = client.get(host_key, value_in_json=True)
+        host.value["status"] = "DEAD"
+        host.value["last_heartbeat"] = datetime.utcnow().isoformat()
+        client.put(host.key, host.value, value_in_json=True)
+
+        # Find all vms that were hosted on this dead host
+        all_vms = client.get_prefix(config("VM_PREFIX"), value_in_json=True)
+        vms_hosted_on_dead_host = filter(
+            lambda _vm: _vm.value["hostname"] == host_key, all_vms
+        )
+        for vm in vms_hosted_on_dead_host:
+            vm.value["host"] = ""
+            vm.value["status"] = "REQUESTED_NEW"
+            client.put(vm.key, vm.value, value_in_json=True)
+
+
+def main(vm_prefix, host_prefix):
+    client = Etcd3Wrapper(
         host=config("ETCD_HOST"), port=int(config("ETCD_PORT"))
     )
 
-    events_iterator, _ = client.watch_prefix(vm_prefix)
+    events_iterator = client.watch_prefix(vm_prefix, timeout=10)
 
-    for event in events_iterator:
+    for e in events_iterator:
-        key = event.key
+        try:
-        value = event.value
+            e.value = json.loads(e.value)
-        if not value:
+        except json.JSONDecodeError:
+            logging.error(f"Invalid JSON {e.value}")
             continue
-        value = json.loads(event.value)
 
-        print(key, value)
+        logging.debug(e.key, e.value)
 
-        if value["status"] == "REQUESTED_NEW":
+        e_status = e.value["status"]
+
+        if e_status == "TIMEOUT":
+            logging.info("Timeout")
+            hosts = client.get_prefix(host_prefix, value_in_json=True)
+            dead_hosts = dead_host_detection(hosts)
+            dead_host_mitigation(client, dead_hosts)
+
+        elif e_status == "REQUESTED_NEW":
             host_name = get_suitable_host(
-                client, vm_prefix, host_prefix, value["specs"]
+                client, vm_prefix, host_prefix, e.value["specs"]
             )
             if host_name:
-                value["status"] = "SCHEDULED_DEPLOY"
+                e.value["status"] = "SCHEDULED_DEPLOY"
-                value["hostname"] = host_name
+                e.value["hostname"] = host_name
-                client.put(key, json.dumps(value))
+                client.put(e.key, json.dumps(e.value))
             else:
                 # email admin
                 print("No Resource Left. Emailing admin....")

tests/test_basics.py

@@ -1,53 +1,83 @@
 import unittest
 import sys
-import etcd3
 import json
+import multiprocessing
+import time
 
-sys.path.insert(0, "../")
+from datetime import datetime
-from main import accumulated_specs, remaining_resources, VmPool
+from os.path import dirname
+
+
+BASE_DIR = dirname(dirname(__file__))
+sys.path.insert(0, BASE_DIR)
+
+from main import (
+    accumulated_specs,
+    remaining_resources,
+    VmPool,
+    dead_host_detection,
+    dead_host_mitigation,
+    main,
+)
+
+from etcd3_wrapper import Etcd3Wrapper
 
 
 class TestFunctions(unittest.TestCase):
-    def setUp(self):
+    @classmethod
-        self.client = etcd3.client()
+    def setUpClass(cls):
-        self.host_prefix = "/v1/host"
+        cls.client = Etcd3Wrapper()
-        self.vm_prefix = "/v1/vm"
+        cls.host_prefix = "/test/host"
+        cls.vm_prefix = "/test/vm"
 
         # These deletion could also be in
         # tearDown() but it is more appropriate here
         # as it enable us to check the ETCD store
         # even after test is run
-        self.client.delete_prefix(self.host_prefix)
+        cls.client.client.delete_prefix(cls.host_prefix)
-        self.client.delete_prefix(self.vm_prefix)
+        cls.client.client.delete_prefix(cls.vm_prefix)
+        cls.create_hosts(cls)
+        cls.create_vms(cls)
 
-        self.create_hosts()
+        cls.p = multiprocessing.Process(
-        self.create_vms()
+            target=main, args=[cls.vm_prefix, cls.host_prefix]
+        )
+        cls.p.start()
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.p.terminate()
 
     def create_hosts(self):
-        host1 = """{
+        host1 = {
             "cpu": 32,
             "ram": 128,
             "hdd": 1024,
-            "sdd": 0
+            "sdd": 0,
-        }"""
+            "status": "ALIVE",
-
+            "last_heartbeat": datetime.utcnow().isoformat(),
-        host2 = """{
+        }
+        host2 = {
             "cpu": 16,
             "ram": 64,
             "hdd": 512,
-            "sdd": 0
+            "sdd": 0,
-        }"""
+            "status": "ALIVE",
+            "last_heartbeat": datetime.utcnow().isoformat(),
+        }
 
-        host3 = """{
+        host3 = {
             "cpu": 16,
             "ram": 32,
             "hdd": 256,
-            "sdd": 256
+            "sdd": 256,
-        }"""
+            "status": "ALIVE",
-        with self.client.lock("lock"):
+            "last_heartbeat": datetime.utcnow().isoformat(),
-            self.client.put(f"{self.host_prefix}/1", host1)
+        }
-            self.client.put(f"{self.host_prefix}/2", host2)
+        with self.client.client.lock("lock"):
-            self.client.put(f"{self.host_prefix}/3", host3)
+            self.client.put(f"{self.host_prefix}/1", host1, value_in_json=True)
+            self.client.put(f"{self.host_prefix}/2", host2, value_in_json=True)
+            self.client.put(f"{self.host_prefix}/3", host3, value_in_json=True)
 
     def create_vms(self):
         vm1 = json.dumps(
@@ -121,35 +151,64 @@ class TestFunctions(unittest.TestCase):
             {"cpu": 8, "ram": 32},
         ]
         self.assertEqual(
-            accumulated_specs(vms),
+            accumulated_specs(vms), {"ssd": 10, "cpu": 16, "ram": 48, "hdd": 10}
-            {"ssd": 10, "cpu": 16, "ram": 48, "hdd": 10},
         )
 
     def test_remaining_resources(self):
         host_specs = {"ssd": 10, "cpu": 16, "ram": 48, "hdd": 10}
         vms_specs = {"ssd": 10, "cpu": 32, "ram": 12, "hdd": 0}
         resultant_specs = {"ssd": 0, "cpu": -16, "ram": 36, "hdd": 10}
-        self.assertEqual(
+        self.assertEqual(remaining_resources(host_specs, vms_specs),
-            remaining_resources(host_specs, vms_specs), resultant_specs
+                         resultant_specs)
-        )
 
     def test_vmpool(self):
+        self.p.join(1)
         vm_pool = VmPool(self.client, self.vm_prefix)
-        print(self.client.get(f"{self.vm_prefix}/1")[1].key)
+
-        self.assertEqual(
+        # vm_pool by host
-            vm_pool.by_host(vm_pool.vms, f"{self.host_prefix}/1"),
+        actual = vm_pool.by_host(vm_pool.vms, f"{self.host_prefix}/3")
-            [
+        ground_truth = [
             (
                 f"{self.vm_prefix}/1",
                 {
                     "owner": "meow",
                     "specs": {"cpu": 4, "ram": 8, "hdd": 100, "sdd": 256},
-                        "hostname": f"{self.host_prefix}/1",
+                    "hostname": f"{self.host_prefix}/3",
                     "status": "SCHEDULED_DEPLOY",
                 },
             )
-            ],
+        ]
+        self.assertEqual(actual[0], ground_truth[0])
+
+        # vm_pool by status
+        actual = vm_pool.by_status(vm_pool.vms, "REQUESTED_NEW")
+        ground_truth = [
+            (
+                f"{self.vm_prefix}/7",
+                {
+                    "owner": "meow",
+                    "specs": {"cpu": 10, "ram": 22, "hdd": 146, "sdd": 0},
+                    "hostname": "",
+                    "status": "REQUESTED_NEW",
+                },
             )
+        ]
+        self.assertEqual(actual[0], ground_truth[0])
+
+        # vm_pool by except status
+        actual = vm_pool.except_status(vm_pool.vms, "SCHEDULED_DEPLOY")
+        ground_truth = [
+            (
+                f"{self.vm_prefix}/7",
+                {
+                    "owner": "meow",
+                    "specs": {"cpu": 10, "ram": 22, "hdd": 146, "sdd": 0},
+                    "hostname": "",
+                    "status": "REQUESTED_NEW",
+                },
+            )
+        ]
+        self.assertEqual(actual[0], ground_truth[0])
 
 
 if __name__ == "__main__":

tests/test_dead_host_mechanism.py

@@ -0,0 +1,83 @@
+import unittest
+import sys
+import json
+import multiprocessing
+import time
+
+from datetime import datetime
+from os.path import dirname
+BASE_DIR = dirname(dirname(__file__))
+sys.path.insert(0, BASE_DIR)
+
+from main import (
+    accumulated_specs,
+    remaining_resources,
+    VmPool,
+    dead_host_detection,
+    dead_host_mitigation,
+    main,
+)
+
+from etcd3_wrapper import Etcd3Wrapper
+
+
+class TestDeadHostMechanism(unittest.TestCase):
+    def setUp(self):
+        self.client = Etcd3Wrapper()
+        self.host_prefix = "/test/host"
+        self.vm_prefix = "/test/vm"
+
+        self.client.client.delete_prefix(self.host_prefix)
+        self.client.client.delete_prefix(self.vm_prefix)
+
+        self.create_hosts()
+
+    def create_hosts(self):
+        host1 = {
+            "cpu": 32,
+            "ram": 128,
+            "hdd": 1024,
+            "sdd": 0,
+            "status": "ALIVE",
+            "last_heartbeat": datetime.utcnow().isoformat(),
+        }
+        host2 = {
+            "cpu": 16,
+            "ram": 64,
+            "hdd": 512,
+            "sdd": 0,
+            "status": "ALIVE",
+            "last_heartbeat": datetime(2011, 1, 1).isoformat(),
+        }
+
+        host3 = {"cpu": 16, "ram": 32, "hdd": 256, "sdd": 256}
+        host4 = {
+            "cpu": 16,
+            "ram": 32,
+            "hdd": 256,
+            "sdd": 256,
+            "status": "DEAD",
+            "last_heartbeat": datetime(2011, 1, 1).isoformat(),
+        }
+        with self.client.client.lock("lock"):
+            self.client.put(f"{self.host_prefix}/1", host1, value_in_json=True)
+            self.client.put(f"{self.host_prefix}/2", host2, value_in_json=True)
+            self.client.put(f"{self.host_prefix}/3", host3, value_in_json=True)
+            self.client.put(f"{self.host_prefix}/4", host4, value_in_json=True)
+
+    def test_dead_host_detection(self):
+        hosts = self.client.get_prefix(self.host_prefix, value_in_json=True)
+        deads = dead_host_detection(hosts)
+        self.assertEqual(deads, ["/test/host/2", "/test/host/3"])
+        return deads
+
+    def test_dead_host_mitigation(self):
+        deads = self.test_dead_host_detection()
+        dead_host_mitigation(self.client, deads)
+        hosts = self.client.get_prefix(self.host_prefix, value_in_json=True)
+        deads = dead_host_detection(hosts)
+        self.assertEqual(deads, [])
+
+
+if __name__ == "__main__":
+    unittest.main()