diff --git a/.gitignore b/.gitignore index 7a23a3b..817f5fa 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ .idea/ .vscode __pycache__/ -venv/ \ No newline at end of file +venv/ +log.txt diff --git a/Pipfile b/Pipfile index d6bf561..291da6d 100644 --- a/Pipfile +++ b/Pipfile @@ -11,6 +11,8 @@ black = "==19.3b0" [packages] etcd3 = "*" python-decouple = "*" +pytest = "*" +coverage = "*" [requires] python_version = "3.7" diff --git a/Pipfile.lock b/Pipfile.lock index a5707af..627e5b1 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "38090ff76faaefba3892389e2d49d44cdb57752738163a27c99193e960d550e1" + "sha256": "8a70e21524bd2bfc041ea7520f90d4a0e981a6c888623f62b76c9df05e9028e1" }, "pipfile-spec": 6, "requires": { @@ -16,6 +16,57 @@ ] }, "default": { + "atomicwrites": { + "hashes": [ + "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4", + "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6" + ], + "version": "==1.3.0" + }, + "attrs": { + "hashes": [ + "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", + "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399" + ], + "version": "==19.1.0" + }, + "coverage": { + "hashes": [ + "sha256:3684fabf6b87a369017756b551cef29e505cb155ddb892a7a29277b978da88b9", + "sha256:39e088da9b284f1bd17c750ac672103779f7954ce6125fd4382134ac8d152d74", + "sha256:3c205bc11cc4fcc57b761c2da73b9b72a59f8d5ca89979afb0c1c6f9e53c7390", + "sha256:465ce53a8c0f3a7950dfb836438442f833cf6663d407f37d8c52fe7b6e56d7e8", + "sha256:48020e343fc40f72a442c8a1334284620f81295256a6b6ca6d8aa1350c763bbe", + "sha256:5296fc86ab612ec12394565c500b412a43b328b3907c0d14358950d06fd83baf", + "sha256:5f61bed2f7d9b6a9ab935150a6b23d7f84b8055524e7be7715b6513f3328138e", + "sha256:68a43a9f9f83693ce0414d17e019daee7ab3f7113a70c79a3dd4c2f704e4d741", + "sha256:6b8033d47fe22506856fe450470ccb1d8ba1ffb8463494a15cfc96392a288c09", + "sha256:7ad7536066b28863e5835e8cfeaa794b7fe352d99a8cded9f43d1161be8e9fbd", + "sha256:7bacb89ccf4bedb30b277e96e4cc68cd1369ca6841bde7b005191b54d3dd1034", + "sha256:839dc7c36501254e14331bcb98b27002aa415e4af7ea039d9009409b9d2d5420", + "sha256:8f9a95b66969cdea53ec992ecea5406c5bd99c9221f539bca1e8406b200ae98c", + "sha256:932c03d2d565f75961ba1d3cec41ddde00e162c5b46d03f7423edcb807734eab", + "sha256:988529edadc49039d205e0aa6ce049c5ccda4acb2d6c3c5c550c17e8c02c05ba", + "sha256:998d7e73548fe395eeb294495a04d38942edb66d1fa61eb70418871bc621227e", + "sha256:9de60893fb447d1e797f6bf08fdf0dbcda0c1e34c1b06c92bd3a363c0ea8c609", + "sha256:9e80d45d0c7fcee54e22771db7f1b0b126fb4a6c0a2e5afa72f66827207ff2f2", + "sha256:a545a3dfe5082dc8e8c3eb7f8a2cf4f2870902ff1860bd99b6198cfd1f9d1f49", + "sha256:a5d8f29e5ec661143621a8f4de51adfb300d7a476224156a39a392254f70687b", + "sha256:aca06bfba4759bbdb09bf52ebb15ae20268ee1f6747417837926fae990ebc41d", + "sha256:bb23b7a6fd666e551a3094ab896a57809e010059540ad20acbeec03a154224ce", + "sha256:bfd1d0ae7e292105f29d7deaa9d8f2916ed8553ab9d5f39ec65bcf5deadff3f9", + "sha256:c62ca0a38958f541a73cf86acdab020c2091631c137bd359c4f5bddde7b75fd4", + "sha256:c709d8bda72cf4cd348ccec2a4881f2c5848fd72903c185f363d361b2737f773", + "sha256:c968a6aa7e0b56ecbd28531ddf439c2ec103610d3e2bf3b75b813304f8cb7723", + "sha256:df785d8cb80539d0b55fd47183264b7002077859028dfe3070cf6359bf8b2d9c", + "sha256:f406628ca51e0ae90ae76ea8398677a921b36f0bd71aab2099dfed08abd0322f", + "sha256:f46087bbd95ebae244a0eda01a618aff11ec7a069b15a3ef8f6b520db523dcf1", + "sha256:f8019c5279eb32360ca03e9fac40a12667715546eed5c5eb59eb381f2f501260", + "sha256:fc5f4d209733750afd2714e9109816a29500718b32dd9a5db01c0cb3a019b96a" + ], + "index": "pypi", + "version": "==4.5.3" + }, "etcd3": { "hashes": [ "sha256:25a524b9f032c6631ff0097532907dea81243eaa63c3744510fd1598cc4e0e87" @@ -25,63 +76,113 @@ }, "grpcio": { "hashes": [ - "sha256:0232add03144dd3cf9b660e2718244cb8e175370dca4d3855cb4e489a7811b53", - "sha256:0f20e6dcb1b8662cdca033bb97c0a8116a5343e3ebc7f71c5fe7f89039978350", - "sha256:10b07a623d33d4966f45c85d410bc6a79c5ac6341f06c3beda6c22be12cbfe07", - "sha256:10c0476d5a52d21f402fc073745dc43b87cc8e080a1f49bbff4e1059019310fb", - "sha256:289dae0b35c59d191c524e976dd0a6f8c995d2062e72621eb866ad0f4472a635", - "sha256:2be726f16142d358a0df1e81d583d6820ee561a7856a79cca2fbe49989308be7", - "sha256:4338d2a81f5b4ca022e085040b3cfce19419a5ce44aa7e6810ac1df05365bed7", - "sha256:4c535b46f20e66bee3097583231977e721acdfcb1671d1490c99b7be8902ce18", - "sha256:557154aef70a0e979700cc9528bc8b606b668084a29a0d57dbc4b06b078a2f1c", - "sha256:5bfdd7e6647498f979dc46583723c852d97b25afe995d55aa1c76a5f9816bc1f", - "sha256:87d8943ae7aa6ca5bbad732867d7f17d2550e4966a0c15b52088e8b579422e47", - "sha256:89d8719d8de4d137678f7caa979e1b0a6fd4026f8096ceef8c2d164bbabefaf2", - "sha256:9c3f4af989ce860710ac1864dc2e867dd87e6cee51a2368df1b253596868e52f", - "sha256:9da52c3c728883aee429bb7c315049f50b2139f680cd86bb1165418e4f93a982", - "sha256:9e9736659987beab42d18525ed10d21f80a1ba8389eac03425fbfd5684e6bbf0", - "sha256:9ebcbb1a054cab362d29d3be571d43d6b9b23302d9fc4b43e5327000da1680a9", - "sha256:a93e08636623e24c939851e2e0c0140b14f524b2980c9cdc4ea52b70a871c7e0", - "sha256:ac322d86d1a079e0a118d544443ee16f320af0062c191b4754c0c6ec2fc79310", - "sha256:b1fb101459868f52df6b61e7bb13375e50badf17a160e39fe1d51ae19e53f461", - "sha256:b39aac96cceac624a23d540473835086a3ffa77c91030189988c073488434493", - "sha256:b65507bc273c6dbf539175a786a344cc0ac78d50e5584f72c6599733f8a3301f", - "sha256:be5bb6e47417e537c884a2e2ff2e1a8b2c064a998fcfdfcc67528d4e63e7ebaf", - "sha256:c92de6a28a909c4f460dc1bbbcb50d676cf0b1f40224b222761f73fdd851b522", - "sha256:c9f5962eb7fa7607b20eb0e4f59ed35829bd600fc0eacb626a6db83229a3e445", - "sha256:d00bdf9c546ed6e649f785c55b05288e8b2dbb6bf2eb74b6c579fa0d591d35bd", - "sha256:da804b1dd8293bd9d61b1e6ea989c887ba042a808a4fbdd80001cfa059aafed2", - "sha256:ead6c5aa3e807345913649c3be395aaca2bbb2d225f18b8f31f37eab225508f6", - "sha256:eb4d81550ce6f826af4ec6e8d98be347fe96291d718bf115c3f254621ae8d98d", - "sha256:ef6a18ec8fd32ec81748fe720544ea2fb2d2dc50fd6d06739d5e2eb8f0626a1c", - "sha256:fad42835656e0b6d3b7ffc900598e776722e30f43b7234a48f2576ca30f31a47", - "sha256:fb98dbfee0d963b49ae5754554028cf62e6bd695f22de16d242ba9d2f0b7339b", - "sha256:fb9cd9bb8d26dc17c2dd715a46bca3a879ec8283879b164e85863110dc6e3b2a" + "sha256:03b78b4e7dcdfe3e257bb528cc93923f9cbbab6d5babf15a60d21e9a4a70b1a2", + "sha256:1ce0ccfbdfe84387dbcbf44adb4ae16ec7ae70e166ffab478993eb1ea1cba3ce", + "sha256:22e167a9406d73dd19ffe8ed6a485f17e6eac82505be8c108897f15e68badcbb", + "sha256:31d0aeca8d8ee2301c62c5c340e0889d653b1280d68f9fa203982cb6337b050e", + "sha256:44c7f99ca17ebbcc96fc54ed00b454d8313f1eac28c563098d8b901025aff941", + "sha256:5471444f53f9db6a1f1f11f5dbc173228881df8446380b6b98f90afb8fd8348e", + "sha256:561bca3b1bde6d6564306eb05848fd155136e9c3a25d2961129b1e2edba22fce", + "sha256:5bf58e1d2c2f55365c06e8cb5abe067b88ca2e5550fb62009c41df4b54505acf", + "sha256:6b7163d1e85d76b0815df63fcc310daec02b44532bb433f743142d4febcb181f", + "sha256:766d79cddad95f5f6020037fe60ea8b98578afdf0c59d5a60c106c1bdd886303", + "sha256:770b7372d5ca68308ff66d7baee53369fa5ce985f84bcb6aa1948c1f2f7b02f2", + "sha256:7ab178da777fc0f55b6aef5a755f99726e8e4b75e3903954df07b27059b54fcf", + "sha256:8078305e77c2f6649d36b24d8778096413e474d9d7892c6f92cfb589c9d71b2e", + "sha256:85600b63a386d860eeaa955e9335e18dd0d7e5477e9214825abf2c2884488369", + "sha256:857d9b939ae128be1c0c792eb885c7ff6a386b9dea899ac4b06f4d90a31f9d87", + "sha256:87a41630c90c179fa5c593400f30a467c498972c702f348d41e19dafeb1d319e", + "sha256:8805d486c6128cc0fcc8ecf16c4095d99a8693a541ef851429ab334e028a4a97", + "sha256:8d71b7a89c306a41ccc7741fc9409b14f5b86727455c2a1c0c7cfcb0f784e1f2", + "sha256:9e1b80bd65f8f160880cb4dad7f55697f6d37b2d7f251fc0c2128e811928f369", + "sha256:9e290c84a145ae2411ee0ec9913c41cd7500e2e7485fe93632434d84ef4fda67", + "sha256:9ec9f88b5bc94bd99372f27cdd53af1c92ba06717380b127733b953cfb181174", + "sha256:a0a02a8b4ba6deadf706d5f849539b3685b72b186a3c9ef5d43e8972ed60fb6f", + "sha256:a4059c59519f5940e01a071f74ae2a60ea8f6185b03d22a09d40c7959a36b16b", + "sha256:a6e028c2a6da2ebfa2365a5b32531d311fbfec0e3600fc27e901b64f0ff7e54e", + "sha256:adcdebf9f8463df4120c427cf6c9aed39258bccd03ed37b6939e7a145d64d6e0", + "sha256:bdec982610259d07156a58f80b8c3e69be7751a9208bc577b059c5193d087fad", + "sha256:cefc4d4251ffb73feb303d4b7e9d6c367cb60f2db16d259ea28b114045f965aa", + "sha256:d4145c8aa6afbac10ad27e408f7ce15992fe89ba5d0b4abca31c0c2729864c03", + "sha256:da76dc5ad719ee99de5ea28a5629ff92172cbb4a70d8a6ae3a5b7a53c7382ce1", + "sha256:dde2452c08ef8b6426ccab6b5b6de9f06d836d9937d6870e68153cbf8cb49348", + "sha256:e3d88091d2539a4868750914a6fe7b9ec50e42b913851fc1b77423b5bd918530", + "sha256:f9c67cfe6278499d7f83559dc6322a8bbb108e307817a3d7acbfea807b3603cc" ], - "version": "==1.21.1" + "version": "==1.22.0" + }, + "importlib-metadata": { + "hashes": [ + "sha256:6dfd58dfe281e8d240937776065dd3624ad5469c835248219bd16cf2e12dbeb7", + "sha256:cb6ee23b46173539939964df59d3d72c3e0c1b5d54b84f1d8a7e912fe43612db" + ], + "version": "==0.18" + }, + "more-itertools": { + "hashes": [ + "sha256:3ad685ff8512bf6dc5a8b82ebf73543999b657eded8c11803d9ba6b648986f4d", + "sha256:8bb43d1f51ecef60d81854af61a3a880555a14643691cc4b64a6ee269c78f09a" + ], + "version": "==7.1.0" + }, + "packaging": { + "hashes": [ + "sha256:0c98a5d0be38ed775798ece1b9727178c4469d9c3b4ada66e8e6b7849f8732af", + "sha256:9e1cbf8c12b1f1ce0bb5344b8d7ecf66a6f8a6e91bcb0c84593ed6d3ab5c4ab3" + ], + "version": "==19.0" + }, + "pluggy": { + "hashes": [ + "sha256:0825a152ac059776623854c1543d65a4ad408eb3d33ee114dff91e57ec6ae6fc", + "sha256:b9817417e95936bf75d85d3f8767f7df6cdde751fc40aed3bb3074cbcb77757c" + ], + "version": "==0.12.0" }, "protobuf": { "hashes": [ - "sha256:03f43eac9d5b651f976e91cf46a25b75e5779d98f0f4114b0abfed83376d75f8", - "sha256:0c94b21e6de01362f91a86b372555d22a60b59708599ca9d5032ae9fdf8e3538", - "sha256:2d2a9f30f61f4063fadd7fb68a2510a6939b43c0d6ceeec5c4704f22225da28e", - "sha256:34a0b05fca061e4abb77dd180209f68d8637115ff319f51e28a6a9382d69853a", - "sha256:358710fd0db25372edcf1150fa691f48376a134a6c69ce29f38f185eea7699e6", - "sha256:41e47198b94c27ba05a08b4a95160656105745c462af574e4bcb0807164065c0", - "sha256:8c61cc8a76e9d381c665aecc5105fa0f1878cf7db8b5cd17202603bcb386d0fc", - "sha256:a6eebc4db759e58fdac02efcd3028b811effac881d8a5bad1996e4e8ee6acb47", - "sha256:a9c12f7c98093da0a46ba76ec40ace725daa1ac4038c41e4b1466afb5c45bb01", - "sha256:cb95068492ba0859b8c9e61fa8ba206a83c64e5d0916fb4543700b2e2b214115", - "sha256:cd98476ce7bb4dcd6a7b101f5eecdc073dafea19f311e36eb8fba1a349346277", - "sha256:ce64cfbea18c535176bdaa10ba740c0fc4c6d998a3f511c17bedb0ae4b3b167c", - "sha256:dcbb59eac73fd454e8f2c5fba9e3d3320fd4707ed6a9d3ea3717924a6f0903ea", - "sha256:dd67f34458ae716029e2a71ede998e9092493b62a519236ca52e3c5202096c87", - "sha256:e3c96056eb5b7284a20e256cb0bf783c8f36ad82a4ae5434a7b7cd02384144a7", - "sha256:f612d584d7a27e2f39e7b17878430a959c1bc09a74ba09db096b468558e5e126", - "sha256:f6de8a7d6122297b81566e5bd4df37fd5d62bec14f8f90ebff8ede1c9726cd0a", - "sha256:fa529d9261682b24c2aaa683667253175c9acebe0a31105394b221090da75832" + "sha256:05c36022fef3c7d3562ac22402965c0c2b9fe8421f459bb377323598996e407f", + "sha256:139b7eadcca0a861d60b523cb37d9475505e0dfb07972436b15407c2b968d87e", + "sha256:15f683006cb77fb849b1f561e509b03dd2b7dcc749086b8dd1831090d0ba4740", + "sha256:2ad566b7b7cdd8717c7af1825e19f09e8fef2787b77fcb979588944657679604", + "sha256:35cfcf97642ef62108e10a9431c77733ec7eaab8e32fe4653de20403429907cb", + "sha256:387822859ecdd012fdc25ec879f7f487da6e1d5b1ae6115e227e6be208836f71", + "sha256:4df14cbe1e7134afcfdbb9f058949e31c466de27d9b2f7fb4da9e0b67231b538", + "sha256:586c4ca37a7146d4822c700059f150ac3445ce0aef6f3ea258640838bb892dc2", + "sha256:58b11e530e954d29ab3180c48dc558a409f705bf16739fd4e0d3e07924ad7add", + "sha256:63c8c98ccb8c95f41c18fb829aeeab21c6249adee4ed75354125bdc44488f30e", + "sha256:72edcbacd0c73eef507d2ff1af99a6c27df18e66a3ff4351e401182e4de62b03", + "sha256:83dc8a561b3b954fd7002c690bb83278b8d1742a1e28abba9aaef28b0c8b437d", + "sha256:913171ecc84c2726b86574e40549a0ea619d569657c5a5ff782a3be7d81401a5", + "sha256:aabb7c741d3416671c3e6fe7c52970a226e6a8274417a97d7d795f953fadef36", + "sha256:b3452bbda12b1cbe2187d416779de07b2ab4c497d83a050e43c344778763721d", + "sha256:c5d5b8d4a9212338297fa1fa44589f69b470c0ba1d38168b432d577176b386a8", + "sha256:d86ee389c2c4fc3cebabb8ce83a8e97b6b3b5dc727b7419c1ccdc7b6e545a233", + "sha256:f2db8c754de788ab8be5e108e1e967c774c0942342b4f8aaaf14063889a6cfdc" ], - "version": "==3.8.0" + "version": "==3.9.0" + }, + "py": { + "hashes": [ + "sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa", + "sha256:dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53" + ], + "version": "==1.8.0" + }, + "pyparsing": { + "hashes": [ + "sha256:1873c03321fc118f4e9746baf201ff990ceb915f433f23b395f5580d1840cb2a", + "sha256:9b6323ef4ab914af344ba97510e966d64ba91055d6b9afa6b30799340e89cc03" + ], + "version": "==2.4.0" + }, + "pytest": { + "hashes": [ + "sha256:6ef6d06de77ce2961156013e9dff62f1b2688aa04d0dc244299fe7d67e09370d", + "sha256:a736fed91c12681a7b34617c8fcefe39ea04599ca72c608751c31d89579a3f77" + ], + "index": "pypi", + "version": "==5.0.1" }, "python-decouple": { "hashes": [ @@ -103,6 +204,20 @@ "sha256:b87c1934daa0b2ccc7db153c37b8bf91d12f165936ade8628e7b962b92dc7705" ], "version": "==5.0.4" + }, + "wcwidth": { + "hashes": [ + "sha256:3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e", + "sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c" + ], + "version": "==0.1.7" + }, + "zipp": { + "hashes": [ + "sha256:4970c3758f4e89a7857a973b1e2a5d75bcdc47794442f2e2dd4fe8e0466e809a", + "sha256:8a5712cfd3bb4248015eb3b0b3c54a5f6ee3f2425963ef2a0125b8bc40aafaec" + ], + "version": "==0.5.2" } }, "develop": { @@ -122,11 +237,11 @@ }, "bandit": { "hashes": [ - "sha256:f89adaff792d1f9b72859784c5f7964c6b5a5f32ca0ca458c9643e02d4fdceac", - "sha256:fa1fee3cb60a3dca89b7a86c0be82af0e830def961728aba9290854fe18c1f90" + "sha256:336620e220cf2d3115877685e264477ff9d9abaeb0afe3dc7264f55fa17a3952", + "sha256:41e75315853507aa145d62a78a2a6c5e3240fe14ee7c601459d0df9418196065" ], "index": "pypi", - "version": "==1.6.1" + "version": "==1.6.2" }, "black": { "hashes": [ @@ -152,11 +267,11 @@ }, "flake8": { "hashes": [ - "sha256:859996073f341f2670741b51ec1e67a01da142831aa1fdc6242dbf88dffbe661", - "sha256:a796a115208f5c03b18f332f7c11729812c8c3ded6c46319c59b53efd3819da8" + "sha256:19241c1cbc971b9962473e4438a2ca19749a7dd002dd1a946eaba171b4114548", + "sha256:8e9dfa3cecb2400b3738a42c54c3043e821682b9c840b0448c0503f781130696" ], "index": "pypi", - "version": "==3.7.7" + "version": "==3.7.8" }, "gitdb2": { "hashes": [ @@ -181,10 +296,10 @@ }, "pbr": { "hashes": [ - "sha256:9181e2a34d80f07a359ff1d0504fad3a47e00e1cf2c475b0aa7dcb030af54c40", - "sha256:94bdc84da376b3dd5061aa0c3b6faffe943ee2e56fa4ff9bd63e1643932f34fc" + "sha256:36ebd78196e8c9588c972f5571230a059ff83783fabbbbedecc07be263ccd7e6", + "sha256:5a03f59455ad54f01a94c15829b8b70065462b7bd8d5d7e983306b59127fc841" ], - "version": "==5.3.1" + "version": "==5.4.0" }, "pycodestyle": { "hashes": [ diff --git a/__init__ b/__init__ new file mode 100644 index 0000000..e69de29 diff --git a/etcd3_wrapper b/etcd3_wrapper index cb2a416..615a070 160000 --- a/etcd3_wrapper +++ b/etcd3_wrapper @@ -1 +1 @@ -Subproject commit cb2a416a17d6789e613ba3b9957917770f4211e1 +Subproject commit 615a0709186e74ee5c9ae78f385fd0c4c4b3834d diff --git a/main.py b/main.py index 5c1b38e..11db59b 100644 --- a/main.py +++ b/main.py @@ -5,13 +5,23 @@ # 3. v3) Introduce a status endpoint of the scheduler - # maybe expose a prometheus compatible output -import etcd3 import json import argparse +import logging from decouple import config from collections import Counter from functools import reduce +from etcd3_wrapper import Etcd3Wrapper +from datetime import datetime + +logging.basicConfig( + level=logging.DEBUG, + filename="log.txt", + filemode="a", + format="%(asctime)s: %(levelname)s - %(message)s", + datefmt="%d-%b-%y %H:%M:%S", +) class VmPool(object): @@ -20,9 +30,7 @@ class VmPool(object): self.vms = [] _vms = self.client.get_prefix(vm_prefix) - self.vms = [ - (vm[1].key.decode("utf-8"), json.loads(vm[0])) for vm in _vms - ] + self.vms = [(vm.key, json.loads(vm.value)) for vm in _vms] @staticmethod def by_host(vms, host): @@ -38,7 +46,7 @@ class VmPool(object): def accumulated_specs(vms_specs): - if vms_specs == []: + if not vms_specs: return {} return reduce((lambda x, y: Counter(x) + Counter(y)), vms_specs) @@ -51,16 +59,13 @@ def remaining_resources(host_specs, vms_specs): return remaining - def get_suitable_host(etcd_client, vm_prefix, host_prefix, vm_specs): vm_pool = VmPool(etcd_client, vm_prefix) - hosts = etcd_client.get_prefix(host_prefix) + hosts = etcd_client.get_prefix(host_prefix, value_in_json=True) + hosts = filter(lambda h: h.value["status"] == "ALIVE", hosts) for host in hosts: - _host_name, host_specs = ( - host[1].key.decode("utf-8"), - json.loads(host[0]), - ) + _host_name, host_value = (host.key, host.value) # Get All Virtual Machines vms = vm_pool.vms @@ -72,19 +77,19 @@ def get_suitable_host(etcd_client, vm_prefix, host_prefix, vm_specs): vms = VmPool.except_status(vms, "REQUESTED_NEW") running_vms_specs = [vm[1]["specs"] for vm in vms] - # Accumulate all of their combined specs running_vms_accumulated_specs = accumulated_specs(running_vms_specs) + print(running_vms_accumulated_specs) # Find out remaining resources after # host_specs - already running vm_specs + print(host_value) remaining = remaining_resources( - host_specs, running_vms_accumulated_specs + host_value["specs"], running_vms_accumulated_specs ) - + print(remaining) # Find out remaining - new_vm_specs remaining = remaining_resources(remaining, vm_specs) - # if remaining resources >= 0 return this host_name if all( map(lambda x: True if remaining[x] >= 0 else False, remaining) @@ -94,31 +99,78 @@ def get_suitable_host(etcd_client, vm_prefix, host_prefix, vm_specs): return None -def main(vm_prefix, host_prefix): +def dead_host_detection(hosts): + dead_hosts_keys = [] + for host in hosts: + # Bring out your dead! - Monty Python and the Holy Grail - client = etcd3.client( + if "status" in host.value and "last_heartbeat" in host.value: + # Don't count that is already buried + if host.value["status"] == "DEAD": + continue + + last_heartbeat = datetime.fromisoformat( + host.value["last_heartbeat"] + ) + delta = datetime.utcnow() - last_heartbeat + if delta.total_seconds() > 60: + dead_hosts_keys.append(host.key) + else: + dead_hosts_keys.append(host.key) + + return dead_hosts_keys + + +def dead_host_mitigation(client: Etcd3Wrapper, dead_hosts_keys): + for host_key in dead_hosts_keys: + host = client.get(host_key, value_in_json=True) + host.value["status"] = "DEAD" + host.value["last_heartbeat"] = datetime.utcnow().isoformat() + client.put(host.key, host.value, value_in_json=True) + + # Find all vms that were hosted on this dead host + all_vms = client.get_prefix(config("VM_PREFIX"), value_in_json=True) + vms_hosted_on_dead_host = filter( + lambda _vm: _vm.value["hostname"] == host_key, all_vms + ) + for vm in vms_hosted_on_dead_host: + vm.value["host"] = "" + vm.value["status"] = "REQUESTED_NEW" + client.put(vm.key, vm.value, value_in_json=True) + + +def main(vm_prefix, host_prefix): + client = Etcd3Wrapper( host=config("ETCD_HOST"), port=int(config("ETCD_PORT")) ) - events_iterator, _ = client.watch_prefix(vm_prefix) + events_iterator = client.watch_prefix(vm_prefix, timeout=10) - for event in events_iterator: - key = event.key - value = event.value - if not value: + for e in events_iterator: + try: + e.value = json.loads(e.value) + except json.JSONDecodeError: + logging.error(f"Invalid JSON {e.value}") continue - value = json.loads(event.value) - print(key, value) + logging.debug(e.key, e.value) - if value["status"] == "REQUESTED_NEW": + e_status = e.value["status"] + + if e_status == "TIMEOUT": + logging.info("Timeout") + hosts = client.get_prefix(host_prefix, value_in_json=True) + dead_hosts = dead_host_detection(hosts) + dead_host_mitigation(client, dead_hosts) + + elif e_status == "REQUESTED_NEW": host_name = get_suitable_host( - client, vm_prefix, host_prefix, value["specs"] + client, vm_prefix, host_prefix, e.value["specs"] ) if host_name: - value["status"] = "SCHEDULED_DEPLOY" - value["hostname"] = host_name - client.put(key, json.dumps(value)) + e.value["status"] = "SCHEDULED_DEPLOY" + e.value["hostname"] = host_name + client.put(e.key, json.dumps(e.value)) else: # email admin print("No Resource Left. Emailing admin....") diff --git a/tests/test_basics.py b/tests/test_basics.py index 0a84263..edb3de6 100644 --- a/tests/test_basics.py +++ b/tests/test_basics.py @@ -1,53 +1,83 @@ import unittest import sys -import etcd3 import json +import multiprocessing +import time -sys.path.insert(0, "../") -from main import accumulated_specs, remaining_resources, VmPool +from datetime import datetime +from os.path import dirname + + +BASE_DIR = dirname(dirname(__file__)) +sys.path.insert(0, BASE_DIR) + +from main import ( + accumulated_specs, + remaining_resources, + VmPool, + dead_host_detection, + dead_host_mitigation, + main, +) + +from etcd3_wrapper import Etcd3Wrapper class TestFunctions(unittest.TestCase): - def setUp(self): - self.client = etcd3.client() - self.host_prefix = "/v1/host" - self.vm_prefix = "/v1/vm" + @classmethod + def setUpClass(cls): + cls.client = Etcd3Wrapper() + cls.host_prefix = "/test/host" + cls.vm_prefix = "/test/vm" # These deletion could also be in # tearDown() but it is more appropriate here # as it enable us to check the ETCD store # even after test is run - self.client.delete_prefix(self.host_prefix) - self.client.delete_prefix(self.vm_prefix) + cls.client.client.delete_prefix(cls.host_prefix) + cls.client.client.delete_prefix(cls.vm_prefix) + cls.create_hosts(cls) + cls.create_vms(cls) - self.create_hosts() - self.create_vms() + cls.p = multiprocessing.Process( + target=main, args=[cls.vm_prefix, cls.host_prefix] + ) + cls.p.start() + + @classmethod + def tearDownClass(cls): + cls.p.terminate() def create_hosts(self): - host1 = """{ + host1 = { "cpu": 32, "ram": 128, "hdd": 1024, - "sdd": 0 - }""" - - host2 = """{ + "sdd": 0, + "status": "ALIVE", + "last_heartbeat": datetime.utcnow().isoformat(), + } + host2 = { "cpu": 16, "ram": 64, "hdd": 512, - "sdd": 0 - }""" + "sdd": 0, + "status": "ALIVE", + "last_heartbeat": datetime.utcnow().isoformat(), + } - host3 = """{ + host3 = { "cpu": 16, "ram": 32, "hdd": 256, - "sdd": 256 - }""" - with self.client.lock("lock"): - self.client.put(f"{self.host_prefix}/1", host1) - self.client.put(f"{self.host_prefix}/2", host2) - self.client.put(f"{self.host_prefix}/3", host3) + "sdd": 256, + "status": "ALIVE", + "last_heartbeat": datetime.utcnow().isoformat(), + } + with self.client.client.lock("lock"): + self.client.put(f"{self.host_prefix}/1", host1, value_in_json=True) + self.client.put(f"{self.host_prefix}/2", host2, value_in_json=True) + self.client.put(f"{self.host_prefix}/3", host3, value_in_json=True) def create_vms(self): vm1 = json.dumps( @@ -121,35 +151,64 @@ class TestFunctions(unittest.TestCase): {"cpu": 8, "ram": 32}, ] self.assertEqual( - accumulated_specs(vms), - {"ssd": 10, "cpu": 16, "ram": 48, "hdd": 10}, + accumulated_specs(vms), {"ssd": 10, "cpu": 16, "ram": 48, "hdd": 10} ) def test_remaining_resources(self): host_specs = {"ssd": 10, "cpu": 16, "ram": 48, "hdd": 10} vms_specs = {"ssd": 10, "cpu": 32, "ram": 12, "hdd": 0} resultant_specs = {"ssd": 0, "cpu": -16, "ram": 36, "hdd": 10} - self.assertEqual( - remaining_resources(host_specs, vms_specs), resultant_specs - ) + self.assertEqual(remaining_resources(host_specs, vms_specs), + resultant_specs) def test_vmpool(self): + self.p.join(1) vm_pool = VmPool(self.client, self.vm_prefix) - print(self.client.get(f"{self.vm_prefix}/1")[1].key) - self.assertEqual( - vm_pool.by_host(vm_pool.vms, f"{self.host_prefix}/1"), - [ - ( - f"{self.vm_prefix}/1", - { - "owner": "meow", - "specs": {"cpu": 4, "ram": 8, "hdd": 100, "sdd": 256}, - "hostname": f"{self.host_prefix}/1", - "status": "SCHEDULED_DEPLOY", - }, - ) - ], - ) + + # vm_pool by host + actual = vm_pool.by_host(vm_pool.vms, f"{self.host_prefix}/3") + ground_truth = [ + ( + f"{self.vm_prefix}/1", + { + "owner": "meow", + "specs": {"cpu": 4, "ram": 8, "hdd": 100, "sdd": 256}, + "hostname": f"{self.host_prefix}/3", + "status": "SCHEDULED_DEPLOY", + }, + ) + ] + self.assertEqual(actual[0], ground_truth[0]) + + # vm_pool by status + actual = vm_pool.by_status(vm_pool.vms, "REQUESTED_NEW") + ground_truth = [ + ( + f"{self.vm_prefix}/7", + { + "owner": "meow", + "specs": {"cpu": 10, "ram": 22, "hdd": 146, "sdd": 0}, + "hostname": "", + "status": "REQUESTED_NEW", + }, + ) + ] + self.assertEqual(actual[0], ground_truth[0]) + + # vm_pool by except status + actual = vm_pool.except_status(vm_pool.vms, "SCHEDULED_DEPLOY") + ground_truth = [ + ( + f"{self.vm_prefix}/7", + { + "owner": "meow", + "specs": {"cpu": 10, "ram": 22, "hdd": 146, "sdd": 0}, + "hostname": "", + "status": "REQUESTED_NEW", + }, + ) + ] + self.assertEqual(actual[0], ground_truth[0]) if __name__ == "__main__": diff --git a/tests/test_dead_host_mechanism.py b/tests/test_dead_host_mechanism.py new file mode 100644 index 0000000..477ae29 --- /dev/null +++ b/tests/test_dead_host_mechanism.py @@ -0,0 +1,83 @@ +import unittest +import sys +import json +import multiprocessing +import time + +from datetime import datetime +from os.path import dirname +BASE_DIR = dirname(dirname(__file__)) +sys.path.insert(0, BASE_DIR) + +from main import ( + accumulated_specs, + remaining_resources, + VmPool, + dead_host_detection, + dead_host_mitigation, + main, +) + +from etcd3_wrapper import Etcd3Wrapper + + +class TestDeadHostMechanism(unittest.TestCase): + def setUp(self): + self.client = Etcd3Wrapper() + self.host_prefix = "/test/host" + self.vm_prefix = "/test/vm" + + self.client.client.delete_prefix(self.host_prefix) + self.client.client.delete_prefix(self.vm_prefix) + + self.create_hosts() + + def create_hosts(self): + host1 = { + "cpu": 32, + "ram": 128, + "hdd": 1024, + "sdd": 0, + "status": "ALIVE", + "last_heartbeat": datetime.utcnow().isoformat(), + } + host2 = { + "cpu": 16, + "ram": 64, + "hdd": 512, + "sdd": 0, + "status": "ALIVE", + "last_heartbeat": datetime(2011, 1, 1).isoformat(), + } + + host3 = {"cpu": 16, "ram": 32, "hdd": 256, "sdd": 256} + host4 = { + "cpu": 16, + "ram": 32, + "hdd": 256, + "sdd": 256, + "status": "DEAD", + "last_heartbeat": datetime(2011, 1, 1).isoformat(), + } + with self.client.client.lock("lock"): + self.client.put(f"{self.host_prefix}/1", host1, value_in_json=True) + self.client.put(f"{self.host_prefix}/2", host2, value_in_json=True) + self.client.put(f"{self.host_prefix}/3", host3, value_in_json=True) + self.client.put(f"{self.host_prefix}/4", host4, value_in_json=True) + + def test_dead_host_detection(self): + hosts = self.client.get_prefix(self.host_prefix, value_in_json=True) + deads = dead_host_detection(hosts) + self.assertEqual(deads, ["/test/host/2", "/test/host/3"]) + return deads + + def test_dead_host_mitigation(self): + deads = self.test_dead_host_detection() + dead_host_mitigation(self.client, deads) + hosts = self.client.get_prefix(self.host_prefix, value_in_json=True) + deads = dead_host_detection(hosts) + self.assertEqual(deads, []) + + +if __name__ == "__main__": + unittest.main()