commit da77ac65ebf7f383f2358e32194a89ad442e5f8b Author: meow Date: Fri Oct 25 11:42:40 2019 +0500 ucloud-{api,scheduler,host,filescanner,imagescanner,metadata} combined diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5430a7a --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +.idea +.vscode +.env + +*/log.txt \ No newline at end of file diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..5db945c --- /dev/null +++ b/Pipfile @@ -0,0 +1,22 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] +prospector = {extras = ["with_everything"],version = "*"} + +[packages] +python-decouple = "*" +requests = "*" +flask = "*" +flask-restful = "*" +bitmath = "*" +ucloud-common = {editable = true,git = "git+https://code.ungleich.ch/ucloud/ucloud_common.git",ref = "wip"} +etcd3-wrapper = {editable = true,git = "git+https://code.ungleich.ch/ungleich-public/etcd3_wrapper.git",ref = "wip"} +python-etcd3 = {editable = true,git = "git+https://github.com/kragniz/python-etcd3.git"} +pyotp = "*" +sshtunnel = "*" + +[requires] +python_version = "3.5" diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 0000000..aaa7369 --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,773 @@ +{ + "_meta": { + "hash": { + "sha256": "b7a8409bec451e017440f063d8436fe66b18affcde7ad5497b433191ae465a52" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.5" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "aniso8601": { + "hashes": [ + "sha256:529dcb1f5f26ee0df6c0a1ee84b7b27197c3c50fc3a6321d66c544689237d072", + "sha256:c033f63d028b9a58e3ab0c2c7d0532ab4bfa7452bfc788fbfe3ddabd327b181a" + ], + "version": "==8.0.0" + }, + "bcrypt": { + "hashes": [ + "sha256:0258f143f3de96b7c14f762c770f5fc56ccd72f8a1857a451c1cd9a655d9ac89", + "sha256:0b0069c752ec14172c5f78208f1863d7ad6755a6fae6fe76ec2c80d13be41e42", + "sha256:19a4b72a6ae5bb467fea018b825f0a7d917789bcfe893e53f15c92805d187294", + "sha256:5432dd7b34107ae8ed6c10a71b4397f1c853bd39a4d6ffa7e35f40584cffd161", + "sha256:6305557019906466fc42dbc53b46da004e72fd7a551c044a827e572c82191752", + "sha256:69361315039878c0680be456640f8705d76cb4a3a3fe1e057e0f261b74be4b31", + "sha256:6fe49a60b25b584e2f4ef175b29d3a83ba63b3a4df1b4c0605b826668d1b6be5", + "sha256:74a015102e877d0ccd02cdeaa18b32aa7273746914a6c5d0456dd442cb65b99c", + "sha256:763669a367869786bb4c8fcf731f4175775a5b43f070f50f46f0b59da45375d0", + "sha256:8b10acde4e1919d6015e1df86d4c217d3b5b01bb7744c36113ea43d529e1c3de", + "sha256:9fe92406c857409b70a38729dbdf6578caf9228de0aef5bc44f859ffe971a39e", + "sha256:a190f2a5dbbdbff4b74e3103cef44344bc30e61255beb27310e2aec407766052", + "sha256:a595c12c618119255c90deb4b046e1ca3bcfad64667c43d1166f2b04bc72db09", + "sha256:c9457fa5c121e94a58d6505cadca8bed1c64444b83b3204928a866ca2e599105", + "sha256:cb93f6b2ab0f6853550b74e051d297c27a638719753eb9ff66d1e4072be67133", + "sha256:ce4e4f0deb51d38b1611a27f330426154f2980e66582dc5f438aad38b5f24fc1", + "sha256:d7bdc26475679dd073ba0ed2766445bb5b20ca4793ca0db32b399dccc6bc84b7", + "sha256:ff032765bb8716d9387fd5376d987a937254b0619eff0972779515b5c98820bc" + ], + "version": "==3.1.7" + }, + "bitmath": { + "hashes": [ + "sha256:293325f01e65defe966853111df11d39215eb705a967cb115851da8c4cfa3eb8" + ], + "index": "pypi", + "version": "==1.3.3.1" + }, + "certifi": { + "hashes": [ + "sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50", + "sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef" + ], + "version": "==2019.9.11" + }, + "cffi": { + "hashes": [ + "sha256:00d890313797d9fe4420506613384b43099ad7d2b905c0752dbcc3a6f14d80fa", + "sha256:0cf9e550ac6c5e57b713437e2f4ac2d7fd0cd10336525a27224f5fc1ec2ee59a", + "sha256:0ea23c9c0cdd6778146a50d867d6405693ac3b80a68829966c98dd5e1bbae400", + "sha256:193697c2918ecdb3865acf6557cddf5076bb39f1f654975e087b67efdff83365", + "sha256:1ae14b542bf3b35e5229439c35653d2ef7d8316c1fffb980f9b7647e544baa98", + "sha256:1e389e069450609c6ffa37f21f40cce36f9be7643bbe5051ab1de99d5a779526", + "sha256:263242b6ace7f9cd4ea401428d2d45066b49a700852334fd55311bde36dcda14", + "sha256:33142ae9807665fa6511cfa9857132b2c3ee6ddffb012b3f0933fc11e1e830d5", + "sha256:364f8404034ae1b232335d8c7f7b57deac566f148f7222cef78cf8ae28ef764e", + "sha256:47368f69fe6529f8f49a5d146ddee713fc9057e31d61e8b6dc86a6a5e38cecc1", + "sha256:4895640844f17bec32943995dc8c96989226974dfeb9dd121cc45d36e0d0c434", + "sha256:558b3afef987cf4b17abd849e7bedf64ee12b28175d564d05b628a0f9355599b", + "sha256:5ba86e1d80d458b338bda676fd9f9d68cb4e7a03819632969cf6d46b01a26730", + "sha256:63424daa6955e6b4c70dc2755897f5be1d719eabe71b2625948b222775ed5c43", + "sha256:6381a7d8b1ebd0bc27c3bc85bc1bfadbb6e6f756b4d4db0aa1425c3719ba26b4", + "sha256:6381ab708158c4e1639da1f2a7679a9bbe3e5a776fc6d1fd808076f0e3145331", + "sha256:6fd58366747debfa5e6163ada468a90788411f10c92597d3b0a912d07e580c36", + "sha256:728ec653964655d65408949b07f9b2219df78badd601d6c49e28d604efe40599", + "sha256:7cfcfda59ef1f95b9f729c56fe8a4041899f96b72685d36ef16a3440a0f85da8", + "sha256:819f8d5197c2684524637f940445c06e003c4a541f9983fd30d6deaa2a5487d8", + "sha256:825ecffd9574557590e3225560a8a9d751f6ffe4a49e3c40918c9969b93395fa", + "sha256:9009e917d8f5ef780c2626e29b6bc126f4cb2a4d43ca67aa2b40f2a5d6385e78", + "sha256:9c77564a51d4d914ed5af096cd9843d90c45b784b511723bd46a8a9d09cf16fc", + "sha256:a19089fa74ed19c4fe96502a291cfdb89223a9705b1d73b3005df4256976142e", + "sha256:a40ed527bffa2b7ebe07acc5a3f782da072e262ca994b4f2085100b5a444bbb2", + "sha256:bb75ba21d5716abc41af16eac1145ab2e471deedde1f22c6f99bd9f995504df0", + "sha256:e22a00c0c81ffcecaf07c2bfb3672fa372c50e2bd1024ffee0da191c1b27fc71", + "sha256:e55b5a746fb77f10c83e8af081979351722f6ea48facea79d470b3731c7b2891", + "sha256:ec2fa3ee81707a5232bf2dfbd6623fdb278e070d596effc7e2d788f2ada71a05", + "sha256:fd82eb4694be712fcae03c717ca2e0fc720657ac226b80bbb597e971fc6928c2" + ], + "version": "==1.13.1" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, + "click": { + "hashes": [ + "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13", + "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7" + ], + "version": "==7.0" + }, + "cryptography": { + "hashes": [ + "sha256:02079a6addc7b5140ba0825f542c0869ff4df9a69c360e339ecead5baefa843c", + "sha256:1df22371fbf2004c6f64e927668734070a8953362cd8370ddd336774d6743595", + "sha256:369d2346db5934345787451504853ad9d342d7f721ae82d098083e1f49a582ad", + "sha256:3cda1f0ed8747339bbdf71b9f38ca74c7b592f24f65cdb3ab3765e4b02871651", + "sha256:44ff04138935882fef7c686878e1c8fd80a723161ad6a98da31e14b7553170c2", + "sha256:4b1030728872c59687badcca1e225a9103440e467c17d6d1730ab3d2d64bfeff", + "sha256:58363dbd966afb4f89b3b11dfb8ff200058fbc3b947507675c19ceb46104b48d", + "sha256:6ec280fb24d27e3d97aa731e16207d58bd8ae94ef6eab97249a2afe4ba643d42", + "sha256:7270a6c29199adc1297776937a05b59720e8a782531f1f122f2eb8467f9aab4d", + "sha256:73fd30c57fa2d0a1d7a49c561c40c2f79c7d6c374cc7750e9ac7c99176f6428e", + "sha256:7f09806ed4fbea8f51585231ba742b58cbcfbfe823ea197d8c89a5e433c7e912", + "sha256:90df0cc93e1f8d2fba8365fb59a858f51a11a394d64dbf3ef844f783844cc793", + "sha256:971221ed40f058f5662a604bd1ae6e4521d84e6cad0b7b170564cc34169c8f13", + "sha256:a518c153a2b5ed6b8cc03f7ae79d5ffad7315ad4569b2d5333a13c38d64bd8d7", + "sha256:b0de590a8b0979649ebeef8bb9f54394d3a41f66c5584fff4220901739b6b2f0", + "sha256:b43f53f29816ba1db8525f006fa6f49292e9b029554b3eb56a189a70f2a40879", + "sha256:d31402aad60ed889c7e57934a03477b572a03af7794fa8fb1780f21ea8f6551f", + "sha256:de96157ec73458a7f14e3d26f17f8128c959084931e8997b9e655a39c8fde9f9", + "sha256:df6b4dca2e11865e6cfbfb708e800efb18370f5a46fd601d3755bc7f85b3a8a2", + "sha256:ecadccc7ba52193963c0475ac9f6fa28ac01e01349a2ca48509667ef41ffd2cf", + "sha256:fb81c17e0ebe3358486cd8cc3ad78adbae58af12fc2bf2bc0bb84e8090fa5ce8" + ], + "version": "==2.8" + }, + "etcd3-wrapper": { + "editable": true, + "git": "https://code.ungleich.ch/ungleich-public/etcd3_wrapper.git", + "ref": "76fb0bdf797199e9ea161dad1d004eea9b4520f8" + }, + "flask": { + "hashes": [ + "sha256:13f9f196f330c7c2c5d7a5cf91af894110ca0215ac051b5844701f2bfd934d52", + "sha256:45eb5a6fd193d6cf7e0cf5d8a5b31f83d5faae0293695626f539a823e93b13f6" + ], + "index": "pypi", + "version": "==1.1.1" + }, + "flask-restful": { + "hashes": [ + "sha256:ecd620c5cc29f663627f99e04f17d1f16d095c83dc1d618426e2ad68b03092f8", + "sha256:f8240ec12349afe8df1db168ea7c336c4e5b0271a36982bff7394f93275f2ca9" + ], + "index": "pypi", + "version": "==0.3.7" + }, + "grpcio": { + "hashes": [ + "sha256:01cb705eafba1108e2a947ba0457da4f6a1e8142c729fc61702b5fdd11009eb1", + "sha256:0b5a79e29f167d3cd06faad6b15babbc2661066daaacf79373c3a8e67ca1fca1", + "sha256:1097a61a0e97b3580642e6e1460a3a1f1ba1815e2a70d6057173bcc495417076", + "sha256:13970e665a4ec4cec7d067d7d3504a0398c657d91d26c581144ad9044e429c9a", + "sha256:1557817cea6e0b87fad2a3e20da385170efb03a313db164e8078955add2dfa1b", + "sha256:1b0fb036a2f9dd93d9a35c57c26420eeb4b571fcb14b51cddf5b1e73ea5d882b", + "sha256:24d9e58d08e8cd545d8a3247a18654aff0e5e60414701696a8098fbb0d792b75", + "sha256:2c38b586163d2b91567fe5e6d9e7798f792012365adc838a64b66b22dce3f4d4", + "sha256:2df3ab4348507de60e1cbf75196403df1b9b4c4d4dc5bd11ac4eb63c46f691c7", + "sha256:32f70f7c90454ea568b868af2e96616743718d9233d23f62407e98caed81dfbf", + "sha256:3af2a49d576820045c9c880ff29a5a96d020fe31b35d248519bfc6ccb8be4eac", + "sha256:4ff7d63800a63db031ebac6a6f581ae84877c959401c24c28f2cc51fd36c47ad", + "sha256:502aaa8be56f0ae69cda66bc27e1fb5531ceaa27ca515ec3c34f6178b1297180", + "sha256:55358ce3ec283222e435f7dbc6603521438458f3c65f7c1cb33b8dabf56d70d8", + "sha256:5583b01c67f85fa64a2c3fb085e5517c88b9c1500a2cce12d473cd99d0ed2e49", + "sha256:58d9a5557d3eb7b734a3cea8b16c891099a522b3953a45a30bd4c034f75fc913", + "sha256:5911f042c4ab177757eec5bcb4e2e9a2e823d888835d24577321bf55f02938fa", + "sha256:5e16ea922f4e5017c04fd94e2639b1006e03097e9dd0cbb7a1c852af3ea8bf2e", + "sha256:656e19d3f1b9050ee01b457f92838a9679d7cf84c995f708780f44484048705e", + "sha256:6a1435449a82008c451c7e1a82a834387b9108f9a8d27910f86e7c482f5568e9", + "sha256:6ff02ca6cbed0ddb76e93ba0f8beb6a8c77d83a84eb7cafe2ae3399a8b9d69ea", + "sha256:76de68f60102f333bf4817f38e81ecbee68b850f5a5da9f355235e948ac40981", + "sha256:7c6d7ddd50fc6548ea1dfe09c62509c4f95b8b40082287747be05aa8feb15ee2", + "sha256:836b9d29507de729129e363276fe7c7d6a34c7961e0f155787025552b15d22c0", + "sha256:869242b2baf8a888a4fe0548f86abc47cb4b48bdfd76ae62d6456e939c202e65", + "sha256:8954b24bd08641d906ee50b2d638efc76df893fbd0913149b80484fd0eac40c9", + "sha256:8cdea65d1abb2e698420db8daf20c8d272fbd9d96a51b26a713c1c76f237d181", + "sha256:90161840b4fe9636f91ed0d3ea1e7e615e488cbea4e77594c889e5f3d7a776db", + "sha256:90fb6316b4d7d36700c40db4335902b78dcae13b5466673c21fd3b08a3c1b0c6", + "sha256:91b34f58db2611c9a93ecf751028f97fba1f06e65f49b38f272f6aa5d2977331", + "sha256:9474944a96a33eb8734fa8dc5805403d57973a3526204a5e1c1780d02e0572b6", + "sha256:9a36275db2a4774ac16c6822e7af816ee048071d5030b4c035fd53942b361935", + "sha256:9cbe26e2976b994c5f7c2d35a63354674d6ca0ce62f5b513f078bf63c1745229", + "sha256:9eaeabb3c0eecd6ddd0c16767fd12d130e2cebb8c2618f959a278b1ff336ddc3", + "sha256:a2bc7e10ebcf4be503ae427f9887e75c0cc24e88ce467a8e6eaca6bd2862406e", + "sha256:a5b42e6292ba51b8e67e09fc256963ba4ca9c04026de004d2fe59cc17e3c3776", + "sha256:bd6ec1233c86c0b9bb5d03ec30dbe3ffbfa53335790320d99a7ae9018c5450f2", + "sha256:bef57530816af54d66b1f4c70a8f851f320cb6f84d4b5a0b422b0e9811ea4e59", + "sha256:c146a63eaadc6589b732780061f3c94cd0574388d372baccbb3c1597a9ebdb7a", + "sha256:c2efd3b130dc639d615b6f58980e1bfd1b177ad821f30827afa5001aa30ddd48", + "sha256:c888b18f7392e6cc79a33a803e7ebd7890ac3318f571fca6b356526f35b53b12", + "sha256:ca30721fda297ae22f16bc37aa7ed244970ddfdcb98247570cdd26daaad4665e", + "sha256:cf5f5340dd682ab034baa52f423a0f91326489c262ac9617fa06309ec05880e9", + "sha256:d0726aa0d9b57c56985db5952e90fb1033a317074f2877db5307cdd6eede1564", + "sha256:df442945b2dd6f8ae0e20b403e0fd4548cd5c2aad69200047cc3251257b78f65", + "sha256:e08e758c31919d167c0867539bd3b2441629ef00aa595e3ea2b635273659f40a", + "sha256:e4864339deeeaefaad34dd3a432ee618a039fca28efb292949c855e00878203c", + "sha256:f4cd049cb94d9f517b1cab5668a3b345968beba093bc79a637e671000b3540ec" + ], + "version": "==1.24.3" + }, + "idna": { + "hashes": [ + "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", + "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" + ], + "version": "==2.8" + }, + "itsdangerous": { + "hashes": [ + "sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19", + "sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749" + ], + "version": "==1.1.0" + }, + "jinja2": { + "hashes": [ + "sha256:74320bb91f31270f9551d46522e33af46a80c3d619f4a4bf42b3164d30b5911f", + "sha256:9fe95f19286cfefaa917656583d020be14e7859c6b0252588391e47db34527de" + ], + "version": "==2.10.3" + }, + "markupsafe": { + "hashes": [ + "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473", + "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161", + "sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235", + "sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5", + "sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff", + "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b", + "sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1", + "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e", + "sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183", + "sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66", + "sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1", + "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1", + "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e", + "sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b", + "sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905", + "sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735", + "sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d", + "sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e", + "sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d", + "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c", + "sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21", + "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2", + "sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5", + "sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b", + "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6", + "sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f", + "sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f", + "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7" + ], + "version": "==1.1.1" + }, + "paramiko": { + "hashes": [ + "sha256:99f0179bdc176281d21961a003ffdb2ec369daac1a1007241f53374e376576cf", + "sha256:f4b2edfa0d226b70bd4ca31ea7e389325990283da23465d572ed1f70a7583041" + ], + "version": "==2.6.0" + }, + "protobuf": { + "hashes": [ + "sha256:125713564d8cfed7610e52444c9769b8dcb0b55e25cc7841f2290ee7bc86636f", + "sha256:1accdb7a47e51503be64d9a57543964ba674edac103215576399d2d0e34eac77", + "sha256:27003d12d4f68e3cbea9eb67427cab3bfddd47ff90670cb367fcd7a3a89b9657", + "sha256:3264f3c431a631b0b31e9db2ae8c927b79fc1a7b1b06b31e8e5bcf2af91fe896", + "sha256:3c5ab0f5c71ca5af27143e60613729e3488bb45f6d3f143dc918a20af8bab0bf", + "sha256:45dcf8758873e3f69feab075e5f3177270739f146255225474ee0b90429adef6", + "sha256:56a77d61a91186cc5676d8e11b36a5feb513873e4ae88d2ee5cf530d52bbcd3b", + "sha256:5984e4947bbcef5bd849d6244aec507d31786f2dd3344139adc1489fb403b300", + "sha256:6b0441da73796dd00821763bb4119674eaf252776beb50ae3883bed179a60b2a", + "sha256:6f6677c5ade94d4fe75a912926d6796d5c71a2a90c2aeefe0d6f211d75c74789", + "sha256:84a825a9418d7196e2acc48f8746cf1ee75877ed2f30433ab92a133f3eaf8fbe", + "sha256:b842c34fe043ccf78b4a6cf1019d7b80113707d68c88842d061fa2b8fb6ddedc", + "sha256:ca33d2f09dae149a1dcf942d2d825ebb06343b77b437198c9e2ef115cf5d5bc1", + "sha256:db83b5c12c0cd30150bb568e6feb2435c49ce4e68fe2d7b903113f0e221e58fe", + "sha256:f50f3b1c5c1c1334ca7ce9cad5992f098f460ffd6388a3cabad10b66c2006b09", + "sha256:f99f127909731cafb841c52f9216e447d3e4afb99b17bebfad327a75aee206de" + ], + "version": "==3.10.0" + }, + "pycparser": { + "hashes": [ + "sha256:9d97450dc26e1d2581c18881d8d1c0a92e84c9ac074961e3dc66e70d745a0643", + "sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3" + ], + "version": "==2.19" + }, + "pynacl": { + "hashes": [ + "sha256:05c26f93964373fc0abe332676cb6735f0ecad27711035b9472751faa8521255", + "sha256:0c6100edd16fefd1557da078c7a31e7b7d7a52ce39fdca2bec29d4f7b6e7600c", + "sha256:0d0a8171a68edf51add1e73d2159c4bc19fc0718e79dec51166e940856c2f28e", + "sha256:1c780712b206317a746ace34c209b8c29dbfd841dfbc02aa27f2084dd3db77ae", + "sha256:2424c8b9f41aa65bbdbd7a64e73a7450ebb4aa9ddedc6a081e7afcc4c97f7621", + "sha256:2d23c04e8d709444220557ae48ed01f3f1086439f12dbf11976e849a4926db56", + "sha256:30f36a9c70450c7878053fa1344aca0145fd47d845270b43a7ee9192a051bf39", + "sha256:37aa336a317209f1bb099ad177fef0da45be36a2aa664507c5d72015f956c310", + "sha256:4943decfc5b905748f0756fdd99d4f9498d7064815c4cf3643820c9028b711d1", + "sha256:53126cd91356342dcae7e209f840212a58dcf1177ad52c1d938d428eebc9fee5", + "sha256:57ef38a65056e7800859e5ba9e6091053cd06e1038983016effaffe0efcd594a", + "sha256:5bd61e9b44c543016ce1f6aef48606280e45f892a928ca7068fba30021e9b786", + "sha256:6482d3017a0c0327a49dddc8bd1074cc730d45db2ccb09c3bac1f8f32d1eb61b", + "sha256:7d3ce02c0784b7cbcc771a2da6ea51f87e8716004512493a2b69016326301c3b", + "sha256:a14e499c0f5955dcc3991f785f3f8e2130ed504fa3a7f44009ff458ad6bdd17f", + "sha256:a39f54ccbcd2757d1d63b0ec00a00980c0b382c62865b61a505163943624ab20", + "sha256:aabb0c5232910a20eec8563503c153a8e78bbf5459490c49ab31f6adf3f3a415", + "sha256:bd4ecb473a96ad0f90c20acba4f0bf0df91a4e03a1f4dd6a4bdc9ca75aa3a715", + "sha256:bf459128feb543cfca16a95f8da31e2e65e4c5257d2f3dfa8c0c1031139c9c92", + "sha256:e2da3c13307eac601f3de04887624939aca8ee3c9488a0bb0eca4fb9401fc6b1", + "sha256:f67814c38162f4deb31f68d590771a29d5ae3b1bd64b75cf232308e5c74777e0" + ], + "version": "==1.3.0" + }, + "pyotp": { + "hashes": [ + "sha256:c88f37fd47541a580b744b42136f387cdad481b560ef410c0d85c957eb2a2bc0", + "sha256:fc537e8acd985c5cbf51e11b7d53c42276fee017a73aec7c07380695671ca1a1" + ], + "index": "pypi", + "version": "==2.3.0" + }, + "python-decouple": { + "hashes": [ + "sha256:1317df14b43efee4337a4aa02914bf004f010cd56d6c4bd894e6474ec8c4fe2d" + ], + "index": "pypi", + "version": "==3.1" + }, + "python-etcd3": { + "editable": true, + "git": "https://github.com/kragniz/python-etcd3.git", + "ref": "247e3952d0b47324091a36ace3ad9717469fb6b9" + }, + "pytz": { + "hashes": [ + "sha256:1c557d7d0e871de1f5ccd5833f60fb2550652da6be2693c1e02300743d21500d", + "sha256:b02c06db6cf09c12dd25137e563b31700d3b80fcc4ad23abb7a315f2789819be" + ], + "version": "==2019.3" + }, + "requests": { + "hashes": [ + "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4", + "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31" + ], + "index": "pypi", + "version": "==2.22.0" + }, + "six": { + "hashes": [ + "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", + "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" + ], + "version": "==1.12.0" + }, + "sshtunnel": { + "hashes": [ + "sha256:c813fdcda8e81c3936ffeac47cb69cfb2d1f5e77ad0de656c6dab56aeebd9249" + ], + "index": "pypi", + "version": "==0.1.5" + }, + "tenacity": { + "hashes": [ + "sha256:6a7511a59145c2e319b7d04ddd93c12d48cc3d3c8fa42c2846d33a620ee91f57", + "sha256:a4eb168dbf55ed2cae27e7c6b2bd48ab54dabaf294177d998330cf59f294c112" + ], + "version": "==5.1.1" + }, + "ucloud-common": { + "editable": true, + "git": "https://code.ungleich.ch/ucloud/ucloud_common.git", + "ref": "9f229eae27f9007e9c6c1021d3d5b12452863763" + }, + "urllib3": { + "hashes": [ + "sha256:3de946ffbed6e6746608990594d08faac602528ac7015ac28d33cee6a45b7398", + "sha256:9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86" + ], + "version": "==1.25.6" + }, + "werkzeug": { + "hashes": [ + "sha256:7280924747b5733b246fe23972186c6b348f9ae29724135a6dfc1e53cea433e7", + "sha256:e5f4a1f98b52b18a93da705a7458e55afb26f32bff83ff5d19189f92462d65c4" + ], + "version": "==0.16.0" + } + }, + "develop": { + "astroid": { + "hashes": [ + "sha256:6560e1e1749f68c64a4b5dee4e091fce798d2f0d84ebe638cf0e0585a343acf4", + "sha256:b65db1bbaac9f9f4d190199bb8680af6f6f84fd3769a5ea883df8a91fe68b4c4" + ], + "version": "==2.2.5" + }, + "certifi": { + "hashes": [ + "sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50", + "sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef" + ], + "version": "==2019.9.11" + }, + "chardet": { + "hashes": [ + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" + ], + "version": "==3.0.4" + }, + "coverage": { + "hashes": [ + "sha256:08907593569fe59baca0bf152c43f3863201efb6113ecb38ce7e97ce339805a6", + "sha256:0be0f1ed45fc0c185cfd4ecc19a1d6532d72f86a2bac9de7e24541febad72650", + "sha256:141f08ed3c4b1847015e2cd62ec06d35e67a3ac185c26f7635f4406b90afa9c5", + "sha256:19e4df788a0581238e9390c85a7a09af39c7b539b29f25c89209e6c3e371270d", + "sha256:23cc09ed395b03424d1ae30dcc292615c1372bfba7141eb85e11e50efaa6b351", + "sha256:245388cda02af78276b479f299bbf3783ef0a6a6273037d7c60dc73b8d8d7755", + "sha256:331cb5115673a20fb131dadd22f5bcaf7677ef758741312bee4937d71a14b2ef", + "sha256:386e2e4090f0bc5df274e720105c342263423e77ee8826002dcffe0c9533dbca", + "sha256:3a794ce50daee01c74a494919d5ebdc23d58873747fa0e288318728533a3e1ca", + "sha256:60851187677b24c6085248f0a0b9b98d49cba7ecc7ec60ba6b9d2e5574ac1ee9", + "sha256:63a9a5fc43b58735f65ed63d2cf43508f462dc49857da70b8980ad78d41d52fc", + "sha256:6b62544bb68106e3f00b21c8930e83e584fdca005d4fffd29bb39fb3ffa03cb5", + "sha256:6ba744056423ef8d450cf627289166da65903885272055fb4b5e113137cfa14f", + "sha256:7494b0b0274c5072bddbfd5b4a6c6f18fbbe1ab1d22a41e99cd2d00c8f96ecfe", + "sha256:826f32b9547c8091679ff292a82aca9c7b9650f9fda3e2ca6bf2ac905b7ce888", + "sha256:93715dffbcd0678057f947f496484e906bf9509f5c1c38fc9ba3922893cda5f5", + "sha256:9a334d6c83dfeadae576b4d633a71620d40d1c379129d587faa42ee3e2a85cce", + "sha256:af7ed8a8aa6957aac47b4268631fa1df984643f07ef00acd374e456364b373f5", + "sha256:bf0a7aed7f5521c7ca67febd57db473af4762b9622254291fbcbb8cd0ba5e33e", + "sha256:bf1ef9eb901113a9805287e090452c05547578eaab1b62e4ad456fcc049a9b7e", + "sha256:c0afd27bc0e307a1ffc04ca5ec010a290e49e3afbe841c5cafc5c5a80ecd81c9", + "sha256:dd579709a87092c6dbee09d1b7cfa81831040705ffa12a1b248935274aee0437", + "sha256:df6712284b2e44a065097846488f66840445eb987eb81b3cc6e4149e7b6982e1", + "sha256:e07d9f1a23e9e93ab5c62902833bf3e4b1f65502927379148b6622686223125c", + "sha256:e2ede7c1d45e65e209d6093b762e98e8318ddeff95317d07a27a2140b80cfd24", + "sha256:e4ef9c164eb55123c62411f5936b5c2e521b12356037b6e1c2617cef45523d47", + "sha256:eca2b7343524e7ba246cab8ff00cab47a2d6d54ada3b02772e908a45675722e2", + "sha256:eee64c616adeff7db37cc37da4180a3a5b6177f5c46b187894e633f088fb5b28", + "sha256:ef824cad1f980d27f26166f86856efe11eff9912c4fed97d3804820d43fa550c", + "sha256:efc89291bd5a08855829a3c522df16d856455297cf35ae827a37edac45f466a7", + "sha256:fa964bae817babece5aa2e8c1af841bebb6d0b9add8e637548809d040443fee0", + "sha256:ff37757e068ae606659c28c3bd0d923f9d29a85de79bf25b2b34b148473b5025" + ], + "version": "==4.5.4" + }, + "coveralls": { + "hashes": [ + "sha256:9bc5a1f92682eef59f688a8f280207190d9a6afb84cef8f567fa47631a784060", + "sha256:fb51cddef4bc458de347274116df15d641a735d3f0a580a9472174e2e62f408c" + ], + "version": "==1.8.2" + }, + "docopt": { + "hashes": [ + "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491" + ], + "version": "==0.6.2" + }, + "docutils": { + "hashes": [ + "sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0", + "sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827", + "sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99" + ], + "version": "==0.15.2" + }, + "dodgy": { + "hashes": [ + "sha256:65e13cf878d7aff129f1461c13cb5fd1bb6dfe66bb5327e09379c3877763280c" + ], + "version": "==0.1.9" + }, + "frosted": { + "hashes": [ + "sha256:c6a30ad502ea373f6fe4cafbcd896ece66948406b04365d14a3eb764cc529b07", + "sha256:d1e5d2b43a064b33c289b9a986a7425fd9a36bed8f519ca430ac7a0915e32b51" + ], + "version": "==1.4.1" + }, + "idna": { + "hashes": [ + "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", + "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" + ], + "version": "==2.8" + }, + "isort": { + "hashes": [ + "sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1", + "sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd" + ], + "version": "==4.3.21" + }, + "lazy-object-proxy": { + "hashes": [ + "sha256:02b260c8deb80db09325b99edf62ae344ce9bc64d68b7a634410b8e9a568edbf", + "sha256:18f9c401083a4ba6e162355873f906315332ea7035803d0fd8166051e3d402e3", + "sha256:1f2c6209a8917c525c1e2b55a716135ca4658a3042b5122d4e3413a4030c26ce", + "sha256:2f06d97f0ca0f414f6b707c974aaf8829c2292c1c497642f63824119d770226f", + "sha256:616c94f8176808f4018b39f9638080ed86f96b55370b5a9463b2ee5c926f6c5f", + "sha256:63b91e30ef47ef68a30f0c3c278fbfe9822319c15f34b7538a829515b84ca2a0", + "sha256:77b454f03860b844f758c5d5c6e5f18d27de899a3db367f4af06bec2e6013a8e", + "sha256:83fe27ba321e4cfac466178606147d3c0aa18e8087507caec78ed5a966a64905", + "sha256:84742532d39f72df959d237912344d8a1764c2d03fe58beba96a87bfa11a76d8", + "sha256:874ebf3caaf55a020aeb08acead813baf5a305927a71ce88c9377970fe7ad3c2", + "sha256:9f5caf2c7436d44f3cec97c2fa7791f8a675170badbfa86e1992ca1b84c37009", + "sha256:a0c8758d01fcdfe7ae8e4b4017b13552efa7f1197dd7358dc9da0576f9d0328a", + "sha256:a4def978d9d28cda2d960c279318d46b327632686d82b4917516c36d4c274512", + "sha256:ad4f4be843dace866af5fc142509e9b9817ca0c59342fdb176ab6ad552c927f5", + "sha256:ae33dd198f772f714420c5ab698ff05ff900150486c648d29951e9c70694338e", + "sha256:b4a2b782b8a8c5522ad35c93e04d60e2ba7f7dcb9271ec8e8c3e08239be6c7b4", + "sha256:c462eb33f6abca3b34cdedbe84d761f31a60b814e173b98ede3c81bb48967c4f", + "sha256:fd135b8d35dfdcdb984828c84d695937e58cc5f49e1c854eb311c4d6aa03f4f1" + ], + "version": "==1.4.2" + }, + "mccabe": { + "hashes": [ + "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", + "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" + ], + "version": "==0.6.1" + }, + "mock": { + "hashes": [ + "sha256:83657d894c90d5681d62155c82bda9c1187827525880eda8ff5df4ec813437c3", + "sha256:d157e52d4e5b938c550f39eb2fd15610db062441a9c2747d3dbfa9298211d0f8" + ], + "version": "==3.0.5" + }, + "mypy": { + "hashes": [ + "sha256:1521c186a3d200c399bd5573c828ea2db1362af7209b2adb1bb8532cea2fb36f", + "sha256:31a046ab040a84a0fc38bc93694876398e62bc9f35eca8ccbf6418b7297f4c00", + "sha256:3b1a411909c84b2ae9b8283b58b48541654b918e8513c20a400bb946aa9111ae", + "sha256:48c8bc99380575deb39f5d3400ebb6a8a1cb5cc669bbba4d3bb30f904e0a0e7d", + "sha256:540c9caa57a22d0d5d3c69047cc9dd0094d49782603eb03069821b41f9e970e9", + "sha256:672e418425d957e276c291930a3921b4a6413204f53fe7c37cad7bc57b9a3391", + "sha256:6ed3b9b3fdc7193ea7aca6f3c20549b377a56f28769783a8f27191903a54170f", + "sha256:9371290aa2cad5ad133e4cdc43892778efd13293406f7340b9ffe99d5ec7c1d9", + "sha256:ace6ac1d0f87d4072f05b5468a084a45b4eda970e4d26704f201e06d47ab2990", + "sha256:b428f883d2b3fe1d052c630642cc6afddd07d5cd7873da948644508be3b9d4a7", + "sha256:d5bf0e6ec8ba346a2cf35cb55bf4adfddbc6b6576fcc9e10863daa523e418dbb", + "sha256:d7574e283f83c08501607586b3167728c58e8442947e027d2d4c7dcd6d82f453", + "sha256:dc889c84241a857c263a2b1cd1121507db7d5b5f5e87e77147097230f374d10b", + "sha256:f4748697b349f373002656bf32fede706a0e713d67bfdcf04edf39b1f61d46eb" + ], + "version": "==0.740" + }, + "mypy-extensions": { + "hashes": [ + "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d", + "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8" + ], + "version": "==0.4.3" + }, + "nose": { + "hashes": [ + "sha256:9ff7c6cc443f8c51994b34a667bbcf45afd6d945be7477b52e97516fd17c53ac", + "sha256:dadcddc0aefbf99eea214e0f1232b94f2fa9bd98fa8353711dacb112bfcbbb2a", + "sha256:f1bffef9cbc82628f6e7d7b40d7e255aefaa1adb6a1b1d26c69a8b79e6208a98" + ], + "version": "==1.3.7" + }, + "pep8-naming": { + "hashes": [ + "sha256:1b419fa45b68b61cd8c5daf4e0c96d28915ad14d3d5f35fcc1e7e95324a33a2e", + "sha256:4eedfd4c4b05e48796f74f5d8628c068ff788b9c2b08471ad408007fc6450e5a" + ], + "version": "==0.4.1" + }, + "pies": { + "hashes": [ + "sha256:79a652dddc64c6fa42c7dfe9686ae7b1d856391094b873e2f52fcd0bd662c102", + "sha256:e8a76923ce0e0f605240901983fe492814a65d3d803efe3013a0e1815b75e4e9" + ], + "version": "==2.6.7" + }, + "prospector": { + "extras": [ + "with_everything" + ], + "hashes": [ + "sha256:aba551e53dc1a5a432afa67385eaa81d7b4cf4c162dc1a4d0ee00b3a0712ad90" + ], + "index": "pypi", + "version": "==1.1.7" + }, + "pycodestyle": { + "hashes": [ + "sha256:cbc619d09254895b0d12c2c691e237b2e91e9b2ecf5e84c26b35400f93dcfb83", + "sha256:cbfca99bd594a10f674d0cd97a3d802a1fdef635d4361e1a2658de47ed261e3a" + ], + "version": "==2.4.0" + }, + "pydocstyle": { + "hashes": [ + "sha256:04c84e034ebb56eb6396c820442b8c4499ac5eb94a3bda88951ac3dc519b6058", + "sha256:66aff87ffe34b1e49bff2dd03a88ce6843be2f3346b0c9814410d34987fbab59" + ], + "version": "==4.0.1" + }, + "pyflakes": { + "hashes": [ + "sha256:08bd6a50edf8cffa9fa09a463063c425ecaaf10d1eb0335a7e8b1401aef89e6f", + "sha256:8d616a382f243dbf19b54743f280b80198be0bca3a5396f1d2e1fca6223e8805" + ], + "version": "==1.6.0" + }, + "pylint": { + "hashes": [ + "sha256:5d77031694a5fb97ea95e828c8d10fc770a1df6eb3906067aaed42201a8a6a09", + "sha256:723e3db49555abaf9bf79dc474c6b9e2935ad82230b10c1138a71ea41ac0fff1" + ], + "version": "==2.3.1" + }, + "pylint-celery": { + "hashes": [ + "sha256:41e32094e7408d15c044178ea828dd524beedbdbe6f83f712c5e35bde1de4beb" + ], + "version": "==0.3" + }, + "pylint-django": { + "hashes": [ + "sha256:75c69d1ec2275918c37f175976da20e2f1e1e62e067098a685cd263ffa833dfd", + "sha256:c7cb6384ea7b33ea77052a5ae07358c10d377807390ef27b2e6ff997303fadb7" + ], + "version": "==2.0.10" + }, + "pylint-flask": { + "hashes": [ + "sha256:f4d97de2216bf7bfce07c9c08b166e978fe9f2725de2a50a9845a97de7e31517" + ], + "version": "==0.6" + }, + "pylint-plugin-utils": { + "hashes": [ + "sha256:2f30510e1c46edf268d3a195b2849bd98a1b9433229bb2ba63b8d776e1fc4d0a", + "sha256:57625dcca20140f43731311cd8fd879318bf45a8b0fd17020717a8781714a25a" + ], + "version": "==0.6" + }, + "pyroma": { + "hashes": [ + "sha256:54d332f540d4828bc5672b75ccf9e12d4b2f72a42a4f304bcec1c73565aecc26", + "sha256:6b94feb609e1896579302f0836ef2fad3f17e0557e3ddcd0d76206cd3e366d27" + ], + "version": "==2.5" + }, + "pyyaml": { + "hashes": [ + "sha256:0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9", + "sha256:01adf0b6c6f61bd11af6e10ca52b7d4057dd0be0343eb9283c878cf3af56aee4", + "sha256:5124373960b0b3f4aa7df1707e63e9f109b5263eca5976c66e08b1c552d4eaf8", + "sha256:5ca4f10adbddae56d824b2c09668e91219bb178a1eee1faa56af6f99f11bf696", + "sha256:7907be34ffa3c5a32b60b95f4d95ea25361c951383a894fec31be7252b2b6f34", + "sha256:7ec9b2a4ed5cad025c2278a1e6a19c011c80a3caaac804fd2d329e9cc2c287c9", + "sha256:87ae4c829bb25b9fe99cf71fbb2140c448f534e24c998cc60f39ae4f94396a73", + "sha256:9de9919becc9cc2ff03637872a440195ac4241c80536632fffeb6a1e25a74299", + "sha256:a5a85b10e450c66b49f98846937e8cfca1db3127a9d5d1e31ca45c3d0bef4c5b", + "sha256:b0997827b4f6a7c286c01c5f60384d218dca4ed7d9efa945c3e1aa623d5709ae", + "sha256:b631ef96d3222e62861443cc89d6563ba3eeb816eeb96b2629345ab795e53681", + "sha256:bf47c0607522fdbca6c9e817a6e81b08491de50f3766a7a0e6a5be7905961b41", + "sha256:f81025eddd0327c7d4cfe9b62cf33190e1e736cc6e97502b3ec425f574b3e7a8" + ], + "version": "==5.1.2" + }, + "requests": { + "hashes": [ + "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4", + "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31" + ], + "index": "pypi", + "version": "==2.22.0" + }, + "requirements-detector": { + "hashes": [ + "sha256:9fbc4b24e8b7c3663aff32e3eba34596848c6b91bd425079b386973bd8d08931" + ], + "version": "==0.6" + }, + "setoptconf": { + "hashes": [ + "sha256:5b0b5d8e0077713f5d5152d4f63be6f048d9a1bb66be15d089a11c898c3cf49c" + ], + "version": "==0.2.0" + }, + "six": { + "hashes": [ + "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", + "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" + ], + "version": "==1.12.0" + }, + "snowballstemmer": { + "hashes": [ + "sha256:209f257d7533fdb3cb73bdbd24f436239ca3b2fa67d56f6ff88e86be08cc5ef0", + "sha256:df3bac3df4c2c01363f3dd2cfa78cce2840a79b9f1c2d2de9ce8d31683992f52" + ], + "version": "==2.0.0" + }, + "typed-ast": { + "hashes": [ + "sha256:1170afa46a3799e18b4c977777ce137bb53c7485379d9706af8a59f2ea1aa161", + "sha256:18511a0b3e7922276346bcb47e2ef9f38fb90fd31cb9223eed42c85d1312344e", + "sha256:262c247a82d005e43b5b7f69aff746370538e176131c32dda9cb0f324d27141e", + "sha256:2b907eb046d049bcd9892e3076c7a6456c93a25bebfe554e931620c90e6a25b0", + "sha256:354c16e5babd09f5cb0ee000d54cfa38401d8b8891eefa878ac772f827181a3c", + "sha256:48e5b1e71f25cfdef98b013263a88d7145879fbb2d5185f2a0c79fa7ebbeae47", + "sha256:4e0b70c6fc4d010f8107726af5fd37921b666f5b31d9331f0bd24ad9a088e631", + "sha256:630968c5cdee51a11c05a30453f8cd65e0cc1d2ad0d9192819df9978984529f4", + "sha256:66480f95b8167c9c5c5c87f32cf437d585937970f3fc24386f313a4c97b44e34", + "sha256:71211d26ffd12d63a83e079ff258ac9d56a1376a25bc80b1cdcdf601b855b90b", + "sha256:7954560051331d003b4e2b3eb822d9dd2e376fa4f6d98fee32f452f52dd6ebb2", + "sha256:838997f4310012cf2e1ad3803bce2f3402e9ffb71ded61b5ee22617b3a7f6b6e", + "sha256:95bd11af7eafc16e829af2d3df510cecfd4387f6453355188342c3e79a2ec87a", + "sha256:bc6c7d3fa1325a0c6613512a093bc2a2a15aeec350451cbdf9e1d4bffe3e3233", + "sha256:cc34a6f5b426748a507dd5d1de4c1978f2eb5626d51326e43280941206c209e1", + "sha256:d755f03c1e4a51e9b24d899561fec4ccaf51f210d52abdf8c07ee2849b212a36", + "sha256:d7c45933b1bdfaf9f36c579671fec15d25b06c8398f113dab64c18ed1adda01d", + "sha256:d896919306dd0aa22d0132f62a1b78d11aaf4c9fc5b3410d3c666b818191630a", + "sha256:fdc1c9bbf79510b76408840e009ed65958feba92a88833cdceecff93ae8fff66", + "sha256:ffde2fbfad571af120fcbfbbc61c72469e72f550d676c3342492a9dfdefb8f12" + ], + "markers": "implementation_name == 'cpython'", + "version": "==1.4.0" + }, + "typing-extensions": { + "hashes": [ + "sha256:2ed632b30bb54fc3941c382decfd0ee4148f5c591651c9272473fea2c6397d95", + "sha256:b1edbbf0652660e32ae780ac9433f4231e7339c7f9a8057d0f042fcbcea49b87", + "sha256:d8179012ec2c620d3791ca6fe2bf7979d979acdbef1fca0bc56b37411db682ed" + ], + "version": "==3.7.4" + }, + "urllib3": { + "hashes": [ + "sha256:3de946ffbed6e6746608990594d08faac602528ac7015ac28d33cee6a45b7398", + "sha256:9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86" + ], + "version": "==1.25.6" + }, + "vulture": { + "hashes": [ + "sha256:17be5f6a7c88ea43f2619f80338af7407275ee46a24000abe2570e59ca44b3d0", + "sha256:23d837cf619c3bb75f87bc498c79cd4f27f0c54031ca88a9e05606c9dd627fef" + ], + "version": "==0.24" + }, + "wrapt": { + "hashes": [ + "sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1" + ], + "version": "==1.11.2" + } + } +} diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..09f6205 --- /dev/null +++ b/TODO.md @@ -0,0 +1,6 @@ +# TODO + +- Check for `etcd3.exceptions.ConnectionFailedError` when calling some etcd operation to + avoid crashing whole application +- Throw KeyError instead of returning None when some key is not found in etcd +- Specify image format when using qemu-img when creating virtual machine \ No newline at end of file diff --git a/api/README.md b/api/README.md new file mode 100755 index 0000000..e28d676 --- /dev/null +++ b/api/README.md @@ -0,0 +1,12 @@ +# ucloud-api +[![Project Status: WIP – Initial development is in progress, but there has not yet been a stable, usable release suitable for the public.](https://www.repostatus.org/badges/latest/wip.svg)](https://www.repostatus.org/#wip) + +## Installation + +**Make sure you have Python >= 3.5 and Pipenv installed.** + +1. Clone the repository and `cd` into it. +2. Run the following commands + - `pipenv install` + - `pipenv shell` + - `python main.py` diff --git a/api/common_fields.py b/api/common_fields.py new file mode 100755 index 0000000..390fe40 --- /dev/null +++ b/api/common_fields.py @@ -0,0 +1,48 @@ +import os + +from config import etcd_client as client +from config import VM_PREFIX + + +class Field: + def __init__(self, _name, _type, _value=None): + self.name = _name + self.value = _value + self.type = _type + self.__errors = [] + + def validation(self): + return True + + def is_valid(self): + if self.value == KeyError: + self.add_error("'{}' field is a required field".format(self.name)) + else: + if not isinstance(self.value, self.type): + self.add_error("Incorrect Type for '{}' field".format(self.name)) + else: + self.validation() + + if self.__errors: + return False + return True + + def get_errors(self): + return self.__errors + + def add_error(self, error): + self.__errors.append(error) + + +class VmUUIDField(Field): + def __init__(self, data): + self.uuid = data.get("uuid", KeyError) + + super().__init__("uuid", str, self.uuid) + + self.validation = self.vm_uuid_validation + + def vm_uuid_validation(self): + r = client.get(os.path.join(VM_PREFIX, self.uuid)) + if not r: + self.add_error("VM with uuid {} does not exists".format(self.uuid)) diff --git a/api/config.py b/api/config.py new file mode 100644 index 0000000..4ae1b66 --- /dev/null +++ b/api/config.py @@ -0,0 +1,32 @@ +import logging + +from etcd3_wrapper import Etcd3Wrapper +from decouple import config + + +from ucloud_common.vm import VmPool +from ucloud_common.host import HostPool +from ucloud_common.request import RequestPool + +logging.basicConfig( + level=logging.DEBUG, + filename="log.txt", + filemode="a", + format="%(asctime)s: %(levelname)s - %(message)s", + datefmt="%d-%b-%y %H:%M:%S", +) + + +WITHOUT_CEPH = config("WITHOUT_CEPH", False, cast=bool) +VM_PREFIX = config("VM_PREFIX") +HOST_PREFIX = config("HOST_PREFIX") +REQUEST_PREFIX = config("REQUEST_PREFIX") +FILE_PREFIX = config("FILE_PREFIX") +IMAGE_PREFIX = config("IMAGE_PREFIX") +IMAGE_STORE_PREFIX = config("IMAGE_STORE_PREFIX") + +etcd_client = Etcd3Wrapper(host=config("ETCD_URL")) + +VM_POOL = VmPool(etcd_client, VM_PREFIX) +HOST_POOL = HostPool(etcd_client, HOST_PREFIX) +REQUEST_POOL = RequestPool(etcd_client, REQUEST_PREFIX) diff --git a/api/create_image_store.py b/api/create_image_store.py new file mode 100755 index 0000000..796cc43 --- /dev/null +++ b/api/create_image_store.py @@ -0,0 +1,17 @@ +import json +import os + +from uuid import uuid4 + +from config import etcd_client as client +from config import IMAGE_STORE_PREFIX + +data = { + "is_public": True, + "type": "ceph", + "name": "images", + "description": "first ever public image-store", + "attributes": {"list": [], "key": [], "pool": "images"}, +} + +client.put(os.path.join(IMAGE_STORE_PREFIX, uuid4().hex), json.dumps(data)) diff --git a/api/helper.py b/api/helper.py new file mode 100755 index 0000000..67a0379 --- /dev/null +++ b/api/helper.py @@ -0,0 +1,70 @@ +import binascii +import requests + +from decouple import config +from pyotp import TOTP +from config import VM_POOL + + +def check_otp(name, realm, token): + try: + data = { + "auth_name": config("AUTH_NAME", ""), + "auth_token": TOTP(config("AUTH_SEED", "")).now(), + "auth_realm": config("AUTH_REALM", ""), + "name": name, + "realm": realm, + "token": token, + } + except binascii.Error: + return 400 + + response = requests.get( + "{OTP_SERVER}{OTP_VERIFY_ENDPOINT}".format( + OTP_SERVER=config("OTP_SERVER", ""), + OTP_VERIFY_ENDPOINT=config("OTP_VERIFY_ENDPOINT", "verify"), + ), + json=data, + ) + return response.status_code + + +def resolve_vm_name(name, owner): + """Return UUID of Virtual Machine of name == name and owner == owner + + Input: name of vm, owner of vm. + Output: uuid of vm if found otherwise None + """ + result = next( + filter( + lambda vm: vm.value["owner"] == owner and vm.value["name"] == name, + VM_POOL.vms, + ), + None, + ) + if result: + return result.key.split("/")[-1] + + return None + +import random + +def random_bytes(num=6): + return [random.randrange(256) for _ in range(num)] + +def generate_mac(uaa=False, multicast=False, oui=None, separator=':', byte_fmt='%02x'): + mac = random_bytes() + if oui: + if type(oui) == str: + oui = [int(chunk) for chunk in oui.split(separator)] + mac = oui + random_bytes(num=6-len(oui)) + else: + if multicast: + mac[0] |= 1 # set bit 0 + else: + mac[0] &= ~1 # clear bit 0 + if uaa: + mac[0] &= ~(1 << 1) # clear bit 1 + else: + mac[0] |= 1 << 1 # set bit 1 + return separator.join(byte_fmt % b for b in mac) diff --git a/api/main.py b/api/main.py new file mode 100644 index 0000000..e8082d5 --- /dev/null +++ b/api/main.py @@ -0,0 +1,380 @@ +import json +import subprocess +import os +from uuid import uuid4 + +from flask import Flask, request +from flask_restful import Resource, Api + +from ucloud_common.vm import VMStatus +from ucloud_common.request import RequestEntry, RequestType + +from helper import generate_mac + +from config import ( + etcd_client, + WITHOUT_CEPH, + VM_PREFIX, + HOST_PREFIX, + FILE_PREFIX, + IMAGE_PREFIX, + logging, + REQUEST_POOL, + VM_POOL, + HOST_POOL, +) +from schemas import ( + CreateVMSchema, + VMStatusSchema, + CreateImageSchema, + VmActionSchema, + OTPSchema, + CreateHostSchema, + VmMigrationSchema, + AddSSHSchema, + RemoveSSHSchema, + GetSSHSchema +) + +app = Flask(__name__) +api = Api(app) + + +class CreateVM(Resource): + @staticmethod + def post(): + data = request.json + print(data) + validator = CreateVMSchema(data) + if validator.is_valid(): + vm_uuid = uuid4().hex + vm_key = os.path.join(VM_PREFIX, vm_uuid) + specs = { + 'cpu': validator.specs['cpu'], + 'ram': validator.specs['ram'], + 'os-ssd': validator.specs['os-ssd'], + 'hdd': validator.specs['hdd'] + } + + vm_entry = { + "name": data["vm_name"], + "owner": data["name"], + "owner_realm": data["realm"], + "specs": specs, + "hostname": "", + "status": "", + "image_uuid": data["image_uuid"], + "log": [], + "vnc_socket": "", + "mac": str(generate_mac()), + "metadata": { + "ssh-keys": [] + } + } + etcd_client.put(vm_key, vm_entry, value_in_json=True) + + # Create ScheduleVM Request + r = RequestEntry.from_scratch(type=RequestType.ScheduleVM, uuid=vm_uuid) + REQUEST_POOL.put(r) + + return {"message": "VM Creation Queued"}, 200 + return validator.get_errors(), 400 + + +class VmStatus(Resource): + @staticmethod + def get(): + data = request.json + validator = VMStatusSchema(data) + if validator.is_valid(): + vm = VM_POOL.get(os.path.join(VM_PREFIX, data["uuid"])) + return json.dumps(str(vm)) + else: + return validator.get_errors(), 400 + + +class CreateImage(Resource): + @staticmethod + def post(): + data = request.json + validator = CreateImageSchema(data) + if validator.is_valid(): + file_entry = etcd_client.get(os.path.join(FILE_PREFIX, data["uuid"])) + file_entry_value = json.loads(file_entry.value) + + image_entry_json = { + "status": "TO_BE_CREATED", + "owner": file_entry_value["owner"], + "filename": file_entry_value["filename"], + "name": data["name"], + "store_name": data["image_store"], + "visibility": "public", + } + etcd_client.put( + os.path.join(IMAGE_PREFIX, data["uuid"]), json.dumps(image_entry_json) + ) + + return {"message": "Image successfully created"} + return validator.get_errors(), 400 + + +class ListPublicImages(Resource): + @staticmethod + def get(): + images = etcd_client.get_prefix(IMAGE_PREFIX) + r = {} + for image in images: + r[image.key.split("/")[-1]] = json.loads(image.value) + return r, 200 + + +class VMAction(Resource): + @staticmethod + def post(): + data = request.json + validator = VmActionSchema(data) + + if validator.is_valid(): + vm_entry = VM_POOL.get(os.path.join(VM_PREFIX, data["uuid"])) + action = data["action"] + + if action == "start": + vm_entry.status = VMStatus.requested_start + VM_POOL.put(vm_entry) + action = "schedule" + + if action == "delete" and vm_entry.hostname == "": + try: + path_without_protocol = vm_entry.path[vm_entry.path.find(":") + 1 :] + + if WITHOUT_CEPH: + command_to_delete = [ + "rm", "-rf", + os.path.join("/var/vm", vm_entry.uuid), + ] + else: + command_to_delete = ["rbd", "rm", path_without_protocol] + + subprocess.check_output(command_to_delete, stderr=subprocess.PIPE) + except subprocess.CalledProcessError as e: + if "No such file" in e.stderr.decode("utf-8"): + etcd_client.client.delete(vm_entry.key) + return {"message": "VM successfully deleted"} + else: + logging.exception(e) + return {"message": "Some error occurred while deleting VM"} + else: + etcd_client.client.delete(vm_entry.key) + return {"message": "VM successfully deleted"} + + r = RequestEntry.from_scratch( + type="{}VM".format(action.title()), + uuid=data["uuid"], + hostname=vm_entry.hostname, + ) + REQUEST_POOL.put(r) + return {"message": "VM {} Queued".format(action.title())}, 200 + else: + return validator.get_errors(), 400 + + +class VMMigration(Resource): + @staticmethod + def post(): + data = request.json + validator = VmMigrationSchema(data) + + if validator.is_valid(): + vm = VM_POOL.get(data["uuid"]) + + r = RequestEntry.from_scratch( + type=RequestType.ScheduleVM, + uuid=vm.uuid, + destination=os.path.join(HOST_PREFIX, data["destination"]), + migration=True, + ) + REQUEST_POOL.put(r) + return {"message": "VM Migration Initialization Queued"}, 200 + else: + return validator.get_errors(), 400 + + +class ListUserVM(Resource): + @staticmethod + def get(): + data = request.json + validator = OTPSchema(data) + + if validator.is_valid(): + vms = etcd_client.get_prefix(VM_PREFIX, value_in_json=True) + return_vms = [] + user_vms = filter(lambda v: v.value["owner"] == data["name"], vms) + for vm in user_vms: + return_vms.append( + { + "name": vm.value["name"], + "vm_uuid": vm.key.split("/")[-1], + "specs": vm.value["specs"], + "status": vm.value["status"], + "hostname": vm.value["hostname"], + "mac": vm.value["mac"], + "vnc_socket": None + if vm.value.get("vnc_socket", None) is None + else vm.value["vnc_socket"], + } + ) + if return_vms: + return {"message": return_vms}, 200 + return {"message": "No VM found"}, 404 + + else: + return validator.get_errors(), 400 + + +class ListUserFiles(Resource): + @staticmethod + def get(): + data = request.json + validator = OTPSchema(data) + + if validator.is_valid(): + files = etcd_client.get_prefix(FILE_PREFIX, value_in_json=True) + return_files = [] + user_files = list(filter(lambda f: f.value["owner"] == data["name"], files)) + for file in user_files: + return_files.append( + { + "filename": file.value["filename"], + "uuid": file.key.split("/")[-1], + } + ) + return {"message": return_files}, 200 + else: + return validator.get_errors(), 400 + + +class CreateHost(Resource): + @staticmethod + def post(): + data = request.json + validator = CreateHostSchema(data) + if validator.is_valid(): + host_key = os.path.join(HOST_PREFIX, uuid4().hex) + host_entry = { + "specs": data["specs"], + "hostname": data["hostname"], + "status": "DEAD", + "last_heartbeat": "", + } + etcd_client.put(host_key, host_entry, value_in_json=True) + + return {"message": "Host Created"}, 200 + + return validator.get_errors(), 400 + + +class ListHost(Resource): + @staticmethod + def get(): + hosts = HOST_POOL.hosts + r = { + host.key: { + "status": host.status, + "specs": host.specs, + "hostname": host.hostname, + } + for host in hosts + } + return r, 200 + + +class GetSSHKeys(Resource): + @staticmethod + def get(): + data = request.json + validator = GetSSHSchema(data) + if validator.is_valid(): + if not validator.key_name.value: + + # {user_prefix}/{realm}/{name}/key/ + etcd_key = os.path.join(USER_PREFIX, data["realm"], data["name"], "key") + etcd_entry = etcd_client.get_prefix(etcd_key, value_in_json=True) + + keys = {key.key.split("/")[-1]: key.value for key in etcd_entry} + return {"keys": keys} + else: + + # {user_prefix}/{realm}/{name}/key/{key_name} + etcd_key = os.path.join(USER_PREFIX, data["realm"], data["name"], + "key", data["key_name"]) + etcd_entry = etcd_client.get(etcd_key, value_in_json=True) + + if etcd_entry: + return {"keys": {etcd_entry.key.split("/")[-1]: etcd_entry.value}} + else: + return {"keys": {}} + else: + return validator.get_errors(), 400 + + +class AddSSHKey(Resource): + @staticmethod + def post(): + data = request.json + validator = AddSSHSchema(data) + if validator.is_valid(): + + # {user_prefix}/{realm}/{name}/key/{key_name} + etcd_key = os.path.join(USER_PREFIX, data["realm"], data["name"], + "key", data["key_name"]) + etcd_entry = etcd_client.get(etcd_key, value_in_json=True) + if etcd_entry: + return {"message": "Key with name '{}' already exists".format(data["key_name"])} + else: + # Key Not Found. It implies user' haven't added any key yet. + etcd_client.put(etcd_key, data["key"], value_in_json=True) + return {"message": "Key added successfully"} + else: + return validator.get_errors(), 400 + + +class RemoveSSHKey(Resource): + @staticmethod + def get(): + data = request.json + validator = RemoveSSHSchema(data) + if validator.is_valid(): + + # {user_prefix}/{realm}/{name}/key/{key_name} + etcd_key = os.path.join(USER_PREFIX, data["realm"], data["name"], + "key", data["key_name"]) + etcd_entry = etcd_client.get(etcd_key, value_in_json=True) + if etcd_entry: + etcd_client.client.delete(etcd_key) + return {"message": "Key successfully removed."} + else: + return {"message": "No Key with name '{}' Exists at all.".format(data["key_name"])} + else: + return validator.get_errors(), 400 + +api.add_resource(CreateVM, "/vm/create") +api.add_resource(VmStatus, "/vm/status") + +api.add_resource(VMAction, "/vm/action") +api.add_resource(VMMigration, "/vm/migrate") + +api.add_resource(CreateImage, "/image/create") +api.add_resource(ListPublicImages, "/image/list-public") + +api.add_resource(ListUserVM, "/user/vms") +api.add_resource(ListUserFiles, "/user/files") + +api.add_resource(AddSSHKey, "/user/add-ssh") +api.add_resource(RemoveSSHKey, "/user/remove-ssh") +api.add_resource(GetSSHKeys, "/user/get-ssh") + +api.add_resource(CreateHost, "/host/create") +api.add_resource(ListHost, "/host/list") + +if __name__ == "__main__": + app.run(host="::", debug=True) diff --git a/api/schemas.py b/api/schemas.py new file mode 100755 index 0000000..8f366bf --- /dev/null +++ b/api/schemas.py @@ -0,0 +1,415 @@ +""" +This module contain classes thats validates and intercept/modify +data coming from ucloud-cli (user) + +It was primarily developed as an alternative to argument parser +of Flask_Restful which is going to be deprecated. I also tried +marshmallow for that purpose but it was an overkill (because it +do validation + serialization + deserialization) and little +inflexible for our purpose. +""" + +# TODO: Fix error message when user's mentioned VM (referred by name) +# does not exists. +# +# Currently, it says uuid is a required field. + +import json +import os +import bitmath + +from ucloud_common.host import HostPool, HostStatus +from ucloud_common.vm import VmPool, VMStatus + +from common_fields import Field, VmUUIDField +from helper import check_otp, resolve_vm_name +from config import etcd_client as client +from config import (HOST_PREFIX, VM_PREFIX, IMAGE_PREFIX, + FILE_PREFIX, IMAGE_STORE_PREFIX) + +HOST_POOL = HostPool(client, HOST_PREFIX) +VM_POOL = VmPool(client, VM_PREFIX) + + +class BaseSchema: + def __init__(self, data, fields=None): + _ = data # suppress linter warning + self.__errors = [] + if fields is None: + self.fields = [] + else: + self.fields = fields + + def validation(self): + # custom validation is optional + return True + + def is_valid(self): + for field in self.fields: + field.is_valid() + self.add_field_errors(field) + + for parent in self.__class__.__bases__: + try: + parent.validation(self) + except AttributeError: + pass + if not self.__errors: + self.validation() + + if self.__errors: + return False + return True + + def get_errors(self): + return {"message": self.__errors} + + def add_field_errors(self, field: Field): + self.__errors += field.get_errors() + + def add_error(self, error): + self.__errors.append(error) + + +class OTPSchema(BaseSchema): + def __init__(self, data: dict, fields=None): + self.name = Field("name", str, data.get("name", KeyError)) + self.realm = Field("realm", str, data.get("realm", KeyError)) + self.token = Field("token", str, data.get("token", KeyError)) + + _fields = [self.name, self.realm, self.token] + if fields: + _fields += fields + super().__init__(data=data, fields=_fields) + + def validation(self): + print(self.name.value, self.realm.value, self.token.value) + if check_otp(self.name.value, self.realm.value, self.token.value) != 200: + self.add_error("Wrong Credentials") + + +########################## Image Operations ############################################### + + +class CreateImageSchema(BaseSchema): + def __init__(self, data): + # Fields + self.uuid = Field("uuid", str, data.get("uuid", KeyError)) + self.name = Field("name", str, data.get("name", KeyError)) + self.image_store = Field("image_store", str, data.get("image_store", KeyError)) + + # Validations + self.uuid.validation = self.file_uuid_validation + self.image_store.validation = self.image_store_name_validation + + # All Fields + fields = [self.uuid, self.name, self.image_store] + super().__init__(data, fields) + + def file_uuid_validation(self): + file_entry = client.get(os.path.join(FILE_PREFIX, self.uuid.value)) + if file_entry is None: + self.add_error( + "Image File with uuid '{}' Not Found".format(self.uuid.value) + ) + + def image_store_name_validation(self): + image_stores = list(client.get_prefix(IMAGE_STORE_PREFIX)) + + image_store = next( + filter( + lambda s: json.loads(s.value)["name"] == self.image_store.value, + image_stores, + ), + None, + ) + if not image_store: + self.add_error("Store '{}' does not exists".format(self.image_store.value)) + + +# Host Operations + +class CreateHostSchema(OTPSchema): + def __init__(self, data): + self.parsed_specs = {} + # Fields + self.specs = Field("specs", dict, data.get("specs", KeyError)) + self.hostname = Field("hostname", str, data.get("hostname", KeyError)) + + # Validation + self.specs.validation = self.specs_validation + + fields = [self.hostname, self.specs] + + super().__init__(data=data, fields=fields) + + def specs_validation(self): + ALLOWED_BASE = 10 + + _cpu = self.specs.value.get('cpu', KeyError) + _ram = self.specs.value.get('ram', KeyError) + _os_ssd = self.specs.value.get('os-ssd', KeyError) + _hdd = self.specs.value.get('hdd', KeyError) + + if KeyError in [_cpu, _ram, _os_ssd, _hdd]: + self.add_error("You must specify CPU, RAM and OS-SSD in your specs") + return None + try: + parsed_ram = bitmath.parse_string_unsafe(_ram) + parsed_os_ssd = bitmath.parse_string_unsafe(_os_ssd) + + if parsed_ram.base != ALLOWED_BASE: + self.add_error("Your specified RAM is not in correct units") + if parsed_os_ssd.base != ALLOWED_BASE: + self.add_error("Your specified OS-SSD is not in correct units") + + if _cpu < 1: + self.add_error("CPU must be atleast 1") + + if parsed_ram < bitmath.GB(1): + self.add_error("RAM must be atleast 1 GB") + + if parsed_os_ssd < bitmath.GB(10): + self.add_error("OS-SSD must be atleast 10 GB") + + parsed_hdd = [] + for hdd in _hdd: + _parsed_hdd = bitmath.parse_string_unsafe(hdd) + if _parsed_hdd.base != ALLOWED_BASE: + self.add_error("Your specified HDD is not in correct units") + break + else: + parsed_hdd.append(str(_parsed_hdd)) + + except ValueError: + # TODO: Find some good error message + self.add_error("Specs are not correct.") + else: + if self.get_errors(): + self.specs = { + 'cpu': _cpu, + 'ram': str(parsed_ram), + 'os-ssd': str(parsed_os_ssd), + 'hdd': parsed_hdd + } + + def validation(self): + if self.realm.value != "ungleich-admin": + self.add_error("Invalid Credentials/Insufficient Permission") + + +# VM Operations + + +class CreateVMSchema(OTPSchema): + def __init__(self, data): + self.parsed_specs = {} + # Fields + self.specs = Field("specs", dict, data.get("specs", KeyError)) + self.vm_name = Field("vm_name", str, data.get("vm_name", KeyError)) + self.image_uuid = Field("image_uuid", str, data.get("image_uuid", KeyError)) + + # Validation + self.image_uuid.validation = self.image_uuid_validation + self.vm_name.validation = self.vm_name_validation + self.specs.validation = self.specs_validation + + fields = [self.vm_name, self.image_uuid, self.specs] + + super().__init__(data=data, fields=fields) + + def image_uuid_validation(self): + images = client.get_prefix(IMAGE_PREFIX) + + if self.image_uuid.value not in [i.key.split("/")[-1] for i in images]: + self.add_error("Image UUID not valid") + + def vm_name_validation(self): + if resolve_vm_name(name=self.vm_name.value, owner=self.name.value): + self.add_error( + 'VM with same name "{}" already exists'.format(self.vm_name.value) + ) + + def specs_validation(self): + ALLOWED_BASE = 10 + + _cpu = self.specs.value.get('cpu', KeyError) + _ram = self.specs.value.get('ram', KeyError) + _os_ssd = self.specs.value.get('os-ssd', KeyError) + _hdd = self.specs.value.get('hdd', KeyError) + + if KeyError in [_cpu, _ram, _os_ssd, _hdd]: + self.add_error("You must specify CPU, RAM and OS-SSD in your specs") + return None + try: + parsed_ram = bitmath.parse_string_unsafe(_ram) + parsed_os_ssd = bitmath.parse_string_unsafe(_os_ssd) + + if parsed_ram.base != ALLOWED_BASE: + self.add_error("Your specified RAM is not in correct units") + if parsed_os_ssd.base != ALLOWED_BASE: + self.add_error("Your specified OS-SSD is not in correct units") + + if _cpu < 1: + self.add_error("CPU must be atleast 1") + + if parsed_ram < bitmath.GB(1): + self.add_error("RAM must be atleast 1 GB") + + if parsed_os_ssd < bitmath.GB(1): + self.add_error("OS-SSD must be atleast 1 GB") + + parsed_hdd = [] + for hdd in _hdd: + _parsed_hdd = bitmath.parse_string_unsafe(hdd) + if _parsed_hdd.base != ALLOWED_BASE: + self.add_error("Your specified HDD is not in correct units") + break + else: + parsed_hdd.append(str(_parsed_hdd)) + + except ValueError: + # TODO: Find some good error message + self.add_error("Specs are not correct.") + else: + if self.get_errors(): + self.specs = { + 'cpu': _cpu, + 'ram': str(parsed_ram), + 'os-ssd': str(parsed_os_ssd), + 'hdd': parsed_hdd + } + + +class VMStatusSchema(OTPSchema): + def __init__(self, data): + data["uuid"] = ( + resolve_vm_name( + name=data.get("vm_name", None), + owner=(data.get("in_support_of", None) or data.get("name", None)), + ) + or KeyError + ) + self.uuid = VmUUIDField(data) + + fields = [self.uuid] + + super().__init__(data, fields) + + def validation(self): + vm = VM_POOL.get(self.uuid.value) + if not ( + vm.value["owner"] == self.name.value or self.realm.value == "ungleich-admin" + ): + self.add_error("Invalid User") + + +class VmActionSchema(OTPSchema): + def __init__(self, data): + data["uuid"] = ( + resolve_vm_name( + name=data.get("vm_name", None), + owner=(data.get("in_support_of", None) or data.get("name", None)), + ) + or KeyError + ) + self.uuid = VmUUIDField(data) + self.action = Field("action", str, data.get("action", KeyError)) + + self.action.validation = self.action_validation + + _fields = [self.uuid, self.action] + + super().__init__(data=data, fields=_fields) + + def action_validation(self): + allowed_actions = ["start", "stop", "delete"] + if self.action.value not in allowed_actions: + self.add_error( + "Invalid Action. Allowed Actions are {}".format(allowed_actions) + ) + + def validation(self): + vm = VM_POOL.get(self.uuid.value) + if not ( + vm.value["owner"] == self.name.value or self.realm.value == "ungleich-admin" + ): + self.add_error("Invalid User") + + if ( + self.action.value == "start" + and vm.status == VMStatus.running + and vm.hostname != "" + ): + self.add_error("VM Already Running") + + if self.action.value == "stop": + if vm.status == VMStatus.stopped: + self.add_error("VM Already Stopped") + elif vm.status != VMStatus.running: + self.add_error("Cannot stop non-running VM") + + +class VmMigrationSchema(OTPSchema): + def __init__(self, data): + data["uuid"] = ( + resolve_vm_name( + name=data.get("vm_name", None), + owner=(data.get("in_support_of", None) or data.get("name", None)), + ) + or KeyError + ) + + self.uuid = VmUUIDField(data) + self.destination = Field("destination", str, data.get("destination", KeyError)) + + self.destination.validation = self.destination_validation + + fields = [self.destination] + super().__init__(data=data, fields=fields) + + def destination_validation(self): + host_key = self.destination.value + host = HOST_POOL.get(host_key) + if not host: + self.add_error("No Such Host ({}) exists".format(self.destination.value)) + elif host.status != HostStatus.alive: + self.add_error("Destination Host is dead") + + def validation(self): + vm = VM_POOL.get(self.uuid.value) + if not ( + vm.value["owner"] == self.name.value or self.realm.value == "ungleich-admin" + ): + self.add_error("Invalid User") + + if vm.status != VMStatus.running: + self.add_error("Can't migrate non-running VM") + + if vm.hostname == os.path.join(HOST_PREFIX, self.destination.value): + self.add_error("Destination host couldn't be same as Source Host") + + +class AddSSHSchema(OTPSchema): + def __init__(self, data): + self.key_name = Field("key_name", str, data.get("key_name", KeyError)) + self.key = Field("key", str, data.get("key_name", KeyError)) + + fields = [self.key_name, self.key] + super().__init__(data=data, fields=fields) + + +class RemoveSSHSchema(OTPSchema): + def __init__(self, data): + self.key_name = Field("key_name", str, data.get("key_name", KeyError)) + + fields = [self.key_name] + super().__init__(data=data, fields=fields) + + +class GetSSHSchema(OTPSchema): + def __init__(self, data): + self.key_name = Field("key_name", str, data.get("key_name", None)) + + fields = [self.key_name] + super().__init__(data=data, fields=fields) \ No newline at end of file diff --git a/filescanner/main.py b/filescanner/main.py new file mode 100755 index 0000000..84ac53e --- /dev/null +++ b/filescanner/main.py @@ -0,0 +1,109 @@ +import os +import glob +import pathlib +import time +import hashlib +import subprocess as sp + +from decouple import config +from etcd3_wrapper import Etcd3Wrapper +from uuid import uuid4 + + + +def getxattr(file, attr): + try: + attr = "user." + attr + value = sp.check_output(['getfattr', file, + '--name', attr, + '--only-values', + '--absolute-names']) + value = value.decode("utf-8") + except sp.CalledProcessError: + value = None + + return value + +def setxattr(file, attr, value): + attr = "user." + attr + sp.check_output(['setfattr', file, + '--name', attr, + '--value', str(value)]) + + +def sha512sum(filename): + _sum = hashlib.sha512() + buffer_size = 2**16 + + with open(filename, "rb") as f: + while True: + data = f.read(buffer_size) + if not data: + break + _sum.update(data) + + return _sum.hexdigest() + + +try: + sp.check_output(['which', 'getfattr']) + sp.check_output(['which', 'setfattr']) +except Exception as e: + print(e) + print('Make sure you have getfattr and setfattr available') + exit(1) + + +BASE_DIR = config("BASE_DIR") + +FILE_PREFIX = config("FILE_PREFIX") + +etcd_client = Etcd3Wrapper(host=config("ETCD_URL")) + +# Recursively Get All Files and Folder below BASE_DIR +files = glob.glob("{}/**".format(BASE_DIR), recursive=True) + +# Retain only Files +files = list(filter(os.path.isfile, files)) + +untracked_files = list( + filter(lambda f: not bool(getxattr(f, "user.utracked")), files) +) + +tracked_files = list( + filter(lambda f: f not in untracked_files, files) +) +for file in untracked_files: + file_id = uuid4() + + # Get Username + owner = pathlib.Path(file).parts[3] + # Get Creation Date of File + # Here, we are assuming that ctime is creation time + # which is mostly not true. + creation_date = time.ctime(os.stat(file).st_ctime) + + # Get File Size + size = os.path.getsize(file) + + # Compute sha512 sum + sha_sum = sha512sum(file) + + # File Path excluding base and username + file_path = pathlib.Path(file).parts[4:] + file_path = os.path.join(*file_path) + + # Create Entry + entry_key = os.path.join(FILE_PREFIX, str(file_id)) + entry_value = { + "filename": file_path, + "owner": owner, + "sha512sum": sha_sum, + "creation_date": creation_date, + "size": size + } + + print("Tracking {}".format(file)) + # Insert Entry + etcd_client.put(entry_key, entry_value, value_in_json=True) + setxattr(file, "user.utracked", True) diff --git a/host/config.py b/host/config.py new file mode 100755 index 0000000..4191828 --- /dev/null +++ b/host/config.py @@ -0,0 +1,32 @@ +import logging + +from etcd3_wrapper import Etcd3Wrapper +from ucloud_common.vm import VmPool +from ucloud_common.host import HostPool +from ucloud_common.request import RequestPool +from decouple import config + +WITHOUT_CEPH = config("WITHOUT_CEPH", False, cast=bool) + +logging.basicConfig( + level=logging.DEBUG, + filename="log.txt", + filemode="a", + format="%(asctime)s: %(levelname)s - %(message)s", + datefmt="%d-%b-%y %H:%M:%S", +) + +etcd_client = Etcd3Wrapper(host=config("ETCD_URL")) + +HOST_PREFIX = config("HOST_PREFIX") +VM_PREFIX = config("VM_PREFIX") +REQUEST_PREFIX = config("REQUEST_PREFIX") +VM_DIR = config("VM_DIR") +IMAGE_DIR = config("IMAGE_DIR") + + +host_pool = HostPool(etcd_client, HOST_PREFIX) +vm_pool = VmPool(etcd_client, VM_PREFIX) +request_pool = RequestPool(etcd_client, REQUEST_PREFIX) + +running_vms = [] diff --git a/host/main.py b/host/main.py new file mode 100755 index 0000000..8fe73c9 --- /dev/null +++ b/host/main.py @@ -0,0 +1,137 @@ +import argparse +import threading +import time +import os +import sys +import virtualmachine + +from ucloud_common.host import HostEntry +from ucloud_common.request import RequestEntry, RequestType + +from config import (vm_pool, host_pool, request_pool, + etcd_client, logging, running_vms, + REQUEST_PREFIX, WITHOUT_CEPH) + + +def update_heartbeat(host: HostEntry): + while True: + host.update_heartbeat() + host_pool.put(host) + time.sleep(10) + + logging.info("Updated last heartbeat time %s", host.last_heartbeat) + + +def maintenance(host): + # To capture vm running according to running_vms list + + # This is to capture successful migration of a VM. + # Suppose, this host is running "vm1" and user initiated + # request to migrate this "vm1" to some other host. On, + # successful migration the destination host would set + # the vm hostname to itself. Thus, we are checking + # whether this host vm is successfully migrated. If yes + # then we shutdown "vm1" on this host. + + for running_vm in running_vms: + with vm_pool.get_put(running_vm.key) as vm_entry: + if vm_entry.hostname != host.key and not vm_entry.in_migration: + running_vm.handle.shutdown() + vm_entry.add_log("VM on source host shutdown.") + # To check vm running according to etcd entries + alleged_running_vms = vm_pool.by_status("RUNNING", vm_pool.by_host(host.key)) + + for vm_entry in alleged_running_vms: + _vm = virtualmachine.get_vm(running_vms, vm_entry.key) + + # Whether, the allegedly running vm is in our + # running_vms list or not if it is said to be + # running on this host but it is not then we + # need to shut it down + + # This is to capture poweroff/shutdown of a VM + # initiated by user inside VM. OR crash of VM by some + # user running process + if (_vm and not _vm.handle.is_running()) or not _vm: + vm_entry.add_log("""{} is not running but is said to be running. + So, shutting it down and declare it killed""".format(vm_entry.key)) + vm_entry.declare_killed() + vm_pool.put(vm_entry) + if _vm: + running_vms.remove(_vm) + + +def main(): + argparser = argparse.ArgumentParser() + argparser.add_argument("hostname", help="Name of this host. e.g /v1/host/1") + args = argparser.parse_args() + + host = host_pool.get(args.hostname) + if not host: + print("No Such Host") + exit(1) + + if WITHOUT_CEPH and not os.path.isdir("/var/vm"): + print("You have set WITHOUT_CEPH to True. So, the /var/vm must exists. But, it don't") + sys.exit(1) + + + logging.info("%s Session Started %s", '*' * 5, '*' * 5) + + # It is seen that under heavy load, timeout event doesn't come + # in a predictive manner (which is intentional because we give + # higher priority to customer's requests) which delays heart + # beat update which in turn misunderstood by scheduler that the + # host is dead when it is actually alive. So, to ensure that we + # update the heart beat in a predictive manner we start Heart + # beat updating mechanism in separated thread + + heartbeat_updating_thread = threading.Thread(target=update_heartbeat, args=(host,)) + try: + heartbeat_updating_thread.start() + except Exception as e: + logging.info("No Need To Go Further. Our heartbeat updating mechanism is not working") + logging.exception(e) + exit(-1) + + for events_iterator in [ + etcd_client.get_prefix(REQUEST_PREFIX, value_in_json=True), + etcd_client.watch_prefix(REQUEST_PREFIX, timeout=10, value_in_json=True), + ]: + for request_event in events_iterator: + request_event = RequestEntry(request_event) + + if request_event.type == "TIMEOUT": + logging.info("Timeout Event") + maintenance(host) + continue + + # If the event is directed toward me OR I am destination of a InitVMMigration + if (request_event.hostname == host.key or request_event.destination == host.key): + logging.debug("EVENT: %s", request_event) + + request_pool.client.client.delete(request_event.key) + vm_entry = vm_pool.get(request_event.uuid) + + if vm_entry: + if request_event.type == RequestType.StartVM: + virtualmachine.start(vm_entry) + + elif request_event.type == RequestType.StopVM: + virtualmachine.stop(vm_entry) + + elif request_event.type == RequestType.DeleteVM: + virtualmachine.delete(vm_entry) + + elif request_event.type == RequestType.InitVMMigration: + virtualmachine.init_migration(vm_entry, host.key) + + elif request_event.type == RequestType.TransferVM: + virtualmachine.transfer(request_event) + else: + logging.info("VM Entry missing") + + logging.info("Running VMs %s", running_vms) + + +main() diff --git a/host/qmp/__init__.py b/host/qmp/__init__.py new file mode 100755 index 0000000..97669bc --- /dev/null +++ b/host/qmp/__init__.py @@ -0,0 +1,534 @@ +# QEMU library +# +# Copyright (C) 2015-2016 Red Hat Inc. +# Copyright (C) 2012 IBM Corp. +# +# Authors: +# Fam Zheng +# +# This work is licensed under the terms of the GNU GPL, version 2. See +# the COPYING file in the top-level directory. +# +# Based on qmp.py. +# + +import errno +import logging +import os +import subprocess +import re +import shutil +import socket +import tempfile + +from . import qmp + + +LOG = logging.getLogger(__name__) + +# Mapping host architecture to any additional architectures it can +# support which often includes its 32 bit cousin. +ADDITIONAL_ARCHES = { + "x86_64" : "i386", + "aarch64" : "armhf" +} + +def kvm_available(target_arch=None): + host_arch = os.uname()[4] + if target_arch and target_arch != host_arch: + if target_arch != ADDITIONAL_ARCHES.get(host_arch): + return False + return os.access("/dev/kvm", os.R_OK | os.W_OK) + + +class QEMUMachineError(Exception): + """ + Exception called when an error in QEMUMachine happens. + """ + + +class QEMUMachineAddDeviceError(QEMUMachineError): + """ + Exception raised when a request to add a device can not be fulfilled + + The failures are caused by limitations, lack of information or conflicting + requests on the QEMUMachine methods. This exception does not represent + failures reported by the QEMU binary itself. + """ + +class MonitorResponseError(qmp.QMPError): + """ + Represents erroneous QMP monitor reply + """ + def __init__(self, reply): + try: + desc = reply["error"]["desc"] + except KeyError: + desc = reply + super(MonitorResponseError, self).__init__(desc) + self.reply = reply + + +class QEMUMachine(object): + """ + A QEMU VM + + Use this object as a context manager to ensure the QEMU process terminates:: + + with VM(binary) as vm: + ... + # vm is guaranteed to be shut down here + """ + + def __init__(self, binary, args=None, wrapper=None, name=None, + test_dir="/var/tmp", monitor_address=None, + socket_scm_helper=None): + ''' + Initialize a QEMUMachine + + @param binary: path to the qemu binary + @param args: list of extra arguments + @param wrapper: list of arguments used as prefix to qemu binary + @param name: prefix for socket and log file names (default: qemu-PID) + @param test_dir: where to create socket and log file + @param monitor_address: address for QMP monitor + @param socket_scm_helper: helper program, required for send_fd_scm() + @note: Qemu process is not started until launch() is used. + ''' + if args is None: + args = [] + if wrapper is None: + wrapper = [] + if name is None: + name = "qemu-%d" % os.getpid() + self._name = name + self._monitor_address = monitor_address + self._vm_monitor = None + self._qemu_log_path = None + self._qemu_log_file = None + self._popen = None + self._binary = binary + self._args = list(args) # Force copy args in case we modify them + self._wrapper = wrapper + self._events = [] + self._iolog = None + self._socket_scm_helper = socket_scm_helper + self._qmp = None + self._qemu_full_args = None + self._test_dir = test_dir + self._temp_dir = None + self._launched = False + self._machine = None + self._console_set = False + self._console_device_type = None + self._console_address = None + self._console_socket = None + + # just in case logging wasn't configured by the main script: + logging.basicConfig(level=logging.DEBUG) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.shutdown() + return False + + # This can be used to add an unused monitor instance. + def add_monitor_null(self): + self._args.append('-monitor') + self._args.append('null') + + def add_fd(self, fd, fdset, opaque, opts=''): + """ + Pass a file descriptor to the VM + """ + options = ['fd=%d' % fd, + 'set=%d' % fdset, + 'opaque=%s' % opaque] + if opts: + options.append(opts) + + # This did not exist before 3.4, but since then it is + # mandatory for our purpose + if hasattr(os, 'set_inheritable'): + os.set_inheritable(fd, True) + + self._args.append('-add-fd') + self._args.append(','.join(options)) + return self + + # Exactly one of fd and file_path must be given. + # (If it is file_path, the helper will open that file and pass its + # own fd) + def send_fd_scm(self, fd=None, file_path=None): + # In iotest.py, the qmp should always use unix socket. + assert self._qmp.is_scm_available() + if self._socket_scm_helper is None: + raise QEMUMachineError("No path to socket_scm_helper set") + if not os.path.exists(self._socket_scm_helper): + raise QEMUMachineError("%s does not exist" % + self._socket_scm_helper) + + # This did not exist before 3.4, but since then it is + # mandatory for our purpose + if hasattr(os, 'set_inheritable'): + os.set_inheritable(self._qmp.get_sock_fd(), True) + if fd is not None: + os.set_inheritable(fd, True) + + fd_param = ["%s" % self._socket_scm_helper, + "%d" % self._qmp.get_sock_fd()] + + if file_path is not None: + assert fd is None + fd_param.append(file_path) + else: + assert fd is not None + fd_param.append(str(fd)) + + devnull = open(os.path.devnull, 'rb') + proc = subprocess.Popen(fd_param, stdin=devnull, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, close_fds=False) + output = proc.communicate()[0] + if output: + LOG.debug(output) + + return proc.returncode + + @staticmethod + def _remove_if_exists(path): + """ + Remove file object at path if it exists + """ + try: + os.remove(path) + except OSError as exception: + if exception.errno == errno.ENOENT: + return + raise + + def is_running(self): + return self._popen is not None and self._popen.poll() is None + + def exitcode(self): + if self._popen is None: + return None + return self._popen.poll() + + def get_pid(self): + if not self.is_running(): + return None + return self._popen.pid + + def _load_io_log(self): + if self._qemu_log_path is not None: + with open(self._qemu_log_path, "r") as iolog: + self._iolog = iolog.read() + + def _base_args(self): + if isinstance(self._monitor_address, tuple): + moncdev = "socket,id=mon,host=%s,port=%s" % ( + self._monitor_address[0], + self._monitor_address[1]) + else: + moncdev = 'socket,id=mon,path=%s' % self._vm_monitor + args = ['-chardev', moncdev, + '-mon', 'chardev=mon,mode=control'] + if self._machine is not None: + args.extend(['-machine', self._machine]) + if self._console_set: + self._console_address = os.path.join(self._temp_dir, + self._name + "-console.sock") + chardev = ('socket,id=console,path=%s,server,nowait' % + self._console_address) + args.extend(['-chardev', chardev]) + if self._console_device_type is None: + args.extend(['-serial', 'chardev:console']) + else: + device = '%s,chardev=console' % self._console_device_type + args.extend(['-device', device]) + return args + + def _pre_launch(self): + self._temp_dir = tempfile.mkdtemp(dir=self._test_dir) + if self._monitor_address is not None: + self._vm_monitor = self._monitor_address + else: + self._vm_monitor = os.path.join(self._temp_dir, + self._name + "-monitor.sock") + self._qemu_log_path = os.path.join(self._temp_dir, self._name + ".log") + self._qemu_log_file = open(self._qemu_log_path, 'wb') + + self._qmp = qmp.QEMUMonitorProtocol(self._vm_monitor, + server=True) + + def _post_launch(self): + self._qmp.accept() + + def _post_shutdown(self): + if self._qemu_log_file is not None: + self._qemu_log_file.close() + self._qemu_log_file = None + + self._qemu_log_path = None + + if self._console_socket is not None: + self._console_socket.close() + self._console_socket = None + + if self._temp_dir is not None: + shutil.rmtree(self._temp_dir) + self._temp_dir = None + + def launch(self): + """ + Launch the VM and make sure we cleanup and expose the + command line/output in case of exception + """ + + if self._launched: + raise QEMUMachineError('VM already launched') + + self._iolog = None + self._qemu_full_args = None + try: + self._launch() + self._launched = True + except: + self.shutdown() + + LOG.debug('Error launching VM') + if self._qemu_full_args: + LOG.debug('Command: %r', ' '.join(self._qemu_full_args)) + if self._iolog: + LOG.debug('Output: %r', self._iolog) + raise + + def _launch(self): + """ + Launch the VM and establish a QMP connection + """ + devnull = open(os.path.devnull, 'rb') + self._pre_launch() + self._qemu_full_args = (self._wrapper + [self._binary] + + self._base_args() + self._args) + LOG.debug('VM launch command: %r', ' '.join(self._qemu_full_args)) + self._popen = subprocess.Popen(self._qemu_full_args, + stdin=devnull, + stdout=self._qemu_log_file, + stderr=subprocess.STDOUT, + shell=False, + close_fds=False) + self._post_launch() + + def wait(self): + """ + Wait for the VM to power off + """ + self._popen.wait() + self._qmp.close() + self._load_io_log() + self._post_shutdown() + + def shutdown(self): + """ + Terminate the VM and clean up + """ + if self.is_running(): + try: + self._qmp.cmd('quit') + self._qmp.close() + except: + self._popen.kill() + self._popen.wait() + + self._load_io_log() + self._post_shutdown() + + exitcode = self.exitcode() + if exitcode is not None and exitcode < 0: + msg = 'qemu received signal %i: %s' + if self._qemu_full_args: + command = ' '.join(self._qemu_full_args) + else: + command = '' + LOG.warn(msg, -exitcode, command) + + self._launched = False + + def qmp(self, cmd, conv_keys=True, **args): + """ + Invoke a QMP command and return the response dict + """ + qmp_args = dict() + for key, value in args.items(): + if conv_keys: + qmp_args[key.replace('_', '-')] = value + else: + qmp_args[key] = value + + return self._qmp.cmd(cmd, args=qmp_args) + + def command(self, cmd, conv_keys=True, **args): + """ + Invoke a QMP command. + On success return the response dict. + On failure raise an exception. + """ + reply = self.qmp(cmd, conv_keys, **args) + if reply is None: + raise qmp.QMPError("Monitor is closed") + if "error" in reply: + raise MonitorResponseError(reply) + return reply["return"] + + def get_qmp_event(self, wait=False): + """ + Poll for one queued QMP events and return it + """ + if len(self._events) > 0: + return self._events.pop(0) + return self._qmp.pull_event(wait=wait) + + def get_qmp_events(self, wait=False): + """ + Poll for queued QMP events and return a list of dicts + """ + events = self._qmp.get_events(wait=wait) + events.extend(self._events) + del self._events[:] + self._qmp.clear_events() + return events + + @staticmethod + def event_match(event, match=None): + """ + Check if an event matches optional match criteria. + + The match criteria takes the form of a matching subdict. The event is + checked to be a superset of the subdict, recursively, with matching + values whenever the subdict values are not None. + + This has a limitation that you cannot explicitly check for None values. + + Examples, with the subdict queries on the left: + - None matches any object. + - {"foo": None} matches {"foo": {"bar": 1}} + - {"foo": None} matches {"foo": 5} + - {"foo": {"abc": None}} does not match {"foo": {"bar": 1}} + - {"foo": {"rab": 2}} matches {"foo": {"bar": 1, "rab": 2}} + """ + if match is None: + return True + + try: + for key in match: + if key in event: + if not QEMUMachine.event_match(event[key], match[key]): + return False + else: + return False + return True + except TypeError: + # either match or event wasn't iterable (not a dict) + return match == event + + def event_wait(self, name, timeout=60.0, match=None): + """ + event_wait waits for and returns a named event from QMP with a timeout. + + name: The event to wait for. + timeout: QEMUMonitorProtocol.pull_event timeout parameter. + match: Optional match criteria. See event_match for details. + """ + return self.events_wait([(name, match)], timeout) + + def events_wait(self, events, timeout=60.0): + """ + events_wait waits for and returns a named event from QMP with a timeout. + + events: a sequence of (name, match_criteria) tuples. + The match criteria are optional and may be None. + See event_match for details. + timeout: QEMUMonitorProtocol.pull_event timeout parameter. + """ + def _match(event): + for name, match in events: + if (event['event'] == name and + self.event_match(event, match)): + return True + return False + + # Search cached events + for event in self._events: + if _match(event): + self._events.remove(event) + return event + + # Poll for new events + while True: + event = self._qmp.pull_event(wait=timeout) + if _match(event): + return event + self._events.append(event) + + return None + + def get_log(self): + """ + After self.shutdown or failed qemu execution, this returns the output + of the qemu process. + """ + return self._iolog + + def add_args(self, *args): + """ + Adds to the list of extra arguments to be given to the QEMU binary + """ + self._args.extend(args) + + def set_machine(self, machine_type): + """ + Sets the machine type + + If set, the machine type will be added to the base arguments + of the resulting QEMU command line. + """ + self._machine = machine_type + + def set_console(self, device_type=None): + """ + Sets the device type for a console device + + If set, the console device and a backing character device will + be added to the base arguments of the resulting QEMU command + line. + + This is a convenience method that will either use the provided + device type, or default to a "-serial chardev:console" command + line argument. + + The actual setting of command line arguments will be be done at + machine launch time, as it depends on the temporary directory + to be created. + + @param device_type: the device type, such as "isa-serial". If + None is given (the default value) a "-serial + chardev:console" command line argument will + be used instead, resorting to the machine's + default device type. + """ + self._console_set = True + self._console_device_type = device_type + + @property + def console_socket(self): + """ + Returns a socket connected to the console + """ + if self._console_socket is None: + self._console_socket = socket.socket(socket.AF_UNIX, + socket.SOCK_STREAM) + self._console_socket.connect(self._console_address) + return self._console_socket diff --git a/host/qmp/__pycache__/__init__.cpython-37.pyc b/host/qmp/__pycache__/__init__.cpython-37.pyc new file mode 100755 index 0000000..3f11efc Binary files /dev/null and b/host/qmp/__pycache__/__init__.cpython-37.pyc differ diff --git a/host/qmp/__pycache__/qmp.cpython-37.pyc b/host/qmp/__pycache__/qmp.cpython-37.pyc new file mode 100755 index 0000000..e9f7c94 Binary files /dev/null and b/host/qmp/__pycache__/qmp.cpython-37.pyc differ diff --git a/host/qmp/qmp.py b/host/qmp/qmp.py new file mode 100755 index 0000000..5c8cf6a --- /dev/null +++ b/host/qmp/qmp.py @@ -0,0 +1,256 @@ +# QEMU Monitor Protocol Python class +# +# Copyright (C) 2009, 2010 Red Hat Inc. +# +# Authors: +# Luiz Capitulino +# +# This work is licensed under the terms of the GNU GPL, version 2. See +# the COPYING file in the top-level directory. + +import json +import errno +import socket +import logging + + +class QMPError(Exception): + pass + + +class QMPConnectError(QMPError): + pass + + +class QMPCapabilitiesError(QMPError): + pass + + +class QMPTimeoutError(QMPError): + pass + + +class QEMUMonitorProtocol(object): + + #: Logger object for debugging messages + logger = logging.getLogger('QMP') + #: Socket's error class + error = socket.error + #: Socket's timeout + timeout = socket.timeout + + def __init__(self, address, server=False): + """ + Create a QEMUMonitorProtocol class. + + @param address: QEMU address, can be either a unix socket path (string) + or a tuple in the form ( address, port ) for a TCP + connection + @param server: server mode listens on the socket (bool) + @raise socket.error on socket connection errors + @note No connection is established, this is done by the connect() or + accept() methods + """ + self.__events = [] + self.__address = address + self.__sock = self.__get_sock() + self.__sockfile = None + if server: + self.__sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self.__sock.bind(self.__address) + self.__sock.listen(1) + + def __get_sock(self): + if isinstance(self.__address, tuple): + family = socket.AF_INET + else: + family = socket.AF_UNIX + return socket.socket(family, socket.SOCK_STREAM) + + def __negotiate_capabilities(self): + greeting = self.__json_read() + if greeting is None or "QMP" not in greeting: + raise QMPConnectError + # Greeting seems ok, negotiate capabilities + resp = self.cmd('qmp_capabilities') + if "return" in resp: + return greeting + raise QMPCapabilitiesError + + def __json_read(self, only_event=False): + while True: + data = self.__sockfile.readline() + if not data: + return + resp = json.loads(data) + if 'event' in resp: + self.logger.debug("<<< %s", resp) + self.__events.append(resp) + if not only_event: + continue + return resp + + def __get_events(self, wait=False): + """ + Check for new events in the stream and cache them in __events. + + @param wait (bool): block until an event is available. + @param wait (float): If wait is a float, treat it as a timeout value. + + @raise QMPTimeoutError: If a timeout float is provided and the timeout + period elapses. + @raise QMPConnectError: If wait is True but no events could be + retrieved or if some other error occurred. + """ + + # Check for new events regardless and pull them into the cache: + self.__sock.setblocking(0) + try: + self.__json_read() + except socket.error as err: + if err[0] == errno.EAGAIN: + # No data available + pass + self.__sock.setblocking(1) + + # Wait for new events, if needed. + # if wait is 0.0, this means "no wait" and is also implicitly false. + if not self.__events and wait: + if isinstance(wait, float): + self.__sock.settimeout(wait) + try: + ret = self.__json_read(only_event=True) + except socket.timeout: + raise QMPTimeoutError("Timeout waiting for event") + except: + raise QMPConnectError("Error while reading from socket") + if ret is None: + raise QMPConnectError("Error while reading from socket") + self.__sock.settimeout(None) + + def connect(self, negotiate=True): + """ + Connect to the QMP Monitor and perform capabilities negotiation. + + @return QMP greeting dict + @raise socket.error on socket connection errors + @raise QMPConnectError if the greeting is not received + @raise QMPCapabilitiesError if fails to negotiate capabilities + """ + self.__sock.connect(self.__address) + self.__sockfile = self.__sock.makefile() + if negotiate: + return self.__negotiate_capabilities() + + def accept(self): + """ + Await connection from QMP Monitor and perform capabilities negotiation. + + @return QMP greeting dict + @raise socket.error on socket connection errors + @raise QMPConnectError if the greeting is not received + @raise QMPCapabilitiesError if fails to negotiate capabilities + """ + self.__sock.settimeout(15) + self.__sock, _ = self.__sock.accept() + self.__sockfile = self.__sock.makefile() + return self.__negotiate_capabilities() + + def cmd_obj(self, qmp_cmd): + """ + Send a QMP command to the QMP Monitor. + + @param qmp_cmd: QMP command to be sent as a Python dict + @return QMP response as a Python dict or None if the connection has + been closed + """ + self.logger.debug(">>> %s", qmp_cmd) + try: + self.__sock.sendall(json.dumps(qmp_cmd).encode('utf-8')) + except socket.error as err: + if err[0] == errno.EPIPE: + return + raise socket.error(err) + resp = self.__json_read() + self.logger.debug("<<< %s", resp) + return resp + + def cmd(self, name, args=None, cmd_id=None): + """ + Build a QMP command and send it to the QMP Monitor. + + @param name: command name (string) + @param args: command arguments (dict) + @param cmd_id: command id (dict, list, string or int) + """ + qmp_cmd = {'execute': name} + if args: + qmp_cmd['arguments'] = args + if cmd_id: + qmp_cmd['id'] = cmd_id + return self.cmd_obj(qmp_cmd) + + def command(self, cmd, **kwds): + """ + Build and send a QMP command to the monitor, report errors if any + """ + ret = self.cmd(cmd, kwds) + if "error" in ret: + raise Exception(ret['error']['desc']) + return ret['return'] + + def pull_event(self, wait=False): + """ + Pulls a single event. + + @param wait (bool): block until an event is available. + @param wait (float): If wait is a float, treat it as a timeout value. + + @raise QMPTimeoutError: If a timeout float is provided and the timeout + period elapses. + @raise QMPConnectError: If wait is True but no events could be + retrieved or if some other error occurred. + + @return The first available QMP event, or None. + """ + self.__get_events(wait) + + if self.__events: + return self.__events.pop(0) + return None + + def get_events(self, wait=False): + """ + Get a list of available QMP events. + + @param wait (bool): block until an event is available. + @param wait (float): If wait is a float, treat it as a timeout value. + + @raise QMPTimeoutError: If a timeout float is provided and the timeout + period elapses. + @raise QMPConnectError: If wait is True but no events could be + retrieved or if some other error occurred. + + @return The list of available QMP events. + """ + self.__get_events(wait) + return self.__events + + def clear_events(self): + """ + Clear current list of pending events. + """ + self.__events = [] + + def close(self): + self.__sock.close() + self.__sockfile.close() + + def settimeout(self, timeout): + self.__sock.settimeout(timeout) + + def get_sock_fd(self): + return self.__sock.fileno() + + def is_scm_available(self): + return self.__sock.family == socket.AF_UNIX diff --git a/host/virtualmachine.py b/host/virtualmachine.py new file mode 100755 index 0000000..f99ffd0 --- /dev/null +++ b/host/virtualmachine.py @@ -0,0 +1,310 @@ +# QEMU Manual +# https://qemu.weilnetz.de/doc/qemu-doc.html + +# For QEMU Monitor Protocol Commands Information, See +# https://qemu.weilnetz.de/doc/qemu-doc.html#pcsys_005fmonitor + +import errno +import os +import subprocess +import tempfile +import time + +from functools import wraps +from os.path import join +from typing import Union + +import bitmath +import sshtunnel +from decouple import config + +import qmp +from config import (WITHOUT_CEPH, VM_PREFIX, VM_DIR, IMAGE_DIR, + etcd_client, logging, request_pool, + running_vms, vm_pool) +from ucloud_common.helpers import get_ipv4_address +from ucloud_common.request import RequestEntry, RequestType +from ucloud_common.vm import VMEntry, VMStatus + + +class VM: + def __init__(self, key, handle, vnc_socket_file): + self.key = key # type: str + self.handle = handle # type: qmp.QEMUMachine + self.vnc_socket_file = vnc_socket_file # type: tempfile.NamedTemporaryFile + + def __repr__(self): + return "VM({})".format(self.key) + + +def get_start_command_args( + vm_entry, vnc_sock_filename: str, migration=False, migration_port=4444 +): + threads_per_core = 1 + vm_memory = int(bitmath.parse_string(vm_entry.specs["ram"]).to_MB()) + vm_cpus = int(vm_entry.specs["cpu"]) + vm_uuid = vm_entry.uuid + + if WITHOUT_CEPH: + command = "-drive file={},format=raw,if=virtio,cache=none".format( + os.path.join(VM_DIR, vm_uuid) + ) + else: + command = "-drive file=rbd:uservms/{},format=raw,if=virtio,cache=none".format( + vm_uuid + ) + + command += " -device virtio-rng-pci -vnc unix:{}".format(vnc_sock_filename) + command += " -m {} -smp cores={},threads={}".format( + vm_memory, vm_cpus, threads_per_core + ) + command += " -name {}".format(vm_uuid) + + if migration: + command += " -incoming tcp:0:{}".format(migration_port) + + command += " -nic tap,model=virtio,mac={}".format(vm_entry.mac) + return command.split(" ") + + +def create_vm_object(vm_entry, migration=False, migration_port=4444): + # NOTE: If migration suddenly stop working, having different + # VNC unix filename on source and destination host can + # be a possible cause of it. + + # REQUIREMENT: Use Unix Socket instead of TCP Port for VNC + vnc_sock_file = tempfile.NamedTemporaryFile() + + qemu_args = get_start_command_args( + vm_entry=vm_entry, + vnc_sock_filename=vnc_sock_file.name, + migration=migration, + migration_port=migration_port, + ) + qemu_machine = qmp.QEMUMachine("/usr/bin/qemu-system-x86_64", args=qemu_args) + return VM(vm_entry.key, qemu_machine, vnc_sock_file) + + +def get_vm(vm_list: list, vm_key) -> Union[VM, None]: + return next((vm for vm in vm_list if vm.key == vm_key), None) + + +def need_running_vm(func): + @wraps(func) + def wrapper(e): + vm = get_vm(running_vms, e.key) + if vm: + try: + status = vm.handle.command("query-status") + logging.debug("VM Status Check - %s", status) + except Exception as exception: + logging.info("%s failed - VM %s %s", func.__name__, e, exception) + else: + return func(e) + + return None + else: + logging.info("%s failed because VM %s is not running", func.__name__, e.key) + return None + + return wrapper + + +def create(vm_entry: VMEntry): + vm_hdd = int(bitmath.parse_string(vm_entry.specs["os-ssd"]).to_MB()) + + if WITHOUT_CEPH: + _command_to_create = [ + "cp", + os.path.join(IMAGE_DIR, vm_entry.image_uuid), + os.path.join(VM_DIR, vm_entry.uuid), + ] + + _command_to_extend = [ + "qemu-img", + "resize", + "-f", "raw", + os.path.join(VM_DIR, vm_entry.uuid), + "{}M".format(vm_hdd), + ] + else: + _command_to_create = [ + "rbd", + "clone", + "images/{}@protected".format(vm_entry.image_uuid), + "uservms/{}".format(vm_entry.uuid), + ] + + _command_to_extend = [ + "rbd", + "resize", + "uservms/{}".format(vm_entry.uuid), + "--size", + vm_hdd, + ] + + try: + subprocess.check_output(_command_to_create) + except subprocess.CalledProcessError as e: + if e.returncode == errno.EEXIST: + logging.debug("Image for vm %s exists", vm_entry.uuid) + # File Already exists. No Problem Continue + return + + # This exception catches all other exceptions + # i.e FileNotFound (BaseImage), pool Does Not Exists etc. + logging.exception(e) + + vm_entry.status = "ERROR" + else: + try: + subprocess.check_output(_command_to_extend) + except Exception as e: + logging.exception(e) + else: + logging.info("New VM Created") + + +def start(vm_entry: VMEntry): + _vm = get_vm(running_vms, vm_entry.key) + + # VM already running. No need to proceed further. + if _vm: + logging.info("VM %s already running", vm_entry.uuid) + return + else: + create(vm_entry) + launch_vm(vm_entry) + + +@need_running_vm +def stop(vm_entry): + vm = get_vm(running_vms, vm_entry.key) + vm.handle.shutdown() + if not vm.handle.is_running(): + vm_entry.add_log("Shutdown successfully") + vm_entry.declare_stopped() + vm_pool.put(vm_entry) + running_vms.remove(vm) + + +def delete(vm_entry): + logging.info("Deleting VM | %s", vm_entry) + stop(vm_entry) + path_without_protocol = vm_entry.path[vm_entry.path.find(":") + 1 :] + + if WITHOUT_CEPH: + vm_deletion_command = ["rm", os.path.join(VM_DIR, vm_entry.uuid)] + else: + vm_deletion_command = ["rbd", "rm", path_without_protocol] + + try: + subprocess.check_output(vm_deletion_command) + except Exception as e: + logging.exception(e) + else: + etcd_client.client.delete(vm_entry.key) + + +def transfer(request_event): + # This function would run on source host i.e host on which the vm + # is running initially. This host would be responsible for transferring + # vm state to destination host. + + _host, _port = request_event.parameters["host"], request_event.parameters["port"] + _uuid = request_event.uuid + _destination = request_event.destination_host_key + vm = get_vm(running_vms, join(VM_PREFIX, _uuid)) + + if vm: + tunnel = sshtunnel.SSHTunnelForwarder( + (_host, 22), + ssh_username=config("ssh_username"), + ssh_pkey=config("ssh_pkey"), + ssh_private_key_password=config("ssh_private_key_password"), + remote_bind_address=("127.0.0.1", _port), + ) + try: + tunnel.start() + except sshtunnel.BaseSSHTunnelForwarderError: + logging.exception("Couldn't establish connection to (%s, 22)", _host) + else: + vm.handle.command( + "migrate", uri="tcp:{}:{}".format(_host, tunnel.local_bind_port) + ) + + status = vm.handle.command("query-migrate")["status"] + while status not in ["failed", "completed"]: + time.sleep(2) + status = vm.handle.command("query-migrate")["status"] + + with vm_pool.get_put(request_event.uuid) as source_vm: + if status == "failed": + source_vm.add_log("Migration Failed") + elif status == "completed": + # If VM is successfully migrated then shutdown the VM + # on this host and update hostname to destination host key + source_vm.add_log("Successfully migrated") + source_vm.hostname = _destination + running_vms.remove(vm) + vm.handle.shutdown() + source_vm.in_migration = False # VM transfer finished + finally: + tunnel.close() + + +def init_migration(vm_entry, destination_host_key): + # This function would run on destination host i.e host on which the vm + # would be transferred after migration. + # This host would be responsible for starting VM that would receive + # state of VM running on source host. + + _vm = get_vm(running_vms, vm_entry.key) + + if _vm: + # VM already running. No need to proceed further. + logging.info("%s Already running", _vm.key) + return + + launch_vm(vm_entry, migration=True, migration_port=4444, + destination_host_key=destination_host_key) + + +def launch_vm(vm_entry, migration=False, migration_port=None, destination_host_key=None): + logging.info("Starting %s", vm_entry.key) + + vm = create_vm_object(vm_entry, migration=migration, migration_port=migration_port) + try: + vm.handle.launch() + except Exception as e: + logging.exception(e) + + if migration: + # We don't care whether MachineError or any other error occurred + vm.handle.shutdown() + else: + # Error during typical launch of a vm + vm_entry.add_log("Error Occurred while starting VM") + vm_entry.declare_killed() + vm_pool.put(vm_entry) + else: + vm_entry.vnc_socket = vm.vnc_socket_file.name + running_vms.append(vm) + + if migration: + vm_entry.in_migration = True + r = RequestEntry.from_scratch( + type=RequestType.TransferVM, + hostname=vm_entry.hostname, + parameters={"host": get_ipv4_address(), "port": 4444}, + uuid=vm_entry.uuid, + destination_host_key=destination_host_key, + ) + request_pool.put(r) + else: + # Typical launching of a vm + vm_entry.status = VMStatus.running + vm_entry.add_log("Started successfully") + + vm_pool.put(vm_entry) + \ No newline at end of file diff --git a/imagescanner/config.py b/imagescanner/config.py new file mode 100755 index 0000000..3ccc06b --- /dev/null +++ b/imagescanner/config.py @@ -0,0 +1,22 @@ +import logging + +from etcd3_wrapper import Etcd3Wrapper +from decouple import config + +BASE_PATH = config("BASE_DIR", "/var/www") +WITHOUT_CEPH = config("WITHOUT_CEPH", False, cast=bool) +ETCD_URL = config("ETCD_URL") +IMAGE_PREFIX = config("IMAGE_PREFIX") +IMAGE_STORE_PREFIX = config("IMAGE_STORE_PREFIX") +IMAGE_DIR = config("IMAGE_DIR") + +logging.basicConfig( + level=logging.DEBUG, + filename="log.txt", + filemode="a", + format="%(asctime)s: %(levelname)s - %(message)s", + datefmt="%d-%b-%y %H:%M:%S", +) + + +client = Etcd3Wrapper(host=ETCD_URL) diff --git a/imagescanner/main.py b/imagescanner/main.py new file mode 100755 index 0000000..f0956ac --- /dev/null +++ b/imagescanner/main.py @@ -0,0 +1,108 @@ +import os +import json +import subprocess +import sys + +from config import (logging, client, IMAGE_DIR, + BASE_PATH, WITHOUT_CEPH, + IMAGE_PREFIX, IMAGE_STORE_PREFIX) + + +def qemu_img_type(path): + qemu_img_info_command = ["qemu-img", "info", "--output", "json", path] + try: + qemu_img_info = subprocess.check_output(qemu_img_info_command) + except Exception as e: + logging.exception(e) + return None + else: + qemu_img_info = json.loads(qemu_img_info.decode("utf-8")) + return qemu_img_info["format"] + +# If you are using WITHOUT_CEPH FLAG in .env +# then please make sure that IMAGE_DIR directory +# exists otherwise this script would fail +if WITHOUT_CEPH and not os.path.isdir(IMAGE_DIR): + print("You have set WITHOUT_CEPH to True. So," + "the {} must exists. But, it don't".format(IMAGE_DIR)) + sys.exit(1) + +try: + subprocess.check_output(['which', 'qemu-img']) +except Exception: + print("qemu-img missing") + sys.exit(1) + +# We want to get images entries that requests images to be created +images = client.get_prefix(IMAGE_PREFIX, value_in_json=True) +images_to_be_created = list(filter(lambda im: im.value['status'] == 'TO_BE_CREATED', images)) + +for image in images_to_be_created: + try: + image_uuid = image.key.split('/')[-1] + image_owner = image.value['owner'] + image_filename = image.value['filename'] + image_store_name = image.value['store_name'] + image_full_path = os.path.join(BASE_PATH, image_owner, image_filename) + + image_stores = client.get_prefix(IMAGE_STORE_PREFIX, value_in_json=True) + user_image_store = next(filter( + lambda s, store_name=image_store_name: s.value["name"] == store_name, + image_stores + )) + + image_store_pool = user_image_store.value['attributes']['pool'] + + except Exception as e: + logging.exception(e) + else: + # At least our basic data is available + + qemu_img_convert_command = ["qemu-img", "convert", "-f", "qcow2", + "-O", "raw", image_full_path, "image.raw"] + + + if WITHOUT_CEPH: + image_import_command = ["mv", "image.raw", os.path.join(IMAGE_DIR, image_uuid)] + snapshot_creation_command = ["true"] + snapshot_protect_command = ["true"] + else: + image_import_command = ["rbd", "import", "image.raw", + "{}/{}".format(image_store_pool, image_uuid)] + snapshot_creation_command = ["rbd", "snap", "create", + "{}/{}@protected".format(image_store_pool, image_uuid)] + snapshot_protect_command = ["rbd", "snap", "protect", + "{}/{}@protected".format(image_store_pool, image_uuid)] + + + # First check whether the image is qcow2 + + if qemu_img_type(image_full_path) == "qcow2": + try: + # Convert .qcow2 to .raw + subprocess.check_output(qemu_img_convert_command) + + # Import image either to ceph/filesystem + subprocess.check_output(image_import_command) + + # Create and Protect Snapshot + subprocess.check_output(snapshot_creation_command) + subprocess.check_output(snapshot_protect_command) + + except Exception as e: + logging.exception(e) + + else: + # Everything is successfully done + image.value["status"] = "CREATED" + client.put(image.key, json.dumps(image.value)) + else: + # The user provided image is either not found or of invalid format + image.value["status"] = "INVALID_IMAGE" + client.put(image.key, json.dumps(image.value)) + + + try: + os.remove("image.raw") + except Exception: + pass diff --git a/metadata/config.py b/metadata/config.py new file mode 100644 index 0000000..0df4102 --- /dev/null +++ b/metadata/config.py @@ -0,0 +1,21 @@ +import logging + +from etcd3_wrapper import Etcd3Wrapper +from decouple import config + +from ucloud_common.vm import VmPool + +logging.basicConfig( + level=logging.DEBUG, + filename="log.txt", + filemode="a", + format="%(asctime)s: %(levelname)s - %(message)s", + datefmt="%d-%b-%y %H:%M:%S", +) + + +VM_PREFIX = config("VM_PREFIX") + +etcd_client = Etcd3Wrapper(host=config("ETCD_URL")) + +VM_POOL = VmPool(etcd_client, VM_PREFIX) diff --git a/metadata/main.py b/metadata/main.py new file mode 100644 index 0000000..d9a0bb7 --- /dev/null +++ b/metadata/main.py @@ -0,0 +1,84 @@ +from flask import Flask, request +from flask_restful import Resource, Api +from config import etcd_client, VM_POOL + +app = Flask(__name__) +api = Api(app) + + +def get_vm_entry(mac_addr): + return next(filter(lambda vm: vm.mac == mac_addr, VM_POOL.vms), None) + + +# https://stackoverflow.com/questions/37140846/how-to-convert-ipv6-link-local-address-to-mac-address-in-python +def ipv62mac(ipv6): + # remove subnet info if given + subnet_index = ipv6.find('/') + if subnet_index != -1: + ipv6 = ipv6[:subnet_index] + + ipv6_parts = ipv6.split(':') + mac_parts = list() + for ipv6_part in ipv6_parts[-4:]: + while len(ipv6_part) < 4: + ipv6_part = '0' + ipv6_part + mac_parts.append(ipv6_part[:2]) + mac_parts.append(ipv6_part[-2:]) + + # modify parts to match MAC value + mac_parts[0] = '%02x' % (int(mac_parts[0], 16) ^ 2) + del mac_parts[4] + del mac_parts[3] + + return ':'.join(mac_parts) + + +class Root(Resource): + @staticmethod + def get(): + data = get_vm_entry(ipv62mac(request.remote_addr)) + + if not data: + return {'message': 'Metadata for such VM does not exists.'}, 404 + else: + + # {user_prefix}/{realm}/{name}/key + etcd_key = os.path.join(USER_PREFIX, data.value['owner_realm'], + data.value['owner'], 'key') + etcd_entry = etcd_client.get_prefix(etcd_key, value_in_json=True) + user_personal_ssh_keys = [key.value for key in etcd_entry] + data.value['metadata']['ssh-keys'] += user_personal_ssh_keys + return data.value['metadata'], 200 + + @staticmethod + def post(): + return {'message': 'Previous Implementation is deprecated.'} + # data = etcd_client.get("/v1/metadata/{}".format(request.remote_addr), value_in_json=True) + # print(data) + # if data: + # for k in request.json: + # if k not in data.value: + # data.value[k] = request.json[k] + # if k.endswith("-list"): + # data.value[k] = [request.json[k]] + # else: + # if k.endswith("-list"): + # data.value[k].append(request.json[k]) + # else: + # data.value[k] = request.json[k] + # etcd_client.put("/v1/metadata/{}".format(request.remote_addr), + # data.value, value_in_json=True) + # else: + # data = {} + # for k in request.json: + # data[k] = request.json[k] + # if k.endswith("-list"): + # data[k] = [request.json[k]] + # etcd_client.put("/v1/metadata/{}".format(request.remote_addr), + # data, value_in_json=True) + + +api.add_resource(Root, '/') + +if __name__ == '__main__': + app.run(debug=True, host="::", port="80") diff --git a/scheduler/config.py b/scheduler/config.py new file mode 100755 index 0000000..81e8503 --- /dev/null +++ b/scheduler/config.py @@ -0,0 +1,25 @@ +import logging + +from decouple import config +from etcd3_wrapper import Etcd3Wrapper +from ucloud_common.vm import VmPool +from ucloud_common.host import HostPool +from ucloud_common.request import RequestPool + +logging.basicConfig( + level=logging.DEBUG, + filename="log.txt", + filemode="a", + format="%(asctime)s: %(levelname)s - %(message)s", + datefmt="%d-%b-%y %H:%M:%S", +) + +vm_prefix = config("VM_PREFIX") +host_prefix = config("HOST_PREFIX") +request_prefix = config("REQUEST_PREFIX") + +etcd_client = Etcd3Wrapper(host=config("ETCD_URL")) + +vm_pool = VmPool(etcd_client, vm_prefix) +host_pool = HostPool(etcd_client, host_prefix) +request_pool = RequestPool(etcd_client, request_prefix) \ No newline at end of file diff --git a/scheduler/helper.py b/scheduler/helper.py new file mode 100755 index 0000000..65bd12d --- /dev/null +++ b/scheduler/helper.py @@ -0,0 +1,123 @@ +import bitmath + +from collections import Counter +from functools import reduce + +from ucloud_common.vm import VmPool, VMStatus +from ucloud_common.host import HostPool, HostStatus +from ucloud_common.request import RequestEntry, RequestPool, RequestType + +from decouple import config +from config import etcd_client as client + +vm_pool = VmPool(client, config("VM_PREFIX")) +host_pool = HostPool(client, config("HOST_PREFIX")) +request_pool = RequestPool(client, config("REQUEST_PREFIX")) + + +def accumulated_specs(vms_specs): + if not vms_specs: + return {} + return reduce((lambda x, y: Counter(x) + Counter(y)), vms_specs) + + +def remaining_resources(host_specs, vms_specs): + # Return remaining resources host_specs - vms + + _vms_specs = Counter(vms_specs) + _remaining = Counter(host_specs) + + for component in _vms_specs: + if isinstance(_vms_specs[component], str): + _vms_specs[component] = int(bitmath.parse_string(_vms_specs[component]).to_MB()) + elif isinstance(_vms_specs[component], list): + _vms_specs[component] = map(lambda x: int(bitmath.parse_string(x).to_MB()), _vms_specs[component]) + _vms_specs[component] = reduce(lambda x, y: x + y, _vms_specs[component], 0) + + for component in _remaining: + if isinstance(_remaining[component], str): + _remaining[component] = int(bitmath.parse_string(_remaining[component]).to_MB()) + elif isinstance(_remaining[component], list): + _remaining[component] = map(lambda x: int(bitmath.parse_string(x).to_MB()), _remaining[component]) + _remaining[component] = reduce(lambda x, y: x + y, _remaining[component], 0) + + print(_vms_specs, _remaining) + _remaining.subtract(_vms_specs) + + return _remaining + + +class NoSuitableHostFound(Exception): + """Exception when no host found that can host a VM.""" + + +def get_suitable_host(vm_specs, hosts=None): + if hosts is None: + hosts = host_pool.by_status(HostStatus.alive) + + for host in hosts: + # Filter them by host_name + vms = vm_pool.by_host(host.key) + + # Filter them by status + vms = vm_pool.by_status(VMStatus.running, vms) + + running_vms_specs = [vm.specs for vm in vms] + + # Accumulate all of their combined specs + running_vms_accumulated_specs = accumulated_specs( + running_vms_specs + ) + + # Find out remaining resources after + # host_specs - already running vm_specs + remaining = remaining_resources( + host.specs, running_vms_accumulated_specs + ) + + # Find out remaining - new_vm_specs + remaining = remaining_resources(remaining, vm_specs) + + if all(map(lambda x: x >= 0, remaining.values())): + return host.key + + raise NoSuitableHostFound + + +def dead_host_detection(): + # Bring out your dead! - Monty Python and the Holy Grail + hosts = host_pool.by_status(HostStatus.alive) + dead_hosts_keys = [] + + for host in hosts: + # Only check those who claims to be alive + if host.status == HostStatus.alive: + if not host.is_alive(): + dead_hosts_keys.append(host.key) + + return dead_hosts_keys + + +def dead_host_mitigation(dead_hosts_keys): + for host_key in dead_hosts_keys: + host = host_pool.get(host_key) + host.declare_dead() + + vms_hosted_on_dead_host = vm_pool.by_host(host_key) + for vm in vms_hosted_on_dead_host: + vm.declare_killed() + vm_pool.put(vm) + host_pool.put(host) + + +def assign_host(vm): + vm.hostname = get_suitable_host(vm.specs) + vm_pool.put(vm) + + r = RequestEntry.from_scratch(type=RequestType.StartVM, + uuid=vm.uuid, + hostname=vm.hostname) + request_pool.put(r) + + vm.log.append("VM scheduled for starting") + return vm.hostname diff --git a/scheduler/main.py b/scheduler/main.py new file mode 100755 index 0000000..4ce178f --- /dev/null +++ b/scheduler/main.py @@ -0,0 +1,89 @@ +# TODO +# 1. send an email to an email address defined by env['admin-email'] +# if resources are finished +# 2. Introduce a status endpoint of the scheduler - +# maybe expose a prometheus compatible output + +import logging + +from ucloud_common.request import RequestEntry, RequestType + +from config import etcd_client as client +from config import (host_pool, request_pool, vm_pool, request_prefix) +from helper import (get_suitable_host, dead_host_mitigation, dead_host_detection, + assign_host, NoSuitableHostFound) + + + +def main(): + pending_vms = [] + + for request_iterator in [ + client.get_prefix(request_prefix, value_in_json=True), + client.watch_prefix(request_prefix, timeout=5, value_in_json=True), + ]: + for request_event in request_iterator: + request_entry = RequestEntry(request_event) + logging.debug("%s, %s", request_entry.key, request_entry.value) + + # Never Run time critical mechanism inside timeout + # mechanism because timeout mechanism only comes + # when no other event is happening. It means under + # heavy load there would not be a timeout event. + if request_entry.type == "TIMEOUT": + + # Detect hosts that are dead and set their status + # to "DEAD", and their VMs' status to "KILLED" + logging.debug("TIMEOUT event occured") + dead_hosts = dead_host_detection() + logging.debug("Dead hosts: %s", dead_hosts) + dead_host_mitigation(dead_hosts) + + # If there are VMs that weren't assigned a host + # because there wasn't a host available which + # meets requirement of that VM then we would + # create a new ScheduleVM request for that VM + # on our behalf. + while pending_vms: + pending_vm_entry = pending_vms.pop() + r = RequestEntry.from_scratch(type="ScheduleVM", + uuid=pending_vm_entry.uuid, + hostname=pending_vm_entry.hostname) + request_pool.put(r) + + elif request_entry.type == RequestType.ScheduleVM: + vm_entry = vm_pool.get(request_entry.uuid) + client.client.delete(request_entry.key) # consume Request + + # If the Request is about a VM which is labelled as "migration" + # and has a destination + if hasattr(request_entry, "migration") and request_entry.migration \ + and hasattr(request_entry, "destination") and request_entry.destination: + try: + get_suitable_host(vm_specs=vm_entry.specs, + hosts=[host_pool.get(request_entry.destination)]) + except NoSuitableHostFound: + logging.info("Requested destination host doesn't have enough capacity" + "to hold %s", vm_entry.uuid) + else: + r = RequestEntry.from_scratch(type=RequestType.InitVMMigration, + uuid=request_entry.uuid, + destination=request_entry.destination) + request_pool.put(r) + + # If the Request is about a VM that just want to get started/created + else: + # assign_host only returns None when we couldn't be able to assign + # a host to a VM because of resource constraints + try: + assign_host(vm_entry) + except NoSuitableHostFound: + vm_entry.log.append("Can't schedule VM. No Resource Left.") + vm_pool.put(vm_entry) + + pending_vms.append(vm_entry) + logging.info("No Resource Left. Emailing admin....") + + +logging.info("%s SESSION STARTED %s", '*' * 5, '*' * 5) +main() diff --git a/scheduler/tests/test_basics.py b/scheduler/tests/test_basics.py new file mode 100755 index 0000000..227e84b --- /dev/null +++ b/scheduler/tests/test_basics.py @@ -0,0 +1,214 @@ +import unittest +import sys +import json +import multiprocessing +import time + +from datetime import datetime +from os.path import dirname + + +BASE_DIR = dirname(dirname(__file__)) +sys.path.insert(0, BASE_DIR) + +from main import ( + accumulated_specs, + remaining_resources, + VmPool, + dead_host_detection, + dead_host_mitigation, + main, +) + +from config import etcd_client + +class TestFunctions(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.client = etcd_client + cls.host_prefix = "/test/host" + cls.vm_prefix = "/test/vm" + + # These deletion could also be in + # tearDown() but it is more appropriate here + # as it enable us to check the ETCD store + # even after test is run + cls.client.client.delete_prefix(cls.host_prefix) + cls.client.client.delete_prefix(cls.vm_prefix) + cls.create_hosts(cls) + cls.create_vms(cls) + + cls.p = multiprocessing.Process( + target=main, args=[cls.vm_prefix, cls.host_prefix] + ) + cls.p.start() + + @classmethod + def tearDownClass(cls): + cls.p.terminate() + + def create_hosts(self): + host1 = { + "cpu": 32, + "ram": 128, + "hdd": 1024, + "sdd": 0, + "status": "ALIVE", + "last_heartbeat": datetime.utcnow().isoformat(), + } + host2 = { + "cpu": 16, + "ram": 64, + "hdd": 512, + "sdd": 0, + "status": "ALIVE", + "last_heartbeat": datetime.utcnow().isoformat(), + } + + host3 = { + "cpu": 16, + "ram": 32, + "hdd": 256, + "sdd": 256, + "status": "ALIVE", + "last_heartbeat": datetime.utcnow().isoformat(), + } + with self.client.client.lock("lock"): + self.client.put(f"{self.host_prefix}/1", host1, value_in_json=True) + self.client.put(f"{self.host_prefix}/2", host2, value_in_json=True) + self.client.put(f"{self.host_prefix}/3", host3, value_in_json=True) + + def create_vms(self): + vm1 = json.dumps( + { + "owner": "meow", + "specs": {"cpu": 4, "ram": 8, "hdd": 100, "sdd": 256}, + "hostname": "", + "status": "REQUESTED_NEW", + } + ) + vm2 = json.dumps( + { + "owner": "meow", + "specs": {"cpu": 16, "ram": 64, "hdd": 512, "sdd": 0}, + "hostname": "", + "status": "REQUESTED_NEW", + } + ) + vm3 = json.dumps( + { + "owner": "meow", + "specs": {"cpu": 16, "ram": 32, "hdd": 128, "sdd": 0}, + "hostname": "", + "status": "REQUESTED_NEW", + } + ) + vm4 = json.dumps( + { + "owner": "meow", + "specs": {"cpu": 16, "ram": 64, "hdd": 512, "sdd": 0}, + "hostname": "", + "status": "REQUESTED_NEW", + } + ) + vm5 = json.dumps( + { + "owner": "meow", + "specs": {"cpu": 2, "ram": 2, "hdd": 10, "sdd": 0}, + "hostname": "", + "status": "REQUESTED_NEW", + } + ) + vm6 = json.dumps( + { + "owner": "meow", + "specs": {"cpu": 10, "ram": 22, "hdd": 146, "sdd": 0}, + "hostname": "", + "status": "REQUESTED_NEW", + } + ) + vm7 = json.dumps( + { + "owner": "meow", + "specs": {"cpu": 10, "ram": 22, "hdd": 146, "sdd": 0}, + "hostname": "", + "status": "REQUESTED_NEW", + } + ) + self.client.put(f"{self.vm_prefix}/1", vm1) + self.client.put(f"{self.vm_prefix}/2", vm2) + self.client.put(f"{self.vm_prefix}/3", vm3) + self.client.put(f"{self.vm_prefix}/4", vm4) + self.client.put(f"{self.vm_prefix}/5", vm5) + self.client.put(f"{self.vm_prefix}/6", vm6) + self.client.put(f"{self.vm_prefix}/7", vm7) + + def test_accumulated_specs(self): + vms = [ + {"ssd": 10, "cpu": 4, "ram": 8}, + {"hdd": 10, "cpu": 4, "ram": 8}, + {"cpu": 8, "ram": 32}, + ] + self.assertEqual( + accumulated_specs(vms), {"ssd": 10, "cpu": 16, "ram": 48, "hdd": 10} + ) + + def test_remaining_resources(self): + host_specs = {"ssd": 10, "cpu": 16, "ram": 48, "hdd": 10} + vms_specs = {"ssd": 10, "cpu": 32, "ram": 12, "hdd": 0} + resultant_specs = {"ssd": 0, "cpu": -16, "ram": 36, "hdd": 10} + self.assertEqual(remaining_resources(host_specs, vms_specs), + resultant_specs) + + def test_vmpool(self): + self.p.join(1) + vm_pool = VmPool(self.client, self.vm_prefix) + + # vm_pool by host + actual = vm_pool.by_host(vm_pool.vms, f"{self.host_prefix}/3") + ground_truth = [ + ( + f"{self.vm_prefix}/1", + { + "owner": "meow", + "specs": {"cpu": 4, "ram": 8, "hdd": 100, "sdd": 256}, + "hostname": f"{self.host_prefix}/3", + "status": "SCHEDULED_DEPLOY", + }, + ) + ] + self.assertEqual(actual[0], ground_truth[0]) + + # vm_pool by status + actual = vm_pool.by_status(vm_pool.vms, "REQUESTED_NEW") + ground_truth = [ + ( + f"{self.vm_prefix}/7", + { + "owner": "meow", + "specs": {"cpu": 10, "ram": 22, "hdd": 146, "sdd": 0}, + "hostname": "", + "status": "REQUESTED_NEW", + }, + ) + ] + self.assertEqual(actual[0], ground_truth[0]) + + # vm_pool by except status + actual = vm_pool.except_status(vm_pool.vms, "SCHEDULED_DEPLOY") + ground_truth = [ + ( + f"{self.vm_prefix}/7", + { + "owner": "meow", + "specs": {"cpu": 10, "ram": 22, "hdd": 146, "sdd": 0}, + "hostname": "", + "status": "REQUESTED_NEW", + }, + ) + ] + self.assertEqual(actual[0], ground_truth[0]) + + +if __name__ == "__main__": + unittest.main() diff --git a/scheduler/tests/test_dead_host_mechanism.py b/scheduler/tests/test_dead_host_mechanism.py new file mode 100755 index 0000000..33bed23 --- /dev/null +++ b/scheduler/tests/test_dead_host_mechanism.py @@ -0,0 +1,81 @@ +import unittest +import sys +import json +import multiprocessing +import time + +from datetime import datetime +from os.path import dirname +BASE_DIR = dirname(dirname(__file__)) +sys.path.insert(0, BASE_DIR) + +from main import ( + accumulated_specs, + remaining_resources, + VmPool, + dead_host_detection, + dead_host_mitigation, + main, + config +) + +class TestDeadHostMechanism(unittest.TestCase): + def setUp(self): + self.client = config.etcd_client + self.host_prefix = "/test/host" + self.vm_prefix = "/test/vm" + + self.client.client.delete_prefix(self.host_prefix) + self.client.client.delete_prefix(self.vm_prefix) + + self.create_hosts() + + def create_hosts(self): + host1 = { + "cpu": 32, + "ram": 128, + "hdd": 1024, + "sdd": 0, + "status": "ALIVE", + "last_heartbeat": datetime.utcnow().isoformat(), + } + host2 = { + "cpu": 16, + "ram": 64, + "hdd": 512, + "sdd": 0, + "status": "ALIVE", + "last_heartbeat": datetime(2011, 1, 1).isoformat(), + } + + host3 = {"cpu": 16, "ram": 32, "hdd": 256, "sdd": 256} + host4 = { + "cpu": 16, + "ram": 32, + "hdd": 256, + "sdd": 256, + "status": "DEAD", + "last_heartbeat": datetime(2011, 1, 1).isoformat(), + } + with self.client.client.lock("lock"): + self.client.put(f"{self.host_prefix}/1", host1, value_in_json=True) + self.client.put(f"{self.host_prefix}/2", host2, value_in_json=True) + self.client.put(f"{self.host_prefix}/3", host3, value_in_json=True) + self.client.put(f"{self.host_prefix}/4", host4, value_in_json=True) + + def test_dead_host_detection(self): + hosts = self.client.get_prefix(self.host_prefix, value_in_json=True) + deads = dead_host_detection(hosts) + self.assertEqual(deads, ["/test/host/2", "/test/host/3"]) + return deads + + def test_dead_host_mitigation(self): + deads = self.test_dead_host_detection() + dead_host_mitigation(self.client, deads) + hosts = self.client.get_prefix(self.host_prefix, value_in_json=True) + deads = dead_host_detection(hosts) + self.assertEqual(deads, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/ucloud.py b/ucloud.py new file mode 100644 index 0000000..5886502 --- /dev/null +++ b/ucloud.py @@ -0,0 +1,16 @@ +import argparse +import subprocess as sp +arg_parser = argparse.ArgumentParser(prog='ucloud', + description='Open Source Cloud Management Software') +arg_parser.add_argument('component', + choices=['api', 'scheduler', 'host', + 'filescanner','imagescanner', + 'metadata']) +arg_parser.add_argument('component_args', nargs='*') +args = arg_parser.parse_args() + +try: + command = ['pipenv', 'run', 'python', 'main.py', *args.component_args] + sp.run(command, cwd=args.component) +except Exception as error: + print(error)