This commit is contained in:
Ahmed Bilal 2019-09-03 18:18:43 +02:00
parent bf5d260e26
commit 745d5f315a
9 changed files with 519 additions and 349 deletions

0
.gitignore vendored Normal file → Executable file
View File

3
Pipfile Normal file → Executable file
View File

@ -6,13 +6,16 @@ verify_ssl = true
[dev-packages]
bandit = "*"
pylama = "*"
prospector = "*"
[packages]
python-decouple = "*"
cython = "*"
pylint = "*"
grpcio = "*"
python-etcd3 = {editable = true,git = "https://github.com/kragniz/python-etcd3"}
sshtunnel = "*"
bitmath = "*"
[requires]
python_version = "3.7"

282
Pipfile.lock generated Normal file → Executable file
View File

@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
"sha256": "8f820ace244a315a831e3330707a6ec24895ed73f8fa1646ce6889915aec0db4"
"sha256": "05c0e0fbcad89f0740bfca9356e4493d95eb898c35017258f750c8c4674034fa"
},
"pipfile-spec": 6,
"requires": {
@ -51,6 +51,13 @@
],
"version": "==3.1.7"
},
"bitmath": {
"hashes": [
"sha256:293325f01e65defe966853111df11d39215eb705a967cb115851da8c4cfa3eb8"
],
"index": "pypi",
"version": "==1.3.3.1"
},
"cffi": {
"hashes": [
"sha256:041c81822e9f84b1d9c401182e174996f0bae9991f33725d059b771744290774",
@ -141,40 +148,40 @@
},
"grpcio": {
"hashes": [
"sha256:03b78b4e7dcdfe3e257bb528cc93923f9cbbab6d5babf15a60d21e9a4a70b1a2",
"sha256:1ce0ccfbdfe84387dbcbf44adb4ae16ec7ae70e166ffab478993eb1ea1cba3ce",
"sha256:22e167a9406d73dd19ffe8ed6a485f17e6eac82505be8c108897f15e68badcbb",
"sha256:31d0aeca8d8ee2301c62c5c340e0889d653b1280d68f9fa203982cb6337b050e",
"sha256:44c7f99ca17ebbcc96fc54ed00b454d8313f1eac28c563098d8b901025aff941",
"sha256:5471444f53f9db6a1f1f11f5dbc173228881df8446380b6b98f90afb8fd8348e",
"sha256:561bca3b1bde6d6564306eb05848fd155136e9c3a25d2961129b1e2edba22fce",
"sha256:5bf58e1d2c2f55365c06e8cb5abe067b88ca2e5550fb62009c41df4b54505acf",
"sha256:6b7163d1e85d76b0815df63fcc310daec02b44532bb433f743142d4febcb181f",
"sha256:766d79cddad95f5f6020037fe60ea8b98578afdf0c59d5a60c106c1bdd886303",
"sha256:770b7372d5ca68308ff66d7baee53369fa5ce985f84bcb6aa1948c1f2f7b02f2",
"sha256:7ab178da777fc0f55b6aef5a755f99726e8e4b75e3903954df07b27059b54fcf",
"sha256:8078305e77c2f6649d36b24d8778096413e474d9d7892c6f92cfb589c9d71b2e",
"sha256:85600b63a386d860eeaa955e9335e18dd0d7e5477e9214825abf2c2884488369",
"sha256:857d9b939ae128be1c0c792eb885c7ff6a386b9dea899ac4b06f4d90a31f9d87",
"sha256:87a41630c90c179fa5c593400f30a467c498972c702f348d41e19dafeb1d319e",
"sha256:8805d486c6128cc0fcc8ecf16c4095d99a8693a541ef851429ab334e028a4a97",
"sha256:8d71b7a89c306a41ccc7741fc9409b14f5b86727455c2a1c0c7cfcb0f784e1f2",
"sha256:9e1b80bd65f8f160880cb4dad7f55697f6d37b2d7f251fc0c2128e811928f369",
"sha256:9e290c84a145ae2411ee0ec9913c41cd7500e2e7485fe93632434d84ef4fda67",
"sha256:9ec9f88b5bc94bd99372f27cdd53af1c92ba06717380b127733b953cfb181174",
"sha256:a0a02a8b4ba6deadf706d5f849539b3685b72b186a3c9ef5d43e8972ed60fb6f",
"sha256:a4059c59519f5940e01a071f74ae2a60ea8f6185b03d22a09d40c7959a36b16b",
"sha256:a6e028c2a6da2ebfa2365a5b32531d311fbfec0e3600fc27e901b64f0ff7e54e",
"sha256:adcdebf9f8463df4120c427cf6c9aed39258bccd03ed37b6939e7a145d64d6e0",
"sha256:bdec982610259d07156a58f80b8c3e69be7751a9208bc577b059c5193d087fad",
"sha256:cefc4d4251ffb73feb303d4b7e9d6c367cb60f2db16d259ea28b114045f965aa",
"sha256:d4145c8aa6afbac10ad27e408f7ce15992fe89ba5d0b4abca31c0c2729864c03",
"sha256:da76dc5ad719ee99de5ea28a5629ff92172cbb4a70d8a6ae3a5b7a53c7382ce1",
"sha256:dde2452c08ef8b6426ccab6b5b6de9f06d836d9937d6870e68153cbf8cb49348",
"sha256:e3d88091d2539a4868750914a6fe7b9ec50e42b913851fc1b77423b5bd918530",
"sha256:f9c67cfe6278499d7f83559dc6322a8bbb108e307817a3d7acbfea807b3603cc"
"sha256:1303578092f1f6e4bfbc354c04ac422856c393723d3ffa032fff0f7cb5cfd693",
"sha256:229c6b313cd82bec8f979b059d87f03cc1a48939b543fe170b5a9c5cf6a6bc69",
"sha256:3cd3d99a8b5568d0d186f9520c16121a0f2a4bcad8e2b9884b76fb88a85a7774",
"sha256:41cfb222db358227521f9638a6fbc397f310042a4db5539a19dea01547c621cd",
"sha256:43330501660f636fd6547d1e196e395cd1e2c2ae57d62219d6184a668ffebda0",
"sha256:45d7a2bd8b4f25a013296683f4140d636cdbb507d94a382ea5029a21e76b1648",
"sha256:47dc935658a13b25108823dabd010194ddea9610357c5c1ef1ad7b3f5157ebee",
"sha256:480aa7e2b56238badce0b9413a96d5b4c90c3bfbd79eba5a0501e92328d9669e",
"sha256:4a0934c8b0f97e1d8c18e76c45afc0d02d33ab03125258179f2ac6c7a13f3626",
"sha256:5624dab19e950f99e560400c59d87b685809e4cfcb2c724103f1ab14c06071f7",
"sha256:60515b1405bb3dadc55e6ca99429072dad3e736afcf5048db5452df5572231ff",
"sha256:610f97ebae742a57d336a69b09a9c7d7de1f62aa54aaa8adc635b38f55ba4382",
"sha256:64ea189b2b0859d1f7b411a09185028744d494ef09029630200cc892e366f169",
"sha256:686090c6c1e09e4f49585b8508d0a31d58bc3895e4049ea55b197d1381e9f70f",
"sha256:7745c365195bb0605e3d47b480a2a4d1baa8a41a5fd0a20de5fa48900e2c886a",
"sha256:79491e0d2b77a1c438116bf9e5f9e2e04e78b78524615e2ce453eff62db59a09",
"sha256:825177dd4c601c487836b7d6b4ba268db59787157911c623ba59a7c03c8d3adc",
"sha256:8a060e1f72fb94eee8a035ed29f1201ce903ad14cbe27bda56b4a22a8abda045",
"sha256:90168cc6353e2766e47b650c963f21cfff294654b10b3a14c67e26a4e3683634",
"sha256:94b7742734bceeff6d8db5edb31ac844cb68fc7f13617eca859ff1b78bb20ba1",
"sha256:962aebf2dd01bbb2cdb64580e61760f1afc470781f9ecd5fe8f3d8dcd8cf4556",
"sha256:9c8d9eacdce840b72eee7924c752c31b675f8aec74790e08cff184a4ea8aa9c1",
"sha256:af5b929debc336f6bab9b0da6915f9ee5e41444012aed6a79a3c7e80d7662fdf",
"sha256:b9cdb87fc77e9a3eabdc42a512368538d648fa0760ad30cf97788076985c790a",
"sha256:c5e6380b90b389454669dc67d0a39fb4dc166416e01308fcddd694236b8329ef",
"sha256:d60c90fe2bfbee735397bf75a2f2c4e70c5deab51cd40c6e4fa98fae018c8db6",
"sha256:d8582c8b1b1063249da1588854251d8a91df1e210a328aeb0ece39da2b2b763b",
"sha256:ddbf86ba3aa0ad8fed2867910d2913ee237d55920b55f1d619049b3399f04efc",
"sha256:e46bc0664c5c8a0545857aa7a096289f8db148e7f9cca2d0b760113e8994bddc",
"sha256:f6437f70ec7fed0ca3a0eef1146591bb754b418bb6c6b21db74f0333d624e135",
"sha256:f71693c3396530c6b00773b029ea85e59272557e9bd6077195a6593e4229892a",
"sha256:f79f7455f8fbd43e8e9d61914ecf7f48ba1c8e271801996fef8d6a8f3cc9f39f"
],
"version": "==1.22.0"
"version": "==1.23.0"
},
"isort": {
"hashes": [
@ -222,26 +229,24 @@
},
"protobuf": {
"hashes": [
"sha256:05c36022fef3c7d3562ac22402965c0c2b9fe8421f459bb377323598996e407f",
"sha256:139b7eadcca0a861d60b523cb37d9475505e0dfb07972436b15407c2b968d87e",
"sha256:15f683006cb77fb849b1f561e509b03dd2b7dcc749086b8dd1831090d0ba4740",
"sha256:2ad566b7b7cdd8717c7af1825e19f09e8fef2787b77fcb979588944657679604",
"sha256:35cfcf97642ef62108e10a9431c77733ec7eaab8e32fe4653de20403429907cb",
"sha256:387822859ecdd012fdc25ec879f7f487da6e1d5b1ae6115e227e6be208836f71",
"sha256:4df14cbe1e7134afcfdbb9f058949e31c466de27d9b2f7fb4da9e0b67231b538",
"sha256:586c4ca37a7146d4822c700059f150ac3445ce0aef6f3ea258640838bb892dc2",
"sha256:58b11e530e954d29ab3180c48dc558a409f705bf16739fd4e0d3e07924ad7add",
"sha256:63c8c98ccb8c95f41c18fb829aeeab21c6249adee4ed75354125bdc44488f30e",
"sha256:72edcbacd0c73eef507d2ff1af99a6c27df18e66a3ff4351e401182e4de62b03",
"sha256:83dc8a561b3b954fd7002c690bb83278b8d1742a1e28abba9aaef28b0c8b437d",
"sha256:913171ecc84c2726b86574e40549a0ea619d569657c5a5ff782a3be7d81401a5",
"sha256:aabb7c741d3416671c3e6fe7c52970a226e6a8274417a97d7d795f953fadef36",
"sha256:b3452bbda12b1cbe2187d416779de07b2ab4c497d83a050e43c344778763721d",
"sha256:c5d5b8d4a9212338297fa1fa44589f69b470c0ba1d38168b432d577176b386a8",
"sha256:d86ee389c2c4fc3cebabb8ce83a8e97b6b3b5dc727b7419c1ccdc7b6e545a233",
"sha256:f2db8c754de788ab8be5e108e1e967c774c0942342b4f8aaaf14063889a6cfdc"
"sha256:00a1b0b352dc7c809749526d1688a64b62ea400c5b05416f93cfb1b11a036295",
"sha256:01acbca2d2c8c3f7f235f1842440adbe01bbc379fa1cbdd80753801432b3fae9",
"sha256:0a795bca65987b62d6b8a2d934aa317fd1a4d06a6dd4df36312f5b0ade44a8d9",
"sha256:0ec035114213b6d6e7713987a759d762dd94e9f82284515b3b7331f34bfaec7f",
"sha256:31b18e1434b4907cb0113e7a372cd4d92c047ce7ba0fa7ea66a404d6388ed2c1",
"sha256:32a3abf79b0bef073c70656e86d5bd68a28a1fbb138429912c4fc07b9d426b07",
"sha256:55f85b7808766e5e3f526818f5e2aeb5ba2edcc45bcccede46a3ccc19b569cb0",
"sha256:64ab9bc971989cbdd648c102a96253fdf0202b0c38f15bd34759a8707bdd5f64",
"sha256:64cf847e843a465b6c1ba90fb6c7f7844d54dbe9eb731e86a60981d03f5b2e6e",
"sha256:917c8662b585470e8fd42f052661fc66d59fccaae450a60044307dcbf82a3335",
"sha256:afed9003d7f2be2c3df20f64220c30faec441073731511728a2cb4cab4cd46a6",
"sha256:bf8e05d638b585d1752c5a84247134a0350d3a8b73d3632489a014a9f6f1e758",
"sha256:d831b047bd69becaf64019a47179eb22118a50dd008340655266a906c69c6417",
"sha256:de2760583ed28749ff885789c1cbc6c9c06d6de92fc825740ab99deb2f25ea4d",
"sha256:eabc4cf1bc19689af8022ba52fd668564a8d96e0d08f3b4732d26a64255216a4",
"sha256:fcff6086c86fb1628d94ea455c7b9de898afc50378042927a59df8065a79a549"
],
"version": "==3.9.0"
"version": "==3.9.1"
},
"pycparser": {
"hashes": [
@ -309,10 +314,10 @@
},
"tenacity": {
"hashes": [
"sha256:a0c3c5f7ae0c33f5556c775ca059c12d6fd8ab7121613a713e8b7d649908571b",
"sha256:b87c1934daa0b2ccc7db153c37b8bf91d12f165936ade8628e7b962b92dc7705"
"sha256:6a7511a59145c2e319b7d04ddd93c12d48cc3d3c8fa42c2846d33a620ee91f57",
"sha256:a4eb168dbf55ed2cae27e7c6b2bd48ab54dabaf294177d998330cf59f294c112"
],
"version": "==5.0.4"
"version": "==5.1.1"
},
"typed-ast": {
"hashes": [
@ -343,6 +348,13 @@
}
},
"develop": {
"astroid": {
"hashes": [
"sha256:6560e1e1749f68c64a4b5dee4e091fce798d2f0d84ebe638cf0e0585a343acf4",
"sha256:b65db1bbaac9f9f4d190199bb8680af6f6f84fd3769a5ea883df8a91fe68b4c4"
],
"version": "==2.2.5"
},
"bandit": {
"hashes": [
"sha256:336620e220cf2d3115877685e264477ff9d9abaeb0afe3dc7264f55fa17a3952",
@ -351,6 +363,19 @@
"index": "pypi",
"version": "==1.6.2"
},
"ddt": {
"hashes": [
"sha256:474546b4020ce8a2f9550ba8899c28aa2c284c7bbf175bddede98be949d1ca7c",
"sha256:d13e6af8f36238e89d00f4ebccf2bda4f6d1878be560a6600689e42077e164e3"
],
"version": "==1.2.1"
},
"dodgy": {
"hashes": [
"sha256:65e13cf878d7aff129f1461c13cb5fd1bb6dfe66bb5327e09379c3877763280c"
],
"version": "==0.1.9"
},
"gitdb2": {
"hashes": [
"sha256:83361131a1836661a155172932a13c08bda2db3674e4caa32368aa6eb02f38c2",
@ -360,10 +385,40 @@
},
"gitpython": {
"hashes": [
"sha256:c15c55ff890cd3a6a8330059e80885410a328f645551b55a91d858bfb3eb2573",
"sha256:df752b6b6f06f11213e91c4925aea7eaf9e37e88fb71c8a7a1aa0a5c10852120"
"sha256:259a8b6d6a4a118738c4a65fa990f8c8c91525bb43970aed2868952ebb86ceb8",
"sha256:73aa7b59e58dd3435121421c33c284e5ef51bc7b2f4373e1a1e4cc06e9c928ec"
],
"version": "==2.1.13"
"version": "==3.0.1"
},
"isort": {
"hashes": [
"sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1",
"sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd"
],
"version": "==4.3.21"
},
"lazy-object-proxy": {
"hashes": [
"sha256:159a745e61422217881c4de71f9eafd9d703b93af95618635849fe469a283661",
"sha256:23f63c0821cc96a23332e45dfaa83266feff8adc72b9bcaef86c202af765244f",
"sha256:3b11be575475db2e8a6e11215f5aa95b9ec14de658628776e10d96fa0b4dac13",
"sha256:3f447aff8bc61ca8b42b73304f6a44fa0d915487de144652816f950a3f1ab821",
"sha256:4ba73f6089cd9b9478bc0a4fa807b47dbdb8fad1d8f31a0f0a5dbf26a4527a71",
"sha256:4f53eadd9932055eac465bd3ca1bd610e4d7141e1278012bd1f28646aebc1d0e",
"sha256:64483bd7154580158ea90de5b8e5e6fc29a16a9b4db24f10193f0c1ae3f9d1ea",
"sha256:6f72d42b0d04bfee2397aa1862262654b56922c20a9bb66bb76b6f0e5e4f9229",
"sha256:7c7f1ec07b227bdc561299fa2328e85000f90179a2f44ea30579d38e037cb3d4",
"sha256:7c8b1ba1e15c10b13cad4171cfa77f5bb5ec2580abc5a353907780805ebe158e",
"sha256:8559b94b823f85342e10d3d9ca4ba5478168e1ac5658a8a2f18c991ba9c52c20",
"sha256:a262c7dfb046f00e12a2bdd1bafaed2408114a89ac414b0af8755c696eb3fc16",
"sha256:acce4e3267610c4fdb6632b3886fe3f2f7dd641158a843cf6b6a68e4ce81477b",
"sha256:be089bb6b83fac7f29d357b2dc4cf2b8eb8d98fe9d9ff89f9ea6012970a853c7",
"sha256:bfab710d859c779f273cc48fb86af38d6e9210f38287df0069a63e40b45a2f5c",
"sha256:c10d29019927301d524a22ced72706380de7cfc50f767217485a912b4c8bd82a",
"sha256:dd6e2b598849b3d7aee2295ac765a578879830fb8966f70be8cd472e6069932e",
"sha256:e408f1eacc0a68fed0c08da45f31d0ebb38079f043328dce69ff133b95c29dc1"
],
"version": "==1.4.1"
},
"mccabe": {
"hashes": [
@ -374,30 +429,45 @@
},
"pbr": {
"hashes": [
"sha256:0ca44dc9fd3b04a22297c2a91082d8df2894862e8f4c86a49dac69eae9e85ca0",
"sha256:4aed6c1b1fa5020def0f22aed663d87b81bb3235f112490b07d2643d7a98c5b5"
"sha256:56e52299170b9492513c64be44736d27a512fa7e606f21942160b68ce510b4bc",
"sha256:9b321c204a88d8ab5082699469f52cc94c5da45c51f114113d01b3d993c24cdf"
],
"version": "==5.4.1"
"version": "==5.4.2"
},
"pep8-naming": {
"hashes": [
"sha256:1b419fa45b68b61cd8c5daf4e0c96d28915ad14d3d5f35fcc1e7e95324a33a2e",
"sha256:4eedfd4c4b05e48796f74f5d8628c068ff788b9c2b08471ad408007fc6450e5a"
],
"version": "==0.4.1"
},
"prospector": {
"hashes": [
"sha256:aba551e53dc1a5a432afa67385eaa81d7b4cf4c162dc1a4d0ee00b3a0712ad90"
],
"index": "pypi",
"version": "==1.1.7"
},
"pycodestyle": {
"hashes": [
"sha256:95a2219d12372f05704562a14ec30bc76b05a5b297b21a5dfe3f6fac3491ae56",
"sha256:e40a936c9a450ad81df37f549d676d127b1b66000a6c500caa2b085bc0ca976c"
"sha256:cbc619d09254895b0d12c2c691e237b2e91e9b2ecf5e84c26b35400f93dcfb83",
"sha256:cbfca99bd594a10f674d0cd97a3d802a1fdef635d4361e1a2658de47ed261e3a"
],
"version": "==2.5.0"
"version": "==2.4.0"
},
"pydocstyle": {
"hashes": [
"sha256:58c421dd605eec0bce65df8b8e5371bb7ae421582cdf0ba8d9435ac5b0ffc36a"
"sha256:04c84e034ebb56eb6396c820442b8c4499ac5eb94a3bda88951ac3dc519b6058",
"sha256:66aff87ffe34b1e49bff2dd03a88ce6843be2f3346b0c9814410d34987fbab59"
],
"version": "==4.0.0"
"version": "==4.0.1"
},
"pyflakes": {
"hashes": [
"sha256:17dbeb2e3f4d772725c777fabc446d5634d1038f234e77343108ce445ea69ce0",
"sha256:d976835886f8c5b31d47970ed689944a0262b5f3afa00a5a7b4dc81e5449f8a2"
"sha256:08bd6a50edf8cffa9fa09a463063c425ecaaf10d1eb0335a7e8b1401aef89e6f",
"sha256:8d616a382f243dbf19b54743f280b80198be0bca3a5396f1d2e1fca6223e8805"
],
"version": "==2.1.1"
"version": "==1.6.0"
},
"pylama": {
"hashes": [
@ -407,6 +477,39 @@
"index": "pypi",
"version": "==7.7.1"
},
"pylint": {
"hashes": [
"sha256:5d77031694a5fb97ea95e828c8d10fc770a1df6eb3906067aaed42201a8a6a09",
"sha256:723e3db49555abaf9bf79dc474c6b9e2935ad82230b10c1138a71ea41ac0fff1"
],
"index": "pypi",
"version": "==2.3.1"
},
"pylint-celery": {
"hashes": [
"sha256:41e32094e7408d15c044178ea828dd524beedbdbe6f83f712c5e35bde1de4beb"
],
"version": "==0.3"
},
"pylint-django": {
"hashes": [
"sha256:75c69d1ec2275918c37f175976da20e2f1e1e62e067098a685cd263ffa833dfd",
"sha256:c7cb6384ea7b33ea77052a5ae07358c10d377807390ef27b2e6ff997303fadb7"
],
"version": "==2.0.10"
},
"pylint-flask": {
"hashes": [
"sha256:f4d97de2216bf7bfce07c9c08b166e978fe9f2725de2a50a9845a97de7e31517"
],
"version": "==0.6"
},
"pylint-plugin-utils": {
"hashes": [
"sha256:8d9e31d5ea8b7b0003e1f0f136b44a5235896a32e47c5bc2ef1143e9f6ba0b74"
],
"version": "==0.5"
},
"pyyaml": {
"hashes": [
"sha256:0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9",
@ -425,6 +528,18 @@
],
"version": "==5.1.2"
},
"requirements-detector": {
"hashes": [
"sha256:9fbc4b24e8b7c3663aff32e3eba34596848c6b91bd425079b386973bd8d08931"
],
"version": "==0.6"
},
"setoptconf": {
"hashes": [
"sha256:5b0b5d8e0077713f5d5152d4f63be6f048d9a1bb66be15d089a11c898c3cf49c"
],
"version": "==0.2.0"
},
"six": {
"hashes": [
"sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
@ -451,6 +566,33 @@
"sha256:7d1ce610a87d26f53c087da61f06f9b7f7e552efad2a7f6d2322632b5f932ea2"
],
"version": "==1.30.1"
},
"typed-ast": {
"hashes": [
"sha256:18511a0b3e7922276346bcb47e2ef9f38fb90fd31cb9223eed42c85d1312344e",
"sha256:262c247a82d005e43b5b7f69aff746370538e176131c32dda9cb0f324d27141e",
"sha256:2b907eb046d049bcd9892e3076c7a6456c93a25bebfe554e931620c90e6a25b0",
"sha256:354c16e5babd09f5cb0ee000d54cfa38401d8b8891eefa878ac772f827181a3c",
"sha256:4e0b70c6fc4d010f8107726af5fd37921b666f5b31d9331f0bd24ad9a088e631",
"sha256:630968c5cdee51a11c05a30453f8cd65e0cc1d2ad0d9192819df9978984529f4",
"sha256:66480f95b8167c9c5c5c87f32cf437d585937970f3fc24386f313a4c97b44e34",
"sha256:71211d26ffd12d63a83e079ff258ac9d56a1376a25bc80b1cdcdf601b855b90b",
"sha256:95bd11af7eafc16e829af2d3df510cecfd4387f6453355188342c3e79a2ec87a",
"sha256:bc6c7d3fa1325a0c6613512a093bc2a2a15aeec350451cbdf9e1d4bffe3e3233",
"sha256:cc34a6f5b426748a507dd5d1de4c1978f2eb5626d51326e43280941206c209e1",
"sha256:d755f03c1e4a51e9b24d899561fec4ccaf51f210d52abdf8c07ee2849b212a36",
"sha256:d7c45933b1bdfaf9f36c579671fec15d25b06c8398f113dab64c18ed1adda01d",
"sha256:d896919306dd0aa22d0132f62a1b78d11aaf4c9fc5b3410d3c666b818191630a",
"sha256:ffde2fbfad571af120fcbfbbc61c72469e72f550d676c3342492a9dfdefb8f12"
],
"markers": "implementation_name == 'cpython'",
"version": "==1.4.0"
},
"wrapt": {
"hashes": [
"sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1"
],
"version": "==1.11.2"
}
}
}

24
config.py Executable file
View File

@ -0,0 +1,24 @@
import logging
from etcd3_wrapper import Etcd3Wrapper
from ucloud_common.vm import VmPool
from ucloud_common.host import HostPool
from ucloud_common.request import RequestPool
from decouple import config
logging.basicConfig(
level=logging.DEBUG,
filename="log.txt",
filemode="a",
format="%(asctime)s: %(levelname)s - %(message)s",
datefmt="%d-%b-%y %H:%M:%S",
)
etcd_client = Etcd3Wrapper(host=config("ETCD_URL"))
host_pool = HostPool(etcd_client, "/v1/host")
vm_pool = VmPool(etcd_client, "/v1/vm")
request_pool = RequestPool(etcd_client, "/v1/request")
running_vms = []

301
main.py Normal file → Executable file
View File

@ -1,76 +1,13 @@
# TODO: Use Unix File Socket for VNC instead of TCP
# QEMU Manual
# https://qemu.weilnetz.de/doc/qemu-doc.html
# For QEMU Monitor Protocol Commands Information, See
# https://qemu.weilnetz.de/doc/qemu-doc.html#pcsys_005fmonitor
import argparse
import qmp
import logging
import subprocess
import threading
import time
import traceback
import sshtunnel
import errno
import virtualmachine
from dataclasses import dataclass
from decouple import config
from typing import Union
from functools import wraps
from string import Template
from os.path import join
from ucloud_common.host import HostEntry
from ucloud_common.request import RequestEntry, RequestType
from etcd3_wrapper import Etcd3Wrapper
from ucloud_common.vm import VmPool, VMStatus, VMEntry
from ucloud_common.host import HostPool, HostEntry
from ucloud_common.request import RequestEntry, RequestPool, RequestType
from ucloud_common.helpers import get_ipv4_address
running_vms = []
vnc_port_pool = list(range(0, 100))
client = Etcd3Wrapper()
vm_pool = None
host_pool = None
request_pool = None
logging.basicConfig(
level=logging.DEBUG,
filename="log.txt",
filemode="a",
format="%(asctime)s: %(levelname)s - %(message)s",
datefmt="%d-%b-%y %H:%M:%S",
)
def get_start_command_args(image_path, vnc_port, migration=False, migration_port=4444):
_args = ("-drive file=$image_path,format=raw,if=virtio,cache=none"
" -m 1024 -device virtio-rng-pci -enable-kvm -vnc :$vnc_port")
if migration:
_args = _args + " -incoming tcp:0:$migration_port"
args_template = Template(_args)
if migration:
args = args_template.substitute(image_path=image_path, vnc_port=vnc_port,
migration_port=migration_port)
else:
args = args_template.substitute(image_path=image_path, vnc_port=vnc_port)
return args.split(" ")
@dataclass
class VM:
key: str
handle: qmp.QEMUMachine
def __repr__(self):
return f"VM({self.key})"
from config import (vm_pool, host_pool, request_pool,
etcd_client, logging, running_vms)
def update_heartbeat(host: HostEntry):
@ -82,108 +19,6 @@ def update_heartbeat(host: HostEntry):
logging.info(f"Updated last heartbeat time {host.last_heartbeat}")
def need_running_vm(func):
@wraps(func)
def wrapper(e):
vm = get_vm(running_vms, e.key)
if vm:
try:
status = vm.handle.command("query-status")
logging.debug(f"VM Status Check - {status}")
except OSError:
logging.info(
f"{func.__name__} failed - VM {e.key} - Unknown Error"
)
return func(e)
else:
logging.info(
f"{func.__name__} failed because VM {e.key} is not running"
)
return
return wrapper
def create_vm(vm_entry: VMEntry):
_command_to_create = f"rbd clone images/{vm_entry.image_uuid}@protected uservms/{vm_entry.uuid}"
try:
subprocess.check_call(_command_to_create.split(" "))
except subprocess.CalledProcessError as e:
if e.returncode == errno.EEXIST:
logging.debug(f"Image for vm {vm_entry.uuid} exists")
# File Already exists. No Problem Continue
return
else:
# This exception catches all other exceptions
# i.e FileNotFound (BaseImage), pool Does Not Exists etc.
logging.exception(f"Can't clone image - {e}")
else:
logging.info("New VM Created")
def start_vm(vm_entry: VMEntry):
_vm = get_vm(running_vms, vm_entry.key)
# VM already running. No need to proceed further.
if _vm:
logging.info(f"VM {vm_entry.uuid} already running")
return
else:
create_vm(vm_entry)
logging.info(f"Starting {vm_entry.key}")
# FIXME: There should be better vnc port allocation scheme
vm = qmp.QEMUMachine(
"/usr/bin/qemu-system-x86_64",
args=get_start_command_args(vm_entry.path, vnc_port_pool.pop(0)),
)
try:
vm.launch()
except (qmp.QEMUMachineError, TypeError, Exception):
vm_entry.declare_killed()
vm_entry.add_log(f"Machine Error occurred - {traceback.format_exc()}")
vm_pool.put(vm_entry)
else:
running_vms.append(VM(vm_entry.key, vm))
vm_entry.status = VMStatus.running
vm_entry.add_log("Started successfully")
vm_pool.put(vm_entry)
@need_running_vm
def stop_vm(vm_entry):
vm = get_vm(running_vms, vm_entry.key)
vm.handle.shutdown()
if not vm.handle.is_running():
vm_entry.add_log("Shutdown successfully")
vm_entry.declare_stopped()
vm_pool.put(vm_entry)
running_vms.remove(vm)
def delete_vm(vm_entry):
logging.info(f"Deleting VM {vm_entry}")
stop_vm(vm_entry)
path_without_protocol = vm_entry.path[vm_entry.path.find(":") + 1:]
try:
rc = subprocess.call(f"rbd rm {path_without_protocol}".split(" "))
except FileNotFoundError as e:
logging.exception(e)
except Exception as e:
logging.exception(f"Unknown error occurred - {e}")
else:
if rc == 0:
client.client.delete(vm_entry.key)
else:
logging.info("Some unknown problem occur while deleting vm file")
def get_vm(vm_list: list, vm_key) -> Union[VM, None]:
return next((vm for vm in vm_list if vm.key == vm_key), None)
def maintenance(host):
# To capture vm running according to running_vms list
@ -195,16 +30,16 @@ def maintenance(host):
# whether this host vm is successfully migrated. If yes
# then we shutdown "vm1" on this host.
for vm in running_vms:
with vm_pool.get_put(vm.key) as vm_entry:
for running_vm in running_vms:
with vm_pool.get_put(running_vm.key) as vm_entry:
if vm_entry.hostname != host.key and not vm_entry.in_migration:
vm.handle.shutdown()
running_vm.handle.shutdown()
vm_entry.add_log("VM on source host shutdown.")
# To check vm running according to etcd entries
alleged_running_vms = vm_pool.by_status("RUNNING", vm_pool.by_host(host.key))
for vm_entry in alleged_running_vms:
_vm = get_vm(running_vms, vm_entry.key)
_vm = virtualmachine.get_vm(running_vms, vm_entry.key)
# Whether, the allegedly running vm is in our
# running_vms list or not if it is said to be
@ -224,104 +59,11 @@ def maintenance(host):
running_vms.remove(_vm)
def transfer_vm(request_event):
# This function would run on source host i.e host on which the vm
# is running initially. This host would be responsible for transferring
# vm state to destination host.
_host, _port = request_event.parameters["host"], request_event.parameters["port"]
_uuid = request_event.uuid
_destination = request_event.destination_host_key
vm = get_vm(running_vms, join("/v1/vm", _uuid))
if vm:
tunnel = sshtunnel.SSHTunnelForwarder(
(_host, 22),
ssh_username=config("ssh_username"),
ssh_pkey=config("ssh_pkey"),
ssh_private_key_password=config("ssh_private_key_password"),
remote_bind_address=('127.0.0.1', _port),
)
try:
tunnel.start()
except sshtunnel.BaseSSHTunnelForwarderError:
logging.exception(f"Couldn't establish connection to ({_host}, 22)")
else:
vm.handle.command("migrate", uri=f"tcp:{_host}:{tunnel.local_bind_port}")
status = vm.handle.command("query-migrate")["status"]
while status not in ["failed", "completed"]:
time.sleep(2)
status = vm.handle.command("query-migrate")["status"]
with vm_pool.get_put(request_event.uuid) as source_vm:
if status == "failed":
source_vm.add_log("Migration Failed")
elif status == "completed":
# If VM is successfully migrated then shutdown the VM
# on this host and update hostname to destination host key
source_vm.add_log("Successfully migrated")
source_vm.hostname = _destination
running_vms.remove(vm)
vm.handle.shutdown()
source_vm.in_migration = False # VM transfer finished
finally:
tunnel.close()
def init_vm_migration(vm_entry, destination_host_key):
# This function would run on destination host i.e host on which the vm
# would be transferred after migration. This host would be responsible
# for starting VM that would receive state of VM running on source host.
_vm = get_vm(running_vms, vm_entry.key)
if _vm:
# VM already running. No need to proceed further.
logging.log(f"{_vm.key} Already running")
return
logging.info(f"Starting {vm_entry.key}")
# FIXME: There should be better vnc port allocation scheme
actual_vm = qmp.QEMUMachine(
"/usr/bin/qemu-system-x86_64",
args=get_start_command_args(vm_entry.path, 100, migration=True, migration_port=4444),
)
try:
actual_vm.launch()
except Exception as e:
# We don't care whether MachineError or any other error occurred
logging.exception(e)
actual_vm.shutdown()
else:
vm_entry.in_migration = True
vm_pool.put(vm_entry)
running_vms.append(VM(vm_entry.key, actual_vm))
r = RequestEntry.from_scratch(type=RequestType.TransferVM,
hostname=vm_entry.hostname,
parameters={
"host": get_ipv4_address(),
"port": 4444,
},
uuid=vm_entry.uuid,
destination_host_key=destination_host_key
)
request_pool.put(r)
def main():
argparser = argparse.ArgumentParser()
argparser.add_argument("hostname", help="Name of this host. e.g /v1/host/1")
args = argparser.parse_args()
global host_pool, vm_pool, request_pool
host_pool = HostPool(client, "/v1/host")
vm_pool = VmPool(client, "/v1/vm")
request_pool = RequestPool(client, "/v1/request")
host = host_pool.get(args.hostname)
if not host:
print("No Such Host")
@ -330,11 +72,12 @@ def main():
logging.info(f"{'*' * 5} Session Started {'*' * 5}")
# It is seen that under heavy load, timeout event doesn't come
# in a predictive manner which delays heart beat update which
# in turn misunderstood by scheduler that the host is dead
# when it is actually alive. So, to ensure that we update the
# heart beat in a predictive manner we start Heart beat updating
# mechanism in separated thread
# in a predictive manner (which is intentional because we give
# higher priority to customer's requests) which delays heart
# beat update which in turn misunderstood by scheduler that the
# host is dead when it is actually alive. So, to ensure that we
# update the heart beat in a predictive manner we start Heart
# beat updating mechanism in separated thread
heartbeat_updating_thread = threading.Thread(target=update_heartbeat, args=(host,))
try:
@ -345,8 +88,8 @@ def main():
exit(-1)
for events_iterator in [
client.get_prefix("/v1/request/", value_in_json=True),
client.watch_prefix("/v1/request/", timeout=10, value_in_json=True),
etcd_client.get_prefix("/v1/request/", value_in_json=True),
etcd_client.watch_prefix("/v1/request/", timeout=10, value_in_json=True),
]:
for request_event in events_iterator:
request_event = RequestEntry(request_event)
@ -365,19 +108,19 @@ def main():
logging.debug(f"EVENT: {request_event}")
if request_event.type == RequestType.StartVM:
start_vm(vm_entry)
virtualmachine.start(vm_entry)
elif request_event.type == RequestType.StopVM:
stop_vm(vm_entry)
virtualmachine.stop(vm_entry)
elif request_event.type == RequestType.DeleteVM:
delete_vm(vm_entry)
virtualmachine.delete(vm_entry)
elif request_event.type == RequestType.InitVMMigration:
init_vm_migration(vm_entry, host.key)
virtualmachine.init_migration(vm_entry, host.key)
elif request_event.type == RequestType.TransferVM:
transfer_vm(request_event)
virtualmachine.transfer(request_event)
logging.info(f"Running VMs {running_vms}")

0
qmp/__init__.py Normal file → Executable file
View File

0
qmp/qmp.py Normal file → Executable file
View File

0
setup.py Normal file → Executable file
View File

258
virtualmachine.py Executable file
View File

@ -0,0 +1,258 @@
# QEMU Manual
# https://qemu.weilnetz.de/doc/qemu-doc.html
# For QEMU Monitor Protocol Commands Information, See
# https://qemu.weilnetz.de/doc/qemu-doc.html#pcsys_005fmonitor
import subprocess
import traceback
import errno
import qmp
import tempfile
import bitmath
import time
from ucloud_common.vm import VMStatus, VMEntry
from config import (vm_pool, request_pool, etcd_client, logging, running_vms)
from typing import Union
from functools import wraps
from dataclasses import dataclass
from ucloud_common.request import RequestEntry, RequestType
import sshtunnel
from decouple import config
from os.path import join
from ucloud_common.helpers import get_ipv4_address
@dataclass
class VM:
key: str
handle: qmp.QEMUMachine
vnc_socket_file: tempfile.NamedTemporaryFile
def __repr__(self):
return f"VM({self.key})"
def get_start_command_args(vm_entry, vnc_sock_filename: str, migration=False, migration_port=4444):
vm_memory = int(bitmath.Byte(int(vm_entry.specs["ram"])).to_MB())
vm_cpus = int(vm_entry.specs["cpu"])
vm_uuid = vm_entry.uuid
threads_per_core = 1
command = (f"-drive file=rbd:uservms/{vm_uuid},format=raw,if=virtio,cache=none"
f" -device virtio-rng-pci -vnc unix:{vnc_sock_filename}"
f" -m {vm_memory} -smp cores={vm_cpus},threads={threads_per_core}"
f" -name {vm_uuid}")
if migration:
command += f" -incoming tcp:0:{migration_port}"
return command.split(" ")
def create_vm_object(vm_entry, migration=False, migration_port=4444):
# NOTE: If migration suddenly stop working, having different
# VNC unix filename on source and destination host can
# be a possible cause of it.
# REQUIREMENT: Use Unix Socket instead of TCP Port for VNC
vnc_sock_file = tempfile.NamedTemporaryFile()
qemu_machine = qmp.QEMUMachine("/usr/bin/qemu-system-x86_64",
args=get_start_command_args(vm_entry,
vnc_sock_file.name,
migration=migration,
migration_port=migration_port
)
)
return VM(vm_entry.key, qemu_machine, vnc_sock_file)
def get_vm(vm_list: list, vm_key) -> Union[VM, None]:
return next((vm for vm in vm_list if vm.key == vm_key), None)
def need_running_vm(func):
@wraps(func)
def wrapper(e):
vm = get_vm(running_vms, e.key)
if vm:
try:
status = vm.handle.command("query-status")
logging.debug(f"VM Status Check - {status}")
except OSError:
logging.info(
f"{func.__name__} failed - VM {e.key} - Unknown Error"
)
return func(e)
else:
logging.info(
f"{func.__name__} failed because VM {e.key} is not running"
)
return
return wrapper
def create(vm_entry: VMEntry):
vm_hdd = int(bitmath.Byte(int(vm_entry.specs["hdd"])).to_MB())
_command_to_create = f"rbd clone images/{vm_entry.image_uuid}@protected uservms/{vm_entry.uuid}"
_command_to_extend = f"rbd resize uservms/{vm_entry.uuid} --size {vm_hdd}"
try:
subprocess.check_call(_command_to_create.split(" "))
except subprocess.CalledProcessError as e:
if e.returncode == errno.EEXIST:
logging.debug(f"Image for vm {vm_entry.uuid} exists")
# File Already exists. No Problem Continue
return
else:
# This exception catches all other exceptions
# i.e FileNotFound (BaseImage), pool Does Not Exists etc.
logging.exception(f"Can't clone image - {e}")
else:
# TODO: Check whether the below suprocess.check_call
# is executed successfully
subprocess.check_call(_command_to_extend.split(" "))
logging.info("New VM Created")
def start(vm_entry: VMEntry):
_vm = get_vm(running_vms, vm_entry.key)
# VM already running. No need to proceed further.
if _vm:
logging.info(f"VM {vm_entry.uuid} already running")
return
else:
create(vm_entry)
logging.info(f"Starting {vm_entry.key}")
vm = create_vm_object(vm_entry)
try:
vm.handle.launch()
except (qmp.QEMUMachineError, TypeError, Exception):
vm_entry.declare_killed()
vm_entry.add_log(f"Machine Error occurred - {traceback.format_exc()}")
vm_pool.put(vm_entry)
else:
running_vms.append(vm)
vm_entry.status = VMStatus.running
vm_entry.add_log("Started successfully")
vm_pool.put(vm_entry)
@need_running_vm
def stop(vm_entry):
vm = get_vm(running_vms, vm_entry.key)
vm.handle.shutdown()
if not vm.handle.is_running():
vm_entry.add_log("Shutdown successfully")
vm_entry.declare_stopped()
vm_pool.put(vm_entry)
running_vms.remove(vm)
def delete(vm_entry):
logging.info(f"Deleting VM {vm_entry}")
stop(vm_entry)
path_without_protocol = vm_entry.path[vm_entry.path.find(":") + 1:]
try:
rc = subprocess.call(f"rbd rm {path_without_protocol}".split(" "))
except FileNotFoundError as e:
logging.exception(e)
except Exception as e:
logging.exception(f"Unknown error occurred - {e}")
else:
if rc == 0:
etcd_client.client.delete(vm_entry.key)
else:
logging.info("Some unknown problem occur while deleting vm file")
def transfer(request_event):
# This function would run on source host i.e host on which the vm
# is running initially. This host would be responsible for transferring
# vm state to destination host.
_host, _port = request_event.parameters["host"], request_event.parameters["port"]
_uuid = request_event.uuid
_destination = request_event.destination_host_key
vm = get_vm(running_vms, join("/v1/vm", _uuid))
if vm:
tunnel = sshtunnel.SSHTunnelForwarder(
(_host, 22),
ssh_username=config("ssh_username"),
ssh_pkey=config("ssh_pkey"),
ssh_private_key_password=config("ssh_private_key_password"),
remote_bind_address=('127.0.0.1', _port),
)
try:
tunnel.start()
except sshtunnel.BaseSSHTunnelForwarderError:
logging.exception(f"Couldn't establish connection to ({_host}, 22)")
else:
vm.handle.command("migrate", uri=f"tcp:{_host}:{tunnel.local_bind_port}")
status = vm.handle.command("query-migrate")["status"]
while status not in ["failed", "completed"]:
time.sleep(2)
status = vm.handle.command("query-migrate")["status"]
with vm_pool.get_put(request_event.uuid) as source_vm:
if status == "failed":
source_vm.add_log("Migration Failed")
elif status == "completed":
# If VM is successfully migrated then shutdown the VM
# on this host and update hostname to destination host key
source_vm.add_log("Successfully migrated")
source_vm.hostname = _destination
running_vms.remove(vm)
vm.handle.shutdown()
source_vm.in_migration = False # VM transfer finished
finally:
tunnel.close()
def init_migration(vm_entry, destination_host_key):
# This function would run on destination host i.e host on which the vm
# would be transferred after migration.
# This host would be responsible for starting VM that would receive
# state of VM running on source host.
_vm = get_vm(running_vms, vm_entry.key)
if _vm:
# VM already running. No need to proceed further.
logging.info(f"{_vm.key} Already running")
return
logging.info(f"Starting {vm_entry.key}")
vm = create_vm_object(vm_entry, migration=True, migration_port=4444)
try:
vm.handle.launch()
except Exception as e:
# We don't care whether MachineError or any other error occurred
logging.exception(e)
vm.handle.shutdown()
else:
vm_entry.in_migration = True
vm_pool.put(vm_entry)
running_vms.append(vm)
r = RequestEntry.from_scratch(type=RequestType.TransferVM,
hostname=vm_entry.hostname,
parameters={
"host": get_ipv4_address(),
"port": 4444,
},
uuid=vm_entry.uuid,
destination_host_key=destination_host_key
)
request_pool.put(r)