single node,w/o ceph networking implemented

This commit is contained in:
ahmadbilalkhalid 2019-11-11 23:42:57 +05:00
parent 2a66be07a6
commit da5a600ccb
23 changed files with 866 additions and 147 deletions

1
.gitignore vendored
View file

@ -4,4 +4,5 @@
__pycache__ __pycache__
docs/build
*/log.txt */log.txt

View file

@ -17,6 +17,8 @@ etcd3-wrapper = {editable = true,git = "git+https://code.ungleich.ch/ungleich-pu
python-etcd3 = {editable = true,git = "git+https://github.com/kragniz/python-etcd3.git"} python-etcd3 = {editable = true,git = "git+https://github.com/kragniz/python-etcd3.git"}
pyotp = "*" pyotp = "*"
sshtunnel = "*" sshtunnel = "*"
helper = "*"
sphinx = "*"
[requires] [requires]
python_version = "3.5" python_version = "3.5"

331
Pipfile.lock generated
View file

@ -1,7 +1,7 @@
{ {
"_meta": { "_meta": {
"hash": { "hash": {
"sha256": "b7a8409bec451e017440f063d8436fe66b18affcde7ad5497b433191ae465a52" "sha256": "45db72f1a666be82e7dc044ced7e7ad7a5b5a6efbb8b8103e6ad04c93a7d017a"
}, },
"pipfile-spec": 6, "pipfile-spec": 6,
"requires": { "requires": {
@ -16,6 +16,13 @@
] ]
}, },
"default": { "default": {
"alabaster": {
"hashes": [
"sha256:446438bdcca0e05bd45ea2de1668c1d9b032e1a9154c2c259092d77031ddd359",
"sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02"
],
"version": "==0.7.12"
},
"aniso8601": { "aniso8601": {
"hashes": [ "hashes": [
"sha256:529dcb1f5f26ee0df6c0a1ee84b7b27197c3c50fc3a6321d66c544689237d072", "sha256:529dcb1f5f26ee0df6c0a1ee84b7b27197c3c50fc3a6321d66c544689237d072",
@ -23,6 +30,13 @@
], ],
"version": "==8.0.0" "version": "==8.0.0"
}, },
"babel": {
"hashes": [
"sha256:af92e6106cb7c55286b25b38ad7695f8b4efb36a90ba483d7f7a6628c46158ab",
"sha256:e86135ae101e31e2c8ec20a4e0c5220f4eed12487d5cf3f78be7e98d3a57fc28"
],
"version": "==2.7.0"
},
"bcrypt": { "bcrypt": {
"hashes": [ "hashes": [
"sha256:0258f143f3de96b7c14f762c770f5fc56ccd72f8a1857a451c1cd9a655d9ac89", "sha256:0258f143f3de96b7c14f762c770f5fc56ccd72f8a1857a451c1cd9a655d9ac89",
@ -62,40 +76,41 @@
}, },
"cffi": { "cffi": {
"hashes": [ "hashes": [
"sha256:00d890313797d9fe4420506613384b43099ad7d2b905c0752dbcc3a6f14d80fa", "sha256:0b49274afc941c626b605fb59b59c3485c17dc776dc3cc7cc14aca74cc19cc42",
"sha256:0cf9e550ac6c5e57b713437e2f4ac2d7fd0cd10336525a27224f5fc1ec2ee59a", "sha256:0e3ea92942cb1168e38c05c1d56b0527ce31f1a370f6117f1d490b8dcd6b3a04",
"sha256:0ea23c9c0cdd6778146a50d867d6405693ac3b80a68829966c98dd5e1bbae400", "sha256:135f69aecbf4517d5b3d6429207b2dff49c876be724ac0c8bf8e1ea99df3d7e5",
"sha256:193697c2918ecdb3865acf6557cddf5076bb39f1f654975e087b67efdff83365", "sha256:19db0cdd6e516f13329cba4903368bff9bb5a9331d3410b1b448daaadc495e54",
"sha256:1ae14b542bf3b35e5229439c35653d2ef7d8316c1fffb980f9b7647e544baa98", "sha256:2781e9ad0e9d47173c0093321bb5435a9dfae0ed6a762aabafa13108f5f7b2ba",
"sha256:1e389e069450609c6ffa37f21f40cce36f9be7643bbe5051ab1de99d5a779526", "sha256:291f7c42e21d72144bb1c1b2e825ec60f46d0a7468f5346841860454c7aa8f57",
"sha256:263242b6ace7f9cd4ea401428d2d45066b49a700852334fd55311bde36dcda14", "sha256:2c5e309ec482556397cb21ede0350c5e82f0eb2621de04b2633588d118da4396",
"sha256:33142ae9807665fa6511cfa9857132b2c3ee6ddffb012b3f0933fc11e1e830d5", "sha256:2e9c80a8c3344a92cb04661115898a9129c074f7ab82011ef4b612f645939f12",
"sha256:364f8404034ae1b232335d8c7f7b57deac566f148f7222cef78cf8ae28ef764e", "sha256:32a262e2b90ffcfdd97c7a5e24a6012a43c61f1f5a57789ad80af1d26c6acd97",
"sha256:47368f69fe6529f8f49a5d146ddee713fc9057e31d61e8b6dc86a6a5e38cecc1", "sha256:3c9fff570f13480b201e9ab69453108f6d98244a7f495e91b6c654a47486ba43",
"sha256:4895640844f17bec32943995dc8c96989226974dfeb9dd121cc45d36e0d0c434", "sha256:415bdc7ca8c1c634a6d7163d43fb0ea885a07e9618a64bda407e04b04333b7db",
"sha256:558b3afef987cf4b17abd849e7bedf64ee12b28175d564d05b628a0f9355599b", "sha256:42194f54c11abc8583417a7cf4eaff544ce0de8187abaf5d29029c91b1725ad3",
"sha256:5ba86e1d80d458b338bda676fd9f9d68cb4e7a03819632969cf6d46b01a26730", "sha256:4424e42199e86b21fc4db83bd76909a6fc2a2aefb352cb5414833c030f6ed71b",
"sha256:63424daa6955e6b4c70dc2755897f5be1d719eabe71b2625948b222775ed5c43", "sha256:4a43c91840bda5f55249413037b7a9b79c90b1184ed504883b72c4df70778579",
"sha256:6381a7d8b1ebd0bc27c3bc85bc1bfadbb6e6f756b4d4db0aa1425c3719ba26b4", "sha256:599a1e8ff057ac530c9ad1778293c665cb81a791421f46922d80a86473c13346",
"sha256:6381ab708158c4e1639da1f2a7679a9bbe3e5a776fc6d1fd808076f0e3145331", "sha256:5c4fae4e9cdd18c82ba3a134be256e98dc0596af1e7285a3d2602c97dcfa5159",
"sha256:6fd58366747debfa5e6163ada468a90788411f10c92597d3b0a912d07e580c36", "sha256:5ecfa867dea6fabe2a58f03ac9186ea64da1386af2159196da51c4904e11d652",
"sha256:728ec653964655d65408949b07f9b2219df78badd601d6c49e28d604efe40599", "sha256:62f2578358d3a92e4ab2d830cd1c2049c9c0d0e6d3c58322993cc341bdeac22e",
"sha256:7cfcfda59ef1f95b9f729c56fe8a4041899f96b72685d36ef16a3440a0f85da8", "sha256:6471a82d5abea994e38d2c2abc77164b4f7fbaaf80261cb98394d5793f11b12a",
"sha256:819f8d5197c2684524637f940445c06e003c4a541f9983fd30d6deaa2a5487d8", "sha256:6d4f18483d040e18546108eb13b1dfa1000a089bcf8529e30346116ea6240506",
"sha256:825ecffd9574557590e3225560a8a9d751f6ffe4a49e3c40918c9969b93395fa", "sha256:71a608532ab3bd26223c8d841dde43f3516aa5d2bf37b50ac410bb5e99053e8f",
"sha256:8a2bcae2258d00fcfc96a9bde4a6177bc4274fe033f79311c5dd3d3148c26518", "sha256:74a1d8c85fb6ff0b30fbfa8ad0ac23cd601a138f7509dc617ebc65ef305bb98d",
"sha256:9009e917d8f5ef780c2626e29b6bc126f4cb2a4d43ca67aa2b40f2a5d6385e78", "sha256:7b93a885bb13073afb0aa73ad82059a4c41f4b7d8eb8368980448b52d4c7dc2c",
"sha256:9c77564a51d4d914ed5af096cd9843d90c45b784b511723bd46a8a9d09cf16fc", "sha256:7d4751da932caaec419d514eaa4215eaf14b612cff66398dd51129ac22680b20",
"sha256:a19089fa74ed19c4fe96502a291cfdb89223a9705b1d73b3005df4256976142e", "sha256:7f627141a26b551bdebbc4855c1157feeef18241b4b8366ed22a5c7d672ef858",
"sha256:a40ed527bffa2b7ebe07acc5a3f782da072e262ca994b4f2085100b5a444bbb2", "sha256:8169cf44dd8f9071b2b9248c35fc35e8677451c52f795daa2bb4643f32a540bc",
"sha256:b8f09f21544b9899defb09afbdaeb200e6a87a2b8e604892940044cf94444644", "sha256:aa00d66c0fab27373ae44ae26a66a9e43ff2a678bf63a9c7c1a9a4d61172827a",
"sha256:bb75ba21d5716abc41af16eac1145ab2e471deedde1f22c6f99bd9f995504df0", "sha256:ccb032fda0873254380aa2bfad2582aedc2959186cce61e3a17abc1a55ff89c3",
"sha256:e22a00c0c81ffcecaf07c2bfb3672fa372c50e2bd1024ffee0da191c1b27fc71", "sha256:d754f39e0d1603b5b24a7f8484b22d2904fa551fe865fd0d4c3332f078d20d4e",
"sha256:e55b5a746fb77f10c83e8af081979351722f6ea48facea79d470b3731c7b2891", "sha256:d75c461e20e29afc0aee7172a0950157c704ff0dd51613506bd7d82b718e7410",
"sha256:ec2fa3ee81707a5232bf2dfbd6623fdb278e070d596effc7e2d788f2ada71a05", "sha256:dcd65317dd15bc0451f3e01c80da2216a31916bdcffd6221ca1202d96584aa25",
"sha256:fd82eb4694be712fcae03c717ca2e0fc720657ac226b80bbb597e971fc6928c2" "sha256:e570d3ab32e2c2861c4ebe6ffcad6a8abf9347432a37608fe1fbd157b3f0036b",
"sha256:fd43a88e045cf992ed09fa724b5315b790525f2676883a6ea64e3263bae6549d"
], ],
"version": "==1.13.1" "version": "==1.13.2"
}, },
"chardet": { "chardet": {
"hashes": [ "hashes": [
@ -137,6 +152,14 @@
], ],
"version": "==2.8" "version": "==2.8"
}, },
"docutils": {
"hashes": [
"sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0",
"sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827",
"sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99"
],
"version": "==0.15.2"
},
"etcd3-wrapper": { "etcd3-wrapper": {
"editable": true, "editable": true,
"git": "https://code.ungleich.ch/ungleich-public/etcd3_wrapper.git", "git": "https://code.ungleich.ch/ungleich-public/etcd3_wrapper.git",
@ -160,56 +183,64 @@
}, },
"grpcio": { "grpcio": {
"hashes": [ "hashes": [
"sha256:01cb705eafba1108e2a947ba0457da4f6a1e8142c729fc61702b5fdd11009eb1", "sha256:0419ae5a45f49c7c40d9ae77ae4de9442431b7822851dfbbe56ee0eacb5e5654",
"sha256:0b5a79e29f167d3cd06faad6b15babbc2661066daaacf79373c3a8e67ca1fca1", "sha256:1e8631eeee0fb0b4230aeb135e4890035f6ef9159c2a3555fa184468e325691a",
"sha256:1097a61a0e97b3580642e6e1460a3a1f1ba1815e2a70d6057173bcc495417076", "sha256:24db2fa5438f3815a4edb7a189035051760ca6aa2b0b70a6a948b28bfc63c76b",
"sha256:13970e665a4ec4cec7d067d7d3504a0398c657d91d26c581144ad9044e429c9a", "sha256:2adb1cdb7d33e91069517b41249622710a94a1faece1fed31cd36904e4201cde",
"sha256:1557817cea6e0b87fad2a3e20da385170efb03a313db164e8078955add2dfa1b", "sha256:2cd51f35692b551aeb1fdeb7a256c7c558f6d78fcddff00640942d42f7aeba5f",
"sha256:1b0fb036a2f9dd93d9a35c57c26420eeb4b571fcb14b51cddf5b1e73ea5d882b", "sha256:3247834d24964589f8c2b121b40cd61319b3c2e8d744a6a82008643ef8a378b1",
"sha256:24d9e58d08e8cd545d8a3247a18654aff0e5e60414701696a8098fbb0d792b75", "sha256:3433cb848b4209717722b62392e575a77a52a34d67c6730138102abc0a441685",
"sha256:2c38b586163d2b91567fe5e6d9e7798f792012365adc838a64b66b22dce3f4d4", "sha256:39671b7ff77a962bd745746d9d2292c8ed227c5748f16598d16d8631d17dd7e5",
"sha256:2df3ab4348507de60e1cbf75196403df1b9b4c4d4dc5bd11ac4eb63c46f691c7", "sha256:40a0b8b2e6f6dd630f8b267eede2f40a848963d0f3c40b1b1f453a4a870f679e",
"sha256:32f70f7c90454ea568b868af2e96616743718d9233d23f62407e98caed81dfbf", "sha256:40f9a74c7aa210b3e76eb1c9d56aa8d08722b73426a77626967019df9bbac287",
"sha256:3af2a49d576820045c9c880ff29a5a96d020fe31b35d248519bfc6ccb8be4eac", "sha256:423f76aa504c84cb94594fb88b8a24027c887f1c488cf58f2173f22f4fbd046c",
"sha256:4ff7d63800a63db031ebac6a6f581ae84877c959401c24c28f2cc51fd36c47ad", "sha256:43bd04cec72281a96eb361e1b0232f0f542b46da50bcfe72ef7e5a1b41d00cb3",
"sha256:502aaa8be56f0ae69cda66bc27e1fb5531ceaa27ca515ec3c34f6178b1297180", "sha256:43e38762635c09e24885d15e3a8e374b72d105d4178ee2cc9491855a8da9c380",
"sha256:55358ce3ec283222e435f7dbc6603521438458f3c65f7c1cb33b8dabf56d70d8", "sha256:4413b11c2385180d7de03add6c8845dd66692b148d36e27ec8c9ef537b2553a1",
"sha256:5583b01c67f85fa64a2c3fb085e5517c88b9c1500a2cce12d473cd99d0ed2e49", "sha256:4450352a87094fd58daf468b04c65a9fa19ad11a0ac8ac7b7ff17d46f873cbc1",
"sha256:58d9a5557d3eb7b734a3cea8b16c891099a522b3953a45a30bd4c034f75fc913", "sha256:49ffda04a6e44de028b3b786278ac9a70043e7905c3eea29eed88b6524d53a29",
"sha256:5911f042c4ab177757eec5bcb4e2e9a2e823d888835d24577321bf55f02938fa", "sha256:4a38c4dde4c9120deef43aaabaa44f19186c98659ce554c29788c4071ab2f0a4",
"sha256:5e16ea922f4e5017c04fd94e2639b1006e03097e9dd0cbb7a1c852af3ea8bf2e", "sha256:50b1febdfd21e2144b56a9aa226829e93a79c354ef22a4e5b013d9965e1ec0ed",
"sha256:656e19d3f1b9050ee01b457f92838a9679d7cf84c995f708780f44484048705e", "sha256:559b1a3a8be7395ded2943ea6c2135d096f8cc7039d6d12127110b6496f251fe",
"sha256:6a1435449a82008c451c7e1a82a834387b9108f9a8d27910f86e7c482f5568e9", "sha256:5de86c182667ec68cf84019aa0d8ceccf01d352cdca19bf9e373725204bdbf50",
"sha256:6ff02ca6cbed0ddb76e93ba0f8beb6a8c77d83a84eb7cafe2ae3399a8b9d69ea", "sha256:5fc069bb481fe3fad0ba24d3baaf69e22dfa6cc1b63290e6dfeaf4ac1e996fb7",
"sha256:76de68f60102f333bf4817f38e81ecbee68b850f5a5da9f355235e948ac40981", "sha256:6a19d654da49516296515d6f65de4bbcbd734bc57913b21a610cfc45e6df3ff1",
"sha256:7c6d7ddd50fc6548ea1dfe09c62509c4f95b8b40082287747be05aa8feb15ee2", "sha256:7535b3e52f498270e7877dde1c8944d6b7720e93e2e66b89c82a11447b5818f5",
"sha256:836b9d29507de729129e363276fe7c7d6a34c7961e0f155787025552b15d22c0", "sha256:7c4e495bcabc308198b8962e60ca12f53b27eb8f03a21ac1d2d711d6dd9ecfca",
"sha256:869242b2baf8a888a4fe0548f86abc47cb4b48bdfd76ae62d6456e939c202e65", "sha256:8a8fc4a0220367cb8370cedac02272d574079ccc32bffbb34d53aaf9e38b5060",
"sha256:8954b24bd08641d906ee50b2d638efc76df893fbd0913149b80484fd0eac40c9", "sha256:8b008515e067232838daca020d1af628bf6520c8cc338bf383284efe6d8bd083",
"sha256:8cdea65d1abb2e698420db8daf20c8d272fbd9d96a51b26a713c1c76f237d181", "sha256:8d1684258e1385e459418f3429e107eec5fb3d75e1f5a8c52e5946b3f329d6ea",
"sha256:90161840b4fe9636f91ed0d3ea1e7e615e488cbea4e77594c889e5f3d7a776db", "sha256:8eb5d54b87fb561dc2e00a5c5226c33ffe8dbc13f2e4033a412bafb7b37b194d",
"sha256:90fb6316b4d7d36700c40db4335902b78dcae13b5466673c21fd3b08a3c1b0c6", "sha256:94cdef0c61bd014bb7af495e21a1c3a369dd0399c3cd1965b1502043f5c88d94",
"sha256:91b34f58db2611c9a93ecf751028f97fba1f06e65f49b38f272f6aa5d2977331", "sha256:9d9f3be69c7a5e84c3549a8c4403fa9ac7672da456863d21e390b2bbf45ccad1",
"sha256:9474944a96a33eb8734fa8dc5805403d57973a3526204a5e1c1780d02e0572b6", "sha256:9fb6fb5975a448169756da2d124a1beb38c0924ff6c0306d883b6848a9980f38",
"sha256:9a36275db2a4774ac16c6822e7af816ee048071d5030b4c035fd53942b361935", "sha256:a5eaae8700b87144d7dfb475aa4675e500ff707292caba3deff41609ddc5b845",
"sha256:9cbe26e2976b994c5f7c2d35a63354674d6ca0ce62f5b513f078bf63c1745229", "sha256:aaeac2d552772b76d24eaff67a5d2325bc5205c74c0d4f9fbe71685d4a971db2",
"sha256:9eaeabb3c0eecd6ddd0c16767fd12d130e2cebb8c2618f959a278b1ff336ddc3", "sha256:bb611e447559b3b5665e12a7da5160c0de6876097f62bf1d23ba66911564868e",
"sha256:a2bc7e10ebcf4be503ae427f9887e75c0cc24e88ce467a8e6eaca6bd2862406e", "sha256:bc0d41f4eb07da8b8d3ea85e50b62f6491ab313834db86ae2345be07536a4e5a",
"sha256:a5b42e6292ba51b8e67e09fc256963ba4ca9c04026de004d2fe59cc17e3c3776", "sha256:bf51051c129b847d1bb63a9b0826346b5f52fb821b15fe5e0d5ef86f268510f5",
"sha256:bd6ec1233c86c0b9bb5d03ec30dbe3ffbfa53335790320d99a7ae9018c5450f2", "sha256:c948c034d8997526011960db54f512756fb0b4be1b81140a15b4ef094c6594a4",
"sha256:bef57530816af54d66b1f4c70a8f851f320cb6f84d4b5a0b422b0e9811ea4e59", "sha256:d435a01334157c3b126b4ee5141401d44bdc8440993b18b05e2f267a6647f92d",
"sha256:c146a63eaadc6589b732780061f3c94cd0574388d372baccbb3c1597a9ebdb7a", "sha256:d46c1f95672b73288e08cdca181e14e84c6229b5879561b7b8cfd48374e09287",
"sha256:c2efd3b130dc639d615b6f58980e1bfd1b177ad821f30827afa5001aa30ddd48", "sha256:d5d58309b42064228b16b0311ff715d6c6e20230e81b35e8d0c8cfa1bbdecad8",
"sha256:c888b18f7392e6cc79a33a803e7ebd7890ac3318f571fca6b356526f35b53b12", "sha256:dc6e2e91365a1dd6314d615d80291159c7981928b88a4c65654e3fefac83a836",
"sha256:ca30721fda297ae22f16bc37aa7ed244970ddfdcb98247570cdd26daaad4665e", "sha256:e0dfb5f7a39029a6cbec23affa923b22a2c02207960fd66f109e01d6f632c1eb",
"sha256:cf5f5340dd682ab034baa52f423a0f91326489c262ac9617fa06309ec05880e9", "sha256:eb4bf58d381b1373bd21d50837a53953d625d1693f1b58fed12743c75d3dd321",
"sha256:d0726aa0d9b57c56985db5952e90fb1033a317074f2877db5307cdd6eede1564", "sha256:ebb211a85248dbc396b29320273c1ffde484b898852432613e8df0164c091006",
"sha256:df442945b2dd6f8ae0e20b403e0fd4548cd5c2aad69200047cc3251257b78f65", "sha256:ec759ece4786ae993a5b7dc3b3dead6e9375d89a6c65dfd6860076d2eb2abe7b",
"sha256:e08e758c31919d167c0867539bd3b2441629ef00aa595e3ea2b635273659f40a", "sha256:f55108397a8fa164268238c3e69cc134e945d1f693572a2f05a028b8d0d2b837",
"sha256:e4864339deeeaefaad34dd3a432ee618a039fca28efb292949c855e00878203c", "sha256:f6c706866d424ff285b85a02de7bbe5ed0ace227766b2c42cbe12f3d9ea5a8aa",
"sha256:f4cd049cb94d9f517b1cab5668a3b345968beba093bc79a637e671000b3540ec" "sha256:f8370ad332b36fbad117440faf0dd4b910e80b9c49db5648afd337abdde9a1b6"
], ],
"version": "==1.24.3" "version": "==1.25.0"
},
"helper": {
"hashes": [
"sha256:33d4a58046018fea9f46da5835a768feb9beab3528d4025d063bf354c4a19750",
"sha256:a63d4a9255ad5071043e7e4ab8000a512627f1db958b1941b63c7d75e56ea65c"
],
"index": "pypi",
"version": "==2.4.2"
}, },
"idna": { "idna": {
"hashes": [ "hashes": [
@ -218,6 +249,13 @@
], ],
"version": "==2.8" "version": "==2.8"
}, },
"imagesize": {
"hashes": [
"sha256:3f349de3eb99145973fefb7dbe38554414e5c30abd0c8e4b970a7c9d09f3a1d8",
"sha256:f3832918bc3c66617f92e35f5d70729187676313caa60c187eb0f28b8fe5e3b5"
],
"version": "==1.1.0"
},
"itsdangerous": { "itsdangerous": {
"hashes": [ "hashes": [
"sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19", "sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19",
@ -265,6 +303,13 @@
], ],
"version": "==1.1.1" "version": "==1.1.1"
}, },
"packaging": {
"hashes": [
"sha256:28b924174df7a2fa32c1953825ff29c61e2f5e082343165438812f00d3a7fc47",
"sha256:d9551545c6d761f3def1677baf08ab2a3ca17c56879e70fecba2fc4dde4ed108"
],
"version": "==19.2"
},
"paramiko": { "paramiko": {
"hashes": [ "hashes": [
"sha256:99f0179bdc176281d21961a003ffdb2ec369daac1a1007241f53374e376576cf", "sha256:99f0179bdc176281d21961a003ffdb2ec369daac1a1007241f53374e376576cf",
@ -299,6 +344,13 @@
], ],
"version": "==2.19" "version": "==2.19"
}, },
"pygments": {
"hashes": [
"sha256:71e430bc85c88a430f000ac1d9b331d2407f681d6f6aec95e8bcfbc3df5b0127",
"sha256:881c4c157e45f30af185c1ffe8d549d48ac9127433f2c380c24b84572ad66297"
],
"version": "==2.4.2"
},
"pynacl": { "pynacl": {
"hashes": [ "hashes": [
"sha256:05c26f93964373fc0abe332676cb6735f0ecad27711035b9472751faa8521255", "sha256:05c26f93964373fc0abe332676cb6735f0ecad27711035b9472751faa8521255",
@ -333,6 +385,13 @@
"index": "pypi", "index": "pypi",
"version": "==2.3.0" "version": "==2.3.0"
}, },
"pyparsing": {
"hashes": [
"sha256:20f995ecd72f2a1f4bf6b072b63b22e2eb457836601e76d6e5dfcd75436acc1f",
"sha256:4ca62001be367f01bd3e92ecbb79070272a9d4964dce6a48a82ff0b8bc7e683a"
],
"version": "==2.4.5"
},
"python-decouple": { "python-decouple": {
"hashes": [ "hashes": [
"sha256:1317df14b43efee4337a4aa02914bf004f010cd56d6c4bd894e6474ec8c4fe2d" "sha256:1317df14b43efee4337a4aa02914bf004f010cd56d6c4bd894e6474ec8c4fe2d"
@ -352,6 +411,24 @@
], ],
"version": "==2019.3" "version": "==2019.3"
}, },
"pyyaml": {
"hashes": [
"sha256:0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9",
"sha256:01adf0b6c6f61bd11af6e10ca52b7d4057dd0be0343eb9283c878cf3af56aee4",
"sha256:5124373960b0b3f4aa7df1707e63e9f109b5263eca5976c66e08b1c552d4eaf8",
"sha256:5ca4f10adbddae56d824b2c09668e91219bb178a1eee1faa56af6f99f11bf696",
"sha256:7907be34ffa3c5a32b60b95f4d95ea25361c951383a894fec31be7252b2b6f34",
"sha256:7ec9b2a4ed5cad025c2278a1e6a19c011c80a3caaac804fd2d329e9cc2c287c9",
"sha256:87ae4c829bb25b9fe99cf71fbb2140c448f534e24c998cc60f39ae4f94396a73",
"sha256:9de9919becc9cc2ff03637872a440195ac4241c80536632fffeb6a1e25a74299",
"sha256:a5a85b10e450c66b49f98846937e8cfca1db3127a9d5d1e31ca45c3d0bef4c5b",
"sha256:b0997827b4f6a7c286c01c5f60384d218dca4ed7d9efa945c3e1aa623d5709ae",
"sha256:b631ef96d3222e62861443cc89d6563ba3eeb816eeb96b2629345ab795e53681",
"sha256:bf47c0607522fdbca6c9e817a6e81b08491de50f3766a7a0e6a5be7905961b41",
"sha256:f81025eddd0327c7d4cfe9b62cf33190e1e736cc6e97502b3ec425f574b3e7a8"
],
"version": "==5.1.2"
},
"requests": { "requests": {
"hashes": [ "hashes": [
"sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4", "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4",
@ -362,10 +439,67 @@
}, },
"six": { "six": {
"hashes": [ "hashes": [
"sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd",
"sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66"
], ],
"version": "==1.12.0" "version": "==1.13.0"
},
"snowballstemmer": {
"hashes": [
"sha256:209f257d7533fdb3cb73bdbd24f436239ca3b2fa67d56f6ff88e86be08cc5ef0",
"sha256:df3bac3df4c2c01363f3dd2cfa78cce2840a79b9f1c2d2de9ce8d31683992f52"
],
"version": "==2.0.0"
},
"sphinx": {
"hashes": [
"sha256:31088dfb95359384b1005619827eaee3056243798c62724fd3fa4b84ee4d71bd",
"sha256:52286a0b9d7caa31efee301ec4300dbdab23c3b05da1c9024b4e84896fb73d79"
],
"index": "pypi",
"version": "==2.2.1"
},
"sphinxcontrib-applehelp": {
"hashes": [
"sha256:edaa0ab2b2bc74403149cb0209d6775c96de797dfd5b5e2a71981309efab3897",
"sha256:fb8dee85af95e5c30c91f10e7eb3c8967308518e0f7488a2828ef7bc191d0d5d"
],
"version": "==1.0.1"
},
"sphinxcontrib-devhelp": {
"hashes": [
"sha256:6c64b077937330a9128a4da74586e8c2130262f014689b4b89e2d08ee7294a34",
"sha256:9512ecb00a2b0821a146736b39f7aeb90759834b07e81e8cc23a9c70bacb9981"
],
"version": "==1.0.1"
},
"sphinxcontrib-htmlhelp": {
"hashes": [
"sha256:4670f99f8951bd78cd4ad2ab962f798f5618b17675c35c5ac3b2132a14ea8422",
"sha256:d4fd39a65a625c9df86d7fa8a2d9f3cd8299a3a4b15db63b50aac9e161d8eff7"
],
"version": "==1.0.2"
},
"sphinxcontrib-jsmath": {
"hashes": [
"sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178",
"sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8"
],
"version": "==1.0.1"
},
"sphinxcontrib-qthelp": {
"hashes": [
"sha256:513049b93031beb1f57d4daea74068a4feb77aa5630f856fcff2e50de14e9a20",
"sha256:79465ce11ae5694ff165becda529a600c754f4bc459778778c7017374d4d406f"
],
"version": "==1.0.2"
},
"sphinxcontrib-serializinghtml": {
"hashes": [
"sha256:c0efb33f8052c04fd7a26c0a07f1678e8512e0faec19f4aa8f2473a8b81d5227",
"sha256:db6615af393650bf1151a6cd39120c29abaf93cc60db8c48eb2dddbfdc3a9768"
],
"version": "==1.1.3"
}, },
"sshtunnel": { "sshtunnel": {
"hashes": [ "hashes": [
@ -376,10 +510,10 @@
}, },
"tenacity": { "tenacity": {
"hashes": [ "hashes": [
"sha256:3a916e734559f1baa2cab965ee00061540c41db71c3bf25375b81540a19758fc", "sha256:72f397c2bb1887e048726603f3f629ea16f88cb3e61e4ed3c57e98582b8e3571",
"sha256:e664bd94f088b17f46da33255ae33911ca6a0fe04b156d334b601a4ef66d3c5f" "sha256:947e728aedf06e8db665bb7898112e90d17e48cc3f3289784a2b9ccf6e56fabc"
], ],
"version": "==5.1.5" "version": "==6.0.0"
}, },
"ucloud-common": { "ucloud-common": {
"editable": true, "editable": true,
@ -627,6 +761,13 @@
], ],
"version": "==1.6.0" "version": "==1.6.0"
}, },
"pygments": {
"hashes": [
"sha256:71e430bc85c88a430f000ac1d9b331d2407f681d6f6aec95e8bcfbc3df5b0127",
"sha256:881c4c157e45f30af185c1ffe8d549d48ac9127433f2c380c24b84572ad66297"
],
"version": "==2.4.2"
},
"pylint": { "pylint": {
"hashes": [ "hashes": [
"sha256:5d77031694a5fb97ea95e828c8d10fc770a1df6eb3906067aaed42201a8a6a09", "sha256:5d77031694a5fb97ea95e828c8d10fc770a1df6eb3906067aaed42201a8a6a09",
@ -662,10 +803,10 @@
}, },
"pyroma": { "pyroma": {
"hashes": [ "hashes": [
"sha256:54d332f540d4828bc5672b75ccf9e12d4b2f72a42a4f304bcec1c73565aecc26", "sha256:351758a81e2a12c970deb73687e239636aad52795cd81429695073d59fff0699",
"sha256:6b94feb609e1896579302f0836ef2fad3f17e0557e3ddcd0d76206cd3e366d27" "sha256:c49c00377219626bf83df42adf018cc231e6162b68cc7aaf2ff1c63803924102"
], ],
"version": "==2.5" "version": "==2.6"
}, },
"pyyaml": { "pyyaml": {
"hashes": [ "hashes": [
@ -707,10 +848,10 @@
}, },
"six": { "six": {
"hashes": [ "hashes": [
"sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd",
"sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66"
], ],
"version": "==1.12.0" "version": "==1.13.0"
}, },
"snowballstemmer": { "snowballstemmer": {
"hashes": [ "hashes": [

View file

@ -4,6 +4,10 @@ from config import etcd_client as client
from config import VM_PREFIX from config import VM_PREFIX
class Optional:
pass
class Field: class Field:
def __init__(self, _name, _type, _value=None): def __init__(self, _name, _type, _value=None):
self.name = _name self.name = _name
@ -18,7 +22,9 @@ class Field:
if self.value == KeyError: if self.value == KeyError:
self.add_error("'{}' field is a required field".format(self.name)) self.add_error("'{}' field is a required field".format(self.name))
else: else:
if not isinstance(self.value, self.type): if isinstance(self.value, Optional):
pass
elif not isinstance(self.value, self.type):
self.add_error("Incorrect Type for '{}' field".format(self.name)) self.add_error("Incorrect Type for '{}' field".format(self.name))
else: else:
self.validation() self.validation()

View file

@ -24,6 +24,7 @@ REQUEST_PREFIX = config("REQUEST_PREFIX")
FILE_PREFIX = config("FILE_PREFIX") FILE_PREFIX = config("FILE_PREFIX")
IMAGE_PREFIX = config("IMAGE_PREFIX") IMAGE_PREFIX = config("IMAGE_PREFIX")
IMAGE_STORE_PREFIX = config("IMAGE_STORE_PREFIX") IMAGE_STORE_PREFIX = config("IMAGE_STORE_PREFIX")
NETWORK_PREFIX = config("NETWORK_PREFIX")
etcd_client = Etcd3Wrapper(host=config("ETCD_URL")) etcd_client = Etcd3Wrapper(host=config("ETCD_URL"))

View file

@ -92,9 +92,11 @@ def resolve_image_name(name, etcd_client):
return image_uuid return image_uuid
def random_bytes(num=6): def random_bytes(num=6):
return [random.randrange(256) for _ in range(num)] return [random.randrange(256) for _ in range(num)]
def generate_mac(uaa=False, multicast=False, oui=None, separator=':', byte_fmt='%02x'): def generate_mac(uaa=False, multicast=False, oui=None, separator=':', byte_fmt='%02x'):
mac = random_bytes() mac = random_bytes()
if oui: if oui:
@ -112,6 +114,7 @@ def generate_mac(uaa=False, multicast=False, oui=None, separator=':', byte_fmt='
mac[0] |= 1 << 1 # set bit 1 mac[0] |= 1 << 1 # set bit 1
return separator.join(byte_fmt % b for b in mac) return separator.join(byte_fmt % b for b in mac)
def get_ip_addr(mac_address, device): def get_ip_addr(mac_address, device):
"""Return IP address of a device provided its mac address / link local address """Return IP address of a device provided its mac address / link local address
and the device with which it is connected. and the device with which it is connected.
@ -140,3 +143,23 @@ def get_ip_addr(mac_address, device):
if ip.is_global and mac_address == mac: if ip.is_global and mac_address == mac:
result.append(ip) result.append(ip)
return result return result
def increment_etcd_counter(etcd_client, key):
kv = etcd_client.get(key)
if kv:
counter = int(kv.value)
counter = counter + 1
else:
counter = 1
etcd_client.put(key, str(counter))
return counter
def get_etcd_counter(etcd_client, key):
kv = etcd_client.get(key)
if kv:
return int(kv.value)
return None

View file

@ -12,7 +12,7 @@ from flask_restful import Resource, Api
from ucloud_common.vm import VMStatus from ucloud_common.vm import VMStatus
from ucloud_common.request import RequestEntry, RequestType from ucloud_common.request import RequestEntry, RequestType
from helper import generate_mac, get_ip_addr from helper import generate_mac, get_ip_addr, get_etcd_counter, increment_etcd_counter
from config import ( from config import (
etcd_client, etcd_client,
@ -21,6 +21,7 @@ from config import (
HOST_PREFIX, HOST_PREFIX,
FILE_PREFIX, FILE_PREFIX,
IMAGE_PREFIX, IMAGE_PREFIX,
NETWORK_PREFIX,
logging, logging,
REQUEST_POOL, REQUEST_POOL,
VM_POOL, VM_POOL,
@ -35,7 +36,6 @@ class CreateVM(Resource):
@staticmethod @staticmethod
def post(): def post():
data = request.json data = request.json
print(data)
validator = schemas.CreateVMSchema(data) validator = schemas.CreateVMSchema(data)
if validator.is_valid(): if validator.is_valid():
vm_uuid = uuid4().hex vm_uuid = uuid4().hex
@ -57,10 +57,10 @@ class CreateVM(Resource):
"image_uuid": validator.image_uuid, "image_uuid": validator.image_uuid,
"log": [], "log": [],
"vnc_socket": "", "vnc_socket": "",
"mac": str(generate_mac()), "network": data["network"],
"metadata": { "metadata": {
"ssh-keys": [] "ssh-keys": []
} },
} }
etcd_client.put(vm_key, vm_entry, value_in_json=True) etcd_client.put(vm_key, vm_entry, value_in_json=True)
@ -80,9 +80,8 @@ class VmStatus(Resource):
if validator.is_valid(): if validator.is_valid():
vm = VM_POOL.get(os.path.join(VM_PREFIX, data["uuid"])) vm = VM_POOL.get(os.path.join(VM_PREFIX, data["uuid"]))
vm_value = vm.value.copy() vm_value = vm.value.copy()
vm_value["ip"] = list(map(str, get_ip_addr(vm.mac, "br0"))) # vm_value["ip"] = list(map(str, get_ip_addr(vm.mac, "br0")))
vm.value = vm_value vm.value = vm_value
print(vm.value)
return vm.value return vm.value
else: else:
return validator.get_errors(), 400 return validator.get_errors(), 400
@ -217,7 +216,7 @@ class ListUserVM(Resource):
"specs": vm.value["specs"], "specs": vm.value["specs"],
"status": vm.value["status"], "status": vm.value["status"],
"hostname": vm.value["hostname"], "hostname": vm.value["hostname"],
"mac": vm.value["mac"], # "mac": vm.value["mac"],
"vnc_socket": None "vnc_socket": None
if vm.value.get("vnc_socket", None) is None if vm.value.get("vnc_socket", None) is None
else vm.value["vnc_socket"], else vm.value["vnc_socket"],
@ -357,6 +356,44 @@ class RemoveSSHKey(Resource):
else: else:
return validator.get_errors(), 400 return validator.get_errors(), 400
class CreateNetwork(Resource):
@staticmethod
def post():
data = request.json
validator = schemas.CreateNetwork(data)
if validator.is_valid():
network_entry = {
"id": increment_etcd_counter(etcd_client, "/v1/counter/vxlan"),
"type": data["type"]
}
network_key = os.path.join(NETWORK_PREFIX, data["name"], data["network_name"])
etcd_client.put(network_key, network_entry, value_in_json=True)
return {"message": "Network successfully added."}
else:
return validator.get_errors(), 400
class ListUserNetwork(Resource):
@staticmethod
def get():
data = request.json
validator = schemas.OTPSchema(data)
if validator.is_valid():
prefix = os.path.join(NETWORK_PREFIX, data["name"])
networks = etcd_client.get_prefix(prefix, value_in_json=True)
user_networks = []
for net in networks:
net.value["name"] = net.key.split("/")[-1]
user_networks.append(net.value)
return {"networks": user_networks}, 200
else:
return validator.get_errors(), 400
api.add_resource(CreateVM, "/vm/create") api.add_resource(CreateVM, "/vm/create")
api.add_resource(VmStatus, "/vm/status") api.add_resource(VmStatus, "/vm/status")
@ -368,6 +405,7 @@ api.add_resource(ListPublicImages, "/image/list-public")
api.add_resource(ListUserVM, "/user/vms") api.add_resource(ListUserVM, "/user/vms")
api.add_resource(ListUserFiles, "/user/files") api.add_resource(ListUserFiles, "/user/files")
api.add_resource(ListUserNetwork, "/user/networks")
api.add_resource(AddSSHKey, "/user/add-ssh") api.add_resource(AddSSHKey, "/user/add-ssh")
api.add_resource(RemoveSSHKey, "/user/remove-ssh") api.add_resource(RemoveSSHKey, "/user/remove-ssh")
@ -376,5 +414,7 @@ api.add_resource(GetSSHKeys, "/user/get-ssh")
api.add_resource(CreateHost, "/host/create") api.add_resource(CreateHost, "/host/create")
api.add_resource(ListHost, "/host/list") api.add_resource(ListHost, "/host/list")
api.add_resource(CreateNetwork, "/network/create")
if __name__ == "__main__": if __name__ == "__main__":
app.run(host="::", debug=True) app.run(host="::", debug=True)

View file

@ -23,11 +23,11 @@ import helper
from ucloud_common.host import HostPool, HostStatus from ucloud_common.host import HostPool, HostStatus
from ucloud_common.vm import VmPool, VMStatus from ucloud_common.vm import VmPool, VMStatus
from common_fields import Field, VmUUIDField from common_fields import Field, VmUUIDField, Optional
from helper import check_otp, resolve_vm_name from helper import check_otp, resolve_vm_name
from config import etcd_client as client from config import etcd_client as client
from config import (HOST_PREFIX, VM_PREFIX, IMAGE_PREFIX, from config import (HOST_PREFIX, VM_PREFIX, IMAGE_PREFIX,
FILE_PREFIX, IMAGE_STORE_PREFIX) FILE_PREFIX, IMAGE_STORE_PREFIX, NETWORK_PREFIX)
HOST_POOL = HostPool(client, HOST_PREFIX) HOST_POOL = HostPool(client, HOST_PREFIX)
VM_POOL = VmPool(client, VM_PREFIX) VM_POOL = VmPool(client, VM_PREFIX)
@ -85,7 +85,6 @@ class OTPSchema(BaseSchema):
super().__init__(data=data, fields=_fields) super().__init__(data=data, fields=_fields)
def validation(self): def validation(self):
print(self.name.value, self.realm.value, self.token.value)
if check_otp(self.name.value, self.realm.value, self.token.value) != 200: if check_otp(self.name.value, self.realm.value, self.token.value) != 200:
self.add_error("Wrong Credentials") self.add_error("Wrong Credentials")
@ -206,20 +205,24 @@ class CreateHostSchema(OTPSchema):
class CreateVMSchema(OTPSchema): class CreateVMSchema(OTPSchema):
def __init__(self, data): def __init__(self, data):
self.parsed_specs = {} self.parsed_specs = {}
# Fields # Fields
self.specs = Field("specs", dict, data.get("specs", KeyError)) self.specs = Field("specs", dict, data.get("specs", KeyError))
self.vm_name = Field("vm_name", str, data.get("vm_name", KeyError)) self.vm_name = Field("vm_name", str, data.get("vm_name", KeyError))
self.image = Field("image", str, data.get("image", KeyError)) self.image = Field("image", str, data.get("image", KeyError))
self.network = Field("network", list, data.get("network", KeyError))
# Validation # Validation
self.image.validation = self.image_validation self.image.validation = self.image_validation
self.vm_name.validation = self.vm_name_validation self.vm_name.validation = self.vm_name_validation
self.specs.validation = self.specs_validation self.specs.validation = self.specs_validation
self.network.validation = self.network_validation
fields = [self.vm_name, self.image, self.specs] fields = [self.vm_name, self.image, self.specs, self.network]
super().__init__(data=data, fields=fields) super().__init__(data=data, fields=fields)
def image_validation(self): def image_validation(self):
try: try:
image_uuid = helper.resolve_image_name(self.image.value, client) image_uuid = helper.resolve_image_name(self.image.value, client)
@ -234,6 +237,18 @@ class CreateVMSchema(OTPSchema):
'VM with same name "{}" already exists'.format(self.vm_name.value) 'VM with same name "{}" already exists'.format(self.vm_name.value)
) )
def network_validation(self):
_network = self.network.value
if _network:
for net in _network:
network = client.get(os.path.join(NETWORK_PREFIX,
self.name.value,
net), value_in_json=True)
if not network:
self.add_error("Network with name {} does not exists"\
.format(net))
def specs_validation(self): def specs_validation(self):
ALLOWED_BASE = 10 ALLOWED_BASE = 10
@ -417,3 +432,29 @@ class GetSSHSchema(OTPSchema):
fields = [self.key_name] fields = [self.key_name]
super().__init__(data=data, fields=fields) super().__init__(data=data, fields=fields)
class CreateNetwork(OTPSchema):
def __init__(self, data):
self.network_name = Field("network_name", str, data.get("network_name", KeyError))
self.type = Field("type", str, data.get("type", KeyError))
self.network_name.validation = self.network_name_validation
self.type.validation = self.network_type_validation
fields = [self.network_name, self.type]
super().__init__(data, fields=fields)
def network_name_validation(self):
network = client.get(os.path.join(NETWORK_PREFIX,
self.name.value,
self.network_name.value),
value_in_json=True)
if network:
self.add_error("Network with name {} already exists"\
.format(self.network_name.value))
def network_type_validation(self):
supported_network_types = ["vxlan"]
if self.type.value not in supported_network_types:
self.add_error("Unsupported Network Type. Supported network types are {}".format(supported_network_types))

20
docs/Makefile Normal file
View file

@ -0,0 +1,20 @@
# Minimal makefile for Sphinx documentation
#
# You can set these variables from the command line, and also
# from the environment for the first two.
SPHINXOPTS ?=
SPHINXBUILD ?= sphinx-build
SOURCEDIR = source
BUILDDIR = build
# Put it first so that "make" without argument is like "make help".
help:
@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
.PHONY: help Makefile
# Catch-all target: route all unknown targets to Sphinx using the new
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
%: Makefile
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

35
docs/make.bat Normal file
View file

@ -0,0 +1,35 @@
@ECHO OFF
pushd %~dp0
REM Command file for Sphinx documentation
if "%SPHINXBUILD%" == "" (
set SPHINXBUILD=sphinx-build
)
set SOURCEDIR=source
set BUILDDIR=build
if "%1" == "" goto help
%SPHINXBUILD% >NUL 2>NUL
if errorlevel 9009 (
echo.
echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
echo.installed, then set the SPHINXBUILD environment variable to point
echo.to the full path of the 'sphinx-build' executable. Alternatively you
echo.may add the Sphinx directory to PATH.
echo.
echo.If you don't have Sphinx installed, grab it from
echo.http://sphinx-doc.org/
exit /b 1
)
%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
goto end
:help
%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
:end
popd

52
docs/source/conf.py Normal file
View file

@ -0,0 +1,52 @@
# Configuration file for the Sphinx documentation builder.
#
# This file only contains a selection of the most common options. For a full
# list see the documentation:
# https://www.sphinx-doc.org/en/master/usage/configuration.html
# -- Path setup --------------------------------------------------------------
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
# import os
# import sys
# sys.path.insert(0, os.path.abspath('.'))
# -- Project information -----------------------------------------------------
project = 'ucloud'
copyright = '2019, Ahmed Bilal Khalid'
author = 'Ahmed Bilal Khalid'
# -- General configuration ---------------------------------------------------
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
]
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path.
exclude_patterns = []
# -- Options for HTML output -------------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
html_theme = 'alabaster'
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

21
docs/source/index.rst Normal file
View file

@ -0,0 +1,21 @@
.. ucloud documentation master file, created by
sphinx-quickstart on Mon Nov 11 19:08:16 2019.
You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive.
Welcome to ucloud's documentation!
==================================
.. toctree::
:maxdepth: 2
:caption: Contents:
introduction/introduction
introduction/installation
Indices and tables
==================
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`

View file

@ -0,0 +1,200 @@
Installation
============
.. note::
The below installation instructions are for single node and without ceph ucloud installation.
The instructions assumes the following things
* User is **root**.
* Base Directory is `/root/`.
Alpine
------
Python Wheel (Binary) Packages does not support Alpine Linux as it is using musl libc instead of glibc.
Therefore, expect longer installation times than other linux distributions.
Enable Edge Repos, Update and Upgrade
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. warning::
The below commands would overwrite your repositories sources and upgrade all packages and their dependencies to match those available in edge repos. **So, be warned**
.. code-block:: sh
:linenos:
cat > /etc/apk/repositories << EOF
http://dl-cdn.alpinelinux.org/alpine/edge/main
http://dl-cdn.alpinelinux.org/alpine/edge/community
http://dl-cdn.alpinelinux.org/alpine/edge/testing
EOF
apk update
apk upgrade
Install Dependencies
~~~~~~~~~~~~~~~~~~~~
.. code-block:: sh
:linenos:
apk add git python3 alpine-sdk python3-dev etcd etcd-ctl openntpd \
libffi-dev openssl-dev make py3-protobuf py3-tempita chrony \
qemu qemu-system-x86_64
pip3 install pipenv
Syncronize Date/Time
~~~~~~~~~~~~~~~~~~~~
.. code-block:: sh
:linenos:
service chronyd start
rc-update add chronyd
Start etcd and enable it
~~~~~~~~~~~~~~~~~~~~~~~~
.. code-block:: sh
:linenos:
start-stop-daemon -b etcd
rc-update add etcd
Install uotp
~~~~~~~~~~~~
.. code-block:: sh
:linenos:
git clone https://code.ungleich.ch/ungleich-public/uotp.git
cd uotp
mv .env.sample .env
pipenv --three --site-packages
pipenv install
pipenv run python app.py
Run :code:`ETCDCTL_API=3 etcdctl get /uotp/admin --print-value-only` to get admin seed. A sample output
.. code-block:: json
{
"seed": "FYTVQ72A2CJJ4TB4",
"realm": ["ungleich-admin"]
}
Now, run :code:`pipenv run python scripts/create-auth.py FYTVQ72A2CJJ4TB4` (Replace **FYTVQ72A2CJJ4TB4** with your admin seed obtained in previous step).
A sample output is as below. It shows seed of auth.
.. code-block:: json
{
"message": "Account Created\nname: auth, realm: ['ungleich-auth'], seed: XZLTUMX26TRAZOXC"
}
.. note::
Please note both **admin** and **auth** seeds as we would need them in setting up ucloud
Install and configure ucloud
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. code-block:: sh
:linenos:
git clone https://code.ungleich.ch/ucloud/ucloud.git
cd ucloud
pipenv --three --site-packages
pipenv install
You just need to update **AUTH_SEED** in the below code to match your auth's seed.
.. code-block:: sh
:linenos:
cat > .env << EOF
AUTH_NAME=auth
AUTH_SEED=XZLTUMX26TRAZOXC
AUTH_REALM=ungleich-auth
REALM_ALLOWED = ["ungleich-admin", "ungleich-user"]
OTP_SERVER="http://127.0.0.1:8000/"
ETCD_URL=localhost
WITHOUT_CEPH=True
BASE_DIR=/var/www
IMAGE_DIR=/var/image
VM_DIR=/var/vm
VM_PREFIX=/v1/vm/
HOST_PREFIX=/v1/host/
REQUEST_PREFIX=/v1/request/
FILE_PREFIX=/v1/file/
IMAGE_PREFIX=/v1/image/
IMAGE_STORE_PREFIX=/v1/image_store/
USER_PREFIX=/v1/user/
NETWORK_PREFIX=/v1/network/
ssh_username=meow
ssh_pkey="~/.ssh/id_rsa"
EOF
Install and configure ucloud-cli
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. code-block:: sh
:linenos:
git clone https://code.ungleich.ch/ucloud/ucloud-cli.git
cd ucloud-cli
pipenv --three --site-packages
pipenv install
cat > .env << EOF
UCLOUD_API_SERVER=http://localhost:5000
EOF
Environment Variables and aliases
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
To ease usage of ucloud and its various componenets put the following in your shell
profile e.g *~/.profile*
.. code-block:: sh
export OTP_NAME=admin
export OTP_REALM=ungleich-admin
export OTP_SEED=FYTVQ72A2CJJ4TB4
alias ucloud='cd /root/ucloud/ && pipenv run python ucloud.py'
alias ucloud-cli='cd /root/ucloud-cli/ && pipenv run python ucloud.py'
alias uotp='cd /root/uotp/ && pipenv run python app.py'
and run :code:`source ~/.profile`
Running ucloud
~~~~~~~~~~~~~~
.. code-block:: sh
ucloud api
We need to create a host by executing the following command
.. code-block:: sh

View file

@ -0,0 +1,23 @@
Introduction
============
**Open** + **Simple** + **Easy to hack** + **IPv6 First**
Tech Stack
----------
* Python 3 as main language.
* Flask for APIs.
* JSON for specifications.
* QEMU (+ kvm acceleration) as hypervisor.
* etcd for key/value storage (specifically all metadata e.g Virtual Machine Specifications, Networks Specifications, Images Specifications etc.).
* Ceph for image storage.
Components
----------
* API
* Scheduler
* Host
* File Scanner
* Image Scanner
* Metadata Server
* VM Init Scripts (dubbed as ucloud-init)

View file

@ -22,6 +22,7 @@ etcd_wrapper_kwargs = {"host": config("ETCD_URL")}
etcd_client = Etcd3Wrapper(*etcd_wrapper_args, **etcd_wrapper_kwargs) etcd_client = Etcd3Wrapper(*etcd_wrapper_args, **etcd_wrapper_kwargs)
HOST_PREFIX = config("HOST_PREFIX") HOST_PREFIX = config("HOST_PREFIX")
NETWORK_PREFIX = config("NETWORK_PREFIX")
VM_PREFIX = config("VM_PREFIX") VM_PREFIX = config("VM_PREFIX")
REQUEST_PREFIX = config("REQUEST_PREFIX") REQUEST_PREFIX = config("REQUEST_PREFIX")
VM_DIR = config("VM_DIR") VM_DIR = config("VM_DIR")

View file

@ -1,5 +1,4 @@
import argparse import argparse
# import threading
import time import time
import os import os
import sys import sys

View file

@ -6,22 +6,23 @@
import errno import errno
import os import os
import subprocess import subprocess as sp
import tempfile import tempfile
import time import time
import random
from functools import wraps from functools import wraps
from os.path import join from os.path import join
from typing import Union from typing import Union
from decouple import config
import bitmath import bitmath
import sshtunnel import sshtunnel
from decouple import config
import qmp import qmp
from config import (WITHOUT_CEPH, VM_PREFIX, VM_DIR, IMAGE_DIR, from config import (WITHOUT_CEPH, VM_PREFIX, VM_DIR, IMAGE_DIR,
etcd_client, logging, request_pool, NETWORK_PREFIX, etcd_client, logging,
running_vms, vm_pool) request_pool, running_vms, vm_pool)
from ucloud_common.helpers import get_ipv4_address from ucloud_common.helpers import get_ipv4_address
from ucloud_common.request import RequestEntry, RequestType from ucloud_common.request import RequestEntry, RequestType
from ucloud_common.vm import VMEntry, VMStatus from ucloud_common.vm import VMEntry, VMStatus
@ -37,13 +38,62 @@ class VM:
return "VM({})".format(self.key) return "VM({})".format(self.key)
def create_dev(script, _id, dev):
assert isinstance(_id, str) and isinstance(dev, str), "_id and dev both must be string"
try:
output = sp.check_output([script, _id, dev], stderr=sp.PIPE)
except Exception as e:
print(e.stderr)
return None
else:
return output.decode("utf-8").strip()
def create_vxlan_br_tap(_id, _dev):
network_script_base = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'network')
vxlan = create_dev(script=os.path.join(network_script_base, 'create-vxlan.sh'),
_id=_id, dev=_dev)
if vxlan:
bridge = create_dev(script=os.path.join(network_script_base, 'create-bridge.sh'),
_id=_id, dev=vxlan)
if bridge:
tap = create_dev(script=os.path.join(network_script_base, 'create-tap.sh'),
_id=str(random.randint(1, 100000)), dev=bridge)
if tap:
return tap
def random_bytes(num=6):
return [random.randrange(256) for _ in range(num)]
def generate_mac(uaa=False, multicast=False, oui=None, separator=':', byte_fmt='%02x'):
mac = random_bytes()
if oui:
if type(oui) == str:
oui = [int(chunk) for chunk in oui.split(separator)]
mac = oui + random_bytes(num=6-len(oui))
else:
if multicast:
mac[0] |= 1 # set bit 0
else:
mac[0] &= ~1 # clear bit 0
if uaa:
mac[0] &= ~(1 << 1) # clear bit 1
else:
mac[0] |= 1 << 1 # set bit 1
return separator.join(byte_fmt % b for b in mac)
def get_start_command_args( def get_start_command_args(
vm_entry, vnc_sock_filename: str, migration=False, migration_port=4444 vm_entry, vnc_sock_filename: str, migration=False, migration_port=4444,
): ):
threads_per_core = 1 threads_per_core = 1
vm_memory = int(bitmath.parse_string(vm_entry.specs["ram"]).to_MB()) vm_memory = int(bitmath.parse_string(vm_entry.specs["ram"]).to_MB())
vm_cpus = int(vm_entry.specs["cpu"]) vm_cpus = int(vm_entry.specs["cpu"])
vm_uuid = vm_entry.uuid vm_uuid = vm_entry.uuid
vm_networks = vm_entry.network
if WITHOUT_CEPH: if WITHOUT_CEPH:
command = "-drive file={},format=raw,if=virtio,cache=none".format( command = "-drive file={},format=raw,if=virtio,cache=none".format(
@ -63,7 +113,20 @@ def get_start_command_args(
if migration: if migration:
command += " -incoming tcp:0:{}".format(migration_port) command += " -incoming tcp:0:{}".format(migration_port)
command += " -nic tap,model=virtio,mac={}".format(vm_entry.mac) tap = None
for network_name in vm_networks:
_key = os.path.join(NETWORK_PREFIX, vm_entry.owner, network_name)
network = etcd_client.get(_key, value_in_json=True)
network_type = network.value["type"]
network_id = str(network.value["id"])
if network_type == "vxlan":
tap = create_vxlan_br_tap(network_id, "eno1")
command += " -netdev tap,id=vmnet{net_id},ifname={tap},script=no,downscript=no"\
" -device virtio-net-pci,netdev=vmnet{net_id},mac={mac}"\
.format(tap=tap, net_id=network_id, mac=generate_mac())
return command.split(" ") return command.split(" ")
@ -144,8 +207,8 @@ def create(vm_entry: VMEntry):
] ]
try: try:
subprocess.check_output(_command_to_create) sp.check_output(_command_to_create)
except subprocess.CalledProcessError as e: except sp.CalledProcessError as e:
if e.returncode == errno.EEXIST: if e.returncode == errno.EEXIST:
logging.debug("Image for vm %s exists", vm_entry.uuid) logging.debug("Image for vm %s exists", vm_entry.uuid)
# File Already exists. No Problem Continue # File Already exists. No Problem Continue
@ -158,7 +221,7 @@ def create(vm_entry: VMEntry):
vm_entry.status = "ERROR" vm_entry.status = "ERROR"
else: else:
try: try:
subprocess.check_output(_command_to_extend) sp.check_output(_command_to_extend)
except Exception as e: except Exception as e:
logging.exception(e) logging.exception(e)
else: else:
@ -199,7 +262,7 @@ def delete(vm_entry):
vm_deletion_command = ["rbd", "rm", path_without_protocol] vm_deletion_command = ["rbd", "rm", path_without_protocol]
try: try:
subprocess.check_output(vm_deletion_command) sp.check_output(vm_deletion_command)
except Exception as e: except Exception as e:
logging.exception(e) logging.exception(e)
else: else:

23
network/create-bridge.sh Executable file
View file

@ -0,0 +1,23 @@
#!/bin/sh
if [ $# -ne 2 ]; then
echo "$0 brid dev"
echo "f.g. $0 100 vxlan100"
echo "Missing arguments" >&2
exit 1
fi
brid=$1; shift
dev=$1; shift
bridge=br${brid}
sysctl net.ipv6.conf.all.forwarding=1 > /dev/null
if ! ip link show $bridge > /dev/null 2> /dev/null; then
ip link add name $bridge type bridge
ip link set $bridge up
ip link set $dev master $bridge
ip address add fd00:/64 dev $bridge
fi
echo $bridge

22
network/create-tap.sh Executable file
View file

@ -0,0 +1,22 @@
#!/bin/sh
if [ $# -ne 2 ]; then
echo "$0 tapid dev"
echo "f.g. $0 100 br100"
echo "Missing arguments" >&2
exit 1
fi
tapid=$1; shift
bridge=$1; shift
vxlan=vxlan${tapid}
tap=tap${tapid}
if ! ip link show $tap > /dev/null 2> /dev/null; then
ip tuntap add $tap mode tap user `whoami`
ip link set $tap up
sleep 0.5s
ip link set $tap master $bridge
fi
echo $tap

View file

@ -1,19 +0,0 @@
#!/bin/sh
if [ $# -ne 2 ]; then
echo "$0 vxlanid dev"
echo "f.i. $0 100 eth0"
exit 1
fi
netid=$1; shift
dev=$1; shift
ip -6 link add vxlan${netid} type vxlan \
id ${netid} \
dstport 4789 \
group ff05::${netid} \
dev ${dev} \
ttl 5
ip link set ${dev} up

26
network/create-vxlan.sh Executable file
View file

@ -0,0 +1,26 @@
#!/bin/sh
if [ $# -ne 2 ]; then
echo "$0 vxlanid dev"
echo "f.i. $0 100 eno1"
echo "Missing arguments" >&2
exit 1
fi
netid=$1; shift
dev=$1; shift
vxlan=vxlan${netid}
if ! ip link show $vxlan > /dev/null 2> /dev/null; then
ip -6 link add $vxlan type vxlan \
id $netid \
dstport 4789 \
group ff05::$netid \
dev $dev \
ttl 5
ip link set $dev up
ip link set $vxlan up
fi
echo $vxlan

View file

@ -41,7 +41,6 @@ def remaining_resources(host_specs, vms_specs):
_remaining[component] = map(lambda x: int(bitmath.parse_string(x).to_MB()), _remaining[component]) _remaining[component] = map(lambda x: int(bitmath.parse_string(x).to_MB()), _remaining[component])
_remaining[component] = reduce(lambda x, y: x + y, _remaining[component], 0) _remaining[component] = reduce(lambda x, y: x + y, _remaining[component], 0)
print(_vms_specs, _remaining)
_remaining.subtract(_vms_specs) _remaining.subtract(_vms_specs)
return _remaining return _remaining
@ -65,15 +64,11 @@ def get_suitable_host(vm_specs, hosts=None):
running_vms_specs = [vm.specs for vm in vms] running_vms_specs = [vm.specs for vm in vms]
# Accumulate all of their combined specs # Accumulate all of their combined specs
running_vms_accumulated_specs = accumulated_specs( running_vms_accumulated_specs = accumulated_specs(running_vms_specs)
running_vms_specs
)
# Find out remaining resources after # Find out remaining resources after
# host_specs - already running vm_specs # host_specs - already running vm_specs
remaining = remaining_resources( remaining = remaining_resources(host.specs, running_vms_accumulated_specs)
host.specs, running_vms_accumulated_specs
)
# Find out remaining - new_vm_specs # Find out remaining - new_vm_specs
remaining = remaining_resources(remaining, vm_specs) remaining = remaining_resources(remaining, vm_specs)

View file

@ -54,6 +54,9 @@ def main():
elif request_entry.type == RequestType.ScheduleVM: elif request_entry.type == RequestType.ScheduleVM:
vm_entry = vm_pool.get(request_entry.uuid) vm_entry = vm_pool.get(request_entry.uuid)
if vm_entry is None:
logging.info("Trying to act on {} but it is deleted".format(request_entry.uuid))
continue
client.client.delete(request_entry.key) # consume Request client.client.delete(request_entry.key) # consume Request
# If the Request is about a VM which is labelled as "migration" # If the Request is about a VM which is labelled as "migration"