From da5a600ccb3434cae1b1c86e182bc541f83bfba5 Mon Sep 17 00:00:00 2001 From: meow Date: Mon, 11 Nov 2019 23:42:57 +0500 Subject: [PATCH] single node,w/o ceph networking implemented --- .gitignore | 1 + Pipfile | 2 + Pipfile.lock | 331 +++++++++++++++------- api/common_fields.py | 8 +- api/config.py | 1 + api/helper.py | 23 ++ api/main.py | 54 +++- api/schemas.py | 51 +++- docs/Makefile | 20 ++ docs/make.bat | 35 +++ docs/source/conf.py | 52 ++++ docs/source/index.rst | 21 ++ docs/source/introduction/installation.rst | 200 +++++++++++++ docs/source/introduction/introduction.rst | 23 ++ host/config.py | 1 + host/main.py | 1 - host/virtualmachine.py | 83 +++++- network/create-bridge.sh | 23 ++ network/create-tap.sh | 22 ++ network/create-vxlan-on-dev.sh | 19 -- network/create-vxlan.sh | 26 ++ scheduler/helper.py | 13 +- scheduler/main.py | 3 + 23 files changed, 866 insertions(+), 147 deletions(-) create mode 100644 docs/Makefile create mode 100644 docs/make.bat create mode 100644 docs/source/conf.py create mode 100644 docs/source/index.rst create mode 100644 docs/source/introduction/installation.rst create mode 100644 docs/source/introduction/introduction.rst create mode 100755 network/create-bridge.sh create mode 100755 network/create-tap.sh delete mode 100644 network/create-vxlan-on-dev.sh create mode 100755 network/create-vxlan.sh diff --git a/.gitignore b/.gitignore index bfdbce1..690d98e 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ __pycache__ +docs/build */log.txt \ No newline at end of file diff --git a/Pipfile b/Pipfile index 5db945c..5aba57b 100644 --- a/Pipfile +++ b/Pipfile @@ -17,6 +17,8 @@ etcd3-wrapper = {editable = true,git = "git+https://code.ungleich.ch/ungleich-pu python-etcd3 = {editable = true,git = "git+https://github.com/kragniz/python-etcd3.git"} pyotp = "*" sshtunnel = "*" +helper = "*" +sphinx = "*" [requires] python_version = "3.5" diff --git a/Pipfile.lock b/Pipfile.lock index 62c17ca..6167f76 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "b7a8409bec451e017440f063d8436fe66b18affcde7ad5497b433191ae465a52" + "sha256": "45db72f1a666be82e7dc044ced7e7ad7a5b5a6efbb8b8103e6ad04c93a7d017a" }, "pipfile-spec": 6, "requires": { @@ -16,6 +16,13 @@ ] }, "default": { + "alabaster": { + "hashes": [ + "sha256:446438bdcca0e05bd45ea2de1668c1d9b032e1a9154c2c259092d77031ddd359", + "sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02" + ], + "version": "==0.7.12" + }, "aniso8601": { "hashes": [ "sha256:529dcb1f5f26ee0df6c0a1ee84b7b27197c3c50fc3a6321d66c544689237d072", @@ -23,6 +30,13 @@ ], "version": "==8.0.0" }, + "babel": { + "hashes": [ + "sha256:af92e6106cb7c55286b25b38ad7695f8b4efb36a90ba483d7f7a6628c46158ab", + "sha256:e86135ae101e31e2c8ec20a4e0c5220f4eed12487d5cf3f78be7e98d3a57fc28" + ], + "version": "==2.7.0" + }, "bcrypt": { "hashes": [ "sha256:0258f143f3de96b7c14f762c770f5fc56ccd72f8a1857a451c1cd9a655d9ac89", @@ -62,40 +76,41 @@ }, "cffi": { "hashes": [ - "sha256:00d890313797d9fe4420506613384b43099ad7d2b905c0752dbcc3a6f14d80fa", - "sha256:0cf9e550ac6c5e57b713437e2f4ac2d7fd0cd10336525a27224f5fc1ec2ee59a", - "sha256:0ea23c9c0cdd6778146a50d867d6405693ac3b80a68829966c98dd5e1bbae400", - "sha256:193697c2918ecdb3865acf6557cddf5076bb39f1f654975e087b67efdff83365", - "sha256:1ae14b542bf3b35e5229439c35653d2ef7d8316c1fffb980f9b7647e544baa98", - "sha256:1e389e069450609c6ffa37f21f40cce36f9be7643bbe5051ab1de99d5a779526", - "sha256:263242b6ace7f9cd4ea401428d2d45066b49a700852334fd55311bde36dcda14", - "sha256:33142ae9807665fa6511cfa9857132b2c3ee6ddffb012b3f0933fc11e1e830d5", - "sha256:364f8404034ae1b232335d8c7f7b57deac566f148f7222cef78cf8ae28ef764e", - "sha256:47368f69fe6529f8f49a5d146ddee713fc9057e31d61e8b6dc86a6a5e38cecc1", - "sha256:4895640844f17bec32943995dc8c96989226974dfeb9dd121cc45d36e0d0c434", - "sha256:558b3afef987cf4b17abd849e7bedf64ee12b28175d564d05b628a0f9355599b", - "sha256:5ba86e1d80d458b338bda676fd9f9d68cb4e7a03819632969cf6d46b01a26730", - "sha256:63424daa6955e6b4c70dc2755897f5be1d719eabe71b2625948b222775ed5c43", - "sha256:6381a7d8b1ebd0bc27c3bc85bc1bfadbb6e6f756b4d4db0aa1425c3719ba26b4", - "sha256:6381ab708158c4e1639da1f2a7679a9bbe3e5a776fc6d1fd808076f0e3145331", - "sha256:6fd58366747debfa5e6163ada468a90788411f10c92597d3b0a912d07e580c36", - "sha256:728ec653964655d65408949b07f9b2219df78badd601d6c49e28d604efe40599", - "sha256:7cfcfda59ef1f95b9f729c56fe8a4041899f96b72685d36ef16a3440a0f85da8", - "sha256:819f8d5197c2684524637f940445c06e003c4a541f9983fd30d6deaa2a5487d8", - "sha256:825ecffd9574557590e3225560a8a9d751f6ffe4a49e3c40918c9969b93395fa", - "sha256:8a2bcae2258d00fcfc96a9bde4a6177bc4274fe033f79311c5dd3d3148c26518", - "sha256:9009e917d8f5ef780c2626e29b6bc126f4cb2a4d43ca67aa2b40f2a5d6385e78", - "sha256:9c77564a51d4d914ed5af096cd9843d90c45b784b511723bd46a8a9d09cf16fc", - "sha256:a19089fa74ed19c4fe96502a291cfdb89223a9705b1d73b3005df4256976142e", - "sha256:a40ed527bffa2b7ebe07acc5a3f782da072e262ca994b4f2085100b5a444bbb2", - "sha256:b8f09f21544b9899defb09afbdaeb200e6a87a2b8e604892940044cf94444644", - "sha256:bb75ba21d5716abc41af16eac1145ab2e471deedde1f22c6f99bd9f995504df0", - "sha256:e22a00c0c81ffcecaf07c2bfb3672fa372c50e2bd1024ffee0da191c1b27fc71", - "sha256:e55b5a746fb77f10c83e8af081979351722f6ea48facea79d470b3731c7b2891", - "sha256:ec2fa3ee81707a5232bf2dfbd6623fdb278e070d596effc7e2d788f2ada71a05", - "sha256:fd82eb4694be712fcae03c717ca2e0fc720657ac226b80bbb597e971fc6928c2" + "sha256:0b49274afc941c626b605fb59b59c3485c17dc776dc3cc7cc14aca74cc19cc42", + "sha256:0e3ea92942cb1168e38c05c1d56b0527ce31f1a370f6117f1d490b8dcd6b3a04", + "sha256:135f69aecbf4517d5b3d6429207b2dff49c876be724ac0c8bf8e1ea99df3d7e5", + "sha256:19db0cdd6e516f13329cba4903368bff9bb5a9331d3410b1b448daaadc495e54", + "sha256:2781e9ad0e9d47173c0093321bb5435a9dfae0ed6a762aabafa13108f5f7b2ba", + "sha256:291f7c42e21d72144bb1c1b2e825ec60f46d0a7468f5346841860454c7aa8f57", + "sha256:2c5e309ec482556397cb21ede0350c5e82f0eb2621de04b2633588d118da4396", + "sha256:2e9c80a8c3344a92cb04661115898a9129c074f7ab82011ef4b612f645939f12", + "sha256:32a262e2b90ffcfdd97c7a5e24a6012a43c61f1f5a57789ad80af1d26c6acd97", + "sha256:3c9fff570f13480b201e9ab69453108f6d98244a7f495e91b6c654a47486ba43", + "sha256:415bdc7ca8c1c634a6d7163d43fb0ea885a07e9618a64bda407e04b04333b7db", + "sha256:42194f54c11abc8583417a7cf4eaff544ce0de8187abaf5d29029c91b1725ad3", + "sha256:4424e42199e86b21fc4db83bd76909a6fc2a2aefb352cb5414833c030f6ed71b", + "sha256:4a43c91840bda5f55249413037b7a9b79c90b1184ed504883b72c4df70778579", + "sha256:599a1e8ff057ac530c9ad1778293c665cb81a791421f46922d80a86473c13346", + "sha256:5c4fae4e9cdd18c82ba3a134be256e98dc0596af1e7285a3d2602c97dcfa5159", + "sha256:5ecfa867dea6fabe2a58f03ac9186ea64da1386af2159196da51c4904e11d652", + "sha256:62f2578358d3a92e4ab2d830cd1c2049c9c0d0e6d3c58322993cc341bdeac22e", + "sha256:6471a82d5abea994e38d2c2abc77164b4f7fbaaf80261cb98394d5793f11b12a", + "sha256:6d4f18483d040e18546108eb13b1dfa1000a089bcf8529e30346116ea6240506", + "sha256:71a608532ab3bd26223c8d841dde43f3516aa5d2bf37b50ac410bb5e99053e8f", + "sha256:74a1d8c85fb6ff0b30fbfa8ad0ac23cd601a138f7509dc617ebc65ef305bb98d", + "sha256:7b93a885bb13073afb0aa73ad82059a4c41f4b7d8eb8368980448b52d4c7dc2c", + "sha256:7d4751da932caaec419d514eaa4215eaf14b612cff66398dd51129ac22680b20", + "sha256:7f627141a26b551bdebbc4855c1157feeef18241b4b8366ed22a5c7d672ef858", + "sha256:8169cf44dd8f9071b2b9248c35fc35e8677451c52f795daa2bb4643f32a540bc", + "sha256:aa00d66c0fab27373ae44ae26a66a9e43ff2a678bf63a9c7c1a9a4d61172827a", + "sha256:ccb032fda0873254380aa2bfad2582aedc2959186cce61e3a17abc1a55ff89c3", + "sha256:d754f39e0d1603b5b24a7f8484b22d2904fa551fe865fd0d4c3332f078d20d4e", + "sha256:d75c461e20e29afc0aee7172a0950157c704ff0dd51613506bd7d82b718e7410", + "sha256:dcd65317dd15bc0451f3e01c80da2216a31916bdcffd6221ca1202d96584aa25", + "sha256:e570d3ab32e2c2861c4ebe6ffcad6a8abf9347432a37608fe1fbd157b3f0036b", + "sha256:fd43a88e045cf992ed09fa724b5315b790525f2676883a6ea64e3263bae6549d" ], - "version": "==1.13.1" + "version": "==1.13.2" }, "chardet": { "hashes": [ @@ -137,6 +152,14 @@ ], "version": "==2.8" }, + "docutils": { + "hashes": [ + "sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0", + "sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827", + "sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99" + ], + "version": "==0.15.2" + }, "etcd3-wrapper": { "editable": true, "git": "https://code.ungleich.ch/ungleich-public/etcd3_wrapper.git", @@ -160,56 +183,64 @@ }, "grpcio": { "hashes": [ - "sha256:01cb705eafba1108e2a947ba0457da4f6a1e8142c729fc61702b5fdd11009eb1", - "sha256:0b5a79e29f167d3cd06faad6b15babbc2661066daaacf79373c3a8e67ca1fca1", - "sha256:1097a61a0e97b3580642e6e1460a3a1f1ba1815e2a70d6057173bcc495417076", - "sha256:13970e665a4ec4cec7d067d7d3504a0398c657d91d26c581144ad9044e429c9a", - "sha256:1557817cea6e0b87fad2a3e20da385170efb03a313db164e8078955add2dfa1b", - "sha256:1b0fb036a2f9dd93d9a35c57c26420eeb4b571fcb14b51cddf5b1e73ea5d882b", - "sha256:24d9e58d08e8cd545d8a3247a18654aff0e5e60414701696a8098fbb0d792b75", - "sha256:2c38b586163d2b91567fe5e6d9e7798f792012365adc838a64b66b22dce3f4d4", - "sha256:2df3ab4348507de60e1cbf75196403df1b9b4c4d4dc5bd11ac4eb63c46f691c7", - "sha256:32f70f7c90454ea568b868af2e96616743718d9233d23f62407e98caed81dfbf", - "sha256:3af2a49d576820045c9c880ff29a5a96d020fe31b35d248519bfc6ccb8be4eac", - "sha256:4ff7d63800a63db031ebac6a6f581ae84877c959401c24c28f2cc51fd36c47ad", - "sha256:502aaa8be56f0ae69cda66bc27e1fb5531ceaa27ca515ec3c34f6178b1297180", - "sha256:55358ce3ec283222e435f7dbc6603521438458f3c65f7c1cb33b8dabf56d70d8", - "sha256:5583b01c67f85fa64a2c3fb085e5517c88b9c1500a2cce12d473cd99d0ed2e49", - "sha256:58d9a5557d3eb7b734a3cea8b16c891099a522b3953a45a30bd4c034f75fc913", - "sha256:5911f042c4ab177757eec5bcb4e2e9a2e823d888835d24577321bf55f02938fa", - "sha256:5e16ea922f4e5017c04fd94e2639b1006e03097e9dd0cbb7a1c852af3ea8bf2e", - "sha256:656e19d3f1b9050ee01b457f92838a9679d7cf84c995f708780f44484048705e", - "sha256:6a1435449a82008c451c7e1a82a834387b9108f9a8d27910f86e7c482f5568e9", - "sha256:6ff02ca6cbed0ddb76e93ba0f8beb6a8c77d83a84eb7cafe2ae3399a8b9d69ea", - "sha256:76de68f60102f333bf4817f38e81ecbee68b850f5a5da9f355235e948ac40981", - "sha256:7c6d7ddd50fc6548ea1dfe09c62509c4f95b8b40082287747be05aa8feb15ee2", - "sha256:836b9d29507de729129e363276fe7c7d6a34c7961e0f155787025552b15d22c0", - "sha256:869242b2baf8a888a4fe0548f86abc47cb4b48bdfd76ae62d6456e939c202e65", - "sha256:8954b24bd08641d906ee50b2d638efc76df893fbd0913149b80484fd0eac40c9", - "sha256:8cdea65d1abb2e698420db8daf20c8d272fbd9d96a51b26a713c1c76f237d181", - "sha256:90161840b4fe9636f91ed0d3ea1e7e615e488cbea4e77594c889e5f3d7a776db", - "sha256:90fb6316b4d7d36700c40db4335902b78dcae13b5466673c21fd3b08a3c1b0c6", - "sha256:91b34f58db2611c9a93ecf751028f97fba1f06e65f49b38f272f6aa5d2977331", - "sha256:9474944a96a33eb8734fa8dc5805403d57973a3526204a5e1c1780d02e0572b6", - "sha256:9a36275db2a4774ac16c6822e7af816ee048071d5030b4c035fd53942b361935", - "sha256:9cbe26e2976b994c5f7c2d35a63354674d6ca0ce62f5b513f078bf63c1745229", - "sha256:9eaeabb3c0eecd6ddd0c16767fd12d130e2cebb8c2618f959a278b1ff336ddc3", - "sha256:a2bc7e10ebcf4be503ae427f9887e75c0cc24e88ce467a8e6eaca6bd2862406e", - "sha256:a5b42e6292ba51b8e67e09fc256963ba4ca9c04026de004d2fe59cc17e3c3776", - "sha256:bd6ec1233c86c0b9bb5d03ec30dbe3ffbfa53335790320d99a7ae9018c5450f2", - "sha256:bef57530816af54d66b1f4c70a8f851f320cb6f84d4b5a0b422b0e9811ea4e59", - "sha256:c146a63eaadc6589b732780061f3c94cd0574388d372baccbb3c1597a9ebdb7a", - "sha256:c2efd3b130dc639d615b6f58980e1bfd1b177ad821f30827afa5001aa30ddd48", - "sha256:c888b18f7392e6cc79a33a803e7ebd7890ac3318f571fca6b356526f35b53b12", - "sha256:ca30721fda297ae22f16bc37aa7ed244970ddfdcb98247570cdd26daaad4665e", - "sha256:cf5f5340dd682ab034baa52f423a0f91326489c262ac9617fa06309ec05880e9", - "sha256:d0726aa0d9b57c56985db5952e90fb1033a317074f2877db5307cdd6eede1564", - "sha256:df442945b2dd6f8ae0e20b403e0fd4548cd5c2aad69200047cc3251257b78f65", - "sha256:e08e758c31919d167c0867539bd3b2441629ef00aa595e3ea2b635273659f40a", - "sha256:e4864339deeeaefaad34dd3a432ee618a039fca28efb292949c855e00878203c", - "sha256:f4cd049cb94d9f517b1cab5668a3b345968beba093bc79a637e671000b3540ec" + "sha256:0419ae5a45f49c7c40d9ae77ae4de9442431b7822851dfbbe56ee0eacb5e5654", + "sha256:1e8631eeee0fb0b4230aeb135e4890035f6ef9159c2a3555fa184468e325691a", + "sha256:24db2fa5438f3815a4edb7a189035051760ca6aa2b0b70a6a948b28bfc63c76b", + "sha256:2adb1cdb7d33e91069517b41249622710a94a1faece1fed31cd36904e4201cde", + "sha256:2cd51f35692b551aeb1fdeb7a256c7c558f6d78fcddff00640942d42f7aeba5f", + "sha256:3247834d24964589f8c2b121b40cd61319b3c2e8d744a6a82008643ef8a378b1", + "sha256:3433cb848b4209717722b62392e575a77a52a34d67c6730138102abc0a441685", + "sha256:39671b7ff77a962bd745746d9d2292c8ed227c5748f16598d16d8631d17dd7e5", + "sha256:40a0b8b2e6f6dd630f8b267eede2f40a848963d0f3c40b1b1f453a4a870f679e", + "sha256:40f9a74c7aa210b3e76eb1c9d56aa8d08722b73426a77626967019df9bbac287", + "sha256:423f76aa504c84cb94594fb88b8a24027c887f1c488cf58f2173f22f4fbd046c", + "sha256:43bd04cec72281a96eb361e1b0232f0f542b46da50bcfe72ef7e5a1b41d00cb3", + "sha256:43e38762635c09e24885d15e3a8e374b72d105d4178ee2cc9491855a8da9c380", + "sha256:4413b11c2385180d7de03add6c8845dd66692b148d36e27ec8c9ef537b2553a1", + "sha256:4450352a87094fd58daf468b04c65a9fa19ad11a0ac8ac7b7ff17d46f873cbc1", + "sha256:49ffda04a6e44de028b3b786278ac9a70043e7905c3eea29eed88b6524d53a29", + "sha256:4a38c4dde4c9120deef43aaabaa44f19186c98659ce554c29788c4071ab2f0a4", + "sha256:50b1febdfd21e2144b56a9aa226829e93a79c354ef22a4e5b013d9965e1ec0ed", + "sha256:559b1a3a8be7395ded2943ea6c2135d096f8cc7039d6d12127110b6496f251fe", + "sha256:5de86c182667ec68cf84019aa0d8ceccf01d352cdca19bf9e373725204bdbf50", + "sha256:5fc069bb481fe3fad0ba24d3baaf69e22dfa6cc1b63290e6dfeaf4ac1e996fb7", + "sha256:6a19d654da49516296515d6f65de4bbcbd734bc57913b21a610cfc45e6df3ff1", + "sha256:7535b3e52f498270e7877dde1c8944d6b7720e93e2e66b89c82a11447b5818f5", + "sha256:7c4e495bcabc308198b8962e60ca12f53b27eb8f03a21ac1d2d711d6dd9ecfca", + "sha256:8a8fc4a0220367cb8370cedac02272d574079ccc32bffbb34d53aaf9e38b5060", + "sha256:8b008515e067232838daca020d1af628bf6520c8cc338bf383284efe6d8bd083", + "sha256:8d1684258e1385e459418f3429e107eec5fb3d75e1f5a8c52e5946b3f329d6ea", + "sha256:8eb5d54b87fb561dc2e00a5c5226c33ffe8dbc13f2e4033a412bafb7b37b194d", + "sha256:94cdef0c61bd014bb7af495e21a1c3a369dd0399c3cd1965b1502043f5c88d94", + "sha256:9d9f3be69c7a5e84c3549a8c4403fa9ac7672da456863d21e390b2bbf45ccad1", + "sha256:9fb6fb5975a448169756da2d124a1beb38c0924ff6c0306d883b6848a9980f38", + "sha256:a5eaae8700b87144d7dfb475aa4675e500ff707292caba3deff41609ddc5b845", + "sha256:aaeac2d552772b76d24eaff67a5d2325bc5205c74c0d4f9fbe71685d4a971db2", + "sha256:bb611e447559b3b5665e12a7da5160c0de6876097f62bf1d23ba66911564868e", + "sha256:bc0d41f4eb07da8b8d3ea85e50b62f6491ab313834db86ae2345be07536a4e5a", + "sha256:bf51051c129b847d1bb63a9b0826346b5f52fb821b15fe5e0d5ef86f268510f5", + "sha256:c948c034d8997526011960db54f512756fb0b4be1b81140a15b4ef094c6594a4", + "sha256:d435a01334157c3b126b4ee5141401d44bdc8440993b18b05e2f267a6647f92d", + "sha256:d46c1f95672b73288e08cdca181e14e84c6229b5879561b7b8cfd48374e09287", + "sha256:d5d58309b42064228b16b0311ff715d6c6e20230e81b35e8d0c8cfa1bbdecad8", + "sha256:dc6e2e91365a1dd6314d615d80291159c7981928b88a4c65654e3fefac83a836", + "sha256:e0dfb5f7a39029a6cbec23affa923b22a2c02207960fd66f109e01d6f632c1eb", + "sha256:eb4bf58d381b1373bd21d50837a53953d625d1693f1b58fed12743c75d3dd321", + "sha256:ebb211a85248dbc396b29320273c1ffde484b898852432613e8df0164c091006", + "sha256:ec759ece4786ae993a5b7dc3b3dead6e9375d89a6c65dfd6860076d2eb2abe7b", + "sha256:f55108397a8fa164268238c3e69cc134e945d1f693572a2f05a028b8d0d2b837", + "sha256:f6c706866d424ff285b85a02de7bbe5ed0ace227766b2c42cbe12f3d9ea5a8aa", + "sha256:f8370ad332b36fbad117440faf0dd4b910e80b9c49db5648afd337abdde9a1b6" ], - "version": "==1.24.3" + "version": "==1.25.0" + }, + "helper": { + "hashes": [ + "sha256:33d4a58046018fea9f46da5835a768feb9beab3528d4025d063bf354c4a19750", + "sha256:a63d4a9255ad5071043e7e4ab8000a512627f1db958b1941b63c7d75e56ea65c" + ], + "index": "pypi", + "version": "==2.4.2" }, "idna": { "hashes": [ @@ -218,6 +249,13 @@ ], "version": "==2.8" }, + "imagesize": { + "hashes": [ + "sha256:3f349de3eb99145973fefb7dbe38554414e5c30abd0c8e4b970a7c9d09f3a1d8", + "sha256:f3832918bc3c66617f92e35f5d70729187676313caa60c187eb0f28b8fe5e3b5" + ], + "version": "==1.1.0" + }, "itsdangerous": { "hashes": [ "sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19", @@ -265,6 +303,13 @@ ], "version": "==1.1.1" }, + "packaging": { + "hashes": [ + "sha256:28b924174df7a2fa32c1953825ff29c61e2f5e082343165438812f00d3a7fc47", + "sha256:d9551545c6d761f3def1677baf08ab2a3ca17c56879e70fecba2fc4dde4ed108" + ], + "version": "==19.2" + }, "paramiko": { "hashes": [ "sha256:99f0179bdc176281d21961a003ffdb2ec369daac1a1007241f53374e376576cf", @@ -299,6 +344,13 @@ ], "version": "==2.19" }, + "pygments": { + "hashes": [ + "sha256:71e430bc85c88a430f000ac1d9b331d2407f681d6f6aec95e8bcfbc3df5b0127", + "sha256:881c4c157e45f30af185c1ffe8d549d48ac9127433f2c380c24b84572ad66297" + ], + "version": "==2.4.2" + }, "pynacl": { "hashes": [ "sha256:05c26f93964373fc0abe332676cb6735f0ecad27711035b9472751faa8521255", @@ -333,6 +385,13 @@ "index": "pypi", "version": "==2.3.0" }, + "pyparsing": { + "hashes": [ + "sha256:20f995ecd72f2a1f4bf6b072b63b22e2eb457836601e76d6e5dfcd75436acc1f", + "sha256:4ca62001be367f01bd3e92ecbb79070272a9d4964dce6a48a82ff0b8bc7e683a" + ], + "version": "==2.4.5" + }, "python-decouple": { "hashes": [ "sha256:1317df14b43efee4337a4aa02914bf004f010cd56d6c4bd894e6474ec8c4fe2d" @@ -352,6 +411,24 @@ ], "version": "==2019.3" }, + "pyyaml": { + "hashes": [ + "sha256:0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9", + "sha256:01adf0b6c6f61bd11af6e10ca52b7d4057dd0be0343eb9283c878cf3af56aee4", + "sha256:5124373960b0b3f4aa7df1707e63e9f109b5263eca5976c66e08b1c552d4eaf8", + "sha256:5ca4f10adbddae56d824b2c09668e91219bb178a1eee1faa56af6f99f11bf696", + "sha256:7907be34ffa3c5a32b60b95f4d95ea25361c951383a894fec31be7252b2b6f34", + "sha256:7ec9b2a4ed5cad025c2278a1e6a19c011c80a3caaac804fd2d329e9cc2c287c9", + "sha256:87ae4c829bb25b9fe99cf71fbb2140c448f534e24c998cc60f39ae4f94396a73", + "sha256:9de9919becc9cc2ff03637872a440195ac4241c80536632fffeb6a1e25a74299", + "sha256:a5a85b10e450c66b49f98846937e8cfca1db3127a9d5d1e31ca45c3d0bef4c5b", + "sha256:b0997827b4f6a7c286c01c5f60384d218dca4ed7d9efa945c3e1aa623d5709ae", + "sha256:b631ef96d3222e62861443cc89d6563ba3eeb816eeb96b2629345ab795e53681", + "sha256:bf47c0607522fdbca6c9e817a6e81b08491de50f3766a7a0e6a5be7905961b41", + "sha256:f81025eddd0327c7d4cfe9b62cf33190e1e736cc6e97502b3ec425f574b3e7a8" + ], + "version": "==5.1.2" + }, "requests": { "hashes": [ "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4", @@ -362,10 +439,67 @@ }, "six": { "hashes": [ - "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", - "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" + "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd", + "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66" ], - "version": "==1.12.0" + "version": "==1.13.0" + }, + "snowballstemmer": { + "hashes": [ + "sha256:209f257d7533fdb3cb73bdbd24f436239ca3b2fa67d56f6ff88e86be08cc5ef0", + "sha256:df3bac3df4c2c01363f3dd2cfa78cce2840a79b9f1c2d2de9ce8d31683992f52" + ], + "version": "==2.0.0" + }, + "sphinx": { + "hashes": [ + "sha256:31088dfb95359384b1005619827eaee3056243798c62724fd3fa4b84ee4d71bd", + "sha256:52286a0b9d7caa31efee301ec4300dbdab23c3b05da1c9024b4e84896fb73d79" + ], + "index": "pypi", + "version": "==2.2.1" + }, + "sphinxcontrib-applehelp": { + "hashes": [ + "sha256:edaa0ab2b2bc74403149cb0209d6775c96de797dfd5b5e2a71981309efab3897", + "sha256:fb8dee85af95e5c30c91f10e7eb3c8967308518e0f7488a2828ef7bc191d0d5d" + ], + "version": "==1.0.1" + }, + "sphinxcontrib-devhelp": { + "hashes": [ + "sha256:6c64b077937330a9128a4da74586e8c2130262f014689b4b89e2d08ee7294a34", + "sha256:9512ecb00a2b0821a146736b39f7aeb90759834b07e81e8cc23a9c70bacb9981" + ], + "version": "==1.0.1" + }, + "sphinxcontrib-htmlhelp": { + "hashes": [ + "sha256:4670f99f8951bd78cd4ad2ab962f798f5618b17675c35c5ac3b2132a14ea8422", + "sha256:d4fd39a65a625c9df86d7fa8a2d9f3cd8299a3a4b15db63b50aac9e161d8eff7" + ], + "version": "==1.0.2" + }, + "sphinxcontrib-jsmath": { + "hashes": [ + "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178", + "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8" + ], + "version": "==1.0.1" + }, + "sphinxcontrib-qthelp": { + "hashes": [ + "sha256:513049b93031beb1f57d4daea74068a4feb77aa5630f856fcff2e50de14e9a20", + "sha256:79465ce11ae5694ff165becda529a600c754f4bc459778778c7017374d4d406f" + ], + "version": "==1.0.2" + }, + "sphinxcontrib-serializinghtml": { + "hashes": [ + "sha256:c0efb33f8052c04fd7a26c0a07f1678e8512e0faec19f4aa8f2473a8b81d5227", + "sha256:db6615af393650bf1151a6cd39120c29abaf93cc60db8c48eb2dddbfdc3a9768" + ], + "version": "==1.1.3" }, "sshtunnel": { "hashes": [ @@ -376,10 +510,10 @@ }, "tenacity": { "hashes": [ - "sha256:3a916e734559f1baa2cab965ee00061540c41db71c3bf25375b81540a19758fc", - "sha256:e664bd94f088b17f46da33255ae33911ca6a0fe04b156d334b601a4ef66d3c5f" + "sha256:72f397c2bb1887e048726603f3f629ea16f88cb3e61e4ed3c57e98582b8e3571", + "sha256:947e728aedf06e8db665bb7898112e90d17e48cc3f3289784a2b9ccf6e56fabc" ], - "version": "==5.1.5" + "version": "==6.0.0" }, "ucloud-common": { "editable": true, @@ -627,6 +761,13 @@ ], "version": "==1.6.0" }, + "pygments": { + "hashes": [ + "sha256:71e430bc85c88a430f000ac1d9b331d2407f681d6f6aec95e8bcfbc3df5b0127", + "sha256:881c4c157e45f30af185c1ffe8d549d48ac9127433f2c380c24b84572ad66297" + ], + "version": "==2.4.2" + }, "pylint": { "hashes": [ "sha256:5d77031694a5fb97ea95e828c8d10fc770a1df6eb3906067aaed42201a8a6a09", @@ -662,10 +803,10 @@ }, "pyroma": { "hashes": [ - "sha256:54d332f540d4828bc5672b75ccf9e12d4b2f72a42a4f304bcec1c73565aecc26", - "sha256:6b94feb609e1896579302f0836ef2fad3f17e0557e3ddcd0d76206cd3e366d27" + "sha256:351758a81e2a12c970deb73687e239636aad52795cd81429695073d59fff0699", + "sha256:c49c00377219626bf83df42adf018cc231e6162b68cc7aaf2ff1c63803924102" ], - "version": "==2.5" + "version": "==2.6" }, "pyyaml": { "hashes": [ @@ -707,10 +848,10 @@ }, "six": { "hashes": [ - "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", - "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" + "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd", + "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66" ], - "version": "==1.12.0" + "version": "==1.13.0" }, "snowballstemmer": { "hashes": [ diff --git a/api/common_fields.py b/api/common_fields.py index 390fe40..c2152c9 100755 --- a/api/common_fields.py +++ b/api/common_fields.py @@ -4,6 +4,10 @@ from config import etcd_client as client from config import VM_PREFIX +class Optional: + pass + + class Field: def __init__(self, _name, _type, _value=None): self.name = _name @@ -18,7 +22,9 @@ class Field: if self.value == KeyError: self.add_error("'{}' field is a required field".format(self.name)) else: - if not isinstance(self.value, self.type): + if isinstance(self.value, Optional): + pass + elif not isinstance(self.value, self.type): self.add_error("Incorrect Type for '{}' field".format(self.name)) else: self.validation() diff --git a/api/config.py b/api/config.py index 4ae1b66..b9e7b82 100644 --- a/api/config.py +++ b/api/config.py @@ -24,6 +24,7 @@ REQUEST_PREFIX = config("REQUEST_PREFIX") FILE_PREFIX = config("FILE_PREFIX") IMAGE_PREFIX = config("IMAGE_PREFIX") IMAGE_STORE_PREFIX = config("IMAGE_STORE_PREFIX") +NETWORK_PREFIX = config("NETWORK_PREFIX") etcd_client = Etcd3Wrapper(host=config("ETCD_URL")) diff --git a/api/helper.py b/api/helper.py index 06b45b1..705800a 100755 --- a/api/helper.py +++ b/api/helper.py @@ -92,9 +92,11 @@ def resolve_image_name(name, etcd_client): return image_uuid + def random_bytes(num=6): return [random.randrange(256) for _ in range(num)] + def generate_mac(uaa=False, multicast=False, oui=None, separator=':', byte_fmt='%02x'): mac = random_bytes() if oui: @@ -112,6 +114,7 @@ def generate_mac(uaa=False, multicast=False, oui=None, separator=':', byte_fmt=' mac[0] |= 1 << 1 # set bit 1 return separator.join(byte_fmt % b for b in mac) + def get_ip_addr(mac_address, device): """Return IP address of a device provided its mac address / link local address and the device with which it is connected. @@ -140,3 +143,23 @@ def get_ip_addr(mac_address, device): if ip.is_global and mac_address == mac: result.append(ip) return result + + +def increment_etcd_counter(etcd_client, key): + kv = etcd_client.get(key) + + if kv: + counter = int(kv.value) + counter = counter + 1 + else: + counter = 1 + + etcd_client.put(key, str(counter)) + return counter + + +def get_etcd_counter(etcd_client, key): + kv = etcd_client.get(key) + if kv: + return int(kv.value) + return None diff --git a/api/main.py b/api/main.py index 1b25802..5096ebf 100644 --- a/api/main.py +++ b/api/main.py @@ -12,7 +12,7 @@ from flask_restful import Resource, Api from ucloud_common.vm import VMStatus from ucloud_common.request import RequestEntry, RequestType -from helper import generate_mac, get_ip_addr +from helper import generate_mac, get_ip_addr, get_etcd_counter, increment_etcd_counter from config import ( etcd_client, @@ -21,6 +21,7 @@ from config import ( HOST_PREFIX, FILE_PREFIX, IMAGE_PREFIX, + NETWORK_PREFIX, logging, REQUEST_POOL, VM_POOL, @@ -35,7 +36,6 @@ class CreateVM(Resource): @staticmethod def post(): data = request.json - print(data) validator = schemas.CreateVMSchema(data) if validator.is_valid(): vm_uuid = uuid4().hex @@ -57,10 +57,10 @@ class CreateVM(Resource): "image_uuid": validator.image_uuid, "log": [], "vnc_socket": "", - "mac": str(generate_mac()), + "network": data["network"], "metadata": { "ssh-keys": [] - } + }, } etcd_client.put(vm_key, vm_entry, value_in_json=True) @@ -80,9 +80,8 @@ class VmStatus(Resource): if validator.is_valid(): vm = VM_POOL.get(os.path.join(VM_PREFIX, data["uuid"])) vm_value = vm.value.copy() - vm_value["ip"] = list(map(str, get_ip_addr(vm.mac, "br0"))) + # vm_value["ip"] = list(map(str, get_ip_addr(vm.mac, "br0"))) vm.value = vm_value - print(vm.value) return vm.value else: return validator.get_errors(), 400 @@ -217,7 +216,7 @@ class ListUserVM(Resource): "specs": vm.value["specs"], "status": vm.value["status"], "hostname": vm.value["hostname"], - "mac": vm.value["mac"], + # "mac": vm.value["mac"], "vnc_socket": None if vm.value.get("vnc_socket", None) is None else vm.value["vnc_socket"], @@ -357,6 +356,44 @@ class RemoveSSHKey(Resource): else: return validator.get_errors(), 400 + +class CreateNetwork(Resource): + @staticmethod + def post(): + data = request.json + validator = schemas.CreateNetwork(data) + + if validator.is_valid(): + + network_entry = { + "id": increment_etcd_counter(etcd_client, "/v1/counter/vxlan"), + "type": data["type"] + } + network_key = os.path.join(NETWORK_PREFIX, data["name"], data["network_name"]) + etcd_client.put(network_key, network_entry, value_in_json=True) + return {"message": "Network successfully added."} + else: + return validator.get_errors(), 400 + + +class ListUserNetwork(Resource): + @staticmethod + def get(): + data = request.json + validator = schemas.OTPSchema(data) + + if validator.is_valid(): + prefix = os.path.join(NETWORK_PREFIX, data["name"]) + networks = etcd_client.get_prefix(prefix, value_in_json=True) + user_networks = [] + for net in networks: + net.value["name"] = net.key.split("/")[-1] + user_networks.append(net.value) + return {"networks": user_networks}, 200 + else: + return validator.get_errors(), 400 + + api.add_resource(CreateVM, "/vm/create") api.add_resource(VmStatus, "/vm/status") @@ -368,6 +405,7 @@ api.add_resource(ListPublicImages, "/image/list-public") api.add_resource(ListUserVM, "/user/vms") api.add_resource(ListUserFiles, "/user/files") +api.add_resource(ListUserNetwork, "/user/networks") api.add_resource(AddSSHKey, "/user/add-ssh") api.add_resource(RemoveSSHKey, "/user/remove-ssh") @@ -376,5 +414,7 @@ api.add_resource(GetSSHKeys, "/user/get-ssh") api.add_resource(CreateHost, "/host/create") api.add_resource(ListHost, "/host/list") +api.add_resource(CreateNetwork, "/network/create") + if __name__ == "__main__": app.run(host="::", debug=True) diff --git a/api/schemas.py b/api/schemas.py index b0e49b7..8aab841 100755 --- a/api/schemas.py +++ b/api/schemas.py @@ -23,11 +23,11 @@ import helper from ucloud_common.host import HostPool, HostStatus from ucloud_common.vm import VmPool, VMStatus -from common_fields import Field, VmUUIDField +from common_fields import Field, VmUUIDField, Optional from helper import check_otp, resolve_vm_name from config import etcd_client as client from config import (HOST_PREFIX, VM_PREFIX, IMAGE_PREFIX, - FILE_PREFIX, IMAGE_STORE_PREFIX) + FILE_PREFIX, IMAGE_STORE_PREFIX, NETWORK_PREFIX) HOST_POOL = HostPool(client, HOST_PREFIX) VM_POOL = VmPool(client, VM_PREFIX) @@ -85,7 +85,6 @@ class OTPSchema(BaseSchema): super().__init__(data=data, fields=_fields) def validation(self): - print(self.name.value, self.realm.value, self.token.value) if check_otp(self.name.value, self.realm.value, self.token.value) != 200: self.add_error("Wrong Credentials") @@ -206,20 +205,24 @@ class CreateHostSchema(OTPSchema): class CreateVMSchema(OTPSchema): def __init__(self, data): self.parsed_specs = {} + # Fields self.specs = Field("specs", dict, data.get("specs", KeyError)) self.vm_name = Field("vm_name", str, data.get("vm_name", KeyError)) self.image = Field("image", str, data.get("image", KeyError)) + self.network = Field("network", list, data.get("network", KeyError)) # Validation self.image.validation = self.image_validation self.vm_name.validation = self.vm_name_validation self.specs.validation = self.specs_validation + self.network.validation = self.network_validation - fields = [self.vm_name, self.image, self.specs] + fields = [self.vm_name, self.image, self.specs, self.network] super().__init__(data=data, fields=fields) + def image_validation(self): try: image_uuid = helper.resolve_image_name(self.image.value, client) @@ -234,6 +237,18 @@ class CreateVMSchema(OTPSchema): 'VM with same name "{}" already exists'.format(self.vm_name.value) ) + def network_validation(self): + _network = self.network.value + + if _network: + for net in _network: + network = client.get(os.path.join(NETWORK_PREFIX, + self.name.value, + net), value_in_json=True) + if not network: + self.add_error("Network with name {} does not exists"\ + .format(net)) + def specs_validation(self): ALLOWED_BASE = 10 @@ -416,4 +431,30 @@ class GetSSHSchema(OTPSchema): self.key_name = Field("key_name", str, data.get("key_name", None)) fields = [self.key_name] - super().__init__(data=data, fields=fields) \ No newline at end of file + super().__init__(data=data, fields=fields) + + +class CreateNetwork(OTPSchema): + def __init__(self, data): + self.network_name = Field("network_name", str, data.get("network_name", KeyError)) + self.type = Field("type", str, data.get("type", KeyError)) + + self.network_name.validation = self.network_name_validation + self.type.validation = self.network_type_validation + + fields = [self.network_name, self.type] + super().__init__(data, fields=fields) + + def network_name_validation(self): + network = client.get(os.path.join(NETWORK_PREFIX, + self.name.value, + self.network_name.value), + value_in_json=True) + if network: + self.add_error("Network with name {} already exists"\ + .format(self.network_name.value)) + + def network_type_validation(self): + supported_network_types = ["vxlan"] + if self.type.value not in supported_network_types: + self.add_error("Unsupported Network Type. Supported network types are {}".format(supported_network_types)) diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d0c3cbf --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..6247f7e --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000..197cfce --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,52 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- Project information ----------------------------------------------------- + +project = 'ucloud' +copyright = '2019, Ahmed Bilal Khalid' +author = 'Ahmed Bilal Khalid' + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = [] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'alabaster' + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] \ No newline at end of file diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..72d77d3 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,21 @@ +.. ucloud documentation master file, created by + sphinx-quickstart on Mon Nov 11 19:08:16 2019. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to ucloud's documentation! +================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + introduction/introduction + introduction/installation + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/source/introduction/installation.rst b/docs/source/introduction/installation.rst new file mode 100644 index 0000000..71269b6 --- /dev/null +++ b/docs/source/introduction/installation.rst @@ -0,0 +1,200 @@ +Installation +============ + +.. note:: + The below installation instructions are for single node and without ceph ucloud installation. + + The instructions assumes the following things + + * User is **root**. + * Base Directory is `/root/`. + +Alpine +------ +Python Wheel (Binary) Packages does not support Alpine Linux as it is using musl libc instead of glibc. +Therefore, expect longer installation times than other linux distributions. + +Enable Edge Repos, Update and Upgrade +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. warning:: + The below commands would overwrite your repositories sources and upgrade all packages and their dependencies to match those available in edge repos. **So, be warned** +.. code-block:: sh + :linenos: + + cat > /etc/apk/repositories << EOF + http://dl-cdn.alpinelinux.org/alpine/edge/main + http://dl-cdn.alpinelinux.org/alpine/edge/community + http://dl-cdn.alpinelinux.org/alpine/edge/testing + EOF + + apk update + apk upgrade + + +Install Dependencies +~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: sh + :linenos: + + apk add git python3 alpine-sdk python3-dev etcd etcd-ctl openntpd \ + libffi-dev openssl-dev make py3-protobuf py3-tempita chrony \ + qemu qemu-system-x86_64 + + pip3 install pipenv + +Syncronize Date/Time +~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: sh + :linenos: + + service chronyd start + rc-update add chronyd + + +Start etcd and enable it +~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: sh + :linenos: + + start-stop-daemon -b etcd + rc-update add etcd + + +Install uotp +~~~~~~~~~~~~ + +.. code-block:: sh + :linenos: + + git clone https://code.ungleich.ch/ungleich-public/uotp.git + cd uotp + mv .env.sample .env + + pipenv --three --site-packages + pipenv install + pipenv run python app.py + +Run :code:`ETCDCTL_API=3 etcdctl get /uotp/admin --print-value-only` to get admin seed. A sample output + +.. code-block:: json + + { + "seed": "FYTVQ72A2CJJ4TB4", + "realm": ["ungleich-admin"] + } + +Now, run :code:`pipenv run python scripts/create-auth.py FYTVQ72A2CJJ4TB4` (Replace **FYTVQ72A2CJJ4TB4** with your admin seed obtained in previous step). +A sample output is as below. It shows seed of auth. + +.. code-block:: json + + { + "message": "Account Created\nname: auth, realm: ['ungleich-auth'], seed: XZLTUMX26TRAZOXC" + } + +.. note:: + Please note both **admin** and **auth** seeds as we would need them in setting up ucloud + + +Install and configure ucloud +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: sh + :linenos: + + git clone https://code.ungleich.ch/ucloud/ucloud.git + cd ucloud + + pipenv --three --site-packages + pipenv install + + +You just need to update **AUTH_SEED** in the below code to match your auth's seed. + + +.. code-block:: sh + :linenos: + + cat > .env << EOF + AUTH_NAME=auth + AUTH_SEED=XZLTUMX26TRAZOXC + AUTH_REALM=ungleich-auth + + REALM_ALLOWED = ["ungleich-admin", "ungleich-user"] + + OTP_SERVER="http://127.0.0.1:8000/" + + ETCD_URL=localhost + + WITHOUT_CEPH=True + + BASE_DIR=/var/www + IMAGE_DIR=/var/image + VM_DIR=/var/vm + + VM_PREFIX=/v1/vm/ + HOST_PREFIX=/v1/host/ + REQUEST_PREFIX=/v1/request/ + FILE_PREFIX=/v1/file/ + IMAGE_PREFIX=/v1/image/ + IMAGE_STORE_PREFIX=/v1/image_store/ + USER_PREFIX=/v1/user/ + NETWORK_PREFIX=/v1/network/ + + ssh_username=meow + ssh_pkey="~/.ssh/id_rsa" + + EOF + + +Install and configure ucloud-cli +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: sh + :linenos: + + git clone https://code.ungleich.ch/ucloud/ucloud-cli.git + cd ucloud-cli + pipenv --three --site-packages + pipenv install + + cat > .env << EOF + UCLOUD_API_SERVER=http://localhost:5000 + EOF + + +Environment Variables and aliases +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To ease usage of ucloud and its various componenets put the following in your shell +profile e.g *~/.profile* + +.. code-block:: sh + + export OTP_NAME=admin + export OTP_REALM=ungleich-admin + export OTP_SEED=FYTVQ72A2CJJ4TB4 + + alias ucloud='cd /root/ucloud/ && pipenv run python ucloud.py' + alias ucloud-cli='cd /root/ucloud-cli/ && pipenv run python ucloud.py' + alias uotp='cd /root/uotp/ && pipenv run python app.py' + +and run :code:`source ~/.profile` + + +Running ucloud +~~~~~~~~~~~~~~ + +.. code-block:: sh + + ucloud api + +We need to create a host by executing the following command + +.. code-block:: sh + + \ No newline at end of file diff --git a/docs/source/introduction/introduction.rst b/docs/source/introduction/introduction.rst new file mode 100644 index 0000000..022713e --- /dev/null +++ b/docs/source/introduction/introduction.rst @@ -0,0 +1,23 @@ +Introduction +============ + +**Open** + **Simple** + **Easy to hack** + **IPv6 First** + +Tech Stack +---------- +* Python 3 as main language. +* Flask for APIs. +* JSON for specifications. +* QEMU (+ kvm acceleration) as hypervisor. +* etcd for key/value storage (specifically all metadata e.g Virtual Machine Specifications, Networks Specifications, Images Specifications etc.). +* Ceph for image storage. + +Components +---------- +* API +* Scheduler +* Host +* File Scanner +* Image Scanner +* Metadata Server +* VM Init Scripts (dubbed as ucloud-init) \ No newline at end of file diff --git a/host/config.py b/host/config.py index 03cee29..c2dbb06 100755 --- a/host/config.py +++ b/host/config.py @@ -22,6 +22,7 @@ etcd_wrapper_kwargs = {"host": config("ETCD_URL")} etcd_client = Etcd3Wrapper(*etcd_wrapper_args, **etcd_wrapper_kwargs) HOST_PREFIX = config("HOST_PREFIX") +NETWORK_PREFIX = config("NETWORK_PREFIX") VM_PREFIX = config("VM_PREFIX") REQUEST_PREFIX = config("REQUEST_PREFIX") VM_DIR = config("VM_DIR") diff --git a/host/main.py b/host/main.py index b86e88e..8a81e86 100755 --- a/host/main.py +++ b/host/main.py @@ -1,5 +1,4 @@ import argparse -# import threading import time import os import sys diff --git a/host/virtualmachine.py b/host/virtualmachine.py index f99ffd0..9e01094 100755 --- a/host/virtualmachine.py +++ b/host/virtualmachine.py @@ -6,22 +6,23 @@ import errno import os -import subprocess +import subprocess as sp import tempfile import time +import random from functools import wraps from os.path import join from typing import Union +from decouple import config import bitmath import sshtunnel -from decouple import config import qmp from config import (WITHOUT_CEPH, VM_PREFIX, VM_DIR, IMAGE_DIR, - etcd_client, logging, request_pool, - running_vms, vm_pool) + NETWORK_PREFIX, etcd_client, logging, + request_pool, running_vms, vm_pool) from ucloud_common.helpers import get_ipv4_address from ucloud_common.request import RequestEntry, RequestType from ucloud_common.vm import VMEntry, VMStatus @@ -37,13 +38,62 @@ class VM: return "VM({})".format(self.key) +def create_dev(script, _id, dev): + assert isinstance(_id, str) and isinstance(dev, str), "_id and dev both must be string" + try: + output = sp.check_output([script, _id, dev], stderr=sp.PIPE) + except Exception as e: + print(e.stderr) + return None + else: + return output.decode("utf-8").strip() + + +def create_vxlan_br_tap(_id, _dev): + network_script_base = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'network') + vxlan = create_dev(script=os.path.join(network_script_base, 'create-vxlan.sh'), + _id=_id, dev=_dev) + if vxlan: + bridge = create_dev(script=os.path.join(network_script_base, 'create-bridge.sh'), + _id=_id, dev=vxlan) + if bridge: + tap = create_dev(script=os.path.join(network_script_base, 'create-tap.sh'), + _id=str(random.randint(1, 100000)), dev=bridge) + if tap: + return tap + + +def random_bytes(num=6): + return [random.randrange(256) for _ in range(num)] + + +def generate_mac(uaa=False, multicast=False, oui=None, separator=':', byte_fmt='%02x'): + mac = random_bytes() + if oui: + if type(oui) == str: + oui = [int(chunk) for chunk in oui.split(separator)] + mac = oui + random_bytes(num=6-len(oui)) + else: + if multicast: + mac[0] |= 1 # set bit 0 + else: + mac[0] &= ~1 # clear bit 0 + if uaa: + mac[0] &= ~(1 << 1) # clear bit 1 + else: + mac[0] |= 1 << 1 # set bit 1 + return separator.join(byte_fmt % b for b in mac) + + def get_start_command_args( - vm_entry, vnc_sock_filename: str, migration=False, migration_port=4444 + vm_entry, vnc_sock_filename: str, migration=False, migration_port=4444, ): threads_per_core = 1 vm_memory = int(bitmath.parse_string(vm_entry.specs["ram"]).to_MB()) vm_cpus = int(vm_entry.specs["cpu"]) vm_uuid = vm_entry.uuid + vm_networks = vm_entry.network + if WITHOUT_CEPH: command = "-drive file={},format=raw,if=virtio,cache=none".format( @@ -62,8 +112,21 @@ def get_start_command_args( if migration: command += " -incoming tcp:0:{}".format(migration_port) + + tap = None + for network_name in vm_networks: + _key = os.path.join(NETWORK_PREFIX, vm_entry.owner, network_name) + network = etcd_client.get(_key, value_in_json=True) + network_type = network.value["type"] + network_id = str(network.value["id"]) + + if network_type == "vxlan": + tap = create_vxlan_br_tap(network_id, "eno1") + + command += " -netdev tap,id=vmnet{net_id},ifname={tap},script=no,downscript=no"\ + " -device virtio-net-pci,netdev=vmnet{net_id},mac={mac}"\ + .format(tap=tap, net_id=network_id, mac=generate_mac()) - command += " -nic tap,model=virtio,mac={}".format(vm_entry.mac) return command.split(" ") @@ -144,8 +207,8 @@ def create(vm_entry: VMEntry): ] try: - subprocess.check_output(_command_to_create) - except subprocess.CalledProcessError as e: + sp.check_output(_command_to_create) + except sp.CalledProcessError as e: if e.returncode == errno.EEXIST: logging.debug("Image for vm %s exists", vm_entry.uuid) # File Already exists. No Problem Continue @@ -158,7 +221,7 @@ def create(vm_entry: VMEntry): vm_entry.status = "ERROR" else: try: - subprocess.check_output(_command_to_extend) + sp.check_output(_command_to_extend) except Exception as e: logging.exception(e) else: @@ -199,7 +262,7 @@ def delete(vm_entry): vm_deletion_command = ["rbd", "rm", path_without_protocol] try: - subprocess.check_output(vm_deletion_command) + sp.check_output(vm_deletion_command) except Exception as e: logging.exception(e) else: diff --git a/network/create-bridge.sh b/network/create-bridge.sh new file mode 100755 index 0000000..78ebbee --- /dev/null +++ b/network/create-bridge.sh @@ -0,0 +1,23 @@ +#!/bin/sh + +if [ $# -ne 2 ]; then + echo "$0 brid dev" + echo "f.g. $0 100 vxlan100" + echo "Missing arguments" >&2 + exit 1 +fi + +brid=$1; shift +dev=$1; shift +bridge=br${brid} + +sysctl net.ipv6.conf.all.forwarding=1 > /dev/null + +if ! ip link show $bridge > /dev/null 2> /dev/null; then + ip link add name $bridge type bridge + ip link set $bridge up + ip link set $dev master $bridge + ip address add fd00:/64 dev $bridge +fi + +echo $bridge \ No newline at end of file diff --git a/network/create-tap.sh b/network/create-tap.sh new file mode 100755 index 0000000..4a5e470 --- /dev/null +++ b/network/create-tap.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +if [ $# -ne 2 ]; then + echo "$0 tapid dev" + echo "f.g. $0 100 br100" + echo "Missing arguments" >&2 + exit 1 +fi + +tapid=$1; shift +bridge=$1; shift +vxlan=vxlan${tapid} +tap=tap${tapid} + +if ! ip link show $tap > /dev/null 2> /dev/null; then + ip tuntap add $tap mode tap user `whoami` + ip link set $tap up + sleep 0.5s + ip link set $tap master $bridge +fi + +echo $tap \ No newline at end of file diff --git a/network/create-vxlan-on-dev.sh b/network/create-vxlan-on-dev.sh deleted file mode 100644 index b366392..0000000 --- a/network/create-vxlan-on-dev.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/sh - -if [ $# -ne 2 ]; then - echo "$0 vxlanid dev" - echo "f.i. $0 100 eth0" - exit 1 -fi - -netid=$1; shift -dev=$1; shift - -ip -6 link add vxlan${netid} type vxlan \ - id ${netid} \ - dstport 4789 \ - group ff05::${netid} \ - dev ${dev} \ - ttl 5 - -ip link set ${dev} up diff --git a/network/create-vxlan.sh b/network/create-vxlan.sh new file mode 100755 index 0000000..1a730f6 --- /dev/null +++ b/network/create-vxlan.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +if [ $# -ne 2 ]; then + echo "$0 vxlanid dev" + echo "f.i. $0 100 eno1" + echo "Missing arguments" >&2 + exit 1 +fi + +netid=$1; shift +dev=$1; shift +vxlan=vxlan${netid} + +if ! ip link show $vxlan > /dev/null 2> /dev/null; then + ip -6 link add $vxlan type vxlan \ + id $netid \ + dstport 4789 \ + group ff05::$netid \ + dev $dev \ + ttl 5 + + ip link set $dev up + ip link set $vxlan up +fi + +echo $vxlan \ No newline at end of file diff --git a/scheduler/helper.py b/scheduler/helper.py index 65bd12d..577bc91 100755 --- a/scheduler/helper.py +++ b/scheduler/helper.py @@ -15,7 +15,7 @@ host_pool = HostPool(client, config("HOST_PREFIX")) request_pool = RequestPool(client, config("REQUEST_PREFIX")) -def accumulated_specs(vms_specs): +def accumulated_specs(vms_specs): if not vms_specs: return {} return reduce((lambda x, y: Counter(x) + Counter(y)), vms_specs) @@ -41,7 +41,6 @@ def remaining_resources(host_specs, vms_specs): _remaining[component] = map(lambda x: int(bitmath.parse_string(x).to_MB()), _remaining[component]) _remaining[component] = reduce(lambda x, y: x + y, _remaining[component], 0) - print(_vms_specs, _remaining) _remaining.subtract(_vms_specs) return _remaining @@ -65,16 +64,12 @@ def get_suitable_host(vm_specs, hosts=None): running_vms_specs = [vm.specs for vm in vms] # Accumulate all of their combined specs - running_vms_accumulated_specs = accumulated_specs( - running_vms_specs - ) + running_vms_accumulated_specs = accumulated_specs(running_vms_specs) # Find out remaining resources after # host_specs - already running vm_specs - remaining = remaining_resources( - host.specs, running_vms_accumulated_specs - ) - + remaining = remaining_resources(host.specs, running_vms_accumulated_specs) + # Find out remaining - new_vm_specs remaining = remaining_resources(remaining, vm_specs) diff --git a/scheduler/main.py b/scheduler/main.py index 6e0ba90..3fb0d1b 100755 --- a/scheduler/main.py +++ b/scheduler/main.py @@ -54,6 +54,9 @@ def main(): elif request_entry.type == RequestType.ScheduleVM: vm_entry = vm_pool.get(request_entry.uuid) + if vm_entry is None: + logging.info("Trying to act on {} but it is deleted".format(request_entry.uuid)) + continue client.client.delete(request_entry.key) # consume Request # If the Request is about a VM which is labelled as "migration"