From a9f8d245b9ae8a14e0311ab61bbe095fd3568367 Mon Sep 17 00:00:00 2001 From: Ahmed Bilal Khalid Date: Fri, 30 Aug 2019 06:23:45 +0500 Subject: [PATCH] a --- Pipfile | 2 +- Pipfile.lock | 90 ++++++++++++++++++++++-------------------------- app/api.py | 36 ++++++++++++------- app/ceph.py | 5 +-- app/file.py | 14 +++++--- app/helper.py | 29 +++++++++++++++- app/host.py | 10 +++--- app/image.py | 15 ++++---- app/scheduler.py | 8 +++-- ucloud-setup.py | 1 + 10 files changed, 127 insertions(+), 83 deletions(-) diff --git a/Pipfile b/Pipfile index 7809482..c8fd0c6 100644 --- a/Pipfile +++ b/Pipfile @@ -3,7 +3,7 @@ name = "pypi" url = "https://pypi.org/simple" verify_ssl = true -[dev-packages] +[dev - packages] pylint = "*" prospector = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 1bc07e9..092d54d 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -4,47 +4,41 @@ "sha256": "889e682e3db0471b72d904bf85ca600a2f783edcf226ef591a5408589adcaf72" }, "pipfile-spec": 6, - "requires": { - "python_version": "3.7" - }, + "requires": {"python_version": "3.7"}, "sources": [ - { - "name": "pypi", - "url": "https://pypi.org/simple", - "verify_ssl": true - } - ] + {"name": "pypi", "url": "https://pypi.org/simple", "verify_ssl": true} + ], }, "default": { "click": { "hashes": [ "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13", - "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7" + "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7", ], "index": "pypi", - "version": "==7.0" + "version": "==7.0", } }, "develop": { "astroid": { "hashes": [ "sha256:6560e1e1749f68c64a4b5dee4e091fce798d2f0d84ebe638cf0e0585a343acf4", - "sha256:b65db1bbaac9f9f4d190199bb8680af6f6f84fd3769a5ea883df8a91fe68b4c4" + "sha256:b65db1bbaac9f9f4d190199bb8680af6f6f84fd3769a5ea883df8a91fe68b4c4", ], - "version": "==2.2.5" + "version": "==2.2.5", }, "dodgy": { "hashes": [ "sha256:65e13cf878d7aff129f1461c13cb5fd1bb6dfe66bb5327e09379c3877763280c" ], - "version": "==0.1.9" + "version": "==0.1.9", }, "isort": { "hashes": [ "sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1", - "sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd" + "sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd", ], - "version": "==4.3.21" + "version": "==4.3.21", }, "lazy-object-proxy": { "hashes": [ @@ -65,84 +59,84 @@ "sha256:ae33dd198f772f714420c5ab698ff05ff900150486c648d29951e9c70694338e", "sha256:b4a2b782b8a8c5522ad35c93e04d60e2ba7f7dcb9271ec8e8c3e08239be6c7b4", "sha256:c462eb33f6abca3b34cdedbe84d761f31a60b814e173b98ede3c81bb48967c4f", - "sha256:fd135b8d35dfdcdb984828c84d695937e58cc5f49e1c854eb311c4d6aa03f4f1" + "sha256:fd135b8d35dfdcdb984828c84d695937e58cc5f49e1c854eb311c4d6aa03f4f1", ], - "version": "==1.4.2" + "version": "==1.4.2", }, "mccabe": { "hashes": [ "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", - "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" + "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f", ], - "version": "==0.6.1" + "version": "==0.6.1", }, "pep8-naming": { "hashes": [ "sha256:1b419fa45b68b61cd8c5daf4e0c96d28915ad14d3d5f35fcc1e7e95324a33a2e", - "sha256:4eedfd4c4b05e48796f74f5d8628c068ff788b9c2b08471ad408007fc6450e5a" + "sha256:4eedfd4c4b05e48796f74f5d8628c068ff788b9c2b08471ad408007fc6450e5a", ], - "version": "==0.4.1" + "version": "==0.4.1", }, "prospector": { "hashes": [ "sha256:aba551e53dc1a5a432afa67385eaa81d7b4cf4c162dc1a4d0ee00b3a0712ad90" ], "index": "pypi", - "version": "==1.1.7" + "version": "==1.1.7", }, "pycodestyle": { "hashes": [ "sha256:cbc619d09254895b0d12c2c691e237b2e91e9b2ecf5e84c26b35400f93dcfb83", - "sha256:cbfca99bd594a10f674d0cd97a3d802a1fdef635d4361e1a2658de47ed261e3a" + "sha256:cbfca99bd594a10f674d0cd97a3d802a1fdef635d4361e1a2658de47ed261e3a", ], - "version": "==2.4.0" + "version": "==2.4.0", }, "pydocstyle": { "hashes": [ "sha256:04c84e034ebb56eb6396c820442b8c4499ac5eb94a3bda88951ac3dc519b6058", - "sha256:66aff87ffe34b1e49bff2dd03a88ce6843be2f3346b0c9814410d34987fbab59" + "sha256:66aff87ffe34b1e49bff2dd03a88ce6843be2f3346b0c9814410d34987fbab59", ], - "version": "==4.0.1" + "version": "==4.0.1", }, "pyflakes": { "hashes": [ "sha256:08bd6a50edf8cffa9fa09a463063c425ecaaf10d1eb0335a7e8b1401aef89e6f", - "sha256:8d616a382f243dbf19b54743f280b80198be0bca3a5396f1d2e1fca6223e8805" + "sha256:8d616a382f243dbf19b54743f280b80198be0bca3a5396f1d2e1fca6223e8805", ], - "version": "==1.6.0" + "version": "==1.6.0", }, "pylint": { "hashes": [ "sha256:5d77031694a5fb97ea95e828c8d10fc770a1df6eb3906067aaed42201a8a6a09", - "sha256:723e3db49555abaf9bf79dc474c6b9e2935ad82230b10c1138a71ea41ac0fff1" + "sha256:723e3db49555abaf9bf79dc474c6b9e2935ad82230b10c1138a71ea41ac0fff1", ], "index": "pypi", - "version": "==2.3.1" + "version": "==2.3.1", }, "pylint-celery": { "hashes": [ "sha256:41e32094e7408d15c044178ea828dd524beedbdbe6f83f712c5e35bde1de4beb" ], - "version": "==0.3" + "version": "==0.3", }, "pylint-django": { "hashes": [ "sha256:75c69d1ec2275918c37f175976da20e2f1e1e62e067098a685cd263ffa833dfd", - "sha256:c7cb6384ea7b33ea77052a5ae07358c10d377807390ef27b2e6ff997303fadb7" + "sha256:c7cb6384ea7b33ea77052a5ae07358c10d377807390ef27b2e6ff997303fadb7", ], - "version": "==2.0.10" + "version": "==2.0.10", }, "pylint-flask": { "hashes": [ "sha256:f4d97de2216bf7bfce07c9c08b166e978fe9f2725de2a50a9845a97de7e31517" ], - "version": "==0.6" + "version": "==0.6", }, "pylint-plugin-utils": { "hashes": [ "sha256:8d9e31d5ea8b7b0003e1f0f136b44a5235896a32e47c5bc2ef1143e9f6ba0b74" ], - "version": "==0.5" + "version": "==0.5", }, "pyyaml": { "hashes": [ @@ -158,34 +152,34 @@ "sha256:b0997827b4f6a7c286c01c5f60384d218dca4ed7d9efa945c3e1aa623d5709ae", "sha256:b631ef96d3222e62861443cc89d6563ba3eeb816eeb96b2629345ab795e53681", "sha256:bf47c0607522fdbca6c9e817a6e81b08491de50f3766a7a0e6a5be7905961b41", - "sha256:f81025eddd0327c7d4cfe9b62cf33190e1e736cc6e97502b3ec425f574b3e7a8" + "sha256:f81025eddd0327c7d4cfe9b62cf33190e1e736cc6e97502b3ec425f574b3e7a8", ], - "version": "==5.1.2" + "version": "==5.1.2", }, "requirements-detector": { "hashes": [ "sha256:9fbc4b24e8b7c3663aff32e3eba34596848c6b91bd425079b386973bd8d08931" ], - "version": "==0.6" + "version": "==0.6", }, "setoptconf": { "hashes": [ "sha256:5b0b5d8e0077713f5d5152d4f63be6f048d9a1bb66be15d089a11c898c3cf49c" ], - "version": "==0.2.0" + "version": "==0.2.0", }, "six": { "hashes": [ "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", - "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" + "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73", ], - "version": "==1.12.0" + "version": "==1.12.0", }, "snowballstemmer": { "hashes": [ "sha256:9f3b9ffe0809d174f7047e121431acf99c89a7040f0ca84f94ba53a498e6d0c9" ], - "version": "==1.9.0" + "version": "==1.9.0", }, "typed-ast": { "hashes": [ @@ -203,16 +197,16 @@ "sha256:d755f03c1e4a51e9b24d899561fec4ccaf51f210d52abdf8c07ee2849b212a36", "sha256:d7c45933b1bdfaf9f36c579671fec15d25b06c8398f113dab64c18ed1adda01d", "sha256:d896919306dd0aa22d0132f62a1b78d11aaf4c9fc5b3410d3c666b818191630a", - "sha256:ffde2fbfad571af120fcbfbbc61c72469e72f550d676c3342492a9dfdefb8f12" + "sha256:ffde2fbfad571af120fcbfbbc61c72469e72f550d676c3342492a9dfdefb8f12", ], "markers": "implementation_name == 'cpython'", - "version": "==1.4.0" + "version": "==1.4.0", }, "wrapt": { "hashes": [ "sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1" ], - "version": "==1.11.2" - } - } + "version": "==1.11.2", + }, + }, } diff --git a/app/api.py b/app/api.py index c3a4d56..96438e8 100644 --- a/app/api.py +++ b/app/api.py @@ -11,6 +11,7 @@ from app.helper import ( PipenvOperation, FileOperation, PackageManagementOperation, + VirtualenvOperation ) @@ -32,15 +33,24 @@ def api(): default="https://otp.ungleich.ch/ungleichotp/", help="URL of ungleich OTP server", ) -def setup(path, auth_name, auth_seed, auth_realm, - realm_allowed, otp_server, etcd_url, - etcd_password): +def setup( + path, + auth_name, + auth_seed, + auth_realm, + realm_allowed, + otp_server, + etcd_url, + etcd_password, +): os.chdir(path) - + repo_name = "ucloud-api" - + # Git Operation Depends on success of previous operation i.e PackageManagementOperation - op = GitOperation.clone(url=f"https://code.ungleich.ch/ungleich-public/{repo_name}.git") + op = GitOperation.clone( + url=f"https://code.ungleich.ch/ungleich-public/{repo_name}.git" + ) content = ( f"AUTH_NAME={auth_name}\n" @@ -53,9 +63,7 @@ def setup(path, auth_name, auth_seed, auth_realm, ) # FileOperation depends on success of previos operation i.e GitOperation.clone - op.add( - FileOperation.write, path=os.path.join(repo_name, ".env"), content=content - ) + op.add(FileOperation.write, path=os.path.join(repo_name, ".env"), content=content) op.add( GitOperation.clone, @@ -66,8 +74,10 @@ def setup(path, auth_name, auth_seed, auth_realm, op.add( GitOperation.clone, path=repo_name, - url="https://code.ungleich.ch/ahmedbilal/etcd3_wrapper" + url="https://code.ungleich.ch/ahmedbilal/etcd3_wrapper", ) - shutil.copytree(src=os.path.join(repo_name, "etcd3_wrapper"), - dst=os.path.join(repo_name, "ucloud_common", "etcd3_wrapper")) - op.add(PipenvOperation.install, path=repo_name) + shutil.copytree( + src=os.path.join(repo_name, "etcd3_wrapper"), + dst=os.path.join(repo_name, "ucloud_common", "etcd3_wrapper"), + ) + op.add(VirtualenvOperation.install, path=repo_name) diff --git a/app/ceph.py b/app/ceph.py index 21f9f5f..02df750 100644 --- a/app/ceph.py +++ b/app/ceph.py @@ -2,6 +2,7 @@ import click import os import subprocess + @click.group() def ceph(): pass @@ -21,7 +22,7 @@ def setup(ceph_url, ssh_username): keys = "\n".join(keys) with open(os.path.expanduser("~/.ssh/known_hosts"), "a") as known_hosts: known_hosts.write(keys) - + os.makedirs("/etc/ceph", exist_ok=True) command = f"sftp -b ./ceph_batch_cmd {ssh_username}@{ceph_url}:/etc/ceph" - subprocess.check_output(command.split()) \ No newline at end of file + subprocess.check_output(command.split()) diff --git a/app/file.py b/app/file.py index a60b676..b28cdcd 100644 --- a/app/file.py +++ b/app/file.py @@ -32,10 +32,12 @@ def setup(path, base_dir, file_prefix, etcd_url, etcd_password): f"https://code.ungleich.ch/ungleich-public/{repo_name}.git" ) - content = f"BASE_DIR={base_dir}\n" \ - f"FILE_PREFIX={file_prefix}\n" \ - f"ETCD_URL={etcd_url}\n" \ - f"ETCD_PASSWORD={etcd_password}\n" + content = ( + f"BASE_DIR={base_dir}\n" + f"FILE_PREFIX={file_prefix}\n" + f"ETCD_URL={etcd_url}\n" + f"ETCD_PASSWORD={etcd_password}\n" + ) op_result.add( FileOperation.write, path=os.path.join(repo_name, ".env"), content=content @@ -51,4 +53,6 @@ def setup(path, base_dir, file_prefix, etcd_url, etcd_password): # Write Crontab entry with open("/etc/crontabs/root", "a") as crontab: - crontab.write(f"*/5\t*\t*\t*\t*\tcd {os.path.join(os.getcwd(), repo_name)} && pipenv run python main.py") \ No newline at end of file + crontab.write( + f"*/5\t*\t*\t*\t*\tcd {os.path.join(os.getcwd(), repo_name)} && pipenv run python main.py" + ) diff --git a/app/helper.py b/app/helper.py index 999d2b4..376cbef 100644 --- a/app/helper.py +++ b/app/helper.py @@ -68,6 +68,33 @@ class GitOperation(object): return Result(output, ResultType.success) +class VirtualenvOperation(object): + @staticmethod + def create(path=".", site_packages=False): + if site_packages: + command = f"virtualenv ." + else: + command = "virtualenv --system-site-packages ." + try: + output = subprocess.check_output(command.split(), cwd=path) + except subprocess.CalledProcessError as e: + return Result(e, ResultType.failure, inspect.currentframe().f_code.co_name) + else: + return Result(output, ResultType.success) + + @staticmethod + def install(path=".", package_name=None): + if package_name: + command = f"pip install {package_name}" + else: + command = f"pip install" + try: + output = subprocess.check_output(command.split(), cwd=path) + except subprocess.CalledProcessError as e: + return Result(e, ResultType.failure, inspect.currentframe().f_code.co_name) + else: + return Result(output, ResultType.success) + class PipenvOperation(object): @staticmethod def create(path=".", site_packages=False): @@ -89,7 +116,7 @@ class PipenvOperation(object): else: command = f"pipenv install" try: - output = subprocess.check_output(command.split(), cwd=path, env={**os.environ, "PIP_INSTALL_OPTION": "--jobs=8"}) + output = subprocess.check_output(command.split(), cwd=path) except subprocess.CalledProcessError as e: return Result(e, ResultType.failure, inspect.currentframe().f_code.co_name) else: diff --git a/app/host.py b/app/host.py index 87983e4..ecf7bad 100644 --- a/app/host.py +++ b/app/host.py @@ -57,8 +57,10 @@ def setup(path, ssh_username, ssh_key_path, ssh_key_pass, etcd_url, etcd_passwor path=repo_name, url="https://code.ungleich.ch/ahmedbilal/etcd3_wrapper", ) - - shutil.copytree(src=os.path.join(repo_name, "etcd3_wrapper"), - dst=os.path.join(repo_name, "ucloud_common", "etcd3_wrapper")) - + + shutil.copytree( + src=os.path.join(repo_name, "etcd3_wrapper"), + dst=os.path.join(repo_name, "ucloud_common", "etcd3_wrapper"), + ) + op_result.add(PipenvOperation.install, path=repo_name) diff --git a/app/image.py b/app/image.py index cdcb2d7..b1fa727 100644 --- a/app/image.py +++ b/app/image.py @@ -31,23 +31,26 @@ def setup(path, base_dir, etcd_url, etcd_password): f"https://code.ungleich.ch/ungleich-public/{repo_name}.git" ) - content = f"BASE_DIR={base_dir}\n" \ - f"ETCD_URL={etcd_url}\n" \ - f"ETCD_PASSWORD={etcd_password}\n" + content = ( + f"BASE_DIR={base_dir}\n" + f"ETCD_URL={etcd_url}\n" + f"ETCD_PASSWORD={etcd_password}\n" + ) op_result.add( FileOperation.write, path=os.path.join(repo_name, ".env"), content=content ) - op_result.add( GitOperation.clone, path=repo_name, url="https://code.ungleich.ch/ahmedbilal/etcd3_wrapper", ) - + op_result.add(PipenvOperation.install, path=repo_name) # Write Crontab entry with open("/etc/crontabs/root", "a") as crontab: - crontab.write(f"*/5\t*\t*\t*\t*\tcd {os.path.join(os.getcwd(), repo_name)} && pipenv run python main.py") \ No newline at end of file + crontab.write( + f"*/5\t*\t*\t*\t*\tcd {os.path.join(os.getcwd(), repo_name)} && pipenv run python main.py" + ) diff --git a/app/scheduler.py b/app/scheduler.py index 7c154f8..48a5e36 100644 --- a/app/scheduler.py +++ b/app/scheduler.py @@ -58,7 +58,9 @@ def setup(path, vm_prefix, host_prefix, request_prefix, etcd_url, etcd_password) url="https://code.ungleich.ch/ahmedbilal/etcd3_wrapper", ) - shutil.copytree(src=os.path.join(repo_name, "etcd3_wrapper"), - dst=os.path.join(repo_name, "ucloud_common", "etcd3_wrapper")) - + shutil.copytree( + src=os.path.join(repo_name, "etcd3_wrapper"), + dst=os.path.join(repo_name, "ucloud_common", "etcd3_wrapper"), + ) + op_result.add(PipenvOperation.install, path=repo_name) diff --git a/ucloud-setup.py b/ucloud-setup.py index 446db2c..c379464 100644 --- a/ucloud-setup.py +++ b/ucloud-setup.py @@ -6,6 +6,7 @@ from app.image import image from app.file import file_scan from app.ceph import ceph + @click.group() def entry_point(): pass