From af00df6068073e6dc42671cd2122e8474ef04657 Mon Sep 17 00:00:00 2001 From: Ahmed Bilal Khalid Date: Thu, 12 Sep 2019 00:16:34 +0500 Subject: [PATCH] a --- app/api.py | 4 +- app/file.py | 4 +- app/helper.py | 225 +++++++++++------------------------------------ app/host.py | 4 +- app/image.py | 4 +- app/scheduler.py | 4 +- 6 files changed, 66 insertions(+), 179 deletions(-) diff --git a/app/api.py b/app/api.py index 8fb07f0..f719018 100644 --- a/app/api.py +++ b/app/api.py @@ -3,7 +3,7 @@ import subprocess as sp import os import shutil -from app.helper import clone_common, clone_etcd_wrapper +from app.helper import clone_common, clone_etcd_wrapper, install_available @click.group() def api(): @@ -53,6 +53,8 @@ def setup(path, auth_name, auth_seed, auth_realm, dst=os.path.join(repo_name, "ucloud_common", "etcd3_wrapper"), ) + install_available(os.path.join(repo_name, "Pipfile")) + # Create virtualenv with site-packages enabled and install all dependencies sp.check_output(['pipenv','--site-packages', '--python', '3'], cwd=repo_name) sp.check_output(['pipenv', 'install'], cwd=repo_name) diff --git a/app/file.py b/app/file.py index 1e9c987..e46ecbd 100644 --- a/app/file.py +++ b/app/file.py @@ -2,7 +2,7 @@ import click import subprocess as sp import os -from app.helper import clone_etcd_wrapper +from app.helper import clone_etcd_wrapper, install_available @click.group() def file_scan(): @@ -35,6 +35,8 @@ def setup(path, base_dir, file_prefix, etcd_url): # Clone etcd wrapper clone_etcd_wrapper(path=repo_name) + install_available(os.path.join(repo_name, "Pipfile")) + # Create virtualenv with site-packages enabled and install all dependencies sp.check_output(['pipenv','--site-packages', '--python', '3'], cwd=repo_name) sp.check_output(['pipenv', 'install'], cwd=repo_name) diff --git a/app/helper.py b/app/helper.py index ffda8a7..b413ae5 100644 --- a/app/helper.py +++ b/app/helper.py @@ -1,28 +1,59 @@ -import subprocess import subprocess as sp -import inspect -import os +import subprocess as sp +import requests +import json -from dataclasses import dataclass -from abc import ABC -from enum import Enum -from pipfile import Pipfile +from decouple import config +def install_available(pipfile): + """ Install Python packages or their dependencies mentioned in + pipfile if they are available in System repos + """ + + def is_available_in_system_repos(package): + try: + sp.check_output(['apk', 'info', f'py3-{package}']) + except: + return False + else: + return True + + if get_distro_name() == "alpine": + dependencies = [] -class ResultType(Enum): - success = 0 - failure = 1 + need_to_be_installed = [] + try: + with open(pipfile) as f: + lines = f.readlines() + try: + start_index = lines.index("[packages]\n") + end_index = lines.index("\n", start_index) + for package in lines[start_index: end_index]: + if '= "*"' in package: + package = package[:package.index(" =")] + if is_available_in_system_repos(package): + need_to_be_installed.append(f"py3-{package}") + else: + dependencies.append(package) + except ValueError: + print("Not Found") + except: + return + else: + for dependency in dependencies: + content = requests.get(f"https://libraries.io/api/pypi/{dependency}/latest/dependencies?api_key={config('LIBRARIES_IO_API')}") + content = json.loads(content.content.decode("utf-8")) + for subdependency in content["dependencies"]: + subdependency = subdependency["name"] + if is_available_in_system_repos(subdependency): + need_to_be_installed.append(f"py3-{subdependency}") -def clone(repo): - command = f"git clone {repo}" - try: - subprocess.check_output(command.split()) - except subprocess.CalledProcessError as e: - return False - else: - return True - + for package in need_to_be_installed: + try: + sp.check_output(["apk", "add", package]) + except: + print(f"Could not install {package}") def clone_common(path='.'): sp.check_output(['git', 'clone', @@ -34,142 +65,6 @@ def clone_etcd_wrapper(path='.'): f'https://code.ungleich.ch/ahmedbilal/etcd3_wrapper'], cwd=path) -class Result(object): - def __init__(self, _result, _type: ResultType, _op=""): - self._type = _type - self._result = str(_result) - self._op = _op - if self._type == ResultType.failure: - print(self._op, "failed") - - def add(self, operation, **kwargs): - if self._type == ResultType.success: - r = operation(**kwargs) - self._type = r._type - self._result = r._result - return self - else: - print("Dependency not satisfied") - exit(-1) - - def __repr__(self): - return f"{self._type}, {self._result}" - - -class Operation(ABC): - pass - - -class GitOperation(object): - @staticmethod - def clone(url, path=".", arguments=""): - command = f"git clone {arguments} {url}" - try: - output = subprocess.check_output(command.split(), cwd=path) - except subprocess.CalledProcessError as e: - return Result(e, ResultType.failure, inspect.currentframe().f_code.co_name) - else: - return Result(output, ResultType.success) - - -class VirtualenvOperation(object): - @staticmethod - def create(path=".", site_packages=False): - if site_packages: - command = f"virtualenv ." - else: - command = "virtualenv --system-site-packages ." - try: - output = subprocess.check_output(command.split(), cwd=path) - except subprocess.CalledProcessError as e: - return Result(e, ResultType.failure, inspect.currentframe().f_code.co_name) - else: - return Result(output, ResultType.success) - - @staticmethod - def install(path=".", package_name=None): - if package_name: - command = f"pip install {package_name}" - else: - with open(os.path.join(path, "requirements.txt"), "w") as f: - subprocess.Popen("pipenv run pip freeze".split(), cwd=path, stdout=f) - command = f"pip install -r requirements.txt" - try: - output = subprocess.check_output(command.split(), cwd=path) - except subprocess.CalledProcessError as e: - return Result(e, ResultType.failure, inspect.currentframe().f_code.co_name) - else: - return Result(output, ResultType.success) - - -def globally_installed_py_packages(): - output = subprocess.check_output("pip3 list --format freeze".split(), env={}) - output = output.decode("utf-8") - output = output.strip() - global_packages = output.split("\n") - return global_packages - - -class PipenvOperation(object): - @staticmethod - def create(path=".", site_packages=False): - if site_packages: - global_packages = globally_installed_py_packages() - - p = Pipfile.load(filename=os.path.join(path, "Pipfile")) - content = "" - with open(os.path.join(path, "Pipfile"), "r") as f: - content = f.read() - - for pip_package in p.data["default"]: - version = p.data["default"][pip_package] - if version == "*": - for package in global_packages: - package = package.lower() - if package.startswith(pip_package.lower()): - pkg, pkg_ver = package.split("==") - substr = f'{pip_package} = "*"' - content = content.replace(substr, f'{pkg} = "=={pkg_ver}"') - - with open(os.path.join(path, "Pipfile"), "w") as f: - f.write(content) - - - command = f"pipenv --site-packages --python 3" - else: - command = "pipenv --python 3" - try: - output = subprocess.check_output(command.split(), cwd=path) - except subprocess.CalledProcessError as e: - return Result(e, ResultType.failure, inspect.currentframe().f_code.co_name) - else: - return Result(output, ResultType.success) - - @staticmethod - def install(path=".", package_name=None): - if package_name: - command = f"pipenv install {package_name}" - else: - command = f"pipenv install" - try: - output = subprocess.check_output(command.split(), cwd=path) - except subprocess.CalledProcessError as e: - return Result(e, ResultType.failure, inspect.currentframe().f_code.co_name) - else: - return Result(output, ResultType.success) - -class FileOperation(object): - @staticmethod - def write(path, content, mode="w"): - try: - with open(path, mode) as file: - file.write(content) - except Exception as e: - return Result(e, ResultType.failure, inspect.currentframe().f_code.co_name) - else: - return Result("", ResultType.success) - - def get_distro_name(): distro_name = None with open("/etc/os-release") as f: @@ -177,22 +72,4 @@ def get_distro_name(): start = content.find("ID=") + 3 end = content.find("\n", start) distro_name = content[start:end] - return distro_name - - -class PackageManagementOperation(object): - @staticmethod - def install(package_name): - try: - distro = get_distro_name() - if distro == "alpine": - command = f"apk add {package_name}" - else: - assert "Unknown Distro" - - subprocess.check_output(command.split()) - except Exception as e: - print(e) - return Result(e, ResultType.failure, inspect.currentframe().f_code.co_name) - else: - return Result("", ResultType.success) + return distro_name \ No newline at end of file diff --git a/app/host.py b/app/host.py index 388c014..7a0553f 100644 --- a/app/host.py +++ b/app/host.py @@ -3,7 +3,7 @@ import subprocess as sp import os import shutil -from app.helper import clone_common, clone_etcd_wrapper +from app.helper import clone_common, clone_etcd_wrapper, install_available @click.group() @@ -49,6 +49,8 @@ def setup(path, ssh_username, ssh_key_path, ssh_key_pass, etcd_url, without_ceph dst=os.path.join(repo_name, "ucloud_common", "etcd3_wrapper"), ) + install_available(os.path.join(repo_name, "Pipfile")) + # Create virtualenv with site-packages enabled and install all dependencies sp.check_output(['pipenv','--site-packages', '--python', '3'], cwd=repo_name) sp.check_output(['pipenv', 'install'], cwd=repo_name) \ No newline at end of file diff --git a/app/image.py b/app/image.py index ec22150..f504659 100644 --- a/app/image.py +++ b/app/image.py @@ -2,7 +2,7 @@ import click import subprocess as sp import os -from app.helper import clone_etcd_wrapper +from app.helper import clone_etcd_wrapper, install_available @click.group() def image(): @@ -35,6 +35,8 @@ def setup(path, base_dir, etcd_url, without_ceph): clone_etcd_wrapper(path=repo_name) + install_available(os.path.join(repo_name, "Pipfile")) + # Create virtualenv with site-packages enabled and install all dependencies sp.check_output(['pipenv','--site-packages', '--python', '3'], cwd=repo_name) sp.check_output(['pipenv', 'install'], cwd=repo_name) diff --git a/app/scheduler.py b/app/scheduler.py index f2fa690..ec61a7a 100644 --- a/app/scheduler.py +++ b/app/scheduler.py @@ -3,7 +3,7 @@ import subprocess as sp import os import shutil -from app.helper import clone_common, clone_etcd_wrapper +from app.helper import clone_common, clone_etcd_wrapper, install_available @click.group() @@ -47,6 +47,8 @@ def setup(path, vm_prefix, host_prefix, request_prefix, etcd_url): dst=os.path.join(repo_name, "ucloud_common", "etcd3_wrapper"), ) + install_available(os.path.join(repo_name, "Pipfile")) + # Create virtualenv with site-packages enabled and install all dependencies sp.check_output(['pipenv','--site-packages', '--python', '3'], cwd=repo_name) sp.check_output(['pipenv', 'install'], cwd=repo_name) \ No newline at end of file