a

app/api.py

@@ -3,7 +3,7 @@ import subprocess as sp
 import os
 import shutil
 
-from app.helper import clone_common, clone_etcd_wrapper
+from app.helper import clone_common, clone_etcd_wrapper, install_available
 
 @click.group()
 def api():
@@ -53,6 +53,8 @@ def setup(path, auth_name, auth_seed, auth_realm,
         dst=os.path.join(repo_name, "ucloud_common", "etcd3_wrapper"),
     )
     
+    install_available(os.path.join(repo_name, "Pipfile"))
+    
     # Create virtualenv with site-packages enabled and install all dependencies
     sp.check_output(['pipenv','--site-packages', '--python', '3'], cwd=repo_name)
     sp.check_output(['pipenv', 'install'], cwd=repo_name)

app/file.py

@@ -2,7 +2,7 @@ import click
 import subprocess as sp
 import os
 
-from app.helper import clone_etcd_wrapper
+from app.helper import clone_etcd_wrapper, install_available
 
 @click.group()
 def file_scan():
@@ -35,6 +35,8 @@ def setup(path, base_dir, file_prefix, etcd_url):
     # Clone etcd wrapper
     clone_etcd_wrapper(path=repo_name)
     
+    install_available(os.path.join(repo_name, "Pipfile"))
+
     # Create virtualenv with site-packages enabled and install all dependencies
     sp.check_output(['pipenv','--site-packages', '--python', '3'], cwd=repo_name)
     sp.check_output(['pipenv', 'install'], cwd=repo_name)

app/helper.py

@@ -1,28 +1,59 @@
-import subprocess
 import subprocess as sp
-import inspect
+import subprocess as sp
-import os
+import requests
+import json
 
-from dataclasses import dataclass
+from decouple import config
-from abc import ABC
-from enum import Enum
-from pipfile import Pipfile
 
+def install_available(pipfile):
+    """ Install Python packages or their dependencies mentioned in
+        pipfile if they are available in System repos
+    """
+    
+    def is_available_in_system_repos(package):
+        try:
+            sp.check_output(['apk', 'info', f'py3-{package}'])
+        except:
+            return False
+        else:
+            return True
+    
+    if get_distro_name() == "alpine":
+        dependencies = []
 
-class ResultType(Enum):
+        need_to_be_installed = []
-    success = 0
-    failure = 1
 
+        try:
+            with open(pipfile) as f:
+                lines = f.readlines()
+                try:
+                    start_index = lines.index("[packages]\n")
+                    end_index = lines.index("\n", start_index)
+                    for package in lines[start_index: end_index]:
+                        if '= "*"' in package:
+                            package = package[:package.index(" =")]
+                            if is_available_in_system_repos(package):
+                                need_to_be_installed.append(f"py3-{package}")
+                            else:
+                                dependencies.append(package)
+                except ValueError:
+                    print("Not Found")
+        except:
+            return
+        else:
+            for dependency in dependencies:
+                content = requests.get(f"https://libraries.io/api/pypi/{dependency}/latest/dependencies?api_key={config('LIBRARIES_IO_API')}")
+                content = json.loads(content.content.decode("utf-8"))
+                for subdependency in content["dependencies"]:
+                    subdependency = subdependency["name"]
+                    if is_available_in_system_repos(subdependency):
+                        need_to_be_installed.append(f"py3-{subdependency}")
 
-def clone(repo):
+            for package in need_to_be_installed:
-    command = f"git clone {repo}"
+                try:
-    try:
+                    sp.check_output(["apk", "add", package])
-        subprocess.check_output(command.split())
+                except:
-    except subprocess.CalledProcessError as e:
+                    print(f"Could not install {package}")
-        return False
-    else:
-        return True
-
 
 def clone_common(path='.'):
     sp.check_output(['git', 'clone',
@@ -34,142 +65,6 @@ def clone_etcd_wrapper(path='.'):
                     f'https://code.ungleich.ch/ahmedbilal/etcd3_wrapper'], cwd=path)
 
 
-class Result(object):
-    def __init__(self, _result, _type: ResultType, _op=""):
-        self._type = _type
-        self._result = str(_result)
-        self._op = _op
-        if self._type == ResultType.failure:
-            print(self._op, "failed")
-
-    def add(self, operation, **kwargs):
-        if self._type == ResultType.success:
-            r = operation(**kwargs)
-            self._type = r._type
-            self._result = r._result
-            return self
-        else:
-            print("Dependency not satisfied")
-            exit(-1)
-
-    def __repr__(self):
-        return f"{self._type}, {self._result}"
-
-
-class Operation(ABC):
-    pass
-
-
-class GitOperation(object):
-    @staticmethod
-    def clone(url, path=".", arguments=""):
-        command = f"git clone {arguments} {url}"
-        try:
-            output = subprocess.check_output(command.split(), cwd=path)
-        except subprocess.CalledProcessError as e:
-            return Result(e, ResultType.failure, inspect.currentframe().f_code.co_name)
-        else:
-            return Result(output, ResultType.success)
-
-
-class VirtualenvOperation(object):
-    @staticmethod
-    def create(path=".", site_packages=False):
-        if site_packages:
-            command = f"virtualenv ."
-        else:
-            command = "virtualenv --system-site-packages ."
-        try:
-            output = subprocess.check_output(command.split(), cwd=path)
-        except subprocess.CalledProcessError as e:
-            return Result(e, ResultType.failure, inspect.currentframe().f_code.co_name)
-        else:
-            return Result(output, ResultType.success)
-
-    @staticmethod
-    def install(path=".", package_name=None):
-        if package_name:
-            command = f"pip install {package_name}"
-        else:
-            with open(os.path.join(path, "requirements.txt"), "w") as f:
-                subprocess.Popen("pipenv run pip freeze".split(), cwd=path, stdout=f)
-            command = f"pip install -r requirements.txt"
-        try:
-            output = subprocess.check_output(command.split(), cwd=path)
-        except subprocess.CalledProcessError as e:
-            return Result(e, ResultType.failure, inspect.currentframe().f_code.co_name)
-        else:
-            return Result(output, ResultType.success)
-
-
-def globally_installed_py_packages():
-    output = subprocess.check_output("pip3 list --format freeze".split(), env={})
-    output = output.decode("utf-8")
-    output = output.strip()
-    global_packages = output.split("\n")
-    return global_packages
-
-
-class PipenvOperation(object):
-    @staticmethod
-    def create(path=".", site_packages=False):
-        if site_packages:
-            global_packages = globally_installed_py_packages()
-
-            p = Pipfile.load(filename=os.path.join(path, "Pipfile"))
-            content = ""
-            with open(os.path.join(path, "Pipfile"), "r") as f:
-                content = f.read()
-
-            for pip_package in p.data["default"]:
-                version = p.data["default"][pip_package]
-                if version == "*":
-                    for package in global_packages:
-                        package = package.lower()
-                        if package.startswith(pip_package.lower()):
-                            pkg, pkg_ver = package.split("==")
-                            substr = f'{pip_package} = "*"'
-                            content = content.replace(substr, f'{pkg} = "=={pkg_ver}"')
-
-            with open(os.path.join(path, "Pipfile"), "w") as f:
-                f.write(content)
-            
-            
-            command = f"pipenv --site-packages --python 3"
-        else:
-            command = "pipenv --python 3"
-        try:
-            output = subprocess.check_output(command.split(), cwd=path)
-        except subprocess.CalledProcessError as e:
-            return Result(e, ResultType.failure, inspect.currentframe().f_code.co_name)
-        else:
-            return Result(output, ResultType.success)
-    
-    @staticmethod
-    def install(path=".", package_name=None):
-        if package_name:
-            command = f"pipenv install {package_name}"
-        else:
-            command = f"pipenv install"
-        try:
-            output = subprocess.check_output(command.split(), cwd=path)
-        except subprocess.CalledProcessError as e:
-            return Result(e, ResultType.failure, inspect.currentframe().f_code.co_name)
-        else:
-            return Result(output, ResultType.success)
-
-class FileOperation(object):
-    @staticmethod
-    def write(path, content, mode="w"):
-        try:
-            with open(path, mode) as file:
-                file.write(content)
-        except Exception as e:
-            return Result(e, ResultType.failure, inspect.currentframe().f_code.co_name)
-        else:
-            return Result("", ResultType.success)
-
-
 def get_distro_name():
     distro_name = None
     with open("/etc/os-release") as f:
@@ -177,22 +72,4 @@ def get_distro_name():
         start = content.find("ID=") + 3
         end = content.find("\n", start)
         distro_name = content[start:end]
-    return distro_name
+    return distro_name
-
-
-class PackageManagementOperation(object):
-    @staticmethod
-    def install(package_name):
-        try:
-            distro = get_distro_name()
-            if distro == "alpine":
-                command = f"apk add {package_name}"
-            else:
-                assert "Unknown Distro"
-
-            subprocess.check_output(command.split())
-        except Exception as e:
-            print(e)
-            return Result(e, ResultType.failure, inspect.currentframe().f_code.co_name)
-        else:
-            return Result("", ResultType.success)

app/host.py

@@ -3,7 +3,7 @@ import subprocess as sp
 import os
 import shutil
 
-from app.helper import clone_common, clone_etcd_wrapper
+from app.helper import clone_common, clone_etcd_wrapper, install_available
 
 
 @click.group()
@@ -49,6 +49,8 @@ def setup(path, ssh_username, ssh_key_path, ssh_key_pass, etcd_url, without_ceph
         dst=os.path.join(repo_name, "ucloud_common", "etcd3_wrapper"),
     )
 
+    install_available(os.path.join(repo_name, "Pipfile"))
+    
     # Create virtualenv with site-packages enabled and install all dependencies
     sp.check_output(['pipenv','--site-packages', '--python', '3'], cwd=repo_name)
     sp.check_output(['pipenv', 'install'], cwd=repo_name)

app/image.py

@@ -2,7 +2,7 @@ import click
 import subprocess as sp
 import os
 
-from app.helper import clone_etcd_wrapper
+from app.helper import clone_etcd_wrapper, install_available
 
 @click.group()
 def image():
@@ -35,6 +35,8 @@ def setup(path, base_dir, etcd_url, without_ceph):
 
     clone_etcd_wrapper(path=repo_name)
     
+    install_available(os.path.join(repo_name, "Pipfile"))
+    
     # Create virtualenv with site-packages enabled and install all dependencies
     sp.check_output(['pipenv','--site-packages', '--python', '3'], cwd=repo_name)
     sp.check_output(['pipenv', 'install'], cwd=repo_name)

app/scheduler.py

@@ -3,7 +3,7 @@ import subprocess as sp
 import os
 import shutil
 
-from app.helper import clone_common, clone_etcd_wrapper
+from app.helper import clone_common, clone_etcd_wrapper, install_available
 
 
 @click.group()
@@ -47,6 +47,8 @@ def setup(path, vm_prefix, host_prefix, request_prefix, etcd_url):
         dst=os.path.join(repo_name, "ucloud_common", "etcd3_wrapper"),
     )
     
+    install_available(os.path.join(repo_name, "Pipfile"))
+    
     # Create virtualenv with site-packages enabled and install all dependencies
     sp.check_output(['pipenv','--site-packages', '--python', '3'], cwd=repo_name)
     sp.check_output(['pipenv', 'install'], cwd=repo_name)