a
This commit is contained in:
parent
0138fb3349
commit
af00df6068
6 changed files with 66 additions and 179 deletions
|
|
@ -3,7 +3,7 @@ import subprocess as sp
|
|||
import os
|
||||
import shutil
|
||||
|
||||
from app.helper import clone_common, clone_etcd_wrapper
|
||||
from app.helper import clone_common, clone_etcd_wrapper, install_available
|
||||
|
||||
@click.group()
|
||||
def api():
|
||||
|
|
@ -53,6 +53,8 @@ def setup(path, auth_name, auth_seed, auth_realm,
|
|||
dst=os.path.join(repo_name, "ucloud_common", "etcd3_wrapper"),
|
||||
)
|
||||
|
||||
install_available(os.path.join(repo_name, "Pipfile"))
|
||||
|
||||
# Create virtualenv with site-packages enabled and install all dependencies
|
||||
sp.check_output(['pipenv','--site-packages', '--python', '3'], cwd=repo_name)
|
||||
sp.check_output(['pipenv', 'install'], cwd=repo_name)
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import click
|
|||
import subprocess as sp
|
||||
import os
|
||||
|
||||
from app.helper import clone_etcd_wrapper
|
||||
from app.helper import clone_etcd_wrapper, install_available
|
||||
|
||||
@click.group()
|
||||
def file_scan():
|
||||
|
|
@ -35,6 +35,8 @@ def setup(path, base_dir, file_prefix, etcd_url):
|
|||
# Clone etcd wrapper
|
||||
clone_etcd_wrapper(path=repo_name)
|
||||
|
||||
install_available(os.path.join(repo_name, "Pipfile"))
|
||||
|
||||
# Create virtualenv with site-packages enabled and install all dependencies
|
||||
sp.check_output(['pipenv','--site-packages', '--python', '3'], cwd=repo_name)
|
||||
sp.check_output(['pipenv', 'install'], cwd=repo_name)
|
||||
|
|
|
|||
225
app/helper.py
225
app/helper.py
|
|
@ -1,28 +1,59 @@
|
|||
import subprocess
|
||||
import subprocess as sp
|
||||
import inspect
|
||||
import os
|
||||
import subprocess as sp
|
||||
import requests
|
||||
import json
|
||||
|
||||
from dataclasses import dataclass
|
||||
from abc import ABC
|
||||
from enum import Enum
|
||||
from pipfile import Pipfile
|
||||
from decouple import config
|
||||
|
||||
def install_available(pipfile):
|
||||
""" Install Python packages or their dependencies mentioned in
|
||||
pipfile if they are available in System repos
|
||||
"""
|
||||
|
||||
def is_available_in_system_repos(package):
|
||||
try:
|
||||
sp.check_output(['apk', 'info', f'py3-{package}'])
|
||||
except:
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
|
||||
if get_distro_name() == "alpine":
|
||||
dependencies = []
|
||||
|
||||
class ResultType(Enum):
|
||||
success = 0
|
||||
failure = 1
|
||||
need_to_be_installed = []
|
||||
|
||||
try:
|
||||
with open(pipfile) as f:
|
||||
lines = f.readlines()
|
||||
try:
|
||||
start_index = lines.index("[packages]\n")
|
||||
end_index = lines.index("\n", start_index)
|
||||
for package in lines[start_index: end_index]:
|
||||
if '= "*"' in package:
|
||||
package = package[:package.index(" =")]
|
||||
if is_available_in_system_repos(package):
|
||||
need_to_be_installed.append(f"py3-{package}")
|
||||
else:
|
||||
dependencies.append(package)
|
||||
except ValueError:
|
||||
print("Not Found")
|
||||
except:
|
||||
return
|
||||
else:
|
||||
for dependency in dependencies:
|
||||
content = requests.get(f"https://libraries.io/api/pypi/{dependency}/latest/dependencies?api_key={config('LIBRARIES_IO_API')}")
|
||||
content = json.loads(content.content.decode("utf-8"))
|
||||
for subdependency in content["dependencies"]:
|
||||
subdependency = subdependency["name"]
|
||||
if is_available_in_system_repos(subdependency):
|
||||
need_to_be_installed.append(f"py3-{subdependency}")
|
||||
|
||||
def clone(repo):
|
||||
command = f"git clone {repo}"
|
||||
try:
|
||||
subprocess.check_output(command.split())
|
||||
except subprocess.CalledProcessError as e:
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
|
||||
for package in need_to_be_installed:
|
||||
try:
|
||||
sp.check_output(["apk", "add", package])
|
||||
except:
|
||||
print(f"Could not install {package}")
|
||||
|
||||
def clone_common(path='.'):
|
||||
sp.check_output(['git', 'clone',
|
||||
|
|
@ -34,142 +65,6 @@ def clone_etcd_wrapper(path='.'):
|
|||
f'https://code.ungleich.ch/ahmedbilal/etcd3_wrapper'], cwd=path)
|
||||
|
||||
|
||||
class Result(object):
|
||||
def __init__(self, _result, _type: ResultType, _op=""):
|
||||
self._type = _type
|
||||
self._result = str(_result)
|
||||
self._op = _op
|
||||
if self._type == ResultType.failure:
|
||||
print(self._op, "failed")
|
||||
|
||||
def add(self, operation, **kwargs):
|
||||
if self._type == ResultType.success:
|
||||
r = operation(**kwargs)
|
||||
self._type = r._type
|
||||
self._result = r._result
|
||||
return self
|
||||
else:
|
||||
print("Dependency not satisfied")
|
||||
exit(-1)
|
||||
|
||||
def __repr__(self):
|
||||
return f"{self._type}, {self._result}"
|
||||
|
||||
|
||||
class Operation(ABC):
|
||||
pass
|
||||
|
||||
|
||||
class GitOperation(object):
|
||||
@staticmethod
|
||||
def clone(url, path=".", arguments=""):
|
||||
command = f"git clone {arguments} {url}"
|
||||
try:
|
||||
output = subprocess.check_output(command.split(), cwd=path)
|
||||
except subprocess.CalledProcessError as e:
|
||||
return Result(e, ResultType.failure, inspect.currentframe().f_code.co_name)
|
||||
else:
|
||||
return Result(output, ResultType.success)
|
||||
|
||||
|
||||
class VirtualenvOperation(object):
|
||||
@staticmethod
|
||||
def create(path=".", site_packages=False):
|
||||
if site_packages:
|
||||
command = f"virtualenv ."
|
||||
else:
|
||||
command = "virtualenv --system-site-packages ."
|
||||
try:
|
||||
output = subprocess.check_output(command.split(), cwd=path)
|
||||
except subprocess.CalledProcessError as e:
|
||||
return Result(e, ResultType.failure, inspect.currentframe().f_code.co_name)
|
||||
else:
|
||||
return Result(output, ResultType.success)
|
||||
|
||||
@staticmethod
|
||||
def install(path=".", package_name=None):
|
||||
if package_name:
|
||||
command = f"pip install {package_name}"
|
||||
else:
|
||||
with open(os.path.join(path, "requirements.txt"), "w") as f:
|
||||
subprocess.Popen("pipenv run pip freeze".split(), cwd=path, stdout=f)
|
||||
command = f"pip install -r requirements.txt"
|
||||
try:
|
||||
output = subprocess.check_output(command.split(), cwd=path)
|
||||
except subprocess.CalledProcessError as e:
|
||||
return Result(e, ResultType.failure, inspect.currentframe().f_code.co_name)
|
||||
else:
|
||||
return Result(output, ResultType.success)
|
||||
|
||||
|
||||
def globally_installed_py_packages():
|
||||
output = subprocess.check_output("pip3 list --format freeze".split(), env={})
|
||||
output = output.decode("utf-8")
|
||||
output = output.strip()
|
||||
global_packages = output.split("\n")
|
||||
return global_packages
|
||||
|
||||
|
||||
class PipenvOperation(object):
|
||||
@staticmethod
|
||||
def create(path=".", site_packages=False):
|
||||
if site_packages:
|
||||
global_packages = globally_installed_py_packages()
|
||||
|
||||
p = Pipfile.load(filename=os.path.join(path, "Pipfile"))
|
||||
content = ""
|
||||
with open(os.path.join(path, "Pipfile"), "r") as f:
|
||||
content = f.read()
|
||||
|
||||
for pip_package in p.data["default"]:
|
||||
version = p.data["default"][pip_package]
|
||||
if version == "*":
|
||||
for package in global_packages:
|
||||
package = package.lower()
|
||||
if package.startswith(pip_package.lower()):
|
||||
pkg, pkg_ver = package.split("==")
|
||||
substr = f'{pip_package} = "*"'
|
||||
content = content.replace(substr, f'{pkg} = "=={pkg_ver}"')
|
||||
|
||||
with open(os.path.join(path, "Pipfile"), "w") as f:
|
||||
f.write(content)
|
||||
|
||||
|
||||
command = f"pipenv --site-packages --python 3"
|
||||
else:
|
||||
command = "pipenv --python 3"
|
||||
try:
|
||||
output = subprocess.check_output(command.split(), cwd=path)
|
||||
except subprocess.CalledProcessError as e:
|
||||
return Result(e, ResultType.failure, inspect.currentframe().f_code.co_name)
|
||||
else:
|
||||
return Result(output, ResultType.success)
|
||||
|
||||
@staticmethod
|
||||
def install(path=".", package_name=None):
|
||||
if package_name:
|
||||
command = f"pipenv install {package_name}"
|
||||
else:
|
||||
command = f"pipenv install"
|
||||
try:
|
||||
output = subprocess.check_output(command.split(), cwd=path)
|
||||
except subprocess.CalledProcessError as e:
|
||||
return Result(e, ResultType.failure, inspect.currentframe().f_code.co_name)
|
||||
else:
|
||||
return Result(output, ResultType.success)
|
||||
|
||||
class FileOperation(object):
|
||||
@staticmethod
|
||||
def write(path, content, mode="w"):
|
||||
try:
|
||||
with open(path, mode) as file:
|
||||
file.write(content)
|
||||
except Exception as e:
|
||||
return Result(e, ResultType.failure, inspect.currentframe().f_code.co_name)
|
||||
else:
|
||||
return Result("", ResultType.success)
|
||||
|
||||
|
||||
def get_distro_name():
|
||||
distro_name = None
|
||||
with open("/etc/os-release") as f:
|
||||
|
|
@ -177,22 +72,4 @@ def get_distro_name():
|
|||
start = content.find("ID=") + 3
|
||||
end = content.find("\n", start)
|
||||
distro_name = content[start:end]
|
||||
return distro_name
|
||||
|
||||
|
||||
class PackageManagementOperation(object):
|
||||
@staticmethod
|
||||
def install(package_name):
|
||||
try:
|
||||
distro = get_distro_name()
|
||||
if distro == "alpine":
|
||||
command = f"apk add {package_name}"
|
||||
else:
|
||||
assert "Unknown Distro"
|
||||
|
||||
subprocess.check_output(command.split())
|
||||
except Exception as e:
|
||||
print(e)
|
||||
return Result(e, ResultType.failure, inspect.currentframe().f_code.co_name)
|
||||
else:
|
||||
return Result("", ResultType.success)
|
||||
return distro_name
|
||||
|
|
@ -3,7 +3,7 @@ import subprocess as sp
|
|||
import os
|
||||
import shutil
|
||||
|
||||
from app.helper import clone_common, clone_etcd_wrapper
|
||||
from app.helper import clone_common, clone_etcd_wrapper, install_available
|
||||
|
||||
|
||||
@click.group()
|
||||
|
|
@ -49,6 +49,8 @@ def setup(path, ssh_username, ssh_key_path, ssh_key_pass, etcd_url, without_ceph
|
|||
dst=os.path.join(repo_name, "ucloud_common", "etcd3_wrapper"),
|
||||
)
|
||||
|
||||
install_available(os.path.join(repo_name, "Pipfile"))
|
||||
|
||||
# Create virtualenv with site-packages enabled and install all dependencies
|
||||
sp.check_output(['pipenv','--site-packages', '--python', '3'], cwd=repo_name)
|
||||
sp.check_output(['pipenv', 'install'], cwd=repo_name)
|
||||
|
|
@ -2,7 +2,7 @@ import click
|
|||
import subprocess as sp
|
||||
import os
|
||||
|
||||
from app.helper import clone_etcd_wrapper
|
||||
from app.helper import clone_etcd_wrapper, install_available
|
||||
|
||||
@click.group()
|
||||
def image():
|
||||
|
|
@ -35,6 +35,8 @@ def setup(path, base_dir, etcd_url, without_ceph):
|
|||
|
||||
clone_etcd_wrapper(path=repo_name)
|
||||
|
||||
install_available(os.path.join(repo_name, "Pipfile"))
|
||||
|
||||
# Create virtualenv with site-packages enabled and install all dependencies
|
||||
sp.check_output(['pipenv','--site-packages', '--python', '3'], cwd=repo_name)
|
||||
sp.check_output(['pipenv', 'install'], cwd=repo_name)
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import subprocess as sp
|
|||
import os
|
||||
import shutil
|
||||
|
||||
from app.helper import clone_common, clone_etcd_wrapper
|
||||
from app.helper import clone_common, clone_etcd_wrapper, install_available
|
||||
|
||||
|
||||
@click.group()
|
||||
|
|
@ -47,6 +47,8 @@ def setup(path, vm_prefix, host_prefix, request_prefix, etcd_url):
|
|||
dst=os.path.join(repo_name, "ucloud_common", "etcd3_wrapper"),
|
||||
)
|
||||
|
||||
install_available(os.path.join(repo_name, "Pipfile"))
|
||||
|
||||
# Create virtualenv with site-packages enabled and install all dependencies
|
||||
sp.check_output(['pipenv','--site-packages', '--python', '3'], cwd=repo_name)
|
||||
sp.check_output(['pipenv', 'install'], cwd=repo_name)
|
||||
Loading…
Reference in a new issue