From d8bbf22c790521baa6dffa99ceb93ba28b5b0764 Mon Sep 17 00:00:00 2001 From: Ahmed Bilal Khalid Date: Sat, 7 Sep 2019 13:10:28 +0500 Subject: [PATCH] A --- app/api.py | 91 ++++++++++++++++++------------------------------ app/file.py | 59 ++++++++++++------------------- app/helper.py | 11 +++--- app/host.py | 65 +++++++++++++--------------------- app/image.py | 55 +++++++++++------------------ app/scheduler.py | 65 +++++++++++++--------------------- init.sh | 41 +++++++++++++++------- 7 files changed, 162 insertions(+), 225 deletions(-) diff --git a/app/api.py b/app/api.py index 1a75541..f6938a0 100644 --- a/app/api.py +++ b/app/api.py @@ -1,19 +1,9 @@ import click -import subprocess +import subprocess as sp import os import shutil -from app.helper import ( - clone, - clone_common, - clone_etcd_wrapper, - GitOperation, - PipenvOperation, - FileOperation, - PackageManagementOperation, - VirtualenvOperation -) - +from app.helper import clone_common, clone_etcd_wrapper @click.group() def api(): @@ -27,58 +17,43 @@ def api(): @click.option("--auth_realm", required=True) @click.option("--realm_allowed", multiple=True, required=True) @click.option("--etcd_url", required=True) -@click.option("--etcd_password", required=True) -@click.option( - "--otp_server", - default="https://otp.ungleich.ch/ungleichotp/", - help="URL of ungleich OTP server") -def setup( - path, - auth_name, - auth_seed, - auth_realm, - realm_allowed, - otp_server, - etcd_url, - etcd_password, -): +@click.option("--otp_server", default="https://otp.ungleich.ch/ungleichotp/", + help="URL of ungleich OTP server") +def setup(path, auth_name, auth_seed, auth_realm, + realm_allowed, otp_server, etcd_url): + os.chdir(path) repo_name = "ucloud-api" + + # Clone main repo + sp.check_output(['git', 'clone', + f'https://code.ungleich.ch/ungleich-public/{repo_name}.git' + '-b', 'wip']) - # Git Operation Depends on success of previous operation i.e PackageManagementOperation - op = GitOperation.clone( - url=f"https://code.ungleich.ch/ungleich-public/{repo_name}.git", - arguments="-b wip" - ) + # Create .env file + with open(os.path.join(repo_name, ".env"), "w") as f: + content = ( + f"AUTH_NAME={auth_name}\n" + f"AUTH_SEED={auth_seed}\n" + f"AUTH_REALM={auth_realm}\n" + f"REALM_ALLOWED={list(realm_allowed)}\n" + f"OTP_SERVER={otp_server}\n" + f"ETCD_URL={etcd_url}\n" + ) + f.writelines(content) + + # Clone Common and Etcd Wrapper + clone_common() + clone_etcd_wrapper() - content = ( - f"AUTH_NAME={auth_name}\n" - f"AUTH_SEED={auth_seed}\n" - f"AUTH_REALM={auth_realm}\n" - f"REALM_ALLOWED={list(realm_allowed)}\n" - f"OTP_SERVER={otp_server}\n" - f"ETCD_URL={etcd_url}\n" - f"ETCD_PASSWORD={etcd_password}\n" - ) - - # FileOperation depends on success of previos operation i.e GitOperation.clone - op.add(FileOperation.write, path=os.path.join(repo_name, ".env"), content=content) - - op.add( - GitOperation.clone, - path=repo_name, - url="https://code.ungleich.ch/ungleich-public/ucloud_common", - ) - - op.add( - GitOperation.clone, - path=repo_name, - url="https://code.ungleich.ch/ahmedbilal/etcd3_wrapper", - ) + # Copy Etcd Wrapper inside ucloud_common as ucloud_common + # also needs etcd wrapper shutil.copytree( src=os.path.join(repo_name, "etcd3_wrapper"), dst=os.path.join(repo_name, "ucloud_common", "etcd3_wrapper"), ) - op.add(PipenvOperation.create, path=repo_name, site_packages=True) - op.add(PipenvOperation.install, path=repo_name) + + # Create virtualenv with site-packages enabled and install all dependencies + sp.check_output(['pipenv','--site-packages', '--python', '3'], path=repo_name) + sp.check_output(['pipenv', 'install'], path=repo_name) diff --git a/app/file.py b/app/file.py index 042d86b..1a3ec17 100644 --- a/app/file.py +++ b/app/file.py @@ -1,16 +1,8 @@ import click -import subprocess +import subprocess as sp import os -from app.helper import ( - clone, - clone_common, - clone_etcd_wrapper, - GitOperation, - PipenvOperation, - FileOperation, -) - +from app.helper import clone_etcd_wrapper @click.group() def file_scan(): @@ -22,37 +14,32 @@ def file_scan(): @click.option("--base_dir", required=True) @click.option("--file_prefix", required=True) @click.option("--etcd_url", required=True) -@click.option("--etcd_password", required=True) -def setup(path, base_dir, file_prefix, etcd_url, etcd_password): +def setup(path, base_dir, file_prefix, etcd_url): os.chdir(path) repo_name = "ucloud-file-scan" - op_result = GitOperation.clone( - f"https://code.ungleich.ch/ungleich-public/{repo_name}.git", - arguments="-b wip" - ) - - content = ( - f"BASE_DIR={base_dir}\n" - f"FILE_PREFIX={file_prefix}\n" - f"ETCD_URL={etcd_url}\n" - f"ETCD_PASSWORD={etcd_password}\n" - ) - - op_result.add( - FileOperation.write, path=os.path.join(repo_name, ".env"), content=content - ) - - op_result.add( - GitOperation.clone, - path=repo_name, - url="https://code.ungleich.ch/ahmedbilal/etcd3_wrapper", - ) - - op_result.add(PipenvOperation.create, path=repo_name, site_packages=True) - op_result.add(PipenvOperation.install, path=repo_name) + # Clone main repository + sp.check_output(['git', 'clone', + f'https://code.ungleich.ch/ungleich-public/{repo_name}.git' + '-b', 'wip']) + # Create .env file + with open(os.path.join(repo_name, ".env"), "w") as f: + content = ( + f"BASE_DIR={base_dir}\n" + f"FILE_PREFIX={file_prefix}\n" + f"ETCD_URL={etcd_url}\n" + ) + f.writelines(content) + + # Clone etcd wrapper + clone_etcd_wrapper() + + # Create virtualenv with site-packages enabled and install all dependencies + sp.check_output(['pipenv','--site-packages', '--python', '3'], path=repo_name) + sp.check_output(['pipenv', 'install'], path=repo_name) + # Write Crontab entry with open("/etc/crontabs/root", "a") as crontab: crontab.write( diff --git a/app/helper.py b/app/helper.py index 0a6e730..b6ef7e8 100644 --- a/app/helper.py +++ b/app/helper.py @@ -1,4 +1,5 @@ import subprocess +import subprocess as sp import inspect import os @@ -23,12 +24,14 @@ def clone(repo): return True -def clone_common(): - return clone("https://code.ungleich.ch/ungleich-public/ucloud_common") +def clone_common(path='.'): + sp.check_output(['git', 'clone', + f'https://code.ungleich.ch/ungleich-public/ucloud_common'], path=path) -def clone_etcd_wrapper(): - return clone("https://code.ungleich.ch/ahmedbilal/etcd3_wrapper") +def clone_etcd_wrapper(path='.'): + sp.check_output(['git', 'clone', + f'https://code.ungleich.ch/ahmedbilal/etcd3_wrapper'], path=path) class Result(object): diff --git a/app/host.py b/app/host.py index 205d0a5..c1455ed 100644 --- a/app/host.py +++ b/app/host.py @@ -1,16 +1,9 @@ import click -import subprocess +import subprocess as sp import os import shutil -from app.helper import ( - clone, - clone_common, - clone_etcd_wrapper, - GitOperation, - PipenvOperation, - FileOperation, -) +from app.helper import clone_common, clone_etcd_wrapper @click.group() @@ -24,45 +17,37 @@ def host(): @click.option("--ssh_key_path", required=True, help="For Example, ~/.ssh/id_rsa") @click.option("--ssh_key_pass", required=True) @click.option("--etcd_url", required=True) -@click.option("--etcd_password", required=True) -def setup(path, ssh_username, ssh_key_path, ssh_key_pass, etcd_url, etcd_password): +def setup(path, ssh_username, ssh_key_path, ssh_key_pass, etcd_url): os.chdir(path) repo_name = "ucloud-vm" + + # Clone main repo + sp.check_output(['git', 'clone', + f'https://code.ungleich.ch/ungleich-public/{repo_name}.git' + '-b', 'wip']) - op_result = GitOperation.clone( - f"https://code.ungleich.ch/ungleich-public/{repo_name}.git", - arguments="-b wip" - ) + # Create .env file + with open(os.path.join(repo_name, ".env"), "w") as f: + content = ( + f"ssh_username={ssh_username}\n" + f"ssh_pkey={ssh_key_path}\n" + f"ssh_private_key_password={ssh_key_pass}\n" + f"ETCD_URL={etcd_url}\n" + ) + f.writelines(content) - content = ( - f"ssh_username={ssh_username}\n" - f"ssh_pkey={ssh_key_path}\n" - f"ssh_private_key_password={ssh_key_pass}\n" - f"ETCD_URL={etcd_url}\n" - f"ETCD_PASSWORD={etcd_password}\n" - ) - - op_result.add( - FileOperation.write, path=os.path.join(repo_name, ".env"), content=content - ) - - op_result.add( - GitOperation.clone, - path=repo_name, - url="https://code.ungleich.ch/ungleich-public/ucloud_common", - ) - - op_result.add( - GitOperation.clone, - path=repo_name, - url="https://code.ungleich.ch/ahmedbilal/etcd3_wrapper", - ) + # Clone Common and Etcd Wrapper + clone_common() + clone_etcd_wrapper() + # Copy Etcd Wrapper inside ucloud_common as ucloud_common + # also needs etcd wrapper shutil.copytree( src=os.path.join(repo_name, "etcd3_wrapper"), dst=os.path.join(repo_name, "ucloud_common", "etcd3_wrapper"), ) - op_result.add(PipenvOperation.create, path=repo_name, site_packages=True) - op_result.add(PipenvOperation.install, path=repo_name) \ No newline at end of file + # Create virtualenv with site-packages enabled and install all dependencies + sp.check_output(['pipenv','--site-packages', '--python', '3'], path=repo_name) + sp.check_output(['pipenv', 'install'], path=repo_name) \ No newline at end of file diff --git a/app/image.py b/app/image.py index b8c3ec8..8771617 100644 --- a/app/image.py +++ b/app/image.py @@ -1,16 +1,8 @@ import click -import subprocess +import subprocess as sp import os -from app.helper import ( - clone, - clone_common, - clone_etcd_wrapper, - GitOperation, - PipenvOperation, - FileOperation, -) - +from app.helper import clone_etcd_wrapper @click.group() def image(): @@ -21,35 +13,30 @@ def image(): @click.option("--path", required=True) @click.option("--base_dir", required=True) @click.option("--etcd_url", required=True) -@click.option("--etcd_password", required=True) -def setup(path, base_dir, etcd_url, etcd_password): +def setup(path, base_dir, etcd_url): os.chdir(path) repo_name = "ucloud-image-scanner" - op_result = GitOperation.clone( - f"https://code.ungleich.ch/ungleich-public/{repo_name}.git", - arguments="-b wip" - ) + # Clone main repo + sp.check_output(['git', 'clone', + f'https://code.ungleich.ch/ungleich-public/{repo_name}.git' + '-b', 'wip']) + + # Create .env file + with open(os.path.join(repo_name, ".env"), "w") as f: + content = ( + f"BASE_DIR={base_dir}\n" + f"ETCD_URL={etcd_url}\n" + ) + f.writelines(content) + - content = ( - f"BASE_DIR={base_dir}\n" - f"ETCD_URL={etcd_url}\n" - f"ETCD_PASSWORD={etcd_password}\n" - ) - - op_result.add( - FileOperation.write, path=os.path.join(repo_name, ".env"), content=content - ) - - op_result.add( - GitOperation.clone, - path=repo_name, - url="https://code.ungleich.ch/ahmedbilal/etcd3_wrapper", - ) - - op_result.add(PipenvOperation.create, path=repo_name, site_packages=True) - op_result.add(PipenvOperation.install, path=repo_name) + clone_etcd_wrapper() + + # Create virtualenv with site-packages enabled and install all dependencies + sp.check_output(['pipenv','--site-packages', '--python', '3'], path=repo_name) + sp.check_output(['pipenv', 'install'], path=repo_name) # Write Crontab entry with open("/etc/crontabs/root", "a") as crontab: diff --git a/app/scheduler.py b/app/scheduler.py index 131fbd9..801e364 100644 --- a/app/scheduler.py +++ b/app/scheduler.py @@ -1,16 +1,9 @@ import click -import subprocess +import subprocess as sp import os import shutil -from app.helper import ( - clone, - clone_common, - clone_etcd_wrapper, - GitOperation, - PipenvOperation, - FileOperation, -) +from app.helper import clone_common, clone_etcd_wrapper @click.group() @@ -24,45 +17,37 @@ def scheduler(): @click.option("--host_prefix", required=True) @click.option("--request_prefix", required=True) @click.option("--etcd_url", required=True) -@click.option("--etcd_password", required=True) -def setup(path, vm_prefix, host_prefix, request_prefix, etcd_url, etcd_password): +def setup(path, vm_prefix, host_prefix, request_prefix, etcd_url): os.chdir(path) repo_name = "ucloud-scheduler" - op_result = GitOperation.clone( - f"https://code.ungleich.ch/ungleich-public/{repo_name}.git", - arguments="-b wip" - ) + # Clone main repo + sp.check_output(['git', 'clone', + f'https://code.ungleich.ch/ungleich-public/{repo_name}.git' + '-b', 'wip']) - content = ( - f"VM_PREFIX={vm_prefix}\n" - f"HOST_PREFIX={host_prefix}\n" - f"REQUEST_PREFIX={request_prefix}\n" - f"ETCD_URL={etcd_url}\n" - f"ETCD_PASSWORD={etcd_password}\n" - ) + # Create .env file + with open(os.path.join(repo_name, ".env"), "w") as f: + content = ( + f"VM_PREFIX={vm_prefix}\n" + f"HOST_PREFIX={host_prefix}\n" + f"REQUEST_PREFIX={request_prefix}\n" + f"ETCD_URL={etcd_url}\n" + ) + f.writelines(content) - op_result.add( - FileOperation.write, path=os.path.join(repo_name, ".env"), content=content - ) - - op_result.add( - GitOperation.clone, - path=repo_name, - url="https://code.ungleich.ch/ungleich-public/ucloud_common", - ) - - op_result.add( - GitOperation.clone, - path=repo_name, - url="https://code.ungleich.ch/ahmedbilal/etcd3_wrapper", - ) + # Clone Common and Etcd Wrapper + clone_common() + clone_etcd_wrapper() + # Copy Etcd Wrapper inside ucloud_common as ucloud_common + # also needs etcd wrapper shutil.copytree( src=os.path.join(repo_name, "etcd3_wrapper"), dst=os.path.join(repo_name, "ucloud_common", "etcd3_wrapper"), ) - - op_result.add(PipenvOperation.create, path=repo_name, site_packages=True) - op_result.add(PipenvOperation.install, path=repo_name) \ No newline at end of file + + # Create virtualenv with site-packages enabled and install all dependencies + sp.check_output(['pipenv','--site-packages', '--python', '3'], path=repo_name) + sp.check_output(['pipenv', 'install'], path=repo_name) \ No newline at end of file diff --git a/init.sh b/init.sh index 148e559..5352453 100644 --- a/init.sh +++ b/init.sh @@ -1,10 +1,17 @@ -# Install QEMU -(git clone https://github.com/ahmedbilal/qemu-with-rbd-alpine.git \ - && cd qemu-with-rbd-alpine && apk add *.apk --allow-untrusted) +get_distro() { + OS=$(cat /etc/*release | grep ID | head -1 | cut -c 4-) + echo $OS +} + +case $(get_distro) in + "alpine") + # Install QEMU + (git clone https://github.com/ahmedbilal/qemu-with-rbd-alpine.git \ + && cd qemu-with-rbd-alpine && apk add *.apk --allow-untrusted) -# Enable Alpine (3.10 + Edge) Repos -cat > /etc/apk/repositories << EOF + # Enable Alpine (3.10 + Edge) Repos + cat > /etc/apk/repositories << EOF http://dl-cdn.alpinelinux.org/alpine/v3.10/main http://dl-cdn.alpinelinux.org/alpine/v3.10/community http://dl-cdn.alpinelinux.org/alpine/edge/main @@ -13,17 +20,25 @@ http://dl-cdn.alpinelinux.org/alpine/edge/testing EOF -# Update Package List and Upgrade System -apk update -apk upgrade + # Update Package List and Upgrade System + apk update + apk upgrade -# Install system packages -apk add python3 ceph py2-pip py3-pip etcd-ctl + # Install system packages + apk add python3 ceph py2-pip py3-pip etcd-ctl -# Some python package dependencies -apk add libffi-dev openssl-dev make alpine-sdk gcc g++ python3-dev + # Some python package dependencies + apk add libffi-dev openssl-dev make alpine-sdk gcc g++ python3-dev -apk add py3-grpcio py3-protobuf py3-tempita + apk add py3-grpcio py3-protobuf py3-tempita + ;; +"devuan") + apt update + apt upgrade + + apt install python3 qemu + +esac pip3 install --upgrade pip pip2 install --upgrade pip