uncloud
/
ucloud-setup
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

A

This commit is contained in:
ahmadbilalkhalid 2019-09-07 13:10:28 +05:00
parent 23b61a7442
commit d8bbf22c79
7 changed files with 162 additions and 225 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
app/api.py
View File

@ -1,19 +1,9 @@
import click import click
import subprocess import subprocess as sp
import os import os
import shutil import shutil
from app.helper import ( from app.helper import clone_common, clone_etcd_wrapper
clone,
clone_common,
clone_etcd_wrapper,
GitOperation,
PipenvOperation,
FileOperation,
PackageManagementOperation,
VirtualenvOperation
)
@click.group() @click.group()
def api(): def api():
@ -27,58 +17,43 @@ def api():
@click.option("--auth_realm", required=True) @click.option("--auth_realm", required=True)
@click.option("--realm_allowed", multiple=True, required=True) @click.option("--realm_allowed", multiple=True, required=True)
@click.option("--etcd_url", required=True) @click.option("--etcd_url", required=True)
@click.option("--etcd_password", required=True) @click.option("--otp_server", default="https://otp.ungleich.ch/ungleichotp/",
@click.option( help="URL of ungleich OTP server")
"--otp_server", def setup(path, auth_name, auth_seed, auth_realm,
default="https://otp.ungleich.ch/ungleichotp/", realm_allowed, otp_server, etcd_url):
help="URL of ungleich OTP server")
def setup(
path,
auth_name,
auth_seed,
auth_realm,
realm_allowed,
otp_server,
etcd_url,
etcd_password,
):
os.chdir(path) os.chdir(path)
repo_name = "ucloud-api" repo_name = "ucloud-api"
# Clone main repo
sp.check_output(['git', 'clone',
f'https://code.ungleich.ch/ungleich-public/{repo_name}.git'
'-b', 'wip'])
# Git Operation Depends on success of previous operation i.e PackageManagementOperation # Create .env file
op = GitOperation.clone( with open(os.path.join(repo_name, ".env"), "w") as f:
url=f"https://code.ungleich.ch/ungleich-public/{repo_name}.git", content = (
arguments="-b wip" f"AUTH_NAME={auth_name}\n"
) f"AUTH_SEED={auth_seed}\n"
f"AUTH_REALM={auth_realm}\n"
f"REALM_ALLOWED={list(realm_allowed)}\n"
f"OTP_SERVER={otp_server}\n"
f"ETCD_URL={etcd_url}\n"
)
f.writelines(content)
# Clone Common and Etcd Wrapper
clone_common()
clone_etcd_wrapper()
content = ( # Copy Etcd Wrapper inside ucloud_common as ucloud_common
f"AUTH_NAME={auth_name}\n" # also needs etcd wrapper
f"AUTH_SEED={auth_seed}\n"
f"AUTH_REALM={auth_realm}\n"
f"REALM_ALLOWED={list(realm_allowed)}\n"
f"OTP_SERVER={otp_server}\n"
f"ETCD_URL={etcd_url}\n"
f"ETCD_PASSWORD={etcd_password}\n"
)
# FileOperation depends on success of previos operation i.e GitOperation.clone
op.add(FileOperation.write, path=os.path.join(repo_name, ".env"), content=content)
op.add(
GitOperation.clone,
path=repo_name,
url="https://code.ungleich.ch/ungleich-public/ucloud_common",
)
op.add(
GitOperation.clone,
path=repo_name,
url="https://code.ungleich.ch/ahmedbilal/etcd3_wrapper",
)
shutil.copytree( shutil.copytree(
src=os.path.join(repo_name, "etcd3_wrapper"), src=os.path.join(repo_name, "etcd3_wrapper"),
dst=os.path.join(repo_name, "ucloud_common", "etcd3_wrapper"), dst=os.path.join(repo_name, "ucloud_common", "etcd3_wrapper"),
) )
op.add(PipenvOperation.create, path=repo_name, site_packages=True)
op.add(PipenvOperation.install, path=repo_name) # Create virtualenv with site-packages enabled and install all dependencies
sp.check_output(['pipenv','--site-packages', '--python', '3'], path=repo_name)
sp.check_output(['pipenv', 'install'], path=repo_name)

59
app/file.py
View File

@ -1,16 +1,8 @@
import click import click
import subprocess import subprocess as sp
import os import os
from app.helper import ( from app.helper import clone_etcd_wrapper
clone,
clone_common,
clone_etcd_wrapper,
GitOperation,
PipenvOperation,
FileOperation,
)
@click.group() @click.group()
def file_scan(): def file_scan():
@ -22,37 +14,32 @@ def file_scan():
@click.option("--base_dir", required=True) @click.option("--base_dir", required=True)
@click.option("--file_prefix", required=True) @click.option("--file_prefix", required=True)
@click.option("--etcd_url", required=True) @click.option("--etcd_url", required=True)
@click.option("--etcd_password", required=True) def setup(path, base_dir, file_prefix, etcd_url):
def setup(path, base_dir, file_prefix, etcd_url, etcd_password):
os.chdir(path) os.chdir(path)
repo_name = "ucloud-file-scan" repo_name = "ucloud-file-scan"
op_result = GitOperation.clone( # Clone main repository
f"https://code.ungleich.ch/ungleich-public/{repo_name}.git", sp.check_output(['git', 'clone',
arguments="-b wip" f'https://code.ungleich.ch/ungleich-public/{repo_name}.git'
) '-b', 'wip'])
content = (
f"BASE_DIR={base_dir}\n"
f"FILE_PREFIX={file_prefix}\n"
f"ETCD_URL={etcd_url}\n"
f"ETCD_PASSWORD={etcd_password}\n"
)
op_result.add(
FileOperation.write, path=os.path.join(repo_name, ".env"), content=content
)
op_result.add(
GitOperation.clone,
path=repo_name,
url="https://code.ungleich.ch/ahmedbilal/etcd3_wrapper",
)
op_result.add(PipenvOperation.create, path=repo_name, site_packages=True)
op_result.add(PipenvOperation.install, path=repo_name)
# Create .env file
with open(os.path.join(repo_name, ".env"), "w") as f:
content = (
f"BASE_DIR={base_dir}\n"
f"FILE_PREFIX={file_prefix}\n"
f"ETCD_URL={etcd_url}\n"
)
f.writelines(content)
# Clone etcd wrapper
clone_etcd_wrapper()
# Create virtualenv with site-packages enabled and install all dependencies
sp.check_output(['pipenv','--site-packages', '--python', '3'], path=repo_name)
sp.check_output(['pipenv', 'install'], path=repo_name)
# Write Crontab entry # Write Crontab entry
with open("/etc/crontabs/root", "a") as crontab: with open("/etc/crontabs/root", "a") as crontab:
crontab.write( crontab.write(

11
app/helper.py
View File

@ -1,4 +1,5 @@
import subprocess import subprocess
import subprocess as sp
import inspect import inspect
import os import os
@ -23,12 +24,14 @@ def clone(repo):
return True return True
def clone_common(): def clone_common(path='.'):
return clone("https://code.ungleich.ch/ungleich-public/ucloud_common") sp.check_output(['git', 'clone',
f'https://code.ungleich.ch/ungleich-public/ucloud_common'], path=path)
def clone_etcd_wrapper(): def clone_etcd_wrapper(path='.'):
return clone("https://code.ungleich.ch/ahmedbilal/etcd3_wrapper") sp.check_output(['git', 'clone',
f'https://code.ungleich.ch/ahmedbilal/etcd3_wrapper'], path=path)
class Result(object): class Result(object):

65
app/host.py
View File

@ -1,16 +1,9 @@
import click import click
import subprocess import subprocess as sp
import os import os
import shutil import shutil
from app.helper import ( from app.helper import clone_common, clone_etcd_wrapper
clone,
clone_common,
clone_etcd_wrapper,
GitOperation,
PipenvOperation,
FileOperation,
)
@click.group() @click.group()
@ -24,45 +17,37 @@ def host():
@click.option("--ssh_key_path", required=True, help="For Example, ~/.ssh/id_rsa") @click.option("--ssh_key_path", required=True, help="For Example, ~/.ssh/id_rsa")
@click.option("--ssh_key_pass", required=True) @click.option("--ssh_key_pass", required=True)
@click.option("--etcd_url", required=True) @click.option("--etcd_url", required=True)
@click.option("--etcd_password", required=True) def setup(path, ssh_username, ssh_key_path, ssh_key_pass, etcd_url):
def setup(path, ssh_username, ssh_key_path, ssh_key_pass, etcd_url, etcd_password):
os.chdir(path) os.chdir(path)
repo_name = "ucloud-vm" repo_name = "ucloud-vm"
# Clone main repo
sp.check_output(['git', 'clone',
f'https://code.ungleich.ch/ungleich-public/{repo_name}.git'
'-b', 'wip'])
op_result = GitOperation.clone( # Create .env file
f"https://code.ungleich.ch/ungleich-public/{repo_name}.git", with open(os.path.join(repo_name, ".env"), "w") as f:
arguments="-b wip" content = (
) f"ssh_username={ssh_username}\n"
f"ssh_pkey={ssh_key_path}\n"
f"ssh_private_key_password={ssh_key_pass}\n"
f"ETCD_URL={etcd_url}\n"
)
f.writelines(content)
content = ( # Clone Common and Etcd Wrapper
f"ssh_username={ssh_username}\n" clone_common()
f"ssh_pkey={ssh_key_path}\n" clone_etcd_wrapper()
f"ssh_private_key_password={ssh_key_pass}\n"
f"ETCD_URL={etcd_url}\n"
f"ETCD_PASSWORD={etcd_password}\n"
)
op_result.add(
FileOperation.write, path=os.path.join(repo_name, ".env"), content=content
)
op_result.add(
GitOperation.clone,
path=repo_name,
url="https://code.ungleich.ch/ungleich-public/ucloud_common",
)
op_result.add(
GitOperation.clone,
path=repo_name,
url="https://code.ungleich.ch/ahmedbilal/etcd3_wrapper",
)
# Copy Etcd Wrapper inside ucloud_common as ucloud_common
# also needs etcd wrapper
shutil.copytree( shutil.copytree(
src=os.path.join(repo_name, "etcd3_wrapper"), src=os.path.join(repo_name, "etcd3_wrapper"),
dst=os.path.join(repo_name, "ucloud_common", "etcd3_wrapper"), dst=os.path.join(repo_name, "ucloud_common", "etcd3_wrapper"),
) )
op_result.add(PipenvOperation.create, path=repo_name, site_packages=True) # Create virtualenv with site-packages enabled and install all dependencies
op_result.add(PipenvOperation.install, path=repo_name) sp.check_output(['pipenv','--site-packages', '--python', '3'], path=repo_name)
sp.check_output(['pipenv', 'install'], path=repo_name)

55
app/image.py
View File

@ -1,16 +1,8 @@
import click import click
import subprocess import subprocess as sp
import os import os
from app.helper import ( from app.helper import clone_etcd_wrapper
clone,
clone_common,
clone_etcd_wrapper,
GitOperation,
PipenvOperation,
FileOperation,
)
@click.group() @click.group()
def image(): def image():
@ -21,35 +13,30 @@ def image():
@click.option("--path", required=True) @click.option("--path", required=True)
@click.option("--base_dir", required=True) @click.option("--base_dir", required=True)
@click.option("--etcd_url", required=True) @click.option("--etcd_url", required=True)
@click.option("--etcd_password", required=True) def setup(path, base_dir, etcd_url):
def setup(path, base_dir, etcd_url, etcd_password):
os.chdir(path) os.chdir(path)
repo_name = "ucloud-image-scanner" repo_name = "ucloud-image-scanner"
op_result = GitOperation.clone( # Clone main repo
f"https://code.ungleich.ch/ungleich-public/{repo_name}.git", sp.check_output(['git', 'clone',
arguments="-b wip" f'https://code.ungleich.ch/ungleich-public/{repo_name}.git'
) '-b', 'wip'])
# Create .env file
with open(os.path.join(repo_name, ".env"), "w") as f:
content = (
f"BASE_DIR={base_dir}\n"
f"ETCD_URL={etcd_url}\n"
)
f.writelines(content)
content = ( clone_etcd_wrapper()
f"BASE_DIR={base_dir}\n"
f"ETCD_URL={etcd_url}\n" # Create virtualenv with site-packages enabled and install all dependencies
f"ETCD_PASSWORD={etcd_password}\n" sp.check_output(['pipenv','--site-packages', '--python', '3'], path=repo_name)
) sp.check_output(['pipenv', 'install'], path=repo_name)
op_result.add(
FileOperation.write, path=os.path.join(repo_name, ".env"), content=content
)
op_result.add(
GitOperation.clone,
path=repo_name,
url="https://code.ungleich.ch/ahmedbilal/etcd3_wrapper",
)
op_result.add(PipenvOperation.create, path=repo_name, site_packages=True)
op_result.add(PipenvOperation.install, path=repo_name)
# Write Crontab entry # Write Crontab entry
with open("/etc/crontabs/root", "a") as crontab: with open("/etc/crontabs/root", "a") as crontab:

65
app/scheduler.py
View File

@ -1,16 +1,9 @@
import click import click
import subprocess import subprocess as sp
import os import os
import shutil import shutil
from app.helper import ( from app.helper import clone_common, clone_etcd_wrapper
clone,
clone_common,
clone_etcd_wrapper,
GitOperation,
PipenvOperation,
FileOperation,
)
@click.group() @click.group()
@ -24,45 +17,37 @@ def scheduler():
@click.option("--host_prefix", required=True) @click.option("--host_prefix", required=True)
@click.option("--request_prefix", required=True) @click.option("--request_prefix", required=True)
@click.option("--etcd_url", required=True) @click.option("--etcd_url", required=True)
@click.option("--etcd_password", required=True) def setup(path, vm_prefix, host_prefix, request_prefix, etcd_url):
def setup(path, vm_prefix, host_prefix, request_prefix, etcd_url, etcd_password):
os.chdir(path) os.chdir(path)
repo_name = "ucloud-scheduler" repo_name = "ucloud-scheduler"
op_result = GitOperation.clone( # Clone main repo
f"https://code.ungleich.ch/ungleich-public/{repo_name}.git", sp.check_output(['git', 'clone',
arguments="-b wip" f'https://code.ungleich.ch/ungleich-public/{repo_name}.git'
) '-b', 'wip'])
content = ( # Create .env file
f"VM_PREFIX={vm_prefix}\n" with open(os.path.join(repo_name, ".env"), "w") as f:
f"HOST_PREFIX={host_prefix}\n" content = (
f"REQUEST_PREFIX={request_prefix}\n" f"VM_PREFIX={vm_prefix}\n"
f"ETCD_URL={etcd_url}\n" f"HOST_PREFIX={host_prefix}\n"
f"ETCD_PASSWORD={etcd_password}\n" f"REQUEST_PREFIX={request_prefix}\n"
) f"ETCD_URL={etcd_url}\n"
)
f.writelines(content)
op_result.add( # Clone Common and Etcd Wrapper
FileOperation.write, path=os.path.join(repo_name, ".env"), content=content clone_common()
) clone_etcd_wrapper()
op_result.add(
GitOperation.clone,
path=repo_name,
url="https://code.ungleich.ch/ungleich-public/ucloud_common",
)
op_result.add(
GitOperation.clone,
path=repo_name,
url="https://code.ungleich.ch/ahmedbilal/etcd3_wrapper",
)
# Copy Etcd Wrapper inside ucloud_common as ucloud_common
# also needs etcd wrapper
shutil.copytree( shutil.copytree(
src=os.path.join(repo_name, "etcd3_wrapper"), src=os.path.join(repo_name, "etcd3_wrapper"),
dst=os.path.join(repo_name, "ucloud_common", "etcd3_wrapper"), dst=os.path.join(repo_name, "ucloud_common", "etcd3_wrapper"),
) )
op_result.add(PipenvOperation.create, path=repo_name, site_packages=True) # Create virtualenv with site-packages enabled and install all dependencies
op_result.add(PipenvOperation.install, path=repo_name) sp.check_output(['pipenv','--site-packages', '--python', '3'], path=repo_name)
sp.check_output(['pipenv', 'install'], path=repo_name)

41
init.sh
View File

@ -1,10 +1,17 @@
# Install QEMU get_distro() {
(git clone https://github.com/ahmedbilal/qemu-with-rbd-alpine.git \ OS=$(cat /etc/*release | grep ID | head -1 | cut -c 4-)
&& cd qemu-with-rbd-alpine && apk add *.apk --allow-untrusted) echo $OS
}
case $(get_distro) in
"alpine")
# Install QEMU
(git clone https://github.com/ahmedbilal/qemu-with-rbd-alpine.git \
&& cd qemu-with-rbd-alpine && apk add *.apk --allow-untrusted)
# Enable Alpine (3.10 + Edge) Repos # Enable Alpine (3.10 + Edge) Repos
cat > /etc/apk/repositories << EOF cat > /etc/apk/repositories << EOF
http://dl-cdn.alpinelinux.org/alpine/v3.10/main http://dl-cdn.alpinelinux.org/alpine/v3.10/main
http://dl-cdn.alpinelinux.org/alpine/v3.10/community http://dl-cdn.alpinelinux.org/alpine/v3.10/community
http://dl-cdn.alpinelinux.org/alpine/edge/main http://dl-cdn.alpinelinux.org/alpine/edge/main
@ -13,17 +20,25 @@ http://dl-cdn.alpinelinux.org/alpine/edge/testing
EOF EOF
# Update Package List and Upgrade System # Update Package List and Upgrade System
apk update apk update
apk upgrade apk upgrade
# Install system packages # Install system packages
apk add python3 ceph py2-pip py3-pip etcd-ctl apk add python3 ceph py2-pip py3-pip etcd-ctl
# Some python package dependencies # Some python package dependencies
apk add libffi-dev openssl-dev make alpine-sdk gcc g++ python3-dev apk add libffi-dev openssl-dev make alpine-sdk gcc g++ python3-dev
apk add py3-grpcio py3-protobuf py3-tempita apk add py3-grpcio py3-protobuf py3-tempita
;;
"devuan")
apt update
apt upgrade
apt install python3 qemu
esac
pip3 install --upgrade pip pip3 install --upgrade pip
pip2 install --upgrade pip pip2 install --upgrade pip