uncloud/ucloud-setup
2
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

a

Browse source
This commit is contained in:
ahmadbilalkhalid 2019-08-28 16:42:50 +05:00
parent 843949e7ca
commit e57772f19e
8 changed files with 301 additions and 119 deletions

1
Pipfile
View file

@ -4,6 +4,7 @@ url = "https://pypi.org/simple"
verify_ssl = true verify_ssl = true
[dev-packages] [dev-packages]
pylint = "*"
[packages] [packages]
click = "*" click = "*"

92
Pipfile.lock generated
View file

@ -1,7 +1,7 @@
{ {
"_meta": { "_meta": {
"hash": { "hash": {
"sha256": "e6728243c3982bd95c23ad63dc24514046979d9cf07efbb6dc60150bdf182788" "sha256": "41ecf42c7fc21789d3e75876efbebbb3b6f7b12e4d823c504444d912ec30edbf"
}, },
"pipfile-spec": 6, "pipfile-spec": 6,
"requires": { "requires": {
@ -25,5 +25,93 @@
"version": "==7.0" "version": "==7.0"
} }
}, },
"develop": {} "develop": {
"astroid": {
"hashes": [
"sha256:6560e1e1749f68c64a4b5dee4e091fce798d2f0d84ebe638cf0e0585a343acf4",
"sha256:b65db1bbaac9f9f4d190199bb8680af6f6f84fd3769a5ea883df8a91fe68b4c4"
],
"version": "==2.2.5"
},
"isort": {
"hashes": [
"sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1",
"sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd"
],
"version": "==4.3.21"
},
"lazy-object-proxy": {
"hashes": [
"sha256:02b260c8deb80db09325b99edf62ae344ce9bc64d68b7a634410b8e9a568edbf",
"sha256:18f9c401083a4ba6e162355873f906315332ea7035803d0fd8166051e3d402e3",
"sha256:1f2c6209a8917c525c1e2b55a716135ca4658a3042b5122d4e3413a4030c26ce",
"sha256:2f06d97f0ca0f414f6b707c974aaf8829c2292c1c497642f63824119d770226f",
"sha256:616c94f8176808f4018b39f9638080ed86f96b55370b5a9463b2ee5c926f6c5f",
"sha256:63b91e30ef47ef68a30f0c3c278fbfe9822319c15f34b7538a829515b84ca2a0",
"sha256:77b454f03860b844f758c5d5c6e5f18d27de899a3db367f4af06bec2e6013a8e",
"sha256:83fe27ba321e4cfac466178606147d3c0aa18e8087507caec78ed5a966a64905",
"sha256:84742532d39f72df959d237912344d8a1764c2d03fe58beba96a87bfa11a76d8",
"sha256:874ebf3caaf55a020aeb08acead813baf5a305927a71ce88c9377970fe7ad3c2",
"sha256:9f5caf2c7436d44f3cec97c2fa7791f8a675170badbfa86e1992ca1b84c37009",
"sha256:a0c8758d01fcdfe7ae8e4b4017b13552efa7f1197dd7358dc9da0576f9d0328a",
"sha256:a4def978d9d28cda2d960c279318d46b327632686d82b4917516c36d4c274512",
"sha256:ad4f4be843dace866af5fc142509e9b9817ca0c59342fdb176ab6ad552c927f5",
"sha256:ae33dd198f772f714420c5ab698ff05ff900150486c648d29951e9c70694338e",
"sha256:b4a2b782b8a8c5522ad35c93e04d60e2ba7f7dcb9271ec8e8c3e08239be6c7b4",
"sha256:c462eb33f6abca3b34cdedbe84d761f31a60b814e173b98ede3c81bb48967c4f",
"sha256:fd135b8d35dfdcdb984828c84d695937e58cc5f49e1c854eb311c4d6aa03f4f1"
],
"version": "==1.4.2"
},
"mccabe": {
"hashes": [
"sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42",
"sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"
],
"version": "==0.6.1"
},
"pylint": {
"hashes": [
"sha256:5d77031694a5fb97ea95e828c8d10fc770a1df6eb3906067aaed42201a8a6a09",
"sha256:723e3db49555abaf9bf79dc474c6b9e2935ad82230b10c1138a71ea41ac0fff1"
],
"index": "pypi",
"version": "==2.3.1"
},
"six": {
"hashes": [
"sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
"sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"
],
"version": "==1.12.0"
},
"typed-ast": {
"hashes": [
"sha256:18511a0b3e7922276346bcb47e2ef9f38fb90fd31cb9223eed42c85d1312344e",
"sha256:262c247a82d005e43b5b7f69aff746370538e176131c32dda9cb0f324d27141e",
"sha256:2b907eb046d049bcd9892e3076c7a6456c93a25bebfe554e931620c90e6a25b0",
"sha256:354c16e5babd09f5cb0ee000d54cfa38401d8b8891eefa878ac772f827181a3c",
"sha256:4e0b70c6fc4d010f8107726af5fd37921b666f5b31d9331f0bd24ad9a088e631",
"sha256:630968c5cdee51a11c05a30453f8cd65e0cc1d2ad0d9192819df9978984529f4",
"sha256:66480f95b8167c9c5c5c87f32cf437d585937970f3fc24386f313a4c97b44e34",
"sha256:71211d26ffd12d63a83e079ff258ac9d56a1376a25bc80b1cdcdf601b855b90b",
"sha256:95bd11af7eafc16e829af2d3df510cecfd4387f6453355188342c3e79a2ec87a",
"sha256:bc6c7d3fa1325a0c6613512a093bc2a2a15aeec350451cbdf9e1d4bffe3e3233",
"sha256:cc34a6f5b426748a507dd5d1de4c1978f2eb5626d51326e43280941206c209e1",
"sha256:d755f03c1e4a51e9b24d899561fec4ccaf51f210d52abdf8c07ee2849b212a36",
"sha256:d7c45933b1bdfaf9f36c579671fec15d25b06c8398f113dab64c18ed1adda01d",
"sha256:d896919306dd0aa22d0132f62a1b78d11aaf4c9fc5b3410d3c666b818191630a",
"sha256:ffde2fbfad571af120fcbfbbc61c72469e72f550d676c3342492a9dfdefb8f12"
],
"markers": "implementation_name == 'cpython'",
"version": "==1.4.0"
},
"wrapt": {
"hashes": [
"sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1",
"sha256:ff17f7c0a4746025a10c0de3d2fb570d8df50248081f88af0b35383b5862a326"
],
"version": "==1.11.2"
}
}
} }

106
app/api.py
View file

@ -2,52 +2,84 @@ import click
import subprocess import subprocess
import os import os
from app.helper import clone, clone_common,\ from app.helper import (
clone_etcd_wrapper, GitOperation,\ clone,
PipenvOperation, FileOperation,\ clone_common,
PackageManagementOperation clone_etcd_wrapper,
GitOperation,
PipenvOperation,
FileOperation,
PackageManagementOperation,
)
@click.group() @click.group()
def api(): def api():
pass pass
@api.command("setup") @api.command("setup")
@click.option("--path", required=True) @click.option("--path", required=True)
@click.option("--auth_name", required=True) @click.option("--auth_name", required=True)
@click.option("--auth_seed", required=True) @click.option("--auth_seed", required=True)
@click.option("--auth_realm", required=True) @click.option("--auth_realm", required=True)
@click.option("--realm_allowed", multiple=True, required=True) @click.option("--realm_allowed", multiple=True, required=True)
@click.option("--otp_server", default="https://otp.ungleich.ch/ungleichotp/", @click.option(
help="URL of ungleich OTP server") "--otp_server",
default="https://otp.ungleich.ch/ungleichotp/",
help="URL of ungleich OTP server",
)
def setup(path, auth_name, auth_seed, auth_realm, realm_allowed, otp_server):
os.chdir(path)
def setup(path, auth_name, auth_seed, auth_realm, # Install etcd
realm_allowed, otp_server): op = PackageManagementOperation.install("etcd")
os.chdir(path)
# Add --pidfile argument to supervise daemon
op.add(
# Install Package subprocess.check_output,
op = PackageManagementOperation.install("etcd") args=r"""sed -i -e 's/supervise_daemon_args="--chdir $ETCD_DATA_DIR"/supervise_daemon_args="--chdir $ETCD_DATA_DIR --pidfile /run/etcd.pid"/g' /etc/init.d/etcd""".split(),
)
# Git Operation Depends on success of previous operation i.e PackageManagementOperation
op.add(GitOperation.clone, # Change address of etcd
url="https://code.ungleich.ch/ungleich-public/ucloud-api.git", op.add(
path=path) subprocess.check_output,
args=r'sed -i -e "s/localhost/[2a0a:e5c0:0:2:0:b3ff:fe39:7994]/g" /etc/etcd/conf.yml'.split(),
content = f"AUTH_NAME={auth_name}\n" \ )
f"AUTH_SEED={auth_seed}\n" \
f"AUTH_REALM={auth_seed}\n" \ # Start and Enable etcd service
f"REALM_ALLOWED={list(realm_allowed)}\n" \ op.add(subprocess.check_output,
f"OTP_SERVER={otp_server}\n" args="service etcd start; rc-update add etcd".split())
# FileOperation depends on success of previos operation i.e GitOperation.clone # Git Operation Depends on success of previous operation i.e PackageManagementOperation
op.add(FileOperation.write, op.add(
path=os.path.join("ucloud-api", ".env"), GitOperation.clone,
content=content) url="https://code.ungleich.ch/ungleich-public/ucloud-api.git",
)
op.add(GitOperation.clone, path="ucloud-api",
url="https://code.ungleich.ch/ungleich-public/ucloud_common") content = (
f"AUTH_NAME={auth_name}\n"
op.add(GitOperation.clone, path="ucloud-api", f"AUTH_SEED={auth_seed}\n"
url="https://code.ungleich.ch/ahmedbilal/etcd3_wrapper") f"AUTH_REALM={auth_seed}\n"
f"REALM_ALLOWED={list(realm_allowed)}\n"
op.add(PipenvOperation.install, path="ucloud-api") f"OTP_SERVER={otp_server}\n"
)
# FileOperation depends on success of previos operation i.e GitOperation.clone
op.add(
FileOperation.write, path=os.path.join("ucloud-api", ".env"), content=content
)
op.add(
GitOperation.clone,
path="ucloud-api",
url="https://code.ungleich.ch/ungleich-public/ucloud_common",
)
op.add(
GitOperation.clone,
path="ucloud-api",
url="https://code.ungleich.ch/ahmedbilal/etcd3_wrapper",
)
op.add(PipenvOperation.install, path="ucloud-api")

42
app/file.py
View file

@ -2,34 +2,46 @@ import click
import subprocess import subprocess
import os import os
from app.helper import clone, clone_common,\ from app.helper import (
clone_etcd_wrapper, GitOperation, PipenvOperation, FileOperation clone,
clone_common,
clone_etcd_wrapper,
GitOperation,
PipenvOperation,
FileOperation,
)
@click.group() @click.group()
def file_scan(): def file_scan():
pass pass
@file_scan.command("setup") @file_scan.command("setup")
@click.option("--path", required=True) @click.option("--path", required=True)
@click.option("--base_dir", required=True) @click.option("--base_dir", required=True)
@click.option("--file_prefix", required=True) @click.option("--file_prefix", required=True)
def setup(path, base_dir, file_prefix): def setup(path, base_dir, file_prefix):
os.chdir(path) os.chdir(path)
repo_name = "ucloud-file-scan" repo_name = "ucloud-file-scan"
op_result = GitOperation.clone(f"https://code.ungleich.ch/ungleich-public/{repo_name}.git")
content = f"BASE_DIR={base_dir}\n" \ op_result = GitOperation.clone(
f"FILE_PREFIX={file_prefix}\n" f"https://code.ungleich.ch/ungleich-public/{repo_name}.git"
)
op_result.add(FileOperation.write,
path=os.path.join(repo_name, ".env"), content = f"BASE_DIR={base_dir}\n" f"FILE_PREFIX={file_prefix}\n"
content=content)
op_result.add(
FileOperation.write, path=os.path.join(repo_name, ".env"), content=content
)
op_result.add(
GitOperation.clone,
path=repo_name,
url="https://code.ungleich.ch/ahmedbilal/etcd3_wrapper",
)
op_result.add(GitOperation.clone, path=repo_name,
url="https://code.ungleich.ch/ahmedbilal/etcd3_wrapper")
op_result.add(PipenvOperation.install, path=repo_name) op_result.add(PipenvOperation.install, path=repo_name)
# Write Crontab entry # Write Crontab entry

21
app/helper.py
View file

@ -4,10 +4,12 @@ from dataclasses import dataclass
from abc import ABC from abc import ABC
from enum import Enum from enum import Enum
class ResultType(Enum): class ResultType(Enum):
success = 0 success = 0
failure = 1 failure = 1
def clone(repo): def clone(repo):
command = f"git clone {repo}" command = f"git clone {repo}"
try: try:
@ -16,14 +18,16 @@ def clone(repo):
return False return False
else: else:
return True return True
def clone_common(): def clone_common():
return clone("https://code.ungleich.ch/ungleich-public/ucloud_common") return clone("https://code.ungleich.ch/ungleich-public/ucloud_common")
def clone_etcd_wrapper(): def clone_etcd_wrapper():
return clone("https://code.ungleich.ch/ahmedbilal/etcd3_wrapper") return clone("https://code.ungleich.ch/ahmedbilal/etcd3_wrapper")
class Result(object): class Result(object):
def __init__(self, _result, _type: ResultType, _op=""): def __init__(self, _result, _type: ResultType, _op=""):
self._type = _type self._type = _type
@ -31,8 +35,7 @@ class Result(object):
self._op = _op self._op = _op
if self._type == ResultType.failure: if self._type == ResultType.failure:
print(self._op, "failed") print(self._op, "failed")
def add(self, operation, **kwargs): def add(self, operation, **kwargs):
if self._type == ResultType.success: if self._type == ResultType.success:
r = operation(**kwargs) r = operation(**kwargs)
@ -42,11 +45,11 @@ class Result(object):
else: else:
print("Dependency not satisfied") print("Dependency not satisfied")
exit(-1) exit(-1)
def __repr__(self): def __repr__(self):
return f"{self._type}, {self._result}" return f"{self._type}, {self._result}"
class Operation(ABC): class Operation(ABC):
pass pass
@ -58,10 +61,11 @@ class GitOperation(object):
try: try:
output = subprocess.check_output(command.split(), cwd=path) output = subprocess.check_output(command.split(), cwd=path)
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
return Result(e, ResultType.failure, inspect.currentframe().f_code.co_name) return Result(e, ResultType.failure, inspect.currentframe().f_code.co_name)
else: else:
return Result(output, ResultType.success) return Result(output, ResultType.success)
class PipenvOperation(object): class PipenvOperation(object):
@staticmethod @staticmethod
def install(path=".", package_name=None): def install(path=".", package_name=None):
@ -72,7 +76,7 @@ class PipenvOperation(object):
try: try:
output = subprocess.check_output(command.split(), cwd=path) output = subprocess.check_output(command.split(), cwd=path)
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
return Result(e, ResultType.failure, inspect.currentframe().f_code.co_name) return Result(e, ResultType.failure, inspect.currentframe().f_code.co_name)
else: else:
return Result(output, ResultType.success) return Result(output, ResultType.success)
@ -98,6 +102,7 @@ def get_distro_name():
distro_name = content[start:end] distro_name = content[start:end]
return distro_name return distro_name
class PackageManagementOperation(object): class PackageManagementOperation(object):
@staticmethod @staticmethod
def install(package_name): def install(package_name):
@ -107,7 +112,7 @@ class PackageManagementOperation(object):
command = f"apk add {package_name}" command = f"apk add {package_name}"
else: else:
assert "Unknown Distro" assert "Unknown Distro"
subprocess.check_output(command.split()) subprocess.check_output(command.split())
except Exception as e: except Exception as e:
print(e) print(e)

64
app/host.py
View file

@ -2,8 +2,15 @@ import click
import subprocess import subprocess
import os import os
from app.helper import clone, clone_common,\ from app.helper import (
clone_etcd_wrapper, GitOperation, PipenvOperation, FileOperation clone,
clone_common,
clone_etcd_wrapper,
GitOperation,
PipenvOperation,
FileOperation,
)
@click.group() @click.group()
def host(): def host():
@ -19,26 +26,35 @@ def host():
@click.option("--etcd_port", required=True) @click.option("--etcd_port", required=True)
def setup(path, ssh_username, ssh_key_path, ssh_key_pass, etcd_host, etcd_port): def setup(path, ssh_username, ssh_key_path, ssh_key_pass, etcd_host, etcd_port):
os.chdir(path) os.chdir(path)
repo_name = "ucloud-vm" repo_name = "ucloud-vm"
op_result = GitOperation.clone(f"https://code.ungleich.ch/ungleich-public/{repo_name}.git") op_result = GitOperation.clone(
f"https://code.ungleich.ch/ungleich-public/{repo_name}.git"
)
content = f"ssh_username={ssh_username}\n" \
f"ssh_pkey={ssh_key_path}\n" \ content = (
f"ssh_private_key_password={ssh_key_pass}\n" \ f"ssh_username={ssh_username}\n"
f"ETCD_HOST={etcd_host}\n" \ f"ssh_pkey={ssh_key_path}\n"
f"ETCD_PORT={etcd_port}\n" f"ssh_private_key_password={ssh_key_pass}\n"
f"ETCD_HOST={etcd_host}\n"
op_result.add(FileOperation.write, f"ETCD_PORT={etcd_port}\n"
path=os.path.join(repo_name, ".env"), )
content=content)
op_result.add(
op_result.add(GitOperation.clone, path=repo_name, FileOperation.write, path=os.path.join(repo_name, ".env"), content=content
url="https://code.ungleich.ch/ungleich-public/ucloud_common") )
op_result.add(GitOperation.clone, path=repo_name, op_result.add(
url="https://code.ungleich.ch/ahmedbilal/etcd3_wrapper") GitOperation.clone,
path=repo_name,
op_result.add(PipenvOperation.install, path=repo_name) url="https://code.ungleich.ch/ungleich-public/ucloud_common",
)
op_result.add(
GitOperation.clone,
path=repo_name,
url="https://code.ungleich.ch/ahmedbilal/etcd3_wrapper",
)
op_result.add(PipenvOperation.install, path=repo_name)

30
app/image.py
View file

@ -2,26 +2,38 @@ import click
import subprocess import subprocess
import os import os
from app.helper import clone, clone_common,\ from app.helper import (
clone_etcd_wrapper, GitOperation, PipenvOperation, FileOperation clone,
clone_common,
clone_etcd_wrapper,
GitOperation,
PipenvOperation,
FileOperation,
)
@click.group() @click.group()
def image(): def image():
pass pass
@image.command("setup") @image.command("setup")
@click.option("--path", required=True) @click.option("--path", required=True)
def setup(path): def setup(path):
os.chdir(path) os.chdir(path)
repo_name = "ucloud-image-scanner" repo_name = "ucloud-image-scanner"
op_result = GitOperation.clone(f"https://code.ungleich.ch/ungleich-public/{repo_name}.git")
op_result = GitOperation.clone(
f"https://code.ungleich.ch/ungleich-public/{repo_name}.git"
)
op_result.add(
GitOperation.clone,
path=repo_name,
url="https://code.ungleich.ch/ahmedbilal/etcd3_wrapper",
)
op_result.add(GitOperation.clone, path=repo_name,
url="https://code.ungleich.ch/ahmedbilal/etcd3_wrapper")
op_result.add(PipenvOperation.install, path=repo_name) op_result.add(PipenvOperation.install, path=repo_name)
# Write Crontab entry # Write Crontab entry

64
app/scheduler.py
View file

@ -2,8 +2,15 @@ import click
import subprocess import subprocess
import os import os
from app.helper import clone, clone_common,\ from app.helper import (
clone_etcd_wrapper, GitOperation, PipenvOperation, FileOperation clone,
clone_common,
clone_etcd_wrapper,
GitOperation,
PipenvOperation,
FileOperation,
)
@click.group() @click.group()
def scheduler(): def scheduler():
@ -17,28 +24,37 @@ def scheduler():
@click.option("--request_prefix", required=True) @click.option("--request_prefix", required=True)
@click.option("--etcd_host", required=True) @click.option("--etcd_host", required=True)
@click.option("--etcd_port", required=True) @click.option("--etcd_port", required=True)
def setup(path, vm_prefix, host_prefix, request_prefix, etcd_host, etcd_port): def setup(path, vm_prefix, host_prefix, request_prefix, etcd_host, etcd_port):
os.chdir(path) os.chdir(path)
repo_name = "ucloud-scheduler" repo_name = "ucloud-scheduler"
op_result = GitOperation.clone(f"https://code.ungleich.ch/ungleich-public/{repo_name}.git") op_result = GitOperation.clone(
f"https://code.ungleich.ch/ungleich-public/{repo_name}.git"
content = f"VM_PREFIX={vm_prefix}\n" \ )
f"HOST_PREFIX={host_prefix}\n" \
f"REQUEST_PREFIX={request_prefix}\n" \ content = (
f"ETCD_HOST={etcd_host}\n" \ f"VM_PREFIX={vm_prefix}\n"
f"ETCD_PORT={etcd_port}\n" f"HOST_PREFIX={host_prefix}\n"
f"REQUEST_PREFIX={request_prefix}\n"
op_result.add(FileOperation.write, f"ETCD_HOST={etcd_host}\n"
path=os.path.join(repo_name, ".env"), f"ETCD_PORT={etcd_port}\n"
content=content) )
op_result.add(GitOperation.clone, path=repo_name, op_result.add(
url="https://code.ungleich.ch/ungleich-public/ucloud_common") FileOperation.write, path=os.path.join(repo_name, ".env"), content=content
)
op_result.add(GitOperation.clone, path=repo_name,
url="https://code.ungleich.ch/ahmedbilal/etcd3_wrapper") op_result.add(
GitOperation.clone,
op_result.add(PipenvOperation.install, path=repo_name) path=repo_name,
url="https://code.ungleich.ch/ungleich-public/ucloud_common",
)
op_result.add(
GitOperation.clone,
path=repo_name,
url="https://code.ungleich.ch/ahmedbilal/etcd3_wrapper",
)
op_result.add(PipenvOperation.install, path=repo_name)