fnux/uncloud
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Shutdown Source VM (PAUSED) on successfull migration + blackened all .py files

Browse source
This commit is contained in:
ahmadbilalkhalid 2019-12-30 14:35:07 +05:00
commit 9bdf4d2180
31 changed files with 1307 additions and 638 deletions
Download patch file Download diff file Expand all files Collapse all files

46
ucloud/common/etcd_wrapper.py
View file

@ -8,7 +8,7 @@ from functools import wraps
from . import logger
PseudoEtcdMeta = namedtuple('PseudoEtcdMeta', ['key'])
PseudoEtcdMeta = namedtuple("PseudoEtcdMeta", ["key"])
class EtcdEntry:
@ -16,8 +16,8 @@ class EtcdEntry:
# value: str
def __init__(self, meta, value, value_in_json=False):
self.key = meta.key.decode('utf-8')
self.value = value.decode('utf-8')
self.key = meta.key.decode("utf-8")
self.value = value.decode("utf-8")
if value_in_json:
self.value = json.loads(self.value)
@ -29,11 +29,18 @@ def readable_errors(func):
try:
return func(*args, **kwargs)
except etcd3.exceptions.ConnectionFailedError as err:
raise etcd3.exceptions.ConnectionFailedError('etcd connection failed.') from err
raise etcd3.exceptions.ConnectionFailedError(
"etcd connection failed."
) from err
except etcd3.exceptions.ConnectionTimeoutError as err:
raise etcd3.exceptions.ConnectionTimeoutError('etcd connection timeout.') from err
raise etcd3.exceptions.ConnectionTimeoutError(
"etcd connection timeout."
) from err
except Exception:
logger.exception('Some etcd error occured. See syslog for details.')
logger.exception(
"Some etcd error occured. See syslog for details."
)
return wrapper
@ -56,7 +63,7 @@ class Etcd3Wrapper:
_value = json.dumps(_value)
if not isinstance(_key, str):
_key = _key.decode('utf-8')
_key = _key.decode("utf-8")
return self.client.put(_key, _value, **kwargs)
@ -70,18 +77,25 @@ class Etcd3Wrapper:
@readable_errors
def watch_prefix(self, key, timeout=0, value_in_json=False):
timeout_event = EtcdEntry(PseudoEtcdMeta(key=b'TIMEOUT'),
value=str.encode(json.dumps({'status': 'TIMEOUT',
'type': 'TIMEOUT'})),
value_in_json=value_in_json)
timeout_event = EtcdEntry(
PseudoEtcdMeta(key=b"TIMEOUT"),
value=str.encode(
json.dumps({"status": "TIMEOUT", "type": "TIMEOUT"})
),
value_in_json=value_in_json,
)
event_queue = queue.Queue()
def add_event_to_queue(event):
if hasattr(event, 'events'):
if hasattr(event, "events"):
for e in event.events:
if e.value:
event_queue.put(EtcdEntry(e, e.value, value_in_json=value_in_json))
event_queue.put(
EtcdEntry(
e, e.value, value_in_json=value_in_json
)
)
self.client.add_watch_prefix_callback(key, add_event_to_queue)
@ -96,4 +110,8 @@ class Etcd3Wrapper:
class PsuedoEtcdEntry(EtcdEntry):
def __init__(self, key, value, value_in_json=False):
super().__init__(PseudoEtcdMeta(key=key.encode('utf-8')), value, value_in_json=value_in_json)
super().__init__(
PseudoEtcdMeta(key=key.encode("utf-8")),
value,
value_in_json=value_in_json,
)

4
ucloud/common/host.py
View file

@ -29,7 +29,9 @@ class HostEntry(SpecificEtcdEntryBase):
self.last_heartbeat = time.strftime("%Y-%m-%d %H:%M:%S")
def is_alive(self):
last_heartbeat = datetime.strptime(self.last_heartbeat, "%Y-%m-%d %H:%M:%S")
last_heartbeat = datetime.strptime(
self.last_heartbeat, "%Y-%m-%d %H:%M:%S"
)
delta = datetime.now() - last_heartbeat
if delta.total_seconds() > 60:
return False

12
ucloud/common/logging.py
View file

@ -7,21 +7,21 @@ class NoTracebackStreamHandler(logging.StreamHandler):
info, cache = record.exc_info, record.exc_text
record.exc_info, record.exc_text = None, None
if record.levelname in ['WARNING', 'WARN']:
if record.levelname in ["WARNING", "WARN"]:
color = colorama.Fore.LIGHTYELLOW_EX
elif record.levelname == 'ERROR':
elif record.levelname == "ERROR":
color = colorama.Fore.LIGHTRED_EX
elif record.levelname == 'INFO':
elif record.levelname == "INFO":
color = colorama.Fore.LIGHTGREEN_EX
elif record.levelname == 'CRITICAL':
elif record.levelname == "CRITICAL":
color = colorama.Fore.LIGHTCYAN_EX
else:
color = colorama.Fore.WHITE
try:
print(color, end='', flush=True)
print(color, end="", flush=True)
super().handle(record)
finally:
record.exc_info = info
record.exc_text = cache
print(colorama.Style.RESET_ALL, end='', flush=True)
print(colorama.Style.RESET_ALL, end="", flush=True)

40
ucloud/common/network.py
View file

@ -11,7 +11,9 @@ def random_bytes(num=6):
return [random.randrange(256) for _ in range(num)]
def generate_mac(uaa=False, multicast=False, oui=None, separator=':', byte_fmt='%02x'):
def generate_mac(
uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x"
):
mac = random_bytes()
if oui:
if type(oui) == str:
@ -30,35 +32,51 @@ def generate_mac(uaa=False, multicast=False, oui=None, separator=':', byte_fmt='
def create_dev(script, _id, dev, ip=None):
command = ['sudo', '-p', 'Enter password to create network devices for vm: ',
script, str(_id), dev]
command = [
"sudo",
"-p",
"Enter password to create network devices for vm: ",
script,
str(_id),
dev,
]
if ip:
command.append(ip)
try:
output = sp.check_output(command, stderr=sp.PIPE)
except Exception:
logger.exception('Creation of interface %s failed.', dev)
logger.exception("Creation of interface %s failed.", dev)
return None
else:
return output.decode('utf-8').strip()
return output.decode("utf-8").strip()
def delete_network_interface(iface):
try:
sp.check_output(
[
'sudo', '-p', 'Enter password to remove {} network device: '.format(iface),
'ip', 'link', 'del', iface
], stderr=sp.PIPE
"sudo",
"-p",
"Enter password to remove {} network device: ".format(
iface
),
"ip",
"link",
"del",
iface,
],
stderr=sp.PIPE,
)
except Exception:
logger.exception('Interface %s Deletion failed', iface)
logger.exception("Interface %s Deletion failed", iface)
def find_free_port():
with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
with closing(
socket.socket(socket.AF_INET, socket.SOCK_STREAM)
) as s:
try:
s.bind(('', 0))
s.bind(("", 0))
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
except Exception:
return None

8
ucloud/common/request.py
View file

@ -17,7 +17,6 @@ class RequestType:
class RequestEntry(SpecificEtcdEntryBase):
def __init__(self, e):
self.destination_sock_path = None
self.destination_host_key = None
@ -30,8 +29,11 @@ class RequestEntry(SpecificEtcdEntryBase):
@classmethod
def from_scratch(cls, request_prefix, **kwargs):
e = PsuedoEtcdEntry(join(request_prefix, uuid4().hex),
value=json.dumps(kwargs).encode("utf-8"), value_in_json=True)
e = PsuedoEtcdEntry(
join(request_prefix, uuid4().hex),
value=json.dumps(kwargs).encode("utf-8"),
value_in_json=True,
)
return cls(e)

10
ucloud/common/schemas.py
View file

@ -14,7 +14,7 @@ class StorageUnit(fields.Field):
class SpecsSchema(Schema):
cpu = fields.Int()
ram = StorageUnit()
os_ssd = StorageUnit(data_key='os-ssd', attribute='os-ssd')
os_ssd = StorageUnit(data_key="os-ssd", attribute="os-ssd")
hdd = fields.List(StorageUnit())
@ -29,11 +29,13 @@ class VMSchema(Schema):
image_uuid = fields.Str()
hostname = fields.Str()
metadata = fields.Dict()
network = fields.List(fields.Tuple((fields.Str(), fields.Str(), fields.Int())))
network = fields.List(
fields.Tuple((fields.Str(), fields.Str(), fields.Int()))
)
in_migration = fields.Bool()
class NetworkSchema(Schema):
_id = fields.Int(data_key='id', attribute='id')
_type = fields.Str(data_key='type', attribute='type')
_id = fields.Int(data_key="id", attribute="id")
_type = fields.Str(data_key="type", attribute="type")
ipv6 = fields.Str()

53
ucloud/common/storage_handlers.py
View file

@ -11,7 +11,7 @@ from ucloud.settings import settings as config
class ImageStorageHandler(ABC):
handler_name = 'base'
handler_name = "base"
def __init__(self, image_base, vm_base):
self.image_base = image_base
@ -55,9 +55,9 @@ class ImageStorageHandler(ABC):
try:
sp.check_output(command, stderr=sp.PIPE)
except sp.CalledProcessError as e:
_stderr = e.stderr.decode('utf-8').strip()
_stderr = e.stderr.decode("utf-8").strip()
if report:
logger.exception('%s:- %s', error_origin, _stderr)
logger.exception("%s:- %s", error_origin, _stderr)
return False
return True
@ -72,14 +72,16 @@ class ImageStorageHandler(ABC):
class FileSystemBasedImageStorageHandler(ImageStorageHandler):
handler_name = 'Filesystem'
handler_name = "Filesystem"
def import_image(self, src, dest, protect=False):
dest = join_path(self.image_base, dest)
try:
shutil.copy(src, dest)
if protect:
os.chmod(dest, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
os.chmod(
dest, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH
)
except Exception as e:
logger.exception(e)
return False
@ -97,7 +99,14 @@ class FileSystemBasedImageStorageHandler(ImageStorageHandler):
def resize_vm_image(self, path, size):
path = join_path(self.vm_base, path)
command = ["qemu-img", "resize", "-f", "raw", path, "{}M".format(size)]
command = [
"qemu-img",
"resize",
"-f",
"raw",
path,
"{}M".format(size),
]
if self.execute_command(command):
return True
else:
@ -126,15 +135,25 @@ class FileSystemBasedImageStorageHandler(ImageStorageHandler):
class CEPHBasedImageStorageHandler(ImageStorageHandler):
handler_name = 'Ceph'
handler_name = "Ceph"
def import_image(self, src, dest, protect=False):
dest = join_path(self.image_base, dest)
import_command = ["rbd", "import", src, dest]
commands = [import_command]
if protect:
snap_create_command = ["rbd", "snap", "create", "{}@protected".format(dest)]
snap_protect_command = ["rbd", "snap", "protect", "{}@protected".format(dest)]
snap_create_command = [
"rbd",
"snap",
"create",
"{}@protected".format(dest),
]
snap_protect_command = [
"rbd",
"snap",
"protect",
"{}@protected".format(dest),
]
commands.append(snap_create_command)
commands.append(snap_protect_command)
@ -174,16 +193,16 @@ class CEPHBasedImageStorageHandler(ImageStorageHandler):
def get_storage_handler():
__storage_backend = config['storage']['storage_backend']
if __storage_backend == 'filesystem':
__storage_backend = config["storage"]["storage_backend"]
if __storage_backend == "filesystem":
return FileSystemBasedImageStorageHandler(
vm_base=config['storage']['vm_dir'],
image_base=config['storage']['image_dir']
vm_base=config["storage"]["vm_dir"],
image_base=config["storage"]["image_dir"],
)
elif __storage_backend == 'ceph':
elif __storage_backend == "ceph":
return CEPHBasedImageStorageHandler(
vm_base=config['storage']['ceph_vm_pool'],
image_base=config['storage']['ceph_image_pool']
vm_base=config["storage"]["ceph_vm_pool"],
image_base=config["storage"]["ceph_image_pool"],
)
else:
raise Exception('Unknown Image Storage Handler')
raise Exception("Unknown Image Storage Handler")

11
ucloud/common/vm.py
View file

@ -13,13 +13,12 @@ class VMStatus:
def declare_stopped(vm):
vm['hostname'] = ''
vm['in_migration'] = False
vm['status'] = VMStatus.stopped
vm["hostname"] = ""
vm["in_migration"] = False
vm["status"] = VMStatus.stopped
class VMEntry(SpecificEtcdEntryBase):
def __init__(self, e):
self.owner = None # type: str
self.specs = None # type: dict
@ -48,7 +47,9 @@ class VMEntry(SpecificEtcdEntryBase):
def add_log(self, msg):
self.log = self.log[:5]
self.log.append("{} - {}".format(datetime.now().isoformat(), msg))
self.log.append(
"{} - {}".format(datetime.now().isoformat(), msg)
)
class VmPool: