Merge branch 'without-ceph' into 'master'

allow ucloud-vm to also be able work without ceph i.e use filesystem

See merge request ungleich-public/ucloud-vm!2

config.py

@@ -6,6 +6,7 @@ from ucloud_common.host import HostPool
 from ucloud_common.request import RequestPool
 from decouple import config
 
+WITHOUT_CEPH = config("WITHOUT_CEPH", False, cast=bool)
 
 logging.basicConfig(
     level=logging.DEBUG,

virtualmachine.py

@@ -11,9 +11,11 @@ import qmp
 import tempfile
 import bitmath
 import time
+import os
 
+from config import (vm_pool, request_pool, etcd_client,
+                    logging, running_vms, WITHOUT_CEPH)
 from ucloud_common.vm import VMStatus, VMEntry
-from config import (vm_pool, request_pool, etcd_client, logging, running_vms)
 from typing import Union
 from functools import wraps
 from dataclasses import dataclass
@@ -37,14 +39,19 @@ class VM:
 
 
 def get_start_command_args(vm_entry, vnc_sock_filename: str, migration=False, migration_port=4444):
+    threads_per_core = 1
     vm_memory = int(bitmath.Byte(int(vm_entry.specs["ram"])).to_MB())
     vm_cpus = int(vm_entry.specs["cpu"])
     vm_uuid = vm_entry.uuid
-    threads_per_core = 1
+
-    command = (f"-drive file=rbd:uservms/{vm_uuid},format=raw,if=virtio,cache=none"
+    if WITHOUT_CEPH:
-               f" -device virtio-rng-pci -vnc unix:{vnc_sock_filename}"
+        command = f"-drive file={os.path.join('/var/vm', vm_uuid)},format=raw,if=virtio,cache=none"
-               f" -m {vm_memory} -smp cores={vm_cpus},threads={threads_per_core}"
+    else:
-               f" -name {vm_uuid}")
+        command = f"-drive file=rbd:uservms/{vm_uuid},format=raw,if=virtio,cache=none"
+
+    command += (f" -device virtio-rng-pci -vnc unix:{vnc_sock_filename}"
+                f" -m {vm_memory} -smp cores={vm_cpus},threads={threads_per_core}"
+                f" -name {vm_uuid}")
     if migration:
         command += f" -incoming tcp:0:{migration_port}"
 
@@ -58,13 +65,13 @@ def create_vm_object(vm_entry, migration=False, migration_port=4444):
 
     # REQUIREMENT: Use Unix Socket instead of TCP Port for VNC
     vnc_sock_file = tempfile.NamedTemporaryFile()
+
+    qemu_args = get_start_command_args(vm_entry=vm_entry,
+                                       vnc_sock_filename=vnc_sock_file.name,
+                                       migration=migration,
+                                       migration_port=migration_port)
     qemu_machine = qmp.QEMUMachine("/usr/bin/qemu-system-x86_64",
-                                   args=get_start_command_args(vm_entry,
+                                   args=qemu_args)
-                                                               vnc_sock_file.name,
-                                                               migration=migration,
-                                                               migration_port=migration_port
-                                                               )
-                                   )
     return VM(vm_entry.key, qemu_machine, vnc_sock_file)
 
 
@@ -80,12 +87,12 @@ def need_running_vm(func):
             try:
                 status = vm.handle.command("query-status")
                 logging.debug(f"VM Status Check - {status}")
-            except OSError:
+            except Exception as exception:
                 logging.info(
-                    f"{func.__name__} failed - VM {e.key} - Unknown Error"
+                    f"{func.__name__} failed - VM {e} {exception} - Unknown Error"
                 )
-
+            else:
-            return func(e)
+                return func(e)
         else:
             logging.info(
                 f"{func.__name__} failed because VM {e.key} is not running"
@@ -97,25 +104,40 @@ def need_running_vm(func):
 
 def create(vm_entry: VMEntry):
     vm_hdd = int(bitmath.Byte(int(vm_entry.specs["hdd"])).to_MB())
-    _command_to_create = f"rbd clone images/{vm_entry.image_uuid}@protected uservms/{vm_entry.uuid}"
+
-    _command_to_extend = f"rbd resize uservms/{vm_entry.uuid} --size {vm_hdd}"
+    if WITHOUT_CEPH:
+        _command_to_create = ["cp",
+                              os.path.join("/var/image", vm_entry.image_uuid),
+                              os.path.join("/var/vm", vm_entry.uuid)]
+
+        _command_to_extend = ["qemu-img", "resize", os.path.join("/var/vm", vm_entry.uuid), vm_entry.specs["hdd"]]
+    else:
+        _command_to_create = ["rbd", "clone",
+                              f"images/{vm_entry.image_uuid}@protected",
+                              f"uservms/{vm_entry.uuid}"]
+
+        _command_to_extend = ["rbd", "resize", f"uservms/{vm_entry.uuid}", "--size", vm_hdd]
 
     try:
-        subprocess.check_call(_command_to_create.split(" "))
+        subprocess.check_output(_command_to_create)
     except subprocess.CalledProcessError as e:
         if e.returncode == errno.EEXIST:
             logging.debug(f"Image for vm {vm_entry.uuid} exists")
             # File Already exists. No Problem Continue
             return
-        else:
+
-            # This exception catches all other exceptions
+        # This exception catches all other exceptions
-            # i.e FileNotFound (BaseImage), pool Does Not Exists etc.
+        # i.e FileNotFound (BaseImage), pool Does Not Exists etc.
-            logging.exception(f"Can't clone image - {e}")
+        logging.exception(e)
+
+        vm_entry.status = "ERROR"
     else:
-        # TODO: Check whether the below suprocess.check_call
+        try:
-        #       is executed successfully
+            subprocess.check_output(_command_to_extend)
-        subprocess.check_call(_command_to_extend.split(" "))
+        except Exception as e:
-        logging.info("New VM Created")
+            logging.exception(e)
+        else:
+            logging.info("New VM Created")
 
 
 def start(vm_entry: VMEntry):
@@ -159,17 +181,18 @@ def delete(vm_entry):
     logging.info(f"Deleting VM {vm_entry}")
     stop(vm_entry)
     path_without_protocol = vm_entry.path[vm_entry.path.find(":") + 1:]
-    try:
+
-        rc = subprocess.call(f"rbd rm {path_without_protocol}".split(" "))
+    if WITHOUT_CEPH:
-    except FileNotFoundError as e:
+        vm_deletion_command = ["rm", os.path.join("/var/vm", vm_entry.uuid)]
-        logging.exception(e)
-    except Exception as e:
-        logging.exception(f"Unknown error occurred - {e}")
     else:
-        if rc == 0:
+        vm_deletion_command = ["rbd", "rm", path_without_protocol]
-            etcd_client.client.delete(vm_entry.key)
+
-        else:
+    try:
-            logging.info("Some unknown problem occur while deleting vm file")
+        subprocess.check_output(vm_deletion_command)
+    except Exception as e:
+        logging.exception(e)
+    else:
+        etcd_client.client.delete(vm_entry.key)
 
 
 def transfer(request_event):