fnux/uncloud
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Shutdown Source VM (PAUSED) on successfull migration + blackened all .py files

Browse source
This commit is contained in:
ahmadbilalkhalid 2019-12-30 14:35:07 +05:00
commit 9bdf4d2180
31 changed files with 1307 additions and 638 deletions
Download patch file Download diff file Expand all files Collapse all files

185
ucloud/vmm/__init__.py
View file

@ -37,7 +37,9 @@ class VMQMPHandles:
self.sock.close()
if exc_type:
logger.error('Couldn\'t get handle for VM.', exc_type, exc_val, exc_tb)
logger.error(
"Couldn't get handle for VM.", exc_type, exc_val, exc_tb
)
raise exc_type("Couldn't get handle for VM.") from exc_type
@ -54,29 +56,46 @@ class TransferVM(Process):
with suppress(FileNotFoundError):
os.remove(self.src_sock_path)
command = ['ssh', '-nNT', '-L', '{}:{}'.format(self.src_sock_path, self.dest_sock_path),
'root@{}'.format(self.host)]
command = [
"ssh",
"-nNT",
"-L",
"{}:{}".format(self.src_sock_path, self.dest_sock_path),
"root@{}".format(self.host),
]
try:
p = sp.Popen(command)
except Exception as e:
logger.error('Couldn\' forward unix socks over ssh.', exc_info=e)
logger.error(
"Couldn' forward unix socks over ssh.", exc_info=e
)
else:
time.sleep(2)
vmm = VMM()
logger.debug('Executing: ssh forwarding command: %s', command)
vmm.execute_command(self.src_uuid, command='migrate',
arguments={'uri': 'unix:{}'.format(self.src_sock_path)})
logger.debug(
"Executing: ssh forwarding command: %s", command
)
vmm.execute_command(
self.src_uuid,
command="migrate",
arguments={"uri": "unix:{}".format(self.src_sock_path)},
)
while p.poll() is None:
success, output = vmm.execute_command(self.src_uuid, command='query-migrate')
success, output = vmm.execute_command(
self.src_uuid, command="query-migrate"
)
if success:
status = output['return']['status']
if status != 'active':
print('Migration Status: ', status)
status = output["return"]["status"]
if status != "active":
print("Migration Status: ", status)
if status == "completed":
vmm.stop(self.src_uuid)
return
else:
print('Migration Status: ', status)
print("Migration Status: ", status)
else:
return
time.sleep(0.2)
@ -84,18 +103,29 @@ class TransferVM(Process):
class VMM:
# Virtual Machine Manager
def __init__(self, qemu_path='/usr/bin/qemu-system-x86_64',
vmm_backend=os.path.expanduser('~/ucloud/vmm/')):
def __init__(
self,
qemu_path="/usr/bin/qemu-system-x86_64",
vmm_backend=os.path.expanduser("~/ucloud/vmm/"),
):
self.qemu_path = qemu_path
self.vmm_backend = vmm_backend
self.socket_dir = os.path.join(self.vmm_backend, 'sock')
self.socket_dir = os.path.join(self.vmm_backend, "sock")
if not os.path.isdir(self.vmm_backend):
logger.info('{} does not exists. Creating it...'.format(self.vmm_backend))
logger.info(
"{} does not exists. Creating it...".format(
self.vmm_backend
)
)
os.makedirs(self.vmm_backend, exist_ok=True)
if not os.path.isdir(self.socket_dir):
logger.info('{} does not exists. Creating it...'.format(self.socket_dir))
logger.info(
"{} does not exists. Creating it...".format(
self.socket_dir
)
)
os.makedirs(self.socket_dir, exist_ok=True)
def is_running(self, uuid):
@ -106,8 +136,12 @@ class VMM:
recv = sock.recv(4096)
except Exception as err:
# unix sock doesn't exists or it is closed
logger.debug('VM {} sock either don\' exists or it is closed. It mean VM is stopped.'.format(uuid),
exc_info=err)
logger.debug(
"VM {} sock either don' exists or it is closed. It mean VM is stopped.".format(
uuid
),
exc_info=err,
)
else:
# if we receive greetings from qmp it mean VM is running
if len(recv) > 0:
@ -122,36 +156,67 @@ class VMM:
# start --> sucess?
migration_args = ()
if migration:
migration_args = ('-incoming', 'unix:{}'.format(os.path.join(self.socket_dir, uuid)))
migration_args = (
"-incoming",
"unix:{}".format(os.path.join(self.socket_dir, uuid)),
)
if self.is_running(uuid):
logger.warning('Cannot start VM. It is already running.')
logger.warning("Cannot start VM. It is already running.")
else:
qmp_arg = ('-qmp', 'unix:{},server,nowait'.format(join_path(self.vmm_backend, uuid)))
vnc_arg = ('-vnc', 'unix:{}'.format(tempfile.NamedTemporaryFile().name))
qmp_arg = (
"-qmp",
"unix:{},server,nowait".format(
join_path(self.vmm_backend, uuid)
),
)
vnc_arg = (
"-vnc",
"unix:{}".format(tempfile.NamedTemporaryFile().name),
)
command = ['sudo', '-p', 'Enter password to start VM {}: '.format(uuid),
self.qemu_path, *args, *qmp_arg, *migration_args, *vnc_arg, '-daemonize']
command = [
"sudo",
"-p",
"Enter password to start VM {}: ".format(uuid),
self.qemu_path,
*args,
*qmp_arg,
*migration_args,
*vnc_arg,
"-daemonize",
]
try:
sp.check_output(command, stderr=sp.PIPE)
except sp.CalledProcessError as err:
logger.exception('Error occurred while starting VM.\nDetail %s', err.stderr.decode('utf-8'))
logger.exception(
"Error occurred while starting VM.\nDetail %s",
err.stderr.decode("utf-8"),
)
else:
with suppress(sp.CalledProcessError):
sp.check_output([
'sudo', '-p',
'Enter password to correct permission for uncloud-vmm\'s directory',
'chmod', '-R', 'o=rwx,g=rwx', self.vmm_backend
])
sp.check_output(
[
"sudo",
"-p",
"Enter password to correct permission for uncloud-vmm's directory",
"chmod",
"-R",
"o=rwx,g=rwx",
self.vmm_backend,
]
)
# TODO: Find some good way to check whether the virtual machine is up and
# running without relying on non-guarenteed ways.
for _ in range(10):
time.sleep(2)
status = self.get_status(uuid)
if status in ['running', 'inmigrate']:
if status in ["running", "inmigrate"]:
return status
logger.warning('Timeout on VM\'s status. Shutting down VM %s', uuid)
logger.warning(
"Timeout on VM's status. Shutting down VM %s", uuid
)
self.stop(uuid)
# TODO: What should we do more. VM can still continue to run in background.
# If we have pid of vm we can kill it using OS.
@ -159,55 +224,73 @@ class VMM:
def execute_command(self, uuid, command, **kwargs):
# execute_command -> sucess?, output
try:
with VMQMPHandles(os.path.join(self.vmm_backend, uuid)) as (sock_handle, file_handle):
command_to_execute = {
'execute': command,
**kwargs
}
sock_handle.sendall(json.dumps(command_to_execute).encode('utf-8'))
with VMQMPHandles(os.path.join(self.vmm_backend, uuid)) as (
sock_handle,
file_handle,
):
command_to_execute = {"execute": command, **kwargs}
sock_handle.sendall(
json.dumps(command_to_execute).encode("utf-8")
)
output = file_handle.readline()
except Exception:
logger.exception('Error occurred while executing command and getting valid output from qmp')
logger.exception(
"Error occurred while executing command and getting valid output from qmp"
)
else:
try:
output = json.loads(output)
except Exception:
logger.exception('QMP Output isn\'t valid JSON. %s', output)
logger.exception(
"QMP Output isn't valid JSON. %s", output
)
else:
return 'return' in output, output
return "return" in output, output
return False, None
def stop(self, uuid):
success, output = self.execute_command(command='quit', uuid=uuid)
success, output = self.execute_command(
command="quit", uuid=uuid
)
return success
def get_status(self, uuid):
success, output = self.execute_command(command='query-status', uuid=uuid)
success, output = self.execute_command(
command="query-status", uuid=uuid
)
if success:
return output['return']['status']
return output["return"]["status"]
else:
# TODO: Think about this for a little more
return 'STOPPED'
return "STOPPED"
def discover(self):
vms = [
uuid for uuid in os.listdir(self.vmm_backend)
uuid
for uuid in os.listdir(self.vmm_backend)
if not isdir(join_path(self.vmm_backend, uuid))
]
return vms
def get_vnc(self, uuid):
success, output = self.execute_command(uuid, command='query-vnc')
success, output = self.execute_command(
uuid, command="query-vnc"
)
if success:
return output['return']['service']
return output["return"]["service"]
return None
def transfer(self, src_uuid, destination_sock_path, host):
p = TransferVM(src_uuid, destination_sock_path, socket_dir=self.socket_dir, host=host)
p = TransferVM(
src_uuid,
destination_sock_path,
socket_dir=self.socket_dir,
host=host,
)
p.start()
# TODO: the following method should clean things that went wrong
# e.g If VM migration fails or didn't start for long time
# i.e 15 minutes we should stop the waiting VM.
def maintenace(self):
pass
pass