From ca1d3f39de0eef0c7d7cf82440ebb02b2393e54e Mon Sep 17 00:00:00 2001 From: Ahmed Bilal <49-ahmedbilal@users.noreply.code.ungleich.ch> Date: Sat, 7 Sep 2019 12:55:12 +0200 Subject: [PATCH] allow ucloud-image-scanner to also be able work without ceph i.e use filesystem --- .gitignore | 1 + config.py | 20 ++++++++ main.py | 144 ++++++++++++++++++++++++++++++++++------------------- 3 files changed, 113 insertions(+), 52 deletions(-) create mode 100644 config.py diff --git a/.gitignore b/.gitignore index c0d30c8..3922966 100755 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .idea/ .vscode/ +.env venv/ log.txt etcd3_wrapper diff --git a/config.py b/config.py new file mode 100644 index 0000000..256d7e9 --- /dev/null +++ b/config.py @@ -0,0 +1,20 @@ +import logging + +from etcd3_wrapper import Etcd3Wrapper +from decouple import config + +BASE_PATH = config("BASE_DIR", "/var/www") +WITHOUT_CEPH = config("WITHOUT_CEPH", False, cast=bool) +ETCD_URL = config("ETCD_URL") + + +logging.basicConfig( + level=logging.DEBUG, + filename="log.txt", + filemode="a", + format="%(asctime)s: %(levelname)s - %(message)s", + datefmt="%d-%b-%y %H:%M:%S", +) + + +client = Etcd3Wrapper(host=ETCD_URL) diff --git a/main.py b/main.py index c159881..12eb0ad 100755 --- a/main.py +++ b/main.py @@ -1,66 +1,106 @@ import os import json import subprocess -import logging -from decouple import config -from etcd3_wrapper import Etcd3Wrapper +from config import logging, client, BASE_PATH, WITHOUT_CEPH -BASE_PATH = config("BASE_DIR") -logging.basicConfig( - level=logging.DEBUG, - filename="log.txt", - filemode="a", - format="%(asctime)s: %(levelname)s - %(message)s", - datefmt="%d-%b-%y %H:%M:%S", -) +# If you are using WITHOUT_CEPH FLAG in .env +# then please make sure that /var/image directory +# exists otherwise this script would fail +if WITHOUT_CEPH and not os.path.isdir("/var/image"): + exit(1) -client = Etcd3Wrapper(host=config("ETCD_URL")) -images = list(client.get_prefix("/v1/image/", value_in_json=True)) -images_to_be_created = list(filter(lambda e: e.value["status"] == "TO_BE_CREATED", images)) + +# We want to get images entries that requests images to be created +images = client.get_prefix("/v1/image/", value_in_json=True) +images_to_be_created = filter(lambda im: im.value["status"] == "TO_BE_CREATED", images) for image in images_to_be_created: - image_uuid = image.key.split("/")[-1] - image_full_path = f"{BASE_PATH}/{image.value['owner']}/{image.value['filename']}" - if os.path.isfile(image_full_path): - output = subprocess.check_output(["qemu-img", "info", image_full_path]).decode("utf-8") - if "qcow2" in output: - logging.info("Converting it to raw") - subprocess.run(["qemu-img", "convert", "-f", "qcow2", - "-O", "raw", image_full_path, f"image.raw"]) - if os.path.isfile(f"image.raw"): - # shutil.move(f"{image_uuid}.raw", f"/var/vm/{image_uuid}.raw") - _store_name = image.value["store_name"] - _image_stores = client.get_prefix("/v1/image_store/", value_in_json=True) - _user_image_store = next(filter(lambda s: s.value["name"] == _store_name, _image_stores)) - if _user_image_store: - _image_store_pool = _user_image_store.value["attributes"]["pool"] - rc = subprocess.check_call(["rbd", "import", "image.raw", - f"{_image_store_pool}/{image_uuid}"]) - if rc == 0: - _snapshot_creation_command = f"rbd snap create {_image_store_pool}/{image_uuid}@protected" - subprocess.check_call(_snapshot_creation_command.split(" ")) + try: + image_uuid = image.key.split('/')[-1] + image_owner = image.value['owner'] + image_filename = image.value['filename'] + image_store_name = image.value['store_name'] + image_full_path = os.path.join(BASE_PATH, image_owner, image_filename) - _snapshot_protect_command = f"rbd snap protect {_image_store_pool}/{image_uuid}@protected" - subprocess.check_call(_snapshot_protect_command.split(" ")) + image_stores = client.get_prefix("/v1/image_store/", value_in_json=True) + user_image_store = next(filter(lambda s: s.value["name"] == image_store_name, image_stores)) - image.value["status"] = "CREATED" - client.put(image.key, json.dumps(image.value)) - else: - logging.info(f"Some error occurred while creating image {image_uuid}") - else: - logging.info(f"Image store {_user_image_store} not found") - else: - logging.info(f"{image_uuid}.raw not found") - else: - logging.info("not qcow2 format") - image.value["status"] = "INVALID_FORMAT" - client.put(image.key, json.dumps(image.value)) + image_store_pool = user_image_store.value['attributes']['pool'] + + except Exception as e: + logging.exception(e) else: - logging.info("File does not exists") - + # At least our basic data is available + + qemu_img_info_command = ["qemu-img", "info", image_full_path] + qemu_img_convert_command = ["qemu-img", "convert", "-f", "qcow2", "-O", "raw", image_full_path, f"image.raw"] + + image_import_command = ["rbd", "import", "image.raw", f"{image_store_pool}/{image_uuid}"] + snapshot_creation_command = ["rbd", "snap", "create", f"{image_store_pool}/{image_uuid}@protected"] + snapshot_protect_command = ["rbd", "snap", "protect", f"{image_store_pool}/{image_uuid}@protected"] + + if WITHOUT_CEPH: + image_import_command = ["mv", "image.raw", os.path.join("/var/image", image_uuid)] + snapshot_creation_command = ["true"] + snapshot_protect_command = ["true"] + + # First check whether the image is qcow2 + # This would also check whether the file exists or not + try: + qemu_img_info = subprocess.Popen(qemu_img_info_command, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL) + except Exception as e: + # Command itself failed i.e maybe qemu-img not found + # Maybe Popen crashes some other ways + logging.exception(e) + else: + # At least we run the qemu-img info command + # It may have not executed successfully. If thats + # the case, our next command grep would also fail + try: + # Fetching the lines containing "file format" + command = ["grep", "-e", "file format"] + output = subprocess.check_output(command, stdin=qemu_img_info.stdout) + except Exception as e: + logging.exception(e) + else: + # grep command runs successfully, it implies + # the previous command qemu-img would have + # run successfully + output = output.decode("utf-8").strip() + if "qcow2" in output: + try: + # Convert .qcow2 to .raw + subprocess.check_output(qemu_img_convert_command) + except Exception as e: + logging.exception(e) + else: + # Image successfully converted + try: + # Import image either to ceph/filesystem + subprocess.check_output(image_import_command) + except Exception as e: + logging.exception(e) + else: + # Image imported successfully + try: + subprocess.check_output(snapshot_creation_command) + subprocess.check_output(snapshot_protect_command) + except Exception as e: + logging.exception(e) + else: + # Everything is successfully done + image.value["status"] = "CREATED" + client.put(image.key, json.dumps(image.value)) + else: + # The user provided image is not in qcow2 format + image.value["status"] = "INVALID_FORMAT" + client.put(image.key, json.dumps(image.value)) + try: os.remove("image.raw") - except OSError: - pass \ No newline at end of file + except Exception: + pass