Merge branch 'without-ceph' into 'master'

allow ucloud-image-scanner to also be able work without ceph i.e use filesystem

See merge request ungleich-public/ucloud-image-scanner!2

.gitignore

@@ -1,5 +1,6 @@
 .idea/
 .vscode/
+.env
 venv/
 log.txt
 etcd3_wrapper

config.py

@@ -0,0 +1,20 @@
+import logging
+
+from etcd3_wrapper import Etcd3Wrapper
+from decouple import config
+
+BASE_PATH = config("BASE_DIR", "/var/www")
+WITHOUT_CEPH = config("WITHOUT_CEPH", False, cast=bool)
+ETCD_URL = config("ETCD_URL")
+
+
+logging.basicConfig(
+    level=logging.DEBUG,
+    filename="log.txt",
+    filemode="a",
+    format="%(asctime)s: %(levelname)s - %(message)s",
+    datefmt="%d-%b-%y %H:%M:%S",
+)
+
+
+client = Etcd3Wrapper(host=ETCD_URL)

main.py

@@ -1,66 +1,106 @@
 import os
 import json
 import subprocess
-import logging
 
-from decouple import config
+from config import logging, client, BASE_PATH, WITHOUT_CEPH
-from etcd3_wrapper import Etcd3Wrapper
 
-BASE_PATH = config("BASE_DIR")
 
-logging.basicConfig(
+# If you are using WITHOUT_CEPH FLAG in .env
-    level=logging.DEBUG,
+# then please make sure that /var/image directory
-    filename="log.txt",
+# exists otherwise this script would fail
-    filemode="a",
+if WITHOUT_CEPH and not os.path.isdir("/var/image"):
-    format="%(asctime)s: %(levelname)s - %(message)s",
+    exit(1)
-    datefmt="%d-%b-%y %H:%M:%S",
-)
 
-client = Etcd3Wrapper(host=config("ETCD_URL"))
+
-images = list(client.get_prefix("/v1/image/", value_in_json=True))
+# We want to get images entries that requests images to be created
-images_to_be_created = list(filter(lambda e: e.value["status"] == "TO_BE_CREATED", images))
+images = client.get_prefix("/v1/image/", value_in_json=True)
+images_to_be_created = filter(lambda im: im.value["status"] == "TO_BE_CREATED", images)
 
 for image in images_to_be_created:
-    image_uuid = image.key.split("/")[-1]
+    try:
-    image_full_path = f"{BASE_PATH}/{image.value['owner']}/{image.value['filename']}"
+        image_uuid = image.key.split('/')[-1]
-    if os.path.isfile(image_full_path):
+        image_owner = image.value['owner']
-        output = subprocess.check_output(["qemu-img", "info", image_full_path]).decode("utf-8")
+        image_filename = image.value['filename']
-        if "qcow2" in output:
+        image_store_name = image.value['store_name']
-            logging.info("Converting it to raw")
+        image_full_path = os.path.join(BASE_PATH, image_owner, image_filename)
-            subprocess.run(["qemu-img", "convert", "-f", "qcow2",
-                            "-O", "raw", image_full_path, f"image.raw"])
-            if os.path.isfile(f"image.raw"):
-                # shutil.move(f"{image_uuid}.raw", f"/var/vm/{image_uuid}.raw")
-                _store_name = image.value["store_name"]
-                _image_stores = client.get_prefix("/v1/image_store/", value_in_json=True)
-                _user_image_store = next(filter(lambda s: s.value["name"] == _store_name, _image_stores))
-                if _user_image_store:
-                    _image_store_pool = _user_image_store.value["attributes"]["pool"]
-                    rc = subprocess.check_call(["rbd", "import", "image.raw",
-                                f"{_image_store_pool}/{image_uuid}"])
-                    if rc == 0:
-                        _snapshot_creation_command = f"rbd snap create {_image_store_pool}/{image_uuid}@protected"
-                        subprocess.check_call(_snapshot_creation_command.split(" "))
 
-                        _snapshot_protect_command = f"rbd snap protect {_image_store_pool}/{image_uuid}@protected"
+        image_stores = client.get_prefix("/v1/image_store/", value_in_json=True)
-                        subprocess.check_call(_snapshot_protect_command.split(" "))
+        user_image_store = next(filter(lambda s: s.value["name"] == image_store_name, image_stores))
 
-                        image.value["status"] = "CREATED"
+        image_store_pool = user_image_store.value['attributes']['pool']
-                        client.put(image.key, json.dumps(image.value))
+
-                    else:
+    except Exception as e:
-                        logging.info(f"Some error occurred while creating image {image_uuid}")
+        logging.exception(e)
-                else:
-                    logging.info(f"Image store {_user_image_store} not found")
-            else:
-                logging.info(f"{image_uuid}.raw not found")
-        else:
-            logging.info("not qcow2 format")
-            image.value["status"] = "INVALID_FORMAT"
-            client.put(image.key, json.dumps(image.value))
     else:
-        logging.info("File does not exists")
+        # At least our basic data is available
-    
+
+        qemu_img_info_command = ["qemu-img", "info", image_full_path]
+        qemu_img_convert_command = ["qemu-img", "convert", "-f", "qcow2", "-O", "raw", image_full_path, f"image.raw"]
+
+        image_import_command = ["rbd", "import", "image.raw", f"{image_store_pool}/{image_uuid}"]
+        snapshot_creation_command = ["rbd", "snap", "create", f"{image_store_pool}/{image_uuid}@protected"]
+        snapshot_protect_command = ["rbd", "snap", "protect", f"{image_store_pool}/{image_uuid}@protected"]
+
+        if WITHOUT_CEPH:
+            image_import_command = ["mv", "image.raw", os.path.join("/var/image", image_uuid)]
+            snapshot_creation_command = ["true"]
+            snapshot_protect_command = ["true"]
+
+        # First check whether the image is qcow2
+        # This would also check whether the file exists or not
+        try:
+            qemu_img_info = subprocess.Popen(qemu_img_info_command,
+                                             stdout=subprocess.PIPE,
+                                             stderr=subprocess.DEVNULL)
+        except Exception as e:
+            # Command itself failed i.e maybe qemu-img not found
+            # Maybe Popen crashes some other ways
+            logging.exception(e)
+        else:
+            # At least we run the qemu-img info command
+            # It may have not executed successfully. If thats
+            # the case, our next command grep would also fail
+            try:
+                # Fetching the lines containing "file format"
+                command = ["grep", "-e", "file format"]
+                output = subprocess.check_output(command, stdin=qemu_img_info.stdout)
+            except Exception as e:
+                logging.exception(e)
+            else:
+                # grep command runs successfully, it implies
+                # the previous command qemu-img would have
+                # run successfully
+                output = output.decode("utf-8").strip()
+                if "qcow2" in output:
+                    try:
+                        # Convert .qcow2 to .raw
+                        subprocess.check_output(qemu_img_convert_command)
+                    except Exception as e:
+                        logging.exception(e)
+                    else:
+                        # Image successfully converted
+                        try:
+                            # Import image either to ceph/filesystem
+                            subprocess.check_output(image_import_command)
+                        except Exception as e:
+                            logging.exception(e)
+                        else:
+                            # Image imported successfully
+                            try:
+                                subprocess.check_output(snapshot_creation_command)
+                                subprocess.check_output(snapshot_protect_command)
+                            except Exception as e:
+                                logging.exception(e)
+                            else:
+                                # Everything is successfully done
+                                image.value["status"] = "CREATED"
+                                client.put(image.key, json.dumps(image.value))
+                else:
+                    # The user provided image is not in qcow2 format
+                    image.value["status"] = "INVALID_FORMAT"
+                    client.put(image.key, json.dumps(image.value))
+
     try:
         os.remove("image.raw")
-    except OSError:
+    except Exception:
-        pass
+        pass