Merge branch 'without-ceph' into 'master'
allow ucloud-image-scanner to also be able work without ceph i.e use filesystem See merge request ungleich-public/ucloud-image-scanner!2
This commit is contained in:
Ahmed Bilal
commit
0a1cdf5bf0
3 changed files with 113 additions and 52 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -1,5 +1,6 @@
|
|||
.idea/
|
||||
.vscode/
|
||||
.env
|
||||
venv/
|
||||
log.txt
|
||||
etcd3_wrapper
|
||||
|
|
|
|||
20
config.py
Normal file
20
config.py
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
import logging
|
||||
|
||||
from etcd3_wrapper import Etcd3Wrapper
|
||||
from decouple import config
|
||||
|
||||
BASE_PATH = config("BASE_DIR", "/var/www")
|
||||
WITHOUT_CEPH = config("WITHOUT_CEPH", False, cast=bool)
|
||||
ETCD_URL = config("ETCD_URL")
|
||||
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.DEBUG,
|
||||
filename="log.txt",
|
||||
filemode="a",
|
||||
format="%(asctime)s: %(levelname)s - %(message)s",
|
||||
datefmt="%d-%b-%y %H:%M:%S",
|
||||
)
|
||||
|
||||
|
||||
client = Etcd3Wrapper(host=ETCD_URL)
|
||||
140
main.py
140
main.py
|
|
@ -1,66 +1,106 @@
|
|||
import os
|
||||
import json
|
||||
import subprocess
|
||||
import logging
|
||||
|
||||
from decouple import config
|
||||
from etcd3_wrapper import Etcd3Wrapper
|
||||
from config import logging, client, BASE_PATH, WITHOUT_CEPH
|
||||
|
||||
BASE_PATH = config("BASE_DIR")
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.DEBUG,
|
||||
filename="log.txt",
|
||||
filemode="a",
|
||||
format="%(asctime)s: %(levelname)s - %(message)s",
|
||||
datefmt="%d-%b-%y %H:%M:%S",
|
||||
)
|
||||
# If you are using WITHOUT_CEPH FLAG in .env
|
||||
# then please make sure that /var/image directory
|
||||
# exists otherwise this script would fail
|
||||
if WITHOUT_CEPH and not os.path.isdir("/var/image"):
|
||||
exit(1)
|
||||
|
||||
client = Etcd3Wrapper(host=config("ETCD_URL"))
|
||||
images = list(client.get_prefix("/v1/image/", value_in_json=True))
|
||||
images_to_be_created = list(filter(lambda e: e.value["status"] == "TO_BE_CREATED", images))
|
||||
|
||||
# We want to get images entries that requests images to be created
|
||||
images = client.get_prefix("/v1/image/", value_in_json=True)
|
||||
images_to_be_created = filter(lambda im: im.value["status"] == "TO_BE_CREATED", images)
|
||||
|
||||
for image in images_to_be_created:
|
||||
image_uuid = image.key.split("/")[-1]
|
||||
image_full_path = f"{BASE_PATH}/{image.value['owner']}/{image.value['filename']}"
|
||||
if os.path.isfile(image_full_path):
|
||||
output = subprocess.check_output(["qemu-img", "info", image_full_path]).decode("utf-8")
|
||||
if "qcow2" in output:
|
||||
logging.info("Converting it to raw")
|
||||
subprocess.run(["qemu-img", "convert", "-f", "qcow2",
|
||||
"-O", "raw", image_full_path, f"image.raw"])
|
||||
if os.path.isfile(f"image.raw"):
|
||||
# shutil.move(f"{image_uuid}.raw", f"/var/vm/{image_uuid}.raw")
|
||||
_store_name = image.value["store_name"]
|
||||
_image_stores = client.get_prefix("/v1/image_store/", value_in_json=True)
|
||||
_user_image_store = next(filter(lambda s: s.value["name"] == _store_name, _image_stores))
|
||||
if _user_image_store:
|
||||
_image_store_pool = _user_image_store.value["attributes"]["pool"]
|
||||
rc = subprocess.check_call(["rbd", "import", "image.raw",
|
||||
f"{_image_store_pool}/{image_uuid}"])
|
||||
if rc == 0:
|
||||
_snapshot_creation_command = f"rbd snap create {_image_store_pool}/{image_uuid}@protected"
|
||||
subprocess.check_call(_snapshot_creation_command.split(" "))
|
||||
try:
|
||||
image_uuid = image.key.split('/')[-1]
|
||||
image_owner = image.value['owner']
|
||||
image_filename = image.value['filename']
|
||||
image_store_name = image.value['store_name']
|
||||
image_full_path = os.path.join(BASE_PATH, image_owner, image_filename)
|
||||
|
||||
_snapshot_protect_command = f"rbd snap protect {_image_store_pool}/{image_uuid}@protected"
|
||||
subprocess.check_call(_snapshot_protect_command.split(" "))
|
||||
image_stores = client.get_prefix("/v1/image_store/", value_in_json=True)
|
||||
user_image_store = next(filter(lambda s: s.value["name"] == image_store_name, image_stores))
|
||||
|
||||
image.value["status"] = "CREATED"
|
||||
client.put(image.key, json.dumps(image.value))
|
||||
else:
|
||||
logging.info(f"Some error occurred while creating image {image_uuid}")
|
||||
else:
|
||||
logging.info(f"Image store {_user_image_store} not found")
|
||||
else:
|
||||
logging.info(f"{image_uuid}.raw not found")
|
||||
else:
|
||||
logging.info("not qcow2 format")
|
||||
image.value["status"] = "INVALID_FORMAT"
|
||||
client.put(image.key, json.dumps(image.value))
|
||||
image_store_pool = user_image_store.value['attributes']['pool']
|
||||
|
||||
except Exception as e:
|
||||
logging.exception(e)
|
||||
else:
|
||||
logging.info("File does not exists")
|
||||
# At least our basic data is available
|
||||
|
||||
qemu_img_info_command = ["qemu-img", "info", image_full_path]
|
||||
qemu_img_convert_command = ["qemu-img", "convert", "-f", "qcow2", "-O", "raw", image_full_path, f"image.raw"]
|
||||
|
||||
image_import_command = ["rbd", "import", "image.raw", f"{image_store_pool}/{image_uuid}"]
|
||||
snapshot_creation_command = ["rbd", "snap", "create", f"{image_store_pool}/{image_uuid}@protected"]
|
||||
snapshot_protect_command = ["rbd", "snap", "protect", f"{image_store_pool}/{image_uuid}@protected"]
|
||||
|
||||
if WITHOUT_CEPH:
|
||||
image_import_command = ["mv", "image.raw", os.path.join("/var/image", image_uuid)]
|
||||
snapshot_creation_command = ["true"]
|
||||
snapshot_protect_command = ["true"]
|
||||
|
||||
# First check whether the image is qcow2
|
||||
# This would also check whether the file exists or not
|
||||
try:
|
||||
qemu_img_info = subprocess.Popen(qemu_img_info_command,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL)
|
||||
except Exception as e:
|
||||
# Command itself failed i.e maybe qemu-img not found
|
||||
# Maybe Popen crashes some other ways
|
||||
logging.exception(e)
|
||||
else:
|
||||
# At least we run the qemu-img info command
|
||||
# It may have not executed successfully. If thats
|
||||
# the case, our next command grep would also fail
|
||||
try:
|
||||
# Fetching the lines containing "file format"
|
||||
command = ["grep", "-e", "file format"]
|
||||
output = subprocess.check_output(command, stdin=qemu_img_info.stdout)
|
||||
except Exception as e:
|
||||
logging.exception(e)
|
||||
else:
|
||||
# grep command runs successfully, it implies
|
||||
# the previous command qemu-img would have
|
||||
# run successfully
|
||||
output = output.decode("utf-8").strip()
|
||||
if "qcow2" in output:
|
||||
try:
|
||||
# Convert .qcow2 to .raw
|
||||
subprocess.check_output(qemu_img_convert_command)
|
||||
except Exception as e:
|
||||
logging.exception(e)
|
||||
else:
|
||||
# Image successfully converted
|
||||
try:
|
||||
# Import image either to ceph/filesystem
|
||||
subprocess.check_output(image_import_command)
|
||||
except Exception as e:
|
||||
logging.exception(e)
|
||||
else:
|
||||
# Image imported successfully
|
||||
try:
|
||||
subprocess.check_output(snapshot_creation_command)
|
||||
subprocess.check_output(snapshot_protect_command)
|
||||
except Exception as e:
|
||||
logging.exception(e)
|
||||
else:
|
||||
# Everything is successfully done
|
||||
image.value["status"] = "CREATED"
|
||||
client.put(image.key, json.dumps(image.value))
|
||||
else:
|
||||
# The user provided image is not in qcow2 format
|
||||
image.value["status"] = "INVALID_FORMAT"
|
||||
client.put(image.key, json.dumps(image.value))
|
||||
|
||||
try:
|
||||
os.remove("image.raw")
|
||||
except OSError:
|
||||
except Exception:
|
||||
pass
|
||||
Loading…
Reference in a new issue