uncloud-mravi/filescanner/main.py
meow 93dee1c9fc New Features + Refactoring
1. User can now use image name instead of image uuid when creation vm.
   For Example, now user can create an alpine vm using the following
   command
   ```shell
   ucloud-cli vm create --vm-name myvm --cpu 2 --ram '2GB' \
       --os-ssd '10GB' --image images:alpine
   ```
2. Instead of directly running code, code is now placed under a function
   main and is called using the following code
   ```python
   if __name__ == "__main__":
       main()
   ```
3. Multiprocess (Process) is used instead of threading (Thread) to update
   heart beat of host.
4. IP Address of vm is included in vm's status which is retrieved by the
   following command
   ```shell
   ucloud-cli vm status --vm-name myvm
   ```
2019-11-02 20:42:24 +05:00

128 lines
No EOL
3.4 KiB
Python
Executable file

import os
import glob
import pathlib
import time
import hashlib
import subprocess as sp
from decouple import config
from etcd3_wrapper import Etcd3Wrapper
from uuid import uuid4
def getxattr(file, attr):
"""Get specified user extended attribute (arg:attr) of a file (arg:file)"""
try:
attr = "user." + attr
value = sp.check_output(['getfattr', file,
'--name', attr,
'--only-values',
'--absolute-names'])
value = value.decode("utf-8")
except sp.CalledProcessError:
value = None
return value
def setxattr(file, attr, value):
"""Set specified user extended attribute (arg:attr) equal to (arg:value)
of a file (arg:file)"""
attr = "user." + attr
sp.check_output(['setfattr', file,
'--name', attr,
'--value', str(value)])
def sha512sum(file: str):
"""Use sha512sum utility to compute sha512 sum of arg:file
IF arg:file does not exists:
raise FileNotFoundError exception
ELSE IF sum successfully computer:
return computed sha512 sum
ELSE:
return None
"""
if not isinstance(file, str): raise TypeError
try:
output = sp.check_output(["sha512sum", file], stderr=sp.PIPE)
except sp.CalledProcessError as e:
error = e.stderr.decode("utf-8")
if "No such file or directory" in error:
raise FileNotFoundError from None
else:
output = output.decode("utf-8").strip()
output = output.split(" ")
return output[0]
return None
try:
sp.check_output(['which', 'getfattr'])
sp.check_output(['which', 'setfattr'])
except Exception as e:
print(e)
print('Make sure you have getfattr and setfattr available')
exit(1)
def main():
BASE_DIR = config("BASE_DIR")
FILE_PREFIX = config("FILE_PREFIX")
etcd_client = Etcd3Wrapper(host=config("ETCD_URL"))
# Recursively Get All Files and Folder below BASE_DIR
files = glob.glob("{}/**".format(BASE_DIR), recursive=True)
# Retain only Files
files = list(filter(os.path.isfile, files))
untracked_files = list(
filter(lambda f: not bool(getxattr(f, "user.utracked")), files)
)
tracked_files = list(
filter(lambda f: f not in untracked_files, files)
)
for file in untracked_files:
file_id = uuid4()
# Get Username
owner = pathlib.Path(file).parts[3]
# Get Creation Date of File
# Here, we are assuming that ctime is creation time
# which is mostly not true.
creation_date = time.ctime(os.stat(file).st_ctime)
# Get File Size
size = os.path.getsize(file)
# Compute sha512 sum
sha_sum = sha512sum(file)
# File Path excluding base and username
file_path = pathlib.Path(file).parts[4:]
file_path = os.path.join(*file_path)
# Create Entry
entry_key = os.path.join(FILE_PREFIX, str(file_id))
entry_value = {
"filename": file_path,
"owner": owner,
"sha512sum": sha_sum,
"creation_date": creation_date,
"size": size
}
print("Tracking {}".format(file))
# Insert Entry
etcd_client.put(entry_key, entry_value, value_in_json=True)
setxattr(file, "user.utracked", True)
if __name__ == "__main__":
main()