mravi/uncloud-mravi
1
0
Fork 0
forked from uncloud/uncloud
Code Issues Pull requests Projects Releases Wiki Activity

No longer using xattrs as they don't work on tmpfs/rootfs

Browse source
This commit is contained in:
ahmadbilalkhalid 2020-01-05 17:21:26 +05:00
commit 180f6f4989
1 changed files with 36 additions and 42 deletions
Download patch file Download diff file Expand all files Collapse all files

68
uncloud/filescanner/main.py
View file

@ -4,6 +4,7 @@ import pathlib
import subprocess as sp import subprocess as sp
import time import time
import argparse import argparse
import bitmath
from uuid import uuid4 from uuid import uuid4
@ -28,66 +29,59 @@ def sha512sum(file: str):
if not isinstance(file, str): if not isinstance(file, str):
raise TypeError raise TypeError
try: try:
output = sp.check_output(["sha512sum", file], stderr=sp.PIPE) output = sp.check_output(['sha512sum', file], stderr=sp.PIPE)
except sp.CalledProcessError as e: except sp.CalledProcessError as e:
error = e.stderr.decode("utf-8") error = e.stderr.decode('utf-8')
if "No such file or directory" in error: if 'No such file or directory' in error:
raise FileNotFoundError from None raise FileNotFoundError from None
else: else:
output = output.decode("utf-8").strip() output = output.decode('utf-8').strip()
output = output.split(" ") output = output.split(' ')
return output[0] return output[0]
return None return None
def track_file(file, base_dir): def track_file(file, base_dir):
file_id = uuid4() file_path = file.relative_to(base_dir)
# Get Username # Get Username
owner = pathlib.Path(file).parts[len(pathlib.Path(base_dir).parts)] try:
owner = file_path.parts[0]
# Get Creation Date of File except IndexError:
# Here, we are assuming that ctime is creation time pass
# which is mostly not true. else:
file_path = file_path.relative_to(owner)
creation_date = time.ctime(os.stat(file).st_ctime) creation_date = time.ctime(os.stat(file).st_ctime)
file_path = pathlib.Path(file).parts[-1] entry_key = os.path.join(settings['etcd']['file_prefix'], str(uuid4()))
# Create Entry
entry_key = os.path.join(
settings["etcd"]["file_prefix"], str(file_id)
)
entry_value = { entry_value = {
"filename": file_path, 'filename': str(file_path),
"owner": owner, 'owner': owner,
"sha512sum": sha512sum(file), 'sha512sum': sha512sum(str(file)),
"creation_date": creation_date, 'creation_date': creation_date,
"size": os.path.getsize(file), 'size': str(bitmath.Byte(os.path.getsize(str(file))).to_MB()),
} }
logger.info("Tracking %s", file) logger.info('Tracking %s', file)
shared.etcd_client.put(entry_key, entry_value, value_in_json=True) shared.etcd_client.put(entry_key, entry_value, value_in_json=True)
os.setxattr(file, "user.utracked", b"True")
def main(debug=False): def main(debug=False):
base_dir = settings["storage"]["file_dir"] base_dir = pathlib.Path(settings['storage']['file_dir'])
# Recursively Get All Files and Folder below BASE_DIR # Recursively Get All Files and Folder below BASE_DIR
files = glob.glob("{}/**".format(base_dir), recursive=True) files = glob.glob('{}/**'.format(base_dir), recursive=True)
files = [pathlib.Path(f) for f in files if pathlib.Path(f).is_file()]
# Retain only Files # Files that are already tracked
files = [file for file in files if os.path.isfile(file)] tracked_files = [
pathlib.Path(os.path.join(base_dir, f.value['owner'], f.value['filename']))
untracked_files = [] for f in shared.etcd_client.get_prefix(settings['etcd']['file_prefix'], value_in_json=True)
for file in files: ]
try: untracked_files = set(files) - set(tracked_files)
os.getxattr(file, "user.utracked") for file in untracked_files:
except OSError:
track_file(file, base_dir) track_file(file, base_dir)
untracked_files.append(file)
if __name__ == "__main__": if __name__ == '__main__':
main() main()