From 180f6f4989133b4a27833d6f9eebc7cc26cf02ce Mon Sep 17 00:00:00 2001 From: meow Date: Sun, 5 Jan 2020 17:21:26 +0500 Subject: [PATCH] No longer using xattrs as they don't work on tmpfs/rootfs --- uncloud/filescanner/main.py | 78 +++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 42 deletions(-) diff --git a/uncloud/filescanner/main.py b/uncloud/filescanner/main.py index c81fbbe..e4d807c 100755 --- a/uncloud/filescanner/main.py +++ b/uncloud/filescanner/main.py @@ -4,6 +4,7 @@ import pathlib import subprocess as sp import time import argparse +import bitmath from uuid import uuid4 @@ -28,66 +29,59 @@ def sha512sum(file: str): if not isinstance(file, str): raise TypeError try: - output = sp.check_output(["sha512sum", file], stderr=sp.PIPE) + output = sp.check_output(['sha512sum', file], stderr=sp.PIPE) except sp.CalledProcessError as e: - error = e.stderr.decode("utf-8") - if "No such file or directory" in error: + error = e.stderr.decode('utf-8') + if 'No such file or directory' in error: raise FileNotFoundError from None else: - output = output.decode("utf-8").strip() - output = output.split(" ") + output = output.decode('utf-8').strip() + output = output.split(' ') return output[0] return None def track_file(file, base_dir): - file_id = uuid4() + file_path = file.relative_to(base_dir) # Get Username - owner = pathlib.Path(file).parts[len(pathlib.Path(base_dir).parts)] + try: + owner = file_path.parts[0] + except IndexError: + pass + else: + file_path = file_path.relative_to(owner) + creation_date = time.ctime(os.stat(file).st_ctime) - # Get Creation Date of File - # Here, we are assuming that ctime is creation time - # which is mostly not true. - creation_date = time.ctime(os.stat(file).st_ctime) + entry_key = os.path.join(settings['etcd']['file_prefix'], str(uuid4())) + entry_value = { + 'filename': str(file_path), + 'owner': owner, + 'sha512sum': sha512sum(str(file)), + 'creation_date': creation_date, + 'size': str(bitmath.Byte(os.path.getsize(str(file))).to_MB()), + } - file_path = pathlib.Path(file).parts[-1] + logger.info('Tracking %s', file) - # Create Entry - entry_key = os.path.join( - settings["etcd"]["file_prefix"], str(file_id) - ) - entry_value = { - "filename": file_path, - "owner": owner, - "sha512sum": sha512sum(file), - "creation_date": creation_date, - "size": os.path.getsize(file), - } - - logger.info("Tracking %s", file) - - shared.etcd_client.put(entry_key, entry_value, value_in_json=True) - os.setxattr(file, "user.utracked", b"True") + shared.etcd_client.put(entry_key, entry_value, value_in_json=True) def main(debug=False): - base_dir = settings["storage"]["file_dir"] - + base_dir = pathlib.Path(settings['storage']['file_dir']) # Recursively Get All Files and Folder below BASE_DIR - files = glob.glob("{}/**".format(base_dir), recursive=True) + files = glob.glob('{}/**'.format(base_dir), recursive=True) + files = [pathlib.Path(f) for f in files if pathlib.Path(f).is_file()] - # Retain only Files - files = [file for file in files if os.path.isfile(file)] - - untracked_files = [] - for file in files: - try: - os.getxattr(file, "user.utracked") - except OSError: - track_file(file, base_dir) - untracked_files.append(file) + # Files that are already tracked + tracked_files = [ + pathlib.Path(os.path.join(base_dir, f.value['owner'], f.value['filename'])) + for f in shared.etcd_client.get_prefix(settings['etcd']['file_prefix'], value_in_json=True) + ] + untracked_files = set(files) - set(tracked_files) + for file in untracked_files: + track_file(file, base_dir) -if __name__ == "__main__": +if __name__ == '__main__': main()