From 3f17ee0da1e855cb144c9ecd7f1ffa902c70568d Mon Sep 17 00:00:00 2001 From: Ahmad Bilal Khalid Date: Sat, 29 Jun 2019 18:16:34 +0500 Subject: [PATCH] initial commit --- .gitignore | 4 ++ .gitmodules | 3 + Pipfile | 14 +++++ Pipfile.lock | 158 ++++++++++++++++++++++++++++++++++++++++++++++++++ etcd3_wrapper | 1 + main.py | 87 +++++++++++++++++++++++++++ 6 files changed, 267 insertions(+) create mode 100644 .gitignore create mode 100644 .gitmodules create mode 100644 Pipfile create mode 100644 Pipfile.lock create mode 160000 etcd3_wrapper create mode 100644 main.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9b30543 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.env +.idea/ +venv/ +var/ \ No newline at end of file diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..004386a --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "etcd3_wrapper"] + path = etcd3_wrapper + url = https://code.ungleich.ch/ahmedbilal/etcd3_wrapper.git diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..ed43d26 --- /dev/null +++ b/Pipfile @@ -0,0 +1,14 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] + +[packages] +python-decouple = "*" +xattr = "*" +etcd3 = "*" + +[requires] +python_version = "3.7" diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 0000000..54b381e --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,158 @@ +{ + "_meta": { + "hash": { + "sha256": "22c474ecb2c9dc36027dceb71527ccb3650d8b59334b3aa49cea4351c6c8d4e9" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.7" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "cffi": { + "hashes": [ + "sha256:041c81822e9f84b1d9c401182e174996f0bae9991f33725d059b771744290774", + "sha256:046ef9a22f5d3eed06334d01b1e836977eeef500d9b78e9ef693f9380ad0b83d", + "sha256:066bc4c7895c91812eff46f4b1c285220947d4aa46fa0a2651ff85f2afae9c90", + "sha256:066c7ff148ae33040c01058662d6752fd73fbc8e64787229ea8498c7d7f4041b", + "sha256:2444d0c61f03dcd26dbf7600cf64354376ee579acad77aef459e34efcb438c63", + "sha256:300832850b8f7967e278870c5d51e3819b9aad8f0a2c8dbe39ab11f119237f45", + "sha256:34c77afe85b6b9e967bd8154e3855e847b70ca42043db6ad17f26899a3df1b25", + "sha256:46de5fa00f7ac09f020729148ff632819649b3e05a007d286242c4882f7b1dc3", + "sha256:4aa8ee7ba27c472d429b980c51e714a24f47ca296d53f4d7868075b175866f4b", + "sha256:4d0004eb4351e35ed950c14c11e734182591465a33e960a4ab5e8d4f04d72647", + "sha256:4e3d3f31a1e202b0f5a35ba3bc4eb41e2fc2b11c1eff38b362de710bcffb5016", + "sha256:50bec6d35e6b1aaeb17f7c4e2b9374ebf95a8975d57863546fa83e8d31bdb8c4", + "sha256:55cad9a6df1e2a1d62063f79d0881a414a906a6962bc160ac968cc03ed3efcfb", + "sha256:5662ad4e4e84f1eaa8efce5da695c5d2e229c563f9d5ce5b0113f71321bcf753", + "sha256:59b4dc008f98fc6ee2bb4fd7fc786a8d70000d058c2bbe2698275bc53a8d3fa7", + "sha256:73e1ffefe05e4ccd7bcea61af76f36077b914f92b76f95ccf00b0c1b9186f3f9", + "sha256:a1f0fd46eba2d71ce1589f7e50a9e2ffaeb739fb2c11e8192aa2b45d5f6cc41f", + "sha256:a2e85dc204556657661051ff4bab75a84e968669765c8a2cd425918699c3d0e8", + "sha256:a5457d47dfff24882a21492e5815f891c0ca35fefae8aa742c6c263dac16ef1f", + "sha256:a8dccd61d52a8dae4a825cdbb7735da530179fea472903eb871a5513b5abbfdc", + "sha256:ae61af521ed676cf16ae94f30fe202781a38d7178b6b4ab622e4eec8cefaff42", + "sha256:b012a5edb48288f77a63dba0840c92d0504aa215612da4541b7b42d849bc83a3", + "sha256:d2c5cfa536227f57f97c92ac30c8109688ace8fa4ac086d19d0af47d134e2909", + "sha256:d42b5796e20aacc9d15e66befb7a345454eef794fdb0737d1af593447c6c8f45", + "sha256:dee54f5d30d775f525894d67b1495625dd9322945e7fee00731952e0368ff42d", + "sha256:e070535507bd6aa07124258171be2ee8dfc19119c28ca94c9dfb7efd23564512", + "sha256:e1ff2748c84d97b065cc95429814cdba39bcbd77c9c85c89344b317dc0d9cbff", + "sha256:ed851c75d1e0e043cbf5ca9a8e1b13c4c90f3fbd863dacb01c0808e2b5204201" + ], + "version": "==1.12.3" + }, + "etcd3": { + "hashes": [ + "sha256:25a524b9f032c6631ff0097532907dea81243eaa63c3744510fd1598cc4e0e87" + ], + "index": "pypi", + "version": "==0.10.0" + }, + "grpcio": { + "hashes": [ + "sha256:0232add03144dd3cf9b660e2718244cb8e175370dca4d3855cb4e489a7811b53", + "sha256:0f20e6dcb1b8662cdca033bb97c0a8116a5343e3ebc7f71c5fe7f89039978350", + "sha256:10b07a623d33d4966f45c85d410bc6a79c5ac6341f06c3beda6c22be12cbfe07", + "sha256:10c0476d5a52d21f402fc073745dc43b87cc8e080a1f49bbff4e1059019310fb", + "sha256:289dae0b35c59d191c524e976dd0a6f8c995d2062e72621eb866ad0f4472a635", + "sha256:2be726f16142d358a0df1e81d583d6820ee561a7856a79cca2fbe49989308be7", + "sha256:4338d2a81f5b4ca022e085040b3cfce19419a5ce44aa7e6810ac1df05365bed7", + "sha256:4c535b46f20e66bee3097583231977e721acdfcb1671d1490c99b7be8902ce18", + "sha256:557154aef70a0e979700cc9528bc8b606b668084a29a0d57dbc4b06b078a2f1c", + "sha256:5bfdd7e6647498f979dc46583723c852d97b25afe995d55aa1c76a5f9816bc1f", + "sha256:87d8943ae7aa6ca5bbad732867d7f17d2550e4966a0c15b52088e8b579422e47", + "sha256:89d8719d8de4d137678f7caa979e1b0a6fd4026f8096ceef8c2d164bbabefaf2", + "sha256:9c3f4af989ce860710ac1864dc2e867dd87e6cee51a2368df1b253596868e52f", + "sha256:9da52c3c728883aee429bb7c315049f50b2139f680cd86bb1165418e4f93a982", + "sha256:9e9736659987beab42d18525ed10d21f80a1ba8389eac03425fbfd5684e6bbf0", + "sha256:9ebcbb1a054cab362d29d3be571d43d6b9b23302d9fc4b43e5327000da1680a9", + "sha256:a93e08636623e24c939851e2e0c0140b14f524b2980c9cdc4ea52b70a871c7e0", + "sha256:ac322d86d1a079e0a118d544443ee16f320af0062c191b4754c0c6ec2fc79310", + "sha256:b1fb101459868f52df6b61e7bb13375e50badf17a160e39fe1d51ae19e53f461", + "sha256:b39aac96cceac624a23d540473835086a3ffa77c91030189988c073488434493", + "sha256:b65507bc273c6dbf539175a786a344cc0ac78d50e5584f72c6599733f8a3301f", + "sha256:be5bb6e47417e537c884a2e2ff2e1a8b2c064a998fcfdfcc67528d4e63e7ebaf", + "sha256:c92de6a28a909c4f460dc1bbbcb50d676cf0b1f40224b222761f73fdd851b522", + "sha256:c9f5962eb7fa7607b20eb0e4f59ed35829bd600fc0eacb626a6db83229a3e445", + "sha256:d00bdf9c546ed6e649f785c55b05288e8b2dbb6bf2eb74b6c579fa0d591d35bd", + "sha256:da804b1dd8293bd9d61b1e6ea989c887ba042a808a4fbdd80001cfa059aafed2", + "sha256:ead6c5aa3e807345913649c3be395aaca2bbb2d225f18b8f31f37eab225508f6", + "sha256:eb4d81550ce6f826af4ec6e8d98be347fe96291d718bf115c3f254621ae8d98d", + "sha256:ef6a18ec8fd32ec81748fe720544ea2fb2d2dc50fd6d06739d5e2eb8f0626a1c", + "sha256:fad42835656e0b6d3b7ffc900598e776722e30f43b7234a48f2576ca30f31a47", + "sha256:fb98dbfee0d963b49ae5754554028cf62e6bd695f22de16d242ba9d2f0b7339b", + "sha256:fb9cd9bb8d26dc17c2dd715a46bca3a879ec8283879b164e85863110dc6e3b2a" + ], + "version": "==1.21.1" + }, + "protobuf": { + "hashes": [ + "sha256:03f43eac9d5b651f976e91cf46a25b75e5779d98f0f4114b0abfed83376d75f8", + "sha256:0c94b21e6de01362f91a86b372555d22a60b59708599ca9d5032ae9fdf8e3538", + "sha256:2d2a9f30f61f4063fadd7fb68a2510a6939b43c0d6ceeec5c4704f22225da28e", + "sha256:34a0b05fca061e4abb77dd180209f68d8637115ff319f51e28a6a9382d69853a", + "sha256:358710fd0db25372edcf1150fa691f48376a134a6c69ce29f38f185eea7699e6", + "sha256:41e47198b94c27ba05a08b4a95160656105745c462af574e4bcb0807164065c0", + "sha256:8c61cc8a76e9d381c665aecc5105fa0f1878cf7db8b5cd17202603bcb386d0fc", + "sha256:a6eebc4db759e58fdac02efcd3028b811effac881d8a5bad1996e4e8ee6acb47", + "sha256:a9c12f7c98093da0a46ba76ec40ace725daa1ac4038c41e4b1466afb5c45bb01", + "sha256:cb95068492ba0859b8c9e61fa8ba206a83c64e5d0916fb4543700b2e2b214115", + "sha256:cd98476ce7bb4dcd6a7b101f5eecdc073dafea19f311e36eb8fba1a349346277", + "sha256:ce64cfbea18c535176bdaa10ba740c0fc4c6d998a3f511c17bedb0ae4b3b167c", + "sha256:dcbb59eac73fd454e8f2c5fba9e3d3320fd4707ed6a9d3ea3717924a6f0903ea", + "sha256:dd67f34458ae716029e2a71ede998e9092493b62a519236ca52e3c5202096c87", + "sha256:e3c96056eb5b7284a20e256cb0bf783c8f36ad82a4ae5434a7b7cd02384144a7", + "sha256:f612d584d7a27e2f39e7b17878430a959c1bc09a74ba09db096b468558e5e126", + "sha256:f6de8a7d6122297b81566e5bd4df37fd5d62bec14f8f90ebff8ede1c9726cd0a", + "sha256:fa529d9261682b24c2aaa683667253175c9acebe0a31105394b221090da75832" + ], + "version": "==3.8.0" + }, + "pycparser": { + "hashes": [ + "sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3" + ], + "version": "==2.19" + }, + "python-decouple": { + "hashes": [ + "sha256:1317df14b43efee4337a4aa02914bf004f010cd56d6c4bd894e6474ec8c4fe2d" + ], + "index": "pypi", + "version": "==3.1" + }, + "six": { + "hashes": [ + "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", + "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" + ], + "version": "==1.12.0" + }, + "tenacity": { + "hashes": [ + "sha256:a0c3c5f7ae0c33f5556c775ca059c12d6fd8ab7121613a713e8b7d649908571b", + "sha256:b87c1934daa0b2ccc7db153c37b8bf91d12f165936ade8628e7b962b92dc7705" + ], + "version": "==5.0.4" + }, + "xattr": { + "hashes": [ + "sha256:117f864caf6428a75ed1263d29e020afb5db30dcbdfc1b79da9f19ec8748d2df", + "sha256:4ea01c6d23252d9caeae613ea63191095924ec14df3a9390a404e84df6a1ddb7", + "sha256:7cb1b28eeab4fe99cc4350e831434142fce658f7d03f173ff7722144e6a47458", + "sha256:c0a5ec84ec3b98f0082cf3bafd299f2fa89a4e54ae4b0b1d6fab5ceea3d0d37a" + ], + "index": "pypi", + "version": "==0.9.6" + } + }, + "develop": {} +} diff --git a/etcd3_wrapper b/etcd3_wrapper new file mode 160000 index 0000000..59974e2 --- /dev/null +++ b/etcd3_wrapper @@ -0,0 +1 @@ +Subproject commit 59974e2047bbea08b72d32129fd905bbea6e9d70 diff --git a/main.py b/main.py new file mode 100644 index 0000000..cf54b20 --- /dev/null +++ b/main.py @@ -0,0 +1,87 @@ +import os +import glob +import xattr +import pathlib +import time +import hashlib + +from decouple import config +from etcd3_wrapper import Etcd3Wrapper +from uuid import uuid4 + + +def getxattr(f, attr, symlink=False): + try: + return xattr.getxattr(f, attr, symlink).decode("utf-8") + except OSError as _: + return None + + +def sha512sum(filename): + _sum = hashlib.sha512() + buffer_size = 2**16 + + with open(filename, "rb") as f: + while True: + data = f.read(buffer_size) + if not data: + break + _sum.update(data) + + return _sum.hexdigest() + + +BASE_DIR = config("BASE_DIR") + +FILE_PREFIX = config("FILE_PREFIX") + +etcd_client = Etcd3Wrapper() + +# Recursively Get All Files and Folder below BASE_DIR +files = glob.glob(f"{BASE_DIR}/**", recursive=True) + +# Retain only Files +files = list(filter(lambda f: os.path.isfile(f), files)) + +untracked_files = list( + filter(lambda f: not bool(getxattr(f, "user.utracked")), files) +) + +tracked_files = list( + filter(lambda f: f not in untracked_files, files) +) +for file in untracked_files: + file_id = uuid4() + + # Get Username + owner = pathlib.Path(file).parts[2] + + # Get Creation Date of File + # Here, we are assuming that ctime is creation time + # which is mostly not true. + creation_date = time.ctime(os.stat(file).st_ctime) + + # Get File Size + size = os.path.getsize(file) + + # Compute sha512 sum + sha_sum = sha512sum(file) + + # File Path excluding base and username + file_path = pathlib.Path(file).parts[3:] + file_path = os.path.join(*file_path) + + # Create Entry + entry_key = f"{FILE_PREFIX}/{file_id}" + entry_value = {"filename": file_path, + "owner": owner, + "sha512sum": sha_sum, + "creation_date": creation_date, + "size": size + } + + print(f"Tracking {file}") + + # Insert Entry + etcd_client.put(entry_key, entry_value, value_in_json=True) + xattr.setxattr(file, b"user.utracked", b"True")