initial commit
This commit is contained in:
commit
3f17ee0da1
6 changed files with 267 additions and 0 deletions
4
.gitignore
vendored
Normal file
4
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,4 @@
|
||||||
|
.env
|
||||||
|
.idea/
|
||||||
|
venv/
|
||||||
|
var/
|
||||||
3
.gitmodules
vendored
Normal file
3
.gitmodules
vendored
Normal file
|
|
@ -0,0 +1,3 @@
|
||||||
|
[submodule "etcd3_wrapper"]
|
||||||
|
path = etcd3_wrapper
|
||||||
|
url = https://code.ungleich.ch/ahmedbilal/etcd3_wrapper.git
|
||||||
14
Pipfile
Normal file
14
Pipfile
Normal file
|
|
@ -0,0 +1,14 @@
|
||||||
|
[[source]]
|
||||||
|
name = "pypi"
|
||||||
|
url = "https://pypi.org/simple"
|
||||||
|
verify_ssl = true
|
||||||
|
|
||||||
|
[dev-packages]
|
||||||
|
|
||||||
|
[packages]
|
||||||
|
python-decouple = "*"
|
||||||
|
xattr = "*"
|
||||||
|
etcd3 = "*"
|
||||||
|
|
||||||
|
[requires]
|
||||||
|
python_version = "3.7"
|
||||||
158
Pipfile.lock
generated
Normal file
158
Pipfile.lock
generated
Normal file
|
|
@ -0,0 +1,158 @@
|
||||||
|
{
|
||||||
|
"_meta": {
|
||||||
|
"hash": {
|
||||||
|
"sha256": "22c474ecb2c9dc36027dceb71527ccb3650d8b59334b3aa49cea4351c6c8d4e9"
|
||||||
|
},
|
||||||
|
"pipfile-spec": 6,
|
||||||
|
"requires": {
|
||||||
|
"python_version": "3.7"
|
||||||
|
},
|
||||||
|
"sources": [
|
||||||
|
{
|
||||||
|
"name": "pypi",
|
||||||
|
"url": "https://pypi.org/simple",
|
||||||
|
"verify_ssl": true
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"default": {
|
||||||
|
"cffi": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:041c81822e9f84b1d9c401182e174996f0bae9991f33725d059b771744290774",
|
||||||
|
"sha256:046ef9a22f5d3eed06334d01b1e836977eeef500d9b78e9ef693f9380ad0b83d",
|
||||||
|
"sha256:066bc4c7895c91812eff46f4b1c285220947d4aa46fa0a2651ff85f2afae9c90",
|
||||||
|
"sha256:066c7ff148ae33040c01058662d6752fd73fbc8e64787229ea8498c7d7f4041b",
|
||||||
|
"sha256:2444d0c61f03dcd26dbf7600cf64354376ee579acad77aef459e34efcb438c63",
|
||||||
|
"sha256:300832850b8f7967e278870c5d51e3819b9aad8f0a2c8dbe39ab11f119237f45",
|
||||||
|
"sha256:34c77afe85b6b9e967bd8154e3855e847b70ca42043db6ad17f26899a3df1b25",
|
||||||
|
"sha256:46de5fa00f7ac09f020729148ff632819649b3e05a007d286242c4882f7b1dc3",
|
||||||
|
"sha256:4aa8ee7ba27c472d429b980c51e714a24f47ca296d53f4d7868075b175866f4b",
|
||||||
|
"sha256:4d0004eb4351e35ed950c14c11e734182591465a33e960a4ab5e8d4f04d72647",
|
||||||
|
"sha256:4e3d3f31a1e202b0f5a35ba3bc4eb41e2fc2b11c1eff38b362de710bcffb5016",
|
||||||
|
"sha256:50bec6d35e6b1aaeb17f7c4e2b9374ebf95a8975d57863546fa83e8d31bdb8c4",
|
||||||
|
"sha256:55cad9a6df1e2a1d62063f79d0881a414a906a6962bc160ac968cc03ed3efcfb",
|
||||||
|
"sha256:5662ad4e4e84f1eaa8efce5da695c5d2e229c563f9d5ce5b0113f71321bcf753",
|
||||||
|
"sha256:59b4dc008f98fc6ee2bb4fd7fc786a8d70000d058c2bbe2698275bc53a8d3fa7",
|
||||||
|
"sha256:73e1ffefe05e4ccd7bcea61af76f36077b914f92b76f95ccf00b0c1b9186f3f9",
|
||||||
|
"sha256:a1f0fd46eba2d71ce1589f7e50a9e2ffaeb739fb2c11e8192aa2b45d5f6cc41f",
|
||||||
|
"sha256:a2e85dc204556657661051ff4bab75a84e968669765c8a2cd425918699c3d0e8",
|
||||||
|
"sha256:a5457d47dfff24882a21492e5815f891c0ca35fefae8aa742c6c263dac16ef1f",
|
||||||
|
"sha256:a8dccd61d52a8dae4a825cdbb7735da530179fea472903eb871a5513b5abbfdc",
|
||||||
|
"sha256:ae61af521ed676cf16ae94f30fe202781a38d7178b6b4ab622e4eec8cefaff42",
|
||||||
|
"sha256:b012a5edb48288f77a63dba0840c92d0504aa215612da4541b7b42d849bc83a3",
|
||||||
|
"sha256:d2c5cfa536227f57f97c92ac30c8109688ace8fa4ac086d19d0af47d134e2909",
|
||||||
|
"sha256:d42b5796e20aacc9d15e66befb7a345454eef794fdb0737d1af593447c6c8f45",
|
||||||
|
"sha256:dee54f5d30d775f525894d67b1495625dd9322945e7fee00731952e0368ff42d",
|
||||||
|
"sha256:e070535507bd6aa07124258171be2ee8dfc19119c28ca94c9dfb7efd23564512",
|
||||||
|
"sha256:e1ff2748c84d97b065cc95429814cdba39bcbd77c9c85c89344b317dc0d9cbff",
|
||||||
|
"sha256:ed851c75d1e0e043cbf5ca9a8e1b13c4c90f3fbd863dacb01c0808e2b5204201"
|
||||||
|
],
|
||||||
|
"version": "==1.12.3"
|
||||||
|
},
|
||||||
|
"etcd3": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:25a524b9f032c6631ff0097532907dea81243eaa63c3744510fd1598cc4e0e87"
|
||||||
|
],
|
||||||
|
"index": "pypi",
|
||||||
|
"version": "==0.10.0"
|
||||||
|
},
|
||||||
|
"grpcio": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:0232add03144dd3cf9b660e2718244cb8e175370dca4d3855cb4e489a7811b53",
|
||||||
|
"sha256:0f20e6dcb1b8662cdca033bb97c0a8116a5343e3ebc7f71c5fe7f89039978350",
|
||||||
|
"sha256:10b07a623d33d4966f45c85d410bc6a79c5ac6341f06c3beda6c22be12cbfe07",
|
||||||
|
"sha256:10c0476d5a52d21f402fc073745dc43b87cc8e080a1f49bbff4e1059019310fb",
|
||||||
|
"sha256:289dae0b35c59d191c524e976dd0a6f8c995d2062e72621eb866ad0f4472a635",
|
||||||
|
"sha256:2be726f16142d358a0df1e81d583d6820ee561a7856a79cca2fbe49989308be7",
|
||||||
|
"sha256:4338d2a81f5b4ca022e085040b3cfce19419a5ce44aa7e6810ac1df05365bed7",
|
||||||
|
"sha256:4c535b46f20e66bee3097583231977e721acdfcb1671d1490c99b7be8902ce18",
|
||||||
|
"sha256:557154aef70a0e979700cc9528bc8b606b668084a29a0d57dbc4b06b078a2f1c",
|
||||||
|
"sha256:5bfdd7e6647498f979dc46583723c852d97b25afe995d55aa1c76a5f9816bc1f",
|
||||||
|
"sha256:87d8943ae7aa6ca5bbad732867d7f17d2550e4966a0c15b52088e8b579422e47",
|
||||||
|
"sha256:89d8719d8de4d137678f7caa979e1b0a6fd4026f8096ceef8c2d164bbabefaf2",
|
||||||
|
"sha256:9c3f4af989ce860710ac1864dc2e867dd87e6cee51a2368df1b253596868e52f",
|
||||||
|
"sha256:9da52c3c728883aee429bb7c315049f50b2139f680cd86bb1165418e4f93a982",
|
||||||
|
"sha256:9e9736659987beab42d18525ed10d21f80a1ba8389eac03425fbfd5684e6bbf0",
|
||||||
|
"sha256:9ebcbb1a054cab362d29d3be571d43d6b9b23302d9fc4b43e5327000da1680a9",
|
||||||
|
"sha256:a93e08636623e24c939851e2e0c0140b14f524b2980c9cdc4ea52b70a871c7e0",
|
||||||
|
"sha256:ac322d86d1a079e0a118d544443ee16f320af0062c191b4754c0c6ec2fc79310",
|
||||||
|
"sha256:b1fb101459868f52df6b61e7bb13375e50badf17a160e39fe1d51ae19e53f461",
|
||||||
|
"sha256:b39aac96cceac624a23d540473835086a3ffa77c91030189988c073488434493",
|
||||||
|
"sha256:b65507bc273c6dbf539175a786a344cc0ac78d50e5584f72c6599733f8a3301f",
|
||||||
|
"sha256:be5bb6e47417e537c884a2e2ff2e1a8b2c064a998fcfdfcc67528d4e63e7ebaf",
|
||||||
|
"sha256:c92de6a28a909c4f460dc1bbbcb50d676cf0b1f40224b222761f73fdd851b522",
|
||||||
|
"sha256:c9f5962eb7fa7607b20eb0e4f59ed35829bd600fc0eacb626a6db83229a3e445",
|
||||||
|
"sha256:d00bdf9c546ed6e649f785c55b05288e8b2dbb6bf2eb74b6c579fa0d591d35bd",
|
||||||
|
"sha256:da804b1dd8293bd9d61b1e6ea989c887ba042a808a4fbdd80001cfa059aafed2",
|
||||||
|
"sha256:ead6c5aa3e807345913649c3be395aaca2bbb2d225f18b8f31f37eab225508f6",
|
||||||
|
"sha256:eb4d81550ce6f826af4ec6e8d98be347fe96291d718bf115c3f254621ae8d98d",
|
||||||
|
"sha256:ef6a18ec8fd32ec81748fe720544ea2fb2d2dc50fd6d06739d5e2eb8f0626a1c",
|
||||||
|
"sha256:fad42835656e0b6d3b7ffc900598e776722e30f43b7234a48f2576ca30f31a47",
|
||||||
|
"sha256:fb98dbfee0d963b49ae5754554028cf62e6bd695f22de16d242ba9d2f0b7339b",
|
||||||
|
"sha256:fb9cd9bb8d26dc17c2dd715a46bca3a879ec8283879b164e85863110dc6e3b2a"
|
||||||
|
],
|
||||||
|
"version": "==1.21.1"
|
||||||
|
},
|
||||||
|
"protobuf": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:03f43eac9d5b651f976e91cf46a25b75e5779d98f0f4114b0abfed83376d75f8",
|
||||||
|
"sha256:0c94b21e6de01362f91a86b372555d22a60b59708599ca9d5032ae9fdf8e3538",
|
||||||
|
"sha256:2d2a9f30f61f4063fadd7fb68a2510a6939b43c0d6ceeec5c4704f22225da28e",
|
||||||
|
"sha256:34a0b05fca061e4abb77dd180209f68d8637115ff319f51e28a6a9382d69853a",
|
||||||
|
"sha256:358710fd0db25372edcf1150fa691f48376a134a6c69ce29f38f185eea7699e6",
|
||||||
|
"sha256:41e47198b94c27ba05a08b4a95160656105745c462af574e4bcb0807164065c0",
|
||||||
|
"sha256:8c61cc8a76e9d381c665aecc5105fa0f1878cf7db8b5cd17202603bcb386d0fc",
|
||||||
|
"sha256:a6eebc4db759e58fdac02efcd3028b811effac881d8a5bad1996e4e8ee6acb47",
|
||||||
|
"sha256:a9c12f7c98093da0a46ba76ec40ace725daa1ac4038c41e4b1466afb5c45bb01",
|
||||||
|
"sha256:cb95068492ba0859b8c9e61fa8ba206a83c64e5d0916fb4543700b2e2b214115",
|
||||||
|
"sha256:cd98476ce7bb4dcd6a7b101f5eecdc073dafea19f311e36eb8fba1a349346277",
|
||||||
|
"sha256:ce64cfbea18c535176bdaa10ba740c0fc4c6d998a3f511c17bedb0ae4b3b167c",
|
||||||
|
"sha256:dcbb59eac73fd454e8f2c5fba9e3d3320fd4707ed6a9d3ea3717924a6f0903ea",
|
||||||
|
"sha256:dd67f34458ae716029e2a71ede998e9092493b62a519236ca52e3c5202096c87",
|
||||||
|
"sha256:e3c96056eb5b7284a20e256cb0bf783c8f36ad82a4ae5434a7b7cd02384144a7",
|
||||||
|
"sha256:f612d584d7a27e2f39e7b17878430a959c1bc09a74ba09db096b468558e5e126",
|
||||||
|
"sha256:f6de8a7d6122297b81566e5bd4df37fd5d62bec14f8f90ebff8ede1c9726cd0a",
|
||||||
|
"sha256:fa529d9261682b24c2aaa683667253175c9acebe0a31105394b221090da75832"
|
||||||
|
],
|
||||||
|
"version": "==3.8.0"
|
||||||
|
},
|
||||||
|
"pycparser": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3"
|
||||||
|
],
|
||||||
|
"version": "==2.19"
|
||||||
|
},
|
||||||
|
"python-decouple": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:1317df14b43efee4337a4aa02914bf004f010cd56d6c4bd894e6474ec8c4fe2d"
|
||||||
|
],
|
||||||
|
"index": "pypi",
|
||||||
|
"version": "==3.1"
|
||||||
|
},
|
||||||
|
"six": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
|
||||||
|
"sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"
|
||||||
|
],
|
||||||
|
"version": "==1.12.0"
|
||||||
|
},
|
||||||
|
"tenacity": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:a0c3c5f7ae0c33f5556c775ca059c12d6fd8ab7121613a713e8b7d649908571b",
|
||||||
|
"sha256:b87c1934daa0b2ccc7db153c37b8bf91d12f165936ade8628e7b962b92dc7705"
|
||||||
|
],
|
||||||
|
"version": "==5.0.4"
|
||||||
|
},
|
||||||
|
"xattr": {
|
||||||
|
"hashes": [
|
||||||
|
"sha256:117f864caf6428a75ed1263d29e020afb5db30dcbdfc1b79da9f19ec8748d2df",
|
||||||
|
"sha256:4ea01c6d23252d9caeae613ea63191095924ec14df3a9390a404e84df6a1ddb7",
|
||||||
|
"sha256:7cb1b28eeab4fe99cc4350e831434142fce658f7d03f173ff7722144e6a47458",
|
||||||
|
"sha256:c0a5ec84ec3b98f0082cf3bafd299f2fa89a4e54ae4b0b1d6fab5ceea3d0d37a"
|
||||||
|
],
|
||||||
|
"index": "pypi",
|
||||||
|
"version": "==0.9.6"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"develop": {}
|
||||||
|
}
|
||||||
1
etcd3_wrapper
Submodule
1
etcd3_wrapper
Submodule
|
|
@ -0,0 +1 @@
|
||||||
|
Subproject commit 59974e2047bbea08b72d32129fd905bbea6e9d70
|
||||||
87
main.py
Normal file
87
main.py
Normal file
|
|
@ -0,0 +1,87 @@
|
||||||
|
import os
|
||||||
|
import glob
|
||||||
|
import xattr
|
||||||
|
import pathlib
|
||||||
|
import time
|
||||||
|
import hashlib
|
||||||
|
|
||||||
|
from decouple import config
|
||||||
|
from etcd3_wrapper import Etcd3Wrapper
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
|
||||||
|
def getxattr(f, attr, symlink=False):
|
||||||
|
try:
|
||||||
|
return xattr.getxattr(f, attr, symlink).decode("utf-8")
|
||||||
|
except OSError as _:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def sha512sum(filename):
|
||||||
|
_sum = hashlib.sha512()
|
||||||
|
buffer_size = 2**16
|
||||||
|
|
||||||
|
with open(filename, "rb") as f:
|
||||||
|
while True:
|
||||||
|
data = f.read(buffer_size)
|
||||||
|
if not data:
|
||||||
|
break
|
||||||
|
_sum.update(data)
|
||||||
|
|
||||||
|
return _sum.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
BASE_DIR = config("BASE_DIR")
|
||||||
|
|
||||||
|
FILE_PREFIX = config("FILE_PREFIX")
|
||||||
|
|
||||||
|
etcd_client = Etcd3Wrapper()
|
||||||
|
|
||||||
|
# Recursively Get All Files and Folder below BASE_DIR
|
||||||
|
files = glob.glob(f"{BASE_DIR}/**", recursive=True)
|
||||||
|
|
||||||
|
# Retain only Files
|
||||||
|
files = list(filter(lambda f: os.path.isfile(f), files))
|
||||||
|
|
||||||
|
untracked_files = list(
|
||||||
|
filter(lambda f: not bool(getxattr(f, "user.utracked")), files)
|
||||||
|
)
|
||||||
|
|
||||||
|
tracked_files = list(
|
||||||
|
filter(lambda f: f not in untracked_files, files)
|
||||||
|
)
|
||||||
|
for file in untracked_files:
|
||||||
|
file_id = uuid4()
|
||||||
|
|
||||||
|
# Get Username
|
||||||
|
owner = pathlib.Path(file).parts[2]
|
||||||
|
|
||||||
|
# Get Creation Date of File
|
||||||
|
# Here, we are assuming that ctime is creation time
|
||||||
|
# which is mostly not true.
|
||||||
|
creation_date = time.ctime(os.stat(file).st_ctime)
|
||||||
|
|
||||||
|
# Get File Size
|
||||||
|
size = os.path.getsize(file)
|
||||||
|
|
||||||
|
# Compute sha512 sum
|
||||||
|
sha_sum = sha512sum(file)
|
||||||
|
|
||||||
|
# File Path excluding base and username
|
||||||
|
file_path = pathlib.Path(file).parts[3:]
|
||||||
|
file_path = os.path.join(*file_path)
|
||||||
|
|
||||||
|
# Create Entry
|
||||||
|
entry_key = f"{FILE_PREFIX}/{file_id}"
|
||||||
|
entry_value = {"filename": file_path,
|
||||||
|
"owner": owner,
|
||||||
|
"sha512sum": sha_sum,
|
||||||
|
"creation_date": creation_date,
|
||||||
|
"size": size
|
||||||
|
}
|
||||||
|
|
||||||
|
print(f"Tracking {file}")
|
||||||
|
|
||||||
|
# Insert Entry
|
||||||
|
etcd_client.put(entry_key, entry_value, value_in_json=True)
|
||||||
|
xattr.setxattr(file, b"user.utracked", b"True")
|
||||||
Loading…
Reference in a new issue