Compare commits
46 commits
Author | SHA1 | Date | |
---|---|---|---|
|
0c04f67b93 | ||
|
41cd0fe6c6 | ||
|
499b85bcfa | ||
|
2f5af0f2fb | ||
|
a5cf8d66df | ||
|
b41a4afa8b | ||
|
ea547d83c5 | ||
|
867cab7f25 | ||
|
40ddd90e9b | ||
|
24a0ee4c04 | ||
|
4d015b6be8 | ||
|
c38d74feb7 | ||
|
4a45525d00 | ||
|
d2186f1f26 | ||
|
9e714732fb | ||
|
5040c33f96 | ||
|
3cc886f057 | ||
|
12df9e5a7e | ||
|
ff0c1a2895 | ||
|
c2d0a49d4f | ||
|
19b8349e8c | ||
|
6d57adc215 | ||
|
395f2234d5 | ||
|
7aaeb3e98f | ||
|
bd88491cb0 | ||
|
cccadf41f7 | ||
|
17fa4e7279 | ||
|
43ebde240d | ||
|
cff9388a4e | ||
|
1be4e6ce4a | ||
|
8cc0897926 | ||
|
beb9734330 | ||
|
1f4cb3e177 | ||
|
a3acd60556 | ||
|
ef57c4a04d | ||
|
c7f34614dd | ||
|
55279e1aee | ||
|
cf0ff95b36 | ||
|
8cb421c6e4 | ||
|
3ddd76fcce | ||
|
63bb9779dd | ||
|
4b8a7d51be | ||
|
a60ede31fb | ||
|
609418975e | ||
|
30620884c9 | ||
|
42202b901b |
9 changed files with 400 additions and 88 deletions
2
.gitignore
vendored
2
.gitignore
vendored
|
@ -1 +1,3 @@
|
||||||
*.pyc
|
*.pyc
|
||||||
|
dist/
|
||||||
|
*.egg-info/
|
||||||
|
|
5
AUTHORS
Normal file
5
AUTHORS
Normal file
|
@ -0,0 +1,5 @@
|
||||||
|
Author:
|
||||||
|
Darko Poljak <darko.poljak@gmail.com>
|
||||||
|
|
||||||
|
Contributors:
|
||||||
|
matthewi
|
22
README.rst
22
README.rst
|
@ -10,29 +10,31 @@ Print duplicates
|
||||||
|
|
||||||
.. code:: python
|
.. code:: python
|
||||||
|
|
||||||
from sweeper import file_dups
|
from sweeper import Sweeper
|
||||||
dups = file_dups(['images1', 'images2'])
|
swp = Sweeper(['images1', 'images2'])
|
||||||
|
dups = swp.file_dups()
|
||||||
print(dups)
|
print(dups)
|
||||||
|
|
||||||
Remove duplicate files
|
Remove duplicate files
|
||||||
|
|
||||||
.. code:: python
|
.. code:: python
|
||||||
|
|
||||||
from sweeper import file_dups
|
from sweeper import Sweeper
|
||||||
rm_file_dups(['images'])
|
swp = Sweeper(['images1', 'images2'])
|
||||||
|
swp.rm()
|
||||||
|
|
||||||
Perform custom action
|
Perform custom action
|
||||||
|
|
||||||
.. code:: python
|
.. code:: python
|
||||||
|
|
||||||
from sweeper import file_dups
|
from sweeper import Sweeper
|
||||||
for files in iter_file_dups(['images']):
|
swp = Sweeper(['images'])
|
||||||
for fname in files:
|
for f, h, dups in swp:
|
||||||
print('found duplicate file with name: %s' % fname)
|
print('encountered {} which duplicates with already found duplicate files {} with hash {}'.format(f, dups, h))
|
||||||
|
|
||||||
As script::
|
As script::
|
||||||
|
|
||||||
python sweeper.py --help
|
python -m sweeper/sweeper --help
|
||||||
|
|
||||||
As installed console script::
|
As installed console script::
|
||||||
|
|
||||||
|
@ -61,5 +63,5 @@ https://github.com/darko-poljak/sweeper
|
||||||
Tested With
|
Tested With
|
||||||
===========
|
===========
|
||||||
|
|
||||||
Python2.7.6, Python3.3.3
|
Python2.7, Python3
|
||||||
|
|
||||||
|
|
0
TODO
Normal file
0
TODO
Normal file
8
setup.py
8
setup.py
|
@ -1,12 +1,17 @@
|
||||||
import os
|
import os
|
||||||
from setuptools import setup
|
from setuptools import setup
|
||||||
|
|
||||||
|
|
||||||
def read(fname):
|
def read(fname):
|
||||||
return open(os.path.join(os.path.dirname(__file__), fname)).read()
|
return open(os.path.join(os.path.dirname(__file__), fname)).read()
|
||||||
|
|
||||||
|
|
||||||
|
import sweeper.sweeper as sw
|
||||||
|
|
||||||
|
|
||||||
setup(
|
setup(
|
||||||
name='sweeper',
|
name='sweeper',
|
||||||
version='0.2.0',
|
version=sw.__version__,
|
||||||
author='Darko Poljak',
|
author='Darko Poljak',
|
||||||
author_email='darko.poljak@gmail.com',
|
author_email='darko.poljak@gmail.com',
|
||||||
description='Find duplicate files and perform action.',
|
description='Find duplicate files and perform action.',
|
||||||
|
@ -30,4 +35,3 @@ setup(
|
||||||
"Topic :: Software Development :: Libraries :: Python Modules",
|
"Topic :: Software Development :: Libraries :: Python Modules",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
from __future__ import absolute_import
|
from __future__ import absolute_import
|
||||||
from .sweeper import file_dups
|
from .sweeper import Sweeper
|
||||||
|
|
||||||
__all__ = ['file_dups', 'mv_file_dups', 'rm_file_dups', 'iter_file_dups']
|
__all__ = ['Sweeper']
|
||||||
|
|
|
@ -2,52 +2,96 @@
|
||||||
# Author: Darko Poljak <darko.poljak@gmail.com>
|
# Author: Darko Poljak <darko.poljak@gmail.com>
|
||||||
# License: GPLv3
|
# License: GPLv3
|
||||||
|
|
||||||
"""Sweeper.
|
"""{0} {1}
|
||||||
|
|
||||||
Usage: sweeper.py [options] [<directory>...]
|
Usage: {0} [options] [<directory>...]
|
||||||
|
|
||||||
Arguments:
|
Arguments:
|
||||||
<directory> directory path to scan for files
|
<directory> directory path to scan for files
|
||||||
|
|
||||||
Options:
|
Options:
|
||||||
-h, --help show this screen
|
-h, --help show this screen
|
||||||
-b <blocksize>, --block-size=<blocksize> size of block used when reading file's
|
-v, --version show version and exit
|
||||||
content [default: 4096]
|
-b <blocksize>, --block-size=<blocksize> size of block used when reading
|
||||||
-d <hashalg>, --digest-alg=<hashalg> secure hash algorithm [default: md5]
|
file's content [default: 4096]
|
||||||
-a <action>, --action=<action> action on duplicate files
|
-d <hashalgs>, --digest-algs=<hashalgs> secure hash algorithm comma separated
|
||||||
(print, remove, move) [default: print]
|
list [default: sha1]
|
||||||
|
note that multiple hashes will slow
|
||||||
|
down sweeper
|
||||||
|
-a <action>, --action=<action> action on duplicate files (pprint,
|
||||||
|
print, remove, move)
|
||||||
|
[default: pprint]
|
||||||
|
-remove removes duplicate files
|
||||||
|
except first or first with specified
|
||||||
|
directory prefix found
|
||||||
|
-move moves duplicate files to
|
||||||
|
duplicates driectory, except first
|
||||||
|
or first with specified directory
|
||||||
|
prefix found
|
||||||
|
-print prints result directory where
|
||||||
|
keys are hash values and values are
|
||||||
|
list of duplicate file paths
|
||||||
|
-pprint prints sets of duplicate file
|
||||||
|
paths each in it's line where sets
|
||||||
|
are separated by blank newline
|
||||||
-m <directory>, --move=<directory> move duplicate files to directory
|
-m <directory>, --move=<directory> move duplicate files to directory
|
||||||
(used with move action) [default: ./dups]
|
(used with move action)
|
||||||
|
[default: ./dups]
|
||||||
|
-k <dirprefix>, --keep=<dirprefix> directory prefix for remove and move
|
||||||
|
actions
|
||||||
|
-s, --simulate if action is remove or move just
|
||||||
|
simulate action by printing, do not
|
||||||
|
actually perform the action
|
||||||
|
-V, --verbose print more info
|
||||||
|
note that verbosity will slow down
|
||||||
|
sweeper due to text printing and
|
||||||
|
gathering additional information
|
||||||
|
-S, --safe-mode enable safe mode: compare hash
|
||||||
|
duplicate files byte by byte too
|
||||||
|
note that it will further slow down
|
||||||
|
sweeper but will overcome hash
|
||||||
|
collisions (although this is
|
||||||
|
unlikely)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
from __future__ import print_function
|
||||||
|
|
||||||
__author__ = 'Darko Poljak <darko.poljak@gmail.com>'
|
__author__ = 'Darko Poljak <darko.poljak@gmail.com>'
|
||||||
__version__ = '0.2.0'
|
__version__ = '0.9.0'
|
||||||
__license__ = 'GPLv3'
|
__license__ = 'GPLv3'
|
||||||
|
|
||||||
__all__ = [
|
__all__ = ['Sweeper']
|
||||||
'file_dups', 'rm_file_dups', 'mv_file_dups', 'iter_file_dups'
|
|
||||||
]
|
|
||||||
|
|
||||||
import sys
|
import sys
|
||||||
import hashlib
|
|
||||||
import os
|
import os
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
|
from functools import partial
|
||||||
|
import hashlib
|
||||||
|
|
||||||
|
|
||||||
# some differences in python versions
|
# some differences in python versions
|
||||||
|
# we prefer iter methods
|
||||||
if sys.version_info[0] == 3:
|
if sys.version_info[0] == 3:
|
||||||
def _do_encode(buf):
|
|
||||||
return buf
|
|
||||||
|
|
||||||
def _dict_iter_items(d):
|
def _dict_iter_items(d):
|
||||||
return d.items()
|
return d.items()
|
||||||
else:
|
|
||||||
def _do_encode(buf):
|
|
||||||
return buf
|
|
||||||
|
|
||||||
|
def _dict_iter_keys(d):
|
||||||
|
return d.keys()
|
||||||
|
|
||||||
|
def _dict_iter_values(d):
|
||||||
|
return d.values()
|
||||||
|
else:
|
||||||
def _dict_iter_items(d):
|
def _dict_iter_items(d):
|
||||||
return d.iteritems()
|
return d.iteritems()
|
||||||
|
|
||||||
|
def _dict_iter_keys(d):
|
||||||
|
return d.iterkeys()
|
||||||
|
|
||||||
|
def _dict_iter_values(d):
|
||||||
|
return d.itervalues()
|
||||||
|
|
||||||
|
range = xrange
|
||||||
|
|
||||||
|
|
||||||
def _filehash(filepath, hashalg, block_size):
|
def _filehash(filepath, hashalg, block_size):
|
||||||
"""Calculate secure hash for given file content using
|
"""Calculate secure hash for given file content using
|
||||||
|
@ -57,56 +101,254 @@ def _filehash(filepath, hashalg, block_size):
|
||||||
md = hashlib.new(hashalg)
|
md = hashlib.new(hashalg)
|
||||||
with open(filepath, "rb") as f:
|
with open(filepath, "rb") as f:
|
||||||
for buf in iter(lambda: f.read(block_size), b''):
|
for buf in iter(lambda: f.read(block_size), b''):
|
||||||
md.update(_do_encode(buf))
|
md.update(buf)
|
||||||
return md.hexdigest()
|
return md.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
def file_dups(topdirs=['./'], hashalg='md5', block_size=4096):
|
def _uniq_list(list_):
|
||||||
"""Find duplicate files in directory list. Return directory
|
return list(set(list_))
|
||||||
with keys equal to file hash value and value as list of
|
|
||||||
file paths whose content is the same.
|
|
||||||
"""
|
def _fbequal(fpath1, fpath2):
|
||||||
dups = defaultdict(list)
|
'''Compare files byte by byte. If files are equal return True,
|
||||||
for topdir in topdirs:
|
False otherwise.
|
||||||
for dirpath, dirnames, filenames in os.walk(topdir):
|
fpath1 and fpath2 are file paths.
|
||||||
for fname in filenames:
|
'''
|
||||||
fpath = os.path.join(dirpath, fname)
|
# nested to work with 2.6
|
||||||
hexmd = _filehash(fpath, hashalg, block_size)
|
with open(fpath1, "rb") as f1:
|
||||||
|
with open(fpath2, "rb") as f2:
|
||||||
|
while True:
|
||||||
|
b1 = f1.read(1)
|
||||||
|
b2 = f2.read(1)
|
||||||
|
if b1 != b2: # different bytes
|
||||||
|
return False
|
||||||
|
if not b1 or not b2: # end in one or both files
|
||||||
|
break
|
||||||
|
if not b1 and not b2: # end in both files, files are equal
|
||||||
|
return True
|
||||||
|
# end in one file but not in the other, files aren't equal
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _remap_keys_to_str(d):
|
||||||
|
'''Iterator that remaps dictionary keys to string in case keys are tuple
|
||||||
|
or list. Leave it unchanged otherwise.
|
||||||
|
Yields string key, value pairs.
|
||||||
|
'''
|
||||||
|
for k in _dict_iter_keys(d):
|
||||||
|
if isinstance(k, tuple) or isinstance(k, list):
|
||||||
|
key = ','.join(k)
|
||||||
|
else:
|
||||||
|
key = k
|
||||||
|
yield (key, d[k])
|
||||||
|
|
||||||
|
|
||||||
|
def _gather_file_list(dirs):
|
||||||
|
'''Gather file paths in directory list dirs.
|
||||||
|
Return tuple (count, files) where count is files
|
||||||
|
list length and files is list of file paths in
|
||||||
|
specified directories.
|
||||||
|
'''
|
||||||
|
files = []
|
||||||
|
for dir_ in dirs:
|
||||||
|
for dirpath, dirnames, filenames in os.walk(dir_):
|
||||||
|
# replace fpath with realpath value (eliminate symbolic links)
|
||||||
|
files.extend([os.path.realpath(os.path.join(dirpath, fname))
|
||||||
|
for fname in filenames])
|
||||||
|
return files
|
||||||
|
|
||||||
|
|
||||||
|
class Sweeper(object):
|
||||||
|
DEF_HASHALGS = ['sha1']
|
||||||
|
|
||||||
|
def __init__(self, topdirs=['./'], hashalgs=DEF_HASHALGS,
|
||||||
|
block_size=4096, verbose=False, safe_mode=False):
|
||||||
|
# replace dir paths with realpath value (eliminate symbolic links)
|
||||||
|
self.topdirs = []
|
||||||
|
for i in range(len(topdirs)):
|
||||||
|
self.topdirs.append(os.path.realpath(topdirs[i]))
|
||||||
|
self.hashalgs = hashalgs
|
||||||
|
self.block_size = block_size
|
||||||
|
self.verbose = verbose
|
||||||
|
self.safe_mode = safe_mode
|
||||||
|
|
||||||
|
# iter through file paths in files list
|
||||||
|
def _files_iter_from_list(self, files):
|
||||||
|
return (fpath for fpath in files)
|
||||||
|
|
||||||
|
# iter through file paths by os.walking
|
||||||
|
def _files_iter_from_disk(self):
|
||||||
|
for topdir in self.topdirs:
|
||||||
|
for dirpath, dirnames, filenames in os.walk(topdir):
|
||||||
|
for fname in filenames:
|
||||||
|
# replace fpath with realpath value
|
||||||
|
# (eliminate symbolic links)
|
||||||
|
fpath = os.path.realpath(os.path.join(dirpath, fname))
|
||||||
|
yield fpath
|
||||||
|
|
||||||
|
def file_dups(self):
|
||||||
|
"""Find duplicate files in directory list. Return directory
|
||||||
|
with keys equal to file hash value and value as list of
|
||||||
|
file paths whose content is the same.
|
||||||
|
If safe_mode is true then you want to play safe: do byte
|
||||||
|
by byte comparison for hash duplicate files.
|
||||||
|
"""
|
||||||
|
dups = defaultdict(list)
|
||||||
|
if self.verbose:
|
||||||
|
if self.safe_mode:
|
||||||
|
print('safe mode is on')
|
||||||
|
print('gathering and counting files...', end='')
|
||||||
|
sys.stdout.flush()
|
||||||
|
files = _gather_file_list(self.topdirs)
|
||||||
|
count = len(files)
|
||||||
|
current = 1
|
||||||
|
print(count)
|
||||||
|
_files_iter = partial(self._files_iter_from_list, files)
|
||||||
|
else:
|
||||||
|
_files_iter = self._files_iter_from_disk
|
||||||
|
|
||||||
|
for fpath in _files_iter():
|
||||||
|
if self.verbose:
|
||||||
|
print('\rprocessing file {0}/{1}: calc hash'.format(current,
|
||||||
|
count),
|
||||||
|
end='')
|
||||||
|
sys.stdout.flush()
|
||||||
|
hexmds = [_filehash(fpath, h, self.block_size)
|
||||||
|
for h in self.hashalgs]
|
||||||
|
hexmd = tuple(hexmds)
|
||||||
|
dup_files = dups[hexmd]
|
||||||
|
files_equals = False
|
||||||
|
if self.safe_mode:
|
||||||
|
if dup_files:
|
||||||
|
if self.verbose:
|
||||||
|
print('\rprocessing file {0}/{1}: byte cmp'.format(
|
||||||
|
current, count), end='')
|
||||||
|
sys.stdout.flush()
|
||||||
|
for f in dup_files:
|
||||||
|
if _fbequal(f, fpath):
|
||||||
|
files_equals = True
|
||||||
|
break
|
||||||
|
if self.verbose and not files_equals:
|
||||||
|
print('\nsame hash value {} but not same bytes for'
|
||||||
|
' file {} with files {}'.format(
|
||||||
|
hexmd, fpath, dup_files))
|
||||||
|
else: # when list is empty in safe mode
|
||||||
|
files_equals = True
|
||||||
|
else:
|
||||||
|
files_equals = True # when safe mode is off
|
||||||
|
if self.verbose:
|
||||||
|
current += 1
|
||||||
|
if files_equals:
|
||||||
dups[hexmd].append(fpath)
|
dups[hexmd].append(fpath)
|
||||||
result = {k: v for k, v in _dict_iter_items(dups) if len(v) > 1}
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
if self.verbose:
|
||||||
|
print('')
|
||||||
|
# make result dict with unique file paths list
|
||||||
|
result = {}
|
||||||
|
for k, v in _dict_iter_items(dups):
|
||||||
|
uniq_v = _uniq_list(v)
|
||||||
|
if len(uniq_v) > 1:
|
||||||
|
result[k] = uniq_v
|
||||||
|
return result
|
||||||
|
|
||||||
def rm_file_dups(topdirs=['./'], hashalg='md5', block_size=4096):
|
def __iter__(self):
|
||||||
"""Remove duplicate files found in specified directory list.
|
"""Find duplicate files in directory list.
|
||||||
First file in list is kept.
|
Yield tuple of file path, hash tuple and list of duplicate files
|
||||||
"""
|
as soon as duplicate file is found.
|
||||||
for files in do_with_file_dups(topdirs, hashalg, block_size):
|
Newly found file is not included in the list at the yield time,
|
||||||
for f in files:
|
but is appended later before next yield.
|
||||||
os.remove(f)
|
This means that not all duplicate files are returned with any
|
||||||
|
return value. Same hash value and sublist could be returned later
|
||||||
|
if file with same content is found.
|
||||||
|
If safe_mode is true then you want to play safe: do byte
|
||||||
|
by byte comparison for hash duplicate files.
|
||||||
|
"""
|
||||||
|
# internaly, file dups dict is still maintained
|
||||||
|
dups = defaultdict(list)
|
||||||
|
_files_iter = self._files_iter_from_disk
|
||||||
|
|
||||||
|
for fpath in _files_iter():
|
||||||
|
hexmds = [_filehash(fpath, h, self.block_size)
|
||||||
|
for h in self.hashalgs]
|
||||||
|
hexmd = tuple(hexmds)
|
||||||
|
dup_files = dups[hexmd]
|
||||||
|
# there were dup list elements (used for yield)
|
||||||
|
if self.safe_mode and dup_files:
|
||||||
|
# compare only with first file in dup_files
|
||||||
|
# all files in dup_files list are already content equal
|
||||||
|
files_equals = _fbequal(dup_files[0], fpath)
|
||||||
|
else: # when list is emtpy in safe mode or when safe mode is off
|
||||||
|
files_equals = True
|
||||||
|
if files_equals:
|
||||||
|
# yield only if current dup files list isn't empty
|
||||||
|
if dup_files:
|
||||||
|
yield (fpath, hexmd, dups[hexmd])
|
||||||
|
# finally append newly found file to dup list
|
||||||
|
dups[hexmd].append(fpath)
|
||||||
|
|
||||||
def mv_file_dups(topdirs=['./'], hashalg='md5', block_size=4096, dest_dir='dups'):
|
def _extract_files_for_action(self, keep_prefix):
|
||||||
"""Move duplicate files found in specified directory list.
|
dups = self.file_dups()
|
||||||
First file in list is kept in the original directory.
|
for files in _dict_iter_values(dups):
|
||||||
"""
|
found = False
|
||||||
if not os.path.exists(dest_dir):
|
if keep_prefix:
|
||||||
os.mkdir(dest_dir)
|
result = []
|
||||||
if not os.path.isdir(dest_dir):
|
for f in files:
|
||||||
raise OSError('%s is not a directory' % dest_dir)
|
if f.startswith(keep_prefix) and not found:
|
||||||
import shutil
|
found = True
|
||||||
for files in do_with_file_dups(topdirs, hashalg, block_size):
|
else:
|
||||||
for i, f in enumerate(files):
|
result.append(f)
|
||||||
if i > 0:
|
if not found:
|
||||||
shutil.move(f, dest_dir)
|
result = list(files)[1:]
|
||||||
|
yield (files, result)
|
||||||
|
|
||||||
|
def _do_action(self, simulate, keep_prefix, action, action_str):
|
||||||
|
for dups, extracted in self._extract_files_for_action(keep_prefix):
|
||||||
|
if simulate or self.verbose:
|
||||||
|
print('found duplicates: \n{}'.format(dups))
|
||||||
|
for f in extracted:
|
||||||
|
if simulate or self.verbose:
|
||||||
|
print(action_str.format(f))
|
||||||
|
if not simulate:
|
||||||
|
action(f)
|
||||||
|
|
||||||
def iter_file_dups(topdirs=['./'], hashalg='md5', block_size=4096):
|
def rm(self, simulate=False, keep_prefix=None):
|
||||||
"""Yield list of duplicate files when found in specified directory list.
|
"""Remove duplicate files found in specified directory list.
|
||||||
"""
|
If keep_prefix is specified then first file with that path
|
||||||
dups = file_dups(topdirs, hashalg, block_size)
|
prefix found is kept in the original directory.
|
||||||
for fpaths in dups.itervalues():
|
Otherwise first file in list is kept in the original directory.
|
||||||
yield fpaths
|
If simulate is True then only print the action, do not actually
|
||||||
|
perform it.
|
||||||
|
If safe_mode is true then do byte by byte comparison for
|
||||||
|
hash duplicate files.
|
||||||
|
"""
|
||||||
|
self._do_action(simulate, keep_prefix, os.remove, 'rm {}')
|
||||||
|
|
||||||
|
def mv(self, dest_dir='dups', simulate=False, keep_prefix=None):
|
||||||
|
"""Move duplicate files found in specified directory list.
|
||||||
|
If keep_prefix is specified then first file with that path
|
||||||
|
prefix found is kept in the original directory.
|
||||||
|
Otherwise first file in list is kept in the original directory.
|
||||||
|
If simulate is True then only print the action, do not actually
|
||||||
|
perform it.
|
||||||
|
If safe_mode is true then do byte by byte comparison for
|
||||||
|
hash duplicate files.
|
||||||
|
"""
|
||||||
|
import shutil
|
||||||
|
|
||||||
|
if not os.path.exists(dest_dir):
|
||||||
|
if simulate:
|
||||||
|
print('mkdir {}'.format(dest_dir))
|
||||||
|
else:
|
||||||
|
os.mkdir(dest_dir)
|
||||||
|
elif not os.path.isdir(dest_dir):
|
||||||
|
errmsg = '{} is not a directory'.format(dest_dir)
|
||||||
|
if simulate:
|
||||||
|
print('would raise:', errmsg)
|
||||||
|
else:
|
||||||
|
raise OSError(errmsg)
|
||||||
|
self._do_action(simulate, keep_prefix,
|
||||||
|
partial(shutil.move, dst=dest_dir),
|
||||||
|
'mv {0} to ' + dest_dir)
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
|
@ -115,32 +357,58 @@ def main():
|
||||||
import json
|
import json
|
||||||
from docopt import docopt
|
from docopt import docopt
|
||||||
|
|
||||||
args = docopt(__doc__)
|
args = docopt(__doc__.format(sys.argv[0], __version__),
|
||||||
|
version=" ".join(('sweeper', __version__)))
|
||||||
|
|
||||||
topdirs = args['<directory>']
|
topdirs = args['<directory>']
|
||||||
if not topdirs:
|
if not topdirs:
|
||||||
topdirs = ['./']
|
topdirs = ['./']
|
||||||
|
|
||||||
action = args['--action']
|
action = args['--action']
|
||||||
|
verbose = args['--verbose']
|
||||||
|
|
||||||
|
# set block size as int
|
||||||
try:
|
try:
|
||||||
bs = int(args['--block-size'])
|
bs = int(args['--block-size'])
|
||||||
args['--block-size'] = bs
|
args['--block-size'] = bs
|
||||||
except ValueError:
|
except ValueError:
|
||||||
print('Invalid block size "%s"' % args['--block-size'])
|
print('Invalid block size "{}"'.format(args['--block-size']))
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
hashalgs = args['--digest-algs'].split(',')
|
||||||
|
hashalgs_uniq = _uniq_list(hashalgs)
|
||||||
|
if len(hashalgs) != len(hashalgs_uniq):
|
||||||
|
print('Duplicate hash algorithms specified')
|
||||||
|
sys.exit(1)
|
||||||
|
block_size = args['--block-size']
|
||||||
|
simulate = args['--simulate']
|
||||||
|
keep_prefix = args['--keep']
|
||||||
|
dest_dir = args['--move']
|
||||||
|
safe_mode = args['--safe-mode']
|
||||||
|
|
||||||
if action == 'print':
|
sweeper = Sweeper(topdirs=topdirs, hashalgs=hashalgs,
|
||||||
dups = file_dups(topdirs, args['--digest-alg'], args['--block-size'])
|
block_size=block_size, verbose=verbose,
|
||||||
print(json.dumps(dict(dups), indent=4))
|
safe_mode=safe_mode)
|
||||||
|
if action == 'print' or action == 'pprint':
|
||||||
|
dups = sweeper.file_dups()
|
||||||
|
# defaultdict(list) -> dict
|
||||||
|
spam = dict(dups)
|
||||||
|
if spam:
|
||||||
|
if action == 'pprint':
|
||||||
|
for _, fpaths in _dict_iter_items(spam):
|
||||||
|
for path in fpaths:
|
||||||
|
print(path)
|
||||||
|
if fpaths:
|
||||||
|
print('')
|
||||||
|
else:
|
||||||
|
print(json.dumps({k: v for k, v in _remap_keys_to_str(spam)},
|
||||||
|
indent=4))
|
||||||
elif action == 'move':
|
elif action == 'move':
|
||||||
mv_file_dups(topdirs, args['--digest-alg'], args['--block-size'],
|
sweeper.mv(dest_dir, simulate, keep_prefix)
|
||||||
args['--move'])
|
|
||||||
elif action == 'remove':
|
elif action == 'remove':
|
||||||
rm_file_dups(topdirs, args['--digest-alg'], args['--block-size'])
|
sweeper.rm(simulate, keep_prefix)
|
||||||
else:
|
else:
|
||||||
print('Invalid action "%s"' % action)
|
print('Invalid action "{}"'.format(action))
|
||||||
|
|
||||||
|
|
||||||
# if used as script call main function
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
main()
|
main()
|
||||||
|
|
Binary file not shown.
|
@ -3,7 +3,7 @@
|
||||||
# License: GPLv3
|
# License: GPLv3
|
||||||
|
|
||||||
import unittest
|
import unittest
|
||||||
from sweeper import file_dups
|
from sweeper import Sweeper
|
||||||
import os
|
import os
|
||||||
|
|
||||||
mydir = os.path.dirname(os.path.realpath(__file__))
|
mydir = os.path.dirname(os.path.realpath(__file__))
|
||||||
|
@ -11,7 +11,8 @@ mydir = os.path.dirname(os.path.realpath(__file__))
|
||||||
|
|
||||||
class TestSweeper(unittest.TestCase):
|
class TestSweeper(unittest.TestCase):
|
||||||
def test_file_dups_dups(self):
|
def test_file_dups_dups(self):
|
||||||
dups = file_dups([os.path.join(mydir, 'testfiles_dups')], 'md5')
|
swp = Sweeper(topdirs=[os.path.join(mydir, 'testfiles_dups')])
|
||||||
|
dups = swp.file_dups()
|
||||||
dups_exist = False
|
dups_exist = False
|
||||||
for h, flist in dups.items():
|
for h, flist in dups.items():
|
||||||
if len(flist) > 1:
|
if len(flist) > 1:
|
||||||
|
@ -19,10 +20,40 @@ class TestSweeper(unittest.TestCase):
|
||||||
self.assertTrue(dups_exist)
|
self.assertTrue(dups_exist)
|
||||||
|
|
||||||
def test_file_dups_nodups(self):
|
def test_file_dups_nodups(self):
|
||||||
dups = file_dups([os.path.join(mydir, 'testfiles_nodups')], 'md5')
|
swp = Sweeper(topdirs=[os.path.join(mydir, 'testfiles_nodups')])
|
||||||
|
dups = swp.file_dups()
|
||||||
for h, flist in dups.items():
|
for h, flist in dups.items():
|
||||||
self.assertTrue(len(flist) == 1)
|
self.assertTrue(len(flist) == 1)
|
||||||
|
|
||||||
|
# does not actually test safe_mode, we would need to find
|
||||||
|
# hash collision
|
||||||
|
def test_file_dups_safe_mode(self):
|
||||||
|
swp = Sweeper(topdirs=[os.path.join(mydir, 'testfiles_dups')],
|
||||||
|
safe_mode=True)
|
||||||
|
dups = swp.file_dups()
|
||||||
|
for h, flist in dups.items():
|
||||||
|
if len(flist) > 1:
|
||||||
|
dups_exist = True
|
||||||
|
self.assertTrue(dups_exist)
|
||||||
|
|
||||||
|
def test_iter_file_dups_dups(self):
|
||||||
|
swp = Sweeper(topdirs=[os.path.join(mydir, 'testfiles_dups')])
|
||||||
|
dups_exist = False
|
||||||
|
for x in swp:
|
||||||
|
dups_exist = True
|
||||||
|
filepath, h, dups = x
|
||||||
|
self.assertNotIn(filepath, dups)
|
||||||
|
self.assertTrue(len(dups) > 0)
|
||||||
|
self.assertTrue(dups_exist)
|
||||||
|
|
||||||
|
def test_iter_file_dups_nodups(self):
|
||||||
|
swp = Sweeper([os.path.join(mydir, 'testfiles_nodups')])
|
||||||
|
dups_exist = False
|
||||||
|
for x in swp:
|
||||||
|
dups_exist = True
|
||||||
|
break
|
||||||
|
self.assertFalse(dups_exist)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|
Loading…
Reference in a new issue