diff --git a/Duplicate-File-Finder/README.md b/Duplicate-File-Finder/README.md new file mode 100644 index 0000000..cc893fe --- /dev/null +++ b/Duplicate-File-Finder/README.md @@ -0,0 +1,56 @@ +# Duplicate File Finder + +A command-line tool that scans a directory (recursively) and finds files +that are byte-for-byte identical, based on content hashing rather than +just filenames. + +## How it works + +1. Files are first grouped by size — files of different sizes can never + be duplicates, so this is a cheap way to skip most comparisons. +2. Remaining candidates are hashed using SHA-256 (read in chunks, so + large files don't get loaded into memory all at once). +3. Files that share a hash are reported as duplicates. + +## Usage + +```bash +python duplicate_finder.py +``` + +Example: + +```bash +python duplicate_finder.py ~/Downloads +``` + +Output: + +``` +Duplicate group (2 files, hash 3a7bd3e2ff...): + /home/user/Downloads/report.pdf + /home/user/Downloads/report (1).pdf + +Total duplicate groups: 1 +Space that could be reclaimed: 245.3 KB +``` + +### Optional: delete duplicates + +```bash +python duplicate_finder.py --delete +``` + +This keeps the first file found in each duplicate group and asks for +confirmation before deleting the rest. + +## Running tests + +```bash +pip install pytest +pytest test_duplicate_finder.py -v +``` + +## Requirements + +None — uses only the Python standard library. diff --git a/Duplicate-File-Finder/duplicate_finder.py b/Duplicate-File-Finder/duplicate_finder.py new file mode 100644 index 0000000..f09892c --- /dev/null +++ b/Duplicate-File-Finder/duplicate_finder.py @@ -0,0 +1,117 @@ +"""Duplicate File Finder. + +Scans a directory tree and reports groups of files that are byte-for-byte +identical, based on content hashing rather than filename or size alone. + +Usage: + python duplicate_finder.py [--delete] + +Example: + python duplicate_finder.py ~/Downloads + python duplicate_finder.py ~/Downloads --delete +""" + +import argparse +import hashlib +import os +from collections import defaultdict + + +def hash_file(file_path: str, chunk_size: int = 8192) -> str: + """Return the SHA-256 hash of a file's contents. + + NOTE: files are read in chunks so large files don't get loaded into + memory all at once. + """ + hasher = hashlib.sha256() + with open(file_path, 'rb') as f: + for chunk in iter(lambda: f.read(chunk_size), b''): + hasher.update(chunk) + return hasher.hexdigest() + + +def find_duplicates(root_dir: str) -> dict[str, list[str]]: + """Walk root_dir and group files by content hash. + + Files are first grouped by size as a cheap pre-filter before hashing, + since files of different sizes can never be duplicates. + """ + size_groups: dict[int, list[str]] = defaultdict(list) + + for dirpath, _, filenames in os.walk(root_dir): + for filename in filenames: + full_path = os.path.join(dirpath, filename) + try: + file_size = os.path.getsize(full_path) + except OSError: + continue + size_groups[file_size].append(full_path) + + hash_groups: dict[str, list[str]] = defaultdict(list) + for candidates in size_groups.values(): + if len(candidates) < 2: + continue + for file_path in candidates: + try: + file_hash = hash_file(file_path) + except OSError: + continue + hash_groups[file_hash].append(file_path) + + return {h: paths for h, paths in hash_groups.items() if len(paths) > 1} + + +def print_report(duplicates: dict[str, list[str]]) -> None: + """Print a human-readable summary of duplicate groups.""" + if not duplicates: + print('No duplicate files found.') + return + + total_wasted_bytes = 0 + for file_hash, paths in duplicates.items(): + wasted = os.path.getsize(paths[0]) * (len(paths) - 1) + total_wasted_bytes += wasted + + print(f'\nDuplicate group ({len(paths)} files, hash {file_hash[:10]}...):') + for path in paths: + print(f' {path}') + + print(f'\nTotal duplicate groups: {len(duplicates)}') + print(f'Space that could be reclaimed: {total_wasted_bytes / 1024:.1f} KB') + + +def delete_duplicates(duplicates: dict[str, list[str]]) -> None: + """Delete all but the first file in each duplicate group.""" + for paths in duplicates.values(): + for path in paths[1:]: + os.remove(path) + print(f'Deleted: {path}') + + +def main() -> None: + parser = argparse.ArgumentParser(description='Find duplicate files by content.') + parser.add_argument('directory', help='Directory to scan for duplicates') + parser.add_argument( + '--delete', + action='store_true', + help='Delete duplicates, keeping only the first file found in each group', + ) + args = parser.parse_args() + + if not os.path.isdir(args.directory): + print(f'Error: {args.directory} is not a valid directory') + return + + duplicates = find_duplicates(args.directory) + print_report(duplicates) + + if args.delete and duplicates: + confirm = input('\nDelete duplicate files listed above? [y/N]: ') + if confirm.lower() == 'y': + delete_duplicates(duplicates) + else: + print('Skipped deletion.') + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/Duplicate-File-Finder/test_duplicate_finder.py b/Duplicate-File-Finder/test_duplicate_finder.py new file mode 100644 index 0000000..f6c9580 --- /dev/null +++ b/Duplicate-File-Finder/test_duplicate_finder.py @@ -0,0 +1,74 @@ +"""Tests for duplicate_finder.py""" + +import os +import tempfile + +from duplicate_finder import find_duplicates, hash_file + + +def test_hash_file_is_consistent(): + with tempfile.NamedTemporaryFile(delete=False, mode='w') as f: + f.write('hello world') + path = f.name + + try: + assert hash_file(path) == hash_file(path) + finally: + os.remove(path) + + +def test_find_duplicates_detects_identical_content(): + with tempfile.TemporaryDirectory() as tmp_dir: + path_a = os.path.join(tmp_dir, 'a.txt') + path_b = os.path.join(tmp_dir, 'b.txt') + path_c = os.path.join(tmp_dir, 'c.txt') + + with open(path_a, 'w') as f: + f.write('same content') + with open(path_b, 'w') as f: + f.write('same content') + with open(path_c, 'w') as f: + f.write('different content') + + duplicates = find_duplicates(tmp_dir) + + assert len(duplicates) == 1 + (group,) = duplicates.values() + assert set(group) == {path_a, path_b} + + +def test_find_duplicates_ignores_unique_files(): + with tempfile.TemporaryDirectory() as tmp_dir: + with open(os.path.join(tmp_dir, 'a.txt'), 'w') as f: + f.write('content one') + with open(os.path.join(tmp_dir, 'b.txt'), 'w') as f: + f.write('content two') + + duplicates = find_duplicates(tmp_dir) + + assert duplicates == {} + + +def test_find_duplicates_handles_nested_directories(): + with tempfile.TemporaryDirectory() as tmp_dir: + nested_dir = os.path.join(tmp_dir, 'nested') + os.makedirs(nested_dir) + + path_a = os.path.join(tmp_dir, 'a.txt') + path_b = os.path.join(nested_dir, 'b.txt') + + with open(path_a, 'w') as f: + f.write('shared content') + with open(path_b, 'w') as f: + f.write('shared content') + + duplicates = find_duplicates(tmp_dir) + + assert len(duplicates) == 1 + (group,) = duplicates.values() + assert set(group) == {path_a, path_b} + + +def test_find_duplicates_empty_directory(): + with tempfile.TemporaryDirectory() as tmp_dir: + assert find_duplicates(tmp_dir) == {} \ No newline at end of file