From 7223e4636113d2f51fe5b2ceb66595dd1d288b72 Mon Sep 17 00:00:00 2001 From: pfn-djf Date: Wed, 16 Apr 2025 14:18:08 +0200 Subject: [PATCH 1/2] ignore pattern in multilevel nested histories --- .github/workflows/build+test.yml | 2 +- ascmhl/commands.py | 64 +- ascmhl/generator.py | 196 ++++- ascmhl/history.py | 21 +- ascmhl/utils.py | 7 + tests/conftest.py | 92 +++ tests/test_ignore.py | 19 + tests/test_ignore_extended.py | 1200 ++++++++++++++++++++++++++++++ 8 files changed, 1592 insertions(+), 9 deletions(-) create mode 100644 tests/test_ignore_extended.py diff --git a/.github/workflows/build+test.yml b/.github/workflows/build+test.yml index 137a9c9..86fbb94 100644 --- a/.github/workflows/build+test.yml +++ b/.github/workflows/build+test.yml @@ -5,7 +5,7 @@ name: ascmhl-build-test on: push: - branches: [ master, dev/windowsPathHandling ] + branches: [ master, ignore_pattern_in_nested_histories ] pull_request: branches: [ master ] diff --git a/ascmhl/commands.py b/ascmhl/commands.py index 824ec07..808c46e 100644 --- a/ascmhl/commands.py +++ b/ascmhl/commands.py @@ -12,6 +12,7 @@ import platform import click +import pathspec from lxml import etree from . import logger @@ -35,6 +36,8 @@ from typing import Dict from collections import namedtuple +from .utils import check_path_is_absolute_to_history + @click.command() @click.argument("root_path", type=click.Path(exists=True)) @@ -231,6 +234,9 @@ def create_for_folder_subcommand( # start a verification session on the existing history session = MHLGenerationCreationSession(existing_history, ignore_spec) + # update the ignore spec and include ignores from nested histories + ignore_spec = get_ignore_spec_including_nested_ignores(existing_history, ignore_list, ignore_spec_file) + num_failed_verifications = 0 # store the directory hashes of sub folders so we can use it when calculating the hash of the parent folder # the mapping lookups will follow the dictionary format of [string: [hash_format: hash_value]] where string @@ -239,7 +245,7 @@ def create_for_folder_subcommand( dir_structure_hash_mapping_lookup = {} hash_format_list = sorted(hash_formats) - for folder_path, children in post_order_lexicographic(root_path, session.ignore_spec.get_path_spec()): + for folder_path, children in post_order_lexicographic(root_path, ignore_spec.get_path_spec()): # generate directory hashes dir_hash_context_lookup = {} @@ -695,7 +701,7 @@ def verify_directory_hash_subcommand( existing_history = MHLHistory.load_from_path(root_path) - ignore_spec = ignore.MHLIgnoreSpec(existing_history.latest_ignore_patterns(), ignore_list, ignore_spec_file) + ignore_spec = get_ignore_spec_including_nested_ignores(existing_history, ignore_list, ignore_spec_file) # FIXME: Update once argument signature has been modified to supply a list of formats hash_formats = [] @@ -1031,7 +1037,7 @@ def diff_entire_folder_against_full_history_subcommand(root_path, verbose, ignor num_failed_verifications = 0 num_new_files = 0 - ignore_spec = ignore.MHLIgnoreSpec(existing_history.latest_ignore_patterns(), ignore_list, ignore_spec_file) + ignore_spec = get_ignore_spec_including_nested_ignores(existing_history, ignore_list, ignore_spec_file) for folder_path, children in post_order_lexicographic(root_path, ignore_spec.get_path_spec()): for item_name, is_dir in children: @@ -1572,3 +1578,55 @@ def seal_file_path(existing_history, file_path, hash_formats: [str], session) -> hash_result_lookup[hash_format] = SealPathResult(current_hash_lookup[hash_format], success) return hash_result_lookup + + +def get_ignore_spec_including_nested_ignores(existing_history, ignore_list, ignore_spec_file=None): + """Get the ignore patterns from nested histories with their respective paths, + so that ignored files from nested histories are also ignored in this session, but are not stored + in the root ascmhl manifest""" + ignore_patterns_cumulated = ignore.default_ignore_list() + # handle non-existent ignores in root history + if existing_history.latest_ignore_patterns() is not None: + for x in existing_history.latest_ignore_patterns(): + if x not in ignore.default_ignore_list(): + ignore_patterns_cumulated.append(x) + + for x in existing_history.latest_ignore_pattern_from_nested_histories(): + ignore_patterns_cumulated.append(x) + + for x in ignore_list: + ignore_patterns_cumulated.append(x) + + patterns_from_file = [] + if ignore_spec_file: + with open(ignore_spec_file, "r") as fh: + patterns_from_file.extend(line.rstrip("\n") for line in fh if line != "\n") + for x in patterns_from_file: + if x not in ignore_patterns_cumulated: + ignore_patterns_cumulated.append(x) + + # we now build the absolute ignore paths for the current session from all nested ignores + # otherwise the post_order_lexicographic() won't ignore these paths + absolute_ignore_paths = [] + path = existing_history.get_root_path() + for pattern in ignore_patterns_cumulated: + if pattern in ignore.default_ignore_list(): + absolute_ignore_paths.append(pattern) + else: + if pattern.find("/") != -1: + if pattern.startswith("/"): + absolute_ignore_paths.append(path + pattern) + elif pattern.startswith("**"): + absolute_ignore_paths.append(pattern) + elif pattern.endswith("/") and pattern[:-1].find("/") == -1: + absolute_ignore_paths.append(pattern) + elif pattern.endswith("/" + "**") and pattern[:-3].find("/") == -1: + absolute_ignore_paths.append(pattern) + else: + absolute_ignore_paths.append(path + os.sep + pattern) + else: + absolute_ignore_paths.append(pattern) + + normalized_paths = [pathspec.util.normalize_file(p) for p in absolute_ignore_paths] + spec = ignore.MHLIgnoreSpec(existing_history.latest_ignore_patterns(), normalized_paths) + return spec diff --git a/ascmhl/generator.py b/ascmhl/generator.py index 170ff8a..d01d248 100644 --- a/ascmhl/generator.py +++ b/ascmhl/generator.py @@ -7,10 +7,12 @@ __email__ = "opensource@pomfort.com" """ +import os from collections import defaultdict +from pathlib import PureWindowsPath from typing import Dict, List -from . import chain_xml_parser +from . import chain_xml_parser, ignore from . import logger from .ignore import MHLIgnoreSpec from .hashlist import MHLHashList, MHLHashEntry, MHLCreatorInfo, MHLProcessInfo @@ -296,9 +298,7 @@ def commit(self, creator_info: MHLCreatorInfo, process_info: MHLProcessInfo): new_hash_list.process_info.root_media_hash = process_info.root_media_hash new_hash_list.process_info.hashlist_custom_basename = process_info.hashlist_custom_basename new_hash_list.process_info.process = process_info.process - new_hash_list.process_info.ignore_spec = MHLIgnoreSpec( - history.latest_ignore_patterns(), self.ignore_spec.get_pattern_list() - ) + new_hash_list.process_info.ignore_spec = self.get_relevant_ignore_pattern(history) history.write_new_generation(new_hash_list) relative_generation_path = self.root_history.get_relative_file_path(new_hash_list.file_path) @@ -307,3 +307,191 @@ def commit(self, creator_info: MHLCreatorInfo, process_info: MHLProcessInfo): referenced_hash_lists[history.parent_history].append(new_hash_list) chain_xml_parser.write_chain(history.chain, new_hash_list) + + def get_relevant_ignore_pattern(self, history) -> MHLIgnoreSpec: + """ + Only store the relevant ignore patterns for an ascmhl-history and ignore others. + This will split the pattern into the relevant bits for the lowest ascmhl-history relative to it + """ + + # get the ignore pattern from the latest history, if there is none, use the default pattern + final_ignores = [] + latest_ignore_patterns = history.latest_ignore_patterns() + if latest_ignore_patterns is None: + final_ignores += ignore.default_ignore_list() + else: + final_ignores += latest_ignore_patterns + + ignore_patterns = self.ignore_spec.get_pattern_list() + + history_path = history.get_root_path() + # get the highest parent history to build the correct relative paths for this generation + if history.parent_history is not None: + parent_history = history.parent_history + while parent_history.parent_history is not None: + parent_history = parent_history.parent_history + + parent_history_path = parent_history.get_root_path() + parent_rel_path = os.path.relpath(history_path, parent_history_path) + for pattern in ignore_patterns: + if not pattern in final_ignores: + if "/" not in pattern: + final_ignores.append(pattern) + elif pattern.startswith("/**/"): + final_ignores.append(pattern) + elif belongs_to_child(pattern, history, parent_history_path): + # if child is ignored itself, we need to append the ignore pattern to the next parent + for child in history.walk_child_histories(history): + child_root: str + if os.name == "nt": + child_root = PureWindowsPath(child.get_root_path()).as_posix() + else: + child_root = child.get_root_path() + if child_root.endswith(pattern): + pattern = extract_ignore_pattern(pattern, history_path) + final_ignores.append(pattern) + continue + elif belongs_to_parent_or_neighbour(pattern, parent_rel_path): + continue + else: + pattern = extract_ignore_pattern(pattern, parent_rel_path) + final_ignores.append(pattern) + else: + continue + else: + for pattern in ignore_patterns: + if not pattern in final_ignores: + if ( + not belongs_to_child(pattern, history, history_path) + and not pattern in ignore.default_ignore_list() + ): + if pattern.startswith("/"): + final_ignores.append(pattern) + elif pattern.find("/") != -1: + final_ignores.append(extract_ignore_pattern(pattern)) + else: + final_ignores.append(pattern) + else: + for child in history.walk_child_histories(history): + child_root: str + if os.name == "nt": + child_root = PureWindowsPath(child.get_root_path()).as_posix() + else: + child_root = child.get_root_path() + if child_root.endswith(pattern): + if history == child.parent_history: + pattern = extract_ignore_pattern(pattern) + final_ignores.append(pattern) + return MHLIgnoreSpec(final_ignores, latest_ignore_patterns) + + +def belongs_to_child(pattern, history, parent_history_path, ignore_child=None) -> bool: + if pattern.startswith("/"): + pattern = pattern[1:] + for child in history.child_histories: + if ignore_child == child: + continue + child_path = child.get_root_path() + parent_rel_path = os.path.relpath(child_path, parent_history_path) + if os.name == "nt": + parent_rel_path = parent_rel_path.replace("\\", "/") + if pattern.startswith(parent_rel_path): + return True + + return False + + +def belongs_to_parent_or_neighbour(pattern, parent_rel_path) -> bool: + if "/" not in pattern: + return False + if pattern.startswith("/") and "/" not in pattern[1:]: + return True + if pattern.endswith("/") and "/" not in pattern[:-1]: + return False + + if pattern.startswith("**/"): + return False + + pattern_parts = pattern.strip("/").split("/") + parent_parts = parent_rel_path.strip(os.sep).split(os.sep) + i = 0 + while i < min(len(pattern_parts), len(parent_parts)): + if pattern_parts[i] != parent_parts[i]: + if i > 0 and i < len(pattern_parts) - 1 and pattern_parts[i] == "**": + return False + return True + else: + i += 1 + if i == len(pattern_parts): + return True + + return False + + +def extract_ignore_pattern(pattern: str, parent_rel_path=None) -> str: + if pattern.startswith("/"): + if "/" in pattern[1:]: + pattern = pattern[1:] + else: + return pattern + + pattern_rel_path = _extract_pattern_relative_to_history(pattern, parent_rel_path) + + if pattern_rel_path is not None: + if pattern.endswith("/"): + if "/" in pattern[:-1]: + return "/" + (pattern if pattern.startswith("**/") else pattern_rel_path) + return pattern + + if pattern.endswith("/**"): + if pattern.startswith("**/"): + return pattern + if "/" in pattern[:-3]: + return pattern_rel_path if pattern.startswith("/") else "/" + pattern_rel_path + return pattern + + if "/" in pattern[:-1]: + return ( + "/" + pattern + if pattern.startswith("**/") + else pattern_rel_path if pattern_rel_path.startswith("/") else "/" + pattern_rel_path + ) + + return pattern + + +def _extract_pattern_relative_to_history(pattern: str, history_path=None) -> str: + if pattern.startswith("**"): + return pattern + if history_path is None: + return pattern + + pattern_parts = pattern.lstrip("/").split("/") + history_path_parts = history_path.lstrip(os.sep).split(os.sep) + + i = j = k = 0 + + while i < len(history_path_parts): + if history_path_parts[i] == pattern_parts[0]: + break + else: + i += 1 + + while j < len(pattern_parts) and i + j < len(history_path_parts): + if history_path_parts[i + j] == pattern_parts[j]: + j += 1 + else: + break + + result = "" + + while j < (len(pattern_parts)): + if k == 0: + result += pattern_parts[j] + j += 1 + k += 1 + else: + result += "/" + pattern_parts[j] + j += 1 + if result != "": + return result diff --git a/ascmhl/history.py b/ascmhl/history.py index e368292..1c8fa1b 100644 --- a/ascmhl/history.py +++ b/ascmhl/history.py @@ -12,7 +12,7 @@ import re from datetime import datetime, date, time -from . import hasher +from . import hasher, ignore from .__version__ import ascmhl_folder_name, ascmhl_file_extension, ascmhl_chainfile_name, ascmhl_collectionfile_name from . import hashlist_xml_parser, chain_xml_parser from .utils import datetime_now_filename_string @@ -98,6 +98,25 @@ def latest_ignore_patterns(self) -> Optional[List[str]]: return None return hash_list.process_info.ignore_spec.get_pattern_list() + def latest_ignore_pattern_from_nested_histories(self) -> Optional[List[str]]: + parent_path = self.get_root_path() + cumulated_ignores = [] + for path, history in self.child_history_mappings.items(): + for pattern in history.latest_ignore_patterns(): + # don't add the default pattern + child_path = history.get_root_path() + path = os.path.relpath(child_path, parent_path) + if pattern in ignore.default_ignore_list(): + continue + else: + # return the directory of the history with the pattern appended + if pattern.find("/") != -1: + cumulated_ignores.append(path + pattern) + else: + cumulated_ignores.append(path + "/**/" + pattern) + + return cumulated_ignores + # methods to query and compare hashes def find_original_hash_entry_for_path(self, relative_path: str) -> Optional[MHLHashEntry]: """Searches the history for the first (original) hash of a file diff --git a/ascmhl/utils.py b/ascmhl/utils.py index 351bc38..c783a89 100644 --- a/ascmhl/utils.py +++ b/ascmhl/utils.py @@ -60,3 +60,10 @@ def convert_posix_to_local_path(path: str) -> str: if os.name == "nt": return str(PureWindowsPath(PurePosixPath(path))) return path + + +def check_path_is_absolute_to_history(base_path, relative_path) -> bool: + base_abs = os.path.abspath(base_path) + relative_abs = os.path.abspath(relative_path) + + return relative_abs.startswith(base_abs) diff --git a/tests/conftest.py b/tests/conftest.py index 49422de..774f223 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -90,3 +90,95 @@ def simple_mhl_folder(fs): # create a simple folder structure with two files fs.create_file("/root/Stuff.txt", contents="stuff\n") fs.create_file("/root/A/A1.txt", contents="A1\n") + + +@pytest.fixture +@freeze_time("2020-01-15 13:00:00") +def post_house_file_structure(fs): + runner = CliRunner() + fs.create_file("/root/ShootingDay1/CameraMedia/A/A001/A1.txt", contents="A1") + fs.create_file("/root/ShootingDay1/CameraMedia/A/A001/A2.txt", contents="A2") + fs.create_file("/root/ShootingDay1/CameraMedia/A/A001/A3.txt", contents="A3") + fs.create_file("/root/ShootingDay1/CameraMedia/A/A001/Sidecar.txt", contents="Sidecar1") + result = runner.invoke(ascmhl.commands.create, [abspath("/root/ShootingDay1/CameraMedia/A"), "-h", "xxh64"]) + assert result.exit_code == 0 + + fs.create_file("/root/ShootingDay1/CameraMedia/B/B002/B1.txt", contents="B1") + fs.create_file("/root/ShootingDay1/CameraMedia/B/B002/B2.txt", contents="B2") + fs.create_file("/root/ShootingDay1/CameraMedia/B/B002/B3.txt", contents="B3") + fs.create_file("/root/ShootingDay1/CameraMedia/B/B002/Sidecar.txt", contents="Sidecar2") + result = runner.invoke(ascmhl.commands.create, [abspath("/root/ShootingDay1/CameraMedia/B"), "-h", "xxh64"]) + assert result.exit_code == 0 + + fs.create_file("/root/ShootingDay1/CameraMedia/Report.pdf", contents="A1-3, B1-3") + result = runner.invoke(ascmhl.commands.create, [abspath("/root/ShootingDay1/CameraMedia"), "-h", "xxh64"]) + assert result.exit_code == 0 + + fs.create_file("/root/ShootingDay1/Sound/Takes/Sound.txt", contents="Sound") + fs.create_file("/root/ShootingDay1/Sound/Sidecar.txt", contents="Sound Sidecar") + result = runner.invoke(ascmhl.commands.create, [abspath("/root/ShootingDay1/Sound"), "-h", "xxh64"]) + assert result.exit_code == 0 + + fs.create_file("/root/ShootingDay1/Report.pdf", contents="A1-3, B1-3, Sound") + result = runner.invoke(ascmhl.commands.create, [abspath("/root/ShootingDay1"), "-h", "xxh64"]) + assert result.exit_code == 0 + + # these are particularly relevant for ignore pattern testing + fs.create_file("/root/ShootingDay1/CameraMedia/A/Proxy/A001/A1_p.txt", contents="A1 Proxy") + fs.create_file("/root/ShootingDay1/CameraMedia/A/Proxy/A001/A2_p.txt", contents="A2 Proxy") + fs.create_file("/root/ShootingDay1/CameraMedia/A/Proxy/A001/A3_p.txt", contents="A3 Proxy") + fs.create_file("/root/ShootingDay1/CameraMedia/A/Proxy/A001/A001.ale", contents="A001 ALE Proxy") + + fs.create_file("/root/ShootingDay1/CameraMedia/B/Proxy/B001/A1_p.txt", contents="A1 Proxy") + fs.create_file("/root/ShootingDay1/CameraMedia/B/Proxy/B001/A2_p.txt", contents="A2 Proxy") + fs.create_file("/root/ShootingDay1/CameraMedia/B/Proxy/B001/A3_p.txt", contents="A3 Proxy") + fs.create_file("/root/ShootingDay1/CameraMedia/B/Proxy/B001/B001.ale", contents="B001 ALE Proxy") + fs.create_file("/root/ShootingDay1/CameraMedia/Proxy", contents="Proxy") + + +@pytest.fixture +@freeze_time("2020-01-15 13:00:00") +def post_house_file_structure_with_range(fs): + runner = CliRunner() + for i in range(1, 5): + for j in range(1, 5): + fs.create_file(f"/root/ShootingDay1/CameraMedia/A/A00{i}/A00{i}C00{j}.mov", contents=f"A00{i}C00{j}") + + fs.create_file("/root/ShootingDay1/CameraMedia/A/A001/A1.txt", contents="A1") + fs.create_file("/root/ShootingDay1/CameraMedia/A/A001/A2.txt", contents="A2") + fs.create_file("/root/ShootingDay1/CameraMedia/A/A001/A3.txt", contents="A3") + fs.create_file("/root/ShootingDay1/CameraMedia/A/A001/Sidecar.txt", contents="Sidecar1") + result = runner.invoke(ascmhl.commands.create, [abspath("/root/ShootingDay1/CameraMedia/A"), "-h", "xxh64"]) + assert result.exit_code == 0 + + fs.create_file("/root/ShootingDay1/CameraMedia/B/B002/B1.txt", contents="B1") + fs.create_file("/root/ShootingDay1/CameraMedia/B/B002/B2.txt", contents="B2") + fs.create_file("/root/ShootingDay1/CameraMedia/B/B002/B3.txt", contents="B3") + fs.create_file("/root/ShootingDay1/CameraMedia/B/B002/Sidecar.txt", contents="Sidecar2") + result = runner.invoke(ascmhl.commands.create, [abspath("/root/ShootingDay1/CameraMedia/B"), "-h", "xxh64"]) + assert result.exit_code == 0 + + fs.create_file("/root/ShootingDay1/CameraMedia/Report.pdf", contents="A1-3, B1-3") + result = runner.invoke(ascmhl.commands.create, [abspath("/root/ShootingDay1/CameraMedia"), "-h", "xxh64"]) + assert result.exit_code == 0 + + fs.create_file("/root/ShootingDay1/Sound/Takes/Sound.txt", contents="Sound") + fs.create_file("/root/ShootingDay1/Sound/Sidecar.txt", contents="Sound Sidecar") + result = runner.invoke(ascmhl.commands.create, [abspath("/root/ShootingDay1/Sound"), "-h", "xxh64"]) + assert result.exit_code == 0 + + fs.create_file("/root/ShootingDay1/Report.pdf", contents="A1-3, B1-3, Sound") + result = runner.invoke(ascmhl.commands.create, [abspath("/root/ShootingDay1"), "-h", "xxh64"]) + assert result.exit_code == 0 + + # these are particularly relevant for ignore pattern testing + fs.create_file("/root/ShootingDay1/CameraMedia/A/Proxy/A001/A1_p.txt", contents="A1 Proxy") + fs.create_file("/root/ShootingDay1/CameraMedia/A/Proxy/A001/A2_p.txt", contents="A2 Proxy") + fs.create_file("/root/ShootingDay1/CameraMedia/A/Proxy/A001/A3_p.txt", contents="A3 Proxy") + fs.create_file("/root/ShootingDay1/CameraMedia/A/Proxy/A001/A001.ale", contents="A001 ALE Proxy") + + fs.create_file("/root/ShootingDay1/CameraMedia/B/Proxy/B001/A1_p.txt", contents="A1 Proxy") + fs.create_file("/root/ShootingDay1/CameraMedia/B/Proxy/B001/A2_p.txt", contents="A2 Proxy") + fs.create_file("/root/ShootingDay1/CameraMedia/B/Proxy/B001/A3_p.txt", contents="A3 Proxy") + fs.create_file("/root/ShootingDay1/CameraMedia/B/Proxy/B001/B001.ale", contents="B001 ALE Proxy") + fs.create_file("/root/ShootingDay1/CameraMedia/Proxy", contents="Proxy") diff --git a/tests/test_ignore.py b/tests/test_ignore.py index 6b66663..bdf1caf 100644 --- a/tests/test_ignore.py +++ b/tests/test_ignore.py @@ -7,6 +7,8 @@ """ import os +from pathlib import Path + from .conftest import path_conversion_tests import pytest @@ -80,6 +82,23 @@ def assert_mhl_file_has_exact_ignore_patterns(mhl_file: str, patterns_to_check: assert patterns_in_file == DEFAULT_IGNORE_SET | patterns_to_check, "mhl file has incorrect ignore patterns" +def assert_pattern_ignored_in_result(pattern: [str], result, negate=False): + if negate: + for pattern in pattern: + if os.name == "posix": + assert f"ignoring filepath {pattern}" not in result.output + else: + pattern = Path(pattern).resolve() + assert f"ignoring filepath {pattern}" not in result.output + else: + for pattern in pattern: + if os.name == "posix": + assert f"ignoring filepath {pattern}" in result.output + else: + pattern = Path(pattern).resolve() + assert f"ignoring filepath {pattern}" in result.output + + def mhl_file_for_gen(mhl_dir: str, mhl_gen: int): """ returns the mhl file associated with a generation number. diff --git a/tests/test_ignore_extended.py b/tests/test_ignore_extended.py new file mode 100644 index 0000000..0e62924 --- /dev/null +++ b/tests/test_ignore_extended.py @@ -0,0 +1,1200 @@ +""" +__author__ = "David Frank" +__copyright__ = "Copyright 2025, Pomfort GmbH" +__license__ = "MIT" +__maintainer__ = "Patrick Renner, Alexander Sahm" +__email__ = "opensource@pomfort.com" +""" + +import os +from os.path import abspath +from pathlib import Path + +from click.testing import CliRunner +from freezegun import freeze_time +import ascmhl.commands +from ascmhl.generator import _extract_pattern_relative_to_history +from ascmhl.history import MHLHistory +from tests.conftest import abspath_conversion_tests +from tests.test_ignore import assert_mhl_file_has_exact_ignore_patterns, assert_pattern_ignored_in_result +from ascmhl.generator import belongs_to_parent_or_neighbour + + +@freeze_time("2020-01-16 09:15:00") +def test_ignore_relative_path_pattern_1(fs, simple_mhl_history): + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64", "-i", "A/A1.txt", "-v"] + ) + assert_mhl_file_has_exact_ignore_patterns("root/ascmhl/0002_root_2020-01-16_091500Z.mhl", {"/A/A1.txt"}) + print(result.output) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/A1.txt"], result) + + +@freeze_time("2020-01-16 09:15:00") +def test_ignore_relative_path_pattern_2(fs, simple_mhl_history): + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64", "-i", "/A/A1.txt", "-v"] + ) + assert_mhl_file_has_exact_ignore_patterns("root/ascmhl/0002_root_2020-01-16_091500Z.mhl", {"/A/A1.txt"}) + print(result.output) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/A1.txt"], result) + + +@freeze_time("2020-01-16 09:15:00") +def test_ignore_relative_path_pattern_3(fs, simple_mhl_history): + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64", "-i", "**/A/A1.txt", "-v"] + ) + assert_mhl_file_has_exact_ignore_patterns("root/ascmhl/0002_root_2020-01-16_091500Z.mhl", {"/**/A/A1.txt"}) + print(result.output) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/A1.txt"], result) + + +@freeze_time("2020-01-16 09:15:00") +def test_ignore_nested_path_pattern_1(fs, nested_mhl_histories): + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, [abspath_conversion_tests("/root/A"), "-h", "xxh64", "-i", "/AA/AA1.txt", "-v"] + ) + print(result.output) + assert_mhl_file_has_exact_ignore_patterns("root/A/AA/ascmhl/0002_AA_2020-01-16_091500Z.mhl", {"/AA1.txt"}) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/AA/AA1.txt"], result) + latest_ignores = MHLHistory.load_from_path(abspath("/root/A")).latest_ignore_patterns() + assert "/AA/AA1.txt" not in latest_ignores + + result = runner.invoke(ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64", "-v"]) + print(result.output) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/AA/AA1.txt"], result) + latest_ignores = MHLHistory.load_from_path(abspath("/root")).latest_ignore_patterns() + assert "/AA/AA1.txt" not in latest_ignores + + +@freeze_time("2020-01-16 09:15:00") +def test_ignore_nested_path_pattern_2(fs, nested_mhl_histories): + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, [abspath_conversion_tests("/root/A"), "-h", "xxh64", "-i", "AA/AA1.txt", "-v"] + ) + print(result.output) + assert_mhl_file_has_exact_ignore_patterns("root/A/AA/ascmhl/0002_AA_2020-01-16_091500Z.mhl", {"/AA1.txt"}) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/AA/AA1.txt"], result) + latest_ignores = MHLHistory.load_from_path(abspath("/root/A")).latest_ignore_patterns() + assert "AA/AA1.txt" not in latest_ignores + + result = runner.invoke(ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64", "-v"]) + print(result.output) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/AA/AA1.txt"], result) + latest_ignores = MHLHistory.load_from_path(abspath("/root")).latest_ignore_patterns() + assert "AA/AA1.txt" not in latest_ignores + + +@freeze_time("2020-01-16 09:15:00") +def test_ignore_nested_path_pattern_3(fs, nested_mhl_histories): + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, [abspath_conversion_tests("/root/A"), "-h", "xxh64", "-i", "**/AA1.txt", "-v"] + ) + print(result.output) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/AA/AA1.txt"], result) + assert_mhl_file_has_exact_ignore_patterns("root/A/ascmhl/0001_A_2020-01-16_091500Z.mhl", {"/**/AA1.txt"}) + assert_mhl_file_has_exact_ignore_patterns("root/A/AA/ascmhl/0002_AA_2020-01-16_091500Z.mhl", {"/**/AA1.txt"}) + + result = runner.invoke(ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64", "-v"]) + print(result.output) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/AA/AA1.txt"], result) + + +@freeze_time("2020-01-16 09:15:00") +def test_ignore_nested_path_pattern_4(fs, nested_mhl_histories): + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64", "-i", "AA1.txt", "-v"] + ) + print(result.output) + assert_mhl_file_has_exact_ignore_patterns("root/ascmhl/0002_root_2020-01-16_091500Z.mhl", {"AA1.txt"}) + assert_mhl_file_has_exact_ignore_patterns("root/A/AA/ascmhl/0002_AA_2020-01-16_091500Z.mhl", {"AA1.txt"}) + assert_mhl_file_has_exact_ignore_patterns("root/B/BB/ascmhl/0002_BB_2020-01-16_091500Z.mhl", {"AA1.txt"}) + assert_mhl_file_has_exact_ignore_patterns("root/B/ascmhl/0002_B_2020-01-16_091500Z.mhl", {"AA1.txt"}) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/AA/AA1.txt"], result) + + +@freeze_time("2020-01-16 09:15:00") +def test_ignore_nested_path_pattern_5(fs, post_house_file_structure): + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, [abspath_conversion_tests("/root/ShootingDay1"), "-h", "xxh64", "-i", "**/A/A001", "-v"] + ) + print(result.output) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/ShootingDay1/CameraMedia/A/A001"], result) + assert_mhl_file_has_exact_ignore_patterns( + "root/ShootingDay1/ascmhl/0002_ShootingDay1_2020-01-16_091500Z.mhl", {"/**/A/A001"} + ) + assert_mhl_file_has_exact_ignore_patterns( + "root/ShootingDay1/CameraMedia/A/ascmhl/0004_A_2020-01-16_091500Z.mhl", {"/**/A/A001"} + ) + assert_mhl_file_has_exact_ignore_patterns( + "root/ShootingDay1/CameraMedia/B/ascmhl/0004_B_2020-01-16_091500Z.mhl", {"/**/A/A001"} + ) + + +@freeze_time("2020-01-16 09:15:00") +def test_ignore_nested_path_pattern_6(fs, post_house_file_structure): + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/ShootingDay1"), "-h", "xxh64", "-i", "**/?/Proxy/", "-v"], + ) + print(result.output) + assert result.exit_code == 0 + pattern = ["/root/ShootingDay1/CameraMedia/A/Proxy", "/root/ShootingDay1/CameraMedia/B/Proxy"] + assert_pattern_ignored_in_result(pattern, result) + assert_mhl_file_has_exact_ignore_patterns( + "root/ShootingDay1/ascmhl/0002_ShootingDay1_2020-01-16_091500Z.mhl", {"/**/?/Proxy/"} + ) + assert_mhl_file_has_exact_ignore_patterns( + "root/ShootingDay1/CameraMedia/A/ascmhl/0004_A_2020-01-16_091500Z.mhl", {"/**/?/Proxy/"} + ) + assert_mhl_file_has_exact_ignore_patterns( + "root/ShootingDay1/CameraMedia/B/ascmhl/0004_B_2020-01-16_091500Z.mhl", {"/**/?/Proxy/"} + ) + + +@freeze_time("2020-01-16 09:15:00") +def test_ignore_future_files(fs, simple_mhl_history): + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64", "-i", "A/A2.txt", "-v"] + ) + assert result.exit_code == 0 + + fs.create_file("/root/A/A2.txt") + result = runner.invoke(ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64", "-v"]) + print(result.output) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/A2.txt"], result) + + +@freeze_time("2020-01-16 09:15:00") +def test_ignore_existing_files(fs, simple_mhl_history): + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64", "-i", "A/A1.txt", "-v"] + ) + assert_mhl_file_has_exact_ignore_patterns("root/ascmhl/0002_root_2020-01-16_091500Z.mhl", {"/A/A1.txt"}) + print(result.output) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/A1.txt"], result) + + +@freeze_time("2020-01-16 09:15:00") +def test_ignore_deleted_files(fs, simple_mhl_history): + runner = CliRunner() + os.remove("/root/A/A1.txt") + result = runner.invoke( + ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64", "-i", "A/A1.txt", "-v"] + ) + assert_mhl_file_has_exact_ignore_patterns("root/ascmhl/0002_root_2020-01-16_091500Z.mhl", {"/A/A1.txt"}) + print(result.output) + assert result.exit_code == 0 + + +@freeze_time("2020-01-16 09:15:00") +def test_ignore_changing_files(fs, simple_mhl_history): + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64", "-i", "A/A1.txt", "-v"] + ) + assert_mhl_file_has_exact_ignore_patterns("root/ascmhl/0002_root_2020-01-16_091500Z.mhl", {"/A/A1.txt"}) + print(result.output) + + os.remove("/root/A/A1.txt") + fs.create_file("/root/A/A1.txt", contents="1234567890") + result = runner.invoke(ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64", "-v"]) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/A1.txt"], result) + + +@freeze_time("2020-01-16 09:15:00") +def test_ignore_in_first_generation(fs): + runner = CliRunner() + + fs.create_file("/root/A/A1.txt", contents="1234567890") + fs.create_file("/root/A/A1.RMD", contents="1234567890") + fs.create_file("/root/A/A2.txt", contents="1234567890") + fs.create_file("/root/A/A2.RMD", contents="1234567890") + fs.create_file("/root/A/ignore.ign", contents="1234567890") + fs.create_file("/root/report.pdf", contents="1234567890") + fs.create_file("/root/report.txt", contents="1234567890") + fs.create_file("/root/ignore.ign", contents="1234567890") + + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root"), "-h", "xxh64", "-v", "-i", "*.txt", "-i", "ignore.ign"], + ) + print(result.output) + assert result.exit_code == 0 + pattern = ["/root/A/A1.txt", "/root/A/A2.txt", "/root/A/ignore.ign", "/root/ignore.ign", "/root/report.txt"] + assert_pattern_ignored_in_result(pattern, result) + assert_mhl_file_has_exact_ignore_patterns("root/ascmhl/0001_root_2020-01-16_091500Z.mhl", {"*.txt", "ignore.ign"}) + + +def test_ignore_paths_are_handled_correctly(fs, nested_mhl_histories): + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, [abspath_conversion_tests("/root/A"), "-h", "xxh64", "-v", "-i", "/AB/AB1.txt"] + ) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/AB/AB1.txt"], result) + + fs.create_file("/root/AB/AB1.txt") + result = runner.invoke( + ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64", "-v", "-i", "Stuff.txt"] + ) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/AB/AB1.txt", "/root/Stuff.txt"], result) + + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64", "-v", "-i", "A/AA/AA1.txt"] + ) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/AA/AA1.txt"], result) + + result = runner.invoke(ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64", "-v"]) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/AA/AA1.txt"], result) + + os.remove("/root/A/AA/AA1.txt") + result = runner.invoke(ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64", "-v"]) + assert result.exit_code == 0 + + os.remove("/root/A/AB/AB1.txt") + os.remove("/root/Stuff.txt") + result = runner.invoke(ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64", "-v"]) + assert result.exit_code == 0 + + os.remove("/root/B/B1.txt") + result = runner.invoke(ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64", "-v"]) + assert result.exit_code == 10 + + +def test_nested_histories_absolute_ignore_patterns(fs, nested_mhl_histories): + runner = CliRunner() + + # existing histories: /root, /root/A/AA, /root/B, /root/B/BB (1st gen) + # create a history with an ignore pattern + result = runner.invoke( + ascmhl.commands.create, [abspath_conversion_tests("/root/A/AA"), "-h", "xxh64", "-i", "/test.txt"] + ) + assert result.exit_code == 0 + + # create two test files + fs.create_file("/root/A/AA/test.txt", contents="testAtSubHistoryRoot") + fs.create_file("/root/A/AA/Subfolder/test.txt", contents="testInSubHistorySubfolder") + + # run the create once in the sub history and see if the ignore pattern works + result = runner.invoke(ascmhl.commands.create, [abspath_conversion_tests("/root/A/AA"), "-h", "xxh64", "-v"]) + assert result.exit_code == 0 + hash_list = MHLHistory.load_from_path(abspath("/root/A/AA")).hash_lists[-1] + + # the ignore pattern /test.txt should ignore this file + assert hash_list.find_media_hash_for_path("test.txt") is None + + # the ignore pattern /test.txt should not ignore this file + assert hash_list.find_media_hash_for_path(f"{Path('Subfolder/test.txt')}") is not None + + # same ignores should be applied when running it on the root history, so we run it again on the root hsitory + result = runner.invoke(ascmhl.commands.create, [abspath("/root"), "-h", "xxh64", "-v"]) + assert result.exit_code == 0 + hash_list = MHLHistory.load_from_path(abspath("/root/A/AA")).hash_lists[-1] + + # the ignore pattern /test.txt from the sub history should ignore this file also when verifying from the root + assert hash_list.find_media_hash_for_path("test.txt") is None + + # the ignore pattern /test.txt from the sub history should not ignore this file + assert hash_list.find_media_hash_for_path(f"{Path('Subfolder/test.txt')}") is not None + + +def test_nested_histories_ignoring_from_root(fs, nested_mhl_histories): + runner = CliRunner() + + # existing histories: /root, /root/A/AA, /root/B, /root/B/BB (1st gen) + # create two test files + fs.create_file("/root/A/AA/test.txt", contents="testAtSubHistoryRoot") + fs.create_file("/root/A/AA/Subfolder/test.txt", contents="testInSubHistorySubfolder") + + # ignore a file in the subhistory from the root hsitory + result = runner.invoke( + ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64", "-i", "/A/AA/test.txt"] + ) + assert result.exit_code == 0 + + hash_list = MHLHistory.load_from_path(abspath("/root/A/AA")).hash_lists[-1] + + # the ignore pattern should ignore this file + assert hash_list.find_media_hash_for_path("test.txt") is None + + # the ignore pattern should also ignore this file + assert hash_list.find_media_hash_for_path(f"{Path('Subfolder/test.txt')}") is not None + + # the ignore pattern inside the sub history should contain the pattern /test.txt not the path we passed into the root history + latest_ignores = MHLHistory.load_from_path(abspath("/root/A/AA")).latest_ignore_patterns() + assert "/test.txt" in latest_ignores + latest_ignores = MHLHistory.load_from_path(abspath("/root")).latest_ignore_patterns() + assert "/test.txt" not in latest_ignores + + # a second verification should not fail + result = runner.invoke(ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64"]) + assert result.exit_code == 0 + + +def test_ignore_multilevel_histories(fs, nested_mhl_histories): + runner = CliRunner() + + result = runner.invoke(ascmhl.commands.create, [abspath_conversion_tests("/root/A"), "-h", "xxh64"]) + assert result.exit_code == 0 + + result = runner.invoke( + ascmhl.commands.create, [abspath_conversion_tests("/root/A/AB"), "-v", "-h", "xxh64", "-i", "AB1.txt"] + ) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/AB/AB1.txt"], result) + latest_ignores = MHLHistory.load_from_path(abspath("/root/A/AB")).latest_ignore_patterns() + assert "AB1.txt" in latest_ignores + + assert result.exit_code == 0 + result = runner.invoke(ascmhl.commands.create, [abspath_conversion_tests("/root/B/BA"), "-h", "xxh64"]) + assert result.exit_code == 0 + result = runner.invoke(ascmhl.commands.create, [abspath_conversion_tests("/root"), "-h", "xxh64"]) + assert result.exit_code == 0 + + result = runner.invoke( + ascmhl.commands.create, [abspath_conversion_tests("/root"), "-v", "-h", "xxh64", "-i", "/A/AA/AA1.txt"] + ) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/AA/AA1.txt"], result) + latest_ignores = MHLHistory.load_from_path(abspath("/root/A/AA")).latest_ignore_patterns() + assert "/AA1.txt" in latest_ignores + + latest_ignores = MHLHistory.load_from_path(abspath("/root/A")).latest_ignore_patterns() + assert "/AA/AA1.txt" not in latest_ignores + assert "/AA1.txt" not in latest_ignores + latest_ignores = MHLHistory.load_from_path(abspath("/root/B")).latest_ignore_patterns() + assert "/AA/AA1.txt" not in latest_ignores + assert "/AA1.txt" not in latest_ignores + latest_ignores = MHLHistory.load_from_path(abspath("/root")).latest_ignore_patterns() + assert "/A/AA/AA1.txt" not in latest_ignores + assert "/AA/AA1.txt" not in latest_ignores + assert "/AA1.txt" not in latest_ignores + + +def test_ignore_files_in_nested_structures(fs, post_house_file_structure): + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/ShootingDay1"), "-v", "-i", "Sidecar.txt", "-h", "xxh64"], + ) + assert result.exit_code == 0 + + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1")).latest_ignore_patterns() + assert "Sidecar.txt" in latest_ignores + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia")).latest_ignore_patterns() + assert "Sidecar.txt" in latest_ignores + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/Sound")).latest_ignore_patterns() + assert "Sidecar.txt" in latest_ignores + + pattern = [ + "/root/ShootingDay1/CameraMedia/A/A001/Sidecar.txt", + "/root/ShootingDay1/CameraMedia/B/B002/Sidecar.txt", + "/root/ShootingDay1/Sound/Sidecar.txt", + ] + assert_pattern_ignored_in_result(pattern, result) + + +def test_ignore_file_type_in_nested_histories(fs, post_house_file_structure): + """ + This should test the '*' functionality + """ + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root"), "-v", "-i", "*.pdf", "-h", "xxh64"], + ) + assert result.exit_code == 0 + + assert_pattern_ignored_in_result( + ["/root/ShootingDay1/CameraMedia/Report.pdf", "/root/ShootingDay1/Report.pdf"], result + ) + + latest_ignores = MHLHistory.load_from_path(abspath("/root")).latest_ignore_patterns() + assert "*.pdf" in latest_ignores + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia")).latest_ignore_patterns() + assert "*.pdf" in latest_ignores + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/Sound")).latest_ignore_patterns() + assert "*.pdf" in latest_ignores + + fs.create_file("/root/ShootingDay1/Sound/Notes.pdf") + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root"), "-v", "-h", "xxh64"], + ) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/ShootingDay1/Sound/Notes.pdf"], result) + + +def test_ignore_file_type_in_path(fs, post_house_file_structure_with_range): + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/ShootingDay1"), "-v", "-i", "CameraMedia/A/A002/*.mov", "-h", "xxh64"], + ) + assert result.exit_code == 0 + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia/A")).latest_ignore_patterns() + assert "/A002/*.mov" in latest_ignores + pattern = [ + "/root/ShootingDay1/CameraMedia/A/A002/A002C001.mov", + "/root/ShootingDay1/CameraMedia/A/A002/A002C002.mov", + "/root/ShootingDay1/CameraMedia/A/A002/A002C003.mov", + "/root/ShootingDay1/CameraMedia/A/A002/A002C004.mov", + ] + assert_pattern_ignored_in_result(pattern, result) + + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1")).latest_ignore_patterns() + assert "/CameraMedia/A/A002/*.mov" not in latest_ignores + assert "CameraMedia/A/A002/*.mov" not in latest_ignores + + # verify it again without providing the ignore pattern + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/ShootingDay1"), "-v", "-h", "xxh64"], + ) + assert result.exit_code == 0 + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia/A")).latest_ignore_patterns() + assert "/A002/*.mov" in latest_ignores + assert_pattern_ignored_in_result(pattern, result) + + +def test_ignore_directories_in_nested_structures_pattern_1(fs, post_house_file_structure): + """ + This pattern should match "Proxy" anywhere in the filepath + """ + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/ShootingDay1/CameraMedia"), "-v", "-i", "Proxy", "-h", "xxh64"], + ) + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia")).latest_ignore_patterns() + assert "Proxy" in latest_ignores + pattern = [ + "/root/ShootingDay1/CameraMedia/A/Proxy", + "/root/ShootingDay1/CameraMedia/B/Proxy", + "/root/ShootingDay1/CameraMedia/Proxy", + ] + assert_pattern_ignored_in_result(pattern, result) + assert result.exit_code == 0 + + +def test_ignore_directories_in_nested_structures_pattern_2(fs, post_house_file_structure): + """ + This pattern should only match directories named "Proxy" anywhere in the directory tree, but not files with that name + """ + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/ShootingDay1/CameraMedia"), "-v", "-i", "Proxy/", "-h", "xxh64"], + ) + pattern = ["/root/ShootingDay1/CameraMedia/A/Proxy", "/root/ShootingDay1/CameraMedia/B/Proxy"] + assert_pattern_ignored_in_result(pattern, result) + assert_pattern_ignored_in_result(["/root/ShootingDay1/CameraMedia/Proxy"], result, negate=True) + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia")).latest_ignore_patterns() + assert "Proxy/" in latest_ignores + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia/A")).latest_ignore_patterns() + assert "Proxy/" in latest_ignores + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia/B")).latest_ignore_patterns() + assert "Proxy/" in latest_ignores + assert result.exit_code == 0 + + +def test_ignore_directories_in_nested_structures_pattern_3(fs, post_house_file_structure): + """ + This pattern should only match /A/Proxy/ relative to CameraMedia and not be applied to other directories + """ + runner = CliRunner() + fs.create_file("/root/ShootingDay1/CameraMedia/B/Proxy/A/Proxy/B001/A3_p.txt", contents="A3 Proxy") + + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/ShootingDay1/CameraMedia"), "-v", "-i", "A/Proxy/", "-h", "xxh64"], + ) + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia/A")).latest_ignore_patterns() + assert "/Proxy/" in latest_ignores + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia/B")).latest_ignore_patterns() + assert "A/Proxy/" not in latest_ignores + assert "/A/Proxy/" not in latest_ignores + assert "//A/Proxy/" not in latest_ignores + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia")).latest_ignore_patterns() + assert "A/Proxy/" not in latest_ignores + assert "/A/Proxy/" not in latest_ignores + assert_pattern_ignored_in_result(["/root/ShootingDay1/CameraMedia/A/Proxy"], result) + assert_pattern_ignored_in_result( + ["/root/ShootingDay1/CameraMedia/B/Proxy", "/root/ShootingDay1/CameraMedia/Proxy"], result, negate=True + ) + assert f"created original hash for {Path('B/Proxy/A/Proxy/B001/A3_p.txt')}" in result.output + assert result.exit_code == 0 + + +def test_ignore_directories_in_nested_structures_pattern_3_wrong_directory(fs, post_house_file_structure): + """ + This pattern should only match /A/Proxy/ relative to the ascmhl and not be applied to other directories + (i.e. have no effect when called on root/ShootingDay1, since it does not contain a directory 'A/Proxy/') + """ + runner = CliRunner() + fs.create_file("/root/ShootingDay1/CameraMedia/B/Proxy/A/Proxy/B001/A3_p.txt", contents="A3 Proxy") + + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/ShootingDay1"), "-v", "-i", "A/Proxy/", "-h", "xxh64"], + ) + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia/A")).latest_ignore_patterns() + assert "/Proxy/" not in latest_ignores + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia/B")).latest_ignore_patterns() + assert "/A/Proxy/" not in latest_ignores + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia")).latest_ignore_patterns() + assert "/A/Proxy/" not in latest_ignores + pattern = [ + "/root/ShootingDay1/CameraMedia/A/Proxy", + "/root/ShootingDay1/CameraMedia/B/Proxy", + "/root/ShootingDay1/CameraMedia/Proxy", + ] + assert_pattern_ignored_in_result(pattern, result, negate=True) + assert f"created original hash for {Path('CameraMedia/B/Proxy/A/Proxy/B001/A3_p.txt')}" in result.output + assert result.exit_code == 0 + + +def test_ignore__directories_in_nested_structures_pattern_4(fs, post_house_file_structure): + """ + This pattern should ignore any occurence of Proxy in the file- or directoryname below root/ShootingDay1 + """ + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/ShootingDay1"), "-v", "-i", "**/Proxy", "-h", "xxh64"], + ) + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1")).latest_ignore_patterns() + assert "/**/Proxy" in latest_ignores + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia")).latest_ignore_patterns() + assert "/**/Proxy" in latest_ignores + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia/A")).latest_ignore_patterns() + assert "/**/Proxy" in latest_ignores + pattern = [ + "/root/ShootingDay1/CameraMedia/A/Proxy", + "/root/ShootingDay1/CameraMedia/B/Proxy", + "/root/ShootingDay1/CameraMedia/Proxy", + ] + assert_pattern_ignored_in_result(pattern, result) + assert result.exit_code == 0 + + +def test_ignore__directories_in_nested_structures_pattern_5(fs, post_house_file_structure): + """ + This pattern should ignore any directory 'Proxy', but not a file named 'Proxy' + """ + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/ShootingDay1"), "-v", "-i", "**/Proxy/", "-h", "xxh64"], + ) + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1")).latest_ignore_patterns() + assert "/**/Proxy/" in latest_ignores + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia")).latest_ignore_patterns() + assert "/**/Proxy/" in latest_ignores + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia/A")).latest_ignore_patterns() + assert "/**/Proxy/" in latest_ignores + pattern = ["/root/ShootingDay1/CameraMedia/A/Proxy", "/root/ShootingDay1/CameraMedia/B/Proxy"] + assert_pattern_ignored_in_result(pattern, result) + assert_pattern_ignored_in_result(["/root/ShootingDay1/CameraMedia/Proxy"], result, negate=True) + assert result.exit_code == 0 + + +def test_ignore__directories_in_nested_structures_pattern_6(fs, post_house_file_structure): + """ + This pattern should ignore any files in any directory 'Proxy', but not a file named 'Proxy' + """ + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/ShootingDay1"), "-v", "-i", "**/Proxy/**", "-h", "xxh64"], + ) + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1")).latest_ignore_patterns() + assert "**/Proxy/**" in latest_ignores + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia")).latest_ignore_patterns() + assert "**/Proxy/**" in latest_ignores + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia/A")).latest_ignore_patterns() + assert "**/Proxy/**" in latest_ignores + pattern = ["/root/ShootingDay1/CameraMedia/A/Proxy", "/root/ShootingDay1/CameraMedia/B/Proxy"] + assert_pattern_ignored_in_result(pattern, result) + assert_pattern_ignored_in_result(["/root/ShootingDay1/CameraMedia/Proxy"], result, negate=True) + assert result.exit_code == 0 + + +def test_ignore_single_character(fs, post_house_file_structure_with_range): + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/ShootingDay1/CameraMedia/A"), "-v", "-i", "A002C00?.mov", "-h", "xxh64"], + ) + assert result.exit_code == 0 + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia/A")).latest_ignore_patterns() + assert "A002C00?.mov" in latest_ignores + pattern = [ + "/root/ShootingDay1/CameraMedia/A/A002/A002C001.mov", + "/root/ShootingDay1/CameraMedia/A/A002/A002C002.mov", + "/root/ShootingDay1/CameraMedia/A/A002/A002C003.mov", + "/root/ShootingDay1/CameraMedia/A/A002/A002C004.mov", + ] + assert_pattern_ignored_in_result(pattern, result) + + +def test_ignore_single_character_in_path(fs, post_house_file_structure_with_range): + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/ShootingDay1"), "-v", "-i", "CameraMedia/A/A002/A002C00?.mov", "-h", "xxh64"], + ) + assert result.exit_code == 0 + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia/A")).latest_ignore_patterns() + assert "/A002/A002C00?.mov" in latest_ignores + pattern = [ + "/root/ShootingDay1/CameraMedia/A/A002/A002C001.mov", + "/root/ShootingDay1/CameraMedia/A/A002/A002C002.mov", + "/root/ShootingDay1/CameraMedia/A/A002/A002C003.mov", + "/root/ShootingDay1/CameraMedia/A/A002/A002C004.mov", + ] + assert_pattern_ignored_in_result(pattern, result) + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1")).latest_ignore_patterns() + assert "CameraMedia/A/A002C00?.mov" not in latest_ignores + + +def test_ignore_multiple_individual_characters_in_path(fs, post_house_file_structure): + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, + [ + abspath_conversion_tests("/root/ShootingDay1"), + "-v", + "-i", + "CameraMedia/?/Proxy/?001/?001.ale", + "-h", + "xxh64", + ], + ) + assert result.exit_code == 0 + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia")).latest_ignore_patterns() + assert "/?/Proxy/?001/?001.ale" in latest_ignores + pattern = [ + "/root/ShootingDay1/CameraMedia/A/Proxy/A001/A001.ale", + "/root/ShootingDay1/CameraMedia/B/Proxy/B001/B001.ale", + ] + assert_pattern_ignored_in_result(pattern, result) + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1")).latest_ignore_patterns() + assert "CameraMedia/?/Proxy/?001/?001.ale" not in latest_ignores + + +def test_ignore_range_of_characters(fs, post_house_file_structure_with_range): + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, + [ + abspath_conversion_tests("/root/ShootingDay1"), + "-v", + "-i", + "CameraMedia/A/A00[1-5]/A00?C00[1-9].mov", + "-h", + "xxh64", + ], + ) + assert result.exit_code == 0 + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia/A")).latest_ignore_patterns() + assert "/A00[1-5]/A00?C00[1-9].mov" in latest_ignores + pattern = [ + "/root/ShootingDay1/CameraMedia/A/A002/A002C001.mov", + "/root/ShootingDay1/CameraMedia/A/A002/A002C002.mov", + "/root/ShootingDay1/CameraMedia/A/A002/A002C003.mov", + "/root/ShootingDay1/CameraMedia/A/A002/A002C004.mov", + ] + assert_pattern_ignored_in_result(pattern, result) + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1")).latest_ignore_patterns() + assert "CameraMedia/A/A00[1-5]/A00?C00[1-9].mov" not in latest_ignores + + +def test_ignore_negate_pattern(fs, post_house_file_structure): + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/ShootingDay1"), "-v", "-i", "Sidecar.txt", "-h", "xxh64"], + ) + assert result.exit_code == 0 + + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1")).latest_ignore_patterns() + assert "Sidecar.txt" in latest_ignores + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia")).latest_ignore_patterns() + assert "Sidecar.txt" in latest_ignores + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/Sound")).latest_ignore_patterns() + assert "Sidecar.txt" in latest_ignores + pattern = [ + "/root/ShootingDay1/CameraMedia/A/A001/Sidecar.txt", + "/root/ShootingDay1/CameraMedia/B/B002/Sidecar.txt", + "/root/ShootingDay1/Sound/Sidecar.txt", + ] + assert_pattern_ignored_in_result(pattern, result) + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/ShootingDay1/Sound"), "-v", "-i", "!Sidecar.txt", "-h", "xxh64"], + ) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/ShootingDay1/Sound/Sidecar.txt"], result, negate=True) + + +@freeze_time("2020-01-16 09:15:00") +def test_ignore_old_deleted_files_in_histories(fs, nested_mhl_histories): + runner = CliRunner() + + # existing histories: /root, /root/A/AA, /root/B, /root/B/BB (1st gen) + + fs.create_file("/root/A/AA/AA1.RMD", contents="Lorem ipsum dolor") + fs.create_file("/root/A/AB/AB1.RMD", contents="sit amet con vota") + fs.create_file("/root/B/BA/B1.RMD", contents="lirum alamru aexti") + fs.create_file("/root/A/AA/AA2.txt", contents="Lorem ipsum dolor") + + result = runner.invoke(ascmhl.commands.create, [abspath("/root"), "-h", "xxh64"]) + assert result.exit_code == 0 + result = runner.invoke(ascmhl.commands.create, [abspath("/root/A/AA"), "-h", "xxh64"]) + assert result.exit_code == 0 + result = runner.invoke(ascmhl.commands.create, [abspath("/root/B"), "-h", "xxh64"]) + assert result.exit_code == 0 + result = runner.invoke(ascmhl.commands.create, [abspath("/root/B/BB"), "-h", "xxh64"]) + assert result.exit_code == 0 + + # existing histories: /root, /root/A/AA, /root/B, /root/B/BB (2nd gen) + + print("First session, no ignores yet\n") + result = runner.invoke(ascmhl.commands.create, [abspath_conversion_tests("/root/A"), "-h", "xxh64", "-v"]) + print(result.output) + assert result.exit_code == 0 + + print("Second Session, should ignore single existing file AB1.txt in root/A/AB/AB1.txt\n") + + result = runner.invoke( + ascmhl.commands.create, [abspath_conversion_tests("/root/A"), "-v", "-h", "xxh64", "-i", "AB/AB1.txt"] + ) + print(result.output) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/AB/AB1.txt"], result) + assert_mhl_file_has_exact_ignore_patterns("root/A/ascmhl/0002_A_2020-01-16_091500Z.mhl", {"/AB/AB1.txt"}) + + print('Third Session, ignore "*.txt" and "/root/A/AB/AB1.RMD "\nRemove root/A/AA/AA1.txt and /root/A/AB/AB1.txt\n') + os.remove("/root/A/AA/AA1.txt") + os.remove("/root/A/AB/AB1.txt") + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/A"), "-v", "-h", "xxh64", "-i", "*.txt", "-i", "AB/AB1.RMD"], + ) + print(result.output) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/AA/AA2.txt", "/root/A/AB/AB1.RMD"], result) + assert_mhl_file_has_exact_ignore_patterns( + "root/A/ascmhl/0003_A_2020-01-16_091500Z.mhl", {"*.txt", "/AB/AB1.RMD", "/AB/AB1.txt"} + ) + + result = runner.invoke(ascmhl.commands.create, [abspath_conversion_tests("/root"), "-v", "-h", "xxh64"]) + print(result.output) + assert result.exit_code == 0 + + os.remove("/root/B/BB/BB1.txt") + + result = runner.invoke(ascmhl.commands.create, [abspath_conversion_tests("/root"), "-v", "-h", "xxh64"]) + print(result.output) + assert result.exit_code == 10 + + +def test_ignore_multiple_pattern_in_single_command(fs, post_house_file_structure): + fs.create_file("/root/ShootingDay1/CameraMedia/A/A001/Report.pdf", contents="A1") + fs.create_file("/root/ShootingDay1/CameraMedia/B/B002/ClipList.pdf", contents="A1") + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, + [ + abspath_conversion_tests("/root/ShootingDay1"), + "-v", + "-h", + "xxh64", + "-i", + "Sidecar.txt", + "-i", + "Proxy/", + "-i", + "CameraMedia/**/*.pdf", + "-i", + "/CameraMedia/B/B002/B3.txt", + ], + ) + assert result.exit_code == 0 + + pattern = [ + "/root/ShootingDay1/CameraMedia/Report.pdf", + "/root/ShootingDay1/CameraMedia/A/A001/Report.pdf", + "/root/ShootingDay1/CameraMedia/B/B002/ClipList.pdf", + "/root/ShootingDay1/CameraMedia/B/B002/Sidecar.txt", + "/root/ShootingDay1/Sound/Sidecar.txt", + "/root/ShootingDay1/CameraMedia/A/Proxy", + "/root/ShootingDay1/CameraMedia/B/Proxy", + ] + assert_pattern_ignored_in_result(pattern, result) + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1")).latest_ignore_patterns() + assert "Sidecar.txt" in latest_ignores + assert "Proxy/" in latest_ignores + assert "/CameraMedia/**/*.pdf" not in latest_ignores + assert "/CameraMedia/B/B002/B3.txt" not in latest_ignores + + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/Sound")).latest_ignore_patterns() + assert "Sidecar.txt" in latest_ignores + assert "Proxy/" in latest_ignores + assert "CameraMedia/**/*.pdf" not in latest_ignores + assert "/CameraMedia/B/B002/B3.txt" not in latest_ignores + + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia")).latest_ignore_patterns() + assert "Sidecar.txt" in latest_ignores + assert "Proxy/" in latest_ignores + assert "/**/*.pdf" in latest_ignores + assert "/CameraMedia/B/B002/B3.txt" not in latest_ignores + + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia/A")).latest_ignore_patterns() + assert "Sidecar.txt" in latest_ignores + assert "Proxy/" in latest_ignores + assert "/**/*.pdf" in latest_ignores + assert "/B002/B3.txt" not in latest_ignores + + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia/B")).latest_ignore_patterns() + assert "Sidecar.txt" in latest_ignores + assert "Proxy/" in latest_ignores + assert "/**/*.pdf" in latest_ignores + assert "/B002/B3.txt" in latest_ignores + + +def test_ignore_diff_nested_multiple_pattern(fs, post_house_file_structure): + fs.create_file("/root/ShootingDay1/CameraMedia/A/A001/Report.pdf", contents="A1") + fs.create_file("/root/ShootingDay1/CameraMedia/B/B002/ClipList.pdf", contents="A1") + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.diff, + [ + abspath_conversion_tests("/root/ShootingDay1"), + "-v", + "-i", + "Sidecar.txt", + "-i", + "Proxy/", + "-i", + "CameraMedia/**/*.pdf", + "-i", + "/CameraMedia/B/B002/B3.txt", + "-i", + "/CameraMedia/Proxy", + ], + ) + assert result.exit_code == 0 + + pattern = [ + "/root/ShootingDay1/CameraMedia/Report.pdf", + "/root/ShootingDay1/CameraMedia/A/A001/Report.pdf", + "/root/ShootingDay1/CameraMedia/B/B002/ClipList.pdf", + "/root/ShootingDay1/CameraMedia/B/B002/Sidecar.txt", + "/root/ShootingDay1/Sound/Sidecar.txt", + "/root/ShootingDay1/CameraMedia/A/Proxy", + "/root/ShootingDay1/CameraMedia/B/Proxy", + ] + assert_pattern_ignored_in_result(pattern, result) + + +def test_ignore_from_file(fs, post_house_file_structure): + fs.create_file( + "/user/ignore.txt", contents="Sidecar.txt\n/CameraMedia/**/Proxy/**\nCameraMedia/A/A001/A1.txt\n/**/temp\n/dir" + ) + + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/ShootingDay1"), "-v", "-h", "xxh64", "-ii", "/user/ignore.txt"], + ) + assert result.exit_code == 0 + + pattern = [ + "/root/ShootingDay1/CameraMedia/A/A001/A1.txt", + "/root/ShootingDay1/CameraMedia/A/A001/Sidecar.txt", + "/root/ShootingDay1/CameraMedia/B/B002/Sidecar.txt", + "/root/ShootingDay1/CameraMedia/A/Proxy/A001", + "/root/ShootingDay1/CameraMedia/B/Proxy/B001", + "/root/ShootingDay1/Sound/Sidecar.txt", + ] + assert_pattern_ignored_in_result(pattern, result) + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia/A")).latest_ignore_patterns() + assert "Sidecar.txt" in latest_ignores + assert "/A001/A1.txt" in latest_ignores + assert "/**/temp" in latest_ignores + assert "/dir" not in latest_ignores + + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia/B")).latest_ignore_patterns() + assert "Sidecar.txt" in latest_ignores + assert "/A001/A1.txt" not in latest_ignores + assert "/**/temp" in latest_ignores + assert "/dir" not in latest_ignores + + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia")).latest_ignore_patterns() + assert "/**/Proxy/**" in latest_ignores + assert "/A001/A1.txt" not in latest_ignores + assert "/**/temp" in latest_ignores + assert "/dir" not in latest_ignores + + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/Sound")).latest_ignore_patterns() + assert "Sidecar.txt" in latest_ignores + assert "/CameraMedia/**/Proxy/**" not in latest_ignores + assert "/CameraMedia/A001/A1.txt" not in latest_ignores + assert "/**/temp" in latest_ignores + assert "/dir" not in latest_ignores + + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1")).latest_ignore_patterns() + assert "Sidecar.txt" in latest_ignores + assert "/CameraMedia/**/Proxy/**" not in latest_ignores + assert "/CameraMedia/A001/A1.txt" not in latest_ignores + assert "/**/temp" in latest_ignores + assert "/dir" in latest_ignores + + +def test_ignore_history_in_ignored_directory(fs): + runner = CliRunner() + fs.create_file("/root/ShootingDay1/CameraMedia/A/A001/A1.txt", contents="A1") + fs.create_file("/root/ShootingDay1/CameraMedia/A/A001/A2.txt", contents="A2") + fs.create_file("/root/ShootingDay1/CameraMedia/A/A001/A3.txt", contents="A3") + fs.create_file("/root/ShootingDay1/CameraMedia/A/A001/Sidecar.txt", contents="Sidecar1") + result = runner.invoke(ascmhl.commands.create, [abspath("/root/ShootingDay1"), "-h", "xxh64"]) + assert result.exit_code == 0 + + result = runner.invoke(ascmhl.commands.create, [abspath("/root/ShootingDay1/CameraMedia"), "-h", "xxh64"]) + assert result.exit_code == 0 + + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/ShootingDay1/CameraMedia/A/A001"), "-v", "-h", "xxh64"], + ) + assert result.exit_code == 0 + + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/ShootingDay1"), "-v", "-i", "/CameraMedia/A/A001", "-h", "xxh64"], + ) + print(result.output) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/ShootingDay1/CameraMedia/A/A001"], result) + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1/CameraMedia")).latest_ignore_patterns() + assert "/A/A001" in latest_ignores + + latest_ignores = MHLHistory.load_from_path(abspath("/root/ShootingDay1")).latest_ignore_patterns() + assert "/CameraMedia/A/A001" not in latest_ignores + + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/ShootingDay1"), "-v", "-h", "xxh64"], + ) + print(result.output) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/ShootingDay1/CameraMedia/A/A001"], result) + + +def test_ignore_only_one_child_history(fs): + fs.create_file("/root/A/A001/A001C001.txt") + fs.create_file("/root/A/A001/A001C002.txt") + fs.create_file("/root/A/A001/A001C003.txt") + + fs.create_file("/root/A/A002/A002C001.txt") + fs.create_file("/root/A/A002/A002C002.txt") + fs.create_file("/root/A/A002/A002C003.txt") + + runner = CliRunner() + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/A/A002"), "-v", "-h", "xxh64"], + ) + assert result.exit_code == 0 + + result = runner.invoke( + ascmhl.commands.create, + [abspath_conversion_tests("/root/A"), "-v", "-i", "/A001/A001C001.txt", "-h", "xxh64"], + ) + assert result.exit_code == 0 + assert_pattern_ignored_in_result(["/root/A/A001/A001C001.txt"], result) + + latest_ignores = MHLHistory.load_from_path(abspath("/root/A")).latest_ignore_patterns() + assert "/A001/A001C001.txt" in latest_ignores + + latest_ignores = MHLHistory.load_from_path(abspath("/root/A/A002")).latest_ignore_patterns() + assert "/A001/A001C001.txt" not in latest_ignores + + +def test_ignore_unit_belongs_to_parent_or_neighbour(): + pattern = "*.txt" + parent_rel_path = f"{Path('/A/AA/AAA')}" + result = ascmhl.generator.belongs_to_parent_or_neighbour(pattern, parent_rel_path) + assert result == False + + pattern = "A.txt" + parent_rel_path = f"{Path('/A/AA/AAA')}" + result = ascmhl.generator.belongs_to_parent_or_neighbour(pattern, parent_rel_path) + assert result == False + + pattern = "A/" + parent_rel_path = f"{Path('/A/AA/AAA')}" + result = ascmhl.generator.belongs_to_parent_or_neighbour(pattern, parent_rel_path) + assert result == False + + pattern = "/A" + parent_rel_path = f"{Path('/A/AA/AAA')}" + result = ascmhl.generator.belongs_to_parent_or_neighbour(pattern, parent_rel_path) + assert result == True + + pattern = "/A/" + parent_rel_path = f"{Path('/A/AA/AAA')}" + result = ascmhl.generator.belongs_to_parent_or_neighbour(pattern, parent_rel_path) + assert result == True + + pattern = "/A/AA" + parent_rel_path = f"{Path('/A/AA/AAA')}" + result = ascmhl.generator.belongs_to_parent_or_neighbour(pattern, parent_rel_path) + assert result == True + + pattern = "/A/AA/" + parent_rel_path = f"{Path('/A/AA/AAA')}" + result = ascmhl.generator.belongs_to_parent_or_neighbour(pattern, parent_rel_path) + assert result == True + + pattern = "/A/AA/AAA" + parent_rel_path = f"{Path('/A/AA/AAA')}" + result = ascmhl.generator.belongs_to_parent_or_neighbour(pattern, parent_rel_path) + assert result == True + + pattern = "/A/AA/AAA.txt" + parent_rel_path = f"{Path('/A/AA/AAA')}" + result = ascmhl.generator.belongs_to_parent_or_neighbour(pattern, parent_rel_path) + assert result == True + + pattern = "/A/AA/AAA/A.txt" + parent_rel_path = f"{Path('/A/AA/AAA')}" + result = ascmhl.generator.belongs_to_parent_or_neighbour(pattern, parent_rel_path) + assert result == False + + pattern = "A/**/AA" + parent_rel_path = f"{Path('/A/AA/AAA')}" + result = ascmhl.generator.belongs_to_parent_or_neighbour(pattern, parent_rel_path) + assert result == False + + pattern = "A/AA/**" + parent_rel_path = f"{Path('/A/AA/AAA')}" + result = ascmhl.generator.belongs_to_parent_or_neighbour(pattern, parent_rel_path) + assert result == True + + pattern = "**/AA/" + parent_rel_path = f"{Path('/A/AA/AAA')}" + result = ascmhl.generator.belongs_to_parent_or_neighbour(pattern, parent_rel_path) + assert result == False + + pattern = "/**/AA" + parent_rel_path = f"{Path('/A/AA/AAA')}" + result = ascmhl.generator.belongs_to_parent_or_neighbour(pattern, parent_rel_path) + assert result == True + + pattern = "/**/AA/" + parent_rel_path = f"{Path('/A/AA/AAA')}" + result = ascmhl.generator.belongs_to_parent_or_neighbour(pattern, parent_rel_path) + assert result == True + + pattern = "A/B/AAA" + parent_rel_path = f"{Path('/A/AA/AAA')}" + result = ascmhl.generator.belongs_to_parent_or_neighbour(pattern, parent_rel_path) + assert result == True + + pattern = "A/AA/AAB" + parent_rel_path = f"{Path('/A/AA/AAA')}" + result = ascmhl.generator.belongs_to_parent_or_neighbour(pattern, parent_rel_path) + assert result == True + + pattern = "A/AA/AAB/AA" + parent_rel_path = f"{Path('/A/AA/AAA')}" + result = ascmhl.generator.belongs_to_parent_or_neighbour(pattern, parent_rel_path) + assert result == True + + pattern = "B/AA/AAA" + parent_rel_path = f"{Path('/A/AA/AAA')}" + result = ascmhl.generator.belongs_to_parent_or_neighbour(pattern, parent_rel_path) + assert result == True + + pattern = "/B/AA/AAA" + parent_rel_path = f"{Path('/A/AA/AAA')}" + result = ascmhl.generator.belongs_to_parent_or_neighbour(pattern, parent_rel_path) + assert result == True + + +def test_ignore_unit_extract_relpath(): + base = f"{Path('/home/user/docs')}" + target = "/home/user/docs/file.txt" + expected = "file.txt" + result = ascmhl.generator._extract_pattern_relative_to_history(target, base) + assert result == expected + + base = f"{Path('/home/user')}" + target = "/home/user/projects/code" + expected = "projects/code" + result = ascmhl.generator._extract_pattern_relative_to_history(target, base) + assert result == expected + + base = f"{Path('/home/user/docs')}" + target = "/home/user" + expected = None + result = ascmhl.generator._extract_pattern_relative_to_history(target, base) + assert result == expected + + base = f"{Path('/home/user/docs')}" + target = "/home/user/photos/image.jpg" + expected = "photos/image.jpg" + result = ascmhl.generator._extract_pattern_relative_to_history(target, base) + assert result == expected + + base = f"{Path('/home/user/docs')}" + target = "/home/user/docs" + expected = None + result = ascmhl.generator._extract_pattern_relative_to_history(target, base) + assert result == expected + + base = f"{Path('/')}" + target = "/var/log" + expected = "var/log" + result = ascmhl.generator._extract_pattern_relative_to_history(target, base) + assert result == expected + + base = f"{Path('/home/user')}" + target = "/home/user/docs/" + expected = "docs/" + result = ascmhl.generator._extract_pattern_relative_to_history(target, base) + assert result == expected From fbf549194a276361d553a3ed3c8725e031daad77 Mon Sep 17 00:00:00 2001 From: pfn-djf Date: Wed, 21 May 2025 15:01:53 +0200 Subject: [PATCH 2/2] Make pathspec work against directories in traversal --- ascmhl/traverse.py | 8 +++++--- tests/test_ignore_extended.py | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/ascmhl/traverse.py b/ascmhl/traverse.py index 1d978e8..e4a9d8f 100644 --- a/ascmhl/traverse.py +++ b/ascmhl/traverse.py @@ -31,12 +31,14 @@ def post_order_lexicographic(top: str, ignore_pathspec: pathspec.PathSpec = None children = [] for name in names: file_path = os.path.join(top, name) + is_directory = isdir(file_path) + if is_directory: + file_path = file_path + "/" if ignore_pathspec and ignore_pathspec.match_file(file_path): if os.path.basename(os.path.normpath(file_path)) != ascmhl_folder_name: - logger.verbose(f"ignoring filepath {file_path}") + logger.verbose(f"ignoring filepath {file_path.rstrip('/')}") continue - path = join(top, name) - children.append((name, isdir(path))) + children.append((name, is_directory)) # if directory, yield children recursively in post order until exhausted. for name, is_dir in children: diff --git a/tests/test_ignore_extended.py b/tests/test_ignore_extended.py index 0e62924..2988b84 100644 --- a/tests/test_ignore_extended.py +++ b/tests/test_ignore_extended.py @@ -946,8 +946,8 @@ def test_ignore_from_file(fs, post_house_file_structure): "/root/ShootingDay1/CameraMedia/A/A001/A1.txt", "/root/ShootingDay1/CameraMedia/A/A001/Sidecar.txt", "/root/ShootingDay1/CameraMedia/B/B002/Sidecar.txt", - "/root/ShootingDay1/CameraMedia/A/Proxy/A001", - "/root/ShootingDay1/CameraMedia/B/Proxy/B001", + "/root/ShootingDay1/CameraMedia/A/Proxy", + "/root/ShootingDay1/CameraMedia/B/Proxy", "/root/ShootingDay1/Sound/Sidecar.txt", ] assert_pattern_ignored_in_result(pattern, result)