diff --git a/ascmhl/commands.py b/ascmhl/commands.py index 824ec07..c75d4e9 100644 --- a/ascmhl/commands.py +++ b/ascmhl/commands.py @@ -34,6 +34,7 @@ from .traverse import post_order_lexicographic from typing import Dict from collections import namedtuple +from .utils import convert_local_path_to_posix @click.command() @@ -1195,18 +1196,30 @@ def flatten_history( if len(existing_history.hash_lists) == 0: raise errors.NoMHLHistoryException(root_path) - for hash_list in existing_history.hash_lists: + flatten_child_histories(existing_history, session, root_path) + + commit_session_for_collection( + session, root_path, author_name, author_email, author_phone, author_role, location, comment + ) + + +def flatten_child_histories(history, session, roothistorypath, pathprefix=""): + for hash_list in history.hash_lists: for media_hash in hash_list.media_hashes: if not media_hash.is_directory: for hash_entry in media_hash.hash_entries: if hash_entry.action != "failed": + # add prefix to media path if subhistory + media_path = media_hash.path + if pathprefix != "": + media_path = convert_local_path_to_posix(pathprefix) + "/" + media_hash.path # check if this entry is newer than the one already in there to avoid duplicate entries - found_media_hash = session.new_hash_lists[collection_history].find_media_hash_for_path( - media_hash.path + found_media_hash = session.new_hash_lists[session.root_history].find_media_hash_for_path( + media_path ) if found_media_hash == None: session.append_file_hash( - media_hash.path, + media_path, media_hash.file_size, media_hash.last_modification_date, hash_entry.hash_format, @@ -1222,7 +1235,7 @@ def flatten_history( if not hashformat_is_already_there: # assuming that hash_entry of same type also has same hash_value .. session.append_file_hash( - media_hash.path, + media_path, media_hash.file_size, media_hash.last_modification_date, hash_entry.hash_format, @@ -1231,9 +1244,14 @@ def flatten_history( hash_date=hash_entry.hash_date, ) - commit_session_for_collection( - session, root_path, author_name, author_email, author_phone, author_role, location, comment - ) + for child_history in history.child_histories: + childpath = child_history.get_root_path() + + # if os.path.isabs(file_path): + childrelativepath = os.path.relpath(childpath, roothistorypath) + + logger.info(f"\nChild History at {childrelativepath}:") + flatten_child_histories(child_history, session, roothistorypath, childrelativepath) @click.command() @@ -1471,7 +1489,7 @@ def commit_session_for_collection( process_info.root_media_hash = root_hash process_info.hashlist_custom_basename = "packinglist_" + os.path.basename(root_path) - session.commit(creator_info, process_info) + session.commit(creator_info, process_info, writeChain=False) """ diff --git a/ascmhl/generator.py b/ascmhl/generator.py index 170ff8a..dfd7bab 100644 --- a/ascmhl/generator.py +++ b/ascmhl/generator.py @@ -7,6 +7,9 @@ __email__ = "opensource@pomfort.com" """ +import os +import shutil + from collections import defaultdict from typing import Dict, List @@ -15,6 +18,7 @@ from .ignore import MHLIgnoreSpec from .hashlist import MHLHashList, MHLHashEntry, MHLCreatorInfo, MHLProcessInfo from .history import MHLHistory +from .utils import convert_posix_to_local_path class MHLGenerationCreationSession: @@ -131,7 +135,13 @@ def append_file_hash( hash_entry = MHLHashEntry(hash_format, hash_string, hash_date=hash_date) if original_hash_entry is None: hash_entry.action = "original" - logger.verbose(f" created original hash for {relative_path} {hash_format}: {hash_string}") + if relative_path != None: + logger.verbose(f" created original hash for {relative_path} {hash_format}: {hash_string}") + else: + # flattening works a bit different, because we don't add to individual (nested) histories + logger.verbose( + f" created original hash for {convert_posix_to_local_path(file_path)} {hash_format}: {hash_string}" + ) else: existing_hash_entry = history.find_first_hash_entry_for_path(history_relative_path, hash_format) if existing_hash_entry is not None: @@ -272,7 +282,7 @@ def append_directory_hashes( hash_entry.structure_hash_string = structure_hash_string parent_media_hash.append_hash_entry(hash_entry) - def commit(self, creator_info: MHLCreatorInfo, process_info: MHLProcessInfo): + def commit(self, creator_info: MHLCreatorInfo, process_info: MHLProcessInfo, writeChain=True): """ this method needs to create the generations of the children bottom up # so each history can reference the children correctly and can get the actual hash of the file @@ -306,4 +316,26 @@ def commit(self, creator_info: MHLCreatorInfo, process_info: MHLProcessInfo): if history.parent_history is not None: referenced_hash_lists[history.parent_history].append(new_hash_list) - chain_xml_parser.write_chain(history.chain, new_hash_list) + if writeChain: + # regular history .... + chain_xml_parser.write_chain(history.chain, new_hash_list) + else: + # ... or flattened history manifest + root_path = os.path.dirname(new_hash_list.file_path) + if not os.path.exists(root_path): + print(f"ERR: folder {root_path} with flattened manifest does not exist") + return + + parent_folder = os.path.dirname(root_path) + + for file_name in os.listdir(root_path): + if file_name.endswith(".mhl"): + src_path = os.path.join(root_path, file_name) + dst_path = os.path.join(parent_folder, file_name) + shutil.move(src_path, dst_path) + + # Remove the folder if empty + if not os.listdir(root_path): + os.rmdir(root_path) + else: + print(f"ERR: temp folder not empty, did not remove {root_path}") diff --git a/tests/test_flatten.py b/tests/test_flatten.py index 7783386..47c5ae9 100644 --- a/tests/test_flatten.py +++ b/tests/test_flatten.py @@ -10,6 +10,7 @@ import os from click.testing import CliRunner from freezegun import freeze_time +from .conftest import path_conversion_tests from .conftest import abspath_conversion_tests import ascmhl.commands @@ -51,3 +52,29 @@ def test_simple_two_hashformats(fs, simple_mhl_history): ascmhl.commands.flatten, [abspath_conversion_tests("/root"), abspath_conversion_tests("/out")] ) assert result.exit_code == 0 + + +@freeze_time("2020-01-16 09:15:00") +def test_nested(fs, nested_mhl_histories): + runner = CliRunner() + + result = runner.invoke( + ascmhl.commands.flatten, ["-v", abspath_conversion_tests("/root"), abspath_conversion_tests("/out")] + ) + assert result.exit_code == 0 + + # check for files in root and sub histories + assert ( + result.output == f"Flattening folder at path: {abspath_conversion_tests('/root')} ...\n" + f" created original hash for Stuff.txt xxh64: 94c399c2a9a21f9a\n" + f"\n" + f"Child History at {path_conversion_tests('A/AA')}:\n" + f" created original hash for {path_conversion_tests('A/AA/AA1.txt')} xxh64: ab6bec9ec04704f6\n" + f"\n" + f"Child History at B:\n" + f" created original hash for {path_conversion_tests('B/B1.txt')} xxh64: 51fb8fb099e92821\n" + f"\n" + f"Child History at {path_conversion_tests('B/BB')}:\n" + f" created original hash for {path_conversion_tests('B/BB/BB1.txt')} xxh64: 5c14eac4f4ad7501\n" + f"Created new generation {path_conversion_tests('collection_2020-01-16/packinglist_root_2020-01-16_091500Z.mhl')}\n" + )