From cc36281620d8ab370a3aa931b0e52f379aadaa59 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 16:23:34 -0500 Subject: [PATCH 01/29] Ignore env directory --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 2da080a4..cc0bed69 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ previous_spacehaven_path.txt quicklaunch_*.jar quicklaunch_*/ extra_mods_path.txt +env/ From 6844c6eb0bd4001a15e83ab4a9325233773c56b1 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 16:31:16 -0500 Subject: [PATCH 02/29] Add vscode launch configuration --- .vscode/launch.json | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..c17b327b --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,15 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Python: Startup file", + "type": "python", + "request": "launch", + "program": "${workspaceFolder}/spacehaven-modloader.py", + "console": "integratedTerminal" + } + ] +} \ No newline at end of file From da8f056017d5632d3dfb6a549ab0ee3f1ed557f8 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 16:33:09 -0500 Subject: [PATCH 03/29] Break reused code into function --- loader/assets/merge.py | 119 ++++++++++++++++++++++------------------- 1 file changed, 64 insertions(+), 55 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index db4e9923..622973eb 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -147,61 +147,7 @@ def _mod_path(filename): if not os.path.exists(mod_file): continue - # Do an element-wise merge (replacing conflicts) - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/Randomizer", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/GOAPAction", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/BackPack", idAttribute="mid") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/Element", idAttribute="mid") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/Product", idAttribute="eid") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/DataLogFragment", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/RandomShip", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/IsoFX", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/Item", idAttribute="mid") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/SubCat", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/Monster", idAttribute="cid") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/PersonalitySettings", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/Encounter", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/CostGroup", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/CharacterSet", idAttribute="cid") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/DifficultySettings", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/Room", idAttribute="rid") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/ObjectiveCollection", idAttribute="nid") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/Notes", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/DialogChoice", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/Faction", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/CelestialObject", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/Character", idAttribute="cid") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/Craft", idAttribute="cid") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/Sector", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/DataLog", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/Plan", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/BackStory", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/DefaultStuff", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/TradingValues", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/CharacterTrait", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/Effect", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/CharacterCondition", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/Ship", idAttribute="rid") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/IdleAnim", idAttribute="id") - mergeDefinitions(coreLibrary, modLibrary, file="library/haven", xpath="/data/MainCat", idAttribute="id") - - mergeDefinitions(coreLibrary, modLibrary, file="library/texts", xpath="/t", idAttribute="id") - - # do that before merging animations and textures because references might have to be remapped! - coreLibrary['_all_modded_textures'].update(_detect_textures(coreLibrary, modLibrary, mod)) - - # this way the last mod loaded will overwrite previous textures - #FIXME reimplement this test - # if region_id in all_modded_textures: - # ui.log.log(" ERROR CONFLICT {}...".format(filename)) - # ui.log.log(" ERROR CONFLICT {}...".format(filename)) - # ui.log.log(" ERROR CONFLICT {}...".format(filename)) - # continue - - mergeDefinitions(coreLibrary, modLibrary, file="library/animations", xpath="/AllAnimations/animations", idAttribute="n") - mergeDefinitions(coreLibrary, modLibrary, file="library/textures", xpath="/AllTexturesAndRegions/textures", idAttribute="i") - mergeDefinitions(coreLibrary, modLibrary, file="library/textures", xpath="/AllTexturesAndRegions/regions", idAttribute="n") - + doMerges(coreLibrary, modLibrary, mod) ui.log.updateLaunchState("Updating XML") @@ -259,6 +205,69 @@ def _mod_path(filename): return extra_assets +def doMerges(coreLib, modLib, mod): + def mergeShim(file, xpath, idAttribute): + mergeDefinitions(coreLib, modLib, file, xpath, idAttribute) + + havenIDLookUpTable = { + "/data/Randomizer": "id", + "/data/GOAPAction": "id", + "/data/BackPack": "mid", + "/data/Element": "mid", + "/data/Product": "eid", + "/data/DataLogFragment": "id", + "/data/RandomShip": "id", + "/data/IsoFX": "id", + "/data/Item": "mid", + "/data/SubCat": "id", + "/data/Monster": "cid", + "/data/PersonalitySettings": "id", + "/data/Encounter": "id", + "/data/CostGroup": "id", + "/data/CharacterSet": "cid", + "/data/DifficultySettings": "id", + "/data/Room": "rid", + "/data/ObjectiveCollection": "nid", + "/data/Notes": "id", + "/data/DialogChoice": "id", + "/data/Faction": "id", + "/data/CelestialObject": "id", + "/data/Character": "cid", + "/data/Craft": "cid", + "/data/Sector": "id", + "/data/DataLog": "id", + "/data/Plan": "id", + "/data/BackStory": "id", + "/data/DefaultStuff": "id", + "/data/TradingValues": "id", + "/data/CharacterTrait": "id", + "/data/Effect": "id", + "/data/CharacterCondition": "id", + "/data/Ship": "rid", + "/data/IdleAnim": "id", + "/data/MainCat": "id", + } + + # Do an element-wise merge (replacing conflicts) + for k,v in havenIDLookUpTable.items(): mergeShim("library/haven", k, v) + mergeShim("library/texts", "/t", idAttribute="id") + + # do that before merging animations and textures because references might have to be remapped! + coreLib['_all_modded_textures'].update(_detect_textures(coreLib, modLib, mod)) + + # this way the last mod loaded will overwrite previous textures + #FIXME reimplement this test + # if region_id in all_modded_textures: + # ui.log.log(" ERROR CONFLICT {}...".format(filename)) + # ui.log.log(" ERROR CONFLICT {}...".format(filename)) + # ui.log.log(" ERROR CONFLICT {}...".format(filename)) + # continue + + mergeShim("library/animations", "/AllAnimations/animations", "n") + mergeShim("library/textures", "/AllTexturesAndRegions/textures", "i") + mergeShim("library/textures", "/AllTexturesAndRegions/regions", "n") + + def mergeDefinitions(baseLibrary, modLibrary, file, xpath, idAttribute): if not file in modLibrary: ui.log.log(" {}: Not present".format(file)) From 88881d8641bde4458e0db3525e06452f5f819816 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 16:42:37 -0500 Subject: [PATCH 04/29] Documentation --- loader/assets/merge.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 622973eb..bba72a22 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -205,10 +205,13 @@ def _mod_path(filename): return extra_assets -def doMerges(coreLib, modLib, mod): - def mergeShim(file, xpath, idAttribute): +def doMerges(coreLib, modLib, mod: str): + """Do merge-based modding sequence""" + def mergeShim(file: str, xpath: str, idAttribute: str): + '''Shim to reduce function call complexity''' mergeDefinitions(coreLib, modLib, file, xpath, idAttribute) + # Lookup table for all nodes in library/haven based on element and the expected ID format havenIDLookUpTable = { "/data/Randomizer": "id", "/data/GOAPAction": "id", From ce66c434f4ef9f92cbbfd68a381280b194dd0b32 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 16:43:50 -0500 Subject: [PATCH 05/29] Prevent indetation error down the line --- loader/assets/merge.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index bba72a22..489bb21b 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -260,11 +260,11 @@ def mergeShim(file: str, xpath: str, idAttribute: str): # this way the last mod loaded will overwrite previous textures #FIXME reimplement this test - # if region_id in all_modded_textures: - # ui.log.log(" ERROR CONFLICT {}...".format(filename)) - # ui.log.log(" ERROR CONFLICT {}...".format(filename)) - # ui.log.log(" ERROR CONFLICT {}...".format(filename)) - # continue + #if region_id in all_modded_textures: + # ui.log.log(" ERROR CONFLICT {}...".format(filename)) + # ui.log.log(" ERROR CONFLICT {}...".format(filename)) + # ui.log.log(" ERROR CONFLICT {}...".format(filename)) + # continue mergeShim("library/animations", "/AllAnimations/animations", "n") mergeShim("library/textures", "/AllTexturesAndRegions/textures", "i") From b3254188fa63a3bf7d56bc5185071ba9eccf3719 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 16:48:11 -0500 Subject: [PATCH 06/29] Sort LUT to make it easier on coders --- loader/assets/merge.py | 54 +++++++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 489bb21b..e99a59c1 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -213,42 +213,42 @@ def mergeShim(file: str, xpath: str, idAttribute: str): # Lookup table for all nodes in library/haven based on element and the expected ID format havenIDLookUpTable = { - "/data/Randomizer": "id", - "/data/GOAPAction": "id", "/data/BackPack": "mid", - "/data/Element": "mid", - "/data/Product": "eid", - "/data/DataLogFragment": "id", - "/data/RandomShip": "id", - "/data/IsoFX": "id", - "/data/Item": "mid", - "/data/SubCat": "id", - "/data/Monster": "cid", - "/data/PersonalitySettings": "id", - "/data/Encounter": "id", - "/data/CostGroup": "id", - "/data/CharacterSet": "cid", - "/data/DifficultySettings": "id", - "/data/Room": "rid", - "/data/ObjectiveCollection": "nid", - "/data/Notes": "id", - "/data/DialogChoice": "id", - "/data/Faction": "id", + "/data/BackStory": "id", "/data/CelestialObject": "id", "/data/Character": "cid", + "/data/CharacterCondition": "id", + "/data/CharacterSet": "cid", + "/data/CharacterTrait": "id", + "/data/CostGroup": "id", "/data/Craft": "cid", - "/data/Sector": "id", "/data/DataLog": "id", - "/data/Plan": "id", - "/data/BackStory": "id", + "/data/DataLogFragment": "id", "/data/DefaultStuff": "id", - "/data/TradingValues": "id", - "/data/CharacterTrait": "id", + "/data/DialogChoice": "id", + "/data/DifficultySettings": "id", "/data/Effect": "id", - "/data/CharacterCondition": "id", - "/data/Ship": "rid", + "/data/Element": "mid", + "/data/Encounter": "id", + "/data/Faction": "id", + "/data/GOAPAction": "id", "/data/IdleAnim": "id", + "/data/IsoFX": "id", + "/data/Item": "mid", "/data/MainCat": "id", + "/data/Monster": "cid", + "/data/Notes": "id", + "/data/ObjectiveCollection": "nid", + "/data/PersonalitySettings": "id", + "/data/Plan": "id", + "/data/Product": "eid", + "/data/RandomShip": "id", + "/data/Randomizer": "id", + "/data/Room": "rid", + "/data/Sector": "id", + "/data/Ship": "rid", + "/data/SubCat": "id", + "/data/TradingValues": "id", } # Do an element-wise merge (replacing conflicts) From 752700f389cc8bf377e85d4ea2937ebe1a357b35 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 16:50:53 -0500 Subject: [PATCH 07/29] Remove blank whitespace --- loader/assets/merge.py | 76 +++++++++++++++++++++--------------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index e99a59c1..4a6a6711 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -13,23 +13,23 @@ def _detect_textures(coreLibrary, modLibrary, mod): textures_path = os.path.join(mod, 'textures') if not os.path.isdir(textures_path): return {} - + mapping_n_region = {} modded_textures = {} seen_textures = set() - + def _add_texture(region_id): filename = region_id + '.png' if filename in seen_textures: return - + path = os.path.join(textures_path, filename) if not os.path.isfile(path): ui.log.log(" ERROR MISSING {}...".format(filename)) ui.log.log(" ERROR MISSING {}...".format(filename)) ui.log.log(" ERROR MISSING {}...".format(filename)) return - + ui.log.log(" Found {}...".format(filename)) if int(region_id) > coreLibrary['_last_core_region_id']: # adding a new texture, this gets tricky as they have to have consecutive numbers. @@ -38,14 +38,14 @@ def _add_texture(region_id): coreLibrary['_next_region_id'] += 1 else: core_region_id = region_id - + seen_textures.add(filename) modded_textures[core_region_id] = { 'mapped_from_id' : region_id, 'filename' : filename, 'path' : path, } - + for filename in os.listdir(textures_path): # also scan the directory for overwriting existing core textures if not filename.endswith('.png'): @@ -56,26 +56,26 @@ def _add_texture(region_id): # wrong format continue _add_texture(filename.split('.')[0]) - + if 'library/textures' not in modLibrary: # no textures.xml file, we're done return modded_textures - + #FIXME verify that there's only one file textures_mod = modLibrary['library/textures'][0] - + for texture_pack in textures_mod.xpath("//t[@i]"): cim_id = texture_pack.get('i') coreLibrary['_custom_textures_cim'][cim_id] = texture_pack.attrib - + for region in textures_mod.xpath("//re[@n]"): region_id = region.get('n') _add_texture(region_id) - + if not mapping_n_region: # no custom mod textures, no need to remap ids return modded_textures - + for animation_chunk in modLibrary['library/animations']: for asset in animation_chunk.xpath("//assetPos[@a]"): mod_local_id = asset.get('a') @@ -84,7 +84,7 @@ def _add_texture(region_id): new_id = mapping_n_region[mod_local_id] ui.log.log(" Mapping animation 'assetPos' {} to {}...".format(mod_local_id, new_id)) asset.set('a', new_id) - + for asset in textures_mod.xpath("//re[@n]"): mod_local_id = asset.get('n') if mod_local_id not in mapping_n_region: @@ -92,7 +92,7 @@ def _add_texture(region_id): new_id = mapping_n_region[mod_local_id] ui.log.log(" Mapping texture 're' {} to {}...".format(mod_local_id, new_id)) asset.set('n', new_id) - + return modded_textures @@ -101,33 +101,33 @@ def mods(corePath, modPaths): coreLibrary = {} def _core_path(filename): return os.path.join(corePath, filename.replace('/', os.sep)) - + for filename in PATCHABLE_XML_FILES: with open(_core_path(filename), 'rb') as f: coreLibrary[filename] = lxml.etree.parse(f, parser=lxml.etree.XMLParser(recover=True)) - + # find the last region in the texture file and remember its index # we will need this to add mod textures with consecutive indexes... coreLibrary['_last_core_region_id'] = int(coreLibrary['library/textures'].find("//re[@n][last()]").get('n')) coreLibrary['_next_region_id'] = coreLibrary['_last_core_region_id'] + 1 coreLibrary['_all_modded_textures'] = {} coreLibrary['_custom_textures_cim'] = {} - + # Merge in modded files for mod in modPaths: ui.log.updateLaunchState("Installing {}".format(os.path.basename(mod))) - + ui.log.log(" Loading mod {}...".format(mod)) - + # Load the mod's library modLibrary = {} def _mod_path(filename): return os.path.join(mod, filename.replace('/', os.sep)) - + mod_files = [] for mod_file in os.listdir(_mod_path('library')): mod_files.append('library/' + mod_file) - + # we allow breaking down mod xml files into smaller pieces for readability for target in PATCHABLE_XML_FILES: for mod_file in mod_files: @@ -138,8 +138,8 @@ def _mod_path(filename): ui.log.log("{} => {}".format(mod_file, target)) with open(_mod_path(mod_file)) as f: modLibrary[target].append(lxml.etree.parse(f, parser=lxml.etree.XMLParser(remove_comments=True))) - - + + mod_file = _mod_path(target) if not os.path.exists(mod_file): # try again with the extension ? @@ -150,26 +150,26 @@ def _mod_path(filename): doMerges(coreLibrary, modLibrary, mod) ui.log.updateLaunchState("Updating XML") - + # Write out the new base library for filename in PATCHABLE_XML_FILES: with open(_core_path(filename), "wb") as f: f.write(lxml.etree.tostring(coreLibrary[filename], pretty_print=True, encoding="UTF-8")) - + ui.log.updateLaunchState("Packing textures") # add or overwrite textures from mods. This is done after all the XML has been merged into the core "textures" file cims = {} reexport_cims = {} extra_assets = [] - + for region in coreLibrary['library/textures'].xpath("//re[@n]"): name = region.get("n") - + if name not in coreLibrary['_all_modded_textures']: continue - + png_file = coreLibrary['_all_modded_textures'][name]['path'] - + page = region.get("t") if not page in cims: cim_name = '{}.cim'.format(page) @@ -181,27 +181,27 @@ def _mod_path(filename): kwargs['height'] = coreLibrary['_custom_textures_cim'][page]['h'] extra_assets.append('library/' + cim_name) cims[page] = Texture(os.path.join(corePath, 'library', cim_name), **kwargs) - + reexport_cims[page] = set() - + # write back the cim file as png for debugging reexport_cims[page].add(os.path.dirname(png_file)) - + x = int(region.get("x")) y = int(region.get("y")) w = int(region.get("w")) h = int(region.get("h")) - + ui.log.log(" Patching {}.cim...".format(page)) cims[page].pack_png(png_file, x, y, w, h) - + # cims contains only the textures files that have actually been modified for page in cims: ui.log.log(" Writing {}.cim...".format(page)) cims[page].export_cim(os.path.join(corePath, 'library', '{}.cim'.format(page))) for path in reexport_cims[page]: cims[page].export_png(os.path.join(path, 'modded_cim_{}.png'.format(page))) - + return extra_assets @@ -282,13 +282,13 @@ def mergeDefinitions(baseLibrary, modLibrary, file, xpath, idAttribute): #that's a big error if we can't find it in the core! ui.log.log(" {}: ERROR CORE NOTHING AT {}".format(file, xpath)) return - + for mod_xml in modLibrary[file]: try: modRoot = mod_xml.xpath(xpath)[0] except: continue - + merged = 0 for element in list(modRoot): conflicts = baseRoot.xpath("*[@{}='{}']".format(idAttribute, element.get(idAttribute))) @@ -298,7 +298,7 @@ def mergeDefinitions(baseLibrary, modLibrary, file, xpath, idAttribute): baseRoot.append(copy.deepcopy(element)) merged += 1 - + if merged: # TODO add source filename ui.log.log(" {}: Merged {} elements into {}".format(file, merged, xpath)) From 8a187103b30d68f212f6ad903c876b56dc4910cb Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 17:32:10 -0500 Subject: [PATCH 08/29] Ignore rope metafiles --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index cc0bed69..bdf9139e 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ quicklaunch_*.jar quicklaunch_*/ extra_mods_path.txt env/ +.vscode/.ropeproject/ From 73c24328d2ac71f53d9b0d7d74caf2459e049e32 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 17:33:49 -0500 Subject: [PATCH 09/29] Put modLibrary code into a function --- loader/assets/merge.py | 56 ++++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 4a6a6711..e3fdfbb4 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -102,6 +102,35 @@ def mods(corePath, modPaths): def _core_path(filename): return os.path.join(corePath, filename.replace('/', os.sep)) + def buildLibrary(location: str): + """Build up a library dict of files in `location`""" + def _mod_path(filename): + return os.path.join(mod, filename.replace('/', os.sep)) + location_library = {} + try: + location_files = [location + '/' + mod_file for mod_file in os.listdir(_mod_path(location))] + except FileNotFoundError: + location_files = [] + + # we allow breaking down mod xml files into smaller pieces for readability + for target in PATCHABLE_XML_FILES: + targetInLocation = target.replace('library', location) + for mod_file in location_files: + if not mod_file.startswith(targetInLocation): continue + if target not in location_library: location_library[target] = [] + + ui.log.log(" {} <= {}".format(target, mod_file)) + with open(_mod_path(mod_file)) as f: + location_library[target].append(lxml.etree.parse(f, parser=lxml.etree.XMLParser(remove_comments=True))) + + mod_file = _mod_path(target) + # try again with the extension ? + if not os.path.exists(mod_file): + mod_file += '.xml' + if not os.path.exists(mod_file): + continue + return location_library + for filename in PATCHABLE_XML_FILES: with open(_core_path(filename), 'rb') as f: coreLibrary[filename] = lxml.etree.parse(f, parser=lxml.etree.XMLParser(recover=True)) @@ -120,32 +149,7 @@ def _core_path(filename): ui.log.log(" Loading mod {}...".format(mod)) # Load the mod's library - modLibrary = {} - def _mod_path(filename): - return os.path.join(mod, filename.replace('/', os.sep)) - - mod_files = [] - for mod_file in os.listdir(_mod_path('library')): - mod_files.append('library/' + mod_file) - - # we allow breaking down mod xml files into smaller pieces for readability - for target in PATCHABLE_XML_FILES: - for mod_file in mod_files: - if not mod_file.startswith(target): - continue - if target not in modLibrary: - modLibrary[target] = [] - ui.log.log("{} => {}".format(mod_file, target)) - with open(_mod_path(mod_file)) as f: - modLibrary[target].append(lxml.etree.parse(f, parser=lxml.etree.XMLParser(remove_comments=True))) - - - mod_file = _mod_path(target) - if not os.path.exists(mod_file): - # try again with the extension ? - mod_file += '.xml' - if not os.path.exists(mod_file): - continue + modLibrary = buildLibrary('library') doMerges(coreLibrary, modLibrary, mod) From 7a6796a39dc81fb894b81c149d5e511ead1779a5 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 17:53:32 -0500 Subject: [PATCH 10/29] Reduce log spam --- loader/assets/merge.py | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index e3fdfbb4..88a2c0f6 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -215,6 +215,10 @@ def mergeShim(file: str, xpath: str, idAttribute: str): '''Shim to reduce function call complexity''' mergeDefinitions(coreLib, modLib, file, xpath, idAttribute) + def mergeAbortMessage(filename: str): + """Shim to standardize error message""" + ui.log.log(f" No merges needed: {filename}") + # Lookup table for all nodes in library/haven based on element and the expected ID format havenIDLookUpTable = { "/data/BackPack": "mid", @@ -256,8 +260,15 @@ def mergeShim(file: str, xpath: str, idAttribute: str): } # Do an element-wise merge (replacing conflicts) - for k,v in havenIDLookUpTable.items(): mergeShim("library/haven", k, v) - mergeShim("library/texts", "/t", idAttribute="id") + currentFile = "library/haven" + if currentFile in modLib: + for path, idText in havenIDLookUpTable.items(): mergeShim(currentFile, path, idText) + else: mergeAbortMessage(currentFile) + + currentFile = "library/texts" + if currentFile in modLib: + mergeShim(currentFile, "/t", idAttribute="id") + else: mergeAbortMessage(currentFile) # do that before merging animations and textures because references might have to be remapped! coreLib['_all_modded_textures'].update(_detect_textures(coreLib, modLib, mod)) @@ -270,9 +281,16 @@ def mergeShim(file: str, xpath: str, idAttribute: str): # ui.log.log(" ERROR CONFLICT {}...".format(filename)) # continue - mergeShim("library/animations", "/AllAnimations/animations", "n") - mergeShim("library/textures", "/AllTexturesAndRegions/textures", "i") - mergeShim("library/textures", "/AllTexturesAndRegions/regions", "n") + + currentFile = "library/animations" + if currentFile in modLib: + mergeShim(currentFile, "/AllAnimations/animations", "n") + else: mergeAbortMessage(currentFile) + + currentFile = "library/textures" + if currentFile in modLib: + mergeShim(currentFile, "/AllTexturesAndRegions/regions", "n") + else: mergeAbortMessage(currentFile) def mergeDefinitions(baseLibrary, modLibrary, file, xpath, idAttribute): From 76e679335ac8ad1bc70e25f172d63b3ed8a3e832 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 17:59:09 -0500 Subject: [PATCH 11/29] Create patch loop --- loader/assets/merge.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 88a2c0f6..5bbdec1e 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -153,6 +153,12 @@ def _mod_path(filename): doMerges(coreLibrary, modLibrary, mod) + # Do patches after merges to avoid clobbers + for mod in modPaths: + ui.log.updateLaunchState(f"Patching {os.path.basename(mod)}") + ui.log.log(f" Loading patches {mod}...") + modPatchesLibrary = buildLibrary('patches') + ui.log.updateLaunchState("Updating XML") # Write out the new base library From 58220f441ddacc69253c9ee78b84a58d2b18642c Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 19:43:29 -0500 Subject: [PATCH 12/29] First framework for XPath-patching --- loader/assets/merge.py | 53 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 5bbdec1e..bf3bc94b 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -158,6 +158,7 @@ def _mod_path(filename): ui.log.updateLaunchState(f"Patching {os.path.basename(mod)}") ui.log.log(f" Loading patches {mod}...") modPatchesLibrary = buildLibrary('patches') + doPatches(coreLibrary, modPatchesLibrary, mod) ui.log.updateLaunchState("Updating XML") @@ -214,6 +215,58 @@ def _mod_path(filename): return extra_assets +def doPatches(coreLib, modLib: dict, mod: str): + # Pretyping + patchList : lxml.etree._ElementTree + patchOperation : lxml.etree._Element + + def doPatchType(patch: lxml.etree._Element, location: str): + """Execute a single patch. Provided to reduce indentation level""" + # Pretyping + currentCoreLib : lxml.etree._ElementTree + + pType = patch.attrib["Class"] + xpath = patch.find('xpath').text + value = patch.find('value') + if "Attribute" in pType: + attribute = patch.find("attribute").text + + currentCoreLib = coreLib[location] + currentCoreLibElems = currentCoreLib.xpath(xpath) + + def AttributeSet(): + ui.log.log(f" Set attr {attribute} on node {xpath}") + for elem in currentCoreLibElems: + elem.set(attribute, value.text) + def AttributeAdd(): pass + def AttributeRemove(): pass + + def NodeSet(): pass + def NodeAdd(): pass + def NodeRemove(): pass + def NodeReplace(): + ui.log.log(f" Replacing node {xpath}") + def BadOp(): pass + + patchDispatcher = { + "AttributeSet" : AttributeSet, + "AttributeAdd" : AttributeAdd, + "AttributeRemove" : AttributeRemove, + "Set": NodeSet, + "Add": NodeAdd, + "Remove": NodeRemove, + "Replace": NodeReplace, + } + + patchDispatcher.get(pType,BadOp)() + + + # Execution + for location in modLib: + for patchList in modLib[location]: + for patchOperation in patchList.getroot(): + doPatchType(patchOperation, location) + def doMerges(coreLib, modLib, mod: str): """Do merge-based modding sequence""" From 5ec1b07c2021db97ddfd23ec07aa76e6cc4d1edf Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 20:08:59 -0500 Subject: [PATCH 13/29] Add the remaining Attribute path operations --- loader/assets/merge.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index bf3bc94b..56d94c30 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -225,6 +225,7 @@ def doPatchType(patch: lxml.etree._Element, location: str): # Pretyping currentCoreLib : lxml.etree._ElementTree + logIndent = " " * 4 pType = patch.attrib["Class"] xpath = patch.find('xpath').text value = patch.find('value') @@ -233,13 +234,23 @@ def doPatchType(patch: lxml.etree._Element, location: str): currentCoreLib = coreLib[location] currentCoreLibElems = currentCoreLib.xpath(xpath) + if len(currentCoreLibElems) == 0: + ui.log.log(f"{logIndent}Unable to perform patch. XPath found no results {xpath}") + return + # AttributeAdd and AttributeSet are able to use the same code due to + # lxml backing library implementation def AttributeSet(): - ui.log.log(f" Set attr {attribute} on node {xpath}") - for elem in currentCoreLibElems: - elem.set(attribute, value.text) - def AttributeAdd(): pass - def AttributeRemove(): pass + elem : lxml.etree._Element + for elem in currentCoreLibElems: elem.set(attribute, value.text) + def AttributeAdd(): + elem : lxml.etree._Element + for elem in currentCoreLibElems: elem.set(attribute, value.text) + def AttributeRemove(): + ui.log.log(f"{logIndent}WARNING: REMOVING ATTRIBUTES MAY BREAK THE GAME") + ui.log.log(f"{logIndent}Remove '{attribute}' from nodes: {xpath}") + elem : lxml.etree._Element + for elem in currentCoreLibElems: elem.attrib.pop(attribute) def NodeSet(): pass def NodeAdd(): pass From da51aa683495f5c5457d482dc547a1c4d24b7297 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 20:26:34 -0500 Subject: [PATCH 14/29] Cleanly handle failing patch --- loader/assets/merge.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 56d94c30..2bceba92 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -276,7 +276,14 @@ def BadOp(): pass for location in modLib: for patchList in modLib[location]: for patchOperation in patchList.getroot(): - doPatchType(patchOperation, location) + try: + doPatchType(patchOperation, location) + except Exception as e: + uri = patchOperation.base + line = patchOperation.sourceline + ui.log.log(f" Failed to apply patch operation {uri}:{line}") + ui.log.log(f" Reason: {repr(e)}") + raise SyntaxError("Issue in patch operation. Check logs for info.") from None def doMerges(coreLib, modLib, mod: str): From e0b9b26e5666d291b5501ac1927e7dba4794c625 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 20:28:29 -0500 Subject: [PATCH 15/29] Add editor typing --- loader/assets/merge.py | 1 + 1 file changed, 1 insertion(+) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 2bceba92..ea79bb11 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -224,6 +224,7 @@ def doPatchType(patch: lxml.etree._Element, location: str): """Execute a single patch. Provided to reduce indentation level""" # Pretyping currentCoreLib : lxml.etree._ElementTree + patchOperation : lxml.etree._Element logIndent = " " * 4 pType = patch.attrib["Class"] From 2f28a1549b05e807f8012f0f1e5337ed9bc3d64f Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 20:47:09 -0500 Subject: [PATCH 16/29] Add per-patch logging --- loader/assets/merge.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index ea79bb11..87433ba7 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -233,6 +233,8 @@ def doPatchType(patch: lxml.etree._Element, location: str): if "Attribute" in pType: attribute = patch.find("attribute").text + ui.log.log(f"{logIndent}XPATH => {location:>15}: {pType:18}{xpath}") + currentCoreLib = coreLib[location] currentCoreLibElems = currentCoreLib.xpath(xpath) if len(currentCoreLibElems) == 0: @@ -249,7 +251,6 @@ def AttributeAdd(): for elem in currentCoreLibElems: elem.set(attribute, value.text) def AttributeRemove(): ui.log.log(f"{logIndent}WARNING: REMOVING ATTRIBUTES MAY BREAK THE GAME") - ui.log.log(f"{logIndent}Remove '{attribute}' from nodes: {xpath}") elem : lxml.etree._Element for elem in currentCoreLibElems: elem.attrib.pop(attribute) From dd85011c41328230b3dc02358439edc177b702a8 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 21:13:48 -0500 Subject: [PATCH 17/29] Implement XPath node replace --- loader/assets/merge.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 87433ba7..47d8c306 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -258,7 +258,11 @@ def NodeSet(): pass def NodeAdd(): pass def NodeRemove(): pass def NodeReplace(): - ui.log.log(f" Replacing node {xpath}") + elem : lxml.etree._Element + parent: lxml.etree._Element + for elem in currentCoreLibElems: + parent = elem.find('./..') + parent.replace(elem, copy.deepcopy(value[0])) def BadOp(): pass patchDispatcher = { From 1aad8d5b3a9ef18b7cc964ea9feab75a1bde2117 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 21:33:26 -0500 Subject: [PATCH 18/29] Change Set operation to Insert, Change pass to NotImplemented --- loader/assets/merge.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 47d8c306..4f4188e3 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -254,30 +254,30 @@ def AttributeRemove(): elem : lxml.etree._Element for elem in currentCoreLibElems: elem.attrib.pop(attribute) - def NodeSet(): pass - def NodeAdd(): pass - def NodeRemove(): pass + def NodeAdd(): raise NotImplementedError() + def NodeInsert(): raise NotImplementedError() + def NodeRemove(): raise NotImplementedError() def NodeReplace(): elem : lxml.etree._Element parent: lxml.etree._Element for elem in currentCoreLibElems: parent = elem.find('./..') parent.replace(elem, copy.deepcopy(value[0])) - def BadOp(): pass + def BadOp(): + raise SyntaxError(f"BAD PATCH OPERATION") patchDispatcher = { "AttributeSet" : AttributeSet, "AttributeAdd" : AttributeAdd, "AttributeRemove" : AttributeRemove, - "Set": NodeSet, "Add": NodeAdd, + "Insert": NodeInsert, "Remove": NodeRemove, "Replace": NodeReplace, } patchDispatcher.get(pType,BadOp)() - # Execution for location in modLib: for patchList in modLib[location]: From 61ad411576340f375b82e1849ea00cab91fb2cb0 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 21:34:14 -0500 Subject: [PATCH 19/29] Add documentation --- loader/assets/merge.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 4f4188e3..0b83e0fa 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -220,11 +220,14 @@ def doPatches(coreLib, modLib: dict, mod: str): patchList : lxml.etree._ElementTree patchOperation : lxml.etree._Element + # Helper functions def doPatchType(patch: lxml.etree._Element, location: str): """Execute a single patch. Provided to reduce indentation level""" # Pretyping currentCoreLib : lxml.etree._ElementTree patchOperation : lxml.etree._Element + xpath: str + value: lxml.etree._Element logIndent = " " * 4 pType = patch.attrib["Class"] From 16dac596f634d810b8912055dced74a3791d9cc6 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 21:40:37 -0500 Subject: [PATCH 20/29] Update documentation --- loader/assets/merge.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 0b83e0fa..2e75ca75 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -244,23 +244,31 @@ def doPatchType(patch: lxml.etree._Element, location: str): ui.log.log(f"{logIndent}Unable to perform patch. XPath found no results {xpath}") return - # AttributeAdd and AttributeSet are able to use the same code due to - # lxml backing library implementation def AttributeSet(): + """Set the attribute on the node, adding if not present""" elem : lxml.etree._Element for elem in currentCoreLibElems: elem.set(attribute, value.text) def AttributeAdd(): + """Adds the attribute to the node IFF the attribute name is not already present""" elem : lxml.etree._Element for elem in currentCoreLibElems: elem.set(attribute, value.text) def AttributeRemove(): + """Remove the attribute from the node""" ui.log.log(f"{logIndent}WARNING: REMOVING ATTRIBUTES MAY BREAK THE GAME") elem : lxml.etree._Element for elem in currentCoreLibElems: elem.attrib.pop(attribute) - def NodeAdd(): raise NotImplementedError() - def NodeInsert(): raise NotImplementedError() - def NodeRemove(): raise NotImplementedError() + def NodeAdd(): + """Adds a provided child node to the selected node""" + raise NotImplementedError() + def NodeInsert(): + """Adds a provided sibling node to the selected node""" + raise NotImplementedError() + def NodeRemove(): + """Deletes the selected node""" + raise NotImplementedError() def NodeReplace(): + """Replaces the selected node with the provided node""" elem : lxml.etree._Element parent: lxml.etree._Element for elem in currentCoreLibElems: From 27459d07205b2292c69cb10d1c9432053b0daa9e Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 21:54:30 -0500 Subject: [PATCH 21/29] Update AttributeAdd to match PatchOperation spec - Spec requires PatchOperationAttributeAdd to fail if attribute alread exists --- loader/assets/merge.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 2e75ca75..01689838 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -251,7 +251,10 @@ def AttributeSet(): def AttributeAdd(): """Adds the attribute to the node IFF the attribute name is not already present""" elem : lxml.etree._Element - for elem in currentCoreLibElems: elem.set(attribute, value.text) + for elem in currentCoreLibElems: + if elem.get(attribute, None) is not None: + raise KeyError(f"Attribute '{attribute}' already exists") + elem.set(attribute, value.text) def AttributeRemove(): """Remove the attribute from the node""" ui.log.log(f"{logIndent}WARNING: REMOVING ATTRIBUTES MAY BREAK THE GAME") From 0bcb097b1c7b0bd8df11c9abcfcf99adddc49fb2 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 21:55:11 -0500 Subject: [PATCH 22/29] Increase indent level --- loader/assets/merge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 01689838..05f29495 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -302,7 +302,7 @@ def BadOp(): uri = patchOperation.base line = patchOperation.sourceline ui.log.log(f" Failed to apply patch operation {uri}:{line}") - ui.log.log(f" Reason: {repr(e)}") + ui.log.log(f" Reason: {repr(e)}") raise SyntaxError("Issue in patch operation. Check logs for info.") from None From 0fe9dac8c9f4121e7b978bfa4593e36fa16ca6fd Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 22:01:26 -0500 Subject: [PATCH 23/29] Implement PatchOperationRemove --- loader/assets/merge.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 05f29495..3d86dc80 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -269,7 +269,11 @@ def NodeInsert(): raise NotImplementedError() def NodeRemove(): """Deletes the selected node""" - raise NotImplementedError() + elem : lxml.etree._Element + parent: lxml.etree._Element + for elem in currentCoreLibElems: + parent = elem.find('./..') + parent.remove(elem) def NodeReplace(): """Replaces the selected node with the provided node""" elem : lxml.etree._Element From 177d4c1b55e55a42f1f9dbf32a6ae03e686268a2 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 22:08:24 -0500 Subject: [PATCH 24/29] Implement PatchOperationInsert --- loader/assets/merge.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 3d86dc80..c70c9db6 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -266,7 +266,12 @@ def NodeAdd(): raise NotImplementedError() def NodeInsert(): """Adds a provided sibling node to the selected node""" - raise NotImplementedError() + elem : lxml.etree._Element + parent: lxml.etree._Element + for elem in currentCoreLibElems: + parent = elem.find('./..') + elemIDX = parent.index(elem) + parent.insert(elemIDX + 1, copy.deepcopy(value[0])) def NodeRemove(): """Deletes the selected node""" elem : lxml.etree._Element From f8cb321b2373264a8c08c72c5e951de660c4060d Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 22:18:14 -0500 Subject: [PATCH 25/29] Implement PatchOperationAdd Insertion action isn't specified in the PatchOperation spec This implementation inserts it at the end of the list. --- loader/assets/merge.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index c70c9db6..32ad4c3c 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -263,7 +263,11 @@ def AttributeRemove(): def NodeAdd(): """Adds a provided child node to the selected node""" - raise NotImplementedError() + elem : lxml.etree._Element + parent: lxml.etree._Element + for elem in currentCoreLibElems: + lastelemIDX = len(elem.getchildren()) + elem.insert(lastelemIDX + 1, copy.deepcopy(value[0])) def NodeInsert(): """Adds a provided sibling node to the selected node""" elem : lxml.etree._Element From 50c82eeede51c00827e29f66db4f114d9f0c951e Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Sun, 29 Nov 2020 22:18:35 -0500 Subject: [PATCH 26/29] Bump version number --- version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.py b/version.py index 9d2c0024..613ac843 100644 --- a/version.py +++ b/version.py @@ -1,2 +1,2 @@ -version = "0.0.7" +version = "0.0.8" From d670ea187d509293168db5e5a5b60de5467e8188 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Mon, 30 Nov 2020 03:59:29 -0500 Subject: [PATCH 27/29] Move PatchOperation functions and switchDict into main scope This should improve performance, though it does mean arguments have to be explicitly passed into each PatchOperation now. Oh well. --- loader/assets/merge.py | 128 ++++++++++++++++++++++------------------- 1 file changed, 69 insertions(+), 59 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 32ad4c3c..7b08e07f 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -215,6 +215,71 @@ def _mod_path(filename): return extra_assets + + +def AttributeSet(currentCoreLibElems, value, attribute): + """Set the attribute on the node, adding if not present""" + elem : lxml.etree._Element + for elem in currentCoreLibElems: elem.set(attribute, value.text) + +def AttributeAdd(currentCoreLibElems, value, attribute): + """Adds the attribute to the node IFF the attribute name is not already present""" + elem : lxml.etree._Element + for elem in currentCoreLibElems: + if elem.get(attribute, None) is not None: + raise KeyError(f"Attribute '{attribute}' already exists") + elem.set(attribute, value.text) + +def AttributeRemove(currentCoreLibElems, value, attribute): + """Remove the attribute from the node""" + ui.log.log(f"{logIndent}WARNING: REMOVING ATTRIBUTES MAY BREAK THE GAME") + elem : lxml.etree._Element + for elem in currentCoreLibElems: elem.attrib.pop(attribute) + +def NodeAdd(currentCoreLibElems, value, attribute): + """Adds a provided child node to the selected node""" + elem : lxml.etree._Element + parent: lxml.etree._Element + for elem in currentCoreLibElems: + lastelemIDX = len(elem.getchildren()) + elem.insert(lastelemIDX + 1, copy.deepcopy(value[0])) + +def NodeInsert(currentCoreLibElems, value, attribute): + """Adds a provided sibling node to the selected node""" + elem : lxml.etree._Element + parent: lxml.etree._Element + for elem in currentCoreLibElems: + parent = elem.find('./..') + elemIDX = parent.index(elem) + parent.insert(elemIDX + 1, copy.deepcopy(value[0])) + +def NodeRemove(currentCoreLibElems, value, attribute): + """Deletes the selected node""" + elem : lxml.etree._Element + parent: lxml.etree._Element + for elem in currentCoreLibElems: + parent = elem.find('./..') + parent.remove(elem) + +def NodeReplace(currentCoreLibElems, value, attribute): + """Replaces the selected node with the provided node""" + elem : lxml.etree._Element + parent: lxml.etree._Element + for elem in currentCoreLibElems: + parent = elem.find('./..') + parent.replace(elem, copy.deepcopy(value[0])) + +patchDispatcher = { + "AttributeSet" : AttributeSet, + "AttributeAdd" : AttributeAdd, + "AttributeRemove" : AttributeRemove, + "Add": NodeAdd, + "Insert": NodeInsert, + "Remove": NodeRemove, + "Replace": NodeReplace, +} + + def doPatches(coreLib, modLib: dict, mod: str): # Pretyping patchList : lxml.etree._ElementTree @@ -243,67 +308,12 @@ def doPatchType(patch: lxml.etree._Element, location: str): if len(currentCoreLibElems) == 0: ui.log.log(f"{logIndent}Unable to perform patch. XPath found no results {xpath}") return - - def AttributeSet(): - """Set the attribute on the node, adding if not present""" - elem : lxml.etree._Element - for elem in currentCoreLibElems: elem.set(attribute, value.text) - def AttributeAdd(): - """Adds the attribute to the node IFF the attribute name is not already present""" - elem : lxml.etree._Element - for elem in currentCoreLibElems: - if elem.get(attribute, None) is not None: - raise KeyError(f"Attribute '{attribute}' already exists") - elem.set(attribute, value.text) - def AttributeRemove(): - """Remove the attribute from the node""" - ui.log.log(f"{logIndent}WARNING: REMOVING ATTRIBUTES MAY BREAK THE GAME") - elem : lxml.etree._Element - for elem in currentCoreLibElems: elem.attrib.pop(attribute) - - def NodeAdd(): - """Adds a provided child node to the selected node""" - elem : lxml.etree._Element - parent: lxml.etree._Element - for elem in currentCoreLibElems: - lastelemIDX = len(elem.getchildren()) - elem.insert(lastelemIDX + 1, copy.deepcopy(value[0])) - def NodeInsert(): - """Adds a provided sibling node to the selected node""" - elem : lxml.etree._Element - parent: lxml.etree._Element - for elem in currentCoreLibElems: - parent = elem.find('./..') - elemIDX = parent.index(elem) - parent.insert(elemIDX + 1, copy.deepcopy(value[0])) - def NodeRemove(): - """Deletes the selected node""" - elem : lxml.etree._Element - parent: lxml.etree._Element - for elem in currentCoreLibElems: - parent = elem.find('./..') - parent.remove(elem) - def NodeReplace(): - """Replaces the selected node with the provided node""" - elem : lxml.etree._Element - parent: lxml.etree._Element - for elem in currentCoreLibElems: - parent = elem.find('./..') - parent.replace(elem, copy.deepcopy(value[0])) - def BadOp(): + def BadOp(currentCoreLibElems, value, attribute): raise SyntaxError(f"BAD PATCH OPERATION") - patchDispatcher = { - "AttributeSet" : AttributeSet, - "AttributeAdd" : AttributeAdd, - "AttributeRemove" : AttributeRemove, - "Add": NodeAdd, - "Insert": NodeInsert, - "Remove": NodeRemove, - "Replace": NodeReplace, - } - - patchDispatcher.get(pType,BadOp)() + + + patchDispatcher.get(pType,BadOp)(currentCoreLibElems, value, attribute) # Execution for location in modLib: From f33a0eace767af131eaf59674866a5a26f3837e7 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Mon, 30 Nov 2020 04:35:05 -0500 Subject: [PATCH 28/29] Simplify PatchOperation function call at cost of extended setup in function --- loader/assets/merge.py | 68 ++++++++++++++++++++++++++++-------------- 1 file changed, 45 insertions(+), 23 deletions(-) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index 7b08e07f..dcc4d1da 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -217,58 +217,85 @@ def _mod_path(filename): -def AttributeSet(currentCoreLibElems, value, attribute): +def AttributeSet(patchArgs): """Set the attribute on the node, adding if not present""" elem : lxml.etree._Element + currentCoreLibElems = patchArgs["coreLibElems"] + attribute = patchArgs["attribute"].text + value = patchArgs["value"] for elem in currentCoreLibElems: elem.set(attribute, value.text) -def AttributeAdd(currentCoreLibElems, value, attribute): + +def AttributeAdd(patchArgs): """Adds the attribute to the node IFF the attribute name is not already present""" elem : lxml.etree._Element + currentCoreLibElems = patchArgs["coreLibElems"] + attribute = patchArgs["attribute"].text + for elem in currentCoreLibElems: if elem.get(attribute, None) is not None: raise KeyError(f"Attribute '{attribute}' already exists") elem.set(attribute, value.text) -def AttributeRemove(currentCoreLibElems, value, attribute): + +def AttributeRemove(patchArgs): """Remove the attribute from the node""" - ui.log.log(f"{logIndent}WARNING: REMOVING ATTRIBUTES MAY BREAK THE GAME") + ui.log.log(f" WARNING: REMOVING ATTRIBUTES MAY BREAK THE GAME") elem : lxml.etree._Element + currentCoreLibElems = patchArgs["coreLibElems"] + attribute = patchArgs["attribute"].text for elem in currentCoreLibElems: elem.attrib.pop(attribute) -def NodeAdd(currentCoreLibElems, value, attribute): + +def NodeAdd(patchArgs): """Adds a provided child node to the selected node""" elem : lxml.etree._Element parent: lxml.etree._Element + currentCoreLibElems = patchArgs["coreLibElems"] + value = patchArgs["value"] for elem in currentCoreLibElems: lastelemIDX = len(elem.getchildren()) elem.insert(lastelemIDX + 1, copy.deepcopy(value[0])) -def NodeInsert(currentCoreLibElems, value, attribute): + +def NodeInsert(patchArgs): """Adds a provided sibling node to the selected node""" elem : lxml.etree._Element parent: lxml.etree._Element + currentCoreLibElems = patchArgs["coreLibElems"] + value = patchArgs["value"] for elem in currentCoreLibElems: parent = elem.find('./..') elemIDX = parent.index(elem) parent.insert(elemIDX + 1, copy.deepcopy(value[0])) -def NodeRemove(currentCoreLibElems, value, attribute): + +def NodeRemove(patchArgs): """Deletes the selected node""" elem : lxml.etree._Element parent: lxml.etree._Element + currentCoreLibElems = patchArgs["coreLibElems"] for elem in currentCoreLibElems: parent = elem.find('./..') parent.remove(elem) -def NodeReplace(currentCoreLibElems, value, attribute): + +def NodeReplace(patchArgs): """Replaces the selected node with the provided node""" elem : lxml.etree._Element parent: lxml.etree._Element + currentCoreLibElems = patchArgs["coreLibElems"] + value = patchArgs["value"] for elem in currentCoreLibElems: parent = elem.find('./..') parent.replace(elem, copy.deepcopy(value[0])) + +# Default case function +def BadOp(patchArgs): + raise SyntaxError(f"BAD PATCH OPERATION") + + patchDispatcher = { "AttributeSet" : AttributeSet, "AttributeAdd" : AttributeAdd, @@ -291,29 +318,24 @@ def doPatchType(patch: lxml.etree._Element, location: str): # Pretyping currentCoreLib : lxml.etree._ElementTree patchOperation : lxml.etree._Element - xpath: str value: lxml.etree._Element - logIndent = " " * 4 - pType = patch.attrib["Class"] + pType = patch.attrib["Class"] xpath = patch.find('xpath').text - value = patch.find('value') - if "Attribute" in pType: - attribute = patch.find("attribute").text - - ui.log.log(f"{logIndent}XPATH => {location:>15}: {pType:18}{xpath}") + currentCoreLibElems = coreLib[location].xpath(xpath) - currentCoreLib = coreLib[location] - currentCoreLibElems = currentCoreLib.xpath(xpath) + ui.log.log(f" XPATH => {location:>15}: {pType:18}{xpath}") if len(currentCoreLibElems) == 0: - ui.log.log(f"{logIndent}Unable to perform patch. XPath found no results {xpath}") + ui.log.log(f" Unable to perform patch. XPath found no results {xpath}") return - def BadOp(currentCoreLibElems, value, attribute): - raise SyntaxError(f"BAD PATCH OPERATION") - + patchArgs = { + "value": patch.find('value'), + "attribute": patch.find("attribute"), # Defer exception throw to later. + "coreLibElems": currentCoreLibElems, + } - patchDispatcher.get(pType,BadOp)(currentCoreLibElems, value, attribute) + patchDispatcher.get(pType,BadOp)(patchArgs) # Execution for location in modLib: From e0da88297af0c282cbbd5e8c62a2baf76ef11841 Mon Sep 17 00:00:00 2001 From: Tahvohck Date: Wed, 9 Dec 2020 04:49:39 -0500 Subject: [PATCH 29/29] Cherry-pick fix from bfb4fce5d1f63b4a82878c1ef704488ecf7aa23d --- loader/assets/merge.py | 1 + 1 file changed, 1 insertion(+) diff --git a/loader/assets/merge.py b/loader/assets/merge.py index dcc4d1da..06d3ac9b 100644 --- a/loader/assets/merge.py +++ b/loader/assets/merge.py @@ -431,6 +431,7 @@ def mergeAbortMessage(filename: str): currentFile = "library/textures" if currentFile in modLib: + mergeShim(currentFile, "/AllTexturesAndRegions/textures", "i") mergeShim(currentFile, "/AllTexturesAndRegions/regions", "n") else: mergeAbortMessage(currentFile)