From 21911117c354bea70601e89c36956fe9520aae49 Mon Sep 17 00:00:00 2001 From: zombachu Date: Wed, 22 Apr 2026 22:14:48 -0700 Subject: [PATCH 1/2] Improve migration command 1. Migrates data in worlds/ and groups/ even if the player file doesn't have a matching player in players/ (possible due to legacy versions mixing and matching UUID and username-based file names) 2. Processes players sequentially to avoid resource exhaustion with thousands of files 3. Adds per-profile and per-player exception handling to avoid errors cancelling the whole migration 4. Continues on serialization errors to avoid invalid items skipping the whole inventory --- .../MigrateInventorySerializationCommand.java | 152 ++++++++++++++---- .../profile/FlatFileProfileDataSource.java | 2 +- .../profile/ProfileFilesLocator.java | 8 +- 3 files changed, 128 insertions(+), 34 deletions(-) diff --git a/src/main/java/org/mvplugins/multiverse/inventories/commands/bulkedit/playerprofile/MigrateInventorySerializationCommand.java b/src/main/java/org/mvplugins/multiverse/inventories/commands/bulkedit/playerprofile/MigrateInventorySerializationCommand.java index 593695ad..e163fbdd 100644 --- a/src/main/java/org/mvplugins/multiverse/inventories/commands/bulkedit/playerprofile/MigrateInventorySerializationCommand.java +++ b/src/main/java/org/mvplugins/multiverse/inventories/commands/bulkedit/playerprofile/MigrateInventorySerializationCommand.java @@ -1,5 +1,6 @@ package org.mvplugins.multiverse.inventories.commands.bulkedit.playerprofile; +import com.google.common.io.Files; import org.jvnet.hk2.annotations.Service; import org.mvplugins.multiverse.core.command.MVCommandIssuer; import org.mvplugins.multiverse.core.command.queue.CommandQueueManager; @@ -12,13 +13,25 @@ import org.mvplugins.multiverse.inventories.commands.InventoriesCommand; import org.mvplugins.multiverse.inventories.config.InventoriesConfig; import org.mvplugins.multiverse.inventories.profile.GlobalProfile; +import org.mvplugins.multiverse.inventories.profile.ProfileCacheManager; import org.mvplugins.multiverse.inventories.profile.ProfileDataSource; import org.mvplugins.multiverse.inventories.profile.key.GlobalProfileKey; import org.mvplugins.multiverse.inventories.profile.key.ProfileKey; +import org.mvplugins.multiverse.inventories.profile.key.ProfileType; import org.mvplugins.multiverse.inventories.profile.key.ProfileTypes; import org.mvplugins.multiverse.inventories.profile.key.ContainerType; +import org.mvplugins.multiverse.inventories.profile.ProfileFilesLocator; +import org.mvplugins.multiverse.inventories.util.ItemStackConverter; -import java.util.Arrays; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicLong; @@ -28,21 +41,32 @@ final class MigrateInventorySerializationCommand extends InventoriesCommand { private final CommandQueueManager commandQueueManager; private final ProfileDataSource profileDataSource; private final InventoriesConfig inventoriesConfig; + private final ProfileCacheManager profileCacheManager; + private final ProfileFilesLocator profileFilesLocator; @Inject MigrateInventorySerializationCommand( @NotNull CommandQueueManager commandQueueManager, @NotNull ProfileDataSource profileDataSource, - @NotNull InventoriesConfig inventoriesConfig + @NotNull InventoriesConfig inventoriesConfig, + @NotNull ProfileCacheManager profileCacheManager, + @NotNull ProfileFilesLocator profileFilesLocator ) { this.commandQueueManager = commandQueueManager; this.profileDataSource = profileDataSource; this.inventoriesConfig = inventoriesConfig; + this.profileCacheManager = profileCacheManager; + this.profileFilesLocator = profileFilesLocator; } @Subcommand("bulkedit migrate inventory-serialization nbt") @CommandPermission("multiverse.inventories.bulkedit") void onNbtCommand(MVCommandIssuer issuer) { + if (!ItemStackConverter.hasByteSerializeSupport()) { + issuer.sendMessage("NBT serialization is only supported on PaperMC 1.20.2 or higher!"); + issuer.sendMessage("Conversion to NBT is not possible on your current server version."); + return; + } commandQueueManager.addToQueue(CommandQueuePayload.issuer(issuer) .prompt(Message.of("Are you sure you want to migrate all player data to NBT?")) .action(() -> doMigration(issuer, true))); @@ -57,20 +81,35 @@ void onBukkitCommand(MVCommandIssuer issuer) { } private void doMigration(MVCommandIssuer issuer, boolean useByteSerialization) { + issuer.sendMessage("Updating config and clearing caches..."); inventoriesConfig.setUseByteSerializationForInventoryData(useByteSerialization); inventoriesConfig.save(); + profileCacheManager.clearAllCache(); long startTime = System.nanoTime(); AtomicLong profileCounter = new AtomicLong(0); - CompletableFuture.allOf(profileDataSource.listGlobalProfileUUIDs() - .stream() - .map(playerUUID -> profileDataSource.getGlobalProfile(GlobalProfileKey.of(playerUUID, "")) - .thenCompose(profile -> run(profile, profileCounter)) - .exceptionally(throwable -> { - issuer.sendMessage("Error updating player " + playerUUID + ": " + throwable.getMessage()); - return null; - })) - .toArray(CompletableFuture[]::new)) + + Map> containerNames = new HashMap<>(); + for (ContainerType type : ContainerType.values()) { + containerNames.put(type, profileDataSource.listContainerDataNames(type)); + } + + Set playerIdentifiers = new HashSet<>(); + // Scan global files + profileFilesLocator.listGlobalFiles().forEach(file -> + playerIdentifiers.add(Files.getNameWithoutExtension(file.getName()))); + + // Scan world and group files + for (ContainerType type : ContainerType.values()) { + for (File folder : profileFilesLocator.listProfileContainerFolders(type)) { + profileFilesLocator.listPlayerProfileFiles(type, folder.getName()).forEach(file -> + playerIdentifiers.add(Files.getNameWithoutExtension(file.getName()))); + } + } + + issuer.sendMessage("Found " + playerIdentifiers.size() + " unique player identifiers to migrate."); + + migrateNextPlayer(issuer, new ArrayList<>(playerIdentifiers), 0, containerNames, profileCounter) .thenRun(() -> { long timeDuration = (System.nanoTime() - startTime) / 1000000; issuer.sendMessage("Updated " + profileCounter.get() + " player profiles."); @@ -78,23 +117,78 @@ private void doMigration(MVCommandIssuer issuer, boolean useByteSerialization) { }); } - private CompletableFuture run(GlobalProfile profile, AtomicLong profileCounter) { - return CompletableFuture.allOf(Arrays.stream(ContainerType.values()) - .flatMap(containerType -> profileDataSource.listContainerDataNames(containerType).stream() - .flatMap(dataName -> ProfileTypes.getTypes().stream() - .map(profileType -> profileDataSource.getPlayerProfile(ProfileKey.of( - containerType, - dataName, - profileType, - profile.getPlayerUUID(), - profile.getLastKnownName() - )).thenCompose(playerProfile -> { - if (playerProfile.getData().isEmpty()) { - return CompletableFuture.completedFuture(null); - } - profileCounter.incrementAndGet(); - return profileDataSource.updatePlayerProfile(playerProfile); - })))) - .toArray(CompletableFuture[]::new)); + private CompletableFuture migrateNextPlayer( + MVCommandIssuer issuer, + List playerIdentifiers, + int index, + Map> containerNames, + AtomicLong profileCounter + ) { + if (index >= playerIdentifiers.size()) { + return CompletableFuture.completedFuture(null); + } + + String playerIdentifier = playerIdentifiers.get(index); + UUID playerUUID; + try { + playerUUID = UUID.fromString(playerIdentifier); + } catch (IllegalArgumentException e) { + playerUUID = UUID.nameUUIDFromBytes(("OfflinePlayer:" + playerIdentifier).getBytes(StandardCharsets.UTF_8)); + } + + if (index % 50 == 0 && index > 0) { + issuer.sendMessage("Processed " + index + " players..."); + } + + return profileDataSource.getGlobalProfile(GlobalProfileKey.of(playerUUID, playerIdentifier)) + .thenCompose(profile -> run(issuer, profile, containerNames, profileCounter)) + .exceptionally(throwable -> { + issuer.sendMessage("Error updating player " + playerIdentifier + ": " + throwable.getMessage()); + return null; + }) + .thenCompose(v -> migrateNextPlayer(issuer, playerIdentifiers, index + 1, containerNames, profileCounter)); + } + + private CompletableFuture run( + MVCommandIssuer issuer, + GlobalProfile profile, + Map> containerNames, + AtomicLong profileCounter + ) { + String playerName = profile.getLastKnownName(); + if (playerName == null || playerName.isEmpty()) { + playerName = profile.getPlayerUUID().toString(); + } + + CompletableFuture future = CompletableFuture.completedFuture(null); + for (ContainerType containerType : ContainerType.values()) { + List dataNames = containerNames.get(containerType); + for (String dataName : dataNames) { + for (ProfileType profileType : ProfileTypes.getTypes()) { + ProfileKey profileKey = ProfileKey.of( + containerType, + dataName, + profileType, + profile.getPlayerUUID(), + playerName + ); + future = future.thenCompose(v -> profileDataSource.getPlayerProfile(profileKey) + .thenCompose(playerProfile -> { + if (playerProfile.getData().isEmpty()) { + return CompletableFuture.completedFuture(null); + } + profileCounter.incrementAndGet(); + return profileDataSource.updatePlayerProfile(playerProfile); + }) + .exceptionally(throwable -> { + issuer.sendMessage(String.format("Error migrating profile %s %s/%s for player %s: %s", + containerType, dataName, profileType, profile.getPlayerUUID(), throwable.getMessage())); + return null; + }) + ); + } + } + } + return future; } } diff --git a/src/main/java/org/mvplugins/multiverse/inventories/profile/FlatFileProfileDataSource.java b/src/main/java/org/mvplugins/multiverse/inventories/profile/FlatFileProfileDataSource.java index ea17a4c1..082fa149 100644 --- a/src/main/java/org/mvplugins/multiverse/inventories/profile/FlatFileProfileDataSource.java +++ b/src/main/java/org/mvplugins/multiverse/inventories/profile/FlatFileProfileDataSource.java @@ -57,7 +57,7 @@ final class FlatFileProfileDataSource implements ProfileDataSource { private FileConfiguration loadFileToJsonConfiguration(File file) { JsonConfiguration jsonConfiguration = new JsonConfiguration(); - jsonConfiguration.options().continueOnSerializationError(false); + jsonConfiguration.options().continueOnSerializationError(true); Try.run(() -> jsonConfiguration.load(file)).getOrElseThrow(e -> { Logging.severe("Could not load file %s : %s", file, e.getMessage()); e.printStackTrace(); diff --git a/src/main/java/org/mvplugins/multiverse/inventories/profile/ProfileFilesLocator.java b/src/main/java/org/mvplugins/multiverse/inventories/profile/ProfileFilesLocator.java index 8cbfc874..0207b076 100644 --- a/src/main/java/org/mvplugins/multiverse/inventories/profile/ProfileFilesLocator.java +++ b/src/main/java/org/mvplugins/multiverse/inventories/profile/ProfileFilesLocator.java @@ -18,7 +18,7 @@ import java.util.UUID; @Service -final class ProfileFilesLocator { +public final class ProfileFilesLocator { private static final String JSON = ".json"; @@ -67,7 +67,7 @@ File getContainerFolder(ContainerType type) { }; } - List listProfileContainerFolders(ContainerType type) { + public List listProfileContainerFolders(ContainerType type) { return Option.of(getContainerFolder(type).listFiles()) .map(filesList -> Arrays.stream(filesList) .filter(File::isDirectory) @@ -83,7 +83,7 @@ File getProfileContainerFolder(ContainerType type, String folderName) { return folder; } - List listPlayerProfileFiles(ContainerType type, String dataName) { + public List listPlayerProfileFiles(ContainerType type, String dataName) { return Option.of(getProfileContainerFolder(type, dataName).listFiles()) .map(filesList -> Arrays.stream(filesList) .filter(File::isFile) @@ -120,7 +120,7 @@ File getGlobalFolder() { return this.globalFolder; } - List listGlobalFiles() { + public List listGlobalFiles() { return Option.of(this.globalFolder.listFiles()) .map(filesList -> Arrays.stream(filesList) .filter(File::isFile) From 04a16571bdd2da1d32435cab17396e132440f016 Mon Sep 17 00:00:00 2001 From: zombachu Date: Fri, 24 Apr 2026 01:08:04 -0700 Subject: [PATCH 2/2] Partial revert of unnecessary changes --- .../MigrateInventorySerializationCommand.java | 154 +++++++----------- .../profile/FlatFileProfileDataSource.java | 2 +- 2 files changed, 57 insertions(+), 99 deletions(-) diff --git a/src/main/java/org/mvplugins/multiverse/inventories/commands/bulkedit/playerprofile/MigrateInventorySerializationCommand.java b/src/main/java/org/mvplugins/multiverse/inventories/commands/bulkedit/playerprofile/MigrateInventorySerializationCommand.java index e163fbdd..81636530 100644 --- a/src/main/java/org/mvplugins/multiverse/inventories/commands/bulkedit/playerprofile/MigrateInventorySerializationCommand.java +++ b/src/main/java/org/mvplugins/multiverse/inventories/commands/bulkedit/playerprofile/MigrateInventorySerializationCommand.java @@ -13,23 +13,19 @@ import org.mvplugins.multiverse.inventories.commands.InventoriesCommand; import org.mvplugins.multiverse.inventories.config.InventoriesConfig; import org.mvplugins.multiverse.inventories.profile.GlobalProfile; -import org.mvplugins.multiverse.inventories.profile.ProfileCacheManager; import org.mvplugins.multiverse.inventories.profile.ProfileDataSource; import org.mvplugins.multiverse.inventories.profile.key.GlobalProfileKey; import org.mvplugins.multiverse.inventories.profile.key.ProfileKey; -import org.mvplugins.multiverse.inventories.profile.key.ProfileType; import org.mvplugins.multiverse.inventories.profile.key.ProfileTypes; import org.mvplugins.multiverse.inventories.profile.key.ContainerType; import org.mvplugins.multiverse.inventories.profile.ProfileFilesLocator; -import org.mvplugins.multiverse.inventories.util.ItemStackConverter; import java.io.File; import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.HashMap; +import java.util.Arrays; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.UUID; import java.util.concurrent.CompletableFuture; @@ -41,7 +37,6 @@ final class MigrateInventorySerializationCommand extends InventoriesCommand { private final CommandQueueManager commandQueueManager; private final ProfileDataSource profileDataSource; private final InventoriesConfig inventoriesConfig; - private final ProfileCacheManager profileCacheManager; private final ProfileFilesLocator profileFilesLocator; @Inject @@ -49,24 +44,17 @@ final class MigrateInventorySerializationCommand extends InventoriesCommand { @NotNull CommandQueueManager commandQueueManager, @NotNull ProfileDataSource profileDataSource, @NotNull InventoriesConfig inventoriesConfig, - @NotNull ProfileCacheManager profileCacheManager, @NotNull ProfileFilesLocator profileFilesLocator ) { this.commandQueueManager = commandQueueManager; this.profileDataSource = profileDataSource; this.inventoriesConfig = inventoriesConfig; - this.profileCacheManager = profileCacheManager; this.profileFilesLocator = profileFilesLocator; } @Subcommand("bulkedit migrate inventory-serialization nbt") @CommandPermission("multiverse.inventories.bulkedit") void onNbtCommand(MVCommandIssuer issuer) { - if (!ItemStackConverter.hasByteSerializeSupport()) { - issuer.sendMessage("NBT serialization is only supported on PaperMC 1.20.2 or higher!"); - issuer.sendMessage("Conversion to NBT is not possible on your current server version."); - return; - } commandQueueManager.addToQueue(CommandQueuePayload.issuer(issuer) .prompt(Message.of("Are you sure you want to migrate all player data to NBT?")) .action(() -> doMigration(issuer, true))); @@ -81,114 +69,84 @@ void onBukkitCommand(MVCommandIssuer issuer) { } private void doMigration(MVCommandIssuer issuer, boolean useByteSerialization) { - issuer.sendMessage("Updating config and clearing caches..."); inventoriesConfig.setUseByteSerializationForInventoryData(useByteSerialization); inventoriesConfig.save(); - profileCacheManager.clearAllCache(); long startTime = System.nanoTime(); AtomicLong profileCounter = new AtomicLong(0); - Map> containerNames = new HashMap<>(); - for (ContainerType type : ContainerType.values()) { - containerNames.put(type, profileDataSource.listContainerDataNames(type)); - } - - Set playerIdentifiers = new HashSet<>(); // Scan global files + Set fileNamesSet = new HashSet<>(); profileFilesLocator.listGlobalFiles().forEach(file -> - playerIdentifiers.add(Files.getNameWithoutExtension(file.getName()))); + fileNamesSet.add(Files.getNameWithoutExtension(file.getName()))); // Scan world and group files for (ContainerType type : ContainerType.values()) { for (File folder : profileFilesLocator.listProfileContainerFolders(type)) { profileFilesLocator.listPlayerProfileFiles(type, folder.getName()).forEach(file -> - playerIdentifiers.add(Files.getNameWithoutExtension(file.getName()))); + fileNamesSet.add(Files.getNameWithoutExtension(file.getName()))); } } - issuer.sendMessage("Found " + playerIdentifiers.size() + " unique player identifiers to migrate."); + List fileNames = new ArrayList<>(fileNamesSet); + issuer.sendMessage("Found " + fileNames.size() + " unique players to migrate."); - migrateNextPlayer(issuer, new ArrayList<>(playerIdentifiers), 0, containerNames, profileCounter) - .thenRun(() -> { - long timeDuration = (System.nanoTime() - startTime) / 1000000; - issuer.sendMessage("Updated " + profileCounter.get() + " player profiles."); - issuer.sendMessage("Bulk edit completed in " + timeDuration + " ms."); - }); - } - - private CompletableFuture migrateNextPlayer( - MVCommandIssuer issuer, - List playerIdentifiers, - int index, - Map> containerNames, - AtomicLong profileCounter - ) { - if (index >= playerIdentifiers.size()) { - return CompletableFuture.completedFuture(null); - } + CompletableFuture future = CompletableFuture.completedFuture(null); + for (int i = 0; i < fileNames.size(); i++) { + final int index = i; + final String fileName = fileNames.get(i); + + future = future.thenCompose(v -> { + UUID playerUUID; + try { + playerUUID = UUID.fromString(fileName); + } catch (IllegalArgumentException e) { + playerUUID = UUID.nameUUIDFromBytes(("OfflinePlayer:" + fileName).getBytes(StandardCharsets.UTF_8)); + } - String playerIdentifier = playerIdentifiers.get(index); - UUID playerUUID; - try { - playerUUID = UUID.fromString(playerIdentifier); - } catch (IllegalArgumentException e) { - playerUUID = UUID.nameUUIDFromBytes(("OfflinePlayer:" + playerIdentifier).getBytes(StandardCharsets.UTF_8)); - } + if (index % 100 == 0) { + issuer.sendMessage("Processed " + index + " players..."); + } - if (index % 50 == 0 && index > 0) { - issuer.sendMessage("Processed " + index + " players..."); + return profileDataSource.getGlobalProfile(GlobalProfileKey.of(playerUUID, fileName)) + .thenCompose(profile -> run(profile, profileCounter)) + .exceptionally(throwable -> { + issuer.sendMessage("Error updating player " + fileName + ": " + throwable.getMessage()); + return null; + }); + }); } - return profileDataSource.getGlobalProfile(GlobalProfileKey.of(playerUUID, playerIdentifier)) - .thenCompose(profile -> run(issuer, profile, containerNames, profileCounter)) - .exceptionally(throwable -> { - issuer.sendMessage("Error updating player " + playerIdentifier + ": " + throwable.getMessage()); - return null; - }) - .thenCompose(v -> migrateNextPlayer(issuer, playerIdentifiers, index + 1, containerNames, profileCounter)); + future.thenRun(() -> { + long timeDuration = (System.nanoTime() - startTime) / 1000000; + issuer.sendMessage("Updated " + profileCounter.get() + " player profiles."); + issuer.sendMessage("Bulk edit completed in " + timeDuration + " ms."); + }); } - private CompletableFuture run( - MVCommandIssuer issuer, - GlobalProfile profile, - Map> containerNames, - AtomicLong profileCounter - ) { - String playerName = profile.getLastKnownName(); - if (playerName == null || playerName.isEmpty()) { - playerName = profile.getPlayerUUID().toString(); - } - - CompletableFuture future = CompletableFuture.completedFuture(null); - for (ContainerType containerType : ContainerType.values()) { - List dataNames = containerNames.get(containerType); - for (String dataName : dataNames) { - for (ProfileType profileType : ProfileTypes.getTypes()) { - ProfileKey profileKey = ProfileKey.of( - containerType, - dataName, - profileType, - profile.getPlayerUUID(), - playerName - ); - future = future.thenCompose(v -> profileDataSource.getPlayerProfile(profileKey) - .thenCompose(playerProfile -> { - if (playerProfile.getData().isEmpty()) { - return CompletableFuture.completedFuture(null); - } - profileCounter.incrementAndGet(); - return profileDataSource.updatePlayerProfile(playerProfile); - }) - .exceptionally(throwable -> { - issuer.sendMessage(String.format("Error migrating profile %s %s/%s for player %s: %s", - containerType, dataName, profileType, profile.getPlayerUUID(), throwable.getMessage())); - return null; - }) - ); - } - } + private CompletableFuture run(GlobalProfile profile, AtomicLong profileCounter) { + String fileName = profile.getLastKnownName(); + if (fileName == null || fileName.isEmpty()) { + fileName = profile.getPlayerUUID().toString(); } - return future; + final String finalFileName = fileName; + + return CompletableFuture.allOf(Arrays.stream(ContainerType.values()) + .flatMap(containerType -> profileDataSource.listContainerDataNames(containerType).stream() + .flatMap(dataName -> ProfileTypes.getTypes().stream() + .map(profileType -> profileDataSource.getPlayerProfile(ProfileKey.of( + containerType, + dataName, + profileType, + profile.getPlayerUUID(), + finalFileName + )).thenCompose(playerProfile -> { + if (playerProfile.getData().isEmpty()) { + return CompletableFuture.completedFuture(null); + } + profileCounter.incrementAndGet(); + return profileDataSource.updatePlayerProfile(playerProfile); + })))) + .toArray(CompletableFuture[]::new)); } } diff --git a/src/main/java/org/mvplugins/multiverse/inventories/profile/FlatFileProfileDataSource.java b/src/main/java/org/mvplugins/multiverse/inventories/profile/FlatFileProfileDataSource.java index 082fa149..ea17a4c1 100644 --- a/src/main/java/org/mvplugins/multiverse/inventories/profile/FlatFileProfileDataSource.java +++ b/src/main/java/org/mvplugins/multiverse/inventories/profile/FlatFileProfileDataSource.java @@ -57,7 +57,7 @@ final class FlatFileProfileDataSource implements ProfileDataSource { private FileConfiguration loadFileToJsonConfiguration(File file) { JsonConfiguration jsonConfiguration = new JsonConfiguration(); - jsonConfiguration.options().continueOnSerializationError(true); + jsonConfiguration.options().continueOnSerializationError(false); Try.run(() -> jsonConfiguration.load(file)).getOrElseThrow(e -> { Logging.severe("Could not load file %s : %s", file, e.getMessage()); e.printStackTrace();