diff --git a/OpenKh.Kh1/Kh1Binl.cs b/OpenKh.Kh1/Kh1Binl.cs new file mode 100644 index 000000000..9d0a3ad62 --- /dev/null +++ b/OpenKh.Kh1/Kh1Binl.cs @@ -0,0 +1,338 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; + +namespace OpenKh.Kh1 +{ + /// + /// Preserving editor for remastered KH1 EvMsg *.binl files. + /// Unknown bytecode and command parameters are kept byte-for-byte. + /// + public sealed class Kh1Binl + { + public sealed class TextEntry + { + private readonly Kh1TextTable _table; + private readonly byte[] _originalBytes; + + internal TextEntry(int index, int offset, int length, byte[] bytes, Kh1TextTable table) + { + Index = index; + Offset = offset; + OriginalLength = length; + _table = table; + _originalBytes = bytes; + Text = DecodeBody(bytes); + OriginalText = Text; + } + + public int Index { get; } + public int Offset { get; } + public int OriginalLength { get; } + public string OriginalText { get; } + public string Text { get; set; } + public bool IsModified => !string.Equals(OriginalText, Text, StringComparison.Ordinal); + internal bool ContainsStructuralCommands => + Text.Contains("{cmd:05", StringComparison.OrdinalIgnoreCase) || + Text.Contains("{cmd:06", StringComparison.OrdinalIgnoreCase) || + Text.Contains("{cmd:0A", StringComparison.OrdinalIgnoreCase); + + internal byte[] EncodeBody() + { + if (!IsModified) + return _originalBytes; + + var output = new List(); + var plainText = new StringBuilder(); + + void FlushPlainText() + { + if (plainText.Length == 0) + return; + output.AddRange(_table.Encode(plainText.ToString())); + plainText.Clear(); + } + + for (var position = 0; position < Text.Length;) + { + if (Text[position] == '\r' || Text[position] == '\n') + { + FlushPlainText(); + if (Text[position] == '\r' && position + 1 < Text.Length && Text[position + 1] == '\n') + position++; + output.Add(0x02); + position++; + continue; + } + + if (TryReadCommandToken(Text, position, out var command, out var tokenLength)) + { + FlushPlainText(); + ValidateCommand(command); + output.AddRange(command); + position += tokenLength; + continue; + } + + plainText.Append(Text[position]); + position++; + } + + FlushPlainText(); + ValidateBody(output); + return output.ToArray(); + } + + private string DecodeBody(byte[] bytes) + { + var output = new StringBuilder(); + for (var offset = 0; offset < bytes.Length;) + { + var opcode = bytes[offset]; + if (opcode == 0x01) + { + output.Append(' '); + offset++; + } + else if (opcode == 0x02) + { + output.AppendLine(); + offset++; + } + else if (opcode <= 0x0E) + { + var length = GetInstructionLength(opcode); + if (offset + length > bytes.Length) + length = bytes.Length - offset; + output.Append("{cmd:") + .Append(string.Join(" ", bytes.AsSpan(offset, length).ToArray().Select(x => x.ToString("X2")))) + .Append('}'); + offset += length; + } + else + { + var nextControl = offset + 1; + while (nextControl < bytes.Length && bytes[nextControl] > 0x0E) + nextControl++; + output.Append(_table.Decode(bytes.AsSpan(offset, nextControl - offset))); + offset = nextControl; + } + } + + return output.ToString(); + } + + private static bool TryReadCommandToken( + string text, + int offset, + out byte[] command, + out int tokenLength) + { + command = null; + tokenLength = 0; + if (!text.AsSpan(offset).StartsWith("{cmd:".AsSpan(), StringComparison.OrdinalIgnoreCase)) + return false; + + var closeBrace = text.IndexOf('}', offset + 5); + if (closeBrace < 0) + throw new InvalidDataException($"Command token at position {offset + 1} has no closing brace."); + + var value = text.Substring(offset + 5, closeBrace - offset - 5) + .Replace(" ", string.Empty); + if (value.Length == 0 || (value.Length & 1) != 0 || !value.All(Uri.IsHexDigit)) + throw new InvalidDataException($"Invalid command token at position {offset + 1}."); + + command = Convert.FromHexString(value); + tokenLength = closeBrace - offset + 1; + return true; + } + + private static void ValidateCommand(byte[] command) + { + if (command.Length == 0 || command[0] > 0x0E) + throw new InvalidDataException("A {cmd:...} token must begin with a BINL opcode from 00 to 0E."); + if (command.Length != GetInstructionLength(command[0])) + throw new InvalidDataException( + $"BINL command {command[0]:X2} must contain {GetInstructionLength(command[0])} byte(s)."); + } + + private static void ValidateBody(IReadOnlyList bytes) + { + for (var offset = 0; offset < bytes.Count;) + { + var opcode = bytes[offset]; + var length = opcode <= 0x0E ? GetInstructionLength(opcode) : 1; + if (offset + length > bytes.Count) + throw new InvalidDataException($"Truncated BINL command {opcode:X2} in edited text."); + if (opcode == 0x05 || opcode == 0x06 || opcode == 0x0A) + throw new InvalidDataException( + $"Structural BINL command {opcode:X2} cannot be inserted into editable text."); + offset += length; + } + } + } + + private readonly byte[] _source; + private readonly int _contentLength; + + private Kh1Binl(byte[] source, int contentLength, IReadOnlyList entries) + { + _source = source; + _contentLength = contentLength; + Entries = entries; + Language = Encoding.ASCII.GetString(source, 5, 2); + } + + public string Language { get; } + public IReadOnlyList Entries { get; } + + public static bool IsValid(Stream stream) + { + ArgumentNullException.ThrowIfNull(stream); + if (!stream.CanSeek || stream.Length < 12) + return false; + + var oldPosition = stream.Position; + Span magic = stackalloc byte[5]; + var read = stream.Read(magic); + stream.Position = oldPosition; + return read == magic.Length && magic.SequenceEqual("EvMsg"u8); + } + + public static Kh1Binl Read(Stream stream, Kh1TextTable table) + { + ArgumentNullException.ThrowIfNull(stream); + ArgumentNullException.ThrowIfNull(table); + if (!IsValid(stream)) + throw new InvalidDataException("The file is not a KH1 EvMsg BINL file."); + + stream.Position = 0; + using var buffer = new MemoryStream(); + stream.CopyTo(buffer); + var source = buffer.ToArray(); + var contentLength = source.Length; + while (contentLength > 11 && source[contentLength - 1] == 0xCD) + contentLength--; + + var recordOffsets = FindRecordOffsets(source, contentLength); + var entries = new List(); + for (var recordIndex = 0; recordIndex < recordOffsets.Count; recordIndex++) + { + var recordStart = recordOffsets[recordIndex]; + var recordEnd = recordIndex + 1 < recordOffsets.Count + ? recordOffsets[recordIndex + 1] + : contentLength; + if (TryFindBody(source, recordStart + 4, recordEnd, out var bodyStart, out var bodyEnd)) + { + var entry = new TextEntry( + entries.Count, + bodyStart, + bodyEnd - bodyStart, + source.AsSpan(bodyStart, bodyEnd - bodyStart).ToArray(), + table); + if (!entry.ContainsStructuralCommands) + entries.Add(entry); + } + } + + return new Kh1Binl(source, contentLength, entries); + } + + public static Kh1Binl Read(Stream stream) => Read(stream, Kh1TextTable.Default); + + public static Kh1Binl Read(string fileName, Kh1TextTable table) + { + using var stream = File.OpenRead(fileName); + return Read(stream, table); + } + + public static Kh1Binl Read(string fileName) => Read(fileName, Kh1TextTable.Default); + + public void Write(Stream stream) + { + ArgumentNullException.ThrowIfNull(stream); + + var replacements = Entries + .Select(x => new { Entry = x, Bytes = x.EncodeBody() }) + .ToList(); + + var cursor = 0; + foreach (var replacement in replacements) + { + stream.Write(_source, cursor, replacement.Entry.Offset - cursor); + stream.Write(replacement.Bytes); + cursor = replacement.Entry.Offset + replacement.Entry.OriginalLength; + } + + stream.Write(_source, cursor, _contentLength - cursor); + while ((stream.Position & 0x0F) != 0) + stream.WriteByte(0xCD); + } + + private static List FindRecordOffsets(byte[] source, int contentLength) + { + var result = new List(); + for (var offset = 11; offset < contentLength;) + { + var opcode = source[offset]; + var length = opcode <= 0x0E ? GetInstructionLength(opcode) : 1; + if (offset + length > contentLength) + throw new InvalidDataException($"Truncated BINL instruction at 0x{offset:X}."); + if (opcode == 0x0A) + result.Add(offset); + offset += length; + } + + return result; + } + + private static bool TryFindBody( + byte[] source, + int start, + int end, + out int bodyStart, + out int bodyEnd) + { + bodyStart = -1; + bodyEnd = -1; + var pendingWhitespace = -1; + for (var offset = start; offset < end;) + { + var opcode = source[offset]; + var length = opcode <= 0x0E ? GetInstructionLength(opcode) : 1; + if (offset + length > end) + return false; + + if (bodyStart < 0) + { + if ((opcode == 0x01 || opcode == 0x02) && pendingWhitespace < 0) + pendingWhitespace = offset; + else if (opcode > 0x0E) + bodyStart = pendingWhitespace >= 0 ? pendingWhitespace : offset; + } + else if (opcode == 0x05 || opcode == 0x06) + { + bodyEnd = offset; + return bodyEnd > bodyStart; + } + + offset += length; + } + + // Records without a 05/06 terminator contain presentation data rather + // than an editable dialogue body. Keeping them out of the entry list + // avoids exposing command parameters as black-square characters. + return false; + } + + private static int GetInstructionLength(byte opcode) => opcode switch + { + 0x05 or 0x06 or 0x07 => 3, + 0x0A or 0x0B or 0x0D => 4, + 0x0C or 0x0E => 2, + _ => 1, + }; + } +} diff --git a/OpenKh.Kh1/Kh1EventMessage.cs b/OpenKh.Kh1/Kh1EventMessage.cs new file mode 100644 index 000000000..16534f4a0 --- /dev/null +++ b/OpenKh.Kh1/Kh1EventMessage.cs @@ -0,0 +1,277 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +namespace OpenKh.Kh1 +{ + /// + /// Preserving editor for the EvMsg bytecode section embedded in remastered + /// KH1 *.ev and *.evdl files. + /// + public sealed class Kh1EventMessage + { + private readonly byte[] _source; + private readonly int _sectionStart; + private readonly int _sectionEnd; + + private Kh1EventMessage( + byte[] source, + int sectionStart, + int sectionEnd, + IReadOnlyList entries) + { + _source = source; + _sectionStart = sectionStart; + _sectionEnd = sectionEnd; + Entries = entries; + } + + public IReadOnlyList Entries { get; } + + public static bool IsValid(Stream stream) + { + ArgumentNullException.ThrowIfNull(stream); + if (!stream.CanSeek || stream.Length < 0x20) + return false; + + var oldPosition = stream.Position; + try + { + Span firstHeader = stackalloc byte[0x20]; + if (stream.Read(firstHeader) != firstHeader.Length) + return false; + + var sectionStart = checked((int)BitConverter.ToUInt32(firstHeader.Slice(0x0C, 4))); + var sectionEnd = checked((int)BitConverter.ToUInt32(firstHeader.Slice(0x10, 4))); + if (sectionStart < 0x14 || + sectionStart > 0x1000 || + (sectionStart & 3) != 0 || + sectionEnd <= sectionStart || + sectionEnd > stream.Length || + ((sectionEnd - sectionStart) & 0x0F) != 0) + return false; + + var header = new byte[sectionStart]; + stream.Position = 0; + stream.ReadExactly(header); + var previous = sectionStart; + for (var offset = 0x10; offset < sectionStart; offset += sizeof(uint)) + { + var value = checked((int)BitConverter.ToUInt32(header, offset)); + if (value <= previous || value > stream.Length) + return false; + previous = value; + } + return true; + } + catch (Exception ex) when (ex is IOException or OverflowException or ArgumentException) + { + return false; + } + finally + { + stream.Position = oldPosition; + } + } + + public static Kh1EventMessage Read(Stream stream, Kh1TextTable table) + { + ArgumentNullException.ThrowIfNull(stream); + ArgumentNullException.ThrowIfNull(table); + if (!IsValid(stream)) + throw new InvalidDataException("The file does not contain a valid KH1 event-message section."); + + stream.Position = 0; + using var buffer = new MemoryStream(); + stream.CopyTo(buffer); + var source = buffer.ToArray(); + var sectionStart = checked((int)BitConverter.ToUInt32(source, 0x0C)); + var sectionEnd = checked((int)BitConverter.ToUInt32(source, 0x10)); + ValidateOffsets(source, sectionStart, sectionEnd); + + var recordOffsets = FindRecordOffsets(source, sectionStart + sizeof(uint), sectionEnd); + var entries = new List(); + for (var recordIndex = 0; recordIndex < recordOffsets.Count; recordIndex++) + { + var recordStart = recordOffsets[recordIndex]; + var recordEnd = recordIndex + 1 < recordOffsets.Count + ? recordOffsets[recordIndex + 1] + : sectionEnd; + var bodySearchStart = source[recordStart] == 0x0A + ? recordStart + GetInstructionLength(0x0A) + : recordStart; + + if (!TryFindBody(source, bodySearchStart, recordEnd, out var bodyStart, out var bodyEnd)) + continue; + + var entry = new Kh1Binl.TextEntry( + entries.Count, + bodyStart, + bodyEnd - bodyStart, + source.AsSpan(bodyStart, bodyEnd - bodyStart).ToArray(), + table); + if (!entry.ContainsStructuralCommands && IsReadableText(entry.Text)) + entries.Add(entry); + } + + return new Kh1EventMessage(source, sectionStart, sectionEnd, entries); + } + + public static Kh1EventMessage Read(Stream stream) => Read(stream, Kh1TextTable.Default); + + public static Kh1EventMessage Read(string fileName, Kh1TextTable table) + { + using var stream = File.OpenRead(fileName); + return Read(stream, table); + } + + public static Kh1EventMessage Read(string fileName) => Read(fileName, Kh1TextTable.Default); + + public void Write(Stream stream) + { + ArgumentNullException.ThrowIfNull(stream); + if (!Entries.Any(x => x.IsModified)) + { + stream.Write(_source); + return; + } + + using var section = new MemoryStream(); + var cursor = _sectionStart; + foreach (var entry in Entries) + { + section.Write(_source, cursor, entry.Offset - cursor); + section.Write(entry.EncodeBody()); + cursor = entry.Offset + entry.OriginalLength; + } + section.Write(_source, cursor, _sectionEnd - cursor); + while ((section.Length & 0x0F) != 0) + section.WriteByte(0x00); + + var sectionBytes = section.ToArray(); + var oldSectionLength = _sectionEnd - _sectionStart; + var delta = checked(sectionBytes.Length - oldSectionLength); + var header = _source.AsSpan(0, _sectionStart).ToArray(); + if (delta != 0) + { + for (var offset = 0x10; offset < _sectionStart; offset += sizeof(uint)) + { + var oldValue = BitConverter.ToUInt32(header, offset); + BitConverter.GetBytes(checked((uint)(oldValue + delta))).CopyTo(header, offset); + } + } + + stream.Write(header); + stream.Write(sectionBytes); + stream.Write(_source, _sectionEnd, _source.Length - _sectionEnd); + } + + private static void ValidateOffsets(byte[] source, int sectionStart, int sectionEnd) + { + var previous = sectionStart; + for (var offset = 0x10; offset < sectionStart; offset += sizeof(uint)) + { + var value = checked((int)BitConverter.ToUInt32(source, offset)); + if (value <= previous || value > source.Length) + throw new InvalidDataException("The KH1 event-message offset table is invalid."); + previous = value; + } + if (BitConverter.ToUInt32(source, 0x10) != sectionEnd) + throw new InvalidDataException("The KH1 event-message section boundary is invalid."); + } + + private static List FindRecordOffsets(byte[] source, int start, int end) + { + var result = new List { start }; + for (var offset = start; offset < end;) + { + var opcode = source[offset]; + var length = opcode <= 0x0E ? GetInstructionLength(opcode) : 1; + if (offset + length > end) + break; + if (opcode == 0x0A && offset != start) + result.Add(offset); + offset += length; + } + return result; + } + + private static bool TryFindBody( + byte[] source, + int start, + int end, + out int bodyStart, + out int bodyEnd) + { + bodyStart = -1; + bodyEnd = -1; + var pendingWhitespace = -1; + for (var offset = start; offset < end;) + { + var opcode = source[offset]; + var length = opcode <= 0x0E ? GetInstructionLength(opcode) : 1; + if (offset + length > end) + return false; + + if (bodyStart < 0) + { + if ((opcode == 0x01 || opcode == 0x02) && pendingWhitespace < 0) + pendingWhitespace = offset; + else if (opcode > 0x0E) + bodyStart = pendingWhitespace >= 0 ? pendingWhitespace : offset; + } + else if (opcode == 0x04 || opcode == 0x05 || opcode == 0x06) + { + bodyEnd = offset; + return bodyEnd > bodyStart; + } + + offset += length; + } + return false; + } + + private static bool IsReadableText(string text) + { + var letters = 0; + var visible = 0; + var squares = 0; + var inToken = false; + foreach (var character in text) + { + if (character == '{') + { + inToken = true; + continue; + } + if (inToken) + { + if (character == '}') + inToken = false; + continue; + } + if (char.IsWhiteSpace(character)) + continue; + visible++; + if (char.IsLetterOrDigit(character)) + letters++; + if (character == '■') + squares++; + } + + return letters >= 2 && + visible > 0 && + letters * 100 / visible >= 35 && + squares <= Math.Max(2, letters / 4); + } + + private static int GetInstructionLength(byte opcode) => opcode switch + { + 0x05 or 0x06 or 0x07 => 3, + 0x0A or 0x0B or 0x0D => 4, + 0x0C or 0x0E => 2, + _ => 1, + }; + } +} diff --git a/OpenKh.Kh1/Kh1Kmb.cs b/OpenKh.Kh1/Kh1Kmb.cs new file mode 100644 index 000000000..68ffb0965 --- /dev/null +++ b/OpenKh.Kh1/Kh1Kmb.cs @@ -0,0 +1,178 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; + +namespace OpenKh.Kh1 +{ + /// + /// Preserving editor for remastered KH1 menu-message *.kmb files. + /// + public sealed class Kh1Kmb + { + public sealed class TextEntry + { + private readonly Kh1TextTable _table; + private readonly byte[] _originalBytes; + + internal TextEntry(int index, int offset, byte[] bytes, Kh1TextTable table) + { + Index = index; + Offset = offset; + OriginalLength = bytes.Length; + _originalBytes = bytes; + _table = table; + Text = Decode(bytes, table); + OriginalText = Text; + } + + public int Index { get; } + public int Offset { get; } + public int OriginalLength { get; } + public string OriginalText { get; } + public string Text { get; set; } + public bool IsModified => !string.Equals(OriginalText, Text, StringComparison.Ordinal); + + internal byte[] Encode() + { + if (!IsModified) + return _originalBytes; + + using var output = new MemoryStream(); + var plainText = new StringBuilder(); + + void FlushPlainText() + { + if (plainText.Length == 0) + return; + output.Write(_table.Encode(plainText.ToString())); + plainText.Clear(); + } + + for (var position = 0; position < Text.Length; position++) + { + if (Text[position] != '\r' && Text[position] != '\n') + { + plainText.Append(Text[position]); + continue; + } + + FlushPlainText(); + if (Text[position] == '\r' && position + 1 < Text.Length && Text[position + 1] == '\n') + position++; + output.WriteByte(0x02); + } + + FlushPlainText(); + var bytes = output.ToArray(); + if (bytes.Contains((byte)0x00)) + throw new InvalidDataException("KMB text cannot contain {eol}; byte 00 terminates the entry."); + return bytes; + } + + private static string Decode(byte[] bytes, Kh1TextTable table) + { + var output = new StringBuilder(); + var textStart = 0; + for (var offset = 0; offset < bytes.Length; offset++) + { + if (bytes[offset] != 0x02) + continue; + + if (offset > textStart) + output.Append(table.Decode(bytes.AsSpan(textStart, offset - textStart))); + output.AppendLine(); + textStart = offset + 1; + } + + if (textStart < bytes.Length) + output.Append(table.Decode(bytes.AsSpan(textStart))); + return output.ToString(); + } + } + + private readonly byte[] _source; + private readonly bool _usesCdPadding; + + private Kh1Kmb(byte[] source, IReadOnlyList entries) + { + _source = source; + _usesCdPadding = source.Length > 0 && source[source.Length - 1] == 0xCD; + Entries = entries; + } + + public IReadOnlyList Entries { get; } + + public static Kh1Kmb Read(Stream stream, Kh1TextTable table) + { + ArgumentNullException.ThrowIfNull(stream); + ArgumentNullException.ThrowIfNull(table); + + using var buffer = new MemoryStream(); + stream.CopyTo(buffer); + var source = buffer.ToArray(); + if (source.Length < sizeof(uint)) + throw new InvalidDataException("The file is too small to be a KH1 KMB file."); + + var count = BitConverter.ToUInt32(source, 0); + if (count > int.MaxValue || count > source.Length - sizeof(uint)) + throw new InvalidDataException("The KMB entry count is invalid."); + + var entries = new List((int)count); + var offset = sizeof(uint); + for (var index = 0; index < count; index++) + { + var end = Array.IndexOf(source, (byte)0x00, offset); + if (end < 0) + throw new InvalidDataException($"KMB entry #{index + 1} has no 00 terminator."); + + entries.Add(new TextEntry( + index, + offset, + source.AsSpan(offset, end - offset).ToArray(), + table)); + offset = end + 1; + } + + return new Kh1Kmb(source, entries); + } + + public static Kh1Kmb Read(Stream stream) => Read(stream, Kh1TextTable.Default); + + public static Kh1Kmb Read(string fileName, Kh1TextTable table) + { + using var stream = File.OpenRead(fileName); + return Read(stream, table); + } + + public static Kh1Kmb Read(string fileName) => Read(fileName, Kh1TextTable.Default); + + public void Write(Stream stream) + { + ArgumentNullException.ThrowIfNull(stream); + + if (!Entries.Any(x => x.IsModified)) + { + stream.Write(_source); + return; + } + + stream.Write(_source, 0, sizeof(uint)); + foreach (var entry in Entries) + { + stream.Write(entry.Encode()); + stream.WriteByte(0x00); + } + + var minimumLength = stream.Position + (_usesCdPadding ? 1 : 0); + var targetLength = Math.Max(_source.Length, Align16(minimumLength)); + if (_usesCdPadding) + stream.WriteByte(0x00); + while (stream.Position < targetLength) + stream.WriteByte(_usesCdPadding ? (byte)0xCD : (byte)0x00); + } + + private static long Align16(long value) => (value + 0x0F) & ~0x0F; + } +} diff --git a/OpenKh.Kh1/Kh1MessageV361.cs b/OpenKh.Kh1/Kh1MessageV361.cs new file mode 100644 index 000000000..67b2b5c8d --- /dev/null +++ b/OpenKh.Kh1/Kh1MessageV361.cs @@ -0,0 +1,253 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; + +namespace OpenKh.Kh1 +{ + /// + /// Preserving editor for the remastered KH1 "Message v361" BINL format. + /// + public sealed class Kh1MessageV361 + { + private static readonly byte[] Magic = Encoding.ASCII.GetBytes("Message v361"); + + public sealed class TextEntry + { + private readonly Kh1TextTable _table; + private readonly byte[] _originalBytes; + + internal TextEntry(int index, int offset, byte[] bytes, Kh1TextTable table) + { + Index = index; + Offset = offset; + OriginalLength = bytes.Length; + _originalBytes = bytes; + _table = table; + Text = Decode(bytes, table); + OriginalText = Text; + } + + public int Index { get; } + public int Offset { get; } + public int OriginalLength { get; } + public string OriginalText { get; } + public string Text { get; set; } + public bool IsModified => !string.Equals(OriginalText, Text, StringComparison.Ordinal); + + internal byte[] Encode() + { + if (!IsModified) + return _originalBytes; + + using var output = new MemoryStream(); + var plainText = new StringBuilder(); + void FlushPlainText() + { + if (plainText.Length == 0) + return; + output.Write(_table.Encode(plainText.ToString())); + plainText.Clear(); + } + + for (var position = 0; position < Text.Length; position++) + { + if (Text[position] != '\r' && Text[position] != '\n') + { + plainText.Append(Text[position]); + continue; + } + + FlushPlainText(); + if (Text[position] == '\r' && position + 1 < Text.Length && Text[position + 1] == '\n') + position++; + output.WriteByte(0x02); + } + + FlushPlainText(); + var bytes = output.ToArray(); + if (bytes.Contains((byte)0x00)) + throw new InvalidDataException("Message v361 text cannot contain {eol}; byte 00 terminates the entry."); + return bytes; + } + + private static string Decode(byte[] bytes, Kh1TextTable table) + { + var output = new StringBuilder(); + var textStart = 0; + for (var offset = 0; offset < bytes.Length; offset++) + { + if (bytes[offset] != 0x02) + continue; + if (offset > textStart) + output.Append(table.Decode(bytes.AsSpan(textStart, offset - textStart))); + output.AppendLine(); + textStart = offset + 1; + } + if (textStart < bytes.Length) + output.Append(table.Decode(bytes.AsSpan(textStart))); + return output.ToString(); + } + } + + private readonly byte[] _source; + private readonly int _offsetTableOffset; + private readonly int _textOffset; + private readonly int _offsetCount; + private readonly bool _hasTrailingSentinel; + private readonly bool _usesCdPadding; + + private Kh1MessageV361( + byte[] source, + int offsetTableOffset, + int textOffset, + int offsetCount, + bool hasTrailingSentinel, + IReadOnlyList entries) + { + _source = source; + _offsetTableOffset = offsetTableOffset; + _textOffset = textOffset; + _offsetCount = offsetCount; + _hasTrailingSentinel = hasTrailingSentinel; + _usesCdPadding = source.Length > 0 && source[source.Length - 1] == 0xCD; + Entries = entries; + } + + public IReadOnlyList Entries { get; } + + public static bool IsValid(Stream stream) + { + ArgumentNullException.ThrowIfNull(stream); + if (!stream.CanSeek || stream.Length < 0x20) + return false; + var oldPosition = stream.Position; + Span magic = stackalloc byte[12]; + var read = stream.Read(magic); + stream.Position = oldPosition; + return read == magic.Length && magic.SequenceEqual(Magic); + } + + public static Kh1MessageV361 Read(Stream stream, Kh1TextTable table) + { + ArgumentNullException.ThrowIfNull(stream); + ArgumentNullException.ThrowIfNull(table); + if (!IsValid(stream)) + throw new InvalidDataException("The file is not a KH1 Message v361 BINL file."); + + stream.Position = 0; + using var buffer = new MemoryStream(); + stream.CopyTo(buffer); + var source = buffer.ToArray(); + var count = BitConverter.ToUInt32(source, 0x0C); + var offsetTableOffset = checked((int)BitConverter.ToUInt32(source, 0x10)); + var textOffset = checked((int)BitConverter.ToUInt32(source, 0x14)); + var offsetTableLength = checked((int)BitConverter.ToUInt32(source, 0x18)); + var textLength = checked((int)BitConverter.ToUInt32(source, 0x1C)); + + var offsetCount = offsetTableLength / sizeof(ushort); + if (count == 0 || count > int.MaxValue || + offsetTableOffset < 0x20 || + (offsetTableLength & 1) != 0 || + (offsetCount != count && offsetCount != count + 1) || + textOffset != offsetTableOffset + offsetTableLength || + textLength < 1 || textOffset + textLength > source.Length || + source[textOffset + textLength - 1] != 0x00) + throw new InvalidDataException("The Message v361 header is invalid."); + + var entries = new List((int)count); + var offsets = new ushort[offsetCount]; + for (var index = 0; index < offsetCount; index++) + { + offsets[index] = BitConverter.ToUInt16(source, offsetTableOffset + index * sizeof(ushort)); + if ((index == 0 && offsets[index] != 0) || + (index > 0 && offsets[index] < offsets[index - 1]) || + offsets[index] >= textLength) + throw new InvalidDataException("The Message v361 offset table is invalid."); + } + + var hasTrailingSentinel = offsetCount > count || + (offsets[count - 1] < textLength - 1 && + source[textOffset + textLength - 2] == 0x00); + var entriesEnd = textLength - (hasTrailingSentinel ? 1 : 0); + if (offsetCount > count && offsets[count] != entriesEnd) + throw new InvalidDataException("The Message v361 final offset is invalid."); + for (var index = 0; index < count; index++) + { + var start = offsets[index]; + var end = index + 1 < count ? offsets[index + 1] : entriesEnd; + var length = end - start; + if (length == 0 || source[textOffset + end - 1] != 0x00) + throw new InvalidDataException($"Message v361 entry #{index + 1} has no 00 terminator."); + entries.Add(new TextEntry( + index, + textOffset + start, + source.AsSpan(textOffset + start, length - 1).ToArray(), + table)); + } + + return new Kh1MessageV361( + source, + offsetTableOffset, + textOffset, + offsetCount, + hasTrailingSentinel, + entries); + } + + public static Kh1MessageV361 Read(Stream stream) => Read(stream, Kh1TextTable.Default); + + public static Kh1MessageV361 Read(string fileName, Kh1TextTable table) + { + using var stream = File.OpenRead(fileName); + return Read(stream, table); + } + + public static Kh1MessageV361 Read(string fileName) => Read(fileName, Kh1TextTable.Default); + + public void Write(Stream stream) + { + ArgumentNullException.ThrowIfNull(stream); + if (!Entries.Any(x => x.IsModified)) + { + stream.Write(_source); + return; + } + + var encoded = Entries.Select(x => x.Encode()).ToList(); + var entriesLength = encoded.Sum(x => x.Length + 1); + var textLength = entriesLength + (_hasTrailingSentinel ? 1 : 0); + if (textLength > ushort.MaxValue) + throw new InvalidDataException("The edited Message v361 text block exceeds 65535 bytes."); + + var header = _source.AsSpan(0, _textOffset).ToArray(); + BitConverter.GetBytes(textLength).CopyTo(header, 0x1C); + var currentOffset = 0; + for (var index = 0; index < encoded.Count; index++) + { + BitConverter.GetBytes((ushort)currentOffset) + .CopyTo(header, _offsetTableOffset + index * sizeof(ushort)); + currentOffset += encoded[index].Length + 1; + } + if (_offsetCount > encoded.Count) + BitConverter.GetBytes((ushort)currentOffset) + .CopyTo(header, _offsetTableOffset + encoded.Count * sizeof(ushort)); + + stream.Write(header); + foreach (var entry in encoded) + { + stream.Write(entry); + stream.WriteByte(0x00); + } + if (_hasTrailingSentinel) + stream.WriteByte(0x00); + + var targetLength = Math.Max(_source.Length, Align16(stream.Position)); + while (stream.Position < targetLength) + stream.WriteByte(_usesCdPadding ? (byte)0xCD : (byte)0x00); + } + + private static long Align16(long value) => (value + 0x0F) & ~0x0F; + } +} diff --git a/OpenKh.Kh1/Kh1TextBin.cs b/OpenKh.Kh1/Kh1TextBin.cs new file mode 100644 index 000000000..bec318020 --- /dev/null +++ b/OpenKh.Kh1/Kh1TextBin.cs @@ -0,0 +1,92 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +namespace OpenKh.Kh1 +{ + /// + /// Preserving editor for the null-terminated KH1 text tables stored in + /// remastered *.bin files. + /// + public sealed class Kh1TextBin + { + private readonly byte[] _source; + + private Kh1TextBin(byte[] source, IReadOnlyList entries) + { + _source = source; + Entries = entries; + } + + public IReadOnlyList Entries { get; } + + public static Kh1TextBin Read(Stream stream, Kh1TextTable table) + { + ArgumentNullException.ThrowIfNull(stream); + ArgumentNullException.ThrowIfNull(table); + + using var buffer = new MemoryStream(); + stream.CopyTo(buffer); + var source = buffer.ToArray(); + var entries = new List(); + var contentLength = source.Length; + while (contentLength > 0 && source[contentLength - 1] == 0xCD) + contentLength--; + + for (var offset = 0; offset < contentLength;) + { + var end = Array.IndexOf(source, (byte)0x00, offset, contentLength - offset); + if (end < 0) + throw new InvalidDataException("The KH1 BIN text table has an unterminated entry."); + + if (end > offset) + { + entries.Add(new Kh1Kmb.TextEntry( + entries.Count, + offset, + source.AsSpan(offset, end - offset).ToArray(), + table)); + } + offset = end + 1; + } + + if (entries.Count == 0) + throw new InvalidDataException("The KH1 BIN file contains no text entries."); + + return new Kh1TextBin(source, entries); + } + + public static Kh1TextBin Read(Stream stream) => Read(stream, Kh1TextTable.Default); + + public static Kh1TextBin Read(string fileName, Kh1TextTable table) + { + using var stream = File.OpenRead(fileName); + return Read(stream, table); + } + + public static Kh1TextBin Read(string fileName) => Read(fileName, Kh1TextTable.Default); + + public void Write(Stream stream) + { + ArgumentNullException.ThrowIfNull(stream); + if (!Entries.Any(x => x.IsModified)) + { + stream.Write(_source); + return; + } + + var cursor = 0; + foreach (var entry in Entries) + { + stream.Write(_source, cursor, entry.Offset - cursor); + stream.Write(entry.Encode()); + cursor = entry.Offset + entry.OriginalLength; + } + stream.Write(_source, cursor, _source.Length - cursor); + + while ((stream.Position & 0x0F) != 0) + stream.WriteByte(0x00); + } + } +} diff --git a/OpenKh.Kh1/Kh1TextTable.cs b/OpenKh.Kh1/Kh1TextTable.cs new file mode 100644 index 000000000..f0660c71c --- /dev/null +++ b/OpenKh.Kh1/Kh1TextTable.cs @@ -0,0 +1,241 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; + +namespace OpenKh.Kh1 +{ + /// + /// Reads the text-table format commonly used to describe the KH1 message encoding. + /// + public sealed class Kh1TextTable + { + private sealed class Entry + { + public byte[] Bytes { get; init; } + public string Text { get; init; } + } + + private readonly List _decodeEntries; + private readonly List _encodeEntries; + private readonly HashSet _ambiguousText; + + /// + /// Built-in international KH1 encoding used by the remastered BINL files. + /// Update when the game's character table changes. + /// + public static Kh1TextTable Default { get; } = CreateDefault(); + + private Kh1TextTable(IEnumerable entries) + { + var entryList = entries.ToList(); + _decodeEntries = entryList + .OrderByDescending(x => x.Bytes.Length) + .ToList(); + + _ambiguousText = entryList + .GroupBy(x => x.Text, StringComparer.Ordinal) + .Where(x => x.Count() > 1) + .Select(x => x.Key) + .ToHashSet(StringComparer.Ordinal); + + _encodeEntries = entryList + .Where(x => !_ambiguousText.Contains(x.Text)) + .OrderByDescending(x => x.Text.Length) + .ToList(); + } + + public static Kh1TextTable Read(Stream stream) + { + ArgumentNullException.ThrowIfNull(stream); + + var entries = new List(); + var byteKeys = new HashSet(StringComparer.Ordinal); + using var reader = new StreamReader(stream, new UTF8Encoding(false, true), true, 1024, true); + var lineNumber = 0; + while (reader.ReadLine() is string line) + { + lineNumber++; + if (string.IsNullOrWhiteSpace(line) || line.TrimStart().StartsWith('#')) + continue; + + var separator = line.IndexOf('='); + if (separator <= 0) + throw new InvalidDataException($"Invalid TBL entry at line {lineNumber}. Expected HEX=text."); + + var hex = line.Substring(0, separator).Trim().Replace(" ", string.Empty); + var text = line.Substring(separator + 1); + if (hex.Length == 0 || (hex.Length & 1) != 0 || !hex.All(Uri.IsHexDigit)) + throw new InvalidDataException($"Invalid hexadecimal key at TBL line {lineNumber}."); + if (text.Length == 0) + throw new InvalidDataException($"Empty text value at TBL line {lineNumber} is not supported."); + + var bytes = Convert.FromHexString(hex); + var byteKey = Convert.ToHexString(bytes); + if (!byteKeys.Add(byteKey)) + throw new InvalidDataException($"Duplicate byte key {byteKey} at TBL line {lineNumber}."); + + entries.Add(new Entry { Bytes = bytes, Text = text }); + } + + if (entries.Count == 0) + throw new InvalidDataException("The TBL file contains no entries."); + + return new Kh1TextTable(entries); + } + + public static Kh1TextTable Read(string fileName) + { + using var stream = File.OpenRead(fileName); + return Read(stream); + } + + private static Kh1TextTable CreateDefault() + { + var entries = new List(); + void Add(byte value, string text) => + entries.Add(new Entry { Bytes = new[] { value }, Text = text }); + + Add(0x00, "{eol}"); + Add(0x01, " "); + Add(0x02, "{lf}"); + Add(0x0F, "{ctrl:0F}"); + Add(0x20, "—"); + + for (var value = 0; value < 10; value++) + Add((byte)(0x21 + value), value.ToString(CultureInfo.InvariantCulture)); + for (var value = 0; value < 26; value++) + { + Add((byte)(0x2B + value), ((char)('A' + value)).ToString()); + Add((byte)(0x45 + value), ((char)('a' + value)).ToString()); + } + + foreach (var (value, text) in new (byte, string)[] + { + (0x5F, "!"), (0x60, "?"), (0x61, "&"), (0x62, "%"), + (0x63, "+"), (0x64, "{-}"), (0x65, "{mX}"), (0x66, "/"), + (0x67, "*"), (0x68, "."), (0x69, ","), (0x6A, "・"), + (0x6B, ":"), (0x6C, ";"), (0x6D, "…"), (0x6E, "-"), + (0x6F, "ー"), (0x70, "~"), (0x71, "'"), (0x72, "\""), + (0x73, "{゛b}"), (0x74, "("), (0x75, ")"), (0x76, "["), + (0x77, "]"), (0x78, "<"), (0x79, ">"), (0x7A, "★"), + (0x7B, "☆"), (0x7C, "↑"), (0x7D, "↓"), (0x7E, "→"), + (0x7F, "←"), (0x80, "●"), (0x81, "■"), + (0x82, "{iPotion}"), (0x83, "{iTent}"), (0x84, "{iGem}"), + (0x85, "{iAbility}"), (0x86, "{iKey}"), (0x87, "{iStaff}"), + (0x88, "{iShield}"), (0x89, "{iRing}"), (0x8A, "{iHat}"), + (0x8B, "{iMickey}"), (0x8C, "○"), (0x8D, "×"), + (0x8E, "△"), (0x8F, "□"), (0x90, "▲"), (0x91, "▼"), + (0x92, "►"), (0x93, "◄"), (0xA9, "®"), + (0xC4, "{III}"), (0xC5, "{VII}"), (0xC6, "{VIII}"), + (0xC7, "{X}"), (0xC8, "Œ"), (0xC9, "œ"), + (0xCA, "¡"), (0xCB, "¿"), (0xCC, "À"), (0xCD, "Á"), + (0xCE, "Â"), (0xCF, "Ä"), (0xD0, "Ç"), (0xD1, "È"), + (0xD2, "É"), (0xD3, "Ê"), (0xD4, "Ë"), (0xD5, "Ì"), + (0xD6, "Í"), (0xD7, "Î"), (0xD8, "Ï"), (0xD9, "Ñ"), + (0xDA, "Ò"), (0xDB, "Ó"), (0xDC, "Ô"), (0xDD, "Ö"), + (0xDE, "Ù"), (0xDF, "Ú"), (0xE0, "Û"), (0xE1, "Ü"), + (0xE2, "ß"), (0xE3, "à"), (0xE4, "á"), (0xE5, "â"), + (0xE6, "ä"), (0xE7, "ç"), (0xE8, "è"), (0xE9, "é"), + (0xEA, "ê"), (0xEB, "ë"), (0xEC, "ì"), (0xED, "í"), + (0xEE, "î"), (0xEF, "ï"), (0xF0, "ñ"), (0xF1, "ò"), + (0xF2, "ó"), (0xF3, "ô"), (0xF4, "ö"), (0xF5, "ù"), + (0xF6, "ú"), (0xF7, "û"), (0xF8, "ü"), (0xF9, "°"), + (0xFA, "{---}"), (0xFB, "》"), (0xFC, "《"), + }) + Add(value, text); + + return new Kh1TextTable(entries); + } + + public string Decode(ReadOnlySpan data) + { + var result = new StringBuilder(); + for (var offset = 0; offset < data.Length;) + { + var entry = FindDecodeEntry(data.Slice(offset)); + if (entry == null) + { + result.AppendFormat(CultureInfo.InvariantCulture, "{{0x{0:X2}}}", data[offset]); + offset++; + } + else + { + if (_ambiguousText.Contains(entry.Text)) + result.Append("{0x").Append(Convert.ToHexString(entry.Bytes)).Append('}'); + else + result.Append(entry.Text); + offset += entry.Bytes.Length; + } + } + + return result.ToString(); + } + + public byte[] Encode(string text) + { + ArgumentNullException.ThrowIfNull(text); + + using var output = new MemoryStream(); + for (var offset = 0; offset < text.Length;) + { + if (TryReadRawByteToken(text, offset, out var rawBytes, out var tokenLength)) + { + output.Write(rawBytes); + offset += tokenLength; + continue; + } + + var entry = _encodeEntries.FirstOrDefault(x => + text.AsSpan(offset).StartsWith(x.Text.AsSpan(), StringComparison.Ordinal)); + if (entry == null) + { + var display = char.ConvertToUtf32(text, offset); + throw new InvalidDataException( + $"Text at position {offset + 1} cannot be encoded by the selected TBL (U+{display:X4})."); + } + + output.Write(entry.Bytes); + offset += entry.Text.Length; + } + + return output.ToArray(); + } + + private Entry FindDecodeEntry(ReadOnlySpan data) + { + foreach (var entry in _decodeEntries) + { + if (data.StartsWith(entry.Bytes)) + return entry; + } + return null; + } + + private static bool TryReadRawByteToken( + string text, + int offset, + out byte[] bytes, + out int tokenLength) + { + bytes = null; + tokenLength = 0; + if (!text.AsSpan(offset).StartsWith("{0x".AsSpan(), StringComparison.OrdinalIgnoreCase)) + return false; + + var closeBrace = text.IndexOf('}', offset + 3); + if (closeBrace < 0) + return false; + + var hex = text.Substring(offset + 3, closeBrace - offset - 3); + if (hex.Length == 0 || (hex.Length & 1) != 0 || !hex.All(Uri.IsHexDigit)) + return false; + + bytes = Convert.FromHexString(hex); + tokenLength = closeBrace - offset + 1; + return true; + } + } +} diff --git a/OpenKh.Tests/Kh1/Kh1TextTableTests.cs b/OpenKh.Tests/Kh1/Kh1TextTableTests.cs new file mode 100644 index 000000000..358b4a456 --- /dev/null +++ b/OpenKh.Tests/Kh1/Kh1TextTableTests.cs @@ -0,0 +1,362 @@ +using OpenKh.Kh1; +using System; +using System.IO; +using System.Text; +using Xunit; + +namespace OpenKh.Tests.Kh1 +{ + public class Kh1TextTableTests + { + private const string TableText = + "00={eol}\n" + + "01= \n" + + "02={lf}\n" + + "2B=A\n" + + "2C=B\n" + + "2D=C\n" + + "45=a\n" + + "68=.\n" + + "94={icon}\n" + + "95={icon}\n"; + + [Fact] + public void SpaceIs01AndZeroIsEol() + { + var table = ReadTable(); + + Assert.Equal(new byte[] { 0x01 }, table.Encode(" ")); + Assert.Equal(new byte[] { 0x00 }, table.Encode("{eol}")); + Assert.Equal(" ", table.Decode(new byte[] { 0x01 })); + Assert.Equal("{eol}", table.Decode(new byte[] { 0x00 })); + } + + [Fact] + public void DefaultTableContainsTheInternationalKh1Encoding() + { + var table = Kh1TextTable.Default; + + Assert.Equal(new byte[] { 0x01 }, table.Encode(" ")); + Assert.Equal(new byte[] { 0xCB, 0x3B, 0x59, 0xE9, 0x60 }, table.Encode("¿Qué?")); + Assert.Equal(new byte[] { 0x0F }, table.Encode("{ctrl:0F}")); + Assert.Equal("{ctrl:0F}", table.Decode(new byte[] { 0x0F })); + Assert.Equal("ÁÉÍÓÚ áéíóú ñ", table.Decode(new byte[] + { + 0xCD, 0xD2, 0xD6, 0xDB, 0xDF, 0x01, + 0xE4, 0xE9, 0xED, 0xF2, 0xF6, 0x01, 0xF0, + })); + } + + [Fact] + public void DefaultTableRoundTripsEveryByte() + { + var table = Kh1TextTable.Default; + + for (var value = 0; value <= byte.MaxValue; value++) + { + var original = new[] { (byte)value }; + Assert.Equal(original, table.Encode(table.Decode(original))); + } + } + + [Fact] + public void AmbiguousTextUsesRawTokensForLosslessRoundTrip() + { + var table = ReadTable(); + + var decoded = table.Decode(new byte[] { 0x94, 0x95 }); + + Assert.Equal("{0x94}{0x95}", decoded); + Assert.Equal(new byte[] { 0x94, 0x95 }, table.Encode(decoded)); + } + + [Fact] + public void BinlRoundTripsCommandsPaddingAndFalseRecordMarkers() + { + var table = ReadTable(); + var original = CreateBinl(); + using var input = new MemoryStream(original); + + var binl = Kh1Binl.Read(input, table); + + Assert.Equal("SP", binl.Language); + Assert.Equal(2, binl.Entries.Count); + Assert.Equal("A B\r\nC.", binl.Entries[0].Text); + Assert.Equal("a{cmd:0C 04}A.", binl.Entries[1].Text); + + using var output = new MemoryStream(); + binl.Write(output); + Assert.Equal(original, output.ToArray()); + } + + [Fact] + public void BinlCanGrowAndBeReadAgain() + { + var table = ReadTable(); + using var input = new MemoryStream(CreateBinl()); + var binl = Kh1Binl.Read(input, table); + binl.Entries[0].Text = "A B A\nC."; + + using var output = new MemoryStream(); + binl.Write(output); + Assert.Equal(0, output.Length % 16); + + output.Position = 0; + var reopened = Kh1Binl.Read(output, table); + Assert.Equal("A B A\r\nC.", reopened.Entries[0].Text); + Assert.Equal(binl.Entries.Count, reopened.Entries.Count); + } + + [Fact] + public void BinlIgnoresPresentationRecordsWithoutATextTerminator() + { + var table = ReadTable(); + var original = CreateBinl(); + using var stream = new MemoryStream(); + stream.Write(original, 0, 11); + stream.Write(new byte[] + { + 0x0A, 0x00, 0x00, 0x00, + 0x07, 0x0C, 0x00, + 0x0B, 0x00, 0x04, 0x00, + 0x81, 0x81, 0x81, + }); + while ((stream.Length & 0x0F) != 0) + stream.WriteByte(0xCD); + stream.Position = 0; + + var binl = Kh1Binl.Read(stream, table); + Assert.Empty(binl.Entries); + + using var output = new MemoryStream(); + binl.Write(output); + Assert.Equal(stream.ToArray(), output.ToArray()); + } + + [Fact] + public void KmbRoundTripsEntriesAndZeroPadding() + { + var table = ReadTable(); + var original = CreateKmb(0x00); + using var input = new MemoryStream(original); + + var kmb = Kh1Kmb.Read(input, table); + + Assert.Equal(3, kmb.Entries.Count); + Assert.Equal("A B", kmb.Entries[0].Text); + Assert.Equal("a\r\nA", kmb.Entries[1].Text); + Assert.Equal("{0x0F}", kmb.Entries[2].Text); + + using var output = new MemoryStream(); + kmb.Write(output); + Assert.Equal(original, output.ToArray()); + } + + [Theory] + [InlineData(0x00)] + [InlineData(0xCD)] + public void KmbCanGrowAndBeReadAgain(byte padding) + { + var table = ReadTable(); + using var input = new MemoryStream(CreateKmb(padding)); + var kmb = Kh1Kmb.Read(input, table); + kmb.Entries[0].Text = "A B A B A B A B A B"; + + using var output = new MemoryStream(); + kmb.Write(output); + Assert.Equal(0, output.Length % 16); + + output.Position = 0; + var reopened = Kh1Kmb.Read(output, table); + Assert.Equal("A B A B A B A B A B", reopened.Entries[0].Text); + Assert.Equal(3, reopened.Entries.Count); + } + + [Fact] + public void MessageV361RoundTripsAndCanGrow() + { + var table = ReadTable(); + var original = CreateMessageV361(); + using var input = new MemoryStream(original); + var message = Kh1MessageV361.Read(input, table); + + Assert.Equal(2, message.Entries.Count); + Assert.Equal("A", message.Entries[0].Text); + Assert.Equal("a\r\nA", message.Entries[1].Text); + + using var unchanged = new MemoryStream(); + message.Write(unchanged); + Assert.Equal(original, unchanged.ToArray()); + + message.Entries[0].Text = "A B A B A B A B A B"; + using var edited = new MemoryStream(); + message.Write(edited); + Assert.Equal(0, edited.Length % 16); + edited.Position = 0; + var reopened = Kh1MessageV361.Read(edited, table); + Assert.Equal("A B A B A B A B A B", reopened.Entries[0].Text); + Assert.Equal("a\r\nA", reopened.Entries[1].Text); + } + + [Fact] + public void TextBinRoundTripsAndCanGrow() + { + var table = ReadTable(); + var original = CreateTextBin(); + using var input = new MemoryStream(original); + var textBin = Kh1TextBin.Read(input, table); + + Assert.Equal(2, textBin.Entries.Count); + Assert.Equal("A B", textBin.Entries[0].Text); + Assert.Equal("a\r\nA", textBin.Entries[1].Text); + + using var unchanged = new MemoryStream(); + textBin.Write(unchanged); + Assert.Equal(original, unchanged.ToArray()); + + textBin.Entries[0].Text = "A B A B A B"; + using var edited = new MemoryStream(); + textBin.Write(edited); + Assert.Equal(0, edited.Length % 16); + edited.Position = 0; + var reopened = Kh1TextBin.Read(edited, table); + Assert.Equal("A B A B A B", reopened.Entries[0].Text); + Assert.Equal("a\r\nA", reopened.Entries[1].Text); + } + + [Fact] + public void EventMessageRoundTripsGrowsAndRelocatesOffsets() + { + var table = ReadTable(); + var original = CreateEventMessage(); + using var input = new MemoryStream(original); + var message = Kh1EventMessage.Read(input, table); + + Assert.Equal(2, message.Entries.Count); + Assert.Equal("A B", message.Entries[0].Text); + Assert.Equal("a A", message.Entries[1].Text); + + using var unchanged = new MemoryStream(); + message.Write(unchanged); + Assert.Equal(original, unchanged.ToArray()); + + var oldFirstBoundary = BitConverter.ToUInt32(original, 0x10); + var oldSecondBoundary = BitConverter.ToUInt32(original, 0x14); + message.Entries[0].Text = "A B A B A B A B A B A B A B A B"; + using var edited = new MemoryStream(); + message.Write(edited); + var editedBytes = edited.ToArray(); + var delta = BitConverter.ToUInt32(editedBytes, 0x10) - oldFirstBoundary; + Assert.True(delta > 0); + Assert.Equal(oldSecondBoundary + delta, BitConverter.ToUInt32(editedBytes, 0x14)); + + edited.Position = 0; + var reopened = Kh1EventMessage.Read(edited, table); + Assert.Equal(message.Entries[0].Text, reopened.Entries[0].Text); + Assert.Equal("a A", reopened.Entries[1].Text); + } + + private static Kh1TextTable ReadTable() + { + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(TableText)); + return Kh1TextTable.Read(stream); + } + + private static byte[] CreateBinl() + { + using var stream = new MemoryStream(); + stream.Write(Encoding.ASCII.GetBytes("EvMsgSP")); + stream.Write(new byte[] { 0x02, 0x00, 0x00, 0x00, 0x04 }); + + stream.Write(new byte[] + { + 0x0A, 0x00, 0x00, 0x00, + 0x07, 0x0C, 0x00, + 0x2B, 0x01, 0x2C, 0x02, 0x2D, 0x68, + 0x05, 0x6E, 0x00, + 0x0D, 0x01, 0x0A, 0x00, + 0x00, + }); + stream.Write(new byte[] + { + 0x0A, 0x00, 0x00, 0x00, + 0x07, 0x0C, 0x00, + 0x45, 0x0C, 0x04, 0x2B, 0x68, + 0x06, 0x3C, 0x00, 0x00, 0x08, + }); + while ((stream.Length & 0x0F) != 0) + stream.WriteByte(0xCD); + return stream.ToArray(); + } + + private static byte[] CreateKmb(byte padding) + { + using var stream = new MemoryStream(); + stream.Write(BitConverter.GetBytes(3)); + stream.Write(new byte[] { 0x2B, 0x01, 0x2C, 0x00 }); + stream.Write(new byte[] { 0x45, 0x02, 0x2B, 0x00 }); + stream.Write(new byte[] { 0x0F, 0x00 }); + while ((stream.Length & 0x0F) != 0) + stream.WriteByte(padding); + return stream.ToArray(); + } + + private static byte[] CreateMessageV361() + { + using var stream = new MemoryStream(); + stream.Write(Encoding.ASCII.GetBytes("Message v361")); + stream.Write(BitConverter.GetBytes(2)); + stream.Write(BitConverter.GetBytes(0x20)); + stream.Write(BitConverter.GetBytes(0x26)); + stream.Write(BitConverter.GetBytes(0x06)); + stream.Write(BitConverter.GetBytes(0x07)); + stream.Write(new byte[] { 0x00, 0x00, 0x02, 0x00, 0x06, 0x00 }); + stream.Write(new byte[] { 0x2B, 0x00, 0x45, 0x02, 0x2B, 0x00, 0x00 }); + while ((stream.Length & 0x0F) != 0) + stream.WriteByte(0xCD); + return stream.ToArray(); + } + + private static byte[] CreateTextBin() + { + using var stream = new MemoryStream(); + stream.Write(new byte[] { 0x2B, 0x01, 0x2C, 0x00 }); + stream.Write(new byte[] { 0x45, 0x02, 0x2B, 0x00 }); + while ((stream.Length & 0x0F) != 0) + stream.WriteByte(0x00); + return stream.ToArray(); + } + + private static byte[] CreateEventMessage() + { + using var section = new MemoryStream(); + section.Write(BitConverter.GetBytes(2)); + section.Write(new byte[] + { + 0x0A, 0x00, 0x00, 0x00, + 0x07, 0x0C, 0x00, + 0x2B, 0x01, 0x2C, + 0x05, 0x10, 0x00, + 0x0A, 0x00, 0x00, 0x00, + 0x07, 0x0C, 0x00, + 0x45, 0x01, 0x2B, + 0x06, 0x20, 0x00, + }); + while ((section.Length & 0x0F) != 0) + section.WriteByte(0x00); + + const int sectionStart = 0x18; + var sectionEnd = sectionStart + checked((int)section.Length); + var secondBoundary = sectionEnd + 0x10; + using var stream = new MemoryStream(); + stream.Write(new byte[0x0C]); + stream.Write(BitConverter.GetBytes(sectionStart)); + stream.Write(BitConverter.GetBytes(sectionEnd)); + stream.Write(BitConverter.GetBytes(secondBoundary)); + section.Position = 0; + section.CopyTo(stream); + stream.Write(new byte[0x20]); + return stream.ToArray(); + } + } +} diff --git a/OpenKh.Tools.Kh1TextEditor/App.xaml b/OpenKh.Tools.Kh1TextEditor/App.xaml new file mode 100644 index 000000000..7f08851b9 --- /dev/null +++ b/OpenKh.Tools.Kh1TextEditor/App.xaml @@ -0,0 +1,7 @@ + + + + diff --git a/OpenKh.Tools.Kh1TextEditor/App.xaml.cs b/OpenKh.Tools.Kh1TextEditor/App.xaml.cs new file mode 100644 index 000000000..c4d9050fe --- /dev/null +++ b/OpenKh.Tools.Kh1TextEditor/App.xaml.cs @@ -0,0 +1,8 @@ +using System.Windows; + +namespace OpenKh.Tools.Kh1TextEditor +{ + public partial class App : Application + { + } +} diff --git a/OpenKh.Tools.Kh1TextEditor/Models/LoadedDocument.cs b/OpenKh.Tools.Kh1TextEditor/Models/LoadedDocument.cs new file mode 100644 index 000000000..3a065804a --- /dev/null +++ b/OpenKh.Tools.Kh1TextEditor/Models/LoadedDocument.cs @@ -0,0 +1,181 @@ +using OpenKh.Kh1; +using System; +using System.Collections.Generic; +using System.IO; + +namespace OpenKh.Tools.Kh1TextEditor.Models +{ + internal sealed class LoadedDocument + { + private readonly Action _write; + + private LoadedDocument( + string fileName, + string relativePath, + string category, + string format, + Action write) + { + FileName = fileName; + RelativePath = relativePath; + Category = category; + Format = format; + _write = write; + } + + public string FileName { get; } + public string RelativePath { get; } + public string Category { get; } + public string Format { get; } + public List Entries { get; } = new(); + + public static LoadedDocument Read(string fileName, string rootPath) + { + var extension = Path.GetExtension(fileName); + var relativePath = Directory.Exists(rootPath) + ? Path.GetRelativePath(rootPath, fileName) + : Path.GetFileName(fileName); + + if (string.Equals(extension, ".binl", StringComparison.OrdinalIgnoreCase)) + { + using var stream = File.OpenRead(fileName); + if (Kh1Binl.IsValid(stream)) + { + var binl = Kh1Binl.Read(stream); + var document = new LoadedDocument(fileName, relativePath, "BINL", "BINL", binl.Write); + foreach (var item in binl.Entries) + { + var entry = item; + document.Entries.Add(new TextOccurrence( + document, + entry.Index, + entry.Offset, + () => entry.Text, + value => entry.Text = value)); + } + return document; + } + + if (Kh1MessageV361.IsValid(stream)) + { + var message = Kh1MessageV361.Read(stream); + var document = new LoadedDocument(fileName, relativePath, "BINL", "BINL-v361", message.Write); + foreach (var item in message.Entries) + { + var entry = item; + document.Entries.Add(new TextOccurrence( + document, + entry.Index, + entry.Offset, + () => entry.Text, + value => entry.Text = value)); + } + return document; + } + + return null; + } + + if (string.Equals(extension, ".kmb", StringComparison.OrdinalIgnoreCase)) + { + var kmb = Kh1Kmb.Read(fileName); + var document = new LoadedDocument(fileName, relativePath, "KMB", "KMB", kmb.Write); + foreach (var item in kmb.Entries) + { + var entry = item; + document.Entries.Add(new TextOccurrence( + document, + entry.Index, + entry.Offset, + () => entry.Text, + value => entry.Text = value)); + } + return document; + } + + if (string.Equals(extension, ".bin", StringComparison.OrdinalIgnoreCase)) + { + if (!IsTextBinFile(fileName)) + return null; + + var textBin = Kh1TextBin.Read(fileName); + var document = new LoadedDocument(fileName, relativePath, "BIN", "BIN", textBin.Write); + foreach (var item in textBin.Entries) + { + var entry = item; + document.Entries.Add(new TextOccurrence( + document, + entry.Index, + entry.Offset, + () => entry.Text, + value => entry.Text = value)); + } + return document; + } + + if (string.Equals(extension, ".ev", StringComparison.OrdinalIgnoreCase) || + string.Equals(extension, ".evdl", StringComparison.OrdinalIgnoreCase)) + { + using var stream = File.OpenRead(fileName); + if (!Kh1EventMessage.IsValid(stream)) + return null; + var eventMessage = Kh1EventMessage.Read(stream); + if (eventMessage.Entries.Count == 0) + return null; + + var category = extension.Equals(".ev", StringComparison.OrdinalIgnoreCase) ? "EV" : "EVDL"; + var document = new LoadedDocument(fileName, relativePath, category, category, eventMessage.Write); + foreach (var item in eventMessage.Entries) + { + var entry = item; + document.Entries.Add(new TextOccurrence( + document, + entry.Index, + entry.Offset, + () => entry.Text, + value => entry.Text = value)); + } + return document; + } + + throw new InvalidDataException("The file is not a supported KH1 remastered text file."); + } + + public static bool IsTextBinFile(string fileName) + { + var name = Path.GetFileName(fileName); + return name.Contains("AbilityHelp", StringComparison.OrdinalIgnoreCase) || + name.Contains("AbilityName", StringComparison.OrdinalIgnoreCase) || + name.Contains("ItemHelp", StringComparison.OrdinalIgnoreCase) || + name.EndsWith("Word.bin", StringComparison.OrdinalIgnoreCase) || + name.EndsWith("SASAMSG.BIN", StringComparison.OrdinalIgnoreCase) || + name.Contains("ChallengeMsg", StringComparison.OrdinalIgnoreCase); + } + + public byte[] BuildFile() + { + using var output = new MemoryStream(); + _write(output); + return output.ToArray(); + } + + public static void WriteFile(string fileName, byte[] data) + { + var directory = Path.GetDirectoryName(Path.GetFullPath(fileName)); + Directory.CreateDirectory(directory); + var temporaryFile = Path.Combine( + directory, + $".{Path.GetFileName(fileName)}.{Guid.NewGuid():N}.tmp"); + try + { + File.WriteAllBytes(temporaryFile, data); + File.Move(temporaryFile, fileName, true); + } + finally + { + if (File.Exists(temporaryFile)) + File.Delete(temporaryFile); + } + } + } +} diff --git a/OpenKh.Tools.Kh1TextEditor/Models/TextOccurrence.cs b/OpenKh.Tools.Kh1TextEditor/Models/TextOccurrence.cs new file mode 100644 index 000000000..f7fc1a609 --- /dev/null +++ b/OpenKh.Tools.Kh1TextEditor/Models/TextOccurrence.cs @@ -0,0 +1,32 @@ +using System; + +namespace OpenKh.Tools.Kh1TextEditor.Models +{ + internal sealed class TextOccurrence + { + private readonly Func _getText; + private readonly Action _setText; + + public TextOccurrence( + LoadedDocument document, + int index, + int offset, + Func getText, + Action setText) + { + Document = document; + Index = index; + Offset = offset; + _getText = getText; + _setText = setText; + } + + public LoadedDocument Document { get; } + public int Index { get; } + public int Offset { get; } + public string Text => _getText(); + public string Location => $"{Document.RelativePath} #{Index + 1:D3} 0x{Offset:X6}"; + + public void SetText(string text) => _setText(text); + } +} diff --git a/OpenKh.Tools.Kh1TextEditor/OpenKh.Tools.Kh1TextEditor.csproj b/OpenKh.Tools.Kh1TextEditor/OpenKh.Tools.Kh1TextEditor.csproj new file mode 100644 index 000000000..d160c2ef6 --- /dev/null +++ b/OpenKh.Tools.Kh1TextEditor/OpenKh.Tools.Kh1TextEditor.csproj @@ -0,0 +1,20 @@ + + + + WinExe + net8.0-windows + latest + true + KH1 Text editor + KH1 Text editor - OpenKH + OpenKH contributors + OpenKH + Copyright (C) OpenKH + https://github.com/OpenKH/OpenKh + + + + + + + diff --git a/OpenKh.Tools.Kh1TextEditor/ViewModels/TextEntryViewModel.cs b/OpenKh.Tools.Kh1TextEditor/ViewModels/TextEntryViewModel.cs new file mode 100644 index 000000000..135296f84 --- /dev/null +++ b/OpenKh.Tools.Kh1TextEditor/ViewModels/TextEntryViewModel.cs @@ -0,0 +1,82 @@ +using OpenKh.Tools.Kh1TextEditor.Models; +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Linq; +using System.Runtime.CompilerServices; + +namespace OpenKh.Tools.Kh1TextEditor.ViewModels +{ + public sealed class TextEntryViewModel : INotifyPropertyChanged + { + private readonly IReadOnlyList _occurrences; + private string _locations; + private string _originalText; + private string _text; + + internal TextEntryViewModel(int index, IGrouping group) + { + Index = index; + _occurrences = group.ToList(); + _text = group.Key; + _originalText = group.Key; + + } + + public int Index { get; } + public string Number => $"#{Index + 1:D4}"; + public string OriginalText => _originalText; + public bool IsModified => !string.Equals(_originalText, Text, StringComparison.Ordinal); + public string Preview => Text.Replace("\r", string.Empty).Replace("\n", " ↵ "); + public int OccurrenceCount => _occurrences.Count; + public int FileCount => _occurrences.Select(x => x.Document).Distinct().Count(); + public string Occurrences => OccurrenceCount == 1 + ? "1 occurrence" + : $"{OccurrenceCount} occurrences in {FileCount} file(s)"; + public string Formats => string.Join("/", _occurrences.Select(x => x.Document.Format).Distinct()); + public string Locations => _locations ??= BuildLocations(); + internal IEnumerable Documents => _occurrences.Select(x => x.Document).Distinct(); + internal bool ContainsLocation(string search) => _occurrences.Any(x => + x.Document.RelativePath.Contains(search, StringComparison.CurrentCultureIgnoreCase)); + + public string Text + { + get => _text; + set + { + if (string.Equals(_text, value, StringComparison.Ordinal)) + return; + _text = value; + OnPropertyChanged(); + OnPropertyChanged(nameof(Preview)); + OnPropertyChanged(nameof(IsModified)); + } + } + + public event PropertyChangedEventHandler PropertyChanged; + + internal void Apply() + { + foreach (var occurrence in _occurrences) + occurrence.SetText(Text); + } + + internal void AcceptChanges() + { + _originalText = Text; + OnPropertyChanged(nameof(OriginalText)); + OnPropertyChanged(nameof(IsModified)); + } + + private string BuildLocations() + { + var locations = _occurrences.Take(200).Select(x => x.Location).ToList(); + if (_occurrences.Count > locations.Count) + locations.Add($"... and {_occurrences.Count - locations.Count} more occurrences"); + return string.Join(Environment.NewLine, locations); + } + + private void OnPropertyChanged([CallerMemberName] string propertyName = null) => + PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(propertyName)); + } +} diff --git a/OpenKh.Tools.Kh1TextEditor/ViewModels/TextFormatTabViewModel.cs b/OpenKh.Tools.Kh1TextEditor/ViewModels/TextFormatTabViewModel.cs new file mode 100644 index 000000000..c2ec4e684 --- /dev/null +++ b/OpenKh.Tools.Kh1TextEditor/ViewModels/TextFormatTabViewModel.cs @@ -0,0 +1,67 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Windows.Data; + +namespace OpenKh.Tools.Kh1TextEditor.ViewModels +{ + public sealed class TextFormatTabViewModel : INotifyPropertyChanged + { + private TextEntryViewModel _selectedEntry; + private string _searchText; + + public TextFormatTabViewModel(string format, List entries) + { + Format = format; + Entries = entries; + EntriesView = CollectionViewSource.GetDefaultView(Entries); + EntriesView.Filter = FilterEntry; + SelectedEntry = Entries.FirstOrDefault(); + } + + public string Format { get; } + public string Header => $"{Format} ({Entries.Count:N0})"; + public List Entries { get; } + public ICollectionView EntriesView { get; } + + public TextEntryViewModel SelectedEntry + { + get => _selectedEntry; + set + { + _selectedEntry = value; + OnPropertyChanged(); + } + } + + public string SearchText + { + get => _searchText; + set + { + if (string.Equals(_searchText, value, StringComparison.Ordinal)) + return; + _searchText = value; + OnPropertyChanged(); + EntriesView.Refresh(); + } + } + + public event PropertyChangedEventHandler PropertyChanged; + + private bool FilterEntry(object item) + { + if (item is not TextEntryViewModel entry || string.IsNullOrWhiteSpace(SearchText)) + return true; + return entry.Text.Contains(SearchText, StringComparison.CurrentCultureIgnoreCase) || + entry.Number.Contains(SearchText, StringComparison.OrdinalIgnoreCase) || + entry.Formats.Contains(SearchText, StringComparison.OrdinalIgnoreCase) || + entry.ContainsLocation(SearchText); + } + + private void OnPropertyChanged([CallerMemberName] string propertyName = null) => + PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(propertyName)); + } +} diff --git a/OpenKh.Tools.Kh1TextEditor/Views/LanguageSelectionWindow.xaml b/OpenKh.Tools.Kh1TextEditor/Views/LanguageSelectionWindow.xaml new file mode 100644 index 000000000..58d0ef18c --- /dev/null +++ b/OpenKh.Tools.Kh1TextEditor/Views/LanguageSelectionWindow.xaml @@ -0,0 +1,22 @@ + + + + + + + + + + +