From 1a54b0585d562da42663c0ac401153f7f7ddc203 Mon Sep 17 00:00:00 2001 From: Michal Lazo Date: Wed, 1 Jul 2026 12:43:37 +0200 Subject: [PATCH] ansi code page --- DFMStabilizerCLI.pas | 76 ++++++++++++++++++++++++++++++--------- DFMTextStabilizerCore.pas | 58 +++++++++++++++++++++++++----- 2 files changed, 109 insertions(+), 25 deletions(-) diff --git a/DFMStabilizerCLI.pas b/DFMStabilizerCLI.pas index 8c6d6af..1cf8a58 100644 --- a/DFMStabilizerCLI.pas +++ b/DFMStabilizerCLI.pas @@ -4,9 +4,14 @@ Command-line interface for the DFM stabilizer tool. Argument syntax: - DFMStabilizerTool [-s] [...] + DFMStabilizerTool [-s] [-a:] [...] -s Recurse into subdirectories when expanding wildcard patterns. + -a: Treat text DFMs with no UTF-8 BOM as being encoded in the + given ANSI code page (e.g. -a:1250) instead of assuming + UTF-8 without BOM. Use this for legacy DFMs that fail with + "No mapping for the Unicode character exists in the target + multi-byte code page". file Exact path to a DFM file. pattern Wildcard pattern: *.dfm, path\*.dfm, etc. @listfile Text file listing one path or pattern per line. @@ -32,22 +37,24 @@ implementation TDFMProcessor = class private FRecursive : Boolean; + FAnsiCodePage: Integer; FSuccessCount: Integer; FFailCount : Integer; procedure ProcessFile(const AFileName: string); procedure ExpandAndProcess(const Pattern: string); procedure ProcessListFile(const AListFileName: string); public - constructor Create(ARecursive: Boolean); + constructor Create(ARecursive: Boolean; AAnsiCodePage: Integer); procedure ProcessArg(const Arg: string); property SuccessCount: Integer read FSuccessCount; property FailCount : Integer read FFailCount; end; -constructor TDFMProcessor.Create(ARecursive: Boolean); +constructor TDFMProcessor.Create(ARecursive: Boolean; AAnsiCodePage: Integer); begin inherited Create; FRecursive := ARecursive; + FAnsiCodePage := AAnsiCodePage; FSuccessCount := 0; FFailCount := 0; end; @@ -56,7 +63,7 @@ procedure TDFMProcessor.ProcessFile(const AFileName: string); begin try Write(' ', AFileName, ' ... '); - if ConvertDFMFile(AFileName) then + if ConvertDFMFile(AFileName, FAnsiCodePage) then Writeln('converted') else Writeln('already up to date'); @@ -151,9 +158,34 @@ procedure TDFMProcessor.ProcessArg(const Arg: string); // --------------------------------------------------------------------------- +const + AnsiCodePageSwitch = '-a:'; + +function IsAnsiCodePageArg(const Arg: string): Boolean; +begin + Result := Arg.StartsWith(AnsiCodePageSwitch, True); +end; + +// Extract the numeric code page from an "-a:" argument. +// Accepts a bare number (-a:1250) or a name containing one (-a:CP1250). +function ExtractAnsiCodePage(const Arg: string): Integer; +var + Value : string; + Digits: string; + C : Char; +begin + Value := Arg.Substring(Length(AnsiCodePageSwitch)); + Digits := ''; + for C in Value do + if CharInSet(C, ['0'..'9']) then + Digits := Digits + C; + if not TryStrToInt(Digits, Result) then + raise Exception.CreateFmt('Invalid ANSI code page: "%s"', [Arg]); +end; + procedure PrintUsage; begin - Writeln('Usage: DFMStabilizerTool [-s] [...]'); + Writeln('Usage: DFMStabilizerTool [-s] [-a:] [...]'); Writeln; Writeln('Converts DFM files in-place to the stabilized UTF-8 text format:'); Writeln(' - strings are not broken at 64 characters (limit raised to 700)'); @@ -163,6 +195,11 @@ procedure PrintUsage; Writeln; Writeln('Options:'); Writeln(' -s Recurse into subdirectories when expanding wildcard patterns'); + Writeln(' -a: Decode text DFMs with no UTF-8 BOM using ANSI code page '); + Writeln(' (e.g. -a:1250 for Central European / Czech) instead of'); + Writeln(' assuming UTF-8 without BOM. Use this if conversion fails with'); + Writeln(' "No mapping for the Unicode character exists in the target'); + Writeln(' multi-byte code page".'); Writeln; Writeln('Arguments:'); Writeln(' file Exact path to a DFM file'); @@ -174,16 +211,18 @@ procedure PrintUsage; Writeln(' DFMStabilizerTool -s *.dfm'); Writeln(' DFMStabilizerTool -s src\*.dfm @extra_forms.txt'); Writeln(' DFMStabilizerTool @all_forms.txt'); + Writeln(' DFMStabilizerTool -a:1250 legacy\*.dfm'); end; // --------------------------------------------------------------------------- procedure Run; var - Recursive: Boolean; - Processor: TDFMProcessor; - I : Integer; - Arg : string; + Recursive : Boolean; + AnsiCodePage: Integer; + Processor : TDFMProcessor; + I : Integer; + Arg : string; begin if ParamCount = 0 then begin @@ -191,20 +230,23 @@ procedure Run; Halt(1); end; - Recursive := False; + Recursive := False; + AnsiCodePage := 0; for I := 1 to ParamCount do - if SameText(ParamStr(I), '-s') then - begin - Recursive := True; - Break; - end; + begin + Arg := ParamStr(I); + if SameText(Arg, '-s') then + Recursive := True + else if IsAnsiCodePageArg(Arg) then + AnsiCodePage := ExtractAnsiCodePage(Arg); + end; - Processor := TDFMProcessor.Create(Recursive); + Processor := TDFMProcessor.Create(Recursive, AnsiCodePage); try for I := 1 to ParamCount do begin Arg := ParamStr(I); - if SameText(Arg, '-s') then + if SameText(Arg, '-s') or IsAnsiCodePageArg(Arg) then Continue; Processor.ProcessArg(Arg); end; diff --git a/DFMTextStabilizerCore.pas b/DFMTextStabilizerCore.pas index 0792212..c86535f 100644 --- a/DFMTextStabilizerCore.pas +++ b/DFMTextStabilizerCore.pas @@ -12,6 +12,14 @@ Accepts both text DFMs (UTF-8 with/without BOM, ANSI) and legacy binary DFMs. The original file is replaced atomically via a temporary file. + + AAnsiCodePage on ConvertDFMFile: when a text DFM has no UTF-8 BOM and + contains non-ASCII bytes, it is normally assumed to be UTF-8 without a BOM. + Some legacy DFMs are instead stored in a single-byte ANSI code page (e.g. + 1250 for Central European / Czech text). Pass that code page explicitly to + decode such files correctly instead of misinterpreting them as UTF-8, which + otherwise fails with "No mapping for the Unicode character exists in the + target multi-byte code page" or produces mojibake. } interface @@ -28,7 +36,10 @@ procedure DFMBinaryToText(const Input, Output: TStream); // Returns True if the file was actually rewritten, False if it was already // in the stabilized format (no disk write performed). // Raises an exception if the file cannot be read, parsed, or written. -function ConvertDFMFile(const AFileName: string): Boolean; +// AAnsiCodePage: Windows code page (e.g. 1250) to use when decoding a text +// DFM that has no UTF-8 BOM but contains non-ASCII bytes. Pass 0 (default) +// to keep the previous auto-detect-as-UTF-8 behavior. +function ConvertDFMFile(const AFileName: string; AAnsiCodePage: Integer = 0): Boolean; implementation @@ -464,7 +475,7 @@ function IsBinaryDFM(Stream: TStream): Boolean; Stream.Position := SavePos; end; -function ConvertDFMFile(const AFileName: string): Boolean; +function ConvertDFMFile(const AFileName: string; AAnsiCodePage: Integer = 0): Boolean; var FileContent : TMemoryStream; BinaryStream: TMemoryStream; @@ -495,12 +506,15 @@ function ConvertDFMFile(const AFileName: string): Boolean; // bytes are misinterpreted as ANSI, causing double-encoding of every // character above U+007F. // - // Three cases: + // Four cases: // 1. BOM present → pass the stream as-is starting from position 0; // TParser skips the BOM itself. - // 2. Non-ASCII bytes without BOM (UTF-8 without BOM) → prepend BOM in - // memory before passing to ObjectTextToBinary. - // 3. Pure ASCII → no BOM needed; pass directly. + // 2. Explicit ANSI code page given (AAnsiCodePage <> 0) → decode the + // raw bytes using that code page, re-encode as + // UTF-8, then prepend BOM before passing on. + // 3. Non-ASCII bytes without BOM, no code page given → assume UTF-8 + // without BOM; prepend BOM in memory. + // 4. Pure ASCII → no BOM needed; pass directly. // // This mirrors the logic in HookedObjectTextToBinary in the IDE plugin. @@ -514,9 +528,37 @@ function ConvertDFMFile(const AFileName: string): Boolean; FileContent.Position := 0; ObjectTextToBinary(FileContent, BinaryStream); end + else if AAnsiCodePage <> 0 then + begin + // Case 2: caller specified the source ANSI code page explicitly — + // decode with it rather than guessing UTF-8, so legacy files (e.g. + // CP1250) are read correctly. + var RawBytes: TBytes; + var AnsiEncoding := TEncoding.GetEncoding(AAnsiCodePage); + try + SetLength(RawBytes, FileContent.Size); + if FileContent.Size > 0 then + Move(FileContent.Memory^, RawBytes[0], FileContent.Size); + var DecodedStr := AnsiEncoding.GetString(RawBytes); + var UTF8Bytes := TEncoding.UTF8.GetBytes(DecodedStr); + + var Patched := TMemoryStream.Create; + try + Patched.Write(BOM[0], BOMLen); + if Length(UTF8Bytes) > 0 then + Patched.Write(UTF8Bytes[0], Length(UTF8Bytes)); + Patched.Position := 0; + ObjectTextToBinary(Patched, BinaryStream); + finally + Patched.Free; + end; + finally + AnsiEncoding.Free; + end; + end else if HasNonAsciiBytes(FileContent.Memory, FileContent.Size) then begin - // Case 2: UTF-8 without BOM — prepend BOM in a temporary stream + // Case 3: UTF-8 without BOM — prepend BOM in a temporary stream var Patched := TMemoryStream.Create; try Patched.Write(BOM[0], BOMLen); @@ -529,7 +571,7 @@ function ConvertDFMFile(const AFileName: string): Boolean; end else begin - // Case 3: pure ASCII (e.g. old ANSI DFM with #NNN escapes) + // Case 4: pure ASCII (e.g. old ANSI DFM with #NNN escapes) FileContent.Position := 0; ObjectTextToBinary(FileContent, BinaryStream); end;