From 1a54b0585d562da42663c0ac401153f7f7ddc203 Mon Sep 17 00:00:00 2001 From: Michal Lazo Date: Wed, 1 Jul 2026 12:43:37 +0200 Subject: [PATCH 01/11] ansi code page --- DFMStabilizerCLI.pas | 76 ++++++++++++++++++++++++++++++--------- DFMTextStabilizerCore.pas | 58 +++++++++++++++++++++++++----- 2 files changed, 109 insertions(+), 25 deletions(-) diff --git a/DFMStabilizerCLI.pas b/DFMStabilizerCLI.pas index 8c6d6af..1cf8a58 100644 --- a/DFMStabilizerCLI.pas +++ b/DFMStabilizerCLI.pas @@ -4,9 +4,14 @@ Command-line interface for the DFM stabilizer tool. Argument syntax: - DFMStabilizerTool [-s] [...] + DFMStabilizerTool [-s] [-a:] [...] -s Recurse into subdirectories when expanding wildcard patterns. + -a: Treat text DFMs with no UTF-8 BOM as being encoded in the + given ANSI code page (e.g. -a:1250) instead of assuming + UTF-8 without BOM. Use this for legacy DFMs that fail with + "No mapping for the Unicode character exists in the target + multi-byte code page". file Exact path to a DFM file. pattern Wildcard pattern: *.dfm, path\*.dfm, etc. @listfile Text file listing one path or pattern per line. @@ -32,22 +37,24 @@ implementation TDFMProcessor = class private FRecursive : Boolean; + FAnsiCodePage: Integer; FSuccessCount: Integer; FFailCount : Integer; procedure ProcessFile(const AFileName: string); procedure ExpandAndProcess(const Pattern: string); procedure ProcessListFile(const AListFileName: string); public - constructor Create(ARecursive: Boolean); + constructor Create(ARecursive: Boolean; AAnsiCodePage: Integer); procedure ProcessArg(const Arg: string); property SuccessCount: Integer read FSuccessCount; property FailCount : Integer read FFailCount; end; -constructor TDFMProcessor.Create(ARecursive: Boolean); +constructor TDFMProcessor.Create(ARecursive: Boolean; AAnsiCodePage: Integer); begin inherited Create; FRecursive := ARecursive; + FAnsiCodePage := AAnsiCodePage; FSuccessCount := 0; FFailCount := 0; end; @@ -56,7 +63,7 @@ procedure TDFMProcessor.ProcessFile(const AFileName: string); begin try Write(' ', AFileName, ' ... '); - if ConvertDFMFile(AFileName) then + if ConvertDFMFile(AFileName, FAnsiCodePage) then Writeln('converted') else Writeln('already up to date'); @@ -151,9 +158,34 @@ procedure TDFMProcessor.ProcessArg(const Arg: string); // --------------------------------------------------------------------------- +const + AnsiCodePageSwitch = '-a:'; + +function IsAnsiCodePageArg(const Arg: string): Boolean; +begin + Result := Arg.StartsWith(AnsiCodePageSwitch, True); +end; + +// Extract the numeric code page from an "-a:" argument. +// Accepts a bare number (-a:1250) or a name containing one (-a:CP1250). +function ExtractAnsiCodePage(const Arg: string): Integer; +var + Value : string; + Digits: string; + C : Char; +begin + Value := Arg.Substring(Length(AnsiCodePageSwitch)); + Digits := ''; + for C in Value do + if CharInSet(C, ['0'..'9']) then + Digits := Digits + C; + if not TryStrToInt(Digits, Result) then + raise Exception.CreateFmt('Invalid ANSI code page: "%s"', [Arg]); +end; + procedure PrintUsage; begin - Writeln('Usage: DFMStabilizerTool [-s] [...]'); + Writeln('Usage: DFMStabilizerTool [-s] [-a:] [...]'); Writeln; Writeln('Converts DFM files in-place to the stabilized UTF-8 text format:'); Writeln(' - strings are not broken at 64 characters (limit raised to 700)'); @@ -163,6 +195,11 @@ procedure PrintUsage; Writeln; Writeln('Options:'); Writeln(' -s Recurse into subdirectories when expanding wildcard patterns'); + Writeln(' -a: Decode text DFMs with no UTF-8 BOM using ANSI code page '); + Writeln(' (e.g. -a:1250 for Central European / Czech) instead of'); + Writeln(' assuming UTF-8 without BOM. Use this if conversion fails with'); + Writeln(' "No mapping for the Unicode character exists in the target'); + Writeln(' multi-byte code page".'); Writeln; Writeln('Arguments:'); Writeln(' file Exact path to a DFM file'); @@ -174,16 +211,18 @@ procedure PrintUsage; Writeln(' DFMStabilizerTool -s *.dfm'); Writeln(' DFMStabilizerTool -s src\*.dfm @extra_forms.txt'); Writeln(' DFMStabilizerTool @all_forms.txt'); + Writeln(' DFMStabilizerTool -a:1250 legacy\*.dfm'); end; // --------------------------------------------------------------------------- procedure Run; var - Recursive: Boolean; - Processor: TDFMProcessor; - I : Integer; - Arg : string; + Recursive : Boolean; + AnsiCodePage: Integer; + Processor : TDFMProcessor; + I : Integer; + Arg : string; begin if ParamCount = 0 then begin @@ -191,20 +230,23 @@ procedure Run; Halt(1); end; - Recursive := False; + Recursive := False; + AnsiCodePage := 0; for I := 1 to ParamCount do - if SameText(ParamStr(I), '-s') then - begin - Recursive := True; - Break; - end; + begin + Arg := ParamStr(I); + if SameText(Arg, '-s') then + Recursive := True + else if IsAnsiCodePageArg(Arg) then + AnsiCodePage := ExtractAnsiCodePage(Arg); + end; - Processor := TDFMProcessor.Create(Recursive); + Processor := TDFMProcessor.Create(Recursive, AnsiCodePage); try for I := 1 to ParamCount do begin Arg := ParamStr(I); - if SameText(Arg, '-s') then + if SameText(Arg, '-s') or IsAnsiCodePageArg(Arg) then Continue; Processor.ProcessArg(Arg); end; diff --git a/DFMTextStabilizerCore.pas b/DFMTextStabilizerCore.pas index 0792212..c86535f 100644 --- a/DFMTextStabilizerCore.pas +++ b/DFMTextStabilizerCore.pas @@ -12,6 +12,14 @@ Accepts both text DFMs (UTF-8 with/without BOM, ANSI) and legacy binary DFMs. The original file is replaced atomically via a temporary file. + + AAnsiCodePage on ConvertDFMFile: when a text DFM has no UTF-8 BOM and + contains non-ASCII bytes, it is normally assumed to be UTF-8 without a BOM. + Some legacy DFMs are instead stored in a single-byte ANSI code page (e.g. + 1250 for Central European / Czech text). Pass that code page explicitly to + decode such files correctly instead of misinterpreting them as UTF-8, which + otherwise fails with "No mapping for the Unicode character exists in the + target multi-byte code page" or produces mojibake. } interface @@ -28,7 +36,10 @@ procedure DFMBinaryToText(const Input, Output: TStream); // Returns True if the file was actually rewritten, False if it was already // in the stabilized format (no disk write performed). // Raises an exception if the file cannot be read, parsed, or written. -function ConvertDFMFile(const AFileName: string): Boolean; +// AAnsiCodePage: Windows code page (e.g. 1250) to use when decoding a text +// DFM that has no UTF-8 BOM but contains non-ASCII bytes. Pass 0 (default) +// to keep the previous auto-detect-as-UTF-8 behavior. +function ConvertDFMFile(const AFileName: string; AAnsiCodePage: Integer = 0): Boolean; implementation @@ -464,7 +475,7 @@ function IsBinaryDFM(Stream: TStream): Boolean; Stream.Position := SavePos; end; -function ConvertDFMFile(const AFileName: string): Boolean; +function ConvertDFMFile(const AFileName: string; AAnsiCodePage: Integer = 0): Boolean; var FileContent : TMemoryStream; BinaryStream: TMemoryStream; @@ -495,12 +506,15 @@ function ConvertDFMFile(const AFileName: string): Boolean; // bytes are misinterpreted as ANSI, causing double-encoding of every // character above U+007F. // - // Three cases: + // Four cases: // 1. BOM present → pass the stream as-is starting from position 0; // TParser skips the BOM itself. - // 2. Non-ASCII bytes without BOM (UTF-8 without BOM) → prepend BOM in - // memory before passing to ObjectTextToBinary. - // 3. Pure ASCII → no BOM needed; pass directly. + // 2. Explicit ANSI code page given (AAnsiCodePage <> 0) → decode the + // raw bytes using that code page, re-encode as + // UTF-8, then prepend BOM before passing on. + // 3. Non-ASCII bytes without BOM, no code page given → assume UTF-8 + // without BOM; prepend BOM in memory. + // 4. Pure ASCII → no BOM needed; pass directly. // // This mirrors the logic in HookedObjectTextToBinary in the IDE plugin. @@ -514,9 +528,37 @@ function ConvertDFMFile(const AFileName: string): Boolean; FileContent.Position := 0; ObjectTextToBinary(FileContent, BinaryStream); end + else if AAnsiCodePage <> 0 then + begin + // Case 2: caller specified the source ANSI code page explicitly — + // decode with it rather than guessing UTF-8, so legacy files (e.g. + // CP1250) are read correctly. + var RawBytes: TBytes; + var AnsiEncoding := TEncoding.GetEncoding(AAnsiCodePage); + try + SetLength(RawBytes, FileContent.Size); + if FileContent.Size > 0 then + Move(FileContent.Memory^, RawBytes[0], FileContent.Size); + var DecodedStr := AnsiEncoding.GetString(RawBytes); + var UTF8Bytes := TEncoding.UTF8.GetBytes(DecodedStr); + + var Patched := TMemoryStream.Create; + try + Patched.Write(BOM[0], BOMLen); + if Length(UTF8Bytes) > 0 then + Patched.Write(UTF8Bytes[0], Length(UTF8Bytes)); + Patched.Position := 0; + ObjectTextToBinary(Patched, BinaryStream); + finally + Patched.Free; + end; + finally + AnsiEncoding.Free; + end; + end else if HasNonAsciiBytes(FileContent.Memory, FileContent.Size) then begin - // Case 2: UTF-8 without BOM — prepend BOM in a temporary stream + // Case 3: UTF-8 without BOM — prepend BOM in a temporary stream var Patched := TMemoryStream.Create; try Patched.Write(BOM[0], BOMLen); @@ -529,7 +571,7 @@ function ConvertDFMFile(const AFileName: string): Boolean; end else begin - // Case 3: pure ASCII (e.g. old ANSI DFM with #NNN escapes) + // Case 4: pure ASCII (e.g. old ANSI DFM with #NNN escapes) FileContent.Position := 0; ObjectTextToBinary(FileContent, BinaryStream); end; From cc315425a411a751f17b3d624b65ccffed3d6b4d Mon Sep 17 00:00:00 2001 From: Michal Lazo Date: Wed, 1 Jul 2026 13:58:34 +0200 Subject: [PATCH 02/11] python version --- dfm_stabilizer_cli.py | 203 +++++++++++++ dfm_stabilizer_tool.py | 9 + dfm_text_stabilizer_core.py | 583 ++++++++++++++++++++++++++++++++++++ 3 files changed, 795 insertions(+) create mode 100644 dfm_stabilizer_cli.py create mode 100644 dfm_stabilizer_tool.py create mode 100644 dfm_text_stabilizer_core.py diff --git a/dfm_stabilizer_cli.py b/dfm_stabilizer_cli.py new file mode 100644 index 0000000..6dba9db --- /dev/null +++ b/dfm_stabilizer_cli.py @@ -0,0 +1,203 @@ +""" +Command-line interface for the Python port of DFMStabilizerTool. + +Argument syntax: + dfm_stabilizer_tool.py [-s] [-a:] [...] + + -s Recurse into subdirectories when expanding wildcard patterns. + -a: Decode text DFMs with no UTF-8 BOM using ANSI code page + (e.g. -a:1250 for Central European / Czech) instead of + assuming UTF-8 without BOM. Use this if conversion fails with + "No mapping for the Unicode character exists in the target + multi-byte code page" style errors. + file Exact path to a DFM file. + pattern Wildcard pattern: *.dfm, path\\*.dfm, etc. + @listfile Text file listing one path or pattern per line. + Lines starting with # are treated as comments. + +Only text DFMs are supported (UTF-8 with/without BOM, or a single-byte ANSI +code page). Legacy binary DFMs (signature 0xFF 0x0A) are reported as failed; +use the original Delphi DFMStabilizerTool for those. + +Exit code: 0 if all files were converted successfully, 1 if any failed. +""" + +from __future__ import annotations + +import fnmatch +import os +import re +import sys +from pathlib import Path +from typing import List, Optional + +from dfm_text_stabilizer_core import DFMError, stabilize_dfm_bytes + +ANSI_CODE_PAGE_SWITCH = "-a:" + + +def is_ansi_code_page_arg(arg: str) -> bool: + return arg.lower().startswith(ANSI_CODE_PAGE_SWITCH) + + +def extract_ansi_code_page(arg: str) -> int: + value = arg[len(ANSI_CODE_PAGE_SWITCH):] + digits = "".join(re.findall(r"\d", value)) + if not digits: + raise DFMError(f'Invalid ANSI code page: "{arg}"') + return int(digits) + + +def convert_dfm_file(file_name: str, ansi_code_page: Optional[int] = None) -> bool: + """ + Convert file_name in-place to the stabilized text format. + Returns True if the file was rewritten, False if already up to date. + Raises DFMError (or an OSError) on failure. + """ + with open(file_name, "rb") as f: + original = f.read() + + stabilized = stabilize_dfm_bytes(original, ansi_code_page) + + if stabilized == original: + return False + + tmp_name = file_name + ".dfmstab.tmp" + try: + with open(tmp_name, "wb") as f: + f.write(stabilized) + os.replace(tmp_name, file_name) + except Exception: + if os.path.exists(tmp_name): + os.remove(tmp_name) + raise + + return True + + +class DFMProcessor: + def __init__(self, recursive: bool, ansi_code_page: Optional[int]): + self.recursive = recursive + self.ansi_code_page = ansi_code_page + self.success_count = 0 + self.fail_count = 0 + + def process_file(self, file_name: str) -> None: + try: + print(f" {file_name} ... ", end="") + if convert_dfm_file(file_name, self.ansi_code_page): + print("converted") + else: + print("already up to date") + self.success_count += 1 + except Exception as exc: # noqa: BLE001 - mirror the Delphi try/except-per-file behavior + print(f"FAILED: {exc}") + self.fail_count += 1 + + def expand_and_process(self, pattern: str) -> None: + directory = os.path.dirname(pattern) or "." + file_pattern = os.path.basename(pattern) + + if os.path.isdir(directory): + try: + entries = sorted(os.listdir(directory)) + except OSError: + entries = [] + for entry in entries: + full_path = os.path.join(directory, entry) + if os.path.isfile(full_path) and fnmatch.fnmatch(entry, file_pattern): + self.process_file(full_path) + + if self.recursive: + for entry in entries: + full_path = os.path.join(directory, entry) + if os.path.isdir(full_path): + self.expand_and_process(os.path.join(full_path, file_pattern)) + + def process_list_file(self, list_file_name: str) -> None: + if not os.path.isfile(list_file_name): + raise FileNotFoundError(f"List file not found: {list_file_name}") + + with open(list_file_name, "r", encoding="utf-8-sig") as f: + lines = f.readlines() + + for line in lines: + line = line.strip() + if line and not line.startswith("#"): + self.process_arg(line) + + def process_arg(self, arg: str) -> None: + if arg.startswith("@"): + self.process_list_file(arg[1:]) + elif "*" in arg or "?" in arg: + self.expand_and_process(arg) + else: + self.process_file(arg) + + +def print_usage() -> None: + print("Usage: dfm_stabilizer_tool.py [-s] [-a:] [...]") + print() + print("Converts DFM files in-place to the stabilized UTF-8 text format:") + print(" - strings are not broken at 64 characters (limit raised to 700)") + print(" - embedded newlines (#13/#10) cause a line break at that position") + print(" - non-ASCII characters are written literally as UTF-8") + print(" - file always starts with a UTF-8 BOM") + print() + print("Only text DFMs are supported (legacy binary DFMs are reported as failed;") + print("use the Delphi DFMStabilizerTool.exe for those).") + print() + print("Options:") + print(" -s Recurse into subdirectories when expanding wildcard patterns") + print(" -a: Decode text DFMs with no UTF-8 BOM using ANSI code page ") + print(" (e.g. -a:1250 for Central European / Czech) instead of") + print(" assuming UTF-8 without BOM. Use this if conversion fails with") + print(' "No mapping for the Unicode character exists in the target') + print(' multi-byte code page".') + print() + print("Arguments:") + print(" file Exact path to a DFM file") + print(" pattern Wildcard pattern (e.g. *.dfm or forms\\*.dfm)") + print(" @listfile Text file with one path/pattern per line (# = comment)") + print() + print("Examples:") + print(" dfm_stabilizer_tool.py MainForm.dfm") + print(" dfm_stabilizer_tool.py -s *.dfm") + print(" dfm_stabilizer_tool.py -s src\\*.dfm @extra_forms.txt") + print(" dfm_stabilizer_tool.py @all_forms.txt") + print(" dfm_stabilizer_tool.py -a:1250 legacy\\*.dfm") + + +def run(argv: Optional[List[str]] = None) -> int: + args = sys.argv[1:] if argv is None else argv + + if not args: + print_usage() + return 1 + + recursive = False + ansi_code_page: Optional[int] = None + for arg in args: + if arg == "-s": + recursive = True + elif is_ansi_code_page_arg(arg): + try: + ansi_code_page = extract_ansi_code_page(arg) + except DFMError as exc: + print(f"Fatal: {exc}", file=sys.stderr) + return 1 + + processor = DFMProcessor(recursive, ansi_code_page) + for arg in args: + if arg == "-s" or is_ansi_code_page_arg(arg): + continue + processor.process_arg(arg) + + print() + total = processor.success_count + processor.fail_count + if total == 0: + print("No files matched.") + else: + print(f"{processor.success_count} converted, {processor.fail_count} failed.") + + return 1 if processor.fail_count > 0 else 0 diff --git a/dfm_stabilizer_tool.py b/dfm_stabilizer_tool.py new file mode 100644 index 0000000..5667f42 --- /dev/null +++ b/dfm_stabilizer_tool.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python3 +"""Entry point for the Python port of DFMStabilizerTool. See dfm_stabilizer_cli for usage.""" + +import sys + +from dfm_stabilizer_cli import run + +if __name__ == "__main__": + sys.exit(run()) diff --git a/dfm_text_stabilizer_core.py b/dfm_text_stabilizer_core.py new file mode 100644 index 0000000..22b630b --- /dev/null +++ b/dfm_text_stabilizer_core.py @@ -0,0 +1,583 @@ +""" +Core DFM text stabilization logic (Python port). + +Ports the *text-DFM* half of DFMTextStabilizerCore.pas: parsing an existing +textual .dfm (UTF-8 with/without BOM, or a legacy single-byte ANSI code page +such as CP1250) and re-emitting it in the stabilized format used by the +Delphi DFMStabilizerTool / DFMBinaryToTextHook: + + - strings are not broken at 64 characters (limit raised to 700) + - embedded newlines (#13/#10) cause a line break at that position + - non-ASCII characters are written literally as UTF-8 + - file always starts with a UTF-8 BOM + +Legacy *binary* DFMs (the old pre-Delphi-5 on-disk format, signature bytes +0xFF 0x0A) are intentionally out of scope for this port -- converting those +requires reimplementing Delphi's TReader/TWriter binary streaming format. +Use the original Delphi DFMStabilizerTool for those files; this module raises +UnsupportedBinaryDFMError so callers can report it clearly. +""" + +from __future__ import annotations + +import codecs +from dataclasses import dataclass, field +from typing import List, Optional, Tuple, Union + +LINE_LENGTH = 700 # matches LineLength in DFMTextStabilizerCore.pas +BYTES_PER_LINE = 32 # matches BytesPerLine in ConvertBinary +UTF8_BOM = codecs.BOM_UTF8 +NEWLINE = "\r\n" # sLineBreak on Windows, matching the original tool's output +INDENT_UNIT = " " + + +class DFMError(Exception): + pass + + +class UnsupportedBinaryDFMError(DFMError): + """Raised when the input is a legacy binary DFM (signature 0xFF 0x0A).""" + + +class DFMParseError(DFMError): + pass + + +# --------------------------------------------------------------------------- +# Decoding raw file bytes to text +# --------------------------------------------------------------------------- + + +def is_binary_dfm(data: bytes) -> bool: + return len(data) >= 2 and data[0] == 0xFF and data[1] == 0x0A + + +def _has_non_ascii_bytes(data: bytes) -> bool: + return any(b > 127 for b in data) + + +def decode_dfm_bytes(data: bytes, ansi_code_page: Optional[int] = None) -> str: + """ + Decode raw .dfm file bytes to text, mirroring the three text-DFM cases in + ConvertDFMFile (BOM / explicit ANSI code page / UTF-8-without-BOM guess / + pure ASCII). Raises UnsupportedBinaryDFMError if this is a legacy binary + DFM instead of a text one. + """ + if is_binary_dfm(data): + raise UnsupportedBinaryDFMError( + "Legacy binary DFM format is not supported by this tool; " + "use the Delphi DFMStabilizerTool.exe for this file." + ) + + if data.startswith(UTF8_BOM): + # Case 1: UTF-8 with BOM. + return data[len(UTF8_BOM):].decode("utf-8") + + if ansi_code_page: + # Case 2: caller specified the source ANSI code page explicitly. + try: + return data.decode(f"cp{ansi_code_page}") + except LookupError as exc: + raise DFMError(f"Unknown ANSI code page: {ansi_code_page}") from exc + + if _has_non_ascii_bytes(data): + # Case 3: assume UTF-8 without BOM. + return data.decode("utf-8") + + # Case 4: pure ASCII. + return data.decode("ascii") + + +# --------------------------------------------------------------------------- +# AST +# --------------------------------------------------------------------------- + +# A value is one of: +# ("scalar", raw_text) -- number / ident / True / nil / etc, verbatim +# ("string", [str]) -- logical Unicode content of a string literal +# ("set", [ident, ...]) -- [Ident1, Ident2] +# ("list", [Value, ...]) -- (Value Value ...) +# ("binary", hex_str) -- {AA00FF...} +# ("collection", [CollectionItem, ...]) -- +Value = Tuple[str, object] + + +@dataclass +class CollectionItem: + index: Optional[int] + properties: List[Tuple[str, "Value"]] + + +@dataclass +class DFMObject: + kind: str # 'object' | 'inherited' | 'inline' + name: str + class_name: str + position: Optional[int] + properties: List[Tuple[str, "Value"]] = field(default_factory=list) + children: List["DFMObject"] = field(default_factory=list) + + +# --------------------------------------------------------------------------- +# Parser +# --------------------------------------------------------------------------- + +_IDENT_START = set("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_") +_IDENT_CONT = _IDENT_START | set("0123456789.") +_WS = set(" \t\r\n") +_SCALAR_STOP = set("'#(),[]<>{}=\t\r\n ") + + +class _Parser: + def __init__(self, text: str): + self.s = text + self.i = 0 + self.n = len(text) + + # -- low-level helpers -------------------------------------------------- + + def skip_ws(self) -> None: + while self.i < self.n and self.s[self.i] in _WS: + self.i += 1 + + def peek(self) -> str: + return self.s[self.i] if self.i < self.n else "" + + def eof(self) -> bool: + self.skip_ws() + return self.i >= self.n + + def expect(self, ch: str) -> None: + self.skip_ws() + if self.peek() != ch: + raise DFMParseError( + f"Expected '{ch}' at offset {self.i}, found " + f"{self.s[self.i:self.i + 20]!r}" + ) + self.i += 1 + + def read_ident(self) -> str: + self.skip_ws() + start = self.i + if self.i >= self.n or self.s[self.i] not in _IDENT_START: + raise DFMParseError( + f"Expected identifier at offset {self.i}, found " + f"{self.s[self.i:self.i + 20]!r}" + ) + self.i += 1 + while self.i < self.n and self.s[self.i] in _IDENT_CONT: + self.i += 1 + return self.s[start:self.i] + + def peek_ident(self) -> str: + save = self.i + self.skip_ws() + start = self.i + while self.i < self.n and self.s[self.i] in _IDENT_CONT: + self.i += 1 + word = self.s[start:self.i] + self.i = save + return word + + # -- object --------------------------------------------------------- + + def parse_objects(self) -> List[DFMObject]: + objects = [] + while not self.eof(): + objects.append(self.parse_object()) + self.skip_ws() + return objects + + def parse_object(self) -> DFMObject: + kind = self.read_ident().lower() + if kind not in ("object", "inherited", "inline"): + raise DFMParseError(f"Expected object/inherited/inline, found {kind!r}") + + first = self.read_ident() + self.skip_ws() + if self.peek() == ":": + self.i += 1 + name = first + class_name = self.read_ident() + else: + name = "" + class_name = first + + position = None + self.skip_ws() + if self.peek() == "[": + self.i += 1 + self.skip_ws() + start = self.i + while self.i < self.n and self.s[self.i] != "]": + self.i += 1 + position = int(self.s[start:self.i].strip()) + self.expect("]") + + node = DFMObject(kind=kind, name=name, class_name=class_name, position=position) + + # Properties, until we see a nested object or the closing 'end'. + while True: + self.skip_ws() + word = self.peek_ident().lower() + if word == "end": + self.read_ident() + return node + if word in ("object", "inherited", "inline"): + break + prop_name = self.read_ident() + self.expect("=") + value = self.parse_value() + node.properties.append((prop_name, value)) + + # Child objects, until the closing 'end'. + while True: + self.skip_ws() + word = self.peek_ident().lower() + if word == "end": + self.read_ident() + return node + if word in ("object", "inherited", "inline"): + node.children.append(self.parse_object()) + continue + raise DFMParseError( + f"Expected nested object or 'end' at offset {self.i}, found " + f"{self.s[self.i:self.i + 20]!r}" + ) + + # -- values ----------------------------------------------------------- + + def parse_value(self) -> Value: + self.skip_ws() + c = self.peek() + if c == "'" or c == "#": + return self.parse_string_value() + if c == "(": + return self.parse_list_value() + if c == "[": + return self.parse_set_value() + if c == "<": + return self.parse_collection_value() + if c == "{": + return self.parse_binary_value() + return self.parse_scalar_value() + + def parse_scalar_value(self) -> Value: + self.skip_ws() + start = self.i + if self.i < self.n and self.s[self.i] == "-": + self.i += 1 + while self.i < self.n and self.s[self.i] not in _SCALAR_STOP: + self.i += 1 + if self.i == start: + raise DFMParseError( + f"Expected a value at offset {self.i}, found " + f"{self.s[self.i:self.i + 20]!r}" + ) + return ("scalar", self.s[start:self.i]) + + def _read_quoted_literal(self) -> str: + self.expect("'") + buf = [] + while True: + if self.i >= self.n: + raise DFMParseError("Unterminated string literal") + c = self.s[self.i] + if c == "'": + if self.i + 1 < self.n and self.s[self.i + 1] == "'": + buf.append("'") + self.i += 2 + continue + self.i += 1 + break + if c in "\r\n": + raise DFMParseError("Unterminated string literal (embedded newline)") + buf.append(c) + self.i += 1 + return "".join(buf) + + def _skip_horizontal_ws(self) -> None: + while self.i < self.n and self.s[self.i] in (" ", "\t"): + self.i += 1 + + def parse_string_value(self) -> Value: + # A newline WITHOUT a preceding '+' ends the string value here (the + # writer always inserts '+' before every line break within a single + # string -- see _serialize_string). Without that marker, a segment on + # the next line belongs to a different value (typically the next + # element of an enclosing list), not a continuation of this string. + # Segments on the *same* line concatenate implicitly, with or without + # a '+' between them. + chars: List[str] = [] + first = True + while True: + if not first: + save = self.i + self._skip_horizontal_ws() + c = self.peek() + if c == "+": + self.i += 1 + self.skip_ws() + elif c in ("'", "#"): + pass # same line, implicit concatenation + else: + self.i = save + break + first = False + c = self.peek() + if c == "'": + chars.append(self._read_quoted_literal()) + elif c == "#": + self.i += 1 + start = self.i + while self.i < self.n and self.s[self.i].isdigit(): + self.i += 1 + if self.i == start: + raise DFMParseError("Expected digits after '#'") + chars.append(chr(int(self.s[start:self.i]))) + else: + break + return ("string", "".join(chars)) + + def parse_list_value(self) -> Value: + self.expect("(") + elements: List[Value] = [] + while True: + self.skip_ws() + if self.peek() == ")": + self.i += 1 + break + elements.append(self.parse_value()) + return ("list", elements) + + def parse_set_value(self) -> Value: + self.expect("[") + idents: List[str] = [] + self.skip_ws() + if self.peek() == "]": + self.i += 1 + return ("set", idents) + while True: + idents.append(self.read_ident()) + self.skip_ws() + if self.peek() == ",": + self.i += 1 + continue + self.expect("]") + break + return ("set", idents) + + def parse_binary_value(self) -> Value: + self.expect("{") + hex_chars = [] + while True: + if self.i >= self.n: + raise DFMParseError("Unterminated binary block") + c = self.s[self.i] + if c == "}": + self.i += 1 + break + if c not in _WS: + hex_chars.append(c) + self.i += 1 + return ("binary", "".join(hex_chars)) + + def parse_collection_value(self) -> Value: + self.expect("<") + items: List[CollectionItem] = [] + while True: + self.skip_ws() + if self.peek() == ">": + self.i += 1 + break + word = self.read_ident().lower() + if word != "item": + raise DFMParseError(f"Expected 'item' inside collection, found {word!r}") + self.skip_ws() + index = None + if self.peek() == "[": + self.i += 1 + self.skip_ws() + start = self.i + while self.i < self.n and self.s[self.i] != "]": + self.i += 1 + index = int(self.s[start:self.i].strip()) + self.expect("]") + + properties: List[Tuple[str, Value]] = [] + while True: + self.skip_ws() + word = self.peek_ident().lower() + if word == "end": + self.read_ident() + break + prop_name = self.read_ident() + self.expect("=") + value = self.parse_value() + properties.append((prop_name, value)) + + items.append(CollectionItem(index=index, properties=properties)) + return ("collection", items) + + +def parse_dfm_text(text: str) -> List[DFMObject]: + parser = _Parser(text) + return parser.parse_objects() + + +# --------------------------------------------------------------------------- +# Serializer (mirrors ConvertHeader/ConvertProperty/ConvertValue/ConvertBinary +# in DFMTextStabilizerCore.pas) +# --------------------------------------------------------------------------- + + +def _indent(level: int) -> str: + return INDENT_UNIT * level + + +def _serialize_string(content: str, level: int, out: List[str]) -> None: + if content == "": + out.append("''") + return + + cur_level = level + 1 + n = len(content) + i = 0 + k = 0 + + if n > LINE_LENGTH: + out.append(NEWLINE + _indent(cur_level)) + + while i < n: + line_break = False + ch = content[i] + if ch >= " " and ch != "'": + j = i + i += 1 + while i < n and content[i] >= " " and content[i] != "'" and (i - k) < LINE_LENGTH: + i += 1 + if (i - k) >= LINE_LENGTH: + line_break = True + out.append("'") + out.append(content[j:i]) + out.append("'") + else: + out.append("#") + out.append(str(ord(ch))) + if ch == "\n": + line_break = True + elif ch == "\r" and (i + 1 >= n or content[i + 1] != "\n"): + line_break = True + i += 1 + if not line_break and (i - k) >= LINE_LENGTH: + line_break = True + + if line_break and i < n: + out.append(" +") + out.append(NEWLINE + _indent(cur_level)) + k = i + + +def _serialize_binary(hex_str: str, level: int, out: List[str]) -> None: + out.append("{") + chunk_chars = BYTES_PER_LINE * 2 + multiline = len(hex_str) >= chunk_chars + inner_indent = _indent(level + 1) + pos = 0 + while pos < len(hex_str): + if multiline: + out.append(NEWLINE + inner_indent) + out.append(hex_str[pos:pos + chunk_chars]) + pos += chunk_chars + out.append("}") + + +def _serialize_collection(items: List[CollectionItem], level: int, out: List[str]) -> None: + out.append("<") + item_indent = _indent(level + 1) + prop_indent = _indent(level + 2) + for item in items: + out.append(NEWLINE + item_indent + "item") + if item.index is not None: + out.append(f" [{item.index}]") + out.append(NEWLINE) + for prop_name, value in item.properties: + out.append(prop_indent + prop_name + " = ") + _serialize_value(value, level + 2, out) + out.append(NEWLINE) + out.append(item_indent + "end") + out.append(">") + + +def _serialize_list(elements: List[Value], level: int, out: List[str]) -> None: + out.append("(") + inner_indent = _indent(level + 1) + for element in elements: + out.append(NEWLINE + inner_indent) + _serialize_value(element, level + 1, out) + out.append(")") + + +def _serialize_value(value: Value, level: int, out: List[str]) -> None: + kind, payload = value + if kind == "scalar": + out.append(payload) + elif kind == "string": + _serialize_string(payload, level, out) + elif kind == "set": + out.append("[" + ", ".join(payload) + "]") + elif kind == "list": + _serialize_list(payload, level, out) + elif kind == "binary": + _serialize_binary(payload, level, out) + elif kind == "collection": + _serialize_collection(payload, level, out) + else: + raise DFMError(f"Unknown value kind: {kind}") + + +def _serialize_object(node: DFMObject, level: int, out: List[str]) -> None: + indent = _indent(level) + out.append(indent) + out.append({"object": "object ", "inherited": "inherited ", "inline": "inline "}[node.kind]) + if node.name: + out.append(node.name) + out.append(": ") + out.append(node.class_name) + if node.position is not None: + out.append(f" [{node.position}]") + out.append(NEWLINE) + + prop_indent = _indent(level + 1) + for prop_name, value in node.properties: + out.append(prop_indent) + out.append(prop_name) + out.append(" = ") + _serialize_value(value, level + 1, out) + out.append(NEWLINE) + + for child in node.children: + _serialize_object(child, level + 1, out) + + out.append(indent) + out.append("end") + out.append(NEWLINE) + + +def stabilize_text(text: str) -> str: + """Parse DFM text and re-serialize it in the stabilized format.""" + objects = parse_dfm_text(text) + out: List[str] = [] + for obj in objects: + _serialize_object(obj, 0, out) + return "".join(out) + + +def stabilize_dfm_bytes(data: bytes, ansi_code_page: Optional[int] = None) -> bytes: + """ + Full in-memory conversion: raw file bytes -> stabilized file bytes + (UTF-8 BOM + stabilized text, CRLF line endings). + """ + text = decode_dfm_bytes(data, ansi_code_page) + stabilized = stabilize_text(text) + return UTF8_BOM + stabilized.encode("utf-8") From c4e984e426a297cd65872db6b90769a4cb38bdb3 Mon Sep 17 00:00:00 2001 From: Michal Lazo Date: Fri, 10 Jul 2026 17:15:19 +0200 Subject: [PATCH 03/11] code_stabilizer.py --- code_stabilizer.py | 204 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100644 code_stabilizer.py diff --git a/code_stabilizer.py b/code_stabilizer.py new file mode 100644 index 0000000..43f44e8 --- /dev/null +++ b/code_stabilizer.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +""" +code_stabilizer.py + +Command-line tool that converts text files in-place from an ANSI codepage +encoding to UTF-8 with a byte-order mark (BOM). + +Argument syntax: + code_stabilizer.py [-s] [-a:] [...] + + -s Recurse into subdirectories when expanding wildcard patterns. + -a: Source ANSI codepage for files that are not already valid + UTF-8 (default: cp1250). Example: -a:cp1252 + file Exact path to a file. + pattern Wildcard pattern: *.pas, path\\*.dfm, etc. + @listfile Text file listing one path or pattern per line. + Lines starting with # are treated as comments. + +Conversion rules: + - Files that already start with a UTF-8 BOM are left untouched. + - Files whose content already decodes as valid UTF-8 (without BOM) simply + get the BOM prepended; their bytes are not re-encoded. + - All other files are decoded using the source codepage (-a) and + re-encoded as UTF-8. + - If the resulting bytes are identical to the original file, the file is + left untouched (no spurious writes / VCS changes). + - Files are replaced atomically via a temporary file. + +Exit code: 0 if all files were converted successfully, 1 if any failed. +""" + +import codecs +import glob +import os +import sys + +UTF8_BOM = codecs.BOM_UTF8 +DEFAULT_CODEPAGE = "cp1250" + + +def normalize_codepage(name): + name = name.strip() + if name.isdigit(): + name = "cp" + name + codecs.lookup(name) # raises LookupError if unknown + return name + + +def convert_file(file_name, codepage): + """Convert file_name in-place to UTF-8 with BOM. + + Returns True if the file was actually rewritten, False if it was + already in the target format (no disk write performed). + """ + with open(file_name, "rb") as f: + raw = f.read() + + body = raw[len(UTF8_BOM):] if raw.startswith(UTF8_BOM) else raw + + try: + text = body.decode("utf-8") + except UnicodeDecodeError: + text = body.decode(codepage) + + output = UTF8_BOM + text.encode("utf-8") + + if output == raw: + return False + + tmp_name = file_name + ".stab.tmp" + try: + with open(tmp_name, "wb") as f: + f.write(output) + os.replace(tmp_name, file_name) + except Exception: + if os.path.exists(tmp_name): + os.remove(tmp_name) + raise + + return True + + +class Processor: + def __init__(self, recursive, codepage): + self.recursive = recursive + self.codepage = codepage + self.success_count = 0 + self.fail_count = 0 + + def process_file(self, file_name): + sys.stdout.write(" {} ... ".format(file_name)) + try: + converted = convert_file(file_name, self.codepage) + sys.stdout.write("converted\n" if converted else "already up to date\n") + self.success_count += 1 + except Exception as e: + sys.stdout.write("FAILED: {}\n".format(e)) + self.fail_count += 1 + + def expand_and_process(self, pattern): + directory = os.path.dirname(pattern) or "." + file_pattern = os.path.basename(pattern) + + for match in sorted(glob.glob(os.path.join(directory, file_pattern))): + if os.path.isfile(match): + self.process_file(match) + + if self.recursive: + try: + subdirs = sorted( + entry.name for entry in os.scandir(directory) if entry.is_dir() + ) + except OSError: + subdirs = [] + for name in subdirs: + self.expand_and_process(os.path.join(directory, name, file_pattern)) + + def process_list_file(self, list_file_name): + if not os.path.isfile(list_file_name): + raise FileNotFoundError("List file not found: {}".format(list_file_name)) + + with open(list_file_name, "r", encoding="utf-8-sig") as f: + for line in f: + line = line.strip() + if line and not line.startswith("#"): + self.process_arg(line) + + def process_arg(self, arg): + if arg.startswith("@"): + self.process_list_file(arg[1:]) + elif "*" in arg or "?" in arg: + self.expand_and_process(arg) + else: + self.process_file(arg) + + +def print_usage(): + print("Usage: code_stabilizer.py [-s] [-a:] [...]") + print() + print("Converts text files in-place from an ANSI codepage to UTF-8 with BOM:") + print(" - files already starting with a UTF-8 BOM are left untouched") + print(" - files that already decode as valid UTF-8 just get the BOM added") + print(" - all other files are decoded using the source codepage and re-encoded") + print(" - file always starts with a UTF-8 BOM after conversion") + print() + print("Options:") + print(" -s Recurse into subdirectories when expanding wildcard patterns") + print(" -a: Source ANSI codepage for non-UTF-8 files (default: {})".format(DEFAULT_CODEPAGE)) + print() + print("Arguments:") + print(" file Exact path to a file") + print(" pattern Wildcard pattern (e.g. *.pas or forms\\*.dfm)") + print(" @listfile Text file with one path/pattern per line (# = comment)") + print() + print("Examples:") + print(" code_stabilizer.py MainForm.pas") + print(" code_stabilizer.py -s *.pas") + print(" code_stabilizer.py -a:cp1250 -s *.pas") + print(" code_stabilizer.py -s src\\*.pas @extra_files.txt") + print(" code_stabilizer.py @all_files.txt") + + +def main(argv): + if len(argv) == 0: + print_usage() + return 1 + + recursive = False + codepage = DEFAULT_CODEPAGE + args = [] + + for arg in argv: + lower = arg.lower() + if lower == "-s": + recursive = True + elif lower.startswith("-a:"): + try: + codepage = normalize_codepage(arg[3:]) + except LookupError: + print("Unknown codepage: {}".format(arg[3:]), file=sys.stderr) + return 1 + else: + args.append(arg) + + if not args: + print_usage() + return 1 + + processor = Processor(recursive, codepage) + for arg in args: + processor.process_arg(arg) + + print() + total = processor.success_count + processor.fail_count + if total == 0: + print("No files matched.") + else: + print("{} converted, {} failed.".format(processor.success_count, processor.fail_count)) + + return 1 if processor.fail_count > 0 else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) From ff8143daaa46792f63b9327cfc4fe4a879ec1d04 Mon Sep 17 00:00:00 2001 From: Michal Lazo Date: Mon, 13 Jul 2026 10:40:23 +0200 Subject: [PATCH 04/11] extend code_stabilizer with support for wide literals --- .gitignore | 1 + code_stabilizer.py | 123 ++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 118 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index f408480..774aac3 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,4 @@ __recovery/ # Logs and temp *.log *.tmp +/__pycache__/*.pyc diff --git a/code_stabilizer.py b/code_stabilizer.py index 43f44e8..5d87b62 100644 --- a/code_stabilizer.py +++ b/code_stabilizer.py @@ -6,9 +6,12 @@ encoding to UTF-8 with a byte-order mark (BOM). Argument syntax: - code_stabilizer.py [-s] [-a:] [...] + code_stabilizer.py [-s] [-w] [-a:] [...] -s Recurse into subdirectories when expanding wildcard patterns. + -w Add the L prefix to narrow string literals ("...") that + contain characters outside ASCII, turning "ahoj" into + L"ahoj" only where the wide prefix is actually needed. -a: Source ANSI codepage for files that are not already valid UTF-8 (default: cp1250). Example: -a:cp1252 file Exact path to a file. @@ -38,6 +41,103 @@ DEFAULT_CODEPAGE = "cp1250" +def widen_line(line, in_block_comment): + """Insert L before narrow "..." literals on one line that contain + a character outside ASCII. + + Understands // and /* */ comments, character literals ('...') and + backslash escapes, so quotes inside those never start a string. + Literals that already have a prefix (L"...", u8"...", macro"...") + are left alone. + + Returns (new_line, in_block_comment) where in_block_comment is the + comment state carried over to the next line. + """ + inserts = [] # indexes of opening quotes that need an L + i = 0 + n = len(line) + + while i < n: + if in_block_comment: + end = line.find("*/", i) + if end == -1: + i = n + else: + in_block_comment = False + i = end + 2 + continue + + c = line[i] + + if c == "/" and i + 1 < n: + if line[i + 1] == "/": + break # rest of the line is a comment + if line[i + 1] == "*": + in_block_comment = True + i += 2 + continue + + if c == "'": + # character literal: skip it, honouring escapes + i += 1 + while i < n: + if line[i] == "\\": + i += 2 + elif line[i] == "'": + i += 1 + break + else: + i += 1 + continue + + if c == '"': + start = i + # already prefixed (L"", u8"", R"", user macro"") -> leave alone + prefixed = start > 0 and (line[start - 1].isalnum() or line[start - 1] == "_") + needs_l = False + i += 1 + while i < n: + if line[i] == "\\": + i += 2 + elif line[i] == '"': + i += 1 + break + else: + if ord(line[i]) > 127: + needs_l = True + i += 1 + if needs_l and not prefixed: + inserts.append(start) + continue + + i += 1 + + if inserts: + parts = [] + prev = 0 + for pos in inserts: + parts.append(line[prev:pos]) + parts.append("L") + prev = pos + parts.append(line[prev:]) + line = "".join(parts) + + return line, in_block_comment + + +def add_wide_prefixes(text): + """Apply widen_line to every line of text, keeping line endings.""" + lines = text.splitlines(keepends=True) + in_block_comment = False + out = [] + for raw_line in lines: + stripped = raw_line.rstrip("\r\n") + eol = raw_line[len(stripped):] + new_line, in_block_comment = widen_line(stripped, in_block_comment) + out.append(new_line + eol) + return "".join(out) + + def normalize_codepage(name): name = name.strip() if name.isdigit(): @@ -46,9 +146,12 @@ def normalize_codepage(name): return name -def convert_file(file_name, codepage): +def convert_file(file_name, codepage, widen=False): """Convert file_name in-place to UTF-8 with BOM. + With widen=True, narrow string literals containing non-ASCII + characters also get the L prefix ("ahoj" -> L"ahoj"). + Returns True if the file was actually rewritten, False if it was already in the target format (no disk write performed). """ @@ -62,6 +165,9 @@ def convert_file(file_name, codepage): except UnicodeDecodeError: text = body.decode(codepage) + if widen: + text = add_wide_prefixes(text) + output = UTF8_BOM + text.encode("utf-8") if output == raw: @@ -81,16 +187,17 @@ def convert_file(file_name, codepage): class Processor: - def __init__(self, recursive, codepage): + def __init__(self, recursive, codepage, widen=False): self.recursive = recursive self.codepage = codepage + self.widen = widen self.success_count = 0 self.fail_count = 0 def process_file(self, file_name): sys.stdout.write(" {} ... ".format(file_name)) try: - converted = convert_file(file_name, self.codepage) + converted = convert_file(file_name, self.codepage, self.widen) sys.stdout.write("converted\n" if converted else "already up to date\n") self.success_count += 1 except Exception as e: @@ -135,7 +242,7 @@ def process_arg(self, arg): def print_usage(): - print("Usage: code_stabilizer.py [-s] [-a:] [...]") + print("Usage: code_stabilizer.py [-s] [-w] [-a:] [...]") print() print("Converts text files in-place from an ANSI codepage to UTF-8 with BOM:") print(" - files already starting with a UTF-8 BOM are left untouched") @@ -145,6 +252,7 @@ def print_usage(): print() print("Options:") print(" -s Recurse into subdirectories when expanding wildcard patterns") + print(' -w Add L prefix to "..." literals containing non-ASCII characters') print(" -a: Source ANSI codepage for non-UTF-8 files (default: {})".format(DEFAULT_CODEPAGE)) print() print("Arguments:") @@ -166,6 +274,7 @@ def main(argv): return 1 recursive = False + widen = False codepage = DEFAULT_CODEPAGE args = [] @@ -173,6 +282,8 @@ def main(argv): lower = arg.lower() if lower == "-s": recursive = True + elif lower == "-w": + widen = True elif lower.startswith("-a:"): try: codepage = normalize_codepage(arg[3:]) @@ -186,7 +297,7 @@ def main(argv): print_usage() return 1 - processor = Processor(recursive, codepage) + processor = Processor(recursive, codepage, widen) for arg in args: processor.process_arg(arg) From 6187c5e242a8a34596d6f3573ef55a4cc206f338 Mon Sep 17 00:00:00 2001 From: Michal Lazo Date: Mon, 13 Jul 2026 11:25:25 +0200 Subject: [PATCH 05/11] don't overwrite files where only UTF BOM will be change --- code_stabilizer.py | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/code_stabilizer.py b/code_stabilizer.py index 5d87b62..39903f2 100644 --- a/code_stabilizer.py +++ b/code_stabilizer.py @@ -12,6 +12,9 @@ -w Add the L prefix to narrow string literals ("...") that contain characters outside ASCII, turning "ahoj" into L"ahoj" only where the wide prefix is actually needed. + A file is only rewritten when its content actually + changes; if the sole difference would be the prepended + UTF-8 BOM, the file is left untouched. -a: Source ANSI codepage for files that are not already valid UTF-8 (default: cp1250). Example: -a:cp1252 file Exact path to a file. @@ -150,10 +153,14 @@ def convert_file(file_name, codepage, widen=False): """Convert file_name in-place to UTF-8 with BOM. With widen=True, narrow string literals containing non-ASCII - characters also get the L prefix ("ahoj" -> L"ahoj"). - - Returns True if the file was actually rewritten, False if it was - already in the target format (no disk write performed). + characters also get the L prefix ("ahoj" -> L"ahoj"), and a file + whose only difference from the result would be the prepended BOM + is left untouched (no rewrite just to add a BOM). + + Returns one of: + "converted" file was rewritten + "unchanged" file was already in the target format + "skipped" widen mode and the only change would be the BOM """ with open(file_name, "rb") as f: raw = f.read() @@ -171,7 +178,10 @@ def convert_file(file_name, codepage, widen=False): output = UTF8_BOM + text.encode("utf-8") if output == raw: - return False + return "unchanged" + + if widen and output == UTF8_BOM + raw: + return "skipped" tmp_name = file_name + ".stab.tmp" try: @@ -183,7 +193,7 @@ def convert_file(file_name, codepage, widen=False): os.remove(tmp_name) raise - return True + return "converted" class Processor: @@ -197,8 +207,13 @@ def __init__(self, recursive, codepage, widen=False): def process_file(self, file_name): sys.stdout.write(" {} ... ".format(file_name)) try: - converted = convert_file(file_name, self.codepage, self.widen) - sys.stdout.write("converted\n" if converted else "already up to date\n") + result = convert_file(file_name, self.codepage, self.widen) + messages = { + "converted": "converted", + "unchanged": "already up to date", + "skipped": "skipped (only BOM would change)", + } + sys.stdout.write(messages[result] + "\n") self.success_count += 1 except Exception as e: sys.stdout.write("FAILED: {}\n".format(e)) @@ -252,7 +267,8 @@ def print_usage(): print() print("Options:") print(" -s Recurse into subdirectories when expanding wildcard patterns") - print(' -w Add L prefix to "..." literals containing non-ASCII characters') + print(' -w Add L prefix to "..." literals containing non-ASCII characters;') + print(" a file whose only change would be the BOM is left untouched") print(" -a: Source ANSI codepage for non-UTF-8 files (default: {})".format(DEFAULT_CODEPAGE)) print() print("Arguments:") From ba329fc54bdb97c2949e95f6eefcf04b97964b82 Mon Sep 17 00:00:00 2001 From: Michal Lazo Date: Mon, 13 Jul 2026 17:16:26 +0200 Subject: [PATCH 06/11] update --- code_stabilizer.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/code_stabilizer.py b/code_stabilizer.py index 39903f2..39058f7 100644 --- a/code_stabilizer.py +++ b/code_stabilizer.py @@ -23,7 +23,8 @@ Lines starting with # are treated as comments. Conversion rules: - - Files that already start with a UTF-8 BOM are left untouched. + - Files that already start with a UTF-8 BOM are considered stabilized + and are left untouched, even when -w is given. - Files whose content already decodes as valid UTF-8 (without BOM) simply get the BOM prepended; their bytes are not re-encoded. - All other files are decoded using the source codepage (-a) and @@ -157,15 +158,22 @@ def convert_file(file_name, codepage, widen=False): whose only difference from the result would be the prepended BOM is left untouched (no rewrite just to add a BOM). + Files that already start with a UTF-8 BOM are considered stabilized + and are never touched, not even by the widen pass. + Returns one of: "converted" file was rewritten "unchanged" file was already in the target format "skipped" widen mode and the only change would be the BOM + "has_bom" file already starts with a UTF-8 BOM, kept as is """ with open(file_name, "rb") as f: raw = f.read() - body = raw[len(UTF8_BOM):] if raw.startswith(UTF8_BOM) else raw + if raw.startswith(UTF8_BOM): + return "has_bom" + + body = raw try: text = body.decode("utf-8") @@ -212,6 +220,7 @@ def process_file(self, file_name): "converted": "converted", "unchanged": "already up to date", "skipped": "skipped (only BOM would change)", + "has_bom": "kept (already has UTF-8 BOM)", } sys.stdout.write(messages[result] + "\n") self.success_count += 1 @@ -260,7 +269,7 @@ def print_usage(): print("Usage: code_stabilizer.py [-s] [-w] [-a:] [...]") print() print("Converts text files in-place from an ANSI codepage to UTF-8 with BOM:") - print(" - files already starting with a UTF-8 BOM are left untouched") + print(" - files already starting with a UTF-8 BOM are left untouched (even with -w)") print(" - files that already decode as valid UTF-8 just get the BOM added") print(" - all other files are decoded using the source codepage and re-encoded") print(" - file always starts with a UTF-8 BOM after conversion") From b521b84da1cec5d664fca4fede9f0e7884bd8a20 Mon Sep 17 00:00:00 2001 From: Michal Lazo Date: Mon, 13 Jul 2026 20:55:31 +0200 Subject: [PATCH 07/11] force --- code_stabilizer.py | 43 +++++++++++++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/code_stabilizer.py b/code_stabilizer.py index 39058f7..3cc5739 100644 --- a/code_stabilizer.py +++ b/code_stabilizer.py @@ -6,7 +6,7 @@ encoding to UTF-8 with a byte-order mark (BOM). Argument syntax: - code_stabilizer.py [-s] [-w] [-a:] [...] + code_stabilizer.py [-s] [-w] [-f] [-a:] [...] -s Recurse into subdirectories when expanding wildcard patterns. -w Add the L prefix to narrow string literals ("...") that @@ -15,6 +15,10 @@ A file is only rewritten when its content actually changes; if the sole difference would be the prepended UTF-8 BOM, the file is left untouched. + -f Force the stabilization even when the file would not + need it otherwise: files already starting with a UTF-8 + BOM are processed too (including the -w widen pass), + and the BOM is written even when it is the only change. -a: Source ANSI codepage for files that are not already valid UTF-8 (default: cp1250). Example: -a:cp1252 file Exact path to a file. @@ -24,7 +28,7 @@ Conversion rules: - Files that already start with a UTF-8 BOM are considered stabilized - and are left untouched, even when -w is given. + and are left untouched, even when -w is given (unless -f forces it). - Files whose content already decodes as valid UTF-8 (without BOM) simply get the BOM prepended; their bytes are not re-encoded. - All other files are decoded using the source codepage (-a) and @@ -150,7 +154,7 @@ def normalize_codepage(name): return name -def convert_file(file_name, codepage, widen=False): +def convert_file(file_name, codepage, widen=False, force=False): """Convert file_name in-place to UTF-8 with BOM. With widen=True, narrow string literals containing non-ASCII @@ -159,7 +163,12 @@ def convert_file(file_name, codepage, widen=False): is left untouched (no rewrite just to add a BOM). Files that already start with a UTF-8 BOM are considered stabilized - and are never touched, not even by the widen pass. + and are not touched, not even by the widen pass. + + With force=True the file is stabilized even when it would not need + it otherwise: files that already have a BOM are still processed + (e.g. by the widen pass), and the BOM is written even when it would + be the only change. Returns one of: "converted" file was rewritten @@ -171,9 +180,11 @@ def convert_file(file_name, codepage, widen=False): raw = f.read() if raw.startswith(UTF8_BOM): - return "has_bom" - - body = raw + if not force: + return "has_bom" + body = raw[len(UTF8_BOM):] + else: + body = raw try: text = body.decode("utf-8") @@ -188,7 +199,7 @@ def convert_file(file_name, codepage, widen=False): if output == raw: return "unchanged" - if widen and output == UTF8_BOM + raw: + if widen and not force and output == UTF8_BOM + raw: return "skipped" tmp_name = file_name + ".stab.tmp" @@ -205,17 +216,18 @@ def convert_file(file_name, codepage, widen=False): class Processor: - def __init__(self, recursive, codepage, widen=False): + def __init__(self, recursive, codepage, widen=False, force=False): self.recursive = recursive self.codepage = codepage self.widen = widen + self.force = force self.success_count = 0 self.fail_count = 0 def process_file(self, file_name): sys.stdout.write(" {} ... ".format(file_name)) try: - result = convert_file(file_name, self.codepage, self.widen) + result = convert_file(file_name, self.codepage, self.widen, self.force) messages = { "converted": "converted", "unchanged": "already up to date", @@ -266,10 +278,11 @@ def process_arg(self, arg): def print_usage(): - print("Usage: code_stabilizer.py [-s] [-w] [-a:] [...]") + print("Usage: code_stabilizer.py [-s] [-w] [-f] [-a:] [...]") print() print("Converts text files in-place from an ANSI codepage to UTF-8 with BOM:") print(" - files already starting with a UTF-8 BOM are left untouched (even with -w)") + print(" unless -f forces their processing") print(" - files that already decode as valid UTF-8 just get the BOM added") print(" - all other files are decoded using the source codepage and re-encoded") print(" - file always starts with a UTF-8 BOM after conversion") @@ -278,6 +291,9 @@ def print_usage(): print(" -s Recurse into subdirectories when expanding wildcard patterns") print(' -w Add L prefix to "..." literals containing non-ASCII characters;') print(" a file whose only change would be the BOM is left untouched") + print(" -f Force stabilization even when the file would not need it:") + print(" processes files that already have a BOM and writes the BOM") + print(" even when it is the only change") print(" -a: Source ANSI codepage for non-UTF-8 files (default: {})".format(DEFAULT_CODEPAGE)) print() print("Arguments:") @@ -300,6 +316,7 @@ def main(argv): recursive = False widen = False + force = False codepage = DEFAULT_CODEPAGE args = [] @@ -309,6 +326,8 @@ def main(argv): recursive = True elif lower == "-w": widen = True + elif lower == "-f": + force = True elif lower.startswith("-a:"): try: codepage = normalize_codepage(arg[3:]) @@ -322,7 +341,7 @@ def main(argv): print_usage() return 1 - processor = Processor(recursive, codepage, widen) + processor = Processor(recursive, codepage, widen, force) for arg in args: processor.process_arg(arg) From acd890c51974c436a5af28ee1f95824cd1925b90 Mon Sep 17 00:00:00 2001 From: Michal Lazo Date: Mon, 13 Jul 2026 22:22:24 +0200 Subject: [PATCH 08/11] handle TEXT --- code_stabilizer.py | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/code_stabilizer.py b/code_stabilizer.py index 3cc5739..de7542c 100644 --- a/code_stabilizer.py +++ b/code_stabilizer.py @@ -12,6 +12,8 @@ -w Add the L prefix to narrow string literals ("...") that contain characters outside ASCII, turning "ahoj" into L"ahoj" only where the wide prefix is actually needed. + Literals wrapped in the TEXT() macro are left alone, + because TEXT() adds the L prefix itself. A file is only rewritten when its content actually changes; if the sole difference would be the prepended UTF-8 BOM, the file is left untouched. @@ -49,6 +51,25 @@ DEFAULT_CODEPAGE = "cp1250" +def preceded_by_text_macro(line, start): + """True if the literal opening at line[start] is the argument of the + TEXT()/_TEXT()/__TEXT() widening macro, e.g. TEXT("ahoj"). + Such literals must stay narrow: the macro expands to L##literal and + an explicit L prefix would produce the invalid token LL"...".""" + i = start - 1 + while i >= 0 and line[i] in " \t": + i -= 1 + if i < 0 or line[i] != "(": + return False + i -= 1 + while i >= 0 and line[i] in " \t": + i -= 1 + end = i + while i >= 0 and (line[i].isalnum() or line[i] == "_"): + i -= 1 + return line[i + 1:end + 1] in ("TEXT", "_TEXT", "__TEXT") + + def widen_line(line, in_block_comment): """Insert L before narrow "..." literals on one line that contain a character outside ASCII. @@ -56,7 +77,7 @@ def widen_line(line, in_block_comment): Understands // and /* */ comments, character literals ('...') and backslash escapes, so quotes inside those never start a string. Literals that already have a prefix (L"...", u8"...", macro"...") - are left alone. + or are wrapped in the TEXT() macro are left alone. Returns (new_line, in_block_comment) where in_block_comment is the comment state carried over to the next line. @@ -114,7 +135,7 @@ def widen_line(line, in_block_comment): if ord(line[i]) > 127: needs_l = True i += 1 - if needs_l and not prefixed: + if needs_l and not prefixed and not preceded_by_text_macro(line, start): inserts.append(start) continue From 6568d2a0baf68280033fc99b744c5dded7b30102 Mon Sep 17 00:00:00 2001 From: Michal Lazo Date: Mon, 13 Jul 2026 22:43:54 +0200 Subject: [PATCH 09/11] upgrade --- code_stabilizer.py | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/code_stabilizer.py b/code_stabilizer.py index de7542c..7908afa 100644 --- a/code_stabilizer.py +++ b/code_stabilizer.py @@ -13,7 +13,10 @@ contain characters outside ASCII, turning "ahoj" into L"ahoj" only where the wide prefix is actually needed. Literals wrapped in the TEXT() macro are left alone, - because TEXT() adds the L prefix itself. + because TEXT() adds the L prefix itself. On lines with + a ternary ? operator where some literal gets the L, + the remaining literals on the line get it too, so both + branches of cond ? "pan" : "paní" are wide. A file is only rewritten when its content actually changes; if the sole difference would be the prepended UTF-8 BOM, the file is left untouched. @@ -79,10 +82,17 @@ def widen_line(line, in_block_comment): Literals that already have a prefix (L"...", u8"...", macro"...") or are wrapped in the TEXT() macro are left alone. + When the line contains a ternary ? operator and at least one literal + gets the L prefix, all remaining eligible literals on the line get it + too, so both branches of cond ? "pan" : "paní" stay the same type. + Returns (new_line, in_block_comment) where in_block_comment is the comment state carried over to the next line. """ - inserts = [] # indexes of opening quotes that need an L + inserts = [] # indexes of opening quotes that need an L + ascii_literals = [] # eligible literals that are pure ASCII + has_ternary = False # saw a ? outside strings/chars/comments + has_wide = False # saw an existing L"..." literal i = 0 n = len(line) @@ -123,6 +133,9 @@ def widen_line(line, in_block_comment): start = i # already prefixed (L"", u8"", R"", user macro"") -> leave alone prefixed = start > 0 and (line[start - 1].isalnum() or line[start - 1] == "_") + if (prefixed and line[start - 1] == "L" + and (start < 2 or not (line[start - 2].isalnum() or line[start - 2] == "_"))): + has_wide = True needs_l = False i += 1 while i < n: @@ -135,12 +148,21 @@ def widen_line(line, in_block_comment): if ord(line[i]) > 127: needs_l = True i += 1 - if needs_l and not prefixed and not preceded_by_text_macro(line, start): - inserts.append(start) + if not prefixed and not preceded_by_text_macro(line, start): + if needs_l: + inserts.append(start) + else: + ascii_literals.append(start) continue + if c == "?": + has_ternary = True + i += 1 + if has_ternary and ascii_literals and (inserts or has_wide): + inserts = sorted(inserts + ascii_literals) + if inserts: parts = [] prev = 0 From 74be40ac17859355ee6e1faa118295b916977d30 Mon Sep 17 00:00:00 2001 From: Michal Lazo Date: Tue, 14 Jul 2026 09:52:47 +0200 Subject: [PATCH 10/11] simplify --- code_stabilizer.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/code_stabilizer.py b/code_stabilizer.py index 7908afa..0bc3d92 100644 --- a/code_stabilizer.py +++ b/code_stabilizer.py @@ -13,10 +13,11 @@ contain characters outside ASCII, turning "ahoj" into L"ahoj" only where the wide prefix is actually needed. Literals wrapped in the TEXT() macro are left alone, - because TEXT() adds the L prefix itself. On lines with - a ternary ? operator where some literal gets the L, - the remaining literals on the line get it too, so both - branches of cond ? "pan" : "paní" are wide. + because TEXT() adds the L prefix itself. When at least + one literal on a line needs the L prefix, the remaining + literals on that line get it too, so initializers like + {"", "minut", "dnů"} and ternary branches like + cond ? "pan" : "paní" stay the same type. A file is only rewritten when its content actually changes; if the sole difference would be the prepended UTF-8 BOM, the file is left untouched. @@ -82,9 +83,11 @@ def widen_line(line, in_block_comment): Literals that already have a prefix (L"...", u8"...", macro"...") or are wrapped in the TEXT() macro are left alone. - When the line contains a ternary ? operator and at least one literal - gets the L prefix, all remaining eligible literals on the line get it - too, so both branches of cond ? "pan" : "paní" stay the same type. + When at least one literal on the line needs the L prefix, all + remaining eligible literals on the line get it too, so aggregate + initializers like {"", "minut", "dnů"} or both branches of + cond ? "pan" : "paní" stay the same type. On lines with a ternary + ? operator an already existing L"..." literal triggers this as well. Returns (new_line, in_block_comment) where in_block_comment is the comment state carried over to the next line. @@ -160,7 +163,7 @@ def widen_line(line, in_block_comment): i += 1 - if has_ternary and ascii_literals and (inserts or has_wide): + if ascii_literals and (inserts or (has_ternary and has_wide)): inserts = sorted(inserts + ascii_literals) if inserts: From 7f7061b5dc71f75df90048a528c4fa3d21774c69 Mon Sep 17 00:00:00 2001 From: Michal Lazo Date: Thu, 16 Jul 2026 10:02:18 +0200 Subject: [PATCH 11/11] converter for converting to utf --- converter.py | 210 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 210 insertions(+) create mode 100644 converter.py diff --git a/converter.py b/converter.py new file mode 100644 index 0000000..5ca3f90 --- /dev/null +++ b/converter.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 +""" +converter.py + +Command-line tool that converts text files in-place from an ANSI codepage +encoding (e.g. cp1250) to UTF-8. Nothing else is changed: no BOM is added +and the text content is kept as it is. + +Argument syntax: + converter.py [-s] [-a:] [...] + + -s Recurse into subdirectories when expanding wildcard patterns. + -a: Source ANSI codepage of the files (default: cp1250). + Example: -a:cp1252 or -a:1252 + file Exact path to a file. + pattern Wildcard pattern: *.pas, path\\*.dfm, etc. + @listfile Text file listing one path or pattern per line. + Lines starting with # are treated as comments. + +Conversion rules: + - Files that already decode as valid UTF-8 (with or without BOM) are + left untouched; converting them again would corrupt the text. + - All other files are decoded using the source codepage (-a) and + re-encoded as UTF-8. + - If the resulting bytes are identical to the original file, the file + is left untouched (no spurious writes / VCS changes). + - Files are replaced atomically via a temporary file. + +Exit code: 0 if all files were converted successfully, 1 if any failed. +""" + +import codecs +import glob +import os +import sys + +UTF8_BOM = codecs.BOM_UTF8 +DEFAULT_CODEPAGE = "cp1250" + + +def normalize_codepage(name): + name = name.strip() + if name.isdigit(): + name = "cp" + name + codecs.lookup(name) # raises LookupError if unknown + return name + + +def convert_file(file_name, codepage): + """Convert file_name in-place from codepage to UTF-8. + + Returns one of: + "converted" file was rewritten + "unchanged" file was already valid UTF-8 (or ASCII), kept as is + """ + with open(file_name, "rb") as f: + raw = f.read() + + body = raw[len(UTF8_BOM):] if raw.startswith(UTF8_BOM) else raw + + try: + body.decode("utf-8") + return "unchanged" + except UnicodeDecodeError: + pass + + text = raw.decode(codepage) + output = text.encode("utf-8") + + if output == raw: + return "unchanged" + + tmp_name = file_name + ".conv.tmp" + try: + with open(tmp_name, "wb") as f: + f.write(output) + os.replace(tmp_name, file_name) + except Exception: + if os.path.exists(tmp_name): + os.remove(tmp_name) + raise + + return "converted" + + +class Processor: + def __init__(self, recursive, codepage): + self.recursive = recursive + self.codepage = codepage + self.success_count = 0 + self.fail_count = 0 + + def process_file(self, file_name): + sys.stdout.write(" {} ... ".format(file_name)) + try: + result = convert_file(file_name, self.codepage) + messages = { + "converted": "converted", + "unchanged": "already valid UTF-8", + } + sys.stdout.write(messages[result] + "\n") + self.success_count += 1 + except Exception as e: + sys.stdout.write("FAILED: {}\n".format(e)) + self.fail_count += 1 + + def expand_and_process(self, pattern): + directory = os.path.dirname(pattern) or "." + file_pattern = os.path.basename(pattern) + + for match in sorted(glob.glob(os.path.join(directory, file_pattern))): + if os.path.isfile(match): + self.process_file(match) + + if self.recursive: + try: + subdirs = sorted( + entry.name for entry in os.scandir(directory) if entry.is_dir() + ) + except OSError: + subdirs = [] + for name in subdirs: + self.expand_and_process(os.path.join(directory, name, file_pattern)) + + def process_list_file(self, list_file_name): + if not os.path.isfile(list_file_name): + raise FileNotFoundError("List file not found: {}".format(list_file_name)) + + with open(list_file_name, "r", encoding="utf-8-sig") as f: + for line in f: + line = line.strip() + if line and not line.startswith("#"): + self.process_arg(line) + + def process_arg(self, arg): + if arg.startswith("@"): + self.process_list_file(arg[1:]) + elif "*" in arg or "?" in arg: + self.expand_and_process(arg) + else: + self.process_file(arg) + + +def print_usage(): + print("Usage: converter.py [-s] [-a:] [...]") + print() + print("Converts text files in-place from an ANSI codepage to UTF-8:") + print(" - files that already decode as valid UTF-8 are left untouched") + print(" - all other files are decoded using the source codepage and re-encoded") + print(" - no BOM is added and the text content is not modified") + print() + print("Options:") + print(" -s Recurse into subdirectories when expanding wildcard patterns") + print(" -a: Source ANSI codepage of the files (default: {})".format(DEFAULT_CODEPAGE)) + print() + print("Arguments:") + print(" file Exact path to a file") + print(" pattern Wildcard pattern (e.g. *.pas or forms\\*.dfm)") + print(" @listfile Text file with one path/pattern per line (# = comment)") + print() + print("Examples:") + print(" converter.py MainForm.pas") + print(" converter.py -s *.pas") + print(" converter.py -a:cp1250 -s *.pas") + print(" converter.py -s src\\*.pas @extra_files.txt") + print(" converter.py @all_files.txt") + + +def main(argv): + if len(argv) == 0: + print_usage() + return 1 + + recursive = False + codepage = DEFAULT_CODEPAGE + args = [] + + for arg in argv: + lower = arg.lower() + if lower == "-s": + recursive = True + elif lower.startswith("-a:"): + try: + codepage = normalize_codepage(arg[3:]) + except LookupError: + print("Unknown codepage: {}".format(arg[3:]), file=sys.stderr) + return 1 + else: + args.append(arg) + + if not args: + print_usage() + return 1 + + processor = Processor(recursive, codepage) + for arg in args: + processor.process_arg(arg) + + print() + total = processor.success_count + processor.fail_count + if total == 0: + print("No files matched.") + else: + print("{} converted, {} failed.".format(processor.success_count, processor.fail_count)) + + return 1 if processor.fail_count > 0 else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:]))