From afe0abd52307d0ed7a4f55f99c3694dee31b866d Mon Sep 17 00:00:00 2001 From: sr9000 Date: Thu, 2 Jul 2026 23:00:51 +0300 Subject: [PATCH 1/3] base 64 limit --- agent.md | 5 ++++ ai-memory/repo-map.md | 3 +++ app/app_settings.py | 23 ++++++++++++++++++ documents/composition/demo_data.py | 6 ++--- settings.py | 6 +++-- state/edit_limits.py | 10 ++++++++ tests/perf/test_decode_amplification.py | 23 +++++++++++++----- .../test_context_menu_base64_file_actions.py | 6 ++--- tests/test_edit_limits_menu.py | 9 ++++++- tests/test_inference_limits.py | 24 +++++++++++++++++++ tests/test_kind_switch_coercion.py | 5 ++-- tests/test_phase_5_3_status_bar_breadcrumb.py | 7 ++++-- tests/test_tree_correctness.py | 4 ++-- tests/test_type_editing.py | 2 +- tree/types.py | 22 +++++++++++++---- 15 files changed, 128 insertions(+), 27 deletions(-) diff --git a/agent.md b/agent.md index fcbd898..8fbf168 100644 --- a/agent.md +++ b/agent.md @@ -42,6 +42,11 @@ Hard rules: 3. **UI uses proxy model** - Map indices proxy↔source before touching tree items. +4. **Base64 auto-inference has a persisted minimum-length guard** + - Automatic string→`BYTES`/`ZLIB`/`GZIP` inference only runs when the string length meets the current + `edit_limits/base64_min_length_chars` threshold (default `100`). + - Short valid base64 stays `STRING` unless the type is pinned explicitly or the threshold is lowered. + ## 4) Isolation constraints (must hold) - `editors/inline/*`, `editors/windowed/*` must not import `app/`, `documents/`, `tree/`. diff --git a/ai-memory/repo-map.md b/ai-memory/repo-map.md index 43be18c..2900226 100644 --- a/ai-memory/repo-map.md +++ b/ai-memory/repo-map.md @@ -41,6 +41,9 @@ Model". primitive in `tree_actions/anchors.py`. This ensures consistency across different UI interactions. - **Type-Centric**: Type inference (`tree/types.py`) and coercion (`tree/item_coercion.py`) are the source of truth for how data is handled. Don't scatter type logic in the UI. +- **Base64 inference guard**: Automatic base64-family detection in `tree/types.py` uses valid syntax + strict decode, + but also a persisted minimum string length from `QSettings` (`edit_limits/base64_min_length_chars`, default `100`) to + reduce false positives on short human-readable strings. - **Surgical Model Updates**: The `DiffApplier` (`undo/diff.py`) is used during Undo/Redo to emit minimal Qt signals. This preserves UI state like selection and expansion that would be lost on a full model reset. **Important**: `DiffApplier.apply()` bypasses `JsonTreeItem.set_data()` — special type handling (e.g., `RAW_FLOAT` diff --git a/app/app_settings.py b/app/app_settings.py index 5a95243..5e73e70 100644 --- a/app/app_settings.py +++ b/app/app_settings.py @@ -14,10 +14,12 @@ from app.dialogs.secret_prefixes_dlg import SecretPrefixesDialog from state.edit_limits import ( get_attach_file_warning_limit_bytes, + get_base64_inference_min_length_chars, get_binary_edit_warning_limit_bytes, get_multiline_edit_warning_limit_chars, get_string_edit_warning_limit_chars, set_attach_file_warning_limit_bytes, + set_base64_inference_min_length_chars, set_binary_edit_warning_limit_bytes, set_multiline_edit_warning_limit_chars, set_string_edit_warning_limit_chars, @@ -60,16 +62,19 @@ def _build_edit_limits_menu(self) -> None: self.limit_multiline_action = QAction(win) self.limit_binary_action = QAction(win) self.limit_attach_action = QAction(win) + self.limit_base64_min_length_action = QAction(win) self.limit_string_action.triggered.connect(self._set_string_warning_limit) self.limit_multiline_action.triggered.connect(self._set_multiline_warning_limit) self.limit_binary_action.triggered.connect(self._set_binary_warning_limit) self.limit_attach_action.triggered.connect(self._set_attach_warning_limit) + self.limit_base64_min_length_action.triggered.connect(self._set_base64_inference_min_length) self.limits_menu.addAction(self.limit_string_action) self.limits_menu.addAction(self.limit_multiline_action) self.limits_menu.addAction(self.limit_binary_action) self.limits_menu.addAction(self.limit_attach_action) + self.limits_menu.addAction(self.limit_base64_min_length_action) self.limits_menu.aboutToShow.connect(self.refresh_edit_limits_menu_entries) self.refresh_edit_limits_menu_entries() @@ -82,6 +87,7 @@ def refresh_edit_limits_menu_entries(self) -> None: multiline_limit = get_multiline_edit_warning_limit_chars() binary_limit = get_binary_edit_warning_limit_bytes() attach_limit = get_attach_file_warning_limit_bytes() + base64_min_length = get_base64_inference_min_length_chars() self.limit_string_action.setText( win.tr("String edit limit... ({value} chars)").format(value=counts(string_limit)) @@ -95,6 +101,9 @@ def refresh_edit_limits_menu_entries(self) -> None: self.limit_attach_action.setText( win.tr("Attach file size limit... ({value})").format(value=format_bytes(attach_limit)) ) + self.limit_base64_min_length_action.setText( + win.tr("Base64 inference minimum length... ({value} chars)").format(value=counts(base64_min_length)) + ) def _prompt_limit_value(self, *, title: str, label: str, current: int) -> int | None: value, ok = QInputDialog.getInt(self._win, title, label, current, 1, 2_147_483_647, 1) @@ -157,3 +166,17 @@ def _set_attach_warning_limit(self) -> None: set_attach_file_warning_limit_bytes(value) self.refresh_edit_limits_menu_entries() win.statusBar.showMessage(win.tr("Updated attach file warning limit"), 2000) + + def _set_base64_inference_min_length(self) -> None: + win = self._win + current = get_base64_inference_min_length_chars() + value = self._prompt_limit_value( + title=win.tr("Base64 Inference Minimum Length"), + label=win.tr("Interpret strings as base64 starting at length (chars):"), + current=current, + ) + if value is None: + return + set_base64_inference_min_length_chars(value) + self.refresh_edit_limits_menu_entries() + win.statusBar.showMessage(win.tr("Updated base64 inference minimum length"), 2000) diff --git a/documents/composition/demo_data.py b/documents/composition/demo_data.py index a4dc6f0..8ea4f95 100644 --- a/documents/composition/demo_data.py +++ b/documents/composition/demo_data.py @@ -30,9 +30,9 @@ def build_demo_data() -> dict[str, Any]: "utf8-text": "Line 1\nLine 2\n\u03a9", "password": "plainsecret", "private_key": "-----BEGIN KEY-----\nabc\n-----END KEY-----", - "bytes": base64.b64encode(b"hello " * 10).decode(), - "zlib": base64.b64encode(zlib.compress(b"hello " * 10)).decode(), - "gzip": base64.b64encode(gzip.compress(b"hello " * 10)).decode(), + "bytes": base64.b64encode(b"hello " * 20).decode(), + "zlib": base64.b64encode(zlib.compress(bytes(range(256)))).decode(), + "gzip": base64.b64encode(gzip.compress(bytes(range(256)))).decode(), "date": "2024-06-01", "time": "12:34", "datetime": "2024-06-01 12:34:56", diff --git a/settings.py b/settings.py index f8a9b09..ee0ad50 100644 --- a/settings.py +++ b/settings.py @@ -9,6 +9,7 @@ BINARY_EDIT_WARNING_LIMIT_BYTES = 100 * 1024 STRING_EDIT_WARNING_LIMIT_CHARS = 10_000 MULTILINE_EDIT_WARNING_LIMIT_CHARS = 100_000 +BASE64_INFERENCE_MIN_LENGTH_CHARS = 100 # Secret field detection and masking defaults. SECRET_WORD_PREFIXES: tuple[str, ...] = ( @@ -55,8 +56,9 @@ # Design decisions: # - No INFERENCE_MAX_TOTAL_CHARS: individual gates (datetime, affix, color) # effectively skip all unnecessary checks; a top-level fast path is redundant. -# - No INFERENCE_MAX_BASE64_PROBE_CHARS: base64 uses content-based syntax -# validation (len mod 4 + alphabet regex) instead of a length cap. +# - No INFERENCE_MAX_BASE64_PROBE_CHARS: base64 does not use a performance +# cap; it uses content-based syntax validation plus a separate minimum-length +# false-positive guard. # - No EDITABLE_DECODE_LIMIT_BYTES: if base64 syntax is valid, decode is allowed. # parse_datetime_text() regex and datetime conversion. diff --git a/state/edit_limits.py b/state/edit_limits.py index 7d23248..6004d0d 100644 --- a/state/edit_limits.py +++ b/state/edit_limits.py @@ -4,6 +4,7 @@ from settings import ( APPLICATION_ID, + BASE64_INFERENCE_MIN_LENGTH_CHARS, BINARY_ATTACH_WARNING_LIMIT_BYTES, BINARY_EDIT_WARNING_LIMIT_BYTES, MULTILINE_EDIT_WARNING_LIMIT_CHARS, @@ -14,6 +15,7 @@ _MULTILINE_LIMIT_KEY = "edit_limits/multiline_chars" _BINARY_EDIT_LIMIT_KEY = "edit_limits/binary_bytes" _ATTACH_LIMIT_KEY = "edit_limits/attach_bytes" +_BASE64_MIN_LENGTH_KEY = "edit_limits/base64_min_length_chars" def _settings() -> QSettings: @@ -58,3 +60,11 @@ def get_attach_file_warning_limit_bytes() -> int: def set_attach_file_warning_limit_bytes(limit: int) -> None: _settings().setValue(_ATTACH_LIMIT_KEY, _coerce_positive_int(limit, default=BINARY_ATTACH_WARNING_LIMIT_BYTES)) + + +def get_base64_inference_min_length_chars() -> int: + return _coerce_positive_int(_settings().value(_BASE64_MIN_LENGTH_KEY), default=BASE64_INFERENCE_MIN_LENGTH_CHARS) + + +def set_base64_inference_min_length_chars(limit: int) -> None: + _settings().setValue(_BASE64_MIN_LENGTH_KEY, _coerce_positive_int(limit, default=BASE64_INFERENCE_MIN_LENGTH_CHARS)) diff --git a/tests/perf/test_decode_amplification.py b/tests/perf/test_decode_amplification.py index e2cfe62..b897088 100644 --- a/tests/perf/test_decode_amplification.py +++ b/tests/perf/test_decode_amplification.py @@ -15,6 +15,8 @@ import pytest +import settings +from state.edit_limits import set_base64_inference_min_length_chars from tests.perf.harness import classify_rows, measure_call, scaling_rows from tests.perf.string_corpus import DEFAULT_SIZES, base64_like, plain_ascii @@ -31,28 +33,37 @@ _collected_rows: list = [] +@pytest.fixture(autouse=True) +def _lower_base64_inference_threshold_for_perf_valid_fixtures(): + previous = settings.BASE64_INFERENCE_MIN_LENGTH_CHARS + set_base64_inference_min_length_chars(20) + try: + yield + finally: + set_base64_inference_min_length_chars(previous) + + # --------------------------------------------------------------------------- # Valid small fixtures for successful decode paths # --------------------------------------------------------------------------- def _make_valid_bytes_fixture() -> str: - """Create a valid base64-encoded BYTES fixture (at least 20 chars for _B64_RE).""" - # Need at least 20 chars to match _B64_RE pattern + """Create a valid base64-encoded BYTES fixture long enough to pass the default minimum-length guard.""" raw = b"Hello, World! This is a longer test message for base64 encoding." return base64.b64encode(raw).decode("ascii") def _make_valid_zlib_fixture() -> str: """Create a valid base64-encoded ZLIB fixture.""" - raw = b"Hello, World! This is compressed data." + raw = b"Hello, World! This is compressed data. " * 8 compressed = zlib.compress(raw) return base64.b64encode(compressed).decode("ascii") def _make_valid_gzip_fixture() -> str: """Create a valid base64-encoded GZIP fixture.""" - raw = b"Hello, World! This is gzip compressed data." + raw = b"Hello, World! This is gzip compressed data. " * 8 compressed = gzip.compress(raw) return base64.b64encode(compressed).decode("ascii") @@ -135,13 +146,13 @@ def test_valid_zlib_fixture_decodes(self): """Valid ZLIB fixture should decompress successfully.""" fixture = _make_valid_zlib_fixture() result = decode_bytes(fixture, JsonType.ZLIB) - assert result == b"Hello, World! This is compressed data." + assert result == (b"Hello, World! This is compressed data. " * 8) def test_valid_gzip_fixture_decodes(self): """Valid GZIP fixture should decompress successfully.""" fixture = _make_valid_gzip_fixture() result = decode_bytes(fixture, JsonType.GZIP) - assert result == b"Hello, World! This is gzip compressed data." + assert result == (b"Hello, World! This is gzip compressed data. " * 8) def test_parse_json_type_detects_bytes(self): """parse_json_type should detect valid base64 as BYTES.""" diff --git a/tests/test_context_menu_base64_file_actions.py b/tests/test_context_menu_base64_file_actions.py index 85f2e95..34616dc 100644 --- a/tests/test_context_menu_base64_file_actions.py +++ b/tests/test_context_menu_base64_file_actions.py @@ -29,7 +29,7 @@ def _value_at(tab: JsonTab, path: tuple[int, ...]) -> str: def test_attach_base64_from_file_replaces_value(qtbot, tmp_path, monkeypatch): - initial = base64.b64encode(b"seed payload for bytes").decode("ascii") + initial = base64.b64encode(b"seed payload for bytes " * 5).decode("ascii") tab = _make_tab(qtbot, {"blob": initial}) _select_value_cell(tab, (0,)) @@ -47,7 +47,7 @@ def test_attach_base64_from_file_replaces_value(qtbot, tmp_path, monkeypatch): def test_attach_base64_from_file_warns_and_can_cancel_large_file(qtbot, tmp_path, monkeypatch): - initial = base64.b64encode(b"seed payload for bytes").decode("ascii") + initial = base64.b64encode(b"seed payload for bytes " * 5).decode("ascii") tab = _make_tab(qtbot, {"blob": initial}) _select_value_cell(tab, (0,)) @@ -74,7 +74,7 @@ def _warn(*_args, **_kwargs): def test_save_base64_as_file_writes_decoded_payload(qtbot, tmp_path, monkeypatch): - payload = b"content to save" + payload = b"content to save " * 5 encoded = base64.b64encode(payload).decode("ascii") tab = _make_tab(qtbot, {"blob": encoded}) _select_value_cell(tab, (0,)) diff --git a/tests/test_edit_limits_menu.py b/tests/test_edit_limits_menu.py index b19191a..4b05ae1 100644 --- a/tests/test_edit_limits_menu.py +++ b/tests/test_edit_limits_menu.py @@ -6,10 +6,12 @@ from settings import APPLICATION_ID from state.edit_limits import ( get_attach_file_warning_limit_bytes, + get_base64_inference_min_length_chars, get_binary_edit_warning_limit_bytes, get_multiline_edit_warning_limit_chars, get_string_edit_warning_limit_chars, set_attach_file_warning_limit_bytes, + set_base64_inference_min_length_chars, set_binary_edit_warning_limit_bytes, set_multiline_edit_warning_limit_chars, set_string_edit_warning_limit_chars, @@ -26,7 +28,7 @@ def test_file_menu_limit_actions_persist_updates(qtbot, monkeypatch): win = MainWindow(yaml_filename="") qtbot.addWidget(win) - picks = iter([111, 222, 333, 444]) + picks = iter([111, 222, 333, 444, 555]) def _pick(*_args, **_kwargs): return next(picks), True @@ -37,11 +39,13 @@ def _pick(*_args, **_kwargs): win._app_settings.limit_multiline_action.trigger() win._app_settings.limit_binary_action.trigger() win._app_settings.limit_attach_action.trigger() + win._app_settings.limit_base64_min_length_action.trigger() assert get_string_edit_warning_limit_chars() == 111 assert get_multiline_edit_warning_limit_chars() == 222 assert get_binary_edit_warning_limit_bytes() == 333 assert get_attach_file_warning_limit_bytes() == 444 + assert get_base64_inference_min_length_chars() == 555 def test_file_menu_limit_actions_restore_after_restart(qtbot): @@ -50,6 +54,7 @@ def test_file_menu_limit_actions_restore_after_restart(qtbot): set_multiline_edit_warning_limit_chars(2002) set_binary_edit_warning_limit_bytes(3003) set_attach_file_warning_limit_bytes(4004) + set_base64_inference_min_length_chars(5005) first = MainWindow(yaml_filename="") qtbot.addWidget(first) @@ -58,6 +63,7 @@ def test_file_menu_limit_actions_restore_after_restart(qtbot): assert "2.00K" in first._app_settings.limit_multiline_action.text() assert "KiB" in first._app_settings.limit_binary_action.text() assert "KiB" in first._app_settings.limit_attach_action.text() + assert "5.00K" in first._app_settings.limit_base64_min_length_action.text() first.close() first.deleteLater() @@ -68,3 +74,4 @@ def test_file_menu_limit_actions_restore_after_restart(qtbot): assert "2.00K" in second._app_settings.limit_multiline_action.text() assert "KiB" in second._app_settings.limit_binary_action.text() assert "KiB" in second._app_settings.limit_attach_action.text() + assert "5.00K" in second._app_settings.limit_base64_min_length_action.text() diff --git a/tests/test_inference_limits.py b/tests/test_inference_limits.py index ff186e0..e2e5bea 100644 --- a/tests/test_inference_limits.py +++ b/tests/test_inference_limits.py @@ -8,6 +8,7 @@ import base64 import settings +from state.edit_limits import set_base64_inference_min_length_chars from tree.inference_limits import ( affix_inference_allowed, base64_syntax_valid, @@ -15,6 +16,7 @@ datetime_inference_allowed, format_preview_decode_allowed, ) +from tree.types import JsonType, _looks_like_base64, parse_json_type class TestDatetimeInferenceAllowed: @@ -129,6 +131,28 @@ def test_too_much_padding(self): assert base64_syntax_valid("Y===") is False +class TestBase64InferenceMinimumLength: + def teardown_method(self): + set_base64_inference_min_length_chars(settings.BASE64_INFERENCE_MIN_LENGTH_CHARS) + + def test_short_valid_base64_is_not_inferred_by_default(self): + assert len("bXkgbG92ZWx5IGJ5dGVzIQ==") < settings.BASE64_INFERENCE_MIN_LENGTH_CHARS + assert _looks_like_base64("bXkgbG92ZWx5IGJ5dGVzIQ==") is False + assert parse_json_type("bXkgbG92ZWx5IGJ5dGVzIQ==") is JsonType.STRING + + def test_valid_base64_at_default_minimum_length_is_inferred(self): + raw = b"x" * 75 + encoded = base64.b64encode(raw).decode("ascii") + assert len(encoded) == settings.BASE64_INFERENCE_MIN_LENGTH_CHARS + assert _looks_like_base64(encoded) is True + assert parse_json_type(encoded) is JsonType.BYTES + + def test_lowered_minimum_allows_shorter_valid_base64(self): + set_base64_inference_min_length_chars(20) + assert _looks_like_base64("bXkgbG92ZWx5IGJ5dGVzIQ==") is True + assert parse_json_type("bXkgbG92ZWx5IGJ5dGVzIQ==") is JsonType.BYTES + + class TestFormatPreviewDecodeAllowed: """Boundary tests for format_preview_decode_allowed.""" diff --git a/tests/test_kind_switch_coercion.py b/tests/test_kind_switch_coercion.py index 81134c7..884adae 100644 --- a/tests/test_kind_switch_coercion.py +++ b/tests/test_kind_switch_coercion.py @@ -524,12 +524,13 @@ def test_bytes_to_string_full_round_trip_through_item(qtbot): """End-to-end: switching BYTES→STRING in a tab surfaces decoded text.""" from documents.tab import JsonTab - raw = b"the answer is 42" + raw = b"the answer is 42 " * 5 bytes_b64 = encode_bytes(raw, JsonType.BYTES) tab = JsonTab(lambda *_: None, data={"blob": bytes_b64}) qtbot.addWidget(tab) item = tab.model.get_item(tab.model.index(0, 0, QModelIndex())) + item.json_type = JsonType.BYTES assert item.json_type is JsonType.BYTES type_idx = tab.model.index(0, 1, QModelIndex()) @@ -537,7 +538,7 @@ def test_bytes_to_string_full_round_trip_through_item(qtbot): item = tab.model.get_item(tab.model.index(0, 0, QModelIndex())) assert item.json_type is JsonType.STRING - assert item.value == "the answer is 42" + assert item.value == (b"the answer is 42 " * 5).decode("utf-8") # --------------------------------------------------------------------------- diff --git a/tests/test_phase_5_3_status_bar_breadcrumb.py b/tests/test_phase_5_3_status_bar_breadcrumb.py index 9009b9d..4fdff57 100644 --- a/tests/test_phase_5_3_status_bar_breadcrumb.py +++ b/tests/test_phase_5_3_status_bar_breadcrumb.py @@ -1,3 +1,5 @@ +import base64 + from PySide6.QtCore import QModelIndex from documents.tab import JsonTab @@ -27,10 +29,11 @@ def test_breadcrumb_callback_updates_on_selection_and_clear(qtbot): def test_breadcrumb_size_hints_for_container_and_binary(qtbot): captured: list[str] = [] + blob = base64.b64encode(b"my lovely bytes!" * 8).decode("ascii") tab = JsonTab( lambda *_: None, - data={"obj": {"k": "v"}, "blob": "bXkgbG92ZWx5IGJ5dGVzIQ=="}, + data={"obj": {"k": "v"}, "blob": blob}, permanent_message_callback=captured.append, ) qtbot.addWidget(tab) @@ -41,4 +44,4 @@ def test_breadcrumb_size_hints_for_container_and_binary(qtbot): blob_name_index = tab.model.index(1, 0, QModelIndex()) tab.view.setCurrentIndex(tab.view_controller.source_to_view(blob_name_index)) - assert captured[-1] == "$.blob (bytes, 16 byte)" + assert captured[-1] == "$.blob (bytes, 128 byte)" diff --git a/tests/test_tree_correctness.py b/tests/test_tree_correctness.py index e82cac7..e5dcc7f 100644 --- a/tests/test_tree_correctness.py +++ b/tests/test_tree_correctness.py @@ -74,8 +74,8 @@ def test_parse_json_type_is_total_and_has_narrower_heuristics(): assert parse_json_type("1234") is JsonType.STRING assert parse_json_type("hello world") is JsonType.STRING - # A pure base64 string (regex + padding + clean decode) is BYTES. - assert parse_json_type("bXkgbG92ZWx5IGJ5dGVzIQ==") is JsonType.BYTES + # Short base64-like strings stay STRING until the base64 minimum-length guard is met. + assert parse_json_type("bXkgbG92ZWx5IGJ5dGVzIQ==") is JsonType.STRING # Strings that aren't valid base64 stay STRING. assert parse_json_type("hi\n") is JsonType.STRING assert parse_json_type("hello") is JsonType.STRING diff --git a/tests/test_type_editing.py b/tests/test_type_editing.py index 968a613..0228504 100644 --- a/tests/test_type_editing.py +++ b/tests/test_type_editing.py @@ -406,7 +406,7 @@ def test_bool_to_string_undo_redo(qtbot): def test_bytes_to_zlib_undo_redo(qtbot): - raw = b"my lovely bytes!" + raw = b"my lovely bytes! " * 5 bytes_b64 = encode_bytes(raw, JsonType.BYTES) tab = JsonTab(lambda *_: None, data={"blob": bytes_b64}) qtbot.addWidget(tab) diff --git a/tree/types.py b/tree/types.py index c5ddf83..4fab02e 100644 --- a/tree/types.py +++ b/tree/types.py @@ -9,20 +9,33 @@ import gmpy2 from pandas import Timestamp +from PySide6.QtCore import QSettings from core.datetime_parsing import parse_datetime_text from core.datetime_parsing.nano_time import NanoTime from core.raw_numeric import RawNumericValue -from settings import INFERENCE_MAX_COLOR_CHARS, NUMBER_AFFIX_MAX_LEN +from settings import APPLICATION_ID, BASE64_INFERENCE_MIN_LENGTH_CHARS, INFERENCE_MAX_COLOR_CHARS, NUMBER_AFFIX_MAX_LEN from tree.inference_limits import base64_syntax_valid from units.number_affix import AffixKind, NumberAffix, parse_number_affix LOGGER = logging.getLogger(__name__) +# Compatibility-only regex kept for perf probes/import stability. +# Actual base64 inference now uses ``base64_syntax_valid()`` plus the persisted +# minimum-length guard in ``_looks_like_base64()``. _B64_RE = re.compile(r"^[A-Za-z0-9+/]{20,}={0,2}$") _COLOR_RGB_RE = re.compile(r"^#(?:[0-9a-fA-F]{3}|[0-9a-fA-F]{6})$") _COLOR_RGBA_RE = re.compile(r"^#(?:[0-9a-fA-F]{4}|[0-9a-fA-F]{8})$") +def _base64_inference_min_length_chars() -> int: + value = QSettings(APPLICATION_ID, "app").value("edit_limits/base64_min_length_chars") + try: + number = int(value) + except (TypeError, ValueError): + return BASE64_INFERENCE_MIN_LENGTH_CHARS + return number if number > 0 else BASE64_INFERENCE_MIN_LENGTH_CHARS + + def looks_like_color_rgb(s: str, *, allow_expensive: bool = False) -> bool: if not allow_expensive and len(s) > INFERENCE_MAX_COLOR_CHARS: return False @@ -39,8 +52,8 @@ def _looks_like_base64(s: str) -> bool: """Return True iff *s* is a syntactically valid, non-empty base64 string. Uses ``base64_syntax_valid`` as a cheap pre-check (len mod 4 + alphabet - regex) before attempting the expensive ``base64.b64decode``. A minimum - length of 20 chars is required to avoid false positives on short strings. + regex) before attempting the expensive ``base64.b64decode``. A configurable + minimum length guard avoids false positives on short strings. No content heuristics are applied: any string that decodes cleanly under strict base64 rules is treated as ``BYTES``. Callers that need to @@ -49,8 +62,7 @@ def _looks_like_base64(s: str) -> bool: """ if not base64_syntax_valid(s): return False - # Require minimum 20 chars to avoid false positives on short strings - if len(s) < 20: + if len(s) < _base64_inference_min_length_chars(): return False try: base64.b64decode(s, validate=True) From 3cac55b169e95503a64612b9d82474f448917b4c Mon Sep 17 00:00:00 2001 From: sr9000 Date: Thu, 2 Jul 2026 23:23:59 +0300 Subject: [PATCH 2/3] fix parser plan item: generalize hyphenated currency boundary --- tests/test_number_affix.py | 13 ++++++++++--- units/number_affix.py | 32 +++++++++++++++++++++++++------- 2 files changed, 35 insertions(+), 10 deletions(-) diff --git a/tests/test_number_affix.py b/tests/test_number_affix.py index 1a65a83..dd6477e 100644 --- a/tests/test_number_affix.py +++ b/tests/test_number_affix.py @@ -29,12 +29,19 @@ def test_parse_units_float_with_space_and_exponent() -> None: assert parsed.number == mpq("-314") -def test_parse_negative_currency_without_space_rejected() -> None: - # "abc-1" should be rejected as currency, requires "abc -1" - assert parse_number_affix("abc-1") is None +def test_parse_hyphenated_currency_without_space_prefers_affix_boundary() -> None: + parsed = parse_number_affix("abc-1") + assert parsed == NumberAffix(AffixKind.CURRENCY, "abc-", False, 1, 0, -1) + assert format_number_affix(parsed) == "abc-1" assert parse_number_affix("abc -1") is not None +def test_parse_hyphenated_currency_without_space_is_not_limited_to_zero_padding() -> None: + parsed = parse_number_affix("prod-200") + assert parsed == NumberAffix(AffixKind.CURRENCY, "prod-", False, 200, 0, -1) + assert format_number_affix(parsed) == "prod-200" + + def test_parse_currency_affix_ending_with_dash_for_zero_padded_int() -> None: parsed = parse_number_affix("abc-001") assert parsed == NumberAffix(AffixKind.CURRENCY, "abc-", False, 1, 3, -1) diff --git a/units/number_affix.py b/units/number_affix.py index dcf26b9..d2d24ca 100644 --- a/units/number_affix.py +++ b/units/number_affix.py @@ -65,21 +65,39 @@ def _parse_number(num_text: str) -> int | mpq: return parsed -def _split_squashed_currency_sign( +def _resolve_currency_no_space_boundary( affix: str, num_text: str, *, has_space: bool, max_affix_len: int, ) -> tuple[str, str] | None: - if has_space or not num_text or num_text[0] != "-": + """Resolve ambiguous ``prefix-number`` boundaries for no-space currency text. + + Prefix currency syntax has an unavoidable ambiguity for strings like + ``prod-200``: the ``-`` can be read either as part of the numeric sign or as + the final character of the affix. This parser treats *spaced* negatives as + the explicit negative form (``prod -200``) and, for *no-space* forms, + prefers the affix boundary when that yields a valid currency affix. + + That keeps parsing structurally consistent for hyphenated affixes instead of + relying on special cases such as zero-padded numbers only. + """ + if has_space or not num_text.startswith("-"): + return None + + # Keep single-symbol / punctuation-led currency prefixes on the historical + # path: ``$-1`` should remain an invalid no-space negative currency form. + # The ambiguity fix is for textual/hyphenatable affixes such as + # ``prod-200`` where the dash naturally belongs to the affix token. + if not affix[-1].isalnum(): return None unsigned = num_text[1:] - if not re.fullmatch(r"0\d+", unsigned): + if not re.fullmatch(r"\d+(?:\.\d*)?(?:[eE][+-]?\d+)?|\.\d+(?:[eE][+-]?\d+)?", unsigned): return None - shifted_affix = affix + num_text[0] + shifted_affix = affix + "-" if not _is_valid_affix(shifted_affix, kind=AffixKind.CURRENCY, max_affix_len=max_affix_len): return None @@ -133,14 +151,14 @@ def parse_number_affix(s: str, *, max_affix_len: int = 16, allow_expensive: bool num_text = m.group("num") if kind == AffixKind.CURRENCY: - squashed = _split_squashed_currency_sign( + resolved = _resolve_currency_no_space_boundary( affix, num_text, has_space=(m.group("sp") == " "), max_affix_len=max_affix_len, ) - if squashed is not None: - affix, num_text = squashed + if resolved is not None: + affix, num_text = resolved elif m.group("sp") == "" and num_text.startswith("-"): return None From d5fbb3b95e9f89e373697fa6bdd9cb0bbdb68f19 Mon Sep 17 00:00:00 2001 From: sr9000 Date: Thu, 2 Jul 2026 23:49:06 +0300 Subject: [PATCH 3/3] preserve affix metadata across string coercion --- tests/test_kind_switch_coercion.py | 30 ++++++++++++++++++++++++ tests/test_type_editing.py | 37 ++++++++++++++++++++++++++++++ tree/item_coercion.py | 19 +++++++++++---- 3 files changed, 82 insertions(+), 4 deletions(-) diff --git a/tests/test_kind_switch_coercion.py b/tests/test_kind_switch_coercion.py index 884adae..0cabb92 100644 --- a/tests/test_kind_switch_coercion.py +++ b/tests/test_kind_switch_coercion.py @@ -664,6 +664,36 @@ def test_affix_to_string_to_affix_undo_redo_round_trip(qtbot): assert item.value == NumberAffix(AffixKind.UNITS, "$", False, 10) +@pytest.mark.parametrize( + ("original", "target_type"), + [ + ( + NumberAffix(AffixKind.CURRENCY, "lvl", True, 7, 4, -1, True), + JsonType.INTEGER_CURRENCY, + ), + ( + NumberAffix(AffixKind.CURRENCY, "lvl", True, mpq("3/2"), 4, 3, True), + JsonType.FLOAT_CURRENCY, + ), + ( + NumberAffix(AffixKind.UNITS, "kg", False, 12, 3, -1, False), + JsonType.INTEGER_UNITS, + ), + ( + NumberAffix(AffixKind.UNITS, "%", False, mpq("1999/20"), 0, 3, False), + JsonType.FLOAT_UNITS, + ), + ], + ids=["int-currency", "float-currency", "int-units", "float-units"], +) +def test_affix_to_string_to_same_affix_type_preserves_metadata(original, target_type): + ok, text = coerce_value_for_type(JsonType.STRING, original, strict=False, old_type=target_type) + assert ok + ok, reparsed = coerce_value_for_type(target_type, text, strict=False, old_type=JsonType.STRING) + assert ok + assert reparsed == original + + def test_bytes_to_integer_returns_decoded_length(): """Switching from a bytes-family kind to INTEGER returns the underlying byte length, which is meaningful (file/blob size).""" diff --git a/tests/test_type_editing.py b/tests/test_type_editing.py index 0228504..4a6f58a 100644 --- a/tests/test_type_editing.py +++ b/tests/test_type_editing.py @@ -514,3 +514,40 @@ def test_integer_currency_to_integer_units_flips_kind_preserving_payload(): assert model.setData(type_idx, JsonType.INTEGER_UNITS) item = model.get_item(model.index(0, 0, QModelIndex())) assert item.value == NumberAffix(AffixKind.UNITS, "$", True, 42) + + +@pytest.mark.parametrize( + ("original", "string_type", "target_type"), + [ + ( + NumberAffix(AffixKind.CURRENCY, "lvl", True, 7, 4, -1, True), + JsonType.STRING, + JsonType.INTEGER_CURRENCY, + ), + ( + NumberAffix(AffixKind.CURRENCY, "lvl", True, mpq("3/2"), 4, 3, True), + JsonType.STRING, + JsonType.FLOAT_CURRENCY, + ), + ( + NumberAffix(AffixKind.UNITS, "kg", False, 12, 3, -1, False), + JsonType.STRING, + JsonType.INTEGER_UNITS, + ), + ( + NumberAffix(AffixKind.UNITS, "%", False, mpq("1999/20"), 0, 3, False), + JsonType.STRING, + JsonType.FLOAT_UNITS, + ), + ], + ids=["int-currency", "float-currency", "int-units", "float-units"], +) +def test_type_switch_string_round_trip_preserves_affix_metadata(original, string_type, target_type): + model = JsonTreeModel({"v": original}) + type_idx = model.index(0, 1, QModelIndex()) + + assert model.setData(type_idx, string_type) + assert model.setData(type_idx, target_type) + + item = model.get_item(model.index(0, 0, QModelIndex())) + assert item.value == original diff --git a/tree/item_coercion.py b/tree/item_coercion.py index 7b53b7e..d4e540a 100644 --- a/tree/item_coercion.py +++ b/tree/item_coercion.py @@ -370,6 +370,17 @@ def _affix_kind_for(target: JsonType) -> AffixKind: return AffixKind.CURRENCY return AffixKind.UNITS + def _rebuild_affix(parsed: NumberAffix, *, kind: AffixKind, number: int | mpq) -> NumberAffix: + return NumberAffix( + kind=kind, + affix=parsed.affix, + space=parsed.space, + number=number, + integral_digits=parsed.integral_digits, + fractional_digits=parsed.fractional_digits, + explicit_plus=parsed.explicit_plus, + ) + # Targeting the RAW_FLOAT pseudo-type: always keep the value as a raw # numeric literal (this path is only reachable programmatically; the type # is not user-selectable). @@ -470,12 +481,12 @@ def _affix_kind_for(target: JsonType) -> AffixKind: truncated = _int_from_truncated(parsed.number) if truncated is None: return False, None - return True, NumberAffix(kind=kind, affix=parsed.affix, space=parsed.space, number=truncated) + return True, _rebuild_affix(parsed, kind=kind, number=truncated) if isinstance(value, NumberAffix): truncated = _int_from_truncated(value.number) if truncated is None: return False, None - return True, NumberAffix(kind=kind, affix=value.affix, space=value.space, number=truncated) + return True, _rebuild_affix(value, kind=kind, number=truncated) truncated = _int_from_truncated(value) if truncated is None: return ( @@ -493,12 +504,12 @@ def _affix_kind_for(target: JsonType) -> AffixKind: q = _to_mpq_or_none(parsed.number) if q is None: return False, None - return True, NumberAffix(kind=kind, affix=parsed.affix, space=parsed.space, number=q) + return True, _rebuild_affix(parsed, kind=kind, number=q) if isinstance(value, NumberAffix): q = _to_mpq_or_none(value.number) if q is None: return False, None - return True, NumberAffix(kind=kind, affix=value.affix, space=value.space, number=q) + return True, _rebuild_affix(value, kind=kind, number=q) q = _to_mpq_or_none(value) if q is None: return (