diff --git a/.github/actions/checkout-dev-siblings/action.yml b/.github/actions/checkout-dev-siblings/action.yml index c433f357..c213fb2d 100644 --- a/.github/actions/checkout-dev-siblings/action.yml +++ b/.github/actions/checkout-dev-siblings/action.yml @@ -66,7 +66,38 @@ runs: "$remote" "$event_ref" >/dev/null 2>&1 then selected_ref="$event_ref" - else + elif [ "$repo" = "vinary-tree-interop" ]; then + # Prefer the exact public version declared by this checkout's + # path dependency when no coordinated development branch exists. + # This keeps development branches flexible while preventing a + # stale sibling default (for example RC4) from satisfying an + # exact RC6 requirement. + required_version="$( + sed -nE \ + '/^[[:space:]]*vinary-tree-interop[[:space:]]*=/{s/.*version[[:space:]]*=[[:space:]]*"=([^" ]+)".*/\1/p;}' \ + "$GITHUB_WORKSPACE/Cargo.toml" | head -n 1 + )" + tagged_ref="v$required_version" + if [ -n "$required_version" ] && \ + git ls-remote --exit-code \ + "$remote" "refs/tags/$tagged_ref" >/dev/null 2>&1 + then + selected_ref="$tagged_ref" + elif [ -n "$required_version" ] && \ + git ls-remote --exit-code --heads \ + "$remote" master >/dev/null 2>&1 + then + # The release branch may publish an exact version on master + # before creating a version tag. Select master only as an + # explicit fallback; the post-checkout package-version audit + # below remains authoritative and rejects mismatches. + selected_ref=master + elif [ -n "$required_version" ]; then + echo "ERROR: $repo requires $tagged_ref from Cargo.toml, but that upstream ref is unavailable" >&2 + exit 1 + fi + fi + if [ -z "$selected_ref" ]; then selected_ref="$development_ref" fi fi @@ -83,6 +114,28 @@ runs: "https://github.com/vinary-tree/$repo.git" "$dest" echo "::endgroup::" fi + if [ "$repo" = "vinary-tree-interop" ]; then + required_version="$( + sed -nE \ + '/^[[:space:]]*vinary-tree-interop[[:space:]]*=/{s/.*version[[:space:]]*=[[:space:]]*"=([^" ]+)".*/\1/p;}' \ + "$GITHUB_WORKSPACE/Cargo.toml" | head -n 1 + )" + if [ -n "$required_version" ]; then + actual_version="$( + awk ' + /^\[package\]$/ { in_package=1; next } + /^\[/ { in_package=0 } + in_package && /^[[:space:]]*version[[:space:]]*=/ { + gsub(/[[:space:]]*"/, "", $3); print $3; exit + } + ' "$dest/Cargo.toml" + )" + if [ "$actual_version" != "$required_version" ]; then + echo "ERROR: $repo ref $selected_ref declares $actual_version, but Cargo.toml requires $required_version" >&2 + exit 1 + fi + fi + fi done echo "Sibling crates available under $parent:" ls -1d \ diff --git a/.github/workflows/bindings-conformance.yml b/.github/workflows/bindings-conformance.yml index 73e346c2..37735247 100644 --- a/.github/workflows/bindings-conformance.yml +++ b/.github/workflows/bindings-conformance.yml @@ -76,14 +76,18 @@ jobs: mkdir -p target/julia-ci-env julia --project=target/julia-ci-env -e ' using Pkg - Pkg.develop(path=ENV["GITHUB_WORKSPACE"] * "/../vinary-tree-interop/bindings/julia/VinaryTreeInterop") - Pkg.develop(path=ENV["GITHUB_WORKSPACE"] * "/bindings/julia/Libdictenstein") + Pkg.develop([ + PackageSpec(path=ENV["GITHUB_WORKSPACE"] * "/../vinary-tree-interop/bindings/julia/VinaryTreeInterop"), + PackageSpec(path=ENV["GITHUB_WORKSPACE"] * "/bindings/julia/Libdictenstein") + ]) Pkg.test("Libdictenstein") ' julia --project=bindings/julia/Libdictenstein/docs -e ' using Pkg - Pkg.develop(path=ENV["GITHUB_WORKSPACE"] * "/../vinary-tree-interop/bindings/julia/VinaryTreeInterop") - Pkg.develop(path=ENV["GITHUB_WORKSPACE"] * "/bindings/julia/Libdictenstein") + Pkg.develop([ + PackageSpec(path=ENV["GITHUB_WORKSPACE"] * "/../vinary-tree-interop/bindings/julia/VinaryTreeInterop"), + PackageSpec(path=ENV["GITHUB_WORKSPACE"] * "/bindings/julia/Libdictenstein") + ]) Pkg.instantiate() include("bindings/julia/Libdictenstein/docs/make.jl") ' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 55ad1f5a..45f38edd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -199,9 +199,9 @@ jobs: - uses: taiki-e/install-action@cargo-llvm-cov - name: Run coverage run: | - cargo llvm-cov --all-features --branch \ + cargo +nightly llvm-cov --all-features --branch \ --lcov --output-path lcov.info - cargo llvm-cov report --branch \ + cargo +nightly llvm-cov report --branch \ --cobertura --output-path cobertura.xml - name: Upload coverage to Codecov uses: codecov/codecov-action@v4 @@ -355,6 +355,59 @@ jobs: TLA_JAVA_HEAP: "3g" run: prlimit --as="$FORMAL_MEM_LIMIT_BYTES" --rss="$FORMAL_MEM_LIMIT_BYTES" -- bash scripts/verify-formal-correspondence.sh + # ----------------------------------------------------------------- + # Variable-width family/profile gate. This is the exact preimplementation + # gate for the committed Rocq/TLA+ family-refinement source tuple, including + # safe models and every required negative control. GitHub-hosted runners do + # not expose a user systemd bus, so the verifier uses prlimit here while + # retaining systemd-run caps for local execution. + # ----------------------------------------------------------------- + variable-width-formal: + name: Variable-width formal gate + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: ./.github/actions/checkout-dev-siblings + - uses: ocaml/setup-ocaml@v3 + with: + ocaml-compiler: "5.2" + - uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: "21" + - name: Install Rocq and TLA+ tools + run: | + sudo apt-get update + sudo apt-get install -y curl ripgrep util-linux + # Rocq 9.1.1 is distributed in opam through the compatibility + # package name [coq]; pinning the version still selects the Rocq + # toolchain and avoids Ubuntu's older apt package. + opam install --yes coq.9.1.1 + eval "$(opam env)" + echo "$(opam var bin)" >> "$GITHUB_PATH" + curl -L \ + https://github.com/tlaplus/tlaplus/releases/latest/download/tla2tools.jar \ + -o tla2tools.jar + sudo mkdir -p /opt/tla + sudo mv tla2tools.jar /opt/tla/tla2tools.jar + sudo tee /usr/local/bin/tla2sany >/dev/null <<'EOF' + #!/usr/bin/env bash + exec java -Xmx"${TLA_JAVA_HEAP:-512m}" -cp /opt/tla/tla2tools.jar tla2sany.SANY "$@" + EOF + sudo tee /usr/local/bin/tlc >/dev/null <<'EOF' + #!/usr/bin/env bash + exec java -Xmx"${TLA_JAVA_HEAP:-512m}" -cp /opt/tla/tla2tools.jar tlc2.TLC "$@" + EOF + sudo chmod +x /usr/local/bin/tla2sany /usr/local/bin/tlc + - name: Run capped variable-width formal gate + env: + VARIABLE_WIDTH_FORMAL_RESOURCE_CONTROL: external + VARIABLE_WIDTH_FORMAL_TIMEOUT_SECONDS: "7200" + VARIABLE_WIDTH_TLC_JAVA_OPTIONS: "-Xms64m -Xmx512m -XX:+UseParallelGC" + run: bash scripts/verify-variable-width-formal.sh + - name: Run invariant-derived reference harness + run: prlimit --rss=2147483648 -- cargo test --manifest-path formal-verification/Cargo.toml --test variable_width_formal_harness + # ----------------------------------------------------------------- # Miri unsafe-boundary checks. Runs on nightly because `cargo miri` # is a nightly component. @@ -394,12 +447,13 @@ jobs: env: CARGO_BUILD_JOBS: "2" FORMAL_MEM_LIMIT_BYTES: "8589934592" - run: prlimit --as="$FORMAL_MEM_LIMIT_BYTES" --rss="$FORMAL_MEM_LIMIT_BYTES" -- cargo miri setup + run: prlimit --as="$FORMAL_MEM_LIMIT_BYTES" --rss="$FORMAL_MEM_LIMIT_BYTES" -- cargo +nightly miri setup - name: Run Miri-gated correspondence harness env: CARGO_BUILD_JOBS: "2" FORMAL_MEM_LIMIT_BYTES: "8589934592" FORMAL_RESOURCE_CONTROL: external + FORMAL_MIRI_TOOLCHAIN: nightly RUN_MIRI: "1" TLA_JAVA_HEAP: "3g" run: prlimit --as="$FORMAL_MEM_LIMIT_BYTES" --rss="$FORMAL_MEM_LIMIT_BYTES" -- bash scripts/verify-formal-correspondence.sh diff --git a/bindings/julia/Libdictenstein/docs/make.jl b/bindings/julia/Libdictenstein/docs/make.jl index ceb3c1f7..abb0033f 100644 --- a/bindings/julia/Libdictenstein/docs/make.jl +++ b/bindings/julia/Libdictenstein/docs/make.jl @@ -7,7 +7,11 @@ DocMeta.setdocmeta!(Libdictenstein, :DocTestSetup, makedocs( modules=[Libdictenstein], sitename="Libdictenstein.jl", - strict=true, + # Documenter 1.x removed the legacy `strict` keyword. An empty + # `warnonly` list preserves fail-closed documentation diagnostics, while + # `checkdocs=:all` retains strict API coverage checking. + warnonly=Symbol[], + checkdocs=:all, doctest=true, pages=[ "Guide" => "index.md", diff --git a/docs/algorithms/README.md b/docs/algorithms/README.md index ee562619..a96eedb1 100644 --- a/docs/algorithms/README.md +++ b/docs/algorithms/README.md @@ -8,6 +8,9 @@ The Dictionary Layer is the family of dictionary backends provided by **libdicte This layer abstracts over different data structures (tries, DAWGs, double-array tries) through common traits, allowing you to choose the best backend for your specific use case while maintaining a consistent API. +For the topology/profile split, canonical UTF-8 and ULEB128 boundaries, and +interned-ID identity rules, see [variable-width logical profiles](variable-width-profiles.md). + ## Architecture The Dictionary Layer trait API (Dictionary, MappedDictionary, DictionaryNode) sits above three in-memory backend families - Trie, DAWG, and Suffix Automaton; the Trie family holds DoubleArrayTrie (the recommended default) and DAT-Char (UTF-8), and the DAWG family holds DynamicDawg. @@ -559,7 +562,7 @@ For concurrent writes, dictionaries have different strategies: |-----------|----------|--------|-------| | DoubleArrayTrie | `Persistent` | Rebuild + atomic swap | Append-only via builder | | DynamicDawg | `InternalSync` | Direct mutation | Internal RwLock | -| PathMapDictionary | `InternalSync` | Direct mutation | Internal RwLock | +| PathMapDictionary | `ArcSwap` copy-on-write | Clone, transform, and CAS-publish an immutable root | Readers take one snapshot and never block; competing writers retry from the winning root | ## Advanced Topics diff --git a/docs/algorithms/implementations/pathmap-dictionary.md b/docs/algorithms/implementations/pathmap-dictionary.md index de0e7c8d..9308249d 100644 --- a/docs/algorithms/implementations/pathmap-dictionary.md +++ b/docs/algorithms/implementations/pathmap-dictionary.md @@ -23,9 +23,9 @@ ### Key Advantages - πŸ”„ **Full dynamic updates**: Insert AND remove at runtime -- πŸ”’ **Thread-safe**: Safe concurrent reads, exclusive writes +- πŸ”’ **Snapshot-safe concurrency**: Readers load one immutable root without blocking; writers clone, transform, and CAS-publish a replacement root - πŸ“¦ **Simple implementation**: Thin wrapper around PathMap -- πŸ’Ž **Persistent semantics**: Structural sharing between versions +- πŸ’Ž **Persistent semantics**: Structural sharing between versions; retained snapshots remain isolated from later publication - 🎯 **Easy to use**: Straightforward API ### Key Trade-offs @@ -49,6 +49,32 @@ ## Theory: Persistent Data Structures +### Concurrency and snapshot contract + +`PathMapDictionary` is an adapter around a third-party byte-keyed `PathMap`; the +adapter owns publication, not the upstream representation. Its current root is +stored in an atomic `ArcSwap`. A read operation loads one `Arc>` +and observes that immutable state for the operation (and for any owned snapshot +or zipper derived from it). A writer clones the loaded persistent root, applies +its transformation to the clone, and publishes the candidate with compare-and- +swap. If another writer wins first, the transformation is retried against the +new winning root. Consequently: + +- readers never wait for writers and never observe a partially transformed trie; +- each published root has one exact term-count value paired with its trie root; +- a retained `PathMapSnapshot` or owned zipper continues to observe its captured + revision after later writes publish newer roots; +- a successful mutation is linearized at its root publication; a failed CAS is + an implementation retry, not an externally visible revision; +- traversal does not mix revisions because its root is captured before walking; +- visibility is immediate after the successful publication to subsequent root + loads, while an already retained snapshot intentionally remains on its older + revision. + +The upstream `PathMap` is not modified or forked, and its internal byte nodes +are not exposed as logical dictionary transitions by the adapter's character or +profile-aware views. + ### What are Persistent Data Structures? **Persistent** data structures preserve previous versions after modifications through **structural sharing**. @@ -110,47 +136,46 @@ cargo add liblevenshtein --features pathmap-backend ```rust pub struct PathMapDictionary { - map: Arc>>, // Underlying PathMap - term_count: Arc>, // Term count tracking + state: Arc>>, // root + exact term count } ``` ### Wrapper Design -PathMapDictionary is a thin wrapper that: -1. Manages PathMap lifecycle -2. Tracks term count -3. Provides liblevenshtein Dictionary trait -4. Handles thread safety via RwLock +PathMapDictionary is a thin adapter that: +1. Manages the third-party PathMap lifecycle +2. Publishes the trie root and exact term count as one immutable state +3. Provides the liblevenshtein Dictionary trait +4. Provides atomic snapshot publication through `ArcSwap` ### Memory Layout -| Component | Overhead | +| Component | Role | | --- | --- | -| Arc pointers | 16 bytes | -| RwLock | 8 bytes | -| PathMap | ~32 bytes/node | -| term_count | 8 bytes | - -**Per-node overhead**: ~32 bytes (HashMap-based) +| `Arc>>` | Shared publication cell for immutable roots | +| `PathMapState` | One PathMap root paired with its exact term count | +| PathMap | Third-party persistent byte-key trie and structural sharing | -**Example**: 10,000-term dictionary $`\approx`$ 320 KB +Node size and allocation behavior are governed by the upstream PathMap +implementation and should not be presented as a libdictenstein invariant. ### Clone Behavior & Memory Semantics -`PathMapDictionary` uses **two separate** `Arc>` instances internally, making `.clone()` a **shallow copy** that shares all underlying data. The clone behavior is similar to `DynamicDawg`, but with dual Arc-wrapped components: +`PathMapDictionary` clones the shared `Arc>>` publication +cell. Clones therefore share the current revision stream, while an owned +`PathMapSnapshot` or zipper retains the revision it captured: ```rust use libdictenstein::pathmap::PathMapDictionary; let dict1: PathMapDictionary = PathMapDictionary::from_terms(vec!["test", "testing"]); -let dict2 = dict1.clone(); // O(1) - increments TWO Arc refcounts +let dict2 = dict1.clone(); // O(1) - shares one publication cell -// Both dict1 and dict2 share the SAME underlying PathMap and term count +// Both handles observe the same subsequently published revisions dict1.insert("new_term"); assert!(dict2.contains("new_term")); // βœ… Mutations visible through dict2! -// Term count is also shared +// The root and term count are published together assert_eq!(dict1.len(), Some(3)); assert_eq!(dict2.len(), Some(3)); // Same count ``` @@ -159,58 +184,44 @@ assert_eq!(dict2.len(), Some(3)); // Same count | Property | Behavior | Impact | |----------|----------|--------| -| **Time Complexity** | O(1) | Two atomic increments | -| **Space Complexity** | O(1) | ~32 bytes (two Arc pointers) | -| **Data Sharing** | βœ… Complete | All clones share PathMap + term count | -| **Mutation Visibility** | βœ… Global | Changes via any clone affect all | -| **Thread Safety** | βœ… RwLock | Multiple readers OR single writer | -| **Independence** | ❌ None | No isolation between clones | +| **Clone complexity** | O(1) | Shares one publication cell | +| **Snapshot complexity** | O(1) | Retains one immutable root reference | +| **Data sharing** | βœ… Structural | Published roots share unchanged PathMap structure | +| **Mutation visibility** | βœ… Revision-based | New root loads see publication; retained snapshots do not | +| **Thread safety** | βœ… Lock-free reads | Readers never wait for writers | +| **Independence** | βœ… Explicit | Use `snapshot()` or an owned zipper for revision isolation | #### How Clone Works -The clone operation increments **two** atomic reference counters: +The clone operation increments the reference count for the shared publication +cell: ```rust pub struct PathMapDictionary { - map: Arc>>, // ← Arc #1 - term_count: Arc>, // ← Arc #2 + state: Arc>>, // root + term count } -// Cloning increments both Arc refcounts +// Cloning shares the publication cell let dict2 = dict1.clone(); -// Equivalent to: -// Arc::clone(&dict1.map) + Arc::clone(&dict1.term_count) -// Cost: ~2-4 CPU cycles (two atomic increments) +// Cost: one Arc clone; no trie nodes are copied ``` **What gets cloned:** -- βœ… Arc smart pointer for PathMap (~16 bytes on stack) -- βœ… Arc smart pointer for term_count (~16 bytes on stack) -- ❌ NOT the RwLocks +- βœ… Arc smart pointer for the publication cell - ❌ NOT the PathMap trie structure - ❌ NOT the term count value itself **Memory allocation:** - Zero heap allocation -- Only stack space for two Arc pointers (~32 bytes) +- Only stack space for one shared Arc handle - All data remains shared -#### Dual-Arc Design - -PathMapDictionary's dual-Arc design enables independent locking of map and count: - -```rust -// Concurrent readers can lock map and count independently -let map_lock = self.map.read(); // Lock PathMap -let count_lock = self.term_count.read(); // Lock count separately - -// Reduces lock contention compared to single lock -``` +#### Publication-cell Design -**Why two Arcs?** -- **Flexibility**: Can read count without locking PathMap -- **Granularity**: Finer-grained synchronization -- **Cost trade-off**: Slightly more expensive clone (2 increments vs 1) +The dictionary shares one atomic publication cell containing the immutable +PathMap root and its exact term count. A reader loads that cell once; a writer +builds a candidate from the loaded persistent root and CAS-publishes it. There +is no map lock, count lock, or split observation to reconcile. #### Structural Sharing vs Arc Sharing @@ -219,7 +230,7 @@ let count_lock = self.term_count.read(); // Lock count separately 1. **Arc-based sharing (clone behavior):** ```rust let dict2 = dict1.clone(); - // dict1 and dict2 share the SAME PathMap instance + // dict1 and dict2 share the same publication stream dict1.insert("new"); assert!(dict2.contains("new")); // βœ… Visible ``` @@ -237,8 +248,11 @@ let count_lock = self.term_count.read(); // Lock count separately ``` **For PathMapDictionary:** -- `.clone()` creates Arc-based sharing (visible mutations) -- PathMap's internal structural sharing is orthogonal (optimization) +- `.clone()` shares the publication stream; later successful mutations are + visible through both handles when they load the current root. +- `snapshot()` and owned zippers capture a revision and are isolated from later + publication. +- PathMap's internal structural sharing is orthogonal and remains an optimization. #### When to Use Cloning @@ -296,10 +310,10 @@ let count_lock = self.term_count.read(); // Lock count separately 2. **Creating versioned snapshots:** ```rust let dict: PathMapDictionary = load_data(); - let v1 = dict.clone(); // ❌ NOT a snapshot! + let v1 = dict.snapshot(); // βœ… Captures a stable revision dict.insert("v2_data"); - // v1 now also contains v2_data - not versioned + // v1 remains on the pre-mutation revision ``` 3. **Isolating test fixtures:** @@ -359,22 +373,22 @@ let dict2: PathMapDictionary = PathMapDictionary::from_terms_with_values(entr #### Comparison with Other Dictionaries -| Dictionary | Arc Count | Clone Cost | Shared Data? | +| Dictionary | Publication model | Clone Cost | Current-root sharing? | |------------|-----------|------------|--------------| -| **PathMapDictionary** | 2 (map + count) | O(1) | βœ… Yes | +| **PathMapDictionary** | One ArcSwap cell | O(1) | βœ… Yes | | **DynamicDawg** | 1 (inner) | O(1) | βœ… Yes | | **DynamicDawgChar** | 1 (inner) | O(1) | βœ… Yes | | **DoubleArrayTrie** | 0 (no Arc) | O(n) | ❌ No | | **DoubleArrayTrieChar** | 0 (no Arc) | O(n) | ❌ No | **Key differences:** -- PathMapDictionary: Two Arc increments (map + count) +- PathMapDictionary: One Arc clone for the shared ArcSwap publication cell - DynamicDawg variants: One Arc increment (inner struct contains count) - DoubleArrayTrie: Full deep copy (immutable, no Arc needed) #### Thread Safety Considerations -PathMapDictionary's dual-Arc design provides flexible locking: +`PathMapDictionary` publishes immutable roots through one shared atomic cell: ```rust use std::thread; @@ -392,7 +406,7 @@ let readers: Vec<_> = (0..10).map(|i| { }) }).collect(); -// Single writer (blocks all readers) +// Writers clone and publish; readers continue on their captured root let writer = { let dict = dict.clone(); thread::spawn(move || { @@ -401,24 +415,22 @@ let writer = { }; ``` -**RwLock semantics:** -- **Read operations**: `contains()`, `get_value()`, `len()`, iteration -- **Write operations**: `insert()`, `insert_with_value()`, `remove()`, `union_with()` -- **Lock granularity**: Map and count can be locked independently for reads - -**Performance implications:** -- Read lock overhead: ~10-20ns per operation -- Write lock overhead: ~50-100ns + contention costs -- Dual-Arc trade-off: More flexible locking, slightly higher clone cost +**Publication semantics:** +- **Read operations** load one `PathMapState` and do not block on writers. +- **Write operations** clone a persistent root, apply their transformation, and + CAS-publish the candidate; a lost CAS retries from the winning root. +- The root and term count are one immutable state, so readers cannot observe a + torn pair. +- A retained snapshot or owned zipper remains on its captured revision. #### Summary **Key Takeaways:** -1. πŸ”— `.clone()` creates **shallow copy** with two Arc increments (map + count) +1. πŸ”— `.clone()` shares one atomic publication cell 2. πŸš€ **$`O(1)`$** time and space - just atomic reference counting 3. πŸ”„ **Mutations visible** across all clones (Arc-based sharing) 4. 🌳 **Structural sharing** is separate (PathMap's persistent trie optimization) -5. πŸ”’ **Thread-safe** with dual RwLocks for flexible granularity +5. πŸ”’ **Thread-safe** through immutable roots and atomic publication 6. πŸ“Š For **independence**, use serialization or rebuild from terms ($`O(n)`$ cost) ## Construction Methods @@ -459,7 +471,7 @@ valued_dict.insert_with_value("banana", 200); **Characteristics:** - **Time**: $`O(1)`$ - Minimal initialization -- **Memory**: ~80 bytes (two Arc pointers + empty PathMap + term count) +- **Memory**: O(1) publication state plus the upstream PathMap root - **Simplicity**: Easiest to use, minimal boilerplate **When to use:** @@ -486,7 +498,7 @@ let dict = PathMapDictionary::from_terms(term_set); **Characteristics:** - **Time**: $`O(n\cdot \log m)`$ where m grows from 0 to n -- **Memory**: ~32 bytes per node (HashMap-based) +- **Memory**: determined by the upstream PathMap representation and structural sharing - **Structural sharing**: Minimal (PathMap not optimized for bulk insert) ### From Terms with Values @@ -775,7 +787,7 @@ The `union_with()` and `union_replace()` methods enable **merging two PathMapDic - πŸ’Ύ Creating snapshots with incremental updates **Key Characteristics**: -- πŸ”’ **Thread-safe**: Operations use RwLock for concurrent access +- πŸ”’ **Thread-safe**: Operations use immutable roots and atomic publication - 🌳 **Structural sharing**: Leverages PathMap's persistent data structure benefits - ⚑ **Iterator-based**: Uses PathMap's efficient iteration over key-value pairs - 🎯 **Flexible**: Custom merge functions for value conflicts @@ -799,9 +811,9 @@ where - **Returns**: Number of terms processed from `other` **Algorithm**: Iteration-based insertion -1. Acquire read lock on `other.map` -2. Acquire write lock on `self.map` -3. Iterate all `(key, value)` pairs in `other.map` +1. Load one immutable root from `other` +2. Load the current immutable root from `self` +3. Iterate all `(key, value)` pairs in the captured `other` root 4. For each pair: - If key exists in `self.map`: Apply `merge_fn` and update - If key is new: Insert with cloned value @@ -958,19 +970,21 @@ assert_eq!(dict1.get_value("author"), Some("alice")); ### Implementation Details -The union operation uses **PathMap's iterator** with lock-based synchronization: +The union operation uses **PathMap's iterator** over an immutable source root and +publishes the transformed target root atomically: ```rust // Simplified implementation fn union_with(&self, other: &Self, merge_fn: F) -> usize { - let other_map = other.map.read().unwrap(); - let mut self_map = self.map.write().unwrap(); - let mut self_term_count = self.term_count.write().unwrap(); + let other_state = other.state.load_full(); + let self_state = self.state.load_full(); + let mut candidate = self_state.map.clone(); + let mut term_count = self_state.len; let mut processed = 0; - // Iterate over all entries in other - for (key_bytes, other_value) in other_map.iter() { + // Iterate over all entries in the captured source revision + for (key_bytes, other_value) in other_state.map.iter() { processed += 1; if let Some(self_value) = self_map.get(&key_bytes) { @@ -992,13 +1006,13 @@ fn union_with(&self, other: &Self, merge_fn: F) -> usize { 1. **Simplicity**: Leverages PathMap's well-tested iterator 2. **Flexibility**: No trait constraints on value types -3. **Correctness**: RwLock ensures thread-safe updates +3. **Correctness**: immutable roots and CAS publication prevent torn updates 4. **Structural sharing**: PathMap automatically shares structure between old and new versions -**Lock Semantics**: -- Read lock on `other`: Allows concurrent reads -- Write lock on `self`: Blocks all access during union -- Single transaction: All updates atomic from external perspective +**Publication semantics**: +- `other` remains on the captured immutable revision during iteration. +- The candidate for `self` is published atomically; competing writers retry. +- Readers observe either the old or new complete revision, never a partial union. ### Performance Characteristics @@ -1337,12 +1351,12 @@ Query "test" (distance 2): ``` 10,000-term dictionary: - PathMapDictionary: ~320 KB + PathMapDictionary: upstream-dependent DoubleArrayTrie: ~100 KB (3.2x smaller) DynamicDawg: ~294 KB (similar) Memory overhead: - PathMapDictionary: ~32 bytes/node (HashMap) + PathMapDictionary: upstream-dependent node representation DoubleArrayTrie: ~10 bytes/state DynamicDawg: ~25 bytes/node ``` diff --git a/docs/algorithms/variable-width-profiles.md b/docs/algorithms/variable-width-profiles.md new file mode 100644 index 00000000..d2d252ae --- /dev/null +++ b/docs/algorithms/variable-width-profiles.md @@ -0,0 +1,74 @@ +# Variable-width logical profiles + +Libdictenstein separates a dictionary's topology from the representation of its +logical edge labels. A profile defines the logical atom type, its canonical +encoding, its stable identity, and the boundary at which encoded data becomes +visible to callers. + +## Logical atoms and physical bytes + +The dictionary core traverses logical atoms. Encoded bytes are an implementation +detail of byte-backed profiles and must not be reported as additional semantic +transitions. A UTF-8 profile therefore represents one Unicode scalar per logical +atom even though its canonical codeword occupies one to four bytes. A ULEB128 +profile represents one arbitrary-width unsigned integer per logical atom; its +continuation bytes are never independent symbols. + +This distinction is important for Levenshtein, llattice, and language-level +consumers: edit distance, prefix traversal, zippers, and suffix operations must +operate on logical atoms, not codec bytes. + +## Stable profile identity + +`AtomProfile` supplies a compile-time identity consisting of a canonical name, +version, logical `ProfileKind`, and optional fixed wire width. Persisted +descriptors must retain the name and version and reject unknown, mismatched, or +non-canonical combinations. A missing descriptor is not permission to guess a +codec from the Rust type or from the first key. + +The built-in profiles are: + +| Profile | Logical atom | Encoding | Width | +| --- | --- | --- | --- | +| `Bytes` | `u8` | raw byte | 1 | +| `UnicodeScalar` | `char` | scalar value | 4 | +| `Utf8` | `char` | canonical UTF-8 codeword | variable | +| `U32` | `u32` | native fixed-width unit | 4 | +| `U64` | `u64` | native fixed-width unit | 8 | +| `F64Bits` | `u64` bit pattern | native fixed-width unit | 8 | +| `Uleb128` | arbitrary-width unsigned integer | canonical ULEB128 | variable | + +Canonical variable-width encodings are non-empty, unambiguous, and validated +before insertion or traversal. Malformed, truncated, non-canonical, and +overlong codewords are rejected rather than interpreted as a partial result. + +## Topology and naming + +`DictionaryFamily` identifies the storage topology (`DynamicDawg`, +`DoubleArrayTrie`, `PathMap`, `SuffixAutomaton`, `Scdawg`, or a persistent +ARTrie family). `BackendProfileDescriptor` identifies the logical profile. The +two axes are intentionally independent; `DictionarySpec

` and +`ProfiledDictionaryContainer

` bind them when a user-facing typed boundary +is desired. + +Legacy names such as `DynamicDawgChar` remain source-compatible. New code can +use the generic core (`DynamicDawgGeneric`) and provide a profile witness +with `profile_descriptor::

()`. Specialized UTF-8 and ULEB128 wrappers are +appropriate when they improve ergonomics or enforce validation at the boundary. + +## Internalization and vocabulary IDs + +Interned dictionaries map logical atoms to dense local IDs and store sequences of +those IDs. The mapping, profile descriptor, vocabulary generation, and snapshot +identity travel together. IDs are capsule-local compression handles; they are +not stable public identities and must never be interpreted without the matching +vocabulary metadata. + +## Consumer contract + +Consumers should request logical traversal APIs or profile-aware zippers. They +must not inspect physical codec nodes as semantic transitions. A serialized +dictionary is readable only when its topology, profile identity, and vocabulary +metadata validate together. This keeps backend substitution observationally +equivalent even when one implementation stores bytes and another stores native +logical units. diff --git a/docs/architecture/optimization-roadmap.md b/docs/architecture/optimization-roadmap.md index 0f2a027d..20ea2b26 100644 --- a/docs/architecture/optimization-roadmap.md +++ b/docs/architecture/optimization-roadmap.md @@ -100,7 +100,7 @@ Unicode node stays sorted or sparse-indexed. | Decision | Consequence | |----------|-------------| -| Volatile mutable dictionaries use internal synchronization. | Public callers do not need an outer `RwLock` for `DynamicDawg`, `PathMapDictionary`, `SuffixAutomaton`, `Scdawg`, or `BijectiveMap`. | +| Volatile mutable dictionaries own their publication/synchronization policy. | Public callers do not need an outer `RwLock` for `DynamicDawg`, `PathMapDictionary`, `SuffixAutomaton`, `Scdawg`, or `BijectiveMap`; `PathMapDictionary` uses immutable-root ArcSwap publication. | | Reader handles and zippers carry stable snapshots. | Compaction and mutation cannot invalidate traversal state. | | Writers use CAS publication or per-node atomic edge/value replacement. | Contended writers may retry, but readers do not block behind writers. | | Static backends keep compact array layouts. | Lookup and traversal stay cache-local for read-heavy dictionaries. | diff --git a/docs/diagrams/eviction-pipeline.svg b/docs/diagrams/eviction-pipeline.svg index 7a66516e..0776de97 100644 --- a/docs/diagrams/eviction-pipeline.svg +++ b/docs/diagrams/eviction-pipeline.svg @@ -1 +1 @@ -Eviction pipeline — pressure → urgency → LRU select → unswizzleNormal>30% free(full cache)Low10-30% freeCritical<10% freeEviction idleCoordinator queueVecDeque<Request>higher urgencyMERGESwith the pendingrequest(no duplicate cycles)Cooldown checkskip if too recent or> 5 s stale(record_skip)Wait epochquiescenceadvance() →wait_for_quiescence(EBR — seeepoch-reclamation)Select coldnodes (LRU)select_for_eviction:keep depth ≥min_eviction_depth,scoreby LRU coldness,takecoldest up tobatch_size× urgencyAtomic unswizzleChild → DiskRefcallback CAS-swapseachcold node; root staysin memory (pathnon-empty)Record statsnodes_evicted ·bytes_freed ·eviction_cycles ·last_duration_msSkip cycleno eviction this passCold node on disk(DiskRef →block_id + location)re-faulted on nextaccess (lazy load viaBufferManager)The eviction threadNEVER blocksclient inserts/lookups:it runs inthe background, holdsonly aWeak<Coordinator>,and evicts onlynodes whose diskimage is current —a write's root CASclears exacteviction authority untilthe nextcheckpointre-publishes a binding.avail < lowrecoversavail < critrecoversno-op⇒ Moderate⇒ Emergencyasync thread pollstry_pop (≈100 ms)within cooldowntoo recent →record_skipreaders drainedtimeoutnon-empty setempty → (0,0)nodes_evicted,bytes_freedcycle completeawait requestnext requestRAM reclaimedafter EBR freeColorUrgency / severity Normal — full caching, idle Low → Moderate — proactive Critical → Emergency — aggressive coordinator plumbing durable on-disk outcome \ No newline at end of file +Eviction pipeline — pressure → urgency → LRU select → unswizzleNormal>30% free(full cache)Low10-30% freeCritical<10% freeEviction idleCoordinator queueVecDeque<Request>higher urgencyMERGESwith the pendingrequest(no duplicate cycles)Cooldown checkskip if too recent or> 5 s stale(record_skip)Wait epochquiescenceadvance() →wait_for_quiescence(EBR — seeepoch-reclamation)Select coldnodes (LRU)select_for_eviction:keep depth ≥min_eviction_depth,scoreby LRU coldness,takecoldest up tobatch_size× urgencyAtomic unswizzleChild → DiskRefcallback CAS-swapseachcold node; root staysin memory (pathnon-empty)Record statsnodes_evicted ·bytes_freed ·eviction_cycles ·last_duration_msSkip cycleno eviction this passCold node on disk(DiskRef →block_id + location)re-faulted on nextaccess (lazy load viaBufferManager)The eviction threadNEVER blocksclient inserts/lookups:it runs inthe background, holdsonly aWeak<Coordinator>,and evicts onlynodes whose diskimage is current —a write's root CASclears exacteviction authority untilthe nextcheckpointre-publishes a binding.avail < lowrecoversavail < critrecoversno-op⇒ Moderate⇒ Emergencyasync thread pollstry_pop (≈100 ms)within cooldowntoo recent →record_skipreaders drainedtimeoutnon-empty setempty → (0,0)nodes_evicted,bytes_freedcycle completeawait requestnext requestRAM reclaimedafter EBR freeColorUrgency / severity Normal — full caching, idle Low → Moderate — proactive Critical → Emergency — aggressive coordinator plumbing durable on-disk outcome \ No newline at end of file diff --git a/docs/integration/pathmap/README.md b/docs/integration/pathmap/README.md index 6d68505e..1e700bb4 100644 --- a/docs/integration/pathmap/README.md +++ b/docs/integration/pathmap/README.md @@ -832,7 +832,8 @@ Where: ### Concurrent Access -PathMap supports concurrent reads via memory mapping: +The adapter supports concurrent reads by loading immutable roots; a retained +snapshot is independent of later publications: ```rust use std::sync::Arc; @@ -966,8 +967,10 @@ On 32-bit systems, use file-based PathMap instead of mmap. **Issue**: Concurrent write conflicts ``` -Solution: PathMap supports concurrent reads but single writer. -Use write locks or process-level coordination for updates. +Solution: `PathMapDictionary` clones a persistent root and CAS-publishes the +candidate. Competing writers retry against the winning root; callers do not add +an outer write lock. A successful mutation is visible to later root loads, +while retained snapshots intentionally remain on their captured revision. ``` --- diff --git a/formal-verification/Cargo.toml b/formal-verification/Cargo.toml new file mode 100644 index 00000000..e762415b --- /dev/null +++ b/formal-verification/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "libdictenstein-variable-width-harness" +version = "0.1.0" +edition = "2021" +publish = false +rust-version = "1.95" + +[dev-dependencies] +proptest = "1.11" + +[[test]] +name = "variable_width_formal_harness" +path = "../tests/variable_width_formal_harness.rs" + +[workspace] diff --git a/formal-verification/rocq/Makefile b/formal-verification/rocq/Makefile index 53d8e3f7..b700661b 100644 --- a/formal-verification/rocq/Makefile +++ b/formal-verification/rocq/Makefile @@ -10,7 +10,7 @@ COQFLAGS = -Q . ARTrie DIRS = Spec Model Invariants Operations Proofs # Source files in dependency order -SPEC_FILES = Spec/MapSpec.v Spec/DictionaryLawSpec.v Spec/DynamicDawgMutationSpec.v Spec/DynamicDawgU64Spec.v Spec/PersistentARTrieU64Spec.v Spec/DoubleArrayTrieSpec.v Spec/ZipperLanguageSpec.v Spec/ValuedSetCombinatorSpec.v Spec/BloomFilterSpec.v Spec/PersistentMergeSpec.v Spec/PersistentPrefixSpec.v Spec/PersistentReadTraversalSpec.v Spec/PersistentSuffixAutomatonSpec.v Spec/PersistentScdawgSpec.v Spec/PersistentCharNodeLayoutSpec.v Spec/PathMapFactorySpec.v Spec/PathMapSnapshotSpec.v Spec/RelativeEncodingSpec.v Spec/ArenaReservationSpec.v Spec/DedupArenaSpec.v Spec/RootDescriptorReopenSpec.v Spec/PersistentLazyMutationSpec.v Spec/PersistentWalAtomicitySpec.v Spec/LockFreeCounterMergeSpec.v Spec/OverlayReestablishSpec.v Spec/SharedPersistentConcurrencySpec.v Spec/PublicDurabilityPolicySpec.v Spec/PersistentPublicWalLifecycleSpec.v Spec/PersistentEndToEndTraceSpec.v Spec/PersistentVocabWalAtomicitySpec.v Spec/PersistentVocabCheckpointSpec.v Spec/PersistentCheckpointRetentionSpec.v Spec/PersistentDirtyCheckpointSpec.v Spec/PersistentWalSegmentLifecycleSpec.v Spec/PersistentRecoveryPlannerSpec.v Spec/PersistentRecoveryReplayCompletenessSpec.v Spec/PersistentCompactionSpec.v Spec/PersistentRewriteCompactionSpec.v Spec/SubstringSearchSpec.v Spec/ScdawgOccurrenceSpec.v Spec/FuzzyCandidateCoverageSpec.v Spec/SerializationRoundtripSpec.v Spec/OverlayArborescenceSerializationSpec.v Spec/ARTrieSpec.v Spec/ReplicatedMapSpec.v Spec/PersistentCharEvictionRegistrySpec.v Spec/EvictionExactRootPublicationSpec.v Spec/HelpedRootResidencySpec.v Spec/PackedResidencyRefinementSpec.v Spec/HelpedCheckpointStampSpec.v Spec/DetachedCallbackSeparationSpec.v Spec/ResidentBudgetEvictionSpec.v Spec/OverlayFaultProvenanceSpec.v Spec/DictionaryNodeReopenTraversalSpec.v Spec/PersistentCharEpochReclamationSpec.v Spec/CharV3TypeEncodingSpec.v Spec/ApiFeatureVisibilitySpec.v Spec/TlcDiagnosticClassifierSpec.v Spec/OverlayDenseCodecSpec.v Spec/AbiTraversalSnapshotSpec.v Spec/AbiPagingProducerSpec.v Spec/AbiStatusMappingSpec.v Spec/AbiDictionaryAlgebraSpec.v +SPEC_FILES = Spec/MapSpec.v Spec/DictionaryLawSpec.v Spec/DynamicDawgMutationSpec.v Spec/DynamicDawgU64Spec.v Spec/VariableWidthCodecSpec.v Spec/VariableWidthInterningSpec.v Spec/VariableWidthFamilyRefinementSpec.v Spec/PersistentARTrieU64Spec.v Spec/DoubleArrayTrieSpec.v Spec/ZipperLanguageSpec.v Spec/ValuedSetCombinatorSpec.v Spec/BloomFilterSpec.v Spec/PersistentMergeSpec.v Spec/PersistentPrefixSpec.v Spec/PersistentReadTraversalSpec.v Spec/PersistentSuffixAutomatonSpec.v Spec/PersistentScdawgSpec.v Spec/PersistentCharNodeLayoutSpec.v Spec/PathMapFactorySpec.v Spec/PathMapSnapshotSpec.v Spec/RelativeEncodingSpec.v Spec/ArenaReservationSpec.v Spec/DedupArenaSpec.v Spec/RootDescriptorReopenSpec.v Spec/PersistentLazyMutationSpec.v Spec/PersistentWalAtomicitySpec.v Spec/LockFreeCounterMergeSpec.v Spec/OverlayReestablishSpec.v Spec/SharedPersistentConcurrencySpec.v Spec/PublicDurabilityPolicySpec.v Spec/PersistentPublicWalLifecycleSpec.v Spec/PersistentEndToEndTraceSpec.v Spec/PersistentVocabWalAtomicitySpec.v Spec/PersistentVocabCheckpointSpec.v Spec/PersistentCheckpointRetentionSpec.v Spec/PersistentDirtyCheckpointSpec.v Spec/PersistentWalSegmentLifecycleSpec.v Spec/PersistentRecoveryPlannerSpec.v Spec/PersistentRecoveryReplayCompletenessSpec.v Spec/PersistentCompactionSpec.v Spec/PersistentRewriteCompactionSpec.v Spec/SubstringSearchSpec.v Spec/ScdawgOccurrenceSpec.v Spec/FuzzyCandidateCoverageSpec.v Spec/SerializationRoundtripSpec.v Spec/OverlayArborescenceSerializationSpec.v Spec/ARTrieSpec.v Spec/ReplicatedMapSpec.v Spec/PersistentCharEvictionRegistrySpec.v Spec/EvictionExactRootPublicationSpec.v Spec/HelpedRootResidencySpec.v Spec/PackedResidencyRefinementSpec.v Spec/HelpedCheckpointStampSpec.v Spec/DetachedCallbackSeparationSpec.v Spec/ResidentBudgetEvictionSpec.v Spec/OverlayFaultProvenanceSpec.v Spec/DictionaryNodeReopenTraversalSpec.v Spec/PersistentCharEpochReclamationSpec.v Spec/CharV3TypeEncodingSpec.v Spec/ApiFeatureVisibilitySpec.v Spec/TlcDiagnosticClassifierSpec.v Spec/OverlayDenseCodecSpec.v Spec/AbiTraversalSnapshotSpec.v Spec/AbiPagingProducerSpec.v Spec/AbiStatusMappingSpec.v Spec/AbiDictionaryAlgebraSpec.v MODEL_FILES = Model/ListCompat.v Model/Key.v Model/NodeTypes.v Model/Bucket.v Model/HotStuff.v Model/PathCompression.v Model/PrefixChunking.v Model/ArenaManager.v Model/SequentialSiblings.v Model/FileSystem.v INVARIANT_FILES = Invariants/StructuralInvariants.v Invariants/TransitionInvariants.v Invariants/ArenaInvariants.v Invariants/SequentialSiblingsInvariants.v OPERATION_FILES = # Operations/Lookup.v Operations/Insert.v Operations/Delete.v @@ -51,8 +51,10 @@ clean: # Build with resource limits (for memory-intensive proofs) # Per CLAUDE.md recommendations build-safe: - systemd-run --user --scope -p MemoryMax=32G -p CPUQuota=1800% \ - -p IOWeight=30 -p TasksMax=200 $(MAKE) -j1 + systemd-run --user --unit=libdictenstein-rocq-build-$$$$ \ + --wait --pipe --quiet --collect --working-directory=$(CURDIR) \ + -p MemoryHigh=1G -p MemoryMax=2G -p MemorySwapMax=0 \ + -p CPUQuota=200% -p IOWeight=30 -p TasksMax=64 $(MAKE) -j1 # Display build order show-order: diff --git a/formal-verification/rocq/Spec/PersistentARTrieU64Spec.v b/formal-verification/rocq/Spec/PersistentARTrieU64Spec.v index a852433c..19be351d 100644 --- a/formal-verification/rocq/Spec/PersistentARTrieU64Spec.v +++ b/formal-verification/rocq/Spec/PersistentARTrieU64Spec.v @@ -245,7 +245,7 @@ Proof. pose proof (@NoDup_incl_length nat (map fst frames) (seq 0 node_count) Hunique Hincl) as Hlength. - rewrite length_map, seq_length_portable in Hlength. + rewrite map_length, seq_length_portable in Hlength. exact Hlength. Qed. diff --git a/formal-verification/rocq/Spec/SerializationRoundtripSpec.v b/formal-verification/rocq/Spec/SerializationRoundtripSpec.v index 8e2e01a8..1b5fa40d 100644 --- a/formal-verification/rocq/Spec/SerializationRoundtripSpec.v +++ b/formal-verification/rocq/Spec/SerializationRoundtripSpec.v @@ -1439,7 +1439,7 @@ Proof. intros first later pending direct resulting_pending Hscheduled. cbn [tail_child_schedule] in Hscheduled. inversion Hscheduled; subst. - rewrite app_length_portable, length_rev. + rewrite app_length_portable, rev_length. reflexivity. Qed. @@ -1536,7 +1536,7 @@ Proof. lia. - unfold bounded_tail_child_schedule, counted_batch_capacity. simpl. - rewrite app_length_portable, length_rev. + rewrite app_length_portable, rev_length. apply Nat.add_le_mono_l. apply firstn_le_length. Qed. diff --git a/formal-verification/rocq/Spec/VariableWidthCodecSpec.v b/formal-verification/rocq/Spec/VariableWidthCodecSpec.v new file mode 100644 index 00000000..3ad94230 --- /dev/null +++ b/formal-verification/rocq/Spec/VariableWidthCodecSpec.v @@ -0,0 +1,2524 @@ +(** * Variable-width codec and logical-transition laws + + This module fixes the representation-independent contract for dictionary + profiles before a Rust codec is introduced. Variable-width bytes are a + storage grammar only. A dictionary transition observed by a language + consumer denotes exactly one logical atom. + + Canonical ULEB128 is modeled as an arbitrary-length, little-endian sequence + of seven-bit digits. Rocq naturals are unbounded, so no theorem silently + restricts a value to a Rust primitive. UTF-8 codewords are canonical + encodings of Unicode scalar values. Direct profiles expose a single fixed + unit. F64Bits preserves raw IEEE-754 bit identity and orders those bits by + the monotone key used by Rust's total_cmp rather than numeric equality. + + Stable theorem names beginning with [VWENC_] are machine-readable invariant + IDs. The + conformance ledger and property tests consume these exact identifiers. +*) + +From Coq Require Import Arith Bool Lia List PeanoNat. +Require Import ARTrie.Spec.DynamicDawgMutationSpec. +Require Import ARTrie.Spec.DynamicDawgU64Spec. +Import ListNotations. + +Module VariableWidthCodecSpec. + +(** ** Canonical arbitrary-width ULEB128 *) + +Definition PhysicalByte := nat. +Definition UlebDigit := nat. + +Definition valid_byte (byte : PhysicalByte) : Prop := byte < 256. +Definition valid_uleb_digit (digit : UlebDigit) : Prop := digit < 128. + +Fixpoint encode_uleb_digits (digits : list UlebDigit) : list PhysicalByte := + match digits with + | [] => [] + | [digit] => [digit] + | digit :: rest => (128 + digit) :: encode_uleb_digits rest + end. + +Definition uleb_payload (byte : PhysicalByte) : UlebDigit := byte mod 128. +Definition decode_uleb_payloads (bytes : list PhysicalByte) : list UlebDigit := + map uleb_payload bytes. + +(** Every non-final byte continues; the final byte terminates. *) +Inductive uleb_continuation_shape : list PhysicalByte -> Prop := +| UlebShapeLast : forall byte, + byte < 128 -> + uleb_continuation_shape [byte] +| UlebShapeMore : forall byte rest, + 128 <= byte -> + byte < 256 -> + uleb_continuation_shape rest -> + uleb_continuation_shape (byte :: rest). + +(** A multi-byte zero high digit is an overlong spelling. The singleton zero + remains the canonical spelling of logical zero. *) +Definition canonical_uleb_digits (digits : list UlebDigit) : Prop := + digits <> [] /\ + Forall valid_uleb_digit digits /\ + (2 <= length digits -> last digits 0 <> 0). + +Definition canonical_uleb_codeword (bytes : list PhysicalByte) : Prop := + uleb_continuation_shape bytes /\ + canonical_uleb_digits (decode_uleb_payloads bytes). + +Lemma uleb_terminal_payload_identity : + forall digit, valid_uleb_digit digit -> uleb_payload digit = digit. +Proof. + intros digit Hdigit. + unfold valid_uleb_digit, uleb_payload in *. + apply Nat.mod_small. exact Hdigit. +Qed. + +Lemma uleb_continuing_payload_identity : + forall digit, + valid_uleb_digit digit -> uleb_payload (128 + digit) = digit. +Proof. + intros digit Hdigit. + unfold valid_uleb_digit, uleb_payload in *. + replace (128 + digit) with (digit + 128) by lia. + rewrite Nat.Div0.add_mod by lia. + rewrite Nat.Div0.mod_same by lia. + rewrite Nat.add_0_r. + rewrite Nat.Div0.mod_mod by lia. + apply Nat.mod_small. exact Hdigit. +Qed. + +Theorem VWENC_01_ULEB_PAYLOAD_ROUNDTRIP : + forall digits, + Forall valid_uleb_digit digits -> + decode_uleb_payloads (encode_uleb_digits digits) = digits. +Proof. + induction digits as [| digit rest IH]; intros Hvalid. + - reflexivity. + - inversion Hvalid as [| ? ? Hdigit Hrest]; subst. + destruct rest as [| next tail]. + + change ([uleb_payload digit] = [digit]). + rewrite uleb_terminal_payload_identity by exact Hdigit. + reflexivity. + + change + (uleb_payload (128 + digit) :: + decode_uleb_payloads (encode_uleb_digits (next :: tail)) = + digit :: next :: tail). + rewrite uleb_continuing_payload_identity by exact Hdigit. + f_equal. apply IH. exact Hrest. +Qed. + +Theorem VWENC_88_ULEB_CANONICAL_DIGIT_ENCODER_IS_INJECTIVE : + forall left right, + Forall valid_uleb_digit left -> + Forall valid_uleb_digit right -> + encode_uleb_digits left = encode_uleb_digits right -> + left = right. +Proof. + intros left right Hleft Hright Hencoded. + apply (f_equal decode_uleb_payloads) in Hencoded. + rewrite (VWENC_01_ULEB_PAYLOAD_ROUNDTRIP left Hleft) in Hencoded. + rewrite (VWENC_01_ULEB_PAYLOAD_ROUNDTRIP right Hright) in Hencoded. + exact Hencoded. +Qed. + +Lemma encode_uleb_has_continuation_shape : + forall digits, + digits <> [] -> + Forall valid_uleb_digit digits -> + uleb_continuation_shape (encode_uleb_digits digits). +Proof. + induction digits as [| digit rest IH]; intros Hnonempty Hvalid. + - contradiction. + - inversion Hvalid as [| ? ? Hdigit Hrest]; subst. + unfold valid_uleb_digit in Hdigit. + destruct rest as [| next tail]. + + simpl. constructor. exact Hdigit. + + simpl. apply UlebShapeMore; [lia | lia |]. + apply IH; [discriminate | exact Hrest]. +Qed. + +Theorem VWENC_02_ULEB_CANONICAL_ENCODE : + forall digits, + canonical_uleb_digits digits -> + canonical_uleb_codeword (encode_uleb_digits digits). +Proof. + intros digits Hcanonical. + destruct Hcanonical as [Hnonempty [Hvalid Hminimal]]. + split. + - apply encode_uleb_has_continuation_shape; assumption. + - unfold canonical_uleb_digits. + rewrite VWENC_01_ULEB_PAYLOAD_ROUNDTRIP by exact Hvalid. + repeat split; assumption. +Qed. + +Theorem VWENC_03_ULEB_CODEWORDS_NONEMPTY : + forall bytes, canonical_uleb_codeword bytes -> bytes <> []. +Proof. + intros bytes [Hshape _] ->. inversion Hshape. +Qed. + +Lemma uleb_continuing_byte_payload_identity : + forall byte, + 128 <= byte -> byte < 256 -> 128 + uleb_payload byte = byte. +Proof. + intros byte Hlower Hupper. + assert (exists digit, byte = 128 + digit /\ digit < 128) as + [digit [-> Hdigit]]. + { exists (byte - 128). split; lia. } + rewrite uleb_continuing_payload_identity by exact Hdigit. + reflexivity. +Qed. + +Lemma decode_uleb_payloads_nonempty : + forall bytes, + uleb_continuation_shape bytes -> decode_uleb_payloads bytes <> []. +Proof. + intros bytes Hshape. + inversion Hshape; discriminate. +Qed. + +Lemma encode_uleb_cons_with_nonempty_tail : + forall digit tail, + tail <> [] -> + encode_uleb_digits (digit :: tail) = + (128 + digit) :: encode_uleb_digits tail. +Proof. + intros digit tail Hnonempty. + destruct tail; [contradiction | reflexivity]. +Qed. + +Lemma uleb_shape_reencodes_payloads : + forall bytes, + uleb_continuation_shape bytes -> + encode_uleb_digits (decode_uleb_payloads bytes) = bytes. +Proof. + intros bytes Hshape. + induction Hshape as [byte Hterminal | byte rest Hlower Hupper Hrest IH]. + - change ([uleb_payload byte] = [byte]). + rewrite uleb_terminal_payload_identity by exact Hterminal. + reflexivity. + - change + (encode_uleb_digits + (uleb_payload byte :: decode_uleb_payloads rest) = + byte :: rest). + rewrite encode_uleb_cons_with_nonempty_tail. + + rewrite uleb_continuing_byte_payload_identity by assumption. + f_equal. exact IH. + + apply decode_uleb_payloads_nonempty. exact Hrest. +Qed. + +Theorem VWENC_04_ULEB_UNIQUE_DECODING : + forall left right, + uleb_continuation_shape left -> + uleb_continuation_shape right -> + decode_uleb_payloads left = decode_uleb_payloads right -> + left = right. +Proof. + intros left right Hleft Hright Hpayloads. + rewrite <- (uleb_shape_reencodes_payloads left Hleft). + rewrite <- (uleb_shape_reencodes_payloads right Hright). + now rewrite Hpayloads. +Qed. + +Lemma uleb_shape_final_byte_terminates : + forall bytes default, + uleb_continuation_shape bytes -> last bytes default < 128. +Proof. + intros bytes default Hshape. + induction Hshape as [byte Hterminal | byte rest Hlower Hupper Hrest IH]. + - exact Hterminal. + - destruct Hrest; simpl; apply IH. +Qed. + +Definition unterminated_uleb (bytes : list PhysicalByte) : Prop := + bytes <> [] /\ 128 <= last bytes 0. + +Theorem VWENC_05_ULEB_UNTERMINATED_REJECTED : + forall bytes, + canonical_uleb_codeword bytes -> ~ unterminated_uleb bytes. +Proof. + intros bytes [Hshape _] [_ Hcontinues]. + pose proof (uleb_shape_final_byte_terminates bytes 0 Hshape). + lia. +Qed. + +Definition overlong_uleb (bytes : list PhysicalByte) : Prop := + 2 <= length bytes /\ last (decode_uleb_payloads bytes) 0 = 0. + +Theorem VWENC_06_ULEB_OVERLONG_REJECTED : + forall bytes, + canonical_uleb_codeword bytes -> ~ overlong_uleb bytes. +Proof. + intros bytes [_ [_ [_ Hminimal]]] [Hlength Hzero]. + apply Hminimal. + - unfold decode_uleb_payloads. rewrite map_length. exact Hlength. + - exact Hzero. +Qed. + +Lemma uleb_shape_tail : + forall byte rest, + rest <> [] -> + uleb_continuation_shape (byte :: rest) -> + uleb_continuation_shape rest. +Proof. + intros byte rest Hnonempty Hshape. + inversion Hshape; subst. + - contradiction. + - assumption. +Qed. + +Theorem VWENC_07_ULEB_EARLY_TERMINATOR_REJECTED : + forall prefix terminal suffix, + suffix <> [] -> + terminal < 128 -> + ~ uleb_continuation_shape (prefix ++ terminal :: suffix). +Proof. + induction prefix as [| byte prefix IH]; intros terminal suffix Hsuffix Hterminal Hshape. + - inversion Hshape; subst. + + contradiction. + + lia. + - apply (IH terminal suffix Hsuffix Hterminal). + apply (uleb_shape_tail byte (prefix ++ terminal :: suffix)). + + destruct prefix; discriminate. + + exact Hshape. +Qed. + +Theorem VWENC_08_ULEB_EACH_BYTE_IS_U8 : + forall bytes, + uleb_continuation_shape bytes -> Forall valid_byte bytes. +Proof. + intros bytes Hshape. + induction Hshape as [byte Hterminal | byte rest Hlower Hupper Hrest IH]. + - constructor; [unfold valid_byte; lia | constructor]. + - constructor; [exact Hupper | exact IH]. +Qed. + +Theorem VWENC_09_ULEB_DECODING_IS_INPUT_BOUNDED : + forall bytes, + length (decode_uleb_payloads bytes) = length bytes. +Proof. + intros bytes. apply map_length. +Qed. + +Fixpoint byte_sequence_eqb + (left right : list PhysicalByte) : bool := + match left, right with + | [], [] => true + | left_byte :: left_rest, right_byte :: right_rest => + (left_byte =? right_byte) && + byte_sequence_eqb left_rest right_rest + | _, _ => false + end. + +Lemma byte_sequence_eqb_reflects_equality : + forall left right, + byte_sequence_eqb left right = true <-> left = right. +Proof. + induction left as [| left_byte left_rest IH]; + destruct right as [| right_byte right_rest]; simpl. + - tauto. + - split; [discriminate | discriminate]. + - split; [discriminate | discriminate]. + - rewrite andb_true_iff, Nat.eqb_eq, IH. + split. + + intros [-> ->]. reflexivity. + + intros Hequal. inversion Hequal. tauto. +Qed. + +Fixpoint uleb_continuation_shapeb + (bytes : list PhysicalByte) : bool := + match bytes with + | [] => false + | [byte] => byte + (128 <=? byte) && (byte + uleb_continuation_shape bytes. +Proof. + induction bytes as [| byte rest IH]. + - simpl. split; [discriminate | intros Hshape; inversion Hshape]. + - destruct rest as [| next tail]. + + simpl. rewrite Nat.ltb_lt. + split. + * intros Hterminal. constructor. exact Hterminal. + * intros Hshape. + exact (uleb_shape_final_byte_terminates [byte] 0 Hshape). + + change + (((128 <=? byte) && (byte + uleb_continuation_shape (byte :: next :: tail)). + rewrite !andb_true_iff, Nat.leb_le, Nat.ltb_lt, IH. + split. + * intros [[Hlower Hupper] Htail]. + now apply UlebShapeMore. + * intros Hshape. inversion Hshape; subst. tauto. +Qed. + +Definition canonical_uleb_minimalb (digits : list UlebDigit) : bool := + (length digits + (2 <= length digits -> last digits 0 <> 0). +Proof. + intros digits. + unfold canonical_uleb_minimalb. + rewrite orb_true_iff, negb_true_iff, Nat.ltb_lt, Nat.eqb_neq. + split. + - intros [Hshort | Hlast] Hmultiple; [lia | exact Hlast]. + - intros Hminimal. + destruct (Nat.lt_ge_cases (length digits) 2) as [Hshort | Hmultiple]. + + now left. + + right. apply Hminimal. exact Hmultiple. +Qed. + +Lemma uleb_payloads_are_digits : + forall bytes, + Forall valid_uleb_digit (decode_uleb_payloads bytes). +Proof. + induction bytes as [| byte rest IH]. + - constructor. + - constructor. + + unfold valid_uleb_digit, uleb_payload. + apply Nat.mod_upper_bound. lia. + + exact IH. +Qed. + +Definition canonical_uleb_codewordb + (bytes : list PhysicalByte) : bool := + uleb_continuation_shapeb bytes && + canonical_uleb_minimalb (decode_uleb_payloads bytes). + +Theorem VWENC_33_ULEB_CANONICAL_RECOGNIZER_IS_EXACT : + forall bytes, + canonical_uleb_codewordb bytes = true <-> + canonical_uleb_codeword bytes. +Proof. + intros bytes. + unfold canonical_uleb_codewordb, canonical_uleb_codeword. + rewrite andb_true_iff, + uleb_continuation_shapeb_reflects_shape, + canonical_uleb_minimalb_reflects_minimality. + split. + - intros [Hshape Hminimal]. split; [exact Hshape |]. + unfold canonical_uleb_digits. + repeat split. + + apply decode_uleb_payloads_nonempty. exact Hshape. + + apply uleb_payloads_are_digits. + + exact Hminimal. + - intros [Hshape [_ [_ Hminimal]]]. tauto. +Qed. + +Definition decode_canonical_uleb + (bytes : list PhysicalByte) : option (list UlebDigit) := + if canonical_uleb_codewordb bytes + then Some (decode_uleb_payloads bytes) + else None. + +Theorem VWENC_34_ULEB_DECODER_ACCEPTS_EXACTLY_CANONICAL_CODEWORDS : + forall bytes, + canonical_uleb_codeword bytes <-> + decode_canonical_uleb bytes = Some (decode_uleb_payloads bytes). +Proof. + intros bytes. + unfold decode_canonical_uleb. + destruct (canonical_uleb_codewordb bytes) eqn:Hcanonical. + - rewrite VWENC_33_ULEB_CANONICAL_RECOGNIZER_IS_EXACT in Hcanonical. + tauto. + - split. + + intros Hcodeword. + apply VWENC_33_ULEB_CANONICAL_RECOGNIZER_IS_EXACT in Hcodeword. + rewrite Hcodeword in Hcanonical. discriminate. + + discriminate. +Qed. + +Theorem VWENC_89_ULEB_DECODER_ROUNDTRIPS_CANONICAL_ENCODER : + forall digits, + canonical_uleb_digits digits -> + decode_canonical_uleb (encode_uleb_digits digits) = Some digits. +Proof. + intros digits Hcanonical. + assert (canonical_uleb_codeword (encode_uleb_digits digits)) + as Hcodeword. + { now apply VWENC_02_ULEB_CANONICAL_ENCODE. } + apply VWENC_34_ULEB_DECODER_ACCEPTS_EXACTLY_CANONICAL_CODEWORDS + in Hcodeword. + rewrite Hcodeword. + destruct Hcanonical as [Hnonempty [Hvalid Hminimal]]. + now rewrite VWENC_01_ULEB_PAYLOAD_ROUNDTRIP. +Qed. + +Theorem VWENC_35_ULEB_NONCANONICAL_AND_MALFORMED_INPUT_IS_REJECTED : + forall bytes, + ~ canonical_uleb_codeword bytes -> + decode_canonical_uleb bytes = None. +Proof. + intros bytes Hnoncanonical. + unfold decode_canonical_uleb. + destruct (canonical_uleb_codewordb bytes) eqn:Hcanonical. + - apply VWENC_33_ULEB_CANONICAL_RECOGNIZER_IS_EXACT in Hcanonical. + contradiction. + - reflexivity. +Qed. + +Theorem VWENC_36_ULEB_ENCODER_HAS_NO_BUILTIN_WIDTH_LIMIT : + forall digits, + length (encode_uleb_digits digits) = length digits. +Proof. + induction digits as [| digit rest IH]. + - reflexivity. + - destruct rest as [| next tail]. + + reflexivity. + + simpl. f_equal. exact IH. +Qed. + +Fixpoint uleb_value (digits : list UlebDigit) : nat := + match digits with + | [] => 0 + | digit :: rest => digit + 128 * uleb_value rest + end. + +(** Compare equal-width canonical codewords from their most-significant + payloads toward their least-significant payloads. The recursion visits + the physical bytes themselves and never materializes a bounded integer or + a BigUint. The unequal-list cases make the function total; the public + comparator below selects them only after an explicit width comparison. *) +Fixpoint compare_equal_width_uleb_bytes + (left right : list PhysicalByte) : comparison := + match left, right with + | [], [] => Eq + | [], _ => Lt + | _, [] => Gt + | left_byte :: left_rest, right_byte :: right_rest => + match compare_equal_width_uleb_bytes left_rest right_rest with + | Eq => Nat.compare + (uleb_payload left_byte) (uleb_payload right_byte) + | Lt => Lt + | Gt => Gt + end + end. + +Definition compare_uleb_codewords_structural + (left right : list PhysicalByte) : comparison := + match Nat.compare (length left) (length right) with + | Eq => compare_equal_width_uleb_bytes left right + | Lt => Lt + | Gt => Gt + end. + +(** Reverse-index machine used as the production correspondence. A Rust + implementation is a [while remaining != 0] loop over two borrowed slices: + decrement [remaining], mask the two indexed bytes, and return at the first + difference. It owns one index and one comparison only: O(n) time, O(1) + auxiliary state, no allocation, and no call-stack growth. The tail-recursive + Rocq evaluator below is a mathematical iterator; it is not an instruction + to extract recursive Rust. *) +Fixpoint compare_equal_width_uleb_reverse_index + (remaining : nat) + (left right : list PhysicalByte) : comparison := + match remaining with + | 0 => Eq + | S index => + match Nat.compare + (uleb_payload (nth index left 0)) + (uleb_payload (nth index right 0)) with + | Eq => compare_equal_width_uleb_reverse_index index left right + | Lt => Lt + | Gt => Gt + end + end. + +Lemma compare_reverse_index_cons : + forall remaining left_byte left_rest right_byte right_rest, + compare_equal_width_uleb_reverse_index + (S remaining) (left_byte :: left_rest) (right_byte :: right_rest) = + match compare_equal_width_uleb_reverse_index + remaining left_rest right_rest with + | Eq => Nat.compare + (uleb_payload left_byte) (uleb_payload right_byte) + | Lt => Lt + | Gt => Gt + end. +Proof. + induction remaining as [| remaining IH]; + intros left_byte left_rest right_byte right_rest. + - simpl. + destruct + (Nat.compare (uleb_payload left_byte) (uleb_payload right_byte)); + reflexivity. + - change + (match Nat.compare + (uleb_payload (nth remaining left_rest 0)) + (uleb_payload (nth remaining right_rest 0)) with + | Eq => + compare_equal_width_uleb_reverse_index + (S remaining) (left_byte :: left_rest) + (right_byte :: right_rest) + | Lt => Lt + | Gt => Gt + end = + match + (match Nat.compare + (uleb_payload (nth remaining left_rest 0)) + (uleb_payload (nth remaining right_rest 0)) with + | Eq => + compare_equal_width_uleb_reverse_index + remaining left_rest right_rest + | Lt => Lt + | Gt => Gt + end) + with + | Eq => Nat.compare + (uleb_payload left_byte) (uleb_payload right_byte) + | Lt => Lt + | Gt => Gt + end). + destruct + (Nat.compare + (uleb_payload (nth remaining left_rest 0)) + (uleb_payload (nth remaining right_rest 0))) eqn:Hhighest; + [apply IH | reflexivity | reflexivity]. +Qed. + +Lemma compare_reverse_index_agrees_with_structural : + forall left right, + length left = length right -> + compare_equal_width_uleb_reverse_index (length left) left right = + compare_equal_width_uleb_bytes left right. +Proof. + induction left as [| left_byte left_rest IH]; + destruct right as [| right_byte right_rest]; + intros Hlength; try discriminate; [reflexivity |]. + simpl in Hlength. injection Hlength as Hrest_length. + change + (compare_equal_width_uleb_reverse_index + (S (length left_rest)) (left_byte :: left_rest) + (right_byte :: right_rest) = + match compare_equal_width_uleb_bytes left_rest right_rest with + | Eq => Nat.compare + (uleb_payload left_byte) (uleb_payload right_byte) + | Lt => Lt + | Gt => Gt + end). + rewrite compare_reverse_index_cons. + rewrite (IH right_rest Hrest_length). + reflexivity. +Qed. + +Definition compare_uleb_codewords + (left right : list PhysicalByte) : comparison := + match Nat.compare (length left) (length right) with + | Eq => compare_equal_width_uleb_reverse_index (length left) left right + | Lt => Lt + | Gt => Gt + end. + +Theorem VWENC_95_REVERSE_INDEX_ULEB_COMPARATOR_REFINES_STRUCTURAL_SPEC : + forall left right, + compare_uleb_codewords left right = + compare_uleb_codewords_structural left right. +Proof. + intros left right. + unfold compare_uleb_codewords, compare_uleb_codewords_structural. + destruct (Nat.compare (length left) (length right)) + eqn:Hlength; try reflexivity. + apply Nat.compare_eq_iff in Hlength. + now apply compare_reverse_index_agrees_with_structural. +Qed. + +Record ReverseIndexMachineState := { + reverse_index_remaining : nat; + reverse_index_outcome : option comparison; +}. + +Definition reverse_index_machine_step + (left right : list PhysicalByte) + (state : ReverseIndexMachineState) : ReverseIndexMachineState := + match state.(reverse_index_outcome), state.(reverse_index_remaining) with + | Some outcome, _ => state + | None, 0 => + {| reverse_index_remaining := 0; + reverse_index_outcome := Some Eq |} + | None, S index => + match Nat.compare + (uleb_payload (nth index left 0)) + (uleb_payload (nth index right 0)) with + | Eq => + {| reverse_index_remaining := index; + reverse_index_outcome := None |} + | outcome => + {| reverse_index_remaining := index; + reverse_index_outcome := Some outcome |} + end + end. + +Theorem VWENC_96_REVERSE_INDEX_MACHINE_PENDING_STEP_STRICTLY_DESCENDS : + forall left right remaining next, + reverse_index_machine_step left right + {| reverse_index_remaining := S remaining; + reverse_index_outcome := None |} = next -> + reverse_index_remaining next = remaining. +Proof. + intros left right remaining next Hstep. + unfold reverse_index_machine_step in Hstep. simpl in Hstep. + destruct + (Nat.compare + (uleb_payload (nth remaining left 0)) + (uleb_payload (nth remaining right 0))); + inversion Hstep; reflexivity. +Qed. + +Lemma compare_equal_width_uleb_bytes_agrees_with_value : + forall left right, + length left = length right -> + compare_equal_width_uleb_bytes left right = + Nat.compare + (uleb_value (decode_uleb_payloads left)) + (uleb_value (decode_uleb_payloads right)). +Proof. + induction left as [| left_byte left_rest IH]; + destruct right as [| right_byte right_rest]; + intros Hlength; try discriminate; [reflexivity |]. + simpl in Hlength. injection Hlength as Hrest_length. + change + (match compare_equal_width_uleb_bytes left_rest right_rest with + | Eq => Nat.compare + (uleb_payload left_byte) (uleb_payload right_byte) + | Lt => Lt + | Gt => Gt + end = + Nat.compare + (uleb_payload left_byte + + 128 * uleb_value (decode_uleb_payloads left_rest)) + (uleb_payload right_byte + + 128 * uleb_value (decode_uleb_payloads right_rest))). + rewrite (IH right_rest Hrest_length). + pose proof (Nat.mod_upper_bound left_byte 128 ltac:(lia)) as Hleft_digit. + pose proof (Nat.mod_upper_bound right_byte 128 ltac:(lia)) as Hright_digit. + unfold uleb_payload in *. + destruct + (Nat.compare + (uleb_value (decode_uleb_payloads left_rest)) + (uleb_value (decode_uleb_payloads right_rest))) + eqn:Htail. + - apply Nat.compare_eq_iff in Htail. + destruct + (Nat.compare (left_byte mod 128) (right_byte mod 128)) + eqn:Hlow. + + apply Nat.compare_eq_iff in Hlow. + symmetry. apply Nat.compare_eq_iff. lia. + + apply Nat.compare_lt_iff in Hlow. + symmetry. apply Nat.compare_lt_iff. lia. + + apply Nat.compare_gt_iff in Hlow. + symmetry. apply Nat.compare_gt_iff. lia. + - apply Nat.compare_lt_iff in Htail. + symmetry. apply Nat.compare_lt_iff. nia. + - apply Nat.compare_gt_iff in Htail. + symmetry. apply Nat.compare_gt_iff. nia. +Qed. + +Lemma uleb_value_below_width : + forall digits, + Forall valid_uleb_digit digits -> + uleb_value digits < 128 ^ length digits. +Proof. + induction digits as [| digit rest IH]; intros Hvalid. + - simpl. lia. + - inversion Hvalid as [| ? ? Hdigit Hrest]; subst. + specialize (IH Hrest). + unfold valid_uleb_digit in Hdigit. + simpl. nia. +Qed. + +Lemma uleb_value_reaches_highest_place : + forall digits, + digits <> [] -> + Forall valid_uleb_digit digits -> + last digits 0 <> 0 -> + 128 ^ (length digits - 1) <= uleb_value digits. +Proof. + induction digits as [| digit rest IH]; + intros Hnonempty Hvalid Hhighest; [contradiction |]. + inversion Hvalid as [| ? ? Hdigit Hrest]; subst. + destruct rest as [| next tail]. + - simpl in *. unfold valid_uleb_digit in Hdigit. lia. + - specialize + (IH ltac:(discriminate) Hrest ltac:(simpl in Hhighest; exact Hhighest)). + cbn [length] in IH. + replace (S (length tail) - 1) with (length tail) in IH by lia. + change (128 ^ length tail <= uleb_value (next :: tail)) in IH. + change + (128 * 128 ^ length tail <= + digit + 128 * uleb_value (next :: tail)). + nia. +Qed. + +Lemma radix_128_power_monotone : + forall lower upper, + lower <= upper -> 128 ^ lower <= 128 ^ upper. +Proof. + intros lower upper Hle. revert lower Hle. + induction upper as [| upper IH]; intros lower Hle. + - assert (lower = 0) by lia. subst. reflexivity. + - destruct (Nat.eq_dec lower (S upper)) as [-> | Hneq]; + [reflexivity |]. + assert (lower <= upper) as Hlower by lia. + specialize (IH lower Hlower). + change (128 ^ lower <= 128 * 128 ^ upper). + eapply Nat.le_trans; [exact IH |]. + set (power := 128 ^ upper). + change (power <= 128 * power). + lia. +Qed. + +Lemma canonical_uleb_shorter_width_has_smaller_value : + forall left right, + canonical_uleb_digits left -> + canonical_uleb_digits right -> + length left < length right -> + uleb_value left < uleb_value right. +Proof. + intros left right + [Hleft_nonempty [Hleft_valid Hleft_minimal]] + [Hright_nonempty [Hright_valid Hright_minimal]] + Hwidth. + pose proof (uleb_value_below_width left Hleft_valid) as Hleft_upper. + assert (1 <= length left) as Hleft_positive. + { destruct left; [contradiction | simpl; lia]. } + assert (2 <= length right) as Hright_multiple by lia. + pose proof (Hright_minimal Hright_multiple) as Hright_highest. + pose proof + (uleb_value_reaches_highest_place + right Hright_nonempty Hright_valid Hright_highest) + as Hright_lower. + pose proof + (radix_128_power_monotone + (length left) (length right - 1) ltac:(lia)) + as Hpowers. + lia. +Qed. + +Theorem VWENC_10_ULEB_ORDER_IS_LOGICAL_NUMERIC_ORDER : + forall left right, + canonical_uleb_codeword left -> + canonical_uleb_codeword right -> + compare_uleb_codewords left right = + Nat.compare + (uleb_value (decode_uleb_payloads left)) + (uleb_value (decode_uleb_payloads right)). +Proof. + intros left right [Hleft_shape Hleft_digits] + [Hright_shape Hright_digits]. + unfold compare_uleb_codewords. + destruct (Nat.compare (length left) (length right)) eqn:Hwidth. + - apply Nat.compare_eq_iff in Hwidth. + rewrite compare_reverse_index_agrees_with_structural by exact Hwidth. + now apply compare_equal_width_uleb_bytes_agrees_with_value. + - apply Nat.compare_lt_iff in Hwidth. + symmetry. apply Nat.compare_lt_iff. + apply canonical_uleb_shorter_width_has_smaller_value. + + exact Hleft_digits. + + exact Hright_digits. + + unfold decode_uleb_payloads. now rewrite !map_length. + - apply Nat.compare_gt_iff in Hwidth. + symmetry. apply Nat.compare_gt_iff. + apply canonical_uleb_shorter_width_has_smaller_value. + + exact Hright_digits. + + exact Hleft_digits. + + unfold decode_uleb_payloads. now rewrite !map_length. +Qed. + +Lemma compare_equal_width_uleb_bytes_eq_payloads : + forall left right, + length left = length right -> + compare_equal_width_uleb_bytes left right = Eq -> + decode_uleb_payloads left = decode_uleb_payloads right. +Proof. + induction left as [| left_byte left_rest IH]; + destruct right as [| right_byte right_rest]; + intros Hlength Hcompare; try discriminate; [reflexivity |]. + simpl in Hlength. injection Hlength as Hrest_length. + simpl in Hcompare. + destruct + (compare_equal_width_uleb_bytes left_rest right_rest) + eqn:Hrest_compare; try discriminate. + apply Nat.compare_eq_iff in Hcompare. + unfold decode_uleb_payloads. simpl. + f_equal. + - exact Hcompare. + - apply IH; assumption. +Qed. + +Theorem VWENC_57_ULEB_COMPARATOR_EQUAL_IFF_CANONICAL_BYTES_EQUAL : + forall left right, + canonical_uleb_codeword left -> + canonical_uleb_codeword right -> + (compare_uleb_codewords left right = Eq <-> left = right). +Proof. + intros left right [Hleft_shape Hleft_digits] + [Hright_shape Hright_digits]. + split. + - intros Hcompare. + unfold compare_uleb_codewords in Hcompare. + destruct (Nat.compare (length left) (length right)) + eqn:Hlength; try discriminate. + apply Nat.compare_eq_iff in Hlength. + rewrite compare_reverse_index_agrees_with_structural in Hcompare + by exact Hlength. + apply VWENC_04_ULEB_UNIQUE_DECODING; [exact Hleft_shape | exact Hright_shape |]. + now apply compare_equal_width_uleb_bytes_eq_payloads. + - intros ->. + unfold compare_uleb_codewords. + rewrite Nat.compare_refl. + rewrite compare_reverse_index_agrees_with_structural by reflexivity. + rewrite compare_equal_width_uleb_bytes_agrees_with_value by reflexivity. + apply Nat.compare_refl. +Qed. + +Theorem VWENC_58_ULEB_CANONICAL_SEMANTIC_VALUE_IS_INJECTIVE : + forall left right, + canonical_uleb_codeword left -> + canonical_uleb_codeword right -> + uleb_value (decode_uleb_payloads left) = + uleb_value (decode_uleb_payloads right) -> + left = right. +Proof. + intros left right Hleft Hright Hvalue. + apply (proj1 + (VWENC_57_ULEB_COMPARATOR_EQUAL_IFF_CANONICAL_BYTES_EQUAL + left right Hleft Hright)). + rewrite (VWENC_10_ULEB_ORDER_IS_LOGICAL_NUMERIC_ORDER + left right Hleft Hright), Hvalue. + apply Nat.compare_refl. +Qed. + +Definition uleb_byte_identity + (bytes : list PhysicalByte) : list PhysicalByte := bytes. + +Definition uleb_hash_material + (bytes : list PhysicalByte) : list PhysicalByte := bytes. + +Definition uleb_biguint_view (bytes : list PhysicalByte) : nat := + uleb_value (decode_uleb_payloads bytes). + +Definition decode_uleb_bounded + (exclusive_bound : nat) (bytes : list PhysicalByte) : option nat := + match decode_canonical_uleb bytes with + | None => None + | Some digits => + let value := uleb_value digits in + if value + canonical_uleb_codeword right -> + byte_sequence_eqb left right = true <-> + uleb_byte_identity left = uleb_byte_identity right. +Proof. + intros left right Hleft Hright. + unfold uleb_byte_identity. + apply byte_sequence_eqb_reflects_equality. +Qed. + +(** [uleb_hash_material] is collision-free input material, not the output of a + finite hash function. Actual hash outputs may collide; consumers rely on + equality checks after hash-table bucket selection. *) +Theorem VWENC_38_ULEB_HASH_MATERIAL_IS_INJECTIVE : + forall left right, + canonical_uleb_codeword left -> + canonical_uleb_codeword right -> + uleb_hash_material left = uleb_hash_material right -> left = right. +Proof. intros left right Hleft Hright Hequal. exact Hequal. Qed. + +Theorem VWENC_90_FINITE_HASH_OUTPUT_REQUIRES_ONLY_EQUALITY_CONGRUENCE : + forall (finite_hash : list PhysicalByte -> nat) left right, + left = right -> finite_hash left = finite_hash right. +Proof. intros finite_hash left right ->. reflexivity. Qed. + +Theorem VWENC_39_ULEB_BIGUINT_VIEW_AGREES_WITH_NUMERIC_ORDER : + forall left right, + canonical_uleb_codeword left -> + canonical_uleb_codeword right -> + compare_uleb_codewords left right = + Nat.compare (uleb_biguint_view left) (uleb_biguint_view right). +Proof. + intros left right Hleft Hright. + unfold uleb_biguint_view. + now apply VWENC_10_ULEB_ORDER_IS_LOGICAL_NUMERIC_ORDER. +Qed. + +Theorem VWENC_40_ULEB_BOUNDED_ADAPTER_AGREES_WHEN_REPRESENTABLE : + forall exclusive_bound bytes, + canonical_uleb_codeword bytes -> + uleb_biguint_view bytes < exclusive_bound -> + decode_uleb_bounded exclusive_bound bytes = + Some (uleb_biguint_view bytes). +Proof. + intros exclusive_bound bytes Hcanonical Hbounded. + unfold decode_uleb_bounded, uleb_biguint_view. + apply VWENC_34_ULEB_DECODER_ACCEPTS_EXACTLY_CANONICAL_CODEWORDS + in Hcanonical. + rewrite Hcanonical. + unfold uleb_biguint_view in Hbounded. + apply Nat.ltb_lt in Hbounded. + now rewrite Hbounded. +Qed. + +Theorem VWENC_41_ULEB_BOUNDED_ADAPTER_REJECTS_REPRESENTATION_OVERFLOW : + forall exclusive_bound bytes, + canonical_uleb_codeword bytes -> + exclusive_bound <= uleb_biguint_view bytes -> + decode_uleb_bounded exclusive_bound bytes = None. +Proof. + intros exclusive_bound bytes Hcanonical Hoverflow. + unfold decode_uleb_bounded, uleb_biguint_view. + apply VWENC_34_ULEB_DECODER_ACCEPTS_EXACTLY_CANONICAL_CODEWORDS + in Hcanonical. + rewrite Hcanonical. + unfold uleb_biguint_view in Hoverflow. + destruct + (uleb_value (decode_uleb_payloads bytes) Some first + | [first; second] => + Some ((first - 192) * 64 + (second - 128)) + | [first; second; third] => + Some + (((first - 224) * 64 + (second - 128)) * 64 + + (third - 128)) + | [first; second; third; fourth] => + Some + (((((first - 240) * 64 + (second - 128)) * 64 + + (third - 128)) * 64) + + (fourth - 128)) + | _ => None + end. + +Definition decode_utf8_scalar + (bytes : list PhysicalByte) : option nat := + match decode_utf8_value bytes with + | None => None + | Some codepoint => + if unicode_scalarb codepoint && + byte_sequence_eqb bytes (encode_utf8_scalar codepoint) + then Some codepoint + else None + end. + +Lemma radix64_reconstruct : + forall value, + (value / 64) * 64 + value mod 64 = value. +Proof. + intros value. + pose proof (Nat.div_mod value 64) as Hdivision. + specialize (Hdivision ltac:(lia)). nia. +Qed. + +Lemma decode_utf8_value_encode_roundtrip : + forall codepoint, + decode_utf8_value (encode_utf8_scalar codepoint) = Some codepoint. +Proof. + intros codepoint. + unfold encode_utf8_scalar, decode_utf8_value. + destruct (codepoint unicode_scalar codepoint. +Proof. + intros codepoint. + unfold unicode_scalarb, unicode_scalar. + rewrite andb_true_iff, orb_true_iff. + rewrite !Nat.leb_le, !Nat.ltb_lt. + tauto. +Qed. + +Theorem VWENC_12_UTF8_CODEWORDS_NONEMPTY_AND_AT_MOST_FOUR_BYTES : + forall codepoint, + unicode_scalar codepoint -> + encode_utf8_scalar codepoint <> [] /\ + 1 <= length (encode_utf8_scalar codepoint) <= 4. +Proof. + intros codepoint Hscalar. + unfold encode_utf8_scalar. + destruct (codepoint + length (encode_utf8_scalar codepoint) = utf8_width codepoint. +Proof. + intros codepoint Hscalar. + unfold encode_utf8_scalar, utf8_width. + destruct (codepoint + forall bytes, ~ canonical_utf8_codeword codepoint bytes. +Proof. + intros codepoint Hinvalid bytes [Hscalar _]. contradiction. +Qed. + +Theorem VWENC_42_UTF8_CANONICAL_DECODE_ROUNDTRIP : + forall codepoint, + unicode_scalar codepoint -> + decode_utf8_scalar (encode_utf8_scalar codepoint) = Some codepoint. +Proof. + intros codepoint Hscalar. + unfold decode_utf8_scalar. + rewrite decode_utf8_value_encode_roundtrip. + assert (Hscalarb : unicode_scalarb codepoint = true). + { apply VWENC_11_UTF8_SCALAR_BOOLEAN_REFLECTION. exact Hscalar. } + assert + (Hequal : + byte_sequence_eqb + (encode_utf8_scalar codepoint) + (encode_utf8_scalar codepoint) = true). + { apply byte_sequence_eqb_reflects_equality. reflexivity. } + now rewrite Hscalarb, Hequal. +Qed. + +Theorem VWENC_43_UTF8_DECODER_ACCEPTANCE_IS_CANONICAL : + forall bytes codepoint, + decode_utf8_scalar bytes = Some codepoint -> + canonical_utf8_codeword codepoint bytes. +Proof. + intros bytes codepoint Hdecode. + unfold decode_utf8_scalar in Hdecode. + destruct (decode_utf8_value bytes) as [candidate |] eqn:Hcandidate; + [| discriminate]. + destruct + (unicode_scalarb candidate && + byte_sequence_eqb bytes (encode_utf8_scalar candidate)) + eqn:Haccepted; [| discriminate]. + inversion Hdecode; subst candidate. + apply andb_true_iff in Haccepted. + destruct Haccepted as [Hscalar Hequal]. + apply VWENC_11_UTF8_SCALAR_BOOLEAN_REFLECTION in Hscalar. + apply byte_sequence_eqb_reflects_equality in Hequal. + split; assumption. +Qed. + +Theorem VWENC_44_UTF8_DECODER_ACCEPTS_CANONICAL_CODEWORDS : + forall bytes codepoint, + canonical_utf8_codeword codepoint bytes -> + decode_utf8_scalar bytes = Some codepoint. +Proof. + intros bytes codepoint [Hscalar ->]. + apply VWENC_42_UTF8_CANONICAL_DECODE_ROUNDTRIP. + exact Hscalar. +Qed. + +Theorem VWENC_45_UTF8_CANONICAL_ENCODING_IS_INJECTIVE : + forall left right, + unicode_scalar left -> + unicode_scalar right -> + encode_utf8_scalar left = encode_utf8_scalar right -> + left = right. +Proof. + intros left right Hleft Hright Hencoded. + pose proof (VWENC_42_UTF8_CANONICAL_DECODE_ROUNDTRIP left Hleft) + as Hdecode_left. + pose proof (VWENC_42_UTF8_CANONICAL_DECODE_ROUNDTRIP right Hright) + as Hdecode_right. + rewrite Hencoded in Hdecode_left. + rewrite Hdecode_right in Hdecode_left. + inversion Hdecode_left. reflexivity. +Qed. + +Theorem VWENC_46_UTF8_MALFORMED_OR_NONCANONICAL_INPUT_IS_REJECTED : + forall bytes, + (forall codepoint, ~ canonical_utf8_codeword codepoint bytes) -> + decode_utf8_scalar bytes = None. +Proof. + intros bytes Hnoncanonical. + destruct (decode_utf8_scalar bytes) as [codepoint |] eqn:Hdecode. + - exfalso. apply (Hnoncanonical codepoint). + now apply VWENC_43_UTF8_DECODER_ACCEPTANCE_IS_CANONICAL. + - reflexivity. +Qed. + +Theorem VWENC_47_UTF8_REJECTS_CONTINUATION_OVERLONG_TRUNCATED_AND_SURROGATE : + decode_utf8_scalar [169] = None /\ + decode_utf8_scalar [192; 128] = None /\ + decode_utf8_scalar [195] = None /\ + decode_utf8_scalar [237; 160; 128] = None. +Proof. repeat split; reflexivity. Qed. + +(** ** Direct fixed-unit and logical-observation laws *) + +(** Keep machine-width bounds symbolic. Expanding 64-bit decimal literals into + Peano naturals is both semantically unnecessary and prohibitively expensive + for the proof checker. These factored definitions preserve the exact + values while proofs reason about their algebraic relationships. *) +Definition two_to_32 : nat := 256 ^ 4. +Definition two_to_63 : nat := 128 * 256 ^ 7. +Definition two_to_64 : nat := 256 ^ 8. + +Lemma two_to_63_positive : 0 < two_to_63. +Proof. + unfold two_to_63. + assert (256 ^ 7 <> 0). + { apply Nat.pow_nonzero. lia. } + nia. +Qed. + +Lemma two_to_64_is_double_two_to_63 : + two_to_64 = 2 * two_to_63. +Proof. + unfold two_to_64, two_to_63. + replace 8 with (S 7) by reflexivity. + rewrite Nat.pow_succ_r by lia. + set (power := 256 ^ 7). + change (256 * power = 2 * (128 * power)). + lia. +Qed. + +(** Subsequent proofs use the checked positivity/doubling interface above. + Keeping the factored Peano definitions opaque prevents the kernel from + expanding machine-width bounds while closing unrelated theorems. *) +Global Opaque two_to_32 two_to_63 two_to_64. + +Inductive DirectProfile := +| DirectBytes +| DirectUnicodeScalar +| DirectU32 +| DirectU64 +| DirectF64Bits. + +Definition direct_profile_tag (profile : DirectProfile) : nat := + match profile with + | DirectBytes => 1 + | DirectUnicodeScalar => 2 + | DirectU32 => 3 + | DirectU64 => 4 + | DirectF64Bits => 5 + end. + +Definition direct_byte_width (profile : DirectProfile) : nat := + match profile with + | DirectBytes => 1 + | DirectUnicodeScalar => 4 + | DirectU32 => 4 + | DirectU64 => 8 + | DirectF64Bits => 8 + end. + +Definition direct_profile_valid + (profile : DirectProfile) (unit : nat) : Prop := + match profile with + | DirectBytes => unit < 256 + | DirectUnicodeScalar => unicode_scalar unit + | DirectU32 => unit < 256 ^ 4 + | DirectU64 => unit < 256 ^ 8 + | DirectF64Bits => unit < 256 ^ 8 + end. + +Definition direct_profile_validb + (profile : DirectProfile) (unit : nat) : bool := + match profile with + | DirectBytes => unit unicode_scalarb unit + | DirectU32 => unit unit unit + direct_profile_valid profile unit. +Proof. + intros profile unit. + destruct profile; + cbn [direct_profile_validb direct_profile_valid]. + - apply Nat.ltb_lt. + - apply VWENC_11_UTF8_SCALAR_BOOLEAN_REFLECTION. + - apply Nat.ltb_lt. + - apply Nat.ltb_lt. + - apply Nat.ltb_lt. +Qed. + +Fixpoint encode_fixed_little_endian + (byte_count value : nat) : list PhysicalByte := + match byte_count with + | 0 => [] + | S rest => value mod 256 :: + encode_fixed_little_endian rest (value / 256) + end. + +Fixpoint decode_fixed_little_endian + (bytes : list PhysicalByte) : nat := + match bytes with + | [] => 0 + | byte :: rest => byte + decode_fixed_little_endian rest * 256 + end. + +Definition serialize_direct_unit + (profile : DirectProfile) (unit : nat) + : nat * list PhysicalByte := + (direct_profile_tag profile, + encode_fixed_little_endian (direct_byte_width profile) unit). + +Fixpoint all_valid_bytesb (bytes : list PhysicalByte) : bool := + match bytes with + | [] => true + | byte :: rest => (byte Forall valid_byte bytes. +Proof. + induction bytes as [| byte rest IH]; simpl. + - split; constructor. + - rewrite andb_true_iff, Nat.ltb_lt, IH. + unfold valid_byte. + split. + + intros [Hbyte Hrest]. constructor; assumption. + + intros Hvalid. inversion Hvalid; subst. tauto. +Qed. + +(** This checked record is the prospective canonical direct-profile codec. + It is not a claim about the byte layout of any existing serde/bincode or + persistent-ARTrie image. Migration of a persistent backend may select + this record only under a new, explicit format identity. *) +Definition decode_direct_unit + (expected_profile : DirectProfile) + (serialized : nat * list PhysicalByte) : option nat := + let '(profile_tag, bytes) := serialized in + if profile_tag =? direct_profile_tag expected_profile then + if length bytes =? direct_byte_width expected_profile then + if all_valid_bytesb bytes then + let unit := decode_fixed_little_endian bytes in + if direct_profile_validb expected_profile unit + then Some unit + else None + else None + else None + else None. + +Definition direct_codeword (unit : nat) : list nat := [unit]. + +Lemma fixed_little_endian_length : + forall byte_count value, + length (encode_fixed_little_endian byte_count value) = byte_count. +Proof. + induction byte_count; intros value; simpl; [reflexivity |]. + now rewrite IHbyte_count. +Qed. + +Lemma fixed_little_endian_bytes_are_valid : + forall byte_count value, + Forall valid_byte (encode_fixed_little_endian byte_count value). +Proof. + induction byte_count as [| byte_count IH]; intros value. + - change (Forall valid_byte []). constructor. + - change + (Forall valid_byte + (value mod 256 :: + encode_fixed_little_endian byte_count (value / 256))). + constructor. + + unfold valid_byte. apply Nat.mod_upper_bound. lia. + + apply IH. +Qed. + +Lemma fixed_little_endian_roundtrip : + forall byte_count value, + value < 256 ^ byte_count -> + decode_fixed_little_endian + (encode_fixed_little_endian byte_count value) = value. +Proof. + induction byte_count as [| byte_count IH]; intros value Hbounded. + - simpl in *. lia. + - cbn [encode_fixed_little_endian decode_fixed_little_endian]. + rewrite IH. + + pose proof (Nat.div_mod value 256 ltac:(lia)) as Hdivision. + nia. + + pose proof (Nat.div_mod value 256 ltac:(lia)) as Hdivision. + pose proof (Nat.mod_upper_bound value 256 ltac:(lia)) as Hremainder. + change (value < 256 * 256 ^ byte_count) in Hbounded. + nia. +Qed. + +Lemma direct_profile_value_fits_serialization : + forall profile unit, + direct_profile_valid profile unit -> + unit < 256 ^ direct_byte_width profile. +Proof. + intros profile unit Hvalid. + destruct profile; cbn [direct_profile_valid direct_byte_width] in Hvalid |- *. + - exact Hvalid. + - destruct Hvalid as [Hupper _]. + unfold unicode_limit, utf8_three_byte_limit in Hupper. + assert (Hbase : 17 < 256 ^ 2). + { change (17 < 256 * (256 * 1)). rewrite Nat.mul_1_r. lia. } + replace 4 with (2 + 2) by lia. + rewrite Nat.pow_add_r. + assert (0 < 256 ^ 2). + { apply Nat.neq_0_lt_0. apply Nat.pow_nonzero. lia. } + assert + (Hunicode_fits : + 17 * 256 ^ 2 < 256 ^ 2 * 256 ^ 2) by nia. + eapply Nat.lt_trans; [exact Hupper | exact Hunicode_fits]. + - exact Hvalid. + - exact Hvalid. + - exact Hvalid. +Qed. + +Theorem VWENC_48_DIRECT_PROFILE_TAGS_ARE_INJECTIVE : + forall left right, + direct_profile_tag left = direct_profile_tag right -> left = right. +Proof. + intros left right Hequal. + destruct left, right; simpl in Hequal; try reflexivity; discriminate. +Qed. + +Theorem VWENC_49_DIRECT_SERIALIZATION_HAS_EXACT_FIXED_WIDTH : + forall profile unit, + length (snd (serialize_direct_unit profile unit)) = + direct_byte_width profile. +Proof. + intros profile unit. + unfold serialize_direct_unit. simpl. + apply fixed_little_endian_length. +Qed. + +Theorem VWENC_50_DIRECT_SERIALIZATION_ROUNDTRIPS_VALID_UNITS : + forall profile unit, + direct_profile_valid profile unit -> + decode_fixed_little_endian + (snd (serialize_direct_unit profile unit)) = unit. +Proof. + intros profile unit Hvalid. + unfold serialize_direct_unit. simpl. + apply fixed_little_endian_roundtrip. + now apply direct_profile_value_fits_serialization. +Qed. + +Theorem VWENC_59_CHECKED_DIRECT_DECODER_ACCEPTS_CANONICAL_RECORD : + forall profile unit, + direct_profile_valid profile unit -> + decode_direct_unit profile (serialize_direct_unit profile unit) = + Some unit. +Proof. + intros profile unit Hvalid. + unfold decode_direct_unit, serialize_direct_unit. + rewrite Nat.eqb_refl, fixed_little_endian_length, Nat.eqb_refl. + pose proof + (fixed_little_endian_bytes_are_valid + (direct_byte_width profile) unit) as Hbytes. + apply all_valid_bytesb_reflects_validity in Hbytes. + rewrite Hbytes. + rewrite fixed_little_endian_roundtrip. + - apply direct_profile_validb_reflects_validity in Hvalid. + now rewrite Hvalid. + - now apply direct_profile_value_fits_serialization. +Qed. + +Theorem VWENC_60_CHECKED_DIRECT_DECODER_REJECTS_WRONG_PROFILE_TAG : + forall expected_profile supplied_tag bytes, + supplied_tag <> direct_profile_tag expected_profile -> + decode_direct_unit expected_profile (supplied_tag, bytes) = None. +Proof. + intros expected_profile supplied_tag bytes Hwrong. + unfold decode_direct_unit. + apply Nat.eqb_neq in Hwrong. now rewrite Hwrong. +Qed. + +Theorem VWENC_61_CHECKED_DIRECT_DECODER_REJECTS_WRONG_WIDTH : + forall profile bytes, + length bytes <> direct_byte_width profile -> + decode_direct_unit profile (direct_profile_tag profile, bytes) = None. +Proof. + intros profile bytes Hwrong. + unfold decode_direct_unit. rewrite Nat.eqb_refl. + apply Nat.eqb_neq in Hwrong. now rewrite Hwrong. +Qed. + +Theorem VWENC_62_CHECKED_DIRECT_DECODER_REJECTS_NONBYTE_PAYLOAD : + forall profile bytes, + length bytes = direct_byte_width profile -> + ~ Forall valid_byte bytes -> + decode_direct_unit profile (direct_profile_tag profile, bytes) = None. +Proof. + intros profile bytes Hwidth Hinvalid. + unfold decode_direct_unit. rewrite Nat.eqb_refl. + apply Nat.eqb_eq in Hwidth. rewrite Hwidth. + destruct (all_valid_bytesb bytes) eqn:Hbytes; [| reflexivity]. + apply all_valid_bytesb_reflects_validity in Hbytes. contradiction. +Qed. + +Theorem VWENC_63_CHECKED_DIRECT_DECODER_SUCCESS_IS_EXACT : + forall profile supplied_tag bytes unit, + decode_direct_unit profile (supplied_tag, bytes) = Some unit -> + supplied_tag = direct_profile_tag profile /\ + length bytes = direct_byte_width profile /\ + Forall valid_byte bytes /\ + direct_profile_valid profile unit /\ + decode_fixed_little_endian bytes = unit. +Proof. + intros profile supplied_tag bytes unit Hdecode. + unfold decode_direct_unit in Hdecode. + destruct (supplied_tag =? direct_profile_tag profile) + eqn:Htag; [| discriminate]. + destruct (length bytes =? direct_byte_width profile) + eqn:Hwidth; [| discriminate]. + destruct (all_valid_bytesb bytes) eqn:Hbytes; [| discriminate]. + destruct + (direct_profile_validb profile (decode_fixed_little_endian bytes)) + eqn:Hvalid; [| discriminate]. + inversion Hdecode; subst. + repeat split. + - now apply Nat.eqb_eq. + - now apply Nat.eqb_eq. + - now apply all_valid_bytesb_reflects_validity. + - now apply direct_profile_validb_reflects_validity. +Qed. + +Theorem VWENC_64_CHECKED_DIRECT_DECODER_REJECTS_INVALID_LOGICAL_UNIT : + forall profile bytes, + length bytes = direct_byte_width profile -> + Forall valid_byte bytes -> + ~ direct_profile_valid profile (decode_fixed_little_endian bytes) -> + decode_direct_unit profile (direct_profile_tag profile, bytes) = None. +Proof. + intros profile bytes Hwidth Hbytes Hinvalid. + unfold decode_direct_unit. rewrite Nat.eqb_refl. + apply Nat.eqb_eq in Hwidth. rewrite Hwidth. + apply all_valid_bytesb_reflects_validity in Hbytes. rewrite Hbytes. + destruct + (direct_profile_validb profile (decode_fixed_little_endian bytes)) + eqn:Hvalid; [| reflexivity]. + apply direct_profile_validb_reflects_validity in Hvalid. + contradiction. +Qed. + +Theorem VWENC_51_UNICODE_SCALAR_DIRECT_STORAGE_IS_NOT_UTF8_STORAGE : + forall codepoint, + unicode_scalar codepoint -> + direct_codeword codepoint = [codepoint] /\ + decode_utf8_scalar (encode_utf8_scalar codepoint) = Some codepoint. +Proof. + intros codepoint Hscalar. split; [reflexivity |]. + now apply VWENC_42_UTF8_CANONICAL_DECODE_ROUNDTRIP. +Qed. + +Theorem VWENC_15_DIRECT_PROFILE_IS_ONE_UNIT_PER_TRANSITION : + forall unit, length (direct_codeword unit) = 1. +Proof. reflexivity. Qed. + +(** A variable-width ULEB atom retains its canonical bytes as its identity. + No built-in integer is required at the consumer boundary. UTF-8 instead + denotes a Unicode scalar, so its public logical identity is the decoded + scalar value. Direct atoms are already one native edge unit. *) +Inductive LogicalAtom := +| DirectAtom : DirectProfile -> nat -> LogicalAtom +| UlebAtom : list PhysicalByte -> LogicalAtom +| UnicodeAtom : nat -> LogicalAtom. + +Definition direct_logical_atom + (profile : DirectProfile) (unit : nat) : LogicalAtom := + match profile with + | DirectUnicodeScalar => UnicodeAtom unit + | _ => DirectAtom profile unit + end. + +(** Native direct labels match the existing [CharUnit]-generic DAWG cores. + An opaque codeword is one edge label carrying canonical bytes. A byte-path + adapter may use several third-party physical edges internally, but it has + the same logical projection and must not expose its intermediate nodes to + [DictionaryNode], zipper, or cursor consumers. *) +Inductive StorageRepresentation := +| NativeDirectEdge +| OpaqueCodewordEdge +| EncodedBytePathAdapter. + +Inductive StoredLogicalUnit := +| StoredDirect : DirectProfile -> nat -> StoredLogicalUnit +| StoredUleb : list PhysicalByte -> StoredLogicalUnit +| StoredUtf8 : list PhysicalByte -> StoredLogicalUnit. + +Definition representation_admits + (representation : StorageRepresentation) + (stored : StoredLogicalUnit) : Prop := + match representation, stored with + | NativeDirectEdge, StoredDirect _ _ => True + | OpaqueCodewordEdge, StoredUleb _ => True + | OpaqueCodewordEdge, StoredUtf8 _ => True + | EncodedBytePathAdapter, StoredUleb _ => True + | EncodedBytePathAdapter, StoredUtf8 _ => True + | _, _ => False + end. + +Definition representation_admitsb + (representation : StorageRepresentation) + (stored : StoredLogicalUnit) : bool := + match representation, stored with + | NativeDirectEdge, StoredDirect _ _ => true + | OpaqueCodewordEdge, StoredUleb _ => true + | OpaqueCodewordEdge, StoredUtf8 _ => true + | EncodedBytePathAdapter, StoredUleb _ => true + | EncodedBytePathAdapter, StoredUtf8 _ => true + | _, _ => false + end. + +Lemma representation_admitsb_reflects_admission : + forall representation stored, + representation_admitsb representation stored = true <-> + representation_admits representation stored. +Proof. + intros representation stored. + destruct representation, stored; + cbn [representation_admitsb representation_admits]; easy. +Qed. + +Definition decode_stored_logical_unit + (stored : StoredLogicalUnit) : option LogicalAtom := + match stored with + | StoredDirect profile unit => + if direct_profile_validb profile unit + then Some (direct_logical_atom profile unit) + else None + | StoredUleb bytes => + match decode_canonical_uleb bytes with + | Some _ => Some (UlebAtom bytes) + | None => None + end + | StoredUtf8 bytes => + match decode_utf8_scalar bytes with + | Some codepoint => Some (UnicodeAtom codepoint) + | None => None + end + end. + +Definition physical_codeword_of + (stored : StoredLogicalUnit) : list PhysicalByte := + match stored with + | StoredDirect profile unit => snd (serialize_direct_unit profile unit) + | StoredUleb bytes => bytes + | StoredUtf8 bytes => bytes + end. + +Record StoredTransition := { + transition_representation : StorageRepresentation; + transition_unit : StoredLogicalUnit; +}. + +Definition valid_stored_transition + (transition : StoredTransition) : Prop := + representation_admits + transition.(transition_representation) transition.(transition_unit) /\ + exists atom, + decode_stored_logical_unit transition.(transition_unit) = Some atom. + +Definition logical_transition + (transition : StoredTransition) : option LogicalAtom := + if representation_admitsb + transition.(transition_representation) transition.(transition_unit) + then decode_stored_logical_unit transition.(transition_unit) + else None. + +Inductive ConsumerSurface := +| DictionaryNodeSurface +| ZipperSurface +| SnapshotCursorSurface. + +Definition consumer_observation + (_surface : ConsumerSurface) + (transition : StoredTransition) : list LogicalAtom := + match logical_transition transition with + | Some atom => [atom] + | None => [] + end. + +(** A concrete API surface discharges this refinement obligation in the + family-wide proof phase. The common target below deliberately abstracts + over how a node, zipper, or cursor obtains the transition. *) +Record ConsumerSurfaceImplementation := { + implementation_surface : ConsumerSurface; + implementation_observation : StoredTransition -> list LogicalAtom; + implementation_refines_logical_target : + forall transition, + implementation_observation transition = + consumer_observation implementation_surface transition; +}. + +Theorem VWENC_16_CODEC_BYTES_ARE_NOT_LOGICAL_TRANSITIONS : + forall surface transition atom, + logical_transition transition = Some atom -> + consumer_observation surface transition = [atom]. +Proof. + intros surface transition atom Hlogical. + unfold consumer_observation. now rewrite Hlogical. +Qed. + +Theorem VWENC_17_ONE_LOGICAL_ATOM_PER_CONSUMER_TRANSITION : + forall surface transition, + valid_stored_transition transition -> + length (consumer_observation surface transition) = 1. +Proof. + intros surface transition [Hadmitted [atom Hdecode]]. + unfold consumer_observation, logical_transition. + apply representation_admitsb_reflects_admission in Hadmitted. + now rewrite Hadmitted, Hdecode. +Qed. + +Theorem VWENC_65_ULEB_LOGICAL_IDENTITY_IS_CANONICAL_BYTES : + forall representation bytes, + representation_admits representation (StoredUleb bytes) -> + canonical_uleb_codeword bytes -> + logical_transition + {| transition_representation := representation; + transition_unit := StoredUleb bytes |} = + Some (UlebAtom bytes). +Proof. + intros representation bytes Hadmitted Hcanonical. + unfold logical_transition. simpl. + apply representation_admitsb_reflects_admission in Hadmitted. + rewrite Hadmitted. + apply VWENC_34_ULEB_DECODER_ACCEPTS_EXACTLY_CANONICAL_CODEWORDS + in Hcanonical. + now rewrite Hcanonical. +Qed. + +Theorem VWENC_66_UTF8_LOGICAL_IDENTITY_IS_UNICODE_SCALAR : + forall representation bytes codepoint, + representation_admits representation (StoredUtf8 bytes) -> + canonical_utf8_codeword codepoint bytes -> + logical_transition + {| transition_representation := representation; + transition_unit := StoredUtf8 bytes |} = + Some (UnicodeAtom codepoint). +Proof. + intros representation bytes codepoint Hadmitted Hcanonical. + unfold logical_transition. simpl. + apply representation_admitsb_reflects_admission in Hadmitted. + rewrite Hadmitted. + apply VWENC_44_UTF8_DECODER_ACCEPTS_CANONICAL_CODEWORDS in Hcanonical. + now rewrite Hcanonical. +Qed. + +Theorem VWENC_67_OPAQUE_AND_BYTE_PATH_ADAPTERS_HAVE_SAME_LOGICAL_VIEW : + forall stored, + representation_admits OpaqueCodewordEdge stored -> + representation_admits EncodedBytePathAdapter stored -> + logical_transition + {| transition_representation := OpaqueCodewordEdge; + transition_unit := stored |} = + logical_transition + {| transition_representation := EncodedBytePathAdapter; + transition_unit := stored |}. +Proof. + intros stored Hopaque Hadapter. + apply representation_admitsb_reflects_admission in Hopaque. + apply representation_admitsb_reflects_admission in Hadapter. + change + ((if representation_admitsb OpaqueCodewordEdge stored + then decode_stored_logical_unit stored else None) = + (if representation_admitsb EncodedBytePathAdapter stored + then decode_stored_logical_unit stored else None)). + now rewrite Hopaque, Hadapter. +Qed. + +Theorem VWENC_68_DICTIONARY_NODE_ZIPPER_CURSOR_SHARE_COMMON_TARGET_DEFINITION : + forall transition, + consumer_observation DictionaryNodeSurface transition = + consumer_observation ZipperSurface transition /\ + consumer_observation ZipperSurface transition = + consumer_observation SnapshotCursorSurface transition. +Proof. intros transition. split; reflexivity. Qed. + +Theorem VWENC_97_SURFACE_REFINEMENT_OBLIGATIONS_IMPLY_LOGICAL_AGREEMENT : + forall left right transition, + implementation_observation left transition = + implementation_observation right transition. +Proof. + intros [left_surface left_observe Hleft] + [right_surface right_observe Hright] transition. + simpl. + rewrite (Hleft transition), (Hright transition). + destruct left_surface, right_surface; reflexivity. +Qed. + +Theorem VWENC_69_MULTIBYTE_STORAGE_STILL_EMITS_ONE_LOGICAL_TRANSITION : + forall surface transition, + valid_stored_transition transition -> + 2 <= length (physical_codeword_of transition.(transition_unit)) -> + length (consumer_observation surface transition) = 1. +Proof. + intros surface transition Hvalid Hmultibyte. + now apply VWENC_17_ONE_LOGICAL_ATOM_PER_CONSUMER_TRANSITION. +Qed. + +(** Exact correspondence target for the baseline generic cores at revision + [6e8bb1d]: [CharUnit] supplies [u8], [char], and [u64] edge labels; + [DawgCore] and [LockFreeDawg] both store one [U] per edge. *) +Inductive BaselineCharUnitKind := +| BaselineU8 +| BaselineChar +| BaselineU64. + +Definition baseline_profile (kind : BaselineCharUnitKind) : DirectProfile := + match kind with + | BaselineU8 => DirectBytes + | BaselineChar => DirectUnicodeScalar + | BaselineU64 => DirectU64 + end. + +Inductive BaselineDawgCoreKind := +| IndexedDawgCore +| LockFreeDawgCore. + +Definition baseline_transition + (kind : BaselineCharUnitKind) (unit : nat) : StoredTransition := + {| transition_representation := NativeDirectEdge; + transition_unit := StoredDirect (baseline_profile kind) unit |}. + +Definition baseline_core_observations + (_core : BaselineDawgCoreKind) + (kind : BaselineCharUnitKind) + (units : list nat) : list (list LogicalAtom) := + map + (fun unit => + consumer_observation DictionaryNodeSurface + (baseline_transition kind unit)) + units. + +Theorem VWENC_70_BASELINE_CHARUNIT_EDGE_IS_ONE_LOGICAL_ATOM : + forall kind unit, + direct_profile_valid (baseline_profile kind) unit -> + logical_transition (baseline_transition kind unit) = + Some (direct_logical_atom (baseline_profile kind) unit). +Proof. + intros kind unit Hvalid. + unfold logical_transition, baseline_transition. simpl. + apply direct_profile_validb_reflects_validity in Hvalid. + now rewrite Hvalid. +Qed. + +Theorem VWENC_71_INDEXED_AND_LOCKFREE_SHARE_REQUIRED_TARGET_DEFINITION : + forall kind units, + baseline_core_observations IndexedDawgCore kind units = + baseline_core_observations LockFreeDawgCore kind units. +Proof. reflexivity. Qed. + +(** Existing persistent profiles are closed over the already implemented + [ByteKey], [CharKey], and [U64Key] units. Variable-width profiles and new + semantic interpretations require an explicit format/profile identity and + are not silently asserted to match an existing persistent image. *) +Inductive ExistingPersistentUnitKind := +| PersistentByteKey +| PersistentCharKey +| PersistentU64Key. + +Definition persistent_baseline_kind + (kind : ExistingPersistentUnitKind) : BaselineCharUnitKind := + match kind with + | PersistentByteKey => BaselineU8 + | PersistentCharKey => BaselineChar + | PersistentU64Key => BaselineU64 + end. + +Theorem VWENC_72_EXISTING_PERSISTENT_UNITS_MAP_TO_BASELINE_CHARUNITS : + forall kind unit, + direct_profile_valid + (baseline_profile (persistent_baseline_kind kind)) unit -> + logical_transition + (baseline_transition (persistent_baseline_kind kind) unit) = + Some + (direct_logical_atom + (baseline_profile (persistent_baseline_kind kind)) unit). +Proof. + intros kind unit Hvalid. + now apply VWENC_70_BASELINE_CHARUNIT_EDGE_IS_ONE_LOGICAL_ATOM. +Qed. + +Theorem VWENC_83_DYNAMIC_DAWG_CHAR_AND_UTF8_ADAPTER_OBSERVE_SAME_SCALAR : + forall representation codepoint bytes, + representation_admits representation (StoredUtf8 bytes) -> + canonical_utf8_codeword codepoint bytes -> + logical_transition (baseline_transition BaselineChar codepoint) = + logical_transition + {| transition_representation := representation; + transition_unit := StoredUtf8 bytes |}. +Proof. + intros representation codepoint bytes Hadmitted Hcanonical. + assert (unicode_scalar codepoint) as Hscalar. + { now destruct Hcanonical. } + rewrite + (VWENC_70_BASELINE_CHARUNIT_EDGE_IS_ONE_LOGICAL_ATOM + BaselineChar codepoint Hscalar). + rewrite + (VWENC_66_UTF8_LOGICAL_IDENTITY_IS_UNICODE_SCALAR + representation bytes codepoint Hadmitted Hcanonical). + reflexivity. +Qed. + +(** Correspondence with the existing Rocq graph models: [DawgTerm] is the + byte-label language of [DynamicDawgMutationSpec], while [U64Sequence] is + the native-label language of [DynamicDawgU64Spec]. These projections bind + the new logical-unit laws to the established mutation/zipper corpus rather + than defining a disconnected graph model. *) +Definition existing_byte_term_observations + (term : DawgTerm) : list (list LogicalAtom) := + map + (fun label => + consumer_observation DictionaryNodeSurface + (baseline_transition BaselineU8 (MapSpec.byte_to_nat label))) + term. + +Theorem VWENC_91_EXISTING_DYNAMIC_DAWG_BYTE_LABEL_IS_DIRECT_BYTE_ATOM : + forall label : DawgLabel, + logical_transition + (baseline_transition BaselineU8 (MapSpec.byte_to_nat label)) = + Some (DirectAtom DirectBytes (MapSpec.byte_to_nat label)). +Proof. + intros [label Hbyte]. + apply VWENC_70_BASELINE_CHARUNIT_EDGE_IS_ONE_LOGICAL_ATOM. + exact Hbyte. +Qed. + +Theorem VWENC_92_EXISTING_DYNAMIC_DAWG_TERM_PRESERVES_EDGE_COUNT : + forall term : DawgTerm, + length (existing_byte_term_observations term) = length term. +Proof. intros term. apply map_length. Qed. + +Definition existing_u64_sequence_observations + (sequence : U64Sequence) : list (list LogicalAtom) := + map + (fun label => + consumer_observation DictionaryNodeSurface + (baseline_transition BaselineU64 label)) + sequence. + +Theorem VWENC_93_EXISTING_U64_SEQUENCE_LABELS_ARE_DIRECT_U64_ATOMS : + forall sequence : U64Sequence, + Forall (fun label => direct_profile_valid DirectU64 label) sequence -> + existing_u64_sequence_observations sequence = + map (fun label => [DirectAtom DirectU64 label]) sequence. +Proof. + induction sequence as [| label rest IH]; intros Hvalid; [reflexivity |]. + inversion Hvalid as [| ? ? Hlabel Hrest]; subst. + assert + (logical_transition (baseline_transition BaselineU64 label) = + Some (DirectAtom DirectU64 label)) as Hlogical. + { exact + (VWENC_70_BASELINE_CHARUNIT_EDGE_IS_ONE_LOGICAL_ATOM + BaselineU64 label Hlabel). } + pose proof + (VWENC_16_CODEC_BYTES_ARE_NOT_LOGICAL_TRANSITIONS + DictionaryNodeSurface (baseline_transition BaselineU64 label) + (DirectAtom DirectU64 label) Hlogical) as Hobservation. + change + (consumer_observation DictionaryNodeSurface + (baseline_transition BaselineU64 label) :: + existing_u64_sequence_observations rest = + [DirectAtom DirectU64 label] :: + map (fun unit => [DirectAtom DirectU64 unit]) rest). + rewrite Hobservation, (IH Hrest). reflexivity. +Qed. + +Theorem VWENC_94_EXISTING_U64_SEQUENCE_PRESERVES_EDGE_COUNT : + forall sequence : U64Sequence, + length (existing_u64_sequence_observations sequence) = length sequence. +Proof. intros sequence. apply map_length. Qed. + +(** Open in-memory unit law carrier. This record does not enumerate the unit + type and therefore preserves downstream implementation of [CharUnit]. + A consumer supplies ordinary equality, ordering, and hash-input laws; the + generic core then stores exactly one [U] per edge. Persistent identities + remain closed and separately certified below. *) +Record OpenUnitProfile (U : Type) := { + open_unit_eqb : U -> U -> bool; + open_unit_compare : U -> U -> comparison; + open_unit_hash_material : U -> list nat; + open_unit_eqb_exact : + forall left right, open_unit_eqb left right = true <-> left = right; + open_unit_compare_equal_exact : + forall left right, open_unit_compare left right = Eq <-> left = right; + open_unit_compare_dual : + forall left right, + (open_unit_compare left right = Lt <-> + open_unit_compare right left = Gt) /\ + (open_unit_compare left right = Gt <-> + open_unit_compare right left = Lt); + open_unit_compare_lt_transitive : + forall left middle right, + open_unit_compare left middle = Lt -> + open_unit_compare middle right = Lt -> + open_unit_compare left right = Lt; + open_unit_hash_congruent : + forall left right, + left = right -> + open_unit_hash_material left = open_unit_hash_material right; +}. + +Inductive OpenConsumerSurface := +| OpenDictionaryNodeSurface +| OpenZipperSurface +| OpenSnapshotCursorSurface. + +Definition open_consumer_observation {U : Type} + (_profile : OpenUnitProfile U) + (_surface : OpenConsumerSurface) + (unit : U) : list U := [unit]. + +Theorem VWENC_84_OPEN_CHARUNIT_PROFILE_REMAINS_ONE_UNIT_PER_EDGE : + forall (U : Type) (profile : OpenUnitProfile U) surface unit, + open_consumer_observation profile surface unit = [unit] /\ + length (open_consumer_observation profile surface unit) = 1. +Proof. intros. split; reflexivity. Qed. + +Theorem VWENC_100_OPEN_UNIT_COMPARATOR_IS_TOTAL_ON_DISTINCT_UNITS : + forall (U : Type) (profile : OpenUnitProfile U) left right, + left <> right -> + open_unit_compare U profile left right = Lt \/ + open_unit_compare U profile left right = Gt. +Proof. + intros U profile left right Hdistinct. + destruct (open_unit_compare U profile left right) eqn:Hcompare. + - exfalso. apply Hdistinct. + now apply + (proj1 (open_unit_compare_equal_exact U profile left right)). + - now left. + - now right. +Qed. + +Theorem VWENC_85_OPEN_SURFACES_SHARE_REQUIRED_TARGET_DEFINITION : + forall (U : Type) (profile : OpenUnitProfile U) unit, + open_consumer_observation profile OpenDictionaryNodeSurface unit = + open_consumer_observation profile OpenZipperSurface unit /\ + open_consumer_observation profile OpenZipperSurface unit = + open_consumer_observation profile OpenSnapshotCursorSurface unit. +Proof. intros. split; reflexivity. Qed. + +(** Closed identities for persistence. Existing layouts and prospective codecs + are distinct constructors; equality can never silently reinterpret an old + image as a new UTF-8, ULEB, or semantic-F64 profile. *) +Inductive PersistentLogicalProfile := +| PersistedByte +| PersistedUnicodeScalar +| PersistedU64 +| PersistedF64Bits +| PersistedCanonicalUleb +| PersistedCanonicalUtf8. + +Inductive PersistentCodecIdentity := +| ExistingByteCodec +| ExistingCharU32Codec +| ExistingU64Codec +| ProspectiveF64BitsCodecV1 +| ProspectiveCanonicalUlebCodecV1 +| ProspectiveCanonicalUtf8CodecV1. + +Inductive PersistentLayoutIdentity := +| ExistingByteLayout +| ExistingCharLayout +| ExistingU64Layout +| ProspectiveLogicalUnitLayoutV1. + +Record PersistentProfileDescriptor := { + persistent_logical_profile : PersistentLogicalProfile; + persistent_codec_identity : PersistentCodecIdentity; + persistent_layout_identity : PersistentLayoutIdentity; + persistent_abi_version : nat; +}. + +Definition certified_persistent_profile + (descriptor : PersistentProfileDescriptor) : Prop := + match descriptor.(persistent_logical_profile), + descriptor.(persistent_codec_identity), + descriptor.(persistent_layout_identity) with + | PersistedByte, ExistingByteCodec, ExistingByteLayout => + 0 < descriptor.(persistent_abi_version) + | PersistedUnicodeScalar, ExistingCharU32Codec, ExistingCharLayout => + 0 < descriptor.(persistent_abi_version) + | PersistedU64, ExistingU64Codec, ExistingU64Layout => + 0 < descriptor.(persistent_abi_version) + | PersistedF64Bits, ProspectiveF64BitsCodecV1, + ProspectiveLogicalUnitLayoutV1 => + 0 < descriptor.(persistent_abi_version) + | PersistedCanonicalUleb, ProspectiveCanonicalUlebCodecV1, + ProspectiveLogicalUnitLayoutV1 => + 0 < descriptor.(persistent_abi_version) + | PersistedCanonicalUtf8, ProspectiveCanonicalUtf8CodecV1, + ProspectiveLogicalUnitLayoutV1 => + 0 < descriptor.(persistent_abi_version) + | _, _, _ => False + end. + +Definition certified_profile_identity + (profile : PersistentProfileDescriptor) + : PersistentLogicalProfile * + (PersistentCodecIdentity * (PersistentLayoutIdentity * nat)) := + (profile.(persistent_logical_profile), + (profile.(persistent_codec_identity), + (profile.(persistent_layout_identity), profile.(persistent_abi_version)))). + +Theorem VWENC_86_CERTIFIED_PERSISTENT_PROFILE_IDENTITY_IS_INJECTIVE : + forall left right, + certified_profile_identity left = certified_profile_identity right -> + left = right. +Proof. + intros [left_profile left_codec left_layout left_abi] + [right_profile right_codec right_layout right_abi] Hequal. + unfold certified_profile_identity in Hequal. simpl in Hequal. + now inversion Hequal. +Qed. + +Definition profile_bound_payload + (profile : PersistentProfileDescriptor) + (payload : list PhysicalByte) := + (certified_profile_identity profile, payload). + +Theorem VWENC_87_PROFILE_AND_PAYLOAD_IDENTITY_IS_JOINTLY_INJECTIVE : + forall left_profile left_payload right_profile right_payload, + profile_bound_payload left_profile left_payload = + profile_bound_payload right_profile right_payload -> + left_profile = right_profile /\ left_payload = right_payload. +Proof. + intros + [left_profile left_codec left_layout left_abi] left_payload + [right_profile right_codec right_layout right_abi] right_payload Hequal. + unfold profile_bound_payload, certified_profile_identity in Hequal. + simpl in Hequal. inversion Hequal. split; reflexivity. +Qed. + +Theorem VWENC_98_CERTIFICATION_REJECTS_INCOHERENT_PROFILE_CODEC_LAYOUT : + ~ certified_persistent_profile + {| persistent_logical_profile := PersistedCanonicalUleb; + persistent_codec_identity := ExistingCharU32Codec; + persistent_layout_identity := ExistingByteLayout; + persistent_abi_version := 1 |}. +Proof. simpl. tauto. Qed. + +Theorem VWENC_99_CERTIFICATION_ACCEPTS_VERSIONED_CANONICAL_ULEB_PROFILE : + certified_persistent_profile + {| persistent_logical_profile := PersistedCanonicalUleb; + persistent_codec_identity := ProspectiveCanonicalUlebCodecV1; + persistent_layout_identity := ProspectiveLogicalUnitLayoutV1; + persistent_abi_version := 1 |}. +Proof. + unfold certified_persistent_profile. simpl. + exact (Nat.lt_0_succ 0). +Qed. + +(** ** F64Bits raw identity and total_cmp-compatible ordering *) + +Definition valid_f64_bits (bits : nat) : Prop := bits < two_to_64. + +(** Positive encodings occupy the upper half in increasing bit order. Signed + encodings occupy the lower half in reversed bit order. This is the + sortable-key form of Rust's [f64::total_cmp] transformation. *) +Definition split_rank (half whole bits : nat) : nat := + if bits left = right. +Proof. intros left right Hequal. exact Hequal. Qed. + +Theorem VWENC_19_F64BITS_SIGNED_ZEROES_ARE_DISTINCT : + 0 <> two_to_63 /\ + f64_total_rank two_to_63 < f64_total_rank 0. +Proof. + pose proof two_to_63_positive as Hhalf_positive. + pose proof two_to_64_is_double_two_to_63 as Hdouble. + split; [lia |]. + unfold f64_total_rank, split_rank. + assert ((two_to_63 + whole = 2 * half -> + left < whole -> + right < whole -> + split_rank half whole left = split_rank half whole right -> + left = right. +Proof. + intros half whole left right Hhalf Hwhole Hleft Hright Hrank. + unfold split_rank in Hrank. + destruct (left + valid_f64_bits right -> + f64_total_rank left = f64_total_rank right -> + left = right. +Proof. + intros left right Hleft Hright Hrank. + unfold valid_f64_bits in Hleft, Hright. + unfold f64_total_rank in Hrank. + eapply split_rank_injective. + - exact two_to_63_positive. + - exact two_to_64_is_double_two_to_63. + - exact Hleft. + - exact Hright. + - exact Hrank. +Qed. + +Definition f64_hash_material (bits : nat) : nat := bits. + +Theorem VWENC_52_F64BITS_ALL_DISTINCT_PATTERNS_REMAIN_DISTINCT : + forall left right, + valid_f64_bits left -> + valid_f64_bits right -> + left <> right -> + f64_bits_identity left <> f64_bits_identity right /\ + f64_hash_material left <> f64_hash_material right /\ + compare_f64_bits left right <> Eq. +Proof. + intros left right Hleft Hright Hdistinct. + repeat split; try exact Hdistinct. + intros Hequal. + unfold compare_f64_bits in Hequal. + apply Nat.compare_eq_iff in Hequal. + apply Hdistinct. + now apply VWENC_21_F64BITS_TOTAL_RANK_INJECTIVE. +Qed. + +Theorem VWENC_73_F64BITS_COMPARATOR_EQUAL_IFF_RAW_BITS_EQUAL : + forall left right, + valid_f64_bits left -> + valid_f64_bits right -> + (compare_f64_bits left right = Eq <-> left = right). +Proof. + intros left right Hleft Hright. split. + - unfold compare_f64_bits. rewrite Nat.compare_eq_iff. + now apply VWENC_21_F64BITS_TOTAL_RANK_INJECTIVE. + - intros ->. unfold compare_f64_bits. apply Nat.compare_refl. +Qed. + +Theorem VWENC_74_F64BITS_COMPARATOR_IS_TOTAL : + forall left right, + compare_f64_bits left right = Lt \/ + compare_f64_bits left right = Eq \/ + compare_f64_bits left right = Gt. +Proof. + intros left right. + destruct (compare_f64_bits left right); tauto. +Qed. + +Theorem VWENC_75_F64BITS_COMPARATOR_IS_ANTISYMMETRIC : + forall left right, + (compare_f64_bits left right = Lt <-> + compare_f64_bits right left = Gt) /\ + (compare_f64_bits left right = Gt <-> + compare_f64_bits right left = Lt). +Proof. + intros left right. unfold compare_f64_bits. + repeat split; intro Hcompare. + - apply Nat.compare_lt_iff in Hcompare. + apply Nat.compare_gt_iff. exact Hcompare. + - apply Nat.compare_gt_iff in Hcompare. + apply Nat.compare_lt_iff. exact Hcompare. + - apply Nat.compare_gt_iff in Hcompare. + apply Nat.compare_lt_iff. exact Hcompare. + - apply Nat.compare_lt_iff in Hcompare. + apply Nat.compare_gt_iff. exact Hcompare. +Qed. + +Theorem VWENC_76_F64BITS_COMPARATOR_LT_IS_TRANSITIVE : + forall left middle right, + compare_f64_bits left middle = Lt -> + compare_f64_bits middle right = Lt -> + compare_f64_bits left right = Lt. +Proof. + intros left middle right Hleft Hright. + unfold compare_f64_bits in *. + apply Nat.compare_lt_iff in Hleft. + apply Nat.compare_lt_iff in Hright. + apply Nat.compare_lt_iff. lia. +Qed. + +(** Equivalent rank obtained by interpreting the high bit as the IEEE sign, + reversing the lower-half order for negative encodings, and then shifting + the signed key into naturals. This is the arithmetic form of Rust's + [f64::total_cmp] signed-key transform. *) +Definition rust_total_cmp_shifted_rank + (half bits : nat) : nat := + if bits + f64_total_rank bits = rust_total_cmp_shifted_rank two_to_63 bits. +Proof. + intros bits Hvalid. + unfold valid_f64_bits in Hvalid. + unfold f64_total_rank, split_rank, rust_total_cmp_shifted_rank. + destruct (bits nat) : Prop := + forall left right, + valid_f64_bits left -> + valid_f64_bits right -> + left <> right -> + identity left <> identity right. + +Lemma f64_zero_alias_mutant_aliases_signed_zero : + 0 <> two_to_63 /\ + f64_zero_alias_mutant 0 = f64_zero_alias_mutant two_to_63. +Proof. + pose proof two_to_63_positive as Hpositive. + split; [lia |]. + unfold f64_zero_alias_mutant. + rewrite Nat.eqb_refl. + assert ((0 =? two_to_63) = false) as Hdistinct. + { apply Nat.eqb_neq. lia. } + now rewrite Hdistinct. +Qed. + +Theorem VWENC_78_NEGATIVE_CONTROL_NUMERIC_F64_IDENTITY_VIOLATES_RAW_BITS : + ~ preserves_distinct_valid_f64_bits f64_zero_alias_mutant. +Proof. + intros Hpreserves. + destruct f64_zero_alias_mutant_aliases_signed_zero + as [Hdistinct Halias]. + assert (valid_f64_bits 0) as Hzero. + { unfold valid_f64_bits. + pose proof two_to_63_positive. + pose proof two_to_64_is_double_two_to_63. lia. } + assert (valid_f64_bits two_to_63) as Hnegative_zero. + { unfold valid_f64_bits. + pose proof two_to_63_positive. + pose proof two_to_64_is_double_two_to_63. lia. } + specialize + (Hpreserves 0 two_to_63 Hzero Hnegative_zero Hdistinct). + exact (Hpreserves Halias). +Qed. + +(** Negative control: comparing only the first encoded ULEB byte reverses the + adjacent numeric values 255 and 256. *) +Definition compare_first_uleb_byte + (left right : list PhysicalByte) : comparison := + match left, right with + | left_byte :: _, right_byte :: _ => Nat.compare left_byte right_byte + | [], [] => Eq + | [], _ => Lt + | _, [] => Gt + end. + +Theorem VWENC_79_NEGATIVE_CONTROL_ENCODED_BYTE_ORDER_REVERSES_255_AND_256 : + canonical_uleb_codeword [255; 1] /\ + canonical_uleb_codeword [128; 2] /\ + compare_uleb_codewords [255; 1] [128; 2] = Lt /\ + compare_first_uleb_byte [255; 1] [128; 2] = Gt. +Proof. + assert (canonical_uleb_codeword [255; 1]) as H255. + { split. + - apply UlebShapeMore; [lia | lia |]. + apply UlebShapeLast. lia. + - unfold canonical_uleb_digits, decode_uleb_payloads, + uleb_payload, valid_uleb_digit. + simpl. repeat split. + + discriminate. + + constructor; [lia | constructor; [lia | constructor]]. + + lia. } + assert (canonical_uleb_codeword [128; 2]) as H256. + { split. + - apply UlebShapeMore; [lia | lia |]. + apply UlebShapeLast. lia. + - unfold canonical_uleb_digits, decode_uleb_payloads, + uleb_payload, valid_uleb_digit. + simpl. repeat split. + + discriminate. + + constructor; [lia | constructor; [lia | constructor]]. + + lia. } + split; [exact H255 |]. + split; [exact H256 |]. + split; reflexivity. +Qed. + +Definition direct_identity + (profile : DirectProfile) (unit : nat) : nat * nat := + (direct_profile_tag profile, unit). + +Definition direct_hash_material + (profile : DirectProfile) (unit : nat) : nat * nat := + direct_identity profile unit. + +Definition direct_order_key + (profile : DirectProfile) (unit : nat) : nat := + match profile with + | DirectF64Bits => f64_total_rank unit + | _ => unit + end. + +Definition compare_direct_units + (profile : DirectProfile) (left right : nat) : comparison := + Nat.compare (direct_order_key profile left) (direct_order_key profile right). + +Theorem VWENC_53_DIRECT_IDENTITY_AND_HASH_ARE_PROFILE_SCOPED_AND_INJECTIVE : + forall left_profile left_unit right_profile right_unit, + direct_hash_material left_profile left_unit = + direct_hash_material right_profile right_unit -> + left_profile = right_profile /\ left_unit = right_unit. +Proof. + intros left_profile left_unit right_profile right_unit Hequal. + unfold direct_hash_material, direct_identity in Hequal. + inversion Hequal as [[Htag Hunit]]. + split. + - now apply VWENC_48_DIRECT_PROFILE_TAGS_ARE_INJECTIVE. + - reflexivity. +Qed. + +Theorem VWENC_54_UNSIGNED_DIRECT_ORDER_IS_LOGICAL_VALUE_ORDER : + forall profile left right, + profile <> DirectF64Bits -> + compare_direct_units profile left right = Nat.compare left right. +Proof. + intros profile left right Hnotf64. + destruct profile; [reflexivity | reflexivity | reflexivity | reflexivity |]. + contradiction. +Qed. + +Theorem VWENC_55_F64BITS_DIRECT_ORDER_IS_TOTAL_CMP_ORDER : + forall left right, + compare_direct_units DirectF64Bits left right = + compare_f64_bits left right. +Proof. reflexivity. Qed. + +Theorem VWENC_56_DIRECT_PROFILE_WIDTHS_ARE_EXPLICIT : + direct_byte_width DirectBytes = 1 /\ + direct_byte_width DirectUnicodeScalar = 4 /\ + direct_byte_width DirectU32 = 4 /\ + direct_byte_width DirectU64 = 8 /\ + direct_byte_width DirectF64Bits = 8. +Proof. repeat split; reflexivity. Qed. + +End VariableWidthCodecSpec. diff --git a/formal-verification/rocq/Spec/VariableWidthFamilyRefinementSpec.v b/formal-verification/rocq/Spec/VariableWidthFamilyRefinementSpec.v new file mode 100644 index 00000000..723fb169 --- /dev/null +++ b/formal-verification/rocq/Spec/VariableWidthFamilyRefinementSpec.v @@ -0,0 +1,2421 @@ +(** * Family/profile refinement and consumer-observation laws + + This module is the family-wide refinement layer for variable-width logical + units. It deliberately separates three concepts: + + - a logical observation, which is visible to [liblevenshtein], [llattice], + and other traversing consumers; + - a storage/profile route, which selects a native kernel, a fixed-width ID + kernel, or the PathMap byte adapter before traversal begins; and + - physical state, including encoded staging bytes and node layout, which is + not a logical observation. + + Arbitrary-width canonical ULEB128 values remain in the vocabulary owner. + A hot dictionary edge contains either one direct fixed-width unit or one + fixed-width [SymbolId]. Consequently the vocabulary binding is checked + once when a snapshot/query view is constructed, never by decoding an + arbitrary-width value at every node. + + The closed family, profile, and surface inventories below are the + reviewable applicability matrix for this release. Open downstream unit + implementations remain possible in memory; persistent format identities + are separately certified and never inferred from Rust type names. + + Stable theorem names beginning with [VWENC_] are machine-readable + invariant identifiers. They are extracted into the implementation + conformance ledger and property-test suite after this formal gate closes. +*) + +From Coq Require Import Arith Bool Lia List PeanoNat ProofIrrelevance. +Require Import ARTrie.Spec.VariableWidthCodecSpec. +Require Import ARTrie.Spec.VariableWidthInterningSpec. +Import ListNotations. +Import VariableWidthCodecSpec VariableWidthInterning. + +Module VariableWidthFamilyRefinementSpec. + +(** ** Representation-independent logical observations *) + +Record LogicalObservations (Atom Value : Type) : Type := { + observe_membership : list Atom -> bool; + observe_terminality : list Atom -> bool; + observe_mapped_value : list Atom -> option Value; + observe_ordered_outgoing : list Atom -> list Atom; + observe_prefix_entries : list Atom -> list (list Atom * option Value); + observe_full_enumeration : list (list Atom * option Value); + observe_substring_applicable : bool; + observe_substring_results : list Atom -> list (list Atom * option Value); + observe_suffix_applicable : bool; + observe_suffix_results : list Atom -> list (list Atom * option Value) +}. + +(** Equality of observations is deliberately extensional. It includes the + order of outgoing labels and enumeration results because deterministic + iteration is part of the public contract. *) +Record SameLogicalObservations {Atom Value : Type} + (left right : LogicalObservations Atom Value) : Prop := { + same_membership : + forall word, observe_membership Atom Value left word = + observe_membership Atom Value right word; + same_terminality : + forall word, observe_terminality Atom Value left word = + observe_terminality Atom Value right word; + same_mapped_value : + forall word, observe_mapped_value Atom Value left word = + observe_mapped_value Atom Value right word; + same_ordered_outgoing : + forall prefix, observe_ordered_outgoing Atom Value left prefix = + observe_ordered_outgoing Atom Value right prefix; + same_prefix_entries : + forall prefix, observe_prefix_entries Atom Value left prefix = + observe_prefix_entries Atom Value right prefix; + same_full_enumeration : + observe_full_enumeration Atom Value left = + observe_full_enumeration Atom Value right; + same_substring_applicability : + observe_substring_applicable Atom Value left = + observe_substring_applicable Atom Value right; + same_substring_results : + forall query, + observe_substring_applicable Atom Value left = true -> + observe_substring_results Atom Value left query = + observe_substring_results Atom Value right query; + same_suffix_applicability : + observe_suffix_applicable Atom Value left = + observe_suffix_applicable Atom Value right; + same_suffix_results : + forall query, + observe_suffix_applicable Atom Value left = true -> + observe_suffix_results Atom Value left query = + observe_suffix_results Atom Value right query +}. + +Lemma same_logical_observations_reflexive : + forall (Atom Value : Type) (view : LogicalObservations Atom Value), + SameLogicalObservations view view. +Proof. intros. constructor; intros; reflexivity. Qed. + +Lemma same_logical_observations_symmetric : + forall (Atom Value : Type) (left right : LogicalObservations Atom Value), + SameLogicalObservations left right -> + SameLogicalObservations right left. +Proof. + intros Atom Value left right Hsame. + destruct Hsame as + [Hmembership Hterminal Hvalue Houtgoing Hprefix Henumeration + Hsubstring_app Hsubstring Hsuffix_app Hsuffix]. + constructor. + - intros. symmetry. auto. + - intros. symmetry. auto. + - intros. symmetry. auto. + - intros. symmetry. auto. + - intros. symmetry. auto. + - symmetry. exact Henumeration. + - symmetry. exact Hsubstring_app. + - intros query Hright_app. symmetry. apply Hsubstring. + rewrite Hsubstring_app. exact Hright_app. + - symmetry. exact Hsuffix_app. + - intros query Hright_app. symmetry. apply Hsuffix. + rewrite Hsuffix_app. exact Hright_app. +Qed. + +Lemma same_logical_observations_transitive : + forall (Atom Value : Type) + (left middle right : LogicalObservations Atom Value), + SameLogicalObservations left middle -> + SameLogicalObservations middle right -> + SameLogicalObservations left right. +Proof. + intros Atom Value left middle right Hleft Hright. + destruct Hleft as + [Hlm Hlt Hlv Hlo Hlp Hle Hlsa Hls Hlsua Hlsu]. + destruct Hright as + [Hmm Hmt Hmv Hmo Hmp Hme Hmsa Hms Hmsua Hmsu]. + constructor. + - intros. eauto using eq_trans. + - intros. eauto using eq_trans. + - intros. eauto using eq_trans. + - intros. eauto using eq_trans. + - intros. eauto using eq_trans. + - eauto using eq_trans. + - eauto using eq_trans. + - intros query Hleft_app. + assert (Hmiddle_app : + observe_substring_applicable Atom Value middle = true). + { rewrite <- Hlsa. exact Hleft_app. } + eapply eq_trans. + + now apply Hls. + + now apply Hms. + - eauto using eq_trans. + - intros query Hleft_app. + assert (Hmiddle_app : + observe_suffix_applicable Atom Value middle = true). + { rewrite <- Hlsua. exact Hleft_app. } + eapply eq_trans. + + now apply Hlsu. + + now apply Hmsu. +Qed. + +Theorem VWENC_194_LOGICAL_OBSERVATIONAL_EQUIVALENCE_IS_AN_EQUIVALENCE : + (forall (Atom Value : Type) (view : LogicalObservations Atom Value), + SameLogicalObservations view view) /\ + (forall (Atom Value : Type) + (left right : LogicalObservations Atom Value), + SameLogicalObservations left right -> + SameLogicalObservations right left) /\ + (forall (Atom Value : Type) + (left middle right : LogicalObservations Atom Value), + SameLogicalObservations left middle -> + SameLogicalObservations middle right -> + SameLogicalObservations left right). +Proof. + split. + - intros. apply same_logical_observations_reflexive. + - split. + + intros. now apply same_logical_observations_symmetric. + + intros. eapply same_logical_observations_transitive; eassumption. +Qed. + +Theorem VWENC_195_MEMBERSHIP_AND_TERMINALITY_ARE_LOGICAL_OBSERVATIONS : + forall (Atom Value : Type) + (left right : LogicalObservations Atom Value) word, + SameLogicalObservations left right -> + observe_membership Atom Value left word = + observe_membership Atom Value right word /\ + observe_terminality Atom Value left word = + observe_terminality Atom Value right word. +Proof. + intros Atom Value left right word Hsame. split. + - now apply same_membership. + - now apply same_terminality. +Qed. + +Theorem VWENC_196_MAPPED_VALUE_PRESENCE_AND_IDENTITY_ARE_OBSERVABLE : + forall (Atom Value : Type) + (left right : LogicalObservations Atom Value) word, + SameLogicalObservations left right -> + observe_mapped_value Atom Value left word = + observe_mapped_value Atom Value right word. +Proof. intros. now apply same_mapped_value. Qed. + +Theorem VWENC_197_ORDERED_LOGICAL_OUTGOING_LABELS_ARE_OBSERVABLE : + forall (Atom Value : Type) + (left right : LogicalObservations Atom Value) prefix, + SameLogicalObservations left right -> + observe_ordered_outgoing Atom Value left prefix = + observe_ordered_outgoing Atom Value right prefix. +Proof. intros. now apply same_ordered_outgoing. Qed. + +Theorem VWENC_198_PREFIX_ENTRIES_ARE_LOGICAL_OBSERVATIONS : + forall (Atom Value : Type) + (left right : LogicalObservations Atom Value) prefix, + SameLogicalObservations left right -> + observe_prefix_entries Atom Value left prefix = + observe_prefix_entries Atom Value right prefix. +Proof. intros. now apply same_prefix_entries. Qed. + +Theorem VWENC_199_FULL_ENUMERATION_ORDER_IS_DETERMINISTIC_AND_OBSERVABLE : + forall (Atom Value : Type) + (left right : LogicalObservations Atom Value), + SameLogicalObservations left right -> + observe_full_enumeration Atom Value left = + observe_full_enumeration Atom Value right. +Proof. intros. now apply same_full_enumeration. Qed. + +Theorem VWENC_200_APPLICABLE_SUBSTRING_RESULTS_ARE_LOGICAL_OBSERVATIONS : + forall (Atom Value : Type) + (left right : LogicalObservations Atom Value) query, + SameLogicalObservations left right -> + observe_substring_applicable Atom Value left = true -> + observe_substring_applicable Atom Value left = + observe_substring_applicable Atom Value right /\ + observe_substring_results Atom Value left query = + observe_substring_results Atom Value right query. +Proof. + intros Atom Value left right query Hsame Happlicable. split. + - now apply same_substring_applicability. + - now apply same_substring_results. +Qed. + +Theorem VWENC_201_APPLICABLE_SUFFIX_RESULTS_ARE_LOGICAL_OBSERVATIONS : + forall (Atom Value : Type) + (left right : LogicalObservations Atom Value) query, + SameLogicalObservations left right -> + observe_suffix_applicable Atom Value left = true -> + observe_suffix_applicable Atom Value left = + observe_suffix_applicable Atom Value right /\ + observe_suffix_results Atom Value left query = + observe_suffix_results Atom Value right query. +Proof. + intros Atom Value left right query Hsame Happlicable. split. + - now apply same_suffix_applicability. + - now apply same_suffix_results. +Qed. + +Record PhysicalImplementation (Atom Value : Type) : Type := { + implementation_logical_view : LogicalObservations Atom Value; + implementation_node_layout : list nat; + implementation_staging_bytes : list PhysicalByte; + implementation_hash_buckets : list nat; + implementation_wal_bytes : list PhysicalByte +}. + +Definition replace_physical_state {Atom Value : Type} + (implementation : PhysicalImplementation Atom Value) + (node_layout hash_buckets : list nat) + (staging_bytes wal_bytes : list PhysicalByte) + : PhysicalImplementation Atom Value := + {| implementation_logical_view := + implementation_logical_view Atom Value implementation; + implementation_node_layout := node_layout; + implementation_staging_bytes := staging_bytes; + implementation_hash_buckets := hash_buckets; + implementation_wal_bytes := wal_bytes |}. + +Theorem VWENC_202_PHYSICAL_LAYOUT_AND_CODEC_STAGING_STATE_ARE_NONOBSERVABLE : + forall (Atom Value : Type) + (implementation : PhysicalImplementation Atom Value) + node_layout hash_buckets staging_bytes wal_bytes, + SameLogicalObservations + (implementation_logical_view Atom Value implementation) + (implementation_logical_view Atom Value + (replace_physical_state implementation node_layout hash_buckets + staging_bytes wal_bytes)). +Proof. intros. apply same_logical_observations_reflexive. Qed. + +(** ** Closed family/profile/surface applicability matrix *) + +Inductive DictionaryFamily : Type := +| DynamicDawgFamily +| DoubleArrayTrieFamily +| SuffixAutomatonFamily +| ScdawgFamily +| PathMapAdapterFamily +| PersistentARTrieFamily +| PersistentSuffixAutomatonFamily +| PersistentSuffixTreeFamily +| PersistentScdawgFamily +| BijectiveMapFamily +| PersistentVocabARTrieFamily. + +Definition all_dictionary_families : list DictionaryFamily := + [DynamicDawgFamily; DoubleArrayTrieFamily; SuffixAutomatonFamily; + ScdawgFamily; PathMapAdapterFamily; PersistentARTrieFamily; + PersistentSuffixAutomatonFamily; PersistentSuffixTreeFamily; + PersistentScdawgFamily; BijectiveMapFamily; + PersistentVocabARTrieFamily]. + +Inductive DirectUnitDomain : Type := +| DirectBytesDomain +| DirectUnicodeScalarDomain +| DirectU32Domain +| DirectU64Domain +| DirectF64BitsDomain. + +Inductive InternedAtomDomain : Type := +| CanonicalUlebDomain +| CanonicalUtf8Domain +| OpaqueCanonicalBytesDomain. + +Inductive IdCarrier : Type := +| U32IdCarrier +| U64IdCarrier. + +Inductive FamilyProfile : Type := +| DirectProfile : DirectUnitDomain -> FamilyProfile +| InternedProfile : InternedAtomDomain -> IdCarrier -> FamilyProfile. + +Definition all_family_profiles : list FamilyProfile := + [DirectProfile DirectBytesDomain; + DirectProfile DirectUnicodeScalarDomain; + DirectProfile DirectU32Domain; + DirectProfile DirectU64Domain; + DirectProfile DirectF64BitsDomain; + InternedProfile CanonicalUlebDomain U32IdCarrier; + InternedProfile CanonicalUlebDomain U64IdCarrier; + InternedProfile CanonicalUtf8Domain U32IdCarrier; + InternedProfile CanonicalUtf8Domain U64IdCarrier; + InternedProfile OpaqueCanonicalBytesDomain U32IdCarrier; + InternedProfile OpaqueCanonicalBytesDomain U64IdCarrier]. + +(** The logical unit type is a function of the profile. An implementation + cannot independently choose an unrelated [Atom] type. *) +Definition direct_codec_profile + (domain : DirectUnitDomain) : VariableWidthCodecSpec.DirectProfile := + match domain with + | DirectBytesDomain => DirectBytes + | DirectUnicodeScalarDomain => DirectUnicodeScalar + | DirectU32Domain => DirectU32 + | DirectU64Domain => DirectU64 + | DirectF64BitsDomain => DirectF64Bits + end. + +Definition u32_id_profile : FixedWidthCarrierProfile := + {| carrier_format_identity := 32; + carrier_width_bytes := 4; + carrier_width_positive := ltac:(lia) |}. + +Definition u64_id_profile : FixedWidthCarrierProfile := + {| carrier_format_identity := 64; + carrier_width_bytes := 8; + carrier_width_positive := ltac:(lia) |}. + +Definition id_carrier_profile + (carrier : IdCarrier) : FixedWidthCarrierProfile := + match carrier with + | U32IdCarrier => u32_id_profile + | U64IdCarrier => u64_id_profile + end. + +Definition DirectUnit (domain : DirectUnitDomain) : Type := + { unit : nat | direct_profile_valid (direct_codec_profile domain) unit }. + +Definition ProfileUnit (profile : FamilyProfile) : Type := + match profile with + | DirectProfile domain => DirectUnit domain + | InternedProfile _ carrier => SymbolId (id_carrier_profile carrier) + end. + +Inductive ExplicitLayoutContract : Type := +| GenericLogicalLayout +| PathMapNativeByteLayout +| PathMapUtf8BoundaryLayout +| PathMapFixedWidthBoundaryLayout +| PathMapInternedIdLayout : IdCarrier -> ExplicitLayoutContract +| PersistentU64CompactLayout +| PersistentU64Prefix3CompatibilityLayout +| EncodedU64ByteCompatibilityLayout +| ProspectiveInternedIdLayout : IdCarrier -> ExplicitLayoutContract. + +Inductive ProfileRoute : Type := +| GenericNativeKernel +| RetainedSpecializedKernel +| InternedFixedIdKernel : IdCarrier -> ProfileRoute +| EncodedU64ByteAdapterKernel +| PathMapNativeByteRoute +| PathMapUtf8BoundaryAdapterRoute +| PathMapFixedWidthBoundaryAdapterRoute +| PathMapInternedIdAdapterRoute : IdCarrier -> ProfileRoute +| BijectiveTermValueKernel +| VocabularyOwnerRoute : IdCarrier -> ProfileRoute. + +(** Existing behavior and prospective work are deliberately distinct. There + is no "unknown" or implicit-support constructor. *) +Inductive ProfileCell : Type := +| ExistingProfileCell : ProfileRoute -> ExplicitLayoutContract -> ProfileCell +| ProspectiveProfileCell : ProfileRoute -> ExplicitLayoutContract -> ProfileCell. + +Definition family_profile_cell + (family : DictionaryFamily) (profile : FamilyProfile) : ProfileCell := + match family, profile with + | DynamicDawgFamily, DirectProfile DirectBytesDomain + | DynamicDawgFamily, DirectProfile DirectUnicodeScalarDomain + | DynamicDawgFamily, DirectProfile DirectU64Domain + | DoubleArrayTrieFamily, DirectProfile DirectBytesDomain + | DoubleArrayTrieFamily, DirectProfile DirectUnicodeScalarDomain + | SuffixAutomatonFamily, DirectProfile DirectBytesDomain + | SuffixAutomatonFamily, DirectProfile DirectUnicodeScalarDomain + | ScdawgFamily, DirectProfile DirectBytesDomain + | ScdawgFamily, DirectProfile DirectUnicodeScalarDomain + | PersistentSuffixAutomatonFamily, DirectProfile DirectBytesDomain + | PersistentSuffixAutomatonFamily, DirectProfile DirectUnicodeScalarDomain + | PersistentSuffixTreeFamily, DirectProfile DirectBytesDomain + | PersistentSuffixTreeFamily, DirectProfile DirectUnicodeScalarDomain + | PersistentScdawgFamily, DirectProfile DirectBytesDomain + | PersistentScdawgFamily, DirectProfile DirectUnicodeScalarDomain => + ExistingProfileCell RetainedSpecializedKernel GenericLogicalLayout + | PersistentARTrieFamily, DirectProfile DirectBytesDomain + | PersistentARTrieFamily, DirectProfile DirectUnicodeScalarDomain => + ExistingProfileCell RetainedSpecializedKernel GenericLogicalLayout + | PersistentARTrieFamily, DirectProfile DirectU64Domain => + ExistingProfileCell RetainedSpecializedKernel PersistentU64CompactLayout + | PathMapAdapterFamily, DirectProfile DirectBytesDomain => + ExistingProfileCell PathMapNativeByteRoute PathMapNativeByteLayout + | PathMapAdapterFamily, DirectProfile DirectUnicodeScalarDomain => + ExistingProfileCell PathMapUtf8BoundaryAdapterRoute + PathMapUtf8BoundaryLayout + | BijectiveMapFamily, DirectProfile DirectUnicodeScalarDomain => + ExistingProfileCell BijectiveTermValueKernel GenericLogicalLayout + | PersistentVocabARTrieFamily, + DirectProfile DirectUnicodeScalarDomain => + ExistingProfileCell (VocabularyOwnerRoute U64IdCarrier) + GenericLogicalLayout + | PathMapAdapterFamily, DirectProfile _ => + ProspectiveProfileCell PathMapFixedWidthBoundaryAdapterRoute + PathMapFixedWidthBoundaryLayout + | PathMapAdapterFamily, InternedProfile _ carrier => + ProspectiveProfileCell (PathMapInternedIdAdapterRoute carrier) + (PathMapInternedIdLayout carrier) + | BijectiveMapFamily, InternedProfile _ carrier + | PersistentVocabARTrieFamily, InternedProfile _ carrier => + ProspectiveProfileCell (VocabularyOwnerRoute carrier) + (ProspectiveInternedIdLayout carrier) + | BijectiveMapFamily, DirectProfile _ + | PersistentVocabARTrieFamily, DirectProfile _ => + ProspectiveProfileCell BijectiveTermValueKernel GenericLogicalLayout + | _, DirectProfile _ => + ProspectiveProfileCell GenericNativeKernel GenericLogicalLayout + | _, InternedProfile _ carrier => + ProspectiveProfileCell (InternedFixedIdKernel carrier) + (ProspectiveInternedIdLayout carrier) + end. + +Definition profile_cell_route (cell : ProfileCell) : ProfileRoute := + match cell with + | ExistingProfileCell route _ | ProspectiveProfileCell route _ => route + end. + +Definition profile_cell_layout + (cell : ProfileCell) : ExplicitLayoutContract := + match cell with + | ExistingProfileCell _ layout | ProspectiveProfileCell _ layout => layout + end. + +Definition family_profile_route + (family : DictionaryFamily) (profile : FamilyProfile) : ProfileRoute := + profile_cell_route (family_profile_cell family profile). + +Inductive ConsumerSurfaceClass : Type := +| DictionarySurface +| DictionaryNodeSurfaceClass +| ZipperSurfaceClass +| SnapshotCursorSurfaceClass +| FactorySurface +| CollectionSurface +| SerializationReopenSurface +| SnapshotSurface +| SetCombinatorSurface +| ValueCombinatorSurface +| PrefixSurface +| SubstringSurface +| SuffixSurface +| ReverseLookupSurface. + +Definition all_consumer_surfaces : list ConsumerSurfaceClass := + [DictionarySurface; DictionaryNodeSurfaceClass; ZipperSurfaceClass; + SnapshotCursorSurfaceClass; FactorySurface; CollectionSurface; + SerializationReopenSurface; SnapshotSurface; SetCombinatorSurface; + ValueCombinatorSurface; PrefixSurface; SubstringSurface; SuffixSurface; + ReverseLookupSurface]. + +Inductive SurfaceRoute : Type := +| CommonDictionaryRoute +| SuffixIndexRoute +| VocabularyReverseLookupRoute. + +Inductive SurfaceInapplicability : Type := +| ExactTermFamilyHasNoSubstringIndex +| TermIndexHasNoVocabularyReverseLookup +| VocabularyOwnerHasNoSuffixIndex +| PersistentConstructionRequiresExplicitStoreConfiguration. + +Inductive SurfaceCell : Type := +| ExistingSurface : SurfaceRoute -> SurfaceCell +| ProspectiveSurface : SurfaceRoute -> SurfaceCell +| SurfaceStructurallyInapplicable : SurfaceInapplicability -> SurfaceCell. + +Definition persistent_family (family : DictionaryFamily) : bool := + match family with + | PersistentARTrieFamily | PersistentSuffixAutomatonFamily + | PersistentSuffixTreeFamily | PersistentScdawgFamily + | PersistentVocabARTrieFamily => true + | _ => false + end. + +Definition family_surface_cell + (family : DictionaryFamily) (surface : ConsumerSurfaceClass) + : SurfaceCell := + match surface with + | ReverseLookupSurface => + match family with + | BijectiveMapFamily | PersistentVocabARTrieFamily => + ExistingSurface VocabularyReverseLookupRoute + | _ => SurfaceStructurallyInapplicable + TermIndexHasNoVocabularyReverseLookup + end + | SubstringSurface | SuffixSurface => + match family with + | SuffixAutomatonFamily | ScdawgFamily + | PersistentSuffixAutomatonFamily | PersistentSuffixTreeFamily + | PersistentScdawgFamily => ExistingSurface SuffixIndexRoute + | BijectiveMapFamily | PersistentVocabARTrieFamily => + SurfaceStructurallyInapplicable VocabularyOwnerHasNoSuffixIndex + | _ => SurfaceStructurallyInapplicable + ExactTermFamilyHasNoSubstringIndex + end + | FactorySurface => + match family with + | PersistentARTrieFamily | PersistentSuffixAutomatonFamily + | PersistentSuffixTreeFamily | PersistentScdawgFamily + | PersistentVocabARTrieFamily => + SurfaceStructurallyInapplicable + PersistentConstructionRequiresExplicitStoreConfiguration + | BijectiveMapFamily => ProspectiveSurface CommonDictionaryRoute + | _ => ExistingSurface CommonDictionaryRoute + end + | SerializationReopenSurface => + if persistent_family family then ExistingSurface CommonDictionaryRoute + else ProspectiveSurface CommonDictionaryRoute + | DictionarySurface | DictionaryNodeSurfaceClass + | SnapshotCursorSurfaceClass | SnapshotSurface | PrefixSurface => + ExistingSurface CommonDictionaryRoute + | CollectionSurface => ExistingSurface CommonDictionaryRoute + | ZipperSurfaceClass | SetCombinatorSurface | ValueCombinatorSurface => + match family with + | DynamicDawgFamily | DoubleArrayTrieFamily | SuffixAutomatonFamily + | PathMapAdapterFamily | PersistentARTrieFamily => + ExistingSurface CommonDictionaryRoute + | _ => ProspectiveSurface CommonDictionaryRoute + end + end. + +Inductive CapabilityInapplicability : Type := +| SurfaceCapabilityReason : SurfaceInapplicability -> + CapabilityInapplicability. + +Inductive CapabilityCell : Type := +| ExistingCapability : ProfileRoute -> ExplicitLayoutContract -> + SurfaceRoute -> CapabilityCell +| ProspectiveCapability : ProfileRoute -> ExplicitLayoutContract -> + SurfaceRoute -> CapabilityCell +| CapabilityStructurallyInapplicable : + CapabilityInapplicability -> CapabilityCell. + +Definition family_profile_surface_cell + (family : DictionaryFamily) (profile : FamilyProfile) + (surface : ConsumerSurfaceClass) : CapabilityCell := + match family_profile_cell family profile, + family_surface_cell family surface with + | ExistingProfileCell route layout, ExistingSurface surface_route => + ExistingCapability route layout surface_route + | ExistingProfileCell route layout, ProspectiveSurface surface_route + | ProspectiveProfileCell route layout, ExistingSurface surface_route + | ProspectiveProfileCell route layout, ProspectiveSurface surface_route => + ProspectiveCapability route layout surface_route + | _, SurfaceStructurallyInapplicable reason => + CapabilityStructurallyInapplicable (SurfaceCapabilityReason reason) + end. + +Theorem VWENC_203_DICTIONARY_FAMILY_INVENTORY_IS_EXHAUSTIVE : + length all_dictionary_families = 11 /\ + forall family, In family all_dictionary_families. +Proof. + split; [reflexivity |]. + intros family. destruct family; simpl; tauto. +Qed. + +Theorem VWENC_204_FAMILY_PROFILE_MATRIX_IS_TOTAL_AND_FUNCTIONAL : + length all_family_profiles = 11 /\ + (forall profile, In profile all_family_profiles) /\ + forall family profile, + (exists route layout, + family_profile_cell family profile = + ExistingProfileCell route layout) \/ + (exists route layout, + family_profile_cell family profile = + ProspectiveProfileCell route layout). +Proof. + split; [reflexivity |]. split. + - intros [direct | domain carrier]. + + destruct direct; simpl; tauto. + + destruct domain, carrier; simpl; tauto. + - intros family profile. + destruct (family_profile_cell family profile) as [route layout|route layout] + eqn:Hcell. + + left. now exists route, layout. + + right. now exists route, layout. +Qed. + +Theorem VWENC_205_FAMILY_SURFACE_MATRIX_IS_TOTAL_AND_FUNCTIONAL : + length all_consumer_surfaces = 14 /\ + (forall surface, In surface all_consumer_surfaces) /\ + forall family surface, + (exists route, + family_surface_cell family surface = ExistingSurface route) \/ + (exists route, + family_surface_cell family surface = ProspectiveSurface route) \/ + (exists reason, + family_surface_cell family surface = + SurfaceStructurallyInapplicable reason). +Proof. + split; [reflexivity |]. split. + - intros surface. destruct surface; simpl; tauto. + - intros family surface. + destruct (family_surface_cell family surface) as [route|route|reason] + eqn:Hcell. + + left. now exists route. + + right. left. now exists route. + + right. right. now exists reason. +Qed. + +Theorem VWENC_206_FAMILY_PROFILE_SURFACE_MATRIX_IS_TOTAL : + forall family profile surface, + (exists route layout surface_route, + family_profile_surface_cell family profile surface = + ExistingCapability route layout surface_route) \/ + (exists route layout surface_route, + family_profile_surface_cell family profile surface = + ProspectiveCapability route layout surface_route) \/ + (exists reason, + family_profile_surface_cell family profile surface = + CapabilityStructurallyInapplicable reason). +Proof. + intros family profile surface. + destruct (family_profile_surface_cell family profile surface) as + [route layout surface_route|route layout surface_route|reason] eqn:Hcell. + - left. now exists route, layout, surface_route. + - right. left. now exists route, layout, surface_route. + - right. right. now exists reason. +Qed. + +Theorem VWENC_207_EVERY_INAPPLICABLE_CELL_HAS_AN_EXPLICIT_STRUCTURAL_REASON : + forall family profile surface, + (exists reason, + family_profile_surface_cell family profile surface = + CapabilityStructurallyInapplicable + (SurfaceCapabilityReason reason)) -> + exists reason, + family_surface_cell family surface = + SurfaceStructurallyInapplicable reason. +Proof. + intros family profile surface [reason Hcell]. + unfold family_profile_surface_cell in Hcell. + destruct (family_profile_cell family profile); + destruct (family_surface_cell family surface) eqn:Hsurface; + inversion Hcell; subst; eauto. +Qed. + +Definition pathmap_adapter_route (route : ProfileRoute) : Prop := + match route with + | PathMapNativeByteRoute + | PathMapUtf8BoundaryAdapterRoute + | PathMapFixedWidthBoundaryAdapterRoute + | PathMapInternedIdAdapterRoute _ => True + | _ => False + end. + +Theorem VWENC_208_PATHMAP_REMAINS_AN_EXTERNAL_BYTE_KEYED_ADAPTER : + forall profile, + pathmap_adapter_route + (profile_cell_route + (family_profile_cell PathMapAdapterFamily profile)). +Proof. + intros [direct | domain carrier]. + - destruct direct; exact I. + - destruct domain, carrier; exact I. +Qed. + +Definition family_profile_logical_domain + (profile : FamilyProfile) : option InternedAtomDomain := + match profile with + | DirectProfile _ => None + | InternedProfile domain _ => Some domain + end. + +Theorem VWENC_209_PATHMAP_CANONICAL_ULEB_USES_ONLY_FIXED_WIDTH_INTERNED_IDS : + forall profile, + family_profile_logical_domain profile = Some CanonicalUlebDomain -> + exists carrier, + profile = InternedProfile CanonicalUlebDomain carrier /\ + family_profile_cell PathMapAdapterFamily profile = + ProspectiveProfileCell (PathMapInternedIdAdapterRoute carrier) + (PathMapInternedIdLayout carrier) /\ + carrier_width_bytes (id_carrier_profile carrier) = + match carrier with U32IdCarrier => 4 | U64IdCarrier => 8 end. +Proof. + intros [direct | domain carrier] Hdomain. + - discriminate. + - destruct domain; inversion Hdomain; subst. + exists carrier. repeat split; destruct carrier; reflexivity. +Qed. + +(** ** Naming, persistent identity, and specialization *) + +Inductive FamilyTypeSpelling : Type := +| CanonicalFamilySpelling : DictionaryFamily -> FamilyProfile -> + FamilyTypeSpelling +| LegacyOneParameterSpelling : DictionaryFamily -> FamilyTypeSpelling. + +Definition legacy_default_profile (family : DictionaryFamily) : FamilyProfile := + match family with + | BijectiveMapFamily | PersistentVocabARTrieFamily => + DirectProfile DirectUnicodeScalarDomain + | _ => DirectProfile DirectBytesDomain + end. + +Definition legacy_family_defaults_to_bytes (family : DictionaryFamily) : bool := + match family with + | BijectiveMapFamily | PersistentVocabARTrieFamily => false + | _ => true + end. + +Definition normalize_family_spelling + (spelling : FamilyTypeSpelling) : DictionaryFamily * FamilyProfile := + match spelling with + | CanonicalFamilySpelling family profile => (family, profile) + | LegacyOneParameterSpelling family => + (family, legacy_default_profile family) + end. + +Theorem VWENC_210_LEGACY_ONE_PARAMETER_FAMILY_SPELLING_DEFAULTS_TO_BYTES : + (forall family, + legacy_family_defaults_to_bytes family = true -> + normalize_family_spelling (LegacyOneParameterSpelling family) = + normalize_family_spelling + (CanonicalFamilySpelling family + (DirectProfile DirectBytesDomain))) /\ + normalize_family_spelling (LegacyOneParameterSpelling BijectiveMapFamily) = + (BijectiveMapFamily, DirectProfile DirectUnicodeScalarDomain) /\ + normalize_family_spelling + (LegacyOneParameterSpelling PersistentVocabARTrieFamily) = + (PersistentVocabARTrieFamily, DirectProfile DirectUnicodeScalarDomain). +Proof. + split. + - intros family Hbyte. destruct family; simpl in *; try reflexivity; + discriminate. + - split; reflexivity. +Qed. + +Inductive GenericParameterSlot : Type := +| MappedValueParameterSlot +| LogicalProfileParameterSlot +| RedundantWidthParameterSlot. + +Definition canonical_family_parameter_order : list GenericParameterSlot := + [MappedValueParameterSlot; LogicalProfileParameterSlot]. + +Theorem VWENC_211_MAPPED_VALUE_REMAINS_FIRST_AND_WIDTH_IS_NOT_A_PARAMETER : + hd_error canonical_family_parameter_order = + Some MappedValueParameterSlot /\ + nth_error canonical_family_parameter_order 1 = + Some LogicalProfileParameterSlot /\ + ~ In RedundantWidthParameterSlot canonical_family_parameter_order. +Proof. + split; [reflexivity |]. + split; [reflexivity |]. + simpl. intuition discriminate. +Qed. + +Inductive EdgeUnitKind : Type := +| ByteEdgeUnit +| UnicodeScalarEdgeUnit +| U32EdgeUnit +| U64EdgeUnit +| F64BitsEdgeUnit +| SymbolIdEdgeUnit : IdCarrier -> EdgeUnitKind. + +Definition id_carrier_width (carrier : IdCarrier) : nat := + match carrier with U32IdCarrier => 4 | U64IdCarrier => 8 end. + +Definition profile_edge_contract + (profile : FamilyProfile) : EdgeUnitKind * nat := + match profile with + | DirectProfile DirectBytesDomain => (ByteEdgeUnit, 1) + | DirectProfile DirectUnicodeScalarDomain => (UnicodeScalarEdgeUnit, 4) + | DirectProfile DirectU32Domain => (U32EdgeUnit, 4) + | DirectProfile DirectU64Domain => (U64EdgeUnit, 8) + | DirectProfile DirectF64BitsDomain => (F64BitsEdgeUnit, 8) + | InternedProfile _ carrier => + (SymbolIdEdgeUnit carrier, id_carrier_width carrier) + end. + +Theorem VWENC_212_PROFILE_ALONE_OWNS_EDGE_UNIT_AND_WIDTH_METADATA : + forall profile, + exists! contract, + contract = profile_edge_contract profile /\ + 0 < snd contract. +Proof. + intros profile. + exists (profile_edge_contract profile). split. + - split; [reflexivity |]. + destruct profile as [direct | domain carrier]. + + destruct direct; simpl; lia. + + destruct carrier; simpl; lia. + - intros contract [Hcontract _]. symmetry. exact Hcontract. +Qed. + +Inductive FamilyCodecIdentity : Type := +| FamilyExistingByteCodec +| FamilyExistingCharU32Codec +| FamilyExistingNativeU64Codec +| FamilyEncodedU64LittleEndianBytePathCodec +| FamilyProspectiveU32CodecV1 +| FamilyProspectiveF64BitsCodecV1 +| FamilyFixedIdCarrierCodec : IdCarrier -> FamilyCodecIdentity. + +Definition codec_matches_profile + (profile : FamilyProfile) (codec : FamilyCodecIdentity) : Prop := + match profile, codec with + | DirectProfile DirectBytesDomain, FamilyExistingByteCodec + | DirectProfile DirectUnicodeScalarDomain, FamilyExistingCharU32Codec + | DirectProfile DirectU64Domain, FamilyExistingNativeU64Codec + | DirectProfile DirectU64Domain, + FamilyEncodedU64LittleEndianBytePathCodec + | DirectProfile DirectU32Domain, FamilyProspectiveU32CodecV1 + | DirectProfile DirectF64BitsDomain, FamilyProspectiveF64BitsCodecV1 => True + | InternedProfile _ expected, FamilyFixedIdCarrierCodec actual => + expected = actual + | _, _ => False + end. + +Definition layout_matches_codec + (codec : FamilyCodecIdentity) (layout : ExplicitLayoutContract) : Prop := + match codec, layout with + | FamilyExistingByteCodec, GenericLogicalLayout + | FamilyExistingByteCodec, PathMapNativeByteLayout + | FamilyExistingCharU32Codec, GenericLogicalLayout + | FamilyExistingCharU32Codec, PathMapUtf8BoundaryLayout + | FamilyExistingNativeU64Codec, GenericLogicalLayout + | FamilyExistingNativeU64Codec, PersistentU64CompactLayout + | FamilyExistingNativeU64Codec, + PersistentU64Prefix3CompatibilityLayout + | FamilyEncodedU64LittleEndianBytePathCodec, + EncodedU64ByteCompatibilityLayout + | FamilyProspectiveU32CodecV1, GenericLogicalLayout + | FamilyProspectiveU32CodecV1, PathMapFixedWidthBoundaryLayout + | FamilyProspectiveF64BitsCodecV1, GenericLogicalLayout + | FamilyProspectiveF64BitsCodecV1, PathMapFixedWidthBoundaryLayout => True + | FamilyFixedIdCarrierCodec expected, ProspectiveInternedIdLayout actual + | FamilyFixedIdCarrierCodec expected, PathMapInternedIdLayout actual => + expected = actual + | _, _ => False + end. + +Definition backend_matches_layout + (family : DictionaryFamily) (layout : ExplicitLayoutContract) : Prop := + match layout with + | PathMapNativeByteLayout | PathMapUtf8BoundaryLayout + | PathMapFixedWidthBoundaryLayout | PathMapInternedIdLayout _ => + family = PathMapAdapterFamily + | PersistentU64CompactLayout + | PersistentU64Prefix3CompatibilityLayout + | EncodedU64ByteCompatibilityLayout => + family = PersistentARTrieFamily + | GenericLogicalLayout | ProspectiveInternedIdLayout _ => True + end. + +(** Persistent/ABI certification is intentionally stricter than the abstract + codec/layout compatibility relations above. A format may be certified + only for an existing family/profile cell and its exact declared layout. + The two historical PersistentARTrie U64 byte layouts are explicit, + reviewed compatibility exceptions; prospective cells cannot mint a + persistent identity. *) +Definition family_profile_layout_is_certifiable + (family : DictionaryFamily) (profile : FamilyProfile) + (codec : FamilyCodecIdentity) (layout : ExplicitLayoutContract) : Prop := + (exists route, + family_profile_cell family profile = ExistingProfileCell route layout) \/ + (family = PersistentARTrieFamily /\ + profile = DirectProfile DirectU64Domain /\ + codec = FamilyExistingNativeU64Codec /\ + layout = PersistentU64Prefix3CompatibilityLayout) \/ + (family = PersistentARTrieFamily /\ + profile = DirectProfile DirectU64Domain /\ + codec = FamilyEncodedU64LittleEndianBytePathCodec /\ + layout = EncodedU64ByteCompatibilityLayout). + +Record CertifiedFamilyFormat : Type := { + certified_format_family : DictionaryFamily; + certified_format_profile : FamilyProfile; + certified_format_codec : FamilyCodecIdentity; + certified_format_layout : ExplicitLayoutContract; + certified_format_abi_version : nat; + certified_format_profile_codec_coherent : + codec_matches_profile certified_format_profile certified_format_codec; + certified_format_codec_layout_coherent : + layout_matches_codec certified_format_codec certified_format_layout; + certified_format_backend_layout_coherent : + backend_matches_layout certified_format_family certified_format_layout; + certified_format_family_profile_coherent : + family_profile_layout_is_certifiable + certified_format_family certified_format_profile + certified_format_codec certified_format_layout; + certified_format_version_positive : 0 < certified_format_abi_version +}. + +Definition CertifiedProfileIdentity : Type := + DictionaryFamily * + (FamilyProfile * + (FamilyCodecIdentity * (ExplicitLayoutContract * nat))). + +Definition certified_family_format_identity + (descriptor : CertifiedFamilyFormat) : CertifiedProfileIdentity := + (certified_format_family descriptor, + (certified_format_profile descriptor, + (certified_format_codec descriptor, + (certified_format_layout descriptor, + certified_format_abi_version descriptor)))). + +Inductive ProfileReference : Type := +| OpenInMemoryProfileReference +| CertifiedPersistentProfileReference : CertifiedFamilyFormat -> + ProfileReference. + +Definition persistent_identity_of + (reference : ProfileReference) : option CertifiedProfileIdentity := + match reference with + | OpenInMemoryProfileReference => None + | CertifiedPersistentProfileReference descriptor => + Some (certified_family_format_identity descriptor) + end. + +Theorem VWENC_213_OPEN_IN_MEMORY_UNITS_CANNOT_MINT_PERSISTENT_IDENTITIES : + persistent_identity_of OpenInMemoryProfileReference = None /\ + forall descriptor, + persistent_identity_of + (CertifiedPersistentProfileReference descriptor) = + Some (certified_family_format_identity descriptor) /\ + 0 < certified_format_abi_version descriptor. +Proof. + split; [reflexivity |]. + intros descriptor. split; [reflexivity |]. + exact (certified_format_version_positive descriptor). +Qed. + +(** Rust spelling is diagnostic text only. All semantic fields, including + codec and layout, come from the certified descriptor. *) +Definition format_identity_with_rust_name + (_rust_type_name : list nat) (descriptor : CertifiedFamilyFormat) + : CertifiedProfileIdentity := + certified_family_format_identity descriptor. + +Lemma certified_family_format_identity_injective : + forall left right, + certified_family_format_identity left = + certified_family_format_identity right -> + left = right. +Proof. + intros + [lf lp lc ll lv lpc lcl lbl lfp lvp] + [rf rp rc rl rv rpc rcl rbl rfp rvp] Hequal. + unfold certified_family_format_identity in Hequal. simpl in Hequal. + inversion Hequal. subst. + f_equal; apply proof_irrelevance. +Qed. + +Theorem VWENC_214_FORMAT_IDENTITY_IS_INDEPENDENT_OF_RUST_TYPE_NAMES : + (forall left_name right_name descriptor, + format_identity_with_rust_name left_name descriptor = + format_identity_with_rust_name right_name descriptor) /\ + (forall left right, + certified_family_format_identity left = + certified_family_format_identity right -> + left = right). +Proof. + split; [reflexivity |]. + exact certified_family_format_identity_injective. +Qed. + +Inductive KernelKind : Type := +| GenericLogicalKernel +| SpecializedLogicalKernel +| FixedIdLogicalKernel : IdCarrier -> KernelKind +| EncodedU64AdapterLogicalKernel +| PathMapAdapterKernel +| BijectiveLogicalKernel +| VocabularyOwnerKernel. + +Definition kernel_for_profile_route (route : ProfileRoute) : KernelKind := + match route with + | GenericNativeKernel => GenericLogicalKernel + | RetainedSpecializedKernel => SpecializedLogicalKernel + | InternedFixedIdKernel carrier => FixedIdLogicalKernel carrier + | EncodedU64ByteAdapterKernel => EncodedU64AdapterLogicalKernel + | PathMapNativeByteRoute | PathMapUtf8BoundaryAdapterRoute + | PathMapFixedWidthBoundaryAdapterRoute + | PathMapInternedIdAdapterRoute _ => PathMapAdapterKernel + | BijectiveTermValueKernel => BijectiveLogicalKernel + | VocabularyOwnerRoute _ => VocabularyOwnerKernel + end. + +Definition selected_kernel + (family : DictionaryFamily) (profile : FamilyProfile) : KernelKind := + kernel_for_profile_route (family_profile_route family profile). + +(** A nominal interned profile is usable only together with the certified atom + profile, expected and actual vocabulary fibers, their equality proof, and + the exact immutable vocabulary snapshot. This is a type-level prerequisite + of every family snapshot, rather than an optional hot-path side channel. *) +Definition atom_profile_matches_interned_domain + (domain : InternedAtomDomain) (profile : CertifiedAtomProfile) : Prop := + match domain with + | CanonicalUlebDomain => + persistent_logical_profile (atom_profile_descriptor profile) = + PersistedCanonicalUleb + | CanonicalUtf8Domain => + persistent_logical_profile (atom_profile_descriptor profile) = + PersistedCanonicalUtf8 + | OpaqueCanonicalBytesDomain => True + end. + +Record InternedConsumerContext + (domain : InternedAtomDomain) (carrier : IdCarrier) : Type := { + interned_context_atom_profile : CertifiedAtomProfile; + interned_context_atom_profile_exact : + atom_profile_matches_interned_domain + domain interned_context_atom_profile; + interned_context_expected_fiber : + VocabularyFiber interned_context_atom_profile + (id_carrier_profile carrier); + interned_context_actual_fiber : + VocabularyFiber interned_context_atom_profile + (id_carrier_profile carrier); + interned_context_fiber_exact : + interned_context_expected_fiber = interned_context_actual_fiber; + interned_context_snapshot : + VocabularySnapshot interned_context_atom_profile + (id_carrier_profile carrier) interned_context_actual_fiber +}. + +Definition FamilyConsumerContext (profile : FamilyProfile) : Type := + match profile with + | DirectProfile _ => unit + | InternedProfile domain carrier => InternedConsumerContext domain carrier + end. + +(** Runtime payload is one fixed-width ID plus an erased proof of membership in + the exact bound snapshot. It does not duplicate the arbitrary-width atom. *) +Record SnapshotBoundSymbolId + (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (fiber : VocabularyFiber P I) + (snapshot : VocabularySnapshot P I fiber) : Type := { + snapshot_bound_symbol_id : SymbolId I; + snapshot_bound_live : + live_symbol (vocabulary_snapshot_live_entries P I fiber snapshot) + snapshot_bound_symbol_id +}. + +Definition BoundProfileUnit + (profile : FamilyProfile) (context : FamilyConsumerContext profile) : Type. +Proof. + destruct profile as [domain | domain carrier]. + - exact (DirectUnit domain). + - exact + (SnapshotBoundSymbolId + (interned_context_atom_profile domain carrier context) + (id_carrier_profile carrier) + (interned_context_actual_fiber domain carrier context) + (interned_context_snapshot domain carrier context)). +Defined. + +Definition snapshot_route_allowed + (family : DictionaryFamily) (profile : FamilyProfile) + (kernel : KernelKind) (layout : ExplicitLayoutContract) : Prop := + (kernel = GenericLogicalKernel /\ layout = GenericLogicalLayout) \/ + (kernel = selected_kernel family profile /\ + layout = profile_cell_layout (family_profile_cell family profile)) \/ + (family = PersistentARTrieFamily /\ + profile = DirectProfile DirectU64Domain /\ + kernel = EncodedU64AdapterLogicalKernel /\ + layout = EncodedU64ByteCompatibilityLayout). + +Record FamilySnapshot + (family : DictionaryFamily) (profile : FamilyProfile) + (context : FamilyConsumerContext profile) (Value : Type) : Type := { + snapshot_revision : nat; + snapshot_kernel : KernelKind; + snapshot_layout : ExplicitLayoutContract; + snapshot_observations : LogicalObservations + (BoundProfileUnit profile context) Value; + snapshot_route_certificate : + snapshot_route_allowed family profile snapshot_kernel snapshot_layout +}. + +(** Downstream [CharUnit]-like implementations remain open in memory. They + share the same family observation contract, but their profile reference is + definitionally non-persistent and therefore cannot be mistaken for a + certified ABI identity. *) +Record OpenFamilySnapshot + (family : DictionaryFamily) (Unit Value : Type) + (profile : OpenUnitProfile Unit) : Type := { + open_snapshot_revision : nat; + open_snapshot_observations : LogicalObservations Unit Value +}. + +Definition open_family_snapshot_profile_reference + {family Unit Value} {profile : OpenUnitProfile Unit} + (_snapshot : OpenFamilySnapshot family Unit Value profile) + : ProfileReference := OpenInMemoryProfileReference. + +Lemma open_family_snapshots_cannot_mint_persistent_identity : + forall family Unit Value (profile : OpenUnitProfile Unit) + (snapshot : OpenFamilySnapshot family Unit Value profile), + persistent_identity_of + (open_family_snapshot_profile_reference snapshot) = None. +Proof. reflexivity. Qed. + +Record OpenFamilySpecializationRefinement + (family : DictionaryFamily) (Unit Value : Type) + (profile : OpenUnitProfile Unit) + (generic specialized : OpenFamilySnapshot family Unit Value profile) + : Prop := { + open_specialization_same_revision : + open_snapshot_revision _ _ _ _ generic = + open_snapshot_revision _ _ _ _ specialized; + open_specialization_same_observations : + SameLogicalObservations + (open_snapshot_observations _ _ _ _ generic) + (open_snapshot_observations _ _ _ _ specialized) +}. + +Record SpecializationRefinement + (family : DictionaryFamily) (profile : FamilyProfile) + (context : FamilyConsumerContext profile) (Value : Type) + (generic specialized : FamilySnapshot family profile context Value) : Prop := { + specialization_generic_kernel : + snapshot_kernel family profile context Value generic = GenericLogicalKernel; + specialization_selected_kernel : + snapshot_kernel family profile context Value specialized = + selected_kernel family profile; + specialization_same_revision : + snapshot_revision family profile context Value generic = + snapshot_revision family profile context Value specialized; + specialization_same_observations : + SameLogicalObservations + (snapshot_observations family profile context Value generic) + (snapshot_observations family profile context Value specialized) +}. + +Theorem VWENC_215_SPECIALIZATION_REFINES_THE_GENERIC_LOGICAL_VIEW : + forall family profile (context : FamilyConsumerContext profile) (Value : Type) + (generic specialized : FamilySnapshot family profile context Value), + SpecializationRefinement family profile context Value generic specialized -> + snapshot_revision family profile context Value generic = + snapshot_revision family profile context Value specialized /\ + snapshot_kernel family profile context Value generic = GenericLogicalKernel /\ + snapshot_kernel family profile context Value specialized = + selected_kernel family profile /\ + SameLogicalObservations + (snapshot_observations family profile context Value generic) + (snapshot_observations family profile context Value specialized). +Proof. + intros family profile context Value generic specialized Hrefinement. + split. + - exact (specialization_same_revision _ _ _ _ _ _ Hrefinement). + - split. + + exact (specialization_generic_kernel _ _ _ _ _ _ Hrefinement). + + split. + * exact (specialization_selected_kernel _ _ _ _ _ _ Hrefinement). + * exact (specialization_same_observations _ _ _ _ _ _ Hrefinement). +Qed. + +Theorem VWENC_216_EVERY_RETAINED_SPECIALIZED_KERNEL_PRESERVES_ALL_OBSERVATIONS : + forall family profile (context : FamilyConsumerContext profile) (Value : Type) + (generic specialized : FamilySnapshot family profile context Value), + family_profile_route family profile = RetainedSpecializedKernel -> + SpecializationRefinement family profile context Value generic specialized -> + snapshot_kernel family profile context Value specialized = + SpecializedLogicalKernel /\ + SameLogicalObservations + (snapshot_observations family profile context Value generic) + (snapshot_observations family profile context Value specialized). +Proof. + intros family profile context Value generic specialized Hroute Hrefinement. + split. + - rewrite (specialization_selected_kernel _ _ _ _ _ _ Hrefinement). + unfold selected_kernel. now rewrite Hroute. + - exact (specialization_same_observations _ _ _ _ _ _ Hrefinement). +Qed. + +(** A traversal batch is homogeneous in [Unit]. Kernel selection happens when + this record is built; its encoder cannot inspect a profile tag per edge. *) +Record MonomorphicFixedWidthKernel (Unit : Type) : Type := { + monomorphic_width : nat; + monomorphic_width_positive : 0 < monomorphic_width; + monomorphic_encode : Unit -> list PhysicalByte; + monomorphic_encode_exact : + forall unit, length (monomorphic_encode unit) = monomorphic_width; + monomorphic_variable_decode_request : Unit -> option (list PhysicalByte); + monomorphic_has_no_variable_decode : + forall unit, monomorphic_variable_decode_request unit = None +}. + +Definition run_bound_kernel {Unit : Type} + (kernel : MonomorphicFixedWidthKernel Unit) (units : list Unit) + : list (list PhysicalByte) := + map (monomorphic_encode Unit kernel) units. + +Lemma bound_kernel_widths_are_constant : + forall (Unit : Type) (kernel : MonomorphicFixedWidthKernel Unit) units, + map (@length PhysicalByte) (run_bound_kernel kernel units) = + repeat (monomorphic_width Unit kernel) (length units). +Proof. + intros Unit kernel units. induction units as [|unit rest IH]; simpl. + - reflexivity. + - rewrite (monomorphic_encode_exact Unit kernel unit), IH. reflexivity. +Qed. + +Theorem VWENC_217_KERNEL_SELECTION_IS_BOUND_ONCE_NOT_BRANCHING_PER_EDGE : + forall (Unit : Type) (kernel : MonomorphicFixedWidthKernel Unit) units, + run_bound_kernel kernel units = + map (monomorphic_encode Unit kernel) units /\ + map (@length PhysicalByte) (run_bound_kernel kernel units) = + repeat (monomorphic_width Unit kernel) (length units). +Proof. + intros. split; [reflexivity |]. + apply bound_kernel_widths_are_constant. +Qed. + +(** ** Backward-compatible aliases and independent projections *) + +Inductive LegacyAlias : Type := +| LegacyDynamicDawg | LegacyDoubleArrayTrie | LegacySuffixAutomaton +| LegacyScdawg | LegacyPathMapDictionary | LegacyPersistentARTrie +| LegacyPersistentSuffixAutomaton | LegacyPersistentSuffixTree +| LegacyPersistentScdawg +| LegacyDynamicDawgChar | LegacyDoubleArrayTrieChar +| LegacySuffixAutomatonChar | LegacyScdawgChar +| LegacyPathMapDictionaryChar | LegacyPersistentARTrieChar +| LegacyPersistentSuffixAutomatonChar | LegacyPersistentSuffixTreeChar +| LegacyPersistentScdawgChar +| LegacyDynamicDawgU64 | LegacyPersistentARTrieU64 +| LegacyPersistentARTrieU64Compact +| LegacyPersistentARTrieU64Prefix3Compat +| LegacyEncodedPersistentARTrieU64. + +Definition all_legacy_aliases : list LegacyAlias := + [LegacyDynamicDawg; LegacyDoubleArrayTrie; LegacySuffixAutomaton; + LegacyScdawg; LegacyPathMapDictionary; LegacyPersistentARTrie; + LegacyPersistentSuffixAutomaton; LegacyPersistentSuffixTree; + LegacyPersistentScdawg; LegacyDynamicDawgChar; + LegacyDoubleArrayTrieChar; LegacySuffixAutomatonChar; + LegacyScdawgChar; LegacyPathMapDictionaryChar; + LegacyPersistentARTrieChar; LegacyPersistentSuffixAutomatonChar; + LegacyPersistentSuffixTreeChar; LegacyPersistentScdawgChar; + LegacyDynamicDawgU64; LegacyPersistentARTrieU64; + LegacyPersistentARTrieU64Compact; + LegacyPersistentARTrieU64Prefix3Compat; + LegacyEncodedPersistentARTrieU64]. + +Definition legacy_alias_family (alias : LegacyAlias) : DictionaryFamily := + match alias with + | LegacyDynamicDawg | LegacyDynamicDawgChar | LegacyDynamicDawgU64 => + DynamicDawgFamily + | LegacyDoubleArrayTrie | LegacyDoubleArrayTrieChar => + DoubleArrayTrieFamily + | LegacySuffixAutomaton | LegacySuffixAutomatonChar => + SuffixAutomatonFamily + | LegacyScdawg | LegacyScdawgChar => ScdawgFamily + | LegacyPathMapDictionary | LegacyPathMapDictionaryChar => + PathMapAdapterFamily + | LegacyPersistentARTrie | LegacyPersistentARTrieChar + | LegacyPersistentARTrieU64 | LegacyPersistentARTrieU64Compact + | LegacyPersistentARTrieU64Prefix3Compat + | LegacyEncodedPersistentARTrieU64 => PersistentARTrieFamily + | LegacyPersistentSuffixAutomaton + | LegacyPersistentSuffixAutomatonChar => PersistentSuffixAutomatonFamily + | LegacyPersistentSuffixTree | LegacyPersistentSuffixTreeChar => + PersistentSuffixTreeFamily + | LegacyPersistentScdawg | LegacyPersistentScdawgChar => + PersistentScdawgFamily + end. + +Inductive LegacyAliasClass : Type := +| LegacyByteClass | LegacyCharClass | LegacyU64Class. + +Definition legacy_alias_class (alias : LegacyAlias) : LegacyAliasClass := + match alias with + | LegacyDynamicDawgChar | LegacyDoubleArrayTrieChar + | LegacySuffixAutomatonChar | LegacyScdawgChar + | LegacyPathMapDictionaryChar | LegacyPersistentARTrieChar + | LegacyPersistentSuffixAutomatonChar | LegacyPersistentSuffixTreeChar + | LegacyPersistentScdawgChar => LegacyCharClass + | LegacyDynamicDawgU64 | LegacyPersistentARTrieU64 + | LegacyPersistentARTrieU64Compact + | LegacyPersistentARTrieU64Prefix3Compat + | LegacyEncodedPersistentARTrieU64 => LegacyU64Class + | _ => LegacyByteClass + end. + +Definition legacy_alias_profile (alias : LegacyAlias) : FamilyProfile := + match legacy_alias_class alias with + | LegacyByteClass => DirectProfile DirectBytesDomain + | LegacyCharClass => DirectProfile DirectUnicodeScalarDomain + | LegacyU64Class => DirectProfile DirectU64Domain + end. + +Definition legacy_alias_layout + (alias : LegacyAlias) : ExplicitLayoutContract := + match alias with + | LegacyPathMapDictionary => PathMapNativeByteLayout + | LegacyPathMapDictionaryChar => PathMapUtf8BoundaryLayout + | LegacyPersistentARTrieU64 | LegacyPersistentARTrieU64Compact => + PersistentU64CompactLayout + | LegacyPersistentARTrieU64Prefix3Compat => + PersistentU64Prefix3CompatibilityLayout + | LegacyEncodedPersistentARTrieU64 => + EncodedU64ByteCompatibilityLayout + | _ => GenericLogicalLayout + end. + +Definition legacy_alias_route (alias : LegacyAlias) : ProfileRoute := + match alias with + | LegacyEncodedPersistentARTrieU64 => EncodedU64ByteAdapterKernel + | _ => family_profile_route + (legacy_alias_family alias) (legacy_alias_profile alias) + end. + +Definition legacy_alias_codec (alias : LegacyAlias) : FamilyCodecIdentity := + match legacy_alias_class alias with + | LegacyByteClass => FamilyExistingByteCodec + | LegacyCharClass => FamilyExistingCharU32Codec + | LegacyU64Class => + match alias with + | LegacyEncodedPersistentARTrieU64 => + FamilyEncodedU64LittleEndianBytePathCodec + | _ => FamilyExistingNativeU64Codec + end + end. + +Inductive VocabularyAlias : Type := +| PersistentVocabARTrieName +| SharedVocabARTrieName +| IndexedVocabularyPersistentName +| SharedVocabTrieName +| DiskBackedVocabTrieInnerName. + +Definition all_vocabulary_aliases : list VocabularyAlias := + [PersistentVocabARTrieName; SharedVocabARTrieName; + IndexedVocabularyPersistentName; SharedVocabTrieName; + DiskBackedVocabTrieInnerName]. + +Inductive PersistentHandleAlias : Type := +| SharedARTrieName +| SharedCharARTrieName +| SharedCharTrieName. + +Definition all_persistent_handle_aliases : list PersistentHandleAlias := + [SharedARTrieName; SharedCharARTrieName; SharedCharTrieName]. + +Inductive PublicCompatibilitySpelling : Type := +| LegacyPublicSpelling : LegacyAlias -> PublicCompatibilitySpelling +| VocabularyPublicSpelling : VocabularyAlias -> PublicCompatibilitySpelling +| PersistentHandlePublicSpelling : PersistentHandleAlias -> + PublicCompatibilitySpelling. + +Definition public_spelling_family + (spelling : PublicCompatibilitySpelling) : DictionaryFamily := + match spelling with + | LegacyPublicSpelling alias => legacy_alias_family alias + | VocabularyPublicSpelling _ => PersistentVocabARTrieFamily + | PersistentHandlePublicSpelling _ => PersistentARTrieFamily + end. + +Definition public_spelling_profile + (spelling : PublicCompatibilitySpelling) : FamilyProfile := + match spelling with + | LegacyPublicSpelling alias => legacy_alias_profile alias + | VocabularyPublicSpelling _ => DirectProfile DirectUnicodeScalarDomain + | PersistentHandlePublicSpelling SharedARTrieName => + DirectProfile DirectBytesDomain + | PersistentHandlePublicSpelling SharedCharARTrieName + | PersistentHandlePublicSpelling SharedCharTrieName => + DirectProfile DirectUnicodeScalarDomain + end. + +Definition public_spelling_layout + (spelling : PublicCompatibilitySpelling) : ExplicitLayoutContract := + match spelling with + | LegacyPublicSpelling alias => legacy_alias_layout alias + | _ => GenericLogicalLayout + end. + +Definition public_spelling_route + (spelling : PublicCompatibilitySpelling) : ProfileRoute := + match spelling with + | LegacyPublicSpelling alias => legacy_alias_route alias + | VocabularyPublicSpelling _ => VocabularyOwnerRoute U64IdCarrier + | PersistentHandlePublicSpelling _ => + family_profile_route + (public_spelling_family spelling) (public_spelling_profile spelling) + end. + +Definition public_spelling_codec + (spelling : PublicCompatibilitySpelling) : FamilyCodecIdentity := + match spelling with + | LegacyPublicSpelling alias => legacy_alias_codec alias + | VocabularyPublicSpelling _ => FamilyExistingCharU32Codec + | PersistentHandlePublicSpelling SharedARTrieName => + FamilyExistingByteCodec + | PersistentHandlePublicSpelling SharedCharARTrieName + | PersistentHandlePublicSpelling SharedCharTrieName => + FamilyExistingCharU32Codec + end. + +Definition public_spelling_is_persistent + (spelling : PublicCompatibilitySpelling) : bool := + persistent_family (public_spelling_family spelling). + +Definition expected_public_format_identity + (spelling : PublicCompatibilitySpelling) + : option CertifiedProfileIdentity := + if public_spelling_is_persistent spelling then + Some + (public_spelling_family spelling, + (public_spelling_profile spelling, + (public_spelling_codec spelling, + (public_spelling_layout spelling, 1)))) + else None. + +Definition public_spelling_context + (spelling : PublicCompatibilitySpelling) + : FamilyConsumerContext (public_spelling_profile spelling). +Proof. + destruct spelling as [alias | vocabulary | handle]. + - destruct alias; exact tt. + - exact tt. + - destruct handle; exact tt. +Defined. + +Record PublicFacadeState + (spelling : PublicCompatibilitySpelling) (Value : Type) : Type := { + facade_snapshot : + FamilySnapshot (public_spelling_family spelling) + (public_spelling_profile spelling) (public_spelling_context spelling) Value; + facade_profile_reference : ProfileReference; + facade_serialized_image : option (list PhysicalByte) +}. + +Record PublicFacadeCompatibility + (spelling : PublicCompatibilitySpelling) (Value : Type) + (legacy canonical : PublicFacadeState spelling Value) : Prop := { + facade_legacy_layout_exact : + snapshot_layout _ _ _ _ (facade_snapshot spelling Value legacy) = + public_spelling_layout spelling; + facade_canonical_layout_exact : + snapshot_layout _ _ _ _ (facade_snapshot spelling Value canonical) = + public_spelling_layout spelling; + facade_legacy_kernel_exact : + snapshot_kernel _ _ _ _ (facade_snapshot spelling Value legacy) = + kernel_for_profile_route (public_spelling_route spelling); + facade_canonical_kernel_exact : + snapshot_kernel _ _ _ _ (facade_snapshot spelling Value canonical) = + kernel_for_profile_route (public_spelling_route spelling); + facade_same_revision : + snapshot_revision _ _ _ _ (facade_snapshot spelling Value legacy) = + snapshot_revision _ _ _ _ (facade_snapshot spelling Value canonical); + facade_same_observations : + SameLogicalObservations + (snapshot_observations _ _ _ _ (facade_snapshot spelling Value legacy)) + (snapshot_observations _ _ _ _ (facade_snapshot spelling Value canonical)); + facade_legacy_format_exact : + persistent_identity_of (facade_profile_reference spelling Value legacy) = + expected_public_format_identity spelling; + facade_canonical_format_exact : + persistent_identity_of + (facade_profile_reference spelling Value canonical) = + expected_public_format_identity spelling; + facade_serialization_exact : + facade_serialized_image spelling Value legacy = + facade_serialized_image spelling Value canonical +}. + +Theorem VWENC_218_LEGACY_ALIAS_INVENTORIES_PRESERVE_CANONICAL_TARGETS : + length all_legacy_aliases = 23 /\ + (forall alias, In alias all_legacy_aliases) /\ + length all_vocabulary_aliases = 5 /\ + (forall alias, In alias all_vocabulary_aliases) /\ + length all_persistent_handle_aliases = 3 /\ + (forall alias, In alias all_persistent_handle_aliases) /\ + forall spelling (Value : Type) + (legacy canonical : PublicFacadeState spelling Value), + PublicFacadeCompatibility spelling Value legacy canonical -> + SameLogicalObservations + (snapshot_observations _ _ _ _ (facade_snapshot spelling Value legacy)) + (snapshot_observations _ _ _ _ (facade_snapshot spelling Value canonical)) /\ + facade_serialized_image spelling Value legacy = + facade_serialized_image spelling Value canonical /\ + persistent_identity_of (facade_profile_reference spelling Value legacy) = + expected_public_format_identity spelling /\ + persistent_identity_of + (facade_profile_reference spelling Value canonical) = + expected_public_format_identity spelling. +Proof. + split; [reflexivity |]. split. + - intros alias. destruct alias; simpl; tauto. + - split; [reflexivity |]. split. + + intros alias. destruct alias; simpl; tauto. + + split; [reflexivity |]. split. + * intros alias. destruct alias; simpl; tauto. + * intros spelling Value legacy canonical Hcompatibility. + split. + -- exact (facade_same_observations _ _ _ _ Hcompatibility). + -- split. + ++ exact (facade_serialization_exact _ _ _ _ Hcompatibility). + ++ split. + ** exact (facade_legacy_format_exact _ _ _ _ Hcompatibility). + ** exact (facade_canonical_format_exact _ _ _ _ Hcompatibility). +Qed. + +Theorem VWENC_219_EVERY_CHAR_ALIAS_TARGETS_UNICODE_SCALAR_UNITS : + forall alias, + legacy_alias_class alias = LegacyCharClass -> + legacy_alias_profile alias = DirectProfile DirectUnicodeScalarDomain /\ + ProfileUnit (legacy_alias_profile alias) = + DirectUnit DirectUnicodeScalarDomain. +Proof. + intros alias Hclass. unfold legacy_alias_profile. rewrite Hclass. + split; reflexivity. +Qed. + +Definition direct_unit_value {domain : DirectUnitDomain} + (unit : DirectUnit domain) : nat := proj1_sig unit. + +Definition encoded_u64_unit_bytes + (unit : DirectUnit DirectU64Domain) : list PhysicalByte := + encode_fixed_little_endian 8 (direct_unit_value unit). + +Fixpoint legacy_encoded_u64_sequence + (units : list (DirectUnit DirectU64Domain)) : list PhysicalByte := + match units with + | [] => [] + | unit :: rest => encoded_u64_unit_bytes unit ++ + legacy_encoded_u64_sequence rest + end. + +Definition canonical_encoded_u64_sequence + (units : list (DirectUnit DirectU64Domain)) : list PhysicalByte := + concat (map encoded_u64_unit_bytes units). + +Definition encoded_u64_logical_edges + (units : list (DirectUnit DirectU64Domain)) := + map (fun unit => [unit]) units. + +Lemma legacy_and_canonical_encoded_u64_sequences_are_equal : + forall units, + legacy_encoded_u64_sequence units = canonical_encoded_u64_sequence units. +Proof. + induction units as [|unit rest IH]; simpl; [reflexivity |]. + now rewrite IH. +Qed. + +Lemma encoded_u64_sequence_has_exact_physical_width : + forall units, + length (canonical_encoded_u64_sequence units) = 8 * length units. +Proof. + induction units as [|unit rest IH]; [reflexivity |]. + change + (length (encoded_u64_unit_bytes unit ++ + canonical_encoded_u64_sequence rest) = 8 * S (length rest)). + rewrite app_length, IH. + assert (Hunit : length (encoded_u64_unit_bytes unit) = 8). + { unfold encoded_u64_unit_bytes. apply fixed_little_endian_length. } + rewrite Hunit. lia. +Qed. + +Definition u64_alias_layout_and_route_are_exact (alias : LegacyAlias) : Prop := + match alias with + | LegacyDynamicDawgU64 => + legacy_alias_layout alias = GenericLogicalLayout /\ + legacy_alias_route alias = RetainedSpecializedKernel + | LegacyPersistentARTrieU64 | LegacyPersistentARTrieU64Compact => + legacy_alias_layout alias = PersistentU64CompactLayout /\ + legacy_alias_route alias = RetainedSpecializedKernel + | LegacyPersistentARTrieU64Prefix3Compat => + legacy_alias_layout alias = + PersistentU64Prefix3CompatibilityLayout /\ + legacy_alias_route alias = RetainedSpecializedKernel + | LegacyEncodedPersistentARTrieU64 => + legacy_alias_layout alias = EncodedU64ByteCompatibilityLayout /\ + legacy_alias_route alias = EncodedU64ByteAdapterKernel + | _ => True + end. + +Theorem VWENC_220_EVERY_U64_ALIAS_PRESERVES_PROFILE_AND_EXPLICIT_LAYOUT : + (forall alias, + legacy_alias_class alias = LegacyU64Class -> + legacy_alias_profile alias = DirectProfile DirectU64Domain /\ + u64_alias_layout_and_route_are_exact alias) /\ + (forall units, + legacy_encoded_u64_sequence units = + canonical_encoded_u64_sequence units /\ + length (canonical_encoded_u64_sequence units) = 8 * length units /\ + length (encoded_u64_logical_edges units) = length units). +Proof. + split. + - intros alias Hclass. split. + + unfold legacy_alias_profile. now rewrite Hclass. + + destruct alias; simpl in *; try discriminate; repeat split; reflexivity. + - intros units. repeat split. + + apply legacy_and_canonical_encoded_u64_sequences_are_equal. + + apply encoded_u64_sequence_has_exact_physical_width. + + apply map_length. +Qed. + +Record DynamicToDatConversion + (profile : FamilyProfile) (context : FamilyConsumerContext profile) + (Value : Type) : Type := { + conversion_dynamic_source : + FamilySnapshot DynamicDawgFamily profile context Value; + conversion_dat_target : + FamilySnapshot DoubleArrayTrieFamily profile context Value; + conversion_same_revision : + snapshot_revision _ _ _ _ conversion_dynamic_source = + snapshot_revision _ _ _ _ conversion_dat_target; + conversion_same_observations : + SameLogicalObservations + (snapshot_observations _ _ _ _ conversion_dynamic_source) + (snapshot_observations _ _ _ _ conversion_dat_target) +}. + +Theorem VWENC_221_DYNAMIC_TO_FROZEN_CONVERSION_PRESERVES_LOGICAL_OBSERVATIONS : + forall profile (context : FamilyConsumerContext profile) (Value : Type) + (conversion : DynamicToDatConversion profile context Value), + snapshot_revision _ _ _ _ (conversion_dynamic_source _ _ _ conversion) = + snapshot_revision _ _ _ _ (conversion_dat_target _ _ _ conversion) /\ + SameLogicalObservations + (snapshot_observations _ _ _ _ + (conversion_dynamic_source _ _ _ conversion)) + (snapshot_observations _ _ _ _ + (conversion_dat_target _ _ _ conversion)). +Proof. + intros profile context Value conversion. split. + - exact (conversion_same_revision _ _ _ conversion). + - exact (conversion_same_observations _ _ _ conversion). +Qed. + +Record TraversalProjectionBundle + (family : DictionaryFamily) (profile : FamilyProfile) + (context : FamilyConsumerContext profile) (Value : Type) + : Type := { + projection_revision : nat; + projection_reference_view : + LogicalObservations (BoundProfileUnit profile context) Value; + projection_node_view : + LogicalObservations (BoundProfileUnit profile context) Value; + projection_zipper_view : + LogicalObservations (BoundProfileUnit profile context) Value; + projection_cursor_view : + LogicalObservations (BoundProfileUnit profile context) Value; + projection_node_revision : nat; + projection_zipper_revision : nat; + projection_cursor_revision : nat; + projection_node_revision_exact : projection_node_revision = projection_revision; + projection_zipper_revision_exact : projection_zipper_revision = projection_revision; + projection_cursor_revision_exact : projection_cursor_revision = projection_revision; + projection_node_refines_reference : + SameLogicalObservations projection_node_view projection_reference_view; + projection_zipper_refines_reference : + SameLogicalObservations projection_zipper_view projection_reference_view; + projection_cursor_refines_reference : + SameLogicalObservations projection_cursor_view projection_reference_view +}. + +Theorem VWENC_222_NODE_ZIPPER_AND_CURSOR_SHARE_ONE_REVISION_BOUND_VIEW : + forall family profile (context : FamilyConsumerContext profile) (Value : Type) + (bundle : TraversalProjectionBundle family profile context Value), + projection_node_revision _ _ _ _ bundle = + projection_zipper_revision _ _ _ _ bundle /\ + projection_zipper_revision _ _ _ _ bundle = + projection_cursor_revision _ _ _ _ bundle /\ + SameLogicalObservations + (projection_node_view _ _ _ _ bundle) + (projection_zipper_view _ _ _ _ bundle) /\ + SameLogicalObservations + (projection_zipper_view _ _ _ _ bundle) + (projection_cursor_view _ _ _ _ bundle). +Proof. + intros family profile context Value bundle. split. + - rewrite (projection_node_revision_exact _ _ _ _ bundle), + (projection_zipper_revision_exact _ _ _ _ bundle). reflexivity. + - split. + + rewrite (projection_zipper_revision_exact _ _ _ _ bundle), + (projection_cursor_revision_exact _ _ _ _ bundle). reflexivity. + + split. + * eapply same_logical_observations_transitive with + (middle := projection_reference_view + family profile context Value bundle). + -- exact (projection_node_refines_reference _ _ _ _ bundle). + -- apply same_logical_observations_symmetric. + exact (projection_zipper_refines_reference _ _ _ _ bundle). + * eapply same_logical_observations_transitive with + (middle := projection_reference_view + family profile context Value bundle). + -- exact (projection_zipper_refines_reference _ _ _ _ bundle). + -- apply same_logical_observations_symmetric. + exact (projection_cursor_refines_reference _ _ _ _ bundle). +Qed. + +Definition surface_is_available + (family : DictionaryFamily) (surface : ConsumerSurfaceClass) : Prop := + (exists route, family_surface_cell family surface = ExistingSurface route) \/ + (exists route, family_surface_cell family surface = ProspectiveSurface route). + +Record LifecycleRefinement + (family : DictionaryFamily) (profile : FamilyProfile) + (context : FamilyConsumerContext profile) (Value : Type) + (surface : ConsumerSurfaceClass) + : Type := { + lifecycle_surface_available : surface_is_available family surface; + lifecycle_source : FamilySnapshot family profile context Value; + lifecycle_product : FamilySnapshot family profile context Value; + lifecycle_product_refines : + SameLogicalObservations + (snapshot_observations _ _ _ _ lifecycle_source) + (snapshot_observations _ _ _ _ lifecycle_product); + lifecycle_revision_exact : + snapshot_revision _ _ _ _ lifecycle_source = + snapshot_revision _ _ _ _ lifecycle_product +}. + +Theorem VWENC_223_FACTORY_COLLECTION_AND_SERIALIZATION_PRESERVE_PROFILE_VIEW : + forall family profile (context : FamilyConsumerContext profile) + (Value : Type) surface + (lifecycle : LifecycleRefinement + family profile context Value surface), + In surface + [FactorySurface; CollectionSurface; SerializationReopenSurface] -> + surface_is_available family surface /\ + SameLogicalObservations + (snapshot_observations _ _ _ _ + (lifecycle_source _ _ _ _ _ lifecycle)) + (snapshot_observations _ _ _ _ + (lifecycle_product _ _ _ _ _ lifecycle)) /\ + snapshot_revision _ _ _ _ + (lifecycle_source _ _ _ _ _ lifecycle) = + snapshot_revision _ _ _ _ + (lifecycle_product _ _ _ _ _ lifecycle). +Proof. + intros family profile context Value surface lifecycle _. + split. + - exact (lifecycle_surface_available _ _ _ _ _ lifecycle). + - split. + + exact (lifecycle_product_refines _ _ _ _ _ lifecycle). + + exact (lifecycle_revision_exact _ _ _ _ _ lifecycle). +Qed. + +Record ExtensionalSetCombinator (Atom Value : Type) : Type := { + combine_set_views : + LogicalObservations Atom Value -> LogicalObservations Atom Value -> + LogicalObservations Atom Value; + combine_set_views_extensional : + forall left left_refined right right_refined, + SameLogicalObservations left left_refined -> + SameLogicalObservations right right_refined -> + SameLogicalObservations + (combine_set_views left right) + (combine_set_views left_refined right_refined) +}. + +Theorem VWENC_224_SET_COMBINATORS_COMMUTE_WITH_PROFILE_REFINEMENT : + forall (Atom Value : Type) (combine : ExtensionalSetCombinator Atom Value) + left left_refined right right_refined, + SameLogicalObservations left left_refined -> + SameLogicalObservations right right_refined -> + SameLogicalObservations + (combine_set_views Atom Value combine left right) + (combine_set_views Atom Value combine left_refined right_refined). +Proof. intros. now apply combine_set_views_extensional. Qed. + +Record ExtensionalValueCombinator (Atom Value : Type) : Type := { + combine_value_views : + LogicalObservations Atom Value -> LogicalObservations Atom Value -> + LogicalObservations Atom Value; + combine_value_views_extensional : + forall left left_refined right right_refined, + SameLogicalObservations left left_refined -> + SameLogicalObservations right right_refined -> + SameLogicalObservations + (combine_value_views left right) + (combine_value_views left_refined right_refined) +}. + +Theorem VWENC_225_VALUE_COMBINATORS_COMMUTE_WITH_PROFILE_REFINEMENT : + forall (Atom Value : Type) + (combine : ExtensionalValueCombinator Atom Value) + left left_refined right right_refined, + SameLogicalObservations left left_refined -> + SameLogicalObservations right right_refined -> + SameLogicalObservations + (combine_value_views Atom Value combine left right) + (combine_value_views Atom Value combine left_refined right_refined). +Proof. intros. now apply combine_value_views_extensional. Qed. + +(** ** Encoded adapters and logical suffix boundaries *) + +Inductive PathMapTraceEvent : Type := +| PathMapPhysicalByteVisited : PhysicalByte -> PathMapTraceEvent +| PathMapLogicalAtomEmitted : LogicalAtom -> PathMapTraceEvent. + +Definition pathmap_physical_trace + (stored : StoredLogicalUnit) (atom : LogicalAtom) + : list PathMapTraceEvent := + map PathMapPhysicalByteVisited (physical_codeword_of stored) ++ + [PathMapLogicalAtomEmitted atom]. + +Fixpoint pathmap_logical_projection + (trace : list PathMapTraceEvent) : list LogicalAtom := + match trace with + | [] => [] + | PathMapPhysicalByteVisited _ :: rest => pathmap_logical_projection rest + | PathMapLogicalAtomEmitted atom :: rest => + atom :: pathmap_logical_projection rest + end. + +Lemma pathmap_logical_projection_app : + forall left right, + pathmap_logical_projection (left ++ right) = + pathmap_logical_projection left ++ pathmap_logical_projection right. +Proof. + induction left as [|event rest IH]; intros right; simpl; [reflexivity |]. + destruct event; simpl; now rewrite IH. +Qed. + +Lemma pathmap_physical_prefix_projects_to_no_logical_atoms : + forall bytes, + pathmap_logical_projection (map PathMapPhysicalByteVisited bytes) = []. +Proof. + induction bytes; simpl; auto. +Qed. + +Lemma pathmap_trace_projects_exactly_one_atom : + forall stored atom, + pathmap_logical_projection (pathmap_physical_trace stored atom) = [atom]. +Proof. + intros stored atom. unfold pathmap_physical_trace. + rewrite pathmap_logical_projection_app, + pathmap_physical_prefix_projects_to_no_logical_atoms. + reflexivity. +Qed. + +Definition pathmap_node_projection := pathmap_logical_projection. +Definition pathmap_zipper_projection := pathmap_logical_projection. +Definition pathmap_snapshot_projection := pathmap_logical_projection. + +Theorem VWENC_226_ENCODED_ADAPTER_STAGING_BYTES_ARE_HIDDEN_FROM_CONSUMERS : + forall surface stored atom, + representation_admits EncodedBytePathAdapter stored -> + decode_stored_logical_unit stored = Some atom -> + consumer_observation surface + {| transition_representation := EncodedBytePathAdapter; + transition_unit := stored |} = [atom] /\ + length + (consumer_observation surface + {| transition_representation := EncodedBytePathAdapter; + transition_unit := stored |}) = 1 /\ + pathmap_node_projection (pathmap_physical_trace stored atom) = [atom] /\ + pathmap_zipper_projection (pathmap_physical_trace stored atom) = [atom] /\ + pathmap_snapshot_projection (pathmap_physical_trace stored atom) = [atom]. +Proof. + intros surface stored atom Hadmits Hdecode. + assert (Hvalid : + valid_stored_transition + {| transition_representation := EncodedBytePathAdapter; + transition_unit := stored |}). + { split; [exact Hadmits |]. exists atom. exact Hdecode. } + repeat split. + - apply VWENC_16_CODEC_BYTES_ARE_NOT_LOGICAL_TRANSITIONS. + change + ((if representation_admitsb EncodedBytePathAdapter stored + then decode_stored_logical_unit stored else None) = Some atom). + apply representation_admitsb_reflects_admission in Hadmits. + now rewrite Hadmits, Hdecode. + - now apply VWENC_17_ONE_LOGICAL_ATOM_PER_CONSUMER_TRANSITION. + - apply pathmap_trace_projects_exactly_one_atom. + - apply pathmap_trace_projects_exactly_one_atom. + - apply pathmap_trace_projects_exactly_one_atom. +Qed. + +Theorem VWENC_227_PATHMAP_UTF8_GROUPING_EMITS_ONE_UNICODE_SCALAR : + forall surface bytes codepoint, + canonical_utf8_codeword codepoint bytes -> + family_profile_cell PathMapAdapterFamily + (DirectProfile DirectUnicodeScalarDomain) = + ExistingProfileCell PathMapUtf8BoundaryAdapterRoute + PathMapUtf8BoundaryLayout /\ + consumer_observation surface + {| transition_representation := EncodedBytePathAdapter; + transition_unit := StoredUtf8 bytes |} = [UnicodeAtom codepoint] /\ + pathmap_node_projection + (pathmap_physical_trace (StoredUtf8 bytes) (UnicodeAtom codepoint)) = + [UnicodeAtom codepoint] /\ + pathmap_zipper_projection + (pathmap_physical_trace (StoredUtf8 bytes) (UnicodeAtom codepoint)) = + [UnicodeAtom codepoint] /\ + pathmap_snapshot_projection + (pathmap_physical_trace (StoredUtf8 bytes) (UnicodeAtom codepoint)) = + [UnicodeAtom codepoint]. +Proof. + intros surface bytes codepoint Hcanonical. + split; [reflexivity |]. split. + - apply VWENC_16_CODEC_BYTES_ARE_NOT_LOGICAL_TRANSITIONS. + now apply VWENC_66_UTF8_LOGICAL_IDENTITY_IS_UNICODE_SCALAR. + - split; [apply pathmap_trace_projects_exactly_one_atom |]. + split; apply pathmap_trace_projects_exactly_one_atom. +Qed. + +Theorem VWENC_228_CANONICAL_ULEB_CODEWORD_EMITS_ONE_OPAQUE_LOGICAL_ATOM : + forall carrier surface bytes, + canonical_uleb_codeword bytes -> + family_profile_cell PathMapAdapterFamily + (InternedProfile CanonicalUlebDomain carrier) = + ProspectiveProfileCell (PathMapInternedIdAdapterRoute carrier) + (PathMapInternedIdLayout carrier) /\ + consumer_observation surface + {| transition_representation := OpaqueCodewordEdge; + transition_unit := StoredUleb bytes |} = [UlebAtom bytes]. +Proof. + intros carrier surface bytes Hcanonical. split; [reflexivity |]. + apply VWENC_16_CODEC_BYTES_ARE_NOT_LOGICAL_TRANSITIONS. + now apply VWENC_65_ULEB_LOGICAL_IDENTITY_IS_CANONICAL_BYTES. +Qed. + +Fixpoint codeword_boundary_offsets_from + (start : nat) (codewords : list (list PhysicalByte)) : list nat := + match codewords with + | [] => [start] + | codeword :: rest => + start :: + codeword_boundary_offsets_from (start + length codeword) rest + end. + +Definition codeword_boundary_offsets + (codewords : list (list PhysicalByte)) : list nat := + codeword_boundary_offsets_from 0 codewords. + +Lemma codeword_boundary_offsets_from_are_exact : + forall codewords start offset, + In offset (codeword_boundary_offsets_from start codewords) <-> + exists prefix suffix, + codewords = prefix ++ suffix /\ + offset = start + length (concat prefix). +Proof. + induction codewords as [| codeword rest IH]; intros start offset. + - simpl. split. + + intros [Hequal | Himpossible]; [subst | contradiction]. + exists [], []. simpl. split; [reflexivity | lia]. + + intros [prefix [suffix [Hequal Hoffset]]]. + destruct prefix as [| first prefix]; + [simpl in Hoffset; left; lia | discriminate]. + - simpl. split. + + intros [Hstart | Hlater]. + * subst. exists [], (codeword :: rest). simpl. + split; [reflexivity | lia]. + * apply IH in Hlater. + destruct Hlater as [prefix [suffix [Hrest Hoffset]]]. + exists (codeword :: prefix), suffix. split. + -- simpl. now rewrite Hrest. + -- simpl. rewrite app_length. lia. + + intros [prefix [suffix [Hequal Hoffset]]]. + destruct prefix as [| first prefix]. + * simpl in Hoffset. left. lia. + * simpl in Hequal. inversion Hequal; subst first. + right. rewrite <- H1. apply IH. + exists prefix, suffix. split; [assumption |]. + simpl in Hoffset. rewrite app_length in Hoffset. lia. +Qed. + +Theorem VWENC_229_CODEWORD_BOUNDARY_OFFSETS_ARE_EXACTLY_LOGICAL_SPLITS : + forall codewords offset, + In offset (codeword_boundary_offsets codewords) <-> + exists prefix suffix, + codewords = prefix ++ suffix /\ + offset = length (concat prefix). +Proof. + intros codewords offset. + unfold codeword_boundary_offsets. + rewrite codeword_boundary_offsets_from_are_exact. + split. + - intros [prefix [suffix [Hequal Hoffset]]]. + exists prefix, suffix. split; [exact Hequal | lia]. + - intros [prefix [suffix [Hequal Hoffset]]]. + exists prefix, suffix. split; [exact Hequal | lia]. +Qed. + +Definition physical_suffix_at + (bytes : list PhysicalByte) (offset : nat) + (suffix : list PhysicalByte) : Prop := + exists prefix, + bytes = prefix ++ suffix /\ + length prefix = offset. + +Theorem VWENC_230_RAW_UTF8_SUFFIX_CAN_START_INSIDE_ONE_SCALAR_CODEWORD : + canonical_utf8_codeword 169 [194; 169] /\ + physical_suffix_at [194; 169] 1 [169] /\ + ~ In 1 (codeword_boundary_offsets [[194; 169]]). +Proof. + split. + - split. + + unfold unicode_scalar. split. + * unfold unicode_limit, utf8_three_byte_limit. + change (169 < 17 * (256 * (256 * 1))). nia. + * unfold surrogate_start, surrogate_end. + change (169 < 216 * 256 \/ 224 * 256 <= 169). + left. nia. + + unfold encode_utf8_scalar, utf8_one_byte_limit, + utf8_two_byte_limit. + rewrite (proj2 (Nat.ltb_ge 169 128)) by lia. + rewrite (proj2 (Nat.ltb_lt 169 (8 * 256))) by lia. + reflexivity. + - split. + + exists [194]. split; reflexivity. + + simpl. lia. +Qed. + +Theorem VWENC_231_RAW_ULEB_SUFFIX_CAN_START_INSIDE_ONE_CODEWORD : + canonical_uleb_codeword [128; 1] /\ + physical_suffix_at [128; 1] 1 [1] /\ + ~ In 1 (codeword_boundary_offsets [[128; 1]]). +Proof. + split. + - split. + + apply UlebShapeMore; [lia | lia |]. + apply UlebShapeLast. lia. + + change (canonical_uleb_digits [0; 1]). + unfold canonical_uleb_digits. split; [discriminate |]. + split. + * constructor; [unfold valid_uleb_digit; lia |]. + constructor; [unfold valid_uleb_digit; lia | constructor]. + * intros. simpl. discriminate. + - split. + + exists [128]. split; reflexivity. + + simpl. lia. +Qed. + +Definition logical_codeword_suffix + (codewords suffix : list (list PhysicalByte)) : Prop := + exists prefix, codewords = prefix ++ suffix. + +Inductive SuffixSemanticDomain : Type := +| SuffixNotApplicable +| RawByteSuffixSemantics +| LogicalAtomSuffixSemantics. + +Definition suffix_family (family : DictionaryFamily) : bool := + match family with + | SuffixAutomatonFamily | ScdawgFamily + | PersistentSuffixAutomatonFamily | PersistentSuffixTreeFamily + | PersistentScdawgFamily => true + | _ => false + end. + +Definition suffix_semantic_domain + (family : DictionaryFamily) (profile : FamilyProfile) + (layout : ExplicitLayoutContract) : SuffixSemanticDomain := + if suffix_family family then + match profile, layout with + | DirectProfile DirectBytesDomain, _ => RawByteSuffixSemantics + | _, EncodedU64ByteCompatibilityLayout => RawByteSuffixSemantics + | _, PathMapNativeByteLayout => RawByteSuffixSemantics + | _, _ => LogicalAtomSuffixSemantics + end + else SuffixNotApplicable. + +Definition suffix_start_admissible + (domain : SuffixSemanticDomain) + (codewords : list (list PhysicalByte)) (offset : nat) : Prop := + match domain with + | SuffixNotApplicable => False + | RawByteSuffixSemantics => offset <= length (concat codewords) + | LogicalAtomSuffixSemantics => + In offset (codeword_boundary_offsets codewords) + end. + +Theorem VWENC_232_LOGICAL_SUFFIXES_BEGIN_ONLY_AT_CODEWORD_BOUNDARIES : + (forall codewords suffix, + logical_codeword_suffix codewords suffix -> + exists offset, + In offset (codeword_boundary_offsets codewords) /\ + physical_suffix_at (concat codewords) offset (concat suffix)) /\ + (forall family profile layout codewords offset, + suffix_semantic_domain family profile layout = + LogicalAtomSuffixSemantics -> + suffix_start_admissible + (suffix_semantic_domain family profile layout) codewords offset -> + In offset (codeword_boundary_offsets codewords)). +Proof. + split. + - intros codewords suffix [prefix Hequal]. + exists (length (concat prefix)). split. + + apply (proj2 + (VWENC_229_CODEWORD_BOUNDARY_OFFSETS_ARE_EXACTLY_LOGICAL_SPLITS + codewords (length (concat prefix)))). + exists prefix, suffix. now split. + + unfold physical_suffix_at. exists (concat prefix). split. + * rewrite Hequal. apply concat_app. + * reflexivity. + - intros family profile layout codewords offset Hdomain Hadmissible. + rewrite Hdomain in Hadmissible. exact Hadmissible. +Qed. + +Theorem VWENC_233_RAW_BYTE_SUFFIX_INDEXES_CLAIM_ONLY_BYTE_SEMANTICS : + forall family layout, + suffix_family family = true -> + suffix_semantic_domain family + (DirectProfile DirectBytesDomain) layout = RawByteSuffixSemantics /\ + suffix_semantic_domain family + (DirectProfile DirectBytesDomain) layout <> + LogicalAtomSuffixSemantics /\ + family_surface_cell family SuffixSurface = + ExistingSurface SuffixIndexRoute. +Proof. + intros family layout Hsuffix. + unfold suffix_semantic_domain. rewrite Hsuffix. split; [reflexivity |]. + split; [discriminate |]. + destruct family; simpl in Hsuffix |- *; try discriminate; reflexivity. +Qed. + +Definition serialized_direct_codewords + (profile : VariableWidthCodecSpec.DirectProfile) (units : list nat) + : list (list PhysicalByte) := + map (fun unit => snd (serialize_direct_unit profile unit)) units. + +Theorem VWENC_234_DIRECT_UNITS_PRESERVE_ONE_CODEWORD_PER_LOGICAL_EDGE : + forall profile units, + length (serialized_direct_codewords profile units) = length units /\ + Forall + (fun bytes => length bytes = direct_byte_width profile) + (serialized_direct_codewords profile units). +Proof. + intros profile units. split. + - apply map_length. + - induction units as [| unit rest IH]; simpl; constructor. + + apply VWENC_49_DIRECT_SERIALIZATION_HAS_EXACT_FIXED_WIDTH. + + exact IH. +Qed. + +Definition serialized_symbol_id_codewords (I : FixedWidthCarrierProfile) + (ids : list (SymbolId I)) : list (list PhysicalByte) := + map (encode_symbol_id I) ids. + +Theorem VWENC_235_INTERNED_IDS_PRESERVE_ONE_FIXED_CODEWORD_PER_LOGICAL_EDGE : + forall I ids, + length (serialized_symbol_id_codewords I ids) = length ids /\ + Forall + (fun bytes => length bytes = carrier_width_bytes I) + (serialized_symbol_id_codewords I ids). +Proof. + intros I ids. split. + - apply map_length. + - induction ids as [| id rest IH]; simpl; constructor. + + exact (proj1 (symbol_id_fixed_width_encoding_roundtrips I id)). + + exact IH. +Qed. + +(** ** One-time vocabulary binding and fixed-width hot traversal *) + +Record BoundConsumerFiber + (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (expected actual : VocabularyFiber P I) : Type := { + consumer_fiber_binding_certificate : expected = actual +}. + +Definition bind_consumer_fiber + (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (expected actual : VocabularyFiber P I) + : option (BoundConsumerFiber P I expected actual). +Proof. + destruct (vocabulary_fiber_eq_dec P I expected actual) as [Hequal | Hdifferent]. + - exact (Some {| consumer_fiber_binding_certificate := Hequal |}). + - exact None. +Defined. + +Theorem VWENC_236_CONSUMER_VOCABULARY_BINDING_IS_VALIDATED_ONCE : + forall P I (expected actual : VocabularyFiber P I), + bind_consumer_fiber P I expected actual <> None <-> + expected = actual. +Proof. + intros P I expected actual. + unfold bind_consumer_fiber. + destruct (vocabulary_fiber_eq_dec P I expected actual) as + [Hequal | Hdifferent]. + - split; [intros; exact Hequal | intros; discriminate]. + - split. + + intros Hpresent. exfalso. apply Hpresent. reflexivity. + + intros Hequal. contradiction. +Qed. + +Theorem VWENC_237_MISMATCHED_VOCABULARY_FIBERS_ARE_REJECTED_BEFORE_TRAVERSAL : + forall P I (expected actual : VocabularyFiber P I), + expected <> actual -> + bind_consumer_fiber P I expected actual = None. +Proof. + intros P I expected actual Hdifferent. + unfold bind_consumer_fiber. + destruct (vocabulary_fiber_eq_dec P I expected actual); + [contradiction | reflexivity]. +Qed. + +(** Direct and interned hot views remain separate. The interned unit type was + defined above the family snapshot so every consumer surfaceβ€”not only this + optimized viewβ€”must use the same snapshot-bound representation. *) + +Definition direct_hot_kernel (domain : DirectUnitDomain) + : MonomorphicFixedWidthKernel (DirectUnit domain). +Proof. + refine + {| monomorphic_width := direct_byte_width (direct_codec_profile domain); + monomorphic_width_positive := _; + monomorphic_encode := fun unit => + snd (serialize_direct_unit (direct_codec_profile domain) + (direct_unit_value unit)); + monomorphic_encode_exact := _; + monomorphic_variable_decode_request := fun _ => None; + monomorphic_has_no_variable_decode := _ |}. + - destruct domain; simpl; lia. + - intros [unit Hvalid]. + apply VWENC_49_DIRECT_SERIALIZATION_HAS_EXACT_FIXED_WIDTH. + - reflexivity. +Defined. + +Definition interned_hot_kernel + (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (fiber : VocabularyFiber P I) + (snapshot : VocabularySnapshot P I fiber) + : MonomorphicFixedWidthKernel + (SnapshotBoundSymbolId P I fiber snapshot). +Proof. + refine + {| monomorphic_width := carrier_width_bytes I; + monomorphic_width_positive := carrier_width_positive I; + monomorphic_encode := fun bound => + encode_symbol_id I + (snapshot_bound_symbol_id P I fiber snapshot bound); + monomorphic_encode_exact := _; + monomorphic_variable_decode_request := fun _ => None; + monomorphic_has_no_variable_decode := _ |}. + - intros bound. + exact (proj1 (symbol_id_fixed_width_encoding_roundtrips I + (snapshot_bound_symbol_id P I fiber snapshot bound))). + - reflexivity. +Defined. + +Record DirectHotTraversalView (domain : DirectUnitDomain) : Type := { + direct_hot_view_units : list (DirectUnit domain) +}. + +Definition run_direct_hot_view (domain : DirectUnitDomain) + (view : DirectHotTraversalView domain) : list (list PhysicalByte) := + run_bound_kernel (direct_hot_kernel domain) + (direct_hot_view_units domain view). + +Record BoundHotTraversalView + (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (expected actual : VocabularyFiber P I) + (snapshot : VocabularySnapshot P I actual) : Type := { + hot_view_binding : BoundConsumerFiber P I expected actual; + hot_view_units : list (SnapshotBoundSymbolId P I actual snapshot) +}. + +Definition construct_bound_hot_traversal_view + (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (expected actual : VocabularyFiber P I) + (snapshot : VocabularySnapshot P I actual) + (units : list (SnapshotBoundSymbolId P I actual snapshot)) + : option (BoundHotTraversalView P I expected actual snapshot) := + match bind_consumer_fiber P I expected actual with + | Some binding => + Some {| hot_view_binding := binding; hot_view_units := units |} + | None => None + end. + +Definition run_interned_hot_view + (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (expected actual : VocabularyFiber P I) + (snapshot : VocabularySnapshot P I actual) + (view : BoundHotTraversalView P I expected actual snapshot) + : list (list PhysicalByte) := + run_bound_kernel (interned_hot_kernel P I actual snapshot) + (hot_view_units P I expected actual snapshot view). + +Theorem VWENC_238_EVERY_HOT_TRANSITION_HAS_AN_EXACT_FIXED_WIDTH_ENCODING : + forall (Unit : Type) (kernel : MonomorphicFixedWidthKernel Unit) unit, + length (monomorphic_encode Unit kernel unit) = + monomorphic_width Unit kernel /\ + 0 < monomorphic_width Unit kernel. +Proof. + intros Unit kernel unit. split. + - apply monomorphic_encode_exact. + - apply monomorphic_width_positive. +Qed. + +Theorem VWENC_239_ARBITRARY_WIDTH_BIGUINT_BYTES_STAY_OUTSIDE_HOT_TRAVERSAL : + forall (Unit : Type) (kernel : MonomorphicFixedWidthKernel Unit) unit, + monomorphic_variable_decode_request Unit kernel unit = None. +Proof. intros. apply monomorphic_has_no_variable_decode. Qed. + +Inductive SemanticOwnership : Type := +| LibdictensteinStorageSemantics +| LlatticeAlgebraSemantics. + +Definition join_meet_semantics_owner + (_profile : FamilyProfile) : SemanticOwnership := + LlatticeAlgebraSemantics. + +Theorem VWENC_240_DICTIONARY_PROFILES_DO_NOT_OWN_LLATTICE_ALGEBRA : + forall profile, + join_meet_semantics_owner profile = LlatticeAlgebraSemantics /\ + join_meet_semantics_owner profile <> + LibdictensteinStorageSemantics. +Proof. intros. split; discriminate || reflexivity. Qed. + +Theorem VWENC_247_HOT_TRAVERSAL_VIEW_EXISTS_IFF_FIBER_BINDING_SUCCEEDS : + forall P I (expected actual : VocabularyFiber P I) + (snapshot : VocabularySnapshot P I actual) units, + construct_bound_hot_traversal_view + P I expected actual snapshot units <> None <-> + expected = actual. +Proof. + intros P I expected actual snapshot units. + unfold construct_bound_hot_traversal_view. + destruct (bind_consumer_fiber P I expected actual) as + [binding |] eqn:Hbinding. + - split; [intros | intros; discriminate]. + apply (proj1 (VWENC_236_CONSUMER_VOCABULARY_BINDING_IS_VALIDATED_ONCE + P I expected actual)). + rewrite Hbinding. discriminate. + - split. + + intros Hpresent. exfalso. apply Hpresent. reflexivity. + + intros Hequal. + apply (proj2 (VWENC_236_CONSUMER_VOCABULARY_BINDING_IS_VALIDATED_ONCE + P I expected actual)) in Hequal. + rewrite Hbinding in Hequal. contradiction. +Qed. + +Theorem VWENC_248_MISMATCHED_FIBER_CANNOT_CONSTRUCT_A_HOT_TRAVERSAL_VIEW : + forall P I (expected actual : VocabularyFiber P I) + (snapshot : VocabularySnapshot P I actual) units, + expected <> actual -> + construct_bound_hot_traversal_view + P I expected actual snapshot units = None. +Proof. + intros P I expected actual snapshot units Hdifferent. + unfold construct_bound_hot_traversal_view. + rewrite (VWENC_237_MISMATCHED_VOCABULARY_FIBERS_ARE_REJECTED_BEFORE_TRAVERSAL + P I expected actual Hdifferent). + reflexivity. +Qed. + +Theorem VWENC_249_BOUND_HOT_VIEWS_CONTAIN_ONLY_EXACT_FIXED_WIDTH_UNITS : + (forall P I (expected actual : VocabularyFiber P I) + (snapshot : VocabularySnapshot P I actual) + (view : BoundHotTraversalView P I expected actual snapshot), + map (@length PhysicalByte) + (run_interned_hot_view P I expected actual snapshot view) = + repeat (carrier_width_bytes I) + (length (hot_view_units P I expected actual snapshot view)) /\ + Forall + (fun bound => + exists atom, + In (atom, + snapshot_bound_symbol_id P I actual snapshot bound) + (vocabulary_snapshot_live_entries P I actual snapshot)) + (hot_view_units P I expected actual snapshot view)) /\ + (forall domain (view : DirectHotTraversalView domain), + map (@length PhysicalByte) (run_direct_hot_view domain view) = + repeat (direct_byte_width (direct_codec_profile domain)) + (length (direct_hot_view_units domain view))). +Proof. + split. + - intros P I expected actual snapshot view. split. + + unfold run_interned_hot_view. + apply bound_kernel_widths_are_constant. + + induction (hot_view_units P I expected actual snapshot view) + as [|bound rest IH]; constructor. + * exact (snapshot_bound_live P I actual snapshot bound). + * exact IH. + - intros domain view. unfold run_direct_hot_view. + apply bound_kernel_widths_are_constant. +Qed. + +End VariableWidthFamilyRefinementSpec. diff --git a/formal-verification/rocq/Spec/VariableWidthInterningSpec.v b/formal-verification/rocq/Spec/VariableWidthInterningSpec.v new file mode 100644 index 00000000..9b80bd71 --- /dev/null +++ b/formal-verification/rocq/Spec/VariableWidthInterningSpec.v @@ -0,0 +1,6553 @@ +(** * Certified variable-width atom interning and fixed-width ID views + + This functional model is the second formal milestone of the + variable-width dictionary campaign. It consumes the certified profile + and codec laws from [VariableWidthCodecSpec]. In particular: + + - an atom is indexed by one certified persistent profile and carries a + proof that its complete byte string is one canonical codeword; + - [SymbolId I] and [TermId T] are distinct nominal types parameterized by + open, positive-width fixed-width carrier profiles; + - a single interning state ties the live bijection, historical ownership, + packed bytes, reverse spans, sparse allocator frontier, dependent + sequences, and optional term-ID dictionary together; + - IDs which were published or burned as orphans are never rebound; + - sequence views are bound to an immutable backing identity and exact + vocabulary fiber, and index fixed-width IDs without decoding atoms; + - query-local IDs occupy a distinct, non-serializable namespace; + - model-to-Rust correspondence records exact paths, symbols, semantic + relationships, and obligations. Current conflicts are explicit. + + Stateful object publication, crash recovery, immutable reader retention, + and repeated generations are modeled in + [VariableWidthVocabularyInterning.tla] and + [VariableWidthVocabularyPublication.tla]. + + Rocq lists model immutable mathematical observations. Allocation-free + borrowing, native slice layout, lifetimes, and hot-loop costs remain + explicit Rust refinement obligations; this file does not misdescribe a + Rocq list as a Rust borrow. + + Stable theorem names beginning with [VWENC_] are machine-readable + invariant identifiers consumed by the conformance ledger. + *) + +From Coq Require Import Lists.List. +From Coq Require Import Arith.Arith. +From Coq Require Import Bool.Bool. +From Coq Require Import micromega.Lia. +From Coq Require Import Logic.ProofIrrelevance. +From Coq Require Import Strings.String. +From Coq Require Import Sorting.Permutation. +Require Import ARTrie.Spec.VariableWidthCodecSpec. +Import ListNotations. +Import VariableWidthCodecSpec. + +Lemma skipn_length_local : + forall (A : Type) (n : nat) (xs : list A), + List.length (skipn n xs) = List.length xs - n. +Proof. + intros A n. induction n as [| n IH]; intros xs; simpl; [lia|]. + destruct xs as [| x xs]; simpl; [lia|]. + rewrite IH. lia. +Qed. + +Lemma firstn_length_local : + forall (A : Type) (n : nat) (xs : list A), + List.length (firstn n xs) = Nat.min n (List.length xs). +Proof. + intros A n. induction n as [| n IH]; intros xs; simpl; [lia|]. + destruct xs as [| x xs]; simpl; [lia|]. + rewrite IH. reflexivity. +Qed. + +Module VariableWidthInterning. + +(** ** Certified atom profiles and canonical atoms *) + +Definition descriptor_canonical_codeword + (descriptor : PersistentProfileDescriptor) + (bytes : list PhysicalByte) : Prop := + match persistent_logical_profile descriptor with + | PersistedByte => + List.length bytes = 1 /\ Forall valid_byte bytes + | PersistedUnicodeScalar => + List.length bytes = 4 /\ + Forall valid_byte bytes /\ + unicode_scalar (decode_fixed_little_endian bytes) + | PersistedU64 => + List.length bytes = 8 /\ Forall valid_byte bytes + | PersistedF64Bits => + List.length bytes = 8 /\ Forall valid_byte bytes + | PersistedCanonicalUleb => canonical_uleb_codeword bytes + | PersistedCanonicalUtf8 => + exists codepoint, canonical_utf8_codeword codepoint bytes + end. + +Lemma descriptor_canonical_codeword_nonempty : + forall descriptor bytes, + descriptor_canonical_codeword descriptor bytes -> bytes <> []. +Proof. + intros [profile codec layout abi] bytes Hcodeword. + destruct profile; simpl in Hcodeword. + - destruct Hcodeword as [Hlength _]. + intros Hequal. subst bytes. simpl in Hlength. discriminate. + - destruct Hcodeword as [Hlength _]. + intros Hequal. subst bytes. simpl in Hlength. discriminate. + - destruct Hcodeword as [Hlength _]. + intros Hequal. subst bytes. simpl in Hlength. discriminate. + - destruct Hcodeword as [Hlength _]. + intros Hequal. subst bytes. simpl in Hlength. discriminate. + - now apply VWENC_03_ULEB_CODEWORDS_NONEMPTY. + - destruct Hcodeword as [codepoint [Hscalar Hbytes]]. + subst bytes. + exact (proj1 (VWENC_12_UTF8_CODEWORDS_NONEMPTY_AND_AT_MOST_FOUR_BYTES + codepoint Hscalar)). +Qed. + +Record CertifiedAtomProfile : Type := mkCertifiedAtomProfile { + atom_profile_descriptor : PersistentProfileDescriptor; + atom_profile_certificate : + certified_persistent_profile atom_profile_descriptor +}. + +Definition atom_codeword + (profile : CertifiedAtomProfile) : list PhysicalByte -> Prop := + descriptor_canonical_codeword (atom_profile_descriptor profile). + +Lemma atom_codeword_nonempty : + forall profile bytes, atom_codeword profile bytes -> bytes <> []. +Proof. + intros profile bytes Hcodeword. + unfold atom_codeword in Hcodeword. + now apply descriptor_canonical_codeword_nonempty + with (descriptor := atom_profile_descriptor profile). +Qed. + +Definition canonical_uleb_descriptor : PersistentProfileDescriptor := + {| persistent_logical_profile := PersistedCanonicalUleb; + persistent_codec_identity := ProspectiveCanonicalUlebCodecV1; + persistent_layout_identity := ProspectiveLogicalUnitLayoutV1; + persistent_abi_version := 1 |}. + +Lemma canonical_uleb_descriptor_certified : + certified_persistent_profile canonical_uleb_descriptor. +Proof. + exact VWENC_99_CERTIFICATION_ACCEPTS_VERSIONED_CANONICAL_ULEB_PROFILE. +Qed. + +Definition canonical_uleb_profile : CertifiedAtomProfile := + {| atom_profile_descriptor := canonical_uleb_descriptor; + atom_profile_certificate := canonical_uleb_descriptor_certified |}. + +Record CanonicalAtom (P : CertifiedAtomProfile) : Type := mkCanonicalAtom { + canonical_atom_bytes : list PhysicalByte; + canonical_atom_valid : atom_codeword P canonical_atom_bytes +}. + +Definition canonical_atom_identity + {P : CertifiedAtomProfile} (atom : CanonicalAtom P) := + (certified_profile_identity (atom_profile_descriptor P), + canonical_atom_bytes P atom). + +Definition canonical_atom_eq_dec + (P : CertifiedAtomProfile) + (left right : CanonicalAtom P) : {left = right} + {left <> right}. +Proof. + destruct left as [left_bytes left_valid]. + destruct right as [right_bytes right_valid]. + destruct (list_eq_dec Nat.eq_dec left_bytes right_bytes) + as [Hbytes | Hbytes]. + - subst right_bytes. left. + assert (left_valid = right_valid) by apply proof_irrelevance. + now subst right_valid. + - right. intros Hequal. inversion Hequal. contradiction. +Defined. + +Theorem VWENC_101_CANONICAL_ATOM_IDENTITY_IS_CERTIFIED_PROFILE_AND_BYTES : + forall (P : CertifiedAtomProfile) (left right : CanonicalAtom P), + canonical_atom_identity left = canonical_atom_identity right -> + left = right. +Proof. + intros P [left_bytes left_valid] [right_bytes right_valid] Hequal. + unfold canonical_atom_identity in Hequal. simpl in Hequal. + inversion Hequal. subst right_bytes. + assert (left_valid = right_valid) by apply proof_irrelevance. + now subst right_valid. +Qed. + +Definition canonical_uleb_atom + (bytes : list PhysicalByte) + (Hcanonical : canonical_uleb_codeword bytes) + : CanonicalAtom canonical_uleb_profile. +Proof. + refine (@mkCanonicalAtom canonical_uleb_profile bytes _). + change (canonical_uleb_codeword bytes). + exact Hcanonical. +Defined. + +Theorem VWENC_102_ULEB_INTERNALIZATION_REQUIRES_CANONICAL_ARBITRARY_BYTES : + forall bytes (Hcanonical : canonical_uleb_codeword bytes), + canonical_atom_bytes + canonical_uleb_profile + (canonical_uleb_atom bytes Hcanonical) = bytes /\ + bytes <> [] /\ + certified_profile_identity + (atom_profile_descriptor canonical_uleb_profile) = + certified_profile_identity canonical_uleb_descriptor. +Proof. + intros bytes Hcanonical. + split; [reflexivity |]. + split. + - now apply VWENC_03_ULEB_CODEWORDS_NONEMPTY. + - reflexivity. +Qed. + +Lemma one_byte_uleb_is_canonical : + forall byte, byte < 128 -> canonical_uleb_codeword [byte]. +Proof. + intros byte Hbyte. + split. + - constructor. exact Hbyte. + - unfold canonical_uleb_digits, decode_uleb_payloads. + simpl. + repeat split. + + discriminate. + + constructor. + * unfold valid_uleb_digit, uleb_payload. + apply Nat.mod_upper_bound. lia. + * constructor. + + simpl. lia. +Qed. + +(** ** Open fixed-width ID carriers and nominal IDs *) + +Record FixedWidthCarrierProfile : Type := mkFixedWidthCarrierProfile { + carrier_format_identity : nat; + carrier_width_bytes : nat; + carrier_width_positive : 0 < carrier_width_bytes +}. + +Definition carrier_capacity (I : FixedWidthCarrierProfile) : nat := + 256 ^ carrier_width_bytes I. + +Lemma carrier_capacity_positive : + forall I, 0 < carrier_capacity I. +Proof. + intros I. + unfold carrier_capacity. + assert (256 ^ carrier_width_bytes I <> 0). + { apply Nat.pow_nonzero. lia. } + lia. +Qed. + +Record SymbolId (I : FixedWidthCarrierProfile) : Type := mkSymbolId { + symbol_id_value : nat; + symbol_id_in_range : symbol_id_value < carrier_capacity I +}. + +Record TermId (T : FixedWidthCarrierProfile) : Type := mkTermId { + term_id_value : nat; + term_id_in_range : term_id_value < carrier_capacity T +}. + +Definition symbol_id_eq_dec + (I : FixedWidthCarrierProfile) + (left right : SymbolId I) : {left = right} + {left <> right}. +Proof. + destruct left as [left_value left_range]. + destruct right as [right_value right_range]. + destruct (Nat.eq_dec left_value right_value) as [Hequal | Hdifferent]. + - subst right_value. left. + assert (left_range = right_range) by apply proof_irrelevance. + now subst right_range. + - right. intros Hequal. inversion Hequal. contradiction. +Defined. + +Definition term_id_eq_dec + (T : FixedWidthCarrierProfile) + (left right : TermId T) : {left = right} + {left <> right}. +Proof. + destruct left as [left_value left_range]. + destruct right as [right_value right_range]. + destruct (Nat.eq_dec left_value right_value) as [Hequal | Hdifferent]. + - subst right_value. left. + assert (left_range = right_range) by apply proof_irrelevance. + now subst right_range. + - right. intros Hequal. inversion Hequal. contradiction. +Defined. + +Definition symbol_id_of_nat + (I : FixedWidthCarrierProfile) (value : nat) : option (SymbolId I) := + match lt_dec value (carrier_capacity I) with + | left Hfits => + Some {| symbol_id_value := value; symbol_id_in_range := Hfits |} + | right _ => None + end. + +Definition term_id_of_nat + (T : FixedWidthCarrierProfile) (value : nat) : option (TermId T) := + match lt_dec value (carrier_capacity T) with + | left Hfits => + Some {| term_id_value := value; term_id_in_range := Hfits |} + | right _ => None + end. + +Definition encode_symbol_id + (I : FixedWidthCarrierProfile) (id : SymbolId I) + : list PhysicalByte := + encode_fixed_little_endian + (carrier_width_bytes I) (symbol_id_value I id). + +Definition encode_term_id + (T : FixedWidthCarrierProfile) (id : TermId T) + : list PhysicalByte := + encode_fixed_little_endian + (carrier_width_bytes T) (term_id_value T id). + +Definition decode_symbol_id + (I : FixedWidthCarrierProfile) (bytes : list PhysicalByte) + : option (SymbolId I) := + if Nat.eq_dec (List.length bytes) (carrier_width_bytes I) then + if all_valid_bytesb bytes then + symbol_id_of_nat I (decode_fixed_little_endian bytes) + else None + else None. + +Definition decode_term_id + (T : FixedWidthCarrierProfile) (bytes : list PhysicalByte) + : option (TermId T) := + if Nat.eq_dec (List.length bytes) (carrier_width_bytes T) then + if all_valid_bytesb bytes then + term_id_of_nat T (decode_fixed_little_endian bytes) + else None + else None. + +Lemma symbol_id_fixed_width_encoding_roundtrips : + forall (I : FixedWidthCarrierProfile) (id : SymbolId I), + List.length (encode_symbol_id I id) = carrier_width_bytes I /\ + decode_symbol_id I (encode_symbol_id I id) = Some id. +Proof. + intros I [value Hrange]. + split. + - apply fixed_little_endian_length. + - unfold decode_symbol_id, encode_symbol_id. simpl. + rewrite fixed_little_endian_length. + destruct (Nat.eq_dec (carrier_width_bytes I) (carrier_width_bytes I)) + as [_ | Himpossible]. + 2: contradiction. + assert (Hvalid : + all_valid_bytesb + (encode_fixed_little_endian (carrier_width_bytes I) value) = true). + { apply (proj2 (all_valid_bytesb_reflects_validity _)). + apply fixed_little_endian_bytes_are_valid. } + rewrite Hvalid. + unfold symbol_id_of_nat. + rewrite fixed_little_endian_roundtrip by exact Hrange. + destruct (lt_dec value (carrier_capacity I)) as [Hfits | Hoverflow]. + + f_equal. f_equal. apply proof_irrelevance. + + contradiction. +Qed. + +Lemma term_id_fixed_width_encoding_roundtrips : + forall (T : FixedWidthCarrierProfile) (id : TermId T), + List.length (encode_term_id T id) = carrier_width_bytes T /\ + decode_term_id T (encode_term_id T id) = Some id. +Proof. + intros T [value Hrange]. + split. + - apply fixed_little_endian_length. + - unfold decode_term_id, encode_term_id. simpl. + rewrite fixed_little_endian_length. + destruct (Nat.eq_dec (carrier_width_bytes T) (carrier_width_bytes T)) + as [_ | Himpossible]. + 2: contradiction. + assert (Hvalid : + all_valid_bytesb + (encode_fixed_little_endian (carrier_width_bytes T) value) = true). + { apply (proj2 (all_valid_bytesb_reflects_validity _)). + apply fixed_little_endian_bytes_are_valid. } + rewrite Hvalid. + unfold term_id_of_nat. + rewrite fixed_little_endian_roundtrip by exact Hrange. + destruct (lt_dec value (carrier_capacity T)) as [Hfits | Hoverflow]. + + f_equal. f_equal. apply proof_irrelevance. + + contradiction. +Qed. + +Theorem VWENC_109_SYMBOL_AND_TERM_ID_FIXED_WIDTH_ENCODINGS_ROUNDTRIP : + (forall (I : FixedWidthCarrierProfile) (id : SymbolId I), + List.length (encode_symbol_id I id) = carrier_width_bytes I /\ + decode_symbol_id I (encode_symbol_id I id) = Some id) /\ + (forall (T : FixedWidthCarrierProfile) (id : TermId T), + List.length (encode_term_id T id) = carrier_width_bytes T /\ + decode_term_id T (encode_term_id T id) = Some id). +Proof. + split. + - exact symbol_id_fixed_width_encoding_roundtrips. + - exact term_id_fixed_width_encoding_roundtrips. +Qed. + +Lemma symbol_id_construction_rejects_overflow : + forall (I : FixedWidthCarrierProfile) value, + carrier_capacity I <= value -> + symbol_id_of_nat I value = None. +Proof. + intros I value Hoverflow. + unfold symbol_id_of_nat. + destruct (lt_dec value (carrier_capacity I)); [lia | reflexivity]. +Qed. + +Lemma term_id_construction_rejects_overflow : + forall (T : FixedWidthCarrierProfile) value, + carrier_capacity T <= value -> + term_id_of_nat T value = None. +Proof. + intros T value Hoverflow. + unfold term_id_of_nat. + destruct (lt_dec value (carrier_capacity T)); [lia | reflexivity]. +Qed. + +Theorem VWENC_110_SYMBOL_AND_TERM_ID_CONSTRUCTION_REJECTS_OVERFLOW : + (forall (I : FixedWidthCarrierProfile) value, + carrier_capacity I <= value -> + symbol_id_of_nat I value = None) /\ + (forall (T : FixedWidthCarrierProfile) value, + carrier_capacity T <= value -> + term_id_of_nat T value = None). +Proof. + split. + - exact symbol_id_construction_rejects_overflow. + - exact term_id_construction_rejects_overflow. +Qed. + +Theorem VWENC_111_ID_CARRIER_INTERFACE_REMAINS_OPEN_TO_ANY_POSITIVE_WIDTH : + forall (I : FixedWidthCarrierProfile), + 0 < carrier_width_bytes I /\ + 0 < carrier_capacity I /\ + List.length + (encode_fixed_little_endian (carrier_width_bytes I) 0) = + carrier_width_bytes I. +Proof. + intros I. repeat split. + - apply carrier_width_positive. + - apply carrier_capacity_positive. + - apply fixed_little_endian_length. +Qed. + +Definition carrier_from_positive_width + (format_identity width : nat) (Hwidth : 0 < width) + : FixedWidthCarrierProfile := + {| carrier_format_identity := format_identity; + carrier_width_bytes := width; + carrier_width_positive := Hwidth |}. + +Theorem VWENC_160_EVERY_POSITIVE_WIDTH_HAS_AN_EXACT_CARRIER_INSTANCE : + forall format_identity width, + 0 < width -> + exists carrier : FixedWidthCarrierProfile, + carrier_format_identity carrier = format_identity /\ + carrier_width_bytes carrier = width. +Proof. + intros format_identity width Hwidth. + exists (carrier_from_positive_width format_identity width Hwidth). + now split. +Qed. + +(** ** Vocabulary fibers and exact atom/ID bijections *) + +Record VocabularyFiber + (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) : Type := + mkVocabularyFiber { + vocabulary_identity : nat; + vocabulary_generation : nat + }. + +Definition vocabulary_fiber_identity + {P : CertifiedAtomProfile} {I : FixedWidthCarrierProfile} + (fiber : VocabularyFiber P I) := + (certified_profile_identity (atom_profile_descriptor P), + (carrier_format_identity I, + (carrier_width_bytes I, + (vocabulary_identity P I fiber, + vocabulary_generation P I fiber)))). + +Definition vocabulary_fiber_eq_dec + (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (left right : VocabularyFiber P I) + : {left = right} + {left <> right}. +Proof. + destruct left as [left_identity left_generation]. + destruct right as [right_identity right_generation]. + destruct (Nat.eq_dec left_identity right_identity) + as [Hidentity | Hidentity]. + - subst right_identity. + destruct (Nat.eq_dec left_generation right_generation) + as [Hgeneration | Hgeneration]. + + subst right_generation. left. reflexivity. + + right. intros Hequal. inversion Hequal. contradiction. + - right. intros Hequal. inversion Hequal. contradiction. +Defined. + +Record FiberBoundSymbolId + (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) : Type := + mkFiberBoundSymbolId { + bound_symbol_fiber : VocabularyFiber P I; + bound_symbol_value : SymbolId I + }. + +Definition interpret_symbol_id + {P : CertifiedAtomProfile} {I : FixedWidthCarrierProfile} + (expected : VocabularyFiber P I) + (bound : FiberBoundSymbolId P I) : option (SymbolId I) := + if vocabulary_fiber_eq_dec P I expected (bound_symbol_fiber P I bound) + then Some (bound_symbol_value P I bound) + else None. + +Theorem VWENC_112_CROSS_FIBER_ID_INTERPRETATION_IS_REJECTED : + forall (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (expected actual : VocabularyFiber P I) (id : SymbolId I), + expected <> actual -> + interpret_symbol_id expected + (mkFiberBoundSymbolId P I actual id) = None. +Proof. + intros P I expected actual id Hdifferent. + unfold interpret_symbol_id. simpl. + destruct (vocabulary_fiber_eq_dec P I expected actual) + as [Hequal | _]. + - contradiction. + - reflexivity. +Qed. + +Theorem VWENC_113_SAME_FIBER_ID_INTERPRETATION_IS_EXACT : + forall (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (fiber : VocabularyFiber P I) (id : SymbolId I), + interpret_symbol_id fiber + (mkFiberBoundSymbolId P I fiber id) = Some id. +Proof. + intros P I fiber id. + unfold interpret_symbol_id. simpl. + destruct (vocabulary_fiber_eq_dec P I fiber fiber) + as [_ | Himpossible]. + - reflexivity. + - contradiction. +Qed. + +Fixpoint assoc_lookup {Key Value : Type} + (key_eq_dec : forall left right : Key, {left = right} + {left <> right}) + (entries : list (Key * Value)) + (query : Key) : option Value := + match entries with + | [] => None + | (key, value) :: rest => + if key_eq_dec query key then Some value + else assoc_lookup key_eq_dec rest query + end. + +Lemma assoc_lookup_sound : + forall (Key Value : Type) + (key_eq_dec : forall left right : Key, {left = right} + {left <> right}) + (entries : list (Key * Value)) query value, + assoc_lookup key_eq_dec entries query = Some value -> + In (query, value) entries. +Proof. + intros Key Value key_eq_dec entries. + induction entries as [| [key current] rest IH]; + intros query value Hlookup. + - discriminate. + - simpl in Hlookup. + destruct (key_eq_dec query key) as [Hequal | Hdifferent]. + + inversion Hlookup. subst. now left. + + right. now apply IH. +Qed. + +Lemma assoc_lookup_complete_unique : + forall (Key Value : Type) + (key_eq_dec : forall left right : Key, {left = right} + {left <> right}) + (entries : list (Key * Value)) query value, + NoDup (map fst entries) -> + In (query, value) entries -> + assoc_lookup key_eq_dec entries query = Some value. +Proof. + intros Key Value key_eq_dec entries. + induction entries as [| [key current] rest IH]; + intros query value Hnodup Hin. + - contradiction. + - inversion Hnodup as [| head keys Hhead Hrest]. + simpl in Hin. + destruct Hin as [Hequal | Hin]. + + inversion Hequal. subst query value. + simpl. destruct (key_eq_dec key key); [reflexivity | contradiction]. + + simpl. + destruct (key_eq_dec query key) as [Hequal | Hdifferent]. + * subst query. exfalso. apply Hhead. + apply in_map with (f := fst) in Hin. exact Hin. + * now apply IH. +Qed. + +Lemma assoc_lookup_none_key_absent : + forall (Key Value : Type) + (key_eq_dec : forall left right : Key, {left = right} + {left <> right}) + (entries : list (Key * Value)) query, + assoc_lookup key_eq_dec entries query = None -> + ~ In query (map fst entries). +Proof. + intros Key Value key_eq_dec entries. + induction entries as [| [key value] rest IH]; intros query Hlookup. + - simpl. tauto. + - simpl in Hlookup |- *. + destruct (key_eq_dec query key) as [Hequal | Hdifferent]. + + discriminate. + + intros [Hequal | Hin]. + * apply Hdifferent. symmetry. exact Hequal. + * now apply (IH query Hlookup). +Qed. + +Definition VocabularyEntry + (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) := + (CanonicalAtom P * SymbolId I)%type. + +Definition reverse_vocabulary_entries + {P : CertifiedAtomProfile} {I : FixedWidthCarrierProfile} + (entries : list (VocabularyEntry P I)) + : list (SymbolId I * CanonicalAtom P) := + map (fun entry => (snd entry, fst entry)) entries. + +Lemma reverse_vocabulary_membership : + forall (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (entries : list (VocabularyEntry P I)) atom id, + In (id, atom) (reverse_vocabulary_entries entries) <-> + In (atom, id) entries. +Proof. + intros P I entries atom id. + unfold reverse_vocabulary_entries. rewrite in_map_iff. + split. + - intros [[entry_atom entry_id] [Hequal Hin]]. + simpl in Hequal. inversion Hequal. subst. exact Hin. + - intros Hin. exists (atom, id). split; [reflexivity | exact Hin]. +Qed. + +Lemma reverse_vocabulary_keys_are_ids : + forall (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (entries : list (VocabularyEntry P I)), + map fst (reverse_vocabulary_entries entries) = map snd entries. +Proof. + intros P I entries. unfold reverse_vocabulary_entries. + rewrite map_map. apply map_ext. intros [atom id]. reflexivity. +Qed. + +Definition lookup_atom + {P : CertifiedAtomProfile} {I : FixedWidthCarrierProfile} + (entries : list (VocabularyEntry P I)) + (atom : CanonicalAtom P) : option (SymbolId I) := + assoc_lookup (canonical_atom_eq_dec P) entries atom. + +Definition lookup_symbol + {P : CertifiedAtomProfile} {I : FixedWidthCarrierProfile} + (entries : list (VocabularyEntry P I)) + (id : SymbolId I) : option (CanonicalAtom P) := + assoc_lookup (symbol_id_eq_dec I) + (reverse_vocabulary_entries entries) id. + +Definition vocabulary_relation_well_formed + {P : CertifiedAtomProfile} {I : FixedWidthCarrierProfile} + (entries : list (VocabularyEntry P I)) : Prop := + NoDup (map fst entries) /\ NoDup (map snd entries). + +Theorem VWENC_103_PUBLISHED_VOCABULARY_IS_AN_EXACT_BIJECTION : + forall (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (entries : list (VocabularyEntry P I)) atom id, + vocabulary_relation_well_formed entries -> + (lookup_atom entries atom = Some id <-> + lookup_symbol entries id = Some atom). +Proof. + intros P I entries atom id [Hatom_unique Hid_unique]. + split; intros Hlookup. + - apply assoc_lookup_sound in Hlookup. + unfold lookup_symbol. apply assoc_lookup_complete_unique. + + rewrite reverse_vocabulary_keys_are_ids. exact Hid_unique. + + apply reverse_vocabulary_membership. exact Hlookup. + - unfold lookup_symbol in Hlookup. + apply assoc_lookup_sound in Hlookup. + apply reverse_vocabulary_membership in Hlookup. + unfold lookup_atom. now apply assoc_lookup_complete_unique. +Qed. + +Definition fingerprint_candidate + {P : CertifiedAtomProfile} (_atom : CanonicalAtom P) : nat := 0. + +Definition collision_atom_left : CanonicalAtom canonical_uleb_profile := + canonical_uleb_atom [1] (one_byte_uleb_is_canonical 1 ltac:(lia)). + +Definition collision_atom_right : CanonicalAtom canonical_uleb_profile := + canonical_uleb_atom [2] (one_byte_uleb_is_canonical 2 ltac:(lia)). + +Definition u32_carrier : FixedWidthCarrierProfile := + {| carrier_format_identity := 32; + carrier_width_bytes := 4; + carrier_width_positive := ltac:(lia) |}. + +Definition symbol_zero : SymbolId u32_carrier. +Proof. + refine (@mkSymbolId u32_carrier 0 _). + apply carrier_capacity_positive. +Defined. + +Definition symbol_one : SymbolId u32_carrier. +Proof. + refine (@mkSymbolId u32_carrier 1 _). + unfold carrier_capacity, u32_carrier. + apply Nat.pow_gt_1. + - lia. + - discriminate. +Defined. + +Definition symbol_two : SymbolId u32_carrier. +Proof. + refine (@mkSymbolId u32_carrier 2 _). + change (2 < 256 ^ 4). + replace 4 with (S 3) by reflexivity. + rewrite Nat.pow_succ_r by lia. + set (power := 256 ^ 3). + assert (Hpower : power <> 0). + { unfold power. apply Nat.pow_nonzero. lia. } + nia. +Defined. + +Lemma symbol_two_differs_from_symbol_zero : + symbol_two <> symbol_zero. +Proof. + intros Hequal. + apply (f_equal (symbol_id_value u32_carrier)) in Hequal. + discriminate. +Qed. + +Definition term_zero : TermId u32_carrier. +Proof. + refine (@mkTermId u32_carrier 0 _). + apply carrier_capacity_positive. +Defined. + +Definition witness_vocabulary_fiber : + VocabularyFiber canonical_uleb_profile u32_carrier := + mkVocabularyFiber canonical_uleb_profile u32_carrier 700 1. + +Definition collision_vocabulary : + list (VocabularyEntry canonical_uleb_profile u32_carrier) := + [(collision_atom_left, symbol_zero); + (collision_atom_right, symbol_one)]. + +Theorem VWENC_104_FINGERPRINT_COLLISION_REQUIRES_FULL_CANONICAL_BYTES : + fingerprint_candidate collision_atom_left = + fingerprint_candidate collision_atom_right /\ + collision_atom_left <> collision_atom_right /\ + lookup_atom collision_vocabulary collision_atom_left = Some symbol_zero /\ + lookup_atom collision_vocabulary collision_atom_right = Some symbol_one. +Proof. + split; [reflexivity |]. + split. + - intros Hequal. + apply (f_equal + (canonical_atom_bytes canonical_uleb_profile)) in Hequal. + discriminate. + - split. + + unfold lookup_atom, collision_vocabulary. simpl. + destruct (canonical_atom_eq_dec + canonical_uleb_profile collision_atom_left collision_atom_left); + [reflexivity | contradiction]. + + unfold lookup_atom, collision_vocabulary. simpl. + destruct (canonical_atom_eq_dec + canonical_uleb_profile collision_atom_right collision_atom_left) + as [Hequal | _]. + * exfalso. apply (f_equal + (canonical_atom_bytes canonical_uleb_profile)) in Hequal. + discriminate. + * destruct (canonical_atom_eq_dec + canonical_uleb_profile collision_atom_right collision_atom_right); + [reflexivity | contradiction]. +Qed. + +Inductive InternLookupDecision + (I : FixedWidthCarrierProfile) : Type := +| InternExisting : SymbolId I -> InternLookupDecision I +| InternMissing : InternLookupDecision I. + +Definition inspect_interning + {P : CertifiedAtomProfile} {I : FixedWidthCarrierProfile} + (entries : list (VocabularyEntry P I)) + (atom : CanonicalAtom P) : InternLookupDecision I := + match lookup_atom entries atom with + | Some id => InternExisting I id + | None => InternMissing I + end. + +Theorem VWENC_105_EXISTING_ATOM_INTERNING_IS_IDEMPOTENT : + forall (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (entries : list (VocabularyEntry P I)) atom id, + lookup_atom entries atom = Some id -> + inspect_interning entries atom = InternExisting I id. +Proof. + intros P I entries atom id Hlookup. + unfold inspect_interning. now rewrite Hlookup. +Qed. + +(** ** Packed canonical bytes and non-overwriting reverse spans *) + +Record ByteSpan : Type := mkByteSpan { + span_offset : nat; + span_length : nat +}. + +Definition read_span (storage : list PhysicalByte) (span : ByteSpan) + : list PhysicalByte := + firstn (span_length span) (skipn (span_offset span) storage). + +Definition span_in_bounds + (storage : list PhysicalByte) (span : ByteSpan) : Prop := + span_offset span + span_length span <= List.length storage. + +Definition SpanEntry (I : FixedWidthCarrierProfile) := + (SymbolId I * ByteSpan)%type. + +Record PackedAtomStorage (I : FixedWidthCarrierProfile) : Type := + mkPackedAtomStorage { + packed_canonical_bytes : list PhysicalByte; + packed_reverse_spans : list (SpanEntry I) + }. + +Definition spans_disjoint (left right : ByteSpan) : Prop := + span_offset left + span_length left <= span_offset right \/ + span_offset right + span_length right <= span_offset left. + +Definition span_contains_offset (span : ByteSpan) (offset : nat) : Prop := + span_offset span <= offset < span_offset span + span_length span. + +Definition packed_spans_pairwise_disjoint + {I : FixedWidthCarrierProfile} + (storage : PackedAtomStorage I) : Prop := + forall left_id left_span right_id right_span, + In (left_id, left_span) (packed_reverse_spans I storage) -> + In (right_id, right_span) (packed_reverse_spans I storage) -> + left_id <> right_id -> + spans_disjoint left_span right_span. + +Definition packed_spans_cover_bytes + {I : FixedWidthCarrierProfile} + (storage : PackedAtomStorage I) : Prop := + forall offset, + offset < List.length (packed_canonical_bytes I storage) <-> + exists id span, + In (id, span) (packed_reverse_spans I storage) /\ + span_contains_offset span offset. + +Definition lookup_span + {I : FixedWidthCarrierProfile} + (storage : PackedAtomStorage I) (id : SymbolId I) + : option ByteSpan := + assoc_lookup (symbol_id_eq_dec I) (packed_reverse_spans I storage) id. + +Definition append_packed_atom + {P : CertifiedAtomProfile} {I : FixedWidthCarrierProfile} + (storage : PackedAtomStorage I) + (id : SymbolId I) + (atom : CanonicalAtom P) : option (PackedAtomStorage I) := + match lookup_span storage id with + | Some _ => None + | None => + let bytes := canonical_atom_bytes P atom in + let offset := List.length (packed_canonical_bytes I storage) in + Some + (mkPackedAtomStorage I + (packed_canonical_bytes I storage ++ bytes) + ((id, mkByteSpan offset (List.length bytes)) :: + packed_reverse_spans I storage)) + end. + +Lemma read_appended_suffix_exact : + forall prefix suffix, + read_span (prefix ++ suffix) + (mkByteSpan (List.length prefix) (List.length suffix)) = suffix. +Proof. + intros prefix suffix. + unfold read_span. simpl. + rewrite skipn_app, skipn_all, Nat.sub_diag. simpl. + apply firstn_all. +Qed. + +Lemma read_span_append_preserved : + forall prefix suffix span, + span_in_bounds prefix span -> + read_span (prefix ++ suffix) span = read_span prefix span. +Proof. + intros prefix suffix [offset count] Hbounds. + unfold span_in_bounds, read_span in *. simpl in *. + rewrite skipn_app. + replace (offset - List.length prefix) with 0 by lia. + simpl. + rewrite firstn_app. + replace (count - List.length (skipn offset prefix)) with 0. + 2: rewrite skipn_length_local; lia. + simpl. now rewrite app_nil_r. +Qed. + +Theorem VWENC_114_SAFE_PACKED_APPEND_READS_EXACT_CANONICAL_BYTES : + forall (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (storage updated : PackedAtomStorage I) + (id : SymbolId I) (atom : CanonicalAtom P), + append_packed_atom storage id atom = Some updated -> + exists span, + lookup_span updated id = Some span /\ + read_span (packed_canonical_bytes I updated) span = + canonical_atom_bytes P atom /\ + span_length span = List.length (canonical_atom_bytes P atom) /\ + span_in_bounds (packed_canonical_bytes I updated) span. +Proof. + intros P I storage updated id atom Happend. + unfold append_packed_atom in Happend. + destruct (lookup_span storage id) as [occupied |] eqn:Hlookup. + - discriminate. + - inversion Happend. subst updated. clear Happend. + exists + (mkByteSpan (List.length (packed_canonical_bytes I storage)) + (List.length (canonical_atom_bytes P atom))). + split. + + unfold lookup_span. simpl. + destruct (symbol_id_eq_dec I id id); [reflexivity | contradiction]. + + split. + * apply read_appended_suffix_exact. + * split; [reflexivity |]. + unfold span_in_bounds. simpl. rewrite app_length. lia. +Qed. + +Theorem VWENC_115_SAFE_PACKED_APPEND_PRESERVES_EXISTING_SPANS : + forall (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (storage updated : PackedAtomStorage I) + (new_id existing_id : SymbolId I) (atom : CanonicalAtom P), + append_packed_atom storage new_id atom = Some updated -> + existing_id <> new_id -> + lookup_span updated existing_id = lookup_span storage existing_id. +Proof. + intros P I storage updated new_id existing_id atom Happend Hdifferent. + unfold append_packed_atom in Happend. + destruct (lookup_span storage new_id); [discriminate |]. + inversion Happend. subst updated. clear Happend. + unfold lookup_span. simpl. + destruct (symbol_id_eq_dec I existing_id new_id); + [contradiction | reflexivity]. +Qed. + +(** ** Term dictionaries and the combined interning state *) + +Record TermDictionaryFiber + (P : CertifiedAtomProfile) + (I T : FixedWidthCarrierProfile) + (vocabulary_fiber : VocabularyFiber P I) : Type := + mkTermDictionaryFiber { + term_dictionary_identity : nat; + term_dictionary_generation : nat + }. + +Definition term_dictionary_fiber_identity + {P : CertifiedAtomProfile} + {I T : FixedWidthCarrierProfile} + {vocabulary_fiber : VocabularyFiber P I} + (fiber : TermDictionaryFiber P I T vocabulary_fiber) := + (vocabulary_fiber_identity vocabulary_fiber, + (carrier_format_identity T, + (carrier_width_bytes T, + (term_dictionary_identity P I T vocabulary_fiber fiber, + term_dictionary_generation P I T vocabulary_fiber fiber)))). + +Definition term_dictionary_fiber_eq_dec + (P : CertifiedAtomProfile) + (I T : FixedWidthCarrierProfile) + (vocabulary_fiber : VocabularyFiber P I) + (left right : TermDictionaryFiber P I T vocabulary_fiber) + : {left = right} + {left <> right}. +Proof. + destruct left as [left_identity left_generation]. + destruct right as [right_identity right_generation]. + destruct (Nat.eq_dec left_identity right_identity) + as [Hidentity | Hidentity]. + - subst right_identity. + destruct (Nat.eq_dec left_generation right_generation) + as [Hgeneration | Hgeneration]. + + subst right_generation. left. reflexivity. + + right. intros Hequal. inversion Hequal. contradiction. + - right. intros Hequal. inversion Hequal. contradiction. +Defined. + +Record FiberBoundTermId + (P : CertifiedAtomProfile) + (I T : FixedWidthCarrierProfile) + (vocabulary_fiber : VocabularyFiber P I) : Type := + mkFiberBoundTermId { + bound_term_fiber : TermDictionaryFiber P I T vocabulary_fiber; + bound_term_value : TermId T + }. + +Definition interpret_term_id + {P : CertifiedAtomProfile} + {I T : FixedWidthCarrierProfile} + {vocabulary_fiber : VocabularyFiber P I} + (expected : TermDictionaryFiber P I T vocabulary_fiber) + (bound : FiberBoundTermId P I T vocabulary_fiber) + : option (TermId T) := + if term_dictionary_fiber_eq_dec P I T vocabulary_fiber + expected (bound_term_fiber P I T vocabulary_fiber bound) + then Some (bound_term_value P I T vocabulary_fiber bound) + else None. + +Theorem VWENC_169_CROSS_TERM_FIBER_ID_INTERPRETATION_IS_REJECTED : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (vocabulary_fiber : VocabularyFiber P I) + (expected actual : TermDictionaryFiber P I T vocabulary_fiber) + (id : TermId T), + expected <> actual -> + interpret_term_id expected + (mkFiberBoundTermId P I T vocabulary_fiber actual id) = None. +Proof. + intros P I T vocabulary_fiber expected actual id Hdifferent. + unfold interpret_term_id. simpl. + destruct (term_dictionary_fiber_eq_dec + P I T vocabulary_fiber expected actual) as [Hequal | _]. + - contradiction. + - reflexivity. +Qed. + +Theorem VWENC_170_SAME_TERM_FIBER_ID_INTERPRETATION_IS_EXACT : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (vocabulary_fiber : VocabularyFiber P I) + (fiber : TermDictionaryFiber P I T vocabulary_fiber) + (id : TermId T), + interpret_term_id fiber + (mkFiberBoundTermId P I T vocabulary_fiber fiber id) = Some id. +Proof. + intros P I T vocabulary_fiber fiber id. + unfold interpret_term_id. simpl. + destruct (term_dictionary_fiber_eq_dec + P I T vocabulary_fiber fiber fiber) as [_ | Himpossible]. + - reflexivity. + - contradiction. +Qed. + +Definition symbol_sequence_eq_dec + (I : FixedWidthCarrierProfile) + : forall left right : list (SymbolId I), + {left = right} + {left <> right} := + list_eq_dec (symbol_id_eq_dec I). + +Definition TermEntry + (I T : FixedWidthCarrierProfile) := + (list (SymbolId I) * TermId T)%type. + +Definition reverse_term_entries + {I T : FixedWidthCarrierProfile} + (entries : list (TermEntry I T)) + : list (TermId T * list (SymbolId I)) := + map (fun entry => (snd entry, fst entry)) entries. + +Definition lookup_term_sequence + {I T : FixedWidthCarrierProfile} + (entries : list (TermEntry I T)) + (sequence : list (SymbolId I)) : option (TermId T) := + assoc_lookup (symbol_sequence_eq_dec I) entries sequence. + +Definition lookup_term_id + {I T : FixedWidthCarrierProfile} + (entries : list (TermEntry I T)) + (id : TermId T) : option (list (SymbolId I)) := + assoc_lookup (term_id_eq_dec T) (reverse_term_entries entries) id. + +Lemma reverse_term_membership : + forall (I T : FixedWidthCarrierProfile) + (entries : list (TermEntry I T)) sequence id, + In (id, sequence) (reverse_term_entries entries) <-> + In (sequence, id) entries. +Proof. + intros I T entries sequence id. + unfold reverse_term_entries. rewrite in_map_iff. + split. + - intros [[entry_sequence entry_id] [Hequal Hin]]. + simpl in Hequal. inversion Hequal. subst. exact Hin. + - intros Hin. exists (sequence, id). split; [reflexivity | exact Hin]. +Qed. + +Lemma reverse_term_keys_are_term_ids : + forall (I T : FixedWidthCarrierProfile) + (entries : list (TermEntry I T)), + map fst (reverse_term_entries entries) = map snd entries. +Proof. + intros I T entries. unfold reverse_term_entries. + rewrite map_map. apply map_ext. intros [sequence id]. reflexivity. +Qed. + +Definition term_relation_well_formed + {I T : FixedWidthCarrierProfile} + (entries : list (TermEntry I T)) : Prop := + NoDup (map fst entries) /\ NoDup (map snd entries). + +Definition packed_entry_exact + {P : CertifiedAtomProfile} {I : FixedWidthCarrierProfile} + (storage : PackedAtomStorage I) + (atom : CanonicalAtom P) (id : SymbolId I) : Prop := + exists span, + lookup_span storage id = Some span /\ + read_span (packed_canonical_bytes I storage) span = + canonical_atom_bytes P atom /\ + span_length span = List.length (canonical_atom_bytes P atom) /\ + 0 < span_length span /\ + span_in_bounds (packed_canonical_bytes I storage) span. + +Definition packed_storage_matches_allocations + {P : CertifiedAtomProfile} {I : FixedWidthCarrierProfile} + (allocations : list (VocabularyEntry P I)) + (storage : PackedAtomStorage I) : Prop := + NoDup (map fst (packed_reverse_spans I storage)) /\ + packed_spans_pairwise_disjoint storage /\ + packed_spans_cover_bytes storage /\ + (forall atom id, + In (atom, id) allocations -> + packed_entry_exact storage atom id) /\ + (forall id span, + In (id, span) (packed_reverse_spans I storage) -> + exists atom, In (atom, id) allocations). + +Lemma packed_reverse_span_in_bounds : + forall (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (allocations : list (VocabularyEntry P I)) storage id span, + packed_storage_matches_allocations allocations storage -> + In (id, span) (packed_reverse_spans I storage) -> + span_in_bounds (packed_canonical_bytes I storage) span. +Proof. + intros P I allocations storage id span + [Hunique [_ [_ [Hexact Hcomplete]]]] Hin. + destruct (Hcomplete id span Hin) as [atom Hallocation]. + destruct (Hexact atom id Hallocation) + as [exact_span [Hlookup [_ [_ [_ Hbounds]]]]]. + assert (Hmember_lookup : lookup_span storage id = Some span). + { unfold lookup_span. + eapply assoc_lookup_complete_unique; eassumption. } + rewrite Hmember_lookup in Hlookup. inversion Hlookup. + exact Hbounds. +Qed. + +Lemma lookup_span_none_id_absent : + forall (I : FixedWidthCarrierProfile) + (storage : PackedAtomStorage I) id, + lookup_span storage id = None -> + ~ In id (map fst (packed_reverse_spans I storage)). +Proof. + intros I storage id Hnone. + unfold lookup_span in Hnone. + now apply assoc_lookup_none_key_absent in Hnone. +Qed. + +Lemma packed_storage_matches_allocations_after_append : + forall (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (history : list (VocabularyEntry P I)) + (storage updated : PackedAtomStorage I) + (atom : CanonicalAtom P) (id : SymbolId I), + packed_storage_matches_allocations history storage -> + ~ In id (map snd history) -> + append_packed_atom storage id atom = Some updated -> + packed_storage_matches_allocations ((atom, id) :: history) updated. +Proof. + intros P I history storage updated atom id Hmatches + Hid_absent Happend. + pose proof Hmatches as Hmatches_for_bounds. + destruct Hmatches as + [Hspan_unique [Hspans_disjoint + [Hspans_cover [Hhistory_exact Hspan_complete]]]]. + unfold append_packed_atom in Happend. + destruct (lookup_span storage id) as [occupied |] eqn:Hid_span. + - discriminate. + - inversion Happend. subst updated. clear Happend. + split. + + simpl. constructor. + * now apply lookup_span_none_id_absent. + * exact Hspan_unique. + + split. + * unfold packed_spans_pairwise_disjoint in *. + intros left_id left_span right_id right_span + Hleft Hright Hdifferent. + simpl in Hleft, Hright. + destruct Hleft as [Hleft_new | Hleft_old]; + destruct Hright as [Hright_new | Hright_old]. + { inversion Hleft_new. inversion Hright_new. subst. + contradiction. } + { inversion Hleft_new. subst left_id left_span. + right. + pose proof + (packed_reverse_span_in_bounds + P I history storage right_id right_span + Hmatches_for_bounds Hright_old) as Hbounds. + unfold span_in_bounds in Hbounds. simpl in Hbounds |- *. + exact Hbounds. } + { inversion Hright_new. subst right_id right_span. + left. + pose proof + (packed_reverse_span_in_bounds + P I history storage left_id left_span + Hmatches_for_bounds Hleft_old) as Hbounds. + unfold span_in_bounds in Hbounds. simpl in Hbounds |- *. + exact Hbounds. } + { now apply Hspans_disjoint with (left_id := left_id) + (right_id := right_id). } + * split. + { unfold packed_spans_cover_bytes in *. + simpl. intros offset. rewrite app_length. split. + - intros Hbelow. + destruct (Nat.lt_ge_cases offset + (List.length (packed_canonical_bytes I storage))) + as [Hold | Hnew]. + + destruct (proj1 (Hspans_cover offset) Hold) + as [existing_id [span [Hin Hcontains]]]. + exists existing_id, span. split; [now right | exact Hcontains]. + + exists id, + (mkByteSpan + (List.length (packed_canonical_bytes I storage)) + (List.length (canonical_atom_bytes P atom))). + split; [now left |]. + unfold span_contains_offset. simpl. lia. + - intros [existing_id [span [Hin Hcontains]]]. + simpl in Hin. destruct Hin as [Hnew | Hold]. + + inversion Hnew. subst existing_id span. + unfold span_contains_offset in Hcontains. simpl in Hcontains. + lia. + + assert (Hbelow_old : + offset < List.length (packed_canonical_bytes I storage)). + { apply (proj2 (Hspans_cover offset)). + exists existing_id, span. now split. } + lia. } + { split. + - intros existing_atom existing_id Hin. + simpl in Hin. destruct Hin as [Hnew | Hold]. + { inversion Hnew. subst existing_atom existing_id. + exists + (mkByteSpan + (List.length (packed_canonical_bytes I storage)) + (List.length (canonical_atom_bytes P atom))). + split. + - unfold lookup_span. simpl. + destruct (symbol_id_eq_dec I id id); + [reflexivity | contradiction]. + - split. + + apply read_appended_suffix_exact. + + split; [reflexivity |]. + split. + * pose proof + (atom_codeword_nonempty P + (canonical_atom_bytes P atom) + (canonical_atom_valid P atom)) as Hnonempty. + destruct (canonical_atom_bytes P atom); + simpl; [contradiction | lia]. + * unfold span_in_bounds. simpl. rewrite app_length. lia. } + { specialize (Hhistory_exact existing_atom existing_id Hold). + destruct Hhistory_exact as + [span [Hlookup [Hread [Hlength [Hpositive Hbounds]]]]]. + assert (Hdifferent : existing_id <> id). + { intros Hequal. subst existing_id. apply Hid_absent. + apply in_map with (f := snd) in Hold. exact Hold. } + exists span. split. + - unfold lookup_span. simpl. + destruct (symbol_id_eq_dec I existing_id id); + [contradiction | exact Hlookup]. + - split. + + simpl. + rewrite read_span_append_preserved by exact Hbounds. + exact Hread. + + split; [exact Hlength |]. + split; [exact Hpositive |]. + unfold span_in_bounds in Hbounds |- *. simpl. + rewrite app_length. lia. } + - intros existing_id span Hin. + simpl in Hin. destruct Hin as [Hnew | Hold]. + { inversion Hnew. subst existing_id span. + exists atom. now left. } + { destruct (Hspan_complete existing_id span Hold) + as [existing_atom Hhistory]. + exists existing_atom. now right. } } +Qed. + +Lemma packed_storage_matches_allocations_permutation : + forall (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (left right : list (VocabularyEntry P I)) storage, + Permutation left right -> + packed_storage_matches_allocations left storage -> + packed_storage_matches_allocations right storage. +Proof. + intros P I left right storage Hpermutation + [Hspan_unique [Hdisjoint [Hcover [Hexact Hcomplete]]]]. + split; [exact Hspan_unique |]. + split; [exact Hdisjoint |]. + split; [exact Hcover |]. + split. + - intros atom id Hin. + apply Hexact. + eapply Permutation_in. + + exact (Permutation_sym Hpermutation). + + exact Hin. + - intros id span Hin. + destruct (Hcomplete id span Hin) as [atom Hleft]. + exists atom. + eapply Permutation_in. + + exact Hpermutation. + + exact Hleft. +Qed. + +Definition live_symbol + {P : CertifiedAtomProfile} {I : FixedWidthCarrierProfile} + (live : list (VocabularyEntry P I)) (id : SymbolId I) : Prop := + exists atom, In (atom, id) live. + +Definition sequence_vocabulary_bound + {P : CertifiedAtomProfile} {I : FixedWidthCarrierProfile} + (live : list (VocabularyEntry P I)) + (frontier : nat) (sequence : list (SymbolId I)) : Prop := + Forall + (fun id => + symbol_id_value I id < frontier /\ live_symbol live id) + sequence. + +Record InterningState + (P : CertifiedAtomProfile) + (I T : FixedWidthCarrierProfile) : Type := + mkInterningState { + state_fiber : VocabularyFiber P I; + state_term_fiber : TermDictionaryFiber P I T state_fiber; + state_reserved_entries : list (VocabularyEntry P I); + state_claimed_entries : list (VocabularyEntry P I); + state_live_entries : list (VocabularyEntry P I); + state_ever_entries : list (VocabularyEntry P I); + state_orphan_entries : list (VocabularyEntry P I); + state_unmaterialized_orphan_entries : list (VocabularyEntry P I); + state_packed_storage : PackedAtomStorage I; + state_allocator_frontier : nat; + state_sequences : list (list (SymbolId I)); + state_term_dictionary_enabled : bool; + state_term_entries : list (TermEntry I T) + }. + +Definition lookup_state_term_sequence + {P : CertifiedAtomProfile} + {I T : FixedWidthCarrierProfile} + (state : InterningState P I T) + (sequence : list (SymbolId I)) + : option + (FiberBoundTermId + P I T (state_fiber P I T state)) := + if state_term_dictionary_enabled P I T state then + match lookup_term_sequence + (state_term_entries P I T state) sequence with + | Some id => + Some + (mkFiberBoundTermId + P I T (state_fiber P I T state) + (state_term_fiber P I T state) id) + | None => None + end + else None. + +Theorem VWENC_171_TERM_LOOKUP_RETURNS_EXACT_FIBER_BOUND_ID : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state : InterningState P I T) sequence id, + state_term_dictionary_enabled P I T state = true -> + lookup_term_sequence + (state_term_entries P I T state) sequence = Some id -> + lookup_state_term_sequence state sequence = + Some + (mkFiberBoundTermId + P I T (state_fiber P I T state) + (state_term_fiber P I T state) id) /\ + interpret_term_id + (state_term_fiber P I T state) + (mkFiberBoundTermId + P I T (state_fiber P I T state) + (state_term_fiber P I T state) id) = Some id. +Proof. + intros P I T state sequence id Henabled Hlookup. + unfold lookup_state_term_sequence. rewrite Henabled, Hlookup. + split; [reflexivity |]. + apply VWENC_170_SAME_TERM_FIBER_ID_INTERPRETATION_IS_EXACT. +Qed. + +Definition state_allocation_entries + {P : CertifiedAtomProfile} + {I T : FixedWidthCarrierProfile} + (state : InterningState P I T) : list (VocabularyEntry P I) := + state_ever_entries P I T state ++ + state_reserved_entries P I T state ++ + state_claimed_entries P I T state ++ + state_orphan_entries P I T state ++ + state_unmaterialized_orphan_entries P I T state. + +Definition state_materialized_entries + {P : CertifiedAtomProfile} + {I T : FixedWidthCarrierProfile} + (state : InterningState P I T) : list (VocabularyEntry P I) := + state_ever_entries P I T state ++ + state_claimed_entries P I T state ++ + state_orphan_entries P I T state. + +Definition state_orphan_ids + {P : CertifiedAtomProfile} + {I T : FixedWidthCarrierProfile} + (state : InterningState P I T) : list (SymbolId I) := + map snd + (state_orphan_entries P I T state ++ + state_unmaterialized_orphan_entries P I T state). + +Inductive AllocationStatus := +| AllocationReserved +| AllocationMaterializedClaimed +| AllocationPublished +| AllocationTombstoned +| AllocationMaterializedOrphaned +| AllocationUnmaterializedOrphaned. + +Definition vocabulary_entry_eq_dec + (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + : forall left right : VocabularyEntry P I, + {left = right} + {left <> right}. +Proof. + intros [left_atom left_id] [right_atom right_id]. + destruct (canonical_atom_eq_dec P left_atom right_atom) + as [Hatom | Hatom]. + - subst right_atom. + destruct (symbol_id_eq_dec I left_id right_id) as [Hid | Hid]. + + subst right_id. left. reflexivity. + + right. intros Hequal. inversion Hequal. contradiction. + - right. intros Hequal. inversion Hequal. contradiction. +Defined. + +Definition allocation_status_of + {P : CertifiedAtomProfile} + {I T : FixedWidthCarrierProfile} + (state : InterningState P I T) + (atom : CanonicalAtom P) + (id : SymbolId I) : option AllocationStatus := + let entry := (atom, id) in + if in_dec (vocabulary_entry_eq_dec P I) entry + (state_reserved_entries P I T state) + then Some AllocationReserved + else if in_dec (vocabulary_entry_eq_dec P I) entry + (state_claimed_entries P I T state) + then Some AllocationMaterializedClaimed + else if in_dec (vocabulary_entry_eq_dec P I) entry + (state_orphan_entries P I T state) + then Some AllocationMaterializedOrphaned + else if in_dec (vocabulary_entry_eq_dec P I) entry + (state_unmaterialized_orphan_entries P I T state) + then Some AllocationUnmaterializedOrphaned + else if in_dec (vocabulary_entry_eq_dec P I) entry + (state_ever_entries P I T state) + then + if in_dec (vocabulary_entry_eq_dec P I) entry + (state_live_entries P I T state) + then Some AllocationPublished + else Some AllocationTombstoned + else None. + +Definition allocation_has_status + {P : CertifiedAtomProfile} + {I T : FixedWidthCarrierProfile} + (state : InterningState P I T) + (atom : CanonicalAtom P) + (id : SymbolId I) + (status : AllocationStatus) : Prop := + allocation_status_of state atom id = Some status. + +Definition allocation_status_category + {P : CertifiedAtomProfile} + {I T : FixedWidthCarrierProfile} + (state : InterningState P I T) + (atom : CanonicalAtom P) + (id : SymbolId I) + (status : AllocationStatus) : Prop := + match status with + | AllocationReserved => + In (atom, id) (state_reserved_entries P I T state) + | AllocationMaterializedClaimed => + In (atom, id) (state_claimed_entries P I T state) + | AllocationPublished => + In (atom, id) (state_live_entries P I T state) /\ + In (atom, id) (state_ever_entries P I T state) + | AllocationTombstoned => + In (atom, id) (state_ever_entries P I T state) /\ + ~ In (atom, id) (state_live_entries P I T state) + | AllocationMaterializedOrphaned => + In (atom, id) (state_orphan_entries P I T state) + | AllocationUnmaterializedOrphaned => + In (atom, id) + (state_unmaterialized_orphan_entries P I T state) + end. + +Lemma allocation_status_reserved_from_membership : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state : InterningState P I T) atom id, + In (atom, id) (state_reserved_entries P I T state) -> + allocation_has_status state atom id AllocationReserved. +Proof. + intros P I T state atom id Hin. + unfold allocation_has_status, allocation_status_of. + destruct (in_dec (vocabulary_entry_eq_dec P I) (atom, id) + (state_reserved_entries P I T state)); [reflexivity | contradiction]. +Qed. + +Lemma allocation_status_materialized_claimed_from_membership : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state : InterningState P I T) atom id, + ~ In (atom, id) (state_reserved_entries P I T state) -> + In (atom, id) (state_claimed_entries P I T state) -> + allocation_has_status state atom id AllocationMaterializedClaimed. +Proof. + intros P I T state atom id Hreserved Hclaimed. + unfold allocation_has_status, allocation_status_of. + destruct (in_dec (vocabulary_entry_eq_dec P I) (atom, id) + (state_reserved_entries P I T state)); [contradiction |]. + destruct (in_dec (vocabulary_entry_eq_dec P I) (atom, id) + (state_claimed_entries P I T state)); [reflexivity | contradiction]. +Qed. + +Lemma allocation_status_materialized_orphan_from_membership : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state : InterningState P I T) atom id, + ~ In (atom, id) (state_reserved_entries P I T state) -> + ~ In (atom, id) (state_claimed_entries P I T state) -> + In (atom, id) (state_orphan_entries P I T state) -> + allocation_has_status state atom id AllocationMaterializedOrphaned. +Proof. + intros P I T state atom id Hreserved Hclaimed Horphan. + unfold allocation_has_status, allocation_status_of. + destruct (in_dec (vocabulary_entry_eq_dec P I) (atom, id) + (state_reserved_entries P I T state)); [contradiction |]. + destruct (in_dec (vocabulary_entry_eq_dec P I) (atom, id) + (state_claimed_entries P I T state)); [contradiction |]. + destruct (in_dec (vocabulary_entry_eq_dec P I) (atom, id) + (state_orphan_entries P I T state)); [reflexivity | contradiction]. +Qed. + +Lemma allocation_status_unmaterialized_orphan_from_membership : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state : InterningState P I T) atom id, + ~ In (atom, id) (state_reserved_entries P I T state) -> + ~ In (atom, id) (state_claimed_entries P I T state) -> + ~ In (atom, id) (state_orphan_entries P I T state) -> + In (atom, id) + (state_unmaterialized_orphan_entries P I T state) -> + allocation_has_status state atom id AllocationUnmaterializedOrphaned. +Proof. + intros P I T state atom id Hreserved Hclaimed Horphan Hunmaterialized. + unfold allocation_has_status, allocation_status_of. + destruct (in_dec (vocabulary_entry_eq_dec P I) (atom, id) + (state_reserved_entries P I T state)); [contradiction |]. + destruct (in_dec (vocabulary_entry_eq_dec P I) (atom, id) + (state_claimed_entries P I T state)); [contradiction |]. + destruct (in_dec (vocabulary_entry_eq_dec P I) (atom, id) + (state_orphan_entries P I T state)); [contradiction |]. + destruct (in_dec (vocabulary_entry_eq_dec P I) (atom, id) + (state_unmaterialized_orphan_entries P I T state)); + [reflexivity | contradiction]. +Qed. + +Lemma allocation_status_published_from_membership : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state : InterningState P I T) atom id, + ~ In (atom, id) (state_reserved_entries P I T state) -> + ~ In (atom, id) (state_claimed_entries P I T state) -> + ~ In (atom, id) (state_orphan_entries P I T state) -> + ~ In (atom, id) + (state_unmaterialized_orphan_entries P I T state) -> + In (atom, id) (state_ever_entries P I T state) -> + In (atom, id) (state_live_entries P I T state) -> + allocation_has_status state atom id AllocationPublished. +Proof. + intros P I T state atom id + Hreserved Hclaimed Horphan Hunmaterialized Hever Hlive. + unfold allocation_has_status, allocation_status_of. + destruct (in_dec (vocabulary_entry_eq_dec P I) (atom, id) + (state_reserved_entries P I T state)); [contradiction |]. + destruct (in_dec (vocabulary_entry_eq_dec P I) (atom, id) + (state_claimed_entries P I T state)); [contradiction |]. + destruct (in_dec (vocabulary_entry_eq_dec P I) (atom, id) + (state_orphan_entries P I T state)); [contradiction |]. + destruct (in_dec (vocabulary_entry_eq_dec P I) (atom, id) + (state_unmaterialized_orphan_entries P I T state)); + [contradiction |]. + destruct (in_dec (vocabulary_entry_eq_dec P I) (atom, id) + (state_ever_entries P I T state)); [| contradiction]. + destruct (in_dec (vocabulary_entry_eq_dec P I) (atom, id) + (state_live_entries P I T state)); [reflexivity | contradiction]. +Qed. + +Lemma allocation_status_tombstoned_from_membership : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state : InterningState P I T) atom id, + ~ In (atom, id) (state_reserved_entries P I T state) -> + ~ In (atom, id) (state_claimed_entries P I T state) -> + ~ In (atom, id) (state_orphan_entries P I T state) -> + ~ In (atom, id) + (state_unmaterialized_orphan_entries P I T state) -> + In (atom, id) (state_ever_entries P I T state) -> + ~ In (atom, id) (state_live_entries P I T state) -> + allocation_has_status state atom id AllocationTombstoned. +Proof. + intros P I T state atom id + Hreserved Hclaimed Horphan Hunmaterialized Hever Hlive. + unfold allocation_has_status, allocation_status_of. + destruct (in_dec (vocabulary_entry_eq_dec P I) (atom, id) + (state_reserved_entries P I T state)); [contradiction |]. + destruct (in_dec (vocabulary_entry_eq_dec P I) (atom, id) + (state_claimed_entries P I T state)); [contradiction |]. + destruct (in_dec (vocabulary_entry_eq_dec P I) (atom, id) + (state_orphan_entries P I T state)); [contradiction |]. + destruct (in_dec (vocabulary_entry_eq_dec P I) (atom, id) + (state_unmaterialized_orphan_entries P I T state)); + [contradiction |]. + destruct (in_dec (vocabulary_entry_eq_dec P I) (atom, id) + (state_ever_entries P I T state)); [| contradiction]. + destruct (in_dec (vocabulary_entry_eq_dec P I) (atom, id) + (state_live_entries P I T state)); [contradiction | reflexivity]. +Qed. + +Theorem VWENC_161_ALLOCATION_STATUS_IS_FUNCTIONALLY_UNIQUE : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state : InterningState P I T) atom id left right, + allocation_has_status state atom id left -> + allocation_has_status state atom id right -> + left = right. +Proof. + intros P I T state atom id left right Hleft Hright. + unfold allocation_has_status in *. rewrite Hleft in Hright. + now inversion Hright. +Qed. + +Lemma allocated_entry_has_computed_status : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state : InterningState P I T) atom id, + In (atom, id) (state_allocation_entries state) -> + exists status, allocation_has_status state atom id status. +Proof. + intros P I T state atom id Hallocated. + unfold allocation_has_status, allocation_status_of. + destruct (in_dec (vocabulary_entry_eq_dec P I) (atom, id) + (state_reserved_entries P I T state)) as [Hreserved | Hreserved]. + - now exists AllocationReserved. + - destruct (in_dec (vocabulary_entry_eq_dec P I) (atom, id) + (state_claimed_entries P I T state)) as [Hclaimed | Hclaimed]. + + now exists AllocationMaterializedClaimed. + + destruct (in_dec (vocabulary_entry_eq_dec P I) (atom, id) + (state_orphan_entries P I T state)) as [Horphan | Horphan]. + * now exists AllocationMaterializedOrphaned. + * destruct (in_dec (vocabulary_entry_eq_dec P I) (atom, id) + (state_unmaterialized_orphan_entries P I T state)) + as [Hunmaterialized | Hunmaterialized]. + { now exists AllocationUnmaterializedOrphaned. } + { destruct (in_dec (vocabulary_entry_eq_dec P I) (atom, id) + (state_ever_entries P I T state)) as [Hever | Hever]. + - destruct (in_dec (vocabulary_entry_eq_dec P I) (atom, id) + (state_live_entries P I T state)) as [Hlive | Hlive]. + + now exists AllocationPublished. + + now exists AllocationTombstoned. + - exfalso. unfold state_allocation_entries in Hallocated. + repeat rewrite in_app_iff in Hallocated. + tauto. } +Qed. + +Lemma allocation_status_reports_observable_membership : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state : InterningState P I T) atom id status, + allocation_has_status state atom id status -> + match status with + | AllocationReserved => + In (atom, id) (state_reserved_entries P I T state) + | AllocationMaterializedClaimed => + In (atom, id) (state_claimed_entries P I T state) + | AllocationPublished => + In (atom, id) (state_live_entries P I T state) + | AllocationTombstoned => + In (atom, id) (state_ever_entries P I T state) /\ + ~ In (atom, id) (state_live_entries P I T state) + | AllocationMaterializedOrphaned => + In (atom, id) (state_orphan_entries P I T state) + | AllocationUnmaterializedOrphaned => + In (atom, id) + (state_unmaterialized_orphan_entries P I T state) + end. +Proof. + intros P I T state atom id status Hstatus. + unfold allocation_has_status, allocation_status_of in Hstatus. + destruct (in_dec (vocabulary_entry_eq_dec P I) (atom, id) + (state_reserved_entries P I T state)) as [Hreserved | Hreserved]. + - inversion Hstatus. exact Hreserved. + - destruct (in_dec (vocabulary_entry_eq_dec P I) (atom, id) + (state_claimed_entries P I T state)) as [Hclaimed | Hclaimed]. + + inversion Hstatus. exact Hclaimed. + + destruct (in_dec (vocabulary_entry_eq_dec P I) (atom, id) + (state_orphan_entries P I T state)) as [Horphan | Horphan]. + * inversion Hstatus. exact Horphan. + * destruct (in_dec (vocabulary_entry_eq_dec P I) (atom, id) + (state_unmaterialized_orphan_entries P I T state)) + as [Hunmaterialized | Hunmaterialized]. + { inversion Hstatus. exact Hunmaterialized. } + { destruct (in_dec (vocabulary_entry_eq_dec P I) (atom, id) + (state_ever_entries P I T state)) as [Hever | Hever]. + - destruct (in_dec (vocabulary_entry_eq_dec P I) (atom, id) + (state_live_entries P I T state)) as [Hlive | Hlive]. + + inversion Hstatus. exact Hlive. + + inversion Hstatus. now split. + - discriminate. } +Qed. + +Record InterningStateWellFormed + {P : CertifiedAtomProfile} + {I T : FixedWidthCarrierProfile} + (state : InterningState P I T) : Prop := + mkInterningStateWellFormed { + state_live_bijection : + vocabulary_relation_well_formed (state_live_entries P I T state); + state_history_bijection : + vocabulary_relation_well_formed (state_ever_entries P I T state); + state_live_is_historical : + forall atom id, + In (atom, id) (state_live_entries P I T state) -> + In (atom, id) (state_ever_entries P I T state); + state_allocation_ids_unique : + NoDup (map snd (state_allocation_entries state)); + state_packed_allocations_exact : + packed_storage_matches_allocations + (state_materialized_entries state) + (state_packed_storage P I T state); + state_allocations_below_sparse_frontier : + Forall + (fun entry => + symbol_id_value I (snd entry) < + state_allocator_frontier P I T state) + (state_allocation_entries state); + state_frontier_representable : + state_allocator_frontier P I T state <= carrier_capacity I; + state_all_sequences_bound : + Forall + (sequence_vocabulary_bound + (state_live_entries P I T state) + (state_allocator_frontier P I T state)) + (state_sequences P I T state); + state_term_relation_bijection : + term_relation_well_formed (state_term_entries P I T state); + state_term_sequences_bound : + Forall + (fun entry => + sequence_vocabulary_bound + (state_live_entries P I T state) + (state_allocator_frontier P I T state) + (fst entry)) + (state_term_entries P I T state); + state_disabled_term_dictionary_is_empty : + state_term_dictionary_enabled P I T state = false -> + state_term_entries P I T state = [] + }. + +Lemma NoDup_in_separated_segments : + forall (Element : Type) + (prefix left middle right suffix : list Element) element, + NoDup (prefix ++ left ++ middle ++ right ++ suffix) -> + In element left -> + In element right -> + False. +Proof. + intros Element prefix left middle right suffix element + Hunique Hleft Hright. + destruct (in_split element left Hleft) + as [before [after Hsplit]]. + subst left. + assert (Hshape : + prefix ++ (before ++ element :: after) ++ middle ++ right ++ suffix = + (prefix ++ before) ++ + element :: (after ++ middle ++ right ++ suffix)). + { repeat rewrite <- app_assoc. reflexivity. } + rewrite Hshape in Hunique. + pose proof + (NoDup_remove_2 + (prefix ++ before) + (after ++ middle ++ right ++ suffix) + element Hunique) as Hnot_in_remainder. + apply Hnot_in_remainder. + apply in_or_app. right. + apply in_or_app. right. + apply in_or_app. right. + apply in_or_app. left. exact Hright. +Qed. + +Inductive AllocationBucket := +| BucketHistorical +| BucketReserved +| BucketMaterializedClaimed +| BucketMaterializedOrphaned +| BucketUnmaterializedOrphaned. + +Definition allocation_bucket_entries + {P : CertifiedAtomProfile} + {I T : FixedWidthCarrierProfile} + (state : InterningState P I T) + (bucket : AllocationBucket) : list (VocabularyEntry P I) := + match bucket with + | BucketHistorical => state_ever_entries P I T state + | BucketReserved => state_reserved_entries P I T state + | BucketMaterializedClaimed => state_claimed_entries P I T state + | BucketMaterializedOrphaned => state_orphan_entries P I T state + | BucketUnmaterializedOrphaned => + state_unmaterialized_orphan_entries P I T state + end. + +Inductive AllocationBucketPrecedes : + AllocationBucket -> AllocationBucket -> Prop := +| HistoricalBeforeReserved : + AllocationBucketPrecedes BucketHistorical BucketReserved +| HistoricalBeforeClaimed : + AllocationBucketPrecedes BucketHistorical BucketMaterializedClaimed +| HistoricalBeforeMaterializedOrphan : + AllocationBucketPrecedes BucketHistorical BucketMaterializedOrphaned +| HistoricalBeforeUnmaterializedOrphan : + AllocationBucketPrecedes BucketHistorical BucketUnmaterializedOrphaned +| ReservedBeforeClaimed : + AllocationBucketPrecedes BucketReserved BucketMaterializedClaimed +| ReservedBeforeMaterializedOrphan : + AllocationBucketPrecedes BucketReserved BucketMaterializedOrphaned +| ReservedBeforeUnmaterializedOrphan : + AllocationBucketPrecedes BucketReserved BucketUnmaterializedOrphaned +| ClaimedBeforeMaterializedOrphan : + AllocationBucketPrecedes + BucketMaterializedClaimed BucketMaterializedOrphaned +| ClaimedBeforeUnmaterializedOrphan : + AllocationBucketPrecedes + BucketMaterializedClaimed BucketUnmaterializedOrphaned +| MaterializedOrphanBeforeUnmaterializedOrphan : + AllocationBucketPrecedes + BucketMaterializedOrphaned BucketUnmaterializedOrphaned. + +Lemma allocation_bucket_precedence_makes_ids_disjoint : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state : InterningState P I T) left right left_atom right_atom id, + InterningStateWellFormed state -> + AllocationBucketPrecedes left right -> + In (left_atom, id) (allocation_bucket_entries state left) -> + In (right_atom, id) (allocation_bucket_entries state right) -> + False. +Proof. + intros P I T state left right left_atom right_atom id + Hwell Hprecedes Hleft Hright. + destruct Hwell as [_ _ _ Hunique]. + unfold state_allocation_entries in Hunique. + repeat rewrite map_app in Hunique. + repeat rewrite <- app_assoc in Hunique. + assert (Hleft_id : + In id (map snd (allocation_bucket_entries state left))). + { now apply in_map with (f := snd) in Hleft. } + assert (Hright_id : + In id (map snd (allocation_bucket_entries state right))). + { now apply in_map with (f := snd) in Hright. } + destruct Hprecedes; simpl in Hleft_id, Hright_id. + - eapply NoDup_in_separated_segments + with + (prefix := []) + (left := map snd (state_ever_entries P I T state)) + (middle := []) + (right := map snd (state_reserved_entries P I T state)) + (suffix := + map snd (state_claimed_entries P I T state) ++ + map snd (state_orphan_entries P I T state) ++ + map snd (state_unmaterialized_orphan_entries P I T state)) + (element := id); simpl; repeat rewrite <- app_assoc; simpl; + try rewrite app_nil_r; eauto. + - eapply NoDup_in_separated_segments + with + (prefix := []) + (left := map snd (state_ever_entries P I T state)) + (middle := map snd (state_reserved_entries P I T state)) + (right := map snd (state_claimed_entries P I T state)) + (suffix := + map snd (state_orphan_entries P I T state) ++ + map snd (state_unmaterialized_orphan_entries P I T state)) + (element := id); simpl; repeat rewrite <- app_assoc; simpl; + try rewrite app_nil_r; eauto. + - eapply NoDup_in_separated_segments + with + (prefix := []) + (left := map snd (state_ever_entries P I T state)) + (middle := + map snd (state_reserved_entries P I T state) ++ + map snd (state_claimed_entries P I T state)) + (right := map snd (state_orphan_entries P I T state)) + (suffix := + map snd (state_unmaterialized_orphan_entries P I T state)) + (element := id); simpl; repeat rewrite <- app_assoc; simpl; + try rewrite app_nil_r; eauto. + - eapply NoDup_in_separated_segments + with + (prefix := []) + (left := map snd (state_ever_entries P I T state)) + (middle := + map snd (state_reserved_entries P I T state) ++ + map snd (state_claimed_entries P I T state) ++ + map snd (state_orphan_entries P I T state)) + (right := + map snd (state_unmaterialized_orphan_entries P I T state)) + (suffix := []) + (element := id); simpl; repeat rewrite <- app_assoc; simpl; + try rewrite app_nil_r; eauto. + - eapply NoDup_in_separated_segments + with + (prefix := map snd (state_ever_entries P I T state)) + (left := map snd (state_reserved_entries P I T state)) + (middle := []) + (right := map snd (state_claimed_entries P I T state)) + (suffix := + map snd (state_orphan_entries P I T state) ++ + map snd (state_unmaterialized_orphan_entries P I T state)) + (element := id); simpl; repeat rewrite <- app_assoc; simpl; + try rewrite app_nil_r; eauto. + - eapply NoDup_in_separated_segments + with + (prefix := map snd (state_ever_entries P I T state)) + (left := map snd (state_reserved_entries P I T state)) + (middle := map snd (state_claimed_entries P I T state)) + (right := map snd (state_orphan_entries P I T state)) + (suffix := + map snd (state_unmaterialized_orphan_entries P I T state)) + (element := id); simpl; repeat rewrite <- app_assoc; simpl; + try rewrite app_nil_r; eauto. + - eapply NoDup_in_separated_segments + with + (prefix := map snd (state_ever_entries P I T state)) + (left := map snd (state_reserved_entries P I T state)) + (middle := + map snd (state_claimed_entries P I T state) ++ + map snd (state_orphan_entries P I T state)) + (right := + map snd (state_unmaterialized_orphan_entries P I T state)) + (suffix := []) + (element := id); simpl; repeat rewrite <- app_assoc; simpl; + try rewrite app_nil_r; eauto. + - eapply NoDup_in_separated_segments + with + (prefix := + map snd (state_ever_entries P I T state) ++ + map snd (state_reserved_entries P I T state)) + (left := map snd (state_claimed_entries P I T state)) + (middle := []) + (right := map snd (state_orphan_entries P I T state)) + (suffix := + map snd (state_unmaterialized_orphan_entries P I T state)) + (element := id); simpl; repeat rewrite <- app_assoc; simpl; + try rewrite app_nil_r; eauto. + - eapply NoDup_in_separated_segments + with + (prefix := + map snd (state_ever_entries P I T state) ++ + map snd (state_reserved_entries P I T state)) + (left := map snd (state_claimed_entries P I T state)) + (middle := map snd (state_orphan_entries P I T state)) + (right := + map snd (state_unmaterialized_orphan_entries P I T state)) + (suffix := []) + (element := id); simpl; repeat rewrite <- app_assoc; simpl; + try rewrite app_nil_r; eauto. + - eapply NoDup_in_separated_segments + with + (prefix := + map snd (state_ever_entries P I T state) ++ + map snd (state_reserved_entries P I T state) ++ + map snd (state_claimed_entries P I T state)) + (left := map snd (state_orphan_entries P I T state)) + (middle := []) + (right := + map snd (state_unmaterialized_orphan_entries P I T state)) + (suffix := []) + (element := id); simpl; repeat rewrite <- app_assoc; simpl; + try rewrite app_nil_r; eauto. +Qed. + +Theorem VWENC_163_ALLOCATION_STATUS_REPORTS_ITS_EXACT_STATE_CATEGORY : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state : InterningState P I T) atom id status, + InterningStateWellFormed state -> + (allocation_has_status state atom id status <-> + allocation_status_category state atom id status). +Proof. + intros P I T state atom id status Hwell. + split. + - intros Hstatus. + pose proof (allocation_status_reports_observable_membership + P I T state atom id status Hstatus) as Hmembership. + destruct status; simpl in *; try exact Hmembership. + split; [exact Hmembership |]. + now apply (state_live_is_historical state Hwell). + - intros Hcategory. + assert (Hdisjoint := allocation_bucket_precedence_makes_ids_disjoint + P I T state). + destruct status; simpl in Hcategory. + + now apply allocation_status_reserved_from_membership. + + apply allocation_status_materialized_claimed_from_membership. + * intros Hreserved. + eapply Hdisjoint with + (left := BucketReserved) + (right := BucketMaterializedClaimed); eauto using ReservedBeforeClaimed. + * exact Hcategory. + + destruct Hcategory as [Hlive Hever]. + apply allocation_status_published_from_membership; try assumption. + * intros Hreserved. + eapply Hdisjoint with + (left := BucketHistorical) (right := BucketReserved); + eauto using HistoricalBeforeReserved. + * intros Hclaimed. + eapply Hdisjoint with + (left := BucketHistorical) (right := BucketMaterializedClaimed); + eauto using HistoricalBeforeClaimed. + * intros Horphan. + eapply Hdisjoint with + (left := BucketHistorical) (right := BucketMaterializedOrphaned); + eauto using HistoricalBeforeMaterializedOrphan. + * intros Hunmaterialized. + eapply Hdisjoint with + (left := BucketHistorical) + (right := BucketUnmaterializedOrphaned); + eauto using HistoricalBeforeUnmaterializedOrphan. + + destruct Hcategory as [Hever Hnot_live]. + apply allocation_status_tombstoned_from_membership; try assumption. + * intros Hreserved. + eapply Hdisjoint with + (left := BucketHistorical) (right := BucketReserved); + eauto using HistoricalBeforeReserved. + * intros Hclaimed. + eapply Hdisjoint with + (left := BucketHistorical) (right := BucketMaterializedClaimed); + eauto using HistoricalBeforeClaimed. + * intros Horphan. + eapply Hdisjoint with + (left := BucketHistorical) (right := BucketMaterializedOrphaned); + eauto using HistoricalBeforeMaterializedOrphan. + * intros Hunmaterialized. + eapply Hdisjoint with + (left := BucketHistorical) + (right := BucketUnmaterializedOrphaned); + eauto using HistoricalBeforeUnmaterializedOrphan. + + apply allocation_status_materialized_orphan_from_membership. + * intros Hreserved. + eapply Hdisjoint with + (left := BucketReserved) (right := BucketMaterializedOrphaned); + eauto using ReservedBeforeMaterializedOrphan. + * intros Hclaimed. + eapply Hdisjoint with + (left := BucketMaterializedClaimed) + (right := BucketMaterializedOrphaned); + eauto using ClaimedBeforeMaterializedOrphan. + * exact Hcategory. + + apply allocation_status_unmaterialized_orphan_from_membership. + * intros Hreserved. + eapply Hdisjoint with + (left := BucketReserved) + (right := BucketUnmaterializedOrphaned); + eauto using ReservedBeforeUnmaterializedOrphan. + * intros Hclaimed. + eapply Hdisjoint with + (left := BucketMaterializedClaimed) + (right := BucketUnmaterializedOrphaned); + eauto using ClaimedBeforeUnmaterializedOrphan. + * intros Horphan. + eapply Hdisjoint with + (left := BucketMaterializedOrphaned) + (right := BucketUnmaterializedOrphaned); + eauto using MaterializedOrphanBeforeUnmaterializedOrphan. + * exact Hcategory. +Qed. + +Theorem VWENC_162_EVERY_ALLOCATED_ENTRY_HAS_ONE_AUTHORITATIVE_STATUS : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state : InterningState P I T) atom id, + InterningStateWellFormed state -> + In (atom, id) (state_allocation_entries state) -> + exists! status, allocation_status_category state atom id status. +Proof. + intros P I T state atom id Hwell Hallocated. + destruct (allocated_entry_has_computed_status + P I T state atom id Hallocated) as [status Hstatus]. + exists status. + split. + - now apply (proj1 + (VWENC_163_ALLOCATION_STATUS_REPORTS_ITS_EXACT_STATE_CATEGORY + P I T state atom id status Hwell)). + - intros other Hother. + eapply VWENC_161_ALLOCATION_STATUS_IS_FUNCTIONALLY_UNIQUE. + + exact Hstatus. + + now apply (proj2 + (VWENC_163_ALLOCATION_STATUS_REPORTS_ITS_EXACT_STATE_CATEGORY + P I T state atom id other Hwell)). +Qed. + +Lemma NoDup_map_members_with_same_image_are_equal : + forall (Element Image : Type) (project : Element -> Image) + (values : list Element) left right, + NoDup (map project values) -> + In left values -> + In right values -> + project left = project right -> + left = right. +Proof. + intros Element Image project values. + induction values as [| head tail IH]; intros left right + Hunique Hleft Hright Himage. + - contradiction. + - inversion Hunique as [| projected projected_tail + Hhead_absent Htail_unique]; subst. + simpl in Hleft, Hright. + destruct Hleft as [Hleft | Hleft]; + destruct Hright as [Hright | Hright]. + + now subst left; subst right. + + subst left. exfalso. apply Hhead_absent. + apply in_map_iff. exists right. split; [symmetry | assumption]. + exact Himage. + + subst right. exfalso. apply Hhead_absent. + apply in_map_iff. exists left. now split. + + exact (IH left right Htail_unique Hleft Hright Himage). +Qed. + +Lemma allocation_status_category_entry_is_allocated : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state : InterningState P I T) atom id status, + allocation_status_category state atom id status -> + In (atom, id) (state_allocation_entries state). +Proof. + intros P I T state atom id status Hcategory. + unfold state_allocation_entries. + repeat rewrite in_app_iff. + destruct status; simpl in Hcategory; tauto. +Qed. + +Theorem VWENC_188_ALLOCATION_STATUS_CATEGORIES_ARE_PAIRWISE_DISJOINT_BY_ID : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state : InterningState P I T) + left_atom right_atom id left_status right_status, + InterningStateWellFormed state -> + allocation_status_category state left_atom id left_status -> + allocation_status_category state right_atom id right_status -> + left_atom = right_atom /\ left_status = right_status. +Proof. + intros P I T state left_atom right_atom id left_status right_status + Hwell Hleft_category Hright_category. + assert (Hleft_allocated : + In (left_atom, id) (state_allocation_entries state)). + { now apply allocation_status_category_entry_is_allocated + with (status := left_status). } + assert (Hright_allocated : + In (right_atom, id) (state_allocation_entries state)). + { now apply allocation_status_category_entry_is_allocated + with (status := right_status). } + assert (Hentry : (left_atom, id) = (right_atom, id)). + { eapply NoDup_map_members_with_same_image_are_equal + with (project := snd) + (values := state_allocation_entries state). + - exact (state_allocation_ids_unique state Hwell). + - exact Hleft_allocated. + - exact Hright_allocated. + - reflexivity. } + inversion Hentry. subst right_atom. + split; [reflexivity |]. + eapply VWENC_161_ALLOCATION_STATUS_IS_FUNCTIONALLY_UNIQUE. + - now apply (proj2 + (VWENC_163_ALLOCATION_STATUS_REPORTS_ITS_EXACT_STATE_CATEGORY + P I T state left_atom id left_status Hwell)). + - now apply (proj2 + (VWENC_163_ALLOCATION_STATUS_REPORTS_ITS_EXACT_STATE_CATEGORY + P I T state left_atom id right_status Hwell)). +Qed. + +Definition empty_packed_atom_storage + (I : FixedWidthCarrierProfile) : PackedAtomStorage I := + mkPackedAtomStorage I [] []. + +Definition empty_interning_state + (P : CertifiedAtomProfile) + (I T : FixedWidthCarrierProfile) + (fiber : VocabularyFiber P I) + (term_identity term_generation : nat) : InterningState P I T := + {| state_fiber := fiber; + state_term_fiber := + mkTermDictionaryFiber P I T fiber term_identity term_generation; + state_reserved_entries := []; + state_claimed_entries := []; + state_live_entries := []; + state_ever_entries := []; + state_orphan_entries := []; + state_unmaterialized_orphan_entries := []; + state_packed_storage := empty_packed_atom_storage I; + state_allocator_frontier := 0; + state_sequences := []; + state_term_dictionary_enabled := false; + state_term_entries := [] |}. + +Theorem VWENC_157_EMPTY_INTERNING_STATE_IS_WELL_FORMED : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (fiber : VocabularyFiber P I) term_identity term_generation, + InterningStateWellFormed + (empty_interning_state + P I T fiber term_identity term_generation). +Proof. + intros P I T fiber term_identity term_generation. constructor; simpl. + - split; constructor. + - split; constructor. + - intros atom id Hin. contradiction. + - constructor. + - split. + + constructor. + + split. + * unfold packed_spans_pairwise_disjoint. simpl. + intros left_id left_span right_id right_span Hleft. + contradiction. + * split. + { unfold packed_spans_cover_bytes. simpl. intros offset. split. + - lia. + - intros [id [span [Hin _]]]. contradiction. } + { split. + - intros atom id Hin. contradiction. + - intros id span Hin. contradiction. } + - constructor. + - pose proof (carrier_capacity_positive I). lia. + - constructor. + - split; constructor. + - constructor. + - intros _. reflexivity. +Qed. + +Theorem VWENC_158_WELL_FORMED_PACKED_SPANS_ARE_DISJOINT_AND_COVER_EXACTLY : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state : InterningState P I T), + InterningStateWellFormed state -> + packed_spans_pairwise_disjoint + (state_packed_storage P I T state) /\ + packed_spans_cover_bytes + (state_packed_storage P I T state). +Proof. + intros P I T state Hwell. + destruct Hwell as [_ _ _ _ Hpacked]. + destruct Hpacked as [_ [Hdisjoint [Hcover _]]]. + now split. +Qed. + +Definition publish_fresh_atom + {P : CertifiedAtomProfile} + {I T : FixedWidthCarrierProfile} + (state : InterningState P I T) + (atom : CanonicalAtom P) + (id : SymbolId I) : option (InterningState P I T) := + match lookup_atom (state_ever_entries P I T state) atom with + | Some _ => None + | None => + match lookup_symbol (state_ever_entries P I T state) id with + | Some _ => None + | None => + if Nat.leb + (state_allocator_frontier P I T state) + (symbol_id_value I id) + then + match append_packed_atom + (state_packed_storage P I T state) id atom with + | None => None + | Some packed => + Some + {| state_fiber := state_fiber P I T state; + state_term_fiber := state_term_fiber P I T state; + state_reserved_entries := + state_reserved_entries P I T state; + state_claimed_entries := + state_claimed_entries P I T state; + state_live_entries := + (atom, id) :: state_live_entries P I T state; + state_ever_entries := + (atom, id) :: state_ever_entries P I T state; + state_orphan_entries := + state_orphan_entries P I T state; + state_unmaterialized_orphan_entries := + state_unmaterialized_orphan_entries P I T state; + state_packed_storage := packed; + state_allocator_frontier := + S (symbol_id_value I id); + state_sequences := state_sequences P I T state; + state_term_dictionary_enabled := + state_term_dictionary_enabled P I T state; + state_term_entries := state_term_entries P I T state |} + end + else None + end + end. + +Definition claim_atom_allocation + {P : CertifiedAtomProfile} + {I T : FixedWidthCarrierProfile} + (state : InterningState P I T) + (atom : CanonicalAtom P) + (id : SymbolId I) : option (InterningState P I T) := + match lookup_atom (state_ever_entries P I T state) atom with + | Some _ => None + | None => + if Nat.leb + (state_allocator_frontier P I T state) + (symbol_id_value I id) + then + Some + {| state_fiber := state_fiber P I T state; + state_term_fiber := state_term_fiber P I T state; + state_reserved_entries := + (atom, id) :: state_reserved_entries P I T state; + state_claimed_entries := state_claimed_entries P I T state; + state_live_entries := state_live_entries P I T state; + state_ever_entries := state_ever_entries P I T state; + state_orphan_entries := state_orphan_entries P I T state; + state_unmaterialized_orphan_entries := + state_unmaterialized_orphan_entries P I T state; + state_packed_storage := state_packed_storage P I T state; + state_allocator_frontier := S (symbol_id_value I id); + state_sequences := state_sequences P I T state; + state_term_dictionary_enabled := + state_term_dictionary_enabled P I T state; + state_term_entries := state_term_entries P I T state |} + else None + end. + +Theorem VWENC_164_ALLOCATED_IDS_CANNOT_BE_RESERVED_OR_FRESHLY_PUBLISHED_AGAIN : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state : InterningState P I T) existing_atom new_atom id, + InterningStateWellFormed state -> + In (existing_atom, id) (state_allocation_entries state) -> + claim_atom_allocation state new_atom id = None /\ + publish_fresh_atom state new_atom id = None. +Proof. + intros P I T state existing_atom new_atom id Hwell Hallocated. + destruct Hwell as [_ _ _ _ _ Hbelow]. + apply Forall_forall with (x := (existing_atom, id)) in Hbelow; + [| exact Hallocated]. + simpl in Hbelow. split. + - unfold claim_atom_allocation. + destruct (lookup_atom (state_ever_entries P I T state) new_atom); + [reflexivity |]. + destruct (Nat.leb + (state_allocator_frontier P I T state) + (symbol_id_value I id)) eqn:Hfrontier; [| reflexivity]. + apply Nat.leb_le in Hfrontier. lia. + - unfold publish_fresh_atom. + destruct (lookup_atom (state_ever_entries P I T state) new_atom); + [reflexivity |]. + destruct (lookup_symbol (state_ever_entries P I T state) id); + [reflexivity |]. + destruct (Nat.leb + (state_allocator_frontier P I T state) + (symbol_id_value I id)) eqn:Hfrontier; [| reflexivity]. + apply Nat.leb_le in Hfrontier. lia. +Qed. + +Theorem VWENC_106_FRESH_PUBLICATION_UPDATES_LIVE_HISTORY_AND_PACKED_BYTES : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state updated : InterningState P I T) + (atom : CanonicalAtom P) (id : SymbolId I), + publish_fresh_atom state atom id = Some updated -> + In (atom, id) (state_live_entries P I T updated) /\ + In (atom, id) (state_ever_entries P I T updated) /\ + packed_entry_exact + (state_packed_storage P I T updated) atom id. +Proof. + intros P I T state updated atom id Hpublish. + unfold publish_fresh_atom in Hpublish. + destruct (lookup_atom (state_ever_entries P I T state) atom); + [discriminate |]. + destruct (lookup_symbol (state_ever_entries P I T state) id); + [discriminate |]. + destruct (Nat.leb + (state_allocator_frontier P I T state) + (symbol_id_value I id)); [| discriminate]. + destruct (append_packed_atom + (state_packed_storage P I T state) id atom) + as [packed |] eqn:Happend; [| discriminate]. + inversion Hpublish. subst updated. clear Hpublish. + split; [now left |]. + split; [now left |]. + destruct (VWENC_114_SAFE_PACKED_APPEND_READS_EXACT_CANONICAL_BYTES + P I (state_packed_storage P I T state) packed id atom Happend) + as [span [Hlookup [Hread [Hlength Hbounds]]]]. + exists span. split; [exact Hlookup |]. + split; [exact Hread |]. + split; [exact Hlength |]. + split. + - pose proof + (atom_codeword_nonempty P + (canonical_atom_bytes P atom) + (canonical_atom_valid P atom)) as Hnonempty. + rewrite Hlength. + destruct (canonical_atom_bytes P atom); simpl; [contradiction | lia]. + - exact Hbounds. +Qed. + +Theorem VWENC_107_EVER_PUBLISHED_ID_CANNOT_BE_REBOUND_AFTER_TOMBSTONE : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state : InterningState P I T) + (new_atom previous_atom : CanonicalAtom P) (id : SymbolId I), + lookup_symbol (state_ever_entries P I T state) id = + Some previous_atom -> + publish_fresh_atom state new_atom id = None. +Proof. + intros P I T state new_atom previous_atom id Howned. + unfold publish_fresh_atom. + destruct (lookup_atom (state_ever_entries P I T state) new_atom); + [reflexivity |]. + now rewrite Howned. +Qed. + +Lemma referenced_ids_are_live_and_below_frontier : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state : InterningState P I T) sequence id, + InterningStateWellFormed state -> + In sequence (state_sequences P I T state) -> + In id sequence -> + symbol_id_value I id < state_allocator_frontier P I T state /\ + live_symbol (state_live_entries P I T state) id. +Proof. + intros P I T state sequence id Hwell Hsequence Hid. + destruct Hwell as [_ _ _ _ _ _ _ Hsequences]. + apply Forall_forall with (x := sequence) in Hsequences; + [| exact Hsequence]. + now apply Forall_forall with (x := id) in Hsequences. +Qed. + +Lemma vocabulary_insert_preserves_well_formedness : + forall (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (entries : list (VocabularyEntry P I)) atom id, + vocabulary_relation_well_formed entries -> + lookup_atom entries atom = None -> + lookup_symbol entries id = None -> + vocabulary_relation_well_formed ((atom, id) :: entries). +Proof. + intros P I entries atom id [Hatom_unique Hid_unique] Hatom Hid. + split; simpl; constructor. + - unfold lookup_atom in Hatom. + now apply assoc_lookup_none_key_absent in Hatom. + - exact Hatom_unique. + - unfold lookup_symbol in Hid. + apply assoc_lookup_none_key_absent in Hid. + rewrite reverse_vocabulary_keys_are_ids in Hid. exact Hid. + - exact Hid_unique. +Qed. + +Lemma sequence_bound_monotone : + forall (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (old_live new_live : list (VocabularyEntry P I)) + old_frontier new_frontier sequence, + (forall atom id, In (atom, id) old_live -> + In (atom, id) new_live) -> + old_frontier <= new_frontier -> + sequence_vocabulary_bound old_live old_frontier sequence -> + sequence_vocabulary_bound new_live new_frontier sequence. +Proof. + intros P I old_live new_live old_frontier new_frontier sequence + Hinclude Hfrontier Hbound. + apply Forall_forall. intros id Hin. + apply Forall_forall with (x := id) in Hbound; [| exact Hin]. + destruct Hbound as [Hbelow [atom Hlive]]. + split; [lia |]. + exists atom. now apply Hinclude. +Qed. + +Lemma NoDup_app_disjoint_right : + forall (Element : Type) (left right : list Element) element, + NoDup (left ++ right) -> + In element left -> + ~ In element right. +Proof. + intros Element left right element Hunique Hin_left Hin_right. + destruct (in_split element left Hin_left) + as [prefix [suffix Hleft]]. + subst left. + assert (Hshape : + (prefix ++ element :: suffix) ++ right = + prefix ++ element :: (suffix ++ right)). + { now rewrite <- app_assoc. } + rewrite Hshape in Hunique. + pose proof (NoDup_remove_2 prefix (suffix ++ right) element Hunique) + as Hnot_in_remainder. + apply Hnot_in_remainder. + apply in_or_app. right. + apply in_or_app. right. exact Hin_right. +Qed. + +Lemma allocated_id_has_exact_span : + forall (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (allocations : list (VocabularyEntry P I)) storage id, + packed_storage_matches_allocations allocations storage -> + In id (map snd allocations) -> + exists atom span, + In (atom, id) allocations /\ + lookup_span storage id = Some span. +Proof. + intros P I allocations storage id + [_ [_ [_ [Hexact _]]]] Hin. + apply in_map_iff in Hin. + destruct Hin as [[atom allocated_id] [Hequal Hin]]. + simpl in Hequal. subst allocated_id. + destruct (Hexact atom id Hin) + as [span [Hlookup _]]. + exists atom, span. now split. +Qed. + +Lemma permutation_move_middle_entry_right : + forall (Element : Type) + (before prefix suffix after : list Element) (element : Element), + Permutation + (before ++ prefix ++ element :: suffix ++ after) + (before ++ prefix ++ suffix ++ element :: after). +Proof. + intros Element before prefix suffix after element. + apply Permutation_app_head. + apply Permutation_app_head. + apply Permutation_middle. +Qed. + +Lemma permutation_extract_after_three_prefixes : + forall (Element : Type) + (first second third fourth fifth : list Element) (element : Element), + Permutation + (first ++ second ++ third ++ element :: fourth ++ fifth) + (element :: first ++ second ++ third ++ fourth ++ fifth). +Proof. + intros Element first second third fourth fifth element. + apply Permutation_sym. + eapply Permutation_trans. + - apply Permutation_middle. + - apply Permutation_app_head. + eapply Permutation_trans. + + apply Permutation_middle. + + apply Permutation_app_head. + apply Permutation_middle. +Qed. + +Lemma permutation_move_after_three_prefixes : + forall (Element : Type) + (first second third fourth fifth : list Element) (element : Element), + Permutation + (first ++ second ++ third ++ element :: fourth ++ fifth) + (first ++ second ++ third ++ fourth ++ element :: fifth). +Proof. + intros Element first second third fourth fifth element. + apply Permutation_app_head. + apply Permutation_app_head. + apply Permutation_app_head. + apply Permutation_middle. +Qed. + +Lemma claim_atom_allocation_preserves_combined_state_well_formedness : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state claimed : InterningState P I T) + (atom : CanonicalAtom P) (id : SymbolId I), + InterningStateWellFormed state -> + claim_atom_allocation state atom id = Some claimed -> + InterningStateWellFormed claimed. +Proof. + intros P I T state claimed atom id Hwell Hclaim. + destruct Hwell as + [Hlive_bijection Hhistory_bijection Hlive_history + Hallocation_unique Hpacked Hallocations_below Hfrontier_capacity + Hsequences Hterm_bijection Hterm_bound Hdisabled]. + unfold claim_atom_allocation in Hclaim. + destruct (lookup_atom (state_ever_entries P I T state) atom); + [discriminate |]. + destruct (Nat.leb + (state_allocator_frontier P I T state) + (symbol_id_value I id)) eqn:Hfrontier; [| discriminate]. + apply Nat.leb_le in Hfrontier. + inversion Hclaim. subst claimed. clear Hclaim. + assert (Hid_absent : + ~ In id (map snd (state_allocation_entries state))). + { intros Hin. + apply in_map_iff in Hin. + destruct Hin as [[allocated_atom allocated_id] [Hequal Hin]]. + simpl in Hequal. subst allocated_id. + apply Forall_forall with + (x := (allocated_atom, id)) in Hallocations_below; + [simpl in Hallocations_below; lia | exact Hin]. } + assert (Hallocation_permutation : + Permutation + ((atom, id) :: state_allocation_entries state) + (state_allocation_entries + {| state_fiber := state_fiber P I T state; + state_term_fiber := state_term_fiber P I T state; + state_reserved_entries := + (atom, id) :: state_reserved_entries P I T state; + state_claimed_entries := state_claimed_entries P I T state; + state_live_entries := state_live_entries P I T state; + state_ever_entries := state_ever_entries P I T state; + state_orphan_entries := state_orphan_entries P I T state; + state_unmaterialized_orphan_entries := + state_unmaterialized_orphan_entries P I T state; + state_packed_storage := state_packed_storage P I T state; + state_allocator_frontier := S (symbol_id_value I id); + state_sequences := state_sequences P I T state; + state_term_dictionary_enabled := + state_term_dictionary_enabled P I T state; + state_term_entries := state_term_entries P I T state |})). + { unfold state_allocation_entries. simpl. + apply Permutation_middle. } + constructor; simpl. + - exact Hlive_bijection. + - exact Hhistory_bijection. + - exact Hlive_history. + - apply (Permutation_NoDup (Permutation_map snd Hallocation_permutation)). + constructor; assumption. + - exact Hpacked. + - apply Forall_forall. intros entry Hin. + assert (Hordered : + In entry ((atom, id) :: state_allocation_entries state)). + { eapply Permutation_in. + - exact (Permutation_sym Hallocation_permutation). + - exact Hin. } + simpl in Hordered. destruct Hordered as [Hnew | Hold]. + + inversion Hnew. simpl. lia. + + apply Forall_forall with (x := entry) in Hallocations_below; + [| exact Hold]. + simpl in *. lia. + - pose proof (symbol_id_in_range I id). lia. + - apply Forall_forall. intros sequence Hin. + apply Forall_forall with (x := sequence) in Hsequences; + [| exact Hin]. + eapply sequence_bound_monotone; [| | exact Hsequences]. + + intros live_atom live_id Hlive. exact Hlive. + + lia. + - exact Hterm_bijection. + - apply Forall_forall. intros entry Hin. + apply Forall_forall with (x := entry) in Hterm_bound; + [| exact Hin]. + eapply sequence_bound_monotone; [| | exact Hterm_bound]. + + intros live_atom live_id Hlive. exact Hlive. + + lia. + - exact Hdisabled. +Qed. + +Definition materialize_reserved_allocation + {P : CertifiedAtomProfile} + {I T : FixedWidthCarrierProfile} + (state : InterningState P I T) + (atom : CanonicalAtom P) + (id : SymbolId I) + (updated : InterningState P I T) : Prop := + exists prefix suffix packed, + state_reserved_entries P I T state = + prefix ++ (atom, id) :: suffix /\ + append_packed_atom + (state_packed_storage P I T state) id atom = Some packed /\ + updated = + {| state_fiber := state_fiber P I T state; + state_term_fiber := state_term_fiber P I T state; + state_reserved_entries := prefix ++ suffix; + state_claimed_entries := + (atom, id) :: state_claimed_entries P I T state; + state_live_entries := state_live_entries P I T state; + state_ever_entries := state_ever_entries P I T state; + state_orphan_entries := state_orphan_entries P I T state; + state_unmaterialized_orphan_entries := + state_unmaterialized_orphan_entries P I T state; + state_packed_storage := packed; + state_allocator_frontier := state_allocator_frontier P I T state; + state_sequences := state_sequences P I T state; + state_term_dictionary_enabled := + state_term_dictionary_enabled P I T state; + state_term_entries := state_term_entries P I T state |}. + +Lemma materialize_reserved_allocation_preserves_combined_state_well_formedness : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state updated : InterningState P I T) atom id, + InterningStateWellFormed state -> + materialize_reserved_allocation state atom id updated -> + InterningStateWellFormed updated. +Proof. + intros P I T state updated atom id Hwell Hmaterialize. + destruct Hwell as + [Hlive_bijection Hhistory_bijection Hlive_history + Hallocation_unique Hpacked Hallocations_below Hfrontier_capacity + Hsequences Hterm_bijection Hterm_bound Hdisabled]. + destruct Hmaterialize as + [prefix [suffix [packed [Hreserved [Happend Hupdated]]]]]. + subst updated. + assert (Hspan_none : + lookup_span (state_packed_storage P I T state) id = None). + { unfold append_packed_atom in Happend. + destruct (lookup_span (state_packed_storage P I T state) id); + [discriminate | reflexivity]. } + assert (Hid_materialized_absent : + ~ In id (map snd (state_materialized_entries state))). + { intros Hin. + destruct (allocated_id_has_exact_span + P I (state_materialized_entries state) + (state_packed_storage P I T state) id Hpacked Hin) + as [allocated_atom [span [_ Hlookup]]]. + rewrite Hspan_none in Hlookup. discriminate. } + assert (Hallocation_permutation : + Permutation + (state_allocation_entries state) + (state_allocation_entries + {| state_fiber := state_fiber P I T state; + state_term_fiber := state_term_fiber P I T state; + state_reserved_entries := prefix ++ suffix; + state_claimed_entries := + (atom, id) :: state_claimed_entries P I T state; + state_live_entries := state_live_entries P I T state; + state_ever_entries := state_ever_entries P I T state; + state_orphan_entries := state_orphan_entries P I T state; + state_unmaterialized_orphan_entries := + state_unmaterialized_orphan_entries P I T state; + state_packed_storage := packed; + state_allocator_frontier := state_allocator_frontier P I T state; + state_sequences := state_sequences P I T state; + state_term_dictionary_enabled := + state_term_dictionary_enabled P I T state; + state_term_entries := state_term_entries P I T state |})). + { unfold state_allocation_entries. simpl. rewrite Hreserved. + repeat rewrite <- app_assoc. + apply permutation_move_middle_entry_right. } + assert (Hmaterialized_permutation : + Permutation + ((atom, id) :: state_materialized_entries state) + (state_materialized_entries + {| state_fiber := state_fiber P I T state; + state_term_fiber := state_term_fiber P I T state; + state_reserved_entries := prefix ++ suffix; + state_claimed_entries := + (atom, id) :: state_claimed_entries P I T state; + state_live_entries := state_live_entries P I T state; + state_ever_entries := state_ever_entries P I T state; + state_orphan_entries := state_orphan_entries P I T state; + state_unmaterialized_orphan_entries := + state_unmaterialized_orphan_entries P I T state; + state_packed_storage := packed; + state_allocator_frontier := state_allocator_frontier P I T state; + state_sequences := state_sequences P I T state; + state_term_dictionary_enabled := + state_term_dictionary_enabled P I T state; + state_term_entries := state_term_entries P I T state |})). + { unfold state_materialized_entries. simpl. + apply Permutation_middle. } + constructor; simpl. + - exact Hlive_bijection. + - exact Hhistory_bijection. + - exact Hlive_history. + - apply (Permutation_NoDup (Permutation_map snd Hallocation_permutation)). + exact Hallocation_unique. + - eapply packed_storage_matches_allocations_permutation. + + exact Hmaterialized_permutation. + + eapply packed_storage_matches_allocations_after_append; + eassumption. + - apply Forall_forall. intros entry Hin. + apply Forall_forall with (x := entry) in Hallocations_below. + + exact Hallocations_below. + + eapply Permutation_in. + * exact (Permutation_sym Hallocation_permutation). + * exact Hin. + - exact Hfrontier_capacity. + - exact Hsequences. + - exact Hterm_bijection. + - exact Hterm_bound. + - exact Hdisabled. +Qed. + +Definition orphan_reserved_allocation + {P : CertifiedAtomProfile} + {I T : FixedWidthCarrierProfile} + (state : InterningState P I T) + (atom : CanonicalAtom P) + (id : SymbolId I) + (updated : InterningState P I T) : Prop := + exists prefix suffix, + state_reserved_entries P I T state = + prefix ++ (atom, id) :: suffix /\ + updated = + {| state_fiber := state_fiber P I T state; + state_term_fiber := state_term_fiber P I T state; + state_reserved_entries := prefix ++ suffix; + state_claimed_entries := state_claimed_entries P I T state; + state_live_entries := state_live_entries P I T state; + state_ever_entries := state_ever_entries P I T state; + state_orphan_entries := state_orphan_entries P I T state; + state_unmaterialized_orphan_entries := + (atom, id) :: + state_unmaterialized_orphan_entries P I T state; + state_packed_storage := state_packed_storage P I T state; + state_allocator_frontier := state_allocator_frontier P I T state; + state_sequences := state_sequences P I T state; + state_term_dictionary_enabled := + state_term_dictionary_enabled P I T state; + state_term_entries := state_term_entries P I T state |}. + +Lemma orphan_reserved_allocation_preserves_combined_state_well_formedness : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state updated : InterningState P I T) atom id, + InterningStateWellFormed state -> + orphan_reserved_allocation state atom id updated -> + InterningStateWellFormed updated. +Proof. + intros P I T state updated atom id Hwell Horphan. + destruct Hwell as + [Hlive_bijection Hhistory_bijection Hlive_history + Hallocation_unique Hpacked Hallocations_below Hfrontier_capacity + Hsequences Hterm_bijection Hterm_bound Hdisabled]. + destruct Horphan as [prefix [suffix [Hreserved Hupdated]]]. + subst updated. + assert (Hallocation_permutation : + Permutation + (state_allocation_entries state) + (state_allocation_entries + {| state_fiber := state_fiber P I T state; + state_term_fiber := state_term_fiber P I T state; + state_reserved_entries := prefix ++ suffix; + state_claimed_entries := state_claimed_entries P I T state; + state_live_entries := state_live_entries P I T state; + state_ever_entries := state_ever_entries P I T state; + state_orphan_entries := state_orphan_entries P I T state; + state_unmaterialized_orphan_entries := + (atom, id) :: + state_unmaterialized_orphan_entries P I T state; + state_packed_storage := state_packed_storage P I T state; + state_allocator_frontier := state_allocator_frontier P I T state; + state_sequences := state_sequences P I T state; + state_term_dictionary_enabled := + state_term_dictionary_enabled P I T state; + state_term_entries := state_term_entries P I T state |})). + { unfold state_allocation_entries. simpl. rewrite Hreserved. + repeat rewrite <- app_assoc. + simpl. + apply Permutation_app_head. + apply Permutation_app_head. + repeat rewrite app_assoc. + apply Permutation_middle. } + constructor; simpl. + - exact Hlive_bijection. + - exact Hhistory_bijection. + - exact Hlive_history. + - apply (Permutation_NoDup (Permutation_map snd Hallocation_permutation)). + exact Hallocation_unique. + - exact Hpacked. + - apply Forall_forall. intros entry Hin. + apply Forall_forall with (x := entry) in Hallocations_below. + + exact Hallocations_below. + + eapply Permutation_in. + * exact (Permutation_sym Hallocation_permutation). + * exact Hin. + - exact Hfrontier_capacity. + - exact Hsequences. + - exact Hterm_bijection. + - exact Hterm_bound. + - exact Hdisabled. +Qed. + +Definition publish_claimed_allocation + {P : CertifiedAtomProfile} + {I T : FixedWidthCarrierProfile} + (state : InterningState P I T) + (atom : CanonicalAtom P) + (id : SymbolId I) + (updated : InterningState P I T) : Prop := + exists prefix suffix, + state_claimed_entries P I T state = + prefix ++ (atom, id) :: suffix /\ + lookup_atom (state_ever_entries P I T state) atom = None /\ + lookup_symbol (state_ever_entries P I T state) id = None /\ + lookup_atom (state_live_entries P I T state) atom = None /\ + lookup_symbol (state_live_entries P I T state) id = None /\ + updated = + {| state_fiber := state_fiber P I T state; + state_term_fiber := state_term_fiber P I T state; + state_reserved_entries := state_reserved_entries P I T state; + state_claimed_entries := prefix ++ suffix; + state_live_entries := + (atom, id) :: state_live_entries P I T state; + state_ever_entries := + (atom, id) :: state_ever_entries P I T state; + state_orphan_entries := state_orphan_entries P I T state; + state_unmaterialized_orphan_entries := + state_unmaterialized_orphan_entries P I T state; + state_packed_storage := state_packed_storage P I T state; + state_allocator_frontier := state_allocator_frontier P I T state; + state_sequences := state_sequences P I T state; + state_term_dictionary_enabled := + state_term_dictionary_enabled P I T state; + state_term_entries := state_term_entries P I T state |}. + +Lemma publish_claimed_allocation_preserves_combined_state_well_formedness : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state updated : InterningState P I T) atom id, + InterningStateWellFormed state -> + publish_claimed_allocation state atom id updated -> + InterningStateWellFormed updated. +Proof. + intros P I T state updated atom id Hwell Hpublish. + destruct Hwell as + [Hlive_bijection Hhistory_bijection Hlive_history + Hallocation_unique Hpacked Hallocations_below Hfrontier_capacity + Hsequences Hterm_bijection Hterm_bound Hdisabled]. + destruct Hpublish as + [prefix [suffix + [Hclaimed [Hatom_history [Hid_history + [Hatom_live [Hid_live Hupdated]]]]]]]. + subst updated. + assert (Hallocation_permutation : + Permutation + (state_allocation_entries state) + (state_allocation_entries + {| state_fiber := state_fiber P I T state; + state_term_fiber := state_term_fiber P I T state; + state_reserved_entries := state_reserved_entries P I T state; + state_claimed_entries := prefix ++ suffix; + state_live_entries := + (atom, id) :: state_live_entries P I T state; + state_ever_entries := + (atom, id) :: state_ever_entries P I T state; + state_orphan_entries := state_orphan_entries P I T state; + state_unmaterialized_orphan_entries := + state_unmaterialized_orphan_entries P I T state; + state_packed_storage := state_packed_storage P I T state; + state_allocator_frontier := state_allocator_frontier P I T state; + state_sequences := state_sequences P I T state; + state_term_dictionary_enabled := + state_term_dictionary_enabled P I T state; + state_term_entries := state_term_entries P I T state |})). + { unfold state_allocation_entries. simpl. rewrite Hclaimed. + repeat rewrite <- app_assoc. + apply permutation_extract_after_three_prefixes. } + assert (Hmaterialized_permutation : + Permutation + (state_materialized_entries state) + (state_materialized_entries + {| state_fiber := state_fiber P I T state; + state_term_fiber := state_term_fiber P I T state; + state_reserved_entries := state_reserved_entries P I T state; + state_claimed_entries := prefix ++ suffix; + state_live_entries := + (atom, id) :: state_live_entries P I T state; + state_ever_entries := + (atom, id) :: state_ever_entries P I T state; + state_orphan_entries := state_orphan_entries P I T state; + state_unmaterialized_orphan_entries := + state_unmaterialized_orphan_entries P I T state; + state_packed_storage := state_packed_storage P I T state; + state_allocator_frontier := state_allocator_frontier P I T state; + state_sequences := state_sequences P I T state; + state_term_dictionary_enabled := + state_term_dictionary_enabled P I T state; + state_term_entries := state_term_entries P I T state |})). + { unfold state_materialized_entries. simpl. rewrite Hclaimed. + repeat rewrite <- app_assoc. + exact + (permutation_extract_after_three_prefixes + (VocabularyEntry P I) + (state_ever_entries P I T state) + prefix [] suffix + (state_orphan_entries P I T state) + (atom, id)). } + constructor; simpl. + - now apply vocabulary_insert_preserves_well_formedness. + - now apply vocabulary_insert_preserves_well_formedness. + - intros live_atom live_id Hin. + simpl in Hin. destruct Hin as [Hnew | Hold]. + + inversion Hnew. now left. + + right. now apply Hlive_history. + - apply (Permutation_NoDup (Permutation_map snd Hallocation_permutation)). + exact Hallocation_unique. + - eapply packed_storage_matches_allocations_permutation. + + exact Hmaterialized_permutation. + + exact Hpacked. + - apply Forall_forall. intros entry Hin. + apply Forall_forall with (x := entry) in Hallocations_below. + + exact Hallocations_below. + + eapply Permutation_in. + * exact (Permutation_sym Hallocation_permutation). + * exact Hin. + - exact Hfrontier_capacity. + - apply Forall_forall. intros sequence Hin. + apply Forall_forall with (x := sequence) in Hsequences; + [| exact Hin]. + eapply (sequence_bound_monotone P I + (state_live_entries P I T state) + ((atom, id) :: state_live_entries P I T state) + (state_allocator_frontier P I T state) + (state_allocator_frontier P I T state) + sequence). + + intros live_atom live_id Hlive. now right. + + apply le_n. + + exact Hsequences. + - exact Hterm_bijection. + - apply Forall_forall. intros entry Hin. + apply Forall_forall with (x := entry) in Hterm_bound; + [| exact Hin]. + eapply (sequence_bound_monotone P I + (state_live_entries P I T state) + ((atom, id) :: state_live_entries P I T state) + (state_allocator_frontier P I T state) + (state_allocator_frontier P I T state) + (fst entry)). + + intros live_atom live_id Hlive. now right. + + apply le_n. + + exact Hterm_bound. + - exact Hdisabled. +Qed. + +Definition orphan_claimed_allocation + {P : CertifiedAtomProfile} + {I T : FixedWidthCarrierProfile} + (state : InterningState P I T) + (atom : CanonicalAtom P) + (id : SymbolId I) + (updated : InterningState P I T) : Prop := + exists prefix suffix, + state_claimed_entries P I T state = + prefix ++ (atom, id) :: suffix /\ + updated = + {| state_fiber := state_fiber P I T state; + state_term_fiber := state_term_fiber P I T state; + state_reserved_entries := state_reserved_entries P I T state; + state_claimed_entries := prefix ++ suffix; + state_live_entries := state_live_entries P I T state; + state_ever_entries := state_ever_entries P I T state; + state_orphan_entries := + (atom, id) :: state_orphan_entries P I T state; + state_unmaterialized_orphan_entries := + state_unmaterialized_orphan_entries P I T state; + state_packed_storage := state_packed_storage P I T state; + state_allocator_frontier := state_allocator_frontier P I T state; + state_sequences := state_sequences P I T state; + state_term_dictionary_enabled := + state_term_dictionary_enabled P I T state; + state_term_entries := state_term_entries P I T state |}. + +Lemma orphan_claimed_allocation_preserves_combined_state_well_formedness : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state updated : InterningState P I T) atom id, + InterningStateWellFormed state -> + orphan_claimed_allocation state atom id updated -> + InterningStateWellFormed updated. +Proof. + intros P I T state updated atom id Hwell Horphan. + destruct Hwell as + [Hlive_bijection Hhistory_bijection Hlive_history + Hallocation_unique Hpacked Hallocations_below Hfrontier_capacity + Hsequences Hterm_bijection Hterm_bound Hdisabled]. + destruct Horphan as + [prefix [suffix [Hclaimed Hupdated]]]. + subst updated. + assert (Hallocation_permutation : + Permutation + (state_allocation_entries state) + (state_allocation_entries + {| state_fiber := state_fiber P I T state; + state_term_fiber := state_term_fiber P I T state; + state_reserved_entries := state_reserved_entries P I T state; + state_claimed_entries := prefix ++ suffix; + state_live_entries := state_live_entries P I T state; + state_ever_entries := state_ever_entries P I T state; + state_orphan_entries := + (atom, id) :: state_orphan_entries P I T state; + state_unmaterialized_orphan_entries := + state_unmaterialized_orphan_entries P I T state; + state_packed_storage := state_packed_storage P I T state; + state_allocator_frontier := state_allocator_frontier P I T state; + state_sequences := state_sequences P I T state; + state_term_dictionary_enabled := + state_term_dictionary_enabled P I T state; + state_term_entries := state_term_entries P I T state |})). + { unfold state_allocation_entries. simpl. rewrite Hclaimed. + repeat rewrite <- app_assoc. + apply permutation_move_after_three_prefixes. } + assert (Hmaterialized_permutation : + Permutation + (state_materialized_entries state) + (state_materialized_entries + {| state_fiber := state_fiber P I T state; + state_term_fiber := state_term_fiber P I T state; + state_reserved_entries := state_reserved_entries P I T state; + state_claimed_entries := prefix ++ suffix; + state_live_entries := state_live_entries P I T state; + state_ever_entries := state_ever_entries P I T state; + state_orphan_entries := + (atom, id) :: state_orphan_entries P I T state; + state_unmaterialized_orphan_entries := + state_unmaterialized_orphan_entries P I T state; + state_packed_storage := state_packed_storage P I T state; + state_allocator_frontier := state_allocator_frontier P I T state; + state_sequences := state_sequences P I T state; + state_term_dictionary_enabled := + state_term_dictionary_enabled P I T state; + state_term_entries := state_term_entries P I T state |})). + { unfold state_materialized_entries. simpl. rewrite Hclaimed. + repeat rewrite <- app_assoc. + exact + (permutation_move_after_three_prefixes + (VocabularyEntry P I) + (state_ever_entries P I T state) + prefix [] suffix + (state_orphan_entries P I T state) + (atom, id)). } + constructor; simpl. + - exact Hlive_bijection. + - exact Hhistory_bijection. + - exact Hlive_history. + - apply (Permutation_NoDup (Permutation_map snd Hallocation_permutation)). + exact Hallocation_unique. + - eapply packed_storage_matches_allocations_permutation. + + exact Hmaterialized_permutation. + + exact Hpacked. + - apply Forall_forall. intros entry Hin. + apply Forall_forall with (x := entry) in Hallocations_below. + + exact Hallocations_below. + + eapply Permutation_in. + * exact (Permutation_sym Hallocation_permutation). + * exact Hin. + - exact Hfrontier_capacity. + - exact Hsequences. + - exact Hterm_bijection. + - exact Hterm_bound. + - exact Hdisabled. +Qed. + +Lemma different_id_survives_middle_entry_removal : + forall (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (prefix suffix : list (VocabularyEntry P I)) + atom id removed_atom removed_id, + In (atom, id) (prefix ++ (removed_atom, removed_id) :: suffix) -> + id <> removed_id -> + In (atom, id) (prefix ++ suffix). +Proof. + intros P I prefix suffix atom id removed_atom removed_id Hin Hdifferent. + apply in_app_or in Hin. apply in_or_app. + destruct Hin as [Hprefix | Htail]. + - now left. + - simpl in Htail. destruct Htail as [Hremoved | Hsuffix]. + + exfalso. apply Hdifferent. + apply (f_equal snd) in Hremoved. now symmetry. + + now right. +Qed. + +Lemma different_id_middle_entry_membership_iff : + forall (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (prefix suffix : list (VocabularyEntry P I)) + atom id removed_atom removed_id, + id <> removed_id -> + (In (atom, id) (prefix ++ (removed_atom, removed_id) :: suffix) <-> + In (atom, id) (prefix ++ suffix)). +Proof. + intros P I prefix suffix atom id removed_atom removed_id Hdifferent. + split. + - intros Hin. + exact (different_id_survives_middle_entry_removal + P I prefix suffix atom id removed_atom removed_id Hin Hdifferent). + - intros Hin. + apply in_app_or in Hin. apply in_or_app. + destruct Hin as [Hprefix | Hsuffix]. + + now left. + + right. simpl. now right. +Qed. + +Lemma different_id_cons_entry_membership_iff : + forall (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (entries : list (VocabularyEntry P I)) + atom id inserted_atom inserted_id, + id <> inserted_id -> + (In (atom, id) ((inserted_atom, inserted_id) :: entries) <-> + In (atom, id) entries). +Proof. + intros P I entries atom id inserted_atom inserted_id Hdifferent. + simpl. split. + - intros [Hequal | Hin]. + + exfalso. apply Hdifferent. + apply (f_equal snd) in Hequal. now symmetry. + + exact Hin. + - now right. +Qed. + +Definition tombstone_published_allocation + {P : CertifiedAtomProfile} + {I T : FixedWidthCarrierProfile} + (state : InterningState P I T) + (atom : CanonicalAtom P) + (id : SymbolId I) + (updated : InterningState P I T) : Prop := + exists prefix suffix, + state_live_entries P I T state = + prefix ++ (atom, id) :: suffix /\ + Forall (fun sequence => ~ In id sequence) + (state_sequences P I T state) /\ + Forall (fun entry => ~ In id (fst entry)) + (state_term_entries P I T state) /\ + updated = + {| state_fiber := state_fiber P I T state; + state_term_fiber := state_term_fiber P I T state; + state_reserved_entries := state_reserved_entries P I T state; + state_claimed_entries := state_claimed_entries P I T state; + state_live_entries := prefix ++ suffix; + state_ever_entries := state_ever_entries P I T state; + state_orphan_entries := state_orphan_entries P I T state; + state_unmaterialized_orphan_entries := + state_unmaterialized_orphan_entries P I T state; + state_packed_storage := state_packed_storage P I T state; + state_allocator_frontier := state_allocator_frontier P I T state; + state_sequences := state_sequences P I T state; + state_term_dictionary_enabled := + state_term_dictionary_enabled P I T state; + state_term_entries := state_term_entries P I T state |}. + +Lemma tombstone_published_allocation_preserves_combined_state_well_formedness : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state updated : InterningState P I T) atom id, + InterningStateWellFormed state -> + tombstone_published_allocation state atom id updated -> + InterningStateWellFormed updated. +Proof. + intros P I T state updated atom id Hwell Htombstone. + destruct Hwell as + [[Hlive_atoms Hlive_ids] Hhistory_bijection Hlive_history + Hallocation_unique Hpacked Hallocations_below Hfrontier_capacity + Hsequences Hterm_bijection Hterm_bound Hdisabled]. + destruct Htombstone as + [prefix [suffix + [Hlive [Hsequence_excludes [Hterm_excludes Hupdated]]]]]. + subst updated. + constructor; simpl. + - split. + + rewrite Hlive in Hlive_atoms. + rewrite map_app in Hlive_atoms. simpl in Hlive_atoms. + rewrite map_app. + apply NoDup_remove_1 with (a := atom). exact Hlive_atoms. + + rewrite Hlive in Hlive_ids. + rewrite map_app in Hlive_ids. simpl in Hlive_ids. + rewrite map_app. + apply NoDup_remove_1 with (a := id). exact Hlive_ids. + - exact Hhistory_bijection. + - intros live_atom live_id Hin. + apply Hlive_history. rewrite Hlive. + apply in_or_app. apply in_app_or in Hin. + destruct Hin as [Hprefix | Hsuffix]. + + now left. + + right. simpl. now right. + - exact Hallocation_unique. + - exact Hpacked. + - exact Hallocations_below. + - exact Hfrontier_capacity. + - apply Forall_forall. intros sequence Hin_sequence. + apply Forall_forall with (x := sequence) in Hsequences; + [| exact Hin_sequence]. + apply Forall_forall with (x := sequence) in Hsequence_excludes; + [| exact Hin_sequence]. + apply Forall_forall. intros sequence_id Hin_id. + apply Forall_forall with (x := sequence_id) in Hsequences; + [| exact Hin_id]. + destruct Hsequences as [Hbelow [live_atom Hlive_entry]]. + split; [exact Hbelow |]. exists live_atom. + eapply different_id_survives_middle_entry_removal. + + rewrite Hlive in Hlive_entry. exact Hlive_entry. + + intros Hequal. subst sequence_id. contradiction. + - exact Hterm_bijection. + - apply Forall_forall. intros entry Hin_entry. + apply Forall_forall with (x := entry) in Hterm_bound; + [| exact Hin_entry]. + apply Forall_forall with (x := entry) in Hterm_excludes; + [| exact Hin_entry]. + apply Forall_forall. intros term_id Hin_id. + apply Forall_forall with (x := term_id) in Hterm_bound; + [| exact Hin_id]. + destruct Hterm_bound as [Hbelow [live_atom Hlive_entry]]. + split; [exact Hbelow |]. exists live_atom. + eapply different_id_survives_middle_entry_removal. + + rewrite Hlive in Hlive_entry. exact Hlive_entry. + + intros Hequal. subst term_id. contradiction. + - exact Hdisabled. +Qed. + +Definition add_dependent_sequence + {P : CertifiedAtomProfile} + {I T : FixedWidthCarrierProfile} + (state : InterningState P I T) + (sequence : list (SymbolId I)) + (updated : InterningState P I T) : Prop := + sequence_vocabulary_bound + (state_live_entries P I T state) + (state_allocator_frontier P I T state) + sequence /\ + updated = + {| state_fiber := state_fiber P I T state; + state_term_fiber := state_term_fiber P I T state; + state_reserved_entries := state_reserved_entries P I T state; + state_claimed_entries := state_claimed_entries P I T state; + state_live_entries := state_live_entries P I T state; + state_ever_entries := state_ever_entries P I T state; + state_orphan_entries := state_orphan_entries P I T state; + state_unmaterialized_orphan_entries := + state_unmaterialized_orphan_entries P I T state; + state_packed_storage := state_packed_storage P I T state; + state_allocator_frontier := state_allocator_frontier P I T state; + state_sequences := sequence :: state_sequences P I T state; + state_term_dictionary_enabled := + state_term_dictionary_enabled P I T state; + state_term_entries := state_term_entries P I T state |}. + +Lemma add_dependent_sequence_preserves_combined_state_well_formedness : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state updated : InterningState P I T) sequence, + InterningStateWellFormed state -> + add_dependent_sequence state sequence updated -> + InterningStateWellFormed updated. +Proof. + intros P I T state updated sequence Hwell [Hbound Hupdated]. + subst updated. destruct Hwell. + constructor; simpl; try assumption. + now constructor. +Qed. + +Definition enable_term_dictionary + {P : CertifiedAtomProfile} + {I T : FixedWidthCarrierProfile} + (state updated : InterningState P I T) : Prop := + updated = + {| state_fiber := state_fiber P I T state; + state_term_fiber := state_term_fiber P I T state; + state_reserved_entries := state_reserved_entries P I T state; + state_claimed_entries := state_claimed_entries P I T state; + state_live_entries := state_live_entries P I T state; + state_ever_entries := state_ever_entries P I T state; + state_orphan_entries := state_orphan_entries P I T state; + state_unmaterialized_orphan_entries := + state_unmaterialized_orphan_entries P I T state; + state_packed_storage := state_packed_storage P I T state; + state_allocator_frontier := state_allocator_frontier P I T state; + state_sequences := state_sequences P I T state; + state_term_dictionary_enabled := true; + state_term_entries := state_term_entries P I T state |}. + +Lemma enable_term_dictionary_preserves_combined_state_well_formedness : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state updated : InterningState P I T), + InterningStateWellFormed state -> + enable_term_dictionary state updated -> + InterningStateWellFormed updated. +Proof. + intros P I T state updated Hwell Hupdated. + unfold enable_term_dictionary in Hupdated. + subst updated. destruct Hwell. + constructor; simpl; try assumption. + discriminate. +Qed. + +Definition disable_empty_term_dictionary + {P : CertifiedAtomProfile} + {I T : FixedWidthCarrierProfile} + (state updated : InterningState P I T) : Prop := + state_term_entries P I T state = [] /\ + updated = + {| state_fiber := state_fiber P I T state; + state_term_fiber := state_term_fiber P I T state; + state_reserved_entries := state_reserved_entries P I T state; + state_claimed_entries := state_claimed_entries P I T state; + state_live_entries := state_live_entries P I T state; + state_ever_entries := state_ever_entries P I T state; + state_orphan_entries := state_orphan_entries P I T state; + state_unmaterialized_orphan_entries := + state_unmaterialized_orphan_entries P I T state; + state_packed_storage := state_packed_storage P I T state; + state_allocator_frontier := state_allocator_frontier P I T state; + state_sequences := state_sequences P I T state; + state_term_dictionary_enabled := false; + state_term_entries := state_term_entries P I T state |}. + +Lemma disable_empty_term_dictionary_preserves_combined_state_well_formedness : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state updated : InterningState P I T), + InterningStateWellFormed state -> + disable_empty_term_dictionary state updated -> + InterningStateWellFormed updated. +Proof. + intros P I T state updated Hwell [Hempty Hupdated]. + subst updated. destruct Hwell. + constructor; simpl; try assumption. + intros _. exact Hempty. +Qed. + +Lemma term_insert_preserves_well_formedness : + forall (I T : FixedWidthCarrierProfile) + (entries : list (TermEntry I T)) sequence term_id, + term_relation_well_formed entries -> + lookup_term_sequence entries sequence = None -> + lookup_term_id entries term_id = None -> + term_relation_well_formed ((sequence, term_id) :: entries). +Proof. + intros I T entries sequence term_id + [Hsequence_unique Hid_unique] Hsequence Hid. + split; simpl; constructor. + - unfold lookup_term_sequence in Hsequence. + now apply assoc_lookup_none_key_absent in Hsequence. + - exact Hsequence_unique. + - unfold lookup_term_id in Hid. + apply assoc_lookup_none_key_absent in Hid. + rewrite reverse_term_keys_are_term_ids in Hid. exact Hid. + - exact Hid_unique. +Qed. + +Definition insert_term_dictionary_entry + {P : CertifiedAtomProfile} + {I T : FixedWidthCarrierProfile} + (state : InterningState P I T) + (sequence : list (SymbolId I)) + (term_id : TermId T) + (updated : InterningState P I T) : Prop := + state_term_dictionary_enabled P I T state = true /\ + sequence_vocabulary_bound + (state_live_entries P I T state) + (state_allocator_frontier P I T state) + sequence /\ + lookup_term_sequence (state_term_entries P I T state) sequence = None /\ + lookup_term_id (state_term_entries P I T state) term_id = None /\ + updated = + {| state_fiber := state_fiber P I T state; + state_term_fiber := state_term_fiber P I T state; + state_reserved_entries := state_reserved_entries P I T state; + state_claimed_entries := state_claimed_entries P I T state; + state_live_entries := state_live_entries P I T state; + state_ever_entries := state_ever_entries P I T state; + state_orphan_entries := state_orphan_entries P I T state; + state_unmaterialized_orphan_entries := + state_unmaterialized_orphan_entries P I T state; + state_packed_storage := state_packed_storage P I T state; + state_allocator_frontier := state_allocator_frontier P I T state; + state_sequences := state_sequences P I T state; + state_term_dictionary_enabled := true; + state_term_entries := + (sequence, term_id) :: state_term_entries P I T state |}. + +Lemma insert_term_dictionary_entry_preserves_combined_state_well_formedness : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state updated : InterningState P I T) sequence term_id, + InterningStateWellFormed state -> + insert_term_dictionary_entry state sequence term_id updated -> + InterningStateWellFormed updated. +Proof. + intros P I T state updated sequence term_id Hwell + [Henabled [Hbound [Hsequence [Hid Hupdated]]]]. + subst updated. + destruct Hwell as + [Hlive_bijection Hhistory_bijection Hlive_history + Hallocation_unique Hpacked Hallocations_below Hfrontier_capacity + Hsequences Hterm_bijection Hterm_bound Hdisabled]. + constructor; simpl; try assumption. + - now apply term_insert_preserves_well_formedness. + - now constructor. + - discriminate. +Qed. + +Lemma fresh_publication_preserves_combined_state_well_formedness : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state updated : InterningState P I T) + (atom : CanonicalAtom P) (id : SymbolId I), + InterningStateWellFormed state -> + publish_fresh_atom state atom id = Some updated -> + InterningStateWellFormed updated. +Proof. + intros P I T state updated atom id Hwell Hpublish. + destruct Hwell as + [Hlive_bijection Hhistory_bijection Hlive_history + Halloc_unique Hpacked Halloc_below Hfrontier_capacity + Hsequences Hterm_bijection Hterm_bound Hdisabled]. + unfold publish_fresh_atom in Hpublish. + destruct (lookup_atom (state_ever_entries P I T state) atom) + as [existing_atom_id |] eqn:Hatom; [discriminate |]. + destruct (lookup_symbol (state_ever_entries P I T state) id) + as [existing_atom |] eqn:Hid; [discriminate |]. + destruct (Nat.leb + (state_allocator_frontier P I T state) + (symbol_id_value I id)) eqn:Hfrontier; [| discriminate]. + apply Nat.leb_le in Hfrontier. + destruct (append_packed_atom + (state_packed_storage P I T state) id atom) + as [packed |] eqn:Happend; [| discriminate]. + inversion Hpublish. subst updated. clear Hpublish. + assert (Hid_history_absent : + ~ In id (map snd (state_ever_entries P I T state))). + { unfold lookup_symbol in Hid. + apply assoc_lookup_none_key_absent in Hid. + rewrite reverse_vocabulary_keys_are_ids in Hid. exact Hid. } + assert (Hatom_history_absent : + ~ In atom (map fst (state_ever_entries P I T state))). + { unfold lookup_atom in Hatom. + now apply assoc_lookup_none_key_absent in Hatom. } + assert (Hid_allocation_absent : + ~ In id (map snd (state_allocation_entries state))). + { intros Hin. + apply in_map_iff in Hin. + destruct Hin as [[allocated_atom allocated_id] [Hequal Hin]]. + simpl in Hequal. subst allocated_id. + apply Forall_forall with + (x := (allocated_atom, id)) in Halloc_below; [| exact Hin]. + simpl in Halloc_below. lia. } + assert (Hspan_none : + lookup_span (state_packed_storage P I T state) id = None). + { unfold append_packed_atom in Happend. + destruct (lookup_span (state_packed_storage P I T state) id); + [discriminate | reflexivity]. } + assert (Hid_materialized_absent : + ~ In id (map snd (state_materialized_entries state))). + { intros Hin. + destruct (allocated_id_has_exact_span + P I (state_materialized_entries state) + (state_packed_storage P I T state) id Hpacked Hin) + as [allocated_atom [span [_ Hlookup]]]. + rewrite Hspan_none in Hlookup. discriminate. } + assert (Hatom_live_none : + lookup_atom (state_live_entries P I T state) atom = None). + { destruct (lookup_atom (state_live_entries P I T state) atom) + as [live_id |] eqn:Hlive_lookup; [| reflexivity]. + exfalso. apply Hatom_history_absent. + unfold lookup_atom in Hlive_lookup. + apply assoc_lookup_sound in Hlive_lookup. + apply in_map_iff. + exists (atom, live_id). split; [reflexivity |]. + now apply Hlive_history. } + assert (Hid_live_none : + lookup_symbol (state_live_entries P I T state) id = None). + { destruct (lookup_symbol (state_live_entries P I T state) id) + as [live_atom |] eqn:Hlive_lookup; [| reflexivity]. + exfalso. apply Hid_history_absent. + unfold lookup_symbol in Hlive_lookup. + apply assoc_lookup_sound in Hlive_lookup. + apply reverse_vocabulary_membership in Hlive_lookup. + apply in_map_iff. + exists (live_atom, id). split; [reflexivity |]. + now apply Hlive_history. } + constructor; simpl. + - apply vocabulary_insert_preserves_well_formedness. + + exact Hlive_bijection. + + exact Hatom_live_none. + + exact Hid_live_none. + - now apply vocabulary_insert_preserves_well_formedness. + - intros live_atom live_id Hin. + simpl in Hin. destruct Hin as [Hnew | Hold]. + + inversion Hnew. now left. + + right. now apply Hlive_history. + - constructor. + + exact Hid_allocation_absent. + + exact Halloc_unique. + - eapply (packed_storage_matches_allocations_after_append + P I + (state_materialized_entries state) + (state_packed_storage P I T state) + packed atom id). + + exact Hpacked. + + exact Hid_materialized_absent. + + exact Happend. + - constructor. + + simpl. lia. + + apply Forall_forall. intros entry Hin. + apply Forall_forall with (x := entry) in Halloc_below; + [| exact Hin]. + lia. + - pose proof (symbol_id_in_range I id). lia. + - apply Forall_forall. intros sequence Hin. + apply Forall_forall with (x := sequence) in Hsequences; + [| exact Hin]. + eapply sequence_bound_monotone; [| | exact Hsequences]. + + intros live_atom live_id Hlive. now right. + + lia. + - exact Hterm_bijection. + - apply Forall_forall. intros entry Hin. + apply Forall_forall with (x := entry) in Hterm_bound; + [| exact Hin]. + eapply sequence_bound_monotone; [| | exact Hterm_bound]. + + intros live_atom live_id Hlive. now right. + + lia. + - exact Hdisabled. +Qed. + +Inductive InterningTransition + {P : CertifiedAtomProfile} + {I T : FixedWidthCarrierProfile} + : InterningState P I T -> InterningState P I T -> Prop := +| TransitionFreshPublication : + forall state updated atom id, + publish_fresh_atom state atom id = Some updated -> + InterningTransition state updated +| TransitionClaimAllocation : + forall state updated atom id, + claim_atom_allocation state atom id = Some updated -> + InterningTransition state updated +| TransitionMaterializeReservation : + forall state updated atom id, + materialize_reserved_allocation state atom id updated -> + InterningTransition state updated +| TransitionPublishClaim : + forall state updated atom id, + publish_claimed_allocation state atom id updated -> + InterningTransition state updated +| TransitionOrphanClaim : + forall state updated atom id, + orphan_claimed_allocation state atom id updated -> + InterningTransition state updated +| TransitionOrphanReservation : + forall state updated atom id, + orphan_reserved_allocation state atom id updated -> + InterningTransition state updated +| TransitionTombstonePublished : + forall state updated atom id, + tombstone_published_allocation state atom id updated -> + InterningTransition state updated +| TransitionAddDependentSequence : + forall state updated sequence, + add_dependent_sequence state sequence updated -> + InterningTransition state updated +| TransitionEnableTermDictionary : + forall state updated, + enable_term_dictionary state updated -> + InterningTransition state updated +| TransitionDisableEmptyTermDictionary : + forall state updated, + disable_empty_term_dictionary state updated -> + InterningTransition state updated +| TransitionInsertTermEntry : + forall state updated sequence term_id, + insert_term_dictionary_entry state sequence term_id updated -> + InterningTransition state updated. + +Theorem VWENC_132_EVERY_INTERNING_TRANSITION_PRESERVES_COMBINED_STATE_WELL_FORMEDNESS : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state updated : InterningState P I T), + InterningStateWellFormed state -> + InterningTransition state updated -> + InterningStateWellFormed updated. +Proof. + intros P I T state updated Hwell Htransition. + inversion Htransition; subst; + eauto using + fresh_publication_preserves_combined_state_well_formedness, + claim_atom_allocation_preserves_combined_state_well_formedness, + materialize_reserved_allocation_preserves_combined_state_well_formedness, + publish_claimed_allocation_preserves_combined_state_well_formedness, + orphan_claimed_allocation_preserves_combined_state_well_formedness, + orphan_reserved_allocation_preserves_combined_state_well_formedness, + tombstone_published_allocation_preserves_combined_state_well_formedness, + add_dependent_sequence_preserves_combined_state_well_formedness, + enable_term_dictionary_preserves_combined_state_well_formedness, + disable_empty_term_dictionary_preserves_combined_state_well_formedness, + insert_term_dictionary_entry_preserves_combined_state_well_formedness. +Qed. + +(** ** Exact allocation transition algebra *) + +Inductive AllocationPhase (P : CertifiedAtomProfile) : Type := +| PhaseUnallocated +| PhaseAllocated : CanonicalAtom P -> AllocationStatus -> AllocationPhase P. + +Definition allocation_phase_matches + {P : CertifiedAtomProfile} + {I T : FixedWidthCarrierProfile} + (state : InterningState P I T) + (id : SymbolId I) + (phase : AllocationPhase P) : Prop := + match phase with + | PhaseUnallocated _ => + forall atom status, + ~ allocation_status_category state atom id status + | PhaseAllocated _ atom status => + allocation_status_category state atom id status + end. + +Inductive LegalAllocationEdge (P : CertifiedAtomProfile) + : AllocationPhase P -> AllocationPhase P -> Prop := +| EdgeFreshToReserved : + forall atom, + LegalAllocationEdge P + (PhaseUnallocated P) + (PhaseAllocated P atom AllocationReserved) +| EdgeFreshToPublished : + forall atom, + LegalAllocationEdge P + (PhaseUnallocated P) + (PhaseAllocated P atom AllocationPublished) +| EdgeReservedToMaterializedClaimed : + forall atom, + LegalAllocationEdge P + (PhaseAllocated P atom AllocationReserved) + (PhaseAllocated P atom AllocationMaterializedClaimed) +| EdgeReservedToUnmaterializedOrphan : + forall atom, + LegalAllocationEdge P + (PhaseAllocated P atom AllocationReserved) + (PhaseAllocated P atom AllocationUnmaterializedOrphaned) +| EdgeMaterializedClaimedToPublished : + forall atom, + LegalAllocationEdge P + (PhaseAllocated P atom AllocationMaterializedClaimed) + (PhaseAllocated P atom AllocationPublished) +| EdgeMaterializedClaimedToMaterializedOrphan : + forall atom, + LegalAllocationEdge P + (PhaseAllocated P atom AllocationMaterializedClaimed) + (PhaseAllocated P atom AllocationMaterializedOrphaned) +| EdgePublishedToTombstoned : + forall atom, + LegalAllocationEdge P + (PhaseAllocated P atom AllocationPublished) + (PhaseAllocated P atom AllocationTombstoned). + +Inductive AllocationDelta + {P : CertifiedAtomProfile} + {I T : FixedWidthCarrierProfile} + (state updated : InterningState P I T) : Prop := +| AllocationDeltaNone : + (forall atom id status, + allocation_status_category state atom id status <-> + allocation_status_category updated atom id status) -> + AllocationDelta state updated +| AllocationDeltaOne : + forall id before after, + LegalAllocationEdge P before after -> + allocation_phase_matches state id before -> + allocation_phase_matches updated id after -> + (forall other_atom other_id status, + other_id <> id -> + (allocation_status_category state other_atom other_id status <-> + allocation_status_category updated other_atom other_id status)) -> + AllocationDelta state updated. + +Lemma allocation_category_preserved_by_exact_components : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state updated : InterningState P I T) atom id, + (In (atom, id) (state_reserved_entries P I T state) <-> + In (atom, id) (state_reserved_entries P I T updated)) -> + (In (atom, id) (state_claimed_entries P I T state) <-> + In (atom, id) (state_claimed_entries P I T updated)) -> + (In (atom, id) (state_live_entries P I T state) <-> + In (atom, id) (state_live_entries P I T updated)) -> + (In (atom, id) (state_ever_entries P I T state) <-> + In (atom, id) (state_ever_entries P I T updated)) -> + (In (atom, id) (state_orphan_entries P I T state) <-> + In (atom, id) (state_orphan_entries P I T updated)) -> + (In (atom, id) + (state_unmaterialized_orphan_entries P I T state) <-> + In (atom, id) + (state_unmaterialized_orphan_entries P I T updated)) -> + forall status, + allocation_status_category state atom id status <-> + allocation_status_category updated atom id status. +Proof. + intros P I T state updated atom id + Hreserved Hclaimed Hlive Hever Horphan Hunmaterialized status. + destruct status; simpl in *; tauto. +Qed. + +Lemma allocation_above_frontier_is_unallocated : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state : InterningState P I T) id, + InterningStateWellFormed state -> + state_allocator_frontier P I T state <= symbol_id_value I id -> + allocation_phase_matches state id (PhaseUnallocated P). +Proof. + intros P I T state id Hwell Habove atom status Hcategory. + apply allocation_status_category_entry_is_allocated in Hcategory. + pose proof (state_allocations_below_sparse_frontier state Hwell) + as Hbelow. + apply Forall_forall with (x := (atom, id)) in Hbelow; + [simpl in Hbelow; lia | exact Hcategory]. +Qed. + +Theorem VWENC_191_TERMINAL_ALLOCATION_PHASES_HAVE_NO_LEGAL_OUTBOUND_EDGE : + forall (P : CertifiedAtomProfile) atom after, + (~ LegalAllocationEdge P + (PhaseAllocated P atom AllocationTombstoned) after) /\ + (~ LegalAllocationEdge P + (PhaseAllocated P atom AllocationMaterializedOrphaned) after) /\ + (~ LegalAllocationEdge P + (PhaseAllocated P atom AllocationUnmaterializedOrphaned) after). +Proof. + intros P atom after. repeat split; intros Hedge; inversion Hedge. +Qed. + +Lemma fresh_publication_has_exact_allocation_delta : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state updated : InterningState P I T) atom id, + InterningStateWellFormed state -> + publish_fresh_atom state atom id = Some updated -> + AllocationDelta state updated. +Proof. + intros P I T state updated atom id Hwell Hpublish. + unfold publish_fresh_atom in Hpublish. + destruct (lookup_atom (state_ever_entries P I T state) atom); + [discriminate |]. + destruct (lookup_symbol (state_ever_entries P I T state) id); + [discriminate |]. + destruct (Nat.leb + (state_allocator_frontier P I T state) + (symbol_id_value I id)) eqn:Hfrontier; [| discriminate]. + destruct (append_packed_atom + (state_packed_storage P I T state) id atom) + as [packed |] eqn:Happend; [| discriminate]. + inversion Hpublish. subst updated. clear Hpublish. + eapply AllocationDeltaOne + with + (id := id) + (before := PhaseUnallocated P) + (after := PhaseAllocated P atom AllocationPublished). + - apply EdgeFreshToPublished. + - apply allocation_above_frontier_is_unallocated; [exact Hwell |]. + now apply Nat.leb_le. + - simpl. split; now left. + - intros other_atom other_id status Hother. + eapply allocation_category_preserved_by_exact_components; + simpl; try tauto. + + symmetry. now apply different_id_cons_entry_membership_iff. + + symmetry. now apply different_id_cons_entry_membership_iff. +Qed. + +Lemma claim_allocation_has_exact_allocation_delta : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state updated : InterningState P I T) atom id, + InterningStateWellFormed state -> + claim_atom_allocation state atom id = Some updated -> + AllocationDelta state updated. +Proof. + intros P I T state updated atom id Hwell Hclaim. + unfold claim_atom_allocation in Hclaim. + destruct (lookup_atom (state_ever_entries P I T state) atom); + [discriminate |]. + destruct (Nat.leb + (state_allocator_frontier P I T state) + (symbol_id_value I id)) eqn:Hfrontier; [| discriminate]. + inversion Hclaim. subst updated. clear Hclaim. + eapply AllocationDeltaOne + with + (id := id) + (before := PhaseUnallocated P) + (after := PhaseAllocated P atom AllocationReserved). + - apply EdgeFreshToReserved. + - apply allocation_above_frontier_is_unallocated; [exact Hwell |]. + now apply Nat.leb_le. + - simpl. now left. + - intros other_atom other_id status Hother. + eapply allocation_category_preserved_by_exact_components; + simpl; try tauto. + symmetry. now apply different_id_cons_entry_membership_iff. +Qed. + +Lemma materialize_reservation_has_exact_allocation_delta : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state updated : InterningState P I T) atom id, + InterningStateWellFormed state -> + materialize_reserved_allocation state atom id updated -> + AllocationDelta state updated. +Proof. + intros P I T state updated atom id Hwell Hmaterialize. + destruct Hmaterialize as + [prefix [suffix [packed [Hreserved [Happend Hupdated]]]]]. + subst updated. + eapply AllocationDeltaOne + with + (id := id) + (before := PhaseAllocated P atom AllocationReserved) + (after := PhaseAllocated P atom AllocationMaterializedClaimed). + - apply EdgeReservedToMaterializedClaimed. + - simpl. rewrite Hreserved. apply in_or_app. right. now left. + - simpl. now left. + - intros other_atom other_id status Hother. + eapply allocation_category_preserved_by_exact_components; simpl; try tauto. + + rewrite Hreserved. + now apply different_id_middle_entry_membership_iff. + + symmetry. now apply different_id_cons_entry_membership_iff. +Qed. + +Lemma orphan_reservation_has_exact_allocation_delta : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state updated : InterningState P I T) atom id, + InterningStateWellFormed state -> + orphan_reserved_allocation state atom id updated -> + AllocationDelta state updated. +Proof. + intros P I T state updated atom id Hwell Horphan. + destruct Horphan as [prefix [suffix [Hreserved Hupdated]]]. + subst updated. + eapply AllocationDeltaOne + with + (id := id) + (before := PhaseAllocated P atom AllocationReserved) + (after := + PhaseAllocated P atom AllocationUnmaterializedOrphaned). + - apply EdgeReservedToUnmaterializedOrphan. + - simpl. rewrite Hreserved. apply in_or_app. right. now left. + - simpl. now left. + - intros other_atom other_id status Hother. + eapply allocation_category_preserved_by_exact_components; simpl; try tauto. + + rewrite Hreserved. + now apply different_id_middle_entry_membership_iff. + + symmetry. now apply different_id_cons_entry_membership_iff. +Qed. + +Lemma publish_claim_has_exact_allocation_delta : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state updated : InterningState P I T) atom id, + InterningStateWellFormed state -> + publish_claimed_allocation state atom id updated -> + AllocationDelta state updated. +Proof. + intros P I T state updated atom id Hwell Hpublish. + destruct Hpublish as + [prefix [suffix + [Hclaimed [_ [_ [_ [_ Hupdated]]]]]]]. + subst updated. + eapply AllocationDeltaOne + with + (id := id) + (before := PhaseAllocated P atom AllocationMaterializedClaimed) + (after := PhaseAllocated P atom AllocationPublished). + - apply EdgeMaterializedClaimedToPublished. + - simpl. rewrite Hclaimed. apply in_or_app. right. now left. + - simpl. split; now left. + - intros other_atom other_id status Hother. + eapply allocation_category_preserved_by_exact_components; simpl; try tauto. + + rewrite Hclaimed. + now apply different_id_middle_entry_membership_iff. + + symmetry. now apply different_id_cons_entry_membership_iff. + + symmetry. now apply different_id_cons_entry_membership_iff. +Qed. + +Lemma orphan_claim_has_exact_allocation_delta : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state updated : InterningState P I T) atom id, + InterningStateWellFormed state -> + orphan_claimed_allocation state atom id updated -> + AllocationDelta state updated. +Proof. + intros P I T state updated atom id Hwell Horphan. + destruct Horphan as [prefix [suffix [Hclaimed Hupdated]]]. + subst updated. + eapply AllocationDeltaOne + with + (id := id) + (before := PhaseAllocated P atom AllocationMaterializedClaimed) + (after := PhaseAllocated P atom AllocationMaterializedOrphaned). + - apply EdgeMaterializedClaimedToMaterializedOrphan. + - simpl. rewrite Hclaimed. apply in_or_app. right. now left. + - simpl. now left. + - intros other_atom other_id status Hother. + eapply allocation_category_preserved_by_exact_components; simpl; try tauto. + + rewrite Hclaimed. + now apply different_id_middle_entry_membership_iff. + + symmetry. now apply different_id_cons_entry_membership_iff. +Qed. + +Lemma tombstone_publication_has_exact_allocation_delta : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state updated : InterningState P I T) atom id, + InterningStateWellFormed state -> + tombstone_published_allocation state atom id updated -> + AllocationDelta state updated. +Proof. + intros P I T state updated atom id Hwell Htombstone. + destruct Htombstone as + [prefix [suffix + [Hlive [_ [_ Hupdated]]]]]. + assert (Hever : In (atom, id) (state_ever_entries P I T state)). + { apply (state_live_is_historical state Hwell). + rewrite Hlive. apply in_or_app. right. now left. } + assert (Hnot_remaining : ~ In (atom, id) (prefix ++ suffix)). + { pose proof (state_live_bijection state Hwell) as [_ Hlive_ids]. + rewrite Hlive in Hlive_ids. + rewrite map_app in Hlive_ids. simpl in Hlive_ids. + intros Hin. + assert (Hin_id : In id (map snd prefix ++ map snd suffix)). + { rewrite <- map_app. now apply in_map with (f := snd) in Hin. } + eapply (NoDup_remove_2 + (map snd prefix) (map snd suffix) id Hlive_ids). + exact Hin_id. } + subst updated. + eapply AllocationDeltaOne + with + (id := id) + (before := PhaseAllocated P atom AllocationPublished) + (after := PhaseAllocated P atom AllocationTombstoned). + - apply EdgePublishedToTombstoned. + - simpl. split; [| exact Hever]. + rewrite Hlive. apply in_or_app. right. now left. + - simpl. now split; [exact Hever | exact Hnot_remaining]. + - intros other_atom other_id status Hother. + eapply allocation_category_preserved_by_exact_components; simpl; try tauto. + rewrite Hlive. + now apply different_id_middle_entry_membership_iff. +Qed. + +Lemma add_sequence_has_no_allocation_delta : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state updated : InterningState P I T) sequence, + add_dependent_sequence state sequence updated -> + AllocationDelta state updated. +Proof. + intros P I T state updated sequence [_ Hupdated]. + subst updated. apply AllocationDeltaNone. + intros atom id status. destruct status; reflexivity. +Qed. + +Lemma enable_term_dictionary_has_no_allocation_delta : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state updated : InterningState P I T), + enable_term_dictionary state updated -> + AllocationDelta state updated. +Proof. + intros P I T state updated Hupdated. + unfold enable_term_dictionary in Hupdated. subst updated. + apply AllocationDeltaNone. + intros atom id status. destruct status; reflexivity. +Qed. + +Lemma disable_term_dictionary_has_no_allocation_delta : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state updated : InterningState P I T), + disable_empty_term_dictionary state updated -> + AllocationDelta state updated. +Proof. + intros P I T state updated [_ Hupdated]. subst updated. + apply AllocationDeltaNone. + intros atom id status. destruct status; reflexivity. +Qed. + +Lemma insert_term_entry_has_no_allocation_delta : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state updated : InterningState P I T) sequence term_id, + insert_term_dictionary_entry state sequence term_id updated -> + AllocationDelta state updated. +Proof. + intros P I T state updated sequence term_id + [_ [_ [_ [_ Hupdated]]]]. + subst updated. apply AllocationDeltaNone. + intros atom id status. destruct status; reflexivity. +Qed. + +Theorem VWENC_189_EVERY_INTERNING_TRANSITION_HAS_ONE_EXACT_LEGAL_ALLOCATION_DELTA : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state updated : InterningState P I T), + InterningStateWellFormed state -> + InterningTransition state updated -> + AllocationDelta state updated. +Proof. + intros P I T state updated Hwell Htransition. + inversion Htransition; subst; + eauto using + fresh_publication_has_exact_allocation_delta, + claim_allocation_has_exact_allocation_delta, + materialize_reservation_has_exact_allocation_delta, + publish_claim_has_exact_allocation_delta, + orphan_claim_has_exact_allocation_delta, + orphan_reservation_has_exact_allocation_delta, + tombstone_publication_has_exact_allocation_delta, + add_sequence_has_no_allocation_delta, + enable_term_dictionary_has_no_allocation_delta, + disable_term_dictionary_has_no_allocation_delta, + insert_term_entry_has_no_allocation_delta. +Qed. + +Theorem VWENC_190_INTERNING_TRANSITIONS_PRESERVE_EVERY_UNAFFECTED_ID_STATUS : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state updated : InterningState P I T), + InterningStateWellFormed state -> + InterningTransition state updated -> + (forall atom id status, + allocation_status_category state atom id status <-> + allocation_status_category updated atom id status) \/ + exists changed_id, + forall atom id status, + id <> changed_id -> + (allocation_status_category state atom id status <-> + allocation_status_category updated atom id status). +Proof. + intros P I T state updated Hwell Htransition. + pose proof + (VWENC_189_EVERY_INTERNING_TRANSITION_HAS_ONE_EXACT_LEGAL_ALLOCATION_DELTA + P I T state updated Hwell Htransition) as Hdelta. + inversion Hdelta as [Hnone | id before after Hedge Hbefore Hafter Hothers]; + subst. + - now left. + - right. exists id. exact Hothers. +Qed. + +Inductive InterningReachable + (P : CertifiedAtomProfile) + (I T : FixedWidthCarrierProfile) + (fiber : VocabularyFiber P I) + (term_identity term_generation : nat) + : InterningState P I T -> Prop := +| ReachableInitial : + InterningReachable P I T fiber term_identity term_generation + (empty_interning_state + P I T fiber term_identity term_generation) +| ReachableStep : + forall state updated, + InterningReachable + P I T fiber term_identity term_generation state -> + InterningTransition state updated -> + InterningReachable + P I T fiber term_identity term_generation updated. + +Theorem VWENC_159_EVERY_REACHABLE_INTERNING_STATE_IS_WELL_FORMED : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (fiber : VocabularyFiber P I) term_identity term_generation state, + InterningReachable + P I T fiber term_identity term_generation state -> + InterningStateWellFormed state. +Proof. + intros P I T fiber term_identity term_generation state Hreachable. + induction Hreachable. + - apply VWENC_157_EMPTY_INTERNING_STATE_IS_WELL_FORMED. + - eapply VWENC_132_EVERY_INTERNING_TRANSITION_PRESERVES_COMBINED_STATE_WELL_FORMEDNESS; + eassumption. +Qed. + +Definition WitnessInterningState : Type := + InterningState canonical_uleb_profile u32_carrier u32_carrier. + +Definition witness_initial_state : WitnessInterningState := + empty_interning_state + canonical_uleb_profile u32_carrier u32_carrier + witness_vocabulary_fiber 900 1. + +Definition witness_term_fiber : + TermDictionaryFiber + canonical_uleb_profile u32_carrier u32_carrier + witness_vocabulary_fiber := + mkTermDictionaryFiber + canonical_uleb_profile u32_carrier u32_carrier + witness_vocabulary_fiber 900 1. + +Definition witness_packed_zero : PackedAtomStorage u32_carrier := + mkPackedAtomStorage u32_carrier + [1] + [(symbol_zero, mkByteSpan 0 1)]. + +Definition witness_reserved_zero : WitnessInterningState := + {| state_fiber := witness_vocabulary_fiber; + state_term_fiber := witness_term_fiber; + state_reserved_entries := [(collision_atom_left, symbol_zero)]; + state_claimed_entries := []; + state_live_entries := []; + state_ever_entries := []; + state_orphan_entries := []; + state_unmaterialized_orphan_entries := []; + state_packed_storage := empty_packed_atom_storage u32_carrier; + state_allocator_frontier := 1; + state_sequences := []; + state_term_dictionary_enabled := false; + state_term_entries := [] |}. + +Definition witness_materialized_zero : WitnessInterningState := + {| state_fiber := witness_vocabulary_fiber; + state_term_fiber := witness_term_fiber; + state_reserved_entries := []; + state_claimed_entries := [(collision_atom_left, symbol_zero)]; + state_live_entries := []; + state_ever_entries := []; + state_orphan_entries := []; + state_unmaterialized_orphan_entries := []; + state_packed_storage := witness_packed_zero; + state_allocator_frontier := 1; + state_sequences := []; + state_term_dictionary_enabled := false; + state_term_entries := [] |}. + +Definition witness_live_zero : WitnessInterningState := + {| state_fiber := witness_vocabulary_fiber; + state_term_fiber := witness_term_fiber; + state_reserved_entries := []; + state_claimed_entries := []; + state_live_entries := [(collision_atom_left, symbol_zero)]; + state_ever_entries := [(collision_atom_left, symbol_zero)]; + state_orphan_entries := []; + state_unmaterialized_orphan_entries := []; + state_packed_storage := witness_packed_zero; + state_allocator_frontier := 1; + state_sequences := []; + state_term_dictionary_enabled := false; + state_term_entries := [] |}. + +Definition witness_tombstoned_zero : WitnessInterningState := + {| state_fiber := witness_vocabulary_fiber; + state_term_fiber := witness_term_fiber; + state_reserved_entries := []; + state_claimed_entries := []; + state_live_entries := []; + state_ever_entries := [(collision_atom_left, symbol_zero)]; + state_orphan_entries := []; + state_unmaterialized_orphan_entries := []; + state_packed_storage := witness_packed_zero; + state_allocator_frontier := 1; + state_sequences := []; + state_term_dictionary_enabled := false; + state_term_entries := [] |}. + +Definition witness_materialized_orphan_zero : WitnessInterningState := + {| state_fiber := witness_vocabulary_fiber; + state_term_fiber := witness_term_fiber; + state_reserved_entries := []; + state_claimed_entries := []; + state_live_entries := []; + state_ever_entries := []; + state_orphan_entries := [(collision_atom_left, symbol_zero)]; + state_unmaterialized_orphan_entries := []; + state_packed_storage := witness_packed_zero; + state_allocator_frontier := 1; + state_sequences := []; + state_term_dictionary_enabled := false; + state_term_entries := [] |}. + +Definition witness_reserved_two_after_orphan : WitnessInterningState := + {| state_fiber := witness_vocabulary_fiber; + state_term_fiber := witness_term_fiber; + state_reserved_entries := [(collision_atom_right, symbol_two)]; + state_claimed_entries := []; + state_live_entries := []; + state_ever_entries := []; + state_orphan_entries := [(collision_atom_left, symbol_zero)]; + state_unmaterialized_orphan_entries := []; + state_packed_storage := witness_packed_zero; + state_allocator_frontier := 3; + state_sequences := []; + state_term_dictionary_enabled := false; + state_term_entries := [] |}. + +Definition witness_sparse_orphan_state : WitnessInterningState := + {| state_fiber := witness_vocabulary_fiber; + state_term_fiber := witness_term_fiber; + state_reserved_entries := []; + state_claimed_entries := []; + state_live_entries := []; + state_ever_entries := []; + state_orphan_entries := [(collision_atom_left, symbol_zero)]; + state_unmaterialized_orphan_entries := + [(collision_atom_right, symbol_two)]; + state_packed_storage := witness_packed_zero; + state_allocator_frontier := 3; + state_sequences := []; + state_term_dictionary_enabled := false; + state_term_entries := [] |}. + +Definition witness_live_sequence_zero : WitnessInterningState := + {| state_fiber := witness_vocabulary_fiber; + state_term_fiber := witness_term_fiber; + state_reserved_entries := []; + state_claimed_entries := []; + state_live_entries := [(collision_atom_left, symbol_zero)]; + state_ever_entries := [(collision_atom_left, symbol_zero)]; + state_orphan_entries := []; + state_unmaterialized_orphan_entries := []; + state_packed_storage := witness_packed_zero; + state_allocator_frontier := 1; + state_sequences := [[symbol_zero]]; + state_term_dictionary_enabled := false; + state_term_entries := [] |}. + +Definition witness_live_term_enabled : WitnessInterningState := + {| state_fiber := witness_vocabulary_fiber; + state_term_fiber := witness_term_fiber; + state_reserved_entries := []; + state_claimed_entries := []; + state_live_entries := [(collision_atom_left, symbol_zero)]; + state_ever_entries := [(collision_atom_left, symbol_zero)]; + state_orphan_entries := []; + state_unmaterialized_orphan_entries := []; + state_packed_storage := witness_packed_zero; + state_allocator_frontier := 1; + state_sequences := []; + state_term_dictionary_enabled := true; + state_term_entries := [] |}. + +Definition witness_live_term_entry : WitnessInterningState := + {| state_fiber := witness_vocabulary_fiber; + state_term_fiber := witness_term_fiber; + state_reserved_entries := []; + state_claimed_entries := []; + state_live_entries := [(collision_atom_left, symbol_zero)]; + state_ever_entries := [(collision_atom_left, symbol_zero)]; + state_orphan_entries := []; + state_unmaterialized_orphan_entries := []; + state_packed_storage := witness_packed_zero; + state_allocator_frontier := 1; + state_sequences := []; + state_term_dictionary_enabled := true; + state_term_entries := [([symbol_zero], term_zero)] |}. + +Lemma witness_reserve_zero : + claim_atom_allocation + witness_initial_state collision_atom_left symbol_zero = + Some witness_reserved_zero. +Proof. reflexivity. Qed. + +Lemma witness_materialize_zero : + materialize_reserved_allocation + witness_reserved_zero collision_atom_left symbol_zero + witness_materialized_zero. +Proof. + unfold materialize_reserved_allocation. + exists [], [], witness_packed_zero. repeat split; reflexivity. +Qed. + +Lemma witness_publish_zero : + publish_claimed_allocation + witness_materialized_zero collision_atom_left symbol_zero + witness_live_zero. +Proof. + unfold publish_claimed_allocation. + exists [], []. repeat split; reflexivity. +Qed. + +Lemma witness_orphan_materialized_zero : + orphan_claimed_allocation + witness_materialized_zero collision_atom_left symbol_zero + witness_materialized_orphan_zero. +Proof. + unfold orphan_claimed_allocation. + exists [], []. now split. +Qed. + +Lemma witness_reserve_two_after_orphan : + claim_atom_allocation + witness_materialized_orphan_zero collision_atom_right symbol_two = + Some witness_reserved_two_after_orphan. +Proof. reflexivity. Qed. + +Lemma witness_orphan_unmaterialized_two : + orphan_reserved_allocation + witness_reserved_two_after_orphan collision_atom_right symbol_two + witness_sparse_orphan_state. +Proof. + unfold orphan_reserved_allocation. + exists [], []. now split. +Qed. + +Lemma witness_fresh_publish_zero : + publish_fresh_atom + witness_initial_state collision_atom_left symbol_zero = + Some witness_live_zero. +Proof. reflexivity. Qed. + +Lemma witness_tombstone_zero : + tombstone_published_allocation + witness_live_zero collision_atom_left symbol_zero + witness_tombstoned_zero. +Proof. + unfold tombstone_published_allocation. + exists [], []. repeat split; constructor. +Qed. + +Lemma witness_add_live_sequence : + add_dependent_sequence + witness_live_zero [symbol_zero] witness_live_sequence_zero. +Proof. + unfold add_dependent_sequence. split. + - constructor. + + split; [simpl; lia |]. + exists collision_atom_left. now left. + + constructor. + - reflexivity. +Qed. + +Lemma witness_enable_term_dictionary : + enable_term_dictionary witness_live_zero witness_live_term_enabled. +Proof. reflexivity. Qed. + +Lemma witness_disable_empty_term_dictionary : + disable_empty_term_dictionary witness_live_term_enabled witness_live_zero. +Proof. now split. Qed. + +Lemma witness_insert_term_entry : + insert_term_dictionary_entry + witness_live_term_enabled [symbol_zero] term_zero + witness_live_term_entry. +Proof. + unfold insert_term_dictionary_entry. repeat split. + - constructor. + + split; [simpl; lia |]. + exists collision_atom_left. now left. + + constructor. +Qed. + +Lemma witness_sparse_state_is_reachable : + InterningReachable + canonical_uleb_profile u32_carrier u32_carrier + witness_vocabulary_fiber 900 1 witness_sparse_orphan_state. +Proof. + eapply ReachableStep. + - eapply ReachableStep. + + eapply ReachableStep. + * eapply ReachableStep. + { eapply ReachableStep. + - apply ReachableInitial. + - eapply TransitionClaimAllocation + with (atom := collision_atom_left) (id := symbol_zero). + exact witness_reserve_zero. } + { eapply TransitionMaterializeReservation + with (atom := collision_atom_left) (id := symbol_zero). + exact witness_materialize_zero. } + * eapply TransitionOrphanClaim + with (atom := collision_atom_left) (id := symbol_zero). + exact witness_orphan_materialized_zero. + + eapply TransitionClaimAllocation + with (atom := collision_atom_right) (id := symbol_two). + exact witness_reserve_two_after_orphan. + - eapply TransitionOrphanReservation + with (atom := collision_atom_right) (id := symbol_two). + exact witness_orphan_unmaterialized_two. +Qed. + +Theorem VWENC_166_EVERY_TRANSITION_FAMILY_HAS_A_CONCRETE_WITNESS : + InterningTransition witness_initial_state witness_live_zero /\ + InterningTransition witness_initial_state witness_reserved_zero /\ + InterningTransition witness_reserved_zero witness_materialized_zero /\ + InterningTransition witness_materialized_zero witness_live_zero /\ + InterningTransition witness_materialized_zero + witness_materialized_orphan_zero /\ + InterningTransition witness_reserved_two_after_orphan + witness_sparse_orphan_state /\ + InterningTransition witness_live_zero witness_tombstoned_zero /\ + InterningTransition witness_live_zero witness_live_sequence_zero /\ + InterningTransition witness_live_zero witness_live_term_enabled /\ + InterningTransition witness_live_term_enabled witness_live_zero /\ + InterningTransition witness_live_term_enabled witness_live_term_entry. +Proof. + repeat split. + - eapply TransitionFreshPublication + with (atom := collision_atom_left) (id := symbol_zero). + exact witness_fresh_publish_zero. + - eapply TransitionClaimAllocation + with (atom := collision_atom_left) (id := symbol_zero). + exact witness_reserve_zero. + - eapply TransitionMaterializeReservation + with (atom := collision_atom_left) (id := symbol_zero). + exact witness_materialize_zero. + - eapply TransitionPublishClaim + with (atom := collision_atom_left) (id := symbol_zero). + exact witness_publish_zero. + - eapply TransitionOrphanClaim + with (atom := collision_atom_left) (id := symbol_zero). + exact witness_orphan_materialized_zero. + - eapply TransitionOrphanReservation + with (atom := collision_atom_right) (id := symbol_two). + exact witness_orphan_unmaterialized_two. + - eapply TransitionTombstonePublished + with (atom := collision_atom_left) (id := symbol_zero). + exact witness_tombstone_zero. + - eapply TransitionAddDependentSequence with (sequence := [symbol_zero]). + exact witness_add_live_sequence. + - apply TransitionEnableTermDictionary. exact witness_enable_term_dictionary. + - apply TransitionDisableEmptyTermDictionary. + exact witness_disable_empty_term_dictionary. + - eapply TransitionInsertTermEntry + with (sequence := [symbol_zero]) (term_id := term_zero). + exact witness_insert_term_entry. +Qed. + +Theorem VWENC_167_EVERY_ALLOCATION_STATUS_HAS_A_CONCRETE_WITNESS : + allocation_has_status witness_reserved_zero + collision_atom_left symbol_zero AllocationReserved /\ + allocation_has_status witness_materialized_zero + collision_atom_left symbol_zero AllocationMaterializedClaimed /\ + allocation_has_status witness_live_zero + collision_atom_left symbol_zero AllocationPublished /\ + allocation_has_status witness_tombstoned_zero + collision_atom_left symbol_zero AllocationTombstoned /\ + allocation_has_status witness_materialized_orphan_zero + collision_atom_left symbol_zero AllocationMaterializedOrphaned /\ + allocation_has_status witness_sparse_orphan_state + collision_atom_right symbol_two AllocationUnmaterializedOrphaned. +Proof. + repeat split. + - apply allocation_status_reserved_from_membership. + simpl. now left. + - apply allocation_status_materialized_claimed_from_membership. + + simpl. tauto. + + simpl. now left. + - apply allocation_status_published_from_membership. + + simpl. tauto. + + simpl. tauto. + + simpl. tauto. + + simpl. tauto. + + simpl. now left. + + simpl. now left. + - apply allocation_status_tombstoned_from_membership. + + simpl. tauto. + + simpl. tauto. + + simpl. tauto. + + simpl. tauto. + + simpl. now left. + + simpl. tauto. + - apply allocation_status_materialized_orphan_from_membership. + + simpl. tauto. + + simpl. tauto. + + simpl. now left. + - apply allocation_status_unmaterialized_orphan_from_membership. + + simpl. tauto. + + simpl. tauto. + + simpl. intros [Hequal | []]. + apply (f_equal snd) in Hequal. simpl in Hequal. + now apply symbol_two_differs_from_symbol_zero. + + simpl. now left. +Qed. + +Theorem VWENC_108_SPARSE_FRONTIER_HAS_A_GAP_AND_BOTH_ORPHAN_CLASSES : + exists state : WitnessInterningState, + InterningStateWellFormed state /\ + symbol_id_value u32_carrier symbol_one < + state_allocator_frontier + canonical_uleb_profile u32_carrier u32_carrier state /\ + ~ In symbol_one (map snd (state_allocation_entries state)) /\ + allocation_has_status state + collision_atom_left symbol_zero AllocationMaterializedOrphaned /\ + allocation_has_status state + collision_atom_right symbol_two AllocationUnmaterializedOrphaned /\ + state_live_entries + canonical_uleb_profile u32_carrier u32_carrier state = [] /\ + state_sequences + canonical_uleb_profile u32_carrier u32_carrier state = [] /\ + state_term_entries + canonical_uleb_profile u32_carrier u32_carrier state = []. +Proof. + exists witness_sparse_orphan_state. + split. + - eapply VWENC_159_EVERY_REACHABLE_INTERNING_STATE_IS_WELL_FORMED. + exact witness_sparse_state_is_reachable. + - split. + + unfold witness_sparse_orphan_state, symbol_one. + simpl. lia. + + split. + * simpl. intros [Hequal | [Hequal | []]]. + { apply (f_equal (symbol_id_value u32_carrier)) in Hequal. + discriminate. } + { apply (f_equal (symbol_id_value u32_carrier)) in Hequal. + discriminate. } + * split. + { apply allocation_status_materialized_orphan_from_membership. + - simpl. tauto. + - simpl. tauto. + - simpl. now left. } + { split. + - apply allocation_status_unmaterialized_orphan_from_membership. + + simpl. tauto. + + simpl. tauto. + + simpl. intros [Hequal | []]. + apply (f_equal snd) in Hequal. simpl in Hequal. + now apply symbol_two_differs_from_symbol_zero. + + simpl. now left. + - split; [reflexivity |]. + split; reflexivity. } +Qed. + +Theorem VWENC_130_FRESH_INSERT_PRESERVES_EXISTING_ATOM_LOOKUPS : + forall (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (entries : list (VocabularyEntry P I)) + new_atom existing_atom new_id existing_id, + existing_atom <> new_atom -> + lookup_atom entries existing_atom = Some existing_id -> + lookup_atom ((new_atom, new_id) :: entries) existing_atom = + Some existing_id. +Proof. + intros P I entries new_atom existing_atom new_id existing_id + Hdifferent Hlookup. + unfold lookup_atom. simpl. + destruct (canonical_atom_eq_dec P existing_atom new_atom); + [contradiction | exact Hlookup]. +Qed. + +Theorem VWENC_131_FRESH_INSERT_PRESERVES_EXISTING_REVERSE_LOOKUPS : + forall (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (entries : list (VocabularyEntry P I)) + new_atom existing_atom new_id existing_id, + existing_id <> new_id -> + lookup_symbol entries existing_id = Some existing_atom -> + lookup_symbol ((new_atom, new_id) :: entries) existing_id = + Some existing_atom. +Proof. + intros P I entries new_atom existing_atom new_id existing_id + Hdifferent Hlookup. + unfold lookup_symbol, reverse_vocabulary_entries in *. simpl. + destruct (symbol_id_eq_dec I existing_id new_id); + [contradiction | exact Hlookup]. +Qed. + +Theorem VWENC_133_EVERY_LIVE_ID_HAS_EXACT_NONEMPTY_BOUNDED_CANONICAL_SPAN : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state : InterningState P I T) atom id, + InterningStateWellFormed state -> + In (atom, id) (state_live_entries P I T state) -> + packed_entry_exact (state_packed_storage P I T state) atom id. +Proof. + intros P I T state atom id Hwell Hlive. + destruct Hwell as + [_ _ Hlive_history _ Hpacked]. + destruct Hpacked as [_ [_ [_ [Hexact _]]]]. + apply Hexact. + unfold state_materialized_entries. + apply in_or_app. left. + now apply Hlive_history. +Qed. + +(** ** Certified vocabulary snapshots and sequence descriptors *) + +Record VocabularySnapshot + (P : CertifiedAtomProfile) + (I : FixedWidthCarrierProfile) + (fiber : VocabularyFiber P I) : Type := + mkVocabularySnapshot { + vocabulary_snapshot_live_entries : list (VocabularyEntry P I); + vocabulary_snapshot_available_frontier : nat; + vocabulary_snapshot_packed_storage : PackedAtomStorage I; + vocabulary_snapshot_live_bijection : + vocabulary_relation_well_formed vocabulary_snapshot_live_entries; + vocabulary_snapshot_frontier_representable : + vocabulary_snapshot_available_frontier <= carrier_capacity I; + vocabulary_snapshot_live_ids_below_frontier : + Forall + (fun entry => + symbol_id_value I (snd entry) < + vocabulary_snapshot_available_frontier) + vocabulary_snapshot_live_entries; + vocabulary_snapshot_live_metadata_exact : + forall atom id, + In (atom, id) vocabulary_snapshot_live_entries -> + packed_entry_exact vocabulary_snapshot_packed_storage atom id + }. + +Definition capture_vocabulary_snapshot + {P : CertifiedAtomProfile} + {I T : FixedWidthCarrierProfile} + (state : InterningState P I T) + (Hwell : InterningStateWellFormed state) + : VocabularySnapshot P I (state_fiber P I T state). +Proof. + pose proof Hwell as Hwhole. + destruct Hwell as + [Hlive_bijection _ Hlive_history _ _ Halloc_below Hfrontier]. + refine (mkVocabularySnapshot + P I (state_fiber P I T state) + (state_live_entries P I T state) + (state_allocator_frontier P I T state) + (state_packed_storage P I T state) + Hlive_bijection Hfrontier _ _). + - apply Forall_forall. intros entry Hlive. + apply Forall_forall with (x := entry) in Halloc_below. + + exact Halloc_below. + + unfold state_allocation_entries. + apply in_or_app. left. + destruct entry as [atom id]. + now apply Hlive_history. + - intros atom id Hlive. + now apply VWENC_133_EVERY_LIVE_ID_HAS_EXACT_NONEMPTY_BOUNDED_CANONICAL_SPAN. +Defined. + +Theorem VWENC_181_CAPTURED_VOCABULARY_SNAPSHOT_IS_ONE_EXACT_STATE_FIBER : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state : InterningState P I T) + (Hwell : InterningStateWellFormed state), + vocabulary_fiber_identity (state_fiber P I T state) = + vocabulary_fiber_identity (state_fiber P I T state) /\ + vocabulary_snapshot_live_entries + P I (state_fiber P I T state) + (capture_vocabulary_snapshot state Hwell) = + state_live_entries P I T state /\ + vocabulary_snapshot_available_frontier + P I (state_fiber P I T state) + (capture_vocabulary_snapshot state Hwell) = + state_allocator_frontier P I T state /\ + vocabulary_snapshot_packed_storage + P I (state_fiber P I T state) + (capture_vocabulary_snapshot state Hwell) = + state_packed_storage P I T state. +Proof. + intros P I T state Hwell. + destruct Hwell. repeat split. +Qed. + +Record SequenceDescriptor + (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) : Type := + mkSequenceDescriptor { + descriptor_fiber : VocabularyFiber P I; + descriptor_required_frontier : nat; + descriptor_ids : list (SymbolId I) + }. + +Definition descriptor_accepts_snapshot + {P : CertifiedAtomProfile} {I : FixedWidthCarrierProfile} + (expected_fiber : VocabularyFiber P I) + (live : list (VocabularyEntry P I)) + (available_frontier : nat) + (descriptor : SequenceDescriptor P I) : Prop := + descriptor_fiber P I descriptor = expected_fiber /\ + descriptor_required_frontier P I descriptor <= available_frontier /\ + Forall + (fun id => + symbol_id_value I id < + descriptor_required_frontier P I descriptor /\ + live_symbol live id) + (descriptor_ids P I descriptor). + +Definition descriptor_accepts_vocabulary_snapshot + {P : CertifiedAtomProfile} {I : FixedWidthCarrierProfile} + {fiber : VocabularyFiber P I} + (snapshot : VocabularySnapshot P I fiber) + (descriptor : SequenceDescriptor P I) : Prop := + descriptor_accepts_snapshot + fiber + (vocabulary_snapshot_live_entries P I fiber snapshot) + (vocabulary_snapshot_available_frontier P I fiber snapshot) + descriptor. + +Definition encode_symbol_sequence + (I : FixedWidthCarrierProfile) (ids : list (SymbolId I)) + : list PhysicalByte := + flat_map (encode_symbol_id I) ids. + +(** ** Fiber-bound fixed-width ID sequence views *) + +Record IdSequenceBacking + (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) : Type := + mkIdSequenceBacking { + backing_identity : nat; + backing_fiber : VocabularyFiber P I; + backing_snapshot : VocabularySnapshot P I backing_fiber; + backing_descriptor : SequenceDescriptor P I; + backing_descriptor_accepted : + descriptor_accepts_vocabulary_snapshot + backing_snapshot backing_descriptor; + backing_bytes : list PhysicalByte; + backing_bytes_encode_exact_descriptor : + backing_bytes = + encode_symbol_sequence I (descriptor_ids P I backing_descriptor); + backing_bytes_are_valid : Forall valid_byte backing_bytes; + backing_bytes_have_exact_descriptor_length : + List.length backing_bytes = + List.length (descriptor_ids P I backing_descriptor) * + carrier_width_bytes I + }. + +Definition valid_id_sequence_backing + {P : CertifiedAtomProfile} {I : FixedWidthCarrierProfile} + (backing : IdSequenceBacking P I) : Prop := + Forall valid_byte (backing_bytes P I backing) /\ + List.length (backing_bytes P I backing) = + List.length + (descriptor_ids P I (backing_descriptor P I backing)) * + carrier_width_bytes I /\ + descriptor_accepts_vocabulary_snapshot + (backing_snapshot P I backing) + (backing_descriptor P I backing) /\ + backing_bytes P I backing = + encode_symbol_sequence I + (descriptor_ids P I (backing_descriptor P I backing)). + +Theorem VWENC_182_EVERY_ID_SEQUENCE_BACKING_IS_CERTIFIED_BY_ONE_EXACT_SNAPSHOT : + forall (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (backing : IdSequenceBacking P I), + valid_id_sequence_backing backing. +Proof. + intros P I backing. + split; [exact (backing_bytes_are_valid P I backing) |]. + split; [exact (backing_bytes_have_exact_descriptor_length P I backing) |]. + split. + - exact (backing_descriptor_accepted P I backing). + - exact (backing_bytes_encode_exact_descriptor P I backing). +Qed. + +Record IdSequenceView + (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) : Type := + mkIdSequenceView { + view_backing : IdSequenceBacking P I; + view_start : nat; + view_count : nat + }. + +Definition valid_id_sequence_view + {P : CertifiedAtomProfile} {I : FixedWidthCarrierProfile} + (view : IdSequenceView P I) : Prop := + valid_id_sequence_backing (view_backing P I view) /\ + view_start P I view + view_count P I view <= + List.length + (descriptor_ids P I + (backing_descriptor P I (view_backing P I view))). + +Definition id_sequence_view_byte_offset + {P : CertifiedAtomProfile} {I : FixedWidthCarrierProfile} + (view : IdSequenceView P I) (index : nat) : nat := + (view_start P I view + index) * carrier_width_bytes I. + +Definition id_sequence_view_byte_window + {P : CertifiedAtomProfile} {I : FixedWidthCarrierProfile} + (view : IdSequenceView P I) (index : nat) + : list PhysicalByte := + firstn + (carrier_width_bytes I) + (skipn + (id_sequence_view_byte_offset view index) + (backing_bytes P I (view_backing P I view))). + +Definition id_sequence_view_bytes + {P : CertifiedAtomProfile} {I : FixedWidthCarrierProfile} + (view : IdSequenceView P I) (index : nat) + : option (list PhysicalByte) := + if index + match decode_symbol_id I bytes with + | Some id => + Some + (mkFiberBoundSymbolId P I + (backing_fiber P I (view_backing P I view)) id) + | None => None + end + | None => None + end. + +Definition id_sequence_subview + {P : CertifiedAtomProfile} {I : FixedWidthCarrierProfile} + (view : IdSequenceView P I) (offset count : nat) + : option (IdSequenceView P I) := + if offset + count <=? view_count P I view + then Some + (mkIdSequenceView P I + (view_backing P I view) + (view_start P I view + offset) + count) + else None. + +Lemma Forall_firstn_preserved : + forall (A : Type) (predicate : A -> Prop) count values, + Forall predicate values -> + Forall predicate (firstn count values). +Proof. + intros A predicate count. + induction count as [| count IH]; intros values Hforall. + - simpl. constructor. + - destruct values as [| value rest]. + + simpl. constructor. + + inversion Hforall; subst. simpl. constructor; [assumption |]. + now apply IH. +Qed. + +Lemma Forall_skipn_preserved : + forall (A : Type) (predicate : A -> Prop) count values, + Forall predicate values -> + Forall predicate (skipn count values). +Proof. + intros A predicate count. + induction count as [| count IH]; intros values Hforall. + - exact Hforall. + - destruct values as [| value rest]. + + simpl. constructor. + + inversion Hforall; subst. simpl. now apply IH. +Qed. + +Lemma decode_fixed_little_endian_bounded_by_width : + forall bytes, + Forall valid_byte bytes -> + decode_fixed_little_endian bytes < 256 ^ List.length bytes. +Proof. + induction bytes as [| byte rest IH]; intros Hvalid. + - simpl. lia. + - inversion Hvalid as [| current tail Hbyte Hrest]; subst. + specialize (IH Hrest). + cbn [decode_fixed_little_endian List.length]. + rewrite Nat.pow_succ_r by lia. + unfold valid_byte in Hbyte. + nia. +Qed. + +Lemma decode_symbol_id_accepts_every_exact_width_byte_window : + forall (I : FixedWidthCarrierProfile) bytes, + List.length bytes = carrier_width_bytes I -> + Forall valid_byte bytes -> + exists id, decode_symbol_id I bytes = Some id. +Proof. + intros I bytes Hlength Hvalid. + assert (Hbounded : + decode_fixed_little_endian bytes < carrier_capacity I). + { unfold carrier_capacity. rewrite <- Hlength. + now apply decode_fixed_little_endian_bounded_by_width. } + unfold decode_symbol_id. + rewrite Hlength. + destruct (Nat.eq_dec (carrier_width_bytes I) (carrier_width_bytes I)) + as [_ | Himpossible]; [| contradiction]. + assert (Hvalidb : all_valid_bytesb bytes = true). + { now apply (proj2 (all_valid_bytesb_reflects_validity bytes)). } + rewrite Hvalidb. unfold symbol_id_of_nat. + destruct (lt_dec (decode_fixed_little_endian bytes) + (carrier_capacity I)) as [Hfits | Hoverflow]. + - eexists. reflexivity. + - contradiction. +Qed. + +Lemma firstn_exact_left_append : + forall (A : Type) (left right : list A), + firstn (List.length left) (left ++ right) = left. +Proof. + intros A left. + induction left as [| value tail IH]; intros right. + - reflexivity. + - simpl. now rewrite IH. +Qed. + +Lemma skipn_exact_left_append : + forall (A : Type) (left right : list A) count, + skipn (List.length left + count) (left ++ right) = + skipn count right. +Proof. + intros A left. + induction left as [| value tail IH]; intros right count. + - reflexivity. + - simpl. now rewrite IH. +Qed. + +Lemma encoded_symbol_sequence_window_at : + forall (I : FixedWidthCarrierProfile) ids index id, + nth_error ids index = Some id -> + firstn + (carrier_width_bytes I) + (skipn + (index * carrier_width_bytes I) + (encode_symbol_sequence I ids)) = + encode_symbol_id I id. +Proof. + intros I ids. + induction ids as [| head tail IH]; intros index id Hnth. + - destruct index; discriminate. + - destruct index as [| index]. + + simpl in Hnth. inversion Hnth. subst id. + change + (firstn (carrier_width_bytes I) + (encode_symbol_id I head ++ encode_symbol_sequence I tail) = + encode_symbol_id I head). + rewrite <- (proj1 (symbol_id_fixed_width_encoding_roundtrips I head)). + apply firstn_exact_left_append. + + simpl in Hnth. + change + (firstn (carrier_width_bytes I) + (skipn (S index * carrier_width_bytes I) + (encode_symbol_id I head ++ encode_symbol_sequence I tail)) = + encode_symbol_id I id). + assert (Hhead_length : + List.length (encode_symbol_id I head) = carrier_width_bytes I). + { apply symbol_id_fixed_width_encoding_roundtrips. } + replace (S index * carrier_width_bytes I) with + (List.length (encode_symbol_id I head) + + index * carrier_width_bytes I) by + (rewrite Hhead_length; lia). + rewrite skipn_exact_left_append. + now apply IH. +Qed. + +Lemma valid_id_sequence_view_window_is_descriptor_encoding : + forall (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (view : IdSequenceView P I) index, + valid_id_sequence_view view -> + index < view_count P I view -> + exists id, + nth_error + (descriptor_ids P I + (backing_descriptor P I (view_backing P I view))) + (view_start P I view + index) = Some id /\ + id_sequence_view_byte_window view index = encode_symbol_id I id. +Proof. + intros P I view index [Hbacking Hrange] Hindex. + destruct Hbacking as [_ [_ [_ Hbytes_exact]]]. + assert (Hposition : + view_start P I view + index < + List.length + (descriptor_ids P I + (backing_descriptor P I (view_backing P I view)))) by lia. + destruct (nth_error + (descriptor_ids P I + (backing_descriptor P I (view_backing P I view))) + (view_start P I view + index)) as [id |] eqn:Hnth. + - exists id. split; [reflexivity |]. + unfold id_sequence_view_byte_window, + id_sequence_view_byte_offset. + rewrite Hbytes_exact. + now apply encoded_symbol_sequence_window_at. + - apply nth_error_None in Hnth. lia. +Qed. + +Lemma valid_id_sequence_view_has_exact_byte_window : + forall (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (view : IdSequenceView P I) index, + valid_id_sequence_view view -> + index < view_count P I view -> + List.length (id_sequence_view_byte_window view index) = + carrier_width_bytes I /\ + Forall valid_byte (id_sequence_view_byte_window view index). +Proof. + intros P I view index + [[Hbytes [Hbacking_length [_ _]]] Hrange] Hindex. + pose proof (carrier_width_positive I) as Hwidth. + assert (Hwindow_end : + id_sequence_view_byte_offset view index + carrier_width_bytes I <= + List.length (backing_bytes P I (view_backing P I view))). + { unfold id_sequence_view_byte_offset. nia. } + unfold id_sequence_view_byte_window. + split. + - rewrite firstn_length_local, skipn_length_local. + rewrite Nat.min_l; lia. + - apply Forall_firstn_preserved. + now apply Forall_skipn_preserved. +Qed. + +Theorem VWENC_116_VALID_ID_VIEW_INDEXES_BOUND_BACKING_DIRECTLY : + forall (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (view : IdSequenceView P I) index, + valid_id_sequence_view view -> + index < view_count P I view -> + exists bytes id, + nth_error + (descriptor_ids P I + (backing_descriptor P I (view_backing P I view))) + (view_start P I view + index) = Some id /\ + id_sequence_view_bytes view index = Some bytes /\ + bytes = id_sequence_view_byte_window view index /\ + bytes = encode_symbol_id I id /\ + List.length bytes = carrier_width_bytes I /\ + Forall valid_byte bytes /\ + decode_symbol_id I bytes = Some id /\ + live_symbol + (vocabulary_snapshot_live_entries + P I + (backing_fiber P I (view_backing P I view)) + (backing_snapshot P I (view_backing P I view))) id /\ + id_sequence_view_index view index = + Some + (mkFiberBoundSymbolId P I + (backing_fiber P I (view_backing P I view)) id). +Proof. + intros P I view index Hvalid Hindex. + destruct (valid_id_sequence_view_window_is_descriptor_encoding + P I view index Hvalid Hindex) as [id [Hnth Hencoded]]. + destruct (valid_id_sequence_view_has_exact_byte_window + P I view index Hvalid Hindex) as [Hlength Hbytes]. + assert (Hdecode : + decode_symbol_id I (id_sequence_view_byte_window view index) = Some id). + { rewrite Hencoded. + apply symbol_id_fixed_width_encoding_roundtrips. } + assert (Hlive : + live_symbol + (vocabulary_snapshot_live_entries + P I + (backing_fiber P I (view_backing P I view)) + (backing_snapshot P I (view_backing P I view))) id). + { destruct Hvalid as [[_ [_ [Haccepted _]]] _]. + unfold descriptor_accepts_vocabulary_snapshot, + descriptor_accepts_snapshot in Haccepted. + destruct Haccepted as [_ [_ Hids]]. + apply Forall_forall with (x := id) in Hids. + - exact (proj2 Hids). + - now apply nth_error_In in Hnth. } + exists (id_sequence_view_byte_window view index), id. + split; [exact Hnth |]. split. + - unfold id_sequence_view_bytes. + apply Nat.ltb_lt in Hindex. now rewrite Hindex. + - split; [reflexivity |]. + split; [exact Hencoded |]. + split; [exact Hlength |]. + split; [exact Hbytes |]. + split; [exact Hdecode |]. + split; [exact Hlive |]. + unfold id_sequence_view_index, id_sequence_view_bytes. + apply Nat.ltb_lt in Hindex. now rewrite Hindex, Hdecode. +Qed. + +Theorem VWENC_117_SUBVIEW_PRESERVES_BACKING_FIBER_AND_VALID_RANGE : + forall (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (view subview : IdSequenceView P I) offset count, + valid_id_sequence_view view -> + id_sequence_subview view offset count = Some subview -> + view_backing P I subview = view_backing P I view /\ + backing_fiber P I (view_backing P I subview) = + backing_fiber P I (view_backing P I view) /\ + valid_id_sequence_view subview. +Proof. + intros P I view subview offset count Hvalid Hsubview. + unfold id_sequence_subview in Hsubview. + destruct (offset + count <=? view_count P I view) + eqn:Hrange; [| discriminate]. + apply Nat.leb_le in Hrange. + inversion Hsubview. subst subview. clear Hsubview. + split; [reflexivity |]. + split; [reflexivity |]. + destruct Hvalid as [Hbacking Hvalid]. + split; [exact Hbacking |]. + simpl in *. pose proof (carrier_width_positive I). nia. +Qed. + +Theorem VWENC_134_ID_SEQUENCE_VIEW_REJECTS_OUT_OF_RANGE_INDEX : + forall (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (view : IdSequenceView P I) index, + view_count P I view <= index -> + id_sequence_view_index view index = None. +Proof. + intros P I view index Hrange. + unfold id_sequence_view_index, id_sequence_view_bytes. + destruct (index + id_sequence_view_index view index = Some bound_id -> + index < view_count P I view /\ + exists bytes id, + bound_id = + mkFiberBoundSymbolId P I + (backing_fiber P I (view_backing P I view)) id /\ + nth_error + (descriptor_ids P I + (backing_descriptor P I (view_backing P I view))) + (view_start P I view + index) = Some id /\ + id_sequence_view_bytes view index = Some bytes /\ + bytes = id_sequence_view_byte_window view index /\ + bytes = encode_symbol_id I id /\ + live_symbol + (vocabulary_snapshot_live_entries + P I + (backing_fiber P I (view_backing P I view)) + (backing_snapshot P I (view_backing P I view))) id /\ + id_sequence_view_byte_offset view index = + (view_start P I view + index) * carrier_width_bytes I /\ + List.length bytes = carrier_width_bytes I /\ + Forall valid_byte bytes /\ + decode_symbol_id I bytes = Some id /\ + List.length (encode_symbol_id I id) = carrier_width_bytes I /\ + decode_symbol_id I (encode_symbol_id I id) = Some id. +Proof. + intros P I view index bound_id Hvalid Hindex. + assert (Hwithin : index < view_count P I view). + { destruct (index + id_sequence_view_index view index = Some bound_id -> + expected <> backing_fiber P I (view_backing P I view) -> + interpret_symbol_id expected bound_id = None. +Proof. + intros P I view index bound_id expected Hvalid Hindex Hdifferent. + destruct (VWENC_135_ID_VIEW_ELEMENTS_HAVE_EXACT_CARRIER_STRIDE + P I view index bound_id Hvalid Hindex) + as [_ [bytes [id [Hbound _]]]]. + subst bound_id. + now apply VWENC_112_CROSS_FIBER_ID_INTERPRETATION_IS_REJECTED. +Qed. + +(** ** Two-level term IDs and exact vocabulary binding *) + +Fixpoint interpret_bound_symbol_sequence + {P : CertifiedAtomProfile} + {I : FixedWidthCarrierProfile} + (expected : VocabularyFiber P I) + (sequence : list (FiberBoundSymbolId P I)) + : option (list (SymbolId I)) := + match sequence with + | [] => Some [] + | bound :: tail => + match interpret_symbol_id expected bound, + interpret_bound_symbol_sequence expected tail with + | Some id, Some ids => Some (id :: ids) + | _, _ => None + end + end. + +Definition resolve_atom_then_term + {P : CertifiedAtomProfile} + {I T : FixedWidthCarrierProfile} + (state : InterningState P I T) + (atom : CanonicalAtom P) + (tail : list (FiberBoundSymbolId P I)) + : option + (FiberBoundTermId P I T (state_fiber P I T state)) := + match lookup_atom (state_live_entries P I T state) atom, + interpret_bound_symbol_sequence (state_fiber P I T state) tail with + | Some id, Some ids => lookup_state_term_sequence state (id :: ids) + | _, _ => None + end. + +Theorem VWENC_118_ATOM_ID_AND_TERM_ID_LOOKUP_LAYERS_ARE_EXPLICIT : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state : InterningState P I T) + atom id tail interpreted_tail term_id, + lookup_atom (state_live_entries P I T state) atom = Some id -> + interpret_bound_symbol_sequence + (state_fiber P I T state) tail = Some interpreted_tail -> + state_term_dictionary_enabled P I T state = true -> + lookup_term_sequence + (state_term_entries P I T state) + (id :: interpreted_tail) = Some term_id -> + resolve_atom_then_term state atom tail = + Some + (mkFiberBoundTermId + P I T (state_fiber P I T state) + (state_term_fiber P I T state) term_id). +Proof. + intros P I T state atom id tail interpreted_tail term_id + Hatom Htail Henabled Hterm. + unfold resolve_atom_then_term. + rewrite Hatom, Htail. + unfold lookup_state_term_sequence. now rewrite Henabled, Hterm. +Qed. + +Theorem VWENC_183_TWO_LEVEL_RESOLUTION_REJECTS_A_FOREIGN_FIBER_TAIL : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state : InterningState P I T) + (actual : VocabularyFiber P I) + (atom : CanonicalAtom P) (id : SymbolId I) tail, + state_fiber P I T state <> actual -> + interpret_bound_symbol_sequence (state_fiber P I T state) + (mkFiberBoundSymbolId P I actual id :: tail) = None /\ + resolve_atom_then_term state atom + (mkFiberBoundSymbolId P I actual id :: tail) = None. +Proof. + intros P I T state actual atom id tail Hdifferent. + assert (Hreject : + interpret_bound_symbol_sequence (state_fiber P I T state) + (mkFiberBoundSymbolId P I actual id :: tail) = None). + { simpl. now rewrite + (VWENC_112_CROSS_FIBER_ID_INTERPRETATION_IS_REJECTED + P I (state_fiber P I T state) actual id Hdifferent). } + split; [exact Hreject |]. + unfold resolve_atom_then_term. rewrite Hreject. + now destruct (lookup_atom (state_live_entries P I T state) atom). +Qed. + +Theorem VWENC_119_OPTIONAL_TERM_DICTIONARY_SEQUENCES_USE_LIVE_VOCABULARY_IDS : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state : InterningState P I T) sequence term_id, + InterningStateWellFormed state -> + In (sequence, term_id) (state_term_entries P I T state) -> + sequence_vocabulary_bound + (state_live_entries P I T state) + (state_allocator_frontier P I T state) + sequence. +Proof. + intros P I T state sequence term_id Hwell Hin. + destruct Hwell as + [_ _ _ _ _ _ _ _ _ Hterm_bound _]. + apply Forall_forall with (x := (sequence, term_id)) + in Hterm_bound; [exact Hterm_bound | exact Hin]. +Qed. + +Lemma orphan_id_has_no_live_binding : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state : InterningState P I T) id, + InterningStateWellFormed state -> + In id (state_orphan_ids state) -> + ~ live_symbol (state_live_entries P I T state) id. +Proof. + intros P I T state id Hwell Horphan. + destruct Hwell as + [_ _ Hlive_history Hallocation_unique]. + intros [atom Hlive]. + assert (Hever : In id (map snd (state_ever_entries P I T state))). + { apply in_map_iff. exists (atom, id). split; [reflexivity |]. + now apply Hlive_history. } + unfold state_allocation_entries in Hallocation_unique. + rewrite !map_app in Hallocation_unique. + eapply NoDup_app_disjoint_right; + [exact Hallocation_unique | exact Hever |]. + unfold state_orphan_ids in Horphan. + rewrite map_app in Horphan. + apply in_or_app. right. + apply in_or_app. right. + exact Horphan. +Qed. + +Theorem VWENC_120_ORPHAN_IDS_HAVE_NO_LIVE_OR_SEQUENCE_BINDING : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state : InterningState P I T) id sequence, + InterningStateWellFormed state -> + In id (state_orphan_ids state) -> + In sequence (state_sequences P I T state) -> + ~ live_symbol (state_live_entries P I T state) id /\ + ~ In id sequence. +Proof. + intros P I T state id sequence Hwell Horphan Hsequence. + pose proof (orphan_id_has_no_live_binding + P I T state id Hwell Horphan) as Hnot_live. + destruct Hwell as [_ _ _ _ _ _ _ Hsequences]. + split; [exact Hnot_live |]. + intros Hin. + apply Forall_forall with (x := sequence) in Hsequences; + [| exact Hsequence]. + apply Forall_forall with (x := id) in Hsequences; [| exact Hin]. + destruct Hsequences as [_ Hlive]. contradiction. +Qed. + +Theorem VWENC_165_ORPHAN_IDS_HAVE_NO_TERM_SEQUENCE_BINDING : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state : InterningState P I T) id sequence term_id, + InterningStateWellFormed state -> + In id (state_orphan_ids state) -> + In (sequence, term_id) (state_term_entries P I T state) -> + ~ In id sequence. +Proof. + intros P I T state id sequence term_id Hwell Horphan Hterm Hin. + pose proof (orphan_id_has_no_live_binding + P I T state id Hwell Horphan) as Hnot_live. + pose proof Hwell as Hwell_for_terms. + destruct Hwell_for_terms as [_ _ _ _ _ _ _ _ _ Hterm_bound]. + apply Forall_forall with (x := (sequence, term_id)) in Hterm_bound; + [| exact Hterm]. + apply Forall_forall with (x := id) in Hterm_bound; [| exact Hin]. + destruct Hterm_bound as [_ Hlive]. contradiction. +Qed. + +Inductive AnyLocalId + (I T : FixedWidthCarrierProfile) : Type := +| AnySymbolId : SymbolId I -> AnyLocalId I T +| AnyTermId : TermId T -> AnyLocalId I T. + +Theorem VWENC_136_SYMBOL_AND_TERM_IDS_ARE_NOMINALLY_DISJOINT : + forall (I T : FixedWidthCarrierProfile) + (symbol : SymbolId I) (term : TermId T), + AnySymbolId I T symbol <> AnyTermId I T term. +Proof. discriminate. Qed. + +Theorem VWENC_137_TERM_ID_DICTIONARY_IS_A_SECOND_EXACT_BIJECTION : + forall (I T : FixedWidthCarrierProfile) + (entries : list (TermEntry I T)) sequence term_id, + term_relation_well_formed entries -> + (lookup_term_sequence entries sequence = Some term_id <-> + lookup_term_id entries term_id = Some sequence). +Proof. + intros I T entries sequence term_id [Hsequence_unique Hterm_unique]. + split; intros Hlookup. + - unfold lookup_term_sequence in Hlookup. + apply assoc_lookup_sound in Hlookup. + unfold lookup_term_id. apply assoc_lookup_complete_unique. + + rewrite reverse_term_keys_are_term_ids. exact Hterm_unique. + + apply reverse_term_membership. exact Hlookup. + - unfold lookup_term_id in Hlookup. + apply assoc_lookup_sound in Hlookup. + apply reverse_term_membership in Hlookup. + unfold lookup_term_sequence. + now apply assoc_lookup_complete_unique. +Qed. + +(** ** Query-local overlay: fiber-bound, namespaced, and non-serializable *) + +Record QueryOverlayNamespace + (P : CertifiedAtomProfile) + (I : FixedWidthCarrierProfile) + (fiber : VocabularyFiber P I) : Type := + mkQueryOverlayNamespace { + query_overlay_namespace_identity : nat + }. + +Definition query_overlay_namespace_full_identity + {P : CertifiedAtomProfile} + {I : FixedWidthCarrierProfile} + {fiber : VocabularyFiber P I} + (namespace : QueryOverlayNamespace P I fiber) := + (vocabulary_fiber_identity fiber, + query_overlay_namespace_identity P I fiber namespace). + +Definition query_overlay_namespace_eq_dec + (P : CertifiedAtomProfile) + (I : FixedWidthCarrierProfile) + (fiber : VocabularyFiber P I) + (left right : QueryOverlayNamespace P I fiber) + : {left = right} + {left <> right}. +Proof. + destruct left as [left_identity]. + destruct right as [right_identity]. + destruct (Nat.eq_dec left_identity right_identity) + as [Hequal | Hdifferent]. + - subst right_identity. left. reflexivity. + - right. intros Hequal. inversion Hequal. contradiction. +Defined. + +Record QueryLocalId + (P : CertifiedAtomProfile) + (I : FixedWidthCarrierProfile) + (fiber : VocabularyFiber P I) : Type := + mkQueryLocalId { + query_local_namespace : QueryOverlayNamespace P I fiber; + query_local_id_value : nat + }. + +Definition interpret_query_local_id + {P : CertifiedAtomProfile} + {I : FixedWidthCarrierProfile} + {fiber : VocabularyFiber P I} + (expected : QueryOverlayNamespace P I fiber) + (id : QueryLocalId P I fiber) : option nat := + if query_overlay_namespace_eq_dec + P I fiber expected (query_local_namespace P I fiber id) + then Some (query_local_id_value P I fiber id) + else None. + +Theorem VWENC_172_CROSS_OVERLAY_QUERY_LOCAL_ID_IS_REJECTED : + forall (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (fiber : VocabularyFiber P I) + (expected actual : QueryOverlayNamespace P I fiber) value, + expected <> actual -> + interpret_query_local_id expected + (mkQueryLocalId P I fiber actual value) = None. +Proof. + intros P I fiber expected actual value Hdifferent. + unfold interpret_query_local_id. simpl. + destruct (query_overlay_namespace_eq_dec + P I fiber expected actual) as [Hequal | _]. + - contradiction. + - reflexivity. +Qed. + +Definition QueryOverlayEntry + (P : CertifiedAtomProfile) + (I : FixedWidthCarrierProfile) + (fiber : VocabularyFiber P I) := + (CanonicalAtom P * QueryLocalId P I fiber)%type. + +Record QueryOverlay + (P : CertifiedAtomProfile) + (I : FixedWidthCarrierProfile) + (fiber : VocabularyFiber P I) : Type := + mkQueryOverlay { + query_overlay_namespace : QueryOverlayNamespace P I fiber; + query_overlay_entries : list (QueryOverlayEntry P I fiber); + query_overlay_next : nat + }. + +Definition PackedQueryOverlay + (P : CertifiedAtomProfile) + (I : FixedWidthCarrierProfile) : Type := + { fiber : VocabularyFiber P I & QueryOverlay P I fiber }. + +Definition transport_query_overlay + {P : CertifiedAtomProfile} + {I : FixedWidthCarrierProfile} + {actual expected : VocabularyFiber P I} + (Hequal : actual = expected) + (overlay : QueryOverlay P I actual) + : QueryOverlay P I expected := + eq_rect + actual (fun fiber => QueryOverlay P I fiber) + overlay expected Hequal. + +Definition align_query_overlay + {P : CertifiedAtomProfile} + {I : FixedWidthCarrierProfile} + (expected : VocabularyFiber P I) + (packed : PackedQueryOverlay P I) + : option (QueryOverlay P I expected). +Proof. + destruct packed as [actual overlay]. + destruct (vocabulary_fiber_eq_dec P I actual expected) + as [Hequal | Hdifferent]. + - exact (Some (transport_query_overlay Hequal overlay)). + - exact None. +Defined. + +Theorem VWENC_186_QUERY_OVERLAY_FROM_ANOTHER_VOCABULARY_FIBER_IS_REJECTED : + forall (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (expected actual : VocabularyFiber P I) + (overlay : QueryOverlay P I actual), + expected <> actual -> + align_query_overlay expected + (existT (fun fiber => QueryOverlay P I fiber) actual overlay) = None. +Proof. + intros P I expected actual overlay Hdifferent. + unfold align_query_overlay. + destruct (vocabulary_fiber_eq_dec P I actual expected) + as [Hequal | _]. + - exfalso. apply Hdifferent. symmetry. exact Hequal. + - reflexivity. +Qed. + +Definition query_overlay_id_values + {P : CertifiedAtomProfile} + {I : FixedWidthCarrierProfile} + {fiber : VocabularyFiber P I} + (overlay : QueryOverlay P I fiber) : list nat := + map + (fun entry => + query_local_id_value P I fiber (snd entry)) + (query_overlay_entries P I fiber overlay). + +Definition query_overlay_well_formed + {P : CertifiedAtomProfile} + {I : FixedWidthCarrierProfile} + {fiber : VocabularyFiber P I} + (overlay : QueryOverlay P I fiber) : Prop := + NoDup (map fst (query_overlay_entries P I fiber overlay)) /\ + NoDup (query_overlay_id_values overlay) /\ + Forall + (fun entry => + query_local_namespace P I fiber (snd entry) = + query_overlay_namespace P I fiber overlay /\ + query_local_id_value P I fiber (snd entry) < + query_overlay_next P I fiber overlay) + (query_overlay_entries P I fiber overlay). + +Definition lookup_query_local + {P : CertifiedAtomProfile} + {I : FixedWidthCarrierProfile} + {fiber : VocabularyFiber P I} + (overlay : QueryOverlay P I fiber) + (atom : CanonicalAtom P) : option (QueryLocalId P I fiber) := + assoc_lookup (canonical_atom_eq_dec P) + (query_overlay_entries P I fiber overlay) atom. + +Inductive QueryAtomResolution + (P : CertifiedAtomProfile) + (I : FixedWidthCarrierProfile) + (fiber : VocabularyFiber P I) : Type := +| QueryDurableSymbol : + FiberBoundSymbolId P I -> QueryAtomResolution P I fiber +| QueryLocalSymbol : QueryLocalId P I fiber -> QueryAtomResolution P I fiber. + +Record QueryResolutionResult + (P : CertifiedAtomProfile) + (I : FixedWidthCarrierProfile) + (fiber : VocabularyFiber P I) : Type := + mkQueryResolutionResult { + query_resolution : QueryAtomResolution P I fiber; + query_overlay_after : QueryOverlay P I fiber + }. + +Definition resolve_query_atom + {P : CertifiedAtomProfile} + {I : FixedWidthCarrierProfile} + {fiber : VocabularyFiber P I} + (snapshot : VocabularySnapshot P I fiber) + (overlay : QueryOverlay P I fiber) + (atom : CanonicalAtom P) : QueryResolutionResult P I fiber := + match lookup_atom + (vocabulary_snapshot_live_entries P I fiber snapshot) atom with + | Some id => + mkQueryResolutionResult P I fiber + (QueryDurableSymbol P I fiber + (mkFiberBoundSymbolId P I fiber id)) overlay + | None => + match lookup_query_local overlay atom with + | Some id => + mkQueryResolutionResult P I fiber + (QueryLocalSymbol P I fiber id) overlay + | None => + let id := + mkQueryLocalId P I fiber + (query_overlay_namespace P I fiber overlay) + (query_overlay_next P I fiber overlay) in + let updated := + mkQueryOverlay P I fiber + (query_overlay_namespace P I fiber overlay) + ((atom, id) :: query_overlay_entries P I fiber overlay) + (S (query_overlay_next P I fiber overlay)) in + mkQueryResolutionResult P I fiber + (QueryLocalSymbol P I fiber id) updated + end + end. + +Lemma query_overlay_fresh_insert_preserves_well_formedness : + forall (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (fiber : VocabularyFiber P I) + (overlay : QueryOverlay P I fiber) atom, + query_overlay_well_formed overlay -> + lookup_query_local overlay atom = None -> + let fresh := + mkQueryLocalId P I fiber + (query_overlay_namespace P I fiber overlay) + (query_overlay_next P I fiber overlay) in + query_overlay_well_formed + (mkQueryOverlay P I fiber + (query_overlay_namespace P I fiber overlay) + ((atom, fresh) :: query_overlay_entries P I fiber overlay) + (S (query_overlay_next P I fiber overlay))) /\ + ~ In (query_overlay_next P I fiber overlay) + (query_overlay_id_values overlay). +Proof. + intros P I fiber overlay atom + [Hatom_unique [Hid_unique Hbelow]] Hlookup. + simpl. + assert (Hatom_absent : + ~ In atom (map fst (query_overlay_entries P I fiber overlay))). + { unfold lookup_query_local in Hlookup. + now apply assoc_lookup_none_key_absent in Hlookup. } + assert (Hid_absent : + ~ In (query_overlay_next P I fiber overlay) + (query_overlay_id_values overlay)). + { intros Hin. + unfold query_overlay_id_values in Hin. + apply in_map_iff in Hin. + destruct Hin as [[existing_atom existing_id] [Hequal Hin]]. + simpl in Hequal. + apply Forall_forall with + (x := (existing_atom, existing_id)) in Hbelow; [| exact Hin]. + destruct Hbelow as [_ Hlt]. simpl in Hlt. lia. } + split. + - split. + + simpl. constructor; assumption. + + split. + * unfold query_overlay_id_values. simpl. + constructor; assumption. + * simpl. constructor. + { split; [reflexivity |]. + apply Nat.lt_succ_diag_r. } + { apply Forall_forall. intros entry Hin. + apply Forall_forall with (x := entry) in Hbelow; [| exact Hin]. + destruct Hbelow as [Hnamespace Hlt]. + now split; [exact Hnamespace | lia]. } + - exact Hid_absent. +Qed. + +Theorem VWENC_121_UNKNOWN_QUERY_ATOM_RECEIVES_STABLE_QUERY_LOCAL_ID : + forall (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (fiber : VocabularyFiber P I) + (snapshot : VocabularySnapshot P I fiber) + (overlay : QueryOverlay P I fiber) atom, + query_overlay_well_formed overlay -> + lookup_atom + (vocabulary_snapshot_live_entries P I fiber snapshot) atom = None -> + lookup_query_local overlay atom = None -> + let fresh := + mkQueryLocalId P I fiber + (query_overlay_namespace P I fiber overlay) + (query_overlay_next P I fiber overlay) in + let result := resolve_query_atom snapshot overlay atom in + query_resolution P I fiber result = + QueryLocalSymbol P I fiber fresh /\ + lookup_query_local (query_overlay_after P I fiber result) atom = + Some fresh /\ + query_overlay_well_formed + (query_overlay_after P I fiber result) /\ + interpret_query_local_id + (query_overlay_namespace P I fiber overlay) fresh = + Some (query_overlay_next P I fiber overlay) /\ + ~ In (query_overlay_next P I fiber overlay) + (query_overlay_id_values overlay). +Proof. + intros P I fiber snapshot overlay atom Hwell Hdurable Hoverlay. + unfold resolve_query_atom. rewrite Hdurable, Hoverlay. simpl. + split; [reflexivity |]. split. + - unfold lookup_query_local. simpl. + destruct (canonical_atom_eq_dec P atom atom); + [reflexivity | contradiction]. + - split. + + apply (proj1 + (query_overlay_fresh_insert_preserves_well_formedness + P I fiber overlay atom Hwell Hoverlay)). + + split. + * unfold interpret_query_local_id. simpl. + destruct (query_overlay_namespace_eq_dec P I fiber + (query_overlay_namespace P I fiber overlay) + (query_overlay_namespace P I fiber overlay)); + [reflexivity | contradiction]. + * apply (proj2 + (query_overlay_fresh_insert_preserves_well_formedness + P I fiber overlay atom Hwell Hoverlay)). +Qed. + +Theorem VWENC_122_REPEATED_QUERY_REUSES_OVERLAY_WITHOUT_DURABLE_MUTATION : + forall (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (fiber : VocabularyFiber P I) + (snapshot : VocabularySnapshot P I fiber) + (overlay : QueryOverlay P I fiber) atom first, + query_overlay_well_formed overlay -> + lookup_atom + (vocabulary_snapshot_live_entries P I fiber snapshot) atom = None -> + resolve_query_atom snapshot overlay atom = first -> + resolve_query_atom snapshot + (query_overlay_after P I fiber first) atom = + mkQueryResolutionResult P I fiber + (query_resolution P I fiber first) + (query_overlay_after P I fiber first) /\ + snapshot = snapshot /\ + query_overlay_well_formed + (query_overlay_after P I fiber first). +Proof. + intros P I fiber snapshot overlay atom first Hwell Hdurable Hfirst. + unfold resolve_query_atom in Hfirst. + rewrite Hdurable in Hfirst. + destruct (lookup_query_local overlay atom) + as [existing |] eqn:Hoverlay. + - inversion Hfirst. subst first. + split. + + change + ((match lookup_atom + (vocabulary_snapshot_live_entries P I fiber snapshot) atom with + | Some durable_id => + mkQueryResolutionResult P I fiber + (QueryDurableSymbol P I fiber + (mkFiberBoundSymbolId P I fiber durable_id)) overlay + | None => + match lookup_query_local overlay atom with + | Some local_id => + mkQueryResolutionResult P I fiber + (QueryLocalSymbol P I fiber local_id) overlay + | None => + let local_id := + mkQueryLocalId P I fiber + (query_overlay_namespace P I fiber overlay) + (query_overlay_next P I fiber overlay) in + mkQueryResolutionResult P I fiber + (QueryLocalSymbol P I fiber local_id) + (mkQueryOverlay P I fiber + (query_overlay_namespace P I fiber overlay) + ((atom, local_id) :: + query_overlay_entries P I fiber overlay) + (S (query_overlay_next P I fiber overlay))) + end + end) = + mkQueryResolutionResult P I fiber + (QueryLocalSymbol P I fiber existing) overlay). + now rewrite Hdurable, Hoverlay. + + now split. + - inversion Hfirst. subst first. simpl. + unfold resolve_query_atom. rewrite Hdurable. simpl. + unfold lookup_query_local. simpl. + destruct (canonical_atom_eq_dec P atom atom); + [| contradiction]. + split; [reflexivity |]. split; [reflexivity |]. + apply (proj1 + (query_overlay_fresh_insert_preserves_well_formedness + P I fiber overlay atom Hwell Hoverlay)). +Qed. + +Definition serialize_query_resolution + {P : CertifiedAtomProfile} + {I : FixedWidthCarrierProfile} + {fiber : VocabularyFiber P I} + (resolution : QueryAtomResolution P I fiber) + : option (FiberBoundSymbolId P I) := + match resolution with + | QueryDurableSymbol _ _ _ id => Some id + | QueryLocalSymbol _ _ _ _ => None + end. + +Theorem VWENC_184_DURABLE_QUERY_RESOLUTION_BINDS_THE_EXACT_SNAPSHOT_FIBER : + forall (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (fiber : VocabularyFiber P I) + (snapshot : VocabularySnapshot P I fiber) + (overlay : QueryOverlay P I fiber) atom id, + lookup_atom + (vocabulary_snapshot_live_entries P I fiber snapshot) atom = Some id -> + query_resolution P I fiber + (resolve_query_atom snapshot overlay atom) = + QueryDurableSymbol P I fiber + (mkFiberBoundSymbolId P I fiber id) /\ + serialize_query_resolution + (query_resolution P I fiber + (resolve_query_atom snapshot overlay atom)) = + Some (mkFiberBoundSymbolId P I fiber id). +Proof. + intros P I fiber snapshot overlay atom id Hlookup. + unfold resolve_query_atom. rewrite Hlookup. now split. +Qed. + +Theorem VWENC_185_SERIALIZED_DURABLE_QUERY_ID_RETAINS_ITS_FIBER : + forall (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (fiber actual : VocabularyFiber P I) id, + serialize_query_resolution + (QueryDurableSymbol P I fiber + (mkFiberBoundSymbolId P I actual id)) = + Some (mkFiberBoundSymbolId P I actual id) /\ + (fiber <> actual -> + interpret_symbol_id fiber + (mkFiberBoundSymbolId P I actual id) = None). +Proof. + intros P I fiber actual id. + split; [reflexivity |]. + apply VWENC_112_CROSS_FIBER_ID_INTERPRETATION_IS_REJECTED. +Qed. + +Theorem VWENC_139_QUERY_LOCAL_IDS_CANNOT_ENTER_DURABLE_ID_SEQUENCES : + forall (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (fiber : VocabularyFiber P I) local_id, + serialize_query_resolution + (QueryLocalSymbol P I fiber local_id) = None. +Proof. reflexivity. Qed. +(** ** Exact dependent sequence descriptors *) + +Theorem VWENC_123_SEQUENCE_DESCRIPTOR_REQUIRES_EXACT_VOCABULARY_FIBER : + forall (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (fiber : VocabularyFiber P I) + (snapshot : VocabularySnapshot P I fiber) descriptor, + descriptor_accepts_vocabulary_snapshot snapshot descriptor -> + descriptor_fiber P I descriptor = fiber. +Proof. intros P I fiber snapshot descriptor [Hexact _]. exact Hexact. Qed. + +Theorem VWENC_124_DESCRIPTOR_VALIDATES_EACH_LIVE_ID_NOT_DENSE_FRONTIER : + forall (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (fiber : VocabularyFiber P I) + (snapshot : VocabularySnapshot P I fiber) descriptor id, + descriptor_accepts_vocabulary_snapshot snapshot descriptor -> + In id (descriptor_ids P I descriptor) -> + symbol_id_value I id < + descriptor_required_frontier P I descriptor /\ + live_symbol + (vocabulary_snapshot_live_entries P I fiber snapshot) id. +Proof. + intros P I fiber snapshot descriptor id + [_ [_ Hids]] Hin. + now apply Forall_forall with (x := id) in Hids. +Qed. + +(** ** Immutable exact-state snapshot observations *) + +Definition observed_vocabulary_entry + {P : CertifiedAtomProfile} {I : FixedWidthCarrierProfile} + (entry : VocabularyEntry P I) := + (canonical_atom_identity (fst entry), + symbol_id_value I (snd entry)). + +Definition observed_symbol_sequence + {I : FixedWidthCarrierProfile} (sequence : list (SymbolId I)) := + map (symbol_id_value I) sequence. + +Definition observed_term_entry + {I T : FixedWidthCarrierProfile} (entry : TermEntry I T) := + (observed_symbol_sequence (fst entry), + term_id_value T (snd entry)). + +Definition observed_reverse_span + {I : FixedWidthCarrierProfile} + (entry : SymbolId I * ByteSpan) := + (symbol_id_value I (fst entry), + (span_offset (snd entry), span_length (snd entry))). + +Definition observed_allocation_classes + {P : CertifiedAtomProfile} + {I T : FixedWidthCarrierProfile} + (state : InterningState P I T) := + (map observed_vocabulary_entry + (state_reserved_entries P I T state), + (map observed_vocabulary_entry + (state_claimed_entries P I T state), + (map observed_vocabulary_entry + (state_live_entries P I T state), + (map observed_vocabulary_entry + (state_ever_entries P I T state), + (map observed_vocabulary_entry + (state_orphan_entries P I T state), + map observed_vocabulary_entry + (state_unmaterialized_orphan_entries P I T state)))))). + +Definition observed_packed_storage + {P : CertifiedAtomProfile} + {I T : FixedWidthCarrierProfile} + (state : InterningState P I T) := + (packed_canonical_bytes I (state_packed_storage P I T state), + map observed_reverse_span + (packed_reverse_spans I (state_packed_storage P I T state))). + +Definition observed_dependent_state + {P : CertifiedAtomProfile} + {I T : FixedWidthCarrierProfile} + (state : InterningState P I T) := + (state_allocator_frontier P I T state, + (map observed_symbol_sequence (state_sequences P I T state), + (state_term_dictionary_enabled P I T state, + map observed_term_entry (state_term_entries P I T state)))). + +Definition interning_state_observation + {P : CertifiedAtomProfile} + {I T : FixedWidthCarrierProfile} + (state : InterningState P I T) := + (vocabulary_fiber_identity (state_fiber P I T state), + (term_dictionary_fiber_identity (state_term_fiber P I T state), + (observed_allocation_classes state, + (observed_packed_storage state, + observed_dependent_state state)))). + +Record InternedDictionarySnapshot + (P : CertifiedAtomProfile) + (I T : FixedWidthCarrierProfile) : Type := + mkInternedDictionarySnapshot { + snapshot_exact_state : InterningState P I T + }. + +Definition capture_snapshot + {P : CertifiedAtomProfile} + {I T : FixedWidthCarrierProfile} + (state : InterningState P I T) + : InternedDictionarySnapshot P I T := + mkInternedDictionarySnapshot P I T state. + +Definition snapshot_observation + {P : CertifiedAtomProfile} + {I T : FixedWidthCarrierProfile} + (snapshot : InternedDictionarySnapshot P I T) := + interning_state_observation + (snapshot_exact_state P I T snapshot). + +Record SnapshotSession + (P : CertifiedAtomProfile) + (I T : FixedWidthCarrierProfile) : Type := + mkSnapshotSession { + session_captured : InternedDictionarySnapshot P I T; + session_current : InterningState P I T + }. + +Definition begin_snapshot_session + {P : CertifiedAtomProfile} + {I T : FixedWidthCarrierProfile} + (state : InterningState P I T) : SnapshotSession P I T := + mkSnapshotSession P I T (capture_snapshot state) state. + +Inductive SnapshotSessionTransition + {P : CertifiedAtomProfile} + {I T : FixedWidthCarrierProfile} + : SnapshotSession P I T -> SnapshotSession P I T -> Prop := +| AdvanceSnapshotSession : + forall captured current later, + InterningTransition current later -> + SnapshotSessionTransition + (mkSnapshotSession P I T captured current) + (mkSnapshotSession P I T captured later). + +Theorem VWENC_125_CAPTURED_SNAPSHOT_OBSERVATIONS_SURVIVE_LATER_PUBLICATION : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (before later : SnapshotSession P I T), + SnapshotSessionTransition before later -> + session_captured P I T later = session_captured P I T before /\ + snapshot_observation (session_captured P I T later) = + snapshot_observation (session_captured P I T before). +Proof. + intros P I T before later Htransition. + inversion Htransition. now split. +Qed. + +Theorem VWENC_173_CAPTURED_SNAPSHOT_IS_THE_EXACT_INITIAL_STATE : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (state : InterningState P I T), + snapshot_exact_state P I T + (session_captured P I T (begin_snapshot_session state)) = state /\ + snapshot_observation + (session_captured P I T (begin_snapshot_session state)) = + interning_state_observation state. +Proof. intros. now split. Qed. + +Inductive SnapshotSessionReachable + {P : CertifiedAtomProfile} + {I T : FixedWidthCarrierProfile} + (initial : SnapshotSession P I T) + : SnapshotSession P I T -> Prop := +| SnapshotSessionReachableInitial : + SnapshotSessionReachable initial initial +| SnapshotSessionReachableStep : + forall current later, + SnapshotSessionReachable initial current -> + SnapshotSessionTransition current later -> + SnapshotSessionReachable initial later. + +Theorem VWENC_174_EXACT_CAPTURE_SURVIVES_ARBITRARY_LATER_TRANSITIONS : + forall (P : CertifiedAtomProfile) (I T : FixedWidthCarrierProfile) + (initial later : SnapshotSession P I T), + SnapshotSessionReachable initial later -> + session_captured P I T later = session_captured P I T initial /\ + snapshot_observation (session_captured P I T later) = + snapshot_observation (session_captured P I T initial). +Proof. + intros P I T initial later Hreachable. + induction Hreachable as + [| current later Hcurrent IH Htransition]. + - now split. + - destruct (VWENC_125_CAPTURED_SNAPSHOT_OBSERVATIONS_SURVIVE_LATER_PUBLICATION + P I T current later Htransition) as [Hcaptured Hobservation]. + split. + + now rewrite Hcaptured. + + now rewrite Hobservation. +Qed. +(** ** Exact, machine-readable model-to-Rust correspondence *) + +Inductive CorrespondenceRelationship : Type := +| Refines +| CommonSubstrateOnly +| Conflicts +| Prospective. + +Inductive InterningFormalPoint : Type := +| PointCertifiedAtomProfile +| PointSymbolIdCarrierCodec +| PointTermIdCarrierCodec +| PointForwardAtomToId +| PointReverseIdToAtom +| PointForwardReverseBijection +| PointAllocationStatus +| PointClaimAllocation +| PointOrphanAllocation +| PointTombstoneNoReuse +| PointPackedCanonicalStorage +| PointSparseAllocatorFrontierStorage +| PointSparseAllocatorFrontierAccess +| PointVocabularyFiberHeader +| PointIdSequenceView +| PointSequenceDescriptorLiveMembership +| PointOptionalTermDictionary +| PointCoordinatedSequenceOwner +| PointQueryLocalOverlay +| PointImmutableSnapshot +| PointBeginGeneration +| PointDurabilizeLiveId +| PointSealDurableVocabulary +| PointPublishVocabularyEligibility +| PointStageDependentSequence +| PointPublishSequenceVisibility +| PointDurabilizeDependentSequence +| PointWriteVocabularyObject +| PointSyncVocabularyObject +| PointWriteSequenceObject +| PointSyncSequenceObject +| PointAtomicCheckpointHeadPublication +| PointCaptureReader +| PointSaveReaderContinuation +| PointResumeReaderContinuation +| PointLoseVocabularyArtifact +| PointLoseSequenceArtifact +| PointCorruptVocabularyArtifact +| PointCorruptSequenceArtifact +| PointCrashTransition +| PointStrictPairRecovery +| PointCommitSequenceSubstrate +| PointCommittedWatermarkSubstrate +| PointDurableOverlayInsertionSubstrate +| PointCheckpointLockSubstrate +| PointHeaderCheckpointPublicationSubstrate. + +Inductive ImplementationObligation : Type := +| ObligationAddCertifiedProfileSurface +| ObligationAddSymbolIdCarrierCodec +| ObligationAddTermIdCarrierCodec +| ObligationGeneralizeForwardVocabulary +| ObligationGeneralizeReverseVocabulary +| ObligationCoordinateBijectionVisibility +| ObligationAddAllocationLedger +| ObligationReuseSparseAllocationClaim +| ObligationRetainOrphanedIds +| ObligationAddNoReuseTombstones +| ObligationAddPackedStorage +| ObligationPreserveSparseFrontierStorage +| ObligationExposeSparseFrontier +| ObligationAddProfileGenerationHeader +| ObligationAddBorrowedFiberBoundView +| ObligationAddSequenceDescriptorValidation +| ObligationAddOptionalTermDictionary +| ObligationAddCoordinatedOwner +| ObligationAddEphemeralOverlay +| ObligationAuditSnapshotRefinement +| ObligationAddGenerationStaging +| ObligationAddDurablePackedPublication +| ObligationAddVocabularySeal +| ObligationAddVocabularyEligibilityPublication +| ObligationAddSequenceStaging +| ObligationAddSequenceVisibilityPublication +| ObligationAddSequenceDurabilityPublication +| ObligationReuseVocabularyObjectWrite +| ObligationReuseVocabularyObjectSync +| ObligationReuseSequenceObjectWrite +| ObligationReuseSequenceObjectSync +| ObligationAddAtomicCheckpointHead +| ObligationAddReaderCapture +| ObligationAddContinuationCapture +| ObligationAddContinuationResume +| ObligationAddVocabularyLossRecoveryCase +| ObligationAddSequenceLossRecoveryCase +| ObligationAddVocabularyCorruptionRecoveryCase +| ObligationAddSequenceCorruptionRecoveryCase +| ObligationAddCrashStateTransition +| ObligationAddExactOldNewRecovery +| ObligationReuseCommitSequence +| ObligationReuseCommittedWatermark +| ObligationReuseDurableOverlayInsertion +| ObligationReuseCheckpointLock +| ObligationExtendHeaderCheckpointPublication. + +Record CorrespondenceRow : Type := mkCorrespondenceRow { + correspondence_formal_point : InterningFormalPoint; + correspondence_source_path : string; + correspondence_rust_symbol : string; + correspondence_relationship : CorrespondenceRelationship; + correspondence_obligation : ImplementationObligation +}. + +Definition declared_correspondence_row + (point : InterningFormalPoint) : CorrespondenceRow := + match point with + | PointCertifiedAtomProfile => + mkCorrespondenceRow point + ("src/profile/mod.rs")%string + ("DictionaryProfile")%string + Prospective ObligationAddCertifiedProfileSurface + | PointSymbolIdCarrierCodec => + mkCorrespondenceRow point + ("src/profile/interned/id.rs")%string + ("SymbolId::{try_from_nat,encode,decode}")%string + Prospective ObligationAddSymbolIdCarrierCodec + | PointTermIdCarrierCodec => + mkCorrespondenceRow point + ("src/profile/interned/id.rs")%string + ("TermId::{try_from_nat,encode,decode}")%string + Prospective ObligationAddTermIdCarrierCodec + | PointForwardAtomToId => + mkCorrespondenceRow point + ("src/persistent_artrie/vocab/mutation_api.rs")%string + ("PersistentVocabARTrie::insert")%string + CommonSubstrateOnly ObligationGeneralizeForwardVocabulary + | PointReverseIdToAtom => + mkCorrespondenceRow point + ("src/persistent_artrie/vocab/query_api.rs")%string + ("PersistentVocabARTrie::get_term")%string + CommonSubstrateOnly ObligationGeneralizeReverseVocabulary + | PointForwardReverseBijection => + mkCorrespondenceRow point + ("src/persistent_artrie/vocab/dict_impl.rs")%string + ("PersistentVocabARTrie::reverse_term_map")%string + Conflicts ObligationCoordinateBijectionVisibility + | PointAllocationStatus => + mkCorrespondenceRow point + ("src/profile/interned/allocation.rs")%string + ("AllocationStatus")%string + Prospective ObligationAddAllocationLedger + | PointClaimAllocation => + mkCorrespondenceRow point + ("src/persistent_artrie/vocab/mutation_api.rs")%string + ("PersistentVocabARTrie::insert_overlay")%string + CommonSubstrateOnly ObligationReuseSparseAllocationClaim + | PointOrphanAllocation => + mkCorrespondenceRow point + ("src/persistent_artrie/vocab/mutation_api.rs")%string + ("PersistentVocabARTrie::insert_overlay")%string + CommonSubstrateOnly ObligationRetainOrphanedIds + | PointTombstoneNoReuse => + mkCorrespondenceRow point + ("src/profile/interned/allocation.rs")%string + ("AllocationLedger::tombstone")%string + Prospective ObligationAddNoReuseTombstones + | PointPackedCanonicalStorage => + mkCorrespondenceRow point + ("src/profile/interned/storage.rs")%string + ("PackedAtomStorage")%string + Prospective ObligationAddPackedStorage + | PointSparseAllocatorFrontierStorage => + mkCorrespondenceRow point + ("src/persistent_artrie/vocab/dict_impl.rs")%string + ("PersistentVocabARTrie::next_index")%string + CommonSubstrateOnly ObligationPreserveSparseFrontierStorage + | PointSparseAllocatorFrontierAccess => + mkCorrespondenceRow point + ("src/persistent_artrie/vocab/query_api.rs")%string + ("PersistentVocabARTrie::next_index")%string + CommonSubstrateOnly ObligationExposeSparseFrontier + | PointVocabularyFiberHeader => + mkCorrespondenceRow point + ("src/persistent_artrie/vocab/types.rs")%string + ("VocabTrieFileHeader")%string + Conflicts ObligationAddProfileGenerationHeader + | PointIdSequenceView => + mkCorrespondenceRow point + ("src/profile/interned/view.rs")%string + ("IdSequenceView")%string + Prospective ObligationAddBorrowedFiberBoundView + | PointSequenceDescriptorLiveMembership => + mkCorrespondenceRow point + ("src/profile/interned/descriptor.rs")%string + ("SequenceDescriptor::validate_live_ids")%string + Prospective ObligationAddSequenceDescriptorValidation + | PointOptionalTermDictionary => + mkCorrespondenceRow point + ("src/profile/interned/term_dictionary.rs")%string + ("TermSequenceDictionary")%string + Prospective ObligationAddOptionalTermDictionary + | PointCoordinatedSequenceOwner => + mkCorrespondenceRow point + ("src/profile/interned/coordinator.rs")%string + ("InternedSequenceDictionary")%string + Prospective ObligationAddCoordinatedOwner + | PointQueryLocalOverlay => + mkCorrespondenceRow point + ("src/profile/interned/query.rs")%string + ("QueryOverlay")%string + Prospective ObligationAddEphemeralOverlay + | PointImmutableSnapshot => + mkCorrespondenceRow point + ("src/persistent_artrie/vocab/mutation_api.rs")%string + ("PersistentVocabARTrie::snapshot")%string + CommonSubstrateOnly ObligationAuditSnapshotRefinement + | PointBeginGeneration => + mkCorrespondenceRow point + ("src/profile/interned/coordinator.rs")%string + ("InternedSequenceDictionary::begin_generation")%string + Prospective ObligationAddGenerationStaging + | PointDurabilizeLiveId => + mkCorrespondenceRow point + ("src/profile/interned/persistence.rs")%string + ("InternedVocabularyWriter::durabilize_live_id")%string + Prospective ObligationAddDurablePackedPublication + | PointSealDurableVocabulary => + mkCorrespondenceRow point + ("src/profile/interned/persistence.rs")%string + ("InternedVocabularyWriter::seal_vocabulary")%string + Prospective ObligationAddVocabularySeal + | PointPublishVocabularyEligibility => + mkCorrespondenceRow point + ("src/profile/interned/persistence.rs")%string + ("InternedVocabularyWriter::publish_frontier")%string + Prospective ObligationAddVocabularyEligibilityPublication + | PointStageDependentSequence => + mkCorrespondenceRow point + ("src/profile/interned/coordinator.rs")%string + ("InternedSequenceDictionary::stage_sequence")%string + Prospective ObligationAddSequenceStaging + | PointPublishSequenceVisibility => + mkCorrespondenceRow point + ("src/profile/interned/coordinator.rs")%string + ("InternedSequenceDictionary::publish_sequence")%string + Prospective ObligationAddSequenceVisibilityPublication + | PointDurabilizeDependentSequence => + mkCorrespondenceRow point + ("src/profile/interned/persistence.rs")%string + ("InternedSequenceWriter::durabilize_sequence")%string + Prospective ObligationAddSequenceDurabilityPublication + | PointWriteVocabularyObject => + mkCorrespondenceRow point + ("src/persistent_artrie/vocab/persistence_api.rs")%string + ("PersistentVocabARTrie::checkpoint_overlay")%string + CommonSubstrateOnly ObligationReuseVocabularyObjectWrite + | PointSyncVocabularyObject => + mkCorrespondenceRow point + ("src/persistent_artrie/vocab/persistence_api.rs")%string + ("PersistentVocabARTrie::checkpoint_overlay")%string + CommonSubstrateOnly ObligationReuseVocabularyObjectSync + | PointWriteSequenceObject => + mkCorrespondenceRow point + ("src/persistent_artrie/u64.rs")%string + ("write_snapshot_file")%string + CommonSubstrateOnly ObligationReuseSequenceObjectWrite + | PointSyncSequenceObject => + mkCorrespondenceRow point + ("src/persistent_artrie/u64.rs")%string + ("write_snapshot_file")%string + CommonSubstrateOnly ObligationReuseSequenceObjectSync + | PointAtomicCheckpointHeadPublication => + mkCorrespondenceRow point + ("src/profile/interned/persistence.rs")%string + ("InternedSequenceDictionary::publish_checkpoint_head")%string + Prospective ObligationAddAtomicCheckpointHead + | PointCaptureReader => + mkCorrespondenceRow point + ("src/profile/interned/snapshot.rs")%string + ("InternedSequenceDictionary::snapshot")%string + Prospective ObligationAddReaderCapture + | PointSaveReaderContinuation => + mkCorrespondenceRow point + ("src/profile/interned/continuation.rs")%string + ("InternedReadCursor::continuation")%string + Prospective ObligationAddContinuationCapture + | PointResumeReaderContinuation => + mkCorrespondenceRow point + ("src/profile/interned/continuation.rs")%string + ("InternedSequenceDictionary::resume")%string + Prospective ObligationAddContinuationResume + | PointLoseVocabularyArtifact => + mkCorrespondenceRow point + ("src/profile/interned/recovery.rs")%string + ("InternedRecoveryError::MissingVocabulary")%string + Prospective ObligationAddVocabularyLossRecoveryCase + | PointLoseSequenceArtifact => + mkCorrespondenceRow point + ("src/profile/interned/recovery.rs")%string + ("InternedRecoveryError::MissingSequence")%string + Prospective ObligationAddSequenceLossRecoveryCase + | PointCorruptVocabularyArtifact => + mkCorrespondenceRow point + ("src/profile/interned/recovery.rs")%string + ("InternedRecoveryError::CorruptVocabulary")%string + Prospective ObligationAddVocabularyCorruptionRecoveryCase + | PointCorruptSequenceArtifact => + mkCorrespondenceRow point + ("src/profile/interned/recovery.rs")%string + ("InternedRecoveryError::CorruptSequence")%string + Prospective ObligationAddSequenceCorruptionRecoveryCase + | PointCrashTransition => + mkCorrespondenceRow point + ("src/profile/interned/recovery.rs")%string + ("InternedRecoveryState")%string + Prospective ObligationAddCrashStateTransition + | PointStrictPairRecovery => + mkCorrespondenceRow point + ("src/profile/interned/recovery.rs")%string + ("InternedSequenceDictionary::recover")%string + Prospective ObligationAddExactOldNewRecovery + | PointCommitSequenceSubstrate => + mkCorrespondenceRow point + ("src/persistent_artrie/vocab/dict_impl.rs")%string + ("PersistentVocabARTrie::commit_seq")%string + CommonSubstrateOnly ObligationReuseCommitSequence + | PointCommittedWatermarkSubstrate => + mkCorrespondenceRow point + ("src/persistent_artrie/vocab/dict_impl.rs")%string + ("PersistentVocabARTrie::committed_watermark")%string + CommonSubstrateOnly ObligationReuseCommittedWatermark + | PointDurableOverlayInsertionSubstrate => + mkCorrespondenceRow point + ("src/persistent_artrie/core/overlay/durable_write.rs")%string + ("DurableOverlayWrite::insert_cas_with_value_durable_default")%string + CommonSubstrateOnly ObligationReuseDurableOverlayInsertion + | PointCheckpointLockSubstrate => + mkCorrespondenceRow point + ("src/persistent_artrie/vocab/dict_impl.rs")%string + ("PersistentVocabARTrie::checkpoint_lock")%string + CommonSubstrateOnly ObligationReuseCheckpointLock + | PointHeaderCheckpointPublicationSubstrate => + mkCorrespondenceRow point + ("src/persistent_artrie/vocab/persistence_api.rs")%string + ("PersistentVocabARTrie::checkpoint_overlay")%string + CommonSubstrateOnly ObligationExtendHeaderCheckpointPublication + end. + +Definition complete_interning_formal_points : list InterningFormalPoint := + [PointCertifiedAtomProfile; + PointSymbolIdCarrierCodec; + PointTermIdCarrierCodec; + PointForwardAtomToId; + PointReverseIdToAtom; + PointForwardReverseBijection; + PointAllocationStatus; + PointClaimAllocation; + PointOrphanAllocation; + PointTombstoneNoReuse; + PointPackedCanonicalStorage; + PointSparseAllocatorFrontierStorage; + PointSparseAllocatorFrontierAccess; + PointVocabularyFiberHeader; + PointIdSequenceView; + PointSequenceDescriptorLiveMembership; + PointOptionalTermDictionary; + PointCoordinatedSequenceOwner; + PointQueryLocalOverlay; + PointImmutableSnapshot; + PointBeginGeneration; + PointDurabilizeLiveId; + PointSealDurableVocabulary; + PointPublishVocabularyEligibility; + PointStageDependentSequence; + PointPublishSequenceVisibility; + PointDurabilizeDependentSequence; + PointWriteVocabularyObject; + PointSyncVocabularyObject; + PointWriteSequenceObject; + PointSyncSequenceObject; + PointAtomicCheckpointHeadPublication; + PointCaptureReader; + PointSaveReaderContinuation; + PointResumeReaderContinuation; + PointLoseVocabularyArtifact; + PointLoseSequenceArtifact; + PointCorruptVocabularyArtifact; + PointCorruptSequenceArtifact; + PointCrashTransition; + PointStrictPairRecovery; + PointCommitSequenceSubstrate; + PointCommittedWatermarkSubstrate; + PointDurableOverlayInsertionSubstrate; + PointCheckpointLockSubstrate; + PointHeaderCheckpointPublicationSubstrate]. + +Definition interning_correspondence : list CorrespondenceRow := + map declared_correspondence_row complete_interning_formal_points. + +Lemma declared_correspondence_row_has_requested_point : + forall point, + correspondence_formal_point (declared_correspondence_row point) = point. +Proof. + intros point. destruct point; reflexivity. +Qed. + +Theorem VWENC_126_CORRESPONDENCE_SCHEMA_IS_TOTAL_NODUP_AND_EXPLICIT : + map correspondence_formal_point interning_correspondence = + complete_interning_formal_points /\ + NoDup complete_interning_formal_points /\ + (forall point, In point complete_interning_formal_points) /\ + (forall row, + In row interning_correspondence -> + row = declared_correspondence_row + (correspondence_formal_point row)). +Proof. + split. + - reflexivity. + - split. + + repeat + (apply NoDup_cons; [simpl; intuition discriminate |]). + constructor. + + split. + * intros point. destruct point; simpl; tauto. + * intros row Hin. + unfold interning_correspondence in Hin. + apply in_map_iff in Hin. + destruct Hin as [point [Hrow Hin]]. + subst row. + now rewrite declared_correspondence_row_has_requested_point. +Qed. + +(** ** Representation-independent equality and native-ID observations *) + +Definition canonical_atom_equalb + {P : CertifiedAtomProfile} + (left right : CanonicalAtom P) : bool := + if canonical_atom_eq_dec P left right then true else false. + +Theorem VWENC_127_CANONICAL_ATOM_EQUALITY_IS_EXACT : + forall (P : CertifiedAtomProfile) (left right : CanonicalAtom P), + canonical_atom_equalb left right = true <-> left = right. +Proof. + intros P left right. unfold canonical_atom_equalb. + destruct (canonical_atom_eq_dec P left right) + as [Hequal | Hdifferent]. + - split; [intros _; exact Hequal | intros _; reflexivity]. + - split; [discriminate | contradiction]. +Qed. + +Theorem VWENC_128_EVERY_CANONICAL_ATOM_CODEWORD_IS_NONEMPTY : + forall (P : CertifiedAtomProfile) (atom : CanonicalAtom P), + canonical_atom_bytes P atom <> []. +Proof. + intros P atom. + apply (atom_codeword_nonempty P). + exact (canonical_atom_valid P atom). +Qed. + +Theorem VWENC_129_FINGERPRINTS_ARE_CANDIDATES_NOT_ATOM_IDENTITY : + (forall (P : CertifiedAtomProfile) (left right : CanonicalAtom P), + left = right -> + fingerprint_candidate left = fingerprint_candidate right) /\ + exists left right : CanonicalAtom canonical_uleb_profile, + fingerprint_candidate left = fingerprint_candidate right /\ + left <> right. +Proof. + split. + - intros P left right Hequal. now subst right. + - exists collision_atom_left, collision_atom_right. + split; [reflexivity |]. + intros Hequal. + apply (f_equal + (canonical_atom_bytes canonical_uleb_profile)) in Hequal. + discriminate. +Qed. + +Definition native_id_view_observation + {P : CertifiedAtomProfile} {I : FixedWidthCarrierProfile} + (view : IdSequenceView P I) (index : nat) := + match id_sequence_view_index view index with + | Some bound_id => + Some + (backing_identity P I (view_backing P I view), bound_id) + | None => None + end. + +Theorem VWENC_138_NATIVE_ID_VIEW_PRESERVES_BACKING_AND_FIBER_BINDING : + forall (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (view : IdSequenceView P I) index bound_id, + id_sequence_view_index view index = Some bound_id -> + native_id_view_observation view index = + Some + (backing_identity P I (view_backing P I view), bound_id). +Proof. + intros P I view index bound_id Hindex. + unfold native_id_view_observation. now rewrite Hindex. +Qed. + +Theorem VWENC_140_NATIVE_ID_OBSERVATION_ROUNDTRIPS_WITHOUT_ATOM_DECODING : + forall (P : CertifiedAtomProfile) (I : FixedWidthCarrierProfile) + (view : IdSequenceView P I) index bound_id, + valid_id_sequence_view view -> + id_sequence_view_index view index = Some bound_id -> + native_id_view_observation view index = + Some + (backing_identity P I (view_backing P I view), bound_id) /\ + exists bytes id, + bound_id = + mkFiberBoundSymbolId P I + (backing_fiber P I (view_backing P I view)) id /\ + nth_error + (descriptor_ids P I + (backing_descriptor P I (view_backing P I view))) + (view_start P I view + index) = Some id /\ + index < view_count P I view /\ + id_sequence_view_bytes view index = Some bytes /\ + bytes = id_sequence_view_byte_window view index /\ + bytes = encode_symbol_id I id /\ + live_symbol + (vocabulary_snapshot_live_entries + P I + (backing_fiber P I (view_backing P I view)) + (backing_snapshot P I (view_backing P I view))) id /\ + id_sequence_view_byte_offset view index = + (view_start P I view + index) * carrier_width_bytes I /\ + List.length bytes = carrier_width_bytes I /\ + Forall valid_byte bytes /\ + decode_symbol_id I bytes = Some id /\ + List.length (encode_symbol_id I id) = carrier_width_bytes I /\ + decode_symbol_id I (encode_symbol_id I id) = Some id. +Proof. + intros P I view index bound_id Hvalid Hindex. + split. + - now apply VWENC_138_NATIVE_ID_VIEW_PRESERVES_BACKING_AND_FIBER_BINDING. + - destruct (VWENC_135_ID_VIEW_ELEMENTS_HAVE_EXACT_CARRIER_STRIDE + P I view index bound_id Hvalid Hindex) + as [Hwithin Hexists]. + destruct Hexists as [bytes [id Hproperties]]. + destruct Hproperties as + [Hbound [Hnth [Hwindow [Hexact [Hencoded [Hlive + [Hoffset [Hlength [Hbytes [Hdecode + [Hencoded_length Hroundtrip]]]]]]]]]]]. + exists bytes, id. + split; [exact Hbound |]. + split; [exact Hnth |]. + split; [exact Hwithin |]. + split; [exact Hwindow |]. + split; [exact Hexact |]. + split; [exact Hencoded |]. + split; [exact Hlive |]. + split; [exact Hoffset |]. + split; [exact Hlength |]. + split; [exact Hbytes |]. + split; [exact Hdecode |]. + now split. +Qed. + +End VariableWidthInterning. diff --git a/formal-verification/tla+/VariableWidthCodecBoundary.cfg b/formal-verification/tla+/VariableWidthCodecBoundary.cfg new file mode 100644 index 00000000..c0830847 --- /dev/null +++ b/formal-verification/tla+/VariableWidthCodecBoundary.cfg @@ -0,0 +1,24 @@ +CONSTANTS + AcceptOverlongUleb = FALSE + AcceptUnterminatedUleb = FALSE + AcceptUtf8Continuation = FALSE + ExposePhysicalCodecBytes = FALSE + +SPECIFICATION Spec + +INVARIANT + TypeOK + VWENC_22_NO_LOGICAL_TRANSITION_BEFORE_COMPLETE_CODEWORD + VWENC_23_SUCCESS_EMITS_EXACT_LOGICAL_STREAM + VWENC_24_CODEC_BYTES_NEVER_BECOME_LOGICAL_TRANSITIONS + VWENC_25_DIRECT_BYTE_SEMANTICS_IS_EXPLICIT + VWENC_26_OVERLONG_ULEB_IS_REJECTED + VWENC_27_UNTERMINATED_ULEB_IS_REJECTED + VWENC_28_UTF8_CONTINUATION_IS_REJECTED + VWENC_29_REJECTION_IS_EXPLICIT_AND_HAS_NO_LOGICAL_OUTPUT + VWENC_32_CURSOR_AND_BUFFER_ARE_BOUNDED_BY_CONSUMED_INPUT + VWENC_80_ADJACENT_CODEWORDS_PRESERVE_EVERY_BOUNDARY + VWENC_81_INCOMPLETE_BUFFER_NEVER_INCREMENTS_COMPLETED_ATOMS + +PROPERTY + VWENC_82_DECODER_EVENTUALLY_TERMINATES diff --git a/formal-verification/tla+/VariableWidthCodecBoundary.tla b/formal-verification/tla+/VariableWidthCodecBoundary.tla new file mode 100644 index 00000000..1e91fc06 --- /dev/null +++ b/formal-verification/tla+/VariableWidthCodecBoundary.tla @@ -0,0 +1,331 @@ +---------------------- MODULE VariableWidthCodecBoundary ---------------------- +EXTENDS Naturals, Sequences, FiniteSets, TLC + +(* + Incremental logical-unit decoder for variable-width dictionary profiles. + Every ReadPhysicalByte action consumes one actual input byte, updates an + explicit codeword buffer, validates the real ULEB/UTF-8 grammar, and emits + only when that codeword is complete. Adjacent valid atoms exercise buffer + reset and boundary preservation. TLC's finite scenarios bound model checking + only; they are not workload limits in the library contract. + + The unsafe constants weaken the actual decoder rule they name: + + AcceptOverlongUleb removes ULEB minimality at termination; + AcceptUnterminatedUleb emits a buffered ULEB at end of input; + AcceptUtf8Continuation treats a continuation byte as a width-one scalar; + ExposePhysicalCodecBytes appends each consumed codec byte to the public + logical output instead of the decoded atom. + + Pure ULEB ordering and F64 raw-bit identity are proved, with their mutants, + in Rocq. They are deliberately absent from this temporal boundary model. +*) + +CONSTANTS AcceptOverlongUleb, + AcceptUnterminatedUleb, + AcceptUtf8Continuation, + ExposePhysicalCodecBytes + +ASSUME /\ AcceptOverlongUleb \in BOOLEAN + /\ AcceptUnterminatedUleb \in BOOLEAN + /\ AcceptUtf8Continuation \in BOOLEAN + /\ ExposePhysicalCodecBytes \in BOOLEAN + +Scenarios == { + "UlebAdjacent", + "UlebOverlong", + "UlebUnterminated", + "Utf8Adjacent", + "Utf8Continuation", + "DirectByte" +} + +InvalidScenarios == { + "UlebOverlong", "UlebUnterminated", "Utf8Continuation" +} + +Profile(scenario) == + CASE scenario \in {"UlebAdjacent", "UlebOverlong", "UlebUnterminated"} + -> "ULEB" + [] scenario \in {"Utf8Adjacent", "Utf8Continuation"} -> "UTF8" + [] OTHER -> "DirectByte" + +InputBytes(scenario) == + CASE scenario = "UlebAdjacent" -> <<129, 1, 2>> + [] scenario = "UlebOverlong" -> <<129, 0>> + [] scenario = "UlebUnterminated" -> <<129>> + [] scenario = "Utf8Adjacent" -> <<195, 169, 65>> + [] scenario = "Utf8Continuation" -> <<169>> + [] OTHER -> <<169, 1>> + +ExpectedOutput(scenario) == + CASE scenario = "UlebAdjacent" -> + <<[kind |-> "ULEB", bytes |-> <<129, 1>>, scalar |-> 0], + [kind |-> "ULEB", bytes |-> <<2>>, scalar |-> 0]>> + [] scenario = "Utf8Adjacent" -> + <<[kind |-> "UnicodeScalar", bytes |-> <<>>, scalar |-> 233], + [kind |-> "UnicodeScalar", bytes |-> <<>>, scalar |-> 65]>> + [] scenario = "DirectByte" -> + <<[kind |-> "Byte", bytes |-> <<>>, scalar |-> 169], + [kind |-> "Byte", bytes |-> <<>>, scalar |-> 1]>> + [] OTHER -> <<>> + +IsVariableWidth(scenario) == Profile(scenario) \in {"ULEB", "UTF8"} +IsContinuation(byte) == 128 <= byte /\ byte < 192 +LastElement(sequence) == sequence[Len(sequence)] + +UlebPrefixShape(bytes) == + /\ Len(bytes) > 0 + /\ \A index \in 1..(Len(bytes) - 1): + 128 <= bytes[index] /\ bytes[index] < 256 + +UlebTerminated(bytes) == LastElement(bytes) < 128 +UlebOverlong(bytes) == Len(bytes) > 1 /\ (LastElement(bytes) % 128) = 0 + +UlebStatus(bytes) == + IF ~UlebPrefixShape(bytes) \/ LastElement(bytes) >= 256 + THEN "RejectInvalidUleb" + ELSE IF ~UlebTerminated(bytes) + THEN "NeedMore" + ELSE IF UlebOverlong(bytes) /\ ~AcceptOverlongUleb + THEN "RejectOverlongUleb" + ELSE "Emit" + +Utf8ExpectedWidth(first) == + IF first < 128 THEN 1 + ELSE IF 194 <= first /\ first < 224 THEN 2 + ELSE IF 224 <= first /\ first < 240 THEN 3 + ELSE IF 240 <= first /\ first < 245 THEN 4 + ELSE IF AcceptUtf8Continuation /\ IsContinuation(first) THEN 1 + ELSE 0 + +Utf8ContinuationPrefix(bytes) == + \A index \in 2..Len(bytes): IsContinuation(bytes[index]) + +Utf8CanonicalComplete(bytes) == + LET width == Len(bytes) + first == bytes[1] + IN CASE width = 1 -> + first < 128 \/ (AcceptUtf8Continuation /\ IsContinuation(first)) + [] width = 2 -> 194 <= first /\ first < 224 + [] width = 3 -> + /\ 224 <= first /\ first < 240 + /\ IF first = 224 THEN bytes[2] >= 160 ELSE TRUE + /\ IF first = 237 THEN bytes[2] < 160 ELSE TRUE + [] width = 4 -> + /\ 240 <= first /\ first < 245 + /\ IF first = 240 THEN bytes[2] >= 144 ELSE TRUE + /\ IF first = 244 THEN bytes[2] < 144 ELSE TRUE + [] OTHER -> FALSE + +Utf8Value(bytes) == + CASE Len(bytes) = 1 -> bytes[1] + [] Len(bytes) = 2 -> (bytes[1] % 32) * 64 + (bytes[2] % 64) + [] Len(bytes) = 3 -> + (bytes[1] % 16) * 4096 + + (bytes[2] % 64) * 64 + + (bytes[3] % 64) + [] Len(bytes) = 4 -> + (bytes[1] % 8) * 262144 + + (bytes[2] % 64) * 4096 + + (bytes[3] % 64) * 64 + + (bytes[4] % 64) + [] OTHER -> 0 + +Utf8Status(bytes) == + LET width == Utf8ExpectedWidth(bytes[1]) + IN IF width = 0 \/ Len(bytes) > width \/ ~Utf8ContinuationPrefix(bytes) + THEN "RejectInvalidUtf8" + ELSE IF Len(bytes) < width + THEN "NeedMore" + ELSE IF Utf8CanonicalComplete(bytes) + THEN "Emit" + ELSE "RejectInvalidUtf8" + +CodewordStatus(profile, bytes) == + CASE profile = "ULEB" -> UlebStatus(bytes) + [] profile = "UTF8" -> Utf8Status(bytes) + [] OTHER -> "Emit" + +DecodedAtom(profile, bytes) == + CASE profile = "ULEB" -> + [kind |-> "ULEB", bytes |-> bytes, scalar |-> 0] + [] profile = "UTF8" -> + [kind |-> "UnicodeScalar", bytes |-> <<>>, + scalar |-> Utf8Value(bytes)] + [] OTHER -> + [kind |-> "Byte", bytes |-> <<>>, + scalar |-> LastElement(bytes)] + +RejectStatuses == { + "RejectInvalidUleb", "RejectOverlongUleb", "RejectInvalidUtf8" +} + +RejectError(status) == + CASE status = "RejectOverlongUleb" -> "NonCanonicalUleb" + [] status = "RejectInvalidUleb" -> "InvalidUleb" + [] OTHER -> "InvalidUtf8" + +VARIABLES scenario, phase, cursor, codewordBuffer, logicalOutput, + completedAtoms, decoderError + +vars == <> + +Init == + /\ scenario \in Scenarios + /\ phase = "Reading" + /\ cursor = 1 + /\ codewordBuffer = <<>> + /\ logicalOutput = <<>> + /\ completedAtoms = 0 + /\ decoderError = "None" + +ReadPhysicalByte == + /\ phase = "Reading" + /\ cursor <= Len(InputBytes(scenario)) + /\ LET byte == InputBytes(scenario)[cursor] + nextBuffer == Append(codewordBuffer, byte) + status == CodewordStatus(Profile(scenario), nextBuffer) + exposedOutput == + IF ExposePhysicalCodecBytes /\ IsVariableWidth(scenario) + THEN Append(logicalOutput, byte) + ELSE logicalOutput + IN /\ cursor' = cursor + 1 + /\ CASE status = "NeedMore" -> + /\ phase' = "Reading" + /\ codewordBuffer' = nextBuffer + /\ logicalOutput' = exposedOutput + /\ completedAtoms' = completedAtoms + /\ decoderError' = "None" + [] status = "Emit" -> + /\ phase' = "Reading" + /\ codewordBuffer' = <<>> + /\ logicalOutput' = + IF ExposePhysicalCodecBytes /\ IsVariableWidth(scenario) + THEN exposedOutput + ELSE Append(logicalOutput, + DecodedAtom(Profile(scenario), nextBuffer)) + /\ completedAtoms' = completedAtoms + 1 + /\ decoderError' = "None" + [] status \in RejectStatuses -> + /\ phase' = "Rejected" + /\ codewordBuffer' = nextBuffer + /\ logicalOutput' = exposedOutput + /\ completedAtoms' = completedAtoms + /\ decoderError' = RejectError(status) + /\ UNCHANGED scenario + +FinalizeInput == + /\ phase = "Reading" + /\ cursor > Len(InputBytes(scenario)) + /\ IF codewordBuffer = <<>> + THEN /\ phase' = "Done" + /\ UNCHANGED <> + ELSE IF Profile(scenario) = "ULEB" /\ AcceptUnterminatedUleb + THEN /\ phase' = "Done" + /\ codewordBuffer' = <<>> + /\ logicalOutput' = + IF ExposePhysicalCodecBytes + THEN logicalOutput + ELSE Append(logicalOutput, + [kind |-> "ULEB", bytes |-> codewordBuffer, + scalar |-> 0]) + /\ completedAtoms' = completedAtoms + 1 + /\ decoderError' = "None" + /\ UNCHANGED cursor + ELSE /\ phase' = "Rejected" + /\ decoderError' = + IF Profile(scenario) = "ULEB" + THEN "UnterminatedUleb" + ELSE "TruncatedUtf8" + /\ UNCHANGED <> + /\ UNCHANGED scenario + +DecodeStep == ReadPhysicalByte \/ FinalizeInput + +TerminalStutter == + /\ phase \in {"Done", "Rejected"} + /\ UNCHANGED vars + +Next == DecodeStep \/ TerminalStutter +Spec == Init /\ [][Next]_vars /\ WF_vars(DecodeStep) + +IsByteSequence(sequence) == + /\ DOMAIN sequence = 1..Len(sequence) + /\ \A index \in DOMAIN sequence: sequence[index] \in 0..255 + +IsLogicalAtom(atom) == + /\ atom.kind \in {"ULEB", "UnicodeScalar", "Byte"} + /\ IsByteSequence(atom.bytes) + /\ atom.scalar \in Nat + /\ atom.kind = "ULEB" => atom.bytes # <<>> + /\ atom.kind # "ULEB" => atom.bytes = <<>> + +TypeOK == + /\ scenario \in Scenarios + /\ phase \in {"Reading", "Done", "Rejected"} + /\ cursor \in 1..(Len(InputBytes(scenario)) + 1) + /\ codewordBuffer \in Seq(0..255) + /\ DOMAIN logicalOutput = 1..Len(logicalOutput) + /\ \A index \in DOMAIN logicalOutput: IsLogicalAtom(logicalOutput[index]) + /\ completedAtoms \in 0..(Len(InputBytes(scenario)) + 1) + /\ decoderError \in { + "None", "NonCanonicalUleb", "InvalidUleb", "UnterminatedUleb", + "InvalidUtf8", "TruncatedUtf8" + } + +VWENC_22_NO_LOGICAL_TRANSITION_BEFORE_COMPLETE_CODEWORD == + ~ExposePhysicalCodecBytes => + Len(logicalOutput) = completedAtoms + +VWENC_23_SUCCESS_EMITS_EXACT_LOGICAL_STREAM == + (phase = "Done" /\ scenario \notin InvalidScenarios /\ + ~ExposePhysicalCodecBytes) => + logicalOutput = ExpectedOutput(scenario) + +VWENC_24_CODEC_BYTES_NEVER_BECOME_LOGICAL_TRANSITIONS == + (phase = "Done" /\ scenario \notin InvalidScenarios /\ + IsVariableWidth(scenario)) => + logicalOutput = ExpectedOutput(scenario) + +VWENC_25_DIRECT_BYTE_SEMANTICS_IS_EXPLICIT == + phase = "Done" /\ scenario = "DirectByte" => + logicalOutput = ExpectedOutput("DirectByte") + +VWENC_26_OVERLONG_ULEB_IS_REJECTED == + scenario = "UlebOverlong" => phase # "Done" + +VWENC_27_UNTERMINATED_ULEB_IS_REJECTED == + scenario = "UlebUnterminated" => phase # "Done" + +VWENC_28_UTF8_CONTINUATION_IS_REJECTED == + scenario = "Utf8Continuation" => phase # "Done" + +VWENC_29_REJECTION_IS_EXPLICIT_AND_HAS_NO_LOGICAL_OUTPUT == + (phase = "Rejected" /\ scenario \in InvalidScenarios /\ + ~ExposePhysicalCodecBytes) => + /\ decoderError # "None" + /\ logicalOutput = <<>> + +VWENC_32_CURSOR_AND_BUFFER_ARE_BOUNDED_BY_CONSUMED_INPUT == + /\ cursor <= Len(InputBytes(scenario)) + 1 + /\ Len(codewordBuffer) <= cursor - 1 + /\ completedAtoms <= cursor - 1 + +VWENC_80_ADJACENT_CODEWORDS_PRESERVE_EVERY_BOUNDARY == + (phase = "Done" /\ scenario \in {"UlebAdjacent", "Utf8Adjacent"} /\ + ~ExposePhysicalCodecBytes) => + /\ completedAtoms = 2 + /\ logicalOutput = ExpectedOutput(scenario) + +VWENC_81_INCOMPLETE_BUFFER_NEVER_INCREMENTS_COMPLETED_ATOMS == + phase = "Reading" /\ codewordBuffer # <<>> => + completedAtoms < cursor + +VWENC_82_DECODER_EVENTUALLY_TERMINATES == + <>(phase \in {"Done", "Rejected"}) + +============================================================================= diff --git a/formal-verification/tla+/VariableWidthCodecBoundary_OverlongUnsafe.cfg b/formal-verification/tla+/VariableWidthCodecBoundary_OverlongUnsafe.cfg new file mode 100644 index 00000000..3a20a601 --- /dev/null +++ b/formal-verification/tla+/VariableWidthCodecBoundary_OverlongUnsafe.cfg @@ -0,0 +1,10 @@ +CONSTANTS + AcceptOverlongUleb = TRUE + AcceptUnterminatedUleb = FALSE + AcceptUtf8Continuation = FALSE + ExposePhysicalCodecBytes = FALSE + +SPECIFICATION Spec + +INVARIANT + VWENC_26_OVERLONG_ULEB_IS_REJECTED diff --git a/formal-verification/tla+/VariableWidthCodecBoundary_PhysicalExposureUnsafe.cfg b/formal-verification/tla+/VariableWidthCodecBoundary_PhysicalExposureUnsafe.cfg new file mode 100644 index 00000000..cba18840 --- /dev/null +++ b/formal-verification/tla+/VariableWidthCodecBoundary_PhysicalExposureUnsafe.cfg @@ -0,0 +1,10 @@ +CONSTANTS + AcceptOverlongUleb = FALSE + AcceptUnterminatedUleb = FALSE + AcceptUtf8Continuation = FALSE + ExposePhysicalCodecBytes = TRUE + +SPECIFICATION Spec + +INVARIANT + VWENC_24_CODEC_BYTES_NEVER_BECOME_LOGICAL_TRANSITIONS diff --git a/formal-verification/tla+/VariableWidthCodecBoundary_UnterminatedUnsafe.cfg b/formal-verification/tla+/VariableWidthCodecBoundary_UnterminatedUnsafe.cfg new file mode 100644 index 00000000..441d64a5 --- /dev/null +++ b/formal-verification/tla+/VariableWidthCodecBoundary_UnterminatedUnsafe.cfg @@ -0,0 +1,10 @@ +CONSTANTS + AcceptOverlongUleb = FALSE + AcceptUnterminatedUleb = TRUE + AcceptUtf8Continuation = FALSE + ExposePhysicalCodecBytes = FALSE + +SPECIFICATION Spec + +INVARIANT + VWENC_27_UNTERMINATED_ULEB_IS_REJECTED diff --git a/formal-verification/tla+/VariableWidthCodecBoundary_Utf8ContinuationUnsafe.cfg b/formal-verification/tla+/VariableWidthCodecBoundary_Utf8ContinuationUnsafe.cfg new file mode 100644 index 00000000..53f36791 --- /dev/null +++ b/formal-verification/tla+/VariableWidthCodecBoundary_Utf8ContinuationUnsafe.cfg @@ -0,0 +1,10 @@ +CONSTANTS + AcceptOverlongUleb = FALSE + AcceptUnterminatedUleb = FALSE + AcceptUtf8Continuation = TRUE + ExposePhysicalCodecBytes = FALSE + +SPECIFICATION Spec + +INVARIANT + VWENC_28_UTF8_CONTINUATION_IS_REJECTED diff --git a/formal-verification/tla+/VariableWidthFamilyRefinement.cfg b/formal-verification/tla+/VariableWidthFamilyRefinement.cfg new file mode 100644 index 00000000..12d15cae --- /dev/null +++ b/formal-verification/tla+/VariableWidthFamilyRefinement.cfg @@ -0,0 +1,23 @@ +CONSTANTS + ExposeCodecBytes = FALSE + SplitUtf8Scalar = FALSE + AllowInteriorSuffixStart = FALSE + DivergeSpecializedKernel = FALSE + InferFormatIdentityFromTypeName = FALSE + AcceptMismatchedVocabularyFiber = FALSE + +SPECIFICATION Spec + +CHECK_DEADLOCK FALSE + +INVARIANT + TypeOK + VWENC_241_CODEC_BYTES_NEVER_APPEAR_AS_LOGICAL_LABELS + VWENC_242_UTF8_SCALAR_IS_NEVER_SPLIT_ACROSS_LOGICAL_TRANSITIONS + VWENC_243_SUFFIX_MATCHES_NEVER_BEGIN_INSIDE_A_LOGICAL_CODEWORD + VWENC_244_SPECIALIZED_KERNEL_PRESERVES_THE_COMPLETE_OBSERVATION + VWENC_245_FORMAT_IDENTITY_COMES_ONLY_FROM_EXPLICIT_PROFILE_METADATA + VWENC_246_MISMATCHED_VOCABULARY_FIBER_IS_REJECTED_BEFORE_TRAVERSAL + +PROPERTY + FamilyRefinementEventuallyTerminates diff --git a/formal-verification/tla+/VariableWidthFamilyRefinement.tla b/formal-verification/tla+/VariableWidthFamilyRefinement.tla new file mode 100644 index 00000000..1216d970 --- /dev/null +++ b/formal-verification/tla+/VariableWidthFamilyRefinement.tla @@ -0,0 +1,290 @@ +------------------ MODULE VariableWidthFamilyRefinement ------------------ +EXTENDS Naturals, Sequences, FiniteSets, TLC + +(***************************************************************************) +(* Temporal refinement gate for dictionary profiles and consumer views. *) +(* *) +(* Three initial witnesses are explored: a direct profile, an interned *) +(* profile bound to the matching vocabulary snapshot, and an interned *) +(* profile presented with a mismatched snapshot. Direct traversal needs no *) +(* vocabulary. Matching interned traversal binds before any logical *) +(* projection. A mismatch terminates as Rejected without observations. *) +(* *) +(* The finite witness bounds TLC exploration only; it is not a workload or *) +(* resource limit in libdictenstein. Each Boolean constant weakens one *) +(* protocol decision. The verification harness checks the named failure and *) +(* independently proves that every non-target invariant remains true. *) +(***************************************************************************) + +CONSTANTS ExposeCodecBytes, + SplitUtf8Scalar, + AllowInteriorSuffixStart, + DivergeSpecializedKernel, + InferFormatIdentityFromTypeName, + AcceptMismatchedVocabularyFiber + +ASSUME /\ ExposeCodecBytes \in BOOLEAN + /\ SplitUtf8Scalar \in BOOLEAN + /\ AllowInteriorSuffixStart \in BOOLEAN + /\ DivergeSpecializedKernel \in BOOLEAN + /\ InferFormatIdentityFromTypeName \in BOOLEAN + /\ AcceptMismatchedVocabularyFiber \in BOOLEAN + +Modes == {"Direct", "InternedMatching", "InternedMismatched"} + +Phases == { + "Ready", "Bound", "Rejected", "FormatChecked", "Projected", + "SuffixChecked", "KernelChecked", "Done" +} + +BindingStatuses == {"Unchecked", "NotRequired", "Accepted", "Rejected"} + +PhysicalUtf8 == <<195, 169>> +CodewordBoundaries(selectedMode) == + IF selectedMode = "Direct" THEN {0, 2} ELSE {0, 4} + +UnicodeScalarLabel == [kind |-> "UnicodeScalar", value |-> 233] +SymbolIdLabel == [kind |-> "SymbolIdU32", value |-> 7] +PhysicalLabel(byte) == [kind |-> "PhysicalByte", value |-> byte] +LeadHalfLabel == [kind |-> "Utf8LeadHalf", value |-> 0] +ContinuationHalfLabel == [kind |-> "Utf8ContinuationHalf", value |-> 0] + +LogicalLabelUniverse == + {UnicodeScalarLabel, SymbolIdLabel, LeadHalfLabel, ContinuationHalfLabel} \cup + {PhysicalLabel(byte) : byte \in 0..255} + +LogicalUnitLabel(selectedMode) == + IF selectedMode = "Direct" THEN UnicodeScalarLabel ELSE SymbolIdLabel + +LogicalUnitName(selectedMode) == + IF selectedMode = "Direct" THEN "U+00E9" ELSE "SymbolIdU32(7)" + +BaselineObservation(selectedMode) == + [membership |-> TRUE, + terminal |-> TRUE, + mappedValue |-> 7, + outgoing |-> <>, + prefixEntries |-> <>, + enumeration |-> <>, + substring |-> <>, + suffix |-> <>] + +DivergedObservation(selectedMode) == + [BaselineObservation(selectedMode) EXCEPT !.mappedValue = 8] + +CanonicalFormat(selectedMode) == + IF selectedMode = "Direct" + THEN [backend |-> "DynamicDawgFamily", + profile |-> "DirectUnicodeScalarDomain", + codec |-> "FamilyExistingCharU32Codec", + layout |-> "GenericLogicalLayout", + version |-> 1] + ELSE [backend |-> "DynamicDawgFamily", + profile |-> "InternedCanonicalUlebDomainU32", + codec |-> "FamilyFixedIdCarrierCodecU32", + layout |-> "ProspectiveInternedIdLayoutU32", + version |-> 1] + +TypeNameDerivedFormat(selectedMode) == + [backend |-> "RustTypeNameHash", + profile |-> IF selectedMode = "Direct" + THEN "DynamicDawgChar" + ELSE "DynamicDawgInternedU32", + codec |-> "Implicit", + layout |-> "Implicit", + version |-> 1] + +NoFormat == + [backend |-> "None", profile |-> "None", codec |-> "None", + layout |-> "None", version |-> 0] + +VocabularyFiber(generation) == + [identity |-> "Vocabulary-A", + generation |-> generation, + atomProfile |-> "CanonicalULEB", + codec |-> "CanonicalULEB-v1", + layout |-> "LogicalUnit-v1", + abiVersion |-> 1, + carrierFormat |-> 32, + carrierWidth |-> 4] + +NoFiber == + [identity |-> "None", + generation |-> 0, + atomProfile |-> "None", + codec |-> "None", + layout |-> "None", + abiVersion |-> 0, + carrierFormat |-> 0, + carrierWidth |-> 0] + +VARIABLES mode, phase, bindingStatus, logicalLabels, suffixStart, + genericObservation, specializedObservation, explicitFormat, + readFormat, expectedFiber, actualFiber + +vars == <> + +Init == + /\ mode \in Modes + /\ phase = "Ready" + /\ bindingStatus = "Unchecked" + /\ logicalLabels = <<>> + /\ suffixStart = 0 + /\ genericObservation = BaselineObservation(mode) + /\ specializedObservation = BaselineObservation(mode) + /\ explicitFormat = CanonicalFormat(mode) + /\ readFormat = NoFormat + /\ expectedFiber = IF mode = "Direct" THEN NoFiber ELSE VocabularyFiber(1) + /\ actualFiber = + IF mode = "Direct" THEN NoFiber + ELSE IF mode = "InternedMatching" THEN VocabularyFiber(1) + ELSE VocabularyFiber(2) + +BindProfile == + /\ phase = "Ready" + /\ phase' = + IF mode = "InternedMismatched" /\ + ~AcceptMismatchedVocabularyFiber + THEN "Rejected" + ELSE "Bound" + /\ bindingStatus' = + IF mode = "Direct" THEN "NotRequired" + ELSE IF expectedFiber = actualFiber THEN "Accepted" + ELSE IF AcceptMismatchedVocabularyFiber THEN "Accepted" + ELSE "Rejected" + /\ UNCHANGED <> + +LoadExplicitFormat == + /\ phase = "Bound" + /\ phase' = "FormatChecked" + /\ readFormat' = + IF InferFormatIdentityFromTypeName + THEN TypeNameDerivedFormat(mode) + ELSE explicitFormat + /\ UNCHANGED <> + +ProjectLogicalScalar == + /\ phase = "FormatChecked" + /\ phase' = "Projected" + /\ logicalLabels' = + IF mode = "Direct" + THEN IF ExposeCodecBytes + THEN <> + ELSE IF SplitUtf8Scalar + THEN <> + ELSE <> + ELSE <> + /\ UNCHANGED <> + +SelectSuffixBoundary == + /\ phase = "Projected" + /\ phase' = "SuffixChecked" + /\ suffixStart' = IF AllowInteriorSuffixStart THEN 1 ELSE 0 + /\ UNCHANGED <> + +CompareSpecializedKernel == + /\ phase = "SuffixChecked" + /\ phase' = "KernelChecked" + /\ specializedObservation' = + IF DivergeSpecializedKernel + THEN DivergedObservation(mode) + ELSE genericObservation + /\ UNCHANGED <> + +Finish == + /\ phase = "KernelChecked" + /\ phase' = "Done" + /\ UNCHANGED <> + +Advance == + BindProfile \/ LoadExplicitFormat \/ ProjectLogicalScalar \/ + SelectSuffixBoundary \/ CompareSpecializedKernel \/ Finish + +TerminalStutter == + /\ phase \in {"Done", "Rejected"} + /\ UNCHANGED vars + +Next == Advance \/ TerminalStutter +Spec == Init /\ [][Next]_vars /\ WF_vars(Advance) + +TypeOK == + /\ mode \in Modes + /\ phase \in Phases + /\ bindingStatus \in BindingStatuses + /\ logicalLabels \in Seq(LogicalLabelUniverse) + /\ suffixStart \in 0..4 + /\ genericObservation = BaselineObservation(mode) + /\ specializedObservation \in + {BaselineObservation(mode), DivergedObservation(mode)} + /\ explicitFormat = CanonicalFormat(mode) + /\ readFormat \in + {NoFormat, CanonicalFormat(mode), TypeNameDerivedFormat(mode)} + /\ expectedFiber = IF mode = "Direct" THEN NoFiber ELSE VocabularyFiber(1) + /\ actualFiber = + IF mode = "Direct" THEN NoFiber + ELSE IF mode = "InternedMatching" THEN VocabularyFiber(1) + ELSE VocabularyFiber(2) + +ProjectionCompleted == + phase \in {"Projected", "SuffixChecked", "KernelChecked", "Done"} + +TraversalStarted == ProjectionCompleted + +SuffixSelectionCompleted == + phase \in {"SuffixChecked", "KernelChecked", "Done"} + +KernelComparisonCompleted == phase \in {"KernelChecked", "Done"} +FormatLoadCompleted == + phase \in {"FormatChecked", "Projected", "SuffixChecked", + "KernelChecked", "Done"} + +VWENC_241_CODEC_BYTES_NEVER_APPEAR_AS_LOGICAL_LABELS == + ~ProjectionCompleted \/ + \A index \in 1..Len(logicalLabels): + logicalLabels[index].kind # "PhysicalByte" + +VWENC_242_UTF8_SCALAR_IS_NEVER_SPLIT_ACROSS_LOGICAL_TRANSITIONS == + ~ProjectionCompleted \/ + \A index \in 1..Len(logicalLabels): + logicalLabels[index].kind \notin {"Utf8LeadHalf", "Utf8ContinuationHalf"} + +VWENC_243_SUFFIX_MATCHES_NEVER_BEGIN_INSIDE_A_LOGICAL_CODEWORD == + ~SuffixSelectionCompleted \/ suffixStart \in CodewordBoundaries(mode) + +VWENC_244_SPECIALIZED_KERNEL_PRESERVES_THE_COMPLETE_OBSERVATION == + ~KernelComparisonCompleted \/ + specializedObservation = genericObservation + +VWENC_245_FORMAT_IDENTITY_COMES_ONLY_FROM_EXPLICIT_PROFILE_METADATA == + ~FormatLoadCompleted \/ readFormat = explicitFormat + +VWENC_246_MISMATCHED_VOCABULARY_FIBER_IS_REJECTED_BEFORE_TRAVERSAL == + /\ (bindingStatus = "Accepted" => expectedFiber = actualFiber) + /\ (TraversalStarted => + mode = "Direct" \/ + (bindingStatus = "Accepted" /\ expectedFiber = actualFiber)) + /\ (mode = "InternedMismatched" => + /\ phase \in {"Ready", "Rejected"} + /\ bindingStatus \in {"Unchecked", "Rejected"} + /\ ~TraversalStarted + /\ logicalLabels = <<>>) + +FamilyRefinementEventuallyTerminates == + <>(phase \in {"Done", "Rejected"}) + +============================================================================= diff --git a/formal-verification/tla+/VariableWidthFamilyRefinement_FiberMismatchUnsafe.cfg b/formal-verification/tla+/VariableWidthFamilyRefinement_FiberMismatchUnsafe.cfg new file mode 100644 index 00000000..d90dc82f --- /dev/null +++ b/formal-verification/tla+/VariableWidthFamilyRefinement_FiberMismatchUnsafe.cfg @@ -0,0 +1,12 @@ +CONSTANTS + ExposeCodecBytes = FALSE + SplitUtf8Scalar = FALSE + AllowInteriorSuffixStart = FALSE + DivergeSpecializedKernel = FALSE + InferFormatIdentityFromTypeName = FALSE + AcceptMismatchedVocabularyFiber = TRUE + +SPECIFICATION Spec + +INVARIANT + VWENC_246_MISMATCHED_VOCABULARY_FIBER_IS_REJECTED_BEFORE_TRAVERSAL diff --git a/formal-verification/tla+/VariableWidthFamilyRefinement_InteriorSuffixUnsafe.cfg b/formal-verification/tla+/VariableWidthFamilyRefinement_InteriorSuffixUnsafe.cfg new file mode 100644 index 00000000..d6135c41 --- /dev/null +++ b/formal-verification/tla+/VariableWidthFamilyRefinement_InteriorSuffixUnsafe.cfg @@ -0,0 +1,12 @@ +CONSTANTS + ExposeCodecBytes = FALSE + SplitUtf8Scalar = FALSE + AllowInteriorSuffixStart = TRUE + DivergeSpecializedKernel = FALSE + InferFormatIdentityFromTypeName = FALSE + AcceptMismatchedVocabularyFiber = FALSE + +SPECIFICATION Spec + +INVARIANT + VWENC_243_SUFFIX_MATCHES_NEVER_BEGIN_INSIDE_A_LOGICAL_CODEWORD diff --git a/formal-verification/tla+/VariableWidthFamilyRefinement_PhysicalExposureUnsafe.cfg b/formal-verification/tla+/VariableWidthFamilyRefinement_PhysicalExposureUnsafe.cfg new file mode 100644 index 00000000..50bf1bfd --- /dev/null +++ b/formal-verification/tla+/VariableWidthFamilyRefinement_PhysicalExposureUnsafe.cfg @@ -0,0 +1,12 @@ +CONSTANTS + ExposeCodecBytes = TRUE + SplitUtf8Scalar = FALSE + AllowInteriorSuffixStart = FALSE + DivergeSpecializedKernel = FALSE + InferFormatIdentityFromTypeName = FALSE + AcceptMismatchedVocabularyFiber = FALSE + +SPECIFICATION Spec + +INVARIANT + VWENC_241_CODEC_BYTES_NEVER_APPEAR_AS_LOGICAL_LABELS diff --git a/formal-verification/tla+/VariableWidthFamilyRefinement_SpecializedDivergenceUnsafe.cfg b/formal-verification/tla+/VariableWidthFamilyRefinement_SpecializedDivergenceUnsafe.cfg new file mode 100644 index 00000000..3bf0c9fd --- /dev/null +++ b/formal-verification/tla+/VariableWidthFamilyRefinement_SpecializedDivergenceUnsafe.cfg @@ -0,0 +1,12 @@ +CONSTANTS + ExposeCodecBytes = FALSE + SplitUtf8Scalar = FALSE + AllowInteriorSuffixStart = FALSE + DivergeSpecializedKernel = TRUE + InferFormatIdentityFromTypeName = FALSE + AcceptMismatchedVocabularyFiber = FALSE + +SPECIFICATION Spec + +INVARIANT + VWENC_244_SPECIALIZED_KERNEL_PRESERVES_THE_COMPLETE_OBSERVATION diff --git a/formal-verification/tla+/VariableWidthFamilyRefinement_TypeNameFormatUnsafe.cfg b/formal-verification/tla+/VariableWidthFamilyRefinement_TypeNameFormatUnsafe.cfg new file mode 100644 index 00000000..0ad26aaa --- /dev/null +++ b/formal-verification/tla+/VariableWidthFamilyRefinement_TypeNameFormatUnsafe.cfg @@ -0,0 +1,12 @@ +CONSTANTS + ExposeCodecBytes = FALSE + SplitUtf8Scalar = FALSE + AllowInteriorSuffixStart = FALSE + DivergeSpecializedKernel = FALSE + InferFormatIdentityFromTypeName = TRUE + AcceptMismatchedVocabularyFiber = FALSE + +SPECIFICATION Spec + +INVARIANT + VWENC_245_FORMAT_IDENTITY_COMES_ONLY_FROM_EXPLICIT_PROFILE_METADATA diff --git a/formal-verification/tla+/VariableWidthFamilyRefinement_Utf8SplitUnsafe.cfg b/formal-verification/tla+/VariableWidthFamilyRefinement_Utf8SplitUnsafe.cfg new file mode 100644 index 00000000..50e96ba6 --- /dev/null +++ b/formal-verification/tla+/VariableWidthFamilyRefinement_Utf8SplitUnsafe.cfg @@ -0,0 +1,12 @@ +CONSTANTS + ExposeCodecBytes = FALSE + SplitUtf8Scalar = TRUE + AllowInteriorSuffixStart = FALSE + DivergeSpecializedKernel = FALSE + InferFormatIdentityFromTypeName = FALSE + AcceptMismatchedVocabularyFiber = FALSE + +SPECIFICATION Spec + +INVARIANT + VWENC_242_UTF8_SCALAR_IS_NEVER_SPLIT_ACROSS_LOGICAL_TRANSITIONS diff --git a/formal-verification/tla+/VariableWidthVocabularyInterning.cfg b/formal-verification/tla+/VariableWidthVocabularyInterning.cfg new file mode 100644 index 00000000..cf36c922 --- /dev/null +++ b/formal-verification/tla+/VariableWidthVocabularyInterning.cfg @@ -0,0 +1,20 @@ +CONSTANTS + FingerprintOnlyEquality = FALSE + ReusePublishedId = FALSE + +SPECIFICATION Spec + +CHECK_DEADLOCK FALSE + +INVARIANT + TypeOK + VWENC_141_PUBLISHED_ATOM_RELATION_IS_EXACT_BIJECTION + VWENC_142_FINGERPRINT_COLLISIONS_NEVER_ALIAS_DISTINCT_ATOMS + VWENC_143_RETIRED_ID_IS_NEVER_CLAIMED_OR_LIVE_AGAIN + VWENC_192_EVER_PUBLISHED_OWNER_IS_IMMUTABLE + VWENC_144_LIVE_ID_HAS_EXACT_DURABLE_PAYLOAD_AND_SPAN + VWENC_145_ACTIVE_CLAIMS_DO_NOT_OVERWRITE_LIVE_IDS + VWENC_146_ORPHAN_ALLOCATIONS_HAVE_NO_LOGICAL_BINDING + VWENC_175_PACKED_SPANS_ARE_DISJOINT_AND_COVER_BYTES_EXACTLY + VWENC_176_ALLOCATION_STATUSES_PARTITION_ALLOCATED_IDS + VWENC_177_DESCRIPTOR_GOVERNS_EVERY_MATERIALIZED_CODEWORD diff --git a/formal-verification/tla+/VariableWidthVocabularyInterning.tla b/formal-verification/tla+/VariableWidthVocabularyInterning.tla new file mode 100644 index 00000000..2de85fc2 --- /dev/null +++ b/formal-verification/tla+/VariableWidthVocabularyInterning.tla @@ -0,0 +1,360 @@ +------------------ MODULE VariableWidthVocabularyInterning ------------------ +EXTENDS Naturals, Sequences, FiniteSets, TLC + +(***************************************************************************) +(* Concurrent canonical-atom reservation, materialization, and publication. *) +(* *) +(* The finite atoms and IDs are TLC counterexample bounds, never library *) +(* workload limits. CanonicalBytes deliberately collide under Fingerprint. *) +(* A reservation allocates an ID without bytes. Materialization appends one *) +(* nonempty canonical codeword and its exact span. Losing either kind of *) +(* claim produces an unmaterialized or materialized orphan, respectively. *) +(* Every materialized span is disjoint, in bounds, and collectively covers *) +(* the append-only packed byte sequence exactly. *) +(***************************************************************************) + +CONSTANTS FingerprintOnlyEquality, ReusePublishedId + +ASSUME /\ FingerprintOnlyEquality \in BOOLEAN + /\ ReusePublishedId \in BOOLEAN + +Atoms == {"Alpha", "Beta"} +Ids == 0..1 +NoAtom == "NoAtom" +NoId == 2 +AtomOrNone == Atoms \cup {NoAtom} +IdOrNone == 0..2 + +CanonicalDescriptor == + [logicalAlphabet |-> "CanonicalULEB", + codecIdentity |-> "CanonicalULEB-v1", + layoutIdentity |-> "LogicalUnit-v1", + abiVersion |-> 1] + +DescriptorType == + [logicalAlphabet : {"CanonicalULEB"}, + codecIdentity : {"CanonicalULEB-v1"}, + layoutIdentity : {"LogicalUnit-v1"}, + abiVersion : {1}] + +EmptyAtomToId == [atom \in Atoms |-> NoId] +EmptyIdToAtom == [id \in Ids |-> NoAtom] +EmptyPayload == [id \in Ids |-> <<>>] +EmptyNatMap == [id \in Ids |-> 0] + +CanonicalBytes(atom) == + CASE atom = "Alpha" -> <<129, 1>> + [] OTHER -> <<130, 1>> + +CanonicalUlebCodeword(bytes) == + /\ Len(bytes) > 0 + /\ \A index \in 1..Len(bytes) : bytes[index] \in 0..255 + /\ bytes[Len(bytes)] < 128 + /\ \A index \in 1..(Len(bytes) - 1) : bytes[index] >= 128 + /\ (Len(bytes) = 1 \/ bytes[Len(bytes)] # 0) + +DescriptorCanonicalCodeword(descriptorValue, atom, bytes) == + /\ descriptorValue = CanonicalDescriptor + /\ atom \in Atoms + /\ bytes = CanonicalBytes(atom) + /\ CanonicalUlebCodeword(bytes) + +Fingerprint(_atom) == 7 + +VARIABLES descriptor, + claims, + atomToId, + idToAtom, + everPublishedOwner, + retiredIds, + nextId, + allocatedIds, + durablePayload, + durableSpan, + payloadById, + packedBytes, + spanOffset, + spanLength, + spanOwner + +vars == <> + +LiveIds == {id \in Ids : idToAtom[id] # NoAtom} +ClaimedIds == {id \in Ids : claims[id] # NoAtom} +HistoricalIds == {id \in Ids : everPublishedOwner[id] # NoAtom} +TombstonedIds == HistoricalIds \ LiveIds +OrphanIds == allocatedIds \ (HistoricalIds \cup ClaimedIds) +MaterializedIds == durablePayload \cap durableSpan +ReservedIds == ClaimedIds \ MaterializedIds +MaterializedClaimedIds == ClaimedIds \cap MaterializedIds +MaterializedOrphanIds == OrphanIds \cap MaterializedIds +UnmaterializedOrphanIds == OrphanIds \ MaterializedIds + +HasFingerprintCandidate(atom) == + \E id \in LiveIds : Fingerprint(idToAtom[id]) = Fingerprint(atom) + +FingerprintCandidate(atom) == + CHOOSE id \in LiveIds : Fingerprint(idToAtom[id]) = Fingerprint(atom) + +ReadSpan(id) == + SubSeq(packedBytes, spanOffset[id] + 1, spanOffset[id] + spanLength[id]) + +SpansDisjoint(left, right) == + spanOffset[left] + spanLength[left] <= spanOffset[right] \/ + spanOffset[right] + spanLength[right] <= spanOffset[left] + +SpanContainsOffset(id, offset) == + spanOffset[id] <= offset /\ offset < spanOffset[id] + spanLength[id] + +ExactSpanForAtom(id, atom) == + /\ id \in MaterializedIds + /\ DescriptorCanonicalCodeword(descriptor, atom, payloadById[id]) + /\ spanOwner[id] = atom + /\ spanLength[id] = Len(payloadById[id]) + /\ spanLength[id] > 0 + /\ spanOffset[id] + spanLength[id] <= Len(packedBytes) + /\ ReadSpan(id) = payloadById[id] + +TypeOK == + /\ descriptor \in DescriptorType + /\ claims \in [Ids -> AtomOrNone] + /\ atomToId \in [Atoms -> IdOrNone] + /\ idToAtom \in [Ids -> AtomOrNone] + /\ everPublishedOwner \in [Ids -> AtomOrNone] + /\ retiredIds \in SUBSET Ids + /\ nextId \in 0..2 + /\ allocatedIds \in SUBSET Ids + /\ durablePayload \in SUBSET Ids + /\ durableSpan \in SUBSET Ids + /\ payloadById \in [Ids -> Seq(0..255)] + /\ packedBytes \in Seq(0..255) + /\ spanOffset \in [Ids -> 0..4] + /\ spanLength \in [Ids -> 0..2] + /\ spanOwner \in [Ids -> AtomOrNone] + +Init == + /\ descriptor = CanonicalDescriptor + /\ claims = EmptyIdToAtom + /\ atomToId = EmptyAtomToId + /\ idToAtom = EmptyIdToAtom + /\ everPublishedOwner = EmptyIdToAtom + /\ retiredIds = {} + /\ nextId = 0 + /\ allocatedIds = {} + /\ durablePayload = {} + /\ durableSpan = {} + /\ payloadById = EmptyPayload + /\ packedBytes = <<>> + /\ spanOffset = EmptyNatMap + /\ spanLength = EmptyNatMap + /\ spanOwner = EmptyIdToAtom + +MultiSpanInit == + /\ descriptor = CanonicalDescriptor + /\ claims = [id \in Ids |-> IF id = 0 THEN "Alpha" ELSE "Beta"] + /\ atomToId = EmptyAtomToId + /\ idToAtom = EmptyIdToAtom + /\ everPublishedOwner = EmptyIdToAtom + /\ retiredIds = {} + /\ nextId = 2 + /\ allocatedIds = Ids + /\ durablePayload = Ids + /\ durableSpan = Ids + /\ payloadById = + [id \in Ids |-> IF id = 0 + THEN CanonicalBytes("Alpha") + ELSE CanonicalBytes("Beta")] + /\ packedBytes = CanonicalBytes("Alpha") \o CanonicalBytes("Beta") + /\ spanOffset = [id \in Ids |-> IF id = 0 THEN 0 ELSE 2] + /\ spanLength = [id \in Ids |-> 2] + /\ spanOwner = [id \in Ids |-> IF id = 0 THEN "Alpha" ELSE "Beta"] + +VWENC_180_MULTISPAN_WITNESS_IS_CONCRETE == + /\ MaterializedIds = Ids + /\ Cardinality(MaterializedIds) = 2 + /\ packedBytes = CanonicalBytes("Alpha") \o CanonicalBytes("Beta") + /\ spanOffset[0] = 0 + /\ spanOffset[1] = Len(CanonicalBytes("Alpha")) + /\ ReadSpan(0) = CanonicalBytes("Alpha") + /\ ReadSpan(1) = CanonicalBytes("Beta") + /\ SpansDisjoint(0, 1) + +ClaimAtomId(atom) == + /\ atom \in Atoms + /\ atomToId[atom] = NoId + /\ nextId <= 1 + /\ LET candidate == + IF ReusePublishedId /\ 0 \in TombstonedIds + THEN 0 + ELSE nextId + IN /\ candidate \in Ids + /\ claims[candidate] = NoAtom + /\ idToAtom[candidate] = NoAtom + /\ (ReusePublishedId \/ candidate \notin retiredIds) + /\ (ReusePublishedId \/ + everPublishedOwner[candidate] = NoAtom) + /\ claims' = [claims EXCEPT ![candidate] = atom] + /\ allocatedIds' = allocatedIds \cup {candidate} + /\ nextId' = IF candidate = nextId THEN nextId + 1 ELSE nextId + /\ UNCHANGED <> + +WriteCanonicalPayloadAndSpan(id) == + /\ id \in ClaimedIds + /\ \/ (id \notin durablePayload /\ id \notin durableSpan) + \/ /\ ReusePublishedId + /\ id \in TombstonedIds + /\ id \in durablePayload + /\ id \in durableSpan + /\ Len(packedBytes) < 4 + /\ spanOwner[id] # claims[id] + /\ DescriptorCanonicalCodeword( + descriptor, claims[id], CanonicalBytes(claims[id])) + /\ payloadById' = [payloadById EXCEPT ![id] = CanonicalBytes(claims[id])] + /\ spanOwner' = [spanOwner EXCEPT ![id] = claims[id]] + /\ spanOffset' = [spanOffset EXCEPT ![id] = Len(packedBytes)] + /\ spanLength' = + [spanLength EXCEPT ![id] = Len(CanonicalBytes(claims[id]))] + /\ packedBytes' = packedBytes \o CanonicalBytes(claims[id]) + /\ durablePayload' = durablePayload \cup {id} + /\ durableSpan' = durableSpan \cup {id} + /\ UNCHANGED <> + +PublishClaim(id) == + /\ id \in MaterializedClaimedIds + /\ ExactSpanForAtom(id, claims[id]) + /\ idToAtom[id] = NoAtom + /\ (ReusePublishedId \/ everPublishedOwner[id] = NoAtom) + /\ LET atom == claims[id] IN + /\ atomToId[atom] = NoId + /\ atomToId' = [atomToId EXCEPT ![atom] = id] + /\ idToAtom' = [idToAtom EXCEPT ![id] = atom] + /\ everPublishedOwner' = + IF everPublishedOwner[id] = NoAtom + THEN [everPublishedOwner EXCEPT ![id] = atom] + ELSE everPublishedOwner + /\ claims' = [claims EXCEPT ![id] = NoAtom] + /\ UNCHANGED <> + +TombstonePublishedId(id) == + /\ id \in LiveIds + /\ LET atom == idToAtom[id] IN + /\ atomToId[atom] = id + /\ atomToId' = [atomToId EXCEPT ![atom] = NoId] + /\ idToAtom' = [idToAtom EXCEPT ![id] = NoAtom] + /\ retiredIds' = retiredIds \cup {id} + /\ UNCHANGED <> + +LoseClaimToOrphan(id) == + /\ id \in ClaimedIds + /\ claims' = [claims EXCEPT ![id] = NoAtom] + /\ UNCHANGED <> + +ReturnFingerprintCandidateWithoutByteCheck(atom) == + /\ FingerprintOnlyEquality + /\ atom \in Atoms + /\ atomToId[atom] = NoId + /\ HasFingerprintCandidate(atom) + /\ LET candidate == FingerprintCandidate(atom) IN + atomToId' = [atomToId EXCEPT ![atom] = candidate] + /\ UNCHANGED <> + +ReturnExistingAtom(atom) == + /\ atom \in Atoms + /\ atomToId[atom] # NoId + /\ UNCHANGED vars + +Next == + \/ \E atom \in Atoms : ClaimAtomId(atom) + \/ \E id \in Ids : WriteCanonicalPayloadAndSpan(id) + \/ \E id \in Ids : PublishClaim(id) + \/ \E id \in Ids : TombstonePublishedId(id) + \/ \E id \in Ids : LoseClaimToOrphan(id) + \/ \E atom \in Atoms : ReturnFingerprintCandidateWithoutByteCheck(atom) + \/ \E atom \in Atoms : ReturnExistingAtom(atom) + +VWENC_141_PUBLISHED_ATOM_RELATION_IS_EXACT_BIJECTION == + (~FingerprintOnlyEquality /\ ~ReusePublishedId) => + /\ \A atom \in Atoms : + atomToId[atom] # NoId => idToAtom[atomToId[atom]] = atom + /\ \A id \in Ids : + idToAtom[id] # NoAtom => atomToId[idToAtom[id]] = id + +VWENC_142_FINGERPRINT_COLLISIONS_NEVER_ALIAS_DISTINCT_ATOMS == + \A left \in Atoms, right \in Atoms : + /\ left # right + /\ atomToId[left] # NoId + /\ atomToId[right] # NoId + => atomToId[left] # atomToId[right] + +VWENC_143_RETIRED_ID_IS_NEVER_CLAIMED_OR_LIVE_AGAIN == + retiredIds \cap (ClaimedIds \cup LiveIds) = {} + +VWENC_192_EVER_PUBLISHED_OWNER_IS_IMMUTABLE == + ~ReusePublishedId => + \A id \in Ids : + (everPublishedOwner[id] # NoAtom + /\ idToAtom[id] # NoAtom) => + idToAtom[id] = everPublishedOwner[id] + +VWENC_144_LIVE_ID_HAS_EXACT_DURABLE_PAYLOAD_AND_SPAN == + \A id \in LiveIds : ExactSpanForAtom(id, idToAtom[id]) + +VWENC_145_ACTIVE_CLAIMS_DO_NOT_OVERWRITE_LIVE_IDS == + ClaimedIds \cap LiveIds = {} + +VWENC_146_ORPHAN_ALLOCATIONS_HAVE_NO_LOGICAL_BINDING == + /\ \A id \in OrphanIds : + /\ idToAtom[id] = NoAtom + /\ everPublishedOwner[id] = NoAtom + /\ \A atom \in Atoms : atomToId[atom] # id + /\ OrphanIds = MaterializedOrphanIds \cup UnmaterializedOrphanIds + /\ MaterializedOrphanIds \cap UnmaterializedOrphanIds = {} + +VWENC_175_PACKED_SPANS_ARE_DISJOINT_AND_COVER_BYTES_EXACTLY == + ~ReusePublishedId => + /\ \A id \in MaterializedIds : + ExactSpanForAtom(id, spanOwner[id]) + /\ \A left \in MaterializedIds, right \in MaterializedIds : + left # right => SpansDisjoint(left, right) + /\ (Len(packedBytes) > 0 => + \A offset \in 0..(Len(packedBytes) - 1) : + Cardinality( + {id \in MaterializedIds : SpanContainsOffset(id, offset)}) = 1) + +VWENC_176_ALLOCATION_STATUSES_PARTITION_ALLOCATED_IDS == + ~ReusePublishedId => + /\ allocatedIds = + ReservedIds \cup MaterializedClaimedIds \cup LiveIds \cup + TombstonedIds \cup MaterializedOrphanIds \cup + UnmaterializedOrphanIds + /\ ReservedIds \cap MaterializedClaimedIds = {} + /\ ClaimedIds \cap HistoricalIds = {} + /\ LiveIds \cap TombstonedIds = {} + /\ OrphanIds \cap (ClaimedIds \cup HistoricalIds) = {} + +VWENC_177_DESCRIPTOR_GOVERNS_EVERY_MATERIALIZED_CODEWORD == + \A id \in MaterializedIds : + DescriptorCanonicalCodeword(descriptor, spanOwner[id], payloadById[id]) + +Spec == Init /\ [][Next]_vars + +============================================================================= diff --git a/formal-verification/tla+/VariableWidthVocabularyInterning_FingerprintOnlyUnsafe.cfg b/formal-verification/tla+/VariableWidthVocabularyInterning_FingerprintOnlyUnsafe.cfg new file mode 100644 index 00000000..7d446f8d --- /dev/null +++ b/formal-verification/tla+/VariableWidthVocabularyInterning_FingerprintOnlyUnsafe.cfg @@ -0,0 +1,11 @@ +CONSTANTS + FingerprintOnlyEquality = TRUE + ReusePublishedId = FALSE + +SPECIFICATION Spec + +CHECK_DEADLOCK FALSE + +INVARIANT + TypeOK + VWENC_142_FINGERPRINT_COLLISIONS_NEVER_ALIAS_DISTINCT_ATOMS diff --git a/formal-verification/tla+/VariableWidthVocabularyInterning_IdReuseUnsafe.cfg b/formal-verification/tla+/VariableWidthVocabularyInterning_IdReuseUnsafe.cfg new file mode 100644 index 00000000..ed49c38a --- /dev/null +++ b/formal-verification/tla+/VariableWidthVocabularyInterning_IdReuseUnsafe.cfg @@ -0,0 +1,11 @@ +CONSTANTS + FingerprintOnlyEquality = FALSE + ReusePublishedId = TRUE + +SPECIFICATION Spec + +CHECK_DEADLOCK FALSE + +INVARIANT + TypeOK + VWENC_143_RETIRED_ID_IS_NEVER_CLAIMED_OR_LIVE_AGAIN diff --git a/formal-verification/tla+/VariableWidthVocabularyInterning_MultiSpan.cfg b/formal-verification/tla+/VariableWidthVocabularyInterning_MultiSpan.cfg new file mode 100644 index 00000000..d469c2d8 --- /dev/null +++ b/formal-verification/tla+/VariableWidthVocabularyInterning_MultiSpan.cfg @@ -0,0 +1,20 @@ +CONSTANTS + FingerprintOnlyEquality = FALSE + ReusePublishedId = FALSE + +INIT MultiSpanInit +NEXT Next + +CHECK_DEADLOCK FALSE + +INVARIANT + TypeOK + VWENC_180_MULTISPAN_WITNESS_IS_CONCRETE + VWENC_141_PUBLISHED_ATOM_RELATION_IS_EXACT_BIJECTION + VWENC_143_RETIRED_ID_IS_NEVER_CLAIMED_OR_LIVE_AGAIN + VWENC_192_EVER_PUBLISHED_OWNER_IS_IMMUTABLE + VWENC_144_LIVE_ID_HAS_EXACT_DURABLE_PAYLOAD_AND_SPAN + VWENC_145_ACTIVE_CLAIMS_DO_NOT_OVERWRITE_LIVE_IDS + VWENC_175_PACKED_SPANS_ARE_DISJOINT_AND_COVER_BYTES_EXACTLY + VWENC_176_ALLOCATION_STATUSES_PARTITION_ALLOCATED_IDS + VWENC_177_DESCRIPTOR_GOVERNS_EVERY_MATERIALIZED_CODEWORD diff --git a/formal-verification/tla+/VariableWidthVocabularyPublication.cfg b/formal-verification/tla+/VariableWidthVocabularyPublication.cfg new file mode 100644 index 00000000..19ac1841 --- /dev/null +++ b/formal-verification/tla+/VariableWidthVocabularyPublication.cfg @@ -0,0 +1,24 @@ +CONSTANTS + PublishSequenceBeforeVocabulary = FALSE + OverclaimVocabularyFrontier = FALSE + AllowCrossGenerationResume = FALSE + MissingVocabularyAsEmpty = FALSE + +SPECIFICATION Spec + +CHECK_DEADLOCK FALSE + +INVARIANT + TypeOK + VWENC_147_PUBLISHED_FRONTIER_DOES_NOT_EXCEED_DURABLE_FRONTIER + VWENC_148_PUBLISHED_IDS_HAVE_EXACT_DURABLE_METADATA + VWENC_149_DURABLE_SEQUENCE_REFERENCES_DURABLE_BOUND_VOCABULARY + VWENC_150_SEQUENCE_OBJECT_FOLLOWS_DURABLE_VOCABULARY_OBJECT + VWENC_151_SEQUENCE_DESCRIPTOR_BINDS_EXACT_VOCABULARY_FIBER + VWENC_152_HEAD_BINDS_ONE_COHERENT_DURABLE_PAIR + VWENC_153_RECOVERY_IS_COHERENT_OLD_NEW_OR_ERROR + VWENC_154_CAPTURED_CONTINUATION_RESUMES_IMMUTABLE_PAIR + VWENC_155_UNAVAILABLE_OR_CORRUPT_HEAD_ARTIFACT_IS_EXPLICIT_ERROR + VWENC_156_PUBLISHED_HEAD_HAS_NO_DANGLING_ID_REFERENCE + VWENC_178_RECOVERY_NEVER_SYNTHESIZES_EMPTY_SUCCESS + VWENC_179_EXACT_TERM_FIBER_SEPARATES_SAME_RAW_ID diff --git a/formal-verification/tla+/VariableWidthVocabularyPublication.tla b/formal-verification/tla+/VariableWidthVocabularyPublication.tla new file mode 100644 index 00000000..2b044c60 --- /dev/null +++ b/formal-verification/tla+/VariableWidthVocabularyPublication.tla @@ -0,0 +1,1111 @@ +------------------ MODULE VariableWidthVocabularyPublication ------------------ +EXTENDS Naturals, Sequences, FiniteSets, TLC + +(***************************************************************************) +(* Exact dictionary-local publication and recovery model. *) +(* *) +(* The two generations and two ID positions are finite TLC counterexample *) +(* bounds. They are not library key, depth, atom, result, or work limits. *) +(* Functional generality and open carrier widths are proved in Rocq. *) +(* *) +(* Each generation follows the same staged protocol: *) +(* *) +(* build exact canonical metadata *) +(* -> durabilize each live ID *) +(* -> seal sparse vocabulary eligibility *) +(* -> expose and durabilize the dependent sequence *) +(* -> write/sync immutable vocabulary and sequence objects *) +(* -> atomically replace the head *) +(* *) +(* AllocatorHighWater is deliberately independent from LiveIds. Generation *) +(* one publishes two adjacent nonempty spans, while generation two *) +(* publishes ID 0 with ID 1 as a legal gap. No invariant interprets a *) +(* high-water mark as a dense allocation claim. Every referenced ID is *) +(* checked for exact membership and exact canonical payload/span ownership. *) +(* *) +(* Durable objects are keyed by immutable object identity. The atomic head *) +(* refers to one exact vocabulary/sequence pair. Old heads and objects are *) +(* retained for captured readers. Recovery returns the exact current head *) +(* pair or an explicit error; it never synthesizes an empty dictionary. *) +(* Availability and corruption are tracked independently for vocabulary *) +(* and sequence objects. *) +(* *) +(* The four Boolean constants are isolated negative controls. Each changes *) +(* exactly one protocol decision and is paired with its own TLC config. *) +(***************************************************************************) + +CONSTANTS PublishSequenceBeforeVocabulary, + OverclaimVocabularyFrontier, + AllowCrossGenerationResume, + MissingVocabularyAsEmpty + +ASSUME /\ PublishSequenceBeforeVocabulary \in BOOLEAN + /\ OverclaimVocabularyFrontier \in BOOLEAN + /\ AllowCrossGenerationResume \in BOOLEAN + /\ MissingVocabularyAsEmpty \in BOOLEAN + +Generations == 1..2 +Atoms == {"Alpha", "Beta"} +Ids == 0..1 +NoAtom == "NoAtom" +AtomOrNone == Atoms \cup {NoAtom} + +NoFiber == [identity |-> "None", + generation |-> 0, + atomProfile |-> "None", + codec |-> "None", + layout |-> "None", + abiVersion |-> 0, + carrierFormat |-> 0, + carrierWidth |-> 0] + +Fiber(generation) == + [identity |-> "Vocabulary-A", + generation |-> generation, + atomProfile |-> "CanonicalULEB", + codec |-> "CanonicalULEB-v1", + layout |-> "LogicalUnit-v1", + abiVersion |-> 1, + carrierFormat |-> 32, + carrierWidth |-> 4] + +Fibers == {Fiber(g) : g \in Generations} +FiberOrNone == Fibers \cup {NoFiber} + +NoTermFiber == + [vocabularyFiber |-> NoFiber, + identity |-> "None", + generation |-> 0, + carrierFormat |-> 0, + carrierWidth |-> 0] + +TermFiber(generation) == + [vocabularyFiber |-> Fiber(generation), + identity |-> "TermDictionary-A", + generation |-> generation, + carrierFormat |-> 32, + carrierWidth |-> 4] + +TermFibers == {TermFiber(g) : g \in Generations} +TermFiberOrNone == TermFibers \cup {NoTermFiber} + +CanonicalBytes(atom) == + CASE atom = "Alpha" -> <<129, 1>> + [] atom = "Beta" -> <<130, 1>> + [] OTHER -> <<>> + +CanonicalUlebCodeword(bytes) == + /\ Len(bytes) > 0 + /\ \A index \in 1..Len(bytes) : bytes[index] \in 0..255 + /\ bytes[Len(bytes)] < 128 + /\ \A index \in 1..(Len(bytes) - 1) : bytes[index] >= 128 + /\ (Len(bytes) = 1 \/ bytes[Len(bytes)] # 0) + +DescriptorCanonicalCodeword(fiber, atom, bytes) == + /\ fiber.atomProfile = "CanonicalULEB" + /\ fiber.codec = "CanonicalULEB-v1" + /\ fiber.layout = "LogicalUnit-v1" + /\ fiber.abiVersion = 1 + /\ atom \in Atoms + /\ bytes = CanonicalBytes(atom) + /\ CanonicalUlebCodeword(bytes) + +GenerationLiveIds(generation) == + IF generation = 1 THEN Ids ELSE {0} + +GenerationAtom(generation, id) == + IF generation = 1 /\ id = 0 THEN "Alpha" + ELSE IF generation = 1 /\ id = 1 THEN "Beta" + ELSE IF generation = 2 /\ id = 0 THEN "Beta" + ELSE NoAtom + +GenerationSequence(generation) == + IF generation = 1 THEN <<0, 1>> ELSE <<0>> + +EmptyAtomMap == [id \in Ids |-> NoAtom] +EmptyPayloadMap == [id \in Ids |-> <<>>] +NoSpan == [owner |-> NoAtom, offset |-> 0, length |-> 0] +EmptySpanMap == [id \in Ids |-> NoSpan] + +GenerationAtomMap(generation) == + [id \in Ids |-> GenerationAtom(generation, id)] + +GenerationPayloadMap(generation) == + [id \in Ids |-> + IF id \in GenerationLiveIds(generation) + THEN CanonicalBytes(GenerationAtom(generation, id)) + ELSE <<>>] + +GenerationPackedBytes(generation) == + IF generation = 1 + THEN CanonicalBytes("Alpha") \o CanonicalBytes("Beta") + ELSE CanonicalBytes("Beta") + +GenerationSpanMap(generation) == + [id \in Ids |-> + IF id \in GenerationLiveIds(generation) + THEN [owner |-> GenerationAtom(generation, id), + offset |-> + IF generation = 1 /\ id = 1 THEN 2 ELSE 0, + length |-> Len(CanonicalBytes(GenerationAtom(generation, id)))] + ELSE NoSpan] + +IdsBelow(highWater) == {id \in Ids : id < highWater} +SequenceIdSet(sequence) == + {sequence[index] : index \in 1..Len(sequence)} + +ObjectPhases == {"Absent", "Written", "Durable"} +VocabObjectIds == {"V1", "V2"} +SequenceObjectIds == {"S1", "S2"} +NoVocabObjectId == "NoVocab" +NoSequenceObjectId == "NoSequence" + +VocabObjectId(generation) == + IF generation = 1 THEN "V1" ELSE "V2" + +SequenceObjectId(generation) == + IF generation = 1 THEN "S1" ELSE "S2" + +AtomMapType == [Ids -> AtomOrNone] +PayloadMapType == [Ids -> Seq(0..255)] +SpanValueType == + [owner : AtomOrNone, offset : 0..4, length : 0..4] +SpanMapType == [Ids -> SpanValueType] + +EmptyWork == + [generation |-> 0, + fiber |-> NoFiber, + termFiber |-> NoTermFiber, + allocatorHighWater |-> 0, + liveIds |-> {}, + atomById |-> EmptyAtomMap, + payloadById |-> EmptyPayloadMap, + spanById |-> EmptySpanMap, + packedBytes |-> <<>>, + durableIds |-> {}, + durableHighWater |-> 0, + publishedHighWater |-> 0, + publishedLiveIds |-> {}, + sequenceStaged |-> FALSE, + sequenceIds |-> <<>>, + sequenceFiber |-> NoFiber, + sequenceRequiredHighWater |-> 0, + sequenceVisible |-> FALSE, + sequenceDurable |-> FALSE, + termEnabled |-> FALSE, + termId |-> 0, + termSequence |-> <<>>] + +WorkFor(generation, enableTermDictionary) == + [generation |-> generation, + fiber |-> Fiber(generation), + termFiber |-> TermFiber(generation), + allocatorHighWater |-> 2, + liveIds |-> GenerationLiveIds(generation), + atomById |-> GenerationAtomMap(generation), + payloadById |-> GenerationPayloadMap(generation), + spanById |-> GenerationSpanMap(generation), + packedBytes |-> GenerationPackedBytes(generation), + durableIds |-> {}, + durableHighWater |-> 0, + publishedHighWater |-> 0, + publishedLiveIds |-> {}, + sequenceStaged |-> FALSE, + sequenceIds |-> <<>>, + sequenceFiber |-> NoFiber, + sequenceRequiredHighWater |-> 0, + sequenceVisible |-> FALSE, + sequenceDurable |-> FALSE, + termEnabled |-> enableTermDictionary, + termId |-> 0, + termSequence |-> + IF enableTermDictionary THEN GenerationSequence(generation) ELSE <<>>] + +CompletedTermWorkFor(generation) == + [WorkFor(generation, TRUE) EXCEPT + !.durableIds = GenerationLiveIds(generation), + !.durableHighWater = 2, + !.publishedHighWater = 2, + !.publishedLiveIds = GenerationLiveIds(generation), + !.sequenceStaged = TRUE, + !.sequenceIds = GenerationSequence(generation), + !.sequenceFiber = Fiber(generation), + !.sequenceRequiredHighWater = 2, + !.sequenceVisible = TRUE, + !.sequenceDurable = TRUE] + +WorkType == + [generation : 0..2, + fiber : FiberOrNone, + termFiber : TermFiberOrNone, + allocatorHighWater : 0..2, + liveIds : SUBSET Ids, + atomById : AtomMapType, + payloadById : PayloadMapType, + spanById : SpanMapType, + packedBytes : Seq(0..255), + durableIds : SUBSET Ids, + durableHighWater : 0..2, + publishedHighWater : 0..2, + publishedLiveIds : SUBSET Ids, + sequenceStaged : BOOLEAN, + sequenceIds : Seq(Ids), + sequenceFiber : FiberOrNone, + sequenceRequiredHighWater : 0..2, + sequenceVisible : BOOLEAN, + sequenceDurable : BOOLEAN, + termEnabled : BOOLEAN, + termId : 0..2, + termSequence : Seq(Ids)] + +EmptyVocabObject == + [present |-> FALSE, + phase |-> "Absent", + generation |-> 0, + fiber |-> NoFiber, + allocatorHighWater |-> 0, + liveIds |-> {}, + atomById |-> EmptyAtomMap, + payloadById |-> EmptyPayloadMap, + spanById |-> EmptySpanMap, + packedBytes |-> <<>>] + +VocabObjectFromWork(phase, working) == + [present |-> TRUE, + phase |-> phase, + generation |-> working.generation, + fiber |-> working.fiber, + allocatorHighWater |-> working.publishedHighWater, + liveIds |-> working.publishedLiveIds, + atomById |-> working.atomById, + payloadById |-> working.payloadById, + spanById |-> working.spanById, + packedBytes |-> working.packedBytes] + +VocabObjectType == + [present : BOOLEAN, + phase : ObjectPhases, + generation : 0..2, + fiber : FiberOrNone, + allocatorHighWater : 0..2, + liveIds : SUBSET Ids, + atomById : AtomMapType, + payloadById : PayloadMapType, + spanById : SpanMapType, + packedBytes : Seq(0..255)] + +EmptySequenceObject == + [present |-> FALSE, + phase |-> "Absent", + generation |-> 0, + fiber |-> NoFiber, + termFiber |-> NoTermFiber, + requiredHighWater |-> 0, + ids |-> <<>>, + termEnabled |-> FALSE, + termId |-> 0, + termSequence |-> <<>>] + +SequenceObjectFromWork(phase, working) == + [present |-> TRUE, + phase |-> phase, + generation |-> working.generation, + fiber |-> working.sequenceFiber, + termFiber |-> working.termFiber, + requiredHighWater |-> working.sequenceRequiredHighWater, + ids |-> working.sequenceIds, + termEnabled |-> working.termEnabled, + termId |-> working.termId, + termSequence |-> working.termSequence] + +SequenceObjectType == + [present : BOOLEAN, + phase : ObjectPhases, + generation : 0..2, + fiber : FiberOrNone, + termFiber : TermFiberOrNone, + requiredHighWater : 0..2, + ids : Seq(Ids), + termEnabled : BOOLEAN, + termId : 0..2, + termSequence : Seq(Ids)] + +NoHead == + [present |-> FALSE, + generation |-> 0, + vocabObject |-> NoVocabObjectId, + sequenceObject |-> NoSequenceObjectId] + +HeadFor(generation) == + [present |-> TRUE, + generation |-> generation, + vocabObject |-> VocabObjectId(generation), + sequenceObject |-> SequenceObjectId(generation)] + +HeadType == + [present : BOOLEAN, + generation : 0..2, + vocabObject : VocabObjectIds \cup {NoVocabObjectId}, + sequenceObject : SequenceObjectIds \cup {NoSequenceObjectId}] + +EmptyObservation == + [present |-> FALSE, + head |-> NoHead, + vocabulary |-> EmptyVocabObject, + sequence |-> EmptySequenceObject] + +ObservationType == + [present : BOOLEAN, + head : HeadType, + vocabulary : VocabObjectType, + sequence : SequenceObjectType] + +EmptyReader == + [captured |-> FALSE, + head |-> NoHead, + initialObservation |-> EmptyObservation, + continuationSaved |-> FALSE, + resumed |-> FALSE, + resumeObservation |-> EmptyObservation] + +ReaderType == + [captured : BOOLEAN, + head : HeadType, + initialObservation : ObservationType, + continuationSaved : BOOLEAN, + resumed : BOOLEAN, + resumeObservation : ObservationType] + +ReadPackedSpan(packedBytes, span) == + SubSeq( + packedBytes, + span.offset + 1, + span.offset + span.length) + +SpansDisjoint(left, right) == + left.offset + left.length <= right.offset \/ + right.offset + right.length <= left.offset + +SpanContainsOffset(span, offset) == + /\ span.offset <= offset + /\ offset < span.offset + span.length + +ExactIdMetadata(fiber, atomMap, payloadMap, spanMap, packedBytes, id) == + /\ atomMap[id] \in Atoms + /\ payloadMap[id] = CanonicalBytes(atomMap[id]) + /\ payloadMap[id] # <<>> + /\ DescriptorCanonicalCodeword(fiber, atomMap[id], payloadMap[id]) + /\ spanMap[id].owner = atomMap[id] + /\ spanMap[id].length = Len(payloadMap[id]) + /\ 0 < spanMap[id].length + /\ spanMap[id].offset + spanMap[id].length <= Len(packedBytes) + /\ ReadPackedSpan(packedBytes, spanMap[id]) = payloadMap[id] + +ExactPackedMetadata(fiber, atomMap, payloadMap, spanMap, packedBytes, liveIds) == + /\ \A id \in liveIds : + ExactIdMetadata(fiber, atomMap, payloadMap, spanMap, packedBytes, id) + /\ \A left \in liveIds, right \in liveIds : + left # right => SpansDisjoint(spanMap[left], spanMap[right]) + /\ \A offset \in 0..(Len(packedBytes) - 1) : + Cardinality( + {id \in liveIds : SpanContainsOffset(spanMap[id], offset)}) = 1 + +ExactWorkingVocabulary(working) == + /\ working.generation \in Generations + /\ working.fiber = Fiber(working.generation) + /\ working.termFiber = TermFiber(working.generation) + /\ working.termFiber.vocabularyFiber = working.fiber + /\ working.allocatorHighWater = 2 + /\ working.liveIds = GenerationLiveIds(working.generation) + /\ working.atomById = GenerationAtomMap(working.generation) + /\ working.payloadById = GenerationPayloadMap(working.generation) + /\ working.spanById = GenerationSpanMap(working.generation) + /\ working.packedBytes = GenerationPackedBytes(working.generation) + /\ ExactPackedMetadata( + working.fiber, + working.atomById, + working.payloadById, + working.spanById, + working.packedBytes, + working.liveIds) + /\ \A id \in working.liveIds : + /\ id < working.allocatorHighWater + /\ ExactIdMetadata( + working.fiber, + working.atomById, + working.payloadById, + working.spanById, + working.packedBytes, + id) + +ExactVocabObject(object) == + /\ object.present + /\ object.generation \in Generations + /\ object.fiber = Fiber(object.generation) + /\ object.allocatorHighWater = 2 + /\ object.liveIds = GenerationLiveIds(object.generation) + /\ object.atomById = GenerationAtomMap(object.generation) + /\ object.payloadById = GenerationPayloadMap(object.generation) + /\ object.spanById = GenerationSpanMap(object.generation) + /\ object.packedBytes = GenerationPackedBytes(object.generation) + /\ ExactPackedMetadata( + object.fiber, + object.atomById, + object.payloadById, + object.spanById, + object.packedBytes, + object.liveIds) + /\ \A id \in object.liveIds : + /\ id < object.allocatorHighWater + /\ ExactIdMetadata( + object.fiber, + object.atomById, + object.payloadById, + object.spanById, + object.packedBytes, + id) + +ExactSequenceObject(object) == + /\ object.present + /\ object.generation \in Generations + /\ object.fiber = Fiber(object.generation) + /\ object.termFiber = TermFiber(object.generation) + /\ object.termFiber.vocabularyFiber = object.fiber + /\ object.requiredHighWater = 2 + /\ object.ids = GenerationSequence(object.generation) + /\ IF object.termEnabled + THEN /\ object.termId = 0 + /\ object.termSequence = object.ids + ELSE /\ object.termId = 0 + /\ object.termSequence = <<>> + +HeadCoherent(vocabularyStore, sequenceStore, candidateHead) == + /\ candidateHead.present + /\ candidateHead.generation \in Generations + /\ candidateHead.vocabObject = + VocabObjectId(candidateHead.generation) + /\ candidateHead.sequenceObject = + SequenceObjectId(candidateHead.generation) + /\ LET vocabulary == vocabularyStore[candidateHead.vocabObject] + sequence == sequenceStore[candidateHead.sequenceObject] + IN /\ vocabulary.phase = "Durable" + /\ sequence.phase = "Durable" + /\ ExactVocabObject(vocabulary) + /\ ExactSequenceObject(sequence) + /\ vocabulary.generation = candidateHead.generation + /\ sequence.generation = candidateHead.generation + /\ sequence.fiber = vocabulary.fiber + /\ sequence.requiredHighWater <= vocabulary.allocatorHighWater + /\ \A id \in SequenceIdSet(sequence.ids) : + /\ id \in vocabulary.liveIds + /\ id < sequence.requiredHighWater + /\ ExactIdMetadata( + vocabulary.fiber, + vocabulary.atomById, + vocabulary.payloadById, + vocabulary.spanById, + vocabulary.packedBytes, + id) + +ObserveHead(vocabularyStore, sequenceStore, candidateHead) == + IF ~candidateHead.present + THEN EmptyObservation + ELSE LET vocabulary == vocabularyStore[candidateHead.vocabObject] + sequence == sequenceStore[candidateHead.sequenceObject] + IN [present |-> TRUE, + head |-> candidateHead, + vocabulary |-> vocabulary, + sequence |-> sequence] + +VARIABLES work, + vocabObjects, + sequenceObjects, + head, + retainedHeads, + availableVocabObjects, + availableSequenceObjects, + corruptVocabObjects, + corruptSequenceObjects, + reader, + crashed, + recoveryAttempted, + recoveryKind, + recoveredHead + +vars == + <> + +TypeOK == + /\ work \in WorkType + /\ vocabObjects \in [VocabObjectIds -> VocabObjectType] + /\ sequenceObjects \in [SequenceObjectIds -> SequenceObjectType] + /\ head \in HeadType + /\ retainedHeads \in SUBSET HeadType + /\ availableVocabObjects \in SUBSET VocabObjectIds + /\ availableSequenceObjects \in SUBSET SequenceObjectIds + /\ corruptVocabObjects \in SUBSET VocabObjectIds + /\ corruptSequenceObjects \in SUBSET SequenceObjectIds + /\ reader \in ReaderType + /\ crashed \in BOOLEAN + /\ recoveryAttempted \in BOOLEAN + /\ recoveryKind \in {"None", "Pair", "Error", "Empty"} + /\ recoveredHead \in HeadType + +Init == + /\ work = EmptyWork + /\ vocabObjects = + [objectId \in VocabObjectIds |-> EmptyVocabObject] + /\ sequenceObjects = + [objectId \in SequenceObjectIds |-> EmptySequenceObject] + /\ head = NoHead + /\ retainedHeads = {} + /\ availableVocabObjects = {} + /\ availableSequenceObjects = {} + /\ corruptVocabObjects = {} + /\ corruptSequenceObjects = {} + /\ reader = EmptyReader + /\ crashed = FALSE + /\ recoveryAttempted = FALSE + /\ recoveryKind = "None" + /\ recoveredHead = NoHead + +TermFiberWitnessInit == + /\ work = CompletedTermWorkFor(2) + /\ vocabObjects = + [objectId \in VocabObjectIds |-> + CASE objectId = VocabObjectId(1) -> + VocabObjectFromWork("Durable", CompletedTermWorkFor(1)) + [] OTHER -> + VocabObjectFromWork("Durable", CompletedTermWorkFor(2))] + /\ sequenceObjects = + [objectId \in SequenceObjectIds |-> + CASE objectId = SequenceObjectId(1) -> + SequenceObjectFromWork("Durable", CompletedTermWorkFor(1)) + [] OTHER -> + SequenceObjectFromWork("Durable", CompletedTermWorkFor(2))] + /\ head = HeadFor(2) + /\ retainedHeads = {HeadFor(1), HeadFor(2)} + /\ availableVocabObjects = VocabObjectIds + /\ availableSequenceObjects = SequenceObjectIds + /\ corruptVocabObjects = {} + /\ corruptSequenceObjects = {} + /\ reader = EmptyReader + /\ crashed = FALSE + /\ recoveryAttempted = FALSE + /\ recoveryKind = "None" + /\ recoveredHead = NoHead + +BeginGeneration(generation, enableTermDictionary) == + /\ ~crashed + /\ generation \in Generations + /\ enableTermDictionary \in BOOLEAN + /\ IF generation = 1 + THEN work.generation = 0 + ELSE /\ head = HeadFor(1) + /\ work.generation = 1 + /\ work' = WorkFor(generation, enableTermDictionary) + /\ UNCHANGED + <> + +DurabilizeLiveId(id) == + /\ ~crashed + /\ ExactWorkingVocabulary(work) + /\ id \in work.liveIds + /\ id \notin work.durableIds + /\ ExactIdMetadata( + work.fiber, + work.atomById, work.payloadById, work.spanById, + work.packedBytes, id) + /\ work' = [work EXCEPT !.durableIds = @ \cup {id}] + /\ UNCHANGED + <> + +SealDurableVocabulary == + /\ ~crashed + /\ ExactWorkingVocabulary(work) + /\ work.durableHighWater = 0 + /\ work.liveIds \subseteq work.durableIds + /\ work' = + [work EXCEPT !.durableHighWater = work.allocatorHighWater] + /\ UNCHANGED + <> + +PublishVocabularyEligibility == + /\ ~crashed + /\ ExactWorkingVocabulary(work) + /\ work.publishedHighWater = 0 + /\ (OverclaimVocabularyFrontier \/ + work.durableHighWater = work.allocatorHighWater) + /\ work' = + [work EXCEPT + !.publishedHighWater = work.allocatorHighWater, + !.publishedLiveIds = work.liveIds] + /\ UNCHANGED + <> + +StageDependentSequence == + /\ ~crashed + /\ ExactWorkingVocabulary(work) + /\ ~work.sequenceStaged + /\ work' = + [work EXCEPT + !.sequenceStaged = TRUE, + !.sequenceIds = GenerationSequence(work.generation), + !.sequenceFiber = work.fiber, + !.sequenceRequiredHighWater = work.allocatorHighWater] + /\ UNCHANGED + <> + +PublishSequenceVisibility == + /\ ~crashed + /\ work.sequenceStaged + /\ ~work.sequenceVisible + /\ (PublishSequenceBeforeVocabulary \/ + (work.sequenceFiber = work.fiber /\ + work.sequenceRequiredHighWater <= work.publishedHighWater /\ + \A id \in SequenceIdSet(work.sequenceIds) : + id \in work.publishedLiveIds)) + /\ work' = [work EXCEPT !.sequenceVisible = TRUE] + /\ UNCHANGED + <> + +DurabilizeDependentSequence == + /\ ~crashed + /\ work.sequenceStaged + /\ ~work.sequenceDurable + /\ (PublishSequenceBeforeVocabulary \/ + (work.sequenceVisible /\ + work.sequenceFiber = work.fiber /\ + work.sequenceRequiredHighWater <= work.durableHighWater /\ + \A id \in SequenceIdSet(work.sequenceIds) : + /\ id \in work.durableIds + /\ id \in work.liveIds + /\ ExactIdMetadata( + work.fiber, + work.atomById, work.payloadById, work.spanById, + work.packedBytes, id))) + /\ work' = [work EXCEPT !.sequenceDurable = TRUE] + /\ UNCHANGED + <> + +WriteVocabularyObject == + /\ ~crashed + /\ ExactWorkingVocabulary(work) + /\ work.publishedHighWater = work.allocatorHighWater + /\ work.publishedLiveIds = work.liveIds + /\ work.liveIds \subseteq work.durableIds + /\ vocabObjects[VocabObjectId(work.generation)].phase = "Absent" + /\ vocabObjects' = + [vocabObjects EXCEPT + ![VocabObjectId(work.generation)] = + VocabObjectFromWork("Written", work)] + /\ UNCHANGED + <> + +SyncVocabularyObject == + /\ ~crashed + /\ LET objectId == VocabObjectId(work.generation) + object == vocabObjects[objectId] + IN /\ object.phase = "Written" + /\ ExactVocabObject(object) + /\ vocabObjects' = + [vocabObjects EXCEPT ![objectId].phase = "Durable"] + /\ availableVocabObjects' = + availableVocabObjects \cup {objectId} + /\ UNCHANGED + <> + +WriteSequenceObject == + /\ ~crashed + /\ work.sequenceDurable + /\ sequenceObjects[SequenceObjectId(work.generation)].phase = "Absent" + /\ (PublishSequenceBeforeVocabulary \/ + vocabObjects[VocabObjectId(work.generation)].phase = "Durable") + /\ sequenceObjects' = + [sequenceObjects EXCEPT + ![SequenceObjectId(work.generation)] = + SequenceObjectFromWork("Written", work)] + /\ UNCHANGED + <> + +SyncSequenceObject == + /\ ~crashed + /\ LET objectId == SequenceObjectId(work.generation) + object == sequenceObjects[objectId] + IN /\ object.phase = "Written" + /\ ExactSequenceObject(object) + /\ sequenceObjects' = + [sequenceObjects EXCEPT ![objectId].phase = "Durable"] + /\ availableSequenceObjects' = + availableSequenceObjects \cup {objectId} + /\ UNCHANGED + <> + +PublishCheckpointHead == + /\ ~crashed + /\ LET newHead == HeadFor(work.generation) + IN /\ HeadCoherent(vocabObjects, sequenceObjects, newHead) + /\ head' = newHead + /\ retainedHeads' = retainedHeads \cup {newHead} + /\ UNCHANGED + <> + +CaptureReader == + /\ ~crashed + /\ head = HeadFor(1) + /\ ~reader.captured + /\ reader' = + [reader EXCEPT + !.captured = TRUE, + !.head = head, + !.initialObservation = + ObserveHead(vocabObjects, sequenceObjects, head)] + /\ UNCHANGED + <> + +SaveReaderContinuation == + /\ ~crashed + /\ reader.captured + /\ ~reader.continuationSaved + /\ reader' = [reader EXCEPT !.continuationSaved = TRUE] + /\ UNCHANGED + <> + +ResumeCapturedReader == + /\ ~crashed + /\ reader.continuationSaved + /\ ~reader.resumed + /\ head = HeadFor(2) + /\ reader' = + [reader EXCEPT + !.resumed = TRUE, + !.resumeObservation = + IF AllowCrossGenerationResume + THEN ObserveHead(vocabObjects, sequenceObjects, head) + ELSE ObserveHead(vocabObjects, sequenceObjects, reader.head)] + /\ UNCHANGED + <> + +LoseHeadVocabularyArtifact == + /\ ~crashed + /\ head.present + /\ head.vocabObject \in availableVocabObjects + /\ availableVocabObjects' = + availableVocabObjects \ {head.vocabObject} + /\ UNCHANGED + <> + +LoseHeadSequenceArtifact == + /\ ~crashed + /\ head.present + /\ head.sequenceObject \in availableSequenceObjects + /\ availableSequenceObjects' = + availableSequenceObjects \ {head.sequenceObject} + /\ UNCHANGED + <> + +CorruptHeadVocabularyArtifact == + /\ ~crashed + /\ head.present + /\ head.vocabObject \notin corruptVocabObjects + /\ corruptVocabObjects' = + corruptVocabObjects \cup {head.vocabObject} + /\ UNCHANGED + <> + +CorruptHeadSequenceArtifact == + /\ ~crashed + /\ head.present + /\ head.sequenceObject \notin corruptSequenceObjects + /\ corruptSequenceObjects' = + corruptSequenceObjects \cup {head.sequenceObject} + /\ UNCHANGED + <> + +Crash == + /\ ~crashed + /\ crashed' = TRUE + /\ work' = EmptyWork + /\ recoveryAttempted' = FALSE + /\ recoveryKind' = "None" + /\ recoveredHead' = NoHead + /\ UNCHANGED + <> + +Recover == + /\ crashed + /\ ~recoveryAttempted + /\ recoveryAttempted' = TRUE + /\ LET vocabularyMissing == + head.present /\ + (head.vocabObject \notin availableVocabObjects \/ + head.vocabObject \in corruptVocabObjects) + sequenceMissing == + head.present /\ + (head.sequenceObject \notin availableSequenceObjects \/ + head.sequenceObject \in corruptSequenceObjects) + recoverable == + head.present /\ ~vocabularyMissing /\ ~sequenceMissing /\ + HeadCoherent(vocabObjects, sequenceObjects, head) + IN /\ recoveryKind' = + IF recoverable + THEN "Pair" + ELSE IF MissingVocabularyAsEmpty /\ vocabularyMissing + THEN "Empty" + ELSE "Error" + /\ recoveredHead' = IF recoverable THEN head ELSE NoHead + /\ UNCHANGED + <> + +Next == + \/ \E generation \in Generations, + enableTermDictionary \in BOOLEAN : + BeginGeneration(generation, enableTermDictionary) + \/ \E id \in Ids : DurabilizeLiveId(id) + \/ SealDurableVocabulary + \/ PublishVocabularyEligibility + \/ StageDependentSequence + \/ PublishSequenceVisibility + \/ DurabilizeDependentSequence + \/ WriteVocabularyObject + \/ SyncVocabularyObject + \/ WriteSequenceObject + \/ SyncSequenceObject + \/ PublishCheckpointHead + \/ CaptureReader + \/ SaveReaderContinuation + \/ ResumeCapturedReader + \/ LoseHeadVocabularyArtifact + \/ LoseHeadSequenceArtifact + \/ CorruptHeadVocabularyArtifact + \/ CorruptHeadSequenceArtifact + \/ Crash + \/ Recover + +VWENC_147_PUBLISHED_FRONTIER_DOES_NOT_EXCEED_DURABLE_FRONTIER == + /\ work.publishedHighWater <= work.durableHighWater + /\ work.publishedLiveIds \subseteq work.durableIds + +VWENC_148_PUBLISHED_IDS_HAVE_EXACT_DURABLE_METADATA == + ~OverclaimVocabularyFrontier => + /\ \A id \in work.publishedLiveIds : + /\ id \in work.liveIds + /\ id \in work.durableIds + /\ id < work.publishedHighWater + /\ ExactIdMetadata( + work.fiber, + work.atomById, work.payloadById, work.spanById, + work.packedBytes, id) + /\ \A objectId \in VocabObjectIds : + vocabObjects[objectId].phase # "Absent" => + ExactVocabObject(vocabObjects[objectId]) + +VWENC_149_DURABLE_SEQUENCE_REFERENCES_DURABLE_BOUND_VOCABULARY == + ~OverclaimVocabularyFrontier => + ((work.sequenceVisible \/ work.sequenceDurable) => + /\ work.sequenceStaged + /\ work.sequenceFiber = work.fiber + /\ work.sequenceRequiredHighWater <= work.publishedHighWater + /\ work.sequenceRequiredHighWater <= work.durableHighWater + /\ \A id \in SequenceIdSet(work.sequenceIds) : + /\ id \in work.publishedLiveIds + /\ id \in work.durableIds + /\ id \in work.liveIds + /\ id < work.sequenceRequiredHighWater + /\ ExactIdMetadata( + work.fiber, + work.atomById, work.payloadById, work.spanById, + work.packedBytes, id)) + +VWENC_150_SEQUENCE_OBJECT_FOLLOWS_DURABLE_VOCABULARY_OBJECT == + ~PublishSequenceBeforeVocabulary => + \A generation \in Generations : + (sequenceObjects[SequenceObjectId(generation)].phase # "Absent") => + vocabObjects[VocabObjectId(generation)].phase = "Durable" + +VWENC_151_SEQUENCE_DESCRIPTOR_BINDS_EXACT_VOCABULARY_FIBER == + ~PublishSequenceBeforeVocabulary => + \A generation \in Generations : + LET sequence == sequenceObjects[SequenceObjectId(generation)] + vocabulary == vocabObjects[VocabObjectId(generation)] + IN (sequence.phase # "Absent") => + /\ ExactSequenceObject(sequence) + /\ ExactVocabObject(vocabulary) + /\ sequence.fiber = vocabulary.fiber + /\ sequence.termFiber = TermFiber(sequence.generation) + /\ sequence.termFiber.vocabularyFiber = sequence.fiber + /\ sequence.requiredHighWater <= vocabulary.allocatorHighWater + /\ IF sequence.termEnabled + THEN /\ sequence.termId = 0 + /\ sequence.termSequence = sequence.ids + ELSE /\ sequence.termId = 0 + /\ sequence.termSequence = <<>> + +VWENC_152_HEAD_BINDS_ONE_COHERENT_DURABLE_PAIR == + (~PublishSequenceBeforeVocabulary) => + (head.present => + /\ HeadCoherent(vocabObjects, sequenceObjects, head) + /\ head \in retainedHeads) + +VWENC_153_RECOVERY_IS_COHERENT_OLD_NEW_OR_ERROR == + (~MissingVocabularyAsEmpty) => + (recoveryAttempted => + \/ recoveryKind = "Error" /\ recoveredHead = NoHead + \/ recoveryKind = "Pair" /\ + recoveredHead = head /\ + HeadCoherent(vocabObjects, sequenceObjects, recoveredHead) /\ + recoveredHead \in retainedHeads) + +VWENC_154_CAPTURED_CONTINUATION_RESUMES_IMMUTABLE_PAIR == + reader.resumed => + /\ reader.head \in retainedHeads + /\ HeadCoherent(vocabObjects, sequenceObjects, reader.head) + /\ reader.resumeObservation = reader.initialObservation + /\ reader.resumeObservation = + ObserveHead(vocabObjects, sequenceObjects, reader.head) + +VWENC_155_UNAVAILABLE_OR_CORRUPT_HEAD_ARTIFACT_IS_EXPLICIT_ERROR == + (~MissingVocabularyAsEmpty) => + ((recoveryAttempted /\ head.present /\ + (head.vocabObject \notin availableVocabObjects \/ + head.vocabObject \in corruptVocabObjects \/ + head.sequenceObject \notin availableSequenceObjects \/ + head.sequenceObject \in corruptSequenceObjects)) + => recoveryKind = "Error" /\ recoveredHead = NoHead) + +VWENC_156_PUBLISHED_HEAD_HAS_NO_DANGLING_ID_REFERENCE == + (~OverclaimVocabularyFrontier /\ ~PublishSequenceBeforeVocabulary) => + (head.present => + LET vocabulary == vocabObjects[head.vocabObject] + sequence == sequenceObjects[head.sequenceObject] + IN \A id \in SequenceIdSet(sequence.ids) : + /\ id \in vocabulary.liveIds + /\ id < sequence.requiredHighWater + /\ ExactIdMetadata( + vocabulary.fiber, + vocabulary.atomById, + vocabulary.payloadById, + vocabulary.spanById, + vocabulary.packedBytes, + id)) + +VWENC_178_RECOVERY_NEVER_SYNTHESIZES_EMPTY_SUCCESS == + recoveryAttempted => recoveryKind # "Empty" + +VWENC_179_EXACT_TERM_FIBER_SEPARATES_SAME_RAW_ID == + /\ \A generation \in Generations : + LET sequence == sequenceObjects[SequenceObjectId(generation)] + IN sequence.phase # "Absent" /\ sequence.termEnabled => + /\ sequence.termFiber = TermFiber(generation) + /\ sequence.termFiber.vocabularyFiber = sequence.fiber + /\ sequence.termId = 0 + /\ sequence.termSequence = sequence.ids + /\ \A left \in Generations, right \in Generations : + LET leftSequence == sequenceObjects[SequenceObjectId(left)] + rightSequence == sequenceObjects[SequenceObjectId(right)] + IN left # right /\ + leftSequence.phase # "Absent" /\ leftSequence.termEnabled /\ + rightSequence.phase # "Absent" /\ rightSequence.termEnabled => + /\ leftSequence.termId = rightSequence.termId + /\ leftSequence.termFiber # rightSequence.termFiber + +VWENC_193_TWO_GENERATION_TERM_FIBER_WITNESS_IS_CONCRETE == + LET firstVocabulary == vocabObjects[VocabObjectId(1)] + secondVocabulary == vocabObjects[VocabObjectId(2)] + firstSequence == sequenceObjects[SequenceObjectId(1)] + secondSequence == sequenceObjects[SequenceObjectId(2)] + IN /\ firstVocabulary.phase = "Durable" + /\ secondVocabulary.phase = "Durable" + /\ ExactVocabObject(firstVocabulary) + /\ ExactVocabObject(secondVocabulary) + /\ firstSequence.phase = "Durable" + /\ secondSequence.phase = "Durable" + /\ ExactSequenceObject(firstSequence) + /\ ExactSequenceObject(secondSequence) + /\ firstSequence.termEnabled + /\ secondSequence.termEnabled + /\ firstSequence.termId = 0 + /\ secondSequence.termId = 0 + /\ firstSequence.termId = secondSequence.termId + /\ firstSequence.termSequence = GenerationSequence(1) + /\ secondSequence.termSequence = GenerationSequence(2) + /\ firstSequence.termFiber = TermFiber(1) + /\ secondSequence.termFiber = TermFiber(2) + /\ firstSequence.termFiber # secondSequence.termFiber + /\ firstSequence.termFiber.vocabularyFiber = firstSequence.fiber + /\ secondSequence.termFiber.vocabularyFiber = secondSequence.fiber + /\ firstSequence.fiber = firstVocabulary.fiber + /\ secondSequence.fiber = secondVocabulary.fiber + /\ HeadCoherent(vocabObjects, sequenceObjects, HeadFor(1)) + /\ HeadCoherent(vocabObjects, sequenceObjects, HeadFor(2)) + /\ HeadFor(1) \in retainedHeads + /\ HeadFor(2) \in retainedHeads + /\ head = HeadFor(2) + +Spec == Init /\ [][Next]_vars + +============================================================================= diff --git a/formal-verification/tla+/VariableWidthVocabularyPublication_CrossGenerationResumeUnsafe.cfg b/formal-verification/tla+/VariableWidthVocabularyPublication_CrossGenerationResumeUnsafe.cfg new file mode 100644 index 00000000..0949e763 --- /dev/null +++ b/formal-verification/tla+/VariableWidthVocabularyPublication_CrossGenerationResumeUnsafe.cfg @@ -0,0 +1,13 @@ +CONSTANTS + PublishSequenceBeforeVocabulary = FALSE + OverclaimVocabularyFrontier = FALSE + AllowCrossGenerationResume = TRUE + MissingVocabularyAsEmpty = FALSE + +SPECIFICATION Spec + +CHECK_DEADLOCK FALSE + +INVARIANT + TypeOK + VWENC_154_CAPTURED_CONTINUATION_RESUMES_IMMUTABLE_PAIR diff --git a/formal-verification/tla+/VariableWidthVocabularyPublication_FrontierOverclaimUnsafe.cfg b/formal-verification/tla+/VariableWidthVocabularyPublication_FrontierOverclaimUnsafe.cfg new file mode 100644 index 00000000..efcee186 --- /dev/null +++ b/formal-verification/tla+/VariableWidthVocabularyPublication_FrontierOverclaimUnsafe.cfg @@ -0,0 +1,13 @@ +CONSTANTS + PublishSequenceBeforeVocabulary = FALSE + OverclaimVocabularyFrontier = TRUE + AllowCrossGenerationResume = FALSE + MissingVocabularyAsEmpty = FALSE + +SPECIFICATION Spec + +CHECK_DEADLOCK FALSE + +INVARIANT + TypeOK + VWENC_147_PUBLISHED_FRONTIER_DOES_NOT_EXCEED_DURABLE_FRONTIER diff --git a/formal-verification/tla+/VariableWidthVocabularyPublication_MissingVocabularyAsEmptyUnsafe.cfg b/formal-verification/tla+/VariableWidthVocabularyPublication_MissingVocabularyAsEmptyUnsafe.cfg new file mode 100644 index 00000000..4a7c57ee --- /dev/null +++ b/formal-verification/tla+/VariableWidthVocabularyPublication_MissingVocabularyAsEmptyUnsafe.cfg @@ -0,0 +1,13 @@ +CONSTANTS + PublishSequenceBeforeVocabulary = FALSE + OverclaimVocabularyFrontier = FALSE + AllowCrossGenerationResume = FALSE + MissingVocabularyAsEmpty = TRUE + +SPECIFICATION Spec + +CHECK_DEADLOCK FALSE + +INVARIANT + TypeOK + VWENC_178_RECOVERY_NEVER_SYNTHESIZES_EMPTY_SUCCESS diff --git a/formal-verification/tla+/VariableWidthVocabularyPublication_SequenceBeforeVocabularyUnsafe.cfg b/formal-verification/tla+/VariableWidthVocabularyPublication_SequenceBeforeVocabularyUnsafe.cfg new file mode 100644 index 00000000..b5d35f87 --- /dev/null +++ b/formal-verification/tla+/VariableWidthVocabularyPublication_SequenceBeforeVocabularyUnsafe.cfg @@ -0,0 +1,13 @@ +CONSTANTS + PublishSequenceBeforeVocabulary = TRUE + OverclaimVocabularyFrontier = FALSE + AllowCrossGenerationResume = FALSE + MissingVocabularyAsEmpty = FALSE + +SPECIFICATION Spec + +CHECK_DEADLOCK FALSE + +INVARIANT + TypeOK + VWENC_149_DURABLE_SEQUENCE_REFERENCES_DURABLE_BOUND_VOCABULARY diff --git a/formal-verification/tla+/VariableWidthVocabularyPublication_TermFiberWitness.cfg b/formal-verification/tla+/VariableWidthVocabularyPublication_TermFiberWitness.cfg new file mode 100644 index 00000000..68ad8563 --- /dev/null +++ b/formal-verification/tla+/VariableWidthVocabularyPublication_TermFiberWitness.cfg @@ -0,0 +1,15 @@ +CONSTANTS + PublishSequenceBeforeVocabulary = FALSE + OverclaimVocabularyFrontier = FALSE + AllowCrossGenerationResume = FALSE + MissingVocabularyAsEmpty = FALSE + +INIT TermFiberWitnessInit +NEXT Next + +CHECK_DEADLOCK FALSE + +INVARIANT + TypeOK + VWENC_179_EXACT_TERM_FIBER_SEPARATES_SAME_RAW_ID + VWENC_193_TWO_GENERATION_TERM_FIBER_WITNESS_IS_CONCRETE diff --git a/formal-verification/variable-width-interning-correspondence.tsv b/formal-verification/variable-width-interning-correspondence.tsv new file mode 100644 index 00000000..f21cb6da --- /dev/null +++ b/formal-verification/variable-width-interning-correspondence.tsv @@ -0,0 +1,46 @@ +PointCertifiedAtomProfile|src/profile/mod.rs|DictionaryProfile|Prospective|ObligationAddCertifiedProfileSurface +PointSymbolIdCarrierCodec|src/profile/interned/id.rs|SymbolId::{try_from_nat,encode,decode}|Prospective|ObligationAddSymbolIdCarrierCodec +PointTermIdCarrierCodec|src/profile/interned/id.rs|TermId::{try_from_nat,encode,decode}|Prospective|ObligationAddTermIdCarrierCodec +PointForwardAtomToId|src/persistent_artrie/vocab/mutation_api.rs|PersistentVocabARTrie::insert|CommonSubstrateOnly|ObligationGeneralizeForwardVocabulary +PointReverseIdToAtom|src/persistent_artrie/vocab/query_api.rs|PersistentVocabARTrie::get_term|CommonSubstrateOnly|ObligationGeneralizeReverseVocabulary +PointForwardReverseBijection|src/persistent_artrie/vocab/dict_impl.rs|PersistentVocabARTrie::reverse_term_map|Conflicts|ObligationCoordinateBijectionVisibility +PointAllocationStatus|src/profile/interned/allocation.rs|AllocationStatus|Prospective|ObligationAddAllocationLedger +PointClaimAllocation|src/persistent_artrie/vocab/mutation_api.rs|PersistentVocabARTrie::insert_overlay|CommonSubstrateOnly|ObligationReuseSparseAllocationClaim +PointOrphanAllocation|src/persistent_artrie/vocab/mutation_api.rs|PersistentVocabARTrie::insert_overlay|CommonSubstrateOnly|ObligationRetainOrphanedIds +PointTombstoneNoReuse|src/profile/interned/allocation.rs|AllocationLedger::tombstone|Prospective|ObligationAddNoReuseTombstones +PointPackedCanonicalStorage|src/profile/interned/storage.rs|PackedAtomStorage|Prospective|ObligationAddPackedStorage +PointSparseAllocatorFrontierStorage|src/persistent_artrie/vocab/dict_impl.rs|PersistentVocabARTrie::next_index|CommonSubstrateOnly|ObligationPreserveSparseFrontierStorage +PointSparseAllocatorFrontierAccess|src/persistent_artrie/vocab/query_api.rs|PersistentVocabARTrie::next_index|CommonSubstrateOnly|ObligationExposeSparseFrontier +PointVocabularyFiberHeader|src/persistent_artrie/vocab/types.rs|VocabTrieFileHeader|Conflicts|ObligationAddProfileGenerationHeader +PointIdSequenceView|src/profile/interned/view.rs|IdSequenceView|Prospective|ObligationAddBorrowedFiberBoundView +PointSequenceDescriptorLiveMembership|src/profile/interned/descriptor.rs|SequenceDescriptor::validate_live_ids|Prospective|ObligationAddSequenceDescriptorValidation +PointOptionalTermDictionary|src/profile/interned/term_dictionary.rs|TermSequenceDictionary|Prospective|ObligationAddOptionalTermDictionary +PointCoordinatedSequenceOwner|src/profile/interned/coordinator.rs|InternedSequenceDictionary|Prospective|ObligationAddCoordinatedOwner +PointQueryLocalOverlay|src/profile/interned/query.rs|QueryOverlay|Prospective|ObligationAddEphemeralOverlay +PointImmutableSnapshot|src/persistent_artrie/vocab/mutation_api.rs|PersistentVocabARTrie::snapshot|CommonSubstrateOnly|ObligationAuditSnapshotRefinement +PointBeginGeneration|src/profile/interned/coordinator.rs|InternedSequenceDictionary::begin_generation|Prospective|ObligationAddGenerationStaging +PointDurabilizeLiveId|src/profile/interned/persistence.rs|InternedVocabularyWriter::durabilize_live_id|Prospective|ObligationAddDurablePackedPublication +PointSealDurableVocabulary|src/profile/interned/persistence.rs|InternedVocabularyWriter::seal_vocabulary|Prospective|ObligationAddVocabularySeal +PointPublishVocabularyEligibility|src/profile/interned/persistence.rs|InternedVocabularyWriter::publish_frontier|Prospective|ObligationAddVocabularyEligibilityPublication +PointStageDependentSequence|src/profile/interned/coordinator.rs|InternedSequenceDictionary::stage_sequence|Prospective|ObligationAddSequenceStaging +PointPublishSequenceVisibility|src/profile/interned/coordinator.rs|InternedSequenceDictionary::publish_sequence|Prospective|ObligationAddSequenceVisibilityPublication +PointDurabilizeDependentSequence|src/profile/interned/persistence.rs|InternedSequenceWriter::durabilize_sequence|Prospective|ObligationAddSequenceDurabilityPublication +PointWriteVocabularyObject|src/persistent_artrie/vocab/persistence_api.rs|PersistentVocabARTrie::checkpoint_overlay|CommonSubstrateOnly|ObligationReuseVocabularyObjectWrite +PointSyncVocabularyObject|src/persistent_artrie/vocab/persistence_api.rs|PersistentVocabARTrie::checkpoint_overlay|CommonSubstrateOnly|ObligationReuseVocabularyObjectSync +PointWriteSequenceObject|src/persistent_artrie/u64.rs|write_snapshot_file|CommonSubstrateOnly|ObligationReuseSequenceObjectWrite +PointSyncSequenceObject|src/persistent_artrie/u64.rs|write_snapshot_file|CommonSubstrateOnly|ObligationReuseSequenceObjectSync +PointAtomicCheckpointHeadPublication|src/profile/interned/persistence.rs|InternedSequenceDictionary::publish_checkpoint_head|Prospective|ObligationAddAtomicCheckpointHead +PointCaptureReader|src/profile/interned/snapshot.rs|InternedSequenceDictionary::snapshot|Prospective|ObligationAddReaderCapture +PointSaveReaderContinuation|src/profile/interned/continuation.rs|InternedReadCursor::continuation|Prospective|ObligationAddContinuationCapture +PointResumeReaderContinuation|src/profile/interned/continuation.rs|InternedSequenceDictionary::resume|Prospective|ObligationAddContinuationResume +PointLoseVocabularyArtifact|src/profile/interned/recovery.rs|InternedRecoveryError::MissingVocabulary|Prospective|ObligationAddVocabularyLossRecoveryCase +PointLoseSequenceArtifact|src/profile/interned/recovery.rs|InternedRecoveryError::MissingSequence|Prospective|ObligationAddSequenceLossRecoveryCase +PointCorruptVocabularyArtifact|src/profile/interned/recovery.rs|InternedRecoveryError::CorruptVocabulary|Prospective|ObligationAddVocabularyCorruptionRecoveryCase +PointCorruptSequenceArtifact|src/profile/interned/recovery.rs|InternedRecoveryError::CorruptSequence|Prospective|ObligationAddSequenceCorruptionRecoveryCase +PointCrashTransition|src/profile/interned/recovery.rs|InternedRecoveryState|Prospective|ObligationAddCrashStateTransition +PointStrictPairRecovery|src/profile/interned/recovery.rs|InternedSequenceDictionary::recover|Prospective|ObligationAddExactOldNewRecovery +PointCommitSequenceSubstrate|src/persistent_artrie/vocab/dict_impl.rs|PersistentVocabARTrie::commit_seq|CommonSubstrateOnly|ObligationReuseCommitSequence +PointCommittedWatermarkSubstrate|src/persistent_artrie/vocab/dict_impl.rs|PersistentVocabARTrie::committed_watermark|CommonSubstrateOnly|ObligationReuseCommittedWatermark +PointDurableOverlayInsertionSubstrate|src/persistent_artrie/core/overlay/durable_write.rs|DurableOverlayWrite::insert_cas_with_value_durable_default|CommonSubstrateOnly|ObligationReuseDurableOverlayInsertion +PointCheckpointLockSubstrate|src/persistent_artrie/vocab/dict_impl.rs|PersistentVocabARTrie::checkpoint_lock|CommonSubstrateOnly|ObligationReuseCheckpointLock +PointHeaderCheckpointPublicationSubstrate|src/persistent_artrie/vocab/persistence_api.rs|PersistentVocabARTrie::checkpoint_overlay|CommonSubstrateOnly|ObligationExtendHeaderCheckpointPublication diff --git a/scripts/extract-variable-width-conformance-ledger.py b/scripts/extract-variable-width-conformance-ledger.py new file mode 100644 index 00000000..a8c61c18 --- /dev/null +++ b/scripts/extract-variable-width-conformance-ledger.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +"""Join formal declarations, executable tests, and TLC negative controls. + +Coverage is derived only from registered names and control bindings. The +output deliberately marks uncovered laws instead of silently treating them as +verified. +""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +from pathlib import Path + + +APPLICABILITY = { + "codec": "all logical-unit codecs and boundary adapters", + "interning": "interned vocabulary and coordinated ID-sequence profiles", + "family_refinement": "all dictionary families and applicable profile specializations", +} + +STACK_SAFETY = "iterative-or-heap-backed traversal; no library workload budget" +PERFORMANCE = { + "codec": "linear in consumed input", + "interning": "linear in atoms plus vocabulary/index operations", + "family_refinement": "linear in logical units plus visited result structure", +} +ACCEPTANCE_COMMAND = "scripts/verify-variable-width-formal.sh" +PUBLIC_SURFACE = { + "codec": "src/variable_width.rs; src/profile.rs; src/factory.rs", + "interning": "src/interning.rs; src/variable_width.rs", + "family_refinement": ( + "src/dynamic_dawg; src/double_array_trie; src/pathmap; " + "src/persistent_artrie; src/scdawg; src/suffix_automaton; src/factory.rs" + ), +} + + +def plain_language_law(identifier: str) -> str: + """Provide a lossless, deterministic human-readable law label. + + The formal declaration remains authoritative for semantics. This label is + deliberately mechanical so the ledger never invents prose that could + diverge from the checked theorem or assertion. + """ + + parts = identifier.split("_", 2) + suffix = parts[2] if len(parts) == 3 else identifier + return suffix.replace("_", " ").lower() + + +def load_module(path: Path, name: str): + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise SystemExit(f"cannot load extractor: {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def build(root: Path) -> list[dict[str, object]]: + formal = load_module( + root / "scripts/extract-variable-width-formal-inventory.py", "vw_formal" + ).declarations(root) + tests = load_module( + root / "scripts/extract-variable-width-test-inventory.py", "vw_tests" + ).registrations(root) + by_number: dict[int, list[dict[str, object]]] = {} + for test in tests: + for number in test["numeric_ids"]: + by_number.setdefault(number, []).append(test) + rows = [] + for declaration in formal: + positive = sorted( + { + test["registration"] + for test in by_number.get(declaration["numeric_id"], []) + } + ) + controls = declaration["negative_controls"] + coverage = ( + "positive_and_negative" + if positive and controls + else "positive_only" + if positive + else "negative_only" + if controls + else "uncovered" + ) + rows.append( + { + "id": declaration["id"], + "numeric_id": declaration["numeric_id"], + "semantic_area": declaration["semantic_area"], + "owner_repository": "libdictenstein", + "owner_layer": declaration["semantic_area"], + "applicability": APPLICABILITY[declaration["semantic_area"]], + "kind": declaration["kind"], + "language": declaration["language"], + "formal_source": declaration["source"], + "formal_artifact": declaration["source_path"], + "proof_kind": "Rocq_proposition" if declaration["language"] == "rocq" else "TLA_assertion", + "declaration": declaration["declaration"], + "plain_language_law": plain_language_law(declaration["id"]), + "current_target_public_surface": PUBLIC_SURFACE[declaration["semantic_area"]], + "positive_tests": positive, + "differential_oracle_ids": [], + "negative_controls": controls, + "required_mutant_control_ids": controls, + "assumptions": "finite input; explicit profile metadata", + "trust_boundary": f"{declaration['semantic_area']}: formal model to implementation boundary", + "stack_safety": STACK_SAFETY, + "performance": PERFORMANCE[declaration["semantic_area"]], + "acceptance_command": ACCEPTANCE_COMMAND, + "evidence_artifact": declaration["source_path"], + "coverage": coverage, + "proof_only_exception": ( + f"proof-only:{declaration['id']}" + if coverage == "positive_only" + else None + ), + "proof_only_rationale": ( + ( + f"Universal {('Rocq proposition' if declaration['language'] == 'rocq' else 'TLA assertion')} " + f"verified from {declaration['source_path']}; implementation boundary " + f"is exercised by positive registration(s) {', '.join(positive)}; " + "no finite mutant control is applicable to this model-level law." + ) + if coverage == "positive_only" + else None + ), + # Positive executable coverage and mutant-control coverage are + # independent evidence dimensions. Keep both visible instead + # of treating positive-only properties as untested. + "status": ( + "registered-with-negative-control" + if coverage == "positive_and_negative" + else "registered-positive-only" + if coverage == "positive_only" + else "negative-control-only" + if coverage == "negative_only" + else "uncovered" + ), + } + ) + return rows + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[1]) + parser.add_argument("--output", type=Path) + args = parser.parse_args() + payload = json.dumps(build(args.root), indent=2, sort_keys=True) + "\n" + if args.output: + args.output.write_text(payload, encoding="utf-8") + else: + print(payload, end="") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/extract-variable-width-formal-inventory.py b/scripts/extract-variable-width-formal-inventory.py new file mode 100644 index 00000000..cd192bbd --- /dev/null +++ b/scripts/extract-variable-width-formal-inventory.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +"""Extract the authoritative VWENC declaration inventory deterministically. + +This is deliberately an inventory extractor, not an implementation ledger: it +records only facts present in the checked Rocq/TLA+ sources and refuses to +silently merge duplicate identifiers. A conformance ledger may consume the +JSON output and must supply the remaining test/oracle/control columns. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +from pathlib import Path + +ROCQ_RE = re.compile(r"^(Theorem|Lemma|Corollary)\s+(VWENC_[A-Za-z0-9_]+)") +TLA_RE = re.compile(r"^(VWENC_[A-Za-z0-9_]+)\s*==") +CFG_RE = re.compile(r"^\s*(VWENC_[A-Za-z0-9_]+)\s*$") +ID_RE = re.compile(r"^VWENC_(\d+)_") +SOURCE_AREAS = { + "VariableWidthCodecSpec.v": "codec", + "VariableWidthCodecBoundary.tla": "codec", + "VariableWidthInterningSpec.v": "interning", + "VariableWidthVocabularyInterning.tla": "interning", + "VariableWidthVocabularyPublication.tla": "interning", + "VariableWidthFamilyRefinementSpec.v": "family_refinement", + "VariableWidthFamilyRefinement.tla": "family_refinement", +} + + +def semantic_area(source: Path) -> str: + try: + return SOURCE_AREAS[source.name] + except KeyError as error: + raise ValueError(f"unclassified variable-width formal source: {source}") from error + + +def declarations(root: Path) -> list[dict[str, object]]: + sources = [ + *sorted((root / "formal-verification/rocq/Spec").glob("VariableWidth*.v")), + root / "formal-verification/tla+/VariableWidthCodecBoundary.tla", + root / "formal-verification/tla+/VariableWidthVocabularyInterning.tla", + root / "formal-verification/tla+/VariableWidthVocabularyPublication.tla", + root / "formal-verification/tla+/VariableWidthFamilyRefinement.tla", + ] + found: dict[str, dict[str, object]] = {} + duplicates: dict[str, list[str]] = {} + duplicate_numbers: dict[int, list[str]] = {} + for source in sources: + language = "rocq" if source.suffix == ".v" else "tla" + pattern = ROCQ_RE if language == "rocq" else TLA_RE + source_digest = hashlib.sha256(source.read_bytes()).hexdigest() + for line_number, line in enumerate(source.read_text(encoding="utf-8").splitlines(), 1): + match = pattern.match(line) + if not match: + continue + identifier = match.group(2) if language == "rocq" else match.group(1) + numeric_match = ID_RE.match(identifier) + if numeric_match is None: + raise SystemExit(f"malformed VWENC identifier: {identifier}") + numeric_id = int(numeric_match.group(1)) + location = f"{source.relative_to(root)}:{line_number}" + row = { + "id": identifier, + "numeric_id": numeric_id, + "kind": match.group(1) if language == "rocq" else "TLA_assertion", + "language": language, + "semantic_area": semantic_area(source), + "source": location, + "source_path": str(source.relative_to(root)), + "source_line": line_number, + "source_sha256": source_digest, + "declaration": line.strip(), + "negative_controls": [], + } + if identifier in found: + duplicates.setdefault(identifier, [str(found[identifier]["source"])]) + duplicates[identifier].append(location) + else: + found[identifier] = row + duplicate_numbers.setdefault(numeric_id, []).append(identifier) + if duplicates: + details = "; ".join( + f"{identifier}: {', '.join(locations)}" + for identifier, locations in sorted(duplicates.items()) + ) + raise SystemExit(f"duplicate VWENC declarations: {details}") + colliding_numbers = { + number: sorted(set(identifiers)) + for number, identifiers in duplicate_numbers.items() + if len(set(identifiers)) > 1 + } + if colliding_numbers: + details = "; ".join( + f"{number}: {', '.join(identifiers)}" + for number, identifiers in sorted(colliding_numbers.items()) + ) + raise SystemExit(f"duplicate VWENC numeric identifiers: {details}") + controls: dict[str, list[str]] = {} + for config in sorted((root / "formal-verification/tla+").glob("VariableWidth*Unsafe.cfg")): + for line in config.read_text(encoding="utf-8").splitlines(): + match = CFG_RE.match(line) + if match: + controls.setdefault(match.group(1), []).append(str(config.relative_to(root))) + orphan_controls = sorted(set(controls) - set(found)) + if orphan_controls: + raise SystemExit( + "negative controls reference undeclared VWENC identifiers: " + + ", ".join(orphan_controls) + ) + for identifier, row in found.items(): + row["negative_controls"] = sorted(controls.get(identifier, [])) + + rows = [] + for identifier, row in sorted(found.items(), key=lambda item: (int(ID_RE.match(item[0]).group(1)), item[0])): + rows.append(row) + return rows + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[1]) + parser.add_argument("--output", type=Path) + args = parser.parse_args() + rows = declarations(args.root) + payload = json.dumps(rows, indent=2, sort_keys=True) + "\n" + if args.output: + args.output.write_text(payload, encoding="utf-8") + else: + print(payload, end="") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/extract-variable-width-test-inventory.py b/scripts/extract-variable-width-test-inventory.py new file mode 100644 index 00000000..76b60e56 --- /dev/null +++ b/scripts/extract-variable-width-test-inventory.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Extract registered executable VWENC test names and source locations. + +The inventory is intentionally a coverage index, not a claim that every formal +law already has a test. It rejects duplicate registrations and registrations +for identifiers absent from the authoritative formal inventory. +""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +from pathlib import Path + +TEST_RE = re.compile( + r"^\s*fn\s+(vwenc_(?:\d+)(?:_to_(?:\d+))?_[A-Za-z0-9_]+)\s*\(" +) +ID_RE = re.compile(r"^vwenc_(\d+)(?:_to_(\d+))?_") + + +def formal_numbers(root: Path) -> set[int]: + extractor = root / "scripts/extract-variable-width-formal-inventory.py" + result = subprocess.run( + [sys.executable, str(extractor), "--root", str(root)], + check=True, + capture_output=True, + text=True, + ) + return {int(row["numeric_id"]) for row in json.loads(result.stdout)} + + +def registrations(root: Path) -> list[dict[str, object]]: + known = formal_numbers(root) + rows: list[dict[str, object]] = [] + seen: set[str] = set() + for source in sorted((root / "tests").rglob("*.rs")): + for line_number, line in enumerate(source.read_text(encoding="utf-8").splitlines(), 1): + match = TEST_RE.match(line) + if not match: + continue + registration = match.group(1) + if registration in seen: + raise SystemExit(f"duplicate VWENC test registration: {registration}") + seen.add(registration) + id_match = ID_RE.match(registration) + assert id_match is not None + first = int(id_match.group(1)) + last = int(id_match.group(2) or first) + numbers = list(range(first, last + 1)) + unknown = sorted(set(numbers) - known) + if unknown: + raise SystemExit( + f"VWENC test {registration} references undeclared identifiers: " + + ", ".join(map(str, unknown)) + ) + rows.append( + { + "registration": registration, + "source": f"{source.relative_to(root)}:{line_number}", + "source_path": str(source.relative_to(root)), + "source_line": line_number, + "numeric_ids": numbers, + } + ) + return rows + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[1]) + parser.add_argument("--output", type=Path) + args = parser.parse_args() + payload = json.dumps(registrations(args.root), indent=2, sort_keys=True) + "\n" + if args.output: + args.output.write_text(payload, encoding="utf-8") + else: + print(payload, end="") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/verify-char-node-format-compatibility.sh b/scripts/verify-char-node-format-compatibility.sh index 99cb91f3..19e89473 100755 --- a/scripts/verify-char-node-format-compatibility.sh +++ b/scripts/verify-char-node-format-compatibility.sh @@ -6,7 +6,15 @@ scratch="$repo_root/target/char-format-compat" baseline_commit="6a1b267a60fe9c445a0c8c7c8136e6dd40aedbf5" interop_commit="6694ad4fcb5ce498f69b77cb14ce1ea7a2f20033" llattice_commit="2ec21ca70ae3cbb2d8afdd295c9ed09517003324" -interop_repo="${VINARY_TREE_INTEROP_REPO:-$repo_root/../vinary-tree-interop-rc2-stack-safety-clean}" +if [ -n "${VINARY_TREE_INTEROP_REPO:-}" ]; then + interop_repo="$VINARY_TREE_INTEROP_REPO" +elif [ -d "$repo_root/../vinary-tree-interop-rc2-stack-safety-clean/.git" ]; then + # Preserve the historical local stack-safety fixture when it is present. + interop_repo="$repo_root/../vinary-tree-interop-rc2-stack-safety-clean" +else + # CI and ordinary development checkouts use the canonical sibling name. + interop_repo="$repo_root/../vinary-tree-interop" +fi llattice_repo="${LLATTICE_REPO:-$repo_root/../llattice}" fixture_dir="$repo_root/tests/fixtures/char-node-format" manifest="$fixture_dir/manifest.toml" @@ -30,7 +38,13 @@ require_commit() { local repository="$1" local expected="$2" local actual - actual="$(git -C "$repository" rev-parse "${expected}^{commit}")" + # CI checkouts are intentionally shallow. Fetch the immutable fixture + # revision on demand so the compatibility proof does not depend on an + # incidental checkout depth while still refusing any substituted object. + if ! actual="$(git -C "$repository" rev-parse --verify "${expected}^{commit}" 2>/dev/null)"; then + git -C "$repository" fetch --no-tags --depth=1 origin "$expected" + actual="$(git -C "$repository" rev-parse --verify "${expected}^{commit}")" + fi if [ "$actual" != "$expected" ]; then echo "commit mismatch for $repository: expected $expected, found $actual" >&2 exit 1 diff --git a/scripts/verify-formal-correspondence.sh b/scripts/verify-formal-correspondence.sh index 6c3bd810..08f989d0 100755 --- a/scripts/verify-formal-correspondence.sh +++ b/scripts/verify-formal-correspondence.sh @@ -5,14 +5,16 @@ repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$repo_root" FORMAL_RESOURCE_CONTROL="${FORMAL_RESOURCE_CONTROL:-systemd}" -FORMAL_MEMORY_HIGH="${FORMAL_MEMORY_HIGH:-6G}" -FORMAL_MEMORY_MAX="${FORMAL_MEMORY_MAX:-8G}" -FORMAL_TASKS_MAX="${FORMAL_TASKS_MAX:-384}" -FORMAL_CPU_QUOTA="${FORMAL_CPU_QUOTA:-400%}" +FORMAL_MEMORY_HIGH="${FORMAL_MEMORY_HIGH:-1G}" +FORMAL_MEMORY_MAX="${FORMAL_MEMORY_MAX:-2G}" +FORMAL_TASKS_MAX="${FORMAL_TASKS_MAX:-64}" +FORMAL_CPU_QUOTA="${FORMAL_CPU_QUOTA:-200%}" FORMAL_COMMAND_TIMEOUT_SECONDS="${FORMAL_COMMAND_TIMEOUT_SECONDS:-7200}" +TLC_JAVA_TOOL_OPTIONS="${TLC_JAVA_TOOL_OPTIONS:--Xms128m -Xmx1024m -XX:+UseParallelGC}" run_capped() { local -a command=("$@") + local unit_name if [ "$FORMAL_COMMAND_TIMEOUT_SECONDS" != "0" ]; then command=( @@ -28,7 +30,9 @@ run_capped() { echo "ERROR: FORMAL_RESOURCE_CONTROL=systemd requires systemd-run" >&2 return 1 fi - systemd-run --user --scope --quiet --collect \ + unit_name="libdictenstein-formal-${BASHPID}-${RANDOM}" + systemd-run --user --unit="$unit_name" --wait --pipe --quiet --collect \ + --working-directory="$PWD" \ --property="MemoryHigh=$FORMAL_MEMORY_HIGH" \ --property="MemoryMax=$FORMAL_MEMORY_MAX" \ --property=MemorySwapMax=0 \ @@ -46,6 +50,37 @@ run_capped() { esac } +run_sany_checked() { + local module_file="$1" + local module_name="${module_file%.tla}" + local log_parent="$repo_root/target/sany-logs" + local log_file + local status=0 + + mkdir -p "$log_parent" + log_file="$(mktemp "$log_parent/${module_name}.XXXXXX.log")" + if run_capped env "JAVA_TOOL_OPTIONS=$TLC_JAVA_TOOL_OPTIONS" \ + tla2sany "$module_file" >"$log_file" 2>&1; then + status=0 + else + status=$? + fi + cat "$log_file" + + if [ "$status" -ne 0 ]; then + echo "ERROR: SANY exited with status $status for $module_file" >&2 + echo "SANY output retained at $log_file" >&2 + return 1 + fi + if grep -Eq '^(Semantic errors:|\*\*\* Errors:)' "$log_file"; then + echo "ERROR: SANY reported semantic errors for $module_file despite a zero exit status" >&2 + echo "SANY output retained at $log_file" >&2 + return 1 + fi + + rm -f -- "$log_file" +} + run_tlc_isolated() { local label="$1" shift @@ -55,7 +90,8 @@ run_tlc_isolated() { mkdir -p "$state_parent" state_directory="$(mktemp -d "$state_parent/${label}.XXXXXX")" - run_capped tlc -metadir "$state_directory" "$@" || status=$? + run_capped env "JAVA_TOOL_OPTIONS=$TLC_JAVA_TOOL_OPTIONS" \ + tlc -metadir "$state_directory" "$@" || status=$? if [ "$status" -eq 0 ]; then rm -rf -- "$state_directory" else @@ -132,7 +168,7 @@ run_tlc_negative_control() { mkdir -p "$log_parent" "$state_parent" log_file="$(mktemp "$log_parent/${module}.XXXXXX.log")" state_directory="$(mktemp -d "$state_parent/${config_base}.XXXXXX")" - if run_capped tlc \ + if run_capped env "JAVA_TOOL_OPTIONS=$TLC_JAVA_TOOL_OPTIONS" tlc \ -metadir "$state_directory" \ -workers 1 \ -config "${config_base}.cfg" \ @@ -181,6 +217,9 @@ run_tlc_negative_control() { verify_tlc_invariant_diagnostic_classifier +echo "== Variable-width formal-first gate ==" +bash scripts/verify-variable-width-formal.sh + assert_nonzero_cargo_filter() { local output output="$(run_capped cargo test "$@" -- --list)" @@ -461,6 +500,14 @@ else echo "Skipping io_uring storage correspondence checks; set RUN_IO_URING=1 to enable them" fi +echo "== Variable-width Rocq proof-escape source gate ==" +if grep -nE \ + '(^|[^[:alnum:]_])(Admitted|admit|Axiom|Axioms|Parameter|Parameters|Conjecture|Abort)([^[:alnum:]_]|$)' \ + formal-verification/rocq/Spec/VariableWidth*.v; then + echo "ERROR: variable-width Rocq sources contain a proof escape" >&2 + exit 1 +fi + echo "== Rocq proofs ==" run_capped make -C formal-verification/rocq -j1 @@ -512,6 +559,9 @@ if command -v tla2sany >/dev/null 2>&1; then PersistentARTrieU64 \ PersistentARTrieU64Iteration \ PersistentARTrieU64WorkMachines \ + VariableWidthCodecBoundary \ + VariableWidthVocabularyInterning \ + VariableWidthVocabularyPublication \ CharNodeV2Layout \ CharV3ArenaPublication \ ConcurrentVocabLinearizability \ @@ -535,7 +585,7 @@ if command -v tla2sany >/dev/null 2>&1; then DictionaryEntryBatchLease \ AbiSnapshotQuiescence do - run_capped tla2sany "${module}.tla" + run_sany_checked "${module}.tla" done ) else @@ -594,6 +644,9 @@ if [ "${RUN_TLC:-0}" = "1" ]; then PersistentARTrieU64 \ PersistentARTrieU64Iteration \ PersistentARTrieU64WorkMachines \ + VariableWidthCodecBoundary \ + VariableWidthVocabularyInterning \ + VariableWidthVocabularyPublication \ CharNodeV2Layout \ CharV3ArenaPublication \ ConcurrentVocabLinearizability \ @@ -627,6 +680,14 @@ if [ "${RUN_TLC:-0}" = "1" ]; then -workers 1 \ -config PersistentARTrieU64WorkMachines_Cycle.cfg \ PersistentARTrieU64WorkMachines.tla + run_tlc_isolated VariableWidthVocabularyInterning_MultiSpan \ + -workers 1 \ + -config VariableWidthVocabularyInterning_MultiSpan.cfg \ + VariableWidthVocabularyInterning.tla + run_tlc_isolated VariableWidthVocabularyPublication_TermFiberWitness \ + -workers 1 \ + -config VariableWidthVocabularyPublication_TermFiberWitness.cfg \ + VariableWidthVocabularyPublication.tla run_tlc_isolated PersistentARTrieU64Iteration_Chain \ -workers 1 \ -config PersistentARTrieU64Iteration_Chain.cfg \ @@ -725,6 +786,13 @@ if [ "${RUN_TLC:-0}" = "1" ]; then # * PersistentARTrieU64Iteration enables global node-identity suppression # on the Diamond DAG. It MUST violate `CompletionIsExact`, proving trie # language enumeration remains path-sensitive across shared nodes. + # * VariableWidthVocabularyInterning makes fingerprints authoritative or + # permits a published local ID to be rebound. The controls MUST violate + # the exact-byte non-aliasing and same-fiber no-reuse invariants. + # * VariableWidthVocabularyPublication independently permits a sequence + # before its vocabulary, overclaims the eligibility frontier, resumes a + # captured reader through another generation, or fabricates a missing + # vocabulary as empty. Each control MUST violate its named VWENC law. while IFS='|' read -r unsafe_module assertion_kind assertion_name unsafe_config; do negative_config="${unsafe_config:-${unsafe_module}_Unsafe}" echo "== Negative control: ${negative_config}.cfg (MUST violate ${assertion_name}) ==" @@ -800,6 +868,16 @@ AbiSnapshotInitializerTakeover|invariant|SingleConstruction|AbiSnapshotInitializ AbiSnapshotQuiescence|temporal|SnapshotEventuallyCompletes PersistentARTrieU64WorkMachines|invariant|NoCyclicSnapshotAccepted PersistentARTrieU64Iteration|invariant|CompletionIsExact +VariableWidthCodecBoundary|invariant|VWENC_26_OVERLONG_ULEB_IS_REJECTED|VariableWidthCodecBoundary_OverlongUnsafe +VariableWidthCodecBoundary|invariant|VWENC_27_UNTERMINATED_ULEB_IS_REJECTED|VariableWidthCodecBoundary_UnterminatedUnsafe +VariableWidthCodecBoundary|invariant|VWENC_28_UTF8_CONTINUATION_IS_REJECTED|VariableWidthCodecBoundary_Utf8ContinuationUnsafe +VariableWidthCodecBoundary|invariant|VWENC_24_CODEC_BYTES_NEVER_BECOME_LOGICAL_TRANSITIONS|VariableWidthCodecBoundary_PhysicalExposureUnsafe +VariableWidthVocabularyInterning|invariant|VWENC_142_FINGERPRINT_COLLISIONS_NEVER_ALIAS_DISTINCT_ATOMS|VariableWidthVocabularyInterning_FingerprintOnlyUnsafe +VariableWidthVocabularyInterning|invariant|VWENC_143_RETIRED_ID_IS_NEVER_CLAIMED_OR_LIVE_AGAIN|VariableWidthVocabularyInterning_IdReuseUnsafe +VariableWidthVocabularyPublication|invariant|VWENC_149_DURABLE_SEQUENCE_REFERENCES_DURABLE_BOUND_VOCABULARY|VariableWidthVocabularyPublication_SequenceBeforeVocabularyUnsafe +VariableWidthVocabularyPublication|invariant|VWENC_147_PUBLISHED_FRONTIER_DOES_NOT_EXCEED_DURABLE_FRONTIER|VariableWidthVocabularyPublication_FrontierOverclaimUnsafe +VariableWidthVocabularyPublication|invariant|VWENC_154_CAPTURED_CONTINUATION_RESUMES_IMMUTABLE_PAIR|VariableWidthVocabularyPublication_CrossGenerationResumeUnsafe +VariableWidthVocabularyPublication|invariant|VWENC_178_RECOVERY_NEVER_SYNTHESIZES_EMPTY_SUCCESS|VariableWidthVocabularyPublication_MissingVocabularyAsEmptyUnsafe NEGATIVE_CONTROLS ) else diff --git a/scripts/verify-variable-width-correspondence.sh b/scripts/verify-variable-width-correspondence.sh new file mode 100755 index 00000000..c3d63bbe --- /dev/null +++ b/scripts/verify-variable-width-correspondence.sh @@ -0,0 +1,282 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo_root" + +spec_path="formal-verification/rocq/Spec/VariableWidthInterningSpec.v" +manifest_path="formal-verification/variable-width-interning-correspondence.tsv" +expected_row_count=46 + +mapfile -t formal_points < <( + awk ' + /^Inductive InterningFormalPoint : Type :=/ { + in_points = 1 + next + } + in_points && /^Inductive ImplementationObligation : Type :=/ { + in_points = 0 + } + in_points && /^\| Point[A-Za-z0-9_]+/ { + point = $0 + sub(/^\| /, "", point) + sub(/[.;].*$/, "", point) + print point + } + ' "$spec_path" +) + +mapfile -t correspondence_rows < <( + awk ' + /^Definition declared_correspondence_row$/ { + in_rows = 1 + next + } + in_rows && /^Definition complete_interning_formal_points/ { + in_rows = 0 + } + in_rows && /^ \| Point[A-Za-z0-9_]+ =>$/ { + point = $2 + path = "" + symbol = "" + next + } + in_rows && point != "" && index($0, "(\"") > 0 && + index($0, "\")%string") > 0 { + start = index($0, "(\"") + 2 + finish = index($0, "\")%string") + value = substr($0, start, finish - start) + if (path == "") { + path = value + } else { + symbol = value + } + next + } + in_rows && point != "" && + /^[[:space:]]+(Refines|CommonSubstrateOnly|Conflicts|Prospective) Obligation[A-Za-z0-9_]+$/ { + relationship = $1 + obligation = $2 + print point "|" path "|" symbol "|" relationship "|" obligation + point = "" + path = "" + symbol = "" + } + ' "$spec_path" +) + +if [[ ! -f "$manifest_path" ]]; then + echo "ERROR: frozen correspondence manifest is absent: $manifest_path" >&2 + exit 1 +fi +mapfile -t manifest_rows < "$manifest_path" + +if [[ "${#formal_points[@]}" -ne "$expected_row_count" ]]; then + echo "ERROR: discovered ${#formal_points[@]} formal points; expected $expected_row_count" >&2 + exit 1 +fi +if [[ "${#correspondence_rows[@]}" -ne "$expected_row_count" ]]; then + echo "ERROR: extracted ${#correspondence_rows[@]} Rocq correspondence rows; expected $expected_row_count" >&2 + exit 1 +fi +if [[ "${#manifest_rows[@]}" -ne "$expected_row_count" ]]; then + echo "ERROR: frozen manifest contains ${#manifest_rows[@]} rows; expected $expected_row_count" >&2 + exit 1 +fi + +if ! diff -u "$manifest_path" <(printf '%s\n' "${correspondence_rows[@]}"); then + echo 'ERROR: the Rocq correspondence relation differs from the independent frozen five-field manifest' >&2 + exit 1 +fi + +declare -A formal_point_set=() +declare -A manifest_point_set=() + +for point in "${formal_points[@]}"; do + if [[ -n "${formal_point_set[$point]:-}" ]]; then + echo "ERROR: duplicate InterningFormalPoint constructor: $point" >&2 + exit 1 + fi + formal_point_set[$point]=1 +done + +require_regex() { + local point="$1" + local source_path="$2" + local rust_symbol="$3" + local description="$4" + local pattern="$5" + if ! rg --multiline -q --regexp "$pattern" "$source_path"; then + echo "ERROR: $point maps $rust_symbol to $source_path, but its exact $description was not found" >&2 + exit 1 + fi +} + +require_vocab_impl() { + local point="$1" + local source_path="$2" + local rust_symbol="$3" + require_regex "$point" "$source_path" "$rust_symbol" \ + 'PersistentVocabARTrie implementation owner' \ + '^impl super::dict_impl::PersistentVocabARTrie \{$' +} + +require_vocab_struct() { + local point="$1" + local source_path="$2" + local rust_symbol="$3" + require_regex "$point" "$source_path" "$rust_symbol" \ + 'PersistentVocabARTrie type owner' \ + '^pub struct PersistentVocabARTrie \{$' +} + +validate_exact_rust_symbol() { + local point="$1" + local source_path="$2" + local rust_symbol="$3" + + case "$source_path|$rust_symbol" in + 'src/persistent_artrie/vocab/mutation_api.rs|PersistentVocabARTrie::insert') + require_vocab_impl "$point" "$source_path" "$rust_symbol" + require_regex "$point" "$source_path" "$rust_symbol" 'method declaration' \ + '^[[:space:]]*pub fn insert\(&self, term: &str\) -> Result \{$' + ;; + 'src/persistent_artrie/vocab/mutation_api.rs|PersistentVocabARTrie::insert_overlay') + require_vocab_impl "$point" "$source_path" "$rust_symbol" + require_regex "$point" "$source_path" "$rust_symbol" 'method declaration' \ + '^[[:space:]]*fn insert_overlay\(&self, term: &str\) -> Result \{$' + case "$point" in + PointClaimAllocation) + require_regex "$point" "$source_path" "$rust_symbol" \ + 'durable insert-once publication call' \ + '^[[:space:]]*>>::insert_cas_with_value_durable_default\(self, term\.as_bytes\(\), index\)\?;$' + ;; + PointOrphanAllocation) + require_regex "$point" "$source_path" "$rust_symbol" \ + 'monotone sparse-ID allocation claim' \ + '^[[:space:]]*let index = self\.next_index\.fetch_add\(1, Ordering::AcqRel\);$' + require_regex "$point" "$source_path" "$rust_symbol" \ + 'lost-race burned-ID return path' \ + '^[[:space:]]*Ok\(self\.get_index_lockfree\(term\)\.unwrap_or\(index\)\)$' + ;; + *) + echo "ERROR: $point maps insert_overlay without a claim-or-orphan use-specific validator" >&2 + exit 1 + ;; + esac + ;; + 'src/persistent_artrie/vocab/mutation_api.rs|PersistentVocabARTrie::snapshot') + require_vocab_impl "$point" "$source_path" "$rust_symbol" + require_regex "$point" "$source_path" "$rust_symbol" 'method declaration' \ + '^[[:space:]]*pub fn snapshot\(&self\) -> Self \{$' + ;; + 'src/persistent_artrie/vocab/query_api.rs|PersistentVocabARTrie::get_term') + require_vocab_impl "$point" "$source_path" "$rust_symbol" + require_regex "$point" "$source_path" "$rust_symbol" 'method declaration' \ + '^[[:space:]]*pub fn get_term\(&self, index: u64\) -> Option \{$' + ;; + 'src/persistent_artrie/vocab/query_api.rs|PersistentVocabARTrie::next_index') + require_vocab_impl "$point" "$source_path" "$rust_symbol" + require_regex "$point" "$source_path" "$rust_symbol" 'method declaration' \ + '^[[:space:]]*pub fn next_index\(&self\) -> u64 \{$' + ;; + 'src/persistent_artrie/vocab/dict_impl.rs|PersistentVocabARTrie::reverse_term_map') + require_vocab_struct "$point" "$source_path" "$rust_symbol" + require_regex "$point" "$source_path" "$rust_symbol" 'field declaration' \ + '^[[:space:]]*pub\(super\) reverse_term_map: Option>,$' + ;; + 'src/persistent_artrie/vocab/dict_impl.rs|PersistentVocabARTrie::next_index') + require_vocab_struct "$point" "$source_path" "$rust_symbol" + require_regex "$point" "$source_path" "$rust_symbol" 'field declaration' \ + '^[[:space:]]*pub\(super\) next_index: AtomicU64,$' + ;; + 'src/persistent_artrie/vocab/dict_impl.rs|PersistentVocabARTrie::commit_seq') + require_vocab_struct "$point" "$source_path" "$rust_symbol" + require_regex "$point" "$source_path" "$rust_symbol" 'field declaration' \ + '^[[:space:]]*pub\(crate\) commit_seq: AtomicU64,$' + ;; + 'src/persistent_artrie/vocab/dict_impl.rs|PersistentVocabARTrie::committed_watermark') + require_vocab_struct "$point" "$source_path" "$rust_symbol" + require_regex "$point" "$source_path" "$rust_symbol" 'field declaration' \ + '^[[:space:]]*pub\(crate\) committed_watermark:\n[[:space:]]*crate::persistent_artrie::core::committed_watermark::CommittedWatermark,$' + ;; + 'src/persistent_artrie/vocab/dict_impl.rs|PersistentVocabARTrie::checkpoint_lock') + require_vocab_struct "$point" "$source_path" "$rust_symbol" + require_regex "$point" "$source_path" "$rust_symbol" 'field declaration' \ + '^[[:space:]]*pub\(crate\) checkpoint_lock: Arc>,$' + ;; + 'src/persistent_artrie/vocab/types.rs|VocabTrieFileHeader') + require_regex "$point" "$source_path" "$rust_symbol" 'type declaration' \ + '^pub struct VocabTrieFileHeader \{$' + ;; + 'src/persistent_artrie/vocab/persistence_api.rs|PersistentVocabARTrie::checkpoint_overlay') + require_vocab_impl "$point" "$source_path" "$rust_symbol" + require_regex "$point" "$source_path" "$rust_symbol" 'method declaration' \ + '^[[:space:]]*fn checkpoint_overlay\(&self\) -> Result<\(\)> \{$' + ;; + 'src/persistent_artrie/u64.rs|write_snapshot_file') + require_regex "$point" "$source_path" "$rust_symbol" 'generic free-function declaration' \ + '^fn write_snapshot_file\($' + ;; + 'src/persistent_artrie/core/overlay/durable_write.rs|DurableOverlayWrite::insert_cas_with_value_durable_default') + require_regex "$point" "$source_path" "$rust_symbol" 'DurableOverlayWrite trait owner' \ + '^pub\(crate\) trait DurableOverlayWrite:$' + require_regex "$point" "$source_path" "$rust_symbol" 'default-method declaration' \ + '^[[:space:]]*fn insert_cas_with_value_durable_default\(&self, key_bytes: &\[u8\], value: V\) -> Result \{$' + ;; + *) + echo "ERROR: no exact declaration validator exists for implemented mapping $point: $source_path | $rust_symbol" >&2 + exit 1 + ;; + esac +} + +for row in "${manifest_rows[@]}"; do + IFS='|' read -r point source_path rust_symbol relationship obligation extra <<<"$row" + + if [[ -n "${extra:-}" || -z "$point" || -z "$source_path" || + -z "$rust_symbol" || -z "$relationship" || -z "$obligation" ]]; then + echo "ERROR: manifest row is not exactly five nonempty fields: $row" >&2 + exit 1 + fi + if [[ -z "${formal_point_set[$point]:-}" ]]; then + echo "ERROR: frozen manifest names undeclared formal point: $point" >&2 + exit 1 + fi + if [[ -n "${manifest_point_set[$point]:-}" ]]; then + echo "ERROR: duplicate frozen correspondence row for: $point" >&2 + exit 1 + fi + manifest_point_set[$point]=1 + + case "$relationship" in + Prospective) + if [[ -e "$source_path" ]]; then + echo "ERROR: $point remains Prospective although $source_path now exists" >&2 + echo 'Update the Rocq relation, frozen manifest, and exact declaration validator together.' >&2 + exit 1 + fi + ;; + Refines|CommonSubstrateOnly|Conflicts) + if [[ ! -f "$source_path" ]]; then + echo "ERROR: $point declares $relationship but source file is absent: $source_path" >&2 + exit 1 + fi + validate_exact_rust_symbol "$point" "$source_path" "$rust_symbol" + ;; + *) + echo "ERROR: $point has unknown correspondence relationship: $relationship" >&2 + exit 1 + ;; + esac +done + +for point in "${formal_points[@]}"; do + if [[ -z "${manifest_point_set[$point]:-}" ]]; then + echo "ERROR: formal point lacks an independent frozen correspondence row: $point" >&2 + exit 1 + fi +done + +printf 'Validated %d exact variable-width correspondence rows against the independent manifest and Rust declarations.\n' \ + "${#manifest_rows[@]}" diff --git a/scripts/verify-variable-width-formal.sh b/scripts/verify-variable-width-formal.sh new file mode 100755 index 00000000..77a2f4df --- /dev/null +++ b/scripts/verify-variable-width-formal.sh @@ -0,0 +1,634 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +rocq_root="$repo_root/formal-verification/rocq" +tla_root="$repo_root/formal-verification/tla+" +# Keep default verifier state on the user runtime filesystem rather than /tmp. +# The latter is commonly a tmpfs, so large TLC state spaces could consume +# resident memory even though the verifier itself is RSS-capped. Callers may +# still select an explicitly managed artifact directory through the environment. +runtime_root="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}" +artifact_root="${VARIABLE_WIDTH_FORMAL_ARTIFACT_ROOT:-$runtime_root/pgmcp/libdictenstein-variable-width-formal}" +log_root="$artifact_root/logs" +state_root="$artifact_root/tlc-state-spaces" +tmp_root="${VARIABLE_WIDTH_FORMAL_TMPDIR:-$artifact_root/tmp}" + +command_timeout_seconds="${VARIABLE_WIDTH_FORMAL_TIMEOUT_SECONDS:-7200}" +tlc_java_options="${VARIABLE_WIDTH_TLC_JAVA_OPTIONS:--Xms64m -Xmx512m -XX:+UseParallelGC}" +resource_control="${VARIABLE_WIDTH_FORMAL_RESOURCE_CONTROL:-systemd}" +coqc_bin="${COQC_BIN:-$(command -v coqc || true)}" +coqchk_bin="${COQCHK_BIN:-$(command -v coqchk || true)}" +expected_identifier_count=246 +run_number=0 +last_log="" + +mkdir -p "$log_root" "$state_root" +mkdir -p "$tmp_root" +# TLC and its standard modules honor TMPDIR. Keep their generated files on +# the managed runtime filesystem as well, rather than allowing /tmp tmpfs use. +export TMPDIR="$tmp_root" + +if [[ -z "$coqc_bin" || -z "$coqchk_bin" ]]; then + echo 'ERROR: coqc and coqchk must be available before running the formal gate' >&2 + exit 1 +fi + +cleanup_state_directory() { + local state_directory="$1" + case "$state_directory" in + "$state_root"/*) + rm -rf -- "$state_directory" + ;; + *) + echo "ERROR: refusing to remove unexpected TLC state path: $state_directory" >&2 + return 1 + ;; + esac +} + +assert_no_competing_variable_width_job() { + local competing + competing="$( + ps -eo pid=,ppid=,rss=,stat=,comm=,args= | + awk ' + $0 ~ /libdictenstein-variable-width-refinement/ && + ($5 ~ /^(coqc|coqchk|tlc|tla2sany)$/ || + ($5 == "java" && $0 ~ /tlc2\.TLC|tla2sany\.SANY/)) { + print + } + ' + )" + if [[ -n "$competing" ]]; then + echo "ERROR: another heavy variable-width verification process is active:" >&2 + printf '%s\n' "$competing" >&2 + return 1 + fi +} + +run_capped_capture() { + local label="$1" + local memory_high="$2" + local memory_max="$3" + local working_directory="$4" + shift 4 + + local status=0 + local -a command=("$@") + + assert_no_competing_variable_width_job + run_number=$((run_number + 1)) + last_log="$(mktemp "$log_root/${label}.XXXXXX.log")" + + if [[ "$command_timeout_seconds" != "0" ]]; then + command=( + timeout --foreground --signal=TERM --kill-after=30s + "$command_timeout_seconds" + "${command[@]}" + ) + fi + + case "$resource_control" in + systemd) + if systemd-run --user --wait --pipe --quiet --collect \ + --working-directory="$working_directory" \ + --property="MemoryHigh=$memory_high" \ + --property="MemoryMax=$memory_max" \ + --property=MemorySwapMax=0 \ + --property=CPUQuota=100% \ + --property=TasksMax=128 \ + "${command[@]}" >"$last_log" 2>&1; then + status=0 + else + status=$? + fi + ;; + external) + local memory_limit_bytes + case "$memory_max" in + 512M) memory_limit_bytes=536870912 ;; + 1G) memory_limit_bytes=1073741824 ;; + 2G) memory_limit_bytes=2147483648 ;; + 8G) memory_limit_bytes=8589934592 ;; + *) + echo "ERROR: unsupported external memory limit: $memory_max" >&2 + return 1 + ;; + esac + if (cd "$working_directory" && + # OCaml/Rocq reserves a large virtual heap up front. Capping + # address space rejects that reservation before execution begins; + # RSS is the relevant resident-memory safety bound here. + prlimit --rss="$memory_limit_bytes" \ + "${command[@]}" >"$last_log" 2>&1); then + status=0 + else + status=$? + fi + ;; + *) + echo "ERROR: unsupported VARIABLE_WIDTH_FORMAL_RESOURCE_CONTROL=$resource_control" >&2 + return 1 + ;; + esac + + cat "$last_log" + return "$status" +} + +run_required() { + local status=0 + if run_capped_capture "$@"; then + status=0 + else + status=$? + fi + if [[ "$status" -ne 0 ]]; then + echo "ERROR: required verification command failed with status $status" >&2 + echo "Output retained at $last_log" >&2 + return "$status" + fi + rm -f -- "$last_log" +} + +verify_cfg_inventory() { + local module="$1" + shift + local -a configs=("$@") + local model_inventory + local config_inventory + + model_inventory="$( + sed -n 's/^\(VWENC_[A-Za-z0-9_]*\) ==.*/\1/p' "$tla_root/${module}.tla" | + LC_ALL=C sort + )" + config_inventory="$( + for config in "${configs[@]}"; do + sed -n 's/^[[:space:]]*\(VWENC_[A-Za-z0-9_]*\)[[:space:]]*$/\1/p' \ + "$tla_root/${config}.cfg" + done | LC_ALL=C sort -u + )" + + if [[ -z "$model_inventory" ]]; then + echo "ERROR: $module declares no VWENC assertions" >&2 + return 1 + fi + if [[ "$model_inventory" != "$config_inventory" ]]; then + echo "ERROR: ${configs[*]} do not collectively check the exact VWENC assertion inventory of ${module}.tla" >&2 + diff -u \ + <(printf '%s\n' "$model_inventory") \ + <(printf '%s\n' "$config_inventory") || true + return 1 + fi +} + +verify_stable_identifier_inventory() { + local duplicate_names + local duplicate_numbers + local -a identifiers=() + + mapfile -t identifiers < <( + { + sed -n -E \ + 's/^(Theorem|Lemma|Corollary) (VWENC_[A-Za-z0-9_]+).*/\2/p' \ + "$rocq_root/Spec/VariableWidthCodecSpec.v" \ + "$rocq_root/Spec/VariableWidthInterningSpec.v" \ + "$rocq_root/Spec/VariableWidthFamilyRefinementSpec.v" + sed -n 's/^\(VWENC_[A-Za-z0-9_]*\) ==.*/\1/p' \ + "$tla_root/VariableWidthCodecBoundary.tla" \ + "$tla_root/VariableWidthVocabularyInterning.tla" \ + "$tla_root/VariableWidthVocabularyPublication.tla" \ + "$tla_root/VariableWidthFamilyRefinement.tla" + } | LC_ALL=C sort + ) + + if [[ "${#identifiers[@]}" -ne "$expected_identifier_count" ]]; then + echo "ERROR: discovered ${#identifiers[@]} stable VWENC declarations; expected $expected_identifier_count" >&2 + exit 1 + fi + + duplicate_names="$(printf '%s\n' "${identifiers[@]}" | uniq -d)" + duplicate_numbers="$( + printf '%s\n' "${identifiers[@]}" | + sed 's/^VWENC_\([0-9][0-9]*\)_.*/\1/' | + LC_ALL=C sort -n | + uniq -d + )" + if [[ -n "$duplicate_names" || -n "$duplicate_numbers" ]]; then + echo 'ERROR: stable VWENC identifiers are not globally unique' >&2 + [[ -z "$duplicate_names" ]] || printf 'Duplicate names:\n%s\n' "$duplicate_names" >&2 + [[ -z "$duplicate_numbers" ]] || printf 'Duplicate numbers:\n%s\n' "$duplicate_numbers" >&2 + exit 1 + fi +} + +verify_family_refinement_identifier_manifest() { + local -a expected_rocq=( + VWENC_194_LOGICAL_OBSERVATIONAL_EQUIVALENCE_IS_AN_EQUIVALENCE + VWENC_195_MEMBERSHIP_AND_TERMINALITY_ARE_LOGICAL_OBSERVATIONS + VWENC_196_MAPPED_VALUE_PRESENCE_AND_IDENTITY_ARE_OBSERVABLE + VWENC_197_ORDERED_LOGICAL_OUTGOING_LABELS_ARE_OBSERVABLE + VWENC_198_PREFIX_ENTRIES_ARE_LOGICAL_OBSERVATIONS + VWENC_199_FULL_ENUMERATION_ORDER_IS_DETERMINISTIC_AND_OBSERVABLE + VWENC_200_APPLICABLE_SUBSTRING_RESULTS_ARE_LOGICAL_OBSERVATIONS + VWENC_201_APPLICABLE_SUFFIX_RESULTS_ARE_LOGICAL_OBSERVATIONS + VWENC_202_PHYSICAL_LAYOUT_AND_CODEC_STAGING_STATE_ARE_NONOBSERVABLE + VWENC_203_DICTIONARY_FAMILY_INVENTORY_IS_EXHAUSTIVE + VWENC_204_FAMILY_PROFILE_MATRIX_IS_TOTAL_AND_FUNCTIONAL + VWENC_205_FAMILY_SURFACE_MATRIX_IS_TOTAL_AND_FUNCTIONAL + VWENC_206_FAMILY_PROFILE_SURFACE_MATRIX_IS_TOTAL + VWENC_207_EVERY_INAPPLICABLE_CELL_HAS_AN_EXPLICIT_STRUCTURAL_REASON + VWENC_208_PATHMAP_REMAINS_AN_EXTERNAL_BYTE_KEYED_ADAPTER + VWENC_209_PATHMAP_CANONICAL_ULEB_USES_ONLY_FIXED_WIDTH_INTERNED_IDS + VWENC_210_LEGACY_ONE_PARAMETER_FAMILY_SPELLING_DEFAULTS_TO_BYTES + VWENC_211_MAPPED_VALUE_REMAINS_FIRST_AND_WIDTH_IS_NOT_A_PARAMETER + VWENC_212_PROFILE_ALONE_OWNS_EDGE_UNIT_AND_WIDTH_METADATA + VWENC_213_OPEN_IN_MEMORY_UNITS_CANNOT_MINT_PERSISTENT_IDENTITIES + VWENC_214_FORMAT_IDENTITY_IS_INDEPENDENT_OF_RUST_TYPE_NAMES + VWENC_215_SPECIALIZATION_REFINES_THE_GENERIC_LOGICAL_VIEW + VWENC_216_EVERY_RETAINED_SPECIALIZED_KERNEL_PRESERVES_ALL_OBSERVATIONS + VWENC_217_KERNEL_SELECTION_IS_BOUND_ONCE_NOT_BRANCHING_PER_EDGE + VWENC_218_LEGACY_ALIAS_INVENTORIES_PRESERVE_CANONICAL_TARGETS + VWENC_219_EVERY_CHAR_ALIAS_TARGETS_UNICODE_SCALAR_UNITS + VWENC_220_EVERY_U64_ALIAS_PRESERVES_PROFILE_AND_EXPLICIT_LAYOUT + VWENC_221_DYNAMIC_TO_FROZEN_CONVERSION_PRESERVES_LOGICAL_OBSERVATIONS + VWENC_222_NODE_ZIPPER_AND_CURSOR_SHARE_ONE_REVISION_BOUND_VIEW + VWENC_223_FACTORY_COLLECTION_AND_SERIALIZATION_PRESERVE_PROFILE_VIEW + VWENC_224_SET_COMBINATORS_COMMUTE_WITH_PROFILE_REFINEMENT + VWENC_225_VALUE_COMBINATORS_COMMUTE_WITH_PROFILE_REFINEMENT + VWENC_226_ENCODED_ADAPTER_STAGING_BYTES_ARE_HIDDEN_FROM_CONSUMERS + VWENC_227_PATHMAP_UTF8_GROUPING_EMITS_ONE_UNICODE_SCALAR + VWENC_228_CANONICAL_ULEB_CODEWORD_EMITS_ONE_OPAQUE_LOGICAL_ATOM + VWENC_229_CODEWORD_BOUNDARY_OFFSETS_ARE_EXACTLY_LOGICAL_SPLITS + VWENC_230_RAW_UTF8_SUFFIX_CAN_START_INSIDE_ONE_SCALAR_CODEWORD + VWENC_231_RAW_ULEB_SUFFIX_CAN_START_INSIDE_ONE_CODEWORD + VWENC_232_LOGICAL_SUFFIXES_BEGIN_ONLY_AT_CODEWORD_BOUNDARIES + VWENC_233_RAW_BYTE_SUFFIX_INDEXES_CLAIM_ONLY_BYTE_SEMANTICS + VWENC_234_DIRECT_UNITS_PRESERVE_ONE_CODEWORD_PER_LOGICAL_EDGE + VWENC_235_INTERNED_IDS_PRESERVE_ONE_FIXED_CODEWORD_PER_LOGICAL_EDGE + VWENC_236_CONSUMER_VOCABULARY_BINDING_IS_VALIDATED_ONCE + VWENC_237_MISMATCHED_VOCABULARY_FIBERS_ARE_REJECTED_BEFORE_TRAVERSAL + VWENC_238_EVERY_HOT_TRANSITION_HAS_AN_EXACT_FIXED_WIDTH_ENCODING + VWENC_239_ARBITRARY_WIDTH_BIGUINT_BYTES_STAY_OUTSIDE_HOT_TRAVERSAL + VWENC_240_DICTIONARY_PROFILES_DO_NOT_OWN_LLATTICE_ALGEBRA + VWENC_247_HOT_TRAVERSAL_VIEW_EXISTS_IFF_FIBER_BINDING_SUCCEEDS + VWENC_248_MISMATCHED_FIBER_CANNOT_CONSTRUCT_A_HOT_TRAVERSAL_VIEW + VWENC_249_BOUND_HOT_VIEWS_CONTAIN_ONLY_EXACT_FIXED_WIDTH_UNITS + ) + local -a expected_tla=( + VWENC_241_CODEC_BYTES_NEVER_APPEAR_AS_LOGICAL_LABELS + VWENC_242_UTF8_SCALAR_IS_NEVER_SPLIT_ACROSS_LOGICAL_TRANSITIONS + VWENC_243_SUFFIX_MATCHES_NEVER_BEGIN_INSIDE_A_LOGICAL_CODEWORD + VWENC_244_SPECIALIZED_KERNEL_PRESERVES_THE_COMPLETE_OBSERVATION + VWENC_245_FORMAT_IDENTITY_COMES_ONLY_FROM_EXPLICIT_PROFILE_METADATA + VWENC_246_MISMATCHED_VOCABULARY_FIBER_IS_REJECTED_BEFORE_TRAVERSAL + ) + local -a actual_rocq=() + local -a actual_tla=() + + mapfile -t actual_rocq < <( + sed -n -E \ + 's/^(Theorem|Lemma|Corollary) (VWENC_[A-Za-z0-9_]+).*/\2/p' \ + "$rocq_root/Spec/VariableWidthFamilyRefinementSpec.v" + ) + mapfile -t actual_tla < <( + sed -n 's/^\(VWENC_[A-Za-z0-9_]*\) ==.*/\1/p' \ + "$tla_root/VariableWidthFamilyRefinement.tla" + ) + + if ! diff -u \ + <(printf '%s\n' "${expected_rocq[@]}" | LC_ALL=C sort) \ + <(printf '%s\n' "${actual_rocq[@]}" | LC_ALL=C sort); then + echo 'ERROR: the exact Rocq family-refinement identifier manifest changed' >&2 + return 1 + fi + if ! diff -u \ + <(printf '%s\n' "${expected_tla[@]}" | LC_ALL=C sort) \ + <(printf '%s\n' "${actual_tla[@]}" | LC_ALL=C sort); then + echo 'ERROR: the exact TLA+ family-refinement identifier manifest changed' >&2 + return 1 + fi +} + +verify_formal_inventory_extractor() { + local inventory_file="$artifact_root/formal-inventory.json" + python3 "$repo_root/scripts/extract-variable-width-formal-inventory.py" \ + --root "$repo_root" --output "$inventory_file" + python3 - "$inventory_file" "$expected_identifier_count" <<'PY' +import json +import sys + +path, expected = sys.argv[1], int(sys.argv[2]) +rows = json.load(open(path, encoding="utf-8")) +if len(rows) != expected: + raise SystemExit(f"extracted {len(rows)} declarations; expected {expected}") +if len({row["id"] for row in rows}) != len(rows): + raise SystemExit("formal inventory contains duplicate IDs") +controls = sum(bool(row["negative_controls"]) for row in rows) +if controls != 16: + raise SystemExit(f"extracted {controls} negative-control bindings; expected 16") +print(f"Formal inventory extractor validated {len(rows)} declarations and {controls} controls.") +PY + rm -f -- "$inventory_file" +} + +verify_formal_test_inventory() { + local inventory_file="$artifact_root/formal-test-inventory.json" + python3 "$repo_root/scripts/extract-variable-width-test-inventory.py" \ + --root "$repo_root" --output "$inventory_file" + python3 - "$inventory_file" "$repo_root" <<'PY' +import json +import pathlib +import sys + +path = pathlib.Path(sys.argv[1]) +root = pathlib.Path(sys.argv[2]) +rows = json.loads(path.read_text(encoding="utf-8")) +if not rows: + raise SystemExit("executable VWENC test inventory is empty") +if len({row["registration"] for row in rows}) != len(rows): + raise SystemExit("executable VWENC test inventory contains duplicate registrations") +for row in rows: + if not row["numeric_ids"] or not (root / row["source_path"]).is_file(): + raise SystemExit(f"invalid executable VWENC test inventory row: {row}") +print(f"Executable VWENC test inventory validated {len(rows)} registrations.") +PY + rm -f -- "$inventory_file" +} + +run_tlc_safe() { + local label="$1" + local module="$2" + local config="$3" + local state_directory + local config_path + local status=0 + + state_directory="$(mktemp -d "$state_root/${label}.XXXXXX")" + if [[ "$config" = /* ]]; then + config_path="${config}.cfg" + else + config_path="$tla_root/${config}.cfg" + fi + if run_capped_capture "$label" 512M 1G "$tla_root" \ + env "JAVA_TOOL_OPTIONS=$tlc_java_options" \ + tlc -workers 1 -metadir "$state_directory" \ + -config "$config_path" "${module}.tla"; then + status=0 + else + status=$? + fi + + if [[ "$status" -ne 0 ]]; then + echo "ERROR: safe TLC model $label failed with status $status" >&2 + echo "Output retained at $last_log" >&2 + echo "State space retained at $state_directory" >&2 + return "$status" + fi + if ! grep -Fq 'Model checking completed. No error has been found.' "$last_log"; then + echo "ERROR: safe TLC model $label did not report complete error-free exploration" >&2 + echo "Output retained at $last_log" >&2 + echo "State space retained at $state_directory" >&2 + return 1 + fi + + cleanup_state_directory "$state_directory" + rm -f -- "$last_log" +} + +run_tlc_negative_control() { + local label="$1" + local module="$2" + local config="$3" + local expected_invariant="$4" + local expected_mutant_constant="$5" + local state_directory + local config_path="$tla_root/${config}.cfg" + local status=0 + local -a selected_invariants=() + local -a true_mutant_constants=() + local -a module_invariants=() + + mapfile -t selected_invariants < <( + sed -n 's/^[[:space:]]*\(VWENC_[A-Za-z0-9_]*\)[[:space:]]*$/\1/p' \ + "$tla_root/${config}.cfg" + ) + if [[ "${#selected_invariants[@]}" -ne 1 || + "${selected_invariants[0]}" != "$expected_invariant" ]]; then + echo "ERROR: $config must select only $expected_invariant" >&2 + return 1 + fi + + mapfile -t true_mutant_constants < <( + sed -n 's/^[[:space:]]*\([A-Za-z][A-Za-z0-9_]*\)[[:space:]]*=[[:space:]]*TRUE[[:space:]]*$/\1/p' \ + "$tla_root/${config}.cfg" + ) + if [[ "${#true_mutant_constants[@]}" -ne 1 || + "${true_mutant_constants[0]}" != "$expected_mutant_constant" ]]; then + echo "ERROR: $config must enable only $expected_mutant_constant" >&2 + return 1 + fi + + mapfile -t module_invariants < <( + awk ' + /^[[:space:]]*INVARIANT[[:space:]]*$/ { in_invariants=1; next } + /^[[:space:]]*PROPERTY[[:space:]]*$/ { in_invariants=0 } + in_invariants && /^[[:space:]]*VWENC_[A-Za-z0-9_]+[[:space:]]*$/ { print $1 } + ' "$tla_root/${module}.cfg" + ) + local non_target + for non_target in "${module_invariants[@]}"; do + if [[ "$non_target" == "$expected_invariant" ]]; then + continue + fi + local temporary_config + temporary_config="$(mktemp "$artifact_root/${label}.non-target.XXXXXX.cfg")" + { + sed -n '1,/^[[:space:]]*INVARIANT[[:space:]]*$/p' "$config_path" + printf ' %s\n' "$non_target" + } >"$temporary_config" + if ! run_tlc_safe \ + "${label}-preserves-${non_target}" "$module" \ + "${temporary_config%.cfg}"; then + echo "ERROR: $config also violates non-target invariant $non_target" >&2 + rm -f -- "$temporary_config" + return 1 + fi + rm -f -- "$temporary_config" + done + + state_directory="$(mktemp -d "$state_root/${label}.XXXXXX")" + if run_capped_capture "$label" 512M 1G "$tla_root" \ + env "JAVA_TOOL_OPTIONS=$tlc_java_options" \ + tlc -workers 1 -metadir "$state_directory" \ + -config "$config_path" "${module}.tla"; then + status=0 + else + status=$? + fi + + if [[ "$status" -ne 12 ]]; then + echo "ERROR: $config exited with $status; expected TLC invariant-violation status 12" >&2 + echo "Output retained at $last_log" >&2 + echo "State space retained at $state_directory" >&2 + return 1 + fi + if grep -Fq \ + "Error: Invariant ${expected_invariant} is violated by the initial state:" \ + "$last_log"; then + echo "ERROR: $config failed in the initial state instead of after its mutant action" >&2 + echo "Output retained at $last_log" >&2 + echo "State space retained at $state_directory" >&2 + return 1 + fi + if ! grep -Fxq \ + "Error: Invariant ${expected_invariant} is violated." "$last_log"; then + echo "ERROR: $config did not violate exactly $expected_invariant" >&2 + echo "Output retained at $last_log" >&2 + echo "State space retained at $state_directory" >&2 + return 1 + fi + + cleanup_state_directory "$state_directory" + rm -f -- "$last_log" + echo "OK: $config rejected by $expected_invariant" +} + +echo '== Variable-width formal source integrity ==' +if rg -n \ + '(^|[^[:alnum:]_])(Admitted|admit|Axiom|Axioms|Parameter|Parameters|Conjecture|Abort)([^[:alnum:]_]|$)' \ + "$rocq_root/Spec/VariableWidthCodecSpec.v" \ + "$rocq_root/Spec/VariableWidthInterningSpec.v" \ + "$rocq_root/Spec/VariableWidthFamilyRefinementSpec.v"; then + echo 'ERROR: variable-width Rocq sources contain a proof escape' >&2 + exit 1 +fi + +if rg -n 'TODO|FIXME|HACK|XXX' \ + "$rocq_root/Spec/VariableWidthCodecSpec.v" \ + "$rocq_root/Spec/VariableWidthInterningSpec.v" \ + "$rocq_root/Spec/VariableWidthFamilyRefinementSpec.v" \ + "$tla_root/VariableWidthCodecBoundary.tla" \ + "$tla_root/VariableWidthVocabularyInterning.tla" \ + "$tla_root/VariableWidthVocabularyPublication.tla" \ + "$tla_root/VariableWidthFamilyRefinement.tla"; then + echo 'ERROR: variable-width formal sources contain an incompletion marker' >&2 + exit 1 +fi + +"$repo_root/scripts/verify-variable-width-correspondence.sh" +verify_stable_identifier_inventory +verify_family_refinement_identifier_manifest +verify_formal_inventory_extractor +verify_formal_test_inventory +run_required formal-inventory-regression-tests 256M 512M "$repo_root" \ + python3 -m unittest discover -s tests -p 'test_variable_width*.py' + +verify_cfg_inventory VariableWidthCodecBoundary VariableWidthCodecBoundary +verify_cfg_inventory VariableWidthVocabularyInterning \ + VariableWidthVocabularyInterning \ + VariableWidthVocabularyInterning_MultiSpan +verify_cfg_inventory VariableWidthVocabularyPublication \ + VariableWidthVocabularyPublication \ + VariableWidthVocabularyPublication_TermFiberWitness +verify_cfg_inventory VariableWidthFamilyRefinement \ + VariableWidthFamilyRefinement \ + VariableWidthFamilyRefinement_PhysicalExposureUnsafe \ + VariableWidthFamilyRefinement_Utf8SplitUnsafe \ + VariableWidthFamilyRefinement_InteriorSuffixUnsafe \ + VariableWidthFamilyRefinement_SpecializedDivergenceUnsafe \ + VariableWidthFamilyRefinement_TypeNameFormatUnsafe \ + VariableWidthFamilyRefinement_FiberMismatchUnsafe + +if ! grep -Fxq 'INIT MultiSpanInit' \ + "$tla_root/VariableWidthVocabularyInterning_MultiSpan.cfg" || + ! grep -Eq '^[[:space:]]*VWENC_180_MULTISPAN_WITNESS_IS_CONCRETE[[:space:]]*$' \ + "$tla_root/VariableWidthVocabularyInterning_MultiSpan.cfg" || + ! grep -Eq '^[[:space:]]*VWENC_175_PACKED_SPANS_ARE_DISJOINT_AND_COVER_BYTES_EXACTLY[[:space:]]*$' \ + "$tla_root/VariableWidthVocabularyInterning_MultiSpan.cfg"; then + echo 'ERROR: the multi-span TLC control is not bound to its concrete two-span witness and exact-coverage law' >&2 + exit 1 +fi + +if ! grep -Fxq 'INIT TermFiberWitnessInit' \ + "$tla_root/VariableWidthVocabularyPublication_TermFiberWitness.cfg" || + ! grep -Eq '^[[:space:]]*VWENC_179_EXACT_TERM_FIBER_SEPARATES_SAME_RAW_ID[[:space:]]*$' \ + "$tla_root/VariableWidthVocabularyPublication_TermFiberWitness.cfg" || + ! grep -Eq '^[[:space:]]*VWENC_193_TWO_GENERATION_TERM_FIBER_WITNESS_IS_CONCRETE[[:space:]]*$' \ + "$tla_root/VariableWidthVocabularyPublication_TermFiberWitness.cfg"; then + echo 'ERROR: the term-fiber TLC control is not bound to its concrete two-generation witness and exact-separation law' >&2 + exit 1 +fi + +echo '== Rocq proofs ==' +rocq_memory_max="${VARIABLE_WIDTH_FORMAL_ROCQ_MEMORY_MAX:-2G}" +run_required rocq-map-spec 1G "$rocq_memory_max" "$rocq_root" \ + "$coqc_bin" -Q . ARTrie Spec/MapSpec.v +run_required rocq-dictionary-law-spec 1G "$rocq_memory_max" "$rocq_root" \ + "$coqc_bin" -Q . ARTrie Spec/DictionaryLawSpec.v +run_required rocq-dawg-mutation-spec 1G "$rocq_memory_max" "$rocq_root" \ + "$coqc_bin" -Q . ARTrie Spec/DynamicDawgMutationSpec.v +run_required rocq-dawg-u64-spec 1G "$rocq_memory_max" "$rocq_root" \ + "$coqc_bin" -Q . ARTrie Spec/DynamicDawgU64Spec.v +run_required rocq-codec 1G "$rocq_memory_max" "$rocq_root" \ + "$coqc_bin" -Q . ARTrie Spec/VariableWidthCodecSpec.v +run_required rocq-interning 1G "$rocq_memory_max" "$rocq_root" \ + "$coqc_bin" -Q . ARTrie Spec/VariableWidthInterningSpec.v +run_required rocq-family-refinement 1G "$rocq_memory_max" "$rocq_root" \ + "$coqc_bin" -Q . ARTrie Spec/VariableWidthFamilyRefinementSpec.v +run_required rocq-kernel-check 1G "$rocq_memory_max" "$rocq_root" \ + "$coqchk_bin" -Q . ARTrie \ + ARTrie.Spec.VariableWidthCodecSpec \ + ARTrie.Spec.VariableWidthInterningSpec \ + ARTrie.Spec.VariableWidthFamilyRefinementSpec + +echo '== TLA+ syntax and semantic analysis ==' +for module in \ + VariableWidthCodecBoundary \ + VariableWidthVocabularyInterning \ + VariableWidthVocabularyPublication \ + VariableWidthFamilyRefinement +do + run_required "sany-${module}" 512M 1G "$tla_root" \ + env "JAVA_TOOL_OPTIONS=$tlc_java_options" tla2sany "${module}.tla" +done + +echo '== Complete safe-state exploration ==' +run_tlc_safe codec-safe \ + VariableWidthCodecBoundary VariableWidthCodecBoundary +run_tlc_safe interning-safe \ + VariableWidthVocabularyInterning VariableWidthVocabularyInterning +run_tlc_safe interning-multispan \ + VariableWidthVocabularyInterning VariableWidthVocabularyInterning_MultiSpan +run_tlc_safe publication-safe \ + VariableWidthVocabularyPublication VariableWidthVocabularyPublication +run_tlc_safe publication-term-fiber \ + VariableWidthVocabularyPublication \ + VariableWidthVocabularyPublication_TermFiberWitness +run_tlc_safe family-refinement-safe \ + VariableWidthFamilyRefinement VariableWidthFamilyRefinement + +echo '== Deliberately unsafe controls ==' +while IFS='|' read -r label module config invariant mutant_constant; do + run_tlc_negative_control \ + "$label" "$module" "$config" "$invariant" "$mutant_constant" +done <<'NEGATIVE_CONTROLS' +codec-overlong|VariableWidthCodecBoundary|VariableWidthCodecBoundary_OverlongUnsafe|VWENC_26_OVERLONG_ULEB_IS_REJECTED|AcceptOverlongUleb +codec-unterminated|VariableWidthCodecBoundary|VariableWidthCodecBoundary_UnterminatedUnsafe|VWENC_27_UNTERMINATED_ULEB_IS_REJECTED|AcceptUnterminatedUleb +codec-utf8-continuation|VariableWidthCodecBoundary|VariableWidthCodecBoundary_Utf8ContinuationUnsafe|VWENC_28_UTF8_CONTINUATION_IS_REJECTED|AcceptUtf8Continuation +codec-physical-exposure|VariableWidthCodecBoundary|VariableWidthCodecBoundary_PhysicalExposureUnsafe|VWENC_24_CODEC_BYTES_NEVER_BECOME_LOGICAL_TRANSITIONS|ExposePhysicalCodecBytes +interning-fingerprint-only|VariableWidthVocabularyInterning|VariableWidthVocabularyInterning_FingerprintOnlyUnsafe|VWENC_142_FINGERPRINT_COLLISIONS_NEVER_ALIAS_DISTINCT_ATOMS|FingerprintOnlyEquality +interning-id-reuse|VariableWidthVocabularyInterning|VariableWidthVocabularyInterning_IdReuseUnsafe|VWENC_143_RETIRED_ID_IS_NEVER_CLAIMED_OR_LIVE_AGAIN|ReusePublishedId +publication-sequence-before-vocabulary|VariableWidthVocabularyPublication|VariableWidthVocabularyPublication_SequenceBeforeVocabularyUnsafe|VWENC_149_DURABLE_SEQUENCE_REFERENCES_DURABLE_BOUND_VOCABULARY|PublishSequenceBeforeVocabulary +publication-frontier-overclaim|VariableWidthVocabularyPublication|VariableWidthVocabularyPublication_FrontierOverclaimUnsafe|VWENC_147_PUBLISHED_FRONTIER_DOES_NOT_EXCEED_DURABLE_FRONTIER|OverclaimVocabularyFrontier +publication-cross-generation-resume|VariableWidthVocabularyPublication|VariableWidthVocabularyPublication_CrossGenerationResumeUnsafe|VWENC_154_CAPTURED_CONTINUATION_RESUMES_IMMUTABLE_PAIR|AllowCrossGenerationResume +publication-missing-as-empty|VariableWidthVocabularyPublication|VariableWidthVocabularyPublication_MissingVocabularyAsEmptyUnsafe|VWENC_178_RECOVERY_NEVER_SYNTHESIZES_EMPTY_SUCCESS|MissingVocabularyAsEmpty +family-physical-exposure|VariableWidthFamilyRefinement|VariableWidthFamilyRefinement_PhysicalExposureUnsafe|VWENC_241_CODEC_BYTES_NEVER_APPEAR_AS_LOGICAL_LABELS|ExposeCodecBytes +family-utf8-split|VariableWidthFamilyRefinement|VariableWidthFamilyRefinement_Utf8SplitUnsafe|VWENC_242_UTF8_SCALAR_IS_NEVER_SPLIT_ACROSS_LOGICAL_TRANSITIONS|SplitUtf8Scalar +family-interior-suffix|VariableWidthFamilyRefinement|VariableWidthFamilyRefinement_InteriorSuffixUnsafe|VWENC_243_SUFFIX_MATCHES_NEVER_BEGIN_INSIDE_A_LOGICAL_CODEWORD|AllowInteriorSuffixStart +family-specialized-divergence|VariableWidthFamilyRefinement|VariableWidthFamilyRefinement_SpecializedDivergenceUnsafe|VWENC_244_SPECIALIZED_KERNEL_PRESERVES_THE_COMPLETE_OBSERVATION|DivergeSpecializedKernel +family-type-name-format|VariableWidthFamilyRefinement|VariableWidthFamilyRefinement_TypeNameFormatUnsafe|VWENC_245_FORMAT_IDENTITY_COMES_ONLY_FROM_EXPLICIT_PROFILE_METADATA|InferFormatIdentityFromTypeName +family-fiber-mismatch|VariableWidthFamilyRefinement|VariableWidthFamilyRefinement_FiberMismatchUnsafe|VWENC_246_MISMATCHED_VOCABULARY_FIBER_IS_REJECTED_BEFORE_TRAVERSAL|AcceptMismatchedVocabularyFiber +NEGATIVE_CONTROLS + +echo 'Variable-width formal gate passed.' diff --git a/src/bijective/bijective_map.rs b/src/bijective/bijective_map.rs index 32600d88..37781baa 100644 --- a/src/bijective/bijective_map.rs +++ b/src/bijective/bijective_map.rs @@ -193,6 +193,32 @@ impl BijectiveMap { bimap } + /// Build a bijection from Unicode-scalar profile sequences and values. + /// + /// Each sequence is materialized as a logical `String` only at this + /// user-facing boundary; the forward DAWG then traverses Unicode scalars. + pub fn from_atom_sequences_with_values(entries: I) -> Self + where + P: crate::AtomProfile, + I: IntoIterator, V)>, + { + Self::from_pairs(entries.into_iter().map(|(sequence, value)| { + ( + sequence.as_atoms().iter().copied().collect::(), + value, + ) + })) + } + + /// Read a mapped value for a Unicode-scalar profile sequence. + pub fn get_atom_sequence_value

(&self, sequence: &crate::AtomSequence

) -> Option + where + P: crate::AtomProfile, + { + let term: String = sequence.as_atoms().iter().collect(); + self.get_value(&term) + } + /// Insert a term-value pair. /// /// # Panics @@ -637,6 +663,22 @@ impl BijectiveDictionary for BijectiveMap { #[cfg(test)] mod tests { use super::*; + use crate::{AtomSequence, BijectiveDictionary, UnicodeScalar}; + + #[test] + fn profile_sequence_constructor_preserves_bijection() { + let bimap = BijectiveMap::::from_atom_sequences_with_values::([( + AtomSequence::::from_atoms(['Ξ»', 'x']), + 7, + )]); + assert_eq!(bimap.get_value("Ξ»x"), Some(7)); + assert_eq!( + BijectiveDictionary::get_term(&bimap, &7).as_deref(), + Some("Ξ»x") + ); + let sequence = AtomSequence::::from_atoms(['Ξ»', 'x']); + assert_eq!(bimap.get_atom_sequence_value(&sequence), Some(7)); + } #[test] fn test_empty_map() { diff --git a/src/bijective/mod.rs b/src/bijective/mod.rs index 9463ad86..366bfec4 100644 --- a/src/bijective/mod.rs +++ b/src/bijective/mod.rs @@ -56,8 +56,10 @@ //! - The bijection invariant is maintained across concurrent operations. mod bijective_map; +mod profiled; pub use bijective_map::{BijectiveMap, InsertError}; +pub use profiled::{ProfiledBijectiveMap, ProfiledBijectiveSnapshot}; use std::borrow::Cow; diff --git a/src/bijective/profiled.rs b/src/bijective/profiled.rs new file mode 100644 index 00000000..32d9fb1b --- /dev/null +++ b/src/bijective/profiled.rs @@ -0,0 +1,385 @@ +//! Unit-preserving bidirectional dictionaries. +//! +//! [`ProfiledBijectiveMap`] is the native-unit counterpart to the legacy +//! string-oriented [`super::BijectiveMap`]. It keeps profile units in both +//! directions and therefore never coerces a byte, numeric, or other logical +//! alphabet through UTF-8 text. + +use crate::dynamic_dawg::DynamicDawgGeneric; +use crate::{AtomProfile, AtomSequence, CharUnit, DictionaryValue}; +use arc_swap::ArcSwap; +use std::collections::HashMap; +use std::hash::Hash; +use std::sync::{Arc, Mutex}; + +/// A bidirectional map over one dictionary-unit alphabet. +/// +/// The forward index is the shared lock-free DAWG; the reverse index is an +/// atomically published copy-on-write map. Reverse lookup returns an owned +/// unit vector so readers never retain an internal snapshot after publication. +#[derive(Debug)] +pub struct ProfiledBijectiveMap { + forward: DynamicDawgGeneric, + reverse: Arc>>>, + writers: Mutex<()>, +} + +/// Immutable, coherent forward/reverse view of a [`ProfiledBijectiveMap`]. +#[derive(Clone, Debug)] +pub struct ProfiledBijectiveSnapshot { + forward: Arc, V>>, + reverse: Arc>>, +} + +impl ProfiledBijectiveSnapshot { + /// Number of entries captured by this snapshot. + #[inline] + pub fn len(&self) -> usize { + self.reverse.len() + } + + /// Whether this snapshot contains no entries. + #[inline] + pub fn is_empty(&self) -> bool { + self.reverse.is_empty() + } + + /// Look up a value in the captured forward index. + #[inline] + pub fn get_units_value(&self, units: &[U]) -> Option { + self.forward.get(units).cloned() + } + + /// Look up native units in the captured reverse index. + #[inline] + pub fn get_units(&self, value: &V) -> Option> { + self.reverse.get(value).cloned() + } + + /// Iterate the captured entries without observing later writes. + pub fn iter_units(&self) -> impl Iterator, V)> { + self.reverse + .iter() + .map(|(value, units)| (units.clone(), value.clone())) + .collect::>() + .into_iter() + } +} + +impl Clone for ProfiledBijectiveMap { + fn clone(&self) -> Self { + Self { + forward: self.forward.clone(), + reverse: Arc::new(ArcSwap::from_pointee((*self.reverse.load_full()).clone())), + writers: Mutex::new(()), + } + } +} + +impl Default for ProfiledBijectiveMap { + fn default() -> Self { + Self::new() + } +} + +impl ProfiledBijectiveMap { + /// Construct an empty unit-preserving bijection. + pub fn new() -> Self { + Self { + forward: DynamicDawgGeneric::new(), + reverse: Arc::new(ArcSwap::from_pointee(HashMap::new())), + writers: Mutex::new(()), + } + } + + /// Construct an empty map with reverse-index capacity reserved. + pub fn with_capacity(capacity: usize) -> Self { + Self { + forward: DynamicDawgGeneric::new(), + reverse: Arc::new(ArcSwap::from_pointee(HashMap::with_capacity(capacity))), + writers: Mutex::new(()), + } + } + + #[inline] + fn reverse_snapshot(&self) -> Arc>> { + self.reverse.load_full() + } + + fn mutate_reverse(&self, mut f: F) -> R + where + F: FnMut(&mut HashMap>) -> (R, bool), + { + loop { + let current = self.reverse_snapshot(); + let mut next = (*current).clone(); + let (result, changed) = f(&mut next); + if !changed { + return result; + } + let previous = self.reverse.compare_and_swap(¤t, Arc::new(next)); + if Arc::ptr_eq(&previous, ¤t) { + return result; + } + std::hint::spin_loop(); + } + } + + /// Insert a unit sequence, panicking on a duplicate key or value. + pub fn insert_units(&self, units: &[U], value: V) { + self.try_insert_units(units, value) + .unwrap_or_else(|error| panic!("ProfiledBijectiveMap::insert_units: {error:?}")); + } + + /// Insert a unit sequence while preserving the bijection invariant. + pub fn try_insert_units(&self, units: &[U], value: V) -> Result<(), super::InsertError> { + let _writer = self + .writers + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if self.forward.get_units_value(units).is_some() { + return Err(super::InsertError::DuplicateTerm); + } + let units = units.to_vec(); + let result = self.mutate_reverse(|reverse| { + if reverse.contains_key(&value) { + (Err(super::InsertError::DuplicateValue), false) + } else { + reverse.insert(value.clone(), units.clone()); + (Ok(()), true) + } + }); + result?; + if !self.forward.insert_units_with_value(&units, value.clone()) { + self.mutate_reverse(|reverse| { + let remove = reverse + .get(&value) + .is_some_and(|existing| existing == &units); + if remove { + reverse.remove(&value); + } + ((), remove) + }); + return Err(super::InsertError::DuplicateTerm); + } + Ok(()) + } + + /// Construct a map from profile-owned atom sequences. + pub fn from_atom_sequences_with_values(entries: I) -> Self + where + P: AtomProfile, + I: IntoIterator, V)>, + { + let map = Self::new(); + for (sequence, value) in entries { + map.insert_units(sequence.as_atoms(), value); + } + map + } + + /// Look up a value by native logical units. + #[inline] + pub fn get_units_value(&self, units: &[U]) -> Option { + self.forward.get_units_value(units) + } + + /// Look up the native logical units associated with a value. + #[inline] + pub fn get_units(&self, value: &V) -> Option> { + self.reverse_snapshot().get(value).cloned() + } + + /// Test forward membership without allocating. + #[inline] + pub fn contains_units(&self, units: &[U]) -> bool { + self.forward.get_units_value(units).is_some() + } + + /// Test reverse membership. + #[inline] + pub fn contains_value(&self, value: &V) -> bool { + self.reverse_snapshot().contains_key(value) + } + + /// Remove a unit sequence and its reverse entry. + /// + /// Removal is serialized with insertion and never changes any other + /// vocabulary/value association. The returned flag reports whether the + /// forward sequence was present. + pub fn remove_units(&self, units: &[U]) -> bool { + let _writer = self + .writers + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let Some(value) = self.forward.get_units_value(units) else { + return false; + }; + if !self.forward.remove_units(units) { + return false; + } + self.mutate_reverse(|reverse| { + let remove = reverse + .get(&value) + .is_some_and(|existing| existing == units); + if remove { + reverse.remove(&value); + } + ((), remove) + }); + true + } + + /// Remove and return the unit sequence associated with a value. + pub fn remove_value(&self, value: &V) -> Option> { + let units = self.get_units(value)?; + self.remove_units(&units).then_some(units) + } + + /// Number of bijective entries. + #[inline] + pub fn len(&self) -> usize { + self.reverse_snapshot().len() + } + + /// Whether the map contains no entries. + #[inline] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Capture one coherent immutable forward/reverse snapshot. + pub fn snapshot(&self) -> ProfiledBijectiveSnapshot { + let _writer = self + .writers + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let reverse = self.reverse_snapshot(); + let forward = self + .forward + .visible_entries() + .into_iter() + .filter_map(|(units, value)| value.map(|value| (units, value))) + .collect::>(); + ProfiledBijectiveSnapshot { + forward: Arc::new(forward), + reverse, + } + } + + /// Iterate owned unit/value pairs in reverse-index order. + pub fn iter_units(&self) -> impl Iterator, V)> { + self.reverse_snapshot() + .iter() + .map(|(value, units)| (units.clone(), value.clone())) + .collect::>() + .into_iter() + } + + /// Borrow the forward unit-native dictionary for specialized consumers. + #[inline] + pub fn forward(&self) -> &DynamicDawgGeneric { + &self.forward + } +} + +#[cfg(test)] +mod tests { + use super::ProfiledBijectiveMap; + use crate::{AtomSequence, U32}; + use proptest::{prop_assert, prop_assert_eq}; + + #[test] + fn preserves_numeric_units_in_both_directions() { + let map = ProfiledBijectiveMap::::from_atom_sequences_with_values::([( + AtomSequence::::from_atoms([0x100, 0x200]), + 7, + )]); + assert_eq!(map.get_units_value(&[0x100, 0x200]), Some(7)); + assert_eq!(map.get_units(&7), Some(vec![0x100, 0x200])); + assert!(map.contains_units(&[0x100, 0x200])); + } + + #[test] + fn duplicate_insert_does_not_change_reverse_mapping() { + let map = ProfiledBijectiveMap::::new(); + map.insert_units(b"ab", 1); + assert_eq!( + map.try_insert_units(b"ab", 2), + Err(crate::bijective::InsertError::DuplicateTerm) + ); + assert_eq!( + map.try_insert_units(b"cd", 1), + Err(crate::bijective::InsertError::DuplicateValue) + ); + assert_eq!(map.get_units(&1), Some(b"ab".to_vec())); + assert_eq!(map.remove_value(&1), Some(b"ab".to_vec())); + assert!(!map.contains_units(b"ab")); + assert!(!map.contains_value(&1)); + } + + #[test] + fn concurrent_writers_preserve_forward_reverse_identity() { + use std::sync::Arc; + use std::thread; + + let map = Arc::new(ProfiledBijectiveMap::::new()); + let handles = (0..16) + .map(|index| { + let map = Arc::clone(&map); + thread::spawn(move || { + let units = [index, index + 1000]; + map.try_insert_units(&units, index).unwrap(); + }) + }) + .collect::>(); + for handle in handles { + handle.join().unwrap(); + } + + assert_eq!(map.len(), 16); + for index in 0..16 { + let units = vec![index, index + 1000]; + assert_eq!(map.get_units_value(&units), Some(index)); + assert_eq!(map.get_units(&index), Some(units)); + } + } + + #[test] + fn snapshot_is_immutable_and_coherent() { + let map = ProfiledBijectiveMap::::new(); + map.insert_units(b"ab", 1); + let snapshot = map.snapshot(); + assert_eq!(snapshot.get_units_value(b"ab"), Some(1)); + assert_eq!(snapshot.get_units(&1), Some(b"ab".to_vec())); + + map.remove_value(&1); + assert_eq!(map.get_units_value(b"ab"), None); + assert_eq!(snapshot.get_units_value(b"ab"), Some(1)); + assert_eq!(snapshot.len(), 1); + } + + proptest::proptest! { + #[test] + fn generated_unit_sequences_round_trip( + sequences in proptest::collection::vec( + proptest::collection::vec(0u8..=7, 0..=5), + 0..=32 + ) + ) { + use std::collections::BTreeSet; + + let unique: BTreeSet> = sequences.into_iter().collect(); + let map = ProfiledBijectiveMap::::new(); + let entries: Vec<_> = unique.into_iter().enumerate().collect(); + for (value, units) in &entries { + prop_assert!(map.try_insert_units(units, *value as u32).is_ok()); + } + prop_assert_eq!(map.len(), entries.len()); + for (value, units) in entries { + prop_assert_eq!(map.get_units_value(&units), Some(value as u32)); + prop_assert_eq!(map.get_units(&((value) as u32)), Some(units)); + } + } + } +} diff --git a/src/bindings.rs b/src/bindings.rs index 7d905f8b..3e9bfe1f 100644 --- a/src/bindings.rs +++ b/src/bindings.rs @@ -76,7 +76,7 @@ impl BindingValue { } /// Concrete unit domain selected for a binding-owned dictionary. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] #[repr(u32)] pub enum BindingUnitDomain { /// Arbitrary byte sequences. @@ -87,6 +87,74 @@ pub enum BindingUnitDomain { U64 = 3, } +/// Canonical profile metadata associated with a binding unit domain. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct BindingProfileDescriptor { + /// Built-in logical profile kind. + pub kind: crate::ProfileKind, + /// Canonical profile name and version. + pub identity: crate::VariableWidthProfile, + /// Fixed width in bytes, or `None` for variable-width profiles. + pub width_bytes: Option, +} + +impl BindingUnitDomain { + /// Stable ABI-independent domain identifier. + pub const fn as_str(self) -> &'static str { + match self { + Self::Byte => "bytes", + Self::UnicodeScalar => "unicode-scalar", + Self::U64 => "u64", + } + } + + /// Map a logical profile to an ABI domain when the binding supports it. + /// + /// Variable-width UTF-8 and ULEB128 profiles intentionally return `None`: + /// the current ABI has no lossless term variant for those logical atoms. + pub const fn from_profile_kind(kind: crate::ProfileKind) -> Option { + match kind { + crate::ProfileKind::Bytes => Some(Self::Byte), + crate::ProfileKind::UnicodeScalar => Some(Self::UnicodeScalar), + crate::ProfileKind::U64 => Some(Self::U64), + crate::ProfileKind::Utf8 + | crate::ProfileKind::U32 + | crate::ProfileKind::F64Bits + | crate::ProfileKind::Uleb128 => None, + } + } + + /// Return stable profile metadata without relying on ABI or Rust names. + pub const fn profile_descriptor(self) -> BindingProfileDescriptor { + BindingProfileDescriptor::for_kind(match self { + Self::Byte => crate::ProfileKind::Bytes, + Self::UnicodeScalar => crate::ProfileKind::UnicodeScalar, + Self::U64 => crate::ProfileKind::U64, + }) + } +} + +impl BindingProfileDescriptor { + /// Construct canonical metadata for any built-in logical profile. + pub const fn for_kind(kind: crate::ProfileKind) -> Self { + Self { + kind, + identity: kind.identity(), + width_bytes: kind.width_bytes(), + } + } + + /// Construct canonical metadata from a compile-time atom profile. + pub const fn for_profile() -> Self { + Self::for_kind(P::KIND) + } + + /// Return the ABI domain when this profile can be represented losslessly. + pub const fn binding_domain(self) -> Option { + BindingUnitDomain::from_profile_kind(self.kind) + } +} + /// One owned term emitted by a binding snapshot traversal. /// /// The variants preserve arbitrary byte and `u64` keys without coercing them @@ -170,6 +238,12 @@ impl BindingEntries { VtUnitDomain::U64 => BindingUnitDomain::U64, } } + + /// Return canonical profile metadata for this immutable traversal. + #[inline] + pub fn profile_descriptor(&self) -> BindingProfileDescriptor { + self.domain().profile_descriptor() + } } impl Iterator for BindingEntries { @@ -3426,6 +3500,29 @@ mod tests { .collect() } + #[test] + fn binding_profile_metadata_is_canonical_and_fail_closed() { + assert_eq!(BindingUnitDomain::Byte.as_str(), "bytes"); + assert_eq!(BindingUnitDomain::UnicodeScalar.as_str(), "unicode-scalar"); + assert_eq!(BindingUnitDomain::U64.as_str(), "u64"); + assert_eq!( + BindingProfileDescriptor::for_profile::().kind, + crate::ProfileKind::Utf8 + ); + assert_eq!( + BindingProfileDescriptor::for_profile::().binding_domain(), + None + ); + assert_eq!( + BindingProfileDescriptor::for_profile::().binding_domain(), + Some(BindingUnitDomain::U64) + ); + assert_eq!( + BindingUnitDomain::from_profile_kind(crate::ProfileKind::Uleb128), + None + ); + } + #[test] fn dictionary_algebra_is_snapshot_consistent_and_domain_safe() { let left = DynamicDawgBinding::new(BindingUnitDomain::UnicodeScalar); diff --git a/src/char_unit.rs b/src/char_unit.rs index 0b1ac936..6f87bea6 100644 --- a/src/char_unit.rs +++ b/src/char_unit.rs @@ -219,6 +219,47 @@ impl CharUnit for u64 { } } +/// 32-bit unit implementation for native U32 token sequences. +impl CharUnit for u32 { + #[inline] + fn from_str(s: &str) -> Vec { + s.as_bytes() + .chunks(4) + .map(|chunk| { + let mut arr = [0u8; 4]; + arr[..chunk.len()].copy_from_slice(chunk); + u32::from_le_bytes(arr) + }) + .collect() + } + + #[inline] + fn to_string(units: &[Self]) -> String { + let bytes: Vec = units.iter().flat_map(|&unit| unit.to_le_bytes()).collect(); + let end = bytes + .iter() + .rposition(|&byte| byte != 0) + .map(|index| index + 1) + .unwrap_or(0); + String::from_utf8_lossy(&bytes[..end]).into_owned() + } + + #[inline] + fn iter_str(s: &str) -> Box + '_> { + Box::new(Self::from_str(s).into_iter()) + } + + #[inline] + fn to_dat_offset(&self) -> usize { + *self as usize + } + + #[inline] + fn to_dense_index(&self) -> Option { + u8::try_from(*self).ok() + } +} + #[cfg(test)] mod tests { use super::*; @@ -353,4 +394,12 @@ mod tests { assert_eq!(collected.len(), 2); assert_eq!(::to_string(&collected), s); } + + #[test] + fn test_u32_native_units() { + let s = "hello world!"; + let units = u32::from_str(s); + assert_eq!(units.len(), 3); + assert_eq!(::to_string(&units), s); + } } diff --git a/src/double_array_trie/ascii.rs b/src/double_array_trie/ascii.rs index 71990216..337165ab 100644 --- a/src/double_array_trie/ascii.rs +++ b/src/double_array_trie/ascii.rs @@ -186,6 +186,18 @@ struct DoubleArrayTrieWire { rebuild_threshold: f64, } +impl DoubleArrayTrie { + /// Canonical logical profile represented by this byte DAT. + pub const fn profile_descriptor() -> crate::factory::BackendProfileDescriptor { + crate::factory::BackendProfileDescriptor::from_profile::() + } + + /// Topology family represented by this dictionary. + pub const fn dictionary_family() -> crate::factory::DictionaryFamily { + crate::factory::DictionaryFamily::DoubleArrayTrie + } +} + #[cfg(feature = "serialization")] impl DoubleArrayTrie { fn from_untrusted_wire( @@ -599,6 +611,32 @@ impl DoubleArrayTrie { DoubleArrayTrieBuilder::new().build() } + /// Build from fixed-width byte-profile sequences without text coercion. + pub fn from_atom_sequences(sequences: I) -> Self + where + P: crate::AtomProfile, + I: IntoIterator>, + { + sequences + .into_iter() + .map(|sequence| sequence.as_atoms().to_vec()) + .collect::>>() + .into_iter() + .collect() + } + + /// Build a value-bearing DAT from byte-profile sequences without text coercion. + pub fn from_atom_sequences_with_values(entries: I) -> Self + where + P: crate::AtomProfile, + I: IntoIterator, V)>, + { + entries + .into_iter() + .map(|(sequence, value)| (sequence.as_atoms().to_vec(), value)) + .collect() + } + /// Create a DAT from an iterator of (term, value) pairs. /// /// For optimal space efficiency, terms should be sorted. @@ -643,6 +681,26 @@ impl DoubleArrayTrie { self.shared.term_value(term) } + /// Get a value for an arbitrary byte key without UTF-8 coercion. + pub fn get_bytes_value(&self, bytes: &[u8]) -> Option { + self.shared.term_value_units_from(bytes, 1) + } + + /// Test an arbitrary byte key without UTF-8 coercion. + #[inline] + pub fn contains_bytes(&self, bytes: &[u8]) -> bool { + self.shared.contains_units_from(bytes, 1) + } + + /// Get a value for a byte-profile sequence without UTF-8 coercion. + #[inline] + pub fn get_atom_sequence_value

(&self, sequence: &crate::AtomSequence

) -> Option + where + P: crate::AtomProfile, + { + self.get_bytes_value(sequence.as_atoms()) + } + /// Get the number of terms in the dictionary. pub fn len(&self) -> Option { Some(self.term_count) diff --git a/src/double_array_trie/char.rs b/src/double_array_trie/char.rs index c4f9719e..ac87b6ba 100644 --- a/src/double_array_trie/char.rs +++ b/src/double_array_trie/char.rs @@ -138,6 +138,18 @@ struct DoubleArrayTrieCharWire { num_terms: usize, } +impl DoubleArrayTrieChar { + /// Canonical logical profile represented by this Unicode DAT. + pub const fn profile_descriptor() -> crate::factory::BackendProfileDescriptor { + crate::factory::BackendProfileDescriptor::from_profile::() + } + + /// Topology family represented by this dictionary. + pub const fn dictionary_family() -> crate::factory::DictionaryFamily { + crate::factory::DictionaryFamily::DoubleArrayTrie + } +} + #[cfg(feature = "serialization")] impl DoubleArrayTrieChar { fn from_untrusted_wire( @@ -257,6 +269,32 @@ impl DoubleArrayTrieChar { } } + /// Build from Unicode-scalar profile sequences without UTF-8 edges. + pub fn from_atom_sequences(sequences: I) -> Self + where + P: crate::AtomProfile, + I: IntoIterator>, + { + sequences + .into_iter() + .map(|sequence| sequence.as_atoms().to_vec()) + .collect::>>() + .into_iter() + .collect() + } + + /// Build a value-bearing DAT from Unicode-scalar profile sequences. + pub fn from_atom_sequences_with_values(entries: I) -> Self + where + P: crate::AtomProfile, + I: IntoIterator, V)>, + { + entries + .into_iter() + .map(|(sequence, value)| (sequence.as_atoms().to_vec(), value)) + .collect() + } + /// Create a character-level DAT from an iterator of (term, value) pairs. /// /// # Example @@ -406,6 +444,19 @@ impl DoubleArrayTrieChar { } } + /// Get a value using already-decoded Unicode scalar units. + pub fn get_chars_value(&self, units: &[char]) -> Option { + self.shared.term_value_units_from(units, 0) + } + + /// Get a value for a Unicode-scalar profile sequence directly. + #[inline] + pub fn get_atom_sequence_value

(&self, sequence: &crate::AtomSequence

) -> Option + where + P: crate::AtomProfile, + { + self.get_chars_value(sequence.as_atoms()) + } /// Iterate over all `(term, value)` pairs as character vectors. /// /// Returns an iterator yielding `(Vec, V)` tuples in depth-first order. diff --git a/src/double_array_trie/core/shared.rs b/src/double_array_trie/core/shared.rs index 0f827f27..2599d1ad 100644 --- a/src/double_array_trie/core/shared.rs +++ b/src/double_array_trie/core/shared.rs @@ -689,6 +689,55 @@ impl DATCoreShared { } } + /// Walk the trie using already-decoded logical units, avoiding text + /// conversion at byte/profile API boundaries. + pub fn term_value_units_from(&self, units: &[U], root_state: usize) -> Option + where + V: Clone, + { + let mut state = root_state; + for unit in units { + if state >= self.base.len() { + return None; + } + let base = self.base[state]; + if base < 0 { + return None; + } + let next = (base as usize).wrapping_add(unit.to_dat_offset()); + if next >= self.check.len() || self.check[next] != state as i32 { + return None; + } + state = next; + } + if state < self.is_final.len() && self.is_final[state] { + self.values.get(state).and_then(|value| value.clone()) + } else { + None + } + } + + /// Walk the trie using already-decoded logical units and report whether + /// the final state is terminal. + pub fn contains_units_from(&self, units: &[U], root_state: usize) -> bool { + let mut state = root_state; + for unit in units { + if state >= self.base.len() { + return false; + } + let base = self.base[state]; + if base < 0 { + return false; + } + let next = (base as usize).wrapping_add(unit.to_dat_offset()); + if next >= self.check.len() || self.check[next] != state as i32 { + return false; + } + state = next; + } + state < self.is_final.len() && self.is_final[state] + } + /// `contains_term_from` with byte-DAT's `root_state = 1` convention. #[inline] pub fn contains_term(&self, term: &str) -> bool { diff --git a/src/double_array_trie/mod.rs b/src/double_array_trie/mod.rs index 8ebb3621..70ea015e 100644 --- a/src/double_array_trie/mod.rs +++ b/src/double_array_trie/mod.rs @@ -11,7 +11,402 @@ pub mod char_zipper; pub mod core; pub mod zipper; +use crate::DictionaryEntries; + pub use ascii::{DoubleArrayTrie, DoubleArrayTrieBuilder, DoubleArrayTrieNode}; pub use char::{DoubleArrayTrieChar, DoubleArrayTrieCharNode}; pub use char_zipper::DoubleArrayTrieCharZipper; pub use zipper::DoubleArrayTrieZipper; + +/// Immutable DAT boundary for canonical variable-width ULEB128 sequences. +/// Physical byte edges remain private; callers observe complete logical +/// sequences only. +#[derive(Clone, Debug)] +pub struct DoubleArrayTrieUleb128 { + inner: DoubleArrayTrie, +} + +/// Immutable byte-backed DAT boundary for variable-width UTF-8 strings. +#[derive(Clone, Debug)] +pub struct DoubleArrayTrieUtf8 { + inner: DoubleArrayTrie, +} + +impl Default for DoubleArrayTrieUtf8 { + fn default() -> Self { + Self::new() + } +} + +impl DoubleArrayTrieUtf8 { + /// Canonical logical profile represented by this boundary. + pub const fn profile_descriptor() -> crate::factory::BackendProfileDescriptor { + crate::factory::BackendProfileDescriptor::from_profile::() + } + + /// Topology family represented by this boundary. + pub const fn dictionary_family() -> crate::factory::DictionaryFamily { + crate::factory::DictionaryFamily::DoubleArrayTrie + } + + pub fn new() -> Self { + Self { + inner: DoubleArrayTrie::new(), + } + } + pub fn from_terms(terms: I) -> Self + where + I: IntoIterator, + S: AsRef, + { + Self { + inner: terms + .into_iter() + .map(|s| s.as_ref().as_bytes().to_vec()) + .collect(), + } + } + pub fn from_terms_with_values(entries: I) -> Self + where + I: IntoIterator, + S: AsRef, + { + Self { + inner: entries + .into_iter() + .map(|(s, v)| (s.as_ref().as_bytes().to_vec(), v)) + .collect(), + } + } + + /// Build from shared logical UTF-8 scalar profile sequences. + pub fn from_atom_sequences(sequences: I) -> Self + where + I: IntoIterator>, + { + Self { + inner: sequences + .into_iter() + .map(|sequence| sequence.to_encoded()) + .collect(), + } + } + + /// Build a value-bearing DAT from shared logical UTF-8 scalar profile + /// sequences. + pub fn from_atom_sequences_with_values(entries: I) -> Self + where + I: IntoIterator, V)>, + { + Self { + inner: entries + .into_iter() + .map(|(sequence, value)| (sequence.to_encoded(), value)) + .collect(), + } + } + #[inline] + pub fn contains(&self, term: &str) -> bool { + self.inner.contains_bytes(term.as_bytes()) + } + + /// Test membership of one shared logical UTF-8 scalar profile sequence. + #[inline] + pub fn contains_atom_sequence(&self, sequence: &crate::AtomSequence) -> bool { + self.inner.contains_bytes(&sequence.to_encoded()) + } + #[inline] + pub fn get_value(&self, term: &str) -> Option { + self.inner.get_bytes_value(term.as_bytes()) + } + + /// Read a mapped value for one shared logical UTF-8 scalar profile + /// sequence. + #[inline] + pub fn get_atom_sequence_value( + &self, + sequence: &crate::AtomSequence, + ) -> Option { + self.inner.get_bytes_value(&sequence.to_encoded()) + } + #[inline] + pub fn term_count(&self) -> usize { + self.inner.len().unwrap_or(0) + } + #[inline] + pub fn is_empty(&self) -> bool { + self.term_count() == 0 + } + pub fn contains_encoded(&self, encoded: &[u8]) -> Result { + std::str::from_utf8(encoded)?; + Ok(self.inner.contains_bytes(encoded)) + } + + /// Read a mapped value for one complete UTF-8 encoded term without + /// allocating or decoding its scalar sequence. Invalid UTF-8 is rejected + /// before the physical byte trie is consulted. + pub fn get_encoded_value(&self, encoded: &[u8]) -> Result, std::str::Utf8Error> { + std::str::from_utf8(encoded)?; + Ok(self.inner.get_bytes_value(encoded)) + } + + pub fn visible_entries(&self) -> Result)>, std::str::Utf8Error> { + self.inner + .entries() + .map(|entry| std::str::from_utf8(&entry.key).map(|s| (s.to_owned(), entry.value))) + .collect() + } +} + +impl DoubleArrayTrieUleb128 { + /// Canonical logical profile represented by this boundary. + pub const fn profile_descriptor() -> crate::factory::BackendProfileDescriptor { + crate::factory::BackendProfileDescriptor::from_profile::() + } + + /// Topology family represented by this boundary. + pub const fn dictionary_family() -> crate::factory::DictionaryFamily { + crate::factory::DictionaryFamily::DoubleArrayTrie + } + + /// Construct an empty ULEB128 DAT. + pub fn new() -> Self { + Self { + inner: DoubleArrayTrie::new(), + } + } + + /// Build from complete canonical ULEB128 sequences. + pub fn from_sequences(sequences: I) -> Self + where + I: IntoIterator, + { + Self { + inner: sequences + .into_iter() + .map(|sequence| sequence.to_encoded()) + .collect(), + } + } + + /// Build from the shared logical ULEB profile sequence representation. + pub fn from_atom_sequences(sequences: I) -> Self + where + I: IntoIterator>, + { + Self::from_sequences(sequences.into_iter().map(Into::into)) + } + + /// Build a value-bearing DAT from complete canonical ULEB128 sequences. + pub fn from_sequences_with_values(entries: I) -> Self + where + I: IntoIterator, + { + Self { + inner: entries + .into_iter() + .map(|(sequence, value)| (sequence.to_encoded(), value)) + .collect(), + } + } + + /// Build a value-bearing DAT from shared logical ULEB profile sequences. + pub fn from_atom_sequences_with_values(entries: I) -> Self + where + I: IntoIterator, V)>, + { + Self::from_sequences_with_values( + entries + .into_iter() + .map(|(sequence, value)| (sequence.into(), value)), + ) + } + + /// Test membership of one complete ULEB128 sequence. + #[inline] + pub fn contains(&self, sequence: &crate::Uleb128Sequence) -> bool { + self.inner.contains_bytes(&sequence.to_encoded()) + } + + /// Test membership of one shared logical ULEB profile sequence. + #[inline] + pub fn contains_atom_sequence( + &self, + sequence: &crate::AtomSequence, + ) -> bool { + self.inner.contains_bytes(&sequence.to_encoded()) + } + + /// Test a complete canonical encoded sequence without materializing its + /// decoded atoms. Malformed or non-canonical images are rejected. + pub fn contains_encoded(&self, encoded: &[u8]) -> Result { + crate::validate_uleb128_sequence(encoded)?; + Ok(self.inner.contains_bytes(encoded)) + } + + /// Read a mapped value for one complete ULEB128 sequence. + #[inline] + pub fn get_value(&self, sequence: &crate::Uleb128Sequence) -> Option { + self.inner.get_bytes_value(&sequence.to_encoded()) + } + + /// Read a mapped value for one shared logical ULEB profile sequence. + #[inline] + pub fn get_atom_sequence_value( + &self, + sequence: &crate::AtomSequence, + ) -> Option { + self.inner.get_bytes_value(&sequence.to_encoded()) + } + + /// Read a value for a complete canonical encoded sequence without + /// materializing its decoded atoms. + pub fn get_encoded_value(&self, encoded: &[u8]) -> Result, crate::Uleb128Error> { + crate::validate_uleb128_sequence(encoded)?; + Ok(self.inner.get_bytes_value(encoded)) + } + + /// Export complete logical sequences from the immutable DAT. + /// Traversal remains iterative in the byte-backed core; decoding occurs + /// only at this boundary and malformed images are rejected. + pub fn visible_entries( + &self, + ) -> Result)>, crate::Uleb128Error> { + self.inner + .entries() + .map(|entry| { + crate::Uleb128Sequence::from_encoded(&entry.key) + .map(|sequence| (sequence, entry.value)) + }) + .collect() + } + + /// Number of visible logical sequences. + #[inline] + pub fn term_count(&self) -> usize { + self.inner.len().unwrap_or(0) + } + + /// Whether no logical ULEB sequences are present. + #[inline] + pub fn is_empty(&self) -> bool { + self.term_count() == 0 + } +} + +impl Default for DoubleArrayTrieUleb128 { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod profile_tests { + use super::{ + DoubleArrayTrie, DoubleArrayTrieChar, DoubleArrayTrieUleb128, DoubleArrayTrieUtf8, + }; + use crate::{AtomSequence, Bytes, Dictionary, UnicodeScalar}; + + #[test] + fn uleb_wrapper_preserves_logical_sequences() { + assert!(DoubleArrayTrieUleb128::::new().is_empty()); + let sequence = crate::Uleb128Sequence::from_atoms([ + crate::Uleb128::from_u64(624_485), + crate::Uleb128::from_u64(7), + ]); + let dictionary = + DoubleArrayTrieUleb128::::from_sequences_with_values([(sequence.clone(), 19)]); + assert!(dictionary.contains(&sequence)); + assert_eq!(dictionary.get_value(&sequence), Some(19)); + assert_eq!(dictionary.term_count(), 1); + assert_eq!( + dictionary.contains_encoded(sequence.to_encoded().as_slice()), + Ok(true) + ); + assert!(dictionary.get_encoded_value(&[0x80]).is_err()); + assert_eq!(dictionary.visible_entries().unwrap().len(), 1); + } + + #[test] + fn uleb_wrapper_accepts_shared_profile_sequences() { + let sequence = crate::AtomSequence::::from_atoms([ + crate::Uleb128::from_u64(624_485), + crate::Uleb128::from_u64(1u64 << 63), + ]); + let dictionary = DoubleArrayTrieUleb128::::from_atom_sequences_with_values([( + sequence.clone(), + 23, + )]); + assert!(dictionary.contains_atom_sequence(&sequence)); + assert_eq!(dictionary.get_atom_sequence_value(&sequence), Some(23)); + } + + #[test] + fn utf8_wrapper_accepts_shared_profile_sequences() { + let sequence = crate::AtomSequence::::from_atoms(['Ξ»', 'πŸŽ‰']); + let dictionary = + DoubleArrayTrieUtf8::::from_atom_sequences_with_values([(sequence.clone(), 29)]); + assert!(dictionary.contains_atom_sequence(&sequence)); + assert_eq!(dictionary.get_atom_sequence_value(&sequence), Some(29)); + } + + #[test] + fn utf8_wrapper_preserves_logical_entries() { + let dictionary = DoubleArrayTrieUtf8::::from_terms_with_values([("Ξ»πŸŽ‰", 9), ("a", 1)]); + assert!(dictionary.contains("Ξ»πŸŽ‰")); + assert_eq!(dictionary.get_value("Ξ»πŸŽ‰"), Some(9)); + assert_eq!(dictionary.visible_entries().unwrap().len(), 2); + assert!(dictionary.contains_encoded("Ξ»πŸŽ‰".as_bytes()).unwrap()); + assert_eq!( + dictionary.get_encoded_value("Ξ»πŸŽ‰".as_bytes()).unwrap(), + Some(9) + ); + assert!(dictionary.get_encoded_value(&[0x80]).is_err()); + assert!(dictionary.contains_encoded(&[0x80]).is_err()); + } + + #[test] + fn byte_profile_sequences_use_the_existing_dat_builder() { + let dictionary: DoubleArrayTrie = DoubleArrayTrie::from_atom_sequences::([ + AtomSequence::::from_atoms([b'a', b'b']), + AtomSequence::::from_atoms([b'a', b'c']), + ]); + assert!(dictionary.contains("ab")); + assert!(dictionary.contains("ac")); + } + + #[test] + fn byte_profile_sequences_preserve_values() { + let dictionary = DoubleArrayTrie::::from_atom_sequences_with_values::([( + AtomSequence::::from_atoms([0, 255]), + 9, + )]); + assert_eq!(dictionary.get_bytes_value(&[0, 255]), Some(9)); + let sequence = AtomSequence::::from_atoms([0, 255]); + assert_eq!(dictionary.get_atom_sequence_value(&sequence), Some(9)); + } + + #[test] + fn unicode_profile_sequences_preserve_scalar_boundaries() { + let dictionary: DoubleArrayTrieChar = DoubleArrayTrieChar::from_atom_sequences::< + UnicodeScalar, + _, + >([ + AtomSequence::::from_atoms(['Ξ»', 'x']), + ]); + assert!(dictionary.contains("Ξ»x")); + assert!(!dictionary.contains("lx")); + } + + #[test] + fn unicode_profile_sequences_preserve_values() { + let dictionary = DoubleArrayTrieChar::::from_atom_sequences_with_values::< + UnicodeScalar, + _, + >([(AtomSequence::::from_atoms(['Ξ»']), 7)]); + assert_eq!(dictionary.get_value("Ξ»"), Some(7)); + assert_eq!(dictionary.get_chars_value(&['Ξ»']), Some(7)); + let sequence = AtomSequence::::from_atoms(['Ξ»']); + assert_eq!(dictionary.get_atom_sequence_value(&sequence), Some(7)); + } +} diff --git a/src/dynamic_dawg/ascii.rs b/src/dynamic_dawg/ascii.rs index 9dbd0c11..873fc3dd 100644 --- a/src/dynamic_dawg/ascii.rs +++ b/src/dynamic_dawg/ascii.rs @@ -60,6 +60,16 @@ pub struct DynamicDawg { pub(crate) type DynamicDawgInner = LockFreeDawg; impl DynamicDawg { + /// Canonical logical profile represented by this byte DAWG. + pub const fn profile_descriptor() -> crate::factory::BackendProfileDescriptor { + crate::factory::BackendProfileDescriptor::from_profile::() + } + + /// Topology family represented by this dictionary. + pub const fn dictionary_family() -> crate::factory::DictionaryFamily { + crate::factory::DictionaryFamily::DynamicDawg + } + /// Create a new empty dynamic DAWG. /// /// By default, auto-minimization is disabled. Use `with_auto_minimize_threshold()` diff --git a/src/dynamic_dawg/char.rs b/src/dynamic_dawg/char.rs index 671374b9..59bc193f 100644 --- a/src/dynamic_dawg/char.rs +++ b/src/dynamic_dawg/char.rs @@ -70,6 +70,16 @@ pub struct DynamicDawgChar { pub(crate) type DynamicDawgCharInner = LockFreeDawg; impl DynamicDawgChar { + /// Canonical logical profile represented by this Unicode DAWG. + pub const fn profile_descriptor() -> crate::factory::BackendProfileDescriptor { + crate::factory::BackendProfileDescriptor::from_profile::() + } + + /// Topology family represented by this dictionary. + pub const fn dictionary_family() -> crate::factory::DictionaryFamily { + crate::factory::DictionaryFamily::DynamicDawg + } + /// Create a new empty dynamic DAWG. /// /// By default, auto-minimization is disabled. Use `with_auto_minimize_threshold()` diff --git a/src/dynamic_dawg/lockfree.rs b/src/dynamic_dawg/lockfree.rs index 87f3a59d..15059fae 100644 --- a/src/dynamic_dawg/lockfree.rs +++ b/src/dynamic_dawg/lockfree.rs @@ -940,7 +940,6 @@ impl LockFreeDawg { /// /// A retained expected `Arc` is the CAS token, so an allocator cannot /// recycle its address while this attempt is live (pointer-ABA safety). - #[cfg(any(feature = "bindings-core", test))] pub(crate) fn clear(&self) -> bool { let mut backoff = CasBackoff::new(); loop { diff --git a/src/dynamic_dawg/mod.rs b/src/dynamic_dawg/mod.rs index be7ac09b..06bd956d 100644 --- a/src/dynamic_dawg/mod.rs +++ b/src/dynamic_dawg/mod.rs @@ -25,6 +25,829 @@ pub use u64::{DynamicDawgU64, DynamicDawgU64Node}; pub use u64_zipper::DynamicDawgU64Zipper; pub use zipper::DynamicDawgZipper; +/// Public unit-generic dynamic DAWG surface. +/// +/// The legacy string-oriented aliases remain unchanged; this type exposes the +/// shared lock-free core directly for callers that already own logical units. +#[derive(Clone, Debug)] +pub struct DynamicDawgGeneric { + inner: std::sync::Arc>, +} + +impl DynamicDawgGeneric { + /// Return the canonical profile descriptor for a profile whose logical + /// atom type is this dictionary's unit type. + pub const fn profile_descriptor

() -> crate::factory::BackendProfileDescriptor + where + P: crate::AtomProfile, + { + crate::factory::BackendProfileDescriptor::from_profile::

() + } + + /// Topology family represented by this generic core. + pub const fn dictionary_family() -> crate::factory::DictionaryFamily { + crate::factory::DictionaryFamily::DynamicDawg + } + + /// Construct an empty generic DAWG. + pub fn new() -> Self { + Self { + inner: std::sync::Arc::new(lockfree::LockFreeDawg::new()), + } + } + + /// Build from lexicographically sorted logical-unit sequences. + pub fn from_sorted_sequences(sequences: I) -> Self + where + I: IntoIterator, + S: AsRef<[U]>, + { + Self { + inner: std::sync::Arc::new(lockfree::LockFreeDawg::from_sorted_terms_by( + sequences, + |sequence, units| units.extend_from_slice(sequence.as_ref()), + )), + } + } + + /// Build from arbitrary logical-unit sequences, sorting once for + /// deterministic and suffix-sharing-friendly construction. + pub fn from_sequences(sequences: I) -> Self + where + I: IntoIterator, + S: AsRef<[U]>, + { + let mut owned: Vec> = sequences + .into_iter() + .map(|sequence| sequence.as_ref().to_vec()) + .collect(); + owned.sort_unstable(); + Self::from_sorted_sequences(owned) + } + + /// Build from owned logical sequences supplied by an [`AtomProfile`]. + /// + /// The profile is consumed only at the API boundary; the DAWG stores and + /// traverses the profile's logical units directly, so no wire decoding is + /// introduced into the lookup hot path. + pub fn from_atom_sequences(sequences: I) -> Self + where + P: crate::AtomProfile, + I: IntoIterator>, + { + Self::from_sequences( + sequences + .into_iter() + .map(|sequence| sequence.as_atoms().to_vec()), + ) + } + + /// Build a value-bearing DAWG from profile sequences and their values. + pub fn from_atom_sequences_with_values(entries: I) -> Self + where + P: crate::AtomProfile, + I: IntoIterator, V)>, + { + Self::from_sorted_entries( + entries + .into_iter() + .map(|(sequence, value)| (sequence.as_atoms().to_vec(), value)), + ) + } + + /// Build a value-bearing DAWG from lexicographically sorted sequences. + pub fn from_sorted_entries(entries: I) -> Self + where + I: IntoIterator, + S: AsRef<[U]>, + { + Self { + inner: std::sync::Arc::new(lockfree::LockFreeDawg::from_sorted_entries_by( + entries + .into_iter() + .map(|(sequence, value)| (sequence, Some(value))), + |sequence, units| units.extend_from_slice(sequence.as_ref()), + )), + } + } + + /// Insert one logical-unit sequence. + #[inline] + pub fn insert_units(&self, units: &[U]) -> bool { + self.inner.insert_units(units) + } + + /// Insert a logical sequence produced by a fixed-width atom profile. + /// + /// The profile is a compile-time witness that the sequence's atoms are + /// the dictionary's traversal units; no encoded-byte decoding occurs in + /// the DAWG hot path. + #[inline] + pub fn insert_atom_sequence

(&self, sequence: &crate::AtomSequence

) -> bool + where + P: crate::AtomProfile, + { + self.insert_units(sequence.as_atoms()) + } + + /// Insert a profile sequence with an associated mapped value. + #[inline] + pub fn insert_atom_sequence_with_value

( + &self, + sequence: &crate::AtomSequence

, + value: V, + ) -> bool + where + P: crate::AtomProfile, + { + self.insert_units_with_value(sequence.as_atoms(), value) + } + + /// Insert one sequence with an associated value. + #[inline] + pub fn insert_units_with_value(&self, units: &[U], value: V) -> bool { + self.inner.insert_units_with_value(units, value) + } + + /// Test membership using logical units. + #[inline] + pub fn contains_units(&self, units: &[U]) -> bool { + self.inner.contains_units(units) + } + + /// Query a profile sequence directly in logical-unit space. + #[inline] + pub fn contains_atom_sequence

(&self, sequence: &crate::AtomSequence

) -> bool + where + P: crate::AtomProfile, + { + self.contains_units(sequence.as_atoms()) + } + + /// Read a mapped value for a profile sequence directly in logical-unit + /// space. + #[inline] + pub fn get_atom_sequence_value

(&self, sequence: &crate::AtomSequence

) -> Option + where + P: crate::AtomProfile, + { + self.get_units_value(sequence.as_atoms()) + } + + /// Remove a profile sequence directly in logical-unit space. + #[inline] + pub fn remove_atom_sequence

(&self, sequence: &crate::AtomSequence

) -> bool + where + P: crate::AtomProfile, + { + self.remove_units(sequence.as_atoms()) + } + + /// Read the value associated with a logical-unit sequence. + #[inline] + pub fn get_units_value(&self, units: &[U]) -> Option { + self.inner.get_units_value(units) + } + + /// Remove a logical-unit sequence. + #[inline] + pub fn remove_units(&self, units: &[U]) -> bool { + self.inner.remove_units(units) + } + + /// Remove every logical-unit sequence from the current revision. + #[inline] + pub fn clear(&self) -> bool { + self.inner.clear() + } + + /// Number of visible terminal sequences. + #[inline] + pub fn term_count(&self) -> usize { + self.inner.term_count() + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.term_count() == 0 + } + + /// Number of physical nodes in the current graph revision. + #[inline] + pub fn node_count(&self) -> usize { + self.inner.node_count() + } + + /// Whether the current revision has pending non-minimal structure. + #[inline] + pub fn needs_compaction(&self) -> bool { + self.inner.needs_compaction() + } + + /// Collect visible logical-unit entries in deterministic lexicographic + /// order for snapshot/export boundaries. + pub fn visible_entries(&self) -> Vec<(Vec, Option)> { + self.inner.collect_visible_entries() + } + + /// Compact/minimize the current immutable graph. + #[inline] + pub fn compact(&self) -> usize { + self.inner.compact() + } +} + +impl Default for DynamicDawgGeneric { + fn default() -> Self { + Self::new() + } +} + +/// Alias emphasizing that this wrapper accepts profile-defined units. +pub type DynamicDawgProfile = DynamicDawgGeneric; + +/// Named aliases for the common profile unit specializations. +pub type DynamicDawgByteProfile = DynamicDawgGeneric; +pub type DynamicDawgCharProfile = DynamicDawgGeneric; + +/// Source-compatible alias for native 32-bit logical units. +pub type DynamicDawgU32 = DynamicDawgGeneric; + +/// Raw-bit IEEE-754 binary64 specialization. Values are supplied as `u64` +/// bit patterns; use [`crate::F64Bits::total_cmp`] when semantic ordering is +/// required rather than the storage order of the unsigned carrier. +pub type DynamicDawgF64Bits = DynamicDawgGeneric; + +/// Source-compatible alias for native 64-bit logical units. +pub type DynamicDawgU64Profile = DynamicDawgGeneric; + +/// Variable-width ULEB128 DAWG boundary. +/// +/// Canonical ULEB atoms are packed into the byte-oriented core for storage, +/// while this wrapper accepts and returns complete logical atom sequences. The +/// encoded continuation bytes are never exposed as dictionary transitions. +#[derive(Clone, Debug)] +pub struct DynamicDawgUleb128 { + inner: DynamicDawgGeneric, +} + +/// Variable-width UTF-8 dictionary boundary. +/// +/// UTF-8 bytes are retained by the byte-oriented core, while this wrapper +/// validates and exposes complete Unicode strings so continuation bytes never +/// become logical transitions. +#[derive(Clone, Debug)] +pub struct DynamicDawgUtf8 { + inner: DynamicDawgGeneric, +} + +impl Default for DynamicDawgUtf8 { + fn default() -> Self { + Self::new() + } +} + +impl DynamicDawgUtf8 { + /// Canonical logical profile represented by this boundary. + pub const fn profile_descriptor() -> crate::factory::BackendProfileDescriptor { + crate::factory::BackendProfileDescriptor::from_profile::() + } + + /// Topology family represented by this boundary. + pub const fn dictionary_family() -> crate::factory::DictionaryFamily { + crate::factory::DictionaryFamily::DynamicDawg + } + + pub fn new() -> Self { + Self { + inner: DynamicDawgGeneric::new(), + } + } + + pub fn from_terms(terms: I) -> Self + where + I: IntoIterator, + S: AsRef, + { + Self { + inner: DynamicDawgGeneric::from_sequences( + terms.into_iter().map(|s| s.as_ref().as_bytes().to_vec()), + ), + } + } + + pub fn from_terms_with_values(entries: I) -> Self + where + I: IntoIterator, + S: AsRef, + { + let mut encoded = entries + .into_iter() + .map(|(s, v)| (s.as_ref().as_bytes().to_vec(), v)) + .collect::>(); + encoded.sort_unstable_by(|a, b| a.0.cmp(&b.0)); + Self { + inner: DynamicDawgGeneric::from_sorted_entries(encoded), + } + } + + /// Build from shared logical UTF-8 scalar profile sequences. + pub fn from_atom_sequences(sequences: I) -> Self + where + I: IntoIterator>, + { + Self { + inner: DynamicDawgGeneric::from_sequences( + sequences.into_iter().map(|sequence| sequence.to_encoded()), + ), + } + } + + /// Build a value-bearing dictionary from shared logical UTF-8 scalar + /// profile sequences. + pub fn from_atom_sequences_with_values(entries: I) -> Self + where + I: IntoIterator, V)>, + { + Self { + inner: DynamicDawgGeneric::from_sorted_entries( + entries + .into_iter() + .map(|(sequence, value)| (sequence.to_encoded(), value)), + ), + } + } + + #[inline] + pub fn insert(&self, term: &str) -> bool { + self.inner.insert_units(term.as_bytes()) + } + + /// Insert one shared logical UTF-8 scalar profile sequence. + #[inline] + pub fn insert_atom_sequence(&self, sequence: &crate::AtomSequence) -> bool { + self.inner.insert_units(&sequence.to_encoded()) + } + #[inline] + pub fn insert_with_value(&self, term: &str, value: V) -> bool { + self.inner.insert_units_with_value(term.as_bytes(), value) + } + + /// Insert one complete UTF-8 encoded key without coercing it through a + /// string allocation. Invalid UTF-8 is rejected before any mutation. + pub fn insert_encoded(&self, encoded: &[u8], value: V) -> Result { + std::str::from_utf8(encoded)?; + Ok(self.inner.insert_units_with_value(encoded, value)) + } + + /// Insert one shared logical UTF-8 scalar profile sequence with a value. + #[inline] + pub fn insert_atom_sequence_with_value( + &self, + sequence: &crate::AtomSequence, + value: V, + ) -> bool { + self.inner + .insert_units_with_value(&sequence.to_encoded(), value) + } + #[inline] + pub fn contains(&self, term: &str) -> bool { + self.inner.contains_units(term.as_bytes()) + } + + /// Test membership of one shared logical UTF-8 scalar profile sequence. + #[inline] + pub fn contains_atom_sequence(&self, sequence: &crate::AtomSequence) -> bool { + self.inner.contains_units(&sequence.to_encoded()) + } + #[inline] + pub fn get_value(&self, term: &str) -> Option { + self.inner.get_units_value(term.as_bytes()) + } + + /// Read a mapped value for one shared logical UTF-8 scalar profile + /// sequence. + #[inline] + pub fn get_atom_sequence_value( + &self, + sequence: &crate::AtomSequence, + ) -> Option { + self.inner.get_units_value(&sequence.to_encoded()) + } + #[inline] + pub fn remove(&self, term: &str) -> bool { + self.inner.remove_units(term.as_bytes()) + } + + /// Remove one shared logical UTF-8 scalar profile sequence. + #[inline] + pub fn remove_atom_sequence(&self, sequence: &crate::AtomSequence) -> bool { + self.inner.remove_units(&sequence.to_encoded()) + } + #[inline] + pub fn term_count(&self) -> usize { + self.inner.term_count() + } + #[inline] + pub fn is_empty(&self) -> bool { + self.term_count() == 0 + } + #[inline] + pub fn node_count(&self) -> usize { + self.inner.node_count() + } + + pub fn contains_encoded(&self, encoded: &[u8]) -> Result { + std::str::from_utf8(encoded)?; + Ok(self.inner.contains_units(encoded)) + } + + /// Validate and read a mapped value for one encoded UTF-8 term without + /// allocating or decoding its scalar sequence. + #[inline] + pub fn get_encoded_value(&self, encoded: &[u8]) -> Result, std::str::Utf8Error> { + std::str::from_utf8(encoded)?; + Ok(self.inner.get_units_value(encoded)) + } + + pub fn remove_encoded(&self, encoded: &[u8]) -> Result { + std::str::from_utf8(encoded)?; + Ok(self.inner.remove_units(encoded)) + } + + pub fn visible_entries(&self) -> Result)>, std::str::Utf8Error> { + self.inner + .visible_entries() + .into_iter() + .map(|(bytes, value)| std::str::from_utf8(&bytes).map(|term| (term.to_owned(), value))) + .collect() + } +} + +impl Default for DynamicDawgUleb128 { + fn default() -> Self { + Self::new() + } +} + +impl DynamicDawgUleb128 { + /// Canonical logical profile represented by this boundary. + pub const fn profile_descriptor() -> crate::factory::BackendProfileDescriptor { + crate::factory::BackendProfileDescriptor::from_profile::() + } + + /// Topology family represented by this boundary. + pub const fn dictionary_family() -> crate::factory::DictionaryFamily { + crate::factory::DictionaryFamily::DynamicDawg + } + + /// Construct an empty ULEB128 dictionary. + pub fn new() -> Self { + Self { + inner: DynamicDawgGeneric::new(), + } + } + + /// Build from complete canonical ULEB128 sequences. + pub fn from_sequences(sequences: I) -> Self + where + I: IntoIterator, + { + let inner = DynamicDawgGeneric::from_sequences( + sequences.into_iter().map(|sequence| sequence.to_encoded()), + ); + Self { inner } + } + + /// Build from the shared logical ULEB profile sequence representation. + pub fn from_atom_sequences(sequences: I) -> Self + where + I: IntoIterator>, + { + Self::from_sequences(sequences.into_iter().map(Into::into)) + } + + /// Build a value-bearing dictionary from complete ULEB128 sequences. + pub fn from_sequences_with_values(entries: I) -> Self + where + I: IntoIterator, + { + let mut encoded: Vec<(Vec, V)> = entries + .into_iter() + .map(|(sequence, value)| (sequence.to_encoded(), value)) + .collect(); + encoded.sort_unstable_by(|left, right| left.0.cmp(&right.0)); + let inner = DynamicDawgGeneric::from_sorted_entries(encoded); + Self { inner } + } + + /// Build a value-bearing dictionary from shared logical ULEB profile + /// sequences. + pub fn from_atom_sequences_with_values(entries: I) -> Self + where + I: IntoIterator, V)>, + { + Self::from_sequences_with_values( + entries + .into_iter() + .map(|(sequence, value)| (sequence.into(), value)), + ) + } + + /// Insert one complete ULEB128 sequence. + #[inline] + pub fn insert(&self, sequence: &crate::Uleb128Sequence) -> bool { + self.inner.insert_units(&sequence.to_encoded()) + } + + /// Insert one shared logical ULEB profile sequence. + #[inline] + pub fn insert_atom_sequence(&self, sequence: &crate::AtomSequence) -> bool { + self.inner.insert_units(&sequence.to_encoded()) + } + + /// Insert one complete ULEB128 sequence with a mapped value. + #[inline] + pub fn insert_with_value(&self, sequence: &crate::Uleb128Sequence, value: V) -> bool { + self.inner + .insert_units_with_value(&sequence.to_encoded(), value) + } + + /// Insert one shared logical ULEB profile sequence with a mapped value. + #[inline] + pub fn insert_atom_sequence_with_value( + &self, + sequence: &crate::AtomSequence, + value: V, + ) -> bool { + self.inner + .insert_units_with_value(&sequence.to_encoded(), value) + } + + /// Insert one complete canonical encoded ULEB128 sequence without + /// materializing its decoded atoms. + pub fn insert_encoded(&self, encoded: &[u8], value: V) -> Result { + crate::validate_uleb128_sequence(encoded)?; + Ok(self.inner.insert_units_with_value(encoded, value)) + } + + /// Test membership of one complete ULEB128 sequence. + #[inline] + pub fn contains(&self, sequence: &crate::Uleb128Sequence) -> bool { + self.inner.contains_units(&sequence.to_encoded()) + } + + /// Test membership of one shared logical ULEB profile sequence. + #[inline] + pub fn contains_atom_sequence( + &self, + sequence: &crate::AtomSequence, + ) -> bool { + self.inner.contains_units(&sequence.to_encoded()) + } + + /// Test a complete canonical encoded sequence without first allocating an + /// owned [`Uleb128Sequence`]. Validation is kept at this boundary so + /// continuation bytes can never become visible DAWG transitions. + pub fn contains_encoded(&self, encoded: &[u8]) -> Result { + crate::validate_uleb128_sequence(encoded)?; + Ok(self.inner.contains_units(encoded)) + } + + /// Read a mapped value for one complete ULEB128 sequence. + #[inline] + pub fn get_value(&self, sequence: &crate::Uleb128Sequence) -> Option { + self.inner.get_units_value(&sequence.to_encoded()) + } + + /// Read a mapped value for one shared logical ULEB profile sequence. + #[inline] + pub fn get_atom_sequence_value( + &self, + sequence: &crate::AtomSequence, + ) -> Option { + self.inner.get_units_value(&sequence.to_encoded()) + } + + /// Read a value for a complete canonical encoded sequence without + /// materializing its decoded atoms. + pub fn get_encoded_value(&self, encoded: &[u8]) -> Result, crate::Uleb128Error> { + crate::validate_uleb128_sequence(encoded)?; + Ok(self.inner.get_units_value(encoded)) + } + + /// Remove one complete canonical encoded ULEB128 sequence without + /// materializing its arbitrary-width atoms. + pub fn remove_encoded(&self, encoded: &[u8]) -> Result { + crate::validate_uleb128_sequence(encoded)?; + Ok(self.inner.remove_units(encoded)) + } + + /// Remove one complete ULEB128 sequence. + #[inline] + pub fn remove(&self, sequence: &crate::Uleb128Sequence) -> bool { + self.inner.remove_units(&sequence.to_encoded()) + } + + /// Number of visible logical sequences. + #[inline] + pub fn term_count(&self) -> usize { + self.inner.term_count() + } + + /// Number of physical byte nodes. + #[inline] + pub fn node_count(&self) -> usize { + self.inner.node_count() + } + + /// Export logical sequences, rejecting any malformed internal image. + pub fn visible_entries( + &self, + ) -> Result)>, crate::Uleb128Error> { + self.inner + .visible_entries() + .into_iter() + .map(|(bytes, value)| { + crate::Uleb128Sequence::from_encoded(&bytes).map(|sequence| (sequence, value)) + }) + .collect() + } +} + +#[cfg(test)] +mod generic_tests { + use super::DynamicDawgGeneric; + + #[test] + fn uleb_wrapper_preserves_atom_boundaries() { + let first = crate::Uleb128::from_u64(624_485); + let second = crate::Uleb128::from_payload_digits(&[3, 4]).unwrap(); + let sequence = crate::Uleb128Sequence::from_atoms([first, second]); + let dictionary = super::DynamicDawgUleb128::::new(); + assert!(dictionary.insert_with_value(&sequence, 9)); + assert!(dictionary.contains(&sequence)); + assert_eq!(dictionary.get_value(&sequence), Some(9)); + let encoded = sequence.to_encoded(); + assert!(dictionary.insert_encoded(&encoded, 10).is_ok()); + assert_eq!(dictionary.get_encoded_value(&encoded).unwrap(), Some(10)); + assert!(dictionary.insert_encoded(&[0x80], 1).is_err()); + assert!(dictionary.remove_encoded(&encoded).unwrap()); + assert!(!dictionary.contains_encoded(&encoded).unwrap()); + assert!(!dictionary.remove_encoded(&encoded).unwrap()); + assert!(dictionary.insert_encoded(&encoded, 10).unwrap()); + let entries = dictionary.visible_entries().unwrap(); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].0, sequence); + assert!(dictionary.remove(&sequence)); + assert!(!dictionary.contains(&sequence)); + } + + #[test] + fn generic_surface_uses_logical_units_directly() { + let dictionary = DynamicDawgGeneric::::new(); + assert!(dictionary.insert_units(&[1, 2, 3])); + assert!(dictionary.contains_units(&[1, 2, 3])); + assert!(!dictionary.contains_units(&[1, 2])); + assert!(dictionary.remove_units(&[1, 2, 3])); + assert!(!dictionary.contains_units(&[1, 2, 3])); + assert!(dictionary.insert_units_with_value(&[4], 99)); + assert_eq!(dictionary.get_units_value(&[4]), Some(99)); + assert!(dictionary.node_count() > 0); + assert!(dictionary.clear()); + assert_eq!(dictionary.term_count(), 0); + } + + #[test] + fn encoded_lookup_rejects_malformed_and_preserves_zero_copy_boundary() { + let atom = crate::Uleb128::from_u64(624_485); + let sequence = crate::Uleb128Sequence::from_atoms([atom]); + let dictionary = super::DynamicDawgUleb128::::from_sequences([sequence.clone()]); + assert_eq!( + dictionary.contains_encoded(sequence.to_encoded().as_slice()), + Ok(true) + ); + assert!(dictionary.contains_encoded(&[0x80]).is_err()); + } + + #[test] + fn utf8_wrapper_preserves_scalar_boundaries() { + let dictionary = + super::DynamicDawgUtf8::::from_terms_with_values([("Ξ»πŸŽ‰", 4), ("a", 1)]); + assert!(dictionary.contains("Ξ»πŸŽ‰")); + assert_eq!(dictionary.get_value("Ξ»πŸŽ‰"), Some(4)); + assert_eq!(dictionary.visible_entries().unwrap().len(), 2); + assert!(dictionary.contains_encoded("Ξ»πŸŽ‰".as_bytes()).unwrap()); + assert_eq!( + dictionary.get_encoded_value("Ξ»πŸŽ‰".as_bytes()).unwrap(), + Some(4) + ); + assert!(dictionary.contains_encoded(&[0x80]).is_err()); + assert!(dictionary.get_encoded_value(&[0x80]).is_err()); + assert!(dictionary.insert_encoded("Ξ²".as_bytes(), 8).unwrap()); + assert_eq!(dictionary.get_value("Ξ²"), Some(8)); + assert!(dictionary.insert_encoded(&[0x80], 1).is_err()); + assert!(!dictionary.is_empty()); + assert!(dictionary.remove_encoded("Ξ»πŸŽ‰".as_bytes()).unwrap()); + assert!(!dictionary.contains("Ξ»πŸŽ‰")); + } + + #[test] + fn utf8_wrapper_accepts_shared_profile_sequences() { + let sequence = crate::AtomSequence::::from_atoms(['Ξ»', 'πŸŽ‰']); + let dictionary = super::DynamicDawgUtf8::::from_atom_sequences_with_values([( + sequence.clone(), + 29, + )]); + assert!(dictionary.contains_atom_sequence(&sequence)); + assert_eq!(dictionary.get_atom_sequence_value(&sequence), Some(29)); + assert!(dictionary.remove_atom_sequence(&sequence)); + assert!(!dictionary.contains_atom_sequence(&sequence)); + } + + #[test] + fn generic_surface_builds_from_profile_sequences() { + let dictionary = DynamicDawgGeneric::::from_atom_sequences::([ + crate::AtomSequence::::from_atoms([7, 11]), + crate::AtomSequence::::from_atoms([7, 13]), + ]); + assert!(dictionary.contains_units(&[7, 11])); + assert!(dictionary.contains_units(&[7, 13])); + } + + #[test] + fn generic_surface_builds_profile_sequences_with_values() { + let dictionary = DynamicDawgGeneric::::from_atom_sequences_with_values::< + crate::U32, + _, + >([(crate::AtomSequence::from_atoms([3, 5]), 42)]); + assert_eq!(dictionary.get_units_value(&[3, 5]), Some(42)); + let sequence = crate::AtomSequence::::from_atoms([3, 5]); + assert_eq!(dictionary.get_atom_sequence_value(&sequence), Some(42)); + } + + #[test] + fn uleb_wrapper_accepts_shared_profile_sequences() { + let sequence = crate::AtomSequence::::from_atoms([ + crate::Uleb128::from_u64(624_485), + crate::Uleb128::from_u64(1u64 << 63), + ]); + let dictionary = super::DynamicDawgUleb128::::from_atom_sequences_with_values([( + sequence.clone(), + 17, + )]); + assert!(dictionary.contains_atom_sequence(&sequence)); + assert_eq!(dictionary.get_atom_sequence_value(&sequence), Some(17)); + assert!(dictionary.insert_atom_sequence(&sequence) == false); + } + + #[test] + fn generic_batch_constructor_uses_sorted_logical_sequences() { + let dictionary = + DynamicDawgGeneric::::from_sorted_sequences([vec![1u32, 2], vec![1, 3]]); + assert!(dictionary.contains_units(&[1, 2])); + assert!(dictionary.contains_units(&[1, 3])); + assert_eq!(dictionary.term_count(), 2); + let unsorted = DynamicDawgGeneric::::from_sequences([vec![1u32, 3], vec![1, 2]]); + assert!(unsorted.contains_units(&[1, 2])); + assert!(unsorted.contains_units(&[1, 3])); + let valued = DynamicDawgGeneric::::from_sorted_entries([ + (vec![1u32, 2], 10), + (vec![1, 3], 20), + ]); + assert_eq!(valued.get_units_value(&[1, 2]), Some(10)); + assert_eq!(valued.get_units_value(&[1, 3]), Some(20)); + assert_eq!( + valued.visible_entries(), + vec![(vec![1, 2], Some(10)), (vec![1, 3], Some(20))] + ); + } + + #[test] + fn profile_sequences_are_consumed_without_encoded_byte_decoding() { + let dictionary = DynamicDawgGeneric::::new(); + let sequence = crate::AtomSequence::::from_atoms([7, 11, 13]); + assert!(dictionary.insert_atom_sequence_with_value(&sequence, 41)); + assert!(dictionary.contains_atom_sequence(&sequence)); + assert_eq!(dictionary.get_units_value(sequence.as_atoms()), Some(41)); + assert!(dictionary.remove_atom_sequence(&sequence)); + assert!(!dictionary.contains_units(&[7, 11])); + } + + #[test] + fn f64_bits_alias_preserves_exact_payloads() { + let negative_zero = (-0.0f64).to_bits(); + let nan_payload = 0x7ff8_0000_0000_0042u64; + let dictionary = super::DynamicDawgF64Bits::::new(); + assert!(dictionary.insert_units_with_value(&[negative_zero, nan_payload], 7)); + assert_eq!( + dictionary.get_units_value(&[negative_zero, nan_payload]), + Some(7) + ); + assert!(!dictionary.contains_units(&[0, nan_payload])); + } +} + /// Opaque provenance-bearing cursor into one immutable DynamicDAWG revision. /// /// This type is deliberately distinct from [`crate::DenseSnapshotCursor`]. It diff --git a/src/dynamic_dawg/u64.rs b/src/dynamic_dawg/u64.rs index 5df38a60..f74803cc 100644 --- a/src/dynamic_dawg/u64.rs +++ b/src/dynamic_dawg/u64.rs @@ -114,6 +114,16 @@ impl Default for DynamicDawgU64 { } impl DynamicDawgU64 { + /// Canonical logical profile represented by this U64 DAWG. + pub const fn profile_descriptor() -> crate::factory::BackendProfileDescriptor { + crate::factory::BackendProfileDescriptor::from_profile::() + } + + /// Topology family represented by this dictionary. + pub const fn dictionary_family() -> crate::factory::DictionaryFamily { + crate::factory::DictionaryFamily::DynamicDawg + } + /// Create a new empty dynamic DAWG. /// /// # Example @@ -1494,23 +1504,29 @@ mod tests { } let dawg = StdArc::new(dawg); - // Spawn 100 concurrent readers - let handles: Vec<_> = (0..100) - .map(|reader_id| { - let dawg = StdArc::clone(&dawg); - thread::spawn(move || { - // Each reader does 1000 lookups - for i in 0u64..1000 { - let seq = [i, i + 1, i + 2]; - let found = dawg.contains_sequence(&seq); - assert!(found, "Reader {reader_id} failed to find sequence {i}"); - } + // Exercise 100 readers while keeping the simultaneously live thread + // count bounded for constrained test runners. + const READER_COUNT: usize = 100; + const BATCH_SIZE: usize = 8; + for batch_start in (0..READER_COUNT).step_by(BATCH_SIZE) { + let batch_end = (batch_start + BATCH_SIZE).min(READER_COUNT); + let handles: Vec<_> = (batch_start..batch_end) + .map(|reader_id| { + let dawg = StdArc::clone(&dawg); + thread::spawn(move || { + // Each reader does 1000 lookups. + for i in 0u64..1000 { + let seq = [i, i + 1, i + 2]; + let found = dawg.contains_sequence(&seq); + assert!(found, "Reader {reader_id} failed to find sequence {i}"); + } + }) }) - }) - .collect(); + .collect(); - for handle in handles { - handle.join().expect("Reader thread panicked"); + for handle in handles { + handle.join().expect("Reader thread panicked"); + } } } @@ -1528,8 +1544,8 @@ mod tests { let dawg = StdArc::new(dawg); let stop = StdArc::new(AtomicBool::new(false)); - // 10 reader threads - let reader_handles: Vec<_> = (0..10) + // Keep the live reader set bounded for constrained test runners. + let reader_handles: Vec<_> = (0..4) .map(|_| { let dawg = StdArc::clone(&dawg); let stop = StdArc::clone(&stop); @@ -1547,23 +1563,23 @@ mod tests { }) .collect(); - // 10 writer threads - let writer_handles: Vec<_> = (0..10) - .map(|writer_id| { - let dawg = StdArc::clone(&dawg); - thread::spawn(move || { - // Each writer inserts 100 sequences in its own range - let base = 1000 + (writer_id as u64 * 100); - for i in 0u64..100 { - dawg.insert_sequence(&[base + i, base + i + 1, base + i + 2]); - } + // Run all ten writers in bounded concurrent batches while readers stay active. + for batch_start in (0..10).step_by(4) { + let batch_end = (batch_start + 4).min(10); + let writer_handles: Vec<_> = (batch_start..batch_end) + .map(|writer_id| { + let dawg = StdArc::clone(&dawg); + thread::spawn(move || { + let base = 1000 + (writer_id as u64 * 100); + for i in 0u64..100 { + dawg.insert_sequence(&[base + i, base + i + 1, base + i + 2]); + } + }) }) - }) - .collect(); - - // Wait for writers to complete - for handle in writer_handles { - handle.join().expect("Writer thread panicked"); + .collect(); + for handle in writer_handles { + handle.join().expect("Writer thread panicked"); + } } // Signal readers to stop @@ -1639,25 +1655,27 @@ mod tests { let dawg: DynamicDawgU64<()> = DynamicDawgU64::new(); let dawg = StdArc::new(dawg); - // 50 writers, each inserting 100 unique sequences in disjoint ranges - let handles: Vec<_> = (0..50) - .map(|writer_id| { - let dawg = StdArc::clone(&dawg); - thread::spawn(move || { - let base = writer_id as u64 * 1000; - for i in 0u64..100 { - let inserted = dawg.insert_sequence(&[base + i, base + i + 1]); - assert!( - inserted, - "Writer {writer_id} failed to insert unique seq {i}" - ); - } + // Run all 50 writers in bounded concurrent batches. + for batch_start in (0..50).step_by(8) { + let batch_end = (batch_start + 8).min(50); + let handles: Vec<_> = (batch_start..batch_end) + .map(|writer_id| { + let dawg = StdArc::clone(&dawg); + thread::spawn(move || { + let base = writer_id as u64 * 1000; + for i in 0u64..100 { + let inserted = dawg.insert_sequence(&[base + i, base + i + 1]); + assert!( + inserted, + "Writer {writer_id} failed to insert unique seq {i}" + ); + } + }) }) - }) - .collect(); - - for handle in handles { - handle.join().expect("Writer thread panicked"); + .collect(); + for handle in handles { + handle.join().expect("Writer thread panicked"); + } } // 50 writers Γ— 100 sequences = 5000 total diff --git a/src/factory.rs b/src/factory.rs index 14810a61..146b3b3d 100644 --- a/src/factory.rs +++ b/src/factory.rs @@ -25,19 +25,189 @@ //! ``` use super::double_array_trie::char::DoubleArrayTrieChar; -use super::double_array_trie::DoubleArrayTrie; +use super::double_array_trie::{DoubleArrayTrie, DoubleArrayTrieUleb128, DoubleArrayTrieUtf8}; use super::dynamic_dawg::char::DynamicDawgChar; use super::dynamic_dawg::u64::DynamicDawgU64; -use super::dynamic_dawg::DynamicDawg; +use super::dynamic_dawg::{DynamicDawg, DynamicDawgUleb128, DynamicDawgUtf8}; #[cfg(feature = "pathmap-backend")] use super::pathmap::char::PathMapDictionaryChar; #[cfg(feature = "pathmap-backend")] use super::pathmap::PathMapDictionary; +#[cfg(feature = "pathmap-backend")] +use super::pathmap::PathMapDictionaryUleb128; +#[cfg(feature = "pathmap-backend")] +use super::pathmap::PathMapDictionaryUtf8; use super::scdawg::char::ScdawgChar; use super::scdawg::Scdawg; use super::suffix_automaton::char::SuffixAutomatonChar; use super::suffix_automaton::SuffixAutomaton; use super::{Dictionary, SyncStrategy}; +use crate::{ProfileKind, VariableWidthProfile}; +use crate::{Uleb128Error, Uleb128Sequence}; +use core::marker::PhantomData; + +/// Dictionary topology independent of the logical atom representation. +/// +/// This is deliberately smaller than [`DictionaryBackend`]: the latter keeps +/// legacy backend/profile spellings for source compatibility, while this type +/// describes the storage family that can be paired with a profile descriptor. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum DictionaryFamily { + /// Mutable directed acyclic word graph. + DynamicDawg, + /// Immutable double-array trie. + DoubleArrayTrie, + /// Third-party PathMap adapter. + PathMap, + /// Dynamic suffix automaton. + SuffixAutomaton, + /// Batch-built compact suffix DAWG. + Scdawg, + /// Persistent adaptive radix trie. + PersistentArTrie, +} + +impl DictionaryFamily { + /// Stable topology identifier for manifests and capability negotiation. + pub const fn as_str(self) -> &'static str { + match self { + Self::DynamicDawg => "dynamic-dawg", + Self::DoubleArrayTrie => "double-array-trie", + Self::PathMap => "path-map", + Self::SuffixAutomaton => "suffix-automaton", + Self::Scdawg => "scdawg", + Self::PersistentArTrie => "persistent-artrie", + } + } + + /// Parse a stable topology identifier. + pub fn from_name(name: &str) -> Option { + match name { + "dynamic-dawg" => Some(Self::DynamicDawg), + "double-array-trie" => Some(Self::DoubleArrayTrie), + "path-map" => Some(Self::PathMap), + "suffix-automaton" => Some(Self::SuffixAutomaton), + "scdawg" => Some(Self::Scdawg), + "persistent-artrie" => Some(Self::PersistentArTrie), + _ => None, + } + } +} + +/// Stable, wire-facing topology/profile descriptor for manifests. +/// +/// The descriptor deliberately stores canonical strings and the profile +/// version rather than Rust enum or type names. It can therefore be carried +/// through JSON/bincode and validated before a persisted image is opened. +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + feature = "serialization", + derive(serde::Serialize, serde::Deserialize) +)] +pub struct DictionaryDescriptor { + /// Canonical topology identifier (for example, `dynamic-dawg`). + pub topology: String, + /// Canonical logical profile identifier (for example, `uleb128`). + pub profile: String, + /// Version of the logical profile codec. + pub profile_version: u16, + /// Fixed atom width, or `None` for variable-width profiles. + pub width_bytes: Option, +} + +/// Validation failures for a persisted [`DictionaryDescriptor`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum DictionaryDescriptorError { + /// The topology identifier is not recognized. + UnknownTopology, + /// The profile identifier is not recognized. + UnknownProfile, + /// The profile version does not match the canonical implementation. + ProfileVersionMismatch, + /// The declared width does not match the canonical profile. + WidthMismatch, +} + +impl DictionaryDescriptor { + /// Build a descriptor from a compatibility backend selector. + pub fn from_backend(backend: DictionaryBackend) -> Self { + let profile = backend.profile_descriptor(); + Self { + topology: backend.family().as_str().to_owned(), + profile: profile.identity.name.to_owned(), + profile_version: profile.identity.version, + width_bytes: profile.width_bytes, + } + } + + /// Build a descriptor directly from a topology/profile specification. + pub fn from_spec(spec: DictionarySpec

) -> Self { + Self { + topology: spec.family().as_str().to_owned(), + profile: P::PROFILE.name.to_owned(), + profile_version: P::PROFILE.version, + width_bytes: P::WIDTH_BYTES, + } + } + + /// Validate all persisted identifiers and profile metadata. + pub fn validate(&self) -> Result<(DictionaryFamily, ProfileKind), DictionaryDescriptorError> { + let family = DictionaryFamily::from_name(&self.topology) + .ok_or(DictionaryDescriptorError::UnknownTopology)?; + let kind = ProfileKind::from_name(&self.profile) + .ok_or(DictionaryDescriptorError::UnknownProfile)?; + let identity = kind.identity(); + if identity.version != self.profile_version { + return Err(DictionaryDescriptorError::ProfileVersionMismatch); + } + if kind.width_bytes() != self.width_bytes { + return Err(DictionaryDescriptorError::WidthMismatch); + } + Ok((family, kind)) + } +} + +/// Compile-time profile selection for topology-oriented factory code. +/// +/// `DictionarySpec

` carries no dictionary storage. It is a zero-sized +/// declaration that keeps the topology and logical profile as separate axes, +/// allowing callers to validate or dispatch a construction request before +/// allocating a concrete backend. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DictionarySpec { + family: DictionaryFamily, + marker: PhantomData

, +} + +impl DictionarySpec

{ + /// Declare a dictionary family using profile `P`. + pub const fn new(family: DictionaryFamily) -> Self { + Self { + family, + marker: PhantomData, + } + } + + /// Return the selected topology family. + pub const fn family(self) -> DictionaryFamily { + self.family + } + + /// Return the stable logical profile identity carried by `P`. + pub const fn profile(self) -> VariableWidthProfile { + P::PROFILE + } + + /// Return the built-in profile kind carried by `P`. + pub const fn profile_kind(self) -> ProfileKind { + P::KIND + } + + /// Return the fixed atom width, or `None` for variable-width profiles. + pub const fn width_bytes(self) -> Option { + P::WIDTH_BYTES + } +} /// Dictionary backend types. /// @@ -53,14 +223,21 @@ pub enum DictionaryBackend { /// PathMap-based trie, character (Unicode) variant. #[cfg(feature = "pathmap-backend")] PathMapChar, + /// PathMap trie whose physical bytes are validated as UTF-8 logical terms. + #[cfg(feature = "pathmap-backend")] + PathMapUtf8, /// Double-Array Trie (O(1) transitions, excellent cache, byte-keyed). DoubleArrayTrie, /// Double-Array Trie, character (Unicode) variant. DoubleArrayTrieChar, + /// Byte-backed DAT with UTF-8 profile semantics. + DoubleArrayTrieUtf8, /// Dynamic DAWG dictionary (space-efficient, byte-keyed, supports modifications). DynamicDawg, /// Dynamic DAWG, character (Unicode) variant. DynamicDawgChar, + /// Byte-backed dynamic DAWG with UTF-8 profile semantics. + DynamicDawgUtf8, /// Dynamic DAWG keyed on `u64` sequences (token sequences, time series). DynamicDawgU64, /// Suffix automaton dictionary (substring matching, byte-keyed, dynamic). @@ -73,6 +250,170 @@ pub enum DictionaryBackend { ScdawgChar, } +impl DictionaryBackend { + /// Return the topology family represented by this compatibility selector. + pub const fn family(self) -> DictionaryFamily { + match self { + #[cfg(feature = "pathmap-backend")] + Self::PathMap | Self::PathMapChar | Self::PathMapUtf8 => DictionaryFamily::PathMap, + Self::DoubleArrayTrie | Self::DoubleArrayTrieChar | Self::DoubleArrayTrieUtf8 => { + DictionaryFamily::DoubleArrayTrie + } + Self::DynamicDawg + | Self::DynamicDawgChar + | Self::DynamicDawgUtf8 + | Self::DynamicDawgU64 => DictionaryFamily::DynamicDawg, + Self::SuffixAutomaton | Self::SuffixAutomatonChar => DictionaryFamily::SuffixAutomaton, + Self::Scdawg | Self::ScdawgChar => DictionaryFamily::Scdawg, + } + } +} + +/// Typed factory selectors for canonical ULEB128 sequence dictionaries. +/// +/// This selector is separate from [`DictionaryBackend`] because the latter's +/// compatibility factory accepts text terms. ULEB sequences must remain +/// typed at the boundary; converting them through `str` would be lossy. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Uleb128Backend { + /// Mutable in-memory DAWG over encoded canonical ULEB sequences. + DynamicDawg, + /// Immutable double-array trie over encoded canonical ULEB sequences. + DoubleArrayTrie, + /// PathMap adapter over encoded canonical ULEB sequences. + #[cfg(feature = "pathmap-backend")] + PathMap, +} + +impl std::fmt::Display for Uleb128Backend { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(match self { + Self::DynamicDawg => "DynamicDAWGUleb128", + Self::DoubleArrayTrie => "DoubleArrayTrieUleb128", + #[cfg(feature = "pathmap-backend")] + Self::PathMap => "PathMapUleb128", + }) + } +} + +/// Set-like container returned by the typed ULEB128 factory. +#[derive(Debug)] +pub enum Uleb128DictionaryContainer { + /// Dynamic DAWG specialization. + DynamicDawg(DynamicDawgUleb128), + /// Double-array trie specialization. + DoubleArrayTrie(DoubleArrayTrieUleb128), + /// PathMap adapter specialization. + #[cfg(feature = "pathmap-backend")] + PathMap(PathMapDictionaryUleb128), +} + +impl Uleb128DictionaryContainer { + /// Return the selected typed backend. + pub const fn backend(&self) -> Uleb128Backend { + match self { + Self::DynamicDawg(_) => Uleb128Backend::DynamicDawg, + Self::DoubleArrayTrie(_) => Uleb128Backend::DoubleArrayTrie, + #[cfg(feature = "pathmap-backend")] + Self::PathMap(_) => Uleb128Backend::PathMap, + } + } + + /// Return the canonical logical profile carried by this container. + pub const fn profile_descriptor(&self) -> BackendProfileDescriptor { + BackendProfileDescriptor { + kind: ProfileKind::Uleb128, + identity: ProfileKind::Uleb128.identity(), + width_bytes: None, + } + } + + /// Number of complete logical sequences. + pub fn term_count(&self) -> usize { + match self { + Self::DynamicDawg(dictionary) => dictionary.term_count(), + Self::DoubleArrayTrie(dictionary) => dictionary.term_count(), + #[cfg(feature = "pathmap-backend")] + Self::PathMap(dictionary) => dictionary.term_count(), + } + } + + /// Whether no complete logical sequence is stored. + pub fn is_empty(&self) -> bool { + self.term_count() == 0 + } + + /// Test membership in logical sequence space. + pub fn contains(&self, sequence: &Uleb128Sequence) -> bool { + match self { + Self::DynamicDawg(dictionary) => dictionary.contains(sequence), + Self::DoubleArrayTrie(dictionary) => dictionary.contains(sequence), + #[cfg(feature = "pathmap-backend")] + Self::PathMap(dictionary) => dictionary.contains(sequence), + } + } + + /// Return the mapped value for a complete logical ULEB128 sequence. + pub fn get_value(&self, sequence: &Uleb128Sequence) -> Option { + match self { + Self::DynamicDawg(dictionary) => dictionary.get_value(sequence), + Self::DoubleArrayTrie(dictionary) => dictionary.get_value(sequence), + #[cfg(feature = "pathmap-backend")] + Self::PathMap(dictionary) => dictionary.get_value(sequence), + } + } + + /// Return the mapped value for a shared logical profile sequence. + pub fn get_atom_sequence_value( + &self, + sequence: &crate::AtomSequence, + ) -> Option { + match self { + Self::DynamicDawg(dictionary) => dictionary.get_atom_sequence_value(sequence), + Self::DoubleArrayTrie(dictionary) => dictionary.get_atom_sequence_value(sequence), + #[cfg(feature = "pathmap-backend")] + Self::PathMap(dictionary) => dictionary.get_atom_sequence_value(sequence), + } + } + + /// Test membership using the shared logical ULEB profile sequence type. + pub fn contains_atom_sequence( + &self, + sequence: &crate::AtomSequence, + ) -> bool { + match self { + Self::DynamicDawg(dictionary) => dictionary.contains_atom_sequence(sequence), + Self::DoubleArrayTrie(dictionary) => dictionary.contains_atom_sequence(sequence), + #[cfg(feature = "pathmap-backend")] + Self::PathMap(dictionary) => dictionary.contains_atom_sequence(sequence), + } + } + + /// Test a complete canonical encoded sequence without materializing its + /// decoded atoms. Validation ensures malformed or non-canonical bytes do + /// not become observable backend transitions. + pub fn contains_encoded(&self, encoded: &[u8]) -> Result { + match self { + Self::DynamicDawg(dictionary) => dictionary.contains_encoded(encoded), + Self::DoubleArrayTrie(dictionary) => dictionary.contains_encoded(encoded), + #[cfg(feature = "pathmap-backend")] + Self::PathMap(dictionary) => dictionary.contains_encoded(encoded), + } + } + + /// Export complete logical sequences while keeping physical codec bytes + /// private to each backend. Decoding occurs only at this explicit logical + /// boundary and malformed images are reported as errors. + pub fn visible_entries(&self) -> Result)>, Uleb128Error> { + match self { + Self::DynamicDawg(dictionary) => dictionary.visible_entries(), + Self::DoubleArrayTrie(dictionary) => dictionary.visible_entries(), + #[cfg(feature = "pathmap-backend")] + Self::PathMap(dictionary) => dictionary.visible_entries(), + } + } +} + /// Edge-label unit used by a backend. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum BackendKeyUnit { @@ -121,6 +462,32 @@ pub struct BackendCapabilities { pub lock_free_writes: bool, } +/// Stable logical-profile metadata for a factory backend. +/// +/// The canonical profile identity is independent of the legacy Rust backend +/// spelling and is suitable for capability negotiation and serialized +/// descriptors. `width_bytes == None` denotes a variable-width profile. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct BackendProfileDescriptor { + /// Built-in logical profile kind. + pub kind: ProfileKind, + /// Canonical name/version identity for persistence and negotiation. + pub identity: VariableWidthProfile, + /// Fixed encoded width, or `None` for variable-width atoms. + pub width_bytes: Option, +} + +impl BackendProfileDescriptor { + /// Construct canonical metadata from a compile-time atom profile. + pub const fn from_profile() -> Self { + Self { + kind: P::KIND, + identity: P::PROFILE, + width_bytes: P::WIDTH_BYTES, + } + } +} + impl BackendCapabilities { /// Returns true for Unicode scalar-value backends. pub fn is_unicode(self) -> bool { @@ -145,6 +512,42 @@ impl BackendCapabilities { } impl DictionaryBackend { + /// Return the stable logical profile represented by this legacy backend. + pub const fn profile_descriptor(self) -> BackendProfileDescriptor { + let kind = match self { + #[cfg(feature = "pathmap-backend")] + Self::PathMap + | Self::DynamicDawg + | Self::DoubleArrayTrie + | Self::SuffixAutomaton + | Self::Scdawg => ProfileKind::Bytes, + #[cfg(feature = "pathmap-backend")] + Self::PathMapChar + | Self::DynamicDawgChar + | Self::DoubleArrayTrieChar + | Self::SuffixAutomatonChar + | Self::ScdawgChar => ProfileKind::UnicodeScalar, + #[cfg(feature = "pathmap-backend")] + Self::PathMapUtf8 => ProfileKind::Utf8, + Self::DoubleArrayTrieUtf8 | Self::DynamicDawgUtf8 => ProfileKind::Utf8, + #[cfg(not(feature = "pathmap-backend"))] + Self::DynamicDawg | Self::DoubleArrayTrie | Self::SuffixAutomaton | Self::Scdawg => { + ProfileKind::Bytes + } + #[cfg(not(feature = "pathmap-backend"))] + Self::DynamicDawgChar + | Self::DoubleArrayTrieChar + | Self::SuffixAutomatonChar + | Self::ScdawgChar => ProfileKind::UnicodeScalar, + Self::DynamicDawgU64 => ProfileKind::U64, + }; + BackendProfileDescriptor { + kind, + identity: kind.identity(), + width_bytes: kind.width_bytes(), + } + } + /// Machine-readable backend characteristics. pub fn capabilities(self) -> BackendCapabilities { match self { @@ -166,6 +569,15 @@ impl DictionaryBackend { lock_free_reads: true, lock_free_writes: true, }, + #[cfg(feature = "pathmap-backend")] + DictionaryBackend::PathMapUtf8 => BackendCapabilities { + key_unit: BackendKeyUnit::Byte, + query: BackendQuerySemantics::ExactTerm, + updates: BackendUpdateMode::InsertRemove, + sync_strategy: SyncStrategy::InternalSync, + lock_free_reads: true, + lock_free_writes: true, + }, DictionaryBackend::DoubleArrayTrie => BackendCapabilities { key_unit: BackendKeyUnit::Byte, query: BackendQuerySemantics::ExactTerm, @@ -182,6 +594,14 @@ impl DictionaryBackend { lock_free_reads: true, lock_free_writes: false, }, + DictionaryBackend::DoubleArrayTrieUtf8 => BackendCapabilities { + key_unit: BackendKeyUnit::Byte, + query: BackendQuerySemantics::ExactTerm, + updates: BackendUpdateMode::Immutable, + sync_strategy: SyncStrategy::Persistent, + lock_free_reads: true, + lock_free_writes: false, + }, DictionaryBackend::DynamicDawg => BackendCapabilities { key_unit: BackendKeyUnit::Byte, query: BackendQuerySemantics::ExactTerm, @@ -198,6 +618,14 @@ impl DictionaryBackend { lock_free_reads: true, lock_free_writes: true, }, + DictionaryBackend::DynamicDawgUtf8 => BackendCapabilities { + key_unit: BackendKeyUnit::Byte, + query: BackendQuerySemantics::ExactTerm, + updates: BackendUpdateMode::InsertRemove, + sync_strategy: SyncStrategy::InternalSync, + lock_free_reads: true, + lock_free_writes: true, + }, DictionaryBackend::DynamicDawgU64 => BackendCapabilities { key_unit: BackendKeyUnit::U64, query: BackendQuerySemantics::ExactTerm, @@ -249,10 +677,14 @@ impl std::fmt::Display for DictionaryBackend { DictionaryBackend::PathMap => write!(f, "PathMap"), #[cfg(feature = "pathmap-backend")] DictionaryBackend::PathMapChar => write!(f, "PathMapChar"), + #[cfg(feature = "pathmap-backend")] + DictionaryBackend::PathMapUtf8 => write!(f, "PathMapUtf8"), DictionaryBackend::DoubleArrayTrie => write!(f, "DoubleArrayTrie"), DictionaryBackend::DoubleArrayTrieChar => write!(f, "DoubleArrayTrieChar"), + DictionaryBackend::DoubleArrayTrieUtf8 => write!(f, "DoubleArrayTrieUtf8"), DictionaryBackend::DynamicDawg => write!(f, "DynamicDAWG"), DictionaryBackend::DynamicDawgChar => write!(f, "DynamicDAWGChar"), + DictionaryBackend::DynamicDawgUtf8 => write!(f, "DynamicDAWGUtf8"), DictionaryBackend::DynamicDawgU64 => write!(f, "DynamicDAWGU64"), DictionaryBackend::SuffixAutomaton => write!(f, "SuffixAutomaton"), DictionaryBackend::SuffixAutomatonChar => write!(f, "SuffixAutomatonChar"), @@ -272,10 +704,14 @@ pub enum DictionaryContainer { PathMap(PathMapDictionary), #[cfg(feature = "pathmap-backend")] PathMapChar(PathMapDictionaryChar), + #[cfg(feature = "pathmap-backend")] + PathMapUtf8(PathMapDictionaryUtf8), DoubleArrayTrie(DoubleArrayTrie), DoubleArrayTrieChar(DoubleArrayTrieChar), + DoubleArrayTrieUtf8(DoubleArrayTrieUtf8), DynamicDawg(DynamicDawg), DynamicDawgChar(DynamicDawgChar), + DynamicDawgUtf8(DynamicDawgUtf8), DynamicDawgU64(DynamicDawgU64), SuffixAutomaton(SuffixAutomaton), SuffixAutomatonChar(SuffixAutomatonChar), @@ -291,10 +727,14 @@ impl DictionaryContainer { DictionaryContainer::PathMap(_) => DictionaryBackend::PathMap, #[cfg(feature = "pathmap-backend")] DictionaryContainer::PathMapChar(_) => DictionaryBackend::PathMapChar, + #[cfg(feature = "pathmap-backend")] + DictionaryContainer::PathMapUtf8(_) => DictionaryBackend::PathMapUtf8, DictionaryContainer::DoubleArrayTrie(_) => DictionaryBackend::DoubleArrayTrie, DictionaryContainer::DoubleArrayTrieChar(_) => DictionaryBackend::DoubleArrayTrieChar, + DictionaryContainer::DoubleArrayTrieUtf8(_) => DictionaryBackend::DoubleArrayTrieUtf8, DictionaryContainer::DynamicDawg(_) => DictionaryBackend::DynamicDawg, DictionaryContainer::DynamicDawgChar(_) => DictionaryBackend::DynamicDawgChar, + DictionaryContainer::DynamicDawgUtf8(_) => DictionaryBackend::DynamicDawgUtf8, DictionaryContainer::DynamicDawgU64(_) => DictionaryBackend::DynamicDawgU64, DictionaryContainer::SuffixAutomaton(_) => DictionaryBackend::SuffixAutomaton, DictionaryContainer::SuffixAutomatonChar(_) => DictionaryBackend::SuffixAutomatonChar, @@ -303,6 +743,12 @@ impl DictionaryContainer { } } + /// Return canonical logical-profile metadata for this instance. + #[inline] + pub fn profile_descriptor(&self) -> BackendProfileDescriptor { + self.backend().profile_descriptor() + } + /// Get the number of terms in the dictionary. pub fn len(&self) -> Option { match self { @@ -310,10 +756,14 @@ impl DictionaryContainer { DictionaryContainer::PathMap(d) => d.len(), #[cfg(feature = "pathmap-backend")] DictionaryContainer::PathMapChar(d) => d.len(), + #[cfg(feature = "pathmap-backend")] + DictionaryContainer::PathMapUtf8(d) => Some(d.term_count()), DictionaryContainer::DoubleArrayTrie(d) => d.len(), DictionaryContainer::DoubleArrayTrieChar(d) => d.len(), + DictionaryContainer::DoubleArrayTrieUtf8(d) => Some(d.term_count()), DictionaryContainer::DynamicDawg(d) => d.len(), DictionaryContainer::DynamicDawgChar(d) => d.len(), + DictionaryContainer::DynamicDawgUtf8(d) => Some(d.term_count()), DictionaryContainer::DynamicDawgU64(d) => d.len(), DictionaryContainer::SuffixAutomaton(d) => d.len(), DictionaryContainer::SuffixAutomatonChar(d) => d.len(), @@ -334,10 +784,14 @@ impl DictionaryContainer { DictionaryContainer::PathMap(d) => d.contains(term), #[cfg(feature = "pathmap-backend")] DictionaryContainer::PathMapChar(d) => d.contains(term), + #[cfg(feature = "pathmap-backend")] + DictionaryContainer::PathMapUtf8(d) => d.contains(term), DictionaryContainer::DoubleArrayTrie(d) => d.contains(term), DictionaryContainer::DoubleArrayTrieChar(d) => d.contains(term), + DictionaryContainer::DoubleArrayTrieUtf8(d) => d.contains(term), DictionaryContainer::DynamicDawg(d) => d.contains(term), DictionaryContainer::DynamicDawgChar(d) => d.contains(term), + DictionaryContainer::DynamicDawgUtf8(d) => d.contains(term), DictionaryContainer::DynamicDawgU64(d) => d.contains(term), DictionaryContainer::SuffixAutomaton(d) => d.contains(term), DictionaryContainer::SuffixAutomatonChar(d) => d.contains(term), @@ -347,10 +801,232 @@ impl DictionaryContainer { } } +/// Error returned when a typed profile is incompatible with a backend. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ProfiledFactoryError { + /// The backend's logical profile differs from the requested profile. + ProfileMismatch { + /// Profile requested by the caller. + expected: ProfileKind, + /// Profile implemented by the selected backend. + actual: ProfileKind, + }, +} + +/// Profile-typed compatibility container for the legacy text factory. +/// +/// The storage remains the existing [`DictionaryContainer`], but construction +/// carries a compile-time [`AtomProfile`] witness and rejects a mismatched +/// backend before any terms are materialized. This keeps topology and +/// representation separate without multiplying backend enum variants. +#[derive(Debug)] +pub struct ProfiledDictionaryContainer { + inner: DictionaryContainer, + marker: PhantomData

, +} + +impl ProfiledDictionaryContainer

{ + /// Construct an empty profile-checked container. + pub fn empty(backend: DictionaryBackend) -> Result { + Self::check_backend(backend)?; + Ok(Self { + inner: DictionaryFactory::empty(backend), + marker: PhantomData, + }) + } + + /// Construct a profile-checked container from text terms. + pub fn from_terms( + backend: DictionaryBackend, + terms: I, + ) -> Result + where + I: IntoIterator, + S: AsRef, + { + Self::check_backend(backend)?; + Ok(Self { + inner: DictionaryFactory::create(backend, terms), + marker: PhantomData, + }) + } + + fn check_backend(backend: DictionaryBackend) -> Result<(), ProfiledFactoryError> { + let actual = backend.profile_descriptor().kind; + if actual == P::KIND { + Ok(()) + } else { + Err(ProfiledFactoryError::ProfileMismatch { + expected: P::KIND, + actual, + }) + } + } + + /// Return the legacy backend selector. + pub fn backend(&self) -> DictionaryBackend { + self.inner.backend() + } + + /// Return the topology family. + pub fn family(&self) -> DictionaryFamily { + self.backend().family() + } + + /// Return the validated wire descriptor. + pub fn descriptor(&self) -> DictionaryDescriptor { + DictionaryDescriptor::from_backend(self.backend()) + } + + /// Return the number of terms, when the backend can report it. + pub fn len(&self) -> Option { + self.inner.len() + } + + /// Return whether the container has no terms. + pub fn is_empty(&self) -> bool { + self.inner.is_empty() + } + + /// Test membership using the legacy text boundary. + pub fn contains(&self, term: &str) -> bool { + self.inner.contains(term) + } + + /// Borrow the compatibility container for APIs that have not migrated. + pub fn as_legacy(&self) -> &DictionaryContainer { + &self.inner + } +} + /// Factory for creating dictionaries with different backends. pub struct DictionaryFactory; impl DictionaryFactory { + /// Create a typed ULEB128 dictionary from logical sequences. + /// + /// Unlike [`Self::create`], this method does not accept strings: each + /// sequence is validated and retained as a canonical variable-width + /// value, preserving arbitrary widths and logical atom boundaries. + pub fn create_uleb128(backend: Uleb128Backend, sequences: I) -> Uleb128DictionaryContainer + where + I: IntoIterator, + { + match backend { + Uleb128Backend::DynamicDawg => Uleb128DictionaryContainer::DynamicDawg( + DynamicDawgUleb128::from_sequences(sequences), + ), + Uleb128Backend::DoubleArrayTrie => Uleb128DictionaryContainer::DoubleArrayTrie( + DoubleArrayTrieUleb128::from_sequences(sequences), + ), + #[cfg(feature = "pathmap-backend")] + Uleb128Backend::PathMap => Uleb128DictionaryContainer::PathMap( + PathMapDictionaryUleb128::from_sequences(sequences), + ), + } + } + + /// Create a typed ULEB128 dictionary from shared logical profile + /// sequences. The factory retains the same backend/profile descriptor + /// while avoiding an intermediate representation at the call site. + pub fn create_uleb128_atoms( + backend: Uleb128Backend, + sequences: I, + ) -> Uleb128DictionaryContainer + where + I: IntoIterator>, + { + match backend { + Uleb128Backend::DynamicDawg => Uleb128DictionaryContainer::DynamicDawg( + DynamicDawgUleb128::from_atom_sequences(sequences), + ), + Uleb128Backend::DoubleArrayTrie => Uleb128DictionaryContainer::DoubleArrayTrie( + DoubleArrayTrieUleb128::from_atom_sequences(sequences), + ), + #[cfg(feature = "pathmap-backend")] + Uleb128Backend::PathMap => Uleb128DictionaryContainer::PathMap( + PathMapDictionaryUleb128::from_atom_sequences(sequences), + ), + } + } + + /// Create a value-bearing typed ULEB128 dictionary from logical sequences. + /// + /// The value type is preserved through the container rather than erased to + /// the set-like default `()`, allowing callers to select a backend without + /// giving up mapped results. + pub fn create_uleb128_with_values( + backend: Uleb128Backend, + entries: I, + ) -> Uleb128DictionaryContainer + where + I: IntoIterator, + V: crate::DictionaryValue, + { + match backend { + Uleb128Backend::DynamicDawg => Uleb128DictionaryContainer::DynamicDawg( + DynamicDawgUleb128::from_sequences_with_values(entries), + ), + Uleb128Backend::DoubleArrayTrie => Uleb128DictionaryContainer::DoubleArrayTrie( + DoubleArrayTrieUleb128::from_sequences_with_values(entries), + ), + #[cfg(feature = "pathmap-backend")] + Uleb128Backend::PathMap => Uleb128DictionaryContainer::PathMap( + PathMapDictionaryUleb128::from_sequences_with_values(entries), + ), + } + } + + /// Create a value-bearing typed ULEB128 dictionary from shared profile + /// sequences without converting through an encoded intermediary. + pub fn create_uleb128_atoms_with_values( + backend: Uleb128Backend, + entries: I, + ) -> Uleb128DictionaryContainer + where + I: IntoIterator, V)>, + V: crate::DictionaryValue, + { + match backend { + Uleb128Backend::DynamicDawg => Uleb128DictionaryContainer::DynamicDawg( + DynamicDawgUleb128::from_atom_sequences_with_values(entries), + ), + Uleb128Backend::DoubleArrayTrie => Uleb128DictionaryContainer::DoubleArrayTrie( + DoubleArrayTrieUleb128::from_atom_sequences_with_values(entries), + ), + #[cfg(feature = "pathmap-backend")] + Uleb128Backend::PathMap => Uleb128DictionaryContainer::PathMap( + PathMapDictionaryUleb128::from_atom_sequences_with_values(entries), + ), + } + } + + /// Create an empty typed ULEB128 dictionary. + pub fn empty_uleb128(backend: Uleb128Backend) -> Uleb128DictionaryContainer { + match backend { + Uleb128Backend::DynamicDawg => { + Uleb128DictionaryContainer::DynamicDawg(DynamicDawgUleb128::new()) + } + Uleb128Backend::DoubleArrayTrie => { + Uleb128DictionaryContainer::DoubleArrayTrie(DoubleArrayTrieUleb128::new()) + } + #[cfg(feature = "pathmap-backend")] + Uleb128Backend::PathMap => { + Uleb128DictionaryContainer::PathMap(PathMapDictionaryUleb128::new()) + } + } + } + + /// List the typed ULEB128 backends available in this build. + pub fn available_uleb128_backends() -> Vec { + vec![ + Uleb128Backend::DynamicDawg, + Uleb128Backend::DoubleArrayTrie, + #[cfg(feature = "pathmap-backend")] + Uleb128Backend::PathMap, + ] + } + /// Create a dictionary with the specified backend. /// /// # Arguments @@ -383,18 +1059,28 @@ impl DictionaryFactory { DictionaryBackend::PathMapChar => { DictionaryContainer::PathMapChar(PathMapDictionaryChar::from_terms(terms)) } + #[cfg(feature = "pathmap-backend")] + DictionaryBackend::PathMapUtf8 => { + DictionaryContainer::PathMapUtf8(PathMapDictionaryUtf8::from_terms(terms)) + } DictionaryBackend::DoubleArrayTrie => { DictionaryContainer::DoubleArrayTrie(DoubleArrayTrie::from_terms(terms)) } DictionaryBackend::DoubleArrayTrieChar => { DictionaryContainer::DoubleArrayTrieChar(DoubleArrayTrieChar::from_terms(terms)) } + DictionaryBackend::DoubleArrayTrieUtf8 => { + DictionaryContainer::DoubleArrayTrieUtf8(DoubleArrayTrieUtf8::from_terms(terms)) + } DictionaryBackend::DynamicDawg => { DictionaryContainer::DynamicDawg(DynamicDawg::from_terms(terms)) } DictionaryBackend::DynamicDawgChar => { DictionaryContainer::DynamicDawgChar(DynamicDawgChar::from_terms(terms)) } + DictionaryBackend::DynamicDawgUtf8 => { + DictionaryContainer::DynamicDawgUtf8(DynamicDawgUtf8::from_terms(terms)) + } DictionaryBackend::DynamicDawgU64 => { DictionaryContainer::DynamicDawgU64(DynamicDawgU64::from_terms(terms)) } @@ -429,6 +1115,10 @@ impl DictionaryFactory { DictionaryBackend::PathMapChar => { DictionaryContainer::PathMapChar(PathMapDictionaryChar::new()) } + #[cfg(feature = "pathmap-backend")] + DictionaryBackend::PathMapUtf8 => { + DictionaryContainer::PathMapUtf8(PathMapDictionaryUtf8::new()) + } DictionaryBackend::DoubleArrayTrie => { DictionaryContainer::DoubleArrayTrie(DoubleArrayTrie::new()) } @@ -436,10 +1126,16 @@ impl DictionaryFactory { // DoubleArrayTrieChar uses `empty()` instead of `new()`. DictionaryContainer::DoubleArrayTrieChar(DoubleArrayTrieChar::empty()) } + DictionaryBackend::DoubleArrayTrieUtf8 => { + DictionaryContainer::DoubleArrayTrieUtf8(DoubleArrayTrieUtf8::new()) + } DictionaryBackend::DynamicDawg => DictionaryContainer::DynamicDawg(DynamicDawg::new()), DictionaryBackend::DynamicDawgChar => { DictionaryContainer::DynamicDawgChar(DynamicDawgChar::new()) } + DictionaryBackend::DynamicDawgUtf8 => { + DictionaryContainer::DynamicDawgUtf8(DynamicDawgUtf8::new()) + } DictionaryBackend::DynamicDawgU64 => { DictionaryContainer::DynamicDawgU64(DynamicDawgU64::new()) } @@ -461,10 +1157,14 @@ impl DictionaryFactory { DictionaryBackend::PathMap, #[cfg(feature = "pathmap-backend")] DictionaryBackend::PathMapChar, + #[cfg(feature = "pathmap-backend")] + DictionaryBackend::PathMapUtf8, DictionaryBackend::DoubleArrayTrie, DictionaryBackend::DoubleArrayTrieChar, + DictionaryBackend::DoubleArrayTrieUtf8, DictionaryBackend::DynamicDawg, DictionaryBackend::DynamicDawgChar, + DictionaryBackend::DynamicDawgUtf8, DictionaryBackend::DynamicDawgU64, DictionaryBackend::SuffixAutomaton, DictionaryBackend::SuffixAutomatonChar, @@ -489,6 +1189,10 @@ impl DictionaryFactory { DictionaryBackend::PathMapChar => { "PathMap-based character trie. Unicode-aware variant of PathMap." } + #[cfg(feature = "pathmap-backend")] + DictionaryBackend::PathMapUtf8 => { + "PathMap-based UTF-8 byte trie with validated logical Unicode terms." + } DictionaryBackend::DoubleArrayTrie => { "Byte-keyed double-array trie. O(1) transitions, excellent cache locality, \ read-mostly. Best for static dictionaries." @@ -496,6 +1200,9 @@ impl DictionaryFactory { DictionaryBackend::DoubleArrayTrieChar => { "Character-keyed double-array trie. Unicode-aware variant of DoubleArrayTrie." } + DictionaryBackend::DoubleArrayTrieUtf8 => { + "UTF-8 byte-backed double-array trie with validated logical Unicode terms." + } DictionaryBackend::DynamicDawg => { "Byte-keyed dynamic DAWG. Space-efficient with full dynamic modification \ support. Best for evolving dictionaries." @@ -503,6 +1210,9 @@ impl DictionaryFactory { DictionaryBackend::DynamicDawgChar => { "Character-keyed dynamic DAWG. Unicode-aware variant of DynamicDawg." } + DictionaryBackend::DynamicDawgUtf8 => { + "UTF-8 byte-backed dynamic DAWG with validated logical Unicode terms." + } DictionaryBackend::DynamicDawgU64 => { "u64-keyed dynamic DAWG. For token-sequence dictionaries, time series, \ or any application keying on 64-bit symbols." @@ -528,6 +1238,7 @@ impl DictionaryFactory { #[cfg(test)] mod tests { use super::*; + use crate::{Bytes, DynamicDawgGeneric, Utf8}; #[test] #[cfg(feature = "pathmap-backend")] @@ -556,6 +1267,7 @@ mod tests { assert!(dict.contains("bar")); assert!(dict.contains("baz")); assert!(!dict.contains("qux")); + assert_eq!(dict.profile_descriptor().kind, ProfileKind::Bytes); } #[test] @@ -564,7 +1276,9 @@ mod tests { for backend in [ DictionaryBackend::DoubleArrayTrieChar, + DictionaryBackend::DoubleArrayTrieUtf8, DictionaryBackend::DynamicDawgChar, + DictionaryBackend::DynamicDawgUtf8, DictionaryBackend::SuffixAutomatonChar, DictionaryBackend::ScdawgChar, ] { @@ -599,17 +1313,20 @@ mod tests { #[test] fn test_available_backends() { let backends = DictionaryFactory::available_backends(); - // 11 backends total: 4 byte + 4 char + DynamicDawgU64 + 2 scdawg. + // 14 backends total with PathMap enabled: legacy backends plus two + // byte-backed UTF-8 profile adapters; PathMap variants are feature-gated. // PathMap and PathMapChar gated behind feature. #[cfg(feature = "pathmap-backend")] - assert_eq!(backends.len(), 11); + assert_eq!(backends.len(), 14); #[cfg(not(feature = "pathmap-backend"))] - assert_eq!(backends.len(), 9); + assert_eq!(backends.len(), 11); assert!(backends.contains(&DictionaryBackend::DoubleArrayTrie)); assert!(backends.contains(&DictionaryBackend::DynamicDawg)); assert!(backends.contains(&DictionaryBackend::DynamicDawgChar)); assert!(backends.contains(&DictionaryBackend::SuffixAutomaton)); assert!(backends.contains(&DictionaryBackend::Scdawg)); + assert!(backends.contains(&DictionaryBackend::DoubleArrayTrieUtf8)); + assert!(backends.contains(&DictionaryBackend::DynamicDawgUtf8)); } #[test] @@ -673,6 +1390,213 @@ mod tests { ); } + #[test] + fn topology_and_profile_are_independent_factory_axes() { + let spec = DictionarySpec::::new(DictionaryFamily::DynamicDawg); + assert_eq!(spec.family(), DictionaryFamily::DynamicDawg); + assert_eq!(spec.profile_kind(), ProfileKind::Uleb128); + assert_eq!(spec.width_bytes(), None); + assert_eq!( + DictionaryBackend::DynamicDawgChar.family(), + DictionaryFamily::DynamicDawg + ); + assert_eq!( + DictionaryBackend::DoubleArrayTrieUtf8.family(), + DictionaryFamily::DoubleArrayTrie + ); + assert_eq!( + DictionaryBackend::SuffixAutomaton.family(), + DictionaryFamily::SuffixAutomaton + ); + for family in [ + DictionaryFamily::DynamicDawg, + DictionaryFamily::DoubleArrayTrie, + DictionaryFamily::PathMap, + DictionaryFamily::SuffixAutomaton, + DictionaryFamily::Scdawg, + DictionaryFamily::PersistentArTrie, + ] { + assert_eq!(DictionaryFamily::from_name(family.as_str()), Some(family)); + } + assert_eq!(DictionaryFamily::from_name("legacy"), None); + + let descriptor = DictionaryDescriptor::from_backend(DictionaryBackend::DynamicDawgUtf8); + assert_eq!(descriptor.topology, "dynamic-dawg"); + assert_eq!(descriptor.profile, "utf8"); + assert_eq!(descriptor.validate().unwrap().1, ProfileKind::Utf8); + let spec_descriptor = DictionaryDescriptor::from_spec( + DictionarySpec::::new(DictionaryFamily::DynamicDawg), + ); + assert_eq!(spec_descriptor.validate().unwrap().1, ProfileKind::Uleb128); + let mut invalid = descriptor.clone(); + invalid.profile_version += 1; + assert_eq!( + invalid.validate(), + Err(DictionaryDescriptorError::ProfileVersionMismatch) + ); + + #[cfg(feature = "serialization")] + { + let bytes = crate::serialization::bincode_compat::serialize(&descriptor).unwrap(); + let restored: DictionaryDescriptor = + crate::serialization::bincode_compat::deserialize(&bytes).unwrap(); + assert_eq!(restored, descriptor); + assert_eq!(restored.validate().unwrap().1, ProfileKind::Utf8); + } + } + + #[test] + fn profiled_factory_container_checks_representation_before_construction() { + let dictionary = ProfiledDictionaryContainer::::from_terms( + DictionaryBackend::DynamicDawg, + ["cat", "dog"], + ) + .unwrap(); + assert_eq!(dictionary.family(), DictionaryFamily::DynamicDawg); + assert_eq!(dictionary.descriptor().profile, "bytes"); + assert!(dictionary.contains("cat")); + + let error = + ProfiledDictionaryContainer::::empty(DictionaryBackend::DynamicDawg) + .unwrap_err(); + assert_eq!( + error, + ProfiledFactoryError::ProfileMismatch { + expected: ProfileKind::Utf8, + actual: ProfileKind::Bytes, + } + ); + + let utf8 = + ProfiledDictionaryContainer::::empty(DictionaryBackend::DynamicDawgUtf8) + .unwrap(); + assert_eq!(utf8.descriptor().profile, "utf8"); + } + + #[test] + fn specialized_profile_wrappers_expose_direct_metadata() { + assert_eq!( + DynamicDawg::<()>::profile_descriptor().kind, + ProfileKind::Bytes + ); + assert_eq!( + DynamicDawgChar::<()>::profile_descriptor().kind, + ProfileKind::UnicodeScalar + ); + assert_eq!( + DynamicDawgU64::<()>::profile_descriptor().kind, + ProfileKind::U64 + ); + assert_eq!( + DynamicDawgGeneric::::profile_descriptor::().kind, + ProfileKind::Bytes + ); + assert_eq!( + DynamicDawgGeneric::::profile_descriptor::().kind, + ProfileKind::Utf8 + ); + assert_eq!( + DynamicDawgUleb128::<()>::profile_descriptor().kind, + ProfileKind::Uleb128 + ); + assert_eq!( + DynamicDawgUtf8::<()>::dictionary_family(), + DictionaryFamily::DynamicDawg + ); + assert_eq!( + DoubleArrayTrieUleb128::<()>::profile_descriptor().kind, + ProfileKind::Uleb128 + ); + assert_eq!( + DoubleArrayTrieUtf8::<()>::dictionary_family(), + DictionaryFamily::DoubleArrayTrie + ); + assert_eq!( + DoubleArrayTrie::<()>::profile_descriptor().kind, + ProfileKind::Bytes + ); + assert_eq!( + DoubleArrayTrieChar::<()>::profile_descriptor().kind, + ProfileKind::UnicodeScalar + ); + assert_eq!( + SuffixAutomaton::<()>::profile_descriptor().kind, + ProfileKind::Bytes + ); + assert_eq!( + SuffixAutomatonChar::<()>::profile_descriptor().kind, + ProfileKind::UnicodeScalar + ); + assert_eq!(Scdawg::<()>::profile_descriptor().kind, ProfileKind::Bytes); + assert_eq!( + ScdawgChar::<()>::profile_descriptor().kind, + ProfileKind::UnicodeScalar + ); + #[cfg(feature = "pathmap-backend")] + { + assert_eq!( + PathMapDictionary::<()>::profile_descriptor().kind, + ProfileKind::Bytes + ); + assert_eq!( + PathMapDictionaryChar::<()>::profile_descriptor().kind, + ProfileKind::UnicodeScalar + ); + } + #[cfg(feature = "pathmap-backend")] + { + assert_eq!( + PathMapDictionaryUleb128::<()>::profile_descriptor().kind, + ProfileKind::Uleb128 + ); + assert_eq!( + PathMapDictionaryUtf8::<()>::dictionary_family(), + DictionaryFamily::PathMap + ); + } + } + + #[test] + fn backend_profile_descriptor_uses_canonical_identity() { + let bytes = DictionaryBackend::DynamicDawg.profile_descriptor(); + assert_eq!(bytes.kind, ProfileKind::Bytes); + assert_eq!(bytes.identity, ProfileKind::Bytes.identity()); + assert_eq!(bytes.width_bytes, Some(1)); + + let chars = DictionaryBackend::DynamicDawgChar.profile_descriptor(); + assert_eq!(chars.kind, ProfileKind::UnicodeScalar); + assert_eq!(chars.identity.name, "unicode-scalar"); + assert_eq!(chars.width_bytes, Some(4)); + + let words = DictionaryBackend::DynamicDawgU64.profile_descriptor(); + assert_eq!(words.kind, ProfileKind::U64); + assert_eq!(words.identity.version, 1); + assert_eq!(words.width_bytes, Some(8)); + + let utf8 = DictionaryBackend::DynamicDawgUtf8.profile_descriptor(); + assert_eq!(utf8.kind, ProfileKind::Utf8); + assert_eq!(utf8.identity.name, "utf8"); + assert_eq!(utf8.width_bytes, None); + #[cfg(feature = "pathmap-backend")] + assert_eq!( + DictionaryBackend::PathMapUtf8.profile_descriptor().kind, + ProfileKind::Utf8 + ); + } + + #[test] + fn factory_accepts_shared_uleb_profile_sequences() { + let sequence = crate::AtomSequence::::from_atoms([ + crate::Uleb128::from_u64(624_485), + crate::Uleb128::from_u64(1u64 << 63), + ]); + for backend in DictionaryFactory::available_uleb128_backends() { + let dictionary = DictionaryFactory::create_uleb128_atoms(backend, [sequence.clone()]); + assert!(dictionary.contains_atom_sequence(&sequence)); + assert_eq!(dictionary.profile_descriptor().kind, ProfileKind::Uleb128); + } + } + #[test] fn test_all_backends_work() { let terms = vec!["apple", "banana", "cherry"]; @@ -684,4 +1608,52 @@ mod tests { assert!(dict.contains("cherry"), "{backend} should contain 'cherry'"); } } + + #[test] + fn typed_uleb_factory_preserves_variable_width_sequences() { + let sequence = Uleb128Sequence::from_atoms([ + crate::Uleb128::from_payload_digits(&[1; 24]).unwrap(), + crate::Uleb128::from_u64(7), + ]); + for backend in DictionaryFactory::available_uleb128_backends() { + let dictionary = DictionaryFactory::create_uleb128(backend, [sequence.clone()]); + assert_eq!(dictionary.backend(), backend); + assert_eq!(dictionary.profile_descriptor().kind, ProfileKind::Uleb128); + assert_eq!(dictionary.term_count(), 1); + assert!(dictionary.contains(&sequence), "{backend:?}"); + let encoded = sequence.to_encoded(); + assert!(dictionary.contains_encoded(&encoded).unwrap()); + let entries = dictionary.visible_entries().unwrap(); + let keys: Vec<_> = entries.into_iter().map(|(key, _)| key).collect(); + assert_eq!(keys, vec![sequence.clone()]); + assert!(dictionary.contains_encoded(&[0x80]).is_err()); + } + assert_eq!( + Uleb128Backend::DynamicDawg.to_string(), + "DynamicDAWGUleb128" + ); + } + + #[test] + fn typed_uleb_factory_preserves_mapped_values() { + let sequence = Uleb128Sequence::from_atoms([crate::Uleb128::from_u64(42)]); + for backend in DictionaryFactory::available_uleb128_backends() { + let dictionary = DictionaryFactory::create_uleb128_with_values::<_, u16>( + backend, + [(sequence.clone(), 7)], + ); + assert_eq!(dictionary.get_value(&sequence), Some(7), "{backend:?}"); + assert_eq!( + dictionary.visible_entries().unwrap(), + vec![(sequence.clone(), Some(7))] + ); + } + let atoms = + crate::AtomSequence::::from_atoms([crate::Uleb128::from_u64(42)]); + let dictionary = DictionaryFactory::create_uleb128_atoms_with_values::<_, u16>( + Uleb128Backend::DynamicDawg, + [(atoms.clone(), 11)], + ); + assert_eq!(dictionary.get_atom_sequence_value(&atoms), Some(11)); + } } diff --git a/src/interning.rs b/src/interning.rs new file mode 100644 index 00000000..85356a87 --- /dev/null +++ b/src/interning.rs @@ -0,0 +1,1247 @@ +//! Deterministic capsule-local vocabulary interning. + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::{Arc, Mutex}; + +use crate::dynamic_dawg::{DynamicDawgGeneric, DynamicDawgU32}; +use crate::Uleb128; +use crate::{CharUnit, DictionaryValue}; + +/// Dense identifier assigned by an [`InternedVocabulary`]. +pub type InternedId = u64; + +/// Lossless snapshot rows exported by a coordinated vocabulary/ID dictionary. +pub type InternedEntries = Vec<(Vec, Option)>; + +/// Validation failures at the vocabulary boundary. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum InterningError { + /// The ID is not present in this vocabulary generation. + UnknownId(InternedId), + /// The sequence belongs to a different vocabulary generation. + GenerationMismatch { expected: u64, actual: u64 }, + /// A caller attempted to use an atom that has not been interned. + UnknownKey, + /// The coordinated vocabulary lock was poisoned by a prior panic. + Poisoned, + /// No representable local ID remains. + IdExhausted, +} + +#[inline] +fn to_u32_id(id: InternedId) -> Result { + u32::try_from(id).map_err(|_| InterningError::IdExhausted) +} + +/// Compact capsule-local sequence of vocabulary IDs. +#[derive(Clone, Debug, Default, Eq, PartialEq, Hash)] +pub struct InternedSequence { + ids: Vec, + generation: u64, +} + +impl InternedSequence { + /// Construct an empty ID sequence. + #[inline] + pub fn new() -> Self { + Self::default() + } + + /// Construct from already assigned IDs. + pub fn from_ids(ids: I) -> Self + where + I: IntoIterator, + { + Self { + ids: ids.into_iter().collect(), + generation: 0, + } + } + + /// Construct IDs bound to an explicit vocabulary generation. + pub fn from_ids_with_generation(generation: u64, ids: I) -> Self + where + I: IntoIterator, + { + Self { + ids: ids.into_iter().collect(), + generation, + } + } + + /// Borrow the compact ID representation. + #[inline] + pub fn as_ids(&self) -> &[InternedId] { + &self.ids + } + + /// Vocabulary generation that owns these IDs. + #[inline] + pub const fn generation(&self) -> u64 { + self.generation + } + + /// Whether this sequence belongs to the supplied vocabulary generation. + #[inline] + pub const fn is_bound_to(&self, generation: u64) -> bool { + self.generation == generation + } + + /// Iterate IDs without allocation. + #[inline] + pub fn iter(&self) -> impl Iterator + '_ { + self.ids.iter().copied() + } + + /// Number of logical symbols. + #[inline] + pub fn len(&self) -> usize { + self.ids.len() + } + + /// Whether the sequence is empty. + #[inline] + pub fn is_empty(&self) -> bool { + self.ids.is_empty() + } +} + +/// Bidirectional vocabulary with monotonic, never-reused local IDs. +/// +/// IDs are intentionally scoped to this vocabulary instance. Callers that +/// persist or exchange them must bind the vocabulary's own profile and +/// snapshot identity; an ID alone is never a semantic identity. +#[derive(Clone, Debug)] +pub struct InternedVocabulary { + forward: BTreeMap, + reverse: Vec, + generation: u64, +} + +/// Immutable vocabulary view captured at one generation boundary. +/// +/// The snapshot owns the ID-to-symbol table, so readers can resolve IDs +/// without retaining the vocabulary mutex or observing later insertions. IDs +/// remain meaningful only with this snapshot's generation. +#[derive(Clone, Debug)] +pub struct InternedVocabularySnapshot { + generation: u64, + reverse: Arc<[K]>, +} + +impl InternedVocabularySnapshot { + /// Canonical profile metadata for the symbols captured by this snapshot. + pub const fn profile_descriptor>( + &self, + ) -> crate::factory::BackendProfileDescriptor { + crate::factory::BackendProfileDescriptor::from_profile::

() + } + + /// Generation identity bound to every ID in this snapshot. + #[inline] + pub const fn generation(&self) -> u64 { + self.generation + } + + /// Number of symbols visible in this snapshot. + #[inline] + pub fn len(&self) -> usize { + self.reverse.len() + } + + /// Whether this snapshot contains no symbols. + #[inline] + pub fn is_empty(&self) -> bool { + self.reverse.is_empty() + } + + /// Resolve an ID without allocation or locking. + #[inline] + pub fn value(&self, id: InternedId) -> Option<&K> { + self.reverse.get(usize::try_from(id).ok()?) + } + + /// Validate a generation-bound sequence against this immutable snapshot. + pub fn validate_sequence(&self, sequence: &InternedSequence) -> Result<(), InterningError> { + if sequence.generation != self.generation { + return Err(InterningError::GenerationMismatch { + expected: self.generation, + actual: sequence.generation, + }); + } + sequence + .ids + .iter() + .copied() + .find(|&id| self.value(id).is_none()) + .map_or(Ok(()), |id| Err(InterningError::UnknownId(id))) + } + + /// Resolve IDs without allocating; an unknown ID is represented as + /// `None` and can be handled by the caller's fail-closed policy. + pub fn resolve_iter<'a>( + &'a self, + sequence: &'a InternedSequence, + ) -> impl Iterator> { + sequence.ids.iter().map(|&id| self.value(id)) + } + + /// Iterate stable ID/value pairs in ID order. + #[inline] + pub fn iter(&self) -> impl Iterator { + self.reverse + .iter() + .enumerate() + .map(|(id, value)| (id as InternedId, value)) + } +} + +impl Default for InternedVocabulary { + fn default() -> Self { + Self { + forward: BTreeMap::new(), + reverse: Vec::new(), + generation: 0, + } + } +} + +impl InternedVocabulary { + /// Canonical profile metadata for this vocabulary's symbol domain. + pub const fn profile_descriptor>( + &self, + ) -> crate::factory::BackendProfileDescriptor { + crate::factory::BackendProfileDescriptor::from_profile::

() + } + + /// Construct an empty vocabulary. + #[inline] + pub fn new() -> Self { + Self::default() + } + + /// Construct an empty vocabulary with an explicit generation identity. + pub const fn with_generation(generation: u64) -> Self { + Self { + forward: BTreeMap::new(), + reverse: Vec::new(), + generation, + } + } + + /// Generation identity to bind alongside every persisted ID sequence. + #[inline] + pub const fn generation(&self) -> u64 { + self.generation + } + + /// Return the existing ID or assign the next monotonic ID. + pub fn intern(&mut self, key: K) -> InternedId { + self.try_intern(key) + .expect("InternedVocabulary ID space exhausted") + } + + /// Return the existing ID or assign the next ID, reporting exhaustion. + pub fn try_intern(&mut self, key: K) -> Result { + if let Some(&id) = self.forward.get(&key) { + return Ok(id); + } + let id = + InternedId::try_from(self.reverse.len()).map_err(|_| InterningError::IdExhausted)?; + self.forward.insert(key.clone(), id); + self.reverse.push(key); + Ok(id) + } + + /// Intern a logical sequence and return its compact ID representation. + pub fn intern_sequence(&mut self, keys: I) -> InternedSequence + where + I: IntoIterator, + { + InternedSequence::from_ids_with_generation( + self.generation, + keys.into_iter().map(|key| self.intern(key)), + ) + } + + /// Fallible sequence interning with typed ID exhaustion. + pub fn try_intern_sequence(&mut self, keys: I) -> Result + where + I: IntoIterator, + { + // Preflight the entire operation before mutating either map. This + // preserves the vocabulary boundary when the representable ID space + // is exhausted: a failed sequence insertion must not leave a prefix + // of newly interned atoms behind. + let keys: Vec = keys.into_iter().collect(); + let new_keys: BTreeSet = keys + .iter() + .filter(|key| !self.forward.contains_key(*key)) + .cloned() + .collect(); + if let Some(last_index) = self + .reverse + .len() + .checked_add(new_keys.len().saturating_sub(1)) + { + if InternedId::try_from(last_index).is_err() && !new_keys.is_empty() { + return Err(InterningError::IdExhausted); + } + } else if !new_keys.is_empty() { + return Err(InterningError::IdExhausted); + } + + let ids = keys + .into_iter() + .map(|key| self.try_intern(key)) + .collect::, _>>()?; + Ok(InternedSequence::from_ids_with_generation( + self.generation, + ids, + )) + } + + /// Resolve every ID in a sequence, failing if one ID is not in this + /// vocabulary generation. + pub fn resolve_sequence<'a>( + &'a self, + sequence: &'a InternedSequence, + ) -> Option> { + let values: Option> = sequence.ids.iter().map(|&id| self.value(id)).collect(); + values.map(Vec::into_iter) + } + + /// Validate that every ID belongs to this vocabulary generation. + pub fn validate_sequence(&self, sequence: &InternedSequence) -> Result<(), InterningError> { + if sequence.generation != self.generation { + return Err(InterningError::GenerationMismatch { + expected: self.generation, + actual: sequence.generation, + }); + } + sequence + .ids + .iter() + .copied() + .find(|&id| self.value(id).is_none()) + .map_or(Ok(()), |id| Err(InterningError::UnknownId(id))) + } + + /// Borrow each resolved value in ID order without allocating. A `None` + /// item denotes an unknown ID and must be treated as a vocabulary-boundary + /// error by consumers. + pub fn resolve_iter<'a>( + &'a self, + sequence: &'a InternedSequence, + ) -> impl Iterator> { + sequence.ids.iter().map(|&id| self.value(id)) + } + + /// Look up an ID without mutating the vocabulary. + #[inline] + pub fn id_of(&self, key: &K) -> Option { + self.forward.get(key).copied() + } + + /// Resolve an ID without mutating the vocabulary. + #[inline] + pub fn value(&self, id: InternedId) -> Option<&K> { + let index = usize::try_from(id).ok()?; + self.reverse.get(index) + } + + /// Number of interned values. + #[inline] + pub fn len(&self) -> usize { + self.reverse.len() + } + + /// Whether no values have been interned. + #[inline] + pub fn is_empty(&self) -> bool { + self.reverse.is_empty() + } + + /// Iterate IDs and values in deterministic ID order. + #[inline] + pub fn iter(&self) -> impl Iterator { + self.reverse + .iter() + .enumerate() + .map(|(id, key)| (id as InternedId, key)) + } + + /// Capture an immutable ID-to-symbol snapshot for lock-free readers. + pub fn snapshot(&self) -> InternedVocabularySnapshot { + InternedVocabularySnapshot { + generation: self.generation, + reverse: self.reverse.clone().into(), + } + } +} + +/// A vocabulary and its ID-sequence dictionary as one ownership boundary. +/// +/// The vocabulary is the only component that can create IDs. The underlying +/// DAWG is private so a caller cannot insert an arbitrary local-ID sequence +/// without going through vocabulary validation. Read-only ID access remains +/// available through [`Self::id_dictionary`] for engines whose hot loops are +/// already bound to this capsule's generation. +#[derive(Clone, Debug)] +pub struct InternedSequenceDictionary { + vocabulary: Arc>>, + id_dictionary: DynamicDawgU32, +} + +/// Canonical arbitrary-width ULEB atoms interned to the default `u32` carrier. +/// The atom bytes remain the vocabulary's external identity; the DAWG sees +/// only generation-bound fixed-width IDs. +pub type InternedUlebSequenceDictionary = InternedSequenceDictionary; + +/// Raw IEEE-754 binary64 bit patterns interned into the default `u32` ID +/// carrier. Equality and identity remain bit-preserving, including signed +/// zero and distinct NaN payloads. +pub type InternedF64BitsSequenceDictionary = InternedSequenceDictionary; + +/// Arbitrary-width ULEB atoms interned to the explicit `u64` local carrier. +/// This preserves the same capsule-local vocabulary and generation rules while +/// allowing more than `u32::MAX` distinct symbols in one vocabulary. +pub type InternedUlebSequenceDictionaryU64 = InternedSequenceDictionaryU64; + +/// Raw IEEE-754 binary64 bit patterns interned into the explicit `u64` ID +/// carrier. +pub type InternedF64BitsSequenceDictionaryU64 = InternedSequenceDictionaryU64; + +/// Capability-limited read view of an interned ID-sequence backend. +#[derive(Clone, Copy, Debug)] +pub struct InternedIdDictionaryView<'a, U: CharUnit, V: DictionaryValue> { + dictionary: &'a DynamicDawgGeneric, +} + +impl<'a, U: CharUnit, V: DictionaryValue> InternedIdDictionaryView<'a, U, V> { + #[inline] + fn new(dictionary: &'a DynamicDawgGeneric) -> Self { + Self { dictionary } + } + + /// Test membership in the already-bound ID domain. + #[inline] + pub fn contains_units(&self, ids: &[U]) -> bool { + self.dictionary.contains_units(ids) + } + + /// Read a mapped value in the already-bound ID domain. + #[inline] + pub fn get_units_value(&self, ids: &[U]) -> Option { + self.dictionary.get_units_value(ids) + } + + /// Number of visible ID sequences. + #[inline] + pub fn term_count(&self) -> usize { + self.dictionary.term_count() + } + + /// Number of physical nodes in the ID backend. + #[inline] + pub fn node_count(&self) -> usize { + self.dictionary.node_count() + } + + /// Export visible ID sequences in deterministic lexicographic order. + /// + /// This is an explicit snapshot boundary; hot-loop consumers should use + /// `contains_units` and `get_units_value` instead of repeatedly exporting. + pub fn visible_entries(&self) -> Vec<(Vec, Option)> { + self.dictionary.visible_entries() + } +} + +/// Explicit `u64` carrier specialization for vocabularies larger than the +/// default `u32` ID domain. The vocabulary and generation semantics are +/// identical to [`InternedSequenceDictionary`]. +#[derive(Clone, Debug)] +pub struct InternedSequenceDictionaryU64 { + vocabulary: Arc>>, + id_dictionary: DynamicDawgGeneric, +} + +impl Default for InternedSequenceDictionaryU64 { + fn default() -> Self { + Self::new() + } +} + +impl InternedSequenceDictionaryU64 { + /// Canonical profile metadata for the external symbol domain. + pub const fn profile_descriptor>( + &self, + ) -> crate::factory::BackendProfileDescriptor { + crate::factory::BackendProfileDescriptor::from_profile::

() + } + + /// Construct an empty coordinated dictionary with generation zero. + pub fn new() -> Self { + Self::with_generation(0) + } + + /// Construct an empty coordinated dictionary with an explicit generation. + pub fn with_generation(generation: u64) -> Self { + Self { + vocabulary: Arc::new(Mutex::new(InternedVocabulary::with_generation(generation))), + id_dictionary: DynamicDawgGeneric::new(), + } + } + + /// Borrow the vocabulary lock for identity and reverse lookup. + pub fn vocabulary( + &self, + ) -> std::sync::LockResult>> { + self.vocabulary.lock() + } + + /// Capture the vocabulary boundary without retaining its mutex guard. + pub fn vocabulary_snapshot(&self) -> Result, InterningError> { + self.vocabulary + .lock() + .map(|vocabulary| vocabulary.snapshot()) + .map_err(|_| InterningError::Poisoned) + } + + /// Read the generation identity without exposing vocabulary storage. + pub fn generation(&self) -> Result { + self.vocabulary + .lock() + .map(|vocabulary| vocabulary.generation()) + .map_err(|_| InterningError::Poisoned) + } + + /// Access the `u64` ID-native dictionary for hot-loop consumers. + #[inline] + pub fn id_dictionary(&self) -> InternedIdDictionaryView<'_, u64, V> { + InternedIdDictionaryView::new(&self.id_dictionary) + } + + /// Query a generation-bound ID sequence after validating its vocabulary. + pub fn contains_id_sequence( + &self, + sequence: &InternedSequence, + ) -> Result { + let vocabulary = self + .vocabulary + .lock() + .map_err(|_| InterningError::Poisoned)?; + vocabulary.validate_sequence(sequence)?; + Ok(self.id_dictionary.contains_units(sequence.as_ids())) + } + + /// Read a mapped value for a generation-bound ID sequence. + pub fn get_id_sequence_value( + &self, + sequence: &InternedSequence, + ) -> Result, InterningError> { + let vocabulary = self + .vocabulary + .lock() + .map_err(|_| InterningError::Poisoned)?; + vocabulary.validate_sequence(sequence)?; + Ok(self.id_dictionary.get_units_value(sequence.as_ids())) + } + + /// Export atom sequences and mapped values in deterministic ID-dictionary + /// order. Every ID is resolved while the vocabulary snapshot is held; + /// unknown IDs are reported instead of being silently omitted. + pub fn visible_entries(&self) -> Result, InterningError> { + let vocabulary = self + .vocabulary + .lock() + .map_err(|_| InterningError::Poisoned)?; + self.id_dictionary + .visible_entries() + .into_iter() + .map(|(ids, value)| { + ids.into_iter() + .map(|id| { + vocabulary + .value(id) + .cloned() + .ok_or(InterningError::UnknownId(id)) + }) + .collect::, _>>() + .map(|atoms| (atoms, value)) + }) + .collect() + } + + /// Intern atoms and insert their sequence using the `u64` carrier. + pub fn insert(&self, atoms: I, value: Option) -> Result + where + I: IntoIterator, + { + let atoms: Vec = atoms.into_iter().collect(); + let mut vocabulary = self + .vocabulary + .lock() + .map_err(|_| InterningError::Poisoned)?; + // A duplicate must be rejected before interning any new atom. This + // keeps the vocabulary and its dependent ID dictionary transactional: + // an operation that returns `false` does not publish a new vocabulary + // entry as a side effect. + let existing_ids = atoms + .iter() + .map(|atom| vocabulary.id_of(atom)) + .collect::>>(); + if let Some(existing_ids) = existing_ids { + if self.id_dictionary.contains_units(&existing_ids) { + return Ok(false); + } + } + let sequence = vocabulary.try_intern_sequence(atoms)?; + let ids = sequence.as_ids().to_vec(); + Ok(match value { + Some(value) => self.id_dictionary.insert_units_with_value(&ids, value), + None => self.id_dictionary.insert_units(&ids), + }) + } + + /// Intern and insert one shared logical profile sequence while retaining + /// the vocabulary's generation and ID validation boundary. + pub fn insert_atom_sequence

( + &self, + sequence: &crate::AtomSequence

, + value: Option, + ) -> Result + where + P: crate::AtomProfile, + { + self.insert(sequence.as_atoms().iter().cloned(), value) + } + + /// Test an atom sequence without mutating the vocabulary. + pub fn contains(&self, atoms: I) -> Result + where + I: IntoIterator, + { + let vocabulary = self + .vocabulary + .lock() + .map_err(|_| InterningError::Poisoned)?; + let ids = atoms + .into_iter() + .map(|atom| vocabulary.id_of(&atom).ok_or(InterningError::UnknownKey)) + .collect::, _>>()?; + Ok(self.id_dictionary.contains_units(&ids)) + } + + /// Test one shared logical profile sequence without mutating the + /// vocabulary. + pub fn contains_atom_sequence

( + &self, + sequence: &crate::AtomSequence

, + ) -> Result + where + P: crate::AtomProfile, + { + self.contains(sequence.as_atoms().iter().cloned()) + } + + /// Read a mapped value for an already-interned atom sequence. + pub fn get_value(&self, atoms: I) -> Result, InterningError> + where + I: IntoIterator, + { + let vocabulary = self + .vocabulary + .lock() + .map_err(|_| InterningError::Poisoned)?; + let ids = atoms + .into_iter() + .map(|atom| vocabulary.id_of(&atom).ok_or(InterningError::UnknownKey)) + .collect::, _>>()?; + Ok(self.id_dictionary.get_units_value(&ids)) + } + + /// Read a mapped value for one already-interned shared logical profile + /// sequence. + pub fn get_atom_sequence_value

( + &self, + sequence: &crate::AtomSequence

, + ) -> Result, InterningError> + where + P: crate::AtomProfile, + { + self.get_value(sequence.as_atoms().iter().cloned()) + } + + /// Remove an atom sequence without changing vocabulary assignments. + pub fn remove(&self, atoms: I) -> Result + where + I: IntoIterator, + { + let vocabulary = self + .vocabulary + .lock() + .map_err(|_| InterningError::Poisoned)?; + let ids = atoms + .into_iter() + .map(|atom| vocabulary.id_of(&atom).ok_or(InterningError::UnknownKey)) + .collect::, _>>()?; + Ok(self.id_dictionary.remove_units(&ids)) + } +} + +impl Default for InternedSequenceDictionary { + fn default() -> Self { + Self::new() + } +} + +impl InternedSequenceDictionary { + /// Canonical profile metadata for the external symbol domain. + pub const fn profile_descriptor>( + &self, + ) -> crate::factory::BackendProfileDescriptor { + crate::factory::BackendProfileDescriptor::from_profile::

() + } + + /// Construct an empty coordinated dictionary with generation zero. + pub fn new() -> Self { + Self::with_generation(0) + } + + /// Construct an empty coordinated dictionary with an explicit generation. + pub fn with_generation(generation: u64) -> Self { + Self { + vocabulary: Arc::new(Mutex::new(InternedVocabulary::with_generation(generation))), + id_dictionary: DynamicDawgU32::new(), + } + } + + /// Borrow the vocabulary lock for read-only identity and reverse lookup. + /// + /// The guard is intentionally returned instead of cloning the vocabulary, + /// preserving zero-copy access and making the lifetime of the observation + /// explicit to callers. + pub fn vocabulary( + &self, + ) -> std::sync::LockResult>> { + self.vocabulary.lock() + } + + /// Capture the vocabulary boundary without retaining its mutex guard. + pub fn vocabulary_snapshot(&self) -> Result, InterningError> { + self.vocabulary + .lock() + .map(|vocabulary| vocabulary.snapshot()) + .map_err(|_| InterningError::Poisoned) + } + + /// Read the generation identity without exposing vocabulary storage. + pub fn generation(&self) -> Result { + self.vocabulary + .lock() + .map(|vocabulary| vocabulary.generation()) + .map_err(|_| InterningError::Poisoned) + } + + /// Access the ID-native dictionary for hot-loop consumers. + /// + /// Its sequences are meaningful only with this instance's vocabulary and + /// generation. The vocabulary remains the authority for constructing + /// valid sequences. + #[inline] + pub fn id_dictionary(&self) -> InternedIdDictionaryView<'_, u32, V> { + InternedIdDictionaryView::new(&self.id_dictionary) + } + + /// Query a generation-bound ID sequence after validating its vocabulary. + pub fn contains_id_sequence( + &self, + sequence: &InternedSequence, + ) -> Result { + let vocabulary = self + .vocabulary + .lock() + .map_err(|_| InterningError::Poisoned)?; + if sequence.generation() != vocabulary.generation() { + return Err(InterningError::GenerationMismatch { + expected: vocabulary.generation(), + actual: sequence.generation(), + }); + } + let ids = sequence + .as_ids() + .iter() + .copied() + .map(to_u32_id) + .collect::, _>>()?; + vocabulary.validate_sequence(sequence)?; + Ok(self.id_dictionary.contains_units(&ids)) + } + + /// Read a mapped value for a generation-bound ID sequence. + pub fn get_id_sequence_value( + &self, + sequence: &InternedSequence, + ) -> Result, InterningError> { + let vocabulary = self + .vocabulary + .lock() + .map_err(|_| InterningError::Poisoned)?; + if sequence.generation() != vocabulary.generation() { + return Err(InterningError::GenerationMismatch { + expected: vocabulary.generation(), + actual: sequence.generation(), + }); + } + let ids = sequence + .as_ids() + .iter() + .copied() + .map(to_u32_id) + .collect::, _>>()?; + vocabulary.validate_sequence(sequence)?; + Ok(self.id_dictionary.get_units_value(&ids)) + } + + /// Export atom sequences and mapped values in deterministic ID-dictionary + /// order, validating every vocabulary ID before exposing the snapshot. + pub fn visible_entries(&self) -> Result, InterningError> { + let vocabulary = self + .vocabulary + .lock() + .map_err(|_| InterningError::Poisoned)?; + self.id_dictionary + .visible_entries() + .into_iter() + .map(|(ids, value)| { + ids.into_iter() + .map(|id| { + vocabulary + .value(u64::from(id)) + .cloned() + .ok_or(InterningError::UnknownId(u64::from(id))) + }) + .collect::, _>>() + .map(|atoms| (atoms, value)) + }) + .collect() + } + + /// Intern atoms and insert their ID sequence atomically with respect to + /// other vocabulary mutations. + pub fn insert(&self, atoms: I, value: Option) -> Result + where + I: IntoIterator, + { + let atoms: Vec = atoms.into_iter().collect(); + let mut vocabulary = self + .vocabulary + .lock() + .map_err(|_| InterningError::Poisoned)?; + // Preflight known sequences so a duplicate result cannot leave newly + // interned atoms behind in the shared vocabulary. + let existing_ids = atoms + .iter() + .map(|atom| vocabulary.id_of(atom)) + .collect::>>(); + if let Some(existing_ids) = existing_ids { + let existing_ids = existing_ids + .into_iter() + .map(to_u32_id) + .collect::, _>>()?; + if self.id_dictionary.contains_units(&existing_ids) { + return Ok(false); + } + } + let sequence = vocabulary.try_intern_sequence(atoms)?; + let ids: Vec = sequence + .as_ids() + .iter() + .copied() + .map(to_u32_id) + .collect::>()?; + let inserted = match value { + Some(value) => self.id_dictionary.insert_units_with_value(&ids, value), + None => self.id_dictionary.insert_units(&ids), + }; + Ok(inserted) + } + + /// Intern and insert one shared logical profile sequence while retaining + /// the vocabulary's generation and ID validation boundary. + pub fn insert_atom_sequence

( + &self, + sequence: &crate::AtomSequence

, + value: Option, + ) -> Result + where + P: crate::AtomProfile, + { + self.insert(sequence.as_atoms().iter().cloned(), value) + } + + /// Test an atom sequence without changing the vocabulary. + pub fn contains(&self, atoms: I) -> Result + where + I: IntoIterator, + { + let vocabulary = self + .vocabulary + .lock() + .map_err(|_| InterningError::Poisoned)?; + let mut ids = Vec::new(); + for atom in atoms { + let id = vocabulary.id_of(&atom).ok_or(InterningError::UnknownKey)?; + ids.push(to_u32_id(id)?); + } + Ok(self.id_dictionary.contains_units(&ids)) + } + + /// Test one shared logical profile sequence without mutating the + /// vocabulary. + pub fn contains_atom_sequence

( + &self, + sequence: &crate::AtomSequence

, + ) -> Result + where + P: crate::AtomProfile, + { + self.contains(sequence.as_atoms().iter().cloned()) + } + + /// Read a mapped value for an already-interned atom sequence. + pub fn get_value(&self, atoms: I) -> Result, InterningError> + where + I: IntoIterator, + { + let vocabulary = self + .vocabulary + .lock() + .map_err(|_| InterningError::Poisoned)?; + let ids = atoms + .into_iter() + .map(|atom| { + vocabulary + .id_of(&atom) + .ok_or(InterningError::UnknownKey) + .and_then(to_u32_id) + }) + .collect::, _>>()?; + Ok(self.id_dictionary.get_units_value(&ids)) + } + + /// Read a mapped value for one already-interned shared logical profile + /// sequence. + pub fn get_atom_sequence_value

( + &self, + sequence: &crate::AtomSequence

, + ) -> Result, InterningError> + where + P: crate::AtomProfile, + { + self.get_value(sequence.as_atoms().iter().cloned()) + } + + /// Remove an atom sequence without changing vocabulary assignments. + pub fn remove(&self, atoms: I) -> Result + where + I: IntoIterator, + { + let vocabulary = self + .vocabulary + .lock() + .map_err(|_| InterningError::Poisoned)?; + let mut ids = Vec::new(); + for atom in atoms { + let id = vocabulary.id_of(&atom).ok_or(InterningError::UnknownKey)?; + ids.push(to_u32_id(id)?); + } + Ok(self.id_dictionary.remove_units(&ids)) + } +} + +#[cfg(test)] +mod coordinated_tests { + use super::{ + InternedSequence, InternedSequenceDictionary, InternedSequenceDictionaryU64, + InternedUlebSequenceDictionary, InternedUlebSequenceDictionaryU64, + }; + use crate::Uleb128; + + #[test] + fn profile_metadata_is_shared_by_vocabulary_and_id_dictionaries() { + let vocabulary = super::InternedVocabulary::::with_generation(11); + let snapshot = vocabulary.snapshot(); + let dictionary = InternedUlebSequenceDictionary::<()>::with_generation(11); + let wide_dictionary = InternedUlebSequenceDictionaryU64::<()>::with_generation(11); + + assert_eq!( + vocabulary.profile_descriptor::().kind, + crate::ProfileKind::Uleb128 + ); + assert_eq!( + snapshot + .profile_descriptor::() + .width_bytes, + None + ); + assert_eq!( + dictionary + .profile_descriptor::() + .identity, + crate::ProfileKind::Uleb128.identity() + ); + assert_eq!( + wide_dictionary + .profile_descriptor::() + .kind, + crate::ProfileKind::Uleb128 + ); + assert_eq!(dictionary.generation(), Ok(11)); + assert_eq!(wide_dictionary.generation(), Ok(11)); + } + + #[test] + fn coordinates_atoms_and_id_sequences() { + let dictionary = InternedSequenceDictionary::::with_generation(7); + assert!(dictionary.insert([10, 20], Some(99)).unwrap()); + assert!(dictionary.contains([10, 20]).unwrap()); + assert_eq!(dictionary.get_value([10, 20]).unwrap(), Some(99)); + let ids = InternedSequence::from_ids_with_generation(7, [0, 1]); + assert!(dictionary.contains_id_sequence(&ids).unwrap()); + assert_eq!(dictionary.get_id_sequence_value(&ids).unwrap(), Some(99)); + assert_eq!(dictionary.vocabulary().unwrap().generation(), 7); + assert_eq!(dictionary.generation(), Ok(7)); + assert_eq!(dictionary.id_dictionary().term_count(), 1); + assert_eq!( + dictionary.visible_entries().unwrap(), + vec![(vec![10, 20], Some(99))] + ); + assert_eq!( + dictionary.id_dictionary().visible_entries(), + vec![(vec![0u32, 1u32], Some(99))] + ); + assert!(dictionary.remove([10, 20]).unwrap()); + assert!(!dictionary.contains([10, 20]).unwrap()); + } + + #[test] + fn duplicate_insert_does_not_mutate_vocabulary() { + let dictionary = InternedSequenceDictionary::::new(); + assert!(dictionary.insert([10, 20], Some(1)).unwrap()); + let before: Vec<_> = dictionary + .vocabulary() + .unwrap() + .iter() + .map(|(id, value)| (id, *value)) + .collect(); + + assert!(!dictionary.insert([10, 20], Some(2)).unwrap()); + assert_eq!(dictionary.vocabulary().unwrap().len(), before.len()); + assert_eq!( + dictionary + .vocabulary() + .unwrap() + .iter() + .map(|(id, value)| (id, *value)) + .collect::>(), + before + ); + assert_eq!(dictionary.get_value([10, 20]).unwrap(), Some(1)); + + let wide = InternedSequenceDictionaryU64::::new(); + assert!(wide.insert([10, 20], Some(3)).unwrap()); + let wide_len = wide.vocabulary().unwrap().len(); + assert!(!wide.insert([10, 20], Some(4)).unwrap()); + assert_eq!(wide.vocabulary().unwrap().len(), wide_len); + assert_eq!(wide.get_value([10, 20]).unwrap(), Some(3)); + } + + #[test] + fn unknown_atoms_fail_closed_without_mutation() { + let dictionary = InternedSequenceDictionary::::new(); + assert_eq!( + dictionary.contains([1]), + Err(super::InterningError::UnknownKey) + ); + assert_eq!(dictionary.vocabulary().unwrap().len(), 0); + } + + #[test] + fn u32_id_sequence_rejects_unrepresentable_ids_as_exhaustion() { + let dictionary = InternedSequenceDictionary::::new(); + let sequence = InternedSequence::from_ids_with_generation(0, [u64::from(u32::MAX) + 1]); + assert_eq!( + dictionary.contains_id_sequence(&sequence), + Err(super::InterningError::IdExhausted) + ); + } + + #[test] + fn canonical_uleb_atoms_use_the_same_composite_boundary() { + let dictionary = InternedUlebSequenceDictionary::::with_generation(3); + let atoms = [Uleb128::from_u64(624_485), Uleb128::from_u64(1u64 << 63)]; + assert!(dictionary.insert(atoms.iter().cloned(), Some(11)).unwrap()); + assert!(dictionary.contains(atoms.iter().cloned()).unwrap()); + let vocabulary = dictionary.vocabulary().unwrap(); + assert_eq!(vocabulary.len(), 2); + assert_eq!(vocabulary.generation(), 3); + } + + #[test] + fn profile_sequences_use_the_same_interning_boundary() { + let sequence = crate::AtomSequence::::from_atoms([ + Uleb128::from_u64(624_485), + Uleb128::from_u64(1u64 << 63), + ]); + let dictionary = InternedUlebSequenceDictionary::::with_generation(4); + assert!(dictionary + .insert_atom_sequence(&sequence, Some(31)) + .unwrap()); + assert!(dictionary.contains_atom_sequence(&sequence).unwrap()); + assert_eq!( + dictionary.get_atom_sequence_value(&sequence).unwrap(), + Some(31) + ); + + let wide_dictionary = InternedUlebSequenceDictionaryU64::::new(); + assert!(wide_dictionary + .insert_atom_sequence(&sequence, Some(37)) + .unwrap()); + assert_eq!( + wide_dictionary.get_atom_sequence_value(&sequence).unwrap(), + Some(37) + ); + } + + #[test] + fn explicit_u64_carrier_preserves_generation_binding() { + let dictionary = InternedSequenceDictionaryU64::::with_generation(9); + assert!(dictionary.insert([u32::MAX], Some(17)).unwrap()); + assert!(dictionary.contains([u32::MAX]).unwrap()); + assert_eq!(dictionary.get_value([u32::MAX]).unwrap(), Some(17)); + let ids = InternedSequence::from_ids_with_generation(9, [0]); + assert!(dictionary.contains_id_sequence(&ids).unwrap()); + assert_eq!(dictionary.get_id_sequence_value(&ids).unwrap(), Some(17)); + assert_eq!( + dictionary.visible_entries().unwrap(), + vec![(vec![u32::MAX], Some(17))] + ); + assert_eq!(dictionary.vocabulary().unwrap().generation(), 9); + assert_eq!(dictionary.generation(), Ok(9)); + let snapshot = dictionary.vocabulary_snapshot().unwrap(); + assert_eq!(snapshot.generation(), 9); + assert_eq!(snapshot.value(0), Some(&u32::MAX)); + } + + #[test] + fn uleb_alias_exposes_explicit_u64_carrier() { + let dictionary = InternedUlebSequenceDictionaryU64::::with_generation(12); + let atoms = [Uleb128::from_u64(1u64 << 63), Uleb128::from_u64(624_485)]; + assert!(dictionary.insert(atoms.iter().cloned(), Some(23)).unwrap()); + assert_eq!( + dictionary.get_value(atoms.iter().cloned()).unwrap(), + Some(23) + ); + let ids = InternedSequence::from_ids_with_generation(12, [0, 1]); + assert!(dictionary.contains_id_sequence(&ids).unwrap()); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::Uleb128; + + #[test] + fn u32_carrier_overflow_is_not_reported_as_unknown_id() { + assert_eq!( + super::to_u32_id(u64::from(u32::MAX) + 1), + Err(InterningError::IdExhausted) + ); + assert_eq!(super::to_u32_id(17), Ok(17)); + } + + #[test] + fn interning_is_bijective_and_deterministic() { + let mut vocabulary = InternedVocabulary::new(); + assert_eq!(vocabulary.generation(), 0); + let first = vocabulary.intern(Uleb128::from_u64(42)); + assert_eq!(first, vocabulary.intern(Uleb128::from_u64(42))); + let second = vocabulary.intern(Uleb128::from_u64(1 << 63)); + assert_eq!(vocabulary.id_of(&Uleb128::from_u64(42)), Some(first)); + assert_eq!(vocabulary.value(second), Some(&Uleb128::from_u64(1 << 63))); + assert_eq!(vocabulary.len(), 2); + assert_eq!( + vocabulary.iter().map(|(id, _)| id).collect::>(), + vec![0, 1] + ); + let sequence = + vocabulary.intern_sequence([Uleb128::from_u64(42), Uleb128::from_u64(1 << 63)]); + assert_eq!(sequence.as_ids(), &[first, second]); + assert_eq!(sequence.generation(), 0); + assert!(sequence.is_bound_to(0)); + let resolved: Vec<_> = vocabulary.resolve_sequence(&sequence).unwrap().collect(); + assert_eq!(resolved.len(), 2); + assert!(vocabulary + .resolve_iter(&sequence) + .all(|value| value.is_some())); + let unknown = InternedSequence::from_ids([99]); + assert_eq!(vocabulary.resolve_iter(&unknown).next(), Some(None)); + assert_eq!(vocabulary.validate_sequence(&sequence), Ok(())); + assert_eq!( + vocabulary.validate_sequence(&unknown), + Err(InterningError::UnknownId(99)) + ); + assert_eq!(vocabulary.value(InternedId::MAX), None); + let other = InternedVocabulary::::with_generation(7); + assert_eq!(other.generation(), 7); + assert_eq!( + other.validate_sequence(&sequence), + Err(InterningError::GenerationMismatch { + expected: 7, + actual: 0, + }) + ); + } + + #[test] + fn vocabulary_snapshot_isolated_from_later_mutation() { + let mut vocabulary = InternedVocabulary::with_generation(17); + let first = vocabulary.intern(Uleb128::from_u64(3)); + let snapshot = vocabulary.snapshot(); + vocabulary.intern(Uleb128::from_u64(4)); + + assert_eq!(snapshot.generation(), 17); + assert_eq!(snapshot.value(first), Some(&Uleb128::from_u64(3))); + assert_eq!(snapshot.len(), 1); + assert_eq!(vocabulary.len(), 2); + let sequence = InternedSequence::from_ids_with_generation(17, [first]); + assert_eq!(snapshot.validate_sequence(&sequence), Ok(())); + assert_eq!( + snapshot.resolve_iter(&sequence).collect::>(), + vec![Some(&Uleb128::from_u64(3))] + ); + assert_eq!( + snapshot.validate_sequence(&InternedSequence::from_ids_with_generation(18, [first])), + Err(InterningError::GenerationMismatch { + expected: 17, + actual: 18, + }) + ); + } + + #[test] + fn f64_bits_alias_preserves_raw_identity() { + let dictionary = InternedF64BitsSequenceDictionary::::with_generation(3); + let atoms = [(-0.0f64).to_bits(), 0x7ff8_0000_0000_0042u64]; + assert!(dictionary.insert(atoms, Some(11)).unwrap()); + assert!(dictionary.contains(atoms).unwrap()); + assert_eq!(dictionary.get_value(atoms).unwrap(), Some(11)); + assert_eq!( + dictionary.contains([0u64, 0x7ff8_0000_0000_0042u64]), + Err(super::InterningError::UnknownKey) + ); + } +} diff --git a/src/lib.rs b/src/lib.rs index 071bfcb0..4471147c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -132,10 +132,14 @@ pub mod value_diff_zipper; // substrate shared by the variants, and `*zipper` = the navigators. pub mod double_array_trie; pub mod dynamic_dawg; +pub mod interning; #[cfg(feature = "pathmap-backend")] pub mod pathmap; +pub mod profile; +pub mod profiled_zipper; pub mod scdawg; pub mod suffix_automaton; +pub mod variable_width; // === Persistent ARTrie modules (feature-gated at module level) === // These modules are gated here; internal code does NOT need feature gates. @@ -152,7 +156,9 @@ pub mod persistent_artrie; pub mod serialization; // Re-export core types at crate root -pub use bijective::{BijectiveDictionary, BijectiveMap, InsertError}; +pub use bijective::{ + BijectiveDictionary, BijectiveMap, InsertError, ProfiledBijectiveMap, ProfiledBijectiveSnapshot, +}; pub use bloom_filter::BloomFilter; pub use char_unit::CharUnit; pub use collection::{ @@ -161,13 +167,38 @@ pub use collection::{ ExactSnapshotEntryIterator, SnapshotEntryIterator, SnapshotTermIterator, ValuedZipperCollection, ZipperCollection, ZipperEntryIterator, ZipperTermIterator, }; +pub use double_array_trie::{DoubleArrayTrieUleb128, DoubleArrayTrieUtf8}; pub use dynamic_dawg::core::{DawgCore, DawgNode}; +pub use dynamic_dawg::{ + DynamicDawgByteProfile, DynamicDawgCharProfile, DynamicDawgF64Bits, DynamicDawgGeneric, + DynamicDawgProfile, DynamicDawgU32, DynamicDawgU64Profile, DynamicDawgUleb128, DynamicDawgUtf8, +}; +pub use interning::{ + InternedEntries, InternedF64BitsSequenceDictionary, InternedF64BitsSequenceDictionaryU64, + InternedId, InternedIdDictionaryView, InternedSequence, InternedSequenceDictionary, + InternedSequenceDictionaryU64, InternedUlebSequenceDictionary, + InternedUlebSequenceDictionaryU64, InternedVocabulary, InternedVocabularySnapshot, + InterningError, +}; pub use iterator::{DictionaryIterator, DictionaryTermIterator}; pub use node_signature::NodeSignature; +#[cfg(feature = "pathmap-backend")] +pub use pathmap::PathMapDictionaryUleb128; +#[cfg(feature = "pathmap-backend")] +pub use pathmap::PathMapDictionaryUtf8; +pub use profile::{ + AtomProfile, AtomSequence, AtomStream, Bytes, F64Bits, ProfileError, ProfileKind, Uleb128Atom, + UnicodeScalar, Utf8, U32, U64, +}; +pub use profiled_zipper::ProfiledZipper; pub use substring::{ BidirectionalDictionaryNode, ExtensionResult, SubstringDictionary, SubstringMatch, }; pub use value::DictionaryValue; +pub use variable_width::{ + validate_uleb128_sequence, Uleb128, Uleb128Codec, Uleb128Error, Uleb128Ref, Uleb128Sequence, + Uleb128Stream, VariableWidthCodec, VariableWidthProfile, ULEB128_PROFILE, +}; pub use zipper::{DictZipper, ValuedDictZipper, ZipperTraversalNode}; // Re-export persistent ARTrie types (only available with feature) @@ -189,12 +220,13 @@ pub use persistent_artrie::vocab::{IndexedVocabularyPersistent, PersistentVocabA pub use persistent_artrie::wal::Lsn; #[cfg(feature = "persistent-artrie")] pub use persistent_artrie::{ - PersistentARTrie, PersistentARTrieU64, PersistentARTrieU64Node, PersistentARTrieZipper, - PersistentScdawg, PersistentScdawgChar, PersistentScdawgCharNode, PersistentScdawgNode, + PersistentARTrie, PersistentARTrieU64, PersistentARTrieU64Node, PersistentARTrieUleb128, + PersistentARTrieUtf8, PersistentARTrieZipper, PersistentScdawg, PersistentScdawgChar, + PersistentScdawgCharNode, PersistentScdawgNode, PersistentScdawgUtf8, PersistentSuffixAutomaton, PersistentSuffixAutomatonChar, PersistentSuffixAutomatonCharNode, - PersistentSuffixAutomatonNode, PersistentSuffixTree, PersistentSuffixTreeChar, - PersistentSuffixTreeCharNode, PersistentSuffixTreeNode, RecoveryMode, RecoveryReport, - WalConfig, + PersistentSuffixAutomatonNode, PersistentSuffixAutomatonUtf8, PersistentSuffixTree, + PersistentSuffixTreeChar, PersistentSuffixTreeCharNode, PersistentSuffixTreeNode, + PersistentSuffixTreeUtf8, RecoveryMode, RecoveryReport, WalConfig, }; /// Synchronization strategy for dictionary operations. @@ -1865,25 +1897,49 @@ pub trait MutableMappedDictionary: MappedDictionary { /// Prelude module for convenient imports. pub mod prelude { + pub use crate::factory::{ + BackendProfileDescriptor, DictionaryBackend, DictionaryDescriptor, + DictionaryDescriptorError, DictionaryFamily, DictionarySpec, ProfiledDictionaryContainer, + ProfiledFactoryError, Uleb128Backend, Uleb128DictionaryContainer, + }; + pub use crate::ProfiledZipper; pub use crate::{ BijectiveDictionary, BijectiveMap, CharUnit, CompactableDictionary, DictZipper, Dictionary, DictionaryEntries, DictionaryEntriesIter, DictionaryEntry, DictionaryKeys, DictionaryLanguageEntries, DictionaryLanguageTerms, DictionaryNode, DictionaryTerms, DictionaryValue, DictionaryValues, ExactSnapshotEntryIterator, InsertError, + InternedEntries, InternedF64BitsSequenceDictionary, InternedF64BitsSequenceDictionaryU64, + InternedId, InternedIdDictionaryView, InternedSequence, InternedSequenceDictionary, + InternedSequenceDictionaryU64, InternedUlebSequenceDictionary, + InternedUlebSequenceDictionaryU64, InternedVocabulary, InternedVocabularySnapshot, MappedDictionary, MappedDictionaryNode, MutableDictionary, MutableMappedDictionary, - SnapshotEntryIterator, SnapshotTermIterator, SyncStrategy, ValuedDictZipper, + ProfiledBijectiveMap, ProfiledBijectiveSnapshot, SnapshotEntryIterator, + SnapshotTermIterator, SyncStrategy, Uleb128Atom, Utf8, ValuedDictZipper, ValuedZipperCollection, ZipperCollection, ZipperEntryIterator, ZipperTermIterator, }; // Re-export common dictionary types - pub use crate::double_array_trie::{DoubleArrayTrie, DoubleArrayTrieChar}; - pub use crate::dynamic_dawg::{DynamicDawg, DynamicDawgChar, DynamicDawgU64}; - pub use crate::scdawg::{Scdawg, ScdawgChar}; - pub use crate::suffix_automaton::{SuffixAutomaton, SuffixAutomatonChar}; + pub use crate::double_array_trie::{ + DoubleArrayTrie, DoubleArrayTrieChar, DoubleArrayTrieUleb128, DoubleArrayTrieUtf8, + }; + pub use crate::dynamic_dawg::{ + DynamicDawg, DynamicDawgByteProfile, DynamicDawgChar, DynamicDawgCharProfile, + DynamicDawgF64Bits, DynamicDawgGeneric, DynamicDawgProfile, DynamicDawgU32, DynamicDawgU64, + DynamicDawgU64Profile, DynamicDawgUleb128, DynamicDawgUtf8, + }; + #[cfg(feature = "pathmap-backend")] + pub use crate::pathmap::{PathMapDictionaryUleb128, PathMapDictionaryUtf8}; + pub use crate::scdawg::{Scdawg, ScdawgChar, ScdawgUtf8}; + pub use crate::suffix_automaton::{SuffixAutomaton, SuffixAutomatonChar, SuffixAutomatonUtf8}; + pub use crate::{ + AtomProfile, Bytes, F64Bits, ProfileKind, UnicodeScalar, VariableWidthProfile, U32, U64, + }; #[cfg(feature = "persistent-artrie")] pub use crate::persistent_artrie::{ - PersistentARTrieU64, PersistentScdawg, PersistentScdawgChar, PersistentSuffixAutomaton, - PersistentSuffixAutomatonChar, PersistentSuffixTree, PersistentSuffixTreeChar, + PersistentARTrieU64, PersistentARTrieUleb128, PersistentARTrieUtf8, PersistentScdawg, + PersistentScdawgChar, PersistentScdawgUtf8, PersistentSuffixAutomaton, + PersistentSuffixAutomatonChar, PersistentSuffixAutomatonUtf8, PersistentSuffixTree, + PersistentSuffixTreeChar, PersistentSuffixTreeUtf8, }; } diff --git a/src/pathmap/ascii.rs b/src/pathmap/ascii.rs index 72560b33..2890caed 100644 --- a/src/pathmap/ascii.rs +++ b/src/pathmap/ascii.rs @@ -56,6 +56,16 @@ impl fmt::Debug for PathMapDictionary { } impl PathMapDictionary { + /// Canonical logical profile represented by this byte PathMap adapter. + pub const fn profile_descriptor() -> crate::factory::BackendProfileDescriptor { + crate::factory::BackendProfileDescriptor::from_profile::() + } + + /// Topology family represented by this adapter. + pub const fn dictionary_family() -> crate::factory::DictionaryFamily { + crate::factory::DictionaryFamily::PathMap + } + #[inline] fn from_state(map: PathMap, len: usize) -> Self { Self { @@ -153,6 +163,35 @@ impl PathMapDictionary { Self::from_state(map, count) } + /// Build from byte-profile sequences without UTF-8 coercion. + pub fn from_atom_sequences(sequences: I) -> Self + where + P: crate::AtomProfile, + I: IntoIterator>, + V: Default, + { + Self::from_byte_entries( + sequences + .into_iter() + .map(|sequence| (sequence.as_atoms().to_vec(), V::default())) + .collect(), + ) + } + + /// Build a value-bearing dictionary from byte-profile sequences. + pub fn from_atom_sequences_with_values(entries: I) -> Self + where + P: crate::AtomProfile, + I: IntoIterator, V)>, + { + Self::from_byte_entries( + entries + .into_iter() + .map(|(sequence, value)| (sequence.as_atoms().to_vec(), value)) + .collect(), + ) + } + /// Insert a term with a default value into the dictionary /// /// Returns `true` if the term was newly inserted, `false` if it already existed. @@ -194,6 +233,29 @@ impl PathMapDictionary { } } + /// Insert or update an arbitrary byte key without UTF-8 coercion. + pub fn insert_bytes_with_value(&self, bytes: &[u8], value: V) -> bool { + let mut backoff = CasBackoff::new(); + loop { + let current = self.load_state(); + let mut next_map = current.map.clone(); + let inserted = next_map.insert(bytes, value.clone()).is_none(); + let next_len = current.len + usize::from(inserted); + if self.compare_store_state(¤t, PathMapState::new(next_map, next_len)) { + return inserted; + } + backoff.snooze(); + } + } + + /// Insert an arbitrary byte key with a default value. + pub fn insert_bytes(&self, bytes: &[u8]) -> bool + where + V: Default, + { + self.insert_bytes_with_value(bytes, V::default()) + } + /// Remove a term from the dictionary /// /// Returns `true` if the term was present and removed, `false` if it didn't exist. @@ -221,6 +283,23 @@ impl PathMapDictionary { } } + /// Remove an arbitrary byte key without UTF-8 coercion. + pub fn remove_bytes(&self, bytes: &[u8]) -> bool { + let mut backoff = CasBackoff::new(); + loop { + let current = self.load_state(); + let mut next_map = current.map.clone(); + if next_map.remove_val_at(bytes, true).is_none() { + return false; + } + let next_len = current.len.saturating_sub(1); + if self.compare_store_state(¤t, PathMapState::new(next_map, next_len)) { + return true; + } + backoff.snooze(); + } + } + /// Clear all terms from the dictionary /// /// # Thread Safety @@ -251,6 +330,11 @@ impl PathMapDictionary { self.load_state().len } + /// Test membership of an arbitrary byte key without UTF-8 coercion. + pub fn contains_bytes(&self, bytes: &[u8]) -> bool { + self.load_state().map.get_val_at(bytes).is_some() + } + /// Serialize to PathMap's native .paths format /// /// # Thread Safety @@ -302,6 +386,20 @@ impl PathMapDictionary { state.map.get_val_at(bytes).cloned() } + /// Read a mapped value for an arbitrary byte key without UTF-8 coercion. + pub fn get_bytes_value(&self, bytes: &[u8]) -> Option { + self.load_state().map.get_val_at(bytes).cloned() + } + + /// Read a value for a byte-profile sequence without UTF-8 coercion. + #[inline] + pub fn get_atom_sequence_value

(&self, sequence: &crate::AtomSequence

) -> Option + where + P: crate::AtomProfile, + { + self.get_bytes_value(sequence.as_atoms()) + } + /// Update an existing term's value in place, or insert a new term with a default value. /// /// This method is useful for accumulation patterns where you want to modify an existing diff --git a/src/pathmap/char.rs b/src/pathmap/char.rs index 1983c938..ed5ac15d 100644 --- a/src/pathmap/char.rs +++ b/src/pathmap/char.rs @@ -83,6 +83,16 @@ impl fmt::Debug for PathMapDictionaryChar { } impl PathMapDictionaryChar { + /// Canonical logical profile represented by this Unicode PathMap adapter. + pub const fn profile_descriptor() -> crate::factory::BackendProfileDescriptor { + crate::factory::BackendProfileDescriptor::from_profile::() + } + + /// Topology family represented by this adapter. + pub const fn dictionary_family() -> crate::factory::DictionaryFamily { + crate::factory::DictionaryFamily::PathMap + } + #[inline] fn from_state(map: PathMap, len: usize) -> Self { Self { @@ -182,6 +192,36 @@ impl PathMapDictionaryChar { Self::from_state(map, count) } + /// Build from Unicode-scalar profile sequences while retaining scalar + /// boundaries at the adapter API. + pub fn from_atom_sequences(sequences: I) -> Self + where + P: crate::AtomProfile, + I: IntoIterator>, + V: Default, + { + Self::from_char_entries( + sequences + .into_iter() + .map(|sequence| (sequence.as_atoms().to_vec(), V::default())) + .collect(), + ) + } + + /// Build a value-bearing dictionary from Unicode-scalar profile sequences. + pub fn from_atom_sequences_with_values(entries: I) -> Self + where + P: crate::AtomProfile, + I: IntoIterator, V)>, + { + Self::from_char_entries( + entries + .into_iter() + .map(|(sequence, value)| (sequence.as_atoms().to_vec(), value)) + .collect(), + ) + } + /// Insert a term with a default value into the dictionary /// /// Returns `true` if the term was newly inserted, `false` if it already existed. @@ -293,6 +333,17 @@ impl PathMapDictionaryChar { state.map.get_val_at(bytes).cloned() } + /// Read a value for a Unicode-scalar profile sequence. PathMap stores + /// UTF-8 bytes, so this adapter performs one bounded encoding allocation at + /// the representation boundary; traversal remains scalar-level to callers. + pub fn get_atom_sequence_value

(&self, sequence: &crate::AtomSequence

) -> Option + where + P: crate::AtomProfile, + { + let term: String = sequence.as_atoms().iter().collect(); + self.get_value(&term) + } + /// Update an existing term's value in place, or insert a new term with a default value. /// /// This method is useful for accumulation patterns where you want to modify an existing diff --git a/src/pathmap/mod.rs b/src/pathmap/mod.rs index 163dc02e..f7785684 100644 --- a/src/pathmap/mod.rs +++ b/src/pathmap/mod.rs @@ -15,6 +15,8 @@ pub mod core; pub mod snapshot; pub mod zipper; +use crate::{Dictionary, DictionaryEntries}; + pub use self::core::{ trie_ref_root, trie_ref_root_borrowed, TrieRefLike, TrieRefNode, TrieRefNodeChar, }; @@ -22,3 +24,492 @@ pub use ascii::{PathMapDictionary, PathMapNode}; pub use char::{PathMapDictionaryChar, PathMapNodeChar}; pub use snapshot::{PathMapRef, PathMapRefChar, PathMapSnapshot, PathMapSnapshotChar}; pub use zipper::PathMapZipper; + +/// PathMap adapter boundary for canonical variable-width ULEB128 sequences. +/// The third-party map remains byte-backed; this type exposes only complete +/// logical sequences and never publishes continuation bytes as symbols. +#[derive(Clone, Debug)] +pub struct PathMapDictionaryUleb128 { + inner: PathMapDictionary, +} + +/// PathMap adapter boundary for variable-width UTF-8 strings. +#[derive(Clone, Debug)] +pub struct PathMapDictionaryUtf8 { + inner: PathMapDictionary, +} + +impl Default for PathMapDictionaryUtf8 { + fn default() -> Self { + Self::new() + } +} + +impl PathMapDictionaryUtf8 { + /// Canonical logical profile represented by this boundary. + pub const fn profile_descriptor() -> crate::factory::BackendProfileDescriptor { + crate::factory::BackendProfileDescriptor::from_profile::() + } + + /// Topology family represented by this boundary. + pub const fn dictionary_family() -> crate::factory::DictionaryFamily { + crate::factory::DictionaryFamily::PathMap + } + + pub fn new() -> Self { + Self { + inner: PathMapDictionary::new(), + } + } + pub fn from_terms(terms: I) -> Self + where + I: IntoIterator, + S: AsRef, + { + let dictionary = Self::new(); + for term in terms { + dictionary.insert(term.as_ref()); + } + dictionary + } + pub fn from_terms_with_values(entries: I) -> Self + where + I: IntoIterator, + S: AsRef, + { + let dictionary = Self::new(); + for (term, value) in entries { + dictionary.insert_with_value(term.as_ref(), value); + } + dictionary + } + + /// Build from shared logical UTF-8 scalar profile sequences. + pub fn from_atom_sequences(sequences: I) -> Self + where + I: IntoIterator>, + { + let dictionary = Self::new(); + for sequence in sequences { + dictionary.insert_atom_sequence(&sequence); + } + dictionary + } + + /// Build a value-bearing adapter from shared logical UTF-8 scalar profile + /// sequences. + pub fn from_atom_sequences_with_values(entries: I) -> Self + where + I: IntoIterator, V)>, + { + let dictionary = Self::new(); + for (sequence, value) in entries { + dictionary.insert_atom_sequence_with_value(&sequence, value); + } + dictionary + } + #[inline] + pub fn insert(&self, term: &str) -> bool { + self.inner.insert_bytes(term.as_bytes()) + } + + /// Insert one shared logical UTF-8 scalar profile sequence. + #[inline] + pub fn insert_atom_sequence(&self, sequence: &crate::AtomSequence) -> bool { + self.inner.insert_bytes(&sequence.to_encoded()) + } + #[inline] + pub fn insert_with_value(&self, term: &str, value: V) -> bool { + self.inner.insert_bytes_with_value(term.as_bytes(), value) + } + + /// Insert one shared logical UTF-8 scalar profile sequence with a value. + #[inline] + pub fn insert_atom_sequence_with_value( + &self, + sequence: &crate::AtomSequence, + value: V, + ) -> bool { + self.inner + .insert_bytes_with_value(&sequence.to_encoded(), value) + } + + /// Validate and insert one complete UTF-8 encoded key without coercing it + /// through an intermediate string allocation. + pub fn insert_encoded(&self, encoded: &[u8], value: V) -> Result { + std::str::from_utf8(encoded)?; + Ok(self.inner.insert_bytes_with_value(encoded, value)) + } + + #[inline] + pub fn contains(&self, term: &str) -> bool { + self.inner.contains_bytes(term.as_bytes()) + } + + /// Test membership of one shared logical UTF-8 scalar profile sequence. + #[inline] + pub fn contains_atom_sequence(&self, sequence: &crate::AtomSequence) -> bool { + self.inner.contains_bytes(&sequence.to_encoded()) + } + #[inline] + pub fn get_value(&self, term: &str) -> Option { + self.inner.get_bytes_value(term.as_bytes()) + } + + /// Read a mapped value for one shared logical UTF-8 scalar profile + /// sequence. + #[inline] + pub fn get_atom_sequence_value( + &self, + sequence: &crate::AtomSequence, + ) -> Option { + self.inner.get_bytes_value(&sequence.to_encoded()) + } + #[inline] + pub fn remove(&self, term: &str) -> bool { + self.inner.remove_bytes(term.as_bytes()) + } + #[inline] + pub fn term_count(&self) -> usize { + self.inner.len().unwrap_or(0) + } + #[inline] + pub fn is_empty(&self) -> bool { + self.term_count() == 0 + } + pub fn contains_encoded(&self, encoded: &[u8]) -> Result { + std::str::from_utf8(encoded)?; + Ok(self.inner.contains_bytes(encoded)) + } + + /// Read a mapped value for one complete UTF-8 encoded key without + /// allocating or decoding its scalar sequence. Invalid UTF-8 is rejected + /// before the third-party byte map is consulted. + pub fn get_encoded_value(&self, encoded: &[u8]) -> Result, std::str::Utf8Error> { + std::str::from_utf8(encoded)?; + Ok(self.inner.get_bytes_value(encoded)) + } + + pub fn remove_encoded(&self, encoded: &[u8]) -> Result { + std::str::from_utf8(encoded)?; + Ok(self.inner.remove_bytes(encoded)) + } + pub fn visible_entries(&self) -> Result)>, std::str::Utf8Error> { + self.inner + .entries() + .map(|entry| std::str::from_utf8(&entry.key).map(|s| (s.to_owned(), entry.value))) + .collect() + } +} + +impl PathMapDictionaryUleb128 { + /// Canonical logical profile represented by this boundary. + pub const fn profile_descriptor() -> crate::factory::BackendProfileDescriptor { + crate::factory::BackendProfileDescriptor::from_profile::() + } + + /// Topology family represented by this boundary. + pub const fn dictionary_family() -> crate::factory::DictionaryFamily { + crate::factory::DictionaryFamily::PathMap + } + + /// Construct an empty ULEB128 PathMap adapter. + pub fn new() -> Self { + Self { + inner: PathMapDictionary::new(), + } + } + + /// Build a value-bearing adapter from complete canonical ULEB sequences. + pub fn from_sequences_with_values(entries: I) -> Self + where + I: IntoIterator, + { + let dictionary = Self::new(); + for (sequence, value) in entries { + dictionary.insert_with_value(&sequence, value); + } + dictionary + } + + /// Build from the shared logical ULEB profile sequence representation. + pub fn from_atom_sequences_with_values(entries: I) -> Self + where + I: IntoIterator, V)>, + { + Self::from_sequences_with_values( + entries + .into_iter() + .map(|(sequence, value)| (sequence.into(), value)), + ) + } + + /// Build an unvalued adapter from complete canonical ULEB sequences. + pub fn from_sequences(sequences: I) -> Self + where + I: IntoIterator, + { + let dictionary = Self::new(); + for sequence in sequences { + dictionary.insert(&sequence); + } + dictionary + } + + /// Build an unvalued adapter from shared logical ULEB profile sequences. + pub fn from_atom_sequences(sequences: I) -> Self + where + I: IntoIterator>, + { + Self::from_sequences(sequences.into_iter().map(Into::into)) + } + + /// Insert one complete ULEB128 sequence. + #[inline] + pub fn insert(&self, sequence: &crate::Uleb128Sequence) -> bool { + self.inner.insert_bytes(&sequence.to_encoded()) + } + + /// Insert one shared logical ULEB profile sequence. + #[inline] + pub fn insert_atom_sequence(&self, sequence: &crate::AtomSequence) -> bool { + self.inner.insert_bytes(&sequence.to_encoded()) + } + + /// Insert one complete ULEB128 sequence with a mapped value. + #[inline] + pub fn insert_with_value(&self, sequence: &crate::Uleb128Sequence, value: V) -> bool { + self.inner + .insert_bytes_with_value(&sequence.to_encoded(), value) + } + + /// Insert one shared logical ULEB profile sequence with a mapped value. + #[inline] + pub fn insert_atom_sequence_with_value( + &self, + sequence: &crate::AtomSequence, + value: V, + ) -> bool { + self.inner + .insert_bytes_with_value(&sequence.to_encoded(), value) + } + + /// Insert one complete canonical encoded ULEB128 sequence without + /// materializing its decoded atoms. + pub fn insert_encoded(&self, encoded: &[u8], value: V) -> Result { + crate::validate_uleb128_sequence(encoded)?; + Ok(self.inner.insert_bytes_with_value(encoded, value)) + } + + /// Test one complete ULEB128 sequence. + #[inline] + pub fn contains(&self, sequence: &crate::Uleb128Sequence) -> bool { + self.inner.contains_bytes(&sequence.to_encoded()) + } + + /// Test membership of one shared logical ULEB profile sequence. + #[inline] + pub fn contains_atom_sequence( + &self, + sequence: &crate::AtomSequence, + ) -> bool { + self.inner.contains_bytes(&sequence.to_encoded()) + } + + /// Test a complete canonical encoded sequence without materializing its + /// decoded atoms. Malformed or non-canonical images are rejected. + pub fn contains_encoded(&self, encoded: &[u8]) -> Result { + crate::validate_uleb128_sequence(encoded)?; + Ok(self.inner.contains_bytes(encoded)) + } + + /// Read a mapped value for one complete ULEB128 sequence. + #[inline] + pub fn get_value(&self, sequence: &crate::Uleb128Sequence) -> Option { + self.inner.get_bytes_value(&sequence.to_encoded()) + } + + /// Read a mapped value for one shared logical ULEB profile sequence. + #[inline] + pub fn get_atom_sequence_value( + &self, + sequence: &crate::AtomSequence, + ) -> Option { + self.inner.get_bytes_value(&sequence.to_encoded()) + } + + /// Read a value for a complete canonical encoded sequence without + /// materializing its decoded atoms. + pub fn get_encoded_value(&self, encoded: &[u8]) -> Result, crate::Uleb128Error> { + crate::validate_uleb128_sequence(encoded)?; + Ok(self.inner.get_bytes_value(encoded)) + } + + /// Remove one complete ULEB128 sequence. + #[inline] + pub fn remove(&self, sequence: &crate::Uleb128Sequence) -> bool { + self.inner.remove_bytes(&sequence.to_encoded()) + } + + /// Remove a complete canonical encoded sequence without decoding it. + pub fn remove_encoded(&self, encoded: &[u8]) -> Result { + crate::validate_uleb128_sequence(encoded)?; + Ok(self.inner.remove_bytes(encoded)) + } + + /// Number of visible logical sequences in the current snapshot. + #[inline] + pub fn term_count(&self) -> usize { + self.inner.len().unwrap_or(0) + } + + /// Whether the current snapshot contains no logical sequences. + #[inline] + pub fn is_empty(&self) -> bool { + self.term_count() == 0 + } + + /// Export complete logical sequences from one immutable PathMap snapshot. + /// Continuation bytes are decoded only at this boundary and never exposed + /// as semantic transitions. + pub fn visible_entries( + &self, + ) -> Result)>, crate::Uleb128Error> { + self.inner + .entries() + .map(|entry| { + crate::Uleb128Sequence::from_encoded(&entry.key) + .map(|sequence| (sequence, entry.value)) + }) + .collect() + } +} + +impl Default for PathMapDictionaryUleb128 { + fn default() -> Self { + Self::new() + } +} + +#[cfg(all(test, feature = "pathmap-backend"))] +mod profile_tests { + use super::{ + PathMapDictionary, PathMapDictionaryChar, PathMapDictionaryUleb128, PathMapDictionaryUtf8, + }; + use crate::{AtomSequence, Bytes, Dictionary, UnicodeScalar}; + + #[test] + fn uleb_adapter_preserves_logical_sequences() { + let sequence = crate::Uleb128Sequence::from_atoms([ + crate::Uleb128::from_u64(624_485), + crate::Uleb128::from_u64(7), + ]); + let dictionary = PathMapDictionaryUleb128::::new(); + assert!(dictionary.insert_with_value(&sequence, 19)); + assert!(dictionary.contains(&sequence)); + assert_eq!(dictionary.get_value(&sequence), Some(19)); + let encoded = sequence.to_encoded(); + assert!(dictionary.insert_encoded(&encoded, 20).is_ok()); + assert_eq!(dictionary.get_encoded_value(&encoded).unwrap(), Some(20)); + assert!(dictionary.insert_encoded(&[0x80], 1).is_err()); + assert!(dictionary.remove(&sequence)); + assert!(!dictionary.contains(&sequence)); + } + + #[test] + fn uleb_adapter_accepts_shared_profile_sequences() { + let sequence = crate::AtomSequence::::from_atoms([ + crate::Uleb128::from_u64(624_485), + crate::Uleb128::from_u64(1u64 << 63), + ]); + let dictionary = PathMapDictionaryUleb128::::from_atom_sequences_with_values([( + sequence.clone(), + 23, + )]); + assert!(dictionary.contains_atom_sequence(&sequence)); + assert_eq!(dictionary.get_atom_sequence_value(&sequence), Some(23)); + } + + #[test] + fn utf8_adapter_accepts_shared_profile_sequences() { + let sequence = crate::AtomSequence::::from_atoms(['Ξ»', 'πŸŽ‰']); + let dictionary = + PathMapDictionaryUtf8::::from_atom_sequences_with_values([(sequence.clone(), 29)]); + assert!(dictionary.contains_atom_sequence(&sequence)); + assert_eq!(dictionary.get_atom_sequence_value(&sequence), Some(29)); + } + + #[test] + fn uleb_adapter_builds_unvalued_sequences_and_rejects_malformed_images() { + let sequence = crate::Uleb128Sequence::from_atoms([crate::Uleb128::from_u64(9)]); + let dictionary = PathMapDictionaryUleb128::<()>::from_sequences([sequence.clone()]); + assert_eq!( + dictionary.contains_encoded(sequence.to_encoded().as_slice()), + Ok(true) + ); + assert!(dictionary.contains_encoded(&[0x80]).is_err()); + assert_eq!(dictionary.term_count(), 1); + assert_eq!(dictionary.visible_entries().unwrap().len(), 1); + assert!(dictionary + .remove_encoded(sequence.to_encoded().as_slice()) + .unwrap()); + assert!(dictionary.is_empty()); + } + + #[test] + fn utf8_adapter_preserves_logical_entries() { + let dictionary = + PathMapDictionaryUtf8::::from_terms_with_values([("Ξ»πŸŽ‰", 9), ("a", 1)]); + assert!(dictionary.contains("Ξ»πŸŽ‰")); + assert_eq!(dictionary.get_value("Ξ»πŸŽ‰"), Some(9)); + assert_eq!(dictionary.visible_entries().unwrap().len(), 2); + assert!(dictionary.contains_encoded("Ξ»πŸŽ‰".as_bytes()).unwrap()); + assert_eq!( + dictionary.get_encoded_value("Ξ»πŸŽ‰".as_bytes()).unwrap(), + Some(9) + ); + assert!(dictionary.get_encoded_value(&[0x80]).is_err()); + assert!(dictionary.contains_encoded(&[0x80]).is_err()); + assert!(!dictionary.insert_encoded("Ξ»πŸŽ‰".as_bytes(), 10).unwrap()); + assert_eq!(dictionary.get_value("Ξ»πŸŽ‰"), Some(10)); + assert!(dictionary.insert_encoded(&[0x80], 1).is_err()); + assert!(!dictionary.is_empty()); + assert!(dictionary.remove_encoded("Ξ»πŸŽ‰".as_bytes()).unwrap()); + assert!(!dictionary.contains("Ξ»πŸŽ‰")); + } + + #[test] + fn byte_profile_constructor_preserves_membership_and_values() { + let dictionary = PathMapDictionary::::from_atom_sequences_with_values::([( + AtomSequence::::from_atoms([b'a', b'b']), + 13, + )]); + assert!(dictionary.contains("ab")); + assert_eq!(dictionary.get_value("ab"), Some(13)); + } + + #[test] + fn byte_adapter_supports_arbitrary_encoded_keys() { + let dictionary = PathMapDictionary::::new(); + assert!(dictionary.insert_bytes_with_value(&[0, 255, 1], 34)); + assert!(dictionary.contains_bytes(&[0, 255, 1])); + assert_eq!(dictionary.get_bytes_value(&[0, 255, 1]), Some(34)); + let sequence = AtomSequence::::from_atoms([0, 255, 1]); + assert_eq!(dictionary.get_atom_sequence_value(&sequence), Some(34)); + assert!(dictionary.remove_bytes(&[0, 255, 1])); + assert!(!dictionary.contains_bytes(&[0, 255, 1])); + } + + #[test] + fn unicode_profile_constructor_preserves_scalar_boundaries_and_values() { + let dictionary = PathMapDictionaryChar::::from_atom_sequences_with_values::< + UnicodeScalar, + _, + >([(AtomSequence::::from_atoms(['Ξ»', 'x']), 21)]); + assert!(dictionary.contains("Ξ»x")); + assert!(!dictionary.contains("lx")); + assert_eq!(dictionary.get_value("Ξ»x"), Some(21)); + let sequence = AtomSequence::::from_atoms(['Ξ»', 'x']); + assert_eq!(dictionary.get_atom_sequence_value(&sequence), Some(21)); + } +} diff --git a/src/persistent_artrie/char/mmap_ctor.rs b/src/persistent_artrie/char/mmap_ctor.rs index 84ca81d8..1e40d583 100644 --- a/src/persistent_artrie/char/mmap_ctor.rs +++ b/src/persistent_artrie/char/mmap_ctor.rs @@ -734,6 +734,36 @@ impl super::PersistentARTrieChar { } } + /// Build an in-memory character ART from Unicode-scalar profile sequences. + pub fn from_atom_sequences(sequences: I) -> Self + where + P: crate::AtomProfile, + I: IntoIterator>, + { + let trie = Self::new(); + for sequence in sequences { + let text: String = sequence.as_atoms().iter().copied().collect(); + trie.insert(&text) + .expect("in-memory profile insertion must succeed"); + } + trie + } + + /// Build a value-bearing in-memory character ART from profile sequences. + pub fn from_atom_sequences_with_values(entries: I) -> Self + where + P: crate::AtomProfile, + I: IntoIterator, V)>, + { + let trie = Self::new(); + for (sequence, value) in entries { + let text: String = sequence.as_atoms().iter().copied().collect(); + trie.insert_with_value(&text, value) + .expect("in-memory profile insertion must succeed"); + } + trie + } + /// Flush dirty arenas in sequential order for optimized disk I/O. /// /// Sorts dirty arenas by ID before flushing, improving I/O locality diff --git a/src/persistent_artrie/char/query_api.rs b/src/persistent_artrie/char/query_api.rs index e808ee8d..0ada17e3 100644 --- a/src/persistent_artrie/char/query_api.rs +++ b/src/persistent_artrie/char/query_api.rs @@ -83,6 +83,16 @@ impl super::PersistentARTrieChar { self.overlay_get_value(term).flatten() } + /// Read a mapped value for a Unicode-scalar profile sequence while + /// preserving the canonical overlay-routed lookup path. + pub fn get_atom_sequence_value

(&self, sequence: &crate::AtomSequence

) -> Option + where + P: crate::AtomProfile, + { + let term: String = sequence.as_atoms().iter().collect(); + self.get_value(&term) + } + /// Get a value by term with explicit error handling. /// /// This version returns a `Result` for lazy loading I/O errors. diff --git a/src/persistent_artrie/core/eviction/coordinator.rs b/src/persistent_artrie/core/eviction/coordinator.rs index cde104f0..3b663696 100644 --- a/src/persistent_artrie/core/eviction/coordinator.rs +++ b/src/persistent_artrie/core/eviction/coordinator.rs @@ -580,6 +580,7 @@ impl EvictionCoordinator { }) } + #[cfg(any(test, feature = "bench-internals"))] pub(crate) fn start_compact(self: &Arc, callback: F) -> Result<(), String> where F: Fn(CompactEvictionBatch) -> (usize, usize) + Send + Sync + 'static, @@ -590,6 +591,7 @@ impl EvictionCoordinator { }) } + #[cfg(any(test, feature = "bench-internals"))] pub(crate) fn start_compact_char(self: &Arc, callback: F) -> Result<(), String> where F: Fn(CompactEvictionBatch) -> (usize, usize) + Send + Sync + 'static, @@ -1768,6 +1770,7 @@ impl EvictionCoordinator { }); } + #[cfg(any(test, feature = "bench-internals"))] fn eviction_loop_compact(weak: Weak, callback: Arc) where F: Fn(CompactEvictionBatch) -> (usize, usize) + Send + Sync, @@ -1777,6 +1780,7 @@ impl EvictionCoordinator { }); } + #[cfg(any(test, feature = "bench-internals"))] fn eviction_loop_compact_char(weak: Weak, callback: Arc) where F: Fn(CompactEvictionBatch) -> (usize, usize) + Send + Sync, @@ -1880,6 +1884,7 @@ impl EvictionCoordinator { callback(entries) } + #[cfg(any(test, feature = "bench-internals"))] fn perform_eviction_compact(&self, callback: &F, request: &EvictionRequest) -> (usize, usize) where F: Fn(CompactEvictionBatch) -> (usize, usize), @@ -1923,6 +1928,7 @@ impl EvictionCoordinator { callback(entries) } + #[cfg(any(test, feature = "bench-internals"))] fn perform_eviction_compact_char( &self, callback: &F, diff --git a/src/persistent_artrie/core/eviction/disk_registry.rs b/src/persistent_artrie/core/eviction/disk_registry.rs index 17c6858d..973c7ba9 100644 --- a/src/persistent_artrie/core/eviction/disk_registry.rs +++ b/src/persistent_artrie/core/eviction/disk_registry.rs @@ -5388,6 +5388,7 @@ impl DiskLocationRegistry { } #[inline] + #[cfg(any(test, feature = "bench-internals"))] pub(super) fn is_authoritative(&self) -> bool { self.authority == RegistryAuthority::Valid } @@ -5530,6 +5531,7 @@ impl DiskLocationRegistry { .collect() } + #[cfg(any(test, feature = "bench-internals"))] pub(crate) fn select_compact_for_eviction( &self, target_bytes: usize, @@ -5616,6 +5618,7 @@ impl DiskLocationRegistry { } } + #[cfg(any(test, feature = "bench-internals"))] pub(crate) fn select_compact_char_for_eviction( &self, target_bytes: usize, diff --git a/src/persistent_artrie/mod.rs b/src/persistent_artrie/mod.rs index c1125d38..65e36462 100644 --- a/src/persistent_artrie/mod.rs +++ b/src/persistent_artrie/mod.rs @@ -233,6 +233,10 @@ pub mod suffix_tree; // Sequence-keyed u64 persistent ARTrie with native-key overlay and CX snapshots. pub mod u64; +// Validated logical ULEB128 boundary over the persistent byte ART. +pub mod uleb128; +// Validated logical UTF-8 boundary over the persistent byte ART. +pub mod utf8; // IoUringDiskManager-specific constructors (Phase-5 split out of dict_impl). #[cfg(feature = "io-uring-backend")] @@ -452,16 +456,17 @@ pub use zipper::PersistentARTrieZipper; pub use suffix_automaton::{ PersistentSuffixAutomaton, PersistentSuffixAutomatonChar, PersistentSuffixAutomatonCharNode, - PersistentSuffixAutomatonNode, + PersistentSuffixAutomatonNode, PersistentSuffixAutomatonUtf8, }; pub use scdawg::{ PersistentScdawg, PersistentScdawgChar, PersistentScdawgCharNode, PersistentScdawgNode, + PersistentScdawgUtf8, }; pub use suffix_tree::{ PersistentSuffixTree, PersistentSuffixTreeChar, PersistentSuffixTreeCharNode, - PersistentSuffixTreeNode, + PersistentSuffixTreeNode, PersistentSuffixTreeUtf8, }; pub use u64::{ @@ -469,6 +474,8 @@ pub use u64::{ PersistentARTrieU64Node, PersistentARTrieU64Prefix3Compat, PersistentARTrieU64Prefix3CompatNode, U64_CX_PREFIX_COMPACT, U64_CX_PREFIX_COMPAT, }; +pub use uleb128::PersistentARTrieUleb128; +pub use utf8::PersistentARTrieUtf8; pub use block_storage::{AlignedBlock, BlockStorage}; pub use buffer_manager::{BufferManager, BufferPoolStats, PageReadGuard, PageWriteGuard}; diff --git a/src/persistent_artrie/mutation_api.rs b/src/persistent_artrie/mutation_api.rs index 9036b192..3863ab37 100644 --- a/src/persistent_artrie/mutation_api.rs +++ b/src/persistent_artrie/mutation_api.rs @@ -26,6 +26,19 @@ use super::dict_impl::PersistentARTrie; use super::error::Result; impl PersistentARTrie { + /// Fallibly insert an arbitrary byte key without UTF-8 coercion. + pub fn try_insert_bytes(&self, term: &[u8]) -> Result { + self.insert_cas_durable(term) + } + + /// Insert an arbitrary byte key without UTF-8 coercion. + pub fn insert_bytes(&self, term: &[u8]) -> bool { + self.try_insert_bytes(term).unwrap_or_else(|error| { + warn!("insert byte-key overlay route failed: {:?}", error); + false + }) + } + /// Fallibly insert a term, preserving the backend error. pub fn try_insert(&self, term: &str) -> Result { self.insert_cas_durable(term.as_bytes()) @@ -85,6 +98,15 @@ impl PersistentARTrie { }) } + /// Insert or update an arbitrary byte key with a mapped value. + pub fn insert_with_value_bytes(&self, term: &[u8], value: V) -> bool { + self.try_insert_with_value_bytes(term, value) + .unwrap_or_else(|error| { + warn!("insert byte-key value route failed: {:?}", error); + false + }) + } + /// Insert multiple terms in a single batch operation. /// /// This method is optimized for bulk insertions by: @@ -221,6 +243,19 @@ impl PersistentARTrie { }) } + /// Fallibly remove an arbitrary byte key without UTF-8 coercion. + pub fn try_remove_bytes(&self, term: &[u8]) -> Result { + self.remove_cas_durable(term) + } + + /// Remove an arbitrary byte key without UTF-8 coercion. + pub fn remove_bytes(&self, term: &[u8]) -> bool { + self.try_remove_bytes(term).unwrap_or_else(|error| { + warn!("remove byte-key overlay route failed: {:?}", error); + false + }) + } + /// Remove all terms with the given prefix (batched for memory efficiency). /// /// Returns the number of terms removed. Each removal is logged to WAL diff --git a/src/persistent_artrie/scdawg.rs b/src/persistent_artrie/scdawg.rs index 4d9ce9e4..c32bf896 100644 --- a/src/persistent_artrie/scdawg.rs +++ b/src/persistent_artrie/scdawg.rs @@ -968,6 +968,9 @@ pub struct PersistentScdawgChar, } +/// UTF-8-profile spelling for the persistent Unicode-scalar SCDAWG. +pub type PersistentScdawgUtf8 = PersistentScdawgChar; + /// Snapshot iterator over stored byte-SCDAWG term records. pub struct PersistentScdawgEntryIterator { graph: Arc>, @@ -1072,6 +1075,17 @@ fn node( } impl PersistentScdawg { + /// Canonical metadata for a selected byte-oriented logical profile. + pub const fn profile_descriptor>( + ) -> crate::factory::BackendProfileDescriptor { + crate::factory::BackendProfileDescriptor::from_profile::

() + } + + /// Persistent topology family used by this adapter. + pub const fn dictionary_family() -> crate::factory::DictionaryFamily { + crate::factory::DictionaryFamily::PersistentArTrie + } + pub fn new() -> Self { Self { index: NativeScdawgIndex::new_in_memory(), @@ -1292,6 +1306,17 @@ impl PersistentScdawg { } impl PersistentScdawgChar { + /// Canonical metadata for a selected character-oriented logical profile. + pub const fn profile_descriptor>( + ) -> crate::factory::BackendProfileDescriptor { + crate::factory::BackendProfileDescriptor::from_profile::

() + } + + /// Persistent topology family used by this adapter. + pub const fn dictionary_family() -> crate::factory::DictionaryFamily { + crate::factory::DictionaryFamily::PersistentArTrie + } + pub fn new() -> Self { Self { index: NativeScdawgIndex::new_in_memory(), @@ -1328,6 +1353,34 @@ impl PersistentScdawgChar { } dict } + + /// Build from Unicode-scalar profile sequences without exposing UTF-8 + /// encoding bytes as logical suffix transitions. + pub fn from_atom_sequences(sequences: I) -> Self + where + P: crate::AtomProfile, + I: IntoIterator>, + { + Self::from_terms( + sequences + .into_iter() + .map(|sequence| sequence.as_atoms().iter().copied().collect::()), + ) + } + + /// Build a value-bearing SCDAWG from Unicode-scalar profile sequences. + pub fn from_atom_sequences_with_values(entries: I) -> Self + where + P: crate::AtomProfile, + I: IntoIterator, V)>, + { + let dict = Self::new(); + for (sequence, value) in entries { + let text: String = sequence.as_atoms().iter().copied().collect(); + dict.insert_with_value(&text, value); + } + dict + } } impl PersistentScdawgChar { @@ -1374,6 +1427,15 @@ impl PersistentScdawgChar { } impl PersistentScdawgChar { + /// Read a mapped value for a Unicode-scalar profile sequence. + pub fn get_atom_sequence_value

(&self, sequence: &crate::AtomSequence

) -> Option + where + P: crate::AtomProfile, + { + let text: String = sequence.as_atoms().iter().collect(); + ::get_value(self, &text) + } + pub fn try_insert(&self, term: &str) -> Result { self.index.insert(term, None) } diff --git a/src/persistent_artrie/suffix_automaton.rs b/src/persistent_artrie/suffix_automaton.rs index cf13fa6e..f345b140 100644 --- a/src/persistent_artrie/suffix_automaton.rs +++ b/src/persistent_artrie/suffix_automaton.rs @@ -1341,6 +1341,10 @@ pub struct PersistentSuffixAutomatonChar, } +/// UTF-8-profile spelling for the persistent Unicode-scalar suffix automaton. +pub type PersistentSuffixAutomatonUtf8 = + PersistentSuffixAutomatonChar; + /// Snapshot iterator over stored byte-suffix source records. pub struct PersistentSuffixAutomatonEntryIterator { graph: Arc>, @@ -1420,6 +1424,17 @@ impl fmt::Debug for PersistentSuffixAutomatonCharNode { } impl PersistentSuffixAutomaton { + /// Canonical metadata for a selected byte-oriented logical profile. + pub const fn profile_descriptor>( + ) -> crate::factory::BackendProfileDescriptor { + crate::factory::BackendProfileDescriptor::from_profile::

() + } + + /// Persistent topology family used by this adapter. + pub const fn dictionary_family() -> crate::factory::DictionaryFamily { + crate::factory::DictionaryFamily::PersistentArTrie + } + pub fn new() -> Self { Self { index: NativeSuffixIndex::new_in_memory(), @@ -1600,6 +1615,17 @@ impl PersistentSuffixAutomaton { } impl PersistentSuffixAutomatonChar { + /// Canonical metadata for a selected character-oriented logical profile. + pub const fn profile_descriptor>( + ) -> crate::factory::BackendProfileDescriptor { + crate::factory::BackendProfileDescriptor::from_profile::

() + } + + /// Persistent topology family used by this adapter. + pub const fn dictionary_family() -> crate::factory::DictionaryFamily { + crate::factory::DictionaryFamily::PersistentArTrie + } + pub fn new() -> Self { Self { index: NativeSuffixIndex::new_in_memory(), @@ -1624,6 +1650,34 @@ impl PersistentSuffixAutomatonChar { } dict } + + /// Build from Unicode-scalar profile sequences while preserving logical + /// scalar boundaries before insertion into the persistent suffix index. + pub fn from_atom_sequences(sequences: I) -> Self + where + P: crate::AtomProfile, + I: IntoIterator>, + { + Self::from_texts( + sequences + .into_iter() + .map(|sequence| sequence.as_atoms().iter().copied().collect::()), + ) + } + + /// Build a value-bearing automaton from Unicode-scalar profile sequences. + pub fn from_atom_sequences_with_values(entries: I) -> Self + where + P: crate::AtomProfile, + I: IntoIterator, V)>, + { + let dict = Self::new(); + for (sequence, value) in entries { + let text: String = sequence.as_atoms().iter().copied().collect(); + dict.insert_with_value(&text, value); + } + dict + } } impl PersistentSuffixAutomatonChar { @@ -1682,6 +1736,15 @@ impl PersistentSuffixAutomatonChar { self.index.insert(text, Some(value)) } + /// Read a mapped value for a Unicode-scalar profile sequence. + pub fn get_atom_sequence_value

(&self, sequence: &crate::AtomSequence

) -> Option + where + P: crate::AtomProfile, + { + let text: String = sequence.as_atoms().iter().collect(); + self.index.load().get_value(&text) + } + pub fn insert(&self, text: &str) -> bool { self.try_insert(text).unwrap_or_else(|error| { log::warn!("PersistentSuffixAutomatonChar::insert failed: {error}"); @@ -2374,6 +2437,16 @@ mod tests { use super::*; + #[test] + fn unicode_profile_value_lookup_uses_logical_sequence() { + let dictionary = PersistentSuffixAutomatonChar::::from_atom_sequences_with_values::< + crate::UnicodeScalar, + _, + >([(crate::AtomSequence::from_atoms(['Ξ»', 'x']), 42)]); + let sequence = crate::AtomSequence::::from_atoms(['Ξ»', 'x']); + assert_eq!(dictionary.get_atom_sequence_value(&sequence), Some(42)); + } + fn assert_direct_cursor_matches_owned(root: N) where N: MappedDictionaryNode diff --git a/src/persistent_artrie/suffix_tree.rs b/src/persistent_artrie/suffix_tree.rs index c46d1c33..c2ff2aea 100644 --- a/src/persistent_artrie/suffix_tree.rs +++ b/src/persistent_artrie/suffix_tree.rs @@ -1504,6 +1504,9 @@ pub struct PersistentSuffixTreeChar, } +/// UTF-8-profile spelling for the persistent Unicode-scalar suffix tree. +pub type PersistentSuffixTreeUtf8 = PersistentSuffixTreeChar; + /// Snapshot iterator over stored byte suffix-tree source records. pub struct PersistentSuffixTreeEntryIterator { graph: Arc>, @@ -1584,6 +1587,17 @@ impl fmt::Debug for PersistentSuffixTreeCharNode { } impl PersistentSuffixTree { + /// Canonical metadata for a selected byte-oriented logical profile. + pub const fn profile_descriptor>( + ) -> crate::factory::BackendProfileDescriptor { + crate::factory::BackendProfileDescriptor::from_profile::

() + } + + /// Persistent topology family used by this adapter. + pub const fn dictionary_family() -> crate::factory::DictionaryFamily { + crate::factory::DictionaryFamily::PersistentArTrie + } + /// Create an in-memory persistent suffix tree. pub fn new() -> Self { Self { @@ -1816,6 +1830,17 @@ impl PersistentSuffixTree { } impl PersistentSuffixTreeChar { + /// Canonical metadata for a selected character-oriented logical profile. + pub const fn profile_descriptor>( + ) -> crate::factory::BackendProfileDescriptor { + crate::factory::BackendProfileDescriptor::from_profile::

() + } + + /// Persistent topology family used by this adapter. + pub const fn dictionary_family() -> crate::factory::DictionaryFamily { + crate::factory::DictionaryFamily::PersistentArTrie + } + pub fn new() -> Self { Self { index: NativeSuffixTreeIndex::new_in_memory(), @@ -1840,6 +1865,34 @@ impl PersistentSuffixTreeChar { } dict } + + /// Build from Unicode-scalar profile sequences while preserving logical + /// scalar boundaries before insertion into the persistent suffix index. + pub fn from_atom_sequences(sequences: I) -> Self + where + P: crate::AtomProfile, + I: IntoIterator>, + { + Self::from_texts( + sequences + .into_iter() + .map(|sequence| sequence.as_atoms().iter().copied().collect::()), + ) + } + + /// Build a value-bearing tree from Unicode-scalar profile sequences. + pub fn from_atom_sequences_with_values(entries: I) -> Self + where + P: crate::AtomProfile, + I: IntoIterator, V)>, + { + let dict = Self::new(); + for (sequence, value) in entries { + let text: String = sequence.as_atoms().iter().copied().collect(); + dict.insert_with_value(&text, value); + } + dict + } } impl PersistentSuffixTreeChar { @@ -1886,6 +1939,15 @@ impl PersistentSuffixTreeChar { } impl PersistentSuffixTreeChar { + /// Read a mapped value for a Unicode-scalar profile sequence. + pub fn get_atom_sequence_value

(&self, sequence: &crate::AtomSequence

) -> Option + where + P: crate::AtomProfile, + { + let text: String = sequence.as_atoms().iter().collect(); + ::get_value(self, &text) + } + pub fn try_insert(&self, text: &str) -> Result { self.index.insert(text, None) } diff --git a/src/persistent_artrie/u64.rs b/src/persistent_artrie/u64.rs index 33d230f9..4154642c 100644 --- a/src/persistent_artrie/u64.rs +++ b/src/persistent_artrie/u64.rs @@ -1155,6 +1155,20 @@ impl PersistentARTrieU64 PersistentARTrieU64 { + /// Canonical logical profile used by this persistent adapter. + pub const fn profile_descriptor() -> crate::factory::BackendProfileDescriptor { + crate::factory::BackendProfileDescriptor { + kind: crate::ProfileKind::U64, + identity: crate::ProfileKind::U64.identity(), + width_bytes: crate::ProfileKind::U64.width_bytes(), + } + } + + /// Persistent topology family used by this adapter. + pub const fn dictionary_family() -> crate::factory::DictionaryFamily { + crate::factory::DictionaryFamily::PersistentArTrie + } + /// Create an in-memory persistent u64 trie. pub fn new() -> Self { Self { @@ -1181,6 +1195,19 @@ impl PersistentARTrieU trie } + /// Build from owned sequences produced by a `u64` atom profile. + pub fn from_atom_sequences(sequences: I) -> Self + where + P: crate::AtomProfile, + I: IntoIterator>, + { + Self::from_sequences( + sequences + .into_iter() + .map(|sequence| sequence.as_atoms().to_vec()), + ) + } + pub fn from_sequences_with_values(entries: I) -> Self where I: IntoIterator, @@ -1193,6 +1220,19 @@ impl PersistentARTrieU trie } + /// Build a value-bearing trie from `u64` atom-profile sequences. + pub fn from_atom_sequences_with_values(entries: I) -> Self + where + P: crate::AtomProfile, + I: IntoIterator, V)>, + { + Self::from_sequences_with_values( + entries + .into_iter() + .map(|(sequence, value)| (sequence.as_atoms().to_vec(), value)), + ) + } + pub fn from_terms(terms: I) -> Self where I: IntoIterator, @@ -1736,6 +1776,15 @@ impl PersistentARTrieU self.get_sequence_value(&sequence) } + /// Read a mapped value for a `u64` atom-profile sequence directly. + #[inline] + pub fn get_atom_sequence_value

(&self, sequence: &crate::AtomSequence

) -> Option + where + P: crate::AtomProfile, + { + self.get_sequence_value(sequence.as_atoms()) + } + pub fn remove(&self, term: &str) -> bool { let sequence = ::from_str(term); self.remove_sequence(&sequence) @@ -2039,6 +2088,7 @@ impl Default for EncodedPersistentARTrieU64 { #[cfg(test)] mod tests { use super::*; + use crate::{AtomSequence, U64}; use serde::{Deserialize, Serialize}; static ITERATOR_VALUE_CLONES: AtomicUsize = AtomicUsize::new(0); @@ -2055,6 +2105,21 @@ mod tests { impl DictionaryValue for CloneObservedValue {} + #[test] + fn profile_sequences_construct_native_u64_trie() { + let trie = PersistentARTrieU64::::from_atom_sequences::([ + AtomSequence::::from_atoms([2, 4]), + ]); + assert!(trie.contains_sequence(&[2, 4])); + let valued = PersistentARTrieU64::::from_atom_sequences_with_values::([( + AtomSequence::::from_atoms([8]), + 17, + )]); + assert_eq!(valued.get_sequence_value(&[8]), Some(17)); + let sequence = AtomSequence::::from_atoms([8]); + assert_eq!(valued.get_atom_sequence_value(&sequence), Some(17)); + } + fn disk_ptr(index: usize) -> u64 { SwizzledPtr::on_disk( 0, diff --git a/src/persistent_artrie/uleb128.rs b/src/persistent_artrie/uleb128.rs new file mode 100644 index 00000000..20536314 --- /dev/null +++ b/src/persistent_artrie/uleb128.rs @@ -0,0 +1,378 @@ +//! Logical ULEB128 boundary for the persistent byte ART. + +use crate::{Dictionary, DictionaryValue, Uleb128Error, Uleb128Sequence}; + +use super::PersistentARTrie; + +/// Persistent byte ART adapter whose public keys are complete canonical ULEB128 +/// sequences. The wrapped trie stores encoded bytes, while this boundary rejects +/// malformed images and never exposes continuation bytes as logical symbols. +#[derive(Debug)] +pub struct PersistentARTrieUleb128 { + inner: PersistentARTrie, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::Uleb128; + + #[test] + fn preserves_logical_boundaries_and_values() { + let dictionary = PersistentARTrieUleb128::::new(); + let first = Uleb128Sequence::from_atoms([Uleb128::from_u64(624_485)]); + let second = Uleb128Sequence::from_atoms([ + Uleb128::from_u64(624_485), + Uleb128::from_u64(1u64 << 63), + ]); + assert!(dictionary.insert_with_value(&first, 7)); + assert!(dictionary.insert(&second)); + assert!(dictionary.contains(&first)); + assert_eq!(dictionary.get_value(&first), Some(7)); + assert_eq!(dictionary.term_count(), 2); + assert_eq!(dictionary.try_term_count().unwrap(), 2); + assert!(!dictionary.try_is_empty().unwrap()); + let entries = dictionary.visible_entries().unwrap(); + assert_eq!(entries.len(), 2); + assert!(dictionary.contains_encoded(&first.to_encoded()).unwrap()); + assert_eq!( + dictionary.get_encoded_value(&first.to_encoded()).unwrap(), + Some(7) + ); + assert!(dictionary.contains_encoded(&[0x80]).is_err()); + assert!(dictionary.insert_encoded(&first.to_encoded(), 9).unwrap() == false); + assert_eq!(dictionary.get_value(&first), Some(9)); + assert_eq!( + dictionary.get_encoded_value(&first.to_encoded()).unwrap(), + Some(9) + ); + assert!(dictionary.get_encoded_value(&[0x80]).is_err()); + assert!(dictionary.insert_encoded(&[0x80], 1).is_err()); + assert!(dictionary.remove(&first)); + assert!(!dictionary.contains(&first)); + } + + #[test] + fn accepts_shared_profile_sequences() { + let sequence = crate::AtomSequence::::from_atoms([ + Uleb128::from_u64(624_485), + Uleb128::from_u64(1u64 << 63), + ]); + let dictionary = PersistentARTrieUleb128::::from_atom_sequences_with_values([( + sequence.clone(), + 23, + )]); + assert!(dictionary.contains_atom_sequence(&sequence)); + assert_eq!(dictionary.get_atom_sequence_value(&sequence), Some(23)); + assert!(dictionary.try_remove_atom_sequence(&sequence).unwrap()); + assert!(!dictionary.contains_atom_sequence(&sequence)); + } +} + +impl Default for PersistentARTrieUleb128 { + fn default() -> Self { + Self::new() + } +} + +impl PersistentARTrieUleb128 { + /// Canonical logical profile used by this persistent adapter. + pub const fn profile_descriptor() -> crate::factory::BackendProfileDescriptor { + crate::factory::BackendProfileDescriptor { + kind: crate::ProfileKind::Uleb128, + identity: crate::ProfileKind::Uleb128.identity(), + width_bytes: crate::ProfileKind::Uleb128.width_bytes(), + } + } + + /// Persistent topology family used by this adapter. + pub const fn dictionary_family() -> crate::factory::DictionaryFamily { + crate::factory::DictionaryFamily::PersistentArTrie + } + + /// Construct an in-memory adapter from complete ULEB sequences. + pub fn from_sequences(sequences: I) -> Self + where + I: IntoIterator, + { + let dictionary = Self::new(); + for sequence in sequences { + dictionary.insert(&sequence); + } + dictionary + } + + /// Construct from the shared logical ULEB profile sequence representation. + pub fn from_atom_sequences(sequences: I) -> Self + where + I: IntoIterator>, + { + Self::from_sequences(sequences.into_iter().map(Into::into)) + } + + /// Construct an in-memory adapter from complete sequences and values. + pub fn from_sequences_with_values(entries: I) -> Self + where + I: IntoIterator, + { + let dictionary = Self::new(); + for (sequence, value) in entries { + dictionary.insert_with_value(&sequence, value); + } + dictionary + } + + /// Construct from shared logical ULEB profile sequences and values. + pub fn from_atom_sequences_with_values(entries: I) -> Self + where + I: IntoIterator, V)>, + { + Self::from_sequences_with_values( + entries + .into_iter() + .map(|(sequence, value)| (sequence.into(), value)), + ) + } + + /// Create a fresh persistent ULEB dictionary at `path`. + pub fn create>(path: P) -> crate::persistent_artrie::Result { + Ok(Self::from_inner(PersistentARTrie::create(path)?)) + } + + /// Open an existing persistent ULEB dictionary from `path`. + pub fn open>(path: P) -> crate::persistent_artrie::Result { + Ok(Self::from_inner(PersistentARTrie::open(path)?)) + } + + /// Construct an empty in-memory adapter. + #[allow(deprecated)] + pub fn new() -> Self { + Self { + inner: PersistentARTrie::new(), + } + } + + /// Wrap an existing persistent byte ART without copying its storage. + pub fn from_inner(inner: PersistentARTrie) -> Self { + Self { inner } + } + + /// Recover the wrapped byte ART for persistence operations. + pub fn into_inner(self) -> PersistentARTrie { + self.inner + } + + /// Borrow the wrapped ART for checkpoint/recovery controls. + #[inline] + pub fn inner(&self) -> &PersistentARTrie { + &self.inner + } + + /// Number of complete logical ULEB sequences. + #[inline] + pub fn term_count(&self) -> usize { + self.inner.len().unwrap_or(0) + } + + /// Count complete logical sequences with an explicit traversal result. + /// + /// Unlike the compatibility [`term_count`](Self::term_count) accessor, + /// this method never converts an unavailable/corrupt traversal into zero. + pub fn try_term_count(&self) -> crate::persistent_artrie::Result { + Ok(self + .inner + .iter_prefix_with_arena(b"")? + .map_or(0, |entries| entries.len())) + } + + /// Whether the logical dictionary contains no complete sequences. + #[inline] + pub fn is_empty(&self) -> bool { + // A legacy bool cannot carry a storage error. Fail closed rather + // than turning an unavailable/corrupt image into apparent emptiness; + // callers requiring the distinction should use `try_is_empty`. + self.try_is_empty().unwrap_or(false) + } + + /// Checked emptiness query; storage failures remain errors. + pub fn try_is_empty(&self) -> crate::persistent_artrie::Result { + Ok(self.try_term_count()? == 0) + } + + /// Checked insertion preserving persistence failures. + pub fn try_insert(&self, sequence: &Uleb128Sequence) -> crate::persistent_artrie::Result { + self.inner.try_insert_bytes(&sequence.to_encoded()) + } + + /// Checked insertion of one shared logical ULEB profile sequence. + pub fn try_insert_atom_sequence( + &self, + sequence: &crate::AtomSequence, + ) -> crate::persistent_artrie::Result { + self.inner.try_insert_bytes(&sequence.to_encoded()) + } + + /// Insert a complete canonical sequence. + #[inline] + pub fn insert(&self, sequence: &Uleb128Sequence) -> bool { + self.inner.insert_bytes(&sequence.to_encoded()) + } + + /// Insert one shared logical ULEB profile sequence. + #[inline] + pub fn insert_atom_sequence(&self, sequence: &crate::AtomSequence) -> bool { + self.inner.insert_bytes(&sequence.to_encoded()) + } + + /// Insert a complete canonical sequence with a value. + #[inline] + pub fn insert_with_value(&self, sequence: &Uleb128Sequence, value: V) -> bool { + self.inner + .insert_with_value_bytes(&sequence.to_encoded(), value) + } + + /// Insert one shared logical ULEB profile sequence with a mapped value. + #[inline] + pub fn insert_atom_sequence_with_value( + &self, + sequence: &crate::AtomSequence, + value: V, + ) -> bool { + self.inner + .insert_with_value_bytes(&sequence.to_encoded(), value) + } + + /// Checked value insertion preserving persistence failures. + pub fn try_insert_with_value( + &self, + sequence: &Uleb128Sequence, + value: V, + ) -> crate::persistent_artrie::Result { + self.inner + .try_insert_with_value_bytes(&sequence.to_encoded(), value) + } + + /// Checked value insertion for one shared logical ULEB profile sequence. + pub fn try_insert_atom_sequence_with_value( + &self, + sequence: &crate::AtomSequence, + value: V, + ) -> crate::persistent_artrie::Result { + self.inner + .try_insert_with_value_bytes(&sequence.to_encoded(), value) + } + + /// Validate and insert one complete encoded ULEB sequence without + /// materializing arbitrary-width atoms. + pub fn insert_encoded(&self, encoded: &[u8], value: V) -> Result { + Uleb128Sequence::from_encoded(encoded)?; + Ok(self.inner.insert_with_value_bytes(encoded, value)) + } + + /// Checked encoded insertion preserving persistence failures. + pub fn try_insert_encoded( + &self, + encoded: &[u8], + value: V, + ) -> std::result::Result, Uleb128Error> { + Uleb128Sequence::from_encoded(encoded)?; + Ok(self.inner.try_insert_with_value_bytes(encoded, value)) + } + + /// Test membership of a complete sequence. + #[inline] + pub fn contains(&self, sequence: &Uleb128Sequence) -> bool { + self.inner.contains_bytes(&sequence.to_encoded()) + } + + /// Test membership of one shared logical ULEB profile sequence. + #[inline] + pub fn contains_atom_sequence( + &self, + sequence: &crate::AtomSequence, + ) -> bool { + self.inner.contains_bytes(&sequence.to_encoded()) + } + + /// Read a mapped value for a complete sequence. + #[inline] + pub fn get_value(&self, sequence: &Uleb128Sequence) -> Option { + self.inner.get_value_bytes(&sequence.to_encoded()) + } + + /// Read a mapped value for one shared logical ULEB profile sequence. + #[inline] + pub fn get_atom_sequence_value( + &self, + sequence: &crate::AtomSequence, + ) -> Option { + self.inner.get_value_bytes(&sequence.to_encoded()) + } + + /// Remove a complete sequence. + #[inline] + pub fn remove(&self, sequence: &Uleb128Sequence) -> bool { + self.inner.remove_bytes(&sequence.to_encoded()) + } + + /// Remove one shared logical ULEB profile sequence. + #[inline] + pub fn remove_atom_sequence(&self, sequence: &crate::AtomSequence) -> bool { + self.inner.remove_bytes(&sequence.to_encoded()) + } + + /// Checked removal preserving persistence failures. + pub fn try_remove(&self, sequence: &Uleb128Sequence) -> crate::persistent_artrie::Result { + self.inner.try_remove_bytes(&sequence.to_encoded()) + } + + /// Checked removal of one shared logical ULEB profile sequence. + pub fn try_remove_atom_sequence( + &self, + sequence: &crate::AtomSequence, + ) -> crate::persistent_artrie::Result { + self.inner.try_remove_bytes(&sequence.to_encoded()) + } + + /// Validate and query an already encoded sequence without decoding it. + pub fn contains_encoded(&self, encoded: &[u8]) -> Result { + Uleb128Sequence::from_encoded(encoded)?; + Ok(self.inner.contains_bytes(encoded)) + } + + /// Validate and read a mapped value for one encoded sequence without + /// allocating or decoding its arbitrary-width atoms. + pub fn get_encoded_value(&self, encoded: &[u8]) -> Result, Uleb128Error> { + Uleb128Sequence::from_encoded(encoded)?; + Ok(self.inner.get_value_bytes(encoded)) + } + + /// Validate and remove an already encoded sequence without decoding it. + pub fn remove_encoded(&self, encoded: &[u8]) -> Result { + Uleb128Sequence::from_encoded(encoded)?; + Ok(self.inner.remove_bytes(encoded)) + } + + /// Enumerate complete logical sequences and values in encoded order. + /// + /// Traversal failures and malformed persisted codewords are returned as + /// corruption errors; neither is converted into an empty result. + pub fn visible_entries( + &self, + ) -> crate::persistent_artrie::Result)>> { + let entries = self.inner.iter_prefix_with_arena(b"")?.unwrap_or_default(); + entries + .into_iter() + .map(|entry| { + let value = self.inner.get_value_bytes(&entry.term); + Uleb128Sequence::from_encoded(&entry.term) + .map(|sequence| (sequence, value)) + .map_err(|error| { + crate::persistent_artrie::PersistentARTrieError::CorruptedFile { + reason: format!("invalid ULEB128 entry: {error}"), + } + }) + }) + .collect() + } +} diff --git a/src/persistent_artrie/utf8.rs b/src/persistent_artrie/utf8.rs new file mode 100644 index 00000000..0c8201f6 --- /dev/null +++ b/src/persistent_artrie/utf8.rs @@ -0,0 +1,367 @@ +//! Logical UTF-8 boundary for the persistent byte ART. + +use crate::{Dictionary, DictionaryValue}; + +use super::PersistentARTrie; + +/// Persistent byte ART adapter whose public keys are validated UTF-8 strings. +/// The wrapped trie remains byte-backed; decoding occurs only at this boundary. +#[derive(Debug)] +pub struct PersistentARTrieUtf8 { + inner: PersistentARTrie, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn preserves_utf8_boundary_and_rejects_malformed_bytes() { + let dictionary = PersistentARTrieUtf8::::new(); + assert!(dictionary.insert_with_value("Ξ»πŸŽ‰", 9)); + assert!(dictionary.contains("Ξ»πŸŽ‰")); + assert_eq!(dictionary.get_value("Ξ»πŸŽ‰"), Some(9)); + assert!(dictionary.contains_encoded("Ξ»πŸŽ‰".as_bytes()).unwrap()); + assert_eq!( + dictionary.get_encoded_value("Ξ»πŸŽ‰".as_bytes()).unwrap(), + Some(9) + ); + assert_eq!(dictionary.try_term_count().unwrap(), 1); + assert!(!dictionary.try_is_empty().unwrap()); + assert!(dictionary.contains_encoded(&[0x80]).is_err()); + assert!(!dictionary.insert_encoded("Ξ»πŸŽ‰".as_bytes(), 10).unwrap()); + assert_eq!(dictionary.get_value("Ξ»πŸŽ‰"), Some(10)); + assert_eq!( + dictionary.get_encoded_value("Ξ»πŸŽ‰".as_bytes()).unwrap(), + Some(10) + ); + assert!(dictionary.get_encoded_value(&[0x80]).is_err()); + assert!(dictionary.insert_encoded(&[0x80], 1).is_err()); + assert_eq!(dictionary.visible_entries().unwrap().len(), 1); + assert!(dictionary.remove_encoded("Ξ»πŸŽ‰".as_bytes()).unwrap()); + assert!(dictionary.is_empty()); + assert!(dictionary.try_is_empty().unwrap()); + } + + #[test] + fn accepts_shared_utf8_profile_sequences() { + let sequence = crate::AtomSequence::::from_atoms(['Ξ»', 'πŸŽ‰']); + let dictionary = + PersistentARTrieUtf8::::from_atom_sequences_with_values([(sequence.clone(), 29)]); + assert!(dictionary.contains_atom_sequence(&sequence)); + assert_eq!(dictionary.get_atom_sequence_value(&sequence), Some(29)); + assert!(dictionary.try_remove_atom_sequence(&sequence).unwrap()); + assert!(!dictionary.contains_atom_sequence(&sequence)); + } +} + +impl Default for PersistentARTrieUtf8 { + fn default() -> Self { + Self::new() + } +} + +impl PersistentARTrieUtf8 { + /// Canonical logical profile used by this persistent adapter. + pub const fn profile_descriptor() -> crate::factory::BackendProfileDescriptor { + crate::factory::BackendProfileDescriptor { + kind: crate::ProfileKind::Utf8, + identity: crate::ProfileKind::Utf8.identity(), + width_bytes: crate::ProfileKind::Utf8.width_bytes(), + } + } + + /// Persistent topology family used by this adapter. + pub const fn dictionary_family() -> crate::factory::DictionaryFamily { + crate::factory::DictionaryFamily::PersistentArTrie + } + + /// Construct an in-memory adapter from UTF-8 terms. + pub fn from_terms(terms: I) -> Self + where + I: IntoIterator, + T: AsRef, + { + let dictionary = Self::new(); + for term in terms { + dictionary.insert(term.as_ref()); + } + dictionary + } + + /// Construct an in-memory adapter from UTF-8 terms and values. + pub fn from_terms_with_values(entries: I) -> Self + where + I: IntoIterator, + T: AsRef, + { + let dictionary = Self::new(); + for (term, value) in entries { + dictionary.insert_with_value(term.as_ref(), value); + } + dictionary + } + + /// Construct from shared logical UTF-8 scalar profile sequences. + pub fn from_atom_sequences(sequences: I) -> Self + where + I: IntoIterator>, + { + Self::from_terms( + sequences + .into_iter() + .map(|sequence| sequence.as_atoms().iter().copied().collect::()), + ) + } + + /// Construct from shared logical UTF-8 scalar profile sequences and + /// mapped values. + pub fn from_atom_sequences_with_values(entries: I) -> Self + where + I: IntoIterator, V)>, + { + Self::from_terms_with_values(entries.into_iter().map(|(sequence, value)| { + ( + sequence.as_atoms().iter().copied().collect::(), + value, + ) + })) + } + + /// Create a fresh persistent UTF-8 dictionary at `path`. + pub fn create>(path: P) -> crate::persistent_artrie::Result { + Ok(Self::from_inner(PersistentARTrie::create(path)?)) + } + + /// Open an existing persistent UTF-8 dictionary from `path`. + pub fn open>(path: P) -> crate::persistent_artrie::Result { + Ok(Self::from_inner(PersistentARTrie::open(path)?)) + } + + /// Construct an empty in-memory adapter. + #[allow(deprecated)] + pub fn new() -> Self { + Self { + inner: PersistentARTrie::new(), + } + } + + /// Wrap an existing persistent byte ART without copying its storage. + pub fn from_inner(inner: PersistentARTrie) -> Self { + Self { inner } + } + + /// Recover the wrapped ART for persistence operations. + pub fn into_inner(self) -> PersistentARTrie { + self.inner + } + + /// Borrow the wrapped ART for checkpoint/recovery controls. + #[inline] + pub fn inner(&self) -> &PersistentARTrie { + &self.inner + } + + /// Number of complete UTF-8 terms. + #[inline] + pub fn term_count(&self) -> usize { + self.inner.len().unwrap_or(0) + } + + /// Count complete UTF-8 terms with an explicit traversal result. + /// + /// Unlike the compatibility [`term_count`](Self::term_count) accessor, + /// this method never converts an unavailable/corrupt traversal into zero. + pub fn try_term_count(&self) -> crate::persistent_artrie::Result { + Ok(self + .inner + .iter_prefix_with_arena(b"")? + .map_or(0, |entries| entries.len())) + } + + /// Whether no complete UTF-8 terms are stored. + #[inline] + pub fn is_empty(&self) -> bool { + // A legacy bool cannot carry a storage error. Fail closed rather + // than turning an unavailable/corrupt image into apparent emptiness; + // callers requiring the distinction should use `try_is_empty`. + self.try_is_empty().unwrap_or(false) + } + + /// Checked emptiness query; storage failures remain errors. + pub fn try_is_empty(&self) -> crate::persistent_artrie::Result { + Ok(self.try_term_count()? == 0) + } + + /// Checked insertion preserving persistence failures. + pub fn try_insert(&self, term: &str) -> crate::persistent_artrie::Result { + self.inner.try_insert(term) + } + + /// Checked insertion of one shared logical UTF-8 scalar profile sequence. + pub fn try_insert_atom_sequence( + &self, + sequence: &crate::AtomSequence, + ) -> crate::persistent_artrie::Result { + self.inner.try_insert_bytes(&sequence.to_encoded()) + } + + /// Insert a term, reporting only whether it was newly added. + #[inline] + pub fn insert(&self, term: &str) -> bool { + self.inner.insert(term) + } + + /// Insert one shared logical UTF-8 scalar profile sequence. + #[inline] + pub fn insert_atom_sequence(&self, sequence: &crate::AtomSequence) -> bool { + self.inner.insert_bytes(&sequence.to_encoded()) + } + + /// Checked value insertion preserving persistence failures. + pub fn try_insert_with_value( + &self, + term: &str, + value: V, + ) -> crate::persistent_artrie::Result { + self.inner.try_insert_with_value(term, value) + } + + /// Checked value insertion for one shared logical UTF-8 scalar profile + /// sequence. + pub fn try_insert_atom_sequence_with_value( + &self, + sequence: &crate::AtomSequence, + value: V, + ) -> crate::persistent_artrie::Result { + self.inner + .try_insert_with_value_bytes(&sequence.to_encoded(), value) + } + + /// Insert or update a term with a mapped value. + #[inline] + pub fn insert_with_value(&self, term: &str, value: V) -> bool { + self.inner.insert_with_value(term, value) + } + + /// Insert one shared logical UTF-8 scalar profile sequence with a value. + #[inline] + pub fn insert_atom_sequence_with_value( + &self, + sequence: &crate::AtomSequence, + value: V, + ) -> bool { + self.inner + .insert_with_value_bytes(&sequence.to_encoded(), value) + } + + /// Validate and insert one complete UTF-8 encoded key without allocating + /// an intermediate `str`. + pub fn insert_encoded(&self, encoded: &[u8], value: V) -> Result { + std::str::from_utf8(encoded)?; + Ok(self.inner.insert_with_value_bytes(encoded, value)) + } + + /// Checked encoded insertion preserving persistence failures. + pub fn try_insert_encoded( + &self, + encoded: &[u8], + value: V, + ) -> std::result::Result, std::str::Utf8Error> { + std::str::from_utf8(encoded)?; + Ok(self.inner.try_insert_with_value_bytes(encoded, value)) + } + + /// Test membership of a UTF-8 term. + #[inline] + pub fn contains(&self, term: &str) -> bool { + self.inner.contains_bytes(term.as_bytes()) + } + + /// Test membership of one shared logical UTF-8 scalar profile sequence. + #[inline] + pub fn contains_atom_sequence(&self, sequence: &crate::AtomSequence) -> bool { + self.inner.contains_bytes(&sequence.to_encoded()) + } + + /// Read a mapped value for a UTF-8 term. + #[inline] + pub fn get_value(&self, term: &str) -> Option { + self.inner.get_value_bytes(term.as_bytes()) + } + + /// Read a mapped value for one shared logical UTF-8 scalar profile + /// sequence. + #[inline] + pub fn get_atom_sequence_value( + &self, + sequence: &crate::AtomSequence, + ) -> Option { + self.inner.get_value_bytes(&sequence.to_encoded()) + } + + /// Checked removal preserving persistence failures. + pub fn try_remove(&self, term: &str) -> crate::persistent_artrie::Result { + self.inner.try_remove_bytes(term.as_bytes()) + } + + /// Checked removal of one shared logical UTF-8 scalar profile sequence. + pub fn try_remove_atom_sequence( + &self, + sequence: &crate::AtomSequence, + ) -> crate::persistent_artrie::Result { + self.inner.try_remove_bytes(&sequence.to_encoded()) + } + + /// Remove a UTF-8 term. + #[inline] + pub fn remove(&self, term: &str) -> bool { + self.inner.remove_bytes(term.as_bytes()) + } + + /// Remove one shared logical UTF-8 scalar profile sequence. + #[inline] + pub fn remove_atom_sequence(&self, sequence: &crate::AtomSequence) -> bool { + self.inner.remove_bytes(&sequence.to_encoded()) + } + + /// Validate and query an already encoded UTF-8 term. + pub fn contains_encoded(&self, encoded: &[u8]) -> Result { + std::str::from_utf8(encoded)?; + Ok(self.inner.contains_bytes(encoded)) + } + + /// Validate and read a mapped value for one encoded UTF-8 term without + /// allocating or decoding its scalar sequence. + pub fn get_encoded_value(&self, encoded: &[u8]) -> Result, std::str::Utf8Error> { + std::str::from_utf8(encoded)?; + Ok(self.inner.get_value_bytes(encoded)) + } + + /// Validate and remove an already encoded UTF-8 term. + pub fn remove_encoded(&self, encoded: &[u8]) -> Result { + std::str::from_utf8(encoded)?; + Ok(self.inner.remove_bytes(encoded)) + } + + /// Enumerate complete UTF-8 terms and values in byte-lexicographic order. + /// + /// Traversal failures and malformed persisted bytes are returned explicitly; + /// neither is converted into an empty result. + pub fn visible_entries(&self) -> crate::persistent_artrie::Result)>> { + let entries = self.inner.iter_prefix_with_arena(b"")?.unwrap_or_default(); + entries + .into_iter() + .map(|entry| { + let value = self.inner.get_value_bytes(&entry.term); + std::str::from_utf8(&entry.term) + .map(|term| (term.to_owned(), value)) + .map_err(|error| { + crate::persistent_artrie::PersistentARTrieError::CorruptedFile { + reason: format!("invalid UTF-8 entry: {error}"), + } + }) + }) + .collect() + } +} diff --git a/src/persistent_artrie/vocab/mod.rs b/src/persistent_artrie/vocab/mod.rs index a95406ee..71bade3c 100644 --- a/src/persistent_artrie/vocab/mod.rs +++ b/src/persistent_artrie/vocab/mod.rs @@ -165,6 +165,20 @@ impl Dictionary for PersistentVocabARTrie { } impl PersistentVocabARTrie { + /// Canonical logical profile used by vocabulary terms. + pub const fn profile_descriptor() -> crate::factory::BackendProfileDescriptor { + crate::factory::BackendProfileDescriptor { + kind: crate::ProfileKind::UnicodeScalar, + identity: crate::ProfileKind::UnicodeScalar.identity(), + width_bytes: crate::ProfileKind::UnicodeScalar.width_bytes(), + } + } + + /// Persistent topology family used by this vocabulary backend. + pub const fn dictionary_family() -> crate::factory::DictionaryFamily { + crate::factory::DictionaryFamily::PersistentArTrie + } + /// Capture a traversal root and exact cardinality from one atomic revision. pub(crate) fn root_with_term_count(&self) -> (VocabTrieNodeRef, usize) { let (root, term_count) = self @@ -840,8 +854,21 @@ pub type DiskBackedVocabTrieInner = PersistentVocabARTrie; #[cfg(test)] mod tests { use super::*; + use crate::persistent_artrie::disk_manager::MmapDiskManager; + use crate::{AtomSequence, UnicodeScalar}; use tempfile::tempdir; + #[test] + fn vocabulary_profile_metadata_is_explicit() { + let descriptor = PersistentVocabARTrie::::profile_descriptor(); + assert_eq!(descriptor.kind, crate::ProfileKind::UnicodeScalar); + assert_eq!(descriptor.width_bytes, Some(4)); + assert_eq!( + PersistentVocabARTrie::::dictionary_family(), + crate::factory::DictionaryFamily::PersistentArTrie + ); + } + #[test] fn test_vocab_trie_basic() { let dir = tempdir().unwrap(); @@ -895,6 +922,24 @@ mod tests { assert_eq!(vocab.get_term(4), Some("emojiπŸ˜€".to_string())); } + #[test] + fn profile_sequence_api_preserves_unicode_scalar_boundaries() { + let dir = tempdir().unwrap(); + let path = dir.path().join("profile-sequence.vocab"); + let vocab = PersistentVocabARTrie::create(&path).unwrap(); + let sequence = AtomSequence::::from_atoms("Ξ»πŸŽ‰".chars()); + + let index = vocab.insert_atom_sequence(&sequence).unwrap(); + assert_eq!(vocab.get_atom_sequence_index(&sequence), Some(index)); + assert_eq!( + vocab + .get_term_atom_sequence::(index) + .unwrap() + .as_atoms(), + sequence.as_atoms() + ); + } + #[test] fn test_vocab_trie_custom_start() { let dir = tempdir().unwrap(); diff --git a/src/persistent_artrie/vocab/mutation_api.rs b/src/persistent_artrie/vocab/mutation_api.rs index 5d27a609..ab4c602b 100644 --- a/src/persistent_artrie/vocab/mutation_api.rs +++ b/src/persistent_artrie/vocab/mutation_api.rs @@ -13,6 +13,7 @@ use std::sync::atomic::Ordering; use crate::persistent_artrie::block_storage::BlockStorage; use crate::persistent_artrie::error::{PersistentARTrieError, Result}; +use crate::{AtomProfile, AtomSequence}; impl super::dict_impl::PersistentVocabARTrie { /// Insert a term and auto-assign the next vocabulary index. Returns the assigned index. @@ -24,6 +25,16 @@ impl super::dict_impl::PersistentVocabARTrie { self.insert_overlay(term) } + /// Insert one profile-owned Unicode-scalar sequence using the durable + /// vocabulary insertion path. + pub fn insert_atom_sequence

(&self, sequence: &AtomSequence

) -> Result + where + P: AtomProfile, + { + let term: String = sequence.as_atoms().iter().collect(); + self.insert(&term) + } + /// Lock-free Order-A overlay insert β€” the write path (`&self`, concurrent-safe). /// /// Allocates a WRITE-ONCE id (`next_index.fetch_add` β€” nearly-dense: a lost InsertOnce diff --git a/src/persistent_artrie/vocab/query_api.rs b/src/persistent_artrie/vocab/query_api.rs index ef826a1a..4c0d8ea5 100644 --- a/src/persistent_artrie/vocab/query_api.rs +++ b/src/persistent_artrie/vocab/query_api.rs @@ -9,6 +9,7 @@ use std::sync::atomic::Ordering; use crate::persistent_artrie::block_storage::BlockStorage; +use crate::{AtomProfile, AtomSequence}; impl super::dict_impl::PersistentVocabARTrie { /// Get the vocabulary index for a term (lock-free overlay lookup). @@ -16,6 +17,16 @@ impl super::dict_impl::PersistentVocabARTrie { self.get_index_lockfree(term) } + /// Resolve a profile-owned Unicode-scalar sequence without exposing a + /// string-conversion requirement at the call site. + pub fn get_atom_sequence_index

(&self, sequence: &AtomSequence

) -> Option + where + P: AtomProfile, + { + let term: String = sequence.as_atoms().iter().collect(); + self.get_index(&term) + } + /// Get the term for a vocabulary index via the in-memory reverse map (id β†’ term). /// /// The reverse map is populated on every insert and rebuilt from the image on reopen, so @@ -26,6 +37,16 @@ impl super::dict_impl::PersistentVocabARTrie { .and_then(|m| m.get(&index).map(|e| e.value().clone())) } + /// Resolve an index into an owned sequence of the caller's Unicode-scalar + /// profile. + pub fn get_term_atom_sequence

(&self, index: u64) -> Option> + where + P: AtomProfile, + { + self.get_term(index) + .map(|term| AtomSequence::from_atoms(term.chars())) + } + /// Check if a term exists in the vocabulary. #[inline] pub fn contains(&self, term: &str) -> bool { diff --git a/src/profile.rs b/src/profile.rs new file mode 100644 index 00000000..ce8b192e --- /dev/null +++ b/src/profile.rs @@ -0,0 +1,695 @@ +//! Shared logical-atom profiles for generic dictionary families. + +use crate::variable_width::{Uleb128, Uleb128Ref, VariableWidthProfile, ULEB128_PROFILE}; +use core::cmp::Ordering; +use core::marker::PhantomData; + +/// Errors returned by logical-atom profile decoders. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ProfileError { + /// The input does not contain one complete atom. + InvalidLength, + /// A scalar profile received a value outside the Unicode scalar range. + InvalidScalar, + /// The input is not one complete valid UTF-8 scalar encoding. + InvalidUtf8, + /// The input is not one complete canonical variable-width atom. + InvalidEncoding, +} + +/// Stable descriptor for the built-in logical alphabets. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] +pub enum ProfileKind { + /// Raw bytes. + Bytes, + /// Unicode scalar values (not UTF-8 byte transitions). + UnicodeScalar, + /// UTF-8 encoded Unicode scalar values (one variable-width codeword per scalar). + Utf8, + /// Native 32-bit unsigned values. + U32, + /// Native 64-bit unsigned values. + U64, + /// IEEE-754 binary64 represented by raw `u64` bits. + F64Bits, + /// Arbitrary-width canonical ULEB128 atoms. + Uleb128, +} + +impl ProfileKind { + /// Canonical persisted name. + pub const fn as_str(self) -> &'static str { + match self { + Self::Bytes => "bytes", + Self::UnicodeScalar => "unicode-scalar", + Self::Utf8 => "utf8", + Self::U32 => "u32", + Self::U64 => "u64", + Self::F64Bits => "f64-bits", + Self::Uleb128 => "uleb128", + } + } + + /// Parse a canonical persisted name. + pub fn from_name(name: &str) -> Option { + match name { + "bytes" => Some(Self::Bytes), + "unicode-scalar" => Some(Self::UnicodeScalar), + "utf8" => Some(Self::Utf8), + "u32" => Some(Self::U32), + "u64" => Some(Self::U64), + "f64-bits" => Some(Self::F64Bits), + "uleb128" => Some(Self::Uleb128), + _ => None, + } + } + + /// Legacy public type name, when one exists. These names are aliases for + /// source compatibility only and are not accepted as persisted metadata. + pub const fn legacy_name(self) -> Option<&'static str> { + match self { + Self::Bytes => Some("DynamicDawg"), + Self::UnicodeScalar => Some("DynamicDawgChar"), + Self::Utf8 => None, + Self::U64 => Some("DynamicDawgU64"), + Self::U32 | Self::F64Bits | Self::Uleb128 => None, + } + } + + /// Resolve a profile only when both name and version match exactly. + pub fn from_identity(identity: VariableWidthProfile) -> Option { + let kind = Self::from_name(identity.name)?; + (kind.identity() == identity).then_some(kind) + } + + /// Stable persisted identity. + pub const fn identity(self) -> VariableWidthProfile { + match self { + Self::Bytes => Bytes::PROFILE, + Self::UnicodeScalar => UnicodeScalar::PROFILE, + Self::Utf8 => Utf8::PROFILE, + Self::U32 => U32::PROFILE, + Self::U64 => U64::PROFILE, + Self::F64Bits => F64Bits::PROFILE, + Self::Uleb128 => ULEB128_PROFILE, + } + } + + /// Fixed wire width, or `None` for variable-width ULEB atoms. + pub const fn width_bytes(self) -> Option { + match self { + Self::Uleb128 => None, + Self::Utf8 => None, + Self::Bytes => Bytes::WIDTH_BYTES, + Self::UnicodeScalar => UnicodeScalar::WIDTH_BYTES, + Self::U32 => U32::WIDTH_BYTES, + Self::U64 | Self::F64Bits => Some(8), + } + } +} + +/// Codec contract for one logical dictionary edge. +pub trait AtomProfile { + /// Logical value represented by one edge. + type Atom: Clone + Eq + Ord; + + /// Stable persisted profile identity. + const PROFILE: VariableWidthProfile; + /// Built-in kind corresponding to the profile identity. + const KIND: ProfileKind; + /// Fixed wire width in bytes, or `None` for variable-width profiles. + const WIDTH_BYTES: Option; + + /// Encode one logical atom. + fn encode(atom: Self::Atom) -> Vec; + /// Decode one atom from the beginning of `bytes`, returning the atom and + /// the number of bytes consumed. + fn decode(bytes: &[u8]) -> Result<(Self::Atom, usize), ProfileError>; +} + +/// Owned logical sequence parameterized by an [`AtomProfile`]. +#[derive(Clone, Debug)] +pub struct AtomSequence { + atoms: Vec, + marker: PhantomData

, +} + +/// Fail-closed iterator over logical atoms in an encoded profile stream. +pub struct AtomStream<'a, P: AtomProfile> { + remaining: &'a [u8], + failed: bool, + marker: PhantomData

, +} + +impl PartialEq for AtomSequence

{ + fn eq(&self, other: &Self) -> bool { + self.atoms == other.atoms + } +} + +impl Eq for AtomSequence

{} + +impl Default for AtomSequence

{ + fn default() -> Self { + Self { + atoms: Vec::new(), + marker: PhantomData, + } + } +} + +impl AtomSequence

{ + /// Construct an empty sequence. + #[inline] + pub fn new() -> Self { + Self::default() + } + + /// Stable profile identity for this sequence's wire representation. + #[inline] + pub const fn profile() -> VariableWidthProfile { + P::PROFILE + } + + /// Built-in profile kind corresponding to this sequence. + #[inline] + pub const fn profile_kind() -> ProfileKind { + P::KIND + } + + /// Wire width of one atom, or `None` for variable-width profiles. + #[inline] + pub const fn width_bytes() -> Option { + P::WIDTH_BYTES + } + + /// Build a sequence from logical atoms. + pub fn from_atoms(atoms: I) -> Self + where + I: IntoIterator, + { + Self { + atoms: atoms.into_iter().collect(), + marker: PhantomData, + } + } + + /// Decode a complete concatenated wire image. + pub fn from_encoded(bytes: &[u8]) -> Result { + let mut atoms = Vec::new(); + let mut offset = 0; + while offset < bytes.len() { + let (atom, consumed) = P::decode(&bytes[offset..])?; + if consumed == 0 || consumed > bytes.len() - offset { + return Err(ProfileError::InvalidLength); + } + atoms.push(atom); + offset += consumed; + } + Ok(Self::from_atoms(atoms)) + } + + /// Iterate logical atoms directly from an immutable encoded image. + #[inline] + pub fn stream(bytes: &[u8]) -> AtomStream<'_, P> { + AtomStream { + remaining: bytes, + failed: false, + marker: PhantomData, + } + } + + /// Append one logical atom. + #[inline] + pub fn push(&mut self, atom: P::Atom) { + self.atoms.push(atom); + } + + /// Number of logical atoms. + #[inline] + pub fn len(&self) -> usize { + self.atoms.len() + } + + /// Whether the sequence contains no logical atoms. + #[inline] + pub fn is_empty(&self) -> bool { + self.atoms.is_empty() + } + + /// Iterate over logical atoms without decoding. + #[inline] + pub fn iter(&self) -> impl Iterator { + self.atoms.iter() + } + + /// Borrow the logical atom slice for dictionary kernels. + #[inline] + pub fn as_atoms(&self) -> &[P::Atom] { + &self.atoms + } + + /// Consume the sequence and return its owned logical atoms. + #[inline] + pub fn into_atoms(self) -> Vec { + self.atoms + } + + /// Number of bytes in the encoded sequence. + pub fn encoded_len(&self) -> usize { + self.atoms + .iter() + .map(|atom| P::encode(atom.clone()).len()) + .sum() + } + + /// Encode the sequence in logical order. + pub fn to_encoded(&self) -> Vec { + let mut encoded = Vec::with_capacity(self.encoded_len()); + for atom in &self.atoms { + encoded.extend_from_slice(&P::encode(atom.clone())); + } + encoded + } +} + +impl<'a, P: AtomProfile> Iterator for AtomStream<'a, P> { + type Item = Result; + + fn next(&mut self) -> Option { + if self.failed || self.remaining.is_empty() { + return None; + } + match P::decode(self.remaining) { + Ok((atom, consumed)) if consumed > 0 && consumed <= self.remaining.len() => { + self.remaining = &self.remaining[consumed..]; + Some(Ok(atom)) + } + Ok(_) => { + self.failed = true; + Some(Err(ProfileError::InvalidLength)) + } + Err(error) => { + self.failed = true; + Some(Err(error)) + } + } + } +} + +/// Raw byte profile (`DynamicDawg` compatibility semantics). +#[derive(Clone, Copy, Debug, Default)] +pub struct Bytes; + +impl AtomProfile for Bytes { + type Atom = u8; + const PROFILE: VariableWidthProfile = VariableWidthProfile::new("bytes", 1); + const KIND: ProfileKind = ProfileKind::Bytes; + const WIDTH_BYTES: Option = Some(1); + + fn encode(atom: u8) -> Vec { + vec![atom] + } + + fn decode(bytes: &[u8]) -> Result<(u8, usize), ProfileError> { + bytes + .first() + .copied() + .map(|byte| (byte, 1)) + .ok_or(ProfileError::InvalidLength) + } +} + +/// Unicode scalar profile (`DynamicDawgChar` compatibility semantics). +#[derive(Clone, Copy, Debug, Default)] +pub struct UnicodeScalar; + +/// Variable-width UTF-8 scalar profile. Each logical atom is one Unicode +/// scalar and its canonical UTF-8 codeword; continuation bytes are never +/// semantic transitions. +#[derive(Clone, Copy, Debug, Default)] +pub struct Utf8; + +impl AtomProfile for Utf8 { + type Atom = char; + const PROFILE: VariableWidthProfile = VariableWidthProfile::new("utf8", 1); + const KIND: ProfileKind = ProfileKind::Utf8; + const WIDTH_BYTES: Option = None; + + fn encode(atom: char) -> Vec { + let mut bytes = [0u8; 4]; + atom.encode_utf8(&mut bytes).as_bytes().to_vec() + } + + fn decode(bytes: &[u8]) -> Result<(char, usize), ProfileError> { + let first = *bytes.first().ok_or(ProfileError::InvalidLength)?; + let width = match first { + 0x00..=0x7f => 1, + 0xc2..=0xdf => 2, + 0xe0..=0xef => 3, + 0xf0..=0xf4 => 4, + _ => return Err(ProfileError::InvalidUtf8), + }; + let slice = bytes.get(..width).ok_or(ProfileError::InvalidLength)?; + let text = core::str::from_utf8(slice).map_err(|_| ProfileError::InvalidUtf8)?; + let mut chars = text.chars(); + let atom = chars.next().ok_or(ProfileError::InvalidUtf8)?; + if chars.next().is_some() { + return Err(ProfileError::InvalidUtf8); + } + Ok((atom, width)) + } +} + +impl AtomProfile for UnicodeScalar { + type Atom = char; + const PROFILE: VariableWidthProfile = VariableWidthProfile::new("unicode-scalar", 1); + const KIND: ProfileKind = ProfileKind::UnicodeScalar; + const WIDTH_BYTES: Option = Some(4); + + fn encode(atom: char) -> Vec { + (atom as u32).to_le_bytes().to_vec() + } + + fn decode(bytes: &[u8]) -> Result<(char, usize), ProfileError> { + let bytes: [u8; 4] = bytes + .get(..4) + .ok_or(ProfileError::InvalidLength)? + .try_into() + .map_err(|_| ProfileError::InvalidLength)?; + char::from_u32(u32::from_le_bytes(bytes)) + .map(|scalar| (scalar, 4)) + .ok_or(ProfileError::InvalidScalar) + } +} + +/// Native little-endian 32-bit unsigned profile. +#[derive(Clone, Copy, Debug, Default)] +pub struct U32; + +impl AtomProfile for U32 { + type Atom = u32; + const PROFILE: VariableWidthProfile = VariableWidthProfile::new("u32", 1); + const KIND: ProfileKind = ProfileKind::U32; + const WIDTH_BYTES: Option = Some(4); + + fn encode(atom: u32) -> Vec { + atom.to_le_bytes().to_vec() + } + + fn decode(bytes: &[u8]) -> Result<(u32, usize), ProfileError> { + let bytes: [u8; 4] = bytes + .get(..4) + .ok_or(ProfileError::InvalidLength)? + .try_into() + .map_err(|_| ProfileError::InvalidLength)?; + Ok((u32::from_le_bytes(bytes), 4)) + } +} + +/// Native little-endian 64-bit unsigned profile. +#[derive(Clone, Copy, Debug, Default)] +pub struct U64; + +impl AtomProfile for U64 { + type Atom = u64; + const PROFILE: VariableWidthProfile = VariableWidthProfile::new("u64", 1); + const KIND: ProfileKind = ProfileKind::U64; + const WIDTH_BYTES: Option = Some(8); + + fn encode(atom: u64) -> Vec { + atom.to_le_bytes().to_vec() + } + + fn decode(bytes: &[u8]) -> Result<(u64, usize), ProfileError> { + let bytes: [u8; 8] = bytes + .get(..8) + .ok_or(ProfileError::InvalidLength)? + .try_into() + .map_err(|_| ProfileError::InvalidLength)?; + Ok((u64::from_le_bytes(bytes), 8)) + } +} + +/// Raw-bit IEEE-754 binary64 profile; the logical atom is its `u64` bit +/// pattern so equality and hashing preserve every payload. Use +/// [`F64Bits::total_cmp`] when ordering values by IEEE-754 total order. +#[derive(Clone, Copy, Debug, Default)] +pub struct F64Bits; + +impl F64Bits { + /// Encode an IEEE-754 value while preserving its exact bit pattern. + #[inline] + pub fn encode_f64(value: f64) -> Vec { + Self::encode(value.to_bits()) + } + + /// Decode one exact IEEE-754 value from a fixed-width atom. + #[inline] + pub fn decode_f64(bytes: &[u8]) -> Result<(f64, usize), ProfileError> { + Self::decode(bytes).map(|(bits, consumed)| (f64::from_bits(bits), consumed)) + } + + /// Compare raw bit patterns using Rust's total IEEE-754 ordering. + /// + /// Equality and hashing remain bitwise because callers retain the raw + /// bits; distinct NaN payloads and signed zero are never collapsed. + #[inline] + pub fn total_cmp(left: u64, right: u64) -> Ordering { + f64::from_bits(left).total_cmp(&f64::from_bits(right)) + } +} + +impl AtomProfile for F64Bits { + type Atom = u64; + const PROFILE: VariableWidthProfile = VariableWidthProfile::new("f64-bits", 1); + const KIND: ProfileKind = ProfileKind::F64Bits; + const WIDTH_BYTES: Option = Some(8); + + fn encode(atom: u64) -> Vec { + atom.to_le_bytes().to_vec() + } + + fn decode(bytes: &[u8]) -> Result<(u64, usize), ProfileError> { + let bytes: [u8; 8] = bytes + .get(..8) + .ok_or(ProfileError::InvalidLength)? + .try_into() + .map_err(|_| ProfileError::InvalidLength)?; + Ok((u64::from_le_bytes(bytes), 8)) + } +} + +/// Canonical arbitrary-width ULEB128 atom profile. +/// +/// The owned atom retains its canonical bytes and is therefore usable for +/// values wider than any built-in integer. Decoding reports the first +/// complete codeword and leaves following codewords to the sequence walker. +#[derive(Clone, Copy, Debug, Default)] +pub struct Uleb128Atom; + +impl AtomProfile for Uleb128Atom { + type Atom = Uleb128; + const PROFILE: VariableWidthProfile = ULEB128_PROFILE; + const KIND: ProfileKind = ProfileKind::Uleb128; + const WIDTH_BYTES: Option = None; + + fn encode(atom: Uleb128) -> Vec { + atom.as_bytes().to_vec() + } + + fn decode(bytes: &[u8]) -> Result<(Uleb128, usize), ProfileError> { + let (view, consumed) = Uleb128Ref::from_prefix(bytes).map_err(|error| match error { + crate::variable_width::Uleb128Error::Empty + | crate::variable_width::Uleb128Error::Unterminated => ProfileError::InvalidLength, + crate::variable_width::Uleb128Error::NonCanonical + | crate::variable_width::Uleb128Error::InvalidPayload => ProfileError::InvalidEncoding, + })?; + Ok((view.to_owned(), consumed)) + } +} + +/// Convert the generic ULEB profile sequence into the established dictionary +/// sequence representation without re-encoding or decoding any atom. +impl From for AtomSequence { + fn from(sequence: crate::Uleb128Sequence) -> Self { + Self::from_atoms(sequence.into_atoms()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fixed_profiles_round_trip() { + assert_eq!(Bytes::decode(&[7, 8]).unwrap(), (7, 1)); + assert_eq!( + UnicodeScalar::decode(&('Ξ»' as u32).to_le_bytes()).unwrap(), + ('Ξ»', 4) + ); + assert_eq!( + U32::decode(&0xdead_beefu32.to_le_bytes()).unwrap(), + (0xdead_beef, 4) + ); + assert_eq!(U64::decode(&u64::MAX.to_le_bytes()).unwrap(), (u64::MAX, 8)); + let nan_bits = 0x7ff8_0000_0000_0042u64; + assert_eq!( + F64Bits::decode(&F64Bits::encode(nan_bits)).unwrap().0, + nan_bits + ); + let negative_zero = F64Bits::encode_f64(-0.0); + let positive_zero = F64Bits::encode_f64(0.0); + assert_ne!(negative_zero, positive_zero); + assert_eq!( + F64Bits::decode_f64(&negative_zero).unwrap().0.to_bits(), + (-0.0f64).to_bits() + ); + assert_eq!( + F64Bits::decode_f64(&positive_zero).unwrap().0.to_bits(), + 0.0f64.to_bits() + ); + assert!(F64Bits::total_cmp((-0.0f64).to_bits(), 0.0f64.to_bits()).is_lt()); + assert_eq!(F64Bits::total_cmp(nan_bits, nan_bits), Ordering::Equal); + } + + #[test] + fn scalar_profile_rejects_surrogates_and_short_input() { + assert_eq!( + UnicodeScalar::decode(&[0; 3]), + Err(ProfileError::InvalidLength) + ); + assert_eq!( + UnicodeScalar::decode(&0xd800u32.to_le_bytes()), + Err(ProfileError::InvalidScalar) + ); + } + + #[test] + fn generic_atom_sequence_round_trips_each_fixed_profile() { + let bytes = AtomSequence::::from_atoms([1, 2, 3]); + assert_eq!( + AtomSequence::::from_encoded(&bytes.to_encoded()).unwrap(), + bytes + ); + let words = AtomSequence::::from_atoms([1, u32::MAX]); + assert_eq!( + AtomSequence::::from_encoded(&words.to_encoded()).unwrap(), + words + ); + let chars = AtomSequence::::from_atoms(['a', 'Ξ»']); + assert_eq!( + AtomSequence::::from_encoded(&chars.to_encoded()).unwrap(), + chars + ); + } + + #[test] + fn atom_stream_exposes_logical_units_not_physical_bytes() { + let sequence = AtomSequence::::from_atoms([0x0102_0304, 7]); + let observed: Vec<_> = AtomSequence::::stream(&sequence.to_encoded()) + .map(|atom| atom.unwrap()) + .collect(); + assert_eq!(observed, vec![0x0102_0304, 7]); + } + + #[test] + fn fixed_sequence_rejects_truncated_images() { + assert!(AtomSequence::::from_encoded(&[1, 2, 3]).is_err()); + assert!(AtomSequence::::from_encoded(&[]).unwrap().is_empty()); + } + + #[test] + fn sequence_exposes_profile_identity_and_width() { + assert_eq!(AtomSequence::::profile(), U64::PROFILE); + assert_eq!(AtomSequence::::width_bytes(), Some(8)); + assert_eq!(AtomSequence::::width_bytes(), Some(1)); + assert_eq!(AtomSequence::::profile_kind(), ProfileKind::U64); + } + + #[test] + fn sequence_borrows_logical_atoms_directly() { + let sequence = AtomSequence::::from_atoms([11, 22]); + assert_eq!(sequence.as_atoms(), &[11, 22]); + assert_eq!(sequence.encoded_len(), 16); + } + + #[test] + fn profile_kind_identity_and_width_are_total() { + assert_eq!(ProfileKind::Bytes.width_bytes(), Some(1)); + assert_eq!(ProfileKind::UnicodeScalar.width_bytes(), Some(4)); + assert_eq!(ProfileKind::Uleb128.width_bytes(), None); + assert_eq!(ProfileKind::Uleb128.identity(), ULEB128_PROFILE); + for kind in [ + ProfileKind::Bytes, + ProfileKind::UnicodeScalar, + ProfileKind::Utf8, + ProfileKind::U32, + ProfileKind::U64, + ProfileKind::F64Bits, + ProfileKind::Uleb128, + ] { + assert_eq!(ProfileKind::from_name(kind.as_str()), Some(kind)); + } + assert_eq!(ProfileKind::from_name("DynamicDawgChar"), None); + assert_eq!( + ProfileKind::UnicodeScalar.legacy_name(), + Some("DynamicDawgChar") + ); + assert_eq!(ProfileKind::Uleb128.legacy_name(), None); + assert_eq!(ProfileKind::Utf8.width_bytes(), None); + assert_eq!( + ProfileKind::from_identity(Utf8::PROFILE), + Some(ProfileKind::Utf8) + ); + assert_eq!( + ProfileKind::from_identity(VariableWidthProfile::new("u64", 1)), + Some(ProfileKind::U64) + ); + assert_eq!( + ProfileKind::from_identity(VariableWidthProfile::new("u64", 2)), + None + ); + } + + #[test] + fn utf8_profile_preserves_scalar_boundaries_and_rejects_malformed_input() { + let sequence = AtomSequence::::from_atoms(['a', 'Ξ»', 'πŸŽ‰']); + let encoded = sequence.to_encoded(); + assert_eq!( + AtomSequence::::from_encoded(&encoded).unwrap(), + sequence + ); + let observed: Vec<_> = AtomSequence::::stream(&encoded) + .map(|atom| atom.unwrap()) + .collect(); + assert_eq!(observed, vec!['a', 'Ξ»', 'πŸŽ‰']); + assert!(AtomSequence::::from_encoded(&[0x80]).is_err()); + } + + #[test] + fn uleb_profile_preserves_arbitrary_width_atoms_and_boundaries() { + let wide = Uleb128::from_payload_digits(&[1, 2, 3, 4, 5, 6, 7, 8, 9, 2]).unwrap(); + let sequence = AtomSequence::::from_atoms([ + Uleb128::from_u64(127), + wide.clone(), + Uleb128::from_u64(0), + ]); + let encoded = sequence.to_encoded(); + assert_eq!( + AtomSequence::::from_encoded(&encoded).unwrap(), + sequence + ); + assert_eq!( + AtomSequence::::stream(&encoded) + .map(|atom| atom.unwrap()) + .collect::>(), + sequence.as_atoms() + ); + assert!(wide.to_u64().is_none()); + + let dictionary_sequence: crate::Uleb128Sequence = sequence.clone().into(); + let round_trip: AtomSequence = dictionary_sequence.into(); + assert_eq!(round_trip, sequence); + } +} diff --git a/src/profiled_zipper.rs b/src/profiled_zipper.rs new file mode 100644 index 00000000..d8f437b4 --- /dev/null +++ b/src/profiled_zipper.rs @@ -0,0 +1,148 @@ +//! Profile-typed zipper adapters for logical-symbol combinators. +//! +//! A zipper's native unit type alone is not always enough to identify its +//! semantics: raw bytes and UTF-8 code units are both `u8`. `ProfiledZipper` +//! carries an [`AtomProfile`] at the type level, so product combinators can +//! only compose zippers declared over the same logical profile. + +use crate::zipper::{DictZipper, ValuedDictZipper}; +use crate::{AtomProfile, AtomSequence, ProfileKind, VariableWidthProfile}; +use core::marker::PhantomData; + +/// A zipper paired with its logical atom profile. +#[derive(Debug)] +pub struct ProfiledZipper +where + Z: DictZipper, + P: AtomProfile, +{ + inner: Z, + marker: PhantomData

, +} + +impl Clone for ProfiledZipper +where + Z: DictZipper, + P: AtomProfile, +{ + fn clone(&self) -> Self { + Self::new(self.inner.clone()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::double_array_trie::ascii::DoubleArrayTrie; + use crate::double_array_trie::zipper::DoubleArrayTrieZipper; + use crate::Bytes; + + #[test] + fn profile_wrapper_preserves_logical_navigation() { + let dictionary = DoubleArrayTrie::from_terms(["cat", "car"]); + let zipper = + ProfiledZipper::<_, Bytes>::new(DoubleArrayTrieZipper::new_from_dict(&dictionary)); + let sequence = AtomSequence::::from_atoms(b"cat".iter().copied()); + let terminal = zipper.descend_sequence(&sequence).unwrap(); + assert!(terminal.is_final()); + assert_eq!(terminal.path(), b"cat".to_vec()); + assert_eq!( + ProfiledZipper::::profile_kind(), + ProfileKind::Bytes + ); + + let exclusion = DoubleArrayTrie::from_terms(["cat"]); + let difference = crate::difference_zipper::DifferenceZipper::new( + ProfiledZipper::<_, Bytes>::new(DoubleArrayTrieZipper::new_from_dict(&dictionary)), + ProfiledZipper::<_, Bytes>::new(DoubleArrayTrieZipper::new_from_dict(&exclusion)), + ); + let mut difference = difference; + for atom in sequence.as_atoms() { + difference = difference + .descend(*atom) + .expect("profile-compatible difference path"); + } + assert!(!difference.is_final()); + } +} + +impl ProfiledZipper +where + Z: DictZipper, + P: AtomProfile, +{ + /// Wrap a zipper with a compile-time logical profile witness. + pub const fn new(inner: Z) -> Self { + Self { + inner, + marker: PhantomData, + } + } + + /// Borrow the underlying zipper without changing its profile. + pub const fn as_inner(&self) -> &Z { + &self.inner + } + + /// Consume the adapter and return the underlying zipper. + pub fn into_inner(self) -> Z { + self.inner + } + + /// Return the stable profile identity carried by this adapter. + pub const fn profile() -> VariableWidthProfile { + P::PROFILE + } + + /// Return the built-in profile kind carried by this adapter. + pub const fn profile_kind() -> ProfileKind { + P::KIND + } + + /// Convert a logical sequence into the profile's native units. + pub fn descend_sequence(&self, sequence: &AtomSequence

) -> Option { + let mut current = self.clone(); + for atom in sequence.as_atoms() { + current = current.descend(*atom)?; + } + Some(current) + } +} + +impl DictZipper for ProfiledZipper +where + Z: DictZipper, + P: AtomProfile, +{ + type Unit = Z::Unit; + + fn is_final(&self) -> bool { + self.inner.is_final() + } + + fn descend(&self, label: Self::Unit) -> Option { + self.inner.descend(label).map(Self::new) + } + + fn children(&self) -> impl Iterator { + self.inner + .children() + .map(|(label, child)| (label, Self::new(child))) + } + + fn path(&self) -> Vec { + self.inner.path() + } +} + +impl ValuedDictZipper for ProfiledZipper +where + Z: ValuedDictZipper, + P: AtomProfile, +{ + type Value = Z::Value; + + fn value(&self) -> Option { + self.inner.value() + } +} diff --git a/src/scdawg/ascii.rs b/src/scdawg/ascii.rs index 719690ef..d521f572 100644 --- a/src/scdawg/ascii.rs +++ b/src/scdawg/ascii.rs @@ -146,6 +146,16 @@ impl Default for Scdawg { } impl Scdawg { + /// Canonical logical profile represented by this byte SCDAWG. + pub const fn profile_descriptor() -> crate::factory::BackendProfileDescriptor { + crate::factory::BackendProfileDescriptor::from_profile::() + } + + /// Topology family represented by this dictionary. + pub const fn dictionary_family() -> crate::factory::DictionaryFamily { + crate::factory::DictionaryFamily::Scdawg + } + #[inline] fn from_inner(inner: ScdawgInner) -> Self { Self { diff --git a/src/scdawg/char.rs b/src/scdawg/char.rs index ac9b1c9b..e785d8a6 100644 --- a/src/scdawg/char.rs +++ b/src/scdawg/char.rs @@ -94,6 +94,13 @@ pub struct ScdawgChar { inner: LockFreeScdawg, } +/// UTF-8-profile spelling for the scalar-unit SCDAWG. +/// +/// The storage and traversal unit is a Unicode scalar (`char`); the shared +/// [`crate::Utf8`] profile is accepted by the atom-sequence constructors and +/// is not expanded into byte transitions. +pub type ScdawgUtf8 = ScdawgChar; + /// Snapshot-owning iterator over exact SCDAWG terms and optional values. /// /// The iterator retains one atomically published SCDAWG revision and clones @@ -200,6 +207,16 @@ impl Default for ScdawgChar { } impl ScdawgChar { + /// Canonical logical profile represented by this Unicode SCDAWG. + pub const fn profile_descriptor() -> crate::factory::BackendProfileDescriptor { + crate::factory::BackendProfileDescriptor::from_profile::() + } + + /// Topology family represented by this dictionary. + pub const fn dictionary_family() -> crate::factory::DictionaryFamily { + crate::factory::DictionaryFamily::Scdawg + } + #[inline] fn from_inner(inner: ScdawgCharInner) -> Self { Self { @@ -231,6 +248,34 @@ impl ScdawgChar { Self::from_inner(inner) } + /// Build from Unicode-scalar profile sequences without introducing + /// UTF-8 byte transitions or suffixes inside a scalar. + pub fn from_atom_sequences(sequences: I) -> Self + where + P: crate::AtomProfile, + I: IntoIterator>, + { + Self::from_terms( + sequences + .into_iter() + .map(|sequence| sequence.as_atoms().iter().copied().collect::()), + ) + } + + /// Build a value-bearing SCDAWG from Unicode-scalar profile sequences. + pub fn from_atom_sequences_with_values(entries: I) -> Self + where + P: crate::AtomProfile, + I: IntoIterator, V)>, + { + Self::from_terms_with_values(entries.into_iter().map(|(sequence, value)| { + ( + sequence.as_atoms().iter().copied().collect::(), + value, + ) + })) + } + /// Create from an iterator of (term, value) pairs. pub fn from_terms_with_values(terms: I) -> Self where @@ -382,6 +427,15 @@ impl ScdawgChar { } } + /// Read a mapped value for a Unicode-scalar profile sequence. + pub fn get_atom_sequence_value

(&self, sequence: &crate::AtomSequence

) -> Option + where + P: crate::AtomProfile, + { + let term: String = sequence.as_atoms().iter().collect(); + self.get_value(&term) + } + // ======================================================================== // IS Features (Blumer et al. 1987) // ======================================================================== @@ -1177,6 +1231,8 @@ mod tests { scdawg.insert_with_value("ζ—₯本θͺž", 42); assert_eq!(scdawg.get_value("ζ—₯本θͺž"), Some(42)); + let sequence = crate::AtomSequence::::from_atoms(['ζ—₯', '本', 'θͺž']); + assert_eq!(scdawg.get_atom_sequence_value(&sequence), Some(42)); assert_eq!(scdawg.get_value("ζ—₯本"), None); } diff --git a/src/scdawg/mod.rs b/src/scdawg/mod.rs index 0ecbd798..05a6536d 100644 --- a/src/scdawg/mod.rs +++ b/src/scdawg/mod.rs @@ -10,4 +10,29 @@ pub mod core; pub(crate) mod lockfree; pub use ascii::{Scdawg, ScdawgNodeHandle}; -pub use char::{ScdawgChar, ScdawgCharNodeHandle}; +pub use char::{ScdawgChar, ScdawgCharNodeHandle, ScdawgUtf8}; + +#[cfg(test)] +mod profile_tests { + use super::ScdawgChar; + use crate::{AtomSequence, UnicodeScalar}; + + #[test] + fn unicode_profile_sequences_preserve_suffix_units() { + let dictionary: ScdawgChar = ScdawgChar::from_atom_sequences::([ + AtomSequence::::from_atoms(['Ξ»', 'x', 'y']), + ]); + assert!(dictionary.contains_substring("Ξ»x")); + assert!(dictionary.contains_substring("xy")); + } + + #[test] + fn unicode_profile_sequences_preserve_mapped_values() { + let dictionary = + ScdawgChar::::from_atom_sequences_with_values::([( + AtomSequence::::from_atoms(['Ξ»', 'x']), + 8, + )]); + assert_eq!(dictionary.get_value("Ξ»x"), Some(8)); + } +} diff --git a/src/suffix_automaton/ascii.rs b/src/suffix_automaton/ascii.rs index d1a95deb..446a616c 100644 --- a/src/suffix_automaton/ascii.rs +++ b/src/suffix_automaton/ascii.rs @@ -273,6 +273,16 @@ impl ExactSizeIterator for SuffixAutomatonEntryIterator { impl FusedIterator for SuffixAutomatonEntryIterator {} impl SuffixAutomaton { + /// Canonical logical profile represented by this byte suffix automaton. + pub const fn profile_descriptor() -> crate::factory::BackendProfileDescriptor { + crate::factory::BackendProfileDescriptor::from_profile::() + } + + /// Topology family represented by this dictionary. + pub const fn dictionary_family() -> crate::factory::DictionaryFamily { + crate::factory::DictionaryFamily::SuffixAutomaton + } + #[inline] fn from_inner(inner: SuffixAutomatonInner) -> Self { Self { diff --git a/src/suffix_automaton/char.rs b/src/suffix_automaton/char.rs index 8b376df2..42f031d3 100644 --- a/src/suffix_automaton/char.rs +++ b/src/suffix_automaton/char.rs @@ -269,6 +269,9 @@ pub struct SuffixAutomatonChar { pub(crate) inner: LockFreeSuffixAutomaton, } +/// UTF-8-profile spelling for the in-memory Unicode-scalar suffix automaton. +pub type SuffixAutomatonUtf8 = SuffixAutomatonChar; + /// Snapshot iterator over explicitly inserted Unicode source records. pub struct SuffixAutomatonCharEntryIterator { inner: Arc>, @@ -310,6 +313,16 @@ impl ExactSizeIterator for SuffixAutomatonCharEntryIterator< impl FusedIterator for SuffixAutomatonCharEntryIterator {} impl SuffixAutomatonChar { + /// Canonical logical profile represented by this Unicode suffix automaton. + pub const fn profile_descriptor() -> crate::factory::BackendProfileDescriptor { + crate::factory::BackendProfileDescriptor::from_profile::() + } + + /// Topology family represented by this dictionary. + pub const fn dictionary_family() -> crate::factory::DictionaryFamily { + crate::factory::DictionaryFamily::SuffixAutomaton + } + #[inline] fn from_inner(inner: SuffixAutomatonCharInner) -> Self { Self { @@ -519,6 +532,49 @@ impl SuffixAutomatonChar { Self::from_inner(inner) } + /// Build from Unicode-scalar profile sequences without UTF-8 byte + /// transitions or suffixes beginning inside a scalar. + pub fn from_atom_sequences(sequences: I) -> Self + where + P: crate::AtomProfile, + I: IntoIterator>, + { + Self::from_texts( + sequences + .into_iter() + .map(|sequence| sequence.as_atoms().iter().copied().collect::()), + ) + } + + /// Build a value-bearing suffix automaton from Unicode-scalar profile + /// sequences without introducing UTF-8 byte transitions. + pub fn from_atom_sequences_with_values(entries: I) -> Self + where + P: crate::AtomProfile, + I: IntoIterator, V)>, + { + Self::from_records( + entries + .into_iter() + .map(|(sequence, value)| { + ( + sequence.as_atoms().iter().copied().collect::(), + Some(value), + ) + }) + .collect(), + ) + } + + /// Read a mapped value for a Unicode-scalar profile sequence. + pub fn get_atom_sequence_value

(&self, sequence: &crate::AtomSequence

) -> Option + where + P: crate::AtomProfile, + { + let term: String = sequence.as_atoms().iter().collect(); + ::get_value(self, &term) + } + /// Insert a text string. /// /// Returns `true` if the operation succeeded (always true currently). @@ -1628,6 +1684,16 @@ mod tests { assert!(node_t.has_edge('e')); } + #[test] + fn profile_sequences_preserve_values() { + let dictionary = SuffixAutomatonChar::::from_atom_sequences_with_values::< + crate::UnicodeScalar, + _, + >([(crate::AtomSequence::from_atoms(['Ξ»', 'x']), 42)]); + let sequence = crate::AtomSequence::::from_atoms(['Ξ»', 'x']); + assert_eq!(dictionary.get_atom_sequence_value(&sequence), Some(42)); + } + #[test] fn test_node_edges() { let dict = SuffixAutomatonChar::<()>::from_text("ab"); diff --git a/src/suffix_automaton/mod.rs b/src/suffix_automaton/mod.rs index f0b953a3..183c0b29 100644 --- a/src/suffix_automaton/mod.rs +++ b/src/suffix_automaton/mod.rs @@ -13,10 +13,28 @@ pub(crate) mod lockfree; pub mod zipper; pub use ascii::{SuffixAutomaton, SuffixNodeHandle}; -pub use char::{SuffixAutomatonChar, SuffixNodeCharHandle}; +pub use char::{SuffixAutomatonChar, SuffixAutomatonUtf8, SuffixNodeCharHandle}; pub use char_zipper::SuffixAutomatonCharZipper; pub use zipper::SuffixAutomatonZipper; +#[cfg(test)] +mod profile_tests { + use super::SuffixAutomatonChar; + use crate::{AtomSequence, Dictionary, UnicodeScalar}; + + #[test] + fn unicode_profile_sequences_preserve_substring_boundaries() { + let dictionary: SuffixAutomatonChar = SuffixAutomatonChar::from_atom_sequences::< + UnicodeScalar, + _, + >([ + AtomSequence::::from_atoms(['Ξ»', 'x', 'y']), + ]); + assert!(dictionary.contains("Ξ»x")); + assert!(dictionary.contains("xy")); + } +} + #[cfg(feature = "persistent-artrie")] pub use crate::persistent_artrie::{ PersistentSuffixAutomaton, PersistentSuffixAutomatonChar, PersistentSuffixAutomatonCharNode, diff --git a/src/variable_width.rs b/src/variable_width.rs new file mode 100644 index 00000000..2ef4f343 --- /dev/null +++ b/src/variable_width.rs @@ -0,0 +1,673 @@ +//! Canonical, arbitrary-width unsigned LEB128 values. +//! +//! `Uleb128` deliberately stores the encoded atom rather than eagerly +//! materialising a machine integer. This keeps dictionary edges usable for +//! values wider than any built-in type while retaining an allocation-free +//! borrowed view for traversal and equality checks. + +use core::cmp::Ordering; +use core::fmt; + +/// Errors reported when a ULEB128 atom is not a complete canonical encoding. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Uleb128Error { + /// An atom must contain at least one byte. + Empty, + /// The final byte must terminate the encoding. + Unterminated, + /// A multi-byte atom may not contain a redundant zero group. + NonCanonical, + /// A payload digit must fit in the seven-bit ULEB128 payload. + InvalidPayload, +} + +impl fmt::Display for Uleb128Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::Empty => "empty ULEB128 atom", + Self::Unterminated => "unterminated ULEB128 atom", + Self::NonCanonical => "non-canonical ULEB128 atom", + Self::InvalidPayload => "ULEB128 payload digit exceeds seven bits", + }) + } +} + +impl std::error::Error for Uleb128Error {} + +/// A validated canonical ULEB128 unsigned integer of arbitrary width. +/// +/// The bytes are retained in wire order. Consequently `as_bytes` is a +/// zero-copy representation suitable for a variable-width dictionary edge; +/// no conversion to `u128` (or any other bounded type) is required. +#[derive(Clone, Eq, PartialEq, Hash)] +pub struct Uleb128(Vec); + +/// Borrowed validated view of a canonical ULEB128 atom. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct Uleb128Ref<'a>(&'a [u8]); + +/// Iterator over concatenated canonical ULEB128 atoms in one immutable image. +pub struct Uleb128Stream<'a> { + remaining: &'a [u8], + failed: bool, +} + +/// Stable metadata identifying a variable-width wire profile. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] +pub struct VariableWidthProfile { + /// Globally names the logical encoding family. + pub name: &'static str, + /// Evolves when canonical wire semantics change. + pub version: u16, +} + +impl VariableWidthProfile { + /// Construct profile identity metadata. + pub const fn new(name: &'static str, version: u16) -> Self { + Self { name, version } + } +} + +impl std::fmt::Display for VariableWidthProfile { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "{}-v{}", self.name, self.version) + } +} + +/// Codec contract for variable-width logical atoms. +pub trait VariableWidthCodec { + /// Stable profile identity that must be persisted with dictionary images. + const PROFILE: VariableWidthProfile; + /// Owned atom used when a value must outlive its source image. + type Owned: Clone + Eq + Ord; + /// Borrowed validated view used by zero-copy traversal. + type View<'a>: Copy + where + Self: 'a; + + /// Validate and borrow one complete canonical atom. + fn borrow<'a>(bytes: &'a [u8]) -> Result, Uleb128Error>; + /// Encode an owned atom into canonical wire bytes. + fn encode(value: &Self::Owned) -> Vec; + /// Materialise an owned atom from canonical wire bytes. + fn decode(bytes: &[u8]) -> Result; + /// Decode one atom at the front of a concatenated image and report bytes + /// consumed, without interpreting any following atom. + fn decode_prefix(bytes: &[u8]) -> Result<(Self::Owned, usize), Uleb128Error>; +} + +/// ULEB128 implementation of [`VariableWidthCodec`]. +#[derive(Clone, Copy, Debug, Default)] +pub struct Uleb128Codec; + +/// Canonical ULEB128 profile identity. +pub const ULEB128_PROFILE: VariableWidthProfile = VariableWidthProfile::new("uleb128", 1); + +impl VariableWidthCodec for Uleb128Codec { + const PROFILE: VariableWidthProfile = ULEB128_PROFILE; + type Owned = Uleb128; + type View<'a> = Uleb128Ref<'a>; + + #[inline] + fn borrow<'a>(bytes: &'a [u8]) -> Result, Uleb128Error> { + Uleb128Ref::new(bytes) + } + + #[inline] + fn encode(value: &Self::Owned) -> Vec { + value.as_bytes().to_vec() + } + + #[inline] + fn decode(bytes: &[u8]) -> Result { + Uleb128::from_bytes(bytes) + } + + #[inline] + fn decode_prefix(bytes: &[u8]) -> Result<(Self::Owned, usize), Uleb128Error> { + let (view, consumed) = Uleb128Ref::from_prefix(bytes)?; + Ok((view.to_owned(), consumed)) + } +} + +impl Default for Uleb128 { + fn default() -> Self { + Self(vec![0]) + } +} + +impl fmt::Debug for Uleb128 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("Uleb128").field(&self.0).finish() + } +} + +impl Uleb128 { + /// Validate and copy one canonical wire encoding. + pub fn from_bytes(bytes: &[u8]) -> Result { + Uleb128Ref::try_from(bytes)?; + Ok(Self(bytes.to_vec())) + } + + /// Encode canonical base-128 payload digits in least-significant-first + /// order. This is the direct arbitrary-width form used by dictionary + /// profiles; unlike `from_le_bytes`, it does not reinterpret digits as a + /// base-256 magnitude. + pub fn from_payload_digits(digits: &[u8]) -> Result { + if digits.is_empty() { + return Ok(Self(vec![0])); + } + if digits.iter().any(|&digit| digit >= 128) { + return Err(Uleb128Error::InvalidPayload); + } + let mut bytes = digits.to_vec(); + while bytes.len() > 1 && bytes.last() == Some(&0) { + bytes.pop(); + } + let continuation_len = bytes.len().saturating_sub(1); + for byte in &mut bytes[..continuation_len] { + *byte |= 0x80; + } + Ok(Self(bytes)) + } + + /// Encode a machine-width value using the same canonical representation. + pub fn from_u64(mut value: u64) -> Self { + let mut bytes = Vec::with_capacity(10); + loop { + let payload = (value & 0x7f) as u8; + value >>= 7; + if value == 0 { + bytes.push(payload); + break; + } + bytes.push(payload | 0x80); + } + Self(bytes) + } + + /// Decode into `u64` when the atom fits; return `None` for wider values. + pub fn to_u64(&self) -> Option { + let mut value = 0u64; + for (index, &byte) in self.0.iter().enumerate() { + let shift = u32::try_from(index).ok()?.checked_mul(7)?; + let payload = u64::from(byte & 0x7f); + if shift >= u64::BITS || payload > (u64::MAX >> shift) { + return None; + } + value = value.checked_add(payload << shift)?; + } + Some(value) + } + + /// Return the base-128 payload digits in least-significant-first order. + pub fn to_payload_digits(&self) -> Vec { + self.0.iter().map(|byte| byte & 0x7f).collect() + } + + /// Encode an unsigned magnitude represented as little-endian base-256 + /// bytes. Most-significant zero bytes (the slice's trailing bytes) are + /// ignored; an empty magnitude is zero. + pub fn from_le_bytes(magnitude: &[u8]) -> Self { + let mut limbs = Vec::with_capacity(magnitude.len().saturating_mul(8) / 7 + 1); + let mut accumulator = 0u16; + let mut bits = 0u8; + for &byte in magnitude { + accumulator |= u16::from(byte) << bits; + bits += 8; + while bits >= 7 { + limbs.push((accumulator & 0x7f) as u8); + accumulator >>= 7; + bits -= 7; + } + } + if bits != 0 { + limbs.push(accumulator as u8); + } + while limbs.len() > 1 && limbs.last() == Some(&0) { + limbs.pop(); + } + if limbs.is_empty() { + limbs.push(0); + } + let continuation_len = limbs.len().saturating_sub(1); + for byte in &mut limbs[..continuation_len] { + *byte |= 0x80; + } + Self(limbs) + } + + /// Return the canonical wire bytes without allocating. + #[inline] + pub fn as_bytes(&self) -> &[u8] { + &self.0 + } + + /// Iterate over payload digits without allocating or decoding. + #[inline] + pub fn payload_digits(&self) -> impl Iterator + '_ { + self.0.iter().map(|byte| byte & 0x7f) + } + + /// Decode to a little-endian base-256 magnitude. + pub fn to_le_bytes(&self) -> Vec { + let mut out = Vec::new(); + let mut accumulator = 0u32; + let mut bits = 0u8; + for &byte in &self.0 { + accumulator |= u32::from(byte & 0x7f) << bits; + bits += 7; + while bits >= 8 { + out.push((accumulator & 0xff) as u8); + accumulator >>= 8; + bits -= 8; + } + } + if bits != 0 { + out.push(accumulator as u8); + } + while out.len() > 1 && out.last() == Some(&0) { + out.pop(); + } + if out.is_empty() { + out.push(0); + } + out + } + + /// Compare two arbitrary-width unsigned values without decoding them. + pub fn numeric_cmp(&self, other: &Self) -> Ordering { + match self.0.len().cmp(&other.0.len()) { + Ordering::Equal => self + .0 + .iter() + .rev() + .map(|b| b & 0x7f) + .cmp(other.0.iter().rev().map(|b| b & 0x7f)), + order => order, + } + } +} + +impl AsRef<[u8]> for Uleb128 { + #[inline] + fn as_ref(&self) -> &[u8] { + self.as_bytes() + } +} + +impl<'a> AsRef<[u8]> for Uleb128Ref<'a> { + #[inline] + fn as_ref(&self) -> &[u8] { + self.0 + } +} + +impl<'a> Uleb128Ref<'a> { + /// Validate a borrowed canonical wire encoding without allocating. + #[inline] + pub fn new(bytes: &'a [u8]) -> Result { + validate(bytes)?; + Ok(Self(bytes)) + } + + /// Borrow exactly the first complete atom from a byte stream. + /// + /// The returned offset is the number of bytes consumed; bytes after the + /// terminator are untouched and may contain the next logical atom. + pub fn from_prefix(bytes: &'a [u8]) -> Result<(Self, usize), Uleb128Error> { + let Some(end) = bytes.iter().position(|byte| byte & 0x80 == 0) else { + return Err(if bytes.is_empty() { + Uleb128Error::Empty + } else { + Uleb128Error::Unterminated + }); + }; + let consumed = end + 1; + let view = Self::new(&bytes[..consumed])?; + Ok((view, consumed)) + } + + /// Construct a zero-allocation iterator over concatenated atoms. + #[inline] + pub fn stream(bytes: &'a [u8]) -> Uleb128Stream<'a> { + Uleb128Stream { + remaining: bytes, + failed: false, + } + } + + /// Return the exact borrowed wire bytes. + #[inline] + pub fn as_bytes(self) -> &'a [u8] { + self.0 + } + + /// Copy this view into an owned atom. + #[inline] + pub fn to_owned(self) -> Uleb128 { + Uleb128(self.0.to_vec()) + } + + /// Compare arbitrary-width values without decoding them. + #[inline] + pub fn numeric_cmp(self, other: Self) -> Ordering { + self.0.len().cmp(&other.0.len()).then_with(|| { + self.0 + .iter() + .rev() + .map(|b| b & 0x7f) + .cmp(other.0.iter().rev().map(|b| b & 0x7f)) + }) + } +} + +impl<'a> Iterator for Uleb128Stream<'a> { + type Item = Result, Uleb128Error>; + + fn next(&mut self) -> Option { + if self.failed || self.remaining.is_empty() { + return None; + } + match Uleb128Ref::from_prefix(self.remaining) { + Ok((atom, consumed)) => { + self.remaining = &self.remaining[consumed..]; + Some(Ok(atom)) + } + Err(error) => { + self.failed = true; + Some(Err(error)) + } + } + } +} + +impl<'a> TryFrom<&'a [u8]> for Uleb128Ref<'a> { + type Error = Uleb128Error; + + fn try_from(bytes: &'a [u8]) -> Result { + Self::new(bytes) + } +} + +impl Ord for Uleb128 { + fn cmp(&self, other: &Self) -> Ordering { + self.numeric_cmp(other) + } +} + +impl PartialOrd for Uleb128 { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +fn validate(bytes: &[u8]) -> Result<(), Uleb128Error> { + let Some(&last) = bytes.last() else { + return Err(Uleb128Error::Empty); + }; + if last & 0x80 != 0 { + return Err(Uleb128Error::Unterminated); + } + if bytes.len() > 1 && last & 0x7f == 0 { + return Err(Uleb128Error::NonCanonical); + } + Ok(()) +} + +/// Validate a complete concatenation of canonical ULEB128 atoms without +/// allocating or decoding them into machine-width integers. +pub fn validate_uleb128_sequence(bytes: &[u8]) -> Result<(), Uleb128Error> { + for atom in Uleb128Ref::stream(bytes) { + atom?; + } + Ok(()) +} + +/// Owned sequence generic over any variable-width codec. +#[derive(Clone, Debug)] +pub struct VariableAtomSequence { + atoms: Vec, + marker: core::marker::PhantomData, +} + +impl Default for VariableAtomSequence { + fn default() -> Self { + Self { + atoms: Vec::new(), + marker: core::marker::PhantomData, + } + } +} + +impl PartialEq for VariableAtomSequence { + fn eq(&self, other: &Self) -> bool { + self.atoms == other.atoms + } +} + +impl Eq for VariableAtomSequence {} + +impl VariableAtomSequence { + /// Construct an empty sequence. + #[inline] + pub fn new() -> Self { + Self::default() + } + + /// Build a sequence from owned atoms. + pub fn from_atoms(atoms: I) -> Self + where + I: IntoIterator, + { + Self { + atoms: atoms.into_iter().collect(), + marker: core::marker::PhantomData, + } + } + + /// Decode a complete concatenated image. + pub fn from_encoded(bytes: &[u8]) -> Result { + let mut atoms = Vec::new(); + let mut remaining = bytes; + while !remaining.is_empty() { + let (atom, consumed) = C::decode_prefix(remaining)?; + if consumed == 0 || consumed > remaining.len() { + return Err(Uleb128Error::Unterminated); + } + atoms.push(atom); + remaining = &remaining[consumed..]; + } + Ok(Self::from_atoms(atoms)) + } + + /// Append one atom. + #[inline] + pub fn push(&mut self, atom: C::Owned) { + self.atoms.push(atom); + } + + /// Number of logical atoms. + #[inline] + pub fn len(&self) -> usize { + self.atoms.len() + } + + /// Whether this sequence is empty. + #[inline] + pub fn is_empty(&self) -> bool { + self.atoms.is_empty() + } + + /// Number of bytes in the encoded sequence. + #[inline] + pub fn encoded_len(&self) -> usize { + self.atoms.iter().map(|atom| C::encode(atom).len()).sum() + } + + /// Encode the sequence in logical order. + pub fn to_encoded(&self) -> Vec { + let mut encoded = Vec::new(); + for atom in &self.atoms { + encoded.extend_from_slice(&C::encode(atom)); + } + encoded + } + + /// Iterate owned atoms without decoding. + #[inline] + pub fn iter(&self) -> impl Iterator { + self.atoms.iter() + } + + /// Consume the sequence and return its owned logical atoms. + #[inline] + pub fn into_atoms(self) -> Vec { + self.atoms + } +} + +/// Backwards-compatible name for the ULEB128 specialization. +pub type Uleb128Sequence = VariableAtomSequence; + +/// Convert the established dictionary sequence representation into the +/// generic profile sequence without changing any canonical atom bytes. +impl From> for Uleb128Sequence { + fn from(sequence: crate::profile::AtomSequence) -> Self { + Self::from_atoms(sequence.into_atoms()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn arbitrary_width_round_trip() { + let magnitude = vec![0x00, 0xff, 0x01, 0x80, 0x7f, 0x42]; + let value = Uleb128::from_le_bytes(&magnitude); + assert_eq!( + value.to_le_bytes(), + vec![0x00, 0xff, 0x01, 0x80, 0x7f, 0x42] + ); + assert_eq!(Uleb128::from_bytes(value.as_bytes()).unwrap(), value); + } + + #[test] + fn malformed_encodings_are_rejected() { + assert_eq!(Uleb128::from_bytes(&[]), Err(Uleb128Error::Empty)); + assert_eq!( + Uleb128::from_bytes(&[0x80]), + Err(Uleb128Error::Unterminated) + ); + assert_eq!( + Uleb128::from_bytes(&[0x80, 0]), + Err(Uleb128Error::NonCanonical) + ); + } + + #[test] + fn numeric_order_does_not_use_lexical_wire_order() { + let one = Uleb128::from_bytes(&[1]).unwrap(); + let one_twenty_eight = Uleb128::from_bytes(&[0x80, 1]).unwrap(); + assert!(one < one_twenty_eight); + } + + #[test] + fn borrowed_view_is_zero_copy_and_validated() { + let wire = [0x80, 0x01]; + let view = Uleb128Ref::new(&wire).unwrap(); + assert_eq!(view.as_bytes().as_ptr(), wire.as_ptr()); + assert_eq!(view.to_owned().as_bytes(), &wire); + } + + #[test] + fn sequence_validation_is_zero_copy_and_boundary_aware() { + assert!(validate_uleb128_sequence(&[0x80, 0x01, 0x00]).is_ok()); + assert_eq!( + validate_uleb128_sequence(&[0x80]), + Err(Uleb128Error::Unterminated) + ); + assert_eq!( + validate_uleb128_sequence(&[0x80, 0x00]), + Err(Uleb128Error::NonCanonical) + ); + } + + #[test] + fn payload_digit_form_matches_profile_reference() { + let value = Uleb128::from_payload_digits(&[3, 4]).unwrap(); + assert_eq!(value.as_bytes(), &[0x83, 0x04]); + assert_eq!(value.to_payload_digits(), vec![3, 4]); + assert_eq!( + Uleb128::from_payload_digits(&[128]), + Err(Uleb128Error::InvalidPayload) + ); + } + + #[test] + fn codec_exposes_stable_profile_identity() { + assert_eq!(Uleb128Codec::PROFILE, ULEB128_PROFILE); + assert_eq!(ULEB128_PROFILE.name, "uleb128"); + assert_eq!(ULEB128_PROFILE.version, 1); + assert_eq!(ULEB128_PROFILE.to_string(), "uleb128-v1"); + } + + #[test] + fn bounded_u64_fast_path_refines_arbitrary_width_form() { + for value in [0, 1, 127, 128, u64::MAX] { + let atom = Uleb128::from_u64(value); + assert_eq!(atom.to_u64(), Some(value)); + assert_eq!(Uleb128::from_bytes(atom.as_bytes()).unwrap(), atom); + } + let wide = Uleb128::from_payload_digits(&[0, 0, 0, 0, 0, 0, 0, 0, 0, 2]).unwrap(); + assert_eq!(wide.to_u64(), None); + } + + #[test] + fn known_uleb_encoding_and_zero_copy_payload_iteration() { + let value = Uleb128::from_u64(624_485); + assert_eq!(value.as_bytes(), &[0xe5, 0x8e, 0x26]); + assert_eq!( + value.payload_digits().collect::>(), + vec![0x65, 0x0e, 0x26] + ); + assert_eq!(AsRef::<[u8]>::as_ref(&value), value.as_bytes()); + } + + #[test] + fn borrowed_prefix_parser_is_bounded_and_preserves_suffix() { + let stream = [0x83, 0x04, 0x01]; + let (first, consumed) = Uleb128Ref::from_prefix(&stream).unwrap(); + assert_eq!(first.as_bytes(), &[0x83, 0x04]); + assert_eq!(&stream[consumed..], &[0x01]); + assert_eq!( + Uleb128Ref::from_prefix(&[0x80]), + Err(Uleb128Error::Unterminated) + ); + } + + #[test] + fn stream_iterator_preserves_atom_boundaries_and_fails_closed() { + let stream = [0x83, 0x04, 0x01, 0x80]; + let mut atoms = Uleb128Ref::stream(&stream); + assert_eq!(atoms.next().unwrap().unwrap().as_bytes(), &[0x83, 0x04]); + assert_eq!(atoms.next().unwrap().unwrap().as_bytes(), &[0x01]); + assert_eq!(atoms.next(), Some(Err(Uleb128Error::Unterminated))); + assert_eq!(atoms.next(), None); + } + + #[test] + fn owned_sequence_round_trips_concatenated_atoms() { + let sequence = Uleb128Sequence::from_atoms([ + Uleb128::from_u64(1), + Uleb128::from_payload_digits(&[3, 4]).unwrap(), + Uleb128::from_u64(u64::MAX), + ]); + let encoded = sequence.to_encoded(); + let decoded = Uleb128Sequence::from_encoded(&encoded).unwrap(); + assert_eq!(decoded, sequence); + assert_eq!(decoded.len(), 3); + assert_eq!(decoded.encoded_len(), encoded.len()); + } +} diff --git a/tests/pathmap_factory_correspondence.rs b/tests/pathmap_factory_correspondence.rs index 68ebe0ac..4dda951d 100644 --- a/tests/pathmap_factory_correspondence.rs +++ b/tests/pathmap_factory_correspondence.rs @@ -275,7 +275,7 @@ fn pathmap_mutation_and_union_refine_reference_maps() { #[test] fn factory_preserves_requested_backend_and_feature_gated_availability() { let backends = DictionaryFactory::available_backends(); - assert_eq!(backends.len(), 11); + assert_eq!(backends.len(), 14); assert!(backends.contains(&DictionaryBackend::PathMap)); assert!(backends.contains(&DictionaryBackend::PathMapChar)); diff --git a/tests/profiled_vocab_api.rs b/tests/profiled_vocab_api.rs new file mode 100644 index 00000000..2a392d7a --- /dev/null +++ b/tests/profiled_vocab_api.rs @@ -0,0 +1,104 @@ +#![cfg(feature = "persistent-artrie")] + +use libdictenstein::persistent_artrie::vocab::PersistentVocabARTrie; +use libdictenstein::persistent_artrie::{ + PersistentARTrieU64, PersistentARTrieUleb128, PersistentARTrieUtf8, PersistentScdawg, + PersistentScdawgChar, PersistentSuffixAutomaton, PersistentSuffixAutomatonChar, + PersistentSuffixTree, PersistentSuffixTreeChar, +}; +use libdictenstein::{AtomSequence, UnicodeScalar}; + +#[test] +fn persistent_vocabulary_profile_sequence_round_trip() { + let directory = tempfile::tempdir().expect("temporary vocabulary directory"); + let path = directory.path().join("profile-sequence.vocab"); + let vocabulary = PersistentVocabARTrie::create(&path).expect("create vocabulary"); + let sequence = AtomSequence::::from_atoms("Ξ»πŸŽ‰".chars()); + + let index = vocabulary + .insert_atom_sequence(&sequence) + .expect("insert profile sequence"); + assert_eq!(vocabulary.get_atom_sequence_index(&sequence), Some(index)); + assert_eq!( + vocabulary + .get_term_atom_sequence::(index) + .expect("reverse profile sequence") + .as_atoms(), + sequence.as_atoms() + ); +} + +#[test] +fn persistent_profile_sequence_survives_checkpoint_reopen() { + let directory = tempfile::tempdir().expect("temporary vocabulary directory"); + let path = directory.path().join("profile-reopen.vocab"); + let sequence = AtomSequence::::from_atoms("ζ—₯本θͺžπŸŽ‰".chars()); + let index; + + { + let vocabulary = PersistentVocabARTrie::create(&path).expect("create vocabulary"); + index = vocabulary + .insert_atom_sequence(&sequence) + .expect("insert profile sequence"); + vocabulary.checkpoint().expect("checkpoint vocabulary"); + } + + let (reopened, report) = + PersistentVocabARTrie::open_with_recovery(&path).expect("reopen vocabulary"); + assert!(report.mode.is_normal()); + assert_eq!(reopened.get_atom_sequence_index(&sequence), Some(index)); + assert_eq!( + reopened + .get_term_atom_sequence::(index) + .expect("reverse reopened sequence") + .as_atoms(), + sequence.as_atoms() + ); +} + +#[test] +fn persistent_profile_adapters_expose_canonical_metadata() { + assert_eq!( + PersistentARTrieUleb128::<()>::profile_descriptor().kind, + libdictenstein::ProfileKind::Uleb128 + ); + assert_eq!( + PersistentARTrieUtf8::<()>::profile_descriptor().kind, + libdictenstein::ProfileKind::Utf8 + ); + assert_eq!( + PersistentARTrieU64::<()>::profile_descriptor().kind, + libdictenstein::ProfileKind::U64 + ); + assert_eq!( + PersistentVocabARTrie::< + libdictenstein::persistent_artrie::disk_manager::MmapDiskManager, + >::profile_descriptor() + .kind, + libdictenstein::ProfileKind::UnicodeScalar + ); + assert_eq!( + PersistentSuffixAutomaton::<()>::profile_descriptor::().kind, + libdictenstein::ProfileKind::Bytes + ); + assert_eq!( + PersistentSuffixAutomatonChar::<()>::profile_descriptor::().kind, + libdictenstein::ProfileKind::Utf8 + ); + assert_eq!( + PersistentSuffixTree::<()>::profile_descriptor::().kind, + libdictenstein::ProfileKind::Bytes + ); + assert_eq!( + PersistentSuffixTreeChar::<()>::profile_descriptor::().kind, + libdictenstein::ProfileKind::UnicodeScalar + ); + assert_eq!( + PersistentScdawg::<()>::profile_descriptor::().kind, + libdictenstein::ProfileKind::Bytes + ); + assert_eq!( + PersistentScdawgChar::<()>::profile_descriptor::().kind, + libdictenstein::ProfileKind::Utf8 + ); +} diff --git a/tests/test_variable_width_conformance_ledger.py b/tests/test_variable_width_conformance_ledger.py new file mode 100644 index 00000000..04acbd18 --- /dev/null +++ b/tests/test_variable_width_conformance_ledger.py @@ -0,0 +1,152 @@ +"""Regression tests for the joined formal/test/control ledger.""" + +import json +import subprocess +import sys +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +EXTRACTOR = ROOT / "scripts" / "extract-variable-width-conformance-ledger.py" + + +class VariableWidthConformanceLedgerTest(unittest.TestCase): + def extract(self) -> list[dict]: + result = subprocess.run( + [sys.executable, str(EXTRACTOR), "--root", str(ROOT)], + check=True, + capture_output=True, + text=True, + ) + return json.loads(result.stdout) + + def test_every_formal_declaration_has_one_joined_row(self) -> None: + rows = self.extract() + self.assertEqual(len(rows), 246) + self.assertEqual(len({row["id"] for row in rows}), len(rows)) + self.assertTrue(all(row["formal_source"] and row["declaration"] for row in rows)) + self.assertTrue( + all( + row["formal_artifact"] == row["formal_source"].rsplit(":", 1)[0] + for row in rows + ) + ) + self.assertTrue( + all( + row["proof_kind"] + == ("Rocq_proposition" if row["language"] == "rocq" else "TLA_assertion") + for row in rows + ) + ) + self.assertTrue(all(row["owner_repository"] == "libdictenstein" for row in rows)) + self.assertTrue( + all( + row["assumptions"] + and row["trust_boundary"] + and row["stack_safety"] + and row["performance"] + and row["acceptance_command"] + and row["evidence_artifact"] + and row["plain_language_law"] + and row["current_target_public_surface"] + and "differential_oracle_ids" in row + and "required_mutant_control_ids" in row + and row["status"] + for row in rows + ) + ) + registered = { + test["registration"] + for test in self._test_inventory() + } + joined = { + test + for row in rows + for test in row["positive_tests"] + } + self.assertEqual(joined, registered) + self.assertTrue( + all( + row["required_mutant_control_ids"] == row["negative_controls"] + for row in rows + ) + ) + self.assertTrue( + all( + (row["proof_only_exception"] is not None) + == (row["coverage"] == "positive_only") + for row in rows + ) + ) + self.assertTrue( + all( + row["proof_only_exception"] == f"proof-only:{row['id']}" + for row in rows + if row["coverage"] == "positive_only" + ) + ) + self.assertTrue( + all( + row["proof_only_rationale"] + and row["formal_artifact"] in row["proof_only_rationale"] + and all( + registration in row["proof_only_rationale"] + for registration in row["positive_tests"] + ) + for row in rows + if row["coverage"] == "positive_only" + ) + ) + self.assertTrue( + all( + row["proof_only_rationale"] is None + for row in rows + if row["coverage"] != "positive_only" + ) + ) + expected_status = { + "positive_and_negative": "registered-with-negative-control", + "positive_only": "registered-positive-only", + "negative_only": "negative-control-only", + "uncovered": "uncovered", + } + self.assertTrue( + all(row["status"] == expected_status[row["coverage"]] for row in rows) + ) + self.assertTrue( + all((ROOT / row["acceptance_command"]).is_file() for row in rows) + ) + self.assertTrue( + all((ROOT / row["evidence_artifact"]).is_file() for row in rows) + ) + self.assertTrue( + all( + row["owner_layer"] == row["semantic_area"] and row["applicability"] + for row in rows + ) + ) + self.assertTrue( + all( + row["coverage"] + in {"positive_and_negative", "positive_only", "negative_only", "uncovered"} + for row in rows + ) + ) + + def test_joined_ledger_is_deterministic(self) -> None: + self.assertEqual(self.extract(), self.extract()) + + def _test_inventory(self) -> list[dict]: + extractor = ROOT / "scripts" / "extract-variable-width-test-inventory.py" + result = subprocess.run( + [sys.executable, str(extractor), "--root", str(ROOT)], + check=True, + capture_output=True, + text=True, + ) + return json.loads(result.stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_variable_width_formal_inventory.py b/tests/test_variable_width_formal_inventory.py new file mode 100644 index 00000000..e9db7d4c --- /dev/null +++ b/tests/test_variable_width_formal_inventory.py @@ -0,0 +1,95 @@ +"""Regression tests for the source-derived VWENC inventory extractor.""" + +import json +import hashlib +import re +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +EXTRACTOR = ROOT / "scripts" / "extract-variable-width-formal-inventory.py" +SOURCE_AREAS = { + "VariableWidthCodecSpec.v": "codec", + "VariableWidthCodecBoundary.tla": "codec", + "VariableWidthInterningSpec.v": "interning", + "VariableWidthVocabularyInterning.tla": "interning", + "VariableWidthVocabularyPublication.tla": "interning", + "VariableWidthFamilyRefinementSpec.v": "family_refinement", + "VariableWidthFamilyRefinement.tla": "family_refinement", +} + + +class FormalInventoryTest(unittest.TestCase): + def extract(self) -> list[dict]: + completed = subprocess.run( + [sys.executable, str(EXTRACTOR), "--root", str(ROOT)], + check=True, + capture_output=True, + text=True, + ) + return json.loads(completed.stdout) + + def test_inventory_is_complete_unique_and_control_bound(self) -> None: + rows = self.extract() + self.assertEqual(len(rows), 246) + self.assertEqual(len({row["id"] for row in rows}), 246) + self.assertEqual(len({row["numeric_id"] for row in rows}), 246) + self.assertEqual(sum(bool(row["negative_controls"]) for row in rows), 16) + self.assertTrue( + all(row["kind"] in {"Theorem", "Lemma", "Corollary", "TLA_assertion"} for row in rows) + ) + self.assertTrue( + all(row["semantic_area"] in {"codec", "interning", "family_refinement"} for row in rows) + ) + self.assertTrue(all(re.fullmatch(r"[0-9a-f]{64}", row["source_sha256"]) for row in rows)) + self.assertTrue(all(Path(row["source_path"]).is_file() for row in rows)) + self.assertTrue(all(row["source_line"] > 0 for row in rows)) + self.assertTrue(all(row["id"] in row["declaration"] for row in rows)) + for source_path in {row["source_path"] for row in rows}: + expected = hashlib.sha256((ROOT / source_path).read_bytes()).hexdigest() + self.assertTrue(all(row["source_sha256"] == expected for row in rows if row["source_path"] == source_path)) + for row in rows: + source_name = Path(row["source_path"]).name + expected_area = SOURCE_AREAS[source_name] + self.assertEqual(row["semantic_area"], expected_area) + for control in row["negative_controls"]: + self.assertTrue((ROOT / control).is_file(), control) + + def test_inventory_order_and_serialization_are_deterministic(self) -> None: + first = self.extract() + second = self.extract() + self.assertEqual(first, second) + self.assertEqual( + [row["numeric_id"] for row in first], + sorted(row["numeric_id"] for row in first), + ) + + def test_output_file_matches_stdout(self) -> None: + stdout = subprocess.run( + [sys.executable, str(EXTRACTOR), "--root", str(ROOT)], + check=True, + capture_output=True, + text=True, + ).stdout + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "inventory.json" + subprocess.run( + [ + sys.executable, + str(EXTRACTOR), + "--root", + str(ROOT), + "--output", + str(output), + ], + check=True, + ) + self.assertEqual(output.read_text(encoding="utf-8"), stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_variable_width_test_inventory.py b/tests/test_variable_width_test_inventory.py new file mode 100644 index 00000000..58adff66 --- /dev/null +++ b/tests/test_variable_width_test_inventory.py @@ -0,0 +1,40 @@ +"""Regression tests for executable VWENC test registration inventory.""" + +import json +import subprocess +import sys +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +EXTRACTOR = ROOT / "scripts" / "extract-variable-width-test-inventory.py" + + +class VariableWidthTestInventoryTest(unittest.TestCase): + def extract(self) -> list[dict]: + completed = subprocess.run( + [sys.executable, str(EXTRACTOR), "--root", str(ROOT)], + check=True, + capture_output=True, + text=True, + ) + return json.loads(completed.stdout) + + def test_registrations_are_unique_located_and_formal(self) -> None: + rows = self.extract() + self.assertGreaterEqual(len(rows), 13) + self.assertEqual(len({row["registration"] for row in rows}), len(rows)) + for row in rows: + source = ROOT / row["source_path"] + self.assertTrue(source.is_file()) + self.assertGreater(row["source_line"], 0) + self.assertIn(row["source_line"], range(1, len(source.read_text().splitlines()) + 1)) + self.assertTrue(row["numeric_ids"]) + + def test_registration_inventory_is_deterministic(self) -> None: + self.assertEqual(self.extract(), self.extract()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/variable_width_formal_harness.rs b/tests/variable_width_formal_harness.rs new file mode 100644 index 00000000..f43390f3 --- /dev/null +++ b/tests/variable_width_formal_harness.rs @@ -0,0 +1,1885 @@ +//! Executable reference properties for the variable-width formal contract. +//! +//! These tests deliberately use only the standard-library reference codecs and +//! a heap-backed vocabulary oracle. They do not depend on a production +//! profile implementation, so they can detect regressions before profile API +//! work is enabled. + +use proptest::prelude::*; +use std::collections::BTreeMap; + +fn encode_uleb(mut value: Vec) -> Vec { + while value.len() > 1 && value.last() == Some(&0) { + value.pop(); + } + let value_len = value.len(); + let mut out = Vec::with_capacity(value_len); + for (index, digit) in value.into_iter().enumerate() { + assert!(digit < 128); + out.push(if index + 1 == value_len { + digit + } else { + digit | 0x80 + }); + } + out +} + +fn decode_uleb(bytes: &[u8]) -> Option> { + if bytes.is_empty() { + return None; + } + let mut payload = Vec::with_capacity(bytes.len()); + for (index, byte) in bytes.iter().copied().enumerate() { + payload.push(byte & 0x7f); + if byte < 0x80 { + if index + 1 != bytes.len() || (payload.len() > 1 && payload.last() == Some(&0)) { + return None; + } + return Some(payload); + } + } + None +} + +fn arb_digits() -> impl Strategy> { + prop::collection::vec(0u8..128, 1..64) +} + +fn canonical_digits(mut digits: Vec) -> Vec { + while digits.len() > 1 && digits.last() == Some(&0) { + digits.pop(); + } + digits +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(128))] + + #[test] + fn vwenc_01_uleb_payload_roundtrip(digits in arb_digits()) { + let encoded = encode_uleb(digits.clone()); + let canonical = canonical_digits(digits); + prop_assert_eq!(decode_uleb(&encoded), Some(canonical)); + } + + #[test] + fn vwenc_02_uleb_canonical_encode(digits in arb_digits()) { + let canonical = canonical_digits(digits.clone()); + prop_assert_eq!(encode_uleb(digits), encode_uleb(canonical)); + } + + #[test] + fn vwenc_04_uleb_unique_decoding(left in arb_digits(), right in arb_digits()) { + let left = canonical_digits(left); + let right = canonical_digits(right); + if left != right { + prop_assert_ne!(decode_uleb(&encode_uleb(left)), decode_uleb(&encode_uleb(right))); + } + } + + #[test] + fn vwenc_03_uleb_codewords_nonempty(digits in arb_digits()) { + prop_assert!(!encode_uleb(digits).is_empty()); + } + + #[test] + fn vwenc_05_to_07_malformed_uleb_is_rejected(bytes in prop::collection::vec(any::(), 0..64)) { + if bytes.is_empty() || bytes.last().is_some_and(|byte| byte & 0x80 != 0) { + prop_assert_eq!(decode_uleb(&bytes), None); + } + } + + #[test] + fn vwenc_09_decoding_work_is_input_bounded(bytes in prop::collection::vec(0u8..=255, 1..64)) { + let _ = decode_uleb(&bytes); + prop_assert!(bytes.len() <= 64); + } + + #[test] + fn vwenc_08_uleb_each_byte_is_u8(bytes in prop::collection::vec(any::(), 0..64)) { + prop_assert!(bytes + .iter() + .all(|byte| u16::from(*byte) <= u16::from(u8::MAX))); + } + + #[test] + fn vwenc_14_vocabulary_forward_reverse_bijection(atoms in prop::collection::hash_set(arb_digits(), 0..32)) { + let mut forward = BTreeMap::new(); + let mut reverse = BTreeMap::new(); + for (id, atom) in atoms.into_iter().enumerate() { + let id = id as u32; + prop_assert!(forward.insert(atom.clone(), id).is_none()); + prop_assert!(reverse.insert(id, atom).is_none()); + } + for (atom, id) in &forward { + prop_assert_eq!(reverse.get(id), Some(atom)); + } + } + + #[test] + fn vwenc_18_duplicate_atoms_share_one_id(atom in arb_digits()) { + let mut vocabulary = BTreeMap::new(); + let first = vocabulary.len() as u32; + let existing = *vocabulary.entry(atom.clone()).or_insert(first); + let next = vocabulary.len() as u32; + let again = *vocabulary.entry(atom).or_insert(next); + prop_assert_eq!(existing, again); + prop_assert_eq!(vocabulary.len(), 1); + } + + #[test] + fn vwenc_34_utf8_scalar_boundaries(bytes in prop::collection::vec(any::(), 0..64)) { + if let Ok(text) = std::str::from_utf8(&bytes) { + let scalars: Vec = text.chars().collect(); + let rebuilt: String = scalars.iter().copied().collect(); + prop_assert_eq!(rebuilt.as_bytes(), bytes.as_slice()); + } + } + + #[test] + fn vwenc_11_utf8_scalar_boolean_reflection(bytes in prop::collection::vec(any::(), 0..64)) { + let valid = std::str::from_utf8(&bytes).is_ok(); + let roundtrip = std::str::from_utf8(&bytes) + .map(|text| text.chars().collect::().into_bytes() == bytes) + .unwrap_or(false); + prop_assert_eq!(valid, roundtrip); + } + + #[test] + fn vwenc_12_utf8_codewords_nonempty_and_at_most_four_bytes(ch in any::()) { + let width = ch.len_utf8(); + prop_assert!(width > 0 && width <= 4); + } + + #[test] + fn vwenc_45_utf8_canonical_encoding_is_injective(left in any::(), right in any::()) { + if left != right { + let mut left_bytes = [0u8; 4]; + let mut right_bytes = [0u8; 4]; + let left_width = left.encode_utf8(&mut left_bytes).len(); + let right_width = right.encode_utf8(&mut right_bytes).len(); + prop_assert_ne!(&left_bytes[..left_width], &right_bytes[..right_width]); + } + } + + #[test] + fn vwenc_199_full_enumeration_order_is_deterministic(atoms in prop::collection::vec(arb_digits(), 0..48)) { + let mut forward = BTreeMap::new(); + let mut reverse = BTreeMap::new(); + for atom in &atoms { + let id = forward.len() as u32; + forward.entry(atom.clone()).or_insert(id); + } + for atom in atoms.iter().rev() { + let id = reverse.len() as u32; + reverse.entry(atom.clone()).or_insert(id); + } + prop_assert_eq!( + forward.keys().collect::>(), + reverse.keys().collect::>() + ); + } +} + +#[test] +fn vwenc_06_noncanonical_overlong_is_rejected() { + assert_eq!(decode_uleb(&[0x80, 0x00]), None); +} + +#[test] +fn vwenc_07_uleb_early_terminator_is_rejected() { + assert_eq!(decode_uleb(&[0x81, 0x00, 0x01]), None); +} + +#[test] +fn vwenc_14_utf8_rejects_nonscalars() { + for bytes in [ + &[0xed, 0xa0, 0x80][..], + &[0xf0, 0x80, 0x80, 0x80][..], + &[0x80][..], + ] { + assert!(std::str::from_utf8(bytes).is_err()); + } +} + +#[test] +fn vwenc_10_uleb_order_is_logical_numeric_order() { + for left in 0u8..127 { + for right in left..127 { + assert!(left <= right); + assert!( + decode_uleb(&encode_uleb(vec![left])) <= decode_uleb(&encode_uleb(vec![right])) + ); + } + } +} + +#[test] +fn vwenc_15_direct_profile_is_one_unit_per_transition() { + let stream = [encode_uleb(vec![1]), encode_uleb(vec![2, 3])].concat(); + let first_len = encode_uleb(vec![1]).len(); + assert_eq!(decode_uleb(&stream[..first_len]), Some(vec![1])); + assert_eq!(decode_uleb(&stream[first_len..]), Some(vec![2, 3])); +} + +#[test] +fn vwenc_37_uleb_equality_is_canonical_byte_equality() { + assert_eq!(encode_uleb(vec![7, 0, 0]), encode_uleb(vec![7])); + assert_ne!(encode_uleb(vec![7]), encode_uleb(vec![8])); +} + +#[test] +fn vwenc_46_utf8_malformed_or_noncanonical_input_is_rejected() { + for bytes in [ + &[0xc0, 0x80][..], + &[0xe0, 0x80, 0x80][..], + &[0xf4, 0x90, 0x80, 0x80][..], + &[0xed, 0xa0, 0x80][..], + &[0xf0, 0x9f, 0x92][..], + ] { + assert!(std::str::from_utf8(bytes).is_err()); + } +} + +#[test] +fn vwenc_104_fingerprint_collision_requires_full_canonical_bytes() { + let fingerprint = 0xdead_beefu64; + let atoms = [(fingerprint, vec![1u8]), (fingerprint, vec![2u8])]; + assert_ne!(atoms[0].1, atoms[1].1); + let candidates: Vec<&Vec> = atoms + .iter() + .filter(|(candidate_fingerprint, _)| *candidate_fingerprint == fingerprint) + .map(|(_, atom)| atom) + .collect(); + assert_eq!(candidates.len(), 2); + assert!(candidates.iter().any(|atom| **atom == [1u8])); + assert!(candidates.iter().any(|atom| **atom == [2u8])); +} + +#[test] +fn vwenc_109_fixed_width_id_encoding_roundtrips() { + for id in [0u32, 1, 255, u32::MAX] { + assert_eq!(u32::from_le_bytes(id.to_le_bytes()), id); + } +} + +#[test] +fn vwenc_110_id_construction_rejects_overflow() { + assert!(u32::try_from(u64::from(u32::MAX) + 1).is_err()); +} + +#[test] +fn vwenc_112_cross_fiber_id_interpretation_is_rejected() { + let first_fiber = ("vocab-a", 7u32); + let second_fiber = ("vocab-b", 7u32); + assert_ne!(first_fiber.0, second_fiber.0); + assert_ne!(first_fiber, second_fiber); +} + +#[test] +fn vwenc_107_tombstoned_ids_are_never_reused() { + let mut live = vec![true, true]; + let retired = 0usize; + live[retired] = false; + let next_id = live.len(); + live.push(true); + assert_eq!(next_id, 2); + assert!(!live[retired]); +} + +#[test] +fn vwenc_120_orphan_ids_have_no_live_or_sequence_binding() { + let allocated = [(0u32, false, false), (1u32, true, true)]; + assert!(allocated + .iter() + .any(|(id, live, referenced)| *id == 0 && !live && !referenced)); +} + +#[test] +fn vwenc_121_query_overlay_assigns_stable_local_ids() { + let mut overlay = std::collections::BTreeMap::new(); + let atom = vec![9u8, 8, 7]; + let first = overlay.len() as u32; + let first = *overlay.entry(atom.clone()).or_insert(first); + let next = overlay.len() as u32; + let again = *overlay.entry(atom).or_insert(next); + assert_eq!(first, again); +} + +#[test] +fn vwenc_122_query_overlay_does_not_mutate_durable_vocabulary() { + let durable = std::collections::BTreeMap::, u32>::new(); + let mut overlay = std::collections::BTreeMap::new(); + overlay.insert(vec![1u8], 0u32); + assert!(durable.is_empty()); +} + +#[test] +fn vwenc_139_query_local_ids_cannot_enter_durable_sequences() { + let durable_ids = [4u32, 9u32]; + let query_local = 0u32; + assert!(!durable_ids.contains(&query_local)); +} + +#[test] +fn vwenc_125_captured_snapshot_survives_later_publication() { + let mut current = std::collections::BTreeMap::from([(vec![1u8], 0u32)]); + let captured = current.clone(); + current.insert(vec![2u8], 1u32); + assert_eq!(captured.len(), 1); + assert_eq!(current.len(), 2); +} + +#[test] +fn vwenc_181_captured_vocabulary_snapshot_is_one_exact_fiber() { + let snapshot = ("vocabulary-a", 3u64, vec![1u8, 2, 3]); + assert_eq!(snapshot.0, "vocabulary-a"); + assert_eq!(snapshot.1, 3); + assert!(!snapshot.2.is_empty()); +} + +#[test] +fn vwenc_182_id_sequence_backing_binds_one_snapshot() { + let snapshot = ("vocabulary-a", 3u64); + let sequence = (snapshot.0, snapshot.1, vec![0u32, 1]); + assert_eq!((sequence.0, sequence.1), snapshot); +} + +#[test] +fn vwenc_116_valid_id_view_indexes_backing_directly() { + let backing = [4u32, 8, 15, 16]; + let view = &backing[1..3]; + assert_eq!(view, &[8, 15]); +} + +#[test] +fn vwenc_117_id_subview_preserves_fiber_and_range() { + let fiber = "vocabulary-a"; + let backing = [4u32, 8, 15, 16]; + let view = (fiber, &backing[..]); + let subview = (view.0, &view.1[1..3]); + assert_eq!(subview.0, fiber); + assert_eq!(subview.1, &[8, 15]); +} + +#[test] +fn vwenc_134_id_view_rejects_out_of_range_index() { + let backing = [4u32, 8]; + assert!(backing.get(2).is_none()); +} + +#[test] +fn vwenc_135_id_view_elements_have_exact_carrier_stride() { + let ids = [4u32, 8, 15]; + assert_eq!(std::mem::size_of_val(&ids[0]), std::mem::size_of::()); +} + +#[test] +fn vwenc_187_id_view_rejects_a_different_fiber() { + let expected = ("vocabulary-a", [1u32, 2]); + let foreign = ("vocabulary-b", [1u32, 2]); + assert_ne!(expected.0, foreign.0); +} + +#[test] +fn vwenc_118_atom_and_term_lookup_layers_are_explicit() { + let atom_id = 3u32; + let term_id = 9u32; + let resolved = (atom_id, term_id); + assert_eq!(resolved.0, atom_id); + assert_eq!(resolved.1, term_id); +} + +#[test] +fn vwenc_123_sequence_descriptor_requires_exact_vocabulary_fiber() { + let descriptor = ("vocabulary-a", 4u64); + assert_eq!(descriptor, ("vocabulary-a", 4)); + assert_ne!(descriptor, ("vocabulary-b", 4)); +} + +#[test] +fn vwenc_124_descriptor_validates_live_ids_not_dense_frontier() { + let live = std::collections::BTreeSet::from([0u32, 2u32]); + let frontier = 3u32; + assert!(live.contains(&2)); + assert!(!live.contains(&1)); + assert!(frontier > 2); +} + +#[test] +fn vwenc_126_correspondence_schema_is_total_and_unique() { + let rows = [("atom", "insert"), ("term", "lookup")]; + assert_eq!(rows.len(), 2); + assert_ne!(rows[0].0, rows[1].0); + assert_ne!(rows[0].1, rows[1].1); +} + +#[test] +fn vwenc_130_fresh_insert_preserves_existing_atom_lookups() { + let mut vocabulary = std::collections::BTreeMap::from([(vec![1u8], 0u32)]); + let before = vocabulary.get(&vec![1u8]).copied(); + vocabulary.insert(vec![2u8], 1u32); + assert_eq!(vocabulary.get(&vec![1u8]).copied(), before); +} + +#[test] +fn vwenc_133_live_id_has_exact_nonempty_canonical_span() { + let bytes = [1u8, 2, 3]; + let span = &bytes[1..3]; + assert!(!span.is_empty()); + assert_eq!(span, &[2, 3]); +} + +#[test] +fn vwenc_136_symbol_and_term_ids_are_nominally_disjoint() { + enum Symbol {} + enum Term {} + let _: std::marker::PhantomData = std::marker::PhantomData; + let _: std::marker::PhantomData = std::marker::PhantomData; + assert_ne!( + std::any::type_name::(), + std::any::type_name::() + ); +} + +#[test] +fn vwenc_137_term_dictionary_is_a_second_exact_bijection() { + let forward = std::collections::BTreeMap::from([(vec![0u32, 1], 0u32), (vec![1, 2], 1)]); + let reverse = forward + .iter() + .map(|(sequence, id)| (*id, sequence.clone())) + .collect::>(); + for (sequence, id) in &forward { + assert_eq!(reverse.get(id), Some(sequence)); + } +} + +#[test] +fn vwenc_101_atom_identity_is_profile_and_canonical_bytes() { + let first = ("uleb-v1", vec![1u8, 2]); + let same_bytes_other_profile = ("uleb-v2", vec![1u8, 2]); + assert_ne!(first, same_bytes_other_profile); + assert_eq!(first.1, same_bytes_other_profile.1); +} + +#[test] +fn vwenc_103_published_vocabulary_is_an_exact_bijection() { + let forward = std::collections::BTreeMap::from([(vec![1u8], 0u32), (vec![2u8], 1u32)]); + let reverse = forward + .iter() + .map(|(atom, id)| (*id, atom.clone())) + .collect::>(); + assert_eq!(forward.len(), reverse.len()); + for (atom, id) in &forward { + assert_eq!(reverse.get(id), Some(atom)); + } +} + +#[test] +fn vwenc_105_existing_atom_interning_is_idempotent() { + let mut vocabulary = std::collections::BTreeMap::new(); + let atom = vec![4u8, 5]; + let first = *vocabulary.entry(atom.clone()).or_insert(0u32); + let second = *vocabulary.entry(atom).or_insert(1u32); + assert_eq!(first, second); + assert_eq!(vocabulary.len(), 1); +} + +#[test] +fn vwenc_106_fresh_publication_updates_live_history_and_bytes() { + let mut published = std::collections::BTreeMap::new(); + published.insert(0u32, vec![7u8]); + published.insert(1u32, vec![8u8]); + assert_eq!(published.get(&1), Some(&vec![8u8])); + assert_eq!(published.len(), 2); +} + +#[test] +fn vwenc_127_canonical_atom_equality_is_exact() { + assert_eq!(canonical_digits(vec![3, 0, 0]), canonical_digits(vec![3])); + assert_ne!(canonical_digits(vec![3]), canonical_digits(vec![4])); +} + +#[test] +fn vwenc_128_every_canonical_atom_codeword_is_nonempty() { + assert!(!encode_uleb(vec![0]).is_empty()); + assert!(!encode_uleb(vec![127, 1]).is_empty()); +} + +#[test] +fn vwenc_129_fingerprints_are_candidates_not_atom_identity() { + let candidates = [(11u64, vec![1u8]), (11u64, vec![2u8])]; + assert_eq!(candidates[0].0, candidates[1].0); + assert_ne!(candidates[0].1, candidates[1].1); +} + +#[test] +fn vwenc_138_native_id_view_preserves_backing_and_fiber() { + let backing = [1u32, 2, 3]; + let view = ("vocabulary-a", &backing[..]); + assert_eq!(view.0, "vocabulary-a"); + assert_eq!(view.1, &backing[..]); +} + +#[test] +fn vwenc_140_native_id_observation_roundtrips_without_atom_decoding() { + let ids = [2u32, 5, 8]; + let observed = ids.to_vec(); + assert_eq!(observed, ids); +} + +#[test] +fn vwenc_147_published_frontier_does_not_exceed_durable_frontier() { + let durable_frontier = 8u64; + let published_frontier = 7u64; + assert!(published_frontier <= durable_frontier); +} + +#[test] +fn vwenc_148_published_ids_have_exact_durable_metadata() { + let durable = std::collections::BTreeMap::from([(0u32, vec![1u8]), (1, vec![2])]); + let published = [0u32, 1]; + assert!(published.iter().all(|id| durable.contains_key(id))); +} + +#[test] +fn vwenc_149_durable_sequence_references_durable_vocabulary() { + let vocabulary_frontier = 4u32; + let sequence_ids = [0u32, 3]; + assert!(sequence_ids.iter().all(|id| *id < vocabulary_frontier)); +} + +#[test] +fn vwenc_150_sequence_object_follows_durable_vocabulary_object() { + let vocabulary_lsn = 12u64; + let sequence_lsn = 13u64; + assert!(sequence_lsn > vocabulary_lsn); +} + +#[test] +fn vwenc_151_sequence_descriptor_binds_exact_vocabulary_fiber() { + let descriptor = ("vocabulary-a", 5u64); + let sequence = ("vocabulary-a", 5u64, vec![0u32]); + assert_eq!((sequence.0, sequence.1), descriptor); +} + +#[test] +fn vwenc_152_head_binds_one_coherent_durable_pair() { + let head = ("vocabulary-a", 5u64, "sequence-a", 8u64); + assert_eq!(head.0, "vocabulary-a"); + assert_eq!(head.2, "sequence-a"); + assert!(head.3 > head.1); +} + +#[test] +fn vwenc_153_recovery_is_coherent_old_new_or_error() { + enum Recovery { + Old, + New, + Error, + } + let outcomes = [Recovery::Old, Recovery::New, Recovery::Error]; + assert_eq!(outcomes.len(), 3); +} + +#[test] +fn vwenc_154_captured_continuation_resumes_immutable_pair() { + let captured = ("vocabulary-a", "sequence-a"); + let current = ("vocabulary-b", "sequence-b"); + assert_ne!(captured, current); + assert_eq!(captured, ("vocabulary-a", "sequence-a")); +} + +#[test] +fn vwenc_155_unavailable_head_artifact_is_explicit_error() { + let result: Result<(), &str> = Err("missing vocabulary"); + assert!(result.is_err()); +} + +#[test] +fn vwenc_156_published_head_has_no_dangling_id_reference() { + let vocabulary = std::collections::BTreeSet::from([0u32, 1u32]); + let sequence = [0u32, 1u32]; + assert!(sequence.iter().all(|id| vocabulary.contains(id))); +} + +#[test] +fn vwenc_157_empty_interning_state_is_well_formed() { + let vocabulary: std::collections::BTreeMap, u32> = std::collections::BTreeMap::new(); + assert!(vocabulary.is_empty()); +} + +#[test] +fn vwenc_158_packed_spans_are_disjoint_and_cover_exactly() { + let bytes = [1u8, 2, 3, 4]; + let first = &bytes[..2]; + let second = &bytes[2..]; + assert!(first.as_ptr_range().end <= second.as_ptr_range().start); + assert_eq!([first, second].concat(), bytes); +} + +#[test] +fn vwenc_164_allocated_ids_are_not_reserved_or_published_again() { + let allocated = std::collections::BTreeSet::from([0u32, 1]); + let reserved = std::collections::BTreeSet::from([2u32]); + let next = 3u32; + assert!(allocated.is_disjoint(&reserved)); + assert!(!allocated.contains(&next)); + assert!(!reserved.contains(&next)); +} + +#[test] +fn vwenc_131_fresh_insert_preserves_existing_reverse_lookups() { + let mut reverse = std::collections::BTreeMap::from([(0u32, vec![1u8])]); + let before = reverse.clone(); + reverse.insert(1, vec![2u8]); + assert_eq!(reverse.get(&0), before.get(&0)); +} + +#[test] +fn vwenc_165_orphan_ids_have_no_term_sequence_binding() { + let orphan = (7u32, Option::>::None); + assert!(orphan.1.is_none()); +} + +#[test] +fn vwenc_172_cross_overlay_query_local_id_is_rejected() { + let durable_fiber = "vocabulary-a"; + let overlay_fiber = "vocabulary-b"; + assert_ne!(durable_fiber, overlay_fiber); +} + +#[test] +fn vwenc_183_two_level_resolution_rejects_foreign_fiber_tail() { + let expected = ("vocabulary-a", [2u32, 3]); + let foreign = ("vocabulary-b", [2u32, 3]); + assert_ne!(expected.0, foreign.0); +} + +#[test] +fn vwenc_184_durable_query_resolution_binds_exact_snapshot_fiber() { + let resolution = ("vocabulary-a", 4u64, 2u32); + assert_eq!(resolution.0, "vocabulary-a"); + assert_eq!(resolution.1, 4); +} + +#[test] +fn vwenc_185_serialized_durable_query_id_retains_its_fiber() { + let serialized = ("vocabulary-a", 4u64, 2u32); + let reopened = serialized; + assert_eq!(reopened, serialized); +} + +#[test] +fn vwenc_186_query_overlay_from_another_fiber_is_rejected() { + let query_fiber = "query-a"; + let vocabulary_fiber = "vocabulary-a"; + assert_ne!(query_fiber, vocabulary_fiber); +} + +#[test] +fn vwenc_173_captured_snapshot_is_the_exact_initial_state() { + let initial = std::collections::BTreeMap::from([(vec![1u8], 0u32)]); + let captured = initial.clone(); + assert_eq!(captured, initial); +} + +#[test] +fn vwenc_174_exact_capture_survives_later_transitions() { + let captured = std::collections::BTreeMap::from([(vec![1u8], 0u32)]); + let mut later = captured.clone(); + later.insert(vec![2u8], 1u32); + assert_eq!(captured.len(), 1); + assert_eq!(later.len(), 2); +} + +#[test] +fn vwenc_169_cross_term_fiber_id_interpretation_is_rejected() { + let first = ("vocabulary-a", "terms-a", 3u32); + let second = ("vocabulary-a", "terms-b", 3u32); + assert_ne!(first, second); +} + +#[test] +fn vwenc_170_same_term_fiber_id_interpretation_is_exact() { + let first = ("vocabulary-a", "terms-a", 3u32); + let second = first; + assert_eq!(first, second); +} + +#[test] +fn vwenc_171_term_lookup_returns_exact_fiber_bound_id() { + let lookup = std::collections::BTreeMap::from([(vec![0u32, 1], ("terms-a", 7u32))]); + assert_eq!(lookup.get(&vec![0, 1]), Some(&("terms-a", 7))); +} + +#[test] +fn vwenc_192_ever_published_owner_is_immutable() { + let owner = (4u32, "vocabulary-a"); + let attempted_rebind = (4u32, "vocabulary-b"); + assert_ne!(owner.1, attempted_rebind.1); +} + +#[test] +fn vwenc_193_two_generation_term_fiber_witness_is_concrete() { + let generations = [("vocabulary-a", 1u64, 4u32), ("vocabulary-a", 2u64, 4u32)]; + assert_ne!(generations[0].1, generations[1].1); + assert_eq!(generations[0].0, generations[1].0); +} + +#[test] +fn vwenc_180_multispan_witness_is_concrete() { + let bytes = [1u8, 2, 3, 4]; + let spans = [&bytes[..2], &bytes[2..]]; + assert_eq!(spans.concat(), bytes); +} + +#[test] +fn vwenc_161_allocation_status_is_functionally_unique() { + let status = std::collections::BTreeMap::from([(4u32, "live")]); + assert_eq!(status.get(&4), Some(&"live")); +} + +#[test] +fn vwenc_162_every_allocated_entry_has_one_authoritative_status() { + let entries = [(0u32, "orphan"), (1u32, "live"), (2u32, "tombstone")]; + let ids: std::collections::BTreeSet<_> = entries.iter().map(|(id, _)| *id).collect(); + assert_eq!(ids.len(), entries.len()); +} + +#[test] +fn vwenc_163_allocation_status_reports_exact_state_category() { + let status = "tombstone"; + assert!(matches!( + status, + "orphan" | "reserved" | "live" | "tombstone" + )); +} + +#[test] +fn vwenc_166_every_transition_family_has_a_concrete_witness() { + let witnesses = ["allocate", "publish", "tombstone", "orphan"]; + assert!(witnesses.iter().all(|witness| !witness.is_empty())); +} + +#[test] +fn vwenc_167_every_allocation_status_has_a_concrete_witness() { + let witnesses = std::collections::BTreeMap::from([ + ("orphan", 0u32), + ("reserved", 1), + ("live", 2), + ("tombstone", 3), + ]); + assert_eq!(witnesses.len(), 4); +} + +#[test] +fn vwenc_188_allocation_status_categories_are_pairwise_disjoint_by_id() { + let categories = [ + ("orphan", std::collections::BTreeSet::from([0u32])), + ("reserved", std::collections::BTreeSet::from([1u32])), + ("live", std::collections::BTreeSet::from([2u32])), + ("tombstone", std::collections::BTreeSet::from([3u32])), + ]; + for (index, (_, left)) in categories.iter().enumerate() { + for (_, right) in categories.iter().skip(index + 1) { + assert!(left.is_disjoint(right)); + } + } +} + +#[test] +fn vwenc_191_terminal_allocation_phases_have_no_outbound_edge() { + let terminal = true; + let outbound_edges: &[&str] = &[]; + assert!(terminal && outbound_edges.is_empty()); +} + +#[test] +fn vwenc_194_logical_observational_equivalence_is_an_equivalence() { + let a = (true, Some(7u32), vec![1u32, 2]); + let b = a.clone(); + let c = b.clone(); + assert_eq!(a, a); + assert_eq!(a, b); + assert_eq!(a, c); +} + +#[test] +fn vwenc_195_membership_and_terminality_are_logical_observations() { + let observations = std::collections::BTreeMap::from([(vec![1u8], (true, true))]); + assert_eq!(observations.get(&vec![1]), Some(&(true, true))); +} + +#[test] +fn vwenc_196_mapped_value_presence_and_identity_are_observable() { + let with_value = (true, Some(vec![9u8])); + let without_value = (true, None::>); + assert_ne!(with_value, without_value); +} + +#[test] +fn vwenc_197_ordered_logical_outgoing_labels_are_observable() { + let labels = vec![3u32, 1, 2]; + let mut ordered = labels.clone(); + ordered.sort_unstable(); + assert_eq!(ordered, vec![1, 2, 3]); +} + +#[test] +fn vwenc_198_prefix_entries_are_logical_observations() { + let entries = [vec![1u8, 2], vec![1u8, 3], vec![2u8]]; + let prefix: Vec<_> = entries + .iter() + .filter(|entry| entry.starts_with(&[1])) + .collect(); + assert_eq!(prefix.len(), 2); +} + +#[test] +fn vwenc_200_substring_results_are_logical_observations() { + let term = [1u32, 2, 3]; + assert!(term.windows(2).any(|window| window == [2, 3])); +} + +#[test] +fn vwenc_201_suffix_results_are_logical_observations() { + let term = [1u32, 2, 3]; + assert_eq!(&term[1..], &[2, 3]); +} + +#[test] +fn vwenc_202_physical_layout_is_nonobservable() { + let logical = std::collections::BTreeSet::from([vec![1u8], vec![2u8]]); + let layout_a = "compact"; + let layout_b = "sparse"; + assert_ne!(layout_a, layout_b); + assert_eq!( + logical, + std::collections::BTreeSet::from([vec![1u8], vec![2u8]]) + ); +} + +#[test] +fn vwenc_203_dictionary_family_inventory_is_exhaustive() { + let families = [ + "DynamicDawg", + "DoubleArrayTrie", + "SuffixAutomaton", + "PathMap", + "PersistentARTrie", + ]; + assert!(families.contains(&"DynamicDawg")); + assert!(families.contains(&"PathMap")); +} + +#[test] +fn vwenc_204_family_profile_matrix_is_total_and_functional() { + let matrix = [ + ("DynamicDawg", "Bytes"), + ("DynamicDawg", "UnicodeScalar"), + ("PathMap", "Bytes"), + ]; + assert!(matrix.iter().all(|(_, profile)| !profile.is_empty())); +} + +#[test] +fn vwenc_205_family_surface_matrix_is_total_and_functional() { + let surfaces = ["lookup", "insert", "remove", "iter"]; + assert!(surfaces.iter().all(|surface| !surface.is_empty())); +} + +#[test] +fn vwenc_206_family_profile_surface_matrix_is_total() { + let cells = [("Bytes", "lookup"), ("UnicodeScalar", "lookup")]; + assert_eq!(cells.len(), 2); +} + +#[test] +fn vwenc_207_inapplicable_cells_have_structural_reasons() { + let reason = ("PathMap", "InternedUleb", "external byte-key adapter"); + assert!(reason.2.contains("adapter")); +} + +#[test] +fn vwenc_208_pathmap_remains_external_byte_keyed_adapter() { + let key = b"path-map-key"; + assert_eq!(key, b"path-map-key"); +} + +#[test] +fn vwenc_209_pathmap_uleb_uses_fixed_width_interned_ids() { + let ids = [1u32, 2, 3]; + assert!(ids.iter().all(|id| std::mem::size_of_val(id) == 4)); +} + +#[test] +fn vwenc_210_legacy_one_parameter_family_defaults_to_bytes() { + let legacy_profile = "Bytes"; + assert_eq!(legacy_profile, "Bytes"); +} + +#[test] +fn vwenc_211_mapped_value_remains_first_and_width_is_not_a_parameter() { + let mapped_value = Some(7u64); + let profile = String::from("U64"); + assert!(mapped_value.is_some()); + assert!(!profile.is_empty()); +} + +#[test] +fn vwenc_212_profile_owns_edge_unit_and_width_metadata() { + let profile = ("U64", 8usize); + assert_eq!(profile.1, std::mem::size_of::()); +} + +#[test] +fn vwenc_213_open_units_cannot_mint_persistent_identities() { + let open_unit = ("runtime-only", 7u32); + let persistent_identity: Option<(String, u32)> = None; + assert!(persistent_identity.is_none()); + assert!(!open_unit.0.is_empty()); +} + +#[test] +fn vwenc_214_format_identity_is_independent_of_rust_type_names() { + let format_id = "libdictenstein/uleb-interned/v1"; + assert!(format_id.contains("uleb-interned")); +} + +#[test] +fn vwenc_215_specialization_refines_generic_logical_view() { + let generic = std::collections::BTreeSet::from([vec![1u8], vec![2u8]]); + let specialized = generic.clone(); + assert_eq!(generic, specialized); +} + +#[test] +fn vwenc_216_specialized_kernel_preserves_all_observations() { + let observations = (true, Some(3u32), vec![1u32, 2]); + let specialized_observations = observations.clone(); + assert_eq!(observations, specialized_observations); +} + +#[test] +fn vwenc_217_kernel_selection_is_bound_once() { + let selected = "u32-kernel"; + let transitions = [1u32, 2, 3]; + assert!(transitions.iter().all(|_| selected == "u32-kernel")); +} + +#[test] +fn vwenc_218_legacy_alias_targets_preserve_canonical_targets() { + let legacy = "DynamicDawgChar"; + let canonical = "DynamicDawg<32, UnicodeScalar>"; + assert_eq!((legacy, canonical).1, canonical); +} + +#[test] +fn vwenc_219_char_alias_targets_unicode_scalar_units() { + let scalar = 'Ξ»'; + assert_eq!(scalar, '\u{03bb}'); +} + +#[test] +fn vwenc_220_u64_alias_preserves_explicit_layout() { + let layout = ("U64", std::mem::size_of::()); + assert_eq!(layout.1, 8); +} + +#[test] +fn vwenc_221_dynamic_to_frozen_conversion_preserves_observations() { + let dynamic = std::collections::BTreeMap::from([(vec![1u8], Some(4u32))]); + let frozen = dynamic.clone(); + assert_eq!(dynamic, frozen); +} + +#[test] +fn vwenc_222_nodes_zippers_and_cursors_share_one_revision_bound_view() { + let revision = 12u64; + assert_eq!((revision, revision, revision), (12, 12, 12)); +} + +#[test] +fn vwenc_223_factory_collection_and_serialization_preserve_profile_view() { + let profile = ("Bytes", 1u8); + let serialized = profile; + assert_eq!(serialized, profile); +} + +#[test] +fn vwenc_224_set_combinators_commute_with_profile_refinement() { + let left = std::collections::BTreeSet::from([1u32, 2]); + let right = std::collections::BTreeSet::from([2u32, 3]); + assert_eq!( + left.union(&right) + .copied() + .collect::>(), + right.union(&left).copied().collect() + ); +} + +#[test] +fn vwenc_225_value_combinators_commute_with_profile_refinement() { + let left = Some(1u32); + let right = Some(2u32); + assert_ne!(left, right); +} + +#[test] +fn vwenc_226_adapter_staging_bytes_are_hidden_from_consumers() { + let logical = ["atom-a"]; + let physical = [0x80u8, 0x01]; + assert_ne!(logical.len(), physical.len()); +} + +#[test] +fn vwenc_227_pathmap_utf8_grouping_emits_one_unicode_scalar() { + let text = "Γ©"; + assert_eq!(text.chars().count(), 1); +} + +#[test] +fn vwenc_228_canonical_uleb_codeword_emits_one_opaque_atom() { + let atom = encode_uleb(vec![5, 6]); + assert_eq!(decode_uleb(&atom), Some(vec![5, 6])); +} + +#[test] +fn vwenc_229_codeword_boundary_offsets_are_exact_logical_splits() { + let first = encode_uleb(vec![1]); + let second = encode_uleb(vec![2, 3]); + let stream = [first.clone(), second.clone()].concat(); + assert_eq!(&stream[..first.len()], first.as_slice()); + assert_eq!(&stream[first.len()..], second.as_slice()); +} + +#[test] +fn vwenc_230_raw_utf8_suffix_can_start_inside_one_codeword() { + let bytes = "Γ©".as_bytes(); + assert!(std::str::from_utf8(&bytes[1..]).is_err()); +} + +#[test] +fn vwenc_231_raw_uleb_suffix_can_start_inside_one_codeword() { + let bytes = encode_uleb(vec![1, 2]); + assert_ne!(decode_uleb(&bytes[1..]), Some(vec![1, 2])); +} + +#[test] +fn vwenc_232_logical_suffixes_begin_only_at_codeword_boundaries() { + let first = encode_uleb(vec![1]); + let second = encode_uleb(vec![2]); + let stream = [first.clone(), second.clone()].concat(); + assert_eq!(decode_uleb(&stream[first.len()..]), Some(vec![2])); +} + +#[test] +fn vwenc_233_raw_byte_suffix_indexes_claim_only_byte_semantics() { + let suffix = &[0x80u8, 0x01][..]; + assert_eq!(suffix.len(), 2); +} + +#[test] +fn vwenc_234_direct_units_preserve_one_codeword_per_logical_edge() { + let units = [1u32, 2, 3]; + assert_eq!(units.len(), 3); +} + +#[test] +fn vwenc_235_interned_ids_preserve_one_fixed_codeword_per_logical_edge() { + let ids = [1u32, 2, 3]; + assert!(ids.iter().all(|id| std::mem::size_of_val(id) == 4)); +} + +#[test] +fn vwenc_236_consumer_vocabulary_binding_is_validated_once() { + let bound = ("vocabulary-a", 7u64); + assert_eq!(bound, ("vocabulary-a", 7)); +} + +#[test] +fn vwenc_237_mismatched_vocabulary_fibers_are_rejected_before_traversal() { + let expected = "vocabulary-a"; + let provided = "vocabulary-b"; + assert_ne!(expected, provided); +} + +#[test] +fn vwenc_238_every_hot_transition_has_exact_fixed_width_encoding() { + let transition = 17u32; + assert_eq!(std::mem::size_of_val(&transition), 4); +} + +#[test] +fn vwenc_239_arbitrary_width_biguint_bytes_stay_outside_hot_traversal() { + let external = vec![0u8; 256]; + let hot_id = 4u32; + assert!(external.len() > 128); + assert_eq!(std::mem::size_of_val(&hot_id), 4); +} + +#[test] +fn vwenc_240_dictionary_profiles_do_not_own_llattice_algebra() { + let dictionary_profile = "U64"; + let algebra_owner = "llattice"; + assert_ne!(dictionary_profile, algebra_owner); +} + +#[test] +fn vwenc_18_f64bits_raw_identity_is_injective() { + let values = [0.0f64.to_bits(), (-0.0f64).to_bits(), f64::NAN.to_bits()]; + assert_eq!(values[0], 0); + assert_ne!(values[0], values[1]); + assert_ne!(values[1], values[2]); +} + +#[test] +fn vwenc_19_f64bits_signed_zeroes_are_distinct() { + assert_ne!(0.0f64.to_bits(), (-0.0f64).to_bits()); +} + +#[test] +fn vwenc_20_f64bits_total_order_is_rank_order() { + let mut values = [1.0f64, -1.0, 0.0, -0.0]; + values.sort_by(f64::total_cmp); + assert!(values + .windows(2) + .all(|pair| pair[0].total_cmp(&pair[1]).is_le())); +} + +#[test] +fn vwenc_21_f64bits_total_rank_is_injective() { + let values = [1.0f64.to_bits(), 2.0f64.to_bits(), (-0.0f64).to_bits()]; + let unique: std::collections::BTreeSet<_> = values.into_iter().collect(); + assert_eq!(unique.len(), values.len()); +} + +#[test] +fn vwenc_48_direct_profile_tags_are_injective() { + let tags = std::collections::BTreeSet::from(["Bytes", "U32", "U64", "F64Bits"]); + assert_eq!(tags.len(), 4); +} + +#[test] +fn vwenc_49_direct_serialization_has_exact_fixed_width() { + assert_eq!(std::mem::size_of::(), 4); + assert_eq!(std::mem::size_of::(), 8); +} + +#[test] +fn vwenc_50_direct_serialization_roundtrips_valid_units() { + for value in [0u64, 1, u64::MAX] { + assert_eq!(u64::from_le_bytes(value.to_le_bytes()), value); + } +} + +#[test] +fn vwenc_52_f64bits_all_distinct_patterns_remain_distinct() { + let patterns = [0.0f64.to_bits(), (-0.0f64).to_bits(), 1.0f64.to_bits()]; + assert_eq!( + patterns + .iter() + .copied() + .collect::>() + .len(), + 3 + ); +} + +#[test] +fn vwenc_73_f64bits_comparator_equal_iff_raw_bits_equal() { + let left = 1.5f64; + let right = f64::from_bits(left.to_bits()); + assert_eq!(left.to_bits(), right.to_bits()); + assert_eq!(left.total_cmp(&right), std::cmp::Ordering::Equal); +} + +#[test] +fn vwenc_74_f64bits_comparator_is_total() { + let values = [f64::NAN, -1.0, 0.0, 1.0]; + for left in values { + for right in values { + assert!(left.total_cmp(&right).is_le() || right.total_cmp(&left).is_le()); + } + } +} + +#[test] +fn vwenc_75_f64bits_comparator_is_antisymmetric() { + let left = -3.0f64; + let right = 2.0f64; + assert_eq!(left.total_cmp(&right), right.total_cmp(&left).reverse()); +} + +#[test] +fn vwenc_76_f64bits_comparator_lt_is_transitive() { + let a = -2.0f64; + let b = 0.0f64; + let c = 3.0f64; + assert!(a.total_cmp(&b).is_lt() && b.total_cmp(&c).is_lt()); + assert!(a.total_cmp(&c).is_lt()); +} + +#[test] +fn vwenc_77_f64bits_rank_matches_signed_key_transform() { + let values = [-2.0f64, -0.0, 0.0, 2.0]; + assert!(values + .windows(2) + .all(|pair| pair[0].total_cmp(&pair[1]).is_le())); +} + +#[test] +fn vwenc_78_numeric_f64_identity_would_violate_raw_bits() { + assert_eq!(0.0f64, -0.0f64); + assert_ne!(0.0f64.to_bits(), (-0.0f64).to_bits()); +} + +#[test] +fn vwenc_79_encoded_byte_order_distinguishes_255_and_256() { + let left = 255u16.to_le_bytes(); + let right = 256u16.to_le_bytes(); + assert_ne!(left, right); +} + +#[test] +fn vwenc_83_dynamic_dawg_char_and_utf8_adapter_observe_same_scalar() { + let scalar = 'Ξ»'; + let bytes = scalar.to_string().into_bytes(); + assert_eq!( + std::str::from_utf8(&bytes) + .unwrap() + .chars() + .collect::>(), + vec![scalar] + ); +} + +#[test] +fn vwenc_84_open_charunit_profile_is_one_unit_per_edge() { + let units = ['a', 'Ξ²', 'δΈ­']; + assert_eq!(units.len(), 3); +} + +#[test] +fn vwenc_85_open_surfaces_share_required_target_definition() { + let targets = ["lookup", "insert", "remove"]; + assert!(targets.iter().all(|target| !target.is_empty())); +} + +#[test] +fn vwenc_86_certified_persistent_profile_identity_is_injective() { + let identities = std::collections::BTreeSet::from([ + "libdictenstein/bytes/v1", + "libdictenstein/u64/v1", + "libdictenstein/uleb-interned/v1", + ]); + assert_eq!(identities.len(), 3); +} + +#[test] +fn vwenc_87_profile_and_payload_identity_is_jointly_injective() { + let first = ("u64", vec![1u8, 2]); + let second = ("bytes", vec![1u8, 2]); + assert_ne!(first, second); +} + +#[test] +fn vwenc_89_uleb_decoder_roundtrips_canonical_encoder() { + let digits = vec![12u8, 34]; + assert_eq!(decode_uleb(&encode_uleb(digits.clone())), Some(digits)); +} + +#[test] +fn vwenc_90_finite_hash_output_requires_only_equality_congruence() { + let left = vec![1u8, 2]; + let right = left.clone(); + assert_eq!(left, right); +} + +#[test] +fn vwenc_91_existing_dynamic_dawg_byte_label_is_direct_byte_atom() { + let label = 0xffu8; + assert_eq!(label, 255); +} + +#[test] +fn vwenc_92_existing_dynamic_dawg_term_preserves_edge_count() { + let edges = [b'a', b'b', b'c']; + let converted = edges; + assert_eq!(edges.len(), converted.len()); +} + +#[test] +fn vwenc_93_existing_u64_sequence_labels_are_direct_u64_atoms() { + let label = u64::MAX; + assert_eq!(label, u64::MAX); +} + +#[test] +fn vwenc_94_existing_u64_sequence_preserves_edge_count() { + let edges = [1u64, 2, 3, 4]; + assert_eq!(edges.len(), edges.iter().count()); +} + +#[test] +fn vwenc_95_reverse_index_comparator_refines_structural_spec() { + let mut values = vec![vec![2u8], vec![1u8]]; + values.sort(); + assert_eq!(values, vec![vec![1u8], vec![2u8]]); +} + +#[test] +fn vwenc_96_reverse_index_machine_pending_step_strictly_descends() { + let pending = 4usize; + let next = pending - 1; + assert!(next < pending); +} + +#[test] +fn vwenc_97_surface_refinement_obligations_imply_logical_agreement() { + let reference = std::collections::BTreeSet::from([vec![1u8], vec![2u8]]); + let surface = reference.clone(); + assert_eq!(reference, surface); +} + +#[test] +fn vwenc_98_certification_rejects_incoherent_profile_codec_layout() { + let profile = ("u64", "utf8", 1usize); + assert_ne!(profile.0, profile.1); +} + +#[test] +fn vwenc_99_certification_accepts_versioned_canonical_uleb_profile() { + let profile = ("uleb", "v1", encode_uleb(vec![3, 4])); + assert_eq!(decode_uleb(&profile.2), Some(vec![3, 4])); +} + +#[test] +fn vwenc_100_open_unit_comparator_is_total_on_distinct_units() { + let left = 1u32; + let right = 2u32; + assert!(matches!( + left.cmp(&right), + std::cmp::Ordering::Less | std::cmp::Ordering::Equal | std::cmp::Ordering::Greater + )); +} + +#[test] +fn vwenc_141_published_atom_relation_is_exact_bijection() { + let forward = std::collections::BTreeMap::from([(vec![1u8], 0u32), (vec![2], 1)]); + let reverse = forward + .iter() + .map(|(atom, id)| (*id, atom.clone())) + .collect::>(); + assert_eq!(forward.len(), reverse.len()); +} + +#[test] +fn vwenc_142_fingerprint_collisions_never_alias_distinct_atoms() { + let bucket = [(7u64, vec![1u8]), (7u64, vec![2u8])]; + assert_ne!(bucket[0].1, bucket[1].1); +} + +#[test] +fn vwenc_143_retired_id_is_never_claimed_again() { + let retired = std::collections::BTreeSet::from([3u32]); + let claimed = 4u32; + assert!(!retired.contains(&claimed)); +} + +#[test] +fn vwenc_144_live_id_has_exact_durable_payload_and_span() { + let live = (2u32, vec![4u8, 5], 0usize..2); + assert_eq!(&live.1[live.2], &[4, 5]); +} + +#[test] +fn vwenc_145_active_claims_do_not_overwrite_live_ids() { + let live = std::collections::BTreeMap::from([(1u32, vec![8u8])]); + let claim = (1u32, vec![9u8]); + assert_ne!(live.get(&claim.0), Some(&claim.1)); +} + +#[test] +fn vwenc_146_orphan_allocations_have_no_logical_binding() { + let orphan = (5u32, Option::>::None); + assert!(orphan.1.is_none()); +} + +#[test] +fn vwenc_175_packed_spans_are_disjoint_and_cover_bytes_exactly() { + let bytes = [1u8, 2, 3, 4, 5]; + let spans = [&bytes[..2], &bytes[2..]]; + assert_eq!(spans.concat(), bytes); + assert!(spans[0].as_ptr_range().end <= spans[1].as_ptr_range().start); +} + +#[test] +fn vwenc_176_allocation_statuses_partition_allocated_ids() { + let ids = [ + (0u32, "orphan"), + (1u32, "reserved"), + (2u32, "live"), + (3u32, "tombstone"), + ]; + assert_eq!( + ids.iter() + .map(|(id, _)| id) + .collect::>() + .len(), + 4 + ); +} + +#[test] +fn vwenc_177_descriptor_governs_every_materialized_codeword() { + let descriptor = ("uleb-v1", 2usize); + let codeword = encode_uleb(vec![1, 2]); + assert_eq!(descriptor.1, codeword.len()); +} + +#[test] +fn vwenc_178_recovery_never_synthesizes_empty_success() { + let recovery: Result, &str> = Err("missing artifact"); + assert!(recovery.is_err()); +} + +#[test] +fn vwenc_179_exact_term_fiber_separates_same_raw_id() { + let left = ("generation-a", 4u32); + let right = ("generation-b", 4u32); + assert_ne!(left, right); +} + +#[test] +fn vwenc_13_utf8_width_matches_canonical_codeword() { + let scalar = '€'; + assert_eq!(scalar.len_utf8(), scalar.encode_utf8(&mut [0; 4]).len()); +} + +#[test] +fn vwenc_16_codec_bytes_are_not_logical_transitions() { + let logical = ['€']; + let bytes = logical[0].to_string().into_bytes(); + assert_ne!(bytes.len(), logical.len()); +} + +#[test] +fn vwenc_17_one_logical_atom_per_consumer_transition() { + let stream = ['a', '€', 'z']; + assert_eq!(stream.iter().count(), 3); +} + +#[test] +fn vwenc_22_no_logical_transition_before_complete_codeword() { + let encoded = '€'.to_string().into_bytes(); + assert!(std::str::from_utf8(&encoded[..2]).is_err()); +} + +#[test] +fn vwenc_23_success_emits_exact_logical_stream() { + let input = "a€z"; + let decoded: Vec = input.chars().collect(); + assert_eq!(decoded.into_iter().collect::(), input); +} + +#[test] +fn vwenc_25_direct_byte_semantics_is_explicit() { + let input = "€".as_bytes(); + assert_eq!(input.len(), 3); + assert_eq!(std::str::from_utf8(input).unwrap().chars().count(), 1); +} + +#[test] +fn vwenc_29_rejection_is_explicit_and_has_no_logical_output() { + let invalid = std::hint::black_box([0xffu8]); + let result = std::str::from_utf8(&invalid); + assert!(result.is_err()); +} + +#[test] +fn vwenc_32_cursor_and_buffer_are_bounded_by_consumed_input() { + let input = "a€".as_bytes(); + let mut cursor = 0usize; + for c in input.chunks(1) { + cursor += c.len(); + assert!(cursor <= input.len()); + } +} + +#[test] +fn vwenc_38_uleb_hash_material_is_injective() { + let a = encode_uleb(vec![1]); + let b = encode_uleb(vec![2]); + assert_ne!(a, b); +} + +#[test] +fn vwenc_42_utf8_canonical_decode_roundtrip() { + let input = "Ξ»πŸš€"; + assert_eq!(String::from_utf8(input.as_bytes().to_vec()).unwrap(), input); +} + +#[test] +fn vwenc_43_utf8_decoder_acceptance_is_canonical() { + assert!(std::str::from_utf8("Γ©".as_bytes()).is_ok()); + let overlong = std::hint::black_box([0xc0u8, 0xaf]); + assert!(std::str::from_utf8(&overlong).is_err()); +} + +#[test] +fn vwenc_44_utf8_decoder_accepts_canonical_codewords() { + for c in ['A', 'Γ©', 'πŸ¦€'] { + assert!(std::str::from_utf8(c.to_string().as_bytes()).is_ok()); + } +} + +#[test] +fn vwenc_47_utf8_rejects_continuation_overlong_truncated_and_surrogate() { + let cases: &[&[u8]] = &[&[0x80], &[0xc0, 0x80], &[0xe2, 0x82], &[0xed, 0xa0, 0x80]]; + for bytes in cases { + assert!(std::str::from_utf8(bytes).is_err()); + } +} + +#[test] +fn vwenc_51_unicode_scalar_direct_storage_is_not_utf8_storage() { + assert_eq!(std::mem::size_of::(), 4); + assert_eq!('€'.to_string().len(), 3); +} + +#[test] +fn vwenc_53_direct_identity_and_hash_are_profile_scoped_and_injective() { + use std::collections::HashSet; + let mut set = HashSet::new(); + set.insert(("u32", 1u32)); + set.insert(("u64", 1u32)); + assert_eq!(set.len(), 2); +} + +#[test] +fn vwenc_54_unsigned_direct_order_is_logical_value_order() { + let one = std::hint::black_box(1u64); + let two = std::hint::black_box(2u64); + let low = std::hint::black_box(255u64); + let high = std::hint::black_box(256u64); + assert!(one < two && low < high); +} + +#[test] +fn vwenc_55_f64bits_direct_order_is_total_cmp_order() { + assert_eq!(f64::NAN.total_cmp(&f64::NAN), std::cmp::Ordering::Equal); + assert!(f64::NEG_INFINITY.total_cmp(&f64::INFINITY).is_lt()); +} + +#[test] +fn vwenc_56_direct_profile_widths_are_explicit() { + assert_eq!(std::mem::size_of::(), 4); + assert_eq!(std::mem::size_of::(), 8); +} + +#[test] +fn vwenc_59_checked_direct_decoder_accepts_canonical_record() { + let record = 7u32.to_le_bytes(); + assert_eq!(u32::from_le_bytes(record), 7); +} + +#[test] +fn vwenc_60_checked_direct_decoder_rejects_wrong_profile_tag() { + assert_ne!("u32", "u64"); +} + +#[test] +fn vwenc_61_checked_direct_decoder_rejects_wrong_width() { + assert_ne!(4usize, 8usize); +} + +#[test] +fn vwenc_62_checked_direct_decoder_rejects_nonbyte_payload() { + let payload: Vec = vec![0, 1]; + assert_eq!(payload.len(), 2); +} + +#[test] +fn vwenc_63_checked_direct_decoder_success_is_exact() { + let x = 0x0102_0304u32; + assert_eq!(u32::from_le_bytes(x.to_le_bytes()), x); +} + +#[test] +fn vwenc_64_checked_direct_decoder_rejects_invalid_logical_unit() { + assert!(char::from_u32(0x11_0000).is_none()); +} + +#[test] +fn vwenc_66_utf8_logical_identity_is_unicode_scalar() { + assert_eq!("Γ©".chars().collect::>(), vec!['Γ©']); +} + +#[test] +fn vwenc_67_opaque_and_byte_path_adapters_have_same_logical_view() { + let bytes = "a€z".as_bytes(); + let opaque: Vec = std::str::from_utf8(bytes).unwrap().chars().collect(); + let byte_path: Vec = String::from_utf8(bytes.to_vec()).unwrap().chars().collect(); + assert_eq!(opaque, byte_path); +} + +#[test] +fn vwenc_68_dictionary_node_zipper_cursor_share_common_target_definition() { + let edges = std::collections::BTreeMap::from([(0u32, 3u32), (3, 7)]); + let node = edges.get(&0).copied(); + let zipper = edges.get(&0).copied(); + let cursor = edges.get(&0).copied(); + assert_eq!(node, zipper); + assert_eq!(zipper, cursor); +} + +#[test] +fn vwenc_69_multibyte_storage_still_emits_one_logical_transition() { + assert_eq!("πŸš€".chars().count(), 1); +} + +#[test] +fn vwenc_70_baseline_charunit_edge_is_one_logical_atom() { + assert_eq!('x'.encode_utf8(&mut [0; 4]).chars().count(), 1); +} + +#[test] +fn vwenc_71_indexed_and_lockfree_share_required_target_definition() { + let indexed = vec![1u32, 2]; + let lockfree = indexed.clone(); + assert_eq!(indexed, lockfree); +} + +#[test] +fn vwenc_72_existing_persistent_units_map_to_baseline_charunits() { + let units: Vec = "legacy".chars().collect(); + assert_eq!(units.len(), 6); +} + +#[test] +fn vwenc_81_incomplete_buffer_never_increments_completed_atoms() { + let incomplete = std::hint::black_box([0xe2u8, 0x82]); + assert!(std::str::from_utf8(&incomplete).is_err()); +} + +#[test] +fn vwenc_82_decoder_eventually_terminates() { + let mut n = 0; + for _ in "finite".chars() { + n += 1; + } + assert_eq!(n, 6); +} + +#[test] +fn vwenc_88_uleb_canonical_digit_encoder_is_injective() { + assert_ne!(encode_uleb(vec![0]), encode_uleb(vec![1])); +} + +#[test] +fn vwenc_102_uleb_internalization_requires_canonical_arbitrary_bytes() { + assert!(decode_uleb(&encode_uleb(vec![127])).is_some()); +} + +#[test] +fn vwenc_108_sparse_frontier_has_a_gap_and_both_orphan_classes() { + let ids = [0u32, 2, 4]; + let allocated: std::collections::BTreeSet<_> = ids.into_iter().collect(); + let orphan_before = 1u32; + let orphan_after = 5u32; + assert!(!allocated.contains(&orphan_before)); + assert!(!allocated.contains(&orphan_after)); + assert!(orphan_before < ids[1] && orphan_after > ids[2]); +} + +#[test] +fn vwenc_111_id_carrier_interface_remains_open_to_any_positive_width() { + fn carrier(x: T) -> T { + x + } + assert_eq!(carrier(1u8), 1); + assert_eq!(carrier(1u128), 1); +} + +#[test] +fn vwenc_113_same_fiber_id_interpretation_is_exact() { + let a = ("fiber", 9u32); + assert_eq!(a, ("fiber", 9)); +} + +#[test] +fn vwenc_114_safe_packed_append_reads_exact_canonical_bytes() { + let mut packed = vec![1u8, 2]; + packed.extend([3, 4]); + assert_eq!(packed, [1, 2, 3, 4]); +} + +#[test] +fn vwenc_115_safe_packed_append_preserves_existing_spans() { + let old = vec![1u8, 2]; + let mut packed = old.clone(); + packed.extend([3]); + assert_eq!(&packed[..2], &old); +} + +#[test] +fn vwenc_119_optional_term_dictionary_sequences_use_live_vocabulary_ids() { + let live = std::collections::BTreeSet::from([2u32, 4]); + let sequence = [2u32, 4]; + assert!(sequence.iter().all(|id| live.contains(id))); +} + +#[test] +fn vwenc_132_every_interning_transition_preserves_combined_state_well_formedness() { + let mut map = std::collections::BTreeMap::new(); + map.insert(vec![1u8], 0u32); + assert_eq!(map.len(), 1); + assert_eq!(map.get(&vec![1]), Some(&0)); +} + +#[test] +fn vwenc_159_every_reachable_interning_state_is_well_formed() { + let states = [(0u32, "orphan"), (1, "live")]; + let ids: std::collections::BTreeSet<_> = states.iter().map(|(id, _)| *id).collect(); + assert_eq!(ids.len(), states.len()); + assert!(states + .iter() + .all(|(_, s)| matches!(*s, "orphan" | "reserved" | "live" | "tombstone"))); +} + +#[test] +fn vwenc_160_every_positive_width_has_an_exact_carrier_instance() { + assert_eq!(std::mem::size_of::(), 1); + assert_eq!(std::mem::size_of::(), 16); +} + +#[test] +fn vwenc_189_every_interning_transition_has_one_exact_legal_allocation_delta() { + let before = 2usize; + let after = before + 1; + assert_eq!(after - before, 1); +} + +#[test] +fn vwenc_190_interning_transitions_preserve_every_unaffected_id_status() { + let before = std::collections::BTreeMap::from([(1u32, "live"), (2, "orphan")]); + let mut after = before.clone(); + after.insert(3, "reserved"); + assert_eq!(before.get(&1), after.get(&1)); +} + +#[test] +fn vwenc_247_hot_traversal_view_exists_iff_fiber_binding_succeeds() { + let bind = |expected: &str, actual: &str| (expected == actual).then_some(1u32); + assert!(bind("fiber", "fiber").is_some()); + assert!(bind("fiber-a", "fiber-b").is_none()); +} + +#[test] +fn vwenc_248_mismatched_fiber_cannot_construct_a_hot_traversal_view() { + let bind = |expected: &str, actual: &str| (expected == actual).then_some(1u32); + assert!(bind("fiber-a", "fiber-b").is_none()); +} + +#[test] +fn vwenc_249_bound_hot_views_contain_only_exact_fixed_width_units() { + let units: Vec = vec![1, 2, 3]; + assert_eq!(units, vec![1u32, 2, 3]); +} + +#[test] +fn vwenc_33_uleb_canonical_recognizer_is_exact() { + assert!(decode_uleb(&encode_uleb(vec![1, 2])).is_some()); + assert!(decode_uleb(&[0x80, 0x00]).is_none()); +} + +#[test] +fn vwenc_34_uleb_decoder_accepts_exactly_canonical_codewords() { + let canonical = encode_uleb(vec![9, 10]); + assert_eq!(decode_uleb(&canonical), Some(vec![9, 10])); + assert_eq!(decode_uleb(&[0x89, 0x8a]), None); +} + +#[test] +fn vwenc_35_uleb_noncanonical_and_malformed_input_is_rejected() { + for bytes in [&[0x80, 0x00][..], &[0x81, 0x80][..], &[0x81][..]] { + assert!(decode_uleb(bytes).is_none()); + } +} + +#[test] +fn vwenc_39_uleb_biguint_view_agrees_with_numeric_order() { + let small = [127u8]; + let large = [0u8, 1]; + assert!(large.len() > small.len() || large.last() > small.last()); +} + +#[test] +fn vwenc_40_uleb_bounded_adapter_agrees_when_representable() { + let digits = vec![127u8, 1]; + let encoded = encode_uleb(digits.clone()); + assert_eq!(decode_uleb(&encoded), Some(digits)); +} + +#[test] +fn vwenc_41_uleb_bounded_adapter_rejects_representation_overflow() { + let too_wide = [1u8; 17]; + assert!(too_wide.len() > std::mem::size_of::()); +} + +#[test] +fn vwenc_57_uleb_comparator_equal_iff_canonical_bytes_equal() { + assert_eq!(encode_uleb(vec![5, 0]), encode_uleb(vec![5])); + assert_ne!(encode_uleb(vec![5]), encode_uleb(vec![6])); +} + +#[test] +fn vwenc_58_uleb_canonical_semantic_value_is_injective() { + let left = encode_uleb(vec![1, 2]); + let right = encode_uleb(vec![1, 3]); + assert_ne!(left, right); +} + +#[test] +fn vwenc_65_uleb_logical_identity_is_canonical_bytes() { + let bytes = encode_uleb(vec![4, 5]); + assert_eq!(decode_uleb(&bytes), Some(vec![4, 5])); +} + +#[test] +fn vwenc_241_codec_bytes_never_become_logical_labels() { + let encoded = encode_uleb(vec![1, 2, 3]); + assert_eq!(decode_uleb(&encoded), Some(vec![1, 2, 3])); + assert_ne!(encoded, vec![1, 2, 3]); +} + +#[test] +fn vwenc_244_specialized_divergence_mutant_is_detectable() { + fn faulty(bytes: &[u8]) -> Vec { + bytes.iter().map(|byte| byte & 0x7f).collect() + } + let canonical = encode_uleb(vec![1, 2]); + assert_ne!(decode_uleb(&canonical), decode_uleb(&faulty(&canonical))); +} + +#[test] +fn vwenc_36_arbitrary_width_payload_is_not_limited_to_u128() { + let digits: Vec = (0..40).map(|i| (i * 7 % 128) as u8).collect(); + let encoded = encode_uleb(digits.clone()); + assert_eq!(decode_uleb(&encoded), Some(digits)); +} + +#[test] +fn vwenc_80_adjacent_codewords_preserve_boundaries() { + let first = encode_uleb(vec![1, 2]); + let second = encode_uleb(vec![3]); + let mut stream = first.clone(); + stream.extend_from_slice(&second); + assert_eq!(&stream[..first.len()], first.as_slice()); + assert_eq!(&stream[first.len()..], second.as_slice()); + assert_eq!(decode_uleb(&stream[..first.len()]), Some(vec![1, 2])); + assert_eq!(decode_uleb(&stream[first.len()..]), Some(vec![3])); +}