Add: support mixed local and remote L4 CommDomains

.github/workflows/ci.yml

@@ -789,6 +789,9 @@ jobs:
     if: false && needs.detect-changes.outputs.docs_only != 'true'
     runs-on: [self-hosted, a5]
     timeout-minutes: 30
+    defaults:
+      run:
+        working-directory: source
     env:
       SIMPLER_SCHEDULER_TIMEOUT_MS: "2000"
       SIMPLER_OP_EXECUTE_TIMEOUT_US: "3000000"
@@ -797,6 +800,11 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v5
+        with:
+          # The legacy workspace root contains an undeletable privileged
+          # .pytest_cache. Use a clean child checkout instead.
+          path: source
+
       - name: Set up environment
         run: |
           python3 -m venv --system-site-packages .venv
@@ -810,7 +818,7 @@ jobs:
         run: |
           source .venv/bin/activate
           DEVICE_LIST=$(python -c "p='${DEVICE_RANGE}'.split('-'); s,e=p[0],p[-1]; print(','.join(str(i) for i in range(int(s),int(e)+1)))")
-          task-submit --timeout 1800 --max-time 1800 --device "$DEVICE_LIST" --run "python -m pytest tests -m requires_hardware --platform a5 --device ${DEVICE_RANGE} -v"
+          task-submit --timeout 1800 --max-time 1800 --device "$DEVICE_LIST" --run "python -m pytest tests -m requires_hardware -p no:cacheprovider --platform a5 --device ${DEVICE_RANGE} -v"
 
       - name: Build and run C++ hardware unit tests (a5)
         run: |
@@ -860,6 +868,9 @@ jobs:
     if: false && needs.detect-changes.outputs.a5_changed == 'true'
     runs-on: [self-hosted, a5]
     timeout-minutes: 60
+    defaults:
+      run:
+        working-directory: source
     env:
       SIMPLER_SCHEDULER_TIMEOUT_MS: "2000"
       SIMPLER_OP_EXECUTE_TIMEOUT_US: "3000000"
@@ -868,6 +879,11 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v5
+        with:
+          # The legacy workspace root contains an undeletable privileged
+          # .pytest_cache. Use a clean child checkout instead.
+          path: source
+
       - name: Set up environment
         run: |
           python3 -m venv --system-site-packages .venv
@@ -881,7 +897,7 @@ jobs:
         run: |
           source .venv/bin/activate
           DEVICE_LIST=$(python -c "p='${DEVICE_RANGE}'.split('-'); s,e=p[0],p[-1]; print(','.join(str(i) for i in range(int(s),int(e)+1)))")
-          PYTEST="python -m pytest examples tests/st --platform a5 --device ${DEVICE_RANGE} -v --require-pto-isa"
+          PYTEST="python -m pytest examples tests/st -p no:cacheprovider --platform a5 --device ${DEVICE_RANGE} -v --require-pto-isa"
           task-submit --timeout 1800 --max-time 1800 --device "$DEVICE_LIST" --run "$PYTEST --pto-session-timeout 1200"
 
       # DFX per-feature smokes — hardware mirror of the st-sim-a5 set. The
@@ -893,7 +909,7 @@ jobs:
           source .venv/bin/activate
           DEVICE_LIST=$(python -c "p='${DEVICE_RANGE}'.split('-'); s,e=p[0],p[-1]; print(','.join(str(i) for i in range(int(s),int(e)+1)))")
           PYTEST="python -m pytest tests/st/a5/tensormap_and_ringbuffer/dfx/dep_gen/test_dep_gen.py \
-            --platform a5 --device ${DEVICE_RANGE} -p no:xdist --pto-session-timeout 600 \
+            --platform a5 --device ${DEVICE_RANGE} -p no:xdist -p no:cacheprovider --pto-session-timeout 600 \
             --require-pto-isa --enable-dep-gen"
           task-submit --timeout 1800 --max-time 1800 --device "$DEVICE_LIST" --run "$PYTEST"
 
@@ -902,7 +918,7 @@ jobs:
           source .venv/bin/activate
           DEVICE_LIST=$(python -c "p='${DEVICE_RANGE}'.split('-'); s,e=p[0],p[-1]; print(','.join(str(i) for i in range(int(s),int(e)+1)))")
           PYTEST="python -m pytest tests/st/a5/tensormap_and_ringbuffer/dfx/l2_swimlane/ \
-            --platform a5 --device ${DEVICE_RANGE} -p no:xdist --pto-session-timeout 600 \
+            --platform a5 --device ${DEVICE_RANGE} -p no:xdist -p no:cacheprovider --pto-session-timeout 600 \
             --require-pto-isa --enable-l2-swimlane --enable-dep-gen"
           task-submit --timeout 1800 --max-time 1800 --device "$DEVICE_LIST" --run "$PYTEST"
 
@@ -911,7 +927,7 @@ jobs:
           source .venv/bin/activate
           DEVICE_LIST=$(python -c "p='${DEVICE_RANGE}'.split('-'); s,e=p[0],p[-1]; print(','.join(str(i) for i in range(int(s),int(e)+1)))")
           PYTEST="python -m pytest tests/st/a5/tensormap_and_ringbuffer/dfx/pmu/test_pmu.py \
-            --platform a5 --device ${DEVICE_RANGE} -p no:xdist --pto-session-timeout 600 \
+            --platform a5 --device ${DEVICE_RANGE} -p no:xdist -p no:cacheprovider --pto-session-timeout 600 \
             --require-pto-isa --enable-pmu 2"
           task-submit --timeout 1800 --max-time 1800 --device "$DEVICE_LIST" --run "$PYTEST"
 
@@ -920,6 +936,6 @@ jobs:
           source .venv/bin/activate
           DEVICE_LIST=$(python -c "p='${DEVICE_RANGE}'.split('-'); s,e=p[0],p[-1]; print(','.join(str(i) for i in range(int(s),int(e)+1)))")
           PYTEST="python -m pytest tests/st/a5/tensormap_and_ringbuffer/dfx/args_dump/test_args_dump.py \
-            --platform a5 --device ${DEVICE_RANGE} -p no:xdist --pto-session-timeout 600 \
+            --platform a5 --device ${DEVICE_RANGE} -p no:xdist -p no:cacheprovider --pto-session-timeout 600 \
             --require-pto-isa --dump-args"
           task-submit --timeout 1800 --max-time 1800 --device "$DEVICE_LIST" --run "$PYTEST"

docs/comm-domain.md

@@ -48,6 +48,68 @@ allocation: if `sum(b.nbytes) > window_size`, `allocate_domain` raises
 Kernels read peer windows through `device_ctx` (which holds every rank's
 window base, local + imported peer); `buffer_ptrs[name]` is the local slice.
 
+### Global CommDomain across local and remote L3 nodes
+
+An L4 worker can build the same `CommContext` shape across any combination of
+forked local L3 workers (`add_worker`) and TCP-connected L3 workers
+(`add_remote_worker`) without `mpirun`:
+
+```python
+with orch.allocate_global_domain(
+    name="tp",
+    members=[(node0_worker_id, 0), (node1_worker_id, 0)],
+    window_size=4096,
+    buffers=[CommBufferSpec("payload", "uint8", 4096, 4096)],
+) as domain:
+    ...
+```
+
+Each member is `(l3_worker_id, local_l2_worker_id)`. The order defines dense
+domain ranks. A remote node reads `comm_profile` and `global_device_ranks`
+from `RemoteWorkerSpec`; a local L3 reads the same fields from its `Worker`
+configuration. All participating nodes must use the same profile.
+
+Global CommDomain capability follows the backend that the node actually
+loads: a platform ending in `sim` supports the `sim` profile, and a real
+`a2a3` platform supports `a3-fabric-v1`. Real A5 and any other
+platform/profile combination currently reject allocation before `PREPARE`.
+Each local or remote L3 repeats the same check during `COMM_INIT`, so an
+unsupported backend never advertises a usable descriptor capability.
+
+The control flow is:
+
+1. L4 sends `COMM_INIT` with cluster, node, global-device, and domain-rank
+   identities.
+2. Each L3 asks its participating L2 children to create a local window and
+   export a transport descriptor.
+3. L4 validates and assembles one complete rank-ordered descriptor table.
+4. L4 returns that table to every L3, which forwards it to each L2 for import.
+5. L4 commits only after all imports succeed. Any earlier failure sends
+   `ABORT` and releases every prepared local window.
+
+The descriptor reports the backend's actual mapped size. A3 Fabric may align
+the requested size to its VMM granularity; buffer carving and bounds checks
+therefore use the returned mapping size. Handles and device pointers never
+cross the public Python API. Remote orchestration code calls
+`orch.get_global_domain(domain_id)` to obtain only its committed L3-local
+contexts.
+
+`copy_to_global_domain` and `copy_from_global_domain` provide bounded
+control-plane staging and smoke checks. Normal communication still runs in
+L2 kernels through the imported `CommContext`.
+
+By default a live Global CommDomain is swept after the current `Worker.run`
+drains. Set `retain_after_run=True` when a communication kernel writes results
+into the window and a second L4 run must inspect them. The later run should
+call `domain.release()` after copying the results; `Worker.close()` is the
+final safety net.
+
+The remote-only end-to-end A3 smoke is
+`tools/a3_l4_tcp_smoke/global_tload_smoke.py`; the local-plus-remote variant is
+`tools/a3_l4_tcp_smoke/mixed_global_tload_smoke.py`. They do not use `mpirun`
+and pass only when every L2 kernel successfully performs peer `TLOAD` through
+the L4-brokered Fabric descriptor table.
+
 ---
 
 ## 2. Lifetime model

docs/remote-l3-worker-design.md

@@ -67,11 +67,18 @@ Implemented:
   imported-handle scheduling eligibility, and deferred owner free.
 - Registry-scope-aware remote callable manifest/control install for dispatcher
   `PYTHON_IMPORT`, inner `PYTHON_IMPORT`, and inner inline `CHIP_CALLABLE`.
+  Pre-init `ChipCallable` registrations on an L4 worker are serialized into
+  each remote session manifest and installed on that L3's L2 children.
+- A no-`mpirun` A3 TCP smoke that keeps a Global CommDomain across two L4
+  runs, executes peer `TLOAD` from each remote L2, and verifies the reduced
+  values before release.
+- Two-server hardware validation covers L4-brokered peer `TLOAD`, one L2
+  compute followed by cross-machine communication, and two-NPU-per-node
+  remote L3 group compute.
 
 Still pending:
 
 - A2 RoCE, A3 HCCS, and A5 UB HCOMM profiles.
-- Remote `CommDomain` allocation/import and hardware-gated validation.
 - Negotiated `PYTHON_SERIALIZED` remote callable payloads and staged
   `CHIP_CALLABLE` blob adapters.
 
@@ -449,10 +456,9 @@ Session execution rules:
   the current one-`WorkerThread`-per-child local scheduling model and keeps
   ordering, buffer lifetime, and callable visibility simple.
 - State-changing CONTROL frames such as register, unregister, buffer free,
-  copy, export/import, and import release serialize with TASK execution on the
-  ordered command lane. They are not applied concurrently with a running TASK
-  on the same endpoint. Future Remote CommDomain controls follow the same
-  ordering rule when they enter scope.
+  copy, export/import, import release, and Global CommDomain transactions
+  serialize with TASK execution on the ordered command lane. They are not
+  applied concurrently with a running TASK on the same endpoint.
 - Bulk data movement may use a separate data plane, but the state change that
   makes staged bytes, callable payloads, or imported handles visible is ordered
   by the command lane.

docs/remote-l3-worker-design/implementation-record.md

@@ -16,7 +16,7 @@ It is updated as each documented feature is completed and verified.
 | 5 | Versioned remote frame codec | In progress | TASK/COMPLETION/CONTROL_REPLY/HELLO/CONTROL/HEALTH exist; core fuzz/bounds coverage is present, with more exhaustive corpus testing still possible. |
 | 6 | Remote callable registry | In progress | Dispatcher `PYTHON_IMPORT`, inner manifest/control `PYTHON_IMPORT`, and inner manifest/control inline `CHIP_CALLABLE` are implemented; serialized payloads and staged chip blobs remain negotiated extensions. |
 | 7 | Fork-safe simulation session runner | In progress | Daemon/session bootstrap and HELLO READY barrier are implemented for sim transport. |
-| 8 | Remote control-plane parity | In progress | Registry, alloc/free/copy, export/import/release-import controls are implemented for sim; Remote CommDomain controls are reserved/unsupported. |
+| 8 | Remote control-plane parity | In progress | Registry, remote buffers, and Global CommDomain prepare/import/commit/release/copy controls are implemented. |
 | 9 | Remote buffer registry | In progress | Sim owner/imported buffers, TASK materialization, public memory API, opaque handles, slot/import-ref capture, and deferred free/release-import are implemented. |
 | 10 | A2 RoCE HCOMM profile | Pending | Hardware-gated profile. |
 | 11 | A3 HCCS HCOMM profile | Pending | Hardware-gated profile. |
@@ -88,13 +88,43 @@ It is updated as each documented feature is completed and verified.
   Imports use shared-memory backed mappings in the session runner, imported
   handles remain opaque on the parent, and owner frees wait for live imports
   and slot refs to drain.
+- Added L4-brokered Global CommDomain setup without MPI. L2 export
+  descriptors are collected by L3, assembled by L4, returned to every L3/L2
+  for import, and released after the L4 DAG drain by default. Domains created
+  with `retain_after_run=True` remain live for a later run until explicitly
+  released or the Worker closes. Sim shm and A3 Fabric V2 use the same
+  descriptor ABI.
+- Added startup-manifest delivery for pre-registered inner `CHIP_CALLABLE`
+  payloads and `tools/a3_l4_tcp_smoke/global_tload_smoke.py`. The smoke uses
+  TCP daemons for launch/control, executes peer Fabric `TLOAD` in L2 kernels,
+  and verifies results without `mpirun`.
+- Added `tools/remote_l4_npu/remote_l4_npu_smoke.py` as the real-device
+  control/data-path baseline. L4 submits one task to each remote L3, each L3
+  runs a two-NPU local group, and L4 copies the results back for a golden
+  check. Remote buffers now use L3-owned child-visible host buffers whenever
+  the L3 has forked chip children, while childless sim sessions keep the
+  shared-memory fallback.
+- Added `tools/a3_l4_tcp_smoke/compute_then_tload_smoke.py`. One persistent L4
+  session first dispatches a local L2 vector-add through every remote L3, then
+  dispatches a cross-machine peer `TLOAD` over the computed values. The smoke
+  checks both the per-rank compute output and the final communication output.
 - Documented the v1 remote registry target/kind matrix, inner
   `INNER_L3_WORKER` visibility rules, remote `CHIP_CALLABLE` staged/inline
   payload contract, partial-register cleanup outcomes, and health-expiry
   scheduling behavior.
 
 ## Verification
 
+- Global CommDomain codec/validation tests and the Linux two-daemon sim
+  transaction test live in `tests/ut/py/test_global_comm_domain.py`.
+- The A3 two-server hardware commands in
+  `tools/a3_l4_tcp_smoke/README.md` passed for both peer `TLOAD` and
+  compute-then-`TLOAD`; every rank reported zero maximum difference.
+- The two-machine real-NPU compute baseline is documented in
+  `tools/remote_l4_npu/README.md`. It intentionally verifies remote L4
+  dispatch and per-machine L3/L2 compute separately from the cross-machine
+  Global CommDomain smoke. The two-NPU-per-node run passed with zero maximum
+  difference for every output.
 - Python focused sidecar/callable tests:
   `tests/ut/py/test_task_interface.py tests/ut/py/test_callable_identity.py`
   passed with `145 passed`.

docs/remote-l3-worker-design/protocol.md

@@ -296,15 +296,29 @@ Required remote controls:
 - `IMPORT_BUFFER`
 - `RELEASE_IMPORT`
 
-Reserved future controls for Remote CommDomain:
+Required Global CommDomain controls:
 
 - `COMM_INIT`
 - `ALLOC_DOMAIN`
 - `RELEASE_DOMAIN`
-
-The first Remote L3 task-dispatch cut rejects the reserved domain controls
-with an unsupported-control reply. They become required only when Remote
-CommDomain enters scope.
+- `COPY_TO_DOMAIN`
+- `COPY_FROM_DOMAIN`
+
+`COMM_INIT` validates the cluster id, node identity, communication profile,
+global device ranks, and dense domain-rank table. `ALLOC_DOMAIN` is a
+transaction with `PREPARE_EXPORT`, `IMPORT`, `COMMIT`, and `ABORT` phases.
+Each L2 exports its local transport descriptor during prepare. L4 assembles
+the complete rank-ordered table and sends it to every L3; each L3 forwards it
+to its L2 children for import. No domain becomes visible to a remote task
+before every node acknowledges `COMMIT`.
+
+`RELEASE_DOMAIN` is idempotent and normally runs after the L4 DAG drain.
+An allocation marked `retain_after_run` may remain live for a later L4 run
+that reads kernel results; explicit release or session shutdown then performs
+the same teardown.
+`COPY_TO_DOMAIN` and `COPY_FROM_DOMAIN` are bounded smoke/control data
+operations for a committed local window. They do not replace kernel data
+movement through `CommContext`.
 
 The register-family controls are registry-scope-aware.
 `PREPARE_REGISTER_CALLABLE` carries:

python/bindings/CMakeLists.txt

@@ -61,6 +61,7 @@ target_include_directories(_task_interface PRIVATE
     ${CMAKE_SOURCE_DIR}/src/common/task_interface
     ${CMAKE_SOURCE_DIR}/src/common/worker
     ${CMAKE_SOURCE_DIR}/src/common/hierarchical
+    ${CMAKE_SOURCE_DIR}/src/common/platform/include
     ${CMAKE_SOURCE_DIR}/src/common/platform/include/common
     ${CMAKE_SOURCE_DIR}/src/common/platform/include/host
     ${CMAKE_CURRENT_SOURCE_DIR}

python/bindings/task_interface.cpp

@@ -1530,6 +1530,36 @@ NB_MODULE(_task_interface, m) {
             nb::arg("allocation_id"), nb::arg("rank_count"), nb::arg("domain_rank"),
             "Pair to comm_alloc_domain_windows: collectively release the per-rank pool."
         )
+        .def(
+            "comm_global_domain_prepare",
+            [](ChipWorker &self, uint64_t domain_id, uint32_t domain_rank, uint32_t rank_count, size_t window_size,
+               uint32_t profile) {
+                auto [descriptor, local_window_base, actual_window_size] =
+                    self.comm_global_domain_prepare(domain_id, domain_rank, rank_count, window_size, profile);
+                return nb::make_tuple(
+                    nb::bytes(reinterpret_cast<const char *>(descriptor.data()), descriptor.size()), local_window_base,
+                    actual_window_size
+                );
+            },
+            nb::arg("domain_id"), nb::arg("domain_rank"), nb::arg("rank_count"), nb::arg("window_size"),
+            nb::arg("profile"), "Create a Global CommDomain local window and return its transport descriptor."
+        )
+        .def(
+            "comm_global_domain_import",
+            [](ChipWorker &self, uint64_t domain_id, nb::bytes descriptors) {
+                std::vector<uint8_t> descriptor_bytes(
+                    reinterpret_cast<const uint8_t *>(descriptors.c_str()),
+                    reinterpret_cast<const uint8_t *>(descriptors.c_str()) + descriptors.size()
+                );
+                return self.comm_global_domain_import(domain_id, descriptor_bytes);
+            },
+            nb::arg("domain_id"), nb::arg("descriptors"),
+            "Import a rank-ordered Global CommDomain descriptor table and return the device context."
+        )
+        .def(
+            "comm_global_domain_release", &ChipWorker::comm_global_domain_release, nb::arg("domain_id"),
+            "Release a prepared or imported Global CommDomain."
+        )
         .def("comm_barrier", &ChipWorker::comm_barrier, nb::arg("comm_handle"), "Synchronize all ranks.")
         .def(
             "comm_destroy", &ChipWorker::comm_destroy, nb::arg("comm_handle"),