✨ Datafix Reconciliation Process

.devcontainer/docker-compose-airgap-preparation.yml

@@ -584,6 +584,7 @@ services:
       WINDMILL_PROBE_ADDR: ${WINDMILL_PROBE_ADDR}
       WINDMILL_PROBE_LIVE_PATH: ${WINDMILL_PROBE_LIVE_PATH}
       WINDMILL_PROBE_READY_PATH: ${WINDMILL_PROBE_READY_PATH}
+      KEYCLOAK_VOTER_GROUP_NAME: ${KEYCLOAK_VOTER_GROUP_NAME}
 
       PUBLIC_ASSETS_PATH: ${PUBLIC_ASSETS_PATH}
        #Demo key

.devcontainer/docker-compose-base.yml

@@ -806,6 +806,7 @@ services:
       WINDMILL_PROBE_READY_PATH: ${WINDMILL_PROBE_READY_PATH}
       APP_VERSION: ${APP_VERSION}
       APP_HASH: ${APP_HASH}
+      KEYCLOAK_VOTER_GROUP_NAME: ${KEYCLOAK_VOTER_GROUP_NAME}
 
       PUBLIC_ASSETS_PATH: ${PUBLIC_ASSETS_PATH}
        #Demo key

.devcontainer/docker-compose-remote.yml

@@ -704,6 +704,7 @@ services:
       DOC_RENDERER_BACKEND: ${DOC_RENDERER_BACKEND}
       BALLOT_VERIFIER_URL: ${BALLOT_VERIFIER_URL}
       ENV_SLUG: ${ENV_SLUG}
+      KEYCLOAK_VOTER_GROUP_NAME: ${KEYCLOAK_VOTER_GROUP_NAME}
 
   beat:
     profiles: ["full", "base"]

.devcontainer/keycloak/import/tenant-90505c8a-23a9-4cdf-a26b-4e19f6a097d5.json

@@ -1814,6 +1814,15 @@
         "containerId": "9c67adb8-ae12-43ad-a3f4-2bc275fc888a",
         "attributes": {}
       },
+      {
+        "id": "2f6b8e4a-9c1d-4e3f-8a7b-5d0c6f2a1b3e",
+        "name": "election-event-voter-list-reconciliation",
+        "description": "",
+        "composite": false,
+        "clientRole": false,
+        "containerId": "9c67adb8-ae12-43ad-a3f4-2bc275fc888a",
+        "attributes": {}
+      },
       {
         "id": "93fc971f-1d2e-4bff-8440-da937436f423",
         "name": "voter-import",
@@ -2535,6 +2544,7 @@
         "voter-export",
         "voter-import",
         "voter-manually-verify",
+        "election-event-voter-list-reconciliation",
         "election-event-tally-columns",
         "election-event-tally-back-button",
         "transmition-ceremony",

CLAUDE.md

@@ -16,6 +16,8 @@ Sequent Voting Platform — an end-to-end verifiable, secure online voting syste
 
 > **Note:** `cargo` is not on PATH by default. Use `devenv shell` (from the repo root) to enter the nix environment, and use the package-specific `rust-local-target/` dir to avoid permission errors (the shared `packages/target/` is owned by root because Docker service containers build into it as root).
 
+> **Do not run `cargo build` manually to check that code compiles.** The `windmill` and `harvest` containers (and `sequent-core`, which both depend on) already auto-rebuild on file changes inside the dev container. After editing code in one of these, check that container's logs instead (`docker logs windmill --tail 100` / `docker logs harvest --tail 100`) to confirm it compiled — don't kick off a separate `cargo build`, which just duplicates that work. `cargo test`, `cargo fmt`, and `cargo clippy` are unaffected by this and still run manually as usual.
+
 ```bash
 # Run any cargo command inside devenv:
 cd /workspaces/step && devenv shell bash -- -c 'cd packages && CARGO_TARGET_DIR=/workspaces/step/packages/<pkg>/rust-local-target cargo <command> -p <package>'
@@ -199,4 +201,4 @@ Dev service URLs (inside dev container):
 - MinIO: http://127.0.0.1:9001
 - RabbitMQ: http://127.0.0.1:15672
 
-**Dev container tips**: When editing Rust code in harvest, windmill, or sequent-core, check the pod logs to see if it compiles successfully — the services auto-rebuild on changes inside the dev container.
+**Dev container tips**: When editing Rust code in harvest, windmill, or sequent-core, don't run `cargo build` to verify it — check the container logs (`docker logs windmill` / `docker logs harvest`) instead, since those services auto-rebuild on changes inside the dev container. See the note under Build Commands → Rust.

REUSE.toml

@@ -44,6 +44,8 @@ path = [
 	"packages/voting-portal/src/gql/**",
 	"packages/voting-portal/tests_output/**",
 	"packages/voting-portal/reports/**",
+	"packages/results-portal/graphql.schema.json",
+	"packages/results-portal/src/gql/**",
 	"packages/windmill/src/graphql/**",
 	"packages/immudb-rs/proto/immudb/**",
 	"packages/keycloak-extensions/message-otp-authenticator/src/main/resources/theme-resources/templates/**",

docs/docusaurus/docs/02-election_managers/02-reference/02-election-event/05-election_management_election-event_voters.md

@@ -86,3 +86,54 @@ View logs of all actions performed by the voter.
 You can add additional fields like the voter's birthday or sex. [Check out the tutorial for that](../../01-tutorials/99-admin_portal_tutorials_add-user-attributes-to-keycloak.md).
 
 **Important:** Additional attributes for voters must be added before the enrollment process. For instance, if the sex attribute is not added, this trait will not be reflected in the reports and statistics.
+
+### External system (i.e. Datafix) voter-list reconciliation
+
+Users with the `election-event-voter-list-reconciliation` permission can open
+the reconciliation wizard from the Voters tab. Run reconciliation only during
+an agreed external-system freeze window:
+
+1. Upload the complete external-system reconciliation CSV. The first line must
+   be the `#META` line and file channels must be uppercase.
+2. Review the external-system and Sequent tables. If the external-system table
+   is non-empty, download its patch, have the external system apply it, and
+   upload the newly generated reconciliation file. Never apply the Sequent
+   side first.
+3. When the external-system table is empty, review the category totals and
+   explicitly apply the Sequent changes.
+4. Review the row-failure table. A completed apply can contain rows that were
+   safely rejected; these are business-level reconciliation results, not a
+   failed background task. Resolve them and retry the same `Sequence`.
+5. Upload the last file again as a diff-only convergence check. Both tables
+   must be empty before the round is considered converged.
+6. Compare the source and patch hashes with the electoral logs before ending
+   the freeze.
+
+An Internet ballot that is still `in-progress` is deliberately reported as a
+row failure. During a hard-down external-system freeze it cannot resolve
+because the review beat must reach the external system. If all remaining
+failures have this cause, complete the freeze hash checks, restore
+external-system connectivity, wait for the review beat to resolve the
+ballots, and retry the same `Sequence`. Apply-time snapshot validation
+prevents that retry from overwriting voter data changed in the meantime.
+
+Both the real-time `/unmark-voted` operation and file reconciliation clear the
+voted-channel marker. They only re-enable a voter when the corresponding
+`MarkVoted` operation owns the disable. An independent administrator disable
+and its comment are preserved, but the channel is still cleared so the voter
+does not remain permanently blocked in an unresolved external-system state.
+The administrator can then decide separately whether the account should be
+re-enabled.
+
+The task result shows at most the first 1,000 apply-time row-failure details
+and always reports the complete failure count. Resolve the common cause and
+retry the same `Sequence` to reveal any remaining failures.
+
+#### Capacity validation
+
+Before production use, UAT must exercise the largest expected voter roll,
+including a first synchronization where most voters are additions. For a
+100,000-voter test, record the generated envelope and audit-artifact sizes,
+backend peak memory, browser peak memory, wizard rendering responsiveness, and
+total Keycloak apply duration. These measurements determine the operational
+batch size and freeze-window duration for that deployment.

docs/permissions.md

@@ -52,3 +52,4 @@ CRUD endpoints for:
 - election-data-tab|approvals-tab
 - election-event-areas-tab|data-tab|keys-tab|logs-tab|publish-tab|reports-tab|scheduled-tab|tally-tab|tasks-tab|voters-tab|approvals-tab
 - election-publish-tab|voters-tab
+- election-event-voter-list-reconciliation

hasura/metadata/actions.graphql

@@ -183,6 +183,20 @@ type Mutation {
     edit_user(body: EditUsersInput!): EditUserOutput!
 }
 
+type Mutation {
+    create_external_reconciliation_import(
+        election_event_id: String!
+        document_id: String!
+    ): taskOutput
+}
+
+type Mutation {
+    apply_external_reconciliation_changes(
+        election_event_id: String!
+        diff_document_id: String!
+    ): taskOutput
+}
+
 type Mutation {
     encrypt_report(
         election_event_id: String!

hasura/metadata/actions.yaml

@@ -387,6 +387,36 @@ actions:
           - role: voter-manually-verify
           - role: voter-write
           - role: admin-user
+    - name: create_external_reconciliation_import
+      definition:
+          kind: synchronous
+          handler: http://{{HARVEST_DOMAIN}}/create-reconciliation-import
+          forward_client_headers: true
+          request_transform:
+              body:
+                  action: transform
+                  template: "{{$body.input}}"
+              template_engine: Kriti
+              version: 2
+      permissions:
+          - role: election-event-voter-list-reconciliation
+          - role: admin-user
+      comment: uploads an external (i.e. Datafix) reconciliation file and kicks off the diff-calculation task
+    - name: apply_external_reconciliation_changes
+      definition:
+          kind: synchronous
+          handler: http://{{HARVEST_DOMAIN}}/apply-reconciliation-changes
+          forward_client_headers: true
+          request_transform:
+              body:
+                  action: transform
+                  template: "{{$body.input}}"
+              template_engine: Kriti
+              version: 2
+      permissions:
+          - role: election-event-voter-list-reconciliation
+          - role: admin-user
+      comment: applies the Sequent-side reconciliation diff for an already-computed import
     - name: update_realm_attributes
       definition:
           kind: synchronous

packages/admin-portal/graphql.schema.json

@@ -11445,6 +11445,51 @@
             "isDeprecated": false,
             "deprecationReason": null
           },
+          {
+            "name": "apply_external_reconciliation_changes",
+            "description": "applies the Sequent-side reconciliation diff for an already-computed import",
+            "args": [
+              {
+                "name": "diff_document_id",
+                "description": null,
+                "type": {
+                  "kind": "NON_NULL",
+                  "name": null,
+                  "ofType": {
+                    "kind": "SCALAR",
+                    "name": "String",
+                    "ofType": null
+                  }
+                },
+                "defaultValue": null,
+                "isDeprecated": false,
+                "deprecationReason": null
+              },
+              {
+                "name": "election_event_id",
+                "description": null,
+                "type": {
+                  "kind": "NON_NULL",
+                  "name": null,
+                  "ofType": {
+                    "kind": "SCALAR",
+                    "name": "String",
+                    "ofType": null
+                  }
+                },
+                "defaultValue": null,
+                "isDeprecated": false,
+                "deprecationReason": null
+              }
+            ],
+            "type": {
+              "kind": "OBJECT",
+              "name": "taskOutput",
+              "ofType": null
+            },
+            "isDeprecated": false,
+            "deprecationReason": null
+          },
           {
             "name": "call_plugin_route",
             "description": null,
@@ -11863,6 +11908,51 @@
             "isDeprecated": false,
             "deprecationReason": null
           },
+          {
+            "name": "create_external_reconciliation_import",
+            "description": "uploads an external (i.e. Datafix) reconciliation file and kicks off the diff-calculation task",
+            "args": [
+              {
+                "name": "document_id",
+                "description": null,
+                "type": {
+                  "kind": "NON_NULL",
+                  "name": null,
+                  "ofType": {
+                    "kind": "SCALAR",
+                    "name": "String",
+                    "ofType": null
+                  }
+                },
+                "defaultValue": null,
+                "isDeprecated": false,
+                "deprecationReason": null
+              },
+              {
+                "name": "election_event_id",
+                "description": null,
+                "type": {
+                  "kind": "NON_NULL",
+                  "name": null,
+                  "ofType": {
+                    "kind": "SCALAR",
+                    "name": "String",
+                    "ofType": null
+                  }
+                },
+                "defaultValue": null,
+                "isDeprecated": false,
+                "deprecationReason": null
+              }
+            ],
+            "type": {
+              "kind": "OBJECT",
+              "name": "taskOutput",
+              "ofType": null
+            },
+            "isDeprecated": false,
+            "deprecationReason": null
+          },
           {
             "name": "create_keys_ceremony",
             "description": "create keys ceremony",