diff --git a/.devcontainer/docker-compose-airgap-preparation.yml b/.devcontainer/docker-compose-airgap-preparation.yml index 2cbc1406d94..7f7e1c04e0e 100644 --- a/.devcontainer/docker-compose-airgap-preparation.yml +++ b/.devcontainer/docker-compose-airgap-preparation.yml @@ -584,6 +584,7 @@ services: WINDMILL_PROBE_ADDR: ${WINDMILL_PROBE_ADDR} WINDMILL_PROBE_LIVE_PATH: ${WINDMILL_PROBE_LIVE_PATH} WINDMILL_PROBE_READY_PATH: ${WINDMILL_PROBE_READY_PATH} + KEYCLOAK_VOTER_GROUP_NAME: ${KEYCLOAK_VOTER_GROUP_NAME} PUBLIC_ASSETS_PATH: ${PUBLIC_ASSETS_PATH} #Demo key diff --git a/.devcontainer/docker-compose-base.yml b/.devcontainer/docker-compose-base.yml index ec265599ca7..110ec824711 100644 --- a/.devcontainer/docker-compose-base.yml +++ b/.devcontainer/docker-compose-base.yml @@ -806,6 +806,7 @@ services: WINDMILL_PROBE_READY_PATH: ${WINDMILL_PROBE_READY_PATH} APP_VERSION: ${APP_VERSION} APP_HASH: ${APP_HASH} + KEYCLOAK_VOTER_GROUP_NAME: ${KEYCLOAK_VOTER_GROUP_NAME} PUBLIC_ASSETS_PATH: ${PUBLIC_ASSETS_PATH} #Demo key diff --git a/.devcontainer/docker-compose-remote.yml b/.devcontainer/docker-compose-remote.yml index 7cbb679df86..118b9e4e091 100644 --- a/.devcontainer/docker-compose-remote.yml +++ b/.devcontainer/docker-compose-remote.yml @@ -704,6 +704,7 @@ services: DOC_RENDERER_BACKEND: ${DOC_RENDERER_BACKEND} BALLOT_VERIFIER_URL: ${BALLOT_VERIFIER_URL} ENV_SLUG: ${ENV_SLUG} + KEYCLOAK_VOTER_GROUP_NAME: ${KEYCLOAK_VOTER_GROUP_NAME} beat: profiles: ["full", "base"] diff --git a/.devcontainer/keycloak/import/tenant-90505c8a-23a9-4cdf-a26b-4e19f6a097d5.json b/.devcontainer/keycloak/import/tenant-90505c8a-23a9-4cdf-a26b-4e19f6a097d5.json index 5207e77b657..f5ade425048 100644 --- a/.devcontainer/keycloak/import/tenant-90505c8a-23a9-4cdf-a26b-4e19f6a097d5.json +++ b/.devcontainer/keycloak/import/tenant-90505c8a-23a9-4cdf-a26b-4e19f6a097d5.json @@ -1814,6 +1814,15 @@ "containerId": "9c67adb8-ae12-43ad-a3f4-2bc275fc888a", "attributes": {} }, + { + "id": "2f6b8e4a-9c1d-4e3f-8a7b-5d0c6f2a1b3e", + "name": "election-event-voter-list-reconciliation", + "description": "", + "composite": false, + "clientRole": false, + "containerId": "9c67adb8-ae12-43ad-a3f4-2bc275fc888a", + "attributes": {} + }, { "id": "93fc971f-1d2e-4bff-8440-da937436f423", "name": "voter-import", @@ -2535,6 +2544,7 @@ "voter-export", "voter-import", "voter-manually-verify", + "election-event-voter-list-reconciliation", "election-event-tally-columns", "election-event-tally-back-button", "transmition-ceremony", diff --git a/CLAUDE.md b/CLAUDE.md index aab62fa1fbc..d7475ff85b5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -16,6 +16,8 @@ Sequent Voting Platform — an end-to-end verifiable, secure online voting syste > **Note:** `cargo` is not on PATH by default. Use `devenv shell` (from the repo root) to enter the nix environment, and use the package-specific `rust-local-target/` dir to avoid permission errors (the shared `packages/target/` is owned by root because Docker service containers build into it as root). +> **Do not run `cargo build` manually to check that code compiles.** The `windmill` and `harvest` containers (and `sequent-core`, which both depend on) already auto-rebuild on file changes inside the dev container. After editing code in one of these, check that container's logs instead (`docker logs windmill --tail 100` / `docker logs harvest --tail 100`) to confirm it compiled — don't kick off a separate `cargo build`, which just duplicates that work. `cargo test`, `cargo fmt`, and `cargo clippy` are unaffected by this and still run manually as usual. + ```bash # Run any cargo command inside devenv: cd /workspaces/step && devenv shell bash -- -c 'cd packages && CARGO_TARGET_DIR=/workspaces/step/packages//rust-local-target cargo -p ' @@ -199,4 +201,4 @@ Dev service URLs (inside dev container): - MinIO: http://127.0.0.1:9001 - RabbitMQ: http://127.0.0.1:15672 -**Dev container tips**: When editing Rust code in harvest, windmill, or sequent-core, check the pod logs to see if it compiles successfully — the services auto-rebuild on changes inside the dev container. +**Dev container tips**: When editing Rust code in harvest, windmill, or sequent-core, don't run `cargo build` to verify it — check the container logs (`docker logs windmill` / `docker logs harvest`) instead, since those services auto-rebuild on changes inside the dev container. See the note under Build Commands → Rust. diff --git a/REUSE.toml b/REUSE.toml index e8663941390..c03d127ec5d 100644 --- a/REUSE.toml +++ b/REUSE.toml @@ -44,6 +44,8 @@ path = [ "packages/voting-portal/src/gql/**", "packages/voting-portal/tests_output/**", "packages/voting-portal/reports/**", + "packages/results-portal/graphql.schema.json", + "packages/results-portal/src/gql/**", "packages/windmill/src/graphql/**", "packages/immudb-rs/proto/immudb/**", "packages/keycloak-extensions/message-otp-authenticator/src/main/resources/theme-resources/templates/**", diff --git a/beyond b/beyond index 58a5f19fc83..5874c35895d 160000 --- a/beyond +++ b/beyond @@ -1 +1 @@ -Subproject commit 58a5f19fc83da0d86279c4dda889c9b08d4e595f +Subproject commit 5874c35895d6df8d9f923762587e397e7560e2a8 diff --git a/docs/docusaurus/docs/02-election_managers/02-reference/02-election-event/05-election_management_election-event_voters.md b/docs/docusaurus/docs/02-election_managers/02-reference/02-election-event/05-election_management_election-event_voters.md index 28c89b34ad1..505e21dd565 100644 --- a/docs/docusaurus/docs/02-election_managers/02-reference/02-election-event/05-election_management_election-event_voters.md +++ b/docs/docusaurus/docs/02-election_managers/02-reference/02-election-event/05-election_management_election-event_voters.md @@ -86,3 +86,54 @@ View logs of all actions performed by the voter. You can add additional fields like the voter's birthday or sex. [Check out the tutorial for that](../../01-tutorials/99-admin_portal_tutorials_add-user-attributes-to-keycloak.md). **Important:** Additional attributes for voters must be added before the enrollment process. For instance, if the sex attribute is not added, this trait will not be reflected in the reports and statistics. + +### External system (i.e. Datafix) voter-list reconciliation + +Users with the `election-event-voter-list-reconciliation` permission can open +the reconciliation wizard from the Voters tab. Run reconciliation only during +an agreed external-system freeze window: + +1. Upload the complete external-system reconciliation CSV. The first line must + be the `#META` line and file channels must be uppercase. +2. Review the external-system and Sequent tables. If the external-system table + is non-empty, download its patch, have the external system apply it, and + upload the newly generated reconciliation file. Never apply the Sequent + side first. +3. When the external-system table is empty, review the category totals and + explicitly apply the Sequent changes. +4. Review the row-failure table. A completed apply can contain rows that were + safely rejected; these are business-level reconciliation results, not a + failed background task. Resolve them and retry the same `Sequence`. +5. Upload the last file again as a diff-only convergence check. Both tables + must be empty before the round is considered converged. +6. Compare the source and patch hashes with the electoral logs before ending + the freeze. + +An Internet ballot that is still `in-progress` is deliberately reported as a +row failure. During a hard-down external-system freeze it cannot resolve +because the review beat must reach the external system. If all remaining +failures have this cause, complete the freeze hash checks, restore +external-system connectivity, wait for the review beat to resolve the +ballots, and retry the same `Sequence`. Apply-time snapshot validation +prevents that retry from overwriting voter data changed in the meantime. + +Both the real-time `/unmark-voted` operation and file reconciliation clear the +voted-channel marker. They only re-enable a voter when the corresponding +`MarkVoted` operation owns the disable. An independent administrator disable +and its comment are preserved, but the channel is still cleared so the voter +does not remain permanently blocked in an unresolved external-system state. +The administrator can then decide separately whether the account should be +re-enabled. + +The task result shows at most the first 1,000 apply-time row-failure details +and always reports the complete failure count. Resolve the common cause and +retry the same `Sequence` to reveal any remaining failures. + +#### Capacity validation + +Before production use, UAT must exercise the largest expected voter roll, +including a first synchronization where most voters are additions. For a +100,000-voter test, record the generated envelope and audit-artifact sizes, +backend peak memory, browser peak memory, wizard rendering responsiveness, and +total Keycloak apply duration. These measurements determine the operational +batch size and freeze-window duration for that deployment. diff --git a/docs/permissions.md b/docs/permissions.md index 01a8b5ba29d..2a6f14393f7 100644 --- a/docs/permissions.md +++ b/docs/permissions.md @@ -52,3 +52,4 @@ CRUD endpoints for: - election-data-tab|approvals-tab - election-event-areas-tab|data-tab|keys-tab|logs-tab|publish-tab|reports-tab|scheduled-tab|tally-tab|tasks-tab|voters-tab|approvals-tab - election-publish-tab|voters-tab +- election-event-voter-list-reconciliation diff --git a/hasura/metadata/actions.graphql b/hasura/metadata/actions.graphql index 356acd40f8f..5ead976bbb8 100644 --- a/hasura/metadata/actions.graphql +++ b/hasura/metadata/actions.graphql @@ -183,6 +183,20 @@ type Mutation { edit_user(body: EditUsersInput!): EditUserOutput! } +type Mutation { + create_external_reconciliation_import( + election_event_id: String! + document_id: String! + ): taskOutput +} + +type Mutation { + apply_external_reconciliation_changes( + election_event_id: String! + diff_document_id: String! + ): taskOutput +} + type Mutation { encrypt_report( election_event_id: String! diff --git a/hasura/metadata/actions.yaml b/hasura/metadata/actions.yaml index 7d80a380182..317dc787ffe 100644 --- a/hasura/metadata/actions.yaml +++ b/hasura/metadata/actions.yaml @@ -387,6 +387,36 @@ actions: - role: voter-manually-verify - role: voter-write - role: admin-user + - name: create_external_reconciliation_import + definition: + kind: synchronous + handler: http://{{HARVEST_DOMAIN}}/create-reconciliation-import + forward_client_headers: true + request_transform: + body: + action: transform + template: "{{$body.input}}" + template_engine: Kriti + version: 2 + permissions: + - role: election-event-voter-list-reconciliation + - role: admin-user + comment: uploads an external (i.e. Datafix) reconciliation file and kicks off the diff-calculation task + - name: apply_external_reconciliation_changes + definition: + kind: synchronous + handler: http://{{HARVEST_DOMAIN}}/apply-reconciliation-changes + forward_client_headers: true + request_transform: + body: + action: transform + template: "{{$body.input}}" + template_engine: Kriti + version: 2 + permissions: + - role: election-event-voter-list-reconciliation + - role: admin-user + comment: applies the Sequent-side reconciliation diff for an already-computed import - name: update_realm_attributes definition: kind: synchronous diff --git a/packages/admin-portal/graphql.schema.json b/packages/admin-portal/graphql.schema.json index e0d038a1779..29ce23d6387 100644 --- a/packages/admin-portal/graphql.schema.json +++ b/packages/admin-portal/graphql.schema.json @@ -11445,6 +11445,51 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "apply_external_reconciliation_changes", + "description": "applies the Sequent-side reconciliation diff for an already-computed import", + "args": [ + { + "name": "diff_document_id", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "election_event_id", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "type": { + "kind": "OBJECT", + "name": "taskOutput", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "call_plugin_route", "description": null, @@ -11863,6 +11908,51 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "create_external_reconciliation_import", + "description": "uploads an external (i.e. Datafix) reconciliation file and kicks off the diff-calculation task", + "args": [ + { + "name": "document_id", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "election_event_id", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "type": { + "kind": "OBJECT", + "name": "taskOutput", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "create_keys_ceremony", "description": "create keys ceremony", diff --git a/packages/admin-portal/rust/sequent-core-0.1.0.tgz b/packages/admin-portal/rust/sequent-core-0.1.0.tgz index 2ba45243109..d6bd3f03b69 100644 Binary files a/packages/admin-portal/rust/sequent-core-0.1.0.tgz and b/packages/admin-portal/rust/sequent-core-0.1.0.tgz differ diff --git a/packages/admin-portal/src/gql/gql.ts b/packages/admin-portal/src/gql/gql.ts index 525814f15fa..6ecadaa4447 100644 --- a/packages/admin-portal/src/gql/gql.ts +++ b/packages/admin-portal/src/gql/gql.ts @@ -14,10 +14,12 @@ import { TypedDocumentNode as DocumentNode } from '@graphql-typed-document-node/ * Learn more about it here: https://the-guild.dev/graphql/codegen/plugins/presets/preset-client#reducing-bundle-size */ type Documents = { + "\n mutation ApplyExternalReconciliationChanges(\n $election_event_id: String!\n $diff_document_id: String!\n ) {\n apply_external_reconciliation_changes(\n election_event_id: $election_event_id\n diff_document_id: $diff_document_id\n ) {\n task_execution {\n id\n }\n }\n }\n": typeof types.ApplyExternalReconciliationChangesDocument, "\n mutation CallPluginRoute($path: String!, $data: jsonb!) {\n call_plugin_route(path: $path, data: $data) {\n data\n }\n }\n": typeof types.CallPluginRouteDocument, "\n mutation ChangeApplicationStatus(\n $election_event_id: String!\n $user_id: String!\n $id: String!\n $tenant_id: String\n $area_id: String\n $rejection_reason: String\n $rejection_message: String\n ) {\n ApplicationChangeStatus(\n body: {\n election_event_id: $election_event_id\n id: $id\n user_id: $user_id\n tenant_id: $tenant_id\n area_id: $area_id\n rejection_reason: $rejection_reason\n rejection_message: $rejection_message\n }\n ) {\n message\n error\n }\n }\n": typeof types.ChangeApplicationStatusDocument, "\n mutation CheckPrivateKey(\n $electionEventId: String!\n $keysCeremonyId: String!\n $privateKeyBase64: String!\n ) {\n check_private_key(\n object: {\n election_event_id: $electionEventId\n keys_ceremony_id: $keysCeremonyId\n private_key_base64: $privateKeyBase64\n }\n ) {\n is_valid\n }\n }\n": typeof types.CheckPrivateKeyDocument, "\n mutation CreateElection(\n $electionEventId: String!\n $externalId: String!\n $presentation: jsonb\n $description: String\n ) {\n create_election(\n election_event_id: $electionEventId\n external_id: $externalId\n presentation: $presentation\n description: $description\n ) {\n id\n }\n }\n": typeof types.CreateElectionDocument, + "\n mutation CreateExternalReconciliationImport(\n $election_event_id: String!\n $document_id: String!\n ) {\n create_external_reconciliation_import(\n election_event_id: $election_event_id\n document_id: $document_id\n ) {\n task_execution {\n id\n }\n }\n }\n": typeof types.CreateExternalReconciliationImportDocument, "\n mutation CreateKeysCeremony(\n $electionEventId: String!\n $threshold: Int!\n $trusteeNames: [String!]\n $electionId: String\n $name: String\n $isAutomaticCeremony: Boolean\n ) {\n create_keys_ceremony(\n object: {\n election_event_id: $electionEventId\n threshold: $threshold\n trustee_names: $trusteeNames\n election_id: $electionId\n name: $name\n is_automatic_ceremony: $isAutomaticCeremony\n }\n ) {\n keys_ceremony_id\n error_message\n }\n }\n": typeof types.CreateKeysCeremonyDocument, "\n mutation CreatePhoneBlacklistEntry(\n $election_event_id: uuid!\n $phone_e164: String!\n $reason: String\n ) {\n create_phone_blacklist_entry(\n election_event_id: $election_event_id\n phone_e164: $phone_e164\n reason: $reason\n ) {\n id\n }\n }\n": typeof types.CreatePhoneBlacklistEntryDocument, "\n mutation InsertReport($object: sequent_backend_report_insert_input!) {\n insert_sequent_backend_report(objects: [$object]) {\n returning {\n id\n election_event_id\n tenant_id\n election_id\n report_type\n template_alias\n cron_config\n encryption_policy\n }\n affected_rows\n }\n }\n": typeof types.InsertReportDocument, @@ -140,10 +142,12 @@ type Documents = { "\n mutation ReviewTallySheet(\n $electionEventId: String!\n $tallySheetId: String!\n $newStatus: String!\n ) {\n review_tally_sheet(\n election_event_id: $electionEventId\n tally_sheet_id: $tallySheetId\n new_status: $newStatus\n ) {\n id\n tenant_id\n election_event_id\n election_id\n contest_id\n area_id\n created_at\n last_updated_at\n labels\n annotations\n reviewed_at\n reviewed_by_user_id\n content\n channel\n deleted_at\n created_by_user_id\n status\n version\n }\n }\n": typeof types.ReviewTallySheetDocument, }; const documents: Documents = { + "\n mutation ApplyExternalReconciliationChanges(\n $election_event_id: String!\n $diff_document_id: String!\n ) {\n apply_external_reconciliation_changes(\n election_event_id: $election_event_id\n diff_document_id: $diff_document_id\n ) {\n task_execution {\n id\n }\n }\n }\n": types.ApplyExternalReconciliationChangesDocument, "\n mutation CallPluginRoute($path: String!, $data: jsonb!) {\n call_plugin_route(path: $path, data: $data) {\n data\n }\n }\n": types.CallPluginRouteDocument, "\n mutation ChangeApplicationStatus(\n $election_event_id: String!\n $user_id: String!\n $id: String!\n $tenant_id: String\n $area_id: String\n $rejection_reason: String\n $rejection_message: String\n ) {\n ApplicationChangeStatus(\n body: {\n election_event_id: $election_event_id\n id: $id\n user_id: $user_id\n tenant_id: $tenant_id\n area_id: $area_id\n rejection_reason: $rejection_reason\n rejection_message: $rejection_message\n }\n ) {\n message\n error\n }\n }\n": types.ChangeApplicationStatusDocument, "\n mutation CheckPrivateKey(\n $electionEventId: String!\n $keysCeremonyId: String!\n $privateKeyBase64: String!\n ) {\n check_private_key(\n object: {\n election_event_id: $electionEventId\n keys_ceremony_id: $keysCeremonyId\n private_key_base64: $privateKeyBase64\n }\n ) {\n is_valid\n }\n }\n": types.CheckPrivateKeyDocument, "\n mutation CreateElection(\n $electionEventId: String!\n $externalId: String!\n $presentation: jsonb\n $description: String\n ) {\n create_election(\n election_event_id: $electionEventId\n external_id: $externalId\n presentation: $presentation\n description: $description\n ) {\n id\n }\n }\n": types.CreateElectionDocument, + "\n mutation CreateExternalReconciliationImport(\n $election_event_id: String!\n $document_id: String!\n ) {\n create_external_reconciliation_import(\n election_event_id: $election_event_id\n document_id: $document_id\n ) {\n task_execution {\n id\n }\n }\n }\n": types.CreateExternalReconciliationImportDocument, "\n mutation CreateKeysCeremony(\n $electionEventId: String!\n $threshold: Int!\n $trusteeNames: [String!]\n $electionId: String\n $name: String\n $isAutomaticCeremony: Boolean\n ) {\n create_keys_ceremony(\n object: {\n election_event_id: $electionEventId\n threshold: $threshold\n trustee_names: $trusteeNames\n election_id: $electionId\n name: $name\n is_automatic_ceremony: $isAutomaticCeremony\n }\n ) {\n keys_ceremony_id\n error_message\n }\n }\n": types.CreateKeysCeremonyDocument, "\n mutation CreatePhoneBlacklistEntry(\n $election_event_id: uuid!\n $phone_e164: String!\n $reason: String\n ) {\n create_phone_blacklist_entry(\n election_event_id: $election_event_id\n phone_e164: $phone_e164\n reason: $reason\n ) {\n id\n }\n }\n": types.CreatePhoneBlacklistEntryDocument, "\n mutation InsertReport($object: sequent_backend_report_insert_input!) {\n insert_sequent_backend_report(objects: [$object]) {\n returning {\n id\n election_event_id\n tenant_id\n election_id\n report_type\n template_alias\n cron_config\n encryption_policy\n }\n affected_rows\n }\n }\n": types.InsertReportDocument, @@ -280,6 +284,10 @@ const documents: Documents = { */ export function graphql(source: string): unknown; +/** + * The graphql function is used to parse GraphQL queries into a document that can be used by GraphQL clients. + */ +export function graphql(source: "\n mutation ApplyExternalReconciliationChanges(\n $election_event_id: String!\n $diff_document_id: String!\n ) {\n apply_external_reconciliation_changes(\n election_event_id: $election_event_id\n diff_document_id: $diff_document_id\n ) {\n task_execution {\n id\n }\n }\n }\n"): (typeof documents)["\n mutation ApplyExternalReconciliationChanges(\n $election_event_id: String!\n $diff_document_id: String!\n ) {\n apply_external_reconciliation_changes(\n election_event_id: $election_event_id\n diff_document_id: $diff_document_id\n ) {\n task_execution {\n id\n }\n }\n }\n"]; /** * The graphql function is used to parse GraphQL queries into a document that can be used by GraphQL clients. */ @@ -296,6 +304,10 @@ export function graphql(source: "\n mutation CheckPrivateKey(\n $elect * The graphql function is used to parse GraphQL queries into a document that can be used by GraphQL clients. */ export function graphql(source: "\n mutation CreateElection(\n $electionEventId: String!\n $externalId: String!\n $presentation: jsonb\n $description: String\n ) {\n create_election(\n election_event_id: $electionEventId\n external_id: $externalId\n presentation: $presentation\n description: $description\n ) {\n id\n }\n }\n"): (typeof documents)["\n mutation CreateElection(\n $electionEventId: String!\n $externalId: String!\n $presentation: jsonb\n $description: String\n ) {\n create_election(\n election_event_id: $electionEventId\n external_id: $externalId\n presentation: $presentation\n description: $description\n ) {\n id\n }\n }\n"]; +/** + * The graphql function is used to parse GraphQL queries into a document that can be used by GraphQL clients. + */ +export function graphql(source: "\n mutation CreateExternalReconciliationImport(\n $election_event_id: String!\n $document_id: String!\n ) {\n create_external_reconciliation_import(\n election_event_id: $election_event_id\n document_id: $document_id\n ) {\n task_execution {\n id\n }\n }\n }\n"): (typeof documents)["\n mutation CreateExternalReconciliationImport(\n $election_event_id: String!\n $document_id: String!\n ) {\n create_external_reconciliation_import(\n election_event_id: $election_event_id\n document_id: $document_id\n ) {\n task_execution {\n id\n }\n }\n }\n"]; /** * The graphql function is used to parse GraphQL queries into a document that can be used by GraphQL clients. */ diff --git a/packages/admin-portal/src/gql/graphql.ts b/packages/admin-portal/src/gql/graphql.ts index b24607ea99e..1141d33f638 100644 --- a/packages/admin-portal/src/gql/graphql.ts +++ b/packages/admin-portal/src/gql/graphql.ts @@ -1237,6 +1237,8 @@ export type Mutation_Root = { ApplicationChangeStatus?: Maybe; /** Verify User Registration Application */ VerifyApplication: Scalars['String']['output']; + /** applies the Sequent-side reconciliation diff for an already-computed import */ + apply_external_reconciliation_changes?: Maybe; call_plugin_route: PluginsRouteOutput; /** check private key */ check_private_key?: Maybe; @@ -1247,6 +1249,8 @@ export type Mutation_Root = { /** create_ballot_receipt */ create_ballot_receipt?: Maybe; create_election?: Maybe; + /** uploads an external (i.e. Datafix) reconciliation file and kicks off the diff-calculation task */ + create_external_reconciliation_import?: Maybe; /** create keys ceremony */ create_keys_ceremony?: Maybe; /** create_new_tally_sheet */ @@ -1967,6 +1971,13 @@ export type Mutation_RootVerifyApplicationArgs = { }; +/** mutation root */ +export type Mutation_RootApply_External_Reconciliation_ChangesArgs = { + diff_document_id: Scalars['String']['input']; + election_event_id: Scalars['String']['input']; +}; + + /** mutation root */ export type Mutation_RootCall_Plugin_RouteArgs = { data: Scalars['jsonb']['input']; @@ -2019,6 +2030,13 @@ export type Mutation_RootCreate_ElectionArgs = { }; +/** mutation root */ +export type Mutation_RootCreate_External_Reconciliation_ImportArgs = { + document_id: Scalars['String']['input']; + election_event_id: Scalars['String']['input']; +}; + + /** mutation root */ export type Mutation_RootCreate_Keys_CeremonyArgs = { object: CreateKeysCeremonyInput; @@ -24994,6 +25012,14 @@ export type Uuid_Comparison_Exp = { _nin?: InputMaybe>; }; +export type ApplyExternalReconciliationChangesMutationVariables = Exact<{ + election_event_id: Scalars['String']['input']; + diff_document_id: Scalars['String']['input']; +}>; + + +export type ApplyExternalReconciliationChangesMutation = { __typename?: 'mutation_root', apply_external_reconciliation_changes?: { __typename?: 'taskOutput', task_execution: { __typename?: 'tasks_execution_type', id: any } } | null }; + export type CallPluginRouteMutationVariables = Exact<{ path: Scalars['String']['input']; data: Scalars['jsonb']['input']; @@ -25034,6 +25060,14 @@ export type CreateElectionMutationVariables = Exact<{ export type CreateElectionMutation = { __typename?: 'mutation_root', create_election?: { __typename?: 'CreateElectionOutput', id: string } | null }; +export type CreateExternalReconciliationImportMutationVariables = Exact<{ + election_event_id: Scalars['String']['input']; + document_id: Scalars['String']['input']; +}>; + + +export type CreateExternalReconciliationImportMutation = { __typename?: 'mutation_root', create_external_reconciliation_import?: { __typename?: 'taskOutput', task_execution: { __typename?: 'tasks_execution_type', id: any } } | null }; + export type CreateKeysCeremonyMutationVariables = Exact<{ electionEventId: Scalars['String']['input']; threshold: Scalars['Int']['input']; @@ -26106,10 +26140,12 @@ export type ReviewTallySheetMutationVariables = Exact<{ export type ReviewTallySheetMutation = { __typename?: 'mutation_root', review_tally_sheet?: { __typename?: 'TallySheetOutput', id: string, tenant_id: string, election_event_id: string, election_id: string, contest_id: string, area_id: string, created_at?: string | null, last_updated_at?: string | null, labels?: any | null, annotations?: any | null, reviewed_at?: string | null, reviewed_by_user_id?: string | null, content?: any | null, channel?: string | null, deleted_at?: string | null, created_by_user_id?: string | null, status: string, version: number } | null }; +export const ApplyExternalReconciliationChangesDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"ApplyExternalReconciliationChanges"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"election_event_id"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"diff_document_id"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"apply_external_reconciliation_changes"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"election_event_id"},"value":{"kind":"Variable","name":{"kind":"Name","value":"election_event_id"}}},{"kind":"Argument","name":{"kind":"Name","value":"diff_document_id"},"value":{"kind":"Variable","name":{"kind":"Name","value":"diff_document_id"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"task_execution"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}}]}}]}}]}}]} as unknown as DocumentNode; export const CallPluginRouteDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"CallPluginRoute"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"path"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"data"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"jsonb"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"call_plugin_route"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"path"},"value":{"kind":"Variable","name":{"kind":"Name","value":"path"}}},{"kind":"Argument","name":{"kind":"Name","value":"data"},"value":{"kind":"Variable","name":{"kind":"Name","value":"data"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"data"}}]}}]}}]} as unknown as DocumentNode; export const ChangeApplicationStatusDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"ChangeApplicationStatus"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"election_event_id"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"user_id"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"id"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"tenant_id"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"area_id"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"rejection_reason"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"rejection_message"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"ApplicationChangeStatus"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"body"},"value":{"kind":"ObjectValue","fields":[{"kind":"ObjectField","name":{"kind":"Name","value":"election_event_id"},"value":{"kind":"Variable","name":{"kind":"Name","value":"election_event_id"}}},{"kind":"ObjectField","name":{"kind":"Name","value":"id"},"value":{"kind":"Variable","name":{"kind":"Name","value":"id"}}},{"kind":"ObjectField","name":{"kind":"Name","value":"user_id"},"value":{"kind":"Variable","name":{"kind":"Name","value":"user_id"}}},{"kind":"ObjectField","name":{"kind":"Name","value":"tenant_id"},"value":{"kind":"Variable","name":{"kind":"Name","value":"tenant_id"}}},{"kind":"ObjectField","name":{"kind":"Name","value":"area_id"},"value":{"kind":"Variable","name":{"kind":"Name","value":"area_id"}}},{"kind":"ObjectField","name":{"kind":"Name","value":"rejection_reason"},"value":{"kind":"Variable","name":{"kind":"Name","value":"rejection_reason"}}},{"kind":"ObjectField","name":{"kind":"Name","value":"rejection_message"},"value":{"kind":"Variable","name":{"kind":"Name","value":"rejection_message"}}}]}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"message"}},{"kind":"Field","name":{"kind":"Name","value":"error"}}]}}]}}]} as unknown as DocumentNode; export const CheckPrivateKeyDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"CheckPrivateKey"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"electionEventId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"keysCeremonyId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"privateKeyBase64"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"check_private_key"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"object"},"value":{"kind":"ObjectValue","fields":[{"kind":"ObjectField","name":{"kind":"Name","value":"election_event_id"},"value":{"kind":"Variable","name":{"kind":"Name","value":"electionEventId"}}},{"kind":"ObjectField","name":{"kind":"Name","value":"keys_ceremony_id"},"value":{"kind":"Variable","name":{"kind":"Name","value":"keysCeremonyId"}}},{"kind":"ObjectField","name":{"kind":"Name","value":"private_key_base64"},"value":{"kind":"Variable","name":{"kind":"Name","value":"privateKeyBase64"}}}]}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"is_valid"}}]}}]}}]} as unknown as DocumentNode; export const CreateElectionDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"CreateElection"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"electionEventId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"externalId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"presentation"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"jsonb"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"description"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"create_election"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"election_event_id"},"value":{"kind":"Variable","name":{"kind":"Name","value":"electionEventId"}}},{"kind":"Argument","name":{"kind":"Name","value":"external_id"},"value":{"kind":"Variable","name":{"kind":"Name","value":"externalId"}}},{"kind":"Argument","name":{"kind":"Name","value":"presentation"},"value":{"kind":"Variable","name":{"kind":"Name","value":"presentation"}}},{"kind":"Argument","name":{"kind":"Name","value":"description"},"value":{"kind":"Variable","name":{"kind":"Name","value":"description"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}}]}}]}}]} as unknown as DocumentNode; +export const CreateExternalReconciliationImportDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"CreateExternalReconciliationImport"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"election_event_id"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"document_id"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"create_external_reconciliation_import"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"election_event_id"},"value":{"kind":"Variable","name":{"kind":"Name","value":"election_event_id"}}},{"kind":"Argument","name":{"kind":"Name","value":"document_id"},"value":{"kind":"Variable","name":{"kind":"Name","value":"document_id"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"task_execution"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}}]}}]}}]}}]} as unknown as DocumentNode; export const CreateKeysCeremonyDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"CreateKeysCeremony"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"electionEventId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"threshold"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"Int"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"trusteeNames"}},"type":{"kind":"ListType","type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"electionId"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"name"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"isAutomaticCeremony"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"Boolean"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"create_keys_ceremony"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"object"},"value":{"kind":"ObjectValue","fields":[{"kind":"ObjectField","name":{"kind":"Name","value":"election_event_id"},"value":{"kind":"Variable","name":{"kind":"Name","value":"electionEventId"}}},{"kind":"ObjectField","name":{"kind":"Name","value":"threshold"},"value":{"kind":"Variable","name":{"kind":"Name","value":"threshold"}}},{"kind":"ObjectField","name":{"kind":"Name","value":"trustee_names"},"value":{"kind":"Variable","name":{"kind":"Name","value":"trusteeNames"}}},{"kind":"ObjectField","name":{"kind":"Name","value":"election_id"},"value":{"kind":"Variable","name":{"kind":"Name","value":"electionId"}}},{"kind":"ObjectField","name":{"kind":"Name","value":"name"},"value":{"kind":"Variable","name":{"kind":"Name","value":"name"}}},{"kind":"ObjectField","name":{"kind":"Name","value":"is_automatic_ceremony"},"value":{"kind":"Variable","name":{"kind":"Name","value":"isAutomaticCeremony"}}}]}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"keys_ceremony_id"}},{"kind":"Field","name":{"kind":"Name","value":"error_message"}}]}}]}}]} as unknown as DocumentNode; export const CreatePhoneBlacklistEntryDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"CreatePhoneBlacklistEntry"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"election_event_id"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"uuid"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"phone_e164"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"reason"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"create_phone_blacklist_entry"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"election_event_id"},"value":{"kind":"Variable","name":{"kind":"Name","value":"election_event_id"}}},{"kind":"Argument","name":{"kind":"Name","value":"phone_e164"},"value":{"kind":"Variable","name":{"kind":"Name","value":"phone_e164"}}},{"kind":"Argument","name":{"kind":"Name","value":"reason"},"value":{"kind":"Variable","name":{"kind":"Name","value":"reason"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}}]}}]}}]} as unknown as DocumentNode; export const InsertReportDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"InsertReport"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"object"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"sequent_backend_report_insert_input"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"insert_sequent_backend_report"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"objects"},"value":{"kind":"ListValue","values":[{"kind":"Variable","name":{"kind":"Name","value":"object"}}]}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"returning"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"election_event_id"}},{"kind":"Field","name":{"kind":"Name","value":"tenant_id"}},{"kind":"Field","name":{"kind":"Name","value":"election_id"}},{"kind":"Field","name":{"kind":"Name","value":"report_type"}},{"kind":"Field","name":{"kind":"Name","value":"template_alias"}},{"kind":"Field","name":{"kind":"Name","value":"cron_config"}},{"kind":"Field","name":{"kind":"Name","value":"encryption_policy"}}]}},{"kind":"Field","name":{"kind":"Name","value":"affected_rows"}}]}}]}}]} as unknown as DocumentNode; diff --git a/packages/admin-portal/src/queries/ApplyExternalReconciliationChanges.ts b/packages/admin-portal/src/queries/ApplyExternalReconciliationChanges.ts new file mode 100644 index 00000000000..3d818aadce3 --- /dev/null +++ b/packages/admin-portal/src/queries/ApplyExternalReconciliationChanges.ts @@ -0,0 +1,20 @@ +// SPDX-FileCopyrightText: 2026 Sequent Tech Inc +// +// SPDX-License-Identifier: AGPL-3.0-only +import {gql} from "@apollo/client" + +export const APPLY_EXTERNAL_RECONCILIATION_CHANGES = gql` + mutation ApplyExternalReconciliationChanges( + $election_event_id: String! + $diff_document_id: String! + ) { + apply_external_reconciliation_changes( + election_event_id: $election_event_id + diff_document_id: $diff_document_id + ) { + task_execution { + id + } + } + } +` diff --git a/packages/admin-portal/src/queries/CreateExternalReconciliationImport.ts b/packages/admin-portal/src/queries/CreateExternalReconciliationImport.ts new file mode 100644 index 00000000000..0307149bde6 --- /dev/null +++ b/packages/admin-portal/src/queries/CreateExternalReconciliationImport.ts @@ -0,0 +1,20 @@ +// SPDX-FileCopyrightText: 2026 Sequent Tech Inc +// +// SPDX-License-Identifier: AGPL-3.0-only +import {gql} from "@apollo/client" + +export const CREATE_EXTERNAL_RECONCILIATION_IMPORT = gql` + mutation CreateExternalReconciliationImport( + $election_event_id: String! + $document_id: String! + ) { + create_external_reconciliation_import( + election_event_id: $election_event_id + document_id: $document_id + ) { + task_execution { + id + } + } + } +` diff --git a/packages/admin-portal/src/resources/User/ListUsers.tsx b/packages/admin-portal/src/resources/User/ListUsers.tsx index 2f8175278fe..61609cf8223 100644 --- a/packages/admin-portal/src/resources/User/ListUsers.tsx +++ b/packages/admin-portal/src/resources/User/ListUsers.tsx @@ -60,7 +60,7 @@ import { import {DELETE_USER} from "@/queries/DeleteUser" import {MANUAL_VERIFICATION} from "@/queries/ManualVerification" import {useMutation, useQuery} from "@apollo/client" -import {IPermissions} from "@/types/keycloak" +import {ATTR_RESET_VALUE, IPermissions} from "@/types/keycloak" import {ResourceListStyles} from "@/components/styles/ResourceListStyles" import {IRole, IUser, translate} from "@sequentech/ui-core" import {SettingsContext} from "@/providers/SettingsContextProvider" @@ -75,6 +75,8 @@ import {USER_PROFILE_ATTRIBUTES} from "@/queries/GetUserProfileAttributes" import {getAttributeLabel, getTranslationLabel, userBasicInfo} from "@/services/UserService" import CustomDateField from "./CustomDateField" import {ListActionsMenu} from "@/components/ListActionsMenu" +import SyncAltIcon from "@mui/icons-material/SyncAlt" +import {ReconciliationWizard} from "@/resources/VoterListSync/ReconciliationWizard" import EditPassword from "./EditPassword" import {styled} from "@mui/material/styles" import {DELETE_USERS} from "@/queries/DeleteUsers" @@ -95,7 +97,6 @@ import {useAliasRenderer} from "@/hooks/useAliasRenderer" export const AUTHORIZED_ELECTION_IDS = "authorized-election-ids" export const VOTED_CHANNEL = "voted-channel" export const DISABLE_COMMENT = "disable-comment" -const ATTR_RESET_VALUE = "NONE" const DataGridContainerStyle = styled(DatagridConfigurable, { shouldForwardProp: (prop) => prop !== "isOpenSideBar", // Prevent `isOpenSideBar` from being passed to the DOM @@ -156,6 +157,7 @@ export const ListUsers: React.FC = ({aside, electionEventId, ele const [deleteId, setDeleteId] = useState() const [openDrawer, setOpenDrawer] = useState(false) const [openImportDrawer, setOpenImportDrawer] = useState(false) + const [openReconciliationWizard, setOpenReconciliationWizard] = useState(false) const [recordIds, setRecordIds] = useState>([]) const [userRecord, setUserRecord] = useState | undefined>() const authContext = useContext(AuthContext) @@ -270,6 +272,7 @@ export const ListUsers: React.FC = ({aside, electionEventId, ele showVotersFilters, showVotersLogs, canSendTemplates, + showVoterListSync, } = useUsersPermissions() /** * Permissions @@ -1122,7 +1125,7 @@ export const ListUsers: React.FC = ({aside, electionEventId, ele }} /> )} - {electionEventId && ( + {electionEventId && showVoterListSync && ( source={`attributes['${DISABLE_COMMENT}']`} label={String( @@ -1226,7 +1229,20 @@ export const ListUsers: React.FC = ({aside, electionEventId, ele /> } withComponent={canCreateVoters} - extraActions={[...listActions]} + extraActions={[ + ...listActions, + ...(electionEventId && showVoterListSync + ? [ + , + ] + : []), + ]} /> } filter={{ @@ -1333,6 +1349,14 @@ export const ListUsers: React.FC = ({aside, electionEventId, ele + {electionEventId && showVoterListSync ? ( + setOpenReconciliationWizard(false)} + /> + ) : null} + setOpenImportDrawer(false)} diff --git a/packages/admin-portal/src/resources/User/useUsersPermissions.tsx b/packages/admin-portal/src/resources/User/useUsersPermissions.tsx index 6ec7d474bec..bf98813beb7 100644 --- a/packages/admin-portal/src/resources/User/useUsersPermissions.tsx +++ b/packages/admin-portal/src/resources/User/useUsersPermissions.tsx @@ -58,6 +58,11 @@ export function useUsersPermissions() { tenantId, IPermissions.NOTIFICATION_SEND ) + const showVoterListSync = authContext.isAuthorized( + true, + tenantId, + IPermissions.ELECTION_EVENT_VOTER_LIST_SYNC + ) /** * Permissions */ @@ -77,5 +82,6 @@ export function useUsersPermissions() { showVotersFilters, showVotersLogs, canSendTemplates, + showVoterListSync, } } diff --git a/packages/admin-portal/src/resources/VoterListSync/CategorySummary.tsx b/packages/admin-portal/src/resources/VoterListSync/CategorySummary.tsx new file mode 100644 index 00000000000..2b23a718a1b --- /dev/null +++ b/packages/admin-portal/src/resources/VoterListSync/CategorySummary.tsx @@ -0,0 +1,50 @@ +// SPDX-FileCopyrightText: 2026 Sequent Tech Inc +// +// SPDX-License-Identifier: AGPL-3.0-only + +import React from "react" +import {Box, Stack, Typography} from "@mui/material" +import {useTranslation} from "react-i18next" +import {ESyncChangeCategory} from "./types" + +interface CategorySummaryProps { + /** Backend-computed per-category counts (the import row's `summary` + * column) — never re-derived from a paginated page of items, which would + * silently undercount once the item grids are server-paginated. */ + counts: Partial> + highlighted?: Set +} + +/** Renders the per-category counts, e.g. "marks N voters as voted via other + * channels, disables M voters, updates K profiles, adds J voters" from the + * reconciliation confirmation dialog acceptance criteria. */ +export const CategorySummary: React.FC = ({counts, highlighted}) => { + const {t} = useTranslation() + const entries = Object.entries(counts).filter(([, count]) => (count ?? 0) > 0) as Array< + [ESyncChangeCategory, number] + > + + return ( + + {entries.map(([category, count]) => ( + + + {t(`reconciliation.categories.${category}`, category)} + + {count} + + ))} + + ) +} + +export default CategorySummary diff --git a/packages/admin-portal/src/resources/VoterListSync/ReconciliationWizard.tsx b/packages/admin-portal/src/resources/VoterListSync/ReconciliationWizard.tsx new file mode 100644 index 00000000000..b4c8907a909 --- /dev/null +++ b/packages/admin-portal/src/resources/VoterListSync/ReconciliationWizard.tsx @@ -0,0 +1,905 @@ +// SPDX-FileCopyrightText: 2026 Sequent Tech Inc +// +// SPDX-License-Identifier: AGPL-3.0-only + +import React, {useCallback, useEffect, useMemo, useState} from "react" +import {useMutation, useQuery} from "@apollo/client" +import { + Alert, + Box, + Button, + CircularProgress, + Dialog, + DialogActions, + DialogContent, + DialogTitle, + Divider, + Drawer, + Stack, + Typography, +} from "@mui/material" +import {useTranslation} from "react-i18next" +import {ATTR_RESET_VALUE} from "@/types/keycloak" +import {DropFile} from "@sequentech/ui-essentials" +import {ETaskExecutionStatus} from "@sequentech/ui-core" +import {SyncDiffTable} from "./SyncDiffTable" +import {CategorySummary} from "./CategorySummary" +import {DownloadDocument} from "@/resources/User/DownloadDocument" +import {ESyncChangeCategory, SyncDiffRow} from "./types" +import {HIGHLIGHTED_CATEGORIES} from "./constants" +import {formatGeneratedAt} from "./utils" +import ElectionHeader from "@/components/ElectionHeader" +import {DrawerStyles} from "@/components/styles/DrawerStyles" +import {ImportStyles} from "@/components/election-event/import-data/ImportScreen" +import {useWidgetStore} from "@/providers/WidgetsContextProvider" +import {ETasksExecution} from "@/types/tasksExecution" +import {GET_UPLOAD_URL} from "@/queries/GetUploadUrl" +import {CREATE_EXTERNAL_RECONCILIATION_IMPORT} from "@/queries/CreateExternalReconciliationImport" +import {APPLY_EXTERNAL_RECONCILIATION_CHANGES} from "@/queries/ApplyExternalReconciliationChanges" +import {GET_TASK_BY_ID} from "@/queries/GetTaskById" +import {FETCH_DOCUMENT} from "@/queries/FetchDocument" + +type WizardStep = "drop" | "processing" | "review" | "applying" | "done" + +interface ReconciliationWizardProps { + open: boolean + electionEventId: string + onClose: () => void +} + +type TranslateFn = ReturnType["t"] + +const POLL_INTERVAL_MS = 3000 + +/** + * A field's own `(old, new)` pair now lives inside the field itself, wire- + * shaped as a single-key object keyed by the Rust variant name (e.g. + * `{"Channel": ["NONE", "INTERNET"]}`, `{"Enabled": [true, false]}`, + * `{"KeycloakUA": [{"voted-channel": "NONE"}, {"voted-channel": "PAPER"}]}`) + * — not a separate `old_value`/`new_value` on the item, since a field is + * never meaningful apart from its own old/new values. `null` when the change + * (e.g. a CountyMun mismatch row failure) has no corresponding field at all. + * See `describeFields` below for how this is turned back into flat + * label/oldValue/newValue rows for display. + */ +type RawFieldValue = Record | null + +interface RawDiffItem { + voter_username: string + target: "datafix" | "sequent" + category: ESyncChangeCategory + field?: RawFieldValue + failure_reason?: string | null +} + +/** + * Content of the diff-envelope document `generate_reconciliation_patches` + * uploads, referenced from the generate task_execution's own + * `annotations.document_id` — there is no `datafix_reconciliation_import` + * table/row anymore, so this document, fetched and parsed client-side, is + * the sole source for the review step. + */ +interface ReconciliationDiffEnvelope { + sequence: number + generated_at: number + /** False for an equal-Sequence convergence check after a successful apply. */ + apply_allowed: boolean + external_patch_document_id: string | null + items: RawDiffItem[] +} + +interface ApplyTaskRowFailure { + voter_id: string + reason: string +} + +interface ApplyTaskRowFailureSummary { + rows: SyncDiffRow[] + totalCount: number + truncated: boolean +} + +/** + * Counts distinct voters per category, not rows: a single voter's change can + * fan out into several `SyncDiffRow`s (e.g. a new voter is 4 rows - area, + * DoB, channel, enabled - all tagged `VOTER_ADDED`), so counting rows would + * multiply-count that one voter under every category with more than one + * field per voter. + */ +const countsByCategory = (rows: SyncDiffRow[]): Partial> => { + const votersByCategory = new Map>() + rows.forEach((row) => { + const voters = votersByCategory.get(row.category) ?? new Set() + voters.add(row.voterId) + votersByCategory.set(row.category, voters) + }) + const counts: Partial> = {} + votersByCategory.forEach((voters, category) => { + counts[category] = voters.size + }) + return counts +} + +const summarizeForConfirmation = (rows: SyncDiffRow[], t: TranslateFn): string => { + const counts = countsByCategory(rows) + + const phrases = [ + [ + ESyncChangeCategory.VOTED_OTHER_CHANNEL, + "reconciliation.wizard.summary.votedOtherChannel", + ], + [ESyncChangeCategory.DISABLED_DELETE_CALL, "reconciliation.wizard.summary.disabled"], + [ESyncChangeCategory.REENABLED, "reconciliation.wizard.summary.reenabled"], + [ESyncChangeCategory.VOTED_UNMARKED, "reconciliation.wizard.summary.votedUnmarked"], + [ESyncChangeCategory.PROFILE_UPDATE, "reconciliation.wizard.summary.profileUpdated"], + [ESyncChangeCategory.VOTER_ADDED, "reconciliation.wizard.summary.voterAdded"], + ] as const + + const parts = phrases + .map(([category, key]) => { + const count = counts[category] ?? 0 + return count > 0 ? t(key, {count}) : null + }) + .filter((part): part is string => part !== null) + + return parts.length > 0 + ? t("reconciliation.wizard.summary.prefix", {parts: parts.join(", ")}) + : t("reconciliation.wizard.summary.empty") +} + +/** Flat label/oldValue/newValue the table actually renders, derived from a + * raw item's `field` (see the `RawFieldValue` doc above). */ +interface FieldDisplay { + label: string + oldValue: string + newValue: string +} + +/** Renders a single old/new element: booleans as "true"/"false" (`Enabled`), + * otherwise the value as-is (every other field, including each individual + * `KeycloakUA` attribute, is already a plain string). */ +const formatFieldValue = (value: unknown): string => { + if (typeof value === "boolean") { + return value ? "true" : "false" + } + return value == null ? ATTR_RESET_VALUE : String(value) +} + +/** + * `KeycloakUA`'s `(old, new)` pair is a bag of Keycloak attributes, not a + * single field - e.g. "voted via other channel" writes `voted-channel` and + * `disable-comment` together in one atomic Keycloak edit (see + * `reconciliation::diff` in windmill), so `new` can carry a key `old` never + * had. Splitting into one row per attribute key (the union of both bags' + * keys, each looked up independently) is what keeps that case readable: + * joining the bags into a single row instead mismatches keys against values + * whenever the two sides don't share the exact same key set. + */ +const describeFields = (field: RawFieldValue, t: TranslateFn): FieldDisplay[] => { + const entry = field ? Object.entries(field)[0] : undefined + if (!entry) { + // `null` when the change (e.g. a CountyMun mismatch row failure) + // has no corresponding field at all - the Reason column explains it. + return [ + { + label: t("reconciliation.table.rowLabel"), + oldValue: ATTR_RESET_VALUE, + newValue: ATTR_RESET_VALUE, + }, + ] + } + const [variantName, tuple] = entry as [string, [unknown, unknown]] + const [oldRaw, newRaw] = tuple + + if (variantName === "KeycloakUA") { + const oldBag = (oldRaw ?? {}) as Record + const newBag = (newRaw ?? {}) as Record + const keys = Array.from(new Set([...Object.keys(oldBag), ...Object.keys(newBag)])) + return keys.map((key) => ({ + label: t(`usersAndRolesScreen.users.fields.${key}`, key), + oldValue: formatFieldValue(oldBag[key] ?? null), + newValue: formatFieldValue(newBag[key] ?? null), + })) + } + + return [ + { + label: variantName, + oldValue: formatFieldValue(oldRaw), + newValue: formatFieldValue(newRaw), + }, + ] +} + +const toRows = ( + items: RawDiffItem[], + target: "datafix" | "sequent", + t: TranslateFn +): SyncDiffRow[] => + items + .filter( + (item) => item.target === target && item.category !== ESyncChangeCategory.ROW_FAILURE + ) + .flatMap((item, index) => + describeFields(item.field ?? null, t).map( + ({label, oldValue, newValue}, fieldIndex) => ({ + id: `${item.voter_username}:${label}:${index}:${fieldIndex}`, + voterId: item.voter_username, + field: label, + label, + oldValue, + newValue, + category: item.category, + target: item.target, + }) + ) + ) + +const toRowFailures = (items: RawDiffItem[], t: TranslateFn): SyncDiffRow[] => + items + .filter((item) => item.category === ESyncChangeCategory.ROW_FAILURE) + .flatMap((item, index) => + describeFields(item.field ?? null, t).map( + ({label, oldValue, newValue}, fieldIndex) => ({ + id: `failure:${item.voter_username}:${index}:${fieldIndex}`, + voterId: item.voter_username, + field: label, + label, + oldValue, + newValue, + category: item.category, + target: item.target, + failureReason: item.failure_reason ?? undefined, + }) + ) + ) + +/** Converts the backend's version-stable structured task result into the same + * rows used for generate-time failures. Human task logs remain presentation + * only and can be reworded without breaking this contract. */ +const taskAnnotationsToRowFailures = ( + annotations: unknown, + t: TranslateFn +): ApplyTaskRowFailureSummary => { + if (!annotations || typeof annotations !== "object") { + return {rows: [], totalCount: 0, truncated: false} + } + const taskResult = annotations as Record + const failures = taskResult.reconciliation_row_failures + if (!Array.isArray(failures)) { + return {rows: [], totalCount: 0, truncated: false} + } + + const rows = failures.flatMap((failure, index) => { + if ( + !failure || + typeof failure !== "object" || + typeof (failure as Partial).voter_id !== "string" || + typeof (failure as Partial).reason !== "string" + ) { + return [] + } + const {voter_id: voterId, reason: failureReason} = failure as ApplyTaskRowFailure + return [ + { + id: `apply-failure:${voterId}:${index}`, + voterId, + field: t("reconciliation.table.rowLabel"), + label: t("reconciliation.table.rowLabel"), + oldValue: ATTR_RESET_VALUE, + newValue: ATTR_RESET_VALUE, + category: ESyncChangeCategory.ROW_FAILURE, + target: "sequent" as const, + failureReason, + }, + ] + }) + const annotatedTotal = taskResult.reconciliation_row_failure_count + const totalCount = + typeof annotatedTotal === "number" && + Number.isSafeInteger(annotatedTotal) && + annotatedTotal >= rows.length + ? annotatedTotal + : rows.length + const truncated = + taskResult.reconciliation_row_failures_truncated === true || totalCount > rows.length + return {rows, totalCount, truncated} +} + +/** + * Reconciliation wizard: Drop the reconciliation file, both diffs (external-side, Sequent-side) are + * calculated at once and shown in separate tables. The external patch is + * downloadable as soon as its diff is non-empty; Apply is disabled until that + * diff is empty (clean import), then applies the Sequent-side diff directly - + * there is no Sequent patch file to download (the Sequent-side items exist + * only as an internal document apply_reconciliation_patch reads from). + * Re-importing an already successfully applied Sequence is an explicit + * diff-only convergence check: its envelope has `apply_allowed = false` and + * this wizard never calls the apply route for it. + * + * There is no `datafix_reconciliation_import` table: the whole diff is a + * document referenced from the generate task_execution's own + * `annotations.document_id`, fetched (via the existing `fetchDocument` + * action, the same one downloads already use) and parsed client-side once. + */ +export const ReconciliationWizard: React.FC = ({ + open, + electionEventId, + onClose, +}) => { + const {t} = useTranslation() + const [addWidget, setWidgetTaskId] = useWidgetStore() + + const [step, setStep] = useState("drop") + const [fileName, setFileName] = useState(null) + const [errorMessage, setErrorMessage] = useState(null) + const [diffDocumentId, setDiffDocumentId] = useState(null) + const [envelope, setEnvelope] = useState(null) + const [generateTaskId, setGenerateTaskId] = useState(null) + const [applyTaskId, setApplyTaskId] = useState(null) + const [confirmOpen, setConfirmOpen] = useState(false) + const [downloadingDocumentId, setDownloadingDocumentId] = useState(null) + + const [getUploadUrl] = useMutation(GET_UPLOAD_URL) + const [createImport] = useMutation(CREATE_EXTERNAL_RECONCILIATION_IMPORT) + const [applyChanges] = useMutation(APPLY_EXTERNAL_RECONCILIATION_CHANGES) + + const {data: generateTaskData} = useQuery(GET_TASK_BY_ID, { + variables: {task_id: generateTaskId}, + skip: !generateTaskId || step !== "processing", + pollInterval: step === "processing" ? POLL_INTERVAL_MS : 0, + }) + const {data: applyTaskData} = useQuery(GET_TASK_BY_ID, { + variables: {task_id: applyTaskId}, + skip: !applyTaskId, + pollInterval: step === "applying" ? POLL_INTERVAL_MS : 0, + }) + + const {data: diffDocumentData} = useQuery(FETCH_DOCUMENT, { + variables: {electionEventId, documentId: diffDocumentId}, + skip: !diffDocumentId || !!envelope, + }) + + // Reads the diff-envelope document's content once its presigned URL is + // available - a plain client-side fetch+parse, the read-back symmetric + // counterpart of the raw PUT upload in handleFiles below, since a + // Document is meant to be fetched by URL, not queried inline. + useEffect(() => { + const url = diffDocumentData?.fetchDocument?.url + if (!url || envelope) { + return + } + fetch(url) + .then((response) => response.json()) + .then((data: ReconciliationDiffEnvelope) => { + setEnvelope(data) + setStep("review") + }) + .catch(() => + setErrorMessage(t("reconciliation.wizard.notifications.envelopeLoadError")) + ) + }, [diffDocumentData?.fetchDocument?.url, envelope, t]) + + const items: RawDiffItem[] = envelope?.items ?? [] + + const datafixRows = useMemo(() => toRows(items, "datafix", t), [items, t]) + const sequentRows = useMemo(() => toRows(items, "sequent", t), [items, t]) + const rowFailures = useMemo(() => toRowFailures(items, t), [items, t]) + const applyTaskAnnotations = applyTaskData?.sequent_backend_tasks_execution?.[0]?.annotations + const applyRowFailureSummary = useMemo( + () => taskAnnotationsToRowFailures(applyTaskAnnotations, t), + [applyTaskAnnotations, t] + ) + const applyRowFailures = applyRowFailureSummary.rows + const finalRowFailures = useMemo( + () => + Array.from( + new Map( + [...rowFailures, ...applyRowFailures].map((row) => [ + `${row.voterId}:${row.failureReason ?? ""}`, + row, + ]) + ).values() + ), + [rowFailures, applyRowFailures] + ) + const finalRowFailureCount = Math.max( + finalRowFailures.length, + applyRowFailureSummary.totalCount + ) + const finalRowFailuresTruncated = + applyRowFailureSummary.truncated && finalRowFailureCount > finalRowFailures.length + const isClean = datafixRows.length === 0 + const summary = useMemo( + () => countsByCategory([...datafixRows, ...sequentRows]), + [datafixRows, sequentRows] + ) + const confirmationSummary = useMemo( + () => summarizeForConfirmation(sequentRows, t), + [sequentRows, t] + ) + + // Drives the wizard's own step transitions - separate from the ambient + // addWidget/setWidgetTaskId call below (which only feeds the app-wide + // floating task widget, shown everywhere else in the app; it doesn't + // know how to advance this dialog's own steps). + const generateStatus = generateTaskData?.sequent_backend_tasks_execution?.[0]?.execution_status + const generateDocumentId = + generateTaskData?.sequent_backend_tasks_execution?.[0]?.annotations?.document_id + + // Runs as an effect (not inline during render) so it fires once per + // status/document transition instead of on every render - calling the + // setters unconditionally during render never stops re-triggering itself + // since `step` only changes once the envelope effect above finishes, + // which React reports as "Too many re-renders". + useEffect(() => { + if (step !== "processing") { + return + } + if (generateStatus === ETaskExecutionStatus.SUCCESS && generateDocumentId) { + setDiffDocumentId(generateDocumentId) + // Stays on "processing" until the effect above finishes fetching + // and parsing the envelope document, then moves to "review" + // itself. + } else if (generateStatus === ETaskExecutionStatus.FAILED) { + setErrorMessage(t("reconciliation.wizard.notifications.generateFailed")) + setStep("drop") + } + }, [step, generateStatus, generateDocumentId, t]) + + const applyStatus = applyTaskData?.sequent_backend_tasks_execution?.[0]?.execution_status + + useEffect(() => { + if (step !== "applying") { + return + } + if (applyStatus === ETaskExecutionStatus.SUCCESS) { + setStep("done") + } else if (applyStatus === ETaskExecutionStatus.FAILED) { + setErrorMessage(t("reconciliation.wizard.notifications.applyFailed")) + setStep("done") + } + }, [step, applyStatus, t]) + + const reset = useCallback(() => { + setStep("drop") + setFileName(null) + setErrorMessage(null) + setDiffDocumentId(null) + setEnvelope(null) + setGenerateTaskId(null) + setApplyTaskId(null) + setConfirmOpen(false) + setDownloadingDocumentId(null) + }, []) + + // The wizard stays mounted across open/close (the parent only flips + // `open`), so without this a closed-then-reopened wizard would still + // show whatever step it was left on - the "done" screen from a prior + // round, or a stale diff from a round closed before applying. + useEffect(() => { + if (open) { + reset() + } + }, [open, reset]) + + const handleFiles = async (files: FileList) => { + const file = files[0] + if (!file) { + return + } + setFileName(file.name) + setErrorMessage(null) + setStep("processing") + + try { + const {data: uploadData} = await getUploadUrl({ + variables: { + name: file.name, + media_type: "text/csv", + size: file.size, + is_public: false, + election_event_id: electionEventId, + }, + }) + const upload = uploadData?.get_upload_url + if (!upload?.url || !upload?.document_id) { + throw new Error(t("reconciliation.wizard.notifications.uploadUrlError")) + } + await fetch(upload.url, { + method: "PUT", + headers: {"Content-Type": "text/csv"}, + body: file, + }) + + const {data: createData} = await createImport({ + variables: { + election_event_id: electionEventId, + document_id: upload.document_id, + }, + }) + const created = createData?.create_external_reconciliation_import + if (!created?.task_execution?.id) { + throw new Error(t("reconciliation.wizard.notifications.generateTaskError")) + } + + setGenerateTaskId(created.task_execution.id) + const widget = addWidget(ETasksExecution.GENERATE_RECONCILIATION_PATCHES, false) + setWidgetTaskId(widget.identifier, created.task_execution.id) + } catch (error) { + setErrorMessage( + error instanceof Error + ? error.message + : t("reconciliation.wizard.notifications.uploadError") + ) + setStep("drop") + } + } + + const handleApply = async () => { + if (!diffDocumentId) { + return + } + setConfirmOpen(false) + + // A successful equal-Sequence retry is a diff-only convergence check + // and must not enqueue apply. A new round with no voter changes still + // does enqueue: that records its Sequence and run-level audit log. + if (!envelope?.apply_allowed) { + setStep("done") + return + } + + setStep("applying") + try { + const {data} = await applyChanges({ + variables: {election_event_id: electionEventId, diff_document_id: diffDocumentId}, + }) + const taskId = data?.apply_external_reconciliation_changes?.task_execution?.id + if (!taskId) { + throw new Error(t("reconciliation.wizard.notifications.applyTaskError")) + } + setApplyTaskId(taskId) + const widget = addWidget(ETasksExecution.APPLY_RECONCILIATION_PATCH, false) + setWidgetTaskId(widget.identifier, taskId) + } catch (error) { + setErrorMessage( + error instanceof Error + ? error.message + : t("reconciliation.wizard.notifications.applyError") + ) + setStep("done") + } + } + + return ( + <> + + + + + + {(step === "drop" || step === "processing") && ( + <> + + {t("reconciliation.wizard.drop.description")} + + {errorMessage && {errorMessage}} + {step === "drop" ? ( + + ) : ( + + + + {t("reconciliation.wizard.drop.uploading", { + fileName, + })} + + + )} + + )} + + {step === "review" && envelope && ( + <> + + + {t("reconciliation.wizard.review.fileSummary", { + fileName, + sequence: envelope.sequence, + generatedAt: formatGeneratedAt( + envelope.generated_at + ), + })} + + + + {!envelope.apply_allowed && + (datafixRows.length > 0 || sequentRows.length > 0) && ( + + {t( + "reconciliation.wizard.review.diffOnlyDifferences" + )} + + )} + + {rowFailures.length > 0 && ( + + + {t( + "reconciliation.wizard.review.rowFailuresWarning", + {count: rowFailures.length} + )} + + + + )} + + {datafixRows.length === 0 && sequentRows.length === 0 ? ( + + {t("reconciliation.wizard.review.noDifferences")} + + ) : ( + <> + + + + + + {t( + "reconciliation.wizard.review.externalDiffTitle" + )} + {datafixRows.length > 0 && + ` (${datafixRows.length})`} + + {datafixRows.length > 0 && + envelope.external_patch_document_id && ( + + )} + + {datafixRows.length > 0 && ( + + {t( + "reconciliation.wizard.review.externalDiffCaption" + )} + + )} + + + + + + + + {t( + "reconciliation.wizard.review.sequentDiffTitle" + )} + {sequentRows.length > 0 && + ` (${sequentRows.length})`} + + + {t( + "reconciliation.wizard.review.sequentDiffCaption" + )} + + + + + )} + + )} + + {(step === "applying" || step === "done") && ( + + {errorMessage && {errorMessage}} + {step === "applying" && ( + + + + {t("reconciliation.wizard.applying.inProgress")} + + + )} + {step === "done" && ( + <> + {finalRowFailureCount > 0 ? ( + + + {t( + "reconciliation.wizard.applying.rowFailures", + {count: finalRowFailureCount} + )} + + {finalRowFailuresTruncated && ( + + {t( + "reconciliation.wizard.applying.rowFailuresTruncated", + { + shown: finalRowFailures.length, + count: finalRowFailureCount, + } + )} + + )} + + + ) : !errorMessage ? ( + + {t("reconciliation.wizard.applying.success")} + + ) : null} + + )} + + )} + + {downloadingDocumentId && ( + setDownloadingDocumentId(null)} + /> + )} + + + + {(step === "drop" || step === "processing") && ( + + {t("reconciliation.wizard.actions.cancel")} + + )} + {step === "review" && ( + <> + + {t("reconciliation.wizard.actions.back")} + + 0) + } + onClick={() => + sequentRows.length > 0 + ? setConfirmOpen(true) + : handleApply() + } + > + {sequentRows.length > 0 + ? t("reconciliation.wizard.actions.apply") + : t("reconciliation.wizard.actions.next")} + + + )} + {step === "applying" && ( + + {t("reconciliation.wizard.actions.back")} + + )} + {step === "done" && ( + <> + + {t("reconciliation.wizard.actions.startOver")} + + + {t("reconciliation.wizard.actions.close")} + + + )} + + + + + + setConfirmOpen(false)}> + {t("reconciliation.wizard.confirm.title")} + + + {confirmationSummary} + + + {t("reconciliation.wizard.confirm.categoriesNote", { + categories: [ + ESyncChangeCategory.VOTED_OTHER_CHANNEL, + ESyncChangeCategory.DISABLED_DELETE_CALL, + ] + .map((category) => + t(`reconciliation.categories.${category}`, category) + ) + .join(", "), + })} + + + + + + + + + + ) +} + +export default ReconciliationWizard diff --git a/packages/admin-portal/src/resources/VoterListSync/SyncDiffTable.tsx b/packages/admin-portal/src/resources/VoterListSync/SyncDiffTable.tsx new file mode 100644 index 00000000000..324db14c6f4 --- /dev/null +++ b/packages/admin-portal/src/resources/VoterListSync/SyncDiffTable.tsx @@ -0,0 +1,90 @@ +// SPDX-FileCopyrightText: 2026 Sequent Tech Inc +// +// SPDX-License-Identifier: AGPL-3.0-only + +import React from "react" +import {Box, Chip, Typography} from "@mui/material" +import {DataGrid, GridColDef, GridRenderCellParams} from "@mui/x-data-grid" +import {useTranslation} from "react-i18next" +import {SyncDiffRow} from "./types" +import {CATEGORY_COLORS} from "./constants" + +interface SyncDiffTableProps { + rows: SyncDiffRow[] + emptyMessage?: string +} + +/** + * Same visual language as ReviewChangesTable (old value struck through, new + * value plain) but built on DataGrid since a reconciliation diff spans many + * voters instead of a single record's fields. Callers split rows by `target` + * into separate external-side/Sequent-side tables, so this component doesn't + * need to render which side a row belongs to. Client-side paginated: the + * whole diff is fetched at once. + */ +export const SyncDiffTable: React.FC = ({rows, emptyMessage}) => { + const {t} = useTranslation() + const columns: GridColDef[] = [ + {field: "voterId", headerName: t("reconciliation.table.voterId"), width: 120}, + {field: "label", headerName: t("reconciliation.table.field"), width: 180}, + { + field: "category", + headerName: t("reconciliation.table.category"), + width: 200, + renderCell: (params: GridRenderCellParams) => ( + + ), + }, + { + field: "oldValue", + headerName: t("reconciliation.table.currentValue"), + width: 160, + renderCell: (params: GridRenderCellParams) => ( + + {params.row.oldValue} + + ), + }, + {field: "newValue", headerName: t("reconciliation.table.newValue"), width: 160}, + { + field: "failureReason", + headerName: t("reconciliation.table.reason"), + flex: 1, + minWidth: 220, + }, + ] + + if (rows.length === 0) { + return ( + + {emptyMessage ?? t("reconciliation.table.noDifferences")} + + ) + } + + return ( + row.id} + initialState={{ + pagination: { + paginationModel: {pageSize: 10}, + }, + }} + pageSizeOptions={[10, 25, 50]} + disableRowSelectionOnClick + /> + ) +} + +export default SyncDiffTable diff --git a/packages/admin-portal/src/resources/VoterListSync/constants.ts b/packages/admin-portal/src/resources/VoterListSync/constants.ts new file mode 100644 index 00000000000..a2d5de9cc6a --- /dev/null +++ b/packages/admin-portal/src/resources/VoterListSync/constants.ts @@ -0,0 +1,29 @@ +// SPDX-FileCopyrightText: 2026 Sequent Tech Inc +// +// SPDX-License-Identifier: AGPL-3.0-only + +import {ESyncChangeCategory} from "./types" + +export const CATEGORY_COLORS: Record< + ESyncChangeCategory, + "default" | "success" | "warning" | "error" | "info" +> = { + [ESyncChangeCategory.VOTED_INTERNET]: "info", + [ESyncChangeCategory.VOTED_OTHER_CHANNEL]: "warning", + [ESyncChangeCategory.DISABLED_DELETE_CALL]: "warning", + [ESyncChangeCategory.DELETION_REVERTED]: "info", + [ESyncChangeCategory.PROFILE_UPDATE]: "default", + [ESyncChangeCategory.VOTER_ADDED]: "success", + [ESyncChangeCategory.REENABLED]: "success", + [ESyncChangeCategory.VOTED_UNMARKED]: "warning", + [ESyncChangeCategory.ROW_FAILURE]: "error", +} + +// Acceptance criteria: the apply-patch confirmation dialog highlights the +// categories that touch voted status or disable voters. +export const HIGHLIGHTED_CATEGORIES = new Set([ + ESyncChangeCategory.VOTED_INTERNET, + ESyncChangeCategory.VOTED_OTHER_CHANNEL, + ESyncChangeCategory.VOTED_UNMARKED, + ESyncChangeCategory.DISABLED_DELETE_CALL, +]) diff --git a/packages/admin-portal/src/resources/VoterListSync/types.ts b/packages/admin-portal/src/resources/VoterListSync/types.ts new file mode 100644 index 00000000000..387931c0703 --- /dev/null +++ b/packages/admin-portal/src/resources/VoterListSync/types.ts @@ -0,0 +1,65 @@ +// SPDX-FileCopyrightText: 2026 Sequent Tech Inc +// +// SPDX-License-Identifier: AGPL-3.0-only + +/** + * Shapes used by the external voter registry reconciliation wizard. + * Wire values match the backend's Rust enums exactly, so these describe + * GraphQL response shapes directly with no translation layer. + */ + +// Source-of-truth categories from the "Handling Inconsistencies" section, +// plus REENABLED (a voter Sequent disabled solely because of a prior +// Datafix delete call is re-enabled when the file no longer reports them +// Deleted). +export enum ESyncChangeCategory { + VOTED_INTERNET = "VOTED_INTERNET", // A: Sequent holds a valid internet ballot, Datafix says NONE + VOTED_OTHER_CHANNEL = "VOTED_OTHER_CHANNEL", // B: Datafix reports a non-INTERNET channel + DISABLED_DELETE_CALL = "DISABLED_DELETE_CALL", // C: Deleted=true in the reconciliation file + DELETION_REVERTED = "DELETION_REVERTED", // C exception: voter has voted, deletion is not applied + PROFILE_UPDATE = "PROFILE_UPDATE", // C: Ward/Poll/SchoolSupportCode/DoB changed + VOTER_ADDED = "VOTER_ADDED", // D: voter missing on one side + REENABLED = "REENABLED", // re-enabled after a Datafix delete-call disable is undone + VOTED_UNMARKED = "VOTED_UNMARKED", // file-driven equivalent of /unmark-voted + ROW_FAILURE = "ROW_FAILURE", // CountyMun mismatch or the voted-other-channel guard +} + +export type ESyncPatchTarget = "datafix" | "sequent" + +export interface SyncFileMeta { + sequence: number + generatedAt: number // unix seconds, UTC +} + +// Fixed column order for the "Patch Files Format" - every patch row carries +// all of these as _old/_new pairs, VoterID is the row key and +// is never one of them (VoterID cannot change). +export const PATCH_FIELDS = [ + "CountyMun", + "DoB", + "Ward", + "Poll", + "SchoolSupportCode", + "Channel", + "Deleted", +] as const + +export type PatchField = (typeof PATCH_FIELDS)[number] + +export interface SyncDiffRow { + id: string + voterId: string + field: string + label: string + oldValue: string + newValue: string + category: ESyncChangeCategory + target: ESyncPatchTarget + /** Set for ROW_FAILURE rows: why the row was excluded from the patch. */ + failureReason?: string +} + +export interface RowFailure { + voterId: string + reason: string +} diff --git a/packages/admin-portal/src/resources/VoterListSync/utils.ts b/packages/admin-portal/src/resources/VoterListSync/utils.ts new file mode 100644 index 00000000000..bd59cd0b290 --- /dev/null +++ b/packages/admin-portal/src/resources/VoterListSync/utils.ts @@ -0,0 +1,11 @@ +// SPDX-FileCopyrightText: 2026 Sequent Tech Inc +// +// SPDX-License-Identifier: AGPL-3.0-only + +/** Formats a `GeneratedAt` unix-seconds value for display. */ +export const formatGeneratedAt = (unixSeconds: number): string => { + if (!unixSeconds) { + return "-" + } + return new Date(unixSeconds * 1000).toLocaleString() +} diff --git a/packages/admin-portal/src/translations/cat.ts b/packages/admin-portal/src/translations/cat.ts index d512e99da5f..e3b8d034350 100644 --- a/packages/admin-portal/src/translations/cat.ts +++ b/packages/admin-portal/src/translations/cat.ts @@ -104,6 +104,104 @@ const catalanTranslation: TranslationType = { next: "Següent", }, }, + reconciliation: { + menuButton: "Sincr. votants ext.", + categories: { + VOTED_INTERNET: "Ha votat per Internet", + VOTED_OTHER_CHANNEL: "Ha votat per un altre canal", + DISABLED_DELETE_CALL: "Votant deshabilitat", + DELETION_REVERTED: "Eliminació revertida", + PROFILE_UPDATE: "Perfil actualitzat", + VOTER_ADDED: "Votant afegit", + REENABLED: "Votant rehabilitat", + VOTED_UNMARKED: "Votant desmarcat com a votat", + ROW_FAILURE: "Error de fila", + }, + table: { + voterId: "ID de Votant", + field: "Camp", + category: "Categoria", + currentValue: "Valor actual", + newValue: "Valor nou", + reason: "Motiu", + rowLabel: "Fila", + noDifferences: "No s'han trobat diferències - els sistemes estan sincronitzats.", + }, + wizard: { + title: "Sincronització de reconciliació externa", + subtitle: "Sincronitza la llista de votants amb el sistema extern", + drop: { + description: + "Deixa anar el fitxer de reconciliació generat pel sistema extern - ambdós diffs (costat extern i costat Sequent) es calculen automàticament i es mostren en taules separades.", + fileFormatLabel: "Fitxer CSV", + uploading: "Pujant {{fileName}} i calculant ambdós diffs...", + }, + review: { + fileSummary: "{{fileName}} - Seqüència {{sequence}}, generat {{generatedAt}}", + rowFailuresWarning: + "{{count}} fila(es) no s'han pogut reconciliar de manera segura i queden excloses d'ambdós diffs - vegeu els detalls a continuació.", + noDifferences: "No hi ha diferències - ambdós sistemes ja estan sincronitzats.", + diffOnlyDifferences: + "Aquesta és una comprovació de convergència d'una seqüència ja aplicada. Les diferències es mostren per fer-ne seguiment, però aquesta ronda no es pot tornar a aplicar.", + externalDiffTitle: "Diff extern", + sequentDiffTitle: "Diff de Sequent", + downloadExternalPatch: "Descarregar pedaç extern", + externalDiffCaption: + "Descarrega el pedaç i lliura'l al sistema extern fora d'aquesta eina. Un cop l'apliqui i generi el següent fitxer de reconciliació, fes clic a 'Enrere' i deixa anar aquest fitxer - 'Aplicar' s'habilita quan aquesta taula estigui buida.", + noExternalDifferences: "No hi ha diferències del costat extern.", + sequentDiffCaption: + "Aplica els canvis directament a Sequent fent clic a 'Aplicar' - no es genera cap fitxer de pedaç per aquests.", + noSequentDifferences: "No hi ha diferències del costat de Sequent.", + }, + applying: { + inProgress: "Aplicant els canvis del costat de Sequent...", + rowFailures: + "{{count}} fila(es) han quedat excloses d'aquesta ronda i necessiten seguiment manual - vegeu els detalls a continuació.", + rowFailuresTruncated: + "Es mostren les primeres {{shown}} de {{count}} errades de fila. Resoleu la causa comuna i torneu-ho a provar per veure les errades restants.", + success: "Tots els canvis del costat de Sequent s'han aplicat correctament.", + }, + actions: { + cancel: "Cancel·lar", + back: "Enrere", + apply: "Aplicar", + next: "Següent", + startOver: "Tornar a començar", + close: "Tancar", + }, + confirm: { + title: "Confirmar canvis de reconciliació", + categoriesNote: + "Les categories ressaltades en taronja ({{categories}}) afecten l'estat de vot o deshabiliten votants.", + applyChanges: "Aplicar canvis", + continue: "Continuar", + }, + summary: { + votedOtherChannel: + "marca {{count}} votant(s) com a votat(s) per un altre canal", + disabled: "deshabilita {{count}} votant(s)", + reenabled: "rehabilita {{count}} votant(s)", + votedUnmarked: "desmarca {{count}} votant(s) com a votat", + profileUpdated: "actualitza {{count}} perfil(s)", + voterAdded: "afegeix {{count}} votant(s)", + prefix: "Això aplicarà canvis que {{parts}}.", + empty: "No hi ha canvis del costat de Sequent per aplicar.", + }, + notifications: { + envelopeLoadError: + "No s'ha pogut carregar el diff de reconciliació - torna-ho a provar.", + generateFailed: + "No s'ha pogut calcular el diff de reconciliació - consulta el widget de tasques per a més detalls.", + applyFailed: + "No s'han pogut aplicar els canvis del costat de Sequent - consulta el widget de tasques per a més detalls.", + uploadUrlError: "No s'ha pogut obtenir una URL de pujada", + generateTaskError: "No s'ha pogut iniciar la tasca de diff de reconciliació", + uploadError: "No s'ha pogut pujar el fitxer de reconciliació", + applyTaskError: "No s'ha pogut iniciar la tasca d'aplicació", + applyError: "No s'han pogut aplicar els canvis de reconciliació", + }, + }, + }, tasksScreen: { noPermissions: "No tens permís per accedir als registres.", title: "Execució de tasques", diff --git a/packages/admin-portal/src/translations/en.ts b/packages/admin-portal/src/translations/en.ts index 68c2ea32bda..f7596c6d774 100644 --- a/packages/admin-portal/src/translations/en.ts +++ b/packages/admin-portal/src/translations/en.ts @@ -103,6 +103,102 @@ const englishTranslation = { next: "Next", }, }, + reconciliation: { + menuButton: "Ext. voters sync", + categories: { + VOTED_INTERNET: "Voted via Internet", + VOTED_OTHER_CHANNEL: "Voted via other channel", + DISABLED_DELETE_CALL: "Voter disabled", + DELETION_REVERTED: "Deletion reverted", + PROFILE_UPDATE: "Profile update", + VOTER_ADDED: "Voter added", + REENABLED: "Voter re-enabled", + VOTED_UNMARKED: "Voter unmarked as voted", + ROW_FAILURE: "Row failure", + }, + table: { + voterId: "Voter ID", + field: "Field", + category: "Category", + currentValue: "Current value", + newValue: "New value", + reason: "Reason", + rowLabel: "Row", + noDifferences: "No differences found - the systems are in sync.", + }, + wizard: { + title: "External reconciliation sync", + subtitle: "Synchronize the voter list with the external system", + drop: { + description: + "Drop the reconciliation file the external system produced - both diffs (external-side and Sequent-side) are calculated automatically and shown in separate tables.", + fileFormatLabel: "CSV file", + uploading: "Uploading {{fileName}} and calculating both diffs...", + }, + review: { + fileSummary: "{{fileName}} - Sequence {{sequence}}, generated {{generatedAt}}", + rowFailuresWarning: + "{{count}} row(s) could not be reconciled safely and are excluded from both diffs - see below for details.", + noDifferences: "No differences - the two systems are already in sync.", + diffOnlyDifferences: + "This is a convergence check for an already applied Sequence. Differences are shown for follow-up, but this round cannot be applied again.", + externalDiffTitle: "External diff", + sequentDiffTitle: "Sequent diff", + downloadExternalPatch: "Download external patch", + externalDiffCaption: + "Download the patch and deliver it to the external system outside this tool. Once it applies the patch and produces the next reconciliation file, click 'Back' and drop that file in - 'Apply' unlocks once this table is empty.", + noExternalDifferences: "No external-side differences.", + sequentDiffCaption: + "Apply changes directly to Sequent by clicking 'Apply' - no patch file is generated for these.", + noSequentDifferences: "No Sequent-side differences.", + }, + applying: { + inProgress: "Applying Sequent-side changes...", + rowFailures: + "{{count}} row(s) were excluded from this round and need manual follow-up - see below for details.", + rowFailuresTruncated: + "Showing the first {{shown}} of {{count}} row failures. Resolve the shared cause and retry to see any remaining failures.", + success: "All Sequent-side changes applied successfully.", + }, + actions: { + cancel: "Cancel", + back: "Back", + apply: "Apply", + next: "Next", + startOver: "Start over", + close: "Close", + }, + confirm: { + title: "Confirm reconciliation changes", + categoriesNote: + "Categories outlined in orange ({{categories}}) touch voted status or disable voters.", + applyChanges: "Apply changes", + continue: "Continue", + }, + summary: { + votedOtherChannel: "marks {{count}} voter(s) as voted via other channels", + disabled: "disables {{count}} voter(s)", + reenabled: "re-enables {{count}} voter(s)", + votedUnmarked: "unmarks {{count}} voter(s) as voted", + profileUpdated: "updates {{count}} profile(s)", + voterAdded: "adds {{count}} voter(s)", + prefix: "This will apply changes that {{parts}}.", + empty: "There are no Sequent-side changes to apply.", + }, + notifications: { + envelopeLoadError: "Failed to load the reconciliation diff - please try again.", + generateFailed: + "Failed to calculate the reconciliation diff - see the task widget for details.", + applyFailed: + "Failed to apply the Sequent-side changes - see the task widget for details.", + uploadUrlError: "Failed to get an upload URL", + generateTaskError: "Failed to start the reconciliation diff task", + uploadError: "Failed to upload the reconciliation file", + applyTaskError: "Failed to start the apply task", + applyError: "Failed to apply the reconciliation changes", + }, + }, + }, logsScreen: { noPermissions: "You don't have permission to access logs.", title: "Logs", diff --git a/packages/admin-portal/src/translations/es.ts b/packages/admin-portal/src/translations/es.ts index a38e27861e2..423a48c6295 100644 --- a/packages/admin-portal/src/translations/es.ts +++ b/packages/admin-portal/src/translations/es.ts @@ -104,6 +104,103 @@ const spanishTranslation: TranslationType = { next: "Siguiente", }, }, + reconciliation: { + menuButton: "Sincr. votantes ext.", + categories: { + VOTED_INTERNET: "Votó por Internet", + VOTED_OTHER_CHANNEL: "Votó por otro canal", + DISABLED_DELETE_CALL: "Votante deshabilitado", + DELETION_REVERTED: "Eliminación revertida", + PROFILE_UPDATE: "Perfil actualizado", + VOTER_ADDED: "Votante añadido", + REENABLED: "Votante rehabilitado", + VOTED_UNMARKED: "Votante desmarcado como votante", + ROW_FAILURE: "Fallo de fila", + }, + table: { + voterId: "ID de Votante", + field: "Campo", + category: "Categoría", + currentValue: "Valor actual", + newValue: "Valor nuevo", + reason: "Motivo", + rowLabel: "Fila", + noDifferences: "No se encontraron diferencias - los sistemas están sincronizados.", + }, + wizard: { + title: "Sincronización de reconciliación externa", + subtitle: "Sincroniza la lista de votantes con el sistema externo", + drop: { + description: + "Suelta el archivo de reconciliación generado por el sistema externo - ambos diffs (lado externo y lado Sequent) se calculan automáticamente y se muestran en tablas separadas.", + fileFormatLabel: "Archivo CSV", + uploading: "Subiendo {{fileName}} y calculando ambos diffs...", + }, + review: { + fileSummary: "{{fileName}} - Secuencia {{sequence}}, generado {{generatedAt}}", + rowFailuresWarning: + "{{count}} fila(s) no se pudieron reconciliar de forma segura y quedan excluidas de ambos diffs - ver detalles abajo.", + noDifferences: "No hay diferencias - ambos sistemas ya están sincronizados.", + diffOnlyDifferences: + "Esta es una comprobación de convergencia de una secuencia ya aplicada. Las diferencias se muestran para su seguimiento, pero esta ronda no se puede volver a aplicar.", + externalDiffTitle: "Diff externo", + sequentDiffTitle: "Diff de Sequent", + downloadExternalPatch: "Descargar parche externo", + externalDiffCaption: + "Descarga el parche y entrégaselo al sistema externo fuera de esta herramienta. Cuando lo aplique y genere el siguiente archivo de reconciliación, haz clic en 'Atrás' y suelta ese archivo - 'Aplicar' se habilita cuando esta tabla esté vacía.", + noExternalDifferences: "No hay diferencias del lado externo.", + sequentDiffCaption: + "Aplica los cambios directamente a Sequent haciendo clic en 'Aplicar' - no se genera archivo de parche para estos.", + noSequentDifferences: "No hay diferencias del lado de Sequent.", + }, + applying: { + inProgress: "Aplicando cambios del lado de Sequent...", + rowFailures: + "{{count}} fila(s) quedaron excluidas de esta ronda y requieren seguimiento manual - ver detalles abajo.", + rowFailuresTruncated: + "Se muestran los primeros {{shown}} de {{count}} fallos de fila. Resuelve la causa común y vuelve a intentarlo para ver los fallos restantes.", + success: "Todos los cambios del lado de Sequent se aplicaron correctamente.", + }, + actions: { + cancel: "Cancelar", + back: "Atrás", + apply: "Aplicar", + next: "Siguiente", + startOver: "Empezar de nuevo", + close: "Cerrar", + }, + confirm: { + title: "Confirmar cambios de reconciliación", + categoriesNote: + "Las categorías resaltadas en naranja ({{categories}}) afectan el estado de voto o deshabilitan votantes.", + applyChanges: "Aplicar cambios", + continue: "Continuar", + }, + summary: { + votedOtherChannel: "marca {{count}} votante(s) como votado(s) por otro canal", + disabled: "deshabilita {{count}} votante(s)", + reenabled: "rehabilita {{count}} votante(s)", + votedUnmarked: "desmarca {{count}} votante(s) como votante", + profileUpdated: "actualiza {{count}} perfil(es)", + voterAdded: "añade {{count}} votante(s)", + prefix: "Esto aplicará cambios que {{parts}}.", + empty: "No hay cambios del lado de Sequent para aplicar.", + }, + notifications: { + envelopeLoadError: + "No se pudo cargar el diff de reconciliación - inténtalo de nuevo.", + generateFailed: + "No se pudo calcular el diff de reconciliación - consulta el widget de tareas para más detalles.", + applyFailed: + "No se pudieron aplicar los cambios del lado de Sequent - consulta el widget de tareas para más detalles.", + uploadUrlError: "No se pudo obtener una URL de subida", + generateTaskError: "No se pudo iniciar la tarea de diff de reconciliación", + uploadError: "No se pudo subir el archivo de reconciliación", + applyTaskError: "No se pudo iniciar la tarea de aplicación", + applyError: "No se pudieron aplicar los cambios de reconciliación", + }, + }, + }, logsScreen: { noPermissions: "No tienes permiso para acceder a las bitácoras.", title: "Bitácoras", diff --git a/packages/admin-portal/src/translations/eu.ts b/packages/admin-portal/src/translations/eu.ts index c885d8d5c48..3c267f64feb 100644 --- a/packages/admin-portal/src/translations/eu.ts +++ b/packages/admin-portal/src/translations/eu.ts @@ -104,6 +104,104 @@ const basqueTranslation: TranslationType = { next: "Hurrengoa", }, }, + reconciliation: { + menuButton: "Kanpo bozk. sink.", + categories: { + VOTED_INTERNET: "Internet bidez bozkatu du", + VOTED_OTHER_CHANNEL: "Beste kanal batetik bozkatu du", + DISABLED_DELETE_CALL: "Bozkatzailea desgaituta", + DELETION_REVERTED: "Ezabaketa desegina", + PROFILE_UPDATE: "Profila eguneratuta", + VOTER_ADDED: "Bozkatzailea gehituta", + REENABLED: "Bozkatzailea berriz gaituta", + VOTED_UNMARKED: "Bozkatzaileari boto-marka kendu zaio", + ROW_FAILURE: "Errenkada-errorea", + }, + table: { + voterId: "Bozkatzailearen IDa", + field: "Eremua", + category: "Kategoria", + currentValue: "Uneko balioa", + newValue: "Balio berria", + reason: "Arrazoia", + rowLabel: "Errenkada", + noDifferences: "Ez da desberdintasunik aurkitu - sistemak sinkronizatuta daude.", + }, + wizard: { + title: "Kanpoko bat-egite sinkronizazioa", + subtitle: "Sinkronizatu bozkatzaileen zerrenda kanpoko sistemarekin", + drop: { + description: + "Jaregin kanpoko sistemak sortutako bat-egite fitxategia - bi diffak (kanpoko aldea eta Sequent aldea) automatikoki kalkulatzen dira eta taula bereizietan erakusten dira.", + fileFormatLabel: "CSV fitxategia", + uploading: "{{fileName}} igotzen eta bi diffak kalkulatzen...", + }, + review: { + fileSummary: "{{fileName}} - {{sequence}} sekuentzia, {{generatedAt}} sortua", + rowFailuresWarning: + "{{count}} errenkada ezin izan dira modu seguruan bateratu eta bi diffetatik kanpo geratzen dira - ikusi xehetasunak behean.", + noDifferences: + "Ez dago desberdintasunik - bi sistemak jada sinkronizatuta daude.", + diffOnlyDifferences: + "Dagoeneko aplikatutako sekuentzia baten konbergentzia-egiaztapena da hau. Aldeak jarraipenerako erakusten dira, baina txanda hau ezin da berriro aplikatu.", + externalDiffTitle: "Kanpoko diff-a", + sequentDiffTitle: "Sequent diff-a", + downloadExternalPatch: "Deskargatu kanpoko adabakia", + externalDiffCaption: + "Deskargatu adabakia eta eman ezazu kanpoko sistemari tresna honetatik kanpo. Adabakia aplikatu eta hurrengo bat-egite fitxategia sortu ondoren, sakatu 'Atzera' eta jaregin fitxategi hori - 'Aplikatu' gaitzen da taula hau hutsik dagoenean.", + noExternalDifferences: "Ez dago desberdintasunik kanpoko aldean.", + sequentDiffCaption: + "Aplikatu aldaketak zuzenean Sequent-i 'Aplikatu' sakatuz - hauetarako ez da adabaki-fitxategirik sortzen.", + noSequentDifferences: "Ez dago desberdintasunik Sequent aldean.", + }, + applying: { + inProgress: "Sequent aldeko aldaketak aplikatzen...", + rowFailures: + "{{count}} errenkada bat-egite honetatik kanpo geratu dira eta eskuzko jarraipena behar dute - ikusi xehetasunak behean.", + rowFailuresTruncated: + "{{count}} errenkada-hutsegiteetatik lehen {{shown}} erakusten dira. Ebatzi kausa komuna eta saiatu berriro gainerakoak ikusteko.", + success: "Sequent aldeko aldaketa guztiak ondo aplikatu dira.", + }, + actions: { + cancel: "Ezeztatu", + back: "Atzera", + apply: "Aplikatu", + next: "Hurrengoa", + startOver: "Hasi berriro", + close: "Itxi", + }, + confirm: { + title: "Berretsi bat-egite aldaketak", + categoriesNote: + "Laranjaz nabarmendutako kategoriek ({{categories}}) boto-egoerari eragiten diote edo bozkatzaileak desgaitzen dituzte.", + applyChanges: "Aplikatu aldaketak", + continue: "Jarraitu", + }, + summary: { + votedOtherChannel: + "{{count}} bozkatzaile beste kanal batetik bozkatu izana adierazten du", + disabled: "{{count}} bozkatzaile desgaitzen ditu", + reenabled: "{{count}} bozkatzaile berriz gaitzen ditu", + votedUnmarked: "{{count}} bozkatzaileari boto-marka kentzen dio", + profileUpdated: "{{count}} profil eguneratzen ditu", + voterAdded: "{{count}} bozkatzaile gehitzen ditu", + prefix: "Honek {{parts}} dituzten aldaketak aplikatuko ditu.", + empty: "Ez dago Sequent aldeko aldaketarik aplikatzeko.", + }, + notifications: { + envelopeLoadError: "Ezin izan da bat-egite diff-a kargatu - saiatu berriro.", + generateFailed: + "Ezin izan da bat-egite diff-a kalkulatu - ikusi zereginen widget-a xehetasunetarako.", + applyFailed: + "Ezin izan dira Sequent aldeko aldaketak aplikatu - ikusi zereginen widget-a xehetasunetarako.", + uploadUrlError: "Ezin izan da igotzeko URLa lortu", + generateTaskError: "Ezin izan da bat-egite diff zeregina abiarazi", + uploadError: "Ezin izan da bat-egite fitxategia igo", + applyTaskError: "Ezin izan da aplikatzeko zeregina abiarazi", + applyError: "Ezin izan dira bat-egite aldaketak aplikatu", + }, + }, + }, logsScreen: { noPermissions: "Ez duzu egunkariak atzitzeko baimenik.", title: "Egunkariak", diff --git a/packages/admin-portal/src/translations/fr.ts b/packages/admin-portal/src/translations/fr.ts index d642fcaadf4..7d809d28979 100644 --- a/packages/admin-portal/src/translations/fr.ts +++ b/packages/admin-portal/src/translations/fr.ts @@ -104,6 +104,104 @@ const frenchTranslation: TranslationType = { next: "Suivant", }, }, + reconciliation: { + menuButton: "Sync. élect. ext.", + categories: { + VOTED_INTERNET: "A voté par Internet", + VOTED_OTHER_CHANNEL: "A voté par un autre canal", + DISABLED_DELETE_CALL: "Électeur désactivé", + DELETION_REVERTED: "Suppression annulée", + PROFILE_UPDATE: "Profil mis à jour", + VOTER_ADDED: "Électeur ajouté", + REENABLED: "Électeur réactivé", + VOTED_UNMARKED: "Vote de l’électeur annulé", + ROW_FAILURE: "Échec de ligne", + }, + table: { + voterId: "ID Électeur", + field: "Champ", + category: "Catégorie", + currentValue: "Valeur actuelle", + newValue: "Nouvelle valeur", + reason: "Motif", + rowLabel: "Ligne", + noDifferences: "Aucune différence trouvée - les systèmes sont synchronisés.", + }, + wizard: { + title: "Synchronisation de réconciliation externe", + subtitle: "Synchronisez la liste des électeurs avec le système externe", + drop: { + description: + "Déposez le fichier de réconciliation généré par le système externe - les deux diffs (côté externe et côté Sequent) sont calculés automatiquement et affichés dans des tableaux séparés.", + fileFormatLabel: "Fichier CSV", + uploading: "Téléversement de {{fileName}} et calcul des deux diffs...", + }, + review: { + fileSummary: "{{fileName}} - Séquence {{sequence}}, généré {{generatedAt}}", + rowFailuresWarning: + "{{count}} ligne(s) n'ont pas pu être réconciliées en toute sécurité et sont exclues des deux diffs - voir les détails ci-dessous.", + noDifferences: "Aucune différence - les deux systèmes sont déjà synchronisés.", + diffOnlyDifferences: + "Il s'agit d'un contrôle de convergence pour une séquence déjà appliquée. Les différences sont affichées pour suivi, mais ce cycle ne peut pas être appliqué à nouveau.", + externalDiffTitle: "Diff externe", + sequentDiffTitle: "Diff Sequent", + downloadExternalPatch: "Télécharger le correctif externe", + externalDiffCaption: + "Téléchargez le correctif et remettez-le au système externe en dehors de cet outil. Une fois qu'il l'applique et produit le fichier de réconciliation suivant, cliquez sur 'Retour' et déposez ce fichier - 'Appliquer' se débloque une fois ce tableau vide.", + noExternalDifferences: "Aucune différence côté externe.", + sequentDiffCaption: + "Appliquez les changements directement à Sequent en cliquant sur 'Appliquer' - aucun fichier de correctif n'est généré pour ceux-ci.", + noSequentDifferences: "Aucune différence côté Sequent.", + }, + applying: { + inProgress: "Application des changements côté Sequent...", + rowFailures: + "{{count}} ligne(s) ont été exclues de ce cycle et nécessitent un suivi manuel - voir les détails ci-dessous.", + rowFailuresTruncated: + "Affichage des {{shown}} premiers échecs de ligne sur {{count}}. Corrigez la cause commune et réessayez pour voir les échecs restants.", + success: "Tous les changements côté Sequent ont été appliqués avec succès.", + }, + actions: { + cancel: "Annuler", + back: "Retour", + apply: "Appliquer", + next: "Suivant", + startOver: "Recommencer", + close: "Fermer", + }, + confirm: { + title: "Confirmer les changements de réconciliation", + categoriesNote: + "Les catégories surlignées en orange ({{categories}}) affectent le statut de vote ou désactivent des électeurs.", + applyChanges: "Appliquer les changements", + continue: "Continuer", + }, + summary: { + votedOtherChannel: + "marque {{count}} électeur(s) comme ayant voté par un autre canal", + disabled: "désactive {{count}} électeur(s)", + reenabled: "réactive {{count}} électeur(s)", + votedUnmarked: "annule le vote de {{count}} électeur(s)", + profileUpdated: "met à jour {{count}} profil(s)", + voterAdded: "ajoute {{count}} électeur(s)", + prefix: "Ceci appliquera des changements qui {{parts}}.", + empty: "Il n'y a aucun changement côté Sequent à appliquer.", + }, + notifications: { + envelopeLoadError: + "Impossible de charger le diff de réconciliation - veuillez réessayer.", + generateFailed: + "Impossible de calculer le diff de réconciliation - consultez le widget de tâches pour plus de détails.", + applyFailed: + "Impossible d'appliquer les changements côté Sequent - consultez le widget de tâches pour plus de détails.", + uploadUrlError: "Impossible d'obtenir une URL de téléversement", + generateTaskError: "Impossible de démarrer la tâche de diff de réconciliation", + uploadError: "Impossible de téléverser le fichier de réconciliation", + applyTaskError: "Impossible de démarrer la tâche d'application", + applyError: "Impossible d'appliquer les changements de réconciliation", + }, + }, + }, logsScreen: { noPermissions: "Vous n'avez pas la permission d'accéder aux journaux.", title: "Journaux", diff --git a/packages/admin-portal/src/translations/gl.ts b/packages/admin-portal/src/translations/gl.ts index 9a469698adb..10f65a0e16a 100644 --- a/packages/admin-portal/src/translations/gl.ts +++ b/packages/admin-portal/src/translations/gl.ts @@ -104,6 +104,103 @@ const galegoTranslation: TranslationType = { next: "Seguinte", }, }, + reconciliation: { + menuButton: "Sincr. votantes ext.", + categories: { + VOTED_INTERNET: "Votou por Internet", + VOTED_OTHER_CHANNEL: "Votou por outra canle", + DISABLED_DELETE_CALL: "Votante desactivado", + DELETION_REVERTED: "Eliminación revertida", + PROFILE_UPDATE: "Perfil actualizado", + VOTER_ADDED: "Votante engadido", + REENABLED: "Votante reactivado", + VOTED_UNMARKED: "Votante desmarcado como votante", + ROW_FAILURE: "Erro de fila", + }, + table: { + voterId: "ID de Votante", + field: "Campo", + category: "Categoría", + currentValue: "Valor actual", + newValue: "Valor novo", + reason: "Motivo", + rowLabel: "Fila", + noDifferences: "Non se atoparon diferenzas - os sistemas están sincronizados.", + }, + wizard: { + title: "Sincronización de reconciliación externa", + subtitle: "Sincroniza a lista de votantes co sistema externo", + drop: { + description: + "Solta o ficheiro de reconciliación xerado polo sistema externo - ambos os diffs (lado externo e lado Sequent) calcúlanse automaticamente e móstranse en táboas separadas.", + fileFormatLabel: "Ficheiro CSV", + uploading: "Subindo {{fileName}} e calculando ambos os diffs...", + }, + review: { + fileSummary: "{{fileName}} - Secuencia {{sequence}}, xerado {{generatedAt}}", + rowFailuresWarning: + "{{count}} fila(s) non se puideron reconciliar de forma segura e quedan excluídas de ambos os diffs - ver detalles abaixo.", + noDifferences: "Non hai diferenzas - ambos os sistemas xa están sincronizados.", + diffOnlyDifferences: + "Esta é unha comprobación de converxencia dunha secuencia xa aplicada. As diferenzas móstranse para o seu seguimento, pero esta rolda non se pode volver aplicar.", + externalDiffTitle: "Diff externo", + sequentDiffTitle: "Diff de Sequent", + downloadExternalPatch: "Descargar parche externo", + externalDiffCaption: + "Descarga o parche e entrégao ao sistema externo fóra desta ferramenta. Unha vez que o aplique e xere o seguinte ficheiro de reconciliación, preme en 'Atrás' e solta ese ficheiro - 'Aplicar' actívase cando esta táboa estea baleira.", + noExternalDifferences: "Non hai diferenzas do lado externo.", + sequentDiffCaption: + "Aplica os cambios directamente a Sequent premendo en 'Aplicar' - non se xera ficheiro de parche para estes.", + noSequentDifferences: "Non hai diferenzas do lado de Sequent.", + }, + applying: { + inProgress: "Aplicando os cambios do lado de Sequent...", + rowFailures: + "{{count}} fila(s) quedaron excluídas desta ronda e precisan seguimento manual - ver detalles abaixo.", + rowFailuresTruncated: + "Amósanse os primeiros {{shown}} de {{count}} erros de fila. Resolve a causa común e téntao de novo para ver os erros restantes.", + success: "Todos os cambios do lado de Sequent aplicáronse correctamente.", + }, + actions: { + cancel: "Cancelar", + back: "Atrás", + apply: "Aplicar", + next: "Seguinte", + startOver: "Comezar de novo", + close: "Pechar", + }, + confirm: { + title: "Confirmar cambios de reconciliación", + categoriesNote: + "As categorías resaltadas en laranxa ({{categories}}) afectan o estado de voto ou desactivan votantes.", + applyChanges: "Aplicar cambios", + continue: "Continuar", + }, + summary: { + votedOtherChannel: "marca {{count}} votante(s) como votado(s) por outra canle", + disabled: "desactiva {{count}} votante(s)", + reenabled: "reactiva {{count}} votante(s)", + votedUnmarked: "desmarca {{count}} votante(s) como votante", + profileUpdated: "actualiza {{count}} perfil(s)", + voterAdded: "engade {{count}} votante(s)", + prefix: "Isto aplicará cambios que {{parts}}.", + empty: "Non hai cambios do lado de Sequent para aplicar.", + }, + notifications: { + envelopeLoadError: + "Non se puido cargar o diff de reconciliación - téntao de novo.", + generateFailed: + "Non se puido calcular o diff de reconciliación - consulta o widget de tarefas para máis detalles.", + applyFailed: + "Non se puideron aplicar os cambios do lado de Sequent - consulta o widget de tarefas para máis detalles.", + uploadUrlError: "Non se puido obter unha URL de subida", + generateTaskError: "Non se puido iniciar a tarefa de diff de reconciliación", + uploadError: "Non se puido subir o ficheiro de reconciliación", + applyTaskError: "Non se puido iniciar a tarefa de aplicación", + applyError: "Non se puideron aplicar os cambios de reconciliación", + }, + }, + }, logsScreen: { noPermissions: "Non tes permiso para acceder aos rexistros.", title: "Rexistros", diff --git a/packages/admin-portal/src/translations/nl.ts b/packages/admin-portal/src/translations/nl.ts index c628b1cac94..4981810c5bb 100644 --- a/packages/admin-portal/src/translations/nl.ts +++ b/packages/admin-portal/src/translations/nl.ts @@ -104,6 +104,105 @@ const dutchTranslation: TranslationType = { next: "Volgende", }, }, + reconciliation: { + menuButton: "Ext. kiezerssync", + categories: { + VOTED_INTERNET: "Gestemd via Internet", + VOTED_OTHER_CHANNEL: "Gestemd via ander kanaal", + DISABLED_DELETE_CALL: "Kiezer gedeactiveerd", + DELETION_REVERTED: "Verwijdering teruggedraaid", + PROFILE_UPDATE: "Profiel bijgewerkt", + VOTER_ADDED: "Kiezer toegevoegd", + REENABLED: "Kiezer heractiveerd", + VOTED_UNMARKED: "Kiezer niet langer gemarkeerd als gestemd", + ROW_FAILURE: "Rijfout", + }, + table: { + voterId: "Kiezer-ID", + field: "Veld", + category: "Categorie", + currentValue: "Huidige waarde", + newValue: "Nieuwe waarde", + reason: "Reden", + rowLabel: "Rij", + noDifferences: "Geen verschillen gevonden - de systemen zijn gesynchroniseerd.", + }, + wizard: { + title: "Externe reconciliatiesynchronisatie", + subtitle: "Synchroniseer de kiezerslijst met het externe systeem", + drop: { + description: + "Sleep het door het externe systeem gegenereerde reconciliatiebestand hierheen - beide diffs (externe zijde en Sequent-zijde) worden automatisch berekend en in aparte tabellen weergegeven.", + fileFormatLabel: "CSV-bestand", + uploading: "{{fileName}} wordt geüpload en beide diffs worden berekend...", + }, + review: { + fileSummary: + "{{fileName}} - Sequentie {{sequence}}, gegenereerd {{generatedAt}}", + rowFailuresWarning: + "{{count}} rij(en) konden niet veilig worden gereconcilieerd en zijn uitgesloten van beide diffs - zie details hieronder.", + noDifferences: "Geen verschillen - beide systemen zijn al gesynchroniseerd.", + diffOnlyDifferences: + "Dit is een convergentiecontrole voor een reeds toegepaste reeks. Verschillen worden ter opvolging getoond, maar deze ronde kan niet opnieuw worden toegepast.", + externalDiffTitle: "Externe diff", + sequentDiffTitle: "Sequent-diff", + downloadExternalPatch: "Externe patch downloaden", + externalDiffCaption: + "Download de patch en lever deze buiten deze tool aan het externe systeem. Zodra het de patch toepast en het volgende reconciliatiebestand produceert, klik op 'Terug' en sleep dat bestand hierheen - 'Toepassen' wordt geactiveerd zodra deze tabel leeg is.", + noExternalDifferences: "Geen verschillen aan de externe zijde.", + sequentDiffCaption: + "Pas wijzigingen direct toe op Sequent door op 'Toepassen' te klikken - hiervoor wordt geen patchbestand gegenereerd.", + noSequentDifferences: "Geen verschillen aan de Sequent-zijde.", + }, + applying: { + inProgress: "Wijzigingen aan de Sequent-zijde worden toegepast...", + rowFailures: + "{{count}} rij(en) zijn uitgesloten van deze ronde en vereisen handmatige opvolging - zie details hieronder.", + rowFailuresTruncated: + "De eerste {{shown}} van {{count}} rijfouten worden getoond. Los de gemeenschappelijke oorzaak op en probeer opnieuw om de resterende fouten te zien.", + success: "Alle wijzigingen aan de Sequent-zijde zijn succesvol toegepast.", + }, + actions: { + cancel: "Annuleren", + back: "Terug", + apply: "Toepassen", + next: "Volgende", + startOver: "Opnieuw beginnen", + close: "Sluiten", + }, + confirm: { + title: "Reconciliatiewijzigingen bevestigen", + categoriesNote: + "Categorieën die oranje zijn omlijnd ({{categories}}) hebben invloed op de stemstatus of deactiveren kiezers.", + applyChanges: "Wijzigingen toepassen", + continue: "Doorgaan", + }, + summary: { + votedOtherChannel: + "markeert {{count}} kiezer(s) als gestemd via een ander kanaal", + disabled: "deactiveert {{count}} kiezer(s)", + reenabled: "heractiveert {{count}} kiezer(s)", + votedUnmarked: "verwijdert de stemmarkering van {{count}} kiezer(s)", + profileUpdated: "werkt {{count}} profiel(en) bij", + voterAdded: "voegt {{count}} kiezer(s) toe", + prefix: "Dit past wijzigingen toe die {{parts}}.", + empty: "Er zijn geen wijzigingen aan de Sequent-zijde om toe te passen.", + }, + notifications: { + envelopeLoadError: + "Kon de reconciliatie-diff niet laden - probeer het opnieuw.", + generateFailed: + "Kon de reconciliatie-diff niet berekenen - zie de taakwidget voor details.", + applyFailed: + "Kon de wijzigingen aan de Sequent-zijde niet toepassen - zie de taakwidget voor details.", + uploadUrlError: "Kon geen upload-URL verkrijgen", + generateTaskError: "Kon de reconciliatie-difftaak niet starten", + uploadError: "Kon het reconciliatiebestand niet uploaden", + applyTaskError: "Kon de toepassingstaak niet starten", + applyError: "Kon de reconciliatiewijzigingen niet toepassen", + }, + }, + }, logsScreen: { noPermissions: "U hebt geen toestemming om toegang te krijgen tot de logs.", title: "Logs", diff --git a/packages/admin-portal/src/translations/tl.ts b/packages/admin-portal/src/translations/tl.ts index c8116e21665..0bcaccfc99f 100644 --- a/packages/admin-portal/src/translations/tl.ts +++ b/packages/admin-portal/src/translations/tl.ts @@ -104,6 +104,104 @@ const tagalogTranslation: TranslationType = { next: "Susunod", }, }, + reconciliation: { + menuButton: "I-sync ang ext. botante", + categories: { + VOTED_INTERNET: "Bumoto sa pamamagitan ng Internet", + VOTED_OTHER_CHANNEL: "Bumoto sa ibang channel", + DISABLED_DELETE_CALL: "Na-disable ang botante", + DELETION_REVERTED: "Na-revert ang pagtanggal", + PROFILE_UPDATE: "Na-update ang profile", + VOTER_ADDED: "Naidagdag ang botante", + REENABLED: "Na-enable muli ang botante", + VOTED_UNMARKED: "Inalis ang markang bumoto ang botante", + ROW_FAILURE: "Nabigo ang row", + }, + table: { + voterId: "ID ng Botante", + field: "Field", + category: "Kategorya", + currentValue: "Kasalukuyang value", + newValue: "Bagong value", + reason: "Dahilan", + rowLabel: "Row", + noDifferences: "Walang nakitang pagkakaiba - naka-sync na ang mga sistema.", + }, + wizard: { + title: "Pag-sync ng External Reconciliation", + subtitle: "I-sync ang listahan ng botante sa external system", + drop: { + description: + "I-drop ang reconciliation file na ginawa ng external system - awtomatikong kinakalkula at ipinapakita sa magkahiwalay na tables ang parehong diffs (external side at Sequent side).", + fileFormatLabel: "CSV file", + uploading: "Ina-upload ang {{fileName}} at kinakalkula ang parehong diffs...", + }, + review: { + fileSummary: + "{{fileName}} - Sequence {{sequence}}, ginawa noong {{generatedAt}}", + rowFailuresWarning: + "Hindi ligtas na ma-reconcile ang {{count}} row(s) at hindi kasama ang mga ito sa parehong diffs - tingnan ang mga detalye sa ibaba.", + noDifferences: "Walang pagkakaiba - naka-sync na ang dalawang sistema.", + diffOnlyDifferences: + "Convergence check ito para sa nailapat nang Sequence. Ipinapakita ang mga pagkakaiba para sa follow-up, ngunit hindi na maaaring ilapat muli ang round na ito.", + externalDiffTitle: "External diff", + sequentDiffTitle: "Sequent diff", + downloadExternalPatch: "I-download ang external patch", + externalDiffCaption: + "I-download ang patch at ibigay ito sa external system sa labas ng tool na ito. Kapag na-apply na nito ang patch at nagawa ang susunod na reconciliation file, i-click ang 'Bumalik' at i-drop ang file na iyon - mag-e-enable ang 'I-apply' kapag walang laman ang table na ito.", + noExternalDifferences: "Walang pagkakaiba sa external side.", + sequentDiffCaption: + "I-apply ang mga pagbabago nang direkta sa Sequent sa pamamagitan ng pag-click sa 'I-apply' - walang ginagawang patch file para dito.", + noSequentDifferences: "Walang pagkakaiba sa Sequent side.", + }, + applying: { + inProgress: "Inilalapat ang mga pagbabago sa Sequent side...", + rowFailures: + "Hindi kasama ang {{count}} row(s) sa round na ito at kailangan ng manual na follow-up - tingnan ang mga detalye sa ibaba.", + rowFailuresTruncated: + "Ipinapakita ang unang {{shown}} sa {{count}} row failure. Ayusin ang magkakaparehong sanhi at subukang muli upang makita ang mga natitirang failure.", + success: "Matagumpay na nailapat ang lahat ng pagbabago sa Sequent side.", + }, + actions: { + cancel: "I-cancel", + back: "Bumalik", + apply: "I-apply", + next: "Susunod", + startOver: "Ulitin mula sa simula", + close: "Isara", + }, + confirm: { + title: "Kumpirmahin ang mga pagbabago sa reconciliation", + categoriesNote: + "Ang mga kategoryang naka-highlight sa orange ({{categories}}) ay nakakaapekto sa status ng pagboto o nagdi-disable ng mga botante.", + applyChanges: "I-apply ang mga pagbabago", + continue: "Magpatuloy", + }, + summary: { + votedOtherChannel: + "mina-mark ang {{count}} botante bilang bumoto sa ibang channel", + disabled: "dini-disable ang {{count}} botante", + reenabled: "ine-enable muli ang {{count}} botante", + votedUnmarked: "inaalis ang markang bumoto sa {{count}} botante", + profileUpdated: "ina-update ang {{count}} profile", + voterAdded: "nagdadagdag ng {{count}} botante", + prefix: "Ilalapat nito ang mga pagbabagong {{parts}}.", + empty: "Walang pagbabago sa Sequent side na ila-apply.", + }, + notifications: { + envelopeLoadError: "Hindi na-load ang reconciliation diff - subukan muli.", + generateFailed: + "Hindi na-calculate ang reconciliation diff - tingnan ang task widget para sa detalye.", + applyFailed: + "Hindi na-apply ang mga pagbabago sa Sequent side - tingnan ang task widget para sa detalye.", + uploadUrlError: "Hindi makakuha ng upload URL", + generateTaskError: "Hindi masimulan ang reconciliation diff task", + uploadError: "Hindi ma-upload ang reconciliation file", + applyTaskError: "Hindi masimulan ang apply task", + applyError: "Hindi na-apply ang mga pagbabago sa reconciliation", + }, + }, + }, logsScreen: { noPermissions: "Wala kang permiso upang ma-access ang mga log.", title: "Mga Log", diff --git a/packages/admin-portal/src/types/keycloak.ts b/packages/admin-portal/src/types/keycloak.ts index 4c23378d62a..d6782a65b18 100644 --- a/packages/admin-portal/src/types/keycloak.ts +++ b/packages/admin-portal/src/types/keycloak.ts @@ -2,6 +2,10 @@ // // SPDX-License-Identifier: AGPL-3.0-only +/** Sentinel value for a cleared/unset Keycloak user attribute — mirrors the + * backend's `sequent_core::types::keycloak::ATTR_RESET_VALUE`. */ +export const ATTR_RESET_VALUE = "NONE" + export enum IPermissions { ADMIN_USER = "admin-user", TENANT_CREATE = "tenant-create", @@ -212,4 +216,5 @@ export enum IPermissions { PHONE_BLACKLIST_CREATE = "phone-blacklist-create", PHONE_BLACKLIST_DELETE = "phone-blacklist-delete", PHONE_BLACKLIST_UPDATE = "phone-blacklist-update", + ELECTION_EVENT_VOTER_LIST_SYNC = "election-event-voter-list-reconciliation", } diff --git a/packages/admin-portal/src/types/tasksExecution.ts b/packages/admin-portal/src/types/tasksExecution.ts index c9a01615719..3ce89531310 100644 --- a/packages/admin-portal/src/types/tasksExecution.ts +++ b/packages/admin-portal/src/types/tasksExecution.ts @@ -29,4 +29,6 @@ export enum ETasksExecution { EXPORT_TALLY_RESULTS_XLSX = "EXPORT_TALLY_RESULTS_XLSX", EXPORT_CERTIFICATE_AUTHORITIES = "EXPORT_CERTIFICATE_AUTHORITIES", PUBLISH_RESULTS_WEBSITE = "PUBLISH_RESULTS_WEBSITE", + GENERATE_RECONCILIATION_PATCHES = "GENERATE_RECONCILIATION_PATCHES", + APPLY_RECONCILIATION_PATCH = "APPLY_RECONCILIATION_PATCH", } diff --git a/packages/ballot-verifier/graphql.schema.json b/packages/ballot-verifier/graphql.schema.json index 389cef0a4fb..7472612b9e2 100644 --- a/packages/ballot-verifier/graphql.schema.json +++ b/packages/ballot-verifier/graphql.schema.json @@ -11337,6 +11337,51 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "apply_external_reconciliation_changes", + "description": "applies the Sequent-side reconciliation diff for an already-computed import", + "args": [ + { + "name": "diff_document_id", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "election_event_id", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "type": { + "kind": "OBJECT", + "name": "taskOutput", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "call_plugin_route", "description": null, @@ -11755,6 +11800,51 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "create_external_reconciliation_import", + "description": "uploads an external (i.e. Datafix) reconciliation file and kicks off the diff-calculation task", + "args": [ + { + "name": "document_id", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "election_event_id", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "type": { + "kind": "OBJECT", + "name": "taskOutput", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "create_keys_ceremony", "description": "create keys ceremony", diff --git a/packages/ballot-verifier/rust/sequent-core-0.1.0.tgz b/packages/ballot-verifier/rust/sequent-core-0.1.0.tgz index 2ba45243109..d6bd3f03b69 100644 Binary files a/packages/ballot-verifier/rust/sequent-core-0.1.0.tgz and b/packages/ballot-verifier/rust/sequent-core-0.1.0.tgz differ diff --git a/packages/ballot-verifier/src/gql/graphql.ts b/packages/ballot-verifier/src/gql/graphql.ts index 4d77603c9c5..174d546c8bc 100644 --- a/packages/ballot-verifier/src/gql/graphql.ts +++ b/packages/ballot-verifier/src/gql/graphql.ts @@ -1229,6 +1229,8 @@ export type Mutation_Root = { ApplicationChangeStatus?: Maybe /** Verify User Registration Application */ VerifyApplication: Scalars["String"]["output"] + /** applies the Sequent-side reconciliation diff for an already-computed import */ + apply_external_reconciliation_changes?: Maybe call_plugin_route: PluginsRouteOutput /** check private key */ check_private_key?: Maybe @@ -1239,6 +1241,8 @@ export type Mutation_Root = { /** create_ballot_receipt */ create_ballot_receipt?: Maybe create_election?: Maybe + /** uploads an external (i.e. Datafix) reconciliation file and kicks off the diff-calculation task */ + create_external_reconciliation_import?: Maybe /** create keys ceremony */ create_keys_ceremony?: Maybe /** create_new_tally_sheet */ @@ -2038,6 +2042,12 @@ export type Mutation_RootVerifyApplicationArgs = { body: ApplicationVerifyBody } +/** mutation root */ +export type Mutation_RootApply_External_Reconciliation_ChangesArgs = { + diff_document_id: Scalars["String"]["input"] + election_event_id: Scalars["String"]["input"] +} + /** mutation root */ export type Mutation_RootCall_Plugin_RouteArgs = { data: Scalars["jsonb"]["input"] @@ -2084,6 +2094,12 @@ export type Mutation_RootCreate_ElectionArgs = { presentation?: InputMaybe } +/** mutation root */ +export type Mutation_RootCreate_External_Reconciliation_ImportArgs = { + document_id: Scalars["String"]["input"] + election_event_id: Scalars["String"]["input"] +} + /** mutation root */ export type Mutation_RootCreate_Keys_CeremonyArgs = { object: CreateKeysCeremonyInput diff --git a/packages/electoral-log/src/messages/message.rs b/packages/electoral-log/src/messages/message.rs index 885e7e8aa92..612bdfba1be 100644 --- a/packages/electoral-log/src/messages/message.rs +++ b/packages/electoral-log/src/messages/message.rs @@ -86,6 +86,53 @@ impl Message { None, ) } + /// Records a third-party voter registry reconciliation run event (patch + /// generation or applying the Sequent-side diff) — see + /// `StatementBody::ExternalReconciliation`. Named for the general + /// capability, not the specific integration (Datafix) that first needed + /// it. + /// Unlike most `Message::*_message` constructors, this calls [`Self::sign`] + /// directly instead of [`Self::from_body`] so `artifact` (the JSON of + /// old/new values applied, for a `ChangesApplied` entry) can be carried — + /// `from_body` always signs with `artifact: None`. + #[instrument(skip_all, err)] + pub fn external_reconciliation_message( + event_id: EventIdString, + kind: ExternalReconciliationKind, + sequence: ExternalReconciliationSequenceString, + generated_at: ExternalReconciliationGeneratedAtString, + input_hash: ExternalReconciliationInputHashString, + output_hash: ExternalReconciliationOutputHashString, + artifact: Option>, + sd: &SigningData, + user_id: Option, + username: Option, + ) -> Result { + let body = StatementBody::ExternalReconciliation( + event_id.clone(), + kind, + sequence, + generated_at, + input_hash, + output_hash, + ); + let head = StatementHead::from_body(event_id, &body); + let statement = Statement::new(head, body); + + Message::sign( + statement, + artifact, + &sd.sender_sk, + &sd.sender_name, + &sd.system_sk, + user_id, + username, + None, /* election_id: a reconciliation run is event-wide, not tied to one election */ + None, /* area_id */ + None, /* ballot_id */ + ) + } + pub fn cast_vote_message( event: EventIdString, election: ElectionIdString, diff --git a/packages/electoral-log/src/messages/newtypes.rs b/packages/electoral-log/src/messages/newtypes.rs index 8dfee9176a0..8b0748da813 100644 --- a/packages/electoral-log/src/messages/newtypes.rs +++ b/packages/electoral-log/src/messages/newtypes.rs @@ -173,6 +173,59 @@ pub struct ExternalApiSubject { pub username: Option, } +/// Which phase of a third-party voter registry reconciliation run a +/// `StatementBody::ExternalReconciliation` entry records — see +/// `external_reconciliation_message`. Named for the general capability (any +/// external voter registry this system reconciles against), not the specific +/// integration (Datafix) that first needed it. +#[derive( + BorshSerialize, + BorshDeserialize, + Deserialize, + Serialize, + Clone, + PartialEq, + Eq, + Hash, + Debug, + Display, +)] +pub enum ExternalReconciliationKind { + /// The external-side diff was computed and (if non-empty) the + /// downloadable patch was generated and uploaded. + PatchGenerated, + /// The Sequent-side diff was applied directly to this system. + ChangesApplied, +} + +/// Decimal `Sequence` of the reconciliation file an `ExternalReconciliation` +/// entry was generated from or applied — kept as a string, like every other +/// *String newtype in this module, rather than a bare integer. +#[derive( + BorshSerialize, BorshDeserialize, Deserialize, Serialize, Clone, PartialEq, Eq, Hash, Debug, +)] +pub struct ExternalReconciliationSequenceString(pub String); + +/// Decimal Unix-seconds `GeneratedAt` of the reconciliation or patch file. +#[derive( + BorshSerialize, BorshDeserialize, Deserialize, Serialize, Clone, PartialEq, Eq, Hash, Debug, +)] +pub struct ExternalReconciliationGeneratedAtString(pub String); + +/// SHA-256 of the reconciliation file that produced this run. +#[derive( + BorshSerialize, BorshDeserialize, Deserialize, Serialize, Clone, PartialEq, Eq, Hash, Debug, +)] +pub struct ExternalReconciliationInputHashString(pub String); + +/// SHA-256 of the generated external-side patch — `None` for a +/// `ChangesApplied` entry (there is no patch file for Sequent's own side, or for a +/// `PatchGenerated` entry whose external-side diff was empty. +#[derive( + BorshSerialize, BorshDeserialize, Deserialize, Serialize, Clone, PartialEq, Eq, Hash, Debug, +)] +pub struct ExternalReconciliationOutputHashString(pub Option); + #[derive( BorshSerialize, BorshDeserialize, Deserialize, Serialize, Clone, PartialEq, Eq, Hash, Debug, )] diff --git a/packages/electoral-log/src/messages/statement.rs b/packages/electoral-log/src/messages/statement.rs index 796752e3f28..6d579818c48 100644 --- a/packages/electoral-log/src/messages/statement.rs +++ b/packages/electoral-log/src/messages/statement.rs @@ -261,6 +261,30 @@ impl StatementHead { ..default_head } } + StatementBody::ExternalReconciliation( + _, + kind, + sequence, + _, + input_hash, + output_hash, + ) => { + let action = match kind { + ExternalReconciliationKind::PatchGenerated => "External patch generated", + ExternalReconciliationKind::ChangesApplied => "Sequent-side changes applied", + }; + StatementHead { + kind: StatementType::ExternalReconciliation, + event_type: StatementEventType::USER, + description: format!( + "{action} for reconciliation Sequence {} (input {}, output {}).", + sequence.0, + input_hash.0, + output_hash.0.as_deref().unwrap_or("none"), + ), + ..default_head + } + } StatementBody::ResultsPublicationAction(details) => { let action = match details.action { ResultsPublicationAction::Publish => "published", @@ -395,6 +419,24 @@ pub enum StatementBody { VoterCountryString, VotingChannelString, ), + /// Records a third-party voter registry reconciliation run event: either + /// the external-side diff/patch being generated, or the Sequent-side diff + /// being applied. Named for the general capability, not the specific + /// integration (Datafix) that first needed it. Doesn't fit + /// `ExternalApiRequest`'s shape: there is no HTTP call to the external + /// system involved, since it is offline for the whole freeze period a + /// reconciliation run happens during. The JSON of every applied voter's + /// old/new values is carried in `Message.artifact` on the + /// `ChangesApplied` entry — there is exactly one entry per phase per run, + /// not one per voter. + ExternalReconciliation( + EventIdString, + ExternalReconciliationKind, + ExternalReconciliationSequenceString, + ExternalReconciliationGeneratedAtString, + ExternalReconciliationInputHashString, + ExternalReconciliationOutputHashString, + ), } // Note: When creating new variants, consider that the length limit STATEMENT_KIND_VARCHAR_LENGTH is 40. @@ -428,6 +470,7 @@ pub enum StatementType { PhoneBlacklistUpdated, ResultsPublicationAction, ExternalApiRequest, + ExternalReconciliation, } #[derive(BorshSerialize, BorshDeserialize, Display, Deserialize, Serialize, Debug, Clone)] @@ -511,6 +554,14 @@ mod statement_compatibility_tests { VoterCountryString(String::new()), VotingChannelString(String::new()), ); + let external_reconciliation = StatementBody::ExternalReconciliation( + EventIdString(String::new()), + ExternalReconciliationKind::PatchGenerated, + ExternalReconciliationSequenceString(String::new()), + ExternalReconciliationGeneratedAtString(String::new()), + ExternalReconciliationInputHashString(String::new()), + ExternalReconciliationOutputHashString(None), + ); assert_eq!(borsh::to_vec(&election_publish).unwrap()[0], 2); assert_eq!(borsh::to_vec(&certificate).unwrap()[0], 23); @@ -520,6 +571,7 @@ mod statement_compatibility_tests { // rebase that introduced `ExternalApiRequest`. assert_eq!(borsh::to_vec(&external).unwrap()[0], 26); assert_eq!(borsh::to_vec(&cast_vote_with_channel).unwrap()[0], 27); + assert_eq!(borsh::to_vec(&external_reconciliation).unwrap()[0], 28); } #[test] diff --git a/packages/graphql.schema.json b/packages/graphql.schema.json index d75ab99a553..4fb4e748adf 100644 --- a/packages/graphql.schema.json +++ b/packages/graphql.schema.json @@ -10293,6 +10293,47 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "apply_external_reconciliation_changes", + "description": "applies the Sequent-side reconciliation diff for an already-computed import", + "args": [ + { + "name": "diff_document_id", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "election_event_id", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "taskOutput", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "call_plugin_route", "description": null, @@ -10667,6 +10708,47 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "create_external_reconciliation_import", + "description": "uploads an external (i.e. Datafix) reconciliation file and kicks off the diff-calculation task", + "args": [ + { + "name": "document_id", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "election_event_id", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "taskOutput", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "create_keys_ceremony", "description": "create keys ceremony", diff --git a/packages/harvest/src/main.rs b/packages/harvest/src/main.rs index 28fddce40c3..87744aca461 100644 --- a/packages/harvest/src/main.rs +++ b/packages/harvest/src/main.rs @@ -124,6 +124,8 @@ async fn rocket() -> _ { routes::tally_sheets::preview_tally_sheet_import, routes::tally_sheets::create_tally_sheet_import, routes::tally_sheets::review_tally_sheet_import, + routes::external_reconciliation::create_reconciliation_import, + routes::external_reconciliation::apply_reconciliation_changes, routes::create_ballot_receipt::create_ballot_receipt, routes::election_dates::manage_election_dates, routes::custom_urls::update_custom_url, diff --git a/packages/harvest/src/routes/api_datafix.rs b/packages/harvest/src/routes/api_datafix.rs index b41ef4fc697..3cb30cc4d4b 100644 --- a/packages/harvest/src/routes/api_datafix.rs +++ b/packages/harvest/src/routes/api_datafix.rs @@ -13,14 +13,14 @@ use serde::Serialize; use tracing::{error, instrument}; use windmill::services; use windmill::services::database::{get_hasura_pool, get_keycloak_pool}; -use windmill::services::datafix::api_datafix::{ +use windmill::services::external::api_datafix::{ acquire_inbound_voter_lock, audit_inbound_operation, audit_inbound_operation_standalone, ensure_inbound_reenable_is_safe, ensure_voter_has_no_active_vote, release_inbound_voter_lock, valid_inbound_voting_channel, InboundVoterLock, }; -use windmill::services::datafix::types::*; -use windmill::services::datafix::utils::get_event_id_and_datafix_annotations; +use windmill::services::external::datafix_types::*; +use windmill::services::external::utils::get_event_id_and_datafix_annotations; #[instrument(skip_all)] #[post("/add-voter", format = "json", data = "")] @@ -61,7 +61,7 @@ pub async fn add_voter( .await?; let realm = get_event_realm(&claims.tenant_id, &election_event_id); - let result = services::datafix::api_datafix::add_datafix_voter( + let result = services::external::api_datafix::add_datafix_voter( &hasura_transaction, &claims.tenant_id, &claims.datafix_event_id, @@ -212,7 +212,7 @@ pub async fn update_voter( release_inbound_voter_lock(lock).await; return Err(err); } - let result = services::datafix::api_datafix::update_datafix_voter( + let result = services::external::api_datafix::update_datafix_voter( &hasura_transaction, &keycloak_transaction, &claims.tenant_id, @@ -353,7 +353,7 @@ pub async fn delete_voter( release_inbound_voter_lock(lock).await; return Err(err); } - let result = services::datafix::api_datafix::disable_datafix_voter( + let result = services::external::api_datafix::disable_datafix_voter( &hasura_transaction, &keycloak_transaction, &claims.tenant_id, @@ -488,7 +488,7 @@ pub async fn unmark_voted( release_inbound_voter_lock(lock).await; return Err(err); } - let result = services::datafix::api_datafix::unmark_voter_as_voted( + let result = services::external::api_datafix::unmark_voter_as_voted( &hasura_transaction, &keycloak_transaction, &claims.tenant_id, @@ -634,7 +634,7 @@ pub async fn mark_voted( release_inbound_voter_lock(lock).await; return Err(err); } - let result = services::datafix::api_datafix::mark_as_voted_via_channel( + let result = services::external::api_datafix::mark_as_voted_via_channel( &hasura_transaction, &keycloak_transaction, &claims.tenant_id, @@ -756,7 +756,7 @@ pub async fn replace_pin( let hasura_transaction = transaction_result.expect("transaction result was checked above"); - let result = services::datafix::api_datafix::replace_voter_pin( + let result = services::external::api_datafix::replace_voter_pin( &hasura_transaction, &keycloak_transaction, &claims.tenant_id, diff --git a/packages/harvest/src/routes/error_catchers.rs b/packages/harvest/src/routes/error_catchers.rs index 08b345865cd..1bbaf4b879a 100644 --- a/packages/harvest/src/routes/error_catchers.rs +++ b/packages/harvest/src/routes/error_catchers.rs @@ -6,7 +6,7 @@ use rocket::serde::json::Json; use rocket::Request; use serde::{Deserialize, Serialize}; use tracing::instrument; -use windmill::services::datafix::types::{ +use windmill::services::external::datafix_types::{ DatafixErrorCode, DatafixResponse, JsonErrorResponse, }; diff --git a/packages/harvest/src/routes/external_reconciliation.rs b/packages/harvest/src/routes/external_reconciliation.rs new file mode 100644 index 00000000000..a94962e44b1 --- /dev/null +++ b/packages/harvest/src/routes/external_reconciliation.rs @@ -0,0 +1,279 @@ +// SPDX-FileCopyrightText: 2026 Sequent Tech Inc +// +// SPDX-License-Identifier: AGPL-3.0-only + +//! Admin-portal-facing Datafix reconciliation routes. Unlike +//! `routes::api_datafix` (the *inbound* Datafix->Sequent API, gated by +//! `DatafixClaims`/`DATAFIX_ACCOUNT`), these are triggered by an admin +//! operator through the wizard and authorized like every other admin action +//! (`JwtClaims` + `authorize`). + +use crate::services::authorization::authorize; +use anyhow::Result; +use deadpool_postgres::Client as DbClient; +use rocket::http::Status; +use rocket::serde::json::Json; +use sequent_core::services::jwt::JwtClaims; +use sequent_core::types::hasura::core::TasksExecution; +use sequent_core::types::permissions::Permissions; +use serde::{Deserialize, Serialize}; +use std::fs::File; +use std::io::BufReader; +use tracing::instrument; +use windmill::postgres::document::get_document; +use windmill::postgres::election_event::{ + get_election_event_by_id, ElectionEventDatafix, +}; +use windmill::services::celery_app::get_celery_app; +use windmill::services::consolidation::eml_generator::ValidateAnnotations; +use windmill::services::database::get_hasura_pool; +use windmill::services::documents::get_document_as_temp_file; +use windmill::services::external::reconciliation::diff::ReconciliationApplyEnvelope; +use windmill::services::external::types::ReconciliationPatchSource; +use windmill::services::tasks_execution::{ + post as post_task_execution, update_fail, +}; +use windmill::tasks::apply_reconciliation_patch::{ + apply_reconciliation_patch, ApplyReconciliationPatchBody, +}; +use windmill::tasks::generate_reconciliation_patches::{ + generate_reconciliation_patches, GenerateReconciliationPatchesBody, +}; +use windmill::types::tasks::ETasksExecution; + +#[derive(Serialize, Debug)] +pub struct DatafixReconciliationTaskOutput { + pub task_execution: TasksExecution, +} + +#[derive(Deserialize, Debug)] +pub struct CreateDatafixReconciliationImportInput { + pub election_event_id: String, + pub document_id: String, +} + +/// Kicks off `generate_reconciliation_patches` for an uploaded reconciliation +/// file. Mirrors `import_users_f`'s shape (insert a `task_execution` row, +/// enqueue the Celery task, return the task_execution for the admin portal's +/// widget) rather than `create_tally_sheet_import`'s synchronous shape, since +/// reconciliation files can be 100k+ rows and always need the async task +/// path. There is no row to insert here — the generate task is the only +/// record of this round until it produces the diff-envelope document. The +/// uploaded file itself isn't read here at all: `generate_reconciliation_patches` +/// downloads it once (to parse it) and hashes those same bytes for the +/// electoral log; re-downloading it here just to check a client-computed +/// hash would be a second, redundant download that doesn't protect anything +/// the electoral log's own hash doesn't already cover — that hash exists so +/// Datafix's own generated hash can be compared against it manually, not as +/// an upload integrity gate. +#[instrument(skip(claims))] +#[post("/create-reconciliation-import", format = "json", data = "")] +pub async fn create_reconciliation_import( + claims: JwtClaims, + body: Json, +) -> Result, (Status, String)> { + let input = body.into_inner(); + let tenant_id = claims.hasura_claims.tenant_id.clone(); + authorize( + &claims, + true, + Some(tenant_id.clone()), + vec![Permissions::ELECTION_EVENT_VOTER_LIST_SYNC], + ) + .map_err(|err| (Status::Forbidden, format!("{err:?}")))?; + + let executer_name = claims + .name + .clone() + .unwrap_or_else(|| claims.hasura_claims.user_id.clone()); + + let task_execution = post_task_execution( + &tenant_id, + Some(&input.election_event_id), + ETasksExecution::GENERATE_RECONCILIATION_PATCHES, + &executer_name, + ) + .await + .map_err(|err| { + ( + Status::InternalServerError, + format!("Failed to insert task execution record: {err:?}"), + ) + })?; + + let task_body = GenerateReconciliationPatchesBody { + tenant_id: tenant_id.clone(), + election_event_id: input.election_event_id.clone(), + source_document_id: input.document_id.clone(), + requested_by_user_id: claims.hasura_claims.user_id.clone(), + requested_by_username: claims.name.clone(), + }; + + let celery_app = get_celery_app().await; + if let Err(err) = celery_app + .send_task(generate_reconciliation_patches::new( + task_body, + task_execution.clone(), + )) + .await + { + let message = + format!("Failed to enqueue reconciliation generation: {err}"); + update_fail(&task_execution, &message).await.ok(); + return Err((Status::InternalServerError, message)); + } + + Ok(Json(DatafixReconciliationTaskOutput { task_execution })) +} + +#[derive(Deserialize, Debug)] +pub struct ApplyDatafixReconciliationChangesInput { + pub election_event_id: String, + /// The `ReconciliationDiff` envelope document id from the generate round + /// being applied (the frontend already fetched and parsed this document + /// to render the review tables, so it has this id in hand). + pub diff_document_id: String, +} + +/// Kicks off `apply_reconciliation_patch`. Re-validates server-side that the +/// referenced round's Datafix-side diff is empty — the same check the +/// frontend uses to enable the "Apply" button, re-run here (by independently +/// re-fetching and re-parsing the diff-envelope document) because the client +/// is never trusted to enforce it. +#[instrument(skip(claims))] +#[post("/apply-reconciliation-changes", format = "json", data = "")] +pub async fn apply_reconciliation_changes( + claims: JwtClaims, + body: Json, +) -> Result, (Status, String)> { + let input = body.into_inner(); + let tenant_id = claims.hasura_claims.tenant_id.clone(); + authorize( + &claims, + true, + Some(tenant_id.clone()), + vec![Permissions::ELECTION_EVENT_VOTER_LIST_SYNC], + ) + .map_err(|err| (Status::Forbidden, format!("{err:?}")))?; + + let mut hasura_db_client: DbClient = get_hasura_pool() + .await + .get() + .await + .map_err(|err| (Status::InternalServerError, format!("{err:?}")))?; + let hasura_transaction = hasura_db_client + .transaction() + .await + .map_err(|err| (Status::InternalServerError, format!("{err:?}")))?; + + let document = get_document( + &hasura_transaction, + &tenant_id, + Some(input.election_event_id.clone()), + &input.diff_document_id, + ) + .await + .map_err(|err| (Status::InternalServerError, format!("{err:?}")))? + .ok_or_else(|| { + ( + Status::NotFound, + "Reconciliation diff not found".to_string(), + ) + })?; + let temp_file = get_document_as_temp_file(&tenant_id, &document) + .await + .map_err(|err| (Status::InternalServerError, format!("{err:?}")))?; + let file = File::open(temp_file.path()).map_err(|err| { + ( + Status::InternalServerError, + format!("Error opening reconciliation diff: {err}"), + ) + })?; + let envelope: ReconciliationApplyEnvelope = + serde_json::from_reader(BufReader::new(file)).map_err(|err| { + ( + Status::InternalServerError, + format!("Error parsing reconciliation diff: {err}"), + ) + })?; + + if envelope.external_patch_document_id.is_some() { + return Err(( + Status::Conflict, + "The external-side diff is not empty — apply the external patch and re-import first".to_string(), + )); + } + if !envelope.apply_allowed { + return Err(( + Status::Conflict, + "This reconciliation envelope is a diff-only convergence check and cannot be applied" + .to_string(), + )); + } + + // Every reconciliation round today comes from Datafix — resolve its + // `CountyMun` so the apply task can record the round's source (see + // `ReconciliationPatchSource`), gating its own Datafix-specific + // bookkeeping without the generic apply logic needing to know about it. + let election_event = get_election_event_by_id( + &hasura_transaction, + &tenant_id, + &input.election_event_id, + ) + .await + .map_err(|err| (Status::InternalServerError, format!("{err:?}")))?; + let datafix_annotations = ElectionEventDatafix(election_event) + .get_annotations() + .map_err(|err| (Status::InternalServerError, format!("{err:?}")))?; + let source = ReconciliationPatchSource::Datafix { + county_mun: datafix_annotations.voterview_request.county_mun, + }; + + hasura_transaction + .commit() + .await + .map_err(|err| (Status::InternalServerError, format!("{err:?}")))?; + + let executer_name = claims + .name + .clone() + .unwrap_or_else(|| claims.hasura_claims.user_id.clone()); + + let task_execution = post_task_execution( + &tenant_id, + Some(&input.election_event_id), + ETasksExecution::APPLY_RECONCILIATION_PATCH, + &executer_name, + ) + .await + .map_err(|err| { + ( + Status::InternalServerError, + format!("Failed to insert task execution record: {err:?}"), + ) + })?; + + let task_body = ApplyReconciliationPatchBody { + tenant_id: tenant_id.clone(), + election_event_id: input.election_event_id.clone(), + source, + diff_document_id: input.diff_document_id.clone(), + applied_by_user_id: claims.hasura_claims.user_id.clone(), + applied_by_username: claims.name.clone(), + }; + + let celery_app = get_celery_app().await; + if let Err(err) = celery_app + .send_task(apply_reconciliation_patch::new( + task_body, + task_execution.clone(), + )) + .await + { + let message = format!("Failed to enqueue reconciliation apply: {err}"); + update_fail(&task_execution, &message).await.ok(); + return Err((Status::InternalServerError, message)); + } + + Ok(Json(DatafixReconciliationTaskOutput { task_execution })) +} diff --git a/packages/harvest/src/routes/mod.rs b/packages/harvest/src/routes/mod.rs index 9ab7c08d9be..bf3991eeaba 100644 --- a/packages/harvest/src/routes/mod.rs +++ b/packages/harvest/src/routes/mod.rs @@ -25,6 +25,7 @@ pub mod export_tally_results; pub mod export_tasks_execution; pub mod export_template; pub mod export_tenant_config; +pub mod external_reconciliation; pub mod fetch_document; pub mod generate_preview_url; pub mod get_certificate_authorities_pem; diff --git a/packages/harvest/src/routes/users.rs b/packages/harvest/src/routes/users.rs index 1d5450bd2b2..8f3c4b6c5c2 100644 --- a/packages/harvest/src/routes/users.rs +++ b/packages/harvest/src/routes/users.rs @@ -27,10 +27,10 @@ use windmill::postgres::election_event::get_election_event_by_id; use windmill::services::cast_votes::get_users_with_vote_info; use windmill::services::celery_app::get_celery_app; use windmill::services::database::{get_hasura_pool, get_keycloak_pool}; -use windmill::services::datafix::utils::datafix_annotations; use windmill::services::export::export_users::{ ExportBody, ExportTenantUsersBody, ExportUsersBody, }; +use windmill::services::external::utils::datafix_annotations; use windmill::services::keycloak_events::list_keycloak_events_by_type; use windmill::services::tasks_execution::*; use windmill::services::users::list_users_has_voted; diff --git a/packages/results-portal/graphql.schema.json b/packages/results-portal/graphql.schema.json index 389cef0a4fb..7472612b9e2 100644 --- a/packages/results-portal/graphql.schema.json +++ b/packages/results-portal/graphql.schema.json @@ -11337,6 +11337,51 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "apply_external_reconciliation_changes", + "description": "applies the Sequent-side reconciliation diff for an already-computed import", + "args": [ + { + "name": "diff_document_id", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "election_event_id", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "type": { + "kind": "OBJECT", + "name": "taskOutput", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "call_plugin_route", "description": null, @@ -11755,6 +11800,51 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "create_external_reconciliation_import", + "description": "uploads an external (i.e. Datafix) reconciliation file and kicks off the diff-calculation task", + "args": [ + { + "name": "document_id", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "election_event_id", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "type": { + "kind": "OBJECT", + "name": "taskOutput", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "create_keys_ceremony", "description": "create keys ceremony", diff --git a/packages/results-portal/src/gql/graphql.ts b/packages/results-portal/src/gql/graphql.ts index b57a0bd060f..dd1b50f4c3d 100644 --- a/packages/results-portal/src/gql/graphql.ts +++ b/packages/results-portal/src/gql/graphql.ts @@ -1228,6 +1228,8 @@ export type Mutation_Root = { ApplicationChangeStatus?: Maybe; /** Verify User Registration Application */ VerifyApplication: Scalars['String']['output']; + /** applies the Sequent-side reconciliation diff for an already-computed import */ + apply_external_reconciliation_changes?: Maybe; call_plugin_route: PluginsRouteOutput; /** check private key */ check_private_key?: Maybe; @@ -1238,6 +1240,8 @@ export type Mutation_Root = { /** create_ballot_receipt */ create_ballot_receipt?: Maybe; create_election?: Maybe; + /** uploads an external (i.e. Datafix) reconciliation file and kicks off the diff-calculation task */ + create_external_reconciliation_import?: Maybe; /** create keys ceremony */ create_keys_ceremony?: Maybe; /** create_new_tally_sheet */ @@ -1958,6 +1962,13 @@ export type Mutation_RootVerifyApplicationArgs = { }; +/** mutation root */ +export type Mutation_RootApply_External_Reconciliation_ChangesArgs = { + diff_document_id: Scalars['String']['input']; + election_event_id: Scalars['String']['input']; +}; + + /** mutation root */ export type Mutation_RootCall_Plugin_RouteArgs = { data: Scalars['jsonb']['input']; @@ -2010,6 +2021,13 @@ export type Mutation_RootCreate_ElectionArgs = { }; +/** mutation root */ +export type Mutation_RootCreate_External_Reconciliation_ImportArgs = { + document_id: Scalars['String']['input']; + election_event_id: Scalars['String']['input']; +}; + + /** mutation root */ export type Mutation_RootCreate_Keys_CeremonyArgs = { object: CreateKeysCeremonyInput; diff --git a/packages/sequent-core/src/types/permissions.rs b/packages/sequent-core/src/types/permissions.rs index 67d5386e11f..81d70283c2a 100644 --- a/packages/sequent-core/src/types/permissions.rs +++ b/packages/sequent-core/src/types/permissions.rs @@ -401,6 +401,8 @@ pub enum Permissions { PHONE_BLACKLIST_UPDATE, #[strum(serialize = "phone-blacklist-delete")] PHONE_BLACKLIST_DELETE, + #[strum(serialize = "election-event-voter-list-reconciliation")] + ELECTION_EVENT_VOTER_LIST_SYNC, } #[allow(non_camel_case_types)] diff --git a/packages/step-cli/src/graphql/schema.json b/packages/step-cli/src/graphql/schema.json index d75ab99a553..4fb4e748adf 100644 --- a/packages/step-cli/src/graphql/schema.json +++ b/packages/step-cli/src/graphql/schema.json @@ -10293,6 +10293,47 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "apply_external_reconciliation_changes", + "description": "applies the Sequent-side reconciliation diff for an already-computed import", + "args": [ + { + "name": "diff_document_id", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "election_event_id", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "taskOutput", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "call_plugin_route", "description": null, @@ -10667,6 +10708,47 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "create_external_reconciliation_import", + "description": "uploads an external (i.e. Datafix) reconciliation file and kicks off the diff-calculation task", + "args": [ + { + "name": "document_id", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + }, + { + "name": "election_event_id", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null + } + ], + "type": { + "kind": "OBJECT", + "name": "taskOutput", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "create_keys_ceremony", "description": "create keys ceremony", diff --git a/packages/ui-core/rust/sequent-core-0.1.0.tgz b/packages/ui-core/rust/sequent-core-0.1.0.tgz index 2ba45243109..d6bd3f03b69 100644 Binary files a/packages/ui-core/rust/sequent-core-0.1.0.tgz and b/packages/ui-core/rust/sequent-core-0.1.0.tgz differ diff --git a/packages/voting-portal/graphql.schema.json b/packages/voting-portal/graphql.schema.json index 389cef0a4fb..7472612b9e2 100644 --- a/packages/voting-portal/graphql.schema.json +++ b/packages/voting-portal/graphql.schema.json @@ -11337,6 +11337,51 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "apply_external_reconciliation_changes", + "description": "applies the Sequent-side reconciliation diff for an already-computed import", + "args": [ + { + "name": "diff_document_id", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "election_event_id", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "type": { + "kind": "OBJECT", + "name": "taskOutput", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "call_plugin_route", "description": null, @@ -11755,6 +11800,51 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "create_external_reconciliation_import", + "description": "uploads an external (i.e. Datafix) reconciliation file and kicks off the diff-calculation task", + "args": [ + { + "name": "document_id", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "election_event_id", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "type": { + "kind": "OBJECT", + "name": "taskOutput", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "create_keys_ceremony", "description": "create keys ceremony", diff --git a/packages/voting-portal/rust/sequent-core-0.1.0.tgz b/packages/voting-portal/rust/sequent-core-0.1.0.tgz index 2ba45243109..d6bd3f03b69 100644 Binary files a/packages/voting-portal/rust/sequent-core-0.1.0.tgz and b/packages/voting-portal/rust/sequent-core-0.1.0.tgz differ diff --git a/packages/voting-portal/src/gql/graphql.ts b/packages/voting-portal/src/gql/graphql.ts index fdb98544482..6280618f0e9 100644 --- a/packages/voting-portal/src/gql/graphql.ts +++ b/packages/voting-portal/src/gql/graphql.ts @@ -1228,6 +1228,8 @@ export type Mutation_Root = { ApplicationChangeStatus?: Maybe; /** Verify User Registration Application */ VerifyApplication: Scalars['String']['output']; + /** applies the Sequent-side reconciliation diff for an already-computed import */ + apply_external_reconciliation_changes?: Maybe; call_plugin_route: PluginsRouteOutput; /** check private key */ check_private_key?: Maybe; @@ -1238,6 +1240,8 @@ export type Mutation_Root = { /** create_ballot_receipt */ create_ballot_receipt?: Maybe; create_election?: Maybe; + /** uploads an external (i.e. Datafix) reconciliation file and kicks off the diff-calculation task */ + create_external_reconciliation_import?: Maybe; /** create keys ceremony */ create_keys_ceremony?: Maybe; /** create_new_tally_sheet */ @@ -1958,6 +1962,13 @@ export type Mutation_RootVerifyApplicationArgs = { }; +/** mutation root */ +export type Mutation_RootApply_External_Reconciliation_ChangesArgs = { + diff_document_id: Scalars['String']['input']; + election_event_id: Scalars['String']['input']; +}; + + /** mutation root */ export type Mutation_RootCall_Plugin_RouteArgs = { data: Scalars['jsonb']['input']; @@ -2010,6 +2021,13 @@ export type Mutation_RootCreate_ElectionArgs = { }; +/** mutation root */ +export type Mutation_RootCreate_External_Reconciliation_ImportArgs = { + document_id: Scalars['String']['input']; + election_event_id: Scalars['String']['input']; +}; + + /** mutation root */ export type Mutation_RootCreate_Keys_CeremonyArgs = { object: CreateKeysCeremonyInput; diff --git a/packages/windmill/src/postgres/area.rs b/packages/windmill/src/postgres/area.rs index 89f4fab493b..a04f1d66cb9 100644 --- a/packages/windmill/src/postgres/area.rs +++ b/packages/windmill/src/postgres/area.rs @@ -499,6 +499,60 @@ pub async fn get_event_areas( Ok(election_events) } +/// Errors if does not find exactly one area with the given name in the given election event. +/// Returns the area-id of the found area. +#[instrument(err, skip_all)] +pub async fn get_area_id_from_event_by_name( + hasura_transaction: &Transaction<'_>, + tenant_id: &str, + election_event_id: &str, + area_name: &str, +) -> Result { + let statement = hasura_transaction + .prepare( + r#" + SELECT + id + FROM + sequent_backend.area + WHERE + tenant_id = $1 AND + election_event_id = $2 AND + name = $3; + "#, + ) + .await?; + + let rows: Vec = hasura_transaction + .query( + &statement, + &[ + &parse_uuid_v4(tenant_id)?, + &parse_uuid_v4(election_event_id)?, + &area_name, + ], + ) + .await?; + + match rows.len() { + 0 => Err(anyhow!( + "No area found with name '{}' in election event '{}'", + area_name, + election_event_id + )), + 1 => { + let area_id: Uuid = rows[0].try_get("id")?; + Ok(area_id.to_string()) + } + count => Err(anyhow!( + "Found {} areas with name '{}' in election event '{}'", + count, + area_name, + election_event_id + )), + } +} + #[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize)] pub struct AreaElection { pub id: String, diff --git a/packages/windmill/src/postgres/cast_vote.rs b/packages/windmill/src/postgres/cast_vote.rs index 57e0c5415da..21966441451 100644 --- a/packages/windmill/src/postgres/cast_vote.rs +++ b/packages/windmill/src/postgres/cast_vote.rs @@ -8,6 +8,7 @@ use sequent_core::ballot::VotingStatusChannel; use sequent_core::services::uuid_validation::parse_uuid_v4; use serde::Serialize; use serde_json::Value; +use std::collections::HashMap; use tokio_postgres::row::Row; use tracing::instrument; use uuid::Uuid; @@ -596,6 +597,64 @@ pub async fn get_cast_votes( Ok(cast_votes) } +/// Precomputes unresolved and valid ballot state for every voter with an +/// active cast vote in an event. Reconciliation needs both states: an +/// `in-progress` ballot is expected during a Datafix freeze and must never be +/// mistaken for "not voted" merely because it is not valid yet. +#[instrument(skip(hasura_transaction), err)] +pub async fn get_voter_cast_vote_states_for_event( + hasura_transaction: &Transaction<'_>, + tenant_id: &str, + election_event_id: &str, +) -> Result> { + let unresolved_status = CastVoteStatus::InProgress.to_string(); + let valid_status = CastVoteStatus::Valid.to_string(); + let statement = hasura_transaction + .prepare( + r#" + SELECT + voter_id_string, + bool_or(status = $3) AS has_unresolved_vote, + bool_or(status = $4) AS has_valid_vote + FROM sequent_backend.cast_vote + WHERE tenant_id = $1 + AND election_event_id = $2 + AND status IN ($3, $4) + AND voter_id_string IS NOT NULL + GROUP BY voter_id_string + "#, + ) + .await?; + + let rows: Vec = hasura_transaction + .query( + &statement, + &[ + &parse_uuid_v4(tenant_id)?, + &parse_uuid_v4(election_event_id)?, + &unresolved_status, + &valid_status, + ], + ) + .await?; + + Ok(rows + .into_iter() + .filter_map(|row| { + row.get::<_, Option>("voter_id_string") + .map(|voter_id| { + ( + voter_id, + VoterCastVoteState { + has_unresolved_vote: row.get("has_unresolved_vote"), + has_valid_vote: row.get("has_valid_vote"), + }, + ) + }) + }) + .collect()) +} + #[instrument(skip(hasura_transaction), err)] pub async fn get_cast_votes_by_election_id( hasura_transaction: &Transaction<'_>, diff --git a/packages/windmill/src/services/cast_votes.rs b/packages/windmill/src/services/cast_votes.rs index edd98bdf43c..e1666924e48 100644 --- a/packages/windmill/src/services/cast_votes.rs +++ b/packages/windmill/src/services/cast_votes.rs @@ -6,10 +6,10 @@ use super::sql_utils::escape_sql_literal; use crate::postgres::cast_vote::{ count_distinct_voters_by_channel_query, count_votes_per_day_query, CastVoteRelation, }; -use crate::services::datafix::utils::{ +use crate::services::electoral_log::ElectoralLog; +use crate::services::external::utils::{ is_datafix_election_event_by_id, voted_via_not_internet_channel, }; -use crate::services::electoral_log::ElectoralLog; use anyhow::{anyhow, Context, Result}; use chrono::NaiveDate; use chrono::{DateTime, Utc}; diff --git a/packages/windmill/src/services/celery_app.rs b/packages/windmill/src/services/celery_app.rs index 36ae7e35cd2..e538142791e 100644 --- a/packages/windmill/src/services/celery_app.rs +++ b/packages/windmill/src/services/celery_app.rs @@ -12,6 +12,7 @@ use tracing::{event, info, instrument, Level}; use crate::services::plugins_manager::plugin_manager::init_plugin_manager; use crate::tasks::activity_logs_report::generate_activity_logs_report; +use crate::tasks::apply_reconciliation_patch::apply_reconciliation_patch; use crate::tasks::create_ballot_receipt::create_ballot_receipt; use crate::tasks::create_keys::create_keys; use crate::tasks::delete_election_event::delete_election_event_t; @@ -30,6 +31,7 @@ use crate::tasks::export_templates::export_templates; use crate::tasks::export_tenant_config::export_tenant_config; use crate::tasks::export_trustees::export_trustees_task; use crate::tasks::export_users::export_users; +use crate::tasks::generate_reconciliation_patches::generate_reconciliation_patches; use crate::tasks::generate_report::generate_report; use crate::tasks::generate_template::generate_template; use crate::tasks::import_application::import_applications; @@ -308,6 +310,8 @@ pub async fn generate_celery_app() -> Result> { post_tally_task, import_templates_task, publish_results_website_task, + generate_reconciliation_patches, + apply_reconciliation_patch, ], task_routes = [ create_keys::NAME => &Queue::Short.queue_name(&slug), @@ -363,6 +367,10 @@ pub async fn generate_celery_app() -> Result> { publish_results_website_task::NAME => &Queue::Reports.queue_name(&slug), process_cast_vote::NAME => &Queue::Communication.queue_name(&slug), edit_user::NAME => &Queue::Short.queue_name(&slug), + // Same queue as import_users/export_users: same order of + // magnitude of work. + generate_reconciliation_patches::NAME => &Queue::ImportExport.queue_name(&slug), + apply_reconciliation_patch::NAME => &Queue::ImportExport.queue_name(&slug), ], prefetch_count = prefetch_count, acks_late = acks_late, diff --git a/packages/windmill/src/services/electoral_log.rs b/packages/windmill/src/services/electoral_log.rs index 6734f6490aa..6c1621f3579 100644 --- a/packages/windmill/src/services/electoral_log.rs +++ b/packages/windmill/src/services/electoral_log.rs @@ -504,6 +504,44 @@ impl ElectoralLog { Ok(()) } + /// Posts a third-party voter registry reconciliation run event (patch + /// generation or applying the Sequent-side diff) — see + /// `windmill::services::external::reconciliation`. Named for the general + /// capability, not the specific integration (Datafix) that first needed + /// it. `artifact` carries the JSON of old/new values applied, for a + /// `ChangesApplied` entry (`None` for `PatchGenerated`, which has nothing + /// to apply yet). + #[instrument(skip(self, artifact), fields(kind = %kind), err)] + pub async fn post_external_reconciliation( + &self, + event_id: String, + kind: ExternalReconciliationKind, + sequence: i64, + generated_at: i64, + input_sha256: String, + output_sha256: Option, + artifact: Option>, + user_id: Option, + username: Option, + ) -> Result<()> { + let event = EventIdString(event_id); + + let message = Message::external_reconciliation_message( + event, + kind, + ExternalReconciliationSequenceString(sequence.to_string()), + ExternalReconciliationGeneratedAtString(generated_at.to_string()), + ExternalReconciliationInputHashString(input_sha256), + ExternalReconciliationOutputHashString(output_sha256), + artifact, + &self.sd, + user_id, + username, + )?; + + self.post(&message).await + } + #[instrument(skip(self))] pub async fn post_results_publication_action( &self, diff --git a/packages/windmill/src/services/datafix/api_datafix.rs b/packages/windmill/src/services/external/api_datafix.rs similarity index 87% rename from packages/windmill/src/services/datafix/api_datafix.rs rename to packages/windmill/src/services/external/api_datafix.rs index bbd63cad787..582972a1d40 100644 --- a/packages/windmill/src/services/datafix/api_datafix.rs +++ b/packages/windmill/src/services/external/api_datafix.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: 2025 Sequent Tech Inc // // SPDX-License-Identifier: AGPL-3.0-only -use super::types::*; +use super::datafix_types::*; use super::utils::*; use crate::postgres::cast_vote::{get_voter_cast_vote_state, VoterCastVoteState}; @@ -247,8 +247,9 @@ pub async fn mark_as_voted_via_channel( Ok(DatafixResponse::ok()) } -/// Unmark a voter as having voted, set the attribute to None -/// Also enables the voter +/// Unmark a voter as having voted. Re-enable only when MarkVoted was the +/// operation that disabled the account; an unrelated administrator disable +/// and its reason must survive this call. #[instrument(skip(hasura_transaction, keycloak_transaction))] pub async fn unmark_voter_as_voted( hasura_transaction: &Transaction<'_>, @@ -264,23 +265,18 @@ pub async fn unmark_voter_as_voted( DatafixResponse::error(DatafixErrorCode::InternalError) })?; - let mut hash_map = HashMap::new(); - hash_map.insert( - VOTED_CHANNEL.to_string(), - vec![ATTR_RESET_VALUE.to_string()], - ); - hash_map.insert( - DISABLE_COMMENT.to_string(), - vec![ATTR_RESET_VALUE.to_string()], - ); - let attributes = Some(hash_map); let user_id = get_user_id(keycloak_transaction, realm, &username).await?; + let current_user = client.get_user(realm, &user_id).await.map_err(|e| { + error!("Error loading user before unmarking voted state: {e:?}"); + DatafixResponse::error(DatafixErrorCode::InternalError) + })?; + let (enabled, attributes) = plan_unmark_voter_edit(¤t_user); let _user = client .edit_user( realm, &user_id, - Some(true), // Enable the voter again - attributes, + enabled, + Some(attributes), None, None, None, @@ -296,6 +292,35 @@ pub async fn unmark_voter_as_voted( Ok(DatafixResponse::ok()) } +/// Pure state transition shared by the inbound operation's tests and kept in +/// lockstep with reconciliation's VOTED_UNMARKED planning. `voted-channel` +/// is always reset. `disable-comment` and `enabled` are owned by this +/// operation only when the current reason is MARKVOTED_CALL (or the account +/// is already enabled and merely carries a stale reason). +fn plan_unmark_voter_edit(user: &User) -> (Option, HashMap>) { + let enabled = user.enabled.unwrap_or(false); + let attributes = user.attributes.as_ref(); + let disable_comment = attributes + .and_then(|values| values.get(DISABLE_COMMENT)) + .and_then(|values| values.last()) + .map(String::as_str) + .unwrap_or(ATTR_RESET_VALUE); + let disabled_by_mark_voted = !enabled && disable_comment == DISABLE_REASON_MARKVOTED_CALL; + + let mut changes = HashMap::from([( + VOTED_CHANNEL.to_string(), + vec![ATTR_RESET_VALUE.to_string()], + )]); + if enabled || disabled_by_mark_voted { + changes.insert( + DISABLE_COMMENT.to_string(), + vec![ATTR_RESET_VALUE.to_string()], + ); + } + + (disabled_by_mark_voted.then_some(true), changes) +} + /// Generate a new password. #[instrument(skip(hasura_transaction, keycloak_transaction, datafix_annotations))] pub async fn replace_voter_pin( @@ -408,7 +433,9 @@ pub async fn acquire_inbound_voter_lock( drop(hasura_client); let realm = get_event_realm(&claims.tenant_id, &election_event_id); let user_id = get_user_id(keycloak_transaction, &realm, username).await?; - let lock_key = datafix_voter_lock_key(&claims.tenant_id, &election_event_id, &user_id); + let user_id_uuid = parse_uuid_v4(&user_id) + .map_err(|_| DatafixResponse::error(DatafixErrorCode::InternalError))?; + let lock_key = external_voter_lock_key(&claims.tenant_id, &election_event_id, &user_id_uuid); let lock = PgLock::acquire( lock_key, Uuid::new_v4().to_string(), @@ -640,11 +667,18 @@ pub fn valid_inbound_voting_channel(channel: &str) -> bool { #[cfg(test)] mod tests { - use super::{active_vote_error, create_user_error_response, valid_inbound_voting_channel}; + use super::{ + active_vote_error, create_user_error_response, plan_unmark_voter_edit, + valid_inbound_voting_channel, + }; use crate::postgres::cast_vote::VoterCastVoteState; - use crate::services::datafix::types::DatafixErrorCode; + use crate::services::external::datafix_types::DatafixErrorCode; use keycloak::KeycloakError; use rocket::http::Status; + use sequent_core::types::keycloak::{ + User, ATTR_RESET_VALUE, DISABLE_COMMENT, DISABLE_REASON_MARKVOTED_CALL, VOTED_CHANNEL, + }; + use std::collections::HashMap; /// Builds the error `create_user` returns when Keycloak answers with the /// given HTTP status. @@ -721,4 +755,38 @@ mod tests { None ); } + + #[test] + fn unmark_only_reenables_accounts_disabled_by_mark_voted() { + let mark_voted_user = User { + enabled: Some(false), + attributes: Some(HashMap::from([( + DISABLE_COMMENT.to_string(), + vec![DISABLE_REASON_MARKVOTED_CALL.to_string()], + )])), + ..User::default() + }; + let (enabled, attributes) = plan_unmark_voter_edit(&mark_voted_user); + assert_eq!(enabled, Some(true)); + assert_eq!( + attributes.get(DISABLE_COMMENT), + Some(&vec![ATTR_RESET_VALUE.to_string()]) + ); + + let manually_disabled_user = User { + enabled: Some(false), + attributes: Some(HashMap::from([( + DISABLE_COMMENT.to_string(), + vec!["Disabled manually".to_string()], + )])), + ..User::default() + }; + let (enabled, attributes) = plan_unmark_voter_edit(&manually_disabled_user); + assert_eq!(enabled, None); + assert!(!attributes.contains_key(DISABLE_COMMENT)); + assert_eq!( + attributes.get(VOTED_CHANNEL), + Some(&vec![ATTR_RESET_VALUE.to_string()]) + ); + } } diff --git a/packages/windmill/src/services/datafix/types.rs b/packages/windmill/src/services/external/datafix_types.rs similarity index 60% rename from packages/windmill/src/services/datafix/types.rs rename to packages/windmill/src/services/external/datafix_types.rs index 38a6d537755..01f8a417436 100644 --- a/packages/windmill/src/services/datafix/types.rs +++ b/packages/windmill/src/services/external/datafix_types.rs @@ -1,7 +1,10 @@ // SPDX-FileCopyrightText: 2025 Sequent Tech Inc // // SPDX-License-Identifier: AGPL-3.0-only -use super::utils::{DATAFIX_ID_KEY, DATAFIX_PSW_POLICY_KEY, DATAFIX_VOTERVIEW_REQ_KEY}; +use super::utils::{ + DATAFIX_ID_KEY, DATAFIX_LAST_APPLIED_SEQUENCE_KEY, DATAFIX_LAST_APPLY_HAD_FAILURES_KEY, + DATAFIX_PSW_POLICY_KEY, DATAFIX_VOTERVIEW_REQ_KEY, +}; use anyhow::{anyhow, Result}; use rand::{distr, Rng}; use rocket::http::Status; @@ -142,6 +145,13 @@ pub struct DatafixAnnotations { pub id: String, pub password_policy: PasswordPolicy, pub voterview_request: VoterviewRequest, + /// See `DATAFIX_LAST_APPLIED_SEQUENCE_KEY`. `None` means no reconciliation + /// has ever been applied; this distinction is required because the + /// legitimate kickoff file uses `Sequence=0`. + pub last_applied_sequence: Option, + /// Whether the last apply finished with per-row failures. Only that state + /// permits another apply at the same Sequence. + pub last_apply_had_failures: bool, } #[derive(Default, Display, Serialize, Deserialize, Debug, Clone, EnumString)] @@ -224,10 +234,30 @@ impl ValidateAnnotations for ElectionEventDatafix { None => return Err(anyhow!("{DATAFIX_VOTERVIEW_REQ_KEY} not found")), }; + let last_applied_sequence = annotations + .get(DATAFIX_LAST_APPLIED_SEQUENCE_KEY) + .map(|value| { + value.parse::().map_err(|err| { + anyhow!("Invalid {DATAFIX_LAST_APPLIED_SEQUENCE_KEY} value '{value}': {err}") + }) + }) + .transpose()?; + let last_apply_had_failures = annotations + .get(DATAFIX_LAST_APPLY_HAD_FAILURES_KEY) + .map(|value| { + value.parse::().map_err(|err| { + anyhow!("Invalid {DATAFIX_LAST_APPLY_HAD_FAILURES_KEY} value '{value}': {err}") + }) + }) + .transpose()? + .unwrap_or(false); + Ok(DatafixAnnotations { id, password_policy, voterview_request, + last_applied_sequence, + last_apply_had_failures, }) } } @@ -286,9 +316,156 @@ pub struct SoapRequestData<'a> { pub timestamp: &'a str, } +// ======================================================================= +// Datafix reconciliation types. These live here (not in sequent_core::types) +// since they don't need WASM exposure, and the direct precedent for this +// feature (everything else in this file) already lives here and is reused by +// harvest via its dependency on this crate. `ReconciliationFileMeta`, +// `ReconciliationChangeCategory` and `ReconciliationPatchTarget` live in +// `super::types` instead — they're not Datafix-specific wire shapes. +// ======================================================================= + +/// The reconciliation file format's own value for an Internet vote in the +/// `Channel` column — always uppercase per the "Accepted Values" spec, and +/// distinct from Keycloak's stored `VOTED_CHANNEL_INTERNET_VALUE` ("Internet"): +/// comparisons and writes must cross the explicit mappings below. +pub const FILE_CHANNEL_INTERNET: &str = "INTERNET"; + +/// Converts the file contract's channel representation into the value stored +/// in Keycloak. Other channels stay uppercase; only Sequent's historical +/// Internet spelling differs. +pub fn file_channel_to_keycloak(channel: &str) -> String { + if channel.eq_ignore_ascii_case(FILE_CHANNEL_INTERNET) { + sequent_core::types::keycloak::VOTED_CHANNEL_INTERNET_VALUE.to_string() + } else { + channel.to_uppercase() + } +} + +/// Converts a Keycloak channel to the canonical reconciliation-file value. +pub fn keycloak_channel_to_file(channel: &str) -> String { + channel.to_uppercase() +} + +pub fn channels_equal(left: &str, right: &str) -> bool { + left.eq_ignore_ascii_case(right) +} + +/// One column of the "Patch Files Format" `_old`/`_new` pair contract, +/// carrying that pair directly (`old`, `new`) instead of leaving it to a +/// separate `old_value`/`new_value` on `DiffItem` — a field and its own +/// old/new values are never meaningful apart from each other, so keeping +/// them on `DiffItem` alongside the field was pure duplication. Wire column +/// names match `PATCH_FIELDS` in the admin portal's `types.ts` exactly. +#[allow(non_camel_case_types)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum DatafixReconciliationField { + CountyMun(String, String), + DoB(String, String), + Ward(String, String), + Poll(String, String), + SchoolSupportCode(String, String), + Channel(String, String), + /// Kept as the literal CSV strings ("true"/"false"/"NONE"), not a `bool` + /// — unlike Sequent's own `Enabled` (a genuine two-state Keycloak flag), + /// the patch CSV's `Deleted` column can legitimately carry `NONE` (no + /// prior value, e.g. reporting a Sequent-only voter Datafix has never + /// seen), which a `bool` can't represent. + Deleted(String, String), +} + +impl DatafixReconciliationField { + /// Every column name, in the fixed order the patch CSV and the "Patch + /// Files Format" spec require regardless of which fields changed — + /// carries no old/new data since it's used to iterate the fixed set of + /// possible columns, not any voter's actual values. + pub const NAMES: [&'static str; 7] = [ + "CountyMun", + "DoB", + "Ward", + "Poll", + "SchoolSupportCode", + "Channel", + "Deleted", + ]; + + /// The column name this instance carries a value for. + pub fn name(&self) -> &'static str { + match self { + Self::CountyMun(..) => "CountyMun", + Self::DoB(..) => "DoB", + Self::Ward(..) => "Ward", + Self::Poll(..) => "Poll", + Self::SchoolSupportCode(..) => "SchoolSupportCode", + Self::Channel(..) => "Channel", + Self::Deleted(..) => "Deleted", + } + } + + /// The `(old, new)` pair as the literal strings the patch CSV writes. + pub fn old_new(&self) -> (&str, &str) { + match self { + Self::CountyMun(old, new) + | Self::DoB(old, new) + | Self::Ward(old, new) + | Self::Poll(old, new) + | Self::SchoolSupportCode(old, new) + | Self::Channel(old, new) + | Self::Deleted(old, new) => (old.as_str(), new.as_str()), + } + } +} + +/// One row of an uploaded reconciliation file, after CSV parsing. Field names +/// match the CSV header (`CountyMun,VoterID,DoB,Ward,Poll,SchoolSupportCode, +/// Channel,Deleted`) via `serde(rename)` so `csv::Reader::deserialize` can +/// build this directly — see `reconciliation::csv`. +#[derive(Deserialize, Serialize, Debug, Clone)] +pub struct ParsedDatafixReconciliationRow { + #[serde(rename = "CountyMun")] + pub county_mun: String, + #[serde(rename = "VoterID")] + pub external_voter_id: String, + #[serde(rename = "DoB")] + pub dob: String, + #[serde(rename = "Ward")] + pub ward: String, + #[serde(rename = "Poll")] + pub poll: String, + #[serde(rename = "SchoolSupportCode")] + pub school_support_code: String, + #[serde(rename = "Channel")] + pub channel: String, + #[serde(rename = "Deleted")] + pub deleted: String, // "true"/"false" — kept as the wire string, parsed where needed +} + +impl ParsedDatafixReconciliationRow { + /// This row's own value for one of `DatafixReconciliationField::NAMES`, + /// by column name — used to fill in a field that didn't change for this + /// voter on the outbound patch CSV with its real current value (this row + /// is exactly that value, since an unchanged field is one Sequent didn't + /// disagree with) instead of a placeholder. + pub fn field_value(&self, name: &str) -> Option<&str> { + match name { + "CountyMun" => Some(&self.county_mun), + "DoB" => Some(&self.dob), + "Ward" => Some(&self.ward), + "Poll" => Some(&self.poll), + "SchoolSupportCode" => Some(&self.school_support_code), + "Channel" => Some(&self.channel), + "Deleted" => Some(&self.deleted), + _ => None, + } + } +} + #[cfg(test)] mod tests { - use super::{DatafixErrorCode, DatafixResponse}; + use super::{ + channels_equal, file_channel_to_keycloak, keycloak_channel_to_file, DatafixErrorCode, + DatafixResponse, FILE_CHANNEL_INTERNET, + }; use rocket::http::Status; #[test] @@ -372,4 +549,12 @@ mod tests { serde_json::json!({"code": 200, "message": "OK"}) ); } + + #[test] + fn channel_boundaries_use_each_systems_canonical_spelling() { + assert!(channels_equal("Internet", FILE_CHANNEL_INTERNET)); + assert_eq!(file_channel_to_keycloak(FILE_CHANNEL_INTERNET), "Internet"); + assert_eq!(keycloak_channel_to_file("Internet"), FILE_CHANNEL_INTERNET); + assert_eq!(file_channel_to_keycloak("paper"), "PAPER"); + } } diff --git a/packages/windmill/src/services/datafix/mod.rs b/packages/windmill/src/services/external/mod.rs similarity index 80% rename from packages/windmill/src/services/datafix/mod.rs rename to packages/windmill/src/services/external/mod.rs index 860db8c6462..36077223647 100644 --- a/packages/windmill/src/services/datafix/mod.rs +++ b/packages/windmill/src/services/external/mod.rs @@ -3,6 +3,8 @@ // SPDX-License-Identifier: AGPL-3.0-only pub mod api_datafix; +pub mod datafix_types; +pub mod reconciliation; pub mod types; pub mod utils; pub mod voterview_requests; diff --git a/packages/windmill/src/services/external/reconciliation/apply.rs b/packages/windmill/src/services/external/reconciliation/apply.rs new file mode 100644 index 00000000000..1664767dbdd --- /dev/null +++ b/packages/windmill/src/services/external/reconciliation/apply.rs @@ -0,0 +1,446 @@ +// SPDX-FileCopyrightText: 2026 Sequent Tech Inc +// +// SPDX-License-Identifier: AGPL-3.0-only + +//! Applies the `target = Sequent` side of a reconciliation diff. Per-voter +//! atomic (spec, "Implementation Requirements": "a row failure does not +//! abort the process") — one voter's failure marks that voter's items +//! FAILED and moves on, exactly like `tally_sheet_import`'s +//! `find_stale_baseline_conflicts` gates a single item without aborting the +//! whole import. +//! +//! Deliberately source-agnostic: every existing-voter category reduces to +//! one generic Keycloak edit (`apply_generic_voter_edit`) built entirely from +//! what the `DiffItem`s already carry (`SequentReconciliationField`) — this +//! module never calls into `api_datafix`, so it has no idea the diff came +//! from Datafix, only that it targets Sequent. `VOTER_ADDED` is handled only +//! by `bulk_create`, so there is one authoritative creation path. +//! +//! The per-voter cast-vote guards (`ensure_voter_has_no_active_vote`-style +//! checks before a disable/re-enable actually lands) are re-implemented here +//! rather than reused from the inbound Datafix API, since those take a +//! `DatafixClaims` (the inbound request guard) that doesn't exist on this +//! path — this module re-validates the same underlying condition +//! (`VoterCastVoteState`, including unresolved and valid votes) directly. + +use crate::postgres::area::{get_area_by_id, get_area_id_from_event_by_name}; +use crate::postgres::cast_vote::get_voter_cast_vote_state; +use crate::services::external::reconciliation::diff::DiffItem; +use crate::services::external::types::{ReconciliationChangeCategory, SequentReconciliationField}; +use crate::services::external::utils::{ + external_voter_lock_key, voted_via_internet, voted_via_not_internet_channel, + DATAFIX_VOTER_LOCK_SECS, +}; +use crate::services::pg_lock::PgLock; +use anyhow::{anyhow, Result}; +use deadpool_postgres::Transaction; +use sequent_core::services::date::ISO8601; +use sequent_core::services::keycloak::KeycloakAdminClient; +use sequent_core::services::uuid_validation::parse_uuid_v4; +use sequent_core::types::keycloak::{User, AREA_ID_ATTR_NAME, ATTR_RESET_VALUE, VOTED_CHANNEL}; +use std::collections::{HashMap, HashSet}; +use tracing::{error, info, instrument}; +use uuid::Uuid; + +/// Outcome of applying one voter's queued changes. +#[derive(Debug)] +pub enum VoterApplyOutcome { + Applied, + Failed { reason: String }, +} + +/// Applies every `target = Sequent` item for one voter (`items` — all sharing +/// `voter_username`, already filtered to exclude `ROW_FAILURE`) under the +/// same per-voter Datafix advisory lock the inbound API and `edit_user` use, +/// so this can never interleave with an inbound Datafix call or an outbound +/// `SetVoted`/`SetNotVoted` for the same voter. Locked on `voter_id`, not +/// `voter_username`, to match the key `edit_user`/the inbound API use — both +/// carried on `items` already, from the `VoterSnapshot` the diff was built +/// from, so this needs no Keycloak lookup of its own. +#[instrument(skip(hasura_transaction, items), fields(voter_username = %voter_username), err)] +pub async fn apply_voter_changes( + hasura_transaction: &Transaction<'_>, + tenant_id: &str, + election_event_id: &str, + realm: &str, + voter_username: &str, + items: &[DiffItem], +) -> Result { + let Some(first_item) = items.first() else { + return Ok(VoterApplyOutcome::Applied); // nothing to do + }; + let Some(user_id) = first_item.voter_id else { + return Ok(VoterApplyOutcome::Failed { + reason: "Diff item is missing the voter's Keycloak id".to_string(), + }); + }; + + let lock = PgLock::acquire( + external_voter_lock_key(tenant_id, election_event_id, &user_id), + Uuid::new_v4().to_string(), + ISO8601::now() + chrono::Duration::seconds(DATAFIX_VOTER_LOCK_SECS), + ) + .await?; + + let result = apply_voter_changes_locked( + hasura_transaction, + tenant_id, + election_event_id, + realm, + &user_id.to_string(), + items, + ) + .await; + + if let Err(err) = lock.release().await { + error!("Error releasing the Datafix voter lock during reconciliation apply: {err}"); + } + + result +} + +/// Validates every item's old snapshot and derives safety guards from the +/// complete category set before writing. A voter can legitimately have +/// mixed categories (for example profile update plus deletion), so dispatch +/// must never depend on item ordering. `items` is guaranteed non-empty by +/// `apply_voter_changes`, its only caller. +#[instrument(skip(hasura_transaction, items), err)] +async fn apply_voter_changes_locked( + hasura_transaction: &Transaction<'_>, + tenant_id: &str, + election_event_id: &str, + realm: &str, + user_id: &str, + items: &[DiffItem], +) -> Result { + let categories: HashSet<_> = items.iter().map(|item| item.category).collect(); + if categories.contains(&ReconciliationChangeCategory::VOTER_ADDED) { + return Ok(VoterApplyOutcome::Failed { + reason: "VOTER_ADDED must be handled by the bulk-create path".to_string(), + }); + } + if categories.iter().any(|category| { + matches!( + category, + ReconciliationChangeCategory::VOTED_INTERNET + | ReconciliationChangeCategory::DELETION_REVERTED + | ReconciliationChangeCategory::ROW_FAILURE + ) + }) { + return Ok(VoterApplyOutcome::Failed { + reason: "Datafix-side or row-failure item reached Sequent apply".to_string(), + }); + } + + let client = KeycloakAdminClient::new() + .await + .map_err(|err| anyhow!("Error getting KeycloakAdminClient: {err:?}"))?; + let current_user = client + .get_user(realm, user_id) + .await + .map_err(|err| anyhow!("Error loading current voter snapshot: {err:?}"))?; + + if let Some(reason) = + validate_old_values(hasura_transaction, tenant_id, ¤t_user, items).await? + { + return Ok(VoterApplyOutcome::Failed { reason }); + } + + let needs_no_active_vote = categories.iter().any(|category| { + matches!( + category, + ReconciliationChangeCategory::VOTED_OTHER_CHANNEL + | ReconciliationChangeCategory::VOTED_UNMARKED + | ReconciliationChangeCategory::DISABLED_DELETE_CALL + | ReconciliationChangeCategory::REENABLED + ) + }); + if needs_no_active_vote { + let state = get_voter_cast_vote_state( + hasura_transaction, + &parse_uuid_v4(tenant_id)?, + &parse_uuid_v4(election_event_id)?, + &user_id, + ) + .await?; + if state.has_unresolved_vote || state.has_valid_vote { + return Ok(VoterApplyOutcome::Failed { + reason: "Voter now has an active Internet ballot; the reconciliation change is stale and was not applied" + .to_string(), + }); + } + } + + if categories.contains(&ReconciliationChangeCategory::REENABLED) { + let attributes = current_user.attributes.clone().unwrap_or_default(); + if voted_via_internet(&attributes) || voted_via_not_internet_channel(&attributes) { + return Ok(VoterApplyOutcome::Failed { + reason: "Voter still has a voted-channel attribute and cannot be re-enabled" + .to_string(), + }); + } + } + + apply_generic_voter_edit( + hasura_transaction, + tenant_id, + election_event_id, + realm, + &user_id, + items, + ) + .await +} + +/// The one generic Keycloak edit every category above (except `VOTER_ADDED`) +/// reduces to: resolve the existing voter's `user_id`, collect whatever +/// `enabled` transition and Keycloak attributes `items` carry, and write them +/// in a single `edit_user` call. Has no notion of *why* — that judgment +/// belongs entirely to whichever origin produced the diff (see +/// `SequentReconciliationField` in `services::external::types`). +#[instrument(skip(hasura_transaction, items), err)] +async fn apply_generic_voter_edit( + hasura_transaction: &Transaction<'_>, + tenant_id: &str, + election_event_id: &str, + realm: &str, + user_id: &str, + items: &[DiffItem], +) -> Result { + let (enabled, mut attributes) = match keycloak_edit_from_items(items) { + Ok(edit) => edit, + Err(reason) => return Ok(VoterApplyOutcome::Failed { reason }), + }; + resolve_area_attribute( + hasura_transaction, + tenant_id, + election_event_id, + items, + &mut attributes, + ) + .await?; + + let client = KeycloakAdminClient::new() + .await + .map_err(|err| anyhow!("Error getting KeycloakAdminClient: {err:?}"))?; + let result = client + .edit_user( + realm, + user_id, + enabled, + Some(attributes), + None, + None, + None, + None, + None, + None, + ) + .await; + Ok(match result { + Ok(_) => VoterApplyOutcome::Applied, + Err(err) => VoterApplyOutcome::Failed { + reason: format!("{err:?}"), + }, + }) +} + +/// Compare-and-set validation for the snapshot captured during generation. +/// A mismatch is a row failure, not a task failure: another voter can still +/// be applied safely. +async fn validate_old_values( + hasura_transaction: &Transaction<'_>, + tenant_id: &str, + current_user: &User, + items: &[DiffItem], +) -> Result> { + let attributes = current_user + .attributes + .as_ref() + .cloned() + .unwrap_or_default(); + let current_attribute = |key: &str| { + attributes + .get(key) + .and_then(|values| values.last()) + .map(String::as_str) + .unwrap_or(ATTR_RESET_VALUE) + }; + + for item in items { + let Some(field) = item.target.sequent_field() else { + continue; + }; + match field { + SequentReconciliationField::Enabled(expected, _) => { + let actual = current_user.enabled.unwrap_or(false); + if actual != *expected { + return Ok(Some(format!( + "Stale snapshot for enabled: expected {expected}, found {actual}" + ))); + } + } + SequentReconciliationField::KeycloakUA(expected, _) => { + for (key, expected_value) in expected { + let actual = current_attribute(key); + let matches = if key == VOTED_CHANNEL { + actual.eq_ignore_ascii_case(expected_value) + } else { + actual == expected_value + }; + if !matches { + return Ok(Some(format!( + "Stale snapshot for attribute '{key}': expected '{expected_value}', found '{actual}'" + ))); + } + } + } + SequentReconciliationField::AreaName(expected, _) => { + let current_area_id = current_attribute(AREA_ID_ATTR_NAME); + let actual = if current_area_id == ATTR_RESET_VALUE { + ATTR_RESET_VALUE.to_string() + } else { + get_area_by_id(hasura_transaction, tenant_id, current_area_id) + .await? + .and_then(|area| area.name) + .unwrap_or_else(|| ATTR_RESET_VALUE.to_string()) + }; + if actual != *expected { + return Ok(Some(format!( + "Stale snapshot for area: expected '{expected}', found '{actual}'" + ))); + } + } + } + } + Ok(None) +} + +/// Merges every `KeycloakUA` attribute across `items` into one map, keyed +/// exactly as Keycloak expects, and derives the `enabled` transition from any +/// `Enabled` item's new value. Purely mechanical: diff.rs already decided +/// every value, this just collects them. +fn keycloak_edit_from_items( + items: &[DiffItem], +) -> std::result::Result<(Option, HashMap>), String> { + let mut enabled = None; + let mut attributes: HashMap> = HashMap::new(); + for item in items { + let Some(field) = item.target.sequent_field() else { + continue; + }; + if let Some(new_enabled) = field.new_enabled() { + if enabled.is_some_and(|current| current != new_enabled) { + return Err(format!( + "Conflicting reconciliation writes for enabled: both {} and {new_enabled}", + enabled.expect("checked as Some above") + )); + } + enabled = Some(new_enabled); + } + if let Some(keycloak_attributes) = field.new_keycloak_attributes() { + for (key, value) in keycloak_attributes { + if let Some(current) = attributes.get(key).and_then(|values| values.last()) { + if current != value { + return Err(format!( + "Conflicting reconciliation writes for Keycloak attribute '{key}': both '{current}' and '{value}'" + )); + } + } + attributes.insert(key.clone(), vec![value.clone()]); + } + } + } + Ok((enabled, attributes)) +} + +/// Resolves any `AreaName` item's composed name to a Sequent `area-id` and +/// merges it into `attributes` — a plain Sequent-domain lookup (an `Area` by +/// its already-composed name), not anything Datafix-specific: the field the +/// file used to compose that name (Ward/Poll/SchoolSupportCode) is Datafix's +/// concern alone, not this apply path's. +#[instrument(skip(hasura_transaction, items))] +async fn resolve_area_attribute( + hasura_transaction: &Transaction<'_>, + tenant_id: &str, + election_event_id: &str, + items: &[DiffItem], + attributes: &mut HashMap>, +) -> Result<()> { + let Some(area_name) = items + .iter() + .find_map(|item| item.target.sequent_field()?.new_area_name()) + else { + return Ok(()); + }; + info!("Resolving area name {area_name:?} to area-id for voter changes"); + let area_id = + get_area_id_from_event_by_name(hasura_transaction, tenant_id, election_event_id, area_name) + .await?; + attributes.insert(AREA_ID_ATTR_NAME.to_string(), vec![area_id]); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::services::external::types::ReconciliationPatchTarget; + use sequent_core::types::keycloak::{DATE_OF_BIRTH, DISABLE_COMMENT}; + + fn item(field: SequentReconciliationField) -> DiffItem { + DiffItem { + voter_username: "voter-1".to_string(), + voter_id: Some(Uuid::new_v4()), + target: ReconciliationPatchTarget::Sequent(Some(field)), + category: ReconciliationChangeCategory::PROFILE_UPDATE, + failure_reason: None, + } + } + + #[test] + fn merges_mixed_category_fields_into_one_keycloak_edit() { + let items = vec![ + item(SequentReconciliationField::Enabled(true, false)), + item(SequentReconciliationField::KeycloakUA( + HashMap::new(), + HashMap::from([(DATE_OF_BIRTH.to_string(), "1990-01-01".to_string())]), + )), + item(SequentReconciliationField::KeycloakUA( + HashMap::new(), + HashMap::from([( + DISABLE_COMMENT.to_string(), + "Disabled by reconciliation".to_string(), + )]), + )), + ]; + + let (enabled, attributes) = keycloak_edit_from_items(&items).unwrap(); + assert_eq!(enabled, Some(false)); + assert_eq!( + attributes.get(DATE_OF_BIRTH), + Some(&vec!["1990-01-01".to_string()]) + ); + assert_eq!( + attributes.get(DISABLE_COMMENT), + Some(&vec!["Disabled by reconciliation".to_string()]) + ); + } + + #[test] + fn rejects_conflicting_attribute_writes_instead_of_using_item_order() { + let items = vec![ + item(SequentReconciliationField::KeycloakUA( + HashMap::new(), + HashMap::from([(DISABLE_COMMENT.to_string(), "MARKVOTED_CALL".to_string())]), + )), + item(SequentReconciliationField::KeycloakUA( + HashMap::new(), + HashMap::from([(DISABLE_COMMENT.to_string(), "DELETE_CALL".to_string())]), + )), + ]; + + let error = keycloak_edit_from_items(&items).unwrap_err(); + assert!(error.contains("Conflicting reconciliation writes")); + assert!(error.contains(DISABLE_COMMENT)); + } +} diff --git a/packages/windmill/src/services/external/reconciliation/bulk_create.rs b/packages/windmill/src/services/external/reconciliation/bulk_create.rs new file mode 100644 index 00000000000..9f7c34f8e02 --- /dev/null +++ b/packages/windmill/src/services/external/reconciliation/bulk_create.rs @@ -0,0 +1,467 @@ +// SPDX-FileCopyrightText: 2026 Sequent Tech Inc +// +// SPDX-License-Identifier: AGPL-3.0-only + +//! Bulk-creates `VOTER_ADDED` voters via direct writes to Keycloak's own +//! Postgres tables (`user_entity`/`user_attribute`/`user_group_membership`), +//! bypassing the Admin REST API entirely — modeled on the existing +//! `services::import::import_users::import_users_file` bulk-import path. +//! A reconciliation round against a realm that's badly out of sync with a +//! 100k+-row file is dominated by `VOTER_ADDED` items; going through +//! Keycloak's REST API one voter at a time at that volume is impractical +//! (confirmed empirically — even after fixing a Keycloak-side connection +//! leak, per-voter `create_user` calls remained far too slow to finish in +//! practice), while this writes many voters per statement. +//! +//! Every other reconciliation category (edits to existing voters) still +//! goes through the Admin API in `apply.rs`, unchanged — this file only +//! ever creates brand-new users, never touches an existing one. +//! +//! Trade-off, worth being explicit about: this bypasses whatever validation +//! Keycloak's own API enforces beyond what we replicate here (the username +//! length rule we've directly observed Keycloak reject — 3 to 40 +//! characters) and Keycloak's event system entirely, so nothing that +//! listens for Keycloak admin events fires for these voters. + +use crate::postgres::keycloak_realm::get_realm_id; +use crate::services::external::reconciliation::diff::DiffItem; +use anyhow::{anyhow, Context, Result}; +use deadpool_postgres::Transaction; +use sequent_core::types::keycloak::{ + AREA_ID_ATTR_NAME, DATE_OF_BIRTH, DISABLE_COMMENT, TENANT_ID_ATTR_NAME, VOTED_CHANNEL, +}; +use std::collections::{HashMap, HashSet}; +use tracing::{instrument, warn}; + +/// Voters are inserted this many at a time — bounds the blast radius of a +/// single bad batch (a constraint violation the database itself can't route +/// around, unlike `ON CONFLICT DO NOTHING` for duplicate usernames) to one +/// batch instead of the whole run, while still writing many voters per +/// round trip instead of one. +const BULK_INSERT_BATCH_SIZE: usize = 2_000; +const KEYCLOAK_USERNAME_MIN_LEN: usize = 3; +const KEYCLOAK_USERNAME_MAX_LEN: usize = 40; + +/// The result of looking an area name up among an election event's areas — +/// area names are expected to be unique, but aren't enforced as such at the +/// database level, so a lookup must be able to report "more than one +/// matched" as distinct from "none matched" rather than silently picking a +/// winner. +enum AreaLookup { + Found(String), + Ambiguous, +} + +/// One `VOTER_ADDED` voter's fields extracted from its `DiffItem`s, area +/// name not yet resolved to an id (batched separately across every pending +/// voter, see `apply_voters_added_bulk`). +struct PendingVoterAdd { + voter_username: String, + enabled: bool, + date_of_birth: Option, + voted_channel: Option, + disable_comment: Option, + area_name: Option, +} + +/// Mirrors `apply::keycloak_edit_from_items`' extraction for the one shape +/// `VOTER_ADDED` items always come in (`AreaName`, two `KeycloakUA` items +/// for `dateOfBirth`/`voted-channel`, `Enabled`) — see `diff::voter_added_to_sequent`. +fn extract_pending_voter_add(voter_username: &str, items: &[DiffItem]) -> PendingVoterAdd { + let mut enabled = true; + let mut date_of_birth = None; + let mut voted_channel = None; + let mut disable_comment = None; + let mut area_name = None; + + for item in items { + let Some(field) = item.target.sequent_field() else { + continue; + }; + if let Some(new_enabled) = field.new_enabled() { + enabled = new_enabled; + } + if let Some(name) = field.new_area_name() { + area_name = Some(name.to_string()); + } + if let Some(attributes) = field.new_keycloak_attributes() { + if let Some(dob) = attributes.get(DATE_OF_BIRTH) { + date_of_birth = Some(dob.clone()); + } + if let Some(channel) = attributes.get(VOTED_CHANNEL) { + voted_channel = Some(channel.clone()); + } + if let Some(comment) = attributes.get(DISABLE_COMMENT) { + disable_comment = Some(comment.clone()); + } + } + } + + PendingVoterAdd { + voter_username: voter_username.to_string(), + enabled, + date_of_birth, + voted_channel, + disable_comment, + area_name, + } +} + +/// Resolves every distinct area name in one query instead of one per voter — +/// mirrors `postgres::area::get_area_id_from_event_by_name`'s own +/// found/not-found/ambiguous cases (a duplicate area name is a data problem +/// in the election event, not something to silently pick a winner for). +#[instrument(skip(hasura_transaction, area_names), err)] +async fn resolve_area_ids_bulk( + hasura_transaction: &Transaction<'_>, + tenant_id: &str, + election_event_id: &str, + area_names: &[String], +) -> Result> { + if area_names.is_empty() { + return Ok(HashMap::new()); + } + + let statement = hasura_transaction + .prepare( + r#" + SELECT + name, + array_agg(id) AS ids + FROM + sequent_backend.area + WHERE + tenant_id = $1 AND + election_event_id = $2 AND + name = ANY($3) + GROUP BY name + "#, + ) + .await + .context("Error preparing bulk area lookup")?; + + let tenant_uuid = sequent_core::services::uuid_validation::parse_uuid_v4(tenant_id)?; + let event_uuid = sequent_core::services::uuid_validation::parse_uuid_v4(election_event_id)?; + let rows = hasura_transaction + .query(&statement, &[&tenant_uuid, &event_uuid, &area_names]) + .await + .context("Error running bulk area lookup")?; + + let mut result = HashMap::new(); + for row in rows { + let name: String = row.try_get("name")?; + let ids: Vec = row.try_get("ids")?; + let lookup = match ids.len() { + 1 => AreaLookup::Found(ids[0].to_string()), + _ => AreaLookup::Ambiguous, + }; + result.insert(name, lookup); + } + Ok(result) +} + +/// Bulk-inserts every `VOTER_ADDED` voter in `voters` directly into +/// Keycloak's tables. Returns the same `(applied_items, row_failures)` shape +/// `run_apply_reconciliation_patch` already collects from the sequential +/// Admin-API path, so the two paths merge into one report. +#[instrument(skip_all, fields(voter_count = voters.len()), err)] +pub async fn apply_voters_added_bulk( + hasura_transaction: &Transaction<'_>, + keycloak_transaction: &Transaction<'_>, + tenant_id: &str, + election_event_id: &str, + realm: &str, + voter_group_name: &str, + voters: &HashMap>, +) -> Result<(Vec, Vec<(String, String)>)> { + let mut applied_items = Vec::new(); + let mut row_failures: Vec<(String, String)> = Vec::new(); + + if voters.is_empty() { + return Ok((applied_items, row_failures)); + } + + let mut pending = Vec::with_capacity(voters.len()); + for (voter_username, items) in voters { + let candidate = extract_pending_voter_add(voter_username, items); + if candidate.voter_username.len() < KEYCLOAK_USERNAME_MIN_LEN + || candidate.voter_username.len() > KEYCLOAK_USERNAME_MAX_LEN + { + row_failures.push(( + voter_username.clone(), + format!( + "VoterID length {} is outside Keycloak's allowed username length ({}-{})", + candidate.voter_username.len(), + KEYCLOAK_USERNAME_MIN_LEN, + KEYCLOAK_USERNAME_MAX_LEN + ), + )); + continue; + } + pending.push(candidate); + } + + let area_names: Vec = pending + .iter() + .filter_map(|candidate| candidate.area_name.clone()) + .collect::>() + .into_iter() + .collect(); + let areas_by_name = resolve_area_ids_bulk( + hasura_transaction, + tenant_id, + election_event_id, + &area_names, + ) + .await?; + + let mut ready: Vec<(PendingVoterAdd, Option)> = Vec::with_capacity(pending.len()); + for candidate in pending { + let area_id = match &candidate.area_name { + None => None, + Some(name) => match areas_by_name.get(name) { + Some(AreaLookup::Found(id)) => Some(id.clone()), + Some(AreaLookup::Ambiguous) => { + row_failures.push(( + candidate.voter_username.clone(), + format!( + "Found multiple areas with name '{name}' in election event '{election_event_id}'" + ), + )); + continue; + } + None => { + row_failures.push(( + candidate.voter_username.clone(), + format!( + "No area found with name '{name}' in election event '{election_event_id}'" + ), + )); + continue; + } + }, + }; + ready.push((candidate, area_id)); + } + + if ready.is_empty() { + return Ok((applied_items, row_failures)); + } + + let realm_id = get_realm_id(keycloak_transaction, realm.to_string()) + .await + .context("Error resolving realm id for bulk voter creation")?; + let group_id = get_group_id(keycloak_transaction, &realm_id, voter_group_name).await?; + + for batch in ready.chunks(BULK_INSERT_BATCH_SIZE) { + match insert_voter_batch(keycloak_transaction, tenant_id, &realm_id, &group_id, batch).await + { + Ok(inserted_usernames) => { + for (candidate, _area_id) in batch { + if inserted_usernames.contains(&candidate.voter_username) { + if let Some(items) = voters.get(&candidate.voter_username) { + applied_items.extend(items.clone()); + } + } else { + row_failures.push(( + candidate.voter_username.clone(), + "Voter already existed in Keycloak (skipped by ON CONFLICT)" + .to_string(), + )); + } + } + } + Err(err) => { + warn!("Bulk voter-add batch of {} failed: {err:?}", batch.len()); + for (candidate, _area_id) in batch { + row_failures.push(( + candidate.voter_username.clone(), + format!("Bulk insert batch failed: {err:?}"), + )); + } + } + } + } + + Ok((applied_items, row_failures)) +} + +#[instrument(skip(keycloak_transaction), err)] +async fn get_group_id( + keycloak_transaction: &Transaction<'_>, + realm_id: &str, + group_name: &str, +) -> Result { + let statement = keycloak_transaction + .prepare( + r#" + SELECT id FROM keycloak_group WHERE realm_id = $1 AND name = $2 + "#, + ) + .await + .context("Error preparing keycloak_group lookup")?; + let row = keycloak_transaction + .query_opt(&statement, &[&realm_id, &group_name]) + .await + .context("Error running keycloak_group lookup")? + .ok_or_else(|| anyhow!("Keycloak group '{group_name}' not found in realm '{realm_id}'"))?; + row.try_get("id").context("Error reading group id") +} + +/// Inserts one batch of ready-to-write voters in three statements (users, +/// attributes, group membership) instead of one round trip per voter. +/// Duplicate usernames are skipped via `ON CONFLICT DO NOTHING` rather than +/// aborting the batch — the caller reports any skipped username as a row +/// failure. Returns the usernames that were actually inserted. +#[instrument(skip(keycloak_transaction, batch), fields(batch_size = batch.len()), err)] +async fn insert_voter_batch( + keycloak_transaction: &Transaction<'_>, + tenant_id: &str, + realm_id: &str, + group_id: &str, + batch: &[(PendingVoterAdd, Option)], +) -> Result> { + let usernames: Vec = batch + .iter() + .map(|(candidate, _)| candidate.voter_username.clone()) + .collect(); + let enabled_flags: Vec = batch + .iter() + .map(|(candidate, _)| candidate.enabled) + .collect(); + + let mut attr_usernames: Vec = Vec::new(); + let mut attr_names: Vec = Vec::new(); + let mut attr_values: Vec = Vec::new(); + for (candidate, area_id) in batch { + let mut push_attr = |name: &str, value: String| { + attr_usernames.push(candidate.voter_username.clone()); + attr_names.push(name.to_string()); + attr_values.push(value); + }; + push_attr(TENANT_ID_ATTR_NAME, tenant_id.to_string()); + if let Some(dob) = &candidate.date_of_birth { + push_attr(DATE_OF_BIRTH, dob.clone()); + } + if let Some(channel) = &candidate.voted_channel { + push_attr(VOTED_CHANNEL, channel.clone()); + } + if let Some(comment) = &candidate.disable_comment { + push_attr(DISABLE_COMMENT, comment.clone()); + } + if let Some(area_id) = area_id { + push_attr(AREA_ID_ATTR_NAME, area_id.clone()); + } + } + + let statement = keycloak_transaction + .prepare( + r#" + WITH input_users AS ( + SELECT * FROM UNNEST($2::text[], $3::boolean[]) AS u(username, enabled) + ), + new_users AS ( + INSERT INTO user_entity ( + id, realm_id, username, enabled, email_verified, + created_timestamp, not_before + ) + SELECT + gen_random_uuid()::text, $1, iu.username, iu.enabled, true, + (extract(epoch from now()) * 1000)::bigint, 0 + FROM input_users iu + ON CONFLICT (realm_id, username) DO NOTHING + RETURNING id, username + ), + input_attrs AS ( + SELECT * FROM UNNEST($4::text[], $5::text[], $6::text[]) AS a(username, name, value) + ), + attrs_inserted AS ( + INSERT INTO user_attribute (id, user_id, name, value) + SELECT gen_random_uuid()::text, nu.id, ia.name, ia.value + FROM input_attrs ia + JOIN new_users nu ON nu.username = ia.username + RETURNING 1 + ), + groups_inserted AS ( + INSERT INTO user_group_membership (group_id, user_id, membership_type) + SELECT $7, nu.id, 'UNMANAGED' + FROM new_users nu + RETURNING 1 + ) + SELECT username FROM new_users + "#, + ) + .await + .context("Error preparing bulk voter insert")?; + + let rows = keycloak_transaction + .query( + &statement, + &[ + &realm_id, + &usernames, + &enabled_flags, + &attr_usernames, + &attr_names, + &attr_values, + &group_id, + ], + ) + .await + .context("Error executing bulk voter insert")?; + + rows.into_iter() + .map(|row| { + row.try_get::<_, String>("username") + .map_err(anyhow::Error::from) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::services::external::types::{ + ReconciliationChangeCategory, ReconciliationPatchTarget, SequentReconciliationField, + }; + use sequent_core::types::keycloak::{ATTR_RESET_VALUE, DISABLE_REASON_MARKVOTED_CALL}; + + fn item(field: SequentReconciliationField) -> DiffItem { + DiffItem { + voter_username: "voter-1".to_string(), + // Real VOTER_ADDED items never carry one; the voter doesn't exist yet. + voter_id: None, + target: ReconciliationPatchTarget::Sequent(Some(field)), + category: ReconciliationChangeCategory::VOTER_ADDED, + failure_reason: None, + } + } + + #[test] + fn extraction_honours_disabled_channel_and_reason() { + let candidate = extract_pending_voter_add( + "voter-1", + &[ + item(SequentReconciliationField::Enabled(false, false)), + item(SequentReconciliationField::KeycloakUA( + HashMap::from([(VOTED_CHANNEL.to_string(), ATTR_RESET_VALUE.to_string())]), + HashMap::from([(VOTED_CHANNEL.to_string(), "PAPER".to_string())]), + )), + item(SequentReconciliationField::KeycloakUA( + HashMap::from([(DISABLE_COMMENT.to_string(), ATTR_RESET_VALUE.to_string())]), + HashMap::from([( + DISABLE_COMMENT.to_string(), + DISABLE_REASON_MARKVOTED_CALL.to_string(), + )]), + )), + ], + ); + + assert!(!candidate.enabled); + assert_eq!(candidate.voted_channel.as_deref(), Some("PAPER")); + assert_eq!( + candidate.disable_comment.as_deref(), + Some(DISABLE_REASON_MARKVOTED_CALL) + ); + } +} diff --git a/packages/windmill/src/services/external/reconciliation/csv.rs b/packages/windmill/src/services/external/reconciliation/csv.rs new file mode 100644 index 00000000000..d317f545ee8 --- /dev/null +++ b/packages/windmill/src/services/external/reconciliation/csv.rs @@ -0,0 +1,356 @@ +// SPDX-FileCopyrightText: 2026 Sequent Tech Inc +// +// SPDX-License-Identifier: AGPL-3.0-only + +//! Parses the reconciliation file format, a `#META` comment line, a CSV header, +//! then one row per voter. Modeled on +//! `services::tally_sheet_import::csv::parse_canonical_csv` — per-row errors +//! are collected, not fatal, since a handful of malformed rows shouldn't sink +//! a 100k-row file; the caller decides whether accumulated errors should +//! still block the whole import. + +use crate::services::external::datafix_types::ParsedDatafixReconciliationRow; +use crate::services::external::types::ReconciliationFileMeta; +use ::csv::{ReaderBuilder, StringRecord}; +use sequent_core::types::keycloak::ATTR_RESET_VALUE; +use std::collections::HashSet; +use std::io::{BufRead, BufReader, Read}; +use std::path::Path; +use tracing::instrument; + +const RECONCILIATION_HEADERS: [&str; 8] = [ + "CountyMun", + "VoterID", + "DoB", + "Ward", + "Poll", + "SchoolSupportCode", + "Channel", + "Deleted", +]; + +/// A row that failed to parse, kept by (1-indexed, header-exclusive) line +/// number so the operator can find it in the original file. Line zero +/// denotes an invalid or missing CSV header. +#[derive(Debug, Clone)] +pub struct RowParseError { + pub line: usize, + pub message: String, +} + +/// Parses the mandatory `#META,Sequence=N,GeneratedAt=T` line. Sequence zero +/// is a legitimate kickoff value, so malformed or missing metadata must be +/// rejected rather than silently mapped to zero. +#[instrument(skip_all)] +pub fn parse_meta_line(line: &str) -> Result { + let line = line.trim_end_matches('\r'); + let fields = line + .strip_prefix("#META,") + .ok_or_else(|| "the first line must start with '#META,'".to_string())?; + let mut sequence = None; + let mut generated_at = None; + let mut seen = HashSet::new(); + for field in fields.split(',') { + let (key, value) = field + .split_once('=') + .ok_or_else(|| format!("invalid metadata field '{field}'"))?; + if !seen.insert(key) { + return Err(format!("duplicate metadata field '{key}'")); + } + match key { + "Sequence" => { + let parsed = value + .parse::() + .map_err(|_| format!("Sequence '{value}' is not an integer"))?; + if parsed < 0 { + return Err("Sequence cannot be negative".to_string()); + } + sequence = Some(parsed); + } + "GeneratedAt" => { + let parsed = value + .parse::() + .map_err(|_| format!("GeneratedAt '{value}' is not an integer"))?; + if parsed < 0 { + return Err("GeneratedAt cannot be negative".to_string()); + } + generated_at = Some(parsed); + } + _ => return Err(format!("unknown metadata field '{key}'")), + } + } + Ok(ReconciliationFileMeta { + sequence: sequence.ok_or_else(|| "metadata is missing Sequence".to_string())?, + generated_at: generated_at.ok_or_else(|| "metadata is missing GeneratedAt".to_string())?, + }) +} + +/// Splits the raw file bytes into the `#META` line and the remaining CSV +/// (header + rows), so callers don't need to know the file starts with a +/// non-CSV comment line before handing the rest to a `csv::Reader`. +#[instrument(skip(bytes))] +pub fn split_meta_and_csv(bytes: &[u8]) -> Result<(ReconciliationFileMeta, &[u8]), String> { + let text_len = bytes.len(); + let newline_pos = bytes.iter().position(|&byte| byte == b'\n'); + let (meta_line_bytes, rest) = match newline_pos { + Some(pos) => (&bytes[..pos], &bytes[(pos + 1).min(text_len)..]), + None => return Err("reconciliation file has no CSV body after #META".to_string()), + }; + let meta_line = std::str::from_utf8(meta_line_bytes) + .map_err(|_| "#META line is not valid UTF-8".to_string())?; + Ok((parse_meta_line(meta_line)?, rest)) +} + +/// Datafix represents unset optional area components both as an empty CSV +/// cell and as `NONE`. Normalize the former at the input boundary so every +/// downstream comparison and generated patch uses the canonical sentinel. +fn normalize_optional_area_fields(row: &mut ParsedDatafixReconciliationRow) { + for value in [&mut row.poll, &mut row.school_support_code] { + if value.is_empty() { + *value = ATTR_RESET_VALUE.to_string(); + } + } +} + +fn validate_row(row: &ParsedDatafixReconciliationRow) -> Result<(), String> { + let values = [ + ("CountyMun", row.county_mun.as_str()), + ("VoterID", row.external_voter_id.as_str()), + ("DoB", row.dob.as_str()), + ("Ward", row.ward.as_str()), + ("Poll", row.poll.as_str()), + ("SchoolSupportCode", row.school_support_code.as_str()), + ("Channel", row.channel.as_str()), + ("Deleted", row.deleted.as_str()), + ]; + for (name, value) in values { + if value.is_empty() { + return Err(format!("{name} cannot be empty; use NONE when unset")); + } + if value.trim() != value { + return Err(format!("{name} cannot have leading or trailing whitespace")); + } + } + if row.channel != row.channel.to_uppercase() { + return Err("Channel must be uppercase".to_string()); + } + if !matches!(row.deleted.as_str(), "true" | "false") { + return Err("Deleted must be exactly 'true' or 'false'".to_string()); + } + Ok(()) +} + +fn validate_headers(headers: &StringRecord) -> Result<(), String> { + if headers.iter().eq(RECONCILIATION_HEADERS) { + Ok(()) + } else { + Err(format!( + "CSV header must be exactly: {}", + RECONCILIATION_HEADERS.join(",") + )) + } +} + +/// Incrementally reads reconciliation rows in fixed-size batches, the +/// production parser, so a 100k+-row file's rows are never all resident in +/// memory at once, only whichever batch is currently being processed. Stops +/// at the first malformed row +/// rather than collecting every error across the whole file: nothing this +/// pipeline does is applied to voter data until a later, separate step, so +/// discovering a bad row late just wastes the processing done so far, it +/// doesn't risk anything being half-applied. +pub struct ReconciliationRowBatches { + reader: ::csv::Reader, + next_line: usize, + headers_validated: bool, +} + +impl ReconciliationRowBatches { + /// `body` must already be positioned past the `#META` line, at the CSV + /// header. + pub fn new(body: R) -> Self { + Self { + reader: ReaderBuilder::new().has_headers(true).from_reader(body), + next_line: 1, + headers_validated: false, + } + } + + /// Reads up to `batch_size` rows. An empty result (with no error) means + /// the file is exhausted. + pub fn next_batch( + &mut self, + batch_size: usize, + ) -> std::result::Result, RowParseError> { + if !self.headers_validated { + let headers = self.reader.headers().map_err(|err| RowParseError { + line: 0, + message: err.to_string(), + })?; + validate_headers(headers).map_err(|message| RowParseError { line: 0, message })?; + self.headers_validated = true; + } + let mut rows = Vec::with_capacity(batch_size); + let mut records = self.reader.deserialize::(); + for _ in 0..batch_size { + match records.next() { + Some(Ok(mut row)) => { + normalize_optional_area_fields(&mut row); + match validate_row(&row) { + Ok(()) => { + rows.push(row); + self.next_line += 1; + } + Err(message) => { + return Err(RowParseError { + line: self.next_line, + message, + }); + } + } + } + Some(Err(err)) => { + return Err(RowParseError { + line: self.next_line, + message: err.to_string(), + }); + } + None => break, + } + } + Ok(rows) + } +} + +impl ReconciliationRowBatches> { + /// Opens `path` and skips exactly one line — the `#META` line, already + /// parsed and validated separately by the caller (via `parse_meta_line` + /// against the same file's bytes) before this is ever called — to + /// position the reader at the CSV header. + pub fn open(path: &Path) -> std::io::Result { + let file = std::fs::File::open(path)?; + let mut reader = BufReader::new(file); + let mut discarded_meta_line = String::new(); + reader.read_line(&mut discarded_meta_line)?; + Ok(Self::new(reader)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn parse_all_streamed( + csv_bytes: &[u8], + ) -> std::result::Result, RowParseError> { + let mut reader = ReconciliationRowBatches::new(csv_bytes); + let mut rows = Vec::new(); + loop { + let batch = reader.next_batch(2)?; + if batch.is_empty() { + return Ok(rows); + } + rows.extend(batch); + } + } + + #[test] + fn parses_sequence_and_generated_at() { + let meta = parse_meta_line("#META,Sequence=42,GeneratedAt=1781780700").unwrap(); + assert_eq!(meta.sequence, 42); + assert_eq!(meta.generated_at, 1781780700); + } + + #[test] + fn rejects_missing_meta_line_and_fields() { + assert!(parse_meta_line("CountyMun,VoterID").is_err()); + assert!(parse_meta_line("#META,GeneratedAt=5").is_err()); + assert!(parse_meta_line("#META,Sequence=0").is_err()); + } + + #[test] + fn rejects_unparseable_or_negative_metadata() { + assert!(parse_meta_line("#META,Sequence=not-a-number,GeneratedAt=5").is_err()); + assert!(parse_meta_line("#META,Sequence=-1,GeneratedAt=5").is_err()); + assert!(parse_meta_line("#META,Sequence=1,GeneratedAt=tomorrow").is_err()); + } + + #[test] + fn splits_meta_line_from_csv_body() { + let file = b"#META,Sequence=1,GeneratedAt=2\nCountyMun,VoterID,DoB,Ward,Poll,SchoolSupportCode,Channel,Deleted\n0014,17695,1963-05-23,04,000,P,NONE,false\n"; + let (meta, csv_bytes) = split_meta_and_csv(file).unwrap(); + assert_eq!(meta.sequence, 1); + let rows = parse_all_streamed(csv_bytes).unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].external_voter_id, "17695"); + assert_eq!(rows[0].channel, "NONE"); + assert_eq!(rows[0].deleted, "false"); + } + + #[test] + fn rejects_a_plain_csv_without_meta_instead_of_swallowing_its_header() { + let file = b"CountyMun,VoterID,DoB,Ward,Poll,SchoolSupportCode,Channel,Deleted\n0014,17695,1963-05-23,04,000,P,NONE,false\n"; + let error = split_meta_and_csv(file).unwrap_err(); + assert!(error.contains("#META")); + } + + #[test] + fn rejects_a_missing_or_reordered_header_even_when_there_are_no_rows() { + for csv_bytes in [ + b"".as_slice(), + b"VoterID,CountyMun,DoB,Ward,Poll,SchoolSupportCode,Channel,Deleted\n".as_slice(), + ] { + let error = parse_all_streamed(csv_bytes).unwrap_err(); + assert_eq!(error.line, 0); + } + } + + #[test] + fn rejects_non_uppercase_channels_and_noncanonical_booleans() { + for (row, expected) in [ + ( + "0014,17695,1963-05-23,04,000,P,Internet,false", + "Channel must be uppercase", + ), + ( + "0014,17696,1963-05-23,04,000,P,NONE,FALSE", + "Deleted must be exactly", + ), + ] { + let csv = format!( + "CountyMun,VoterID,DoB,Ward,Poll,SchoolSupportCode,Channel,Deleted\n{row}\n" + ); + let error = parse_all_streamed(csv.as_bytes()).unwrap_err(); + assert!(error.message.contains(expected)); + } + } + + #[test] + fn normalizes_empty_optional_area_fields_to_none() { + let csv_bytes = b"CountyMun,VoterID,DoB,Ward,Poll,SchoolSupportCode,Channel,Deleted\n0014,17695,1963-05-23,04,,P,NONE,false\n0014,17696,1963-05-23,04,000,,NONE,false\n0014,17697,1963-05-23,04,,,NONE,false\n"; + let rows = parse_all_streamed(csv_bytes).unwrap(); + + assert_eq!(rows[0].poll, ATTR_RESET_VALUE); + assert_eq!(rows[0].school_support_code, "P"); + assert_eq!(rows[1].poll, "000"); + assert_eq!(rows[1].school_support_code, ATTR_RESET_VALUE); + assert_eq!(rows[2].poll, ATTR_RESET_VALUE); + assert_eq!(rows[2].school_support_code, ATTR_RESET_VALUE); + } + + #[test] + fn still_rejects_empty_required_fields() { + let csv_bytes = b"CountyMun,VoterID,DoB,Ward,Poll,SchoolSupportCode,Channel,Deleted\n0014,,1963-05-23,04,000,P,NONE,false\n"; + let error = parse_all_streamed(csv_bytes).unwrap_err(); + assert!(error.message.contains("VoterID cannot be empty")); + } + + #[test] + fn streaming_reader_stops_at_the_first_malformed_row() { + let csv_bytes = b"CountyMun,VoterID,DoB,Ward,Poll,SchoolSupportCode,Channel,Deleted\n0014,17695,1963-05-23,04,000,P,NONE,false\n0014,79535,1948-04-06\n0014,68684,1978-02-28,03,000,P,NONE,false\n"; + let mut reader = ReconciliationRowBatches::new(csv_bytes.as_slice()); + assert_eq!(reader.next_batch(1).unwrap().len(), 1); + let error = reader.next_batch(1).unwrap_err(); + assert_eq!(error.line, 2); + } +} diff --git a/packages/windmill/src/services/external/reconciliation/diff.rs b/packages/windmill/src/services/external/reconciliation/diff.rs new file mode 100644 index 00000000000..aad54dc8b7a --- /dev/null +++ b/packages/windmill/src/services/external/reconciliation/diff.rs @@ -0,0 +1,1350 @@ +// SPDX-FileCopyrightText: 2026 Sequent Tech Inc +// +// SPDX-License-Identifier: AGPL-3.0-only + +use crate::services::external::datafix_types::{ + channels_equal, file_channel_to_keycloak, keycloak_channel_to_file, DatafixReconciliationField, + ParsedDatafixReconciliationRow, FILE_CHANNEL_INTERNET, +}; +use crate::services::external::types::{ + ReconciliationChangeCategory, ReconciliationPatchSource, ReconciliationPatchTarget, + SequentReconciliationField, +}; +use crate::services::users::VoterSnapshot; +use sequent_core::types::keycloak::{ + ATTR_RESET_VALUE, DATE_OF_BIRTH, DISABLE_COMMENT, DISABLE_REASON_DELETE_CALL, + DISABLE_REASON_MARKVOTED_CALL, VOTED_CHANNEL, +}; +use serde::{Deserialize, Serialize}; +use std::collections::{HashMap, HashSet}; +use tracing::{info, instrument}; +use uuid::Uuid; + +/// One (voter, field) change destined for either the Datafix patch or a +/// direct Sequent apply. Never persisted on its own — always as part of a +/// serialized into one of the two documents `generate_reconciliation_patches` +/// uploads: the full `ReconciliationDiff` envelope (both sides, for review) +/// and the Sequent-side NDJSON apply stream. Each is written once and +/// never mutated afterward, so unlike an earlier draft of this type there's +/// no `apply_status` field here — the outcome of applying lives in the +/// row-failures document and the electoral log artifact instead, not back on +/// the item that was applied. The field this item changes, and its own +/// `(old, new)` pair, both live on `target` itself +/// (`ReconciliationPatchTarget::Datafix`/`Sequent`) — not as separate +/// `old_value`/`new_value` members here, since a field is never meaningful +/// apart from its own old/new values and which field enum applies depends on +/// which side `target` is. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct DiffItem { + pub voter_username: String, + /// Keycloak's own internal user id (`VoterSnapshot::voter_id`), captured + /// here so `apply.rs` never has to resolve it again per voter. `None` + /// where no `VoterSnapshot` was ever looked up for this item: the + /// row-level `CountyMun` mismatch failure, and `VOTER_ADDED` items — the + /// voter doesn't exist in Sequent yet. + #[serde(default)] + pub voter_id: Option, + #[serde(flatten)] + pub target: ReconciliationPatchTarget, + pub category: ReconciliationChangeCategory, + pub failure_reason: Option, +} + +/// The full content of the "diff envelope" document `generate_reconciliation_patches` +/// uploads once per round, referenced from `task_execution.annotations.document_id`. +/// There is no `external_reconciliation_import` table or row of any kind — this +/// document *is* the record. The frontend fetches and parses the whole thing +/// to render both diff tables. Server-side readers — the apply task and the +/// Harvest apply route — deserialize its [`ReconciliationApplyEnvelope`] +/// projection (never trusting client-supplied fields) to find the Sequent +/// patch document and re-validate that the external side is clean. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ReconciliationDiff { + pub sequence: i64, + pub generated_at: i64, + pub source_sha256: String, + /// `None` when the external-side diff was empty. Apply is only allowed + /// once this is `None` — a non-empty external side means that system's + /// own process hasn't converged yet. + pub external_patch_document_id: Option, + pub external_patch_sha256: Option, + /// Every `target = Sequent` item, serialized as an NDJSON stream so apply + /// can process one voter at a time. It includes `ROW_FAILURE` items for + /// structured reporting; those entries are never treated as mutations. + pub sequent_patch_document_id: String, + /// False for an already-successful Sequence: the envelope is a diff-only + /// convergence check and must not be applied again. True for a new round + /// or a same-Sequence retry whose previous apply had row failures. + #[serde(default = "default_apply_allowed")] + pub apply_allowed: bool, + /// Every item, both sides, including `ROW_FAILURE`s — what the review UI + /// renders. + pub items: Vec, +} + +/// Immutable apply metadata projected from a [`ReconciliationDiff`]. The +/// review-only `items` array is deliberately absent, so synchronous callers +/// and the apply task can deserialize a large envelope without retaining the +/// complete diff in memory. +#[derive(Debug, Clone, Deserialize)] +pub struct ReconciliationApplyEnvelope { + pub sequence: i64, + pub generated_at: i64, + pub source_sha256: String, + pub external_patch_document_id: Option, + pub sequent_patch_document_id: String, + #[serde(default = "default_apply_allowed")] + pub apply_allowed: bool, +} + +fn default_apply_allowed() -> bool { + true +} + +/// Original Datafix area columns indexed by Sequent's composed area name. +/// The composed name is not generally reversible (components may contain +/// hyphens and optional components are omitted), so the reverse voter pass +/// learns this mapping from rows in the same reconciliation file. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DatafixAreaFields { + pub ward: String, + pub poll: String, + pub school_support_code: String, +} + +pub type DatafixAreaFieldsByName = HashMap>; + +pub fn index_datafix_area_fields( + index: &mut DatafixAreaFieldsByName, + rows: &[ParsedDatafixReconciliationRow], +) { + for row in rows { + let name = composed_area_name(row); + let fields = DatafixAreaFields { + ward: row.ward.clone(), + poll: row.poll.clone(), + school_support_code: row.school_support_code.clone(), + }; + index + .entry(name) + .and_modify(|current| { + if current.as_ref() != Some(&fields) { + *current = None; + } + }) + .or_insert(Some(fields)); + } +} + +/// Runs the forward pass for one batch of file rows (see +/// `services::external::reconciliation::csv::ReconciliationRowBatches`): +/// classifies each row against its Sequent snapshot, if this batch's +/// `fetch_realm_voter_snapshots_by_usernames` call found one — +/// `snapshots_by_username` holds only this batch's matches, not the whole +/// realm, so a missing entry means this file row's voter doesn't exist in +/// Sequent at all (D, forward direction, handled inside `classify_file_row` +/// same as before). `source` carries whatever config the file's own origin +/// needs for classification (for Datafix, its `CountyMun`, from the event's +/// own `VoterviewRequest::county_mun`). +#[instrument(skip_all, fields(batch_size = file_rows.len()))] +pub fn diff_file_row_batch( + file_rows: &[ParsedDatafixReconciliationRow], + snapshots_by_username: &HashMap, + source: &ReconciliationPatchSource, +) -> Vec { + file_rows + .iter() + .flat_map(|row| { + classify_file_row( + row, + snapshots_by_username.get(&row.external_voter_id), + source, + ) + }) + .collect() +} + +/// D, reverse direction, run against one page of Sequent's own voter walk +/// (`users::fetch_realm_voter_snapshots_page`) after every batch of the file +/// has been scanned, so `all_file_usernames` (every `VoterID` seen across +/// every batch) is complete: an enabled voter whose username was never seen +/// in the file at all is reported into the Datafix patch via +/// `voter_missing_from_file`. Disabled voters need no report here — if +/// Sequent already considers them gone, there's nothing for Datafix to +/// catch up on regardless of whether the file mentions them. +#[instrument(skip_all, fields(page_size = snapshots_page.len()))] +pub fn diff_unmatched_sequent_voters( + snapshots_page: &[VoterSnapshot], + all_file_usernames: &HashSet, + source: &ReconciliationPatchSource, + area_fields_by_name: &DatafixAreaFieldsByName, +) -> Vec { + snapshots_page + .iter() + .filter(|snapshot| snapshot.enabled && !all_file_usernames.contains(&snapshot.username)) + .flat_map(|snapshot| { + voter_missing_from_file(&snapshot.username, snapshot, source, area_fields_by_name) + }) + .collect() +} + +/// Classifies a single file row against the voter's current Sequent snapshot +/// (`None` if Sequent doesn't have this voter at all — source D, forward +/// direction). Returns zero or more `DiffItem`s: most rows produce 0-2 items, +/// but the "voted via other channel on a voter holding a valid Internet +/// ballot" exception produces **two** — a `ROW_FAILURE` (excluded from +/// apply) *and* a `VOTED_INTERNET`-style correction so the Datafix patch +/// still keeps `Channel=INTERNET` on their side for the rest of the round +/// (spec, "Concrete examples" B) — easy to miss, worth a dedicated test. +#[instrument(skip_all, fields(voter_username = %row.external_voter_id))] +fn classify_file_row( + row: &ParsedDatafixReconciliationRow, + snapshot: Option<&VoterSnapshot>, + source: &ReconciliationPatchSource, +) -> Vec { + let ReconciliationPatchSource::Datafix { county_mun } = source; + let event_county_mun = county_mun.as_str(); + if row.county_mun != event_county_mun && row.deleted != "true" { + return vec![DiffItem { + voter_username: row.external_voter_id.clone(), + // No `VoterSnapshot` lookup has happened yet at this point. + voter_id: None, + // Excluded from both diffs regardless; `CountyMun` has no Sequent + // equivalent field, so there is nothing to name here. + target: ReconciliationPatchTarget::Sequent(None), + category: ReconciliationChangeCategory::ROW_FAILURE, + failure_reason: Some(format!( + "CountyMun ({}) does not match this election event's CountyMun ({event_county_mun}) \ + and the row is not marked Deleted — Datafix processing error.", + row.county_mun, + )), + }]; + } + + let Some(snapshot) = snapshot else { + // A file cannot manufacture an Internet ballot for a voter Sequent + // does not even know. Correct Datafix first; the next clean file will + // add the voter with Channel=NONE. + if channels_equal(&row.channel, FILE_CHANNEL_INTERNET) { + return vec![diff_item( + &row.external_voter_id, + ReconciliationPatchTarget::Datafix(DatafixReconciliationField::Channel( + row.channel.clone(), + ATTR_RESET_VALUE.to_string(), + )), + ReconciliationChangeCategory::VOTED_UNMARKED, + )]; + } + return voter_added_to_sequent(row); + }; + + let mut items = Vec::new(); + let file_says_none = channels_equal(&row.channel, ATTR_RESET_VALUE); + let file_says_internet = channels_equal(&row.channel, FILE_CHANNEL_INTERNET); + let stored_channel = snapshot + .voted_channel + .as_deref() + .unwrap_or(ATTR_RESET_VALUE); + let stored_disable_comment = snapshot + .disable_comment + .as_deref() + .unwrap_or(ATTR_RESET_VALUE); + let unmarking_stored_channel = file_says_none + && !snapshot.has_valid_internet_vote + && !snapshot.has_unresolved_internet_vote + && !channels_equal(stored_channel, ATTR_RESET_VALUE); + let applying_other_channel = !file_says_none + && !file_says_internet + && !snapshot.has_valid_internet_vote + && !snapshot.has_unresolved_internet_vote; + + // A) Sequent holds a valid Internet ballot; Datafix says NONE. + if file_says_none && snapshot.has_valid_internet_vote { + items.push(diff_item( + &row.external_voter_id, + ReconciliationPatchTarget::Datafix(DatafixReconciliationField::Channel( + row.channel.clone(), + FILE_CHANNEL_INTERNET.to_string(), + )), + ReconciliationChangeCategory::VOTED_INTERNET, + )); + } else if file_says_none && snapshot.has_unresolved_internet_vote { + items.push(row_failure( + &row.external_voter_id, + "Voter has an in-progress Internet ballot; Channel=NONE cannot be reconciled until the ballot resolves", + )); + } else if unmarking_stored_channel { + // File-driven equivalent of `/unmark-voted`. Compute the final + // disable reason here, together with the channel reset, so a + // Deleted=true row never emits two competing writes to the same + // attribute. A manual/admin disable is not undone by an unmark: only + // MARKVOTED_CALL owns the enable transition and its comment. + let desired_disable_comment = if row.deleted == "true" { + DISABLE_REASON_DELETE_CALL + } else if !snapshot.enabled && stored_disable_comment != DISABLE_REASON_MARKVOTED_CALL { + stored_disable_comment + } else { + ATTR_RESET_VALUE + }; + let mut old_attributes = + HashMap::from([(VOTED_CHANNEL.to_string(), stored_channel.to_string())]); + let mut new_attributes = + HashMap::from([(VOTED_CHANNEL.to_string(), ATTR_RESET_VALUE.to_string())]); + if desired_disable_comment != stored_disable_comment { + old_attributes.insert( + DISABLE_COMMENT.to_string(), + stored_disable_comment.to_string(), + ); + new_attributes.insert( + DISABLE_COMMENT.to_string(), + desired_disable_comment.to_string(), + ); + } + items.push(diff_item( + &row.external_voter_id, + ReconciliationPatchTarget::Sequent(Some(SequentReconciliationField::KeycloakUA( + old_attributes, + new_attributes, + ))), + ReconciliationChangeCategory::VOTED_UNMARKED, + )); + if !snapshot.enabled + && row.deleted != "true" + && stored_disable_comment == DISABLE_REASON_MARKVOTED_CALL + { + items.push(diff_item( + &row.external_voter_id, + ReconciliationPatchTarget::Sequent(Some(SequentReconciliationField::Enabled( + false, true, + ))), + ReconciliationChangeCategory::VOTED_UNMARKED, + )); + } + } else if file_says_internet && snapshot.has_unresolved_internet_vote { + items.push(row_failure( + &row.external_voter_id, + "Voter has an in-progress Internet ballot; Channel=INTERNET cannot be confirmed until the ballot resolves", + )); + } else if file_says_internet && !snapshot.has_valid_internet_vote { + // The reverse half of source-of-truth A: Datafix cannot claim an + // Internet vote that Sequent has no active record of. + items.push(diff_item( + &row.external_voter_id, + ReconciliationPatchTarget::Datafix(DatafixReconciliationField::Channel( + row.channel.clone(), + ATTR_RESET_VALUE.to_string(), + )), + ReconciliationChangeCategory::VOTED_UNMARKED, + )); + } + + // B) Datafix reports another channel. The same active-ballot guard used + // by the real-time API applies to both valid and in-progress ballots. + if !file_says_none && !file_says_internet { + if snapshot.has_valid_internet_vote { + // Exception: cannot resolve automatically — row failure now, and + // Sequent still wins for the rest of the round (spec, example B). + items.push(DiffItem { + voter_username: row.external_voter_id.clone(), + voter_id: Some(snapshot.voter_id), + target: ReconciliationPatchTarget::Sequent(Some( + SequentReconciliationField::KeycloakUA( + HashMap::from([(VOTED_CHANNEL.to_string(), stored_channel.to_string())]), + HashMap::from([(VOTED_CHANNEL.to_string(), row.channel.clone())]), + ), + )), + category: ReconciliationChangeCategory::ROW_FAILURE, + failure_reason: Some(format!( + "Voter holds a valid Internet ballot; voted-via-other-channel ({}) cannot be \ + resolved automatically — release the voter via edit_user after the freeze \ + ends.", + row.channel, + )), + }); + items.push(diff_item( + &row.external_voter_id, + ReconciliationPatchTarget::Datafix(DatafixReconciliationField::Channel( + row.channel.clone(), + FILE_CHANNEL_INTERNET.to_string(), + )), + ReconciliationChangeCategory::VOTED_INTERNET, + )); + } else if snapshot.has_unresolved_internet_vote { + items.push(row_failure( + &row.external_voter_id, + &format!( + "Voter has an in-progress Internet ballot; voted-via-other-channel ({}) cannot be applied until it resolves", + row.channel + ), + )); + } else { + // Both the voted channel and why the voter is being disabled are + // plain Keycloak attributes — carried together so `apply` writes + // them in a single edit without knowing this came from Datafix. + // Deleted=true takes precedence over MARKVOTED_CALL in the same + // canonical item so the delete block below has nothing to + // overwrite later. + let desired_disable_comment = if row.deleted == "true" { + DISABLE_REASON_DELETE_CALL + } else { + DISABLE_REASON_MARKVOTED_CALL + }; + if !channels_equal(stored_channel, &row.channel) + || stored_disable_comment != desired_disable_comment + { + items.push(diff_item( + &row.external_voter_id, + ReconciliationPatchTarget::Sequent(Some( + SequentReconciliationField::KeycloakUA( + HashMap::from([ + (VOTED_CHANNEL.to_string(), stored_channel.to_string()), + ( + DISABLE_COMMENT.to_string(), + stored_disable_comment.to_string(), + ), + ]), + HashMap::from([ + ( + VOTED_CHANNEL.to_string(), + file_channel_to_keycloak(&row.channel), + ), + ( + DISABLE_COMMENT.to_string(), + desired_disable_comment.to_string(), + ), + ]), + ), + )), + ReconciliationChangeCategory::VOTED_OTHER_CHANNEL, + )); + } + if snapshot.enabled { + items.push(diff_item( + &row.external_voter_id, + ReconciliationPatchTarget::Sequent(Some(SequentReconciliationField::Enabled( + true, false, + ))), + ReconciliationChangeCategory::VOTED_OTHER_CHANNEL, + )); + } + } + } + + // C) Profile fields — Datafix wins. Ward/Poll/SchoolSupportCode are + // compared as a single composed area name (see the module doc in + // snapshot.rs for why they can't be compared field-by-field), surfaced + // here as an AreaName-field change for display/patch purposes. + let file_area_name = composed_area_name(row); + if snapshot.area_name.as_deref() != Some(file_area_name.as_str()) { + items.push(diff_item( + &row.external_voter_id, + ReconciliationPatchTarget::Sequent(Some(SequentReconciliationField::AreaName( + snapshot + .area_name + .clone() + .unwrap_or_else(|| ATTR_RESET_VALUE.to_string()), + file_area_name.clone(), + ))), + ReconciliationChangeCategory::PROFILE_UPDATE, + )); + } + if Some(row.dob.as_str()) != snapshot.dob.as_deref() { + items.push(diff_item( + &row.external_voter_id, + ReconciliationPatchTarget::Sequent(Some(SequentReconciliationField::KeycloakUA( + HashMap::from([( + DATE_OF_BIRTH.to_string(), + snapshot + .dob + .clone() + .unwrap_or_else(|| ATTR_RESET_VALUE.to_string()), + )]), + HashMap::from([(DATE_OF_BIRTH.to_string(), row.dob.clone())]), + ))), + ReconciliationChangeCategory::PROFILE_UPDATE, + )); + } + + // C) Deleted=true — Datafix wins unless an active Internet ballot makes + // the transition unsafe. Profile changes above still reconcile disabled + // voters in the same round. + if row.deleted == "true" { + if snapshot.has_valid_internet_vote { + items.push(diff_item( + &row.external_voter_id, + ReconciliationPatchTarget::Datafix(DatafixReconciliationField::Deleted( + "true".to_string(), + "false".to_string(), + )), + ReconciliationChangeCategory::DELETION_REVERTED, + )); + } else if snapshot.has_unresolved_internet_vote { + items.push(row_failure( + &row.external_voter_id, + "Voter has an in-progress Internet ballot and cannot be deleted until it resolves", + )); + } else { + // An authoritative non-Internet channel already emitted the + // same enabled=false transition; every other Deleted=true path + // gets it here exactly once. + if snapshot.enabled && !applying_other_channel { + items.push(diff_item( + &row.external_voter_id, + ReconciliationPatchTarget::Sequent(Some(SequentReconciliationField::Enabled( + true, false, + ))), + ReconciliationChangeCategory::DISABLED_DELETE_CALL, + )); + } + // Channel transitions in A/B already assigned DELETE_CALL as the + // canonical final reason. Do not emit a duplicate attribute + // write whose winner would otherwise depend on item ordering. + if !unmarking_stored_channel + && !applying_other_channel + && stored_disable_comment != DISABLE_REASON_DELETE_CALL + { + items.push(diff_item( + &row.external_voter_id, + ReconciliationPatchTarget::Sequent(Some( + SequentReconciliationField::KeycloakUA( + HashMap::from([( + DISABLE_COMMENT.to_string(), + stored_disable_comment.to_string(), + )]), + HashMap::from([( + DISABLE_COMMENT.to_string(), + DISABLE_REASON_DELETE_CALL.to_string(), + )]), + ), + )), + ReconciliationChangeCategory::DISABLED_DELETE_CALL, + )); + } + } + } else if !snapshot.enabled + && stored_disable_comment == DISABLE_REASON_DELETE_CALL + && file_says_none + && channels_equal(stored_channel, ATTR_RESET_VALUE) + { + items.push(diff_item( + &row.external_voter_id, + ReconciliationPatchTarget::Sequent(Some(SequentReconciliationField::Enabled( + false, true, + ))), + ReconciliationChangeCategory::REENABLED, + )); + } + + // A row-level safety failure excludes every Sequent mutation for that + // voter in this round. Datafix corrections may remain (for example a + // valid Internet ballot correcting PAPER back to INTERNET), because the + // external patch must still move the source file toward convergence. + let failure_reasons: Vec = items + .iter() + .filter(|item| item.category == ReconciliationChangeCategory::ROW_FAILURE) + .filter_map(|item| item.failure_reason.clone()) + .collect(); + if !failure_reasons.is_empty() { + items.retain(|item| { + item.category != ReconciliationChangeCategory::ROW_FAILURE && item.target.is_datafix() + }); + items.push(row_failure( + &row.external_voter_id, + &failure_reasons.join("; "), + )); + } + + // Every item above was classified against `snapshot`, so they all share + // its voter id — set once here instead of at each push site above. + items + .into_iter() + .map(|item| DiffItem { + voter_id: Some(snapshot.voter_id), + ..item + }) + .collect() +} + +/// D, forward direction: the file has a voter Sequent doesn't. Per "Patch +/// Files Format", every field Sequent actually stores is present with `_old = +/// NONE` since Sequent has no prior record at all — expanded here into one +/// `DiffItem` per `SequentReconciliationField`, all sharing `VOTER_ADDED`/ +/// `target = Sequent`. `CountyMun` has no Sequent equivalent, so unlike the +/// `DatafixReconciliationField::NAMES` set used for the outbound patch CSV, +/// this only covers the fields Sequent itself understands. The bulk apply +/// path consumes the emitted `Enabled` and `disable-comment` values exactly, +/// including file rows already marked deleted or voted via another channel. +#[instrument(skip_all, fields(voter_username = %row.external_voter_id))] +fn voter_added_to_sequent(row: &ParsedDatafixReconciliationRow) -> Vec { + let file_area_name = composed_area_name(row); + let voted_other_channel = !channels_equal(&row.channel, ATTR_RESET_VALUE); + let file_enabled = row.deleted != "true" && !voted_other_channel; + let disable_comment = if row.deleted == "true" { + Some(DISABLE_REASON_DELETE_CALL) + } else if voted_other_channel { + Some(DISABLE_REASON_MARKVOTED_CALL) + } else { + None + }; + let mut items = vec![ + diff_item( + &row.external_voter_id, + ReconciliationPatchTarget::Sequent(Some(SequentReconciliationField::AreaName( + ATTR_RESET_VALUE.to_string(), + file_area_name, + ))), + ReconciliationChangeCategory::VOTER_ADDED, + ), + diff_item( + &row.external_voter_id, + ReconciliationPatchTarget::Sequent(Some(SequentReconciliationField::KeycloakUA( + HashMap::from([(DATE_OF_BIRTH.to_string(), ATTR_RESET_VALUE.to_string())]), + HashMap::from([(DATE_OF_BIRTH.to_string(), row.dob.clone())]), + ))), + ReconciliationChangeCategory::VOTER_ADDED, + ), + diff_item( + &row.external_voter_id, + ReconciliationPatchTarget::Sequent(Some(SequentReconciliationField::KeycloakUA( + HashMap::from([(VOTED_CHANNEL.to_string(), ATTR_RESET_VALUE.to_string())]), + HashMap::from([( + VOTED_CHANNEL.to_string(), + file_channel_to_keycloak(&row.channel), + )]), + ))), + ReconciliationChangeCategory::VOTER_ADDED, + ), + diff_item( + &row.external_voter_id, + ReconciliationPatchTarget::Sequent(Some(SequentReconciliationField::Enabled( + false, + file_enabled, + ))), + ReconciliationChangeCategory::VOTER_ADDED, + ), + ]; + if let Some(disable_comment) = disable_comment { + items.push(diff_item( + &row.external_voter_id, + ReconciliationPatchTarget::Sequent(Some(SequentReconciliationField::KeycloakUA( + HashMap::from([(DISABLE_COMMENT.to_string(), ATTR_RESET_VALUE.to_string())]), + HashMap::from([(DISABLE_COMMENT.to_string(), disable_comment.to_string())]), + ))), + ReconciliationChangeCategory::VOTER_ADDED, + )); + } + items +} + +/// D, reverse direction: Sequent has an enabled voter the file doesn't +/// mention at all — reported into the Datafix patch so their side adds it. +#[instrument(skip_all)] +fn voter_missing_from_file( + username: &str, + snapshot: &VoterSnapshot, + source: &ReconciliationPatchSource, + area_fields_by_name: &DatafixAreaFieldsByName, +) -> Vec { + let ReconciliationPatchSource::Datafix { county_mun } = source; + let Some(area_name) = snapshot.area_name.as_deref() else { + return vec![row_failure( + username, + "Sequent-only voter has no resolvable area; cannot build a complete Datafix add patch", + )]; + }; + let Some(Some(area_fields)) = area_fields_by_name.get(area_name) else { + return vec![row_failure( + username, + &format!( + "Cannot recover Ward/Poll/SchoolSupportCode for Sequent area '{area_name}' from this reconciliation file" + ), + )]; + }; + let fields = [ + DatafixReconciliationField::CountyMun(ATTR_RESET_VALUE.to_string(), county_mun.clone()), + DatafixReconciliationField::Ward(ATTR_RESET_VALUE.to_string(), area_fields.ward.clone()), + DatafixReconciliationField::Poll(ATTR_RESET_VALUE.to_string(), area_fields.poll.clone()), + DatafixReconciliationField::SchoolSupportCode( + ATTR_RESET_VALUE.to_string(), + area_fields.school_support_code.clone(), + ), + DatafixReconciliationField::DoB( + ATTR_RESET_VALUE.to_string(), + snapshot + .dob + .clone() + .unwrap_or_else(|| ATTR_RESET_VALUE.to_string()), + ), + DatafixReconciliationField::Channel( + ATTR_RESET_VALUE.to_string(), + if snapshot.has_valid_internet_vote { + FILE_CHANNEL_INTERNET.to_string() + } else { + snapshot + .voted_channel + .as_deref() + .map(keycloak_channel_to_file) + .unwrap_or_else(|| ATTR_RESET_VALUE.to_string()) + }, + ), + DatafixReconciliationField::Deleted(ATTR_RESET_VALUE.to_string(), "false".to_string()), + ]; + fields + .into_iter() + .map(|field| DiffItem { + voter_id: Some(snapshot.voter_id), + ..diff_item( + username, + ReconciliationPatchTarget::Datafix(field), + ReconciliationChangeCategory::VOTER_ADDED, + ) + }) + .collect() +} + +#[instrument(skip_all)] +fn diff_item( + voter_username: &str, + target: ReconciliationPatchTarget, + category: ReconciliationChangeCategory, +) -> DiffItem { + DiffItem { + voter_username: voter_username.to_string(), + voter_id: None, + target, + category, + failure_reason: None, + } +} + +#[instrument(skip_all)] +fn row_failure(voter_username: &str, reason: &str) -> DiffItem { + DiffItem { + voter_username: voter_username.to_string(), + voter_id: None, + target: ReconciliationPatchTarget::Sequent(None), + category: ReconciliationChangeCategory::ROW_FAILURE, + failure_reason: Some(reason.to_string()), + } +} + +/// Composes a file row's Ward-SchoolSupportCode-Poll into the same format as +/// `Area::name`, reusing `external::utils::compose_area_name` (via a minimal +/// `VoterInformationBody`) so the join/uppercase rule is defined in exactly +/// one place. +#[instrument(skip_all)] +fn composed_area_name(row: &ParsedDatafixReconciliationRow) -> String { + use crate::services::external::datafix_types::VoterInformationBody; + use crate::services::external::utils::compose_area_name; + + // The file's own "no value" sentinel (`ATTR_RESET_VALUE`) must not be + // concatenated into the composed name as a literal segment — translate it + // to `None` first so `compose_area_name`'s own empty-value handling + // (built for the inbound API's `Option` contract) omits it. + let optional_field = |value: &str| (value != ATTR_RESET_VALUE).then(|| value.to_string()); + + let composed_area = compose_area_name(&VoterInformationBody { + voter_id: row.external_voter_id.clone(), + ward: row.ward.clone(), + schoolboard: optional_field(&row.school_support_code), + poll: optional_field(&row.poll), + birthdate: None, + enabled: None, + }); + info!(%composed_area, "Composed area name from file row"); + composed_area +} + +#[cfg(test)] +mod tests { + use super::*; + + fn row(voter_id: &str, channel: &str, deleted: &str) -> ParsedDatafixReconciliationRow { + ParsedDatafixReconciliationRow { + county_mun: "0014".to_string(), + external_voter_id: voter_id.to_string(), + dob: "1990-01-01".to_string(), + ward: "01".to_string(), + poll: "000".to_string(), + school_support_code: "P".to_string(), + channel: channel.to_string(), + deleted: deleted.to_string(), + } + } + + fn datafix_source(county_mun: &str) -> ReconciliationPatchSource { + ReconciliationPatchSource::Datafix { + county_mun: county_mun.to_string(), + } + } + + fn enabled_snapshot() -> VoterSnapshot { + VoterSnapshot { + username: "voter-1".to_string(), + voter_id: Uuid::new_v4(), + enabled: true, + area_name: Some("01-P-000".to_string()), + dob: Some("1990-01-01".to_string()), + voted_channel: None, + has_valid_internet_vote: false, + has_unresolved_internet_vote: false, + disable_comment: None, + } + } + + #[test] + fn county_mun_mismatch_is_a_row_failure() { + let mut bad_row = row("voter-1", ATTR_RESET_VALUE, "false"); + bad_row.county_mun = "0099".to_string(); + let items = classify_file_row(&bad_row, Some(&enabled_snapshot()), &datafix_source("0014")); + assert_eq!(items.len(), 1); + assert_eq!(items[0].category, ReconciliationChangeCategory::ROW_FAILURE); + assert_eq!(items[0].target, ReconciliationPatchTarget::Sequent(None)); + } + + #[test] + fn county_mun_mismatch_is_ignored_when_deleted() { + let mut bad_row = row("voter-1", ATTR_RESET_VALUE, "true"); + bad_row.county_mun = "0099".to_string(); + let items = classify_file_row(&bad_row, Some(&enabled_snapshot()), &datafix_source("0014")); + assert!(items + .iter() + .all(|item| item.category != ReconciliationChangeCategory::ROW_FAILURE)); + } + + #[test] + fn a_voted_internet_ballot_wins_over_none_in_the_file() { + let snapshot = VoterSnapshot { + has_valid_internet_vote: true, + ..enabled_snapshot() + }; + let items = classify_file_row( + &row("voter-1", ATTR_RESET_VALUE, "false"), + Some(&snapshot), + &datafix_source("0014"), + ); + assert_eq!(items.len(), 1); + assert_eq!( + items[0].category, + ReconciliationChangeCategory::VOTED_INTERNET + ); + assert_eq!( + items[0].target, + ReconciliationPatchTarget::Datafix(DatafixReconciliationField::Channel( + ATTR_RESET_VALUE.to_string(), + FILE_CHANNEL_INTERNET.to_string() + )) + ); + } + + #[test] + fn other_channel_wins_when_sequent_has_not_voted() { + let items = classify_file_row( + &row("voter-1", "PAPER", "false"), + Some(&enabled_snapshot()), + &datafix_source("0014"), + ); + assert_eq!(items.len(), 2); + assert!(items + .iter() + .all(|item| item.category == ReconciliationChangeCategory::VOTED_OTHER_CHANNEL)); + let attributes_item = items + .iter() + .find_map(|item| item.target.sequent_field()?.new_keycloak_attributes()) + .expect("one item carries the Keycloak attributes"); + assert_eq!( + attributes_item.get(VOTED_CHANNEL), + Some(&"PAPER".to_string()) + ); + assert_eq!( + attributes_item.get(DISABLE_COMMENT), + Some(&DISABLE_REASON_MARKVOTED_CALL.to_string()) + ); + assert!(items.iter().any(|item| { + item.target + == ReconciliationPatchTarget::Sequent(Some(SequentReconciliationField::Enabled( + true, false, + ))) + })); + } + + #[test] + fn other_channel_wins_and_keeps_the_voters_prior_channel_as_the_old_value() { + // A voter already recorded with some channel (not the one the file now + // reports) must not show "NONE" as the diff's old value — only a + // voter with no prior channel at all should. + let snapshot = VoterSnapshot { + voted_channel: Some("PHONE".to_string()), + ..enabled_snapshot() + }; + let items = classify_file_row( + &row("voter-1", "PAPER", "false"), + Some(&snapshot), + &datafix_source("0014"), + ); + let keycloak_field = items + .iter() + .find_map(|item| match item.target.sequent_field() { + Some(field @ SequentReconciliationField::KeycloakUA(..)) => Some(field), + _ => None, + }) + .expect("one item carries the Keycloak attributes"); + let SequentReconciliationField::KeycloakUA(old, new) = keycloak_field else { + unreachable!("matched above"); + }; + assert_eq!(old.get(VOTED_CHANNEL), Some(&"PHONE".to_string())); + assert_eq!(new.get(VOTED_CHANNEL), Some(&"PAPER".to_string())); + } + + #[test] + fn other_channel_on_a_valid_internet_ballot_produces_a_failure_and_a_correction() { + let snapshot = VoterSnapshot { + has_valid_internet_vote: true, + ..enabled_snapshot() + }; + let items = classify_file_row( + &row("voter-1", "PAPER", "false"), + Some(&snapshot), + &datafix_source("0014"), + ); + assert_eq!(items.len(), 2); + assert!(items + .iter() + .any(|item| item.category == ReconciliationChangeCategory::ROW_FAILURE)); + assert!(items.iter().any(|item| { + item.category == ReconciliationChangeCategory::VOTED_INTERNET + && item.target + == ReconciliationPatchTarget::Datafix(DatafixReconciliationField::Channel( + "PAPER".to_string(), + FILE_CHANNEL_INTERNET.to_string(), + )) + })); + } + + #[test] + fn deleted_true_disables_a_voter_who_has_not_voted() { + let items = classify_file_row( + &row("voter-1", ATTR_RESET_VALUE, "true"), + Some(&enabled_snapshot()), + &datafix_source("0014"), + ); + assert!(items.iter().any(|item| { + item.category == ReconciliationChangeCategory::DISABLED_DELETE_CALL + && item.target + == ReconciliationPatchTarget::Sequent(Some( + SequentReconciliationField::Enabled(true, false), + )) + })); + } + + #[test] + fn deleted_true_is_reverted_for_a_voter_who_has_voted() { + let snapshot = VoterSnapshot { + has_valid_internet_vote: true, + ..enabled_snapshot() + }; + let items = classify_file_row( + &row("voter-1", ATTR_RESET_VALUE, "true"), + Some(&snapshot), + &datafix_source("0014"), + ); + assert!(items.iter().any(|item| { + item.category == ReconciliationChangeCategory::DELETION_REVERTED + && item.target + == ReconciliationPatchTarget::Datafix(DatafixReconciliationField::Deleted( + "true".to_string(), + "false".to_string(), + )) + })); + assert!(items + .iter() + .all(|item| item.category != ReconciliationChangeCategory::DISABLED_DELETE_CALL)); + } + + #[test] + fn unknown_voter_is_added_to_sequent_with_none_old_values() { + let items = classify_file_row( + &row("voter-1", ATTR_RESET_VALUE, "false"), + None, + &datafix_source("0014"), + ); + assert_eq!(items.len(), 4); // one per SequentReconciliationField + assert!(items + .iter() + .all(|item| item.category == ReconciliationChangeCategory::VOTER_ADDED)); + assert!(items.iter().all(|item| item.target.is_sequent())); + let area_item = items + .iter() + .find_map(|item| item.target.sequent_field()) + .and_then(|field| field.new_area_name().map(|_| field)) + .expect("one item carries the area name"); + assert!(matches!( + area_item, + SequentReconciliationField::AreaName(old, _) if old == ATTR_RESET_VALUE + )); + assert!(items.iter().any(|item| { + item.target + == ReconciliationPatchTarget::Sequent(Some(SequentReconciliationField::Enabled( + false, true, + ))) + })); + } + + #[test] + fn disabled_voter_deleted_externally_converges_when_file_still_says_deleted() { + let snapshot = VoterSnapshot { + enabled: false, + disable_comment: Some(DISABLE_REASON_DELETE_CALL.to_string()), + ..enabled_snapshot() + }; + let items = classify_file_row( + &row("voter-1", ATTR_RESET_VALUE, "true"), + Some(&snapshot), + &datafix_source("0014"), + ); + assert!(items.is_empty()); + } + + #[test] + fn disabled_voter_deleted_externally_is_reenabled_when_file_says_not_deleted() { + let snapshot = VoterSnapshot { + enabled: false, + disable_comment: Some(DISABLE_REASON_DELETE_CALL.to_string()), + ..enabled_snapshot() + }; + let items = classify_file_row( + &row("voter-1", ATTR_RESET_VALUE, "false"), + Some(&snapshot), + &datafix_source("0014"), + ); + assert_eq!(items.len(), 1); + assert_eq!(items[0].category, ReconciliationChangeCategory::REENABLED); + assert_eq!( + items[0].target, + ReconciliationPatchTarget::Sequent(Some(SequentReconciliationField::Enabled( + false, true + ))) + ); + } + + #[test] + fn disabled_voter_marked_voted_via_other_channel_is_retagged_as_delete_call() { + let snapshot = VoterSnapshot { + enabled: false, + disable_comment: Some(DISABLE_REASON_MARKVOTED_CALL.to_string()), + ..enabled_snapshot() + }; + let items = classify_file_row( + &row("voter-1", "PAPER", "true"), + Some(&snapshot), + &datafix_source("0014"), + ); + assert_eq!(items.len(), 1); + let attributes_item = items + .iter() + .find_map(|item| item.target.sequent_field()?.new_keycloak_attributes()) + .expect("the canonical channel item carries the delete reason"); + assert_eq!( + attributes_item.get(DISABLE_COMMENT), + Some(&DISABLE_REASON_DELETE_CALL.to_string()) + ); + } + + #[test] + fn admin_disabled_voter_with_no_sentinel_comment_is_also_retagged_as_delete_call() { + let snapshot = VoterSnapshot { + enabled: false, + disable_comment: Some("Released after voting online".to_string()), + ..enabled_snapshot() + }; + let items = classify_file_row( + &row("voter-1", ATTR_RESET_VALUE, "true"), + Some(&snapshot), + &datafix_source("0014"), + ); + assert_eq!(items.len(), 1); + assert!(items + .iter() + .all(|item| item.category == ReconciliationChangeCategory::DISABLED_DELETE_CALL)); + } + + #[test] + fn voter_missing_from_file_is_reported_externally() { + let area_fields = HashMap::from([( + "01-P-000".to_string(), + Some(DatafixAreaFields { + ward: "01".to_string(), + poll: "000".to_string(), + school_support_code: "P".to_string(), + }), + )]); + let items = voter_missing_from_file( + "voter-1", + &enabled_snapshot(), + &datafix_source("0014"), + &area_fields, + ); + assert_eq!(items.len(), 7); + assert!(items + .iter() + .all(|item| item.category == ReconciliationChangeCategory::VOTER_ADDED)); + assert!(items.iter().all(|item| item.target.is_datafix())); + assert!(items.iter().any(|item| { + item.target + == ReconciliationPatchTarget::Datafix(DatafixReconciliationField::CountyMun( + ATTR_RESET_VALUE.to_string(), + "0014".to_string(), + )) + })); + assert!(items.iter().any(|item| { + item.target + == ReconciliationPatchTarget::Datafix(DatafixReconciliationField::Poll( + ATTR_RESET_VALUE.to_string(), + "000".to_string(), + )) + })); + } + + #[test] + fn keycloak_internet_casing_converges_and_reverse_patch_is_uppercase() { + let snapshot = VoterSnapshot { + voted_channel: Some("Internet".to_string()), + ..enabled_snapshot() + }; + let items = classify_file_row( + &row("voter-1", FILE_CHANNEL_INTERNET, "false"), + Some(&snapshot), + &datafix_source("0014"), + ); + assert_eq!(items.len(), 1); + assert_eq!( + items[0].target, + ReconciliationPatchTarget::Datafix(DatafixReconciliationField::Channel( + FILE_CHANNEL_INTERNET.to_string(), + ATTR_RESET_VALUE.to_string(), + )) + ); + + let area_fields = HashMap::from([( + "01-P-000".to_string(), + Some(DatafixAreaFields { + ward: "01".to_string(), + poll: "000".to_string(), + school_support_code: "P".to_string(), + }), + )]); + let reverse = + voter_missing_from_file("voter-1", &snapshot, &datafix_source("0014"), &area_fields); + assert!(reverse.iter().any(|item| { + item.target + == ReconciliationPatchTarget::Datafix(DatafixReconciliationField::Channel( + ATTR_RESET_VALUE.to_string(), + FILE_CHANNEL_INTERNET.to_string(), + )) + })); + } + + #[test] + fn file_none_unmarks_a_stored_non_internet_vote() { + let snapshot = VoterSnapshot { + enabled: false, + voted_channel: Some("PAPER".to_string()), + disable_comment: Some(DISABLE_REASON_MARKVOTED_CALL.to_string()), + ..enabled_snapshot() + }; + let items = classify_file_row( + &row("voter-1", ATTR_RESET_VALUE, "false"), + Some(&snapshot), + &datafix_source("0014"), + ); + assert_eq!(items.len(), 2); + assert!(items + .iter() + .all(|item| item.category == ReconciliationChangeCategory::VOTED_UNMARKED)); + assert!(items.iter().any(|item| { + item.target + == ReconciliationPatchTarget::Sequent(Some(SequentReconciliationField::Enabled( + false, true, + ))) + })); + } + + #[test] + fn unmark_and_delete_emit_one_canonical_disable_comment() { + let snapshot = VoterSnapshot { + enabled: false, + voted_channel: Some("PAPER".to_string()), + disable_comment: Some(DISABLE_REASON_MARKVOTED_CALL.to_string()), + ..enabled_snapshot() + }; + let items = classify_file_row( + &row("voter-1", ATTR_RESET_VALUE, "true"), + Some(&snapshot), + &datafix_source("0014"), + ); + + assert_eq!(items.len(), 1); + let SequentReconciliationField::KeycloakUA(_, new_attributes) = items[0] + .target + .sequent_field() + .expect("unmark emits a Keycloak attribute change") + else { + panic!("expected Keycloak attributes"); + }; + assert_eq!( + new_attributes.get(VOTED_CHANNEL), + Some(&ATTR_RESET_VALUE.to_string()) + ); + assert_eq!( + new_attributes.get(DISABLE_COMMENT), + Some(&DISABLE_REASON_DELETE_CALL.to_string()) + ); + } + + #[test] + fn unmark_preserves_an_existing_delete_reason_and_converges_in_one_round() { + let snapshot = VoterSnapshot { + enabled: false, + voted_channel: Some("PAPER".to_string()), + disable_comment: Some(DISABLE_REASON_DELETE_CALL.to_string()), + ..enabled_snapshot() + }; + let file_row = row("voter-1", ATTR_RESET_VALUE, "true"); + let items = classify_file_row(&file_row, Some(&snapshot), &datafix_source("0014")); + + assert_eq!(items.len(), 1); + let SequentReconciliationField::KeycloakUA(_, new_attributes) = items[0] + .target + .sequent_field() + .expect("unmark emits a Keycloak attribute change") + else { + panic!("expected Keycloak attributes"); + }; + assert_eq!( + new_attributes, + &HashMap::from([(VOTED_CHANNEL.to_string(), ATTR_RESET_VALUE.to_string())]) + ); + + let after_apply = VoterSnapshot { + voted_channel: Some(ATTR_RESET_VALUE.to_string()), + ..snapshot + }; + assert!( + classify_file_row(&file_row, Some(&after_apply), &datafix_source("0014")).is_empty() + ); + } + + #[test] + fn unmark_does_not_enable_or_clear_an_admin_disabled_voter() { + let manual_reason = "Disabled manually"; + let snapshot = VoterSnapshot { + enabled: false, + voted_channel: Some("PAPER".to_string()), + disable_comment: Some(manual_reason.to_string()), + ..enabled_snapshot() + }; + let items = classify_file_row( + &row("voter-1", ATTR_RESET_VALUE, "false"), + Some(&snapshot), + &datafix_source("0014"), + ); + + assert_eq!(items.len(), 1); + let SequentReconciliationField::KeycloakUA(_, new_attributes) = items[0] + .target + .sequent_field() + .expect("unmark emits a Keycloak attribute change") + else { + panic!("expected Keycloak attributes"); + }; + assert_eq!( + new_attributes, + &HashMap::from([(VOTED_CHANNEL.to_string(), ATTR_RESET_VALUE.to_string())]) + ); + assert!(items.iter().all(|item| !matches!( + item.target.sequent_field(), + Some(SequentReconciliationField::Enabled(_, _)) + ))); + } + + #[test] + fn file_internet_without_an_active_ballot_is_corrected_to_none() { + let items = classify_file_row( + &row("voter-1", FILE_CHANNEL_INTERNET, "false"), + Some(&enabled_snapshot()), + &datafix_source("0014"), + ); + assert_eq!(items.len(), 1); + assert_eq!( + items[0].category, + ReconciliationChangeCategory::VOTED_UNMARKED + ); + assert_eq!( + items[0].target, + ReconciliationPatchTarget::Datafix(DatafixReconciliationField::Channel( + FILE_CHANNEL_INTERNET.to_string(), + ATTR_RESET_VALUE.to_string(), + )) + ); + } + + #[test] + fn unresolved_internet_ballot_blocks_other_channel_and_deletion() { + let snapshot = VoterSnapshot { + has_unresolved_internet_vote: true, + ..enabled_snapshot() + }; + for file_row in [ + row("voter-1", "PAPER", "false"), + row("voter-1", ATTR_RESET_VALUE, "true"), + row("voter-1", FILE_CHANNEL_INTERNET, "false"), + ] { + let items = classify_file_row(&file_row, Some(&snapshot), &datafix_source("0014")); + assert!(items + .iter() + .any(|item| item.category == ReconciliationChangeCategory::ROW_FAILURE)); + assert!(items.iter().all(|item| { + item.category != ReconciliationChangeCategory::VOTED_OTHER_CHANNEL + && item.category != ReconciliationChangeCategory::DISABLED_DELETE_CALL + })); + } + } + + #[test] + fn a_new_other_channel_voter_is_created_disabled_with_keycloak_casing_and_reason() { + let items = classify_file_row( + &row("voter-1", "PAPER", "false"), + None, + &datafix_source("0014"), + ); + assert_eq!(items.len(), 5); + assert!(items.iter().any(|item| { + item.target + == ReconciliationPatchTarget::Sequent(Some(SequentReconciliationField::Enabled( + false, false, + ))) + })); + assert!(items.iter().any(|item| { + item.target.sequent_field().is_some_and(|field| { + field.new_keycloak_attributes().is_some_and(|attributes| { + attributes.get(DISABLE_COMMENT) + == Some(&DISABLE_REASON_MARKVOTED_CALL.to_string()) + }) + }) + })); + } + + #[test] + fn composed_area_name_omits_the_files_none_sentinel() { + // The file's own "NONE" sentinel for an absent SchoolSupportCode/Poll + // must not end up as a literal "-NONE-" segment in the composed name. + let mut file_row = row("voter-1", ATTR_RESET_VALUE, "false"); + assert_eq!(composed_area_name(&file_row), "01-P-000"); + + file_row.school_support_code = ATTR_RESET_VALUE.to_string(); + assert_eq!(composed_area_name(&file_row), "01-000"); + + file_row.poll = ATTR_RESET_VALUE.to_string(); + assert_eq!(composed_area_name(&file_row), "01"); + } +} diff --git a/packages/windmill/src/services/external/reconciliation/mod.rs b/packages/windmill/src/services/external/reconciliation/mod.rs new file mode 100644 index 00000000000..244bd22fc6f --- /dev/null +++ b/packages/windmill/src/services/external/reconciliation/mod.rs @@ -0,0 +1,20 @@ +// SPDX-FileCopyrightText: 2026 Sequent Tech Inc +// +// SPDX-License-Identifier: AGPL-3.0-only + +//! Datafix reconciliation: importing a CSV file Datafix produces, diffing it +//! against Sequent's current voter data per the source-of-truth rules, +//! generating a downloadable patch for Datafix, and applying the +//! Sequent-side changes directly. +//! +//! Reuses the existing Datafix API helpers throughout (per the spec's +//! explicit instruction to do so): `utils::external_voter_lock_key`/`PgLock`, +//! `utils::post_operation_result_to_electoral_log`, and +//! `utils::compose_area_name`. The per-voter cast-vote guard is +//! re-implemented rather than reused — see `apply`'s module doc. + +pub mod apply; +pub mod bulk_create; +pub mod csv; +pub mod diff; +pub mod patch; diff --git a/packages/windmill/src/services/external/reconciliation/patch.rs b/packages/windmill/src/services/external/reconciliation/patch.rs new file mode 100644 index 00000000000..79b2ede9fce --- /dev/null +++ b/packages/windmill/src/services/external/reconciliation/patch.rs @@ -0,0 +1,441 @@ +// SPDX-FileCopyrightText: 2026 Sequent Tech Inc +// +// SPDX-License-Identifier: AGPL-3.0-only + +//! Builds the streamed documents `generate_reconciliation_patches` produces +//! from the computed diff: the downloadable Datafix patch CSV, the review +//! envelope's JSON item array, and the internal Sequent-patch NDJSON stream +//! `apply_reconciliation_patch` later consumes. Unlike +//! the diff itself (only the fields that changed), the CSV needs every +//! `DatafixReconciliationField` present per voter regardless of which ones changed, +//! per the "Patch Files Format" spec. + +use crate::services::external::datafix_types::{ + DatafixReconciliationField, ParsedDatafixReconciliationRow, FILE_CHANNEL_INTERNET, +}; +use crate::services::external::reconciliation::diff::DiffItem; +use crate::services::external::types::ReconciliationPatchTarget; +use sequent_core::types::keycloak::ATTR_RESET_VALUE; +use sha2::{Digest, Sha256}; +use std::collections::HashMap; +use std::io::Write; +use tracing::instrument; + +/// Incrementally builds the Datafix patch CSV described in "Patch Files +/// Format" across many batches — the streaming counterpart to serializing +/// the whole diff's items in one call, so a 100k+-row reconciliation never +/// needs the whole diff resident in memory to produce this document. One row +/// per changed voter, every `DatafixReconciliationField` as an `_old`/`_new` +/// pair — unchanged fields repeat this batch's real row value (read off +/// `file_rows_by_username`, this same batch's parsed rows) in both columns, +/// per spec. `NONE` is only used when the voter has no row in the file at +/// all (D, reverse direction — added to Datafix), the spec's own example of +/// a legitimate `NONE`. Unlike the non-streaming version this used to be, +/// voters are written in whatever order their batch encounters them, not +/// sorted across the whole file — each row is independently valid regardless +/// of order, so this only affects the file's own organization, not +/// correctness. +pub struct ExternalPatchCsvWriter { + writer: ::csv::Writer, + wrote_any_row: bool, +} + +impl ExternalPatchCsvWriter { + #[instrument(skip(writer), err)] + pub fn start(mut writer: W, sequence: i64, generated_at: i64) -> std::io::Result { + writeln!( + writer, + "#META,Sequence={sequence},GeneratedAt={generated_at}" + )?; + let header: Vec = std::iter::once("VoterID".to_string()) + .chain( + DatafixReconciliationField::NAMES + .iter() + .flat_map(|name| [format!("{name}_old"), format!("{name}_new")]), + ) + .collect(); + let mut writer = ::csv::WriterBuilder::new() + .has_headers(false) + .from_writer(writer); + writer.write_record(header)?; + Ok(Self { + writer, + wrote_any_row: false, + }) + } + + /// Appends this batch's `target = Datafix` items as patch rows. + /// `file_rows_by_username` must be this same batch's parsed rows. + #[instrument(skip_all, err)] + pub fn write_batch( + &mut self, + items: &[DiffItem], + file_rows_by_username: &HashMap, + ) -> std::io::Result<()> { + let mut by_voter: HashMap<&str, HashMap<&'static str, (&str, &str)>> = HashMap::new(); + for item in items { + let Some(field) = item.target.datafix_field() else { + continue; + }; + by_voter + .entry(item.voter_username.as_str()) + .or_default() + .insert(field.name(), field.old_new()); + } + + let mut voters: Vec<&str> = by_voter.keys().copied().collect(); + voters.sort_unstable(); + + for voter_username in voters { + let fields = &by_voter[voter_username]; + let file_row = file_rows_by_username.get(voter_username); + let values: Vec = DatafixReconciliationField::NAMES + .iter() + .flat_map(|name| match fields.get(name) { + Some((old_value, new_value)) => { + vec![old_value.to_string(), new_value.to_string()] + } + None => { + let value = file_row + .and_then(|row| row.field_value(name)) + .unwrap_or(ATTR_RESET_VALUE); + vec![value.to_string(), value.to_string()] + } + }) + .collect(); + self.writer.write_record( + std::iter::once(voter_username).chain(values.iter().map(String::as_str)), + )?; + self.wrote_any_row = true; + } + Ok(()) + } + + /// Returns the underlying writer if any row was ever written, `None` + /// (matching "nothing to patch") otherwise — the caller decides whether + /// a Datafix patch document exists at all based on this. + pub fn finish(mut self) -> std::io::Result> { + if !self.wrote_any_row { + return Ok(None); + } + self.writer.flush()?; + self.writer + .into_inner() + .map(Some) + .map_err(|error| error.into_error()) + } +} + +/// Incrementally writes a JSON array of `DiffItem`s across many batches, +/// without ever holding the whole array in memory at once. Used for the diff +/// envelope's `items` field and the apply audit artifact; the executable +/// Sequent stream uses `DiffItemNdjsonWriter` below. +pub struct DiffItemArrayWriter { + writer: W, + wrote_any: bool, +} + +/// Incrementally writes the internal apply stream as newline-delimited JSON. +/// Unlike a JSON array, NDJSON can be consumed one voter at a time with +/// `serde_json::StreamDeserializer`, keeping apply memory bounded by one +/// voter (or one bulk-create batch) instead of the complete patch. +pub struct DiffItemNdjsonWriter { + writer: W, +} + +impl DiffItemNdjsonWriter { + pub fn new(writer: W) -> Self { + Self { writer } + } + + #[instrument(skip_all, err)] + pub fn write_batch<'a>( + &mut self, + items: impl Iterator, + ) -> anyhow::Result<()> { + for item in items { + serde_json::to_writer(&mut self.writer, item)?; + self.writer.write_all(b"\n")?; + } + Ok(()) + } + + pub fn finish(self) -> W { + self.writer + } +} + +impl DiffItemArrayWriter { + #[instrument(skip(writer), err)] + pub fn start(mut writer: W) -> std::io::Result { + writer.write_all(b"[")?; + Ok(Self { + writer, + wrote_any: false, + }) + } + + #[instrument(skip_all, err)] + pub fn write_batch<'a>( + &mut self, + items: impl Iterator, + ) -> anyhow::Result<()> { + for item in items { + if self.wrote_any { + self.writer.write_all(b",")?; + } + serde_json::to_writer(&mut self.writer, item)?; + self.wrote_any = true; + } + Ok(()) + } + + /// Closes the array and returns the underlying writer. + pub fn finish(mut self) -> std::io::Result { + self.writer.write_all(b"]")?; + Ok(self.writer) + } +} + +/// The internal apply stream contains every Sequent-side item, including +/// `ROW_FAILURE`: failures are not mutations, but carrying them in the same +/// bounded stream lets apply report generate-time failures without loading +/// the full review envelope into memory. +pub fn is_sequent_apply_stream_item(item: &DiffItem) -> bool { + item.target.is_sequent() +} + +#[instrument(skip_all)] +pub fn sha256_hex(content: &[u8]) -> String { + let mut hasher = Sha256::new(); + hasher.update(content); + hex::encode(hasher.finalize()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::services::external::types::{ + ReconciliationChangeCategory, SequentReconciliationField, + }; + + fn item(voter: &str, target: ReconciliationPatchTarget) -> DiffItem { + DiffItem { + voter_username: voter.to_string(), + voter_id: Some(uuid::Uuid::new_v4()), + target, + category: ReconciliationChangeCategory::PROFILE_UPDATE, + failure_reason: None, + } + } + + fn file_row(voter_id: &str) -> ParsedDatafixReconciliationRow { + ParsedDatafixReconciliationRow { + county_mun: "0014".to_string(), + external_voter_id: voter_id.to_string(), + dob: "1990-01-01".to_string(), + ward: "01".to_string(), + poll: "000".to_string(), + school_support_code: "P".to_string(), + channel: ATTR_RESET_VALUE.to_string(), + deleted: "false".to_string(), + } + } + + /// Drives `ExternalPatchCsvWriter` over an in-memory buffer for a single + /// batch, mirroring the removed `build_external_patch_csv`'s own + /// `Option` shape so the tests below stay close to what they + /// were checking before this became a streaming writer. + fn write_csv( + items: &[DiffItem], + file_rows_by_username: &HashMap, + sequence: i64, + generated_at: i64, + ) -> Option { + let mut writer = ExternalPatchCsvWriter::start(Vec::new(), sequence, generated_at) + .expect("writing to a Vec cannot fail"); + writer + .write_batch(items, file_rows_by_username) + .expect("writing to a Vec cannot fail"); + writer + .finish() + .ok() + .flatten() + .map(|buffer| String::from_utf8(buffer).expect("writer only emits valid UTF-8")) + } + + #[test] + fn returns_none_when_there_is_nothing_for_the_external_system() { + let items = vec![item( + "v1", + ReconciliationPatchTarget::Sequent(Some(SequentReconciliationField::AreaName( + "01".to_string(), + "02".to_string(), + ))), + )]; + assert!(write_csv(&items, &HashMap::new(), 1, 100).is_none()); + } + + #[test] + fn includes_the_meta_line_and_header() { + let items = vec![item( + "v1", + ReconciliationPatchTarget::Datafix(DatafixReconciliationField::Channel( + ATTR_RESET_VALUE.to_string(), + FILE_CHANNEL_INTERNET.to_string(), + )), + )]; + let csv = write_csv(&items, &HashMap::new(), 5, 1781780700).unwrap(); + let mut lines = csv.lines(); + assert_eq!( + lines.next(), + Some("#META,Sequence=5,GeneratedAt=1781780700") + ); + assert!(lines + .next() + .unwrap() + .starts_with("VoterID,CountyMun_old,CountyMun_new")); + assert!(csv.contains("v1,")); + } + + #[test] + fn unchanged_fields_carry_the_voters_real_row_value_not_none() { + let items = vec![item( + "v1", + ReconciliationPatchTarget::Datafix(DatafixReconciliationField::Channel( + ATTR_RESET_VALUE.to_string(), + FILE_CHANNEL_INTERNET.to_string(), + )), + )]; + let file_rows = HashMap::from([("v1".to_string(), file_row("v1"))]); + let csv = write_csv(&items, &file_rows, 5, 1781780700).unwrap(); + let data_line = csv.lines().nth(2).unwrap(); + assert!(data_line.contains("0014,0014")); // CountyMun_old,CountyMun_new + assert!(data_line.contains("1990-01-01,1990-01-01")); // DoB_old,DoB_new + assert!(data_line.contains("false,false")); // Deleted_old,Deleted_new + // The changed field (Channel) legitimately carries "NONE" as its real + // old value here — only the placeholder pair for an unchanged field + // would indicate the bug this test guards against. + assert!(!data_line.contains("NONE,NONE")); + } + + #[test] + fn voter_missing_from_the_file_falls_back_to_none_for_unchanged_fields() { + let items = vec![item( + "v2", + ReconciliationPatchTarget::Datafix(DatafixReconciliationField::Ward( + ATTR_RESET_VALUE.to_string(), + "01-P-000".to_string(), + )), + )]; + let csv = write_csv(&items, &HashMap::new(), 5, 1781780700).unwrap(); + let data_line = csv.lines().nth(2).unwrap(); + assert!(data_line.contains("NONE,NONE")); // e.g. CountyMun_old,CountyMun_new + } + + #[test] + fn external_patch_csv_writer_accumulates_across_batches() { + let batch_one = vec![item( + "v1", + ReconciliationPatchTarget::Datafix(DatafixReconciliationField::Channel( + ATTR_RESET_VALUE.to_string(), + FILE_CHANNEL_INTERNET.to_string(), + )), + )]; + let batch_two = vec![item( + "v2", + ReconciliationPatchTarget::Datafix(DatafixReconciliationField::Ward( + ATTR_RESET_VALUE.to_string(), + "01-P-000".to_string(), + )), + )]; + let mut writer = ExternalPatchCsvWriter::start(Vec::new(), 5, 1781780700).unwrap(); + writer.write_batch(&batch_one, &HashMap::new()).unwrap(); + writer.write_batch(&batch_two, &HashMap::new()).unwrap(); + let csv = String::from_utf8(writer.finish().unwrap().unwrap()).unwrap(); + assert!(csv.contains("v1,")); + assert!(csv.contains("v2,")); + } + + #[test] + fn external_patch_quotes_values_that_contain_csv_delimiters() { + let items = vec![item( + "v1", + ReconciliationPatchTarget::Datafix(DatafixReconciliationField::Channel( + ATTR_RESET_VALUE.to_string(), + "PAPER,ASSISTED".to_string(), + )), + )]; + let csv = write_csv(&items, &HashMap::new(), 5, 1781780700).unwrap(); + assert!(csv.contains("\"PAPER,ASSISTED\"")); + + let body = csv.split_once('\n').unwrap().1; + let mut reader = ::csv::Reader::from_reader(body.as_bytes()); + let record = reader.records().next().unwrap().unwrap(); + assert_eq!(record.get(12), Some("PAPER,ASSISTED")); + } + + #[test] + fn diff_item_writers_produce_valid_array_and_stream_formats() { + let sequent_item = item( + "v1", + ReconciliationPatchTarget::Sequent(Some(SequentReconciliationField::AreaName( + "01".to_string(), + "02".to_string(), + ))), + ); + let row_failure = DiffItem { + voter_username: "v2".to_string(), + voter_id: None, + target: ReconciliationPatchTarget::Sequent(None), + category: ReconciliationChangeCategory::ROW_FAILURE, + failure_reason: Some("bad row".to_string()), + }; + let datafix_item = item( + "v3", + ReconciliationPatchTarget::Datafix(DatafixReconciliationField::Channel( + ATTR_RESET_VALUE.to_string(), + FILE_CHANNEL_INTERNET.to_string(), + )), + ); + + // The review envelope remains a normal JSON array. + let mut writer = DiffItemArrayWriter::start(Vec::new()).unwrap(); + writer + .write_batch([&sequent_item, &row_failure, &datafix_item].into_iter()) + .unwrap(); + let bytes = writer.finish().unwrap(); + let parsed: Vec = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(parsed.len(), 3); + + // The internal apply document is NDJSON and carries generate-time + // row failures as structured, non-mutating stream entries. + let mut writer = DiffItemNdjsonWriter::new(Vec::new()); + writer + .write_batch( + [&sequent_item, &row_failure, &datafix_item] + .into_iter() + .filter(|item| is_sequent_apply_stream_item(item)), + ) + .unwrap(); + let bytes = writer.finish(); + let parsed: Vec = serde_json::Deserializer::from_slice(&bytes) + .into_iter() + .collect::>() + .unwrap(); + assert_eq!(parsed.len(), 2); + assert_eq!(parsed[0].voter_username, "v1"); + assert_eq!( + parsed[1].category, + ReconciliationChangeCategory::ROW_FAILURE + ); + } + + #[test] + fn sha256_hex_is_stable() { + assert_eq!( + sha256_hex(b"hello"), + "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824" + ); + } +} diff --git a/packages/windmill/src/services/external/types.rs b/packages/windmill/src/services/external/types.rs new file mode 100644 index 00000000000..2e29d2f1041 --- /dev/null +++ b/packages/windmill/src/services/external/types.rs @@ -0,0 +1,164 @@ +// SPDX-FileCopyrightText: 2026 Sequent Tech Inc +// +// SPDX-License-Identifier: AGPL-3.0-only +use super::datafix_types::DatafixReconciliationField; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use strum_macros::{Display, EnumString}; + +/// The mandatory `#META,Sequence=N,GeneratedAt=T` line every reconciliation +/// and patch file starts with. `Sequence` is the ordering authority for +/// stale-file protection; `GeneratedAt` is informational only. +#[derive(Debug, Clone, Copy)] +pub struct ReconciliationFileMeta { + pub sequence: i64, + pub generated_at: i64, +} + +/// Source-of-truth category a reconciliation change falls under. Wire +/// values match the admin portal's `ESyncChangeCategory` exactly. +#[allow(non_camel_case_types)] +#[derive(Display, Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Hash, EnumString)] +pub enum ReconciliationChangeCategory { + /// A: Sequent holds a valid Internet ballot, Datafix reported `NONE`. + VOTED_INTERNET, + /// B: Datafix reports a non-`INTERNET` channel Sequent doesn't have. + VOTED_OTHER_CHANNEL, + /// C: `Deleted=true` in the file for a voter who has not voted. Named + /// for the Keycloak `disable-comment` reason this drives + /// (`DISABLE_REASON_DELETE_CALL`), set generically via `KeycloakUA` — + /// see `SequentReconciliationField`. + DISABLED_DELETE_CALL, + /// C exception: `Deleted=true` in the file for a voter who *has* voted — + /// the deletion is not applied, the Datafix patch reverts it. + DELETION_REVERTED, + /// C: `Ward`/`Poll`/`SchoolSupportCode`/`DoB` changed on the Datafix side. + PROFILE_UPDATE, + /// D: a voter present on one side, missing on the other. + VOTER_ADDED, + /// A voter Sequent disabled solely because of a Datafix `/delete-voter` + /// call (`disable-comment = DISABLE_REASON_DELETE_CALL`) is re-enabled + /// because the file no longer reports them `Deleted` — see the match + /// block in `reconciliation::diff::classify_file_row`. + REENABLED, + /// Datafix reports `Channel=NONE` for a voter Sequent still records as + /// voted. Mirrors the real-time `/unmark-voted` outcome: clear the voted + /// channel and disable reason, then re-enable the voter. + VOTED_UNMARKED, + /// Excluded from both diffs/patch: a `CountyMun` processing error, or the + /// "voted via other channel but holds a valid Internet ballot" guard. + ROW_FAILURE, +} + +/// Sequent-side field a reconciliation change applies to, when `target = +/// Sequent`, carrying its own `(old, new)` pair directly instead of leaving +/// it to a separate `old_value`/`new_value` on `DiffItem` — a field and its +/// old/new values are never meaningful apart from each other. Distinct from +/// `DatafixReconciliationField`: +/// - `AreaName` replaces `Ward`/`Poll`/`SchoolSupportCode` — Sequent only +/// stores the composed area name (see `Area::name` / +/// `reconciliation::diff::composed_area_name`), resolved to an `area-id` +/// attribute by looking the name up, not by decomposing it back into parts. +/// - `Enabled` drives the Keycloak `enabled` flag directly, `(old, new)` in +/// Keycloak's own enabled terms (not `Deleted`'s inverted CSV sense) — a +/// file row with `Deleted=true` for a voter currently enabled in Sequent +/// means `Enabled(true, false)`. +/// - Everything else Sequent stores as a plain Keycloak user attribute +/// (date of birth, voted channel, the disable-comment reason, ...) is +/// carried verbatim in `KeycloakUA`, keyed exactly as Keycloak expects. +/// The origin (today, only Datafix — see `reconciliation::diff`) decides +/// these key/value pairs; `reconciliation::apply` only ever merges and +/// writes the new set, with no knowledge of *why* an attribute has a given +/// value. This is what lets `apply` stay a plain, source-agnostic Keycloak +/// edit. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum SequentReconciliationField { + AreaName(String, String), + Enabled(bool, bool), + KeycloakUA(HashMap, HashMap), +} + +impl SequentReconciliationField { + /// The new area name, if this is an `AreaName` change. + pub fn new_area_name(&self) -> Option<&str> { + match self { + Self::AreaName(_, new) => Some(new.as_str()), + Self::Enabled(..) | Self::KeycloakUA(..) => None, + } + } + + /// The target `enabled` state, if this is an `Enabled` change. + pub fn new_enabled(&self) -> Option { + match self { + Self::Enabled(_, new) => Some(*new), + Self::AreaName(..) | Self::KeycloakUA(..) => None, + } + } + + /// The Keycloak attributes to write, if this is a `KeycloakUA` change. + pub fn new_keycloak_attributes(&self) -> Option<&HashMap> { + match self { + Self::KeycloakUA(_, new) => Some(new), + Self::AreaName(..) | Self::Enabled(..) => None, + } + } +} + +/// Which side of the reconciliation a change is applied to, carrying the +/// field it applies to in the shape that side actually understands: `Datafix` +/// changes are written into the downloadable patch for Datafix to apply on +/// their end, described in terms of `DatafixReconciliationField`; `Sequent` +/// changes are applied directly to this system, described in terms of +/// `SequentReconciliationField` — `None` when the change (e.g. a `CountyMun` +/// mismatch `ROW_FAILURE`) doesn't correspond to any field Sequent actually +/// stores. Kept generic on the Sequent side (an area name and freeform +/// Keycloak attributes rather than Datafix's raw Ward/Poll/SchoolSupportCode +/// columns) so a future voter interface provider other than Datafix can +/// populate the same shape. +/// +/// Wire shape: adjacently tagged (`{"target": "datafix"|"sequent", "field": +/// ...}`), matching the admin portal's separate `target`/`field` columns. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +#[serde(tag = "target", content = "field", rename_all = "lowercase")] +pub enum ReconciliationPatchTarget { + Datafix(DatafixReconciliationField), + Sequent(Option), +} + +impl ReconciliationPatchTarget { + pub fn is_datafix(&self) -> bool { + matches!(self, Self::Datafix(_)) + } + + pub fn is_sequent(&self) -> bool { + matches!(self, Self::Sequent(_)) + } + + pub fn datafix_field(&self) -> Option<&DatafixReconciliationField> { + match self { + Self::Datafix(field) => Some(field), + Self::Sequent(_) => None, + } + } + + pub fn sequent_field(&self) -> Option<&SequentReconciliationField> { + match self { + Self::Sequent(field) => field.as_ref(), + Self::Datafix(_) => None, + } + } +} + +/// Which external voter registry a reconciliation round came from (generate) +/// or is being applied against (apply) — same shape as +/// `ReconciliationPatchTarget`: today only `Datafix` exists, carrying +/// whatever config that source's own classification (`CountyMun`, see +/// `reconciliation::diff::classify_file_row`) and apply-time bookkeeping +/// (its own `Sequence` tracking, see `reconciliation::apply`) needs — so a +/// future non-Datafix voter registry source can add its own variant without +/// any of the generic reconciliation code needing to change. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +#[serde(tag = "source", rename_all = "lowercase")] +pub enum ReconciliationPatchSource { + Datafix { county_mun: String }, +} diff --git a/packages/windmill/src/services/datafix/utils.rs b/packages/windmill/src/services/external/utils.rs similarity index 74% rename from packages/windmill/src/services/datafix/utils.rs rename to packages/windmill/src/services/external/utils.rs index e89de47be9c..85c3862789f 100644 --- a/packages/windmill/src/services/datafix/utils.rs +++ b/packages/windmill/src/services/external/utils.rs @@ -1,9 +1,10 @@ // SPDX-FileCopyrightText: 2025 Sequent Tech Inc // // SPDX-License-Identifier: AGPL-3.0-only -use super::types::*; +use super::datafix_types::*; use crate::postgres::area::get_event_areas; use crate::postgres::election_event::get_election_event_by_id; +use crate::postgres::election_event::update_election_event_annotations; use crate::postgres::election_event::{get_all_tenant_election_events, ElectionEventDatafix}; use crate::services::consolidation::eml_generator::ValidateAnnotations; use crate::services::electoral_log::ElectoralLog; @@ -12,6 +13,8 @@ use crate::services::users::get_users_by_username; use anyhow::{anyhow, Result}; use deadpool_postgres::Transaction; use electoral_log::messages::newtypes::{ExtApiName, ExtApiRequestDirection}; +use sequent_core::ballot::Annotations; +use sequent_core::serialization::deserialize_with_path::deserialize_value; use sequent_core::types::hasura::core::ElectionEvent; use sequent_core::types::keycloak::UserArea; use sequent_core::types::keycloak::{ @@ -19,19 +22,35 @@ use sequent_core::types::keycloak::{ }; use std::collections::HashMap; use tracing::{error, info, instrument, warn}; +use uuid::Uuid; pub const DATAFIX_ID_KEY: &str = "datafix:id"; pub const DATAFIX_PSW_POLICY_KEY: &str = "datafix:password_policy"; pub const DATAFIX_VOTERVIEW_REQ_KEY: &str = "datafix:voterview_request"; +/// Last `Sequence` (from a reconciliation file's `#META` line) actually applied +/// for this event — the monotonic gate against importing a stale file. Kept +/// per-provider under its own annotation key (like the three above), not a +/// dedicated `election_event` column, since a future non-Datafix voter +/// registry integration would need its own independent sequence, not share +/// this one. +pub const DATAFIX_LAST_APPLIED_SEQUENCE_KEY: &str = "datafix:last_applied_sequence"; +/// Whether the most recent reconciliation apply had per-row failures. A true +/// value permits retrying that same Sequence; a successful apply clears it. +pub const DATAFIX_LAST_APPLY_HAD_FAILURES_KEY: &str = "datafix:last_apply_had_failures"; /// Lifetime of the per-voter Datafix advisory lock. Must exceed the slowest /// VoterView round-trip so the lock outlives an in-flight SOAP call. pub const DATAFIX_VOTER_LOCK_SECS: i64 = 300; -/// Advisory-lock key that serializes all Datafix work for one voter within an -/// event, so outbound `SetVoted`, disable-release and inbound mark/unmark cannot -/// interleave for the same voter. +/// Advisory-lock key that serializes all external-voter-registry work for one +/// voter within an event — outbound `SetVoted`, disable-release, inbound +/// mark/unmark, and reconciliation apply all take this same lock, so none of +/// them can interleave for the same voter. #[instrument] -pub fn datafix_voter_lock_key(tenant_id: &str, election_event_id: &str, voter_id: &str) -> String { +pub fn external_voter_lock_key( + tenant_id: &str, + election_event_id: &str, + voter_id: &Uuid, +) -> String { format!("datafix-voter-{tenant_id}-{election_event_id}-{voter_id}") } @@ -41,7 +60,7 @@ pub fn datafix_voter_lock_key(tenant_id: &str, election_event_id: &str, voter_id pub fn voted_via_internet(attributes: &HashMap>) -> bool { match attributes.iter().find(|tupple| tupple.0.eq(VOTED_CHANNEL)) { Some((_, v)) => { - matches!(v.last(), Some(channel) if channel.eq(VOTED_CHANNEL_INTERNET_VALUE)) + matches!(v.last(), Some(channel) if channel.eq_ignore_ascii_case(VOTED_CHANNEL_INTERNET_VALUE)) } None => false, } @@ -53,7 +72,7 @@ pub fn voted_via_internet(attributes: &HashMap>) -> bool { pub fn voted_via_not_internet_channel(attributes: &HashMap>) -> bool { match attributes.iter().find(|tupple| tupple.0.eq(VOTED_CHANNEL)) { Some((_, v)) => { - matches!(v.last(), Some(channel) if channel != ATTR_RESET_VALUE && channel != VOTED_CHANNEL_INTERNET_VALUE && !channel.is_empty()) + matches!(v.last(), Some(channel) if !channel.eq_ignore_ascii_case(ATTR_RESET_VALUE) && !channel.eq_ignore_ascii_case(VOTED_CHANNEL_INTERNET_VALUE) && !channel.is_empty()) } None => false, } @@ -113,8 +132,12 @@ pub async fn get_event_id_and_datafix_annotations( /// a concatenation of `Ward-SchoolSupportCode-Poll`. `None` (or empty) values are /// ignored (e.g. `WARD-POLL` when there is no SchoolSupportCode, /// `WARD-SCHOOL` when there is no Poll). All values are uppercased. +/// `pub(crate)` (rather than private) so `reconciliation::diff` can reuse the +/// exact same Ward-SchoolSupportCode-Poll composition/uppercasing rule when +/// comparing a reconciliation file row's area against a voter's resolved +/// `Area::name`. #[instrument(skip_all)] -fn compose_area_name(voter_info: &VoterInformationBody) -> String { +pub(crate) fn compose_area_name(voter_info: &VoterInformationBody) -> String { let mut parts = vec![voter_info.ward.clone()]; if let Some(schoolboard) = &voter_info.schoolboard { @@ -243,6 +266,57 @@ pub fn datafix_annotations(election_event: &ElectionEvent) -> Result, + tenant_id: &str, + election_event_id: &str, + sequence: i64, + had_failures: bool, +) -> Result<()> { + let election_event = + get_election_event_by_id(hasura_transaction, tenant_id, election_event_id).await?; + let annotations_value = election_event + .annotations + .clone() + .ok_or_else(|| anyhow!("Missing election event annotations"))?; + let mut annotations: Annotations = deserialize_value(annotations_value)?; + + let current: Option = annotations + .get(DATAFIX_LAST_APPLIED_SEQUENCE_KEY) + .and_then(|value| value.parse().ok()); + if current.is_some_and(|current| current > sequence) { + return Err(anyhow!( + "Cannot move Datafix reconciliation state backwards from Sequence {} to {sequence}", + current.unwrap_or_default() + )); + } + + annotations.insert( + DATAFIX_LAST_APPLIED_SEQUENCE_KEY.to_string(), + sequence.to_string(), + ); + annotations.insert( + DATAFIX_LAST_APPLY_HAD_FAILURES_KEY.to_string(), + had_failures.to_string(), + ); + let annotations_value = serde_json::to_value(&annotations)?; + update_election_event_annotations( + hasura_transaction, + tenant_id, + election_event_id, + annotations_value, + ) + .await +} + #[instrument(skip_all, fields(direction = %direction), err)] pub async fn post_operation_result_to_electoral_log( hasura_transaction: &Transaction<'_>, diff --git a/packages/windmill/src/services/datafix/voterview_requests.rs b/packages/windmill/src/services/external/voterview_requests.rs similarity index 99% rename from packages/windmill/src/services/datafix/voterview_requests.rs rename to packages/windmill/src/services/external/voterview_requests.rs index 6afa1ee54f7..f49e9bd9bea 100644 --- a/packages/windmill/src/services/datafix/voterview_requests.rs +++ b/packages/windmill/src/services/external/voterview_requests.rs @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0-only -use super::types::{ +use super::datafix_types::{ DatafixAnnotations, SoapRequest, SoapRequestData, SoapRequestResponse, SoapRequestResult, }; use crate::postgres::election_event::ElectionEventDatafix; diff --git a/packages/windmill/src/services/insert_cast_vote.rs b/packages/windmill/src/services/insert_cast_vote.rs index 68903b732e9..fca5fe7632e 100644 --- a/packages/windmill/src/services/insert_cast_vote.rs +++ b/packages/windmill/src/services/insert_cast_vote.rs @@ -9,11 +9,12 @@ use crate::postgres::election_event::get_election_event_by_id; use crate::postgres::scheduled_event::find_scheduled_event_by_election_event_id; use crate::services::cast_votes::{CastVote, CastVoteStatus}; use crate::services::database::{get_hasura_pool, get_keycloak_pool}; -use crate::services::datafix::utils::{ - datafix_annotations, datafix_voter_lock_key, is_datafix_election_event, DATAFIX_VOTER_LOCK_SECS, -}; use crate::services::election_event_board::get_election_event_board; use crate::services::electoral_log::ElectoralLog; +use crate::services::external::utils::{ + datafix_annotations, external_voter_lock_key, is_datafix_election_event, + DATAFIX_VOTER_LOCK_SECS, +}; use crate::services::pg_lock::PgLock; use crate::services::protocol_manager::get_protocol_manager; use crate::services::users::get_username_by_id; @@ -189,8 +190,10 @@ async fn insert_datafix_cast_vote_locked<'a>( is_early_voting_area: bool, initial_status: CastVoteStatus, ) -> Result<(CastVote, VotingStatusChannel), CastVoteError> { + let voter_id_uuid = parse_uuid_v4(ids.voter_id) + .map_err(|err| CastVoteError::VoterStateLocked(format!("Invalid voter id: {err}")))?; let lock = PgLock::acquire( - datafix_voter_lock_key(ids.tenant_id, ids.election_event_id, ids.voter_id), + external_voter_lock_key(ids.tenant_id, ids.election_event_id, &voter_id_uuid), Uuid::new_v4().to_string(), ISO8601::now() + Duration::seconds(DATAFIX_VOTER_LOCK_SECS), ) diff --git a/packages/windmill/src/services/mod.rs b/packages/windmill/src/services/mod.rs index 4dfa5cc0e97..56e554dd16a 100644 --- a/packages/windmill/src/services/mod.rs +++ b/packages/windmill/src/services/mod.rs @@ -13,7 +13,6 @@ pub mod compress; pub mod consolidation; pub mod custom_url; pub mod database; -pub mod datafix; pub mod delete_election_event; pub mod documents; pub mod election; @@ -27,6 +26,7 @@ pub mod electoral_log; pub mod ess_xml_converter; pub mod event_list; pub mod export; +pub mod external; pub mod folders; pub mod generate_preview_url; pub mod google_meet; diff --git a/packages/windmill/src/services/tasks_execution.rs b/packages/windmill/src/services/tasks_execution.rs index ff3ccc28d1f..95fde90b288 100644 --- a/packages/windmill/src/services/tasks_execution.rs +++ b/packages/windmill/src/services/tasks_execution.rs @@ -52,6 +52,21 @@ pub async fn update( document_id: Option, ) -> Result<(), anyhow::Error> { let annotations = serde_json::to_value(TaskAnnotations { document_id })?; + update_with_annotations(tenant_id, task_id, status, logs, annotations).await +} + +/// Updates a task with caller-owned structured annotations. Most tasks only +/// need `document_id` and should use `update`; workflows with typed result +/// data (for example reconciliation row failures) use this instead of +/// encoding machine-readable state in human log strings. +#[instrument(skip_all, err)] +pub async fn update_with_annotations( + tenant_id: &str, + task_id: &str, + status: TasksExecutionStatus, + logs: serde_json::Value, + annotations: serde_json::Value, +) -> Result<(), anyhow::Error> { update_task_execution_status(tenant_id, task_id, status, Some(logs), annotations) .await .context("Failed to update task execution record")?; diff --git a/packages/windmill/src/services/users.rs b/packages/windmill/src/services/users.rs index 7603d412513..ce5edf9d612 100644 --- a/packages/windmill/src/services/users.rs +++ b/packages/windmill/src/services/users.rs @@ -41,6 +41,186 @@ pub const VALIDATE_ID_ATTR_NAME: &str = "sequent.read-only.id-card-number-valida pub const DELEGATE_TO_ATTR_NAME: &str = "delegate-vote-to"; pub const VALIDATE_ID_REGISTERED_VOTER: &str = "VERIFIED"; +/// A voter's current Sequent-side state, as far as reconciliation cares. +#[derive(Debug, Clone)] +pub struct VoterSnapshot { + pub username: String, + /// Keycloak's own internal user id (`user_entity.id`) — same value + /// `cast_vote.voter_id_string` carries (as text there), so this is what + /// actually matches against the event-wide cast-vote state map; + /// `username` is a separate, mutable-in-theory identifier not safe to key + /// that lookup on. + pub voter_id: Uuid, + pub enabled: bool, + /// The voter's resolved area name (`WARD-SCHOOLBOARD-POLL`, uppercased — + /// see `reconciliation::snapshot`), `None` if their `area-id` attribute + /// doesn't resolve to a known area (or is unset). + pub area_name: Option, + pub dob: Option, + /// Raw `voted-channel` attribute value; `None` means not voted. File-side + /// comparisons normalize this value explicitly at the boundary. + pub voted_channel: Option, + pub has_valid_internet_vote: bool, + pub has_unresolved_internet_vote: bool, + /// Raw `disable-comment` attribute value (`sequent_core::types::keycloak::DISABLE_COMMENT`), + /// used by `diff.rs` to classify an already-disabled voter — see the + /// classification rules in `reconciliation::diff::classify_file_row`. + pub disable_comment: Option, +} + +const VOTER_SNAPSHOT_PAGE_SIZE: i64 = 5_000; + +/// One keyset-paginated page of the realm's voter-group users, ordered by +/// username, with every attribute. Restricting the query to the configured +/// voter group prevents service accounts and administrators in the same +/// realm from being emitted as Sequent-only voters. `after_username` is the +/// last username of the previous page (`None` for the first page); an empty +/// result means the scan is done. +/// +/// Kept as a direct query against Keycloak's own Postgres rather than the +/// Admin REST API (`GET /admin/realms/{realm}/users`, see +/// `sequent_core::services::keycloak::admin_client`): that endpoint only +/// offers offset-based (`first`/`max`) pagination, which is not +/// concurrency-safe — a user created or deleted elsewhere in the realm while +/// a scan is in progress shifts every later offset, silently skipping or +/// duplicating voters. The keyset scan here (`username > $2`) doesn't have +/// that failure mode, which reconciliation (needing to see every voter +/// exactly once) depends on. +#[instrument(skip(keycloak_transaction, areas_by_id), err)] +pub async fn fetch_realm_voter_snapshots_page( + keycloak_transaction: &Transaction<'_>, + realm: &str, + voter_group_name: &str, + after_username: Option<&str>, + areas_by_id: &HashMap, +) -> Result> { + let statement = keycloak_transaction + .prepare( + r#" + SELECT + u.id AS voter_id, + u.username, + u.enabled, + json_object_agg(ua.name, ua.value) FILTER (WHERE ua.name IS NOT NULL) AS attributes + FROM user_entity u + INNER JOIN realm AS ra ON ra.id = u.realm_id + INNER JOIN user_group_membership ugm ON ugm.user_id = u.id + INNER JOIN keycloak_group kg ON kg.id = ugm.group_id AND kg.realm_id = u.realm_id + LEFT JOIN user_attribute ua ON ua.user_id = u.id + WHERE ra.name = $1 + AND kg.name = $2 + AND u.username > $3 + GROUP BY u.id, u.username, u.enabled + ORDER BY u.username + LIMIT $4 + "#, + ) + .await?; + + let rows: Vec = keycloak_transaction + .query( + &statement, + &[ + &realm, + &voter_group_name, + &after_username.unwrap_or(""), + &VOTER_SNAPSHOT_PAGE_SIZE, + ], + ) + .await?; + + Ok(rows + .into_iter() + .filter_map(|row| voter_snapshot_row_to_snapshot(row, areas_by_id)) + .collect()) +} + +/// Batch-fetches voter-group snapshots for exactly the usernames named in +/// `usernames` (typically one reconciliation-file batch's worth of +/// `VoterID`s), in a single round trip via `= ANY($2)` — the file-driven +/// counterpart to `fetch_realm_voter_snapshots_page`'s Sequent-driven +/// pagination, so reconciliation never needs the whole file's rows resident +/// in memory to look voters up. A username with no matching row simply +/// isn't present in the result; the caller treats that as "this file row's +/// voter doesn't exist in Sequent" (D, forward direction). +#[instrument(skip(keycloak_transaction, areas_by_id, usernames), err)] +pub async fn fetch_realm_voter_snapshots_by_usernames( + keycloak_transaction: &Transaction<'_>, + realm: &str, + voter_group_name: &str, + usernames: &[String], + areas_by_id: &HashMap, +) -> Result> { + if usernames.is_empty() { + return Ok(Vec::new()); + } + + let statement = keycloak_transaction + .prepare( + r#" + SELECT + u.id AS voter_id, + u.username, + u.enabled, + json_object_agg(ua.name, ua.value) FILTER (WHERE ua.name IS NOT NULL) AS attributes + FROM user_entity u + INNER JOIN realm AS ra ON ra.id = u.realm_id + INNER JOIN user_group_membership ugm ON ugm.user_id = u.id + INNER JOIN keycloak_group kg ON kg.id = ugm.group_id AND kg.realm_id = u.realm_id + LEFT JOIN user_attribute ua ON ua.user_id = u.id + WHERE ra.name = $1 + AND kg.name = $2 + AND u.username = ANY($3) + GROUP BY u.id, u.username, u.enabled + "#, + ) + .await?; + + let rows: Vec = keycloak_transaction + .query(&statement, &[&realm, &voter_group_name, &usernames]) + .await?; + + Ok(rows + .into_iter() + .filter_map(|row| voter_snapshot_row_to_snapshot(row, areas_by_id)) + .collect()) +} + +#[instrument(skip(row, areas_by_id))] +fn voter_snapshot_row_to_snapshot( + row: Row, + areas_by_id: &HashMap, +) -> Option { + let voter_id: String = row.get("voter_id"); + let voter_id = Uuid::parse_str(&voter_id).ok()?; + let username: String = row.get("username"); + let enabled: bool = row.get("enabled"); + let attributes: Option = row.get("attributes"); + let attributes = attributes.unwrap_or(serde_json::Value::Null); + + let attr = |key: &str| -> Option { + attributes + .get(key) + .and_then(|value| value.as_str()) + .map(str::to_string) + }; + + let area_id = attr(AREA_ID_ATTR_NAME); + let area_name = area_id.and_then(|id| areas_by_id.get(&id).cloned()); + + Some(VoterSnapshot { + username, + voter_id, + enabled, + area_name, + dob: attr(DATE_OF_BIRTH), + voted_channel: attr(VOTED_CHANNEL), + has_valid_internet_vote: false, // filled in by reconciliation's event-wide vote-state query + has_unresolved_internet_vote: false, // filled in by reconciliation's event-wide vote-state query + disable_comment: attr(DISABLE_COMMENT), + }) +} + #[instrument(skip(hasura_transaction), err)] async fn get_area_ids( hasura_transaction: &Transaction<'_>, diff --git a/packages/windmill/src/tasks/apply_reconciliation_patch.rs b/packages/windmill/src/tasks/apply_reconciliation_patch.rs new file mode 100644 index 00000000000..d6ebc9b9e92 --- /dev/null +++ b/packages/windmill/src/tasks/apply_reconciliation_patch.rs @@ -0,0 +1,691 @@ +// SPDX-FileCopyrightText: 2026 Sequent Tech Inc +// +// SPDX-License-Identifier: AGPL-3.0-only + +//! Applies the `target = Sequent` diff of an already-computed reconciliation +//! round — the Sequent-patch document `generate_reconciliation_patches` +//! produced, not recomputed (spec: "calculated in the first diff, not +//! recalculated"). Per-voter atomic; failures are collected and reported at +//! the end rather than aborting (spec, "Implementation Requirements"). There +//! is no `datafix_reconciliation_import` row to mutate, and no downloadable +//! row-failures document either — the outcome is reported as a bounded, +//! structured task annotation and a single electoral log entry, not written +//! back onto anything else. + +use crate::postgres::document::get_document; +use crate::postgres::election_event::{get_election_event_by_id, ElectionEventDatafix}; +use crate::services::consolidation::eml_generator::ValidateAnnotations; +use crate::services::database::{get_hasura_pool, get_keycloak_pool}; +use crate::services::documents::get_document_as_temp_file; +use crate::services::electoral_log::ElectoralLog; +use crate::services::external::reconciliation::apply::{apply_voter_changes, VoterApplyOutcome}; +use crate::services::external::reconciliation::bulk_create::apply_voters_added_bulk; +use crate::services::external::reconciliation::diff::{DiffItem, ReconciliationApplyEnvelope}; +use crate::services::external::reconciliation::patch::DiffItemArrayWriter; +use crate::services::external::types::{ReconciliationChangeCategory, ReconciliationPatchSource}; +use crate::services::external::utils::set_datafix_reconciliation_state; +use crate::services::protocol_manager::get_event_board; +use crate::services::serialize_tasks_logs::append_general_log; +use crate::services::tasks_execution::{update_fail, update_with_annotations}; +use crate::types::error::{Error, Result}; +use celery::error::TaskError; +use electoral_log::messages::newtypes::ExternalReconciliationKind; +use sequent_core::services::keycloak::get_event_realm; +use sequent_core::types::hasura::core::TasksExecution; +use sequent_core::types::hasura::extra::TasksExecutionStatus; +use serde::{Deserialize, Serialize}; +use std::collections::{HashMap, HashSet}; +use std::fs::File; +use std::io::{BufReader, BufWriter, Write}; +use tracing::{info, instrument}; + +const VOTER_ADD_APPLY_BATCH_SIZE: usize = 5_000; +const MAX_RECONCILIATION_ROW_FAILURE_DETAILS: usize = 1_000; + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct ApplyReconciliationPatchBody { + pub tenant_id: String, + pub election_event_id: String, + /// Which external system produced the round being applied — gates the + /// source-specific bookkeeping below (e.g. Datafix's own `Sequence` + /// tracking); the per-voter Sequent apply itself doesn't care. + pub source: ReconciliationPatchSource, + /// The `ReconciliationDiff` envelope document from the generate round + /// being applied — re-fetched and re-parsed here, never trusted from the + /// client, for the safety checks below (staleness, external side clean). + pub diff_document_id: String, + pub applied_by_user_id: String, + pub applied_by_username: Option, +} + +#[derive(Serialize)] +struct ReconciliationTaskRowFailure { + voter_id: String, + reason: String, +} + +#[derive(Serialize)] +struct ApplyReconciliationTaskAnnotations { + /// Reserved for task-result document compatibility. Row-failure details + /// are currently carried as the bounded sample below. + document_id: Option, + reconciliation_row_failure_count: usize, + reconciliation_row_failures_truncated: bool, + reconciliation_row_failures: Vec, +} + +/// Bounded task-facing failure summary. Retry eligibility and operator-facing +/// totals use `total_count`; only the first N details are retained for the +/// task annotation and browser table. +#[derive(Debug, Default)] +struct RowFailureSummary { + total_count: usize, + details: Vec<(String, String)>, +} + +impl RowFailureSummary { + fn record(&mut self, voter_id: String, reason: String) { + self.total_count += 1; + if self.details.len() < MAX_RECONCILIATION_ROW_FAILURE_DETAILS { + self.details.push((voter_id, reason)); + } + } + + fn extend(&mut self, failures: impl IntoIterator) { + for (voter_id, reason) in failures { + self.record(voter_id, reason); + } + } + + fn has_failures(&self) -> bool { + self.total_count > 0 + } + + fn is_truncated(&self) -> bool { + self.total_count > self.details.len() + } +} + +/// Enforces the NDJSON contract while retaining only the completed voter ids. +/// A repeated non-contiguous voter is a malformed apply artifact, not another +/// independent row, because its old-value validation would observe mutations +/// made by the first group. +#[derive(Debug, Default)] +struct VoterGroupTracker { + current: Option, + completed: HashSet, +} + +impl VoterGroupTracker { + fn switch_to(&mut self, voter_id: &str) -> std::result::Result, String> { + if self.current.as_deref() == Some(voter_id) { + return Ok(None); + } + if self.completed.contains(voter_id) { + return Err(format!( + "Invalid Sequent apply stream: voter {voter_id} appears in more than one non-contiguous group" + )); + } + + let previous = self.current.replace(voter_id.to_string()); + if let Some(completed) = previous.as_ref() { + self.completed.insert(completed.clone()); + } + Ok(previous) + } + + fn finish(self) -> Option { + self.current + } +} + +/// Applies every `target = Sequent` item of the round `diff_document_id` +/// points to, one voter at a time, per-row atomic (spec: "a row failure does +/// not abort the process" — every voter is still attempted). Row failures are +/// reported as structured task annotations and rendered by the wizard. A +/// completed per-row apply is `SUCCESS` even when some rows were safely +/// rejected; `FAILED` is reserved for an infrastructure/orchestration error. +/// Same-Sequence retry eligibility is tracked independently on the election +/// event whenever the complete row-failure count is non-zero. +#[instrument( + skip_all, + fields( + tenant_id = %body.tenant_id, + election_event_id = %body.election_event_id, + diff_document_id = %body.diff_document_id + ), + err +)] +#[wrap_map_err::wrap_map_err(TaskError)] +#[celery::task(max_retries = 0)] +pub async fn apply_reconciliation_patch( + body: ApplyReconciliationPatchBody, + task_execution: TasksExecution, +) -> Result<()> { + match run_apply_reconciliation_patch(&body).await { + Ok((applied_count, row_failures)) => { + info!( + "Reconciliation apply completed: {applied_count} row(s) applied, {} row failure(s)", + row_failures.total_count + ); + + let mut logs = task_execution.logs.clone(); + logs = append_task_log(&logs, &format!("Applied {applied_count} row(s).")); + let closing_message = if !row_failures.has_failures() { + "Task completed successfully".to_string() + } else { + format!( + "Task completed with {} row failure(s). See the reconciliation result.", + row_failures.total_count + ) + }; + logs = append_task_log(&logs, &closing_message); + + let failures_truncated = row_failures.is_truncated(); + let failure_count = row_failures.total_count; + let annotations = serde_json::to_value(ApplyReconciliationTaskAnnotations { + document_id: None, + reconciliation_row_failure_count: failure_count, + reconciliation_row_failures_truncated: failures_truncated, + reconciliation_row_failures: row_failures + .details + .into_iter() + .map(|(voter_id, reason)| ReconciliationTaskRowFailure { voter_id, reason }) + .collect(), + }) + .map_err(|err| { + Error::String(format!( + "Error serializing reconciliation task result: {err}" + )) + })?; + update_with_annotations( + &task_execution.tenant_id, + &task_execution.id, + TasksExecutionStatus::SUCCESS, + logs.unwrap_or_else(|| serde_json::Value::Array(vec![])), + annotations, + ) + .await + .map_err(|err| { + Error::String(format!("Error storing reconciliation task result: {err}")) + })?; + Ok(()) + } + Err(message) => { + update_fail(&task_execution, &message).await.ok(); + Err(Error::String(message)) + } + } +} + +/// Appends one log line to `current_logs`, same shape `update_complete`/ +/// `update_fail` write — but callable multiple times before a single terminal +/// `update`, since this task has more than one line to add (the applied +/// count and a final result summary) instead of the single summary message +/// those two helpers are built for. +fn append_task_log( + current_logs: &Option, + message: &str, +) -> Option { + serde_json::to_value(append_general_log(current_logs, message)).ok() +} + +#[instrument(skip(body), err)] +async fn run_apply_reconciliation_patch( + body: &ApplyReconciliationPatchBody, +) -> std::result::Result<(usize, RowFailureSummary), String> { + let mut hasura_client = get_hasura_pool() + .await + .get() + .await + .map_err(|err| format!("Error getting Hasura client: {err}"))?; + let hasura_transaction = hasura_client + .transaction() + .await + .map_err(|err| format!("Error starting Hasura transaction: {err}"))?; + + let envelope: ReconciliationApplyEnvelope = fetch_json_document( + &hasura_transaction, + &body.tenant_id, + &body.election_event_id, + &body.diff_document_id, + ) + .await + .map_err(|err| format!("Error loading the reconciliation diff: {err:?}"))?; + + if envelope.external_patch_document_id.is_some() { + return Err( + "The external-side diff is not empty — apply the external patch and re-import first" + .to_string(), + ); + } + + // Source-specific bookkeeping: today only Datafix exists, tracking its + // own last-applied `Sequence` per event annotation — a future + // non-Datafix source would keep its own independent tracking here + // instead, under its own arm, without touching the generic apply below. + if let ReconciliationPatchSource::Datafix { .. } = &body.source { + let election_event = get_election_event_by_id( + &hasura_transaction, + &body.tenant_id, + &body.election_event_id, + ) + .await + .map_err(|err| format!("Error loading election event: {err:?}"))?; + let datafix_annotations = ElectionEventDatafix(election_event) + .get_annotations() + .map_err(|err| format!("Error reading Datafix configuration: {err}"))?; + + // Apply-time Sequence check: reject only if this round is stale + // relative to the current one. Unlike the earlier table-backed design, + // there's no "already fully applied at this same Sequence" row to detect + // a bare retry against — re-running apply against the same + // diff_document_id simply re-applies the same Sequent-side items, which + // is safe (the underlying per-voter operations are themselves + // idempotent) but will post a second electoral log entry. Acceptable: + // the frontend only calls apply when there's something outstanding to + // apply in the first place, so an accidental retry here is a rare + // double-click, not a normal path. + match datafix_annotations.last_applied_sequence { + Some(last_applied) if envelope.sequence < last_applied => { + return Err(format!( + "Reconciliation round Sequence {} is older than the current round ({last_applied})", + envelope.sequence + )); + } + Some(last_applied) + if envelope.sequence == last_applied + && !datafix_annotations.last_apply_had_failures => + { + return Err(format!( + "Reconciliation round Sequence {} was already applied successfully; this envelope is diff-only", + envelope.sequence + )); + } + _ => {} + } + } + + if !envelope.apply_allowed { + return Err(format!( + "Reconciliation round Sequence {} is a diff-only convergence check and cannot be applied", + envelope.sequence + )); + } + + let realm = get_event_realm(&body.tenant_id, &body.election_event_id); + let patch_document = get_document( + &hasura_transaction, + &body.tenant_id, + Some(body.election_event_id.clone()), + &envelope.sequent_patch_document_id, + ) + .await + .map_err(|err| format!("Error loading the Sequent apply stream: {err:?}"))? + .ok_or_else(|| { + format!( + "Sequent apply stream document {} not found", + envelope.sequent_patch_document_id + ) + })?; + let patch_temp = get_document_as_temp_file(&body.tenant_id, &patch_document) + .await + .map_err(|err| format!("Error downloading the Sequent apply stream: {err:?}"))?; + + let mut keycloak_client = get_keycloak_pool() + .await + .get() + .await + .map_err(|err| format!("Error getting Keycloak client: {err}"))?; + let keycloak_transaction = keycloak_client + .transaction() + .await + .map_err(|err| format!("Error starting Keycloak transaction: {err}"))?; + + let voter_group_name = std::env::var("KEYCLOAK_VOTER_GROUP_NAME") + .map_err(|err| format!("Error getting env var KEYCLOAK_VOTER_GROUP_NAME: {err:?}"))?; + let mut row_failures = RowFailureSummary::default(); + let mut applied_voters_count: usize = 0; + let mut pending_voters_added: HashMap> = HashMap::new(); + + // Applied old/new values are also streamed to disk. The electoral-log + // API ultimately needs one byte artifact, but no second Vec is + // retained while the apply itself runs. + let audit_temp = tempfile::NamedTempFile::new() + .map_err(|err| format!("Error creating reconciliation audit artifact: {err}"))?; + let audit_file = audit_temp + .reopen() + .map_err(|err| format!("Error opening reconciliation audit artifact: {err}"))?; + let mut audit_writer = DiffItemArrayWriter::start(BufWriter::new(audit_file)) + .map_err(|err| format!("Error starting reconciliation audit artifact: {err}"))?; + + // Consume one contiguous voter group at a time. `VoterGroupTracker` + // rejects any voter that reappears after its first group, making the + // generator/apply ordering contract self-enforcing. + let patch_file = File::open(patch_temp.path()) + .map_err(|err| format!("Error opening the Sequent apply stream: {err}"))?; + let stream = + serde_json::Deserializer::from_reader(BufReader::new(patch_file)).into_iter::(); + let mut voter_groups = VoterGroupTracker::default(); + let mut current_items: Vec = Vec::new(); + for item in stream { + let item = item.map_err(|err| format!("Invalid Sequent apply stream item: {err}"))?; + if let Some(completed_voter) = voter_groups.switch_to(&item.voter_username)? { + process_voter_group( + &hasura_transaction, + &keycloak_transaction, + body, + &realm, + &voter_group_name, + completed_voter, + std::mem::take(&mut current_items), + &mut pending_voters_added, + &mut audit_writer, + &mut applied_voters_count, + &mut row_failures, + ) + .await?; + } + current_items.push(item); + } + if let Some(voter_username) = voter_groups.finish() { + process_voter_group( + &hasura_transaction, + &keycloak_transaction, + body, + &realm, + &voter_group_name, + voter_username, + current_items, + &mut pending_voters_added, + &mut audit_writer, + &mut applied_voters_count, + &mut row_failures, + ) + .await?; + } + flush_voters_added( + &hasura_transaction, + &keycloak_transaction, + body, + &realm, + &voter_group_name, + &mut pending_voters_added, + &mut audit_writer, + &mut applied_voters_count, + &mut row_failures, + ) + .await?; + + // Electoral log: every apply attempt gets a run-level entry, including a + // run where every row failed. The artifact contains only old/new items + // that were actually applied. + let mut audit_file = audit_writer + .finish() + .map_err(|err| format!("Error finishing reconciliation audit artifact: {err}"))?; + audit_file + .flush() + .map_err(|err| format!("Error flushing reconciliation audit artifact: {err}"))?; + let artifact = std::fs::read(audit_temp.path()) + .map_err(|err| format!("Error reading reconciliation audit artifact: {err}"))?; + let slug = std::env::var("ENV_SLUG").map_err(|err| format!("Missing ENV_SLUG: {err}"))?; + let board_name = get_event_board(&body.tenant_id, &body.election_event_id, &slug); + let electoral_log = ElectoralLog::new( + &hasura_transaction, + &body.tenant_id, + Some(&body.election_event_id), + &board_name, + ) + .await + .map_err(|err| format!("Error initializing reconciliation electoral log: {err:?}"))?; + electoral_log + .post_external_reconciliation( + body.election_event_id.clone(), + ExternalReconciliationKind::ChangesApplied, + envelope.sequence, + envelope.generated_at, + envelope.source_sha256.clone(), + None, + Some(artifact), + Some(body.applied_by_user_id.clone()), + body.applied_by_username.clone(), + ) + .await + .map_err(|err| format!("Error storing reconciliation electoral log: {err:?}"))?; + + if let ReconciliationPatchSource::Datafix { .. } = &body.source { + set_datafix_reconciliation_state( + &hasura_transaction, + &body.tenant_id, + &body.election_event_id, + envelope.sequence, + row_failures.has_failures(), + ) + .await + .map_err(|err| format!("Error storing Datafix reconciliation state: {err:?}"))?; + } + + keycloak_transaction + .commit() + .await + .map_err(|err| format!("Error committing Keycloak transaction: {err}"))?; + + hasura_transaction + .commit() + .await + .map_err(|err| format!("Error committing transaction: {err}"))?; + + Ok((applied_voters_count, row_failures)) +} + +#[allow(clippy::too_many_arguments)] +async fn process_voter_group( + hasura_transaction: &deadpool_postgres::Transaction<'_>, + keycloak_transaction: &deadpool_postgres::Transaction<'_>, + body: &ApplyReconciliationPatchBody, + realm: &str, + voter_group_name: &str, + voter_username: String, + voter_items: Vec, + pending_voters_added: &mut HashMap>, + audit_writer: &mut DiffItemArrayWriter, + applied_voters_count: &mut usize, + row_failures: &mut RowFailureSummary, +) -> std::result::Result<(), String> { + let generated_failures: Vec = voter_items + .iter() + .filter(|item| item.category == ReconciliationChangeCategory::ROW_FAILURE) + .map(|item| { + item.failure_reason + .clone() + .unwrap_or_else(|| "Row was excluded while generating the diff".to_string()) + }) + .collect(); + if !generated_failures.is_empty() { + row_failures.extend( + generated_failures + .into_iter() + .map(|reason| (voter_username.clone(), reason)), + ); + return Ok(()); + } + + if voter_items + .iter() + .all(|item| item.category == ReconciliationChangeCategory::VOTER_ADDED) + { + pending_voters_added.insert(voter_username, voter_items); + if pending_voters_added.len() >= VOTER_ADD_APPLY_BATCH_SIZE { + flush_voters_added( + hasura_transaction, + keycloak_transaction, + body, + realm, + voter_group_name, + pending_voters_added, + audit_writer, + applied_voters_count, + row_failures, + ) + .await?; + } + return Ok(()); + } + + match apply_voter_changes( + hasura_transaction, + &body.tenant_id, + &body.election_event_id, + realm, + &voter_username, + &voter_items, + ) + .await + { + Ok(VoterApplyOutcome::Applied) => { + *applied_voters_count += 1; + audit_writer + .write_batch(voter_items.iter()) + .map_err(|err| format!("Error writing reconciliation audit artifact: {err}"))?; + } + Ok(VoterApplyOutcome::Failed { reason }) => { + row_failures.record(voter_username, reason); + } + Err(err) => row_failures.record(voter_username, format!("{err:?}")), + } + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +async fn flush_voters_added( + hasura_transaction: &deadpool_postgres::Transaction<'_>, + keycloak_transaction: &deadpool_postgres::Transaction<'_>, + body: &ApplyReconciliationPatchBody, + realm: &str, + voter_group_name: &str, + pending_voters_added: &mut HashMap>, + audit_writer: &mut DiffItemArrayWriter, + applied_voters_count: &mut usize, + row_failures: &mut RowFailureSummary, +) -> std::result::Result<(), String> { + if pending_voters_added.is_empty() { + return Ok(()); + } + let voters_added = std::mem::take(pending_voters_added); + let voters_added_count = voters_added.len(); + let (bulk_applied, bulk_failures) = apply_voters_added_bulk( + hasura_transaction, + keycloak_transaction, + &body.tenant_id, + &body.election_event_id, + realm, + voter_group_name, + &voters_added, + ) + .await + .map_err(|err| format!("Error bulk-creating added voters: {err:?}"))?; + *applied_voters_count += voters_added_count - bulk_failures.len(); + audit_writer + .write_batch(bulk_applied.iter()) + .map_err(|err| format!("Error writing reconciliation audit artifact: {err}"))?; + row_failures.extend(bulk_failures); + Ok(()) +} + +/// Downloads a `Document` and deserializes its content as JSON — shared by +/// the diff-envelope and Sequent-patch reads above. +#[instrument(skip(hasura_transaction), err)] +async fn fetch_json_document( + hasura_transaction: &deadpool_postgres::Transaction<'_>, + tenant_id: &str, + election_event_id: &str, + document_id: &str, +) -> anyhow::Result { + let document = get_document( + hasura_transaction, + tenant_id, + Some(election_event_id.to_string()), + document_id, + ) + .await? + .ok_or_else(|| anyhow::anyhow!("Document {document_id} not found"))?; + let temp_file = get_document_as_temp_file(tenant_id, &document).await?; + let file = File::open(temp_file.path())?; + Ok(serde_json::from_reader(BufReader::new(file))?) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn apply_metadata_ignores_the_large_review_item_array() { + let json = r#"{ + "items":[{"arbitrary":"review-only"}], + "sequence":7, + "generated_at":123, + "source_sha256":"abc", + "external_patch_document_id":null, + "sequent_patch_document_id":"stream-id", + "apply_allowed":true + }"#; + let envelope: ReconciliationApplyEnvelope = serde_json::from_str(json).unwrap(); + assert_eq!(envelope.sequence, 7); + assert_eq!(envelope.sequent_patch_document_id, "stream-id"); + } + + #[test] + fn task_annotations_expose_version_stable_structured_row_failures() { + let annotations = serde_json::to_value(ApplyReconciliationTaskAnnotations { + document_id: None, + reconciliation_row_failure_count: 1, + reconciliation_row_failures_truncated: false, + reconciliation_row_failures: vec![ReconciliationTaskRowFailure { + voter_id: "voter-1".to_string(), + reason: "stale snapshot".to_string(), + }], + }) + .unwrap(); + + assert_eq!( + annotations["reconciliation_row_failures"][0]["voter_id"], + "voter-1" + ); + assert_eq!( + annotations["reconciliation_row_failures"][0]["reason"], + "stale snapshot" + ); + assert_eq!(annotations["reconciliation_row_failure_count"], 1); + assert_eq!(annotations["reconciliation_row_failures_truncated"], false); + } + + #[test] + fn row_failure_summary_caps_details_without_losing_the_total() { + let mut summary = RowFailureSummary::default(); + for index in 0..MAX_RECONCILIATION_ROW_FAILURE_DETAILS + 2 { + summary.record(format!("voter-{index}"), "failed".to_string()); + } + + assert_eq!( + summary.total_count, + MAX_RECONCILIATION_ROW_FAILURE_DETAILS + 2 + ); + assert_eq!( + summary.details.len(), + MAX_RECONCILIATION_ROW_FAILURE_DETAILS + ); + assert!(summary.is_truncated()); + } + + #[test] + fn voter_group_tracker_rejects_a_non_contiguous_repeat() { + let mut tracker = VoterGroupTracker::default(); + assert_eq!(tracker.switch_to("voter-1").unwrap(), None); + assert_eq!(tracker.switch_to("voter-1").unwrap(), None); + assert_eq!( + tracker.switch_to("voter-2").unwrap(), + Some("voter-1".to_string()) + ); + assert!(tracker.switch_to("voter-1").is_err()); + } +} diff --git a/packages/windmill/src/tasks/edit_user.rs b/packages/windmill/src/tasks/edit_user.rs index e41786ce3fa..1877e7370da 100644 --- a/packages/windmill/src/tasks/edit_user.rs +++ b/packages/windmill/src/tasks/edit_user.rs @@ -7,13 +7,15 @@ use crate::postgres::cast_vote::{ }; use crate::postgres::election_event::{get_election_event_by_id, ElectionEventDatafix}; use crate::services::database::get_hasura_pool; -use crate::services::datafix; -use crate::services::datafix::types::{SoapRequest, SoapRequestResponse, SoapRequestResult}; -use crate::services::datafix::utils::{ - datafix_voter_lock_key, post_operation_result_to_electoral_log, voted_via_internet, +use crate::services::external; +use crate::services::external::datafix_types::{ + SoapRequest, SoapRequestResponse, SoapRequestResult, +}; +use crate::services::external::utils::{ + external_voter_lock_key, post_operation_result_to_electoral_log, voted_via_internet, voted_via_not_internet_channel, DATAFIX_VOTER_LOCK_SECS, }; -use crate::services::datafix::voterview_requests::SoapSendError; +use crate::services::external::voterview_requests::SoapSendError; use crate::services::pg_lock::PgLock; use crate::services::tasks_execution::{update_complete, update_fail}; use crate::types::error::{Error, Result}; @@ -26,7 +28,9 @@ use sequent_core::services::date::ISO8601; use sequent_core::services::keycloak::{get_event_realm, KeycloakAdminClient}; use sequent_core::services::uuid_validation::parse_uuid_v4; use sequent_core::types::hasura::core::{ElectionEvent, TasksExecution}; -use sequent_core::types::keycloak::{User, VOTED_CHANNEL, VOTED_CHANNEL_INTERNET_VALUE}; +use sequent_core::types::keycloak::{ + User, ATTR_RESET_VALUE, VOTED_CHANNEL, VOTED_CHANNEL_INTERNET_VALUE, +}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use tracing::{error, info, instrument}; @@ -65,10 +69,10 @@ pub struct EditUserOutput { /// the (potentially slow, retried) VoterView round-trip. The release logic runs /// in [`apply_datafix_voter_edit`]; its outcome is recorded on `task_execution`, /// which backs the operator's task widget. A Datafix voter's ballots are -/// discarded after the Keycloak disable. A retry also resumes a partial -/// disable when Keycloak is already disabled but active ballots remain. -/// Whether `SetNotVoted` converges is still handled by the separate manual -/// reconciliation process. +/// discarded and its voted-channel attribute reset after the Keycloak +/// disable. A retry also resumes a partial disable when Keycloak is already +/// disabled but active ballots remain. Whether `SetNotVoted` converges is +/// still handled by the separate manual reconciliation process. #[instrument( skip_all, fields( @@ -150,7 +154,6 @@ fn plan_voter_release( if reenable_transition && (cast_vote_state.has_unresolved_vote || cast_vote_state.has_valid_vote - || voted_via_internet(current_attributes) || voted_via_not_internet_channel(current_attributes)) { return Err( @@ -260,6 +263,39 @@ async fn discard_voter_ballots( Ok(()) } +/// Resets `VOTED_CHANNEL` back to `NONE` after a release discards the voter's +/// ballots, mirroring the reset `unmark_voter_as_voted` already does for the +/// inbound `/unmark-voted` call. Without this the attribute — set once, when a +/// vote first resolves to `Valid`, and otherwise never touched — stays stale +/// after the ballot it described is gone, wrongly blocking a later re-enable +/// and feeding a stale channel into the reconciliation patch for a voter +/// Datafix has no record of. Only ever runs after `plan_voter_release` has +/// already confirmed the voter isn't recorded as voted through another +/// channel, so this can only be clearing a stale `INTERNET` value or a no-op. +#[instrument(skip(ctx))] +async fn clear_voted_channel(ctx: &DatafixEditCtx<'_>) -> anyhow::Result<()> { + let client = KeycloakAdminClient::new().await?; + let attributes = HashMap::from([( + VOTED_CHANNEL.to_string(), + vec![ATTR_RESET_VALUE.to_string()], + )]); + client + .edit_user( + &ctx.realm, + &ctx.body.user_id, + None, + Some(attributes), + None, + None, + None, + None, + None, + None, + ) + .await + .map(|_| ()) +} + /// Records the outcome of a disabled-voter release in the electoral log. /// Failures are logged and swallowed so auditing never fails the user edit. #[instrument(skip(ctx))] @@ -309,7 +345,7 @@ async fn send_set_not_voted( election_event: ElectionEvent, username: &str, ) { - let prepared = match datafix::voterview_requests::prepare( + let prepared = match external::voterview_requests::prepare( SoapRequest::SetNotVoted, ElectionEventDatafix(election_event), &Some(username.to_string()), @@ -329,7 +365,7 @@ async fn send_set_not_voted( } }; let template_sha256 = prepared.template_sha256().to_string(); - let operation = match datafix::voterview_requests::send_prepared(prepared).await { + let operation = match external::voterview_requests::send_prepared(prepared).await { Ok(SoapRequestResult { response, template_sha256, @@ -415,6 +451,9 @@ async fn run_datafix_voter_edit( discard_voter_ballots(&body.tenant_id, &body.election_event_id, &body.user_id) .await .map_err(|err| format!("Error discarding Datafix cast votes: {err:?}"))?; + clear_voted_channel(ctx).await.map_err(|err| { + format!("Could not reset the voter's voted-channel attribute after discard: {err}") + })?; if !plan.owes_set_not_voted { return Ok(()); @@ -440,8 +479,10 @@ async fn apply_datafix_voter_edit(body: &EditUserTaskBody) -> std::result::Resul .await .map_err(|err| format!("Error loading election event: {err:?}"))?; + let user_id_uuid = + parse_uuid_v4(&body.user_id).map_err(|err| format!("Invalid voter id: {err}"))?; let lock = PgLock::acquire( - datafix_voter_lock_key(&body.tenant_id, &body.election_event_id, &body.user_id), + external_voter_lock_key(&body.tenant_id, &body.election_event_id, &user_id_uuid), Uuid::new_v4().to_string(), ISO8601::now() + Duration::seconds(DATAFIX_VOTER_LOCK_SECS), ) @@ -567,10 +608,25 @@ mod tests { } #[test] - fn reenabling_is_refused_while_marked_voted_via_internet() { + fn reenabling_a_voter_with_only_discarded_internet_ballots_is_allowed() { + // The voted-channel attribute is never cleared by a discard, so once a + // voter has ever cast an internet ballot it stays "Internet" forever — + // re-enable must key off the live `VoterCastVoteState`, not this stale + // attribute, or a fully-resolved (discarded) voter could never be + // re-enabled. + let plan = plan_voter_release(Some(false), Some(true), &no_cast_votes(), &internet_voter()) + .unwrap(); + assert!(!plan.release_attempt); + } + + #[test] + fn reenabling_is_refused_while_marked_voted_via_another_channel() { + // Unlike an internet ballot, a non-internet channel has no + // corresponding `cast_vote` row — the attribute is the only record of + // it, and only Datafix's own `/unmark-voted` call may reverse it. + let attributes = HashMap::from([(VOTED_CHANNEL.to_string(), vec!["PAPER".to_string()])]); assert!( - plan_voter_release(Some(false), Some(true), &no_cast_votes(), &internet_voter()) - .is_err() + plan_voter_release(Some(false), Some(true), &no_cast_votes(), &attributes).is_err() ); } diff --git a/packages/windmill/src/tasks/generate_reconciliation_patches.rs b/packages/windmill/src/tasks/generate_reconciliation_patches.rs new file mode 100644 index 00000000000..83d34b42c98 --- /dev/null +++ b/packages/windmill/src/tasks/generate_reconciliation_patches.rs @@ -0,0 +1,828 @@ +// SPDX-FileCopyrightText: 2026 Sequent Tech Inc +// +// SPDX-License-Identifier: AGPL-3.0-only + +//! Parses an uploaded reconciliation file, computes both diffs at once +//! (spec: "Both diffs... are calculated at once"), and uploads three +//! documents: the Sequent-patch NDJSON stream `apply_reconciliation_patch` later +//! applies from, the downloadable Datafix patch CSV (only if that side is +//! non-empty), and a "diff envelope" JSON referencing both plus every item — +//! see `reconciliation::diff::ReconciliationDiff`. There is no +//! `datafix_reconciliation_import` table or row of any kind; the envelope +//! document *is* the record, and its id is the one thing recorded on +//! `task_execution.annotations.document_id`. Named `GENERATE_RECONCILIATION_PATCHES` +//! to match the `ETasksExecution` value already committed on the frontend, +//! even though it also computes the diff, not just the patch. +//! +//! Both the input (the uploaded file) and the output (the three documents +//! above) are handled in fixed-size batches rather than fully materialized +//! in memory: the file is read incrementally via +//! `reconciliation::csv::ReconciliationRowBatches`, each batch's matching +//! Sequent voters are fetched in one round trip via +//! `users::fetch_realm_voter_snapshots_by_usernames`, and each batch's +//! resulting `DiffItem`s are written straight into the three open output +//! files via `reconciliation::patch::DiffItemArrayWriter`, +//! `DiffItemNdjsonWriter`, and `ExternalPatchCsvWriter` — nothing here ever +//! holds the whole diff (or the whole file) resident in memory at once, which +//! a 100k+-row reconciliation run otherwise would. + +use crate::postgres::area::get_event_areas; +use crate::postgres::cast_vote::get_voter_cast_vote_states_for_event; +use crate::postgres::document::get_document; +use crate::postgres::election_event::{get_election_event_by_id, ElectionEventDatafix}; +use crate::services::consolidation::eml_generator::ValidateAnnotations; +use crate::services::database::{get_hasura_pool, get_keycloak_pool}; +use crate::services::documents::{get_document_as_temp_file, upload_and_return_document}; +use crate::services::electoral_log::ElectoralLog; +use crate::services::external::reconciliation::csv::{ + split_meta_and_csv, ReconciliationRowBatches, +}; +use crate::services::external::reconciliation::diff::{ + diff_file_row_batch, diff_unmatched_sequent_voters, index_datafix_area_fields, + DatafixAreaFieldsByName, DiffItem, +}; +use crate::services::external::reconciliation::patch::{ + is_sequent_apply_stream_item, sha256_hex, DiffItemArrayWriter, DiffItemNdjsonWriter, + ExternalPatchCsvWriter, +}; +use crate::services::external::types::ReconciliationPatchSource; +use crate::services::protocol_manager::get_event_board; +use crate::services::serialize_tasks_logs::append_general_log; +use crate::services::tally_sheet_import::hash::hash_bytes; +use crate::services::tasks_execution::{update, update_complete, update_fail}; +use crate::services::users::{ + fetch_realm_voter_snapshots_by_usernames, fetch_realm_voter_snapshots_page, VoterSnapshot, +}; +use crate::types::error::{Error, Result}; +use celery::error::TaskError; +use electoral_log::messages::newtypes::ExternalReconciliationKind; +use sequent_core::services::keycloak::get_event_realm; +use sequent_core::types::hasura::core::TasksExecution; +use sequent_core::types::hasura::extra::TasksExecutionStatus; +use serde::{Deserialize, Serialize}; +use std::collections::{HashMap, HashSet}; +use std::io::{BufWriter, Write}; +use tracing::instrument; + +/// Rows are read from the file and matched against Keycloak this many at a +/// time — matches `users::VOTER_SNAPSHOT_PAGE_SIZE`, so the file-driven +/// forward pass and the Sequent-driven reverse pass move the same amount of +/// data per round trip. +const RECONCILIATION_BATCH_SIZE: usize = 5_000; + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct GenerateReconciliationPatchesBody { + pub tenant_id: String, + pub election_event_id: String, + pub source_document_id: String, + pub requested_by_user_id: String, + pub requested_by_username: Option, +} + +/// Computes both reconciliation diffs for one uploaded file and uploads the +/// three documents described in the module doc. Nothing here mutates voter +/// data (that's `apply_reconciliation_patch`'s job). +#[instrument( + skip_all, + fields( + tenant_id = %body.tenant_id, + election_event_id = %body.election_event_id + ), + err +)] +#[wrap_map_err::wrap_map_err(TaskError)] +#[celery::task(max_retries = 0)] +pub async fn generate_reconciliation_patches( + body: GenerateReconciliationPatchesBody, + task_execution: TasksExecution, +) -> Result<()> { + let mut task_execution = task_execution; + match run_generate_reconciliation_patches(&body, &mut task_execution).await { + Ok(diff_document_id) => { + update_complete(&task_execution, Some(diff_document_id)) + .await + .ok(); + Ok(()) + } + Err(message) => { + update_fail(&task_execution, &message).await.ok(); + Err(Error::String(message)) + } + } +} + +#[instrument(skip(body, task_execution), err)] +async fn run_generate_reconciliation_patches( + body: &GenerateReconciliationPatchesBody, + task_execution: &mut TasksExecution, +) -> std::result::Result { + let mut hasura_client = get_hasura_pool() + .await + .get() + .await + .map_err(|err| format!("Error getting Hasura client: {err}"))?; + let hasura_transaction = hasura_client + .transaction() + .await + .map_err(|err| format!("Error starting Hasura transaction: {err}"))?; + + let document = get_document( + &hasura_transaction, + &body.tenant_id, + Some(body.election_event_id.clone()), + &body.source_document_id, + ) + .await + .map_err(|err| format!("Error loading uploaded document: {err:?}"))? + .ok_or_else(|| "Uploaded reconciliation file not found".to_string())?; + let temp_file = get_document_as_temp_file(&body.tenant_id, &document) + .await + .map_err(|err| format!("Error downloading uploaded reconciliation file: {err:?}"))?; + + // Hash the whole file once (so Datafix's own generated hash can later be + // compared against it manually) and read its `#META` line, then drop the + // bytes — the batch loop below re-reads the same temp file path + // incrementally instead of holding the whole file while processing the output. + let file_bytes = std::fs::read(temp_file.path()) + .map_err(|err| format!("Error reading uploaded file: {err}"))?; + let source_sha256 = hash_bytes(&file_bytes); + let (meta, _) = split_meta_and_csv(&file_bytes) + .map_err(|err| format!("Invalid reconciliation metadata: {err}"))?; + drop(file_bytes); + + let election_event = get_election_event_by_id( + &hasura_transaction, + &body.tenant_id, + &body.election_event_id, + ) + .await + .map_err(|err| format!("Error loading election event: {err:?}"))?; + let datafix_annotations = ElectionEventDatafix(election_event) + .get_annotations() + .map_err(|err| format!("Election event has no valid Datafix configuration: {err}"))?; + + let apply_allowed = apply_permission_for_sequence( + meta.sequence, + datafix_annotations.last_applied_sequence, + datafix_annotations.last_apply_had_failures, + )?; + checkpoint( + task_execution, + &format!("Sequence {} accepted; scanning the file.", meta.sequence), + ) + .await; + + let source = ReconciliationPatchSource::Datafix { + county_mun: datafix_annotations.voterview_request.county_mun.clone(), + }; + let realm = get_event_realm(&body.tenant_id, &body.election_event_id); + let voter_group_name = std::env::var("KEYCLOAK_VOTER_GROUP_NAME") + .map_err(|err| format!("Error getting env var KEYCLOAK_VOTER_GROUP_NAME: {err:?}"))?; + + let mut keycloak_client = get_keycloak_pool() + .await + .get() + .await + .map_err(|err| format!("Error getting Keycloak client: {err}"))?; + let keycloak_transaction = keycloak_client + .transaction() + .await + .map_err(|err| format!("Error starting Keycloak transaction: {err}"))?; + + let areas_by_id: HashMap = get_event_areas( + &hasura_transaction, + &body.tenant_id, + &body.election_event_id, + ) + .await + .map_err(|err| format!("Error loading event areas: {err:?}"))? + .into_iter() + .filter_map(|area| area.name.map(|name| (area.id, name))) + .collect(); + + let voter_cast_vote_states = get_voter_cast_vote_states_for_event( + &hasura_transaction, + &body.tenant_id, + &body.election_event_id, + ) + .await + .map_err(|err| format!("Error loading active voter ballot states: {err:?}"))?; + + // Three output documents, each written incrementally as batches are + // processed below, instead of serialized once from one fully-materialized + // diff. The internal Sequent stream is NDJSON so apply can deserialize it + // one voter at a time; the review envelope remains a JSON object/array. + let sequent_patch_temp = tempfile::NamedTempFile::new() + .map_err(|err| format!("Error creating the Sequent patch temp file: {err}"))?; + let mut sequent_patch_writer = DiffItemNdjsonWriter::new(BufWriter::new( + sequent_patch_temp + .reopen() + .map_err(|err| format!("Error reopening the Sequent patch temp file: {err}"))?, + )); + + let envelope_temp = tempfile::NamedTempFile::new() + .map_err(|err| format!("Error creating the diff envelope temp file: {err}"))?; + let mut envelope_writer = BufWriter::new( + envelope_temp + .reopen() + .map_err(|err| format!("Error reopening the diff envelope temp file: {err}"))?, + ); + // `items` is written first (before `sequence`/`external_patch_document_id`/ + // etc., unlike `ReconciliationDiff`'s own field order) precisely so it + // can be streamed before the fields that aren't known until the whole + // file has been scanned. Field order has no bearing on the server-side + // readers deserializing the `ReconciliationApplyEnvelope` projection by + // name; the frontend likewise reads the envelope's named properties. + envelope_writer + .write_all(b"{\"items\":") + .map_err(|err| format!("Error starting the diff envelope: {err}"))?; + let mut envelope_items_writer = DiffItemArrayWriter::start(envelope_writer) + .map_err(|err| format!("Error starting the diff envelope items: {err}"))?; + + let external_patch_temp = tempfile::NamedTempFile::new() + .map_err(|err| format!("Error creating the Datafix patch temp file: {err}"))?; + let mut external_patch_writer = ExternalPatchCsvWriter::start( + BufWriter::new( + external_patch_temp + .reopen() + .map_err(|err| format!("Error reopening the Datafix patch temp file: {err}"))?, + ), + meta.sequence, + meta.generated_at, + ) + .map_err(|err| format!("Error starting the Datafix patch: {err}"))?; + + // Forward pass: read the file in batches, batch-fetch the matching + // Sequent snapshots for exactly this batch's VoterIDs (one round trip + // per batch via `= ANY($usernames)`, not one per row and not the whole + // realm at once), classify, and stream each batch's items straight into + // the three writers above. `all_file_usernames` only keeps the usernames + // (not the full parsed rows) across batches, for the reverse pass below. + let mut file_reader = ReconciliationRowBatches::open(temp_file.path()) + .map_err(|err| format!("Error opening the reconciliation file for reading: {err}"))?; + let mut all_file_usernames: HashSet = HashSet::new(); + let mut area_fields_by_name = DatafixAreaFieldsByName::new(); + let mut total_rows: usize = 0; + + loop { + let file_rows = file_reader + .next_batch(RECONCILIATION_BATCH_SIZE) + .map_err(|err| { + if err.line == 0 { + format!( + "Reconciliation file has an invalid CSV header: {}", + err.message + ) + } else { + format!( + "Reconciliation file has a malformed row at line {}: {}", + err.line, err.message + ) + } + })?; + if file_rows.is_empty() { + break; + } + total_rows += file_rows.len(); + + let usernames: Vec = file_rows + .iter() + .map(|row| row.external_voter_id.clone()) + .collect(); + for username in &usernames { + if !all_file_usernames.insert(username.clone()) { + return Err(format!( + "Reconciliation file contains duplicate VoterID '{username}'" + )); + } + } + index_datafix_area_fields(&mut area_fields_by_name, &file_rows); + + let mut snapshots = fetch_realm_voter_snapshots_by_usernames( + &keycloak_transaction, + &realm, + &voter_group_name, + &usernames, + &areas_by_id, + ) + .await + .map_err(|err| format!("Error fetching voter snapshots for a file batch: {err:?}"))?; + for snapshot in snapshots.iter_mut() { + if let Some(state) = voter_cast_vote_states.get(&snapshot.voter_id.to_string()) { + snapshot.has_valid_internet_vote = state.has_valid_vote; + snapshot.has_unresolved_internet_vote = state.has_unresolved_vote; + } + } + let snapshots_by_username: HashMap = snapshots + .into_iter() + .map(|snapshot| (snapshot.username.clone(), snapshot)) + .collect(); + + let batch_items = diff_file_row_batch(&file_rows, &snapshots_by_username, &source); + write_batch_to_all_outputs( + &mut envelope_items_writer, + &mut sequent_patch_writer, + &mut external_patch_writer, + &batch_items, + &file_rows, + )?; + + checkpoint( + task_execution, + &format!("Processed {total_rows} row(s) so far."), + ) + .await; + } + + // Reverse pass: page through Sequent's own voters (unchanged pagination) + // to find enabled voters the file never mentioned in any batch — the + // `voter_missing_from_file` case, wired up here for the first time (see + // `diff::diff_unmatched_sequent_voters`'s doc). + let mut after_username: Option = None; + loop { + let mut page = fetch_realm_voter_snapshots_page( + &keycloak_transaction, + &realm, + &voter_group_name, + after_username.as_deref(), + &areas_by_id, + ) + .await + .map_err(|err| format!("Error fetching voter snapshot page: {err:?}"))?; + if page.is_empty() { + break; + } + for snapshot in page.iter_mut() { + if let Some(state) = voter_cast_vote_states.get(&snapshot.voter_id.to_string()) { + snapshot.has_valid_internet_vote = state.has_valid_vote; + snapshot.has_unresolved_internet_vote = state.has_unresolved_vote; + } + } + after_username = page.last().map(|snapshot| snapshot.username.clone()); + + let reverse_items = diff_unmatched_sequent_voters( + &page, + &all_file_usernames, + &source, + &area_fields_by_name, + ); + write_batch_to_all_outputs( + &mut envelope_items_writer, + &mut sequent_patch_writer, + &mut external_patch_writer, + &reverse_items, + &[], // no file rows exist for these voters; every field falls back to NONE + )?; + } + checkpoint(task_execution, "Computed the full diff.").await; + + // Finish and upload the Sequent patch (always produced, never + // downloadable, purely apply_reconciliation_patch's input). + let sequent_patch_writer_inner = sequent_patch_writer.finish(); + flush_writer(sequent_patch_writer_inner) + .map_err(|err| format!("Error finishing the Sequent patch: {err}"))?; + let sequent_patch_size = file_size(sequent_patch_temp.path()) + .map_err(|err| format!("Error sizing the Sequent patch: {err}"))?; + let sequent_patch_document_id = upload_document_from_temp_file( + &hasura_transaction, + &body.tenant_id, + &body.election_event_id, + &format!("sequent_patch_seq{}.ndjson", meta.sequence), + "application/x-ndjson", + sequent_patch_size, + sequent_patch_temp.path(), + ) + .await + .map_err(|err| format!("Error uploading the Sequent patch: {err:?}"))?; + checkpoint( + task_execution, + &format!("Uploaded the Sequent patch ({sequent_patch_size} bytes)."), + ) + .await; + + // Finish and upload the downloadable external (Datafix) patch CSV — + // only if non-empty. + let mut external_patch_document_id = None; + let mut external_patch_sha256 = None; + if let Some(external_patch_writer_inner) = external_patch_writer + .finish() + .map_err(|err| format!("Error finishing the Datafix patch: {err}"))? + { + flush_writer(external_patch_writer_inner) + .map_err(|err| format!("Error finishing the Datafix patch: {err}"))?; + let csv_size = file_size(external_patch_temp.path()) + .map_err(|err| format!("Error sizing the Datafix patch: {err}"))?; + let csv_bytes = std::fs::read(external_patch_temp.path()) + .map_err(|err| format!("Error reading back the Datafix patch for hashing: {err}"))?; + let hash = sha256_hex(&csv_bytes); + drop(csv_bytes); + let file_name = format!("datafix_patch_seq{}.csv", meta.sequence); + let uploaded_id = upload_document_from_temp_file( + &hasura_transaction, + &body.tenant_id, + &body.election_event_id, + &file_name, + "text/csv", + csv_size, + external_patch_temp.path(), + ) + .await + .map_err(|err| format!("Error uploading Datafix patch: {err:?}"))?; + external_patch_document_id = Some(uploaded_id); + external_patch_sha256 = Some(hash); + checkpoint( + task_execution, + &format!("Uploaded the Datafix patch ({csv_size} bytes)."), + ) + .await; + } + + // Finish the diff envelope: close the `items` array, then append the + // fields that could only be known once every batch (and the Datafix + // patch, if any) had been processed. + let envelope_writer_inner = envelope_items_writer + .finish() + .map_err(|err| format!("Error finishing the diff envelope: {err}"))?; + let mut envelope_writer_inner = envelope_writer_inner; + write_envelope_tail( + &mut envelope_writer_inner, + meta.sequence, + meta.generated_at, + &source_sha256, + external_patch_document_id.as_deref(), + external_patch_sha256.as_deref(), + &sequent_patch_document_id, + apply_allowed, + ) + .map_err(|err| format!("Error finishing the diff envelope: {err}"))?; + flush_writer(envelope_writer_inner) + .map_err(|err| format!("Error finishing the diff envelope: {err}"))?; + let envelope_size = file_size(envelope_temp.path()) + .map_err(|err| format!("Error sizing the diff envelope: {err}"))?; + let envelope_document_id = upload_document_from_temp_file( + &hasura_transaction, + &body.tenant_id, + &body.election_event_id, + &format!("diff_seq{}.json", meta.sequence), + "application/json", + envelope_size, + envelope_temp.path(), + ) + .await + .map_err(|err| format!("Error uploading the diff envelope: {err:?}"))?; + + // Electoral log: "patch generated" run-level entry. + let slug = std::env::var("ENV_SLUG").map_err(|err| format!("Missing ENV_SLUG: {err}"))?; + let board_name = get_event_board(&body.tenant_id, &body.election_event_id, &slug); + let electoral_log = ElectoralLog::new( + &hasura_transaction, + &body.tenant_id, + Some(&body.election_event_id), + &board_name, + ) + .await + .map_err(|err| format!("Error initializing reconciliation electoral log: {err:?}"))?; + electoral_log + .post_external_reconciliation( + body.election_event_id.clone(), + ExternalReconciliationKind::PatchGenerated, + meta.sequence, + meta.generated_at, + source_sha256.clone(), + external_patch_sha256.clone(), + None, + Some(body.requested_by_user_id.clone()), + body.requested_by_username.clone(), + ) + .await + .map_err(|err| format!("Error storing reconciliation electoral log: {err:?}"))?; + + hasura_transaction + .commit() + .await + .map_err(|err| format!("Error committing transaction: {err}"))?; + + Ok(envelope_document_id) +} + +/// Enforces the ticket's `<=` stale rule while preserving its two explicit +/// equal-Sequence cases: row-failure retries may apply; successful rounds are +/// generated only for a diff-only convergence check. +fn apply_permission_for_sequence( + sequence: i64, + last_applied_sequence: Option, + last_apply_had_failures: bool, +) -> std::result::Result { + match last_applied_sequence { + Some(last_applied) if sequence < last_applied => Err(format!( + "Reconciliation file Sequence {sequence} is older than the last applied Sequence {last_applied}" + )), + Some(last_applied) if sequence == last_applied => Ok(last_apply_had_failures), + _ => Ok(true), + } +} + +/// Writes one batch's items into all three open output writers — the +/// per-batch step shared by both the forward (file-driven) and reverse +/// (Sequent-driven) passes above. +fn write_batch_to_all_outputs( + envelope_items_writer: &mut DiffItemArrayWriter, + sequent_patch_writer: &mut DiffItemNdjsonWriter, + external_patch_writer: &mut ExternalPatchCsvWriter, + batch_items: &[DiffItem], + file_rows: &[crate::services::external::datafix_types::ParsedDatafixReconciliationRow], +) -> std::result::Result<(), String> { + envelope_items_writer + .write_batch(batch_items.iter()) + .map_err(|err| format!("Error writing diff envelope batch: {err:?}"))?; + sequent_patch_writer + .write_batch( + batch_items + .iter() + .filter(|item| is_sequent_apply_stream_item(item)), + ) + .map_err(|err| format!("Error writing Sequent patch batch: {err:?}"))?; + + let file_rows_by_username: HashMap = file_rows + .iter() + .map(|row| (row.external_voter_id.clone(), row.clone())) + .collect(); + external_patch_writer + .write_batch(batch_items, &file_rows_by_username) + .map_err(|err| format!("Error writing Datafix patch batch: {err}"))?; + Ok(()) +} + +/// Appends the diff envelope's metadata fields after the already-closed +/// `items` array and closes the object. Kept as one explicit hand-written +/// object rather than `serde_json::to_writer(&ReconciliationDiff {..})` +/// because `items` must be streamed before these fields are known — see the +/// call site's comment. +fn write_envelope_tail( + writer: &mut W, + sequence: i64, + generated_at: i64, + source_sha256: &str, + external_patch_document_id: Option<&str>, + external_patch_sha256: Option<&str>, + sequent_patch_document_id: &str, + apply_allowed: bool, +) -> std::io::Result<()> { + write!( + writer, + ",\"sequence\":{sequence},\"generated_at\":{generated_at}," + )?; + write!(writer, "\"source_sha256\":{},", json_string(source_sha256))?; + write!( + writer, + "\"external_patch_document_id\":{},", + json_optional_string(external_patch_document_id) + )?; + write!( + writer, + "\"external_patch_sha256\":{},", + json_optional_string(external_patch_sha256) + )?; + write!( + writer, + "\"sequent_patch_document_id\":{},\"apply_allowed\":{apply_allowed}}}", + json_string(sequent_patch_document_id), + )?; + Ok(()) +} + +fn json_string(value: &str) -> String { + serde_json::to_string(value).unwrap_or_else(|_| "null".to_string()) +} + +fn json_optional_string(value: Option<&str>) -> String { + match value { + Some(value) => json_string(value), + None => "null".to_string(), + } +} + +/// Flushes a `BufWriter` so every byte written through it is guaranteed to +/// have reached the underlying file before its size is read back from disk — +/// shared by every output document's finishing step below. +fn flush_writer(mut writer: BufWriter) -> std::io::Result<()> { + writer.flush() +} + +/// Reads back the size of an already-flushed output file from disk, since +/// `BufWriter` itself doesn't track total bytes written. +fn file_size(path: &std::path::Path) -> std::io::Result { + Ok(std::fs::metadata(path)?.len()) +} + +/// Uploads an already-written temp file as a `Document` — the counterpart to +/// a "serialize to one buffer, then upload" helper, for documents built +/// incrementally by the streaming writers above instead. +#[instrument(skip(hasura_transaction), err)] +async fn upload_document_from_temp_file( + hasura_transaction: &deadpool_postgres::Transaction<'_>, + tenant_id: &str, + election_event_id: &str, + file_name: &str, + media_type: &str, + file_size: u64, + temp_file_path: &std::path::Path, +) -> anyhow::Result { + let uploaded = upload_and_return_document( + hasura_transaction, + temp_file_path.to_str().unwrap_or_default(), + file_size, + media_type, + tenant_id, + Some(election_event_id.to_string()), + file_name, + None, + false, + ) + .await?; + Ok(uploaded.id) +} + +/// Appends `message` to the task's log and persists it immediately (status +/// stays `IN_PROGRESS`), so a crash partway through this task leaves a +/// record of how far processing got instead of nothing beyond "Task +/// started". Mutates `task_execution.logs` in place: the final +/// `update_complete`/`update_fail` call is built from that same field +/// (celery task arguments are captured once at enqueue time and don't +/// reflect this task's own DB writes), so without this the last checkpoint +/// would otherwise be overwritten by a summary built from the task's +/// original, pre-run logs. Persisting a checkpoint is best-effort: a +/// failure here is logged, not propagated — a diagnostic write must never +/// abort the reconciliation run it's only there to report on. +async fn checkpoint(task_execution: &mut TasksExecution, message: &str) { + let new_logs = match serde_json::to_value(append_general_log(&task_execution.logs, message)) { + Ok(value) => value, + Err(err) => { + tracing::warn!("Error serializing reconciliation checkpoint log: {err:?}"); + return; + } + }; + task_execution.logs = Some(new_logs.clone()); + if let Err(err) = update( + &task_execution.tenant_id, + &task_execution.id, + TasksExecutionStatus::IN_PROGRESS, + new_logs, + None, + ) + .await + { + tracing::warn!("Error persisting reconciliation checkpoint log: {err:?}"); + } +} + +#[cfg(test)] +mod tests { + use super::{apply_permission_for_sequence, write_envelope_tail}; + use crate::services::external::reconciliation::diff::{ + DiffItem, ReconciliationApplyEnvelope, ReconciliationDiff, + }; + use crate::services::external::reconciliation::patch::DiffItemArrayWriter; + use crate::services::external::types::{ + ReconciliationChangeCategory, ReconciliationPatchTarget, SequentReconciliationField, + }; + use std::io::Write; + + #[allow(clippy::too_many_arguments)] + fn write_test_envelope( + items: &[DiffItem], + sequence: i64, + generated_at: i64, + source_sha256: &str, + external_patch_document_id: Option<&str>, + external_patch_sha256: Option<&str>, + sequent_patch_document_id: &str, + apply_allowed: bool, + ) -> Vec { + let mut buffer = Vec::new(); + buffer.write_all(b"{\"items\":").unwrap(); + let mut items_writer = DiffItemArrayWriter::start(buffer).unwrap(); + items_writer.write_batch(items.iter()).unwrap(); + let mut buffer = items_writer.finish().unwrap(); + write_envelope_tail( + &mut buffer, + sequence, + generated_at, + source_sha256, + external_patch_document_id, + external_patch_sha256, + sequent_patch_document_id, + apply_allowed, + ) + .unwrap(); + buffer + } + + fn test_item(voter_username: &str, old_enabled: bool, new_enabled: bool) -> DiffItem { + DiffItem { + voter_username: voter_username.to_string(), + voter_id: Some(uuid::Uuid::new_v4()), + target: ReconciliationPatchTarget::Sequent(Some(SequentReconciliationField::Enabled( + old_enabled, + new_enabled, + ))), + category: ReconciliationChangeCategory::PROFILE_UPDATE, + failure_reason: None, + } + } + + #[test] + fn sequence_gate_distinguishes_retry_from_convergence_check() { + assert_eq!(apply_permission_for_sequence(0, None, false), Ok(true)); + assert!(apply_permission_for_sequence(4, Some(5), true).is_err()); + assert_eq!(apply_permission_for_sequence(5, Some(5), true), Ok(true)); + assert_eq!(apply_permission_for_sequence(5, Some(5), false), Ok(false)); + assert_eq!(apply_permission_for_sequence(6, Some(5), false), Ok(true)); + } + + #[test] + fn envelope_writer_round_trips_multiple_items_and_patch_metadata() { + let items = vec![ + test_item("voter-1", true, false), + test_item("voter-2", false, true), + ]; + let bytes = write_test_envelope( + &items, + 7, + 1_725_000_000, + "source-sha256", + Some("external-patch-document"), + Some("external-patch-sha256"), + "sequent-patch-document", + true, + ); + + let apply: ReconciliationApplyEnvelope = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(apply.sequence, 7); + assert_eq!(apply.generated_at, 1_725_000_000); + assert_eq!(apply.source_sha256, "source-sha256"); + assert_eq!( + apply.external_patch_document_id.as_deref(), + Some("external-patch-document") + ); + assert_eq!(apply.sequent_patch_document_id, "sequent-patch-document"); + assert!(apply.apply_allowed); + + let documented: ReconciliationDiff = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(documented.items, items); + assert_eq!(documented.sequence, 7); + assert_eq!(documented.generated_at, 1_725_000_000); + assert_eq!(documented.source_sha256, "source-sha256"); + assert_eq!( + documented.external_patch_document_id.as_deref(), + Some("external-patch-document") + ); + assert_eq!( + documented.external_patch_sha256.as_deref(), + Some("external-patch-sha256") + ); + assert_eq!( + documented.sequent_patch_document_id, + "sequent-patch-document" + ); + assert!(documented.apply_allowed); + } + + #[test] + fn envelope_writer_round_trips_an_empty_diff_without_patch_metadata() { + let bytes = write_test_envelope( + &[], + 8, + 1_725_000_001, + "empty-source-sha256", + None, + None, + "empty-sequent-patch-document", + false, + ); + + let apply: ReconciliationApplyEnvelope = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(apply.sequence, 8); + assert_eq!(apply.generated_at, 1_725_000_001); + assert_eq!(apply.source_sha256, "empty-source-sha256"); + assert_eq!(apply.external_patch_document_id, None); + assert_eq!( + apply.sequent_patch_document_id, + "empty-sequent-patch-document" + ); + assert!(!apply.apply_allowed); + + let documented: ReconciliationDiff = serde_json::from_slice(&bytes).unwrap(); + assert!(documented.items.is_empty()); + assert_eq!(documented.sequence, 8); + assert_eq!(documented.generated_at, 1_725_000_001); + assert_eq!(documented.source_sha256, "empty-source-sha256"); + assert_eq!(documented.external_patch_document_id, None); + assert_eq!(documented.external_patch_sha256, None); + assert_eq!( + documented.sequent_patch_document_id, + "empty-sequent-patch-document" + ); + assert!(!documented.apply_allowed); + } +} diff --git a/packages/windmill/src/tasks/mod.rs b/packages/windmill/src/tasks/mod.rs index 902ef5e5290..483a23a1bed 100644 --- a/packages/windmill/src/tasks/mod.rs +++ b/packages/windmill/src/tasks/mod.rs @@ -3,6 +3,7 @@ // SPDX-License-Identifier: AGPL-3.0-only pub mod activity_logs_report; +pub mod apply_reconciliation_patch; pub mod create_ballot_receipt; pub mod create_keys; pub mod delete_election_event; @@ -19,6 +20,7 @@ pub mod export_templates; pub mod export_tenant_config; pub mod export_trustees; pub mod export_users; +pub mod generate_reconciliation_patches; pub mod generate_report; pub mod generate_template; pub mod import_application; diff --git a/packages/windmill/src/tasks/process_cast_vote.rs b/packages/windmill/src/tasks/process_cast_vote.rs index fc9abddfba8..f90a1691e40 100644 --- a/packages/windmill/src/tasks/process_cast_vote.rs +++ b/packages/windmill/src/tasks/process_cast_vote.rs @@ -8,13 +8,13 @@ use crate::postgres::cast_vote::{ use crate::postgres::election_event::{get_election_event_by_id, ElectionEventDatafix}; use crate::services::cast_votes::{CastVote, CastVoteStatus}; use crate::services::database::get_hasura_pool; -use crate::services::datafix; -use crate::services::datafix::types::{SoapRequest, SoapRequestResponse}; -use crate::services::datafix::utils::{ - datafix_annotations, datafix_voter_lock_key, post_operation_result_to_electoral_log, +use crate::services::external; +use crate::services::external::datafix_types::{SoapRequest, SoapRequestResponse}; +use crate::services::external::utils::{ + datafix_annotations, external_voter_lock_key, post_operation_result_to_electoral_log, voted_via_internet, voted_via_not_internet_channel, DATAFIX_VOTER_LOCK_SECS, }; -use crate::services::datafix::voterview_requests::SoapSendError; +use crate::services::external::voterview_requests::SoapSendError; use crate::services::pg_lock::PgLock; use crate::types::error::Result; use celery::error::TaskError; @@ -61,8 +61,13 @@ pub async fn process_cast_vote( .voter_id_string .as_deref() .ok_or("Voter id not found")?; + let voter_id = Uuid::parse_str(voter_id).map_err(|err| format!("Invalid voter id: {err}"))?; let lock = match PgLock::acquire( - datafix_voter_lock_key(&cast_vote.tenant_id, &cast_vote.election_event_id, voter_id), + external_voter_lock_key( + &cast_vote.tenant_id, + &cast_vote.election_event_id, + &voter_id, + ), Uuid::new_v4().to_string(), ISO8601::now() + Duration::seconds(DATAFIX_VOTER_LOCK_SECS), ) @@ -167,7 +172,7 @@ async fn process_locked_cast_vote( return Ok(()); } - let prepared = datafix::voterview_requests::prepare( + let prepared = external::voterview_requests::prepare( SoapRequest::SetVoted, ElectionEventDatafix(election_event), &Some(username.clone()), @@ -180,7 +185,7 @@ async fn process_locked_cast_vote( .map_err(|err| format!("Datafix voter lock was lost before SetVoted: {err}"))?; let template_sha256 = prepared.template_sha256().to_string(); - let result = datafix::voterview_requests::send_prepared(prepared).await; + let result = external::voterview_requests::send_prepared(prepared).await; match &result { Ok(result) => info!( template_sha256 = %result.template_sha256, @@ -478,16 +483,18 @@ mod tests { #[test] fn voter_lock_is_event_wide() { - let first = datafix_voter_lock_key("tenant", "event", "voter"); - let second = datafix_voter_lock_key("tenant", "event", "voter"); + let voter = Uuid::new_v4(); + let other_voter = Uuid::new_v4(); + let first = external_voter_lock_key("tenant", "event", &voter); + let second = external_voter_lock_key("tenant", "event", &voter); assert_eq!(first, second); assert_ne!( first, - datafix_voter_lock_key("tenant", "other-event", "voter") + external_voter_lock_key("tenant", "other-event", &voter) ); assert_ne!( first, - datafix_voter_lock_key("tenant", "event", "other-voter") + external_voter_lock_key("tenant", "event", &other_voter) ); } } diff --git a/packages/windmill/src/types/tasks.rs b/packages/windmill/src/types/tasks.rs index f16c74b1802..56d8495f182 100644 --- a/packages/windmill/src/types/tasks.rs +++ b/packages/windmill/src/types/tasks.rs @@ -34,6 +34,8 @@ pub enum ETasksExecution { EXPORT_TALLY_RESULTS_XLSX, EXPORT_CERTIFICATE_AUTHORITIES, PUBLISH_RESULTS_WEBSITE, + GENERATE_RECONCILIATION_PATCHES, + APPLY_RECONCILIATION_PATCH, } impl ETasksExecution { @@ -67,6 +69,8 @@ impl ETasksExecution { ETasksExecution::EXPORT_TALLY_RESULTS_XLSX => "Export Tally Results To XLSX", ETasksExecution::EXPORT_CERTIFICATE_AUTHORITIES => "Export Certificate Authorities", ETasksExecution::PUBLISH_RESULTS_WEBSITE => "Publish Results Website", + ETasksExecution::GENERATE_RECONCILIATION_PATCHES => "Generate Reconciliation Patches", + ETasksExecution::APPLY_RECONCILIATION_PATCH => "Apply Reconciliation Changes", } } } diff --git a/packages/yarn.lock b/packages/yarn.lock index 32bdd97283c..ddd09073a4b 100644 --- a/packages/yarn.lock +++ b/packages/yarn.lock @@ -17506,15 +17506,15 @@ sentence-case@^3.0.4: "sequent-core@file:./admin-portal/rust/sequent-core-0.1.0.tgz": version "0.1.0" - resolved "file:./admin-portal/rust/sequent-core-0.1.0.tgz#e39ca4aaa84b575cba2071177a674c6b6751d10d" + resolved "file:./admin-portal/rust/sequent-core-0.1.0.tgz#2afdd078136501957d06ee35d21e6d204aaa78d5" "sequent-core@file:./ballot-verifier/rust/sequent-core-0.1.0.tgz": version "0.1.0" - resolved "file:./ballot-verifier/rust/sequent-core-0.1.0.tgz#e39ca4aaa84b575cba2071177a674c6b6751d10d" + resolved "file:./ballot-verifier/rust/sequent-core-0.1.0.tgz#2afdd078136501957d06ee35d21e6d204aaa78d5" "sequent-core@file:./voting-portal/rust/sequent-core-0.1.0.tgz": version "0.1.0" - resolved "file:./voting-portal/rust/sequent-core-0.1.0.tgz#e39ca4aaa84b575cba2071177a674c6b6751d10d" + resolved "file:./voting-portal/rust/sequent-core-0.1.0.tgz#2afdd078136501957d06ee35d21e6d204aaa78d5" serialize-javascript@^6.0.0, serialize-javascript@^6.0.2: version "6.0.2"