From d9a99912c17c8d99c928616ac20d5b67d8264d7d Mon Sep 17 00:00:00 2001 From: Noah Schatz Date: Tue, 11 Aug 2026 14:30:30 +0000 Subject: [PATCH 1/6] park: UNVERIFIED, UNGRADED work in progress on the Safe Harbor date residual Parked by the coordinator after every worker in the fleet died on an API auth error (Login expired) mid-slice. This commit exists ONLY so the work is not lost. It MUST NOT be treated as landed: - scripts/verify.sh NEVER RAN on it - the conformance-refuter NEVER GRADED it - it changes what leaves the box in a de-identification toolkit, which is the highest-consequence surface in this tree Intended scope was the founder decision of 2026-08-11: make policy safe-harbor actually Safe Harbor by dropping PV1-19, PV1-44, PV1-45, OBR-7 and DG1-5 from retention under SAFE_HARBOR_PROFILE while keeping them under LIMITED_DATA_SET_PROFILE, plus seeding the leak corpus that is structurally blind to the class and correcting docs-content/limitations.md. Whether this tree does any of that is UNKNOWN and unmeasured. src/retention.ts and test/fixtures/hl7/adt-a03.hl7 are new and ungraded; probe.local.mts is worker scratch. Re-measure from scratch before building on it. Do not cherry-pick it green. --- probe.local.mts | 34 ++++++++ scripts/phi-allow-list.txt | 24 ++++++ src/deidentify.ts | 36 ++++++++ src/hl7/apply.ts | 8 +- src/hl7/extract.ts | 74 ++++++++++++++-- src/hl7/index.ts | 39 +++++++-- src/hl7/retain.ts | 86 +++++++++++++++++-- src/index.ts | 8 ++ src/locus.ts | 12 +++ src/manifest.ts | Bin 3320 -> 3744 bytes src/policy.ts | 7 +- src/profile.ts | 75 +++++++++++++++-- src/report.ts | 41 ++++++--- src/retention.ts | 100 ++++++++++++++++++++++ test/fixtures/hl7/adt-a03.hl7 | 7 ++ test/hl7/deidentify-hl7.test.ts | 144 +++++++++++++++++++++++++++++++- 16 files changed, 651 insertions(+), 44 deletions(-) create mode 100644 probe.local.mts create mode 100644 src/retention.ts create mode 100644 test/fixtures/hl7/adt-a03.hl7 diff --git a/probe.local.mts b/probe.local.mts new file mode 100644 index 0000000..cef069d --- /dev/null +++ b/probe.local.mts @@ -0,0 +1,34 @@ +import { parseHL7 } from "@cosyte/hl7"; +import { deidentifyHl7 } from "./src/hl7/index.js"; +import { createDeidContext, SAFE_HARBOR_PROFILE, LIMITED_DATA_SET_PROFILE, profileOptions } from "./src/index.js"; + +const wire = [ + "MSH|^~\\&|A|B|C|D|20200101120000||ADT^A01|M1|P|2.5", + "PID|1||ZZMRN001^^^HOSP^MR||ZZFAM^ZZGIV||19850302|F", + "PV1|1|I|WARD^ROOM^BED|||||ATTEND^DOCFAMILY^DOCGIVEN|||||||||||ZZVISIT999^^^HOSP^VN|||||||||||||||||||||||||20200103040500|20200109060700", + "OBR|1|ZZPLACER777|ZZFILLER888|2951-2^Sodium^LN|||20200104080000", + "ORC|NW|ZZPLACER777|ZZFILLER888", + "DG1|1|I10|E11.9^Type 2 diabetes^I10||20200105090000", +].join("\r"); + +const ctx = createDeidContext({ key: "probe-key", patientId: "p1" }); + +for (const [label, profile] of [["safe-harbor", SAFE_HARBOR_PROFILE], ["limited-data-set", LIMITED_DATA_SET_PROFILE]] as const) { + const { document, manifest } = deidentifyHl7(parseHL7(wire), profileOptions(profile, ctx)); + const out = document.toString(); + console.log(`\n=== ${label} ===`); + const sentinels = { + "PV1-19 visit number": "ZZVISIT999", + "PV1-44 admit date": "20200103040500", + "PV1-45 discharge date": "20200109060700", + "OBR-7 observation date": "20200104080000", + "DG1-5 diagnosis date": "20200105090000", + "placer order number": "ZZPLACER777", + "filler order number": "ZZFILLER888", + }; + for (const [name, s] of Object.entries(sentinels)) { + console.log(` ${out.includes(s) ? "SURVIVES" : "gone "} ${name}`); + } + console.log(" manifest:", manifest.map((m) => `${m.locus}:${m.disposition}`).join(" ")); + console.log(" wire:", out.split("\r").join("\n ")); +} diff --git a/scripts/phi-allow-list.txt b/scripts/phi-allow-list.txt index c29236f..00570a6 100644 --- a/scripts/phi-allow-list.txt +++ b/scripts/phi-allow-list.txt @@ -81,6 +81,17 @@ NAME ZZLABGIVEN NAME ZZLABSTREET NAME ZZLABCITY +# --- Synthetic HL7 v2 encounter fixture tokens (test/fixtures/hl7/adt-a03.hl7) --- +# The discharge ADT that seeds the encounter loci carved out of the retained visit / order segments: +# the visit number, the admit + discharge + observation + diagnosis dates, and the placer / filler +# order numbers, alongside the usual patient demographics. Every one is a ZZ-tagged sentinel or a +# fictional 555 phone; each is asserted ABSENT under the Safe Harbor profile and PRESENT under the +# limited-data-set profile, which is why the fixture has to carry them in the clear to begin with. +NAME ZZENCFAMILY +NAME ZZENCGIVEN +NAME ZZENCSTREET +NAME ZZENCCITY + # --- Synthetic C-CDA header person-name / street / city / county tokens (test/fixtures/ccda) --- # Patient + guardian + author + informant person names, organization names, and address parts. All # obviously-synthetic ZZ-tagged tokens: checked structurally by the C-CDA header detector. @@ -186,6 +197,12 @@ DOB 19811203 # The two birth dates of the longitudinal date-shift fixtures (test/hl7/deidentify-hl7.test.ts). DOB 20200110 DOB 20200210 +# The encounter fixture's admit / discharge / observation / diagnosis timestamps. Not birth dates, but +# they are elements of dates directly related to the individual, declared here for the same reason. +DOB 20200103040500 +DOB 20200109060700 +DOB 20200104080000 +DOB 20200105090000 # # ▶ `19800101` IS DELIBERATELY ABSENT AND MUST STAY ABSENT. It is the undeclared DOB # `test/scripts/phi-scan.test.ts` uses to prove the HL7, C-CDA, X12 and NCPDP detectors CATCH a @@ -197,6 +214,13 @@ DOB 20200210 ID ZZMRN ID ZZMRN001 ID ZZMRN002 +ID ZZMRN003 +# The encounter fixture's visit number, order numbers, and account number (test/fixtures/hl7/adt-a03). +ID ZZVISIT700 +ID ZZPLACER700 +ID ZZFILLER700 +ID ZZACCT300 +ID 5550000020 # Inline HL7 identifiers in test modules / scripts/smoke.mjs: the five identifier-type placeholders # of the PID-3 repetition test, the surviving MRN of the MRG merge case, and the smoke MRN. ID I1 diff --git a/src/deidentify.ts b/src/deidentify.ts index 07aef82..d417473 100644 --- a/src/deidentify.ts +++ b/src/deidentify.ts @@ -27,6 +27,7 @@ import type { DeidDocument, GenericLocus, TransformedLocus } from "./locus.js"; import { ManifestBuilder, type DeidManifestEntry, type DeidResult } from "./manifest.js"; import { resolvePolicy, type DeidPolicy, type TransformName } from "./policy.js"; import type { FreeTextRedactor } from "./redactor.js"; +import type { RetainedLocusClass } from "./retention.js"; import { dateShift, generalizeAge, @@ -50,6 +51,15 @@ import { export interface DeidOptions { /** The policy to apply. Defaults to the built-in Safe Harbor policy. */ readonly policy?: DeidPolicy | "safe-harbor"; + /** + * The **retention classes** the configured profile permits: the named groups of identifying loci a + * format adapter may pass through unchanged (each one still recorded as a residual). Build this + * with {@link profileOptions} rather than by hand; the widen-never-narrow contract on a derived + * profile is what keeps a site preset from adding one. + * + * **Absent or empty retains nothing**, so a bare options bag gets the strict treatment. + */ + readonly retainedLoci?: readonly RetainedLocusClass[]; /** The context carrying the consumer's key, required only when the policy uses a keyed transform. */ readonly context?: DeidContext; /** @@ -84,6 +94,26 @@ function blocked( }; } +/** + * Build the outcome for a locus the configured profile's **retention set** keeps: the value passes + * through unchanged, and the fact is **recorded** as a residual so it reaches the retained-quasi- + * identifier inventory a determiner reads. An unclassified retained locus is recorded as the + * catch-all (R), never as "nothing happened here". + */ +function retainedResidual(locus: GenericLocus, category: SafeHarborCategory): LocusOutcome { + return { + value: locus.value, + disposition: "retained", + manifest: { + category, + transform: "retain", + locus: locus.path, + disposition: "retained", + code: DEID_DISPOSITION_CODES.DEID_RESIDUAL_RETAINED, + }, + }; +} + /** Choose the right generalization for a locus from its kind, then its category. `null` = can't. */ function generalizeLocus( locus: GenericLocus, @@ -285,6 +315,12 @@ function handleLocus( DEID_DISPOSITION_CODES.DEID_LOCUS_BLOCKED, ); } + // The configured profile's retention set deliberately keeps this class of identifying locus. It is + // reached only AFTER the three fail-closed guards above, so free text and unrecognized structure can + // never be retained by this route however an adapter marks them; and it is always recorded. + if (locus.retainedByPolicy === true) { + return retainedResidual(locus, locus.category); + } return applyTransform(policy.transforms[locus.category], locus, locus.category, context); } diff --git a/src/hl7/apply.ts b/src/hl7/apply.ts index 403ce92..acab25b 100644 --- a/src/hl7/apply.ts +++ b/src/hl7/apply.ts @@ -6,7 +6,9 @@ * * Removal is clean: a redacted or blocked field's repetitions are dropped to `[]` (it serializes as an * empty field), never zero-length-padded into `^^^` residue. A pseudonymized identifier replaces only - * the id-number component (CX.1); a generalized address keeps only the Safe Harbor 3-digit ZIP. + * the id-number component (CX.1); a generalized address keeps only the Safe Harbor 3-digit ZIP. A + * locus the profile's retention set kept carries the `none` edit and is not written at all, so it + * survives byte-identical. * * @packageDocumentation */ @@ -150,6 +152,10 @@ export function applyHl7( case "address-zip": applyAddressZip(field, coord.rep, t); break; + case "none": + // A locus the profile's retention set kept: byte-identical is the whole point, so nothing is + // written back. A whole-field write would flatten its components/repetitions into one value. + break; } } diff --git a/src/hl7/extract.ts b/src/hl7/extract.ts index 8c8f731..1a6f3de 100644 --- a/src/hl7/extract.ts +++ b/src/hl7/extract.ts @@ -10,8 +10,10 @@ * else: a recognized segment is retained only if it is on the explicit {@link RETAIN_SEGMENTS} * clinical/administrative list, so a *known* patient-identity segment absent from the map (MRG / FAM / * ACC / PEO / PDA) is blocked exactly like a Z-segment or a segment unknown to the parser. A non-mapped - * field inside a mapped segment, and a retained clinical segment, are left untouched (the over-scrub - * guard); OBX-5 is retained only when OBX-2 positively types it as a structured clinical value. + * field inside a mapped segment is left untouched (the over-scrub guard); OBX-5 is retained only when + * OBX-2 positively types it as a structured clinical value. Inside a **retained** segment the + * {@link RETAINED_LOCUS_RULES} carve-out still applies: its identifying dates and encounter / order + * numbers are handed to the engine unless the configured profile names their retention class. * * @packageDocumentation */ @@ -21,11 +23,16 @@ import { type Hl7Message, type Segment } from "@cosyte/hl7"; import { SAFE_HARBOR_CATEGORIES } from "../categories.js"; import { safeLocusToken } from "../derived-token.js"; import type { GenericLocus } from "../locus.js"; +import { retains, type RetainedLocusClass } from "../retention.js"; import { HL7_LOCUS_MAP, categoryForIdentifierType, type Hl7FieldRule } from "./locus-map.js"; -import { RETAIN_SEGMENTS } from "./retain.js"; +import { RETAIN_SEGMENTS, RETAINED_LOCUS_RULES } from "./retain.js"; -/** How the applier writes a transformed locus back onto the cloned raw tree. */ -export type Hl7EditKind = "whole-field" | "id-number" | "address-zip"; +/** + * How the applier writes a transformed locus back onto the cloned raw tree. `none` writes **nothing**: + * it is the coordinate of a locus the profile's retention set kept, which must stay byte-identical + * (a whole-field write would flatten its components and repetitions into a single value). + */ +export type Hl7EditKind = "whole-field" | "id-number" | "address-zip" | "none"; /** * A write-back coordinate: the exact structural location of one extracted locus in the message's raw @@ -42,6 +49,15 @@ export interface Hl7Coord { readonly edit: Hl7EditKind; } +/** + * Options for {@link extractHl7Loci}. The retention set comes from the configured profile via + * {@link profileOptions}; **omitting it retains nothing**, which is the fail-closed default. + */ +export interface Hl7ExtractOptions { + /** The retention classes the profile permits. Absent or empty keeps no identifying locus. */ + readonly retainedLoci?: readonly RetainedLocusClass[]; +} + /** The paired output of {@link extractHl7Loci}: loci for the engine + coordinates for the applier. */ export interface Hl7Extraction { /** The located candidate values, in document order. */ @@ -202,6 +218,44 @@ function extractRule( } } +/** + * Extract the identifying fields carved out of a **retained** segment: the encounter dates and the + * encounter / order identifiers. When the configured profile names the rule's retention class the + * locus is marked `retainedByPolicy` (passed through unchanged, and recorded as a residual) and given + * a **`none`** coordinate so the applier writes nothing; otherwise it is an ordinary locus the policy + * acts on: a date generalizes to its year, an identifier is blocked as the (R) catch-all. + */ +function extractRetainedLoci( + out: Hl7Extraction, + seg: Segment, + type: string, + occ: number, + retainedLoci: readonly RetainedLocusClass[] | undefined, +): void { + const rules = RETAINED_LOCUS_RULES[type]; + if (rules === undefined) return; + for (const rule of rules) { + if (!hasContent(seg, rule.field)) continue; + const kept = retains(retainedLoci, rule.retention); + push( + out, + { + path: fieldPath(type, occ, rule.field), + kind: rule.kind, + category: rule.category, + ...(kept ? { retainedByPolicy: true } : {}), + value: seg.field(rule.field).value, + }, + { + segIndex: seg.absoluteIndex, + field: rule.field, + rep: 0, + edit: kept ? "none" : "whole-field", + }, + ); + } +} + /** Extract the OBX-5 locus, failing closed unless OBX-2 positively types it as a structured value. */ function extractObx(out: Hl7Extraction, seg: Segment, occ: number): void { if (!hasContent(seg, 5)) return; @@ -254,6 +308,7 @@ function extractUnknownSegment(out: Hl7Extraction, seg: Segment, type: string, o * the `@cosyte/hl7` model. Never mutates the message. * * @param msg - The parsed HL7 v2 message. + * @param options - The configured profile's retention classes. Omitted retains nothing (fail closed). * @returns The loci (for the engine) and their index-aligned write-back coordinates. * @example * ```ts @@ -264,7 +319,7 @@ function extractUnknownSegment(out: Hl7Extraction, seg: Segment, type: string, o * loci.length; // number of located candidate values * ``` */ -export function extractHl7Loci(msg: Hl7Message): Hl7Extraction { +export function extractHl7Loci(msg: Hl7Message, options: Hl7ExtractOptions = {}): Hl7Extraction { const out: Hl7Extraction = { loci: [], coords: [] }; const occurrences = new Map(); @@ -297,7 +352,12 @@ export function extractHl7Loci(msg: Hl7Message): Hl7Extraction { // retain-list. Everything else is blocked: a Z-segment, a segment unknown to the parser, OR a // *known* patient/relative-identity segment absent from the map and the retain-list (MRG / ACC / // FAM / PEO / PDA). A merge message's prior name + MRN can never ride through in the clear. - if (RETAIN_SEGMENTS.has(type)) continue; + if (RETAIN_SEGMENTS.has(type)) { + // Retaining the SEGMENT does not retain every field in it: the identifying dates and the + // encounter / order identifiers are carved back out and handed to the engine. + extractRetainedLoci(out, seg, type, occ, options.retainedLoci); + continue; + } extractUnknownSegment(out, seg, type, occ); } diff --git a/src/hl7/index.ts b/src/hl7/index.ts index 79b68b5..354a941 100644 --- a/src/hl7/index.ts +++ b/src/hl7/index.ts @@ -20,9 +20,15 @@ * line is unchanged: the output is **"Safe-Harbor-transformed per the configured policy"**, never * "de-identified". * - * **Known limitations.** Free text is block-only (no scrub); within **retained** clinical / - * visit segments, patient-related dates (OBR/DG1/PV1 timestamps), visit identifiers (PV1-19), and - * provider names (PV1-7/8, OBR-16) are **not** de-identified; the address generalization keeps only the + * **Inside a retained segment**, the identifying dates and the encounter / order numbers are carved + * back out ({@link RETAINED_LOCUS_RULES}): under a Safe-Harbor-labelled policy the admit (PV1-44), + * discharge (PV1-45), observation (OBR-7) and diagnosis (DG1-5) dates generalize to their **year**, and + * the visit number (PV1-19) and the placer / filler order numbers (OBR-2/3, ORC-2/3) are **blocked** as + * the (R) catch-all. A profile that names their retention class keeps them **unchanged and recorded**. + * + * **Known limitations.** Free text is block-only (no scrub); a field of a retained segment that the + * carve-out does not name is still passed through untouched and unrecorded (the specimen collection + * date SPM-17, and the provider names PV1-7/8 and OBR-16); the address generalization keeps only the * Safe Harbor 3-digit ZIP and conservatively drops the (permitted) state as well. * * @packageDocumentation @@ -63,9 +69,10 @@ export interface Hl7DeidResult { * de-identified, and Expert Determination is not rendered. * * @param msg - The parsed HL7 v2 message to de-identify. - * @param options - The policy and (for keyed transforms, MRN / account / beneficiary pseudonymization) - * the key context. A keyed transform with no context is a fatal `DEID_NO_KEY`, never an unkeyed - * fallback. + * @param options - The policy, the profile's retention classes, and (for keyed transforms, MRN / + * account / beneficiary pseudonymization) the key context. A keyed transform with no context is a + * fatal `DEID_NO_KEY`, never an unkeyed fallback. **Retention defaults to nothing**, so a bare + * options bag removes the encounter dates and identifiers rather than keeping them. * @returns The de-identified message and the value-free manifest. * @throws {@link DeidError} `DEID_NO_KEY` when a keyed transform is required for a category present in * the message but no key context was supplied. @@ -82,7 +89,10 @@ export interface Hl7DeidResult { * ``` */ export function deidentifyHl7(msg: Hl7Message, options: DeidOptions = {}): Hl7DeidResult { - const { loci, coords } = extractHl7Loci(msg); + const { loci, coords } = extractHl7Loci( + msg, + options.retainedLoci !== undefined ? { retainedLoci: options.retainedLoci } : {}, + ); const { document, manifest } = deidentify({ loci }, options); const deidentified = applyHl7(msg, document.loci, coords); return { document: deidentified, manifest }; @@ -94,7 +104,18 @@ export { type Hl7FieldRule, type Hl7FieldMode, } from "./locus-map.js"; -export { extractHl7Loci, type Hl7Coord, type Hl7Extraction, type Hl7EditKind } from "./extract.js"; +export { + extractHl7Loci, + type Hl7Coord, + type Hl7Extraction, + type Hl7EditKind, + type Hl7ExtractOptions, +} from "./extract.js"; export { applyHl7 } from "./apply.js"; -export { RETAIN_SEGMENTS } from "./retain.js"; +export { RETAIN_SEGMENTS, RETAINED_LOCUS_RULES, type Hl7RetainedFieldRule } from "./retain.js"; +export { + RETAINED_LOCUS_CLASSES, + retains, + type RetainedLocusClass, +} from "../retention.js"; export { SAFE_HARBOR_CATEGORIES, type SafeHarborCategory } from "../categories.js"; diff --git a/src/hl7/retain.ts b/src/hl7/retain.ts index d895e2a..2173442 100644 --- a/src/hl7/retain.ts +++ b/src/hl7/retain.ts @@ -13,16 +13,24 @@ * **FAM** (family history, a relative), **PEO**, and **PDA** are deliberately **absent** from this list, * so they **fail closed** and are blocked. * - * **Documented limitation.** Retained clinical/visit segments may still carry patient-related - * *dates* (OBR observation date, DG1 diagnosis date, PV1 admit/discharge date, SPM collection date) and - * *visit identifiers* (PV1-19), and *provider* names (PV1-7/8, OBR-16). Selective scrubbing of those loci - * is **not** performed; this adapter covers the PID-family patient/relative demographics and the - * free-text / unknown-structure fail-closed defaults. Forgetting a clinical segment here fails - * **safe**: it is blocked, not leaked. + * **Retaining the segment is not retaining every field in it.** {@link RETAINED_LOCUS_RULES} carves + * the patient-related *dates* and the *encounter / order identifiers* back out of these segments and + * hands them to the engine, so under a Safe-Harbor-labelled policy an admission date is reduced to its + * year and a visit number is blocked as category (R). They survive only under a profile that names + * their retention class, and even then they are **recorded**. + * + * **Documented limitation, and it is narrower than it was but real.** A field inside a retained segment + * that is on **neither** list is still passed through untouched and is **not** recorded anywhere: the + * specimen collection date (SPM-17), the attending / referring *provider* names (PV1-7/8, OBR-16), and + * every other unmapped position. Forgetting a clinical segment here fails **safe**: it is blocked, not + * leaked. Forgetting a *field* of a retained segment does not. * * @packageDocumentation */ +import { SAFE_HARBOR_CATEGORIES, type SafeHarborCategory } from "../categories.js"; +import { RETAINED_LOCUS_CLASSES, type RetainedLocusClass } from "../retention.js"; + /** * Recognized segments retained (passed through) by the HL7 v2 de-identifier. Anything not on this list, * and not a mapped PID-family segment or OBX/NTE: fails closed. @@ -53,7 +61,7 @@ export const RETAIN_SEGMENTS: ReadonlySet = new Set([ "RDT", "EQL", "OMC", - // Visit / additional demographics (deferred date/visit-id limitation) + // Visit / additional demographics (the visit number and admit/discharge dates are carved out below) "PV1", "PV2", "PD1", @@ -128,3 +136,67 @@ export const RETAIN_SEGMENTS: ReadonlySet = new Set([ "CSP", "CSS", ]); + +/** + * One field carved back out of a retained segment: an identifying locus a profile may keep, but only + * by naming its {@link RetainedLocusClass}. Absent that, the engine acts on it under the policy. + */ +export interface Hl7RetainedFieldRule { + /** 1-based HL7 field number (e.g. `44` for PV1-44). */ + readonly field: number; + /** The retention class a profile must name for this locus to survive. */ + readonly retention: RetainedLocusClass; + /** The Safe Harbor category the locus carries when the policy acts on it. */ + readonly category: SafeHarborCategory; + /** `date` generalizes to year under Safe Harbor; `identifier` is blocked as the (R) catch-all. */ + readonly kind: "date" | "identifier"; +} + +const R = RETAINED_LOCUS_CLASSES; +const C_DATES = SAFE_HARBOR_CATEGORIES.DATES; +const C_OTHER = SAFE_HARBOR_CATEGORIES.OTHER_UNIQUE_ID; + +/** + * The **carve-out table**: for each retained segment, the fields that are identifying rather than + * clinical. Every position is grounded in the HL7 v2.x segment definitions. + * + * The dates are elements of dates directly related to the individual, which §164.514(b)(2)(i)(C) + * removes (admission and discharge are named in the regulation text itself); the visit number and the + * placer / filler order numbers are unique identifying codes, which §164.514(b)(2)(i)(R) removes. Both + * groups are absent from §164.514(e)(2)'s sixteen direct identifiers, so a limited data set may keep + * them, which is what the retention classes express. + * + * @example + * ```ts + * import { RETAINED_LOCUS_RULES } from "@cosyte/deid/hl7"; + * + * RETAINED_LOCUS_RULES.PV1?.find((r) => r.field === 44)?.retention; // => "encounter-dates" + * ``` + */ +export const RETAINED_LOCUS_RULES: Readonly> = + Object.freeze({ + PV1: [ + // PV1-19 Visit Number (CX): the encounter identifier. + { field: 19, retention: R.ENCOUNTER_IDENTIFIERS, category: C_OTHER, kind: "identifier" }, + // PV1-44 Admit Date/Time (TS) and PV1-45 Discharge Date/Time (TS). + { field: 44, retention: R.ENCOUNTER_DATES, category: C_DATES, kind: "date" }, + { field: 45, retention: R.ENCOUNTER_DATES, category: C_DATES, kind: "date" }, + ], + OBR: [ + // OBR-2 Placer Order Number (EI) and OBR-3 Filler Order Number (EI). + { field: 2, retention: R.ENCOUNTER_IDENTIFIERS, category: C_OTHER, kind: "identifier" }, + { field: 3, retention: R.ENCOUNTER_IDENTIFIERS, category: C_OTHER, kind: "identifier" }, + // OBR-7 Observation Date/Time (TS): the service date. + { field: 7, retention: R.ENCOUNTER_DATES, category: C_DATES, kind: "date" }, + ], + ORC: [ + // ORC-2 Placer Order Number (EI) and ORC-3 Filler Order Number (EI): the same two identifiers + // the order-control segment carries alongside OBR. + { field: 2, retention: R.ENCOUNTER_IDENTIFIERS, category: C_OTHER, kind: "identifier" }, + { field: 3, retention: R.ENCOUNTER_IDENTIFIERS, category: C_OTHER, kind: "identifier" }, + ], + DG1: [ + // DG1-5 Diagnosis Date/Time (TS). + { field: 5, retention: R.ENCOUNTER_DATES, category: C_DATES, kind: "date" }, + ], + }); diff --git a/src/index.ts b/src/index.ts index c8a4d85..94ca4b1 100644 --- a/src/index.ts +++ b/src/index.ts @@ -87,6 +87,14 @@ export { type DeidStandard, } from "./profile.js"; +// ── Policy-scoped retention: the named classes of identifying locus a profile may keep, and record. +export { + RETAINED_LOCUS_CLASSES, + NO_RETAINED_LOCI, + retains, + type RetainedLocusClass, +} from "./retention.js"; + // ── The generic locus model. export { type LocusKind, diff --git a/src/locus.ts b/src/locus.ts index dbbfcf1..9e15bd4 100644 --- a/src/locus.ts +++ b/src/locus.ts @@ -49,6 +49,18 @@ export interface GenericLocus { readonly kind: LocusKind; /** The Safe Harbor category, when known. Omit to force fail-closed handling as category (R). */ readonly category?: SafeHarborCategory; + /** + * Set by an adapter when the **configured policy's retention set** names the class this locus + * belongs to: the value is passed through **unchanged** and recorded as a residual, rather than + * transformed. It is how a limited-data-set preset keeps an admission date or an encounter number + * that Safe Harbor removes. + * + * **This is the one flag that makes the engine emit an identifying value.** It is opt-in and + * fails closed: absent (the default) the locus takes its normal policy transform. It is **not** the + * over-scrub guard: a `clinical` locus is not an identifier and is retained without a manifest row, + * whereas a locus marked here **is** identifying and is always recorded. + */ + readonly retainedByPolicy?: boolean; /** The value at the locus. Consumed by the engine, never copied into the manifest. */ readonly value: string; } diff --git a/src/manifest.ts b/src/manifest.ts index a606c4d823a50dcac89954b1c282669623152ff3..443099f58a98606c7d1a0796dd2e292dacff3ae9 100644 GIT binary patch delta 459 zcmYL`y-EW?6os)cMc%+dIM}F*#@8@CAGX!OFL? zwDfL9up7Adp7WhEKS%G!AK!<&F@OT|j-WZLUEs*hPO$J461?z1u4O}~0U|$~xSd6y2u+4-@+y+) z@j~l}WSdUeCFu#OI4~%)Reiu(vB+V|0wGvgcTvu&^7&!*Sd~xHJ9*5f)%~nopmCZm z5FAui{R3t4LZF(Cb{W)DV4ai!+l_EJ-$P(JS&KrEO=^;O!vY7c4NY)%YuIulR~uy# zg=|sx`?}nQ{AFl-s = new Set([ diff --git a/src/profile.ts b/src/profile.ts index b93778f..e4d870a 100644 --- a/src/profile.ts +++ b/src/profile.ts @@ -18,6 +18,16 @@ * **never** re-weaken a category the base scrubs. A site preset can therefore only ever *tighten* the * base standard, never quietly loosen it (fail-closed, {@link FATAL_CODES.DEID_PROFILE_INVALID}). * + * **The contract is about the strength of the result, not the size of any list, and retention reads + * the opposite way round from a transform override.** A profile's `retainedLoci` names identifying + * loci it *keeps*, so **dropping** a class removes more and is the widening the contract permits, + * while **adding** one keeps more and is the narrowing it refuses. Reading it as "the derived list may + * only grow" would invert the guarantee, which is why the check is a **subset** test and not a rank + * comparison. Note the scope, too: the contract binds a **derived** profile against its base. It says + * nothing about the built-in presets themselves, so strengthening one of those (retaining less than it + * used to) is not a contract question at all, and every profile derived from it inherits the stronger + * base. + * * @packageDocumentation */ @@ -33,13 +43,19 @@ import { type TransformName, } from "./policy.js"; import { type FreeTextRedactor } from "./redactor.js"; +import { + NO_RETAINED_LOCI, + RETAINED_LOCUS_CLASSES, + type RetainedLocusClass, +} from "./retention.js"; /** * The **protection rank** of a transform: higher means the residual is *less* identifying, so the * transform is *stronger* de-identification. Used to enforce the widen-never-narrow contract: * `block` (value withheld) is strongest; `date-shift` (a full-precision shifted **real** date) is the * weakest transform that still acts. `byo-redact` is ranked with `block` because the policy map never - * performs it: it fails closed to a block. + * performs it: it fails closed to a block. `retain` ranks **0**, below every transform, because it + * changes nothing: no base transform is weaker, so no override can ever move a category onto it. */ const TRANSFORM_RANK: Readonly> = Object.freeze({ block: 5, @@ -49,6 +65,7 @@ const TRANSFORM_RANK: Readonly> = Object.freeze({ hash: 3, generalize: 2, "date-shift": 1, + retain: 0, }); /** The named standard a profile targets, surfaced so output labelling can never overclaim. */ @@ -82,6 +99,13 @@ export interface DeidProfile { * any category uses a keyed transform such as `date-shift` on a category that is always present). */ readonly requiresContext: boolean; + /** + * The **retention classes** this profile permits: named groups of *identifying* loci a format + * adapter passes through unchanged, each one still recorded as a `DEID_RESIDUAL_RETAINED` residual. + * Empty on {@link SAFE_HARBOR_PROFILE}; {@link LIMITED_DATA_SET_PROFILE} carries the two classes + * §164.514(e)(2) permits a limited data set to keep. + */ + readonly retainedLoci: readonly RetainedLocusClass[]; /** An optional default free-text redactor the profile carries into {@link profileOptions}. */ readonly redactor?: FreeTextRedactor; } @@ -107,8 +131,11 @@ export const SAFE_HARBOR_PROFILE: DeidProfile = Object.freeze({ policy: SAFE_HARBOR_POLICY, description: "HIPAA Safe Harbor (§164.514(b)(2)) transform set: the 18 categories removed/pseudonymized/" + - "generalized, dates to year, the (R) catch-all blocked. Fails closed. Not a certification.", + "generalized, dates to year, the (R) catch-all blocked. Retains no identifying locus: an " + + "admission/discharge/service date is reduced to its year and an encounter or order number is " + + "blocked as (R). Fails closed. Not a certification.", requiresContext: false, + retainedLoci: NO_RETAINED_LOCI, }); /** @@ -116,6 +143,13 @@ export const SAFE_HARBOR_PROFILE: DeidProfile = Object.freeze({ * are **date-shifted** (a single consistent per-patient offset, intervals preserved) rather than * generalized to year, so time-series analysis survives. * + * It also **keeps**, unchanged, the two classes of identifying locus §164.514(e)(2) permits a limited + * data set to carry and Safe Harbor does not: **encounter dates** (admission / discharge / service / + * diagnosis) and **encounter and order identifiers** (visit number, placer and filler order numbers). + * That list of sixteen direct identifiers names no date and has **no catch-all**, which is precisely + * why these survive here and are removed under Safe Harbor. Each kept locus is still **recorded** as a + * `DEID_RESIDUAL_RETAINED` residual, so it reaches the determiner's residual inventory. + * * **This is deliberately less protective than Safe Harbor and is NOT Safe Harbor.** A shifted-but-real * date is still "an element of a date" (§164.514(b)(2)(i)(C)), so this profile: * @@ -141,9 +175,14 @@ export const LIMITED_DATA_SET_PROFILE: DeidProfile = Object.freeze({ }), description: "Longitudinal research preset: Safe-Harbor identifier handling, but dates are DATE-SHIFTED " + - "(interval-preserving), not generalized. Retains shifted real dates: Expert-Determination " + + "(interval-preserving), not generalized, and the encounter dates and encounter/order identifiers " + + "§164.514(e)(2) permits are KEPT UNCHANGED and recorded as residuals. Expert-Determination " + "territory, NOT Safe Harbor, NOT a certified de-identification. Requires a keyed per-patient context.", requiresContext: true, + retainedLoci: Object.freeze([ + RETAINED_LOCUS_CLASSES.ENCOUNTER_DATES, + RETAINED_LOCUS_CLASSES.ENCOUNTER_IDENTIFIERS, + ]), }); /** @@ -170,6 +209,12 @@ export interface DeidProfileSpec { * transform than the base (widen-never-narrow); a weakening override is rejected. */ readonly transforms?: Partial>>; + /** + * The retention classes the derived profile permits. Must be a **subset** of the base's: dropping a + * class removes more (a widening, allowed), adding one keeps more (a narrowing, rejected). Omit to + * inherit the base's set unchanged. + */ + readonly retainedLoci?: readonly RetainedLocusClass[]; /** An optional default free-text redactor the derived profile carries. */ readonly redactor?: FreeTextRedactor; /** An optional human description; a default is synthesized from the base when omitted. */ @@ -188,9 +233,10 @@ const RESERVED_NAMES: ReadonlySet = new Set(["safe-harbor", "limited-dat * * @param spec - The profile name, base, per-category overrides, and optional redactor. * @returns A frozen {@link DeidProfile}. - * @throws {@link DeidError} `DEID_PROFILE_INVALID` if an override weakens a category or the name - * reclaims a reserved standard label; `DEID_POLICY_INVALID` if the derived policy violates the - * key/label contract (e.g. a `safe-harbor`-labelled policy that date-shifts). + * @throws {@link DeidError} `DEID_PROFILE_INVALID` if an override weakens a category, if + * `retainedLoci` adds a retention class the base does not retain, or if the name reclaims a reserved + * standard label; `DEID_POLICY_INVALID` if the derived policy violates the key/label contract (e.g. + * a `safe-harbor`-labelled policy that date-shifts). * @example * ```ts * import { defineDeidProfile, SAFE_HARBOR_CATEGORIES } from "@cosyte/deid"; @@ -213,6 +259,17 @@ export function defineDeidProfile(spec: DeidProfileSpec): DeidProfile { ); } + const retainedLoci = spec.retainedLoci ?? base.retainedLoci; + const added = retainedLoci.filter((cls) => !base.retainedLoci.includes(cls)); + if (added.length > 0) { + throw new DeidError( + FATAL_CODES.DEID_PROFILE_INVALID, + `retention class(es) ${added.map((c) => `"${c}"`).join(", ")} are not retained by the base ` + + `profile "${base.name}"; a profile may only DROP a retained class (keep less, remove more), ` + + "never add one", + ); + } + for (const [category, transform] of Object.entries(overrides) as [ SafeHarborCategory, TransformName, @@ -246,13 +303,16 @@ export function defineDeidProfile(spec: DeidProfileSpec): DeidProfile { spec.description ?? `Custom site profile derived from "${base.name}" (tightened, never loosened).`, requiresContext, + retainedLoci: Object.freeze([...retainedLoci]), ...(spec.redactor !== undefined ? { redactor: spec.redactor } : {}), }); } /** * Build the {@link DeidOptions} to pass to any adapter (`deidentifyHl7`, `deidentifyFhir`, …) from a - * profile: its policy, the supplied key context, and the profile's default redactor (unless overridden). + * profile: its policy, its retention set, the supplied key context, and the profile's default redactor + * (unless overridden). **Going through here is what carries the retention set**: an options bag built + * by hand from `profile.policy` alone retains nothing, which is the fail-closed direction. * * @param profile - The profile to apply. * @param context - The keyed per-patient context (required by profiles whose `requiresContext` is true). @@ -275,6 +335,7 @@ export function profileOptions( const redactor = overrides?.redactor ?? profile.redactor; return { policy: profile.policy, + retainedLoci: profile.retainedLoci, ...(ctx !== undefined ? { context: ctx } : {}), ...(redactor !== undefined ? { redactor } : {}), }; diff --git a/src/report.ts b/src/report.ts index 8c601c5..49b6388 100644 --- a/src/report.ts +++ b/src/report.ts @@ -53,8 +53,12 @@ export const EXPERT_DETERMINATION_DISCLAIMER = "re-identification risk score and reaches no conclusion. It is descriptive input a determiner " + "consumes and documents, never the determination itself."; -/** A manifest disposition: the three outcomes a locus can have. */ -export type ReportDisposition = "transformed" | "removed" | "blocked"; +/** + * A manifest disposition: the four outcomes an acted-on locus can have. `retained` is an + * **identifying** locus the configured profile's retention set deliberately kept unchanged; it is + * always paired with `DEID_RESIDUAL_RETAINED`, so it also appears in the residual inventory below. + */ +export type ReportDisposition = "transformed" | "removed" | "blocked" | "retained"; /** * Per-category coverage, for one of the 18 Safe Harbor categories (45 CFR §164.514(b)(2)(i)(A)–(R)), @@ -97,10 +101,18 @@ export interface CategoryCoverage { } /** - * One entry in the **retained-quasi-identifier residual inventory**: a coarse identifying element the - * pass **deliberately kept** for analytic utility and **recorded** as `DEID_RESIDUAL_RETAINED`: a - * year-only date, a retained safe 3-digit ZIP prefix, an exact age ≤ 89. These are exactly the residuals - * an expert reasons about under the §164.514(b)(2)(ii) actual-knowledge test. + * One entry in the **retained-quasi-identifier residual inventory**: an identifying element the pass + * **deliberately kept** for analytic utility and **recorded** as `DEID_RESIDUAL_RETAINED`. Two things + * land here: + * + * - a **coarse residual** left by a generalization: a year-only date, a retained safe 3-digit ZIP + * prefix, an exact age ≤ 89; + * - a **whole value kept by the profile's retention set**: under a limited-data-set preset, an + * admission / discharge / service date or an encounter or order number, which a Safe-Harbor-labelled + * policy removes instead. + * + * These are exactly the residuals an expert reasons about under the §164.514(b)(2)(ii) + * actual-knowledge test, and the second kind is the stronger one: it is a full, unreduced value. * * @example * ```ts @@ -144,6 +156,8 @@ export interface DispositionSummary { readonly removed: number; /** Loci failed closed (blocked; value withheld). */ readonly blocked: number; + /** Identifying loci the profile's retention set kept unchanged (each one a recorded residual). */ + readonly retained: number; /** Of the transformed values, how many retained a coarse residual (`DEID_RESIDUAL_RETAINED`). */ readonly residualRetained: number; /** Free-text loci blocked by default (`DEID_FREETEXT_BLOCKED`). */ @@ -335,6 +349,7 @@ function coverageFor( transformed: 0, removed: 0, blocked: 0, + retained: 0, }; const transforms = new Set(); const codes = new Set(); @@ -369,6 +384,7 @@ function summarize(entries: readonly DeidManifestEntry[]): DispositionSummary { transformed: 0, removed: 0, blocked: 0, + retained: 0, residualRetained: 0, freeTextBlocked: 0, freeTextConsumerRedacted: 0, @@ -514,7 +530,8 @@ export function formatExpertDeterminationSupportReport( ); const d = report.dispositionSummary; lines.push( - `- Dispositions: transformed: ${String(d.transformed)}, removed: ${String(d.removed)}, blocked: ${String(d.blocked)}` + + `- Dispositions: transformed: ${String(d.transformed)}, removed: ${String(d.removed)}, blocked: ${String(d.blocked)},` + + ` retained: ${String(d.retained)}` + ` (free-text blocked: ${String(d.freeTextBlocked)}, consumer-redacted: ${String(d.freeTextConsumerRedacted)})`, ); lines.push(""); @@ -537,14 +554,18 @@ export function formatExpertDeterminationSupportReport( "_None recorded._ Coarse residuals (year-only dates, safe 3-digit ZIP prefixes, exact", ); lines.push( - "ages ≤ 89) would appear here when a generalization keeps one. Clinical values retained", + "ages ≤ 89) appear here when a generalization keeps one, and so does every whole value a", + ); + lines.push( + "profile's retention set kept. Clinical values retained untouched by the over-scrub guard", ); lines.push( - "untouched by the over-scrub guard are not identifiers and are not enumerated in the", + "are not identifiers and are not enumerated. What is NOT enumerated anywhere is a field", ); lines.push( - "value-free manifest; see each format's retained-segment limitations for residual dates.", + "inside a retained structure that no locus map reaches: those are named, per format, in the", ); + lines.push("published limitations, and they are not visible to this report."); } else { lines.push( "These are residual identifying elements the pass kept for utility. They are an actual-knowledge", diff --git a/src/retention.ts b/src/retention.ts new file mode 100644 index 0000000..15cd8ef --- /dev/null +++ b/src/retention.ts @@ -0,0 +1,100 @@ +/** + * **Policy-scoped retention classes**: the named, enumerated groups of *identifying* loci that a + * profile may deliberately pass through **untouched**, and that a stricter profile removes. + * + * This is the format-agnostic half of a decision the HL7 v2 adapter (and, in time, the other format + * adapters) makes at extraction: whether a locus that lives inside an otherwise-retained clinical or + * visit structure is *kept* or *acted on*. It exists because the two standards this library models + * draw the line in **different places**, and reading either one off the other is a compliance trap: + * + * - **Safe Harbor, §164.514(b)(2)(i)(C)**, requires removal of *all elements of dates (except year) + * directly related to an individual*, and names **admission and discharge dates** among them. The + * catch-all, **(R)**, then requires removal of *any other unique identifying number, characteristic, + * or code*, which is what a visit/encounter number and a placer/filler order number are. So under a + * Safe-Harbor-labelled policy **neither class may be retained**, and the built-in Safe Harbor + * profile retains **nothing**. + * - **A limited data set, §164.514(e)(2)**, excludes an enumerated list of **sixteen** direct + * identifiers. That list contains **no dates at all** and **no catch-all**: it names names, postal + * address detail, telephone, fax, email, social security, medical record, health plan beneficiary, + * account, certificate/licence, vehicle, device, URL, IP, biometric, and full-face-image + * identifiers, and stops. Admission, discharge and service dates, and an encounter or order number, + * are therefore **permitted to remain** in a limited data set. + * + * **Retention is never silent.** A locus retained under a class is still recorded in the value-free + * manifest as a `DEID_RESIDUAL_RETAINED` residual, so it reaches the retained-quasi-identifier + * inventory a determiner reads. A retained identifier that no artifact names is invisible twice over, + * which is the failure mode this module is designed against. + * + * **The default is retain-nothing.** Every entry point defaults to an empty retention set, so an + * options bag that never mentions retention gets the strict treatment (fail closed). + * + * @packageDocumentation + */ + +/** + * The stable registry of retention classes. `key === value` so the full set survives an + * `Object.values(...)` snapshot into a stability tripwire. These are part of the public contract: + * renaming or removing one is a **breaking change**; new classes may be **added** in a later release. + * + * @example + * ```ts + * import { RETAINED_LOCUS_CLASSES } from "@cosyte/deid"; + * + * RETAINED_LOCUS_CLASSES.ENCOUNTER_DATES; // => "encounter-dates" + * ``` + */ +export const RETAINED_LOCUS_CLASSES = { + /** + * Patient-related **dates** carried by retained clinical / visit structures: admission, discharge, + * observation / service, and diagnosis dates. These are elements of dates directly related to the + * individual, so **Safe Harbor removes them** (only the year may remain); a limited data set + * **may keep them**, because §164.514(e)(2)'s direct-identifier list names no date. + */ + ENCOUNTER_DATES: "encounter-dates", + /** + * **Encounter and order identifiers**: the visit / encounter number, and the placer and filler order + * numbers. These are not one of the seventeen concrete Safe Harbor identifier types, so Safe Harbor + * reaches them through the **(R)** catch-all and they are blocked; §164.514(e)(2) has **no** + * catch-all, so a limited data set **may keep them**. + */ + ENCOUNTER_IDENTIFIERS: "encounter-identifiers", +} as const; + +/** + * A value from {@link RETAINED_LOCUS_CLASSES}: the class a profile lists to keep, and an adapter + * checks before it passes a locus through. + * + * @example + * ```ts + * import { RETAINED_LOCUS_CLASSES, type RetainedLocusClass } from "@cosyte/deid"; + * + * const cls: RetainedLocusClass = RETAINED_LOCUS_CLASSES.ENCOUNTER_IDENTIFIERS; + * ``` + */ +export type RetainedLocusClass = + (typeof RETAINED_LOCUS_CLASSES)[keyof typeof RETAINED_LOCUS_CLASSES]; + +/** The empty retention set: the fail-closed default every entry point uses. */ +export const NO_RETAINED_LOCI: readonly RetainedLocusClass[] = Object.freeze([]); + +/** + * Test whether a retention class is enabled by a (possibly absent) retention set. An absent or empty + * set retains **nothing**, so a caller that never mentions retention gets the strict treatment. + * + * @param classes - The retention classes a profile or options bag enabled, if any. + * @param cls - The class to test. + * @returns `true` only when `classes` is present and lists `cls`. + * @example + * ```ts + * import { retains, RETAINED_LOCUS_CLASSES } from "@cosyte/deid"; + * + * retains(undefined, RETAINED_LOCUS_CLASSES.ENCOUNTER_DATES); // => false (fail closed) + * retains(["encounter-dates"], RETAINED_LOCUS_CLASSES.ENCOUNTER_DATES); // => true + * ``` + */ +export function retains( + classes: readonly RetainedLocusClass[] | undefined, + cls: RetainedLocusClass, +): boolean { + return classes !== undefined && classes.includes(cls); +} diff --git a/test/fixtures/hl7/adt-a03.hl7 b/test/fixtures/hl7/adt-a03.hl7 new file mode 100644 index 0000000..7d4a044 --- /dev/null +++ b/test/fixtures/hl7/adt-a03.hl7 @@ -0,0 +1,7 @@ +MSH|^~\&|ADTAPP|ADTFAC|EHRAPP|EHRFAC|20200110120000||ADT^A03|SYNTHMSG003|P|2.5 +PID|1||ZZMRN003^^^HOSP^MR||ZZENCFAMILY^ZZENCGIVEN||19900215|M|||ZZENCSTREET^^ZZENCCITY^MA^90210||5550000020|||||ZZACCT300 +PV1|1|I|WARD^ROOM^BED|||||ATTEND^DOCFAMILY^DOCGIVEN|||||||||||ZZVISIT700^^^HOSP^VN|||||||||||||||||||||||||20200103040500|20200109060700 +ORC|NW|ZZPLACER700|ZZFILLER700 +OBR|1|ZZPLACER700|ZZFILLER700|2951-2^Sodium^LN|||20200104080000 +OBX|1|NM|2951-2^Sodium^LN|1|140|mmol/L|135-145|N|||F +DG1|1|I10|E11.9^Type 2 diabetes mellitus^I10||20200105090000 diff --git a/test/hl7/deidentify-hl7.test.ts b/test/hl7/deidentify-hl7.test.ts index 9ae8763..a4c93ce 100644 --- a/test/hl7/deidentify-hl7.test.ts +++ b/test/hl7/deidentify-hl7.test.ts @@ -16,9 +16,12 @@ import { parseHL7 } from "@cosyte/hl7"; import { DEID_DISPOSITION_CODES, FATAL_CODES, + LIMITED_DATA_SET_PROFILE, SAFE_HARBOR_CATEGORIES, + SAFE_HARBOR_PROFILE, createDeidContext, defineDeidPolicy, + profileOptions, } from "../../src/index.js"; import { deidentifyHl7 } from "../../src/hl7/index.js"; @@ -39,8 +42,9 @@ const ctx = createDeidContext({ key: "hl7-test-key", patientId: "patient-1" }); /** * The patient / relative / guarantor / insured PHI sentinels seeded across `adt-a01.hl7`. Every one * must be GONE after a de-id pass. (Retained-by-design values, the MSH/EVN envelope timestamps, the - * PV1 provider, the insurer org name/address and the OBR order numbers, are deliberately not `ZZ`-tagged - * and are not in this list; provider/order loci are an explicit Phase-2 scope boundary.) + * PV1 provider name and the insurer org name/address, are deliberately not `ZZ`-tagged and are not in + * this list. The encounter dates and the encounter / order identifiers are NOT among them: they are + * removed under this profile, and `adt-a03.hl7` is the fixture that seeds and proves it.) */ const ADT_SENTINELS: readonly string[] = [ "ZZMRN001", @@ -385,3 +389,139 @@ describe("deidentifyHl7, fatal + policy + immutability", () => { expect(manifest).toEqual([]); }); }); + +/** + * The encounter loci carved out of the RETAINED visit / order / diagnosis segments, seeded in + * `adt-a03.hl7`. Each row is one of the seven the two profiles must treat differently. + * + * Under `safe-harbor` every one must be GONE: the dates are elements of dates directly related to the + * individual (§164.514(b)(2)(i)(C) names admission and discharge in the regulation text itself, and + * permits only the year), and the visit / order numbers are unique identifying codes the (R) catch-all + * reaches. Under `limited-data-set` every one must SURVIVE BYTE-IDENTICAL: §164.514(e)(2)'s list of + * sixteen direct identifiers names no date and carries no catch-all, so a limited data set may keep + * them. + * + * Both directions are asserted, because a removal test that passes because the detector never found the + * locus is indistinguishable from one that passes because the locus was removed. + */ +const ENCOUNTER_LOCI: readonly { readonly what: string; readonly locus: string; readonly seeded: string }[] = + [ + { what: "visit number", locus: "PV1-19", seeded: "ZZVISIT700" }, + { what: "admit date/time", locus: "PV1-44", seeded: "20200103040500" }, + { what: "discharge date/time", locus: "PV1-45", seeded: "20200109060700" }, + { what: "placer order number (order control)", locus: "ORC-2", seeded: "ZZPLACER700" }, + { what: "filler order number (order control)", locus: "ORC-3", seeded: "ZZFILLER700" }, + { what: "placer order number (observation request)", locus: "OBR-2", seeded: "ZZPLACER700" }, + { what: "filler order number (observation request)", locus: "OBR-3", seeded: "ZZFILLER700" }, + { what: "observation (service) date/time", locus: "OBR-7", seeded: "20200104080000" }, + { what: "diagnosis date/time", locus: "DG1-5", seeded: "20200105090000" }, + ]; + +describe("deidentifyHl7, the encounter loci inside retained segments (§164.514(b)(2) vs §164.514(e))", () => { + it("PRE-CONDITION: every seeded encounter value is really present in the original wire", () => { + const raw = loadFixture("adt-a03"); + const missing = ENCOUNTER_LOCI.filter((l) => !raw.includes(l.seeded)); + expect(missing).toEqual([]); + }); + + it("safe-harbor: every encounter date and encounter/order identifier is REMOVED from the wire", () => { + const wire = deidentifyHl7( + parseHL7(loadFixture("adt-a03")), + profileOptions(SAFE_HARBOR_PROFILE, ctx), + ).document.toString(); + const survivors = ENCOUNTER_LOCI.filter((l) => wire.includes(l.seeded)); + expect(survivors).toEqual([]); + }); + + it("safe-harbor: the dates keep their YEAR (permitted) and the identifiers are blocked as (R)", () => { + const { document, manifest } = deidentifyHl7( + parseHL7(loadFixture("adt-a03")), + profileOptions(SAFE_HARBOR_PROFILE, ctx), + ); + // Dates: generalized to the four-digit year, recorded as a coarse residual. + for (const path of ["PV1.44.1", "PV1.45.1", "OBR.7.1", "DG1.5.1"]) { + expect(document.get(path)).toBe("2020"); + } + for (const locus of ["PV1-44", "PV1-45", "OBR-7", "DG1-5"]) { + const entry = manifest.find((m) => m.locus === locus); + expect(entry?.category).toBe(C.DATES); + expect(entry?.disposition).toBe("transformed"); + expect(entry?.code).toBe(D.DEID_RESIDUAL_RETAINED); + } + // Identifiers: the whole field is gone, assigning authority and all, recorded as blocked (R). + for (const path of ["PV1.19.1", "ORC.2.1", "ORC.3.1", "OBR.2.1", "OBR.3.1"]) { + expect(document.get(path)).toBeUndefined(); + } + for (const locus of ["PV1-19", "ORC-2", "ORC-3", "OBR-2", "OBR-3"]) { + const entry = manifest.find((m) => m.locus === locus); + expect(entry?.category).toBe(C.OTHER_UNIQUE_ID); + expect(entry?.disposition).toBe("blocked"); + expect(entry?.code).toBe(D.DEID_LOCUS_BLOCKED); + } + }); + + it("limited-data-set: every one SURVIVES byte-identical, and every one is RECORDED", () => { + const original = parseHL7(loadFixture("adt-a03")); + const { document, manifest } = deidentifyHl7( + parseHL7(loadFixture("adt-a03")), + profileOptions(LIMITED_DATA_SET_PROFILE, ctx), + ); + const wire = document.toString(); + const removed = ENCOUNTER_LOCI.filter((l) => !wire.includes(l.seeded)); + expect(removed).toEqual([]); + // Byte-identical, not merely "the token appears somewhere": the composite survives intact, so the + // visit number keeps its assigning authority and identifier-type components. + for (const path of ["PV1.19.1", "PV1.19.4", "PV1.19.5", "PV1.44.1", "OBR.2.1", "DG1.5.1"]) { + expect(document.get(path)).toBe(original.get(path)); + } + // Recorded, every one: a kept identifier that no artifact names is invisible twice over. + for (const locus of ["PV1-19", "PV1-44", "PV1-45", "ORC-2", "ORC-3", "OBR-2", "OBR-3", "OBR-7", "DG1-5"]) { + const entry = manifest.find((m) => m.locus === locus); + expect(entry?.disposition).toBe("retained"); + expect(entry?.transform).toBe("retain"); + expect(entry?.code).toBe(D.DEID_RESIDUAL_RETAINED); + } + }); + + it("limited-data-set still removes the PATIENT identifiers §164.514(e)(2) DOES name", () => { + const wire = deidentifyHl7( + parseHL7(loadFixture("adt-a03")), + profileOptions(LIMITED_DATA_SET_PROFILE, ctx), + ).document.toString(); + // Names, address detail, phone, and the raw medical record number are all on the limited-data-set + // exclusion list, so keeping the encounter loci must not have loosened any of them. + for (const s of ["ZZENCFAMILY", "ZZENCGIVEN", "ZZENCSTREET", "ZZENCCITY", "5550000020", "ZZMRN003"]) { + expect(wire.includes(s)).toBe(false); + } + }); + + it("retention is OPT-IN: a bare options bag keeps nothing (fail closed)", () => { + // A consumer who builds options by hand from the limited-data-set POLICY, without the profile's + // retention set, gets the strict treatment rather than a silent pass-through. + const wire = deidentifyHl7(parseHL7(loadFixture("adt-a03")), { + policy: LIMITED_DATA_SET_PROFILE.policy, + context: ctx, + }).document.toString(); + const survivors = ENCOUNTER_LOCI.filter((l) => wire.includes(l.seeded)); + expect(survivors).toEqual([]); + }); + + it("an absent encounter field is not invented as a locus (no manifest row, no over-scrub)", () => { + // adt-a01 carries a PV1 with no visit number and no admit/discharge date. + const { manifest } = deidentifyHl7(parseHL7(loadFixture("adt-a01")), { + context: ctx, + }); + expect(manifest.filter((m) => m.locus.startsWith("PV1-"))).toEqual([]); + }); + + it("the clinical value alongside the encounter loci survives byte-identical (over-scrub guard)", () => { + const original = parseHL7(loadFixture("adt-a03")); + const { document } = deidentifyHl7( + parseHL7(loadFixture("adt-a03")), + profileOptions(SAFE_HARBOR_PROFILE, ctx), + ); + for (const path of ["OBX[0].5", "OBX[0].6", "OBX[0].3.1", "OBR.4.1", "DG1.3.1", "PV1.3.1"]) { + expect(document.get(path)).toBe(original.get(path)); + } + }); +}); From 5392ee27ef784c0fd43e753e6bdae0c0bbb0eb23 Mon Sep 17 00:00:00 2001 From: Noah Schatz Date: Tue, 11 Aug 2026 14:31:53 +0000 Subject: [PATCH 2/6] park (addendum): the leak-corpus seeding, also UNVERIFIED and UNGRADED 119 lines the worker flushed to test/corpus/leak-corpus.test.ts after the first park commit. Same status as everything else on this branch: verify never ran, no refuter graded it. Worth keeping specifically because seeding this corpus was the DEEPER half of the slice. The gate was structurally blind to the class its own roadmap says it must catch -- no HL7 fixture carried PV1-19/44/45 -- so a fix without a fixture would leave nothing to catch a regression. --- test/corpus/leak-corpus.test.ts | 119 ++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) diff --git a/test/corpus/leak-corpus.test.ts b/test/corpus/leak-corpus.test.ts index 248fbd1..07ef55b 100644 --- a/test/corpus/leak-corpus.test.ts +++ b/test/corpus/leak-corpus.test.ts @@ -42,6 +42,9 @@ import { serializeDicom } from "@cosyte/dicom"; import { buildExpertDeterminationSupportReport, createDeidContext, + LIMITED_DATA_SET_PROFILE, + profileOptions, + SAFE_HARBOR_PROFILE, type DeidManifestEntry, } from "../../src/index.js"; import { deidentifyHl7 } from "../../src/hl7/index.js"; @@ -118,6 +121,57 @@ function hl7Case(): CorpusCase { }; } +/** + * The HL7 v2 **encounter** case: the loci carved out of the RETAINED visit / order / diagnosis + * segments, which no other fixture in this corpus carries. Without it the headline gate is + * structurally blind to this whole class: the leak sweep can only report on sentinels a fixture + * actually seeds, so an adapter that passes an admission date or a visit number straight through + * produces exactly the same green as one that removes it. + * + * Swept under the **Safe Harbor** profile, where §164.514(b)(2)(i)(C) permits only the year of a date + * directly related to the individual and (R) reaches the encounter and order numbers. The other + * direction, that the limited-data-set profile still CARRIES them, is asserted below: a corpus that + * only ever proves absence cannot tell a removal from a preset that scrubs everything. + */ +function hl7EncounterCase(): CorpusCase { + const ctx = createDeidContext({ key: "hl7-enc-corpus", patientId: "p-hl7-enc" }); + const raw = hl7Wire("adt-a03"); + const { document, manifest } = deidentifyHl7( + parseHL7(raw), + profileOptions(SAFE_HARBOR_PROFILE, ctx), + ); + return { + name: "hl7-encounter", + deidWire: document.toString(), + manifest, + originalWire: raw, + sentinels: [ + // The encounter loci: the visit number, the admit / discharge / observation / diagnosis dates, + // and the placer + filler order numbers. + "ZZVISIT700", + "20200103040500", + "20200109060700", + "20200104080000", + "20200105090000", + "ZZPLACER700", + "ZZFILLER700", + // The patient demographics carried alongside them, so this fixture is a whole document. + "ZZMRN003", + "ZZENCFAMILY", + "ZZENCGIVEN", + "ZZENCSTREET", + "ZZENCCITY", + "90210", + "5550000020", + "ZZACCT300", + "19900215", + ], + // Distinctive clinical survivors only: the LOINC code, the unit, the diagnosis code, and the + // patient-location text. The bare "140" is excluded for the same reason as the other cases. + survivors: ["2951-2", "mmol/L", "E11.9", "WARD"], + }; +} + // ── C-CDA ───────────────────────────────────────────────────────────────────────────────────────── function ccdaCase(): CorpusCase { const ctx = createDeidContext({ key: "ccda-corpus", patientId: "p-ccda" }); @@ -269,6 +323,7 @@ function dicomCase(): CorpusCase { const CASES: readonly CorpusCase[] = [ hl7Case(), + hl7EncounterCase(), ccdaCase(), fhirCase(), x12Case(), @@ -322,6 +377,70 @@ describe("corpus non-vacuity, the sweep and the corpus both have teeth", () => { } }); +/** + * The **other direction**, and it is not optional. Every gate above proves a value is ABSENT, and a + * detector that reports zero can be a gap rather than a clearance: an adapter that dropped the whole + * PV1 segment, or a fixture whose loci the extractor never reached, would pass all of them. This + * proves the same loci are still CARRIED by the profile that is entitled to carry them, so the + * absences above are a decision the policy made and not an accident of the harness. + */ +describe("encounter loci positive control, the limited-data-set profile still carries them", () => { + const ENCOUNTER_SENTINELS: readonly string[] = [ + "ZZVISIT700", + "20200103040500", + "20200109060700", + "20200104080000", + "20200105090000", + "ZZPLACER700", + "ZZFILLER700", + ]; + + const ctx = createDeidContext({ key: "hl7-lds-corpus", patientId: "p-hl7-lds" }); + const { document, manifest } = deidentifyHl7( + parseHL7(hl7Wire("adt-a03")), + profileOptions(LIMITED_DATA_SET_PROFILE, ctx), + ); + const wire = document.toString(); + + it("every encounter sentinel survives the limited-data-set pass", () => { + const removed = ENCOUNTER_SENTINELS.filter((s) => !wire.includes(s)); + expect(removed).toEqual([]); + }); + + it("every surviving encounter locus is RECORDED as a retained residual", () => { + const retained = manifest.filter((m) => m.disposition === "retained"); + expect(retained.map((m) => m.locus).sort()).toEqual([ + "DG1-5", + "OBR-2", + "OBR-3", + "OBR-7", + "ORC-2", + "ORC-3", + "PV1-19", + "PV1-44", + "PV1-45", + ]); + expect(retained.every((m) => m.code === "DEID_RESIDUAL_RETAINED")).toBe(true); + }); + + it("and each one reaches the determiner's residual inventory in the support report", () => { + const report = buildExpertDeterminationSupportReport(manifest, { + policy: LIMITED_DATA_SET_PROFILE.policy, + }); + const inventoried = new Set(report.retainedQuasiIdentifiers.map((r) => r.locus)); + for (const locus of ["PV1-19", "PV1-44", "PV1-45", "OBR-2", "OBR-3", "OBR-7", "ORC-2", "ORC-3", "DG1-5"]) { + expect(inventoried.has(locus)).toBe(true); + } + expect(report.dispositionSummary.retained).toBe(9); + }); + + it("the patient identifiers §164.514(e)(2) DOES name are still gone", () => { + for (const s of ["ZZENCFAMILY", "ZZENCGIVEN", "ZZENCSTREET", "ZZENCCITY", "5550000020", "ZZMRN003"]) { + expect(wire.includes(s)).toBe(false); + } + }); +}); + describe("consolidated over-scrub corpus, clinical/financial values survive", () => { for (const c of CASES) { if (c.survivors.length > 0) { From cf1e2f8df7e9f07855a4718de3637848d26d6391 Mon Sep 17 00:00:00 2001 From: Noah Schatz Date: Tue, 11 Aug 2026 14:35:55 +0000 Subject: [PATCH 3/6] fix(hl7): make policy safe-harbor actually Safe Harbor for encounter loci (DEID-SAFE-HARBOR-DATES) Founder decision, 2026-08-11, on a measured finding: under `policy: "safe-harbor"` seven identifying loci survived BYTE-IDENTICAL with no manifest entry at all, because RETAIN_SEGMENTS retains a segment whole and nothing carved its identifying fields back out: PV1-19 visit number, PV1-44 admit, PV1-45 discharge, OBR-7 observation date, DG1-5 diagnosis date, and the placer/filler order numbers (OBR-2/3, ORC-2/3). The manifest read only `PID-3[0]:transformed PID-5:removed PID-7:transformed`. HIPAA Safe Harbor 164.514(b)(2)(i)(C) requires removal of all elements of dates except year directly related to an individual, and names admission and discharge dates in the regulation text itself; the visit and order numbers are unique identifying codes the (R) catch-all reaches. A policy NAMED safe-harbor that returns them is a compliance trap for anyone who trusts the name. WHAT CHANGES - New format-agnostic retention model (src/retention.ts): two named classes, `encounter-dates` and `encounter-identifiers`, each grounded in a DIFFERENT regulation. 164.514(e)(2)'s limited-data-set exclusion list enumerates sixteen direct identifiers and contains no date and NO CATCH-ALL, which is exactly why these two classes may stay in a limited data set and may not stay under Safe Harbor. - SAFE_HARBOR_PROFILE retains neither class; LIMITED_DATA_SET_PROFILE retains both. - New HL7 carve-out table (RETAINED_LOCUS_RULES): retaining a SEGMENT no longer means retaining every field in it. Under safe-harbor the four dates generalize to their year and the five identifier loci are blocked as (R). - Anything still retained is RECORDED. A kept locus emits a manifest entry with disposition `retained`, transform `retain`, code DEID_RESIDUAL_RETAINED, so it reaches the Expert-Determination support report's residual inventory instead of being invisible twice over. DeidManifestEntry.disposition and ReportDisposition gain "retained"; DispositionSummary gains a `retained` count. - widen-never-narrow now covers retention, and it reads the OPPOSITE way round from a transform override: DROPPING a class removes more (allowed), ADDING one keeps more (fatal DEID_PROFILE_INVALID). It is a subset test, not a rank comparison. - Fail closed by default: retention rides on DeidOptions and only profileOptions() puts it there, so a bare options bag, or one built by hand from a profile's `policy`, keeps nothing. The engine's retention branch also runs AFTER the clinical / free-text / unrecognized-structure guards, so the flag can never keep prose or unknown structure. - New applier edit kind `none`: a retained locus is not written back at all, so it survives byte-identical rather than being flattened into a single-component field. THE LEAK CORPUS WAS STRUCTURALLY BLIND TO THIS CLASS, WHICH IS THE DEEPER DEFECT No HL7 fixture carried PV1-19/44/45 at all, so the headline gate could not have caught a regression: a sweep only reports on sentinels a fixture actually seeds. test/fixtures/hl7/adt-a03.hl7 seeds all seven plus the usual demographics, and is wired into test/corpus/leak-corpus.test.ts. Proven RED on the pre-fix source: the new fixture + tests over f8b7f56's src/ fail 8 tests, including the headline leak gate reporting all seven encounter sentinels surviving. POSITIVE CONTROL IN BOTH DIRECTIONS, because a detector that reports zero can be a gap rather than a clearance: safe-harbor removes all seven, limited-data-set still carries all seven byte-identical (composites intact, assigning authority and all) while still removing the patient identifiers 164.514(e)(2) does name. DOCS docs-content/limitations.md's "loci absent from the parser models fail closed" row read as the opposite of the truth for a retained segment, so it is DELETED rather than caveated. Its place is taken by an accurate section that names what is retained AND names what is still passed through and recorded nowhere: SPM-17, PV1-7/8, OBR-16. The support report and its guide no longer redirect a determiner to "each format's retained-segment notes" for something the report can now inventory. --- README.md | 17 ++- docs-content/guides-expert-determination.md | 17 ++- docs-content/guides-hl7.md | 33 ++--- docs-content/limitations.md | 64 +++++++-- probe.local.mts | 34 ----- test/profile.test.ts | 141 ++++++++++++++++++++ 6 files changed, 234 insertions(+), 72 deletions(-) delete mode 100644 probe.local.mts diff --git a/README.md b/README.md index e40c9dc..909483b 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,7 @@ const { document, manifest } = deidentifyHl7(parseHL7(rawMessage), { context }); document.toString(); // spec-clean, de-identified HL7 wire // PID-5 (name), NK1/GT1/IN1/IN2 relatives, SSN, phone → removed; MRN/account → consistent surrogate; // DOB → year; address → safe 3-digit ZIP. OBX-5/NTE free text and Z-segments fail closed (blocked). +// Admit/discharge/observation/diagnosis dates → year; visit and order numbers blocked as (R). // Structured clinical OBX values, units, codes, and statuses survive untouched. ``` @@ -112,11 +113,17 @@ so a known patient-identity segment absent from the map (e.g. **MRG** prior name coded / date); narrative (`TX`/`FT`), ambiguous String (`ST`), and any empty/unknown OBX-2 fail closed, as do **NTE-3** comments. Structured clinical values, units, codes, and statuses survive untouched. +**Inside a retained segment**, the identifying loci are carved back out: under a Safe-Harbor-labelled +policy the admit (PV1-44), discharge (PV1-45), observation (OBR-7) and diagnosis (DG1-5) dates keep only +their **year**, and the visit number (PV1-19) with the placer and filler order numbers (OBR-2/3, ORC-2/3) +are **blocked** as the (R) catch-all. A profile that names their retention class, as the limited-data-set +preset does, keeps them **unchanged and recorded**. + **Known limitations (this release).** Free text is block-by-default (no built-in scrub; opt-in BYO -redaction: see [Free text](#free-text-block-by-default--byo-redaction)); within **retained** clinical / -visit segments, patient-related _dates_ (OBR/DG1/PV1 timestamps), _visit identifiers_ (PV1-19), and -_provider_ names (PV1-7/8, OBR-16) are **not** de-identified; the address generalization keeps only the -Safe Harbor 3-digit ZIP. +redaction: see [Free text](#free-text-block-by-default--byo-redaction)); a field of a retained segment +that the carve-out does not name is **not** de-identified and is recorded nowhere (the specimen +collection date SPM-17, and the _provider_ names PV1-7/8 and OBR-16); the address generalization keeps +only the Safe Harbor 3-digit ZIP. ## De-identify a C-CDA document @@ -371,7 +378,7 @@ Determination** (§164.514(b)(1), a qualified statistician's risk judgment). `@c the latter and **never renders** it. `buildExpertDeterminationSupportReport(manifest)` structures the value-free manifest into what an expert reasons about: per-locus dispositions, coverage across all 18 categories, and the **retained-quasi-identifier inventory** (year-only dates, safe 3-digit ZIP prefixes, -exact ages ≤ 89), then hands it over. +exact ages ≤ 89, and any whole value a profile's retention set kept), then hands it over. ```ts import { buildExpertDeterminationSupportReport } from "@cosyte/deid"; diff --git a/docs-content/guides-expert-determination.md b/docs-content/guides-expert-determination.md index ac1ddbc..bcd0016 100644 --- a/docs-content/guides-expert-determination.md +++ b/docs-content/guides-expert-determination.md @@ -80,12 +80,17 @@ report.totals.categoriesActedOn; // => 2 report.retainedQuasiIdentifiers[0].locus; // => "PID-7" ``` -The **retained-quasi-identifier inventory** is the residual an expert cares about most: the coarse -identifying elements the pass kept for utility and **recorded** as `DEID_RESIDUAL_RETAINED`, a year-only -date, a safe 3-digit ZIP prefix, an exact age ≤ 89. These are the §164.514(b)(2)(ii) actual-knowledge -considerations. (Clinical values retained untouched by the over-scrub guard are not identifiers and are -not enumerated in the value-free manifest; consult each format's retained-segment notes for residual -dates in retained clinical segments.) +The **retained-quasi-identifier inventory** is the residual an expert cares about most: the identifying +elements the pass kept for utility and **recorded** as `DEID_RESIDUAL_RETAINED`. Two kinds land there: a +coarse residual left by a generalization (a year-only date, a safe 3-digit ZIP prefix, an exact age +≤ 89), and a **whole unreduced value** a profile's retention set kept, such as the admission, discharge +and service dates and the encounter and order numbers a limited-data-set preset carries. The second kind +is the stronger residual, and it is inventoried here rather than left to a footnote. These are the +§164.514(b)(2)(ii) actual-knowledge considerations. + +Clinical values retained untouched by the over-scrub guard are not identifiers and are not enumerated. +What is enumerated **nowhere** is a field inside a retained structure that no locus map reaches; those +are named per format in the published limitations, and this report cannot see them. ## The optional k-anonymity indicator: caller-supplied, descriptive only diff --git a/docs-content/guides-hl7.md b/docs-content/guides-hl7.md index 44e632f..f182db4 100644 --- a/docs-content/guides-hl7.md +++ b/docs-content/guides-hl7.md @@ -30,8 +30,8 @@ import { createDeidContext } from "@cosyte/deid"; const context = createDeidContext({ key: process.env.DEID_KEY! }); const { document, manifest } = deidentifyHl7(parseHL7(rawMessage), { context }); -document.toString(); // spec-clean, de-identified HL7 wire -manifest; // value-free audit: category + locus + disposition, never a value +document.toString(); // spec-clean, de-identified HL7 wire +manifest; // value-free audit: category + locus + disposition, never a value ``` A keyed transform (MRN / account / beneficiary pseudonymization) requires a `context`; calling without @@ -40,14 +40,16 @@ surrogate. ## What is located, and how it is transformed -| Segment | Loci | Transform | -|---|---|---| -| **PID** | name (5/6/9), DOB (7/29), address (11), SSN (19), phone (13/14), driver's licence (20), MRN/account/mother-id (2/3/4/18/21), county (12), birth place (23) | names/phone/SSN/licence **removed**; MRN/account → consistent **surrogate** (keyed HMAC); DOB → **year**; ZIP → safe **3-digit** (or `000`); county/birth place fail closed | -| **NK1 / GT1 / IN1 / IN2** | relatives / guarantor / insured names, addresses, phones, SSNs, DOBs, member/policy/Medicare/Medicaid ids | same category transforms: Safe Harbor removes identifiers of **relatives, employers, and household members**, not only the patient | -| **OBX-5, NTE-3** | narrative / ambiguous free text (OBX-5 unless OBX-2 types it structured) | **fail closed**: blocked, never regex-scrubbed | -| **MRG / ACC / FAM / PEO / PDA** | known patient-identity / relative / geographic segments absent from the map | **fail closed**: blocked (e.g. a merge message's prior name + MRN) | -| **Z-segments / unknown structure** | every populated field | **fail closed**: blocked | -| Retained clinical/administrative segments (an explicit allow-list: OBR, ORC, AL1, DG1, PV1, RX*, …) | n/a | **retained untouched** (the over-scrub guard) | +| Segment | Loci | Transform | +| ---------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **PID** | name (5/6/9), DOB (7/29), address (11), SSN (19), phone (13/14), driver's licence (20), MRN/account/mother-id (2/3/4/18/21), county (12), birth place (23) | names/phone/SSN/licence **removed**; MRN/account → consistent **surrogate** (keyed HMAC); DOB → **year**; ZIP → safe **3-digit** (or `000`); county/birth place fail closed | +| **NK1 / GT1 / IN1 / IN2** | relatives / guarantor / insured names, addresses, phones, SSNs, DOBs, member/policy/Medicare/Medicaid ids | same category transforms: Safe Harbor removes identifiers of **relatives, employers, and household members**, not only the patient | +| **OBX-5, NTE-3** | narrative / ambiguous free text (OBX-5 unless OBX-2 types it structured) | **fail closed**: blocked, never regex-scrubbed | +| **MRG / ACC / FAM / PEO / PDA** | known patient-identity / relative / geographic segments absent from the map | **fail closed**: blocked (e.g. a merge message's prior name + MRN) | +| **Z-segments / unknown structure** | every populated field | **fail closed**: blocked | +| **PV1-19, OBR-2/3, ORC-2/3** | visit number, placer + filler order numbers, inside retained segments | **blocked** as the (R) catch-all (kept, and recorded, only under a profile that names the class) | +| **PV1-44/45, OBR-7, DG1-5** | admit, discharge, observation and diagnosis dates, inside retained segments | → **year** (§164.514(b)(2)(i)(C) names admission and discharge); kept whole, and recorded, only under a profile that names the class | +| Retained clinical/administrative segments (an explicit allow-list: OBR, ORC, AL1, DG1, PV1, RX\*, …) | every field except the two rows above | **retained untouched** (the over-scrub guard) | A recognized segment is retained **only** if it is on the explicit retain-list; anything else fails closed. OBX-5 is retained only when OBX-2 positively types it as a structured clinical value (numeric, @@ -59,9 +61,9 @@ handled differently, structurally, from the parser's typing. ## The two guarantees -- **No leak.** Every seeded PHI sentinel across PID/NK1/GT1/IN1/IN2, the free-text loci, and Z-segments - is gone from the serialized output. An unmapped locus that could carry PHI is blocked, never passed - through in the clear. +- **No leak.** Every seeded PHI sentinel across PID/NK1/GT1/IN1/IN2, the encounter dates and order + identifiers, the free-text loci, and Z-segments is gone from the serialized output under the Safe + Harbor profile. An unmapped locus that could carry PHI is blocked, never passed through in the clear. - **No over-scrub.** Structured clinical OBX values, units, LOINC/coded observation identifiers, reference ranges, and result statuses are retained byte-identical: the de-identifier never degenerates into a blanket-blanking "safe but useless" scrubber. @@ -69,8 +71,9 @@ handled differently, structurally, from the parser's typing. ## Known limitations (this release) - Free text is **block-only**: there is no built-in NLP scrub. -- Within **retained** clinical / visit segments, patient-related **dates** (OBR / DG1 / PV1 timestamps), - **visit identifiers** (PV1-19), and **provider** names (PV1-7/8, OBR-16) are **not** de-identified. +- Within **retained** clinical / visit segments, a field the carve-out above does not name is **not** + de-identified and is **not recorded**: the specimen collection date (SPM-17) and the **provider** + names (PV1-7/8, OBR-16). Retaining a segment is not auditing every field in it. - The address generalization keeps only the Safe Harbor 3-digit ZIP (the permitted state is also dropped, conservative, never a leak). diff --git a/docs-content/limitations.md b/docs-content/limitations.md index 2f4eb6c..39df865 100644 --- a/docs-content/limitations.md +++ b/docs-content/limitations.md @@ -25,17 +25,16 @@ Read this page before you rely on the library for anything that leaves your cont ## What it does NOT do -| It does **not**… | Because… | -|---|---| -| Certify HIPAA de-identification | The library **transforms and evidences**; it never certifies. Output is *"Safe-Harbor-transformed per the configured policy,"* never *"de-identified."* | -| Discharge the §164.514(b)(2)(ii) **actual-knowledge** clause | That is an organizational judgment about what a recipient knows: the library surfaces the residual (kept year, safe-3-digit ZIP) so a human can apply it, but cannot make it. | -| Render or certify **Expert Determination** (§164.514(b)(1)) | *"The risk is very small"* is a qualified statistician's contextual judgment about a dataset **and its recipient**. The [ED support report](#expert-determination) emits value-free facts as **input**; `determination` is always `null` and it computes no risk score. | -| De-identify **free text / narrative** | Free-text loci (HL7 OBX-5/NTE, C-CDA narrative ``, FHIR notes/`div`, X12 MSG/NTE, NCPDP free text) are **blocked by default**. A [BYO redactor](#free-text) is **consumer-asserted**, never the library's guarantee; a naive built-in regex scrub is deliberately **refused** as a false-safety hazard. | -| Clean **DICOM burned-in pixels** or full-face images (category Q) | v1 is **metadata-only** (delegated PS3.15 Annex E). Burned-in annotation raises `DICOM_BURNED_IN_ANNOTATION_NOT_REMOVED` and `burnedInAnnotationHazard`; pixel decode is a future `@cosyte/dicom-pixel`. **Do not release an image on metadata alone.** | -| Handle **NCPDP SCRIPT** ePrescribing | **Deferred.** The current parser surface (lossy serialize + an address-less `Patient` model) cannot support a faithful structural de-id, so SCRIPT is **not** silently half-handled. NCPDP **Telecom** is supported. | -| Handle loci/formats absent from the parser models | A locus the parser does not model, or a format not in the suite, **fails closed**, never silently passed. Vendor-proprietary loci absent from public specs are deferred, not invented. | -| Guarantee against a determined re-identification attack | De-identification reduces risk to the regulatory bar; it is not a cryptographic guarantee. **Key custody is the consumer's**: a leaked HMAC key or date-shift offset re-identifies. | -| Do anything the manifest does not record | If it is not in the manifest, the library did not do it. The manifest is the complete, value-free audit. | +| It does **not**… | Because… | +| ----------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Certify HIPAA de-identification | The library **transforms and evidences**; it never certifies. Output is _"Safe-Harbor-transformed per the configured policy,"_ never _"de-identified."_ | +| Discharge the §164.514(b)(2)(ii) **actual-knowledge** clause | That is an organizational judgment about what a recipient knows: the library surfaces the residual (kept year, safe-3-digit ZIP) so a human can apply it, but cannot make it. | +| Render or certify **Expert Determination** (§164.514(b)(1)) | _"The risk is very small"_ is a qualified statistician's contextual judgment about a dataset **and its recipient**. The [ED support report](#expert-determination) emits value-free facts as **input**; `determination` is always `null` and it computes no risk score. | +| De-identify **free text / narrative** | Free-text loci (HL7 OBX-5/NTE, C-CDA narrative ``, FHIR notes/`div`, X12 MSG/NTE, NCPDP free text) are **blocked by default**. A [BYO redactor](#free-text) is **consumer-asserted**, never the library's guarantee; a naive built-in regex scrub is deliberately **refused** as a false-safety hazard. | +| Clean **DICOM burned-in pixels** or full-face images (category Q) | v1 is **metadata-only** (delegated PS3.15 Annex E). Burned-in annotation raises `DICOM_BURNED_IN_ANNOTATION_NOT_REMOVED` and `burnedInAnnotationHazard`; pixel decode is a future `@cosyte/dicom-pixel`. **Do not release an image on metadata alone.** | +| Handle **NCPDP SCRIPT** ePrescribing | **Deferred.** The current parser surface (lossy serialize + an address-less `Patient` model) cannot support a faithful structural de-id, so SCRIPT is **not** silently half-handled. NCPDP **Telecom** is supported. | +| Guarantee against a determined re-identification attack | De-identification reduces risk to the regulatory bar; it is not a cryptographic guarantee. **Key custody is the consumer's**: a leaked HMAC key or date-shift offset re-identifies. | +| Do anything the manifest does not record | If it is not in the manifest, the library did not do it. The manifest is the complete, value-free audit. | ## Fail-closed posture @@ -50,11 +49,38 @@ A keyed transform with no key is a **fatal** `DEID_NO_KEY`, never a silent unkey unkeyed hash of an identifier is re-identifiable). A context configured with `maxShiftDays: 0` is a fatal `DEID_CONTEXT_INVALID`, a zero-bound shift is a guaranteed no-op, i.e. the original real dates. + + +## What is retained, and what "fails closed" does NOT cover + +**Fail-closed is a rule about _structures_, not about every field.** An unrecognized segment, resource, +loop, or extension is blocked. But a segment on a format's **retain-list** is passed through, and +retaining the structure is not the same as auditing every field inside it. Read this before you assume +an unmapped position was removed. + +Two things follow, and the second is the one that surprises people: + +- **The identifying loci inside retained HL7 v2 structures are carved back out and acted on.** Under a + Safe-Harbor-labelled policy the admit (PV1-44), discharge (PV1-45), observation (OBR-7) and diagnosis + (DG1-5) dates are reduced to their **year**, and the visit number (PV1-19) with the placer and filler + order numbers (OBR-2/3, ORC-2/3) are **blocked** as the (R) catch-all. §164.514(b)(2)(i)(C) names + admission and discharge dates in the regulation text itself. +- **A field of a retained structure that no locus map names is still passed through, and is recorded + nowhere.** Named, because a limitation you cannot enumerate is not a limitation: the specimen + collection date (SPM-17) and the attending / referring **provider** names (PV1-7/8, OBR-16). They are + not in the manifest, so they are not in the support report either. If your threat model includes + them, filter them yourself. + +Vendor-proprietary loci absent from public specs are deferred, **not invented**: a quirk is encoded +only when a real de-identified document grounds it. + + ## Policy profiles, and the Limited Data Set caveat - **`SAFE_HARBOR_PROFILE`**: the fail-closed default, dates generalized to year, the (R) catch-all - blocked. + blocked. It retains **no** identifying locus: an admission, discharge or service date keeps only its + year, and an encounter or order number is removed. - **`LIMITED_DATA_SET_PROFILE`**: a **research / longitudinal** preset that **date-shifts** dates (interval-preserving) rather than generalizing them. It is deliberately **less protective than Safe Harbor** for dates: a shifted-but-real date is still "an element of a date." Therefore it is **not** @@ -63,11 +89,24 @@ fatal `DEID_CONTEXT_INVALID`, a zero-bound shift is a guaranteed no-op, i.e. the its own, a HIPAA §164.514(e) Limited Data Set. Disclosing an actual Limited Data Set additionally requires a **Data Use Agreement**, which is the consumer's responsibility. + It also **keeps unchanged** the two classes §164.514(e)(2) permits and Safe Harbor does not: the + **encounter dates** (admission / discharge / service / diagnosis) and the **encounter and order + identifiers** (visit number, placer and filler order numbers). That list of sixteen direct + identifiers names no date and has no catch-all, which is exactly why these survive here and are + removed under Safe Harbor. Every one is still **recorded** as a `DEID_RESIDUAL_RETAINED` residual and + appears in the support report's inventory, so nothing is kept silently. + +`defineDeidProfile()`'s widen-never-narrow contract covers retention too, and it reads the opposite way +round from a transform override: a derived profile may **drop** a retained class (keep less, remove +more) but may never **add** one. Retention is also opt-in at the call: options built by hand from a +profile's `policy` alone keep nothing. + `defineDeidProfile()` derives a per-site profile under a **widen-never-narrow** contract: a site may move a category to an equal-or-stronger transform (more removal), but **never** re-weaken a category, a weakening override is a fatal `DEID_PROFILE_INVALID`. A site preset can only tighten the base. + ## Free text is the consumer's responsibility The library bundles **no** NLP/PHI detector. With no redactor, free-text loci are **blocked**. With a @@ -76,6 +115,7 @@ and is **not re-verified** by the library: "no findings" from a redactor is not structural PHI the adapters remove is unaffected either way. + ## The Expert-Determination report makes no determination The report is descriptive input a determiner consumes and documents; it reaches no conclusion: diff --git a/probe.local.mts b/probe.local.mts deleted file mode 100644 index cef069d..0000000 --- a/probe.local.mts +++ /dev/null @@ -1,34 +0,0 @@ -import { parseHL7 } from "@cosyte/hl7"; -import { deidentifyHl7 } from "./src/hl7/index.js"; -import { createDeidContext, SAFE_HARBOR_PROFILE, LIMITED_DATA_SET_PROFILE, profileOptions } from "./src/index.js"; - -const wire = [ - "MSH|^~\\&|A|B|C|D|20200101120000||ADT^A01|M1|P|2.5", - "PID|1||ZZMRN001^^^HOSP^MR||ZZFAM^ZZGIV||19850302|F", - "PV1|1|I|WARD^ROOM^BED|||||ATTEND^DOCFAMILY^DOCGIVEN|||||||||||ZZVISIT999^^^HOSP^VN|||||||||||||||||||||||||20200103040500|20200109060700", - "OBR|1|ZZPLACER777|ZZFILLER888|2951-2^Sodium^LN|||20200104080000", - "ORC|NW|ZZPLACER777|ZZFILLER888", - "DG1|1|I10|E11.9^Type 2 diabetes^I10||20200105090000", -].join("\r"); - -const ctx = createDeidContext({ key: "probe-key", patientId: "p1" }); - -for (const [label, profile] of [["safe-harbor", SAFE_HARBOR_PROFILE], ["limited-data-set", LIMITED_DATA_SET_PROFILE]] as const) { - const { document, manifest } = deidentifyHl7(parseHL7(wire), profileOptions(profile, ctx)); - const out = document.toString(); - console.log(`\n=== ${label} ===`); - const sentinels = { - "PV1-19 visit number": "ZZVISIT999", - "PV1-44 admit date": "20200103040500", - "PV1-45 discharge date": "20200109060700", - "OBR-7 observation date": "20200104080000", - "DG1-5 diagnosis date": "20200105090000", - "placer order number": "ZZPLACER777", - "filler order number": "ZZFILLER888", - }; - for (const [name, s] of Object.entries(sentinels)) { - console.log(` ${out.includes(s) ? "SURVIVES" : "gone "} ${name}`); - } - console.log(" manifest:", manifest.map((m) => `${m.locus}:${m.disposition}`).join(" ")); - console.log(" wire:", out.split("\r").join("\n ")); -} diff --git a/test/profile.test.ts b/test/profile.test.ts index 29f0de5..c49ed43 100644 --- a/test/profile.test.ts +++ b/test/profile.test.ts @@ -9,9 +9,12 @@ import { createDeidContext, defineDeidProfile, deidentify, + DeidError, FATAL_CODES, LIMITED_DATA_SET_PROFILE, profileOptions, + RETAINED_LOCUS_CLASSES, + retains, SAFE_HARBOR_CATEGORIES, SAFE_HARBOR_POLICY, SAFE_HARBOR_PROFILE, @@ -126,3 +129,141 @@ describe("profileOptions composition", () => { expect(opts.policy).toBe(SAFE_HARBOR_PROFILE.policy); }); }); + +describe("policy-scoped retention, and what widen-never-narrow means for it", () => { + const R = RETAINED_LOCUS_CLASSES; + + it("Safe Harbor retains NOTHING; the limited data set retains the two §164.514(e)(2) permits", () => { + expect(SAFE_HARBOR_PROFILE.retainedLoci).toEqual([]); + expect([...LIMITED_DATA_SET_PROFILE.retainedLoci].sort()).toEqual([ + "encounter-dates", + "encounter-identifiers", + ]); + }); + + it("retains() fails closed on an absent or empty set", () => { + expect(retains(undefined, R.ENCOUNTER_DATES)).toBe(false); + expect(retains([], R.ENCOUNTER_DATES)).toBe(false); + expect(retains([R.ENCOUNTER_DATES], R.ENCOUNTER_DATES)).toBe(true); + expect(retains([R.ENCOUNTER_DATES], R.ENCOUNTER_IDENTIFIERS)).toBe(false); + }); + + it("profileOptions carries the retention set; a hand-built options bag does not", () => { + expect(profileOptions(SAFE_HARBOR_PROFILE).retainedLoci).toEqual([]); + expect(profileOptions(LIMITED_DATA_SET_PROFILE).retainedLoci).toHaveLength(2); + // The fail-closed direction: reading the policy off a profile loses the retention set entirely. + const handBuilt = { policy: LIMITED_DATA_SET_PROFILE.policy }; + expect(handBuilt).not.toHaveProperty("retainedLoci"); + }); + + it("DROPPING a retained class is a WIDENING and is allowed (keep less, remove more)", () => { + const tighter = defineDeidProfile({ + name: "site-dates-only", + base: LIMITED_DATA_SET_PROFILE, + retainedLoci: [R.ENCOUNTER_DATES], + }); + expect(tighter.retainedLoci).toEqual([R.ENCOUNTER_DATES]); + + const strictest = defineDeidProfile({ + name: "site-retain-nothing", + base: LIMITED_DATA_SET_PROFILE, + retainedLoci: [], + }); + expect(strictest.retainedLoci).toEqual([]); + }); + + it("ADDING a retained class is a NARROWING and is REJECTED (fatal DEID_PROFILE_INVALID)", () => { + // Derived from Safe Harbor, which retains nothing: any class at all is an addition. + expect(() => + defineDeidProfile({ + name: "site-keeps-visit-numbers", + retainedLoci: [R.ENCOUNTER_IDENTIFIERS], + }), + ).toThrow(DeidError); + try { + defineDeidProfile({ name: "site-keeps-dates", retainedLoci: [R.ENCOUNTER_DATES] }); + expect.unreachable("adding a retention class must be rejected"); + } catch (err) { + expect((err as DeidError).code).toBe(FATAL_CODES.DEID_PROFILE_INVALID); + } + // And a profile derived from the LDS base may not add one it does not already have either: with + // both classes on the base there is nothing left to add, so drop one first and then try to re-add. + const dropped = defineDeidProfile({ + name: "site-dropped", + base: LIMITED_DATA_SET_PROFILE, + retainedLoci: [R.ENCOUNTER_DATES], + }); + expect(() => + defineDeidProfile({ + name: "site-re-added", + base: dropped, + retainedLoci: [R.ENCOUNTER_DATES, R.ENCOUNTER_IDENTIFIERS], + }), + ).toThrow(DeidError); + }); + + it("omitting retainedLoci inherits the base's set unchanged", () => { + const derived = defineDeidProfile({ name: "site-inherit", base: LIMITED_DATA_SET_PROFILE }); + expect([...derived.retainedLoci].sort()).toEqual( + [...LIMITED_DATA_SET_PROFILE.retainedLoci].sort(), + ); + }); + + it("a retained locus is passed through UNCHANGED and always RECORDED as a residual", () => { + const ctx = createDeidContext({ key: "retain-key", patientId: "p1" }); + const { document, manifest } = deidentify( + { + loci: [ + { + path: "PV1-44", + kind: "date", + category: C.DATES, + retainedByPolicy: true, + value: "20200103040500", + }, + ], + }, + profileOptions(LIMITED_DATA_SET_PROFILE, ctx), + ); + expect(document.loci[0]?.value).toBe("20200103040500"); + expect(document.loci[0]?.disposition).toBe("retained"); + expect(manifest[0]?.disposition).toBe("retained"); + expect(manifest[0]?.transform).toBe("retain"); + expect(manifest[0]?.code).toBe("DEID_RESIDUAL_RETAINED"); + }); + + it("the retention flag can never keep free text or unrecognized structure (guard order)", () => { + const ctx = createDeidContext({ key: "retain-key", patientId: "p1" }); + const { document, manifest } = deidentify( + { + loci: [ + { + path: "NTE-3", + kind: "freetext", + category: C.OTHER_UNIQUE_ID, + retainedByPolicy: true, + value: "ZZPROSE", + }, + { + path: "ZPI-1", + kind: "unknown", + category: C.OTHER_UNIQUE_ID, + retainedByPolicy: true, + value: "ZZUNKNOWN", + }, + { path: "PID-5", kind: "identifier", retainedByPolicy: true, value: "ZZUNCLASSIFIED" }, + ], + }, + profileOptions(LIMITED_DATA_SET_PROFILE, ctx), + ); + // All three fail closed regardless of the flag: the three guards run BEFORE retention. + expect(document.loci.map((l) => l.value)).toEqual([null, null, null]); + expect(manifest.every((m) => m.disposition === "blocked")).toBe(true); + }); + + it("`retain` is not policy-assignable: assigning it to a category fails closed to a block", () => { + expect(() => + defineDeidProfile({ name: "site-retain-names", transforms: { [C.NAMES]: "retain" } }), + ).toThrow(DeidError); + }); +}); From e2c200adb0f7434ee5e817f209a06dd091451bcf Mon Sep 17 00:00:00 2001 From: Noah Schatz Date: Tue, 11 Aug 2026 14:41:10 +0000 Subject: [PATCH 4/6] chore: exhaustive switch on the new transform, formatting, and the changeset `retain` joins the fail-closed arm of applyTransform explicitly rather than falling into `default`, so the exhaustiveness rule is satisfied by naming it and the comment records WHY reaching that arm must still block: it means a policy asked for retention per-category, which is not how retention is decided. verify.sh is green on this tree: typecheck, lint, format:check, check, phi-scan, check:no-internal-refs, test:coverage, build, attw, smoke. --- .changeset/safe-harbor-encounter-loci.md | 15 ++++++++ src/deidentify.ts | 9 +++-- src/hl7/index.ts | 6 +-- src/profile.ts | 6 +-- test/corpus/leak-corpus.test.ts | 21 ++++++++++- test/hl7/deidentify-hl7.test.ts | 48 +++++++++++++++++------- 6 files changed, 76 insertions(+), 29 deletions(-) create mode 100644 .changeset/safe-harbor-encounter-loci.md diff --git a/.changeset/safe-harbor-encounter-loci.md b/.changeset/safe-harbor-encounter-loci.md new file mode 100644 index 0000000..3778418 --- /dev/null +++ b/.changeset/safe-harbor-encounter-loci.md @@ -0,0 +1,15 @@ +--- +"@cosyte/deid": patch +--- + +The `safe-harbor` policy now actually removes the encounter loci Safe Harbor requires it to remove. Seven identifying values previously survived a `safe-harbor` pass byte-identical, with no manifest entry at all: the visit number (PV1-19), the admit and discharge dates (PV1-44/45), the observation date (OBR-7), the diagnosis date (DG1-5), and the placer and filler order numbers (OBR-2/3, ORC-2/3). Retaining an HL7 segment retained every field inside it, and nothing carved these back out. + +45 CFR 164.514(b)(2)(i)(C) requires removal of all elements of dates except year that are directly related to an individual, and names admission and discharge dates in the regulation text itself; a visit or order number is a unique identifying code the (R) catch-all reaches. A policy named `safe-harbor` that returned them was a trap for anyone who trusted the name, so this is a deliberate breaking change while the package is pre-alpha. + +**What changes.** Under `SAFE_HARBOR_PROFILE` the four dates now generalize to their year and the five identifier loci are blocked as category (R). Under `LIMITED_DATA_SET_PROFILE` all seven are kept unchanged, because 164.514(e)(2)'s limited-data-set exclusion list enumerates sixteen direct identifiers, contains no date, and has no catch-all. The split is expressed by two named retention classes, `encounter-dates` and `encounter-identifiers`, on the new `retainedLoci` field of a profile. + +**Anything still retained is now recorded.** A kept locus emits a manifest entry with disposition `retained`, transform `retain`, and code `DEID_RESIDUAL_RETAINED`, so it reaches the Expert-Determination support report's residual inventory rather than being invisible in both artifacts. `DeidManifestEntry["disposition"]` and `ReportDisposition` gain `"retained"`, and `DispositionSummary` gains a `retained` count. + +**`defineDeidProfile()`'s widen-never-narrow contract now covers retention, and it reads the opposite way round from a transform override:** dropping a retained class removes more and is allowed; adding one keeps more and is a fatal `DEID_PROFILE_INVALID`. It is a subset test, not a rank comparison. Retention is opt-in at the call site too: only `profileOptions()` carries it, so an options bag built by hand from a profile's `policy` keeps nothing. + +**Corrected claims.** The published limitations page claimed that loci absent from the parser models fail closed, which read as the opposite of the truth for a retained segment; that claim is deleted. In its place the page names what is retained, and names what is still passed through and recorded nowhere: the specimen collection date (SPM-17) and the provider names (PV1-7/8, OBR-16). Retaining a segment is not auditing every field in it. diff --git a/src/deidentify.ts b/src/deidentify.ts index d417473..1612c74 100644 --- a/src/deidentify.ts +++ b/src/deidentify.ts @@ -226,11 +226,14 @@ function applyTransform( }, }; case "byo-redact": + case "retain": case "block": default: - // `byo-redact` is not a category transform: free-text redaction is driven by the `redactor` - // option, not the policy map. If a policy assigns it (or `block`, or anything unknown) to a - // category, fail closed (block). + // Neither `byo-redact` nor `retain` is a category transform: free-text redaction is driven by + // the `redactor` option and retention by the profile's retention set, not by the policy map. If + // a policy assigns either (or `block`, or anything unknown) to a category, fail closed (block). + // `retain` in particular must never keep a value from here: reaching this arm means a policy + // asked for it per-category, which is not how retention is decided. return blocked(locus.path, category, DEID_DISPOSITION_CODES.DEID_LOCUS_BLOCKED); } } diff --git a/src/hl7/index.ts b/src/hl7/index.ts index 354a941..ec54dc7 100644 --- a/src/hl7/index.ts +++ b/src/hl7/index.ts @@ -113,9 +113,5 @@ export { } from "./extract.js"; export { applyHl7 } from "./apply.js"; export { RETAIN_SEGMENTS, RETAINED_LOCUS_RULES, type Hl7RetainedFieldRule } from "./retain.js"; -export { - RETAINED_LOCUS_CLASSES, - retains, - type RetainedLocusClass, -} from "../retention.js"; +export { RETAINED_LOCUS_CLASSES, retains, type RetainedLocusClass } from "../retention.js"; export { SAFE_HARBOR_CATEGORIES, type SafeHarborCategory } from "../categories.js"; diff --git a/src/profile.ts b/src/profile.ts index e4d870a..ffe9971 100644 --- a/src/profile.ts +++ b/src/profile.ts @@ -43,11 +43,7 @@ import { type TransformName, } from "./policy.js"; import { type FreeTextRedactor } from "./redactor.js"; -import { - NO_RETAINED_LOCI, - RETAINED_LOCUS_CLASSES, - type RetainedLocusClass, -} from "./retention.js"; +import { NO_RETAINED_LOCI, RETAINED_LOCUS_CLASSES, type RetainedLocusClass } from "./retention.js"; /** * The **protection rank** of a transform: higher means the residual is *less* identifying, so the diff --git a/test/corpus/leak-corpus.test.ts b/test/corpus/leak-corpus.test.ts index 07ef55b..5b42c24 100644 --- a/test/corpus/leak-corpus.test.ts +++ b/test/corpus/leak-corpus.test.ts @@ -428,14 +428,31 @@ describe("encounter loci positive control, the limited-data-set profile still ca policy: LIMITED_DATA_SET_PROFILE.policy, }); const inventoried = new Set(report.retainedQuasiIdentifiers.map((r) => r.locus)); - for (const locus of ["PV1-19", "PV1-44", "PV1-45", "OBR-2", "OBR-3", "OBR-7", "ORC-2", "ORC-3", "DG1-5"]) { + for (const locus of [ + "PV1-19", + "PV1-44", + "PV1-45", + "OBR-2", + "OBR-3", + "OBR-7", + "ORC-2", + "ORC-3", + "DG1-5", + ]) { expect(inventoried.has(locus)).toBe(true); } expect(report.dispositionSummary.retained).toBe(9); }); it("the patient identifiers §164.514(e)(2) DOES name are still gone", () => { - for (const s of ["ZZENCFAMILY", "ZZENCGIVEN", "ZZENCSTREET", "ZZENCCITY", "5550000020", "ZZMRN003"]) { + for (const s of [ + "ZZENCFAMILY", + "ZZENCGIVEN", + "ZZENCSTREET", + "ZZENCCITY", + "5550000020", + "ZZMRN003", + ]) { expect(wire.includes(s)).toBe(false); } }); diff --git a/test/hl7/deidentify-hl7.test.ts b/test/hl7/deidentify-hl7.test.ts index a4c93ce..1b7071b 100644 --- a/test/hl7/deidentify-hl7.test.ts +++ b/test/hl7/deidentify-hl7.test.ts @@ -404,18 +404,21 @@ describe("deidentifyHl7, fatal + policy + immutability", () => { * Both directions are asserted, because a removal test that passes because the detector never found the * locus is indistinguishable from one that passes because the locus was removed. */ -const ENCOUNTER_LOCI: readonly { readonly what: string; readonly locus: string; readonly seeded: string }[] = - [ - { what: "visit number", locus: "PV1-19", seeded: "ZZVISIT700" }, - { what: "admit date/time", locus: "PV1-44", seeded: "20200103040500" }, - { what: "discharge date/time", locus: "PV1-45", seeded: "20200109060700" }, - { what: "placer order number (order control)", locus: "ORC-2", seeded: "ZZPLACER700" }, - { what: "filler order number (order control)", locus: "ORC-3", seeded: "ZZFILLER700" }, - { what: "placer order number (observation request)", locus: "OBR-2", seeded: "ZZPLACER700" }, - { what: "filler order number (observation request)", locus: "OBR-3", seeded: "ZZFILLER700" }, - { what: "observation (service) date/time", locus: "OBR-7", seeded: "20200104080000" }, - { what: "diagnosis date/time", locus: "DG1-5", seeded: "20200105090000" }, - ]; +const ENCOUNTER_LOCI: readonly { + readonly what: string; + readonly locus: string; + readonly seeded: string; +}[] = [ + { what: "visit number", locus: "PV1-19", seeded: "ZZVISIT700" }, + { what: "admit date/time", locus: "PV1-44", seeded: "20200103040500" }, + { what: "discharge date/time", locus: "PV1-45", seeded: "20200109060700" }, + { what: "placer order number (order control)", locus: "ORC-2", seeded: "ZZPLACER700" }, + { what: "filler order number (order control)", locus: "ORC-3", seeded: "ZZFILLER700" }, + { what: "placer order number (observation request)", locus: "OBR-2", seeded: "ZZPLACER700" }, + { what: "filler order number (observation request)", locus: "OBR-3", seeded: "ZZFILLER700" }, + { what: "observation (service) date/time", locus: "OBR-7", seeded: "20200104080000" }, + { what: "diagnosis date/time", locus: "DG1-5", seeded: "20200105090000" }, +]; describe("deidentifyHl7, the encounter loci inside retained segments (§164.514(b)(2) vs §164.514(e))", () => { it("PRE-CONDITION: every seeded encounter value is really present in the original wire", () => { @@ -475,7 +478,17 @@ describe("deidentifyHl7, the encounter loci inside retained segments (§164.514( expect(document.get(path)).toBe(original.get(path)); } // Recorded, every one: a kept identifier that no artifact names is invisible twice over. - for (const locus of ["PV1-19", "PV1-44", "PV1-45", "ORC-2", "ORC-3", "OBR-2", "OBR-3", "OBR-7", "DG1-5"]) { + for (const locus of [ + "PV1-19", + "PV1-44", + "PV1-45", + "ORC-2", + "ORC-3", + "OBR-2", + "OBR-3", + "OBR-7", + "DG1-5", + ]) { const entry = manifest.find((m) => m.locus === locus); expect(entry?.disposition).toBe("retained"); expect(entry?.transform).toBe("retain"); @@ -490,7 +503,14 @@ describe("deidentifyHl7, the encounter loci inside retained segments (§164.514( ).document.toString(); // Names, address detail, phone, and the raw medical record number are all on the limited-data-set // exclusion list, so keeping the encounter loci must not have loosened any of them. - for (const s of ["ZZENCFAMILY", "ZZENCGIVEN", "ZZENCSTREET", "ZZENCCITY", "5550000020", "ZZMRN003"]) { + for (const s of [ + "ZZENCFAMILY", + "ZZENCGIVEN", + "ZZENCSTREET", + "ZZENCCITY", + "5550000020", + "ZZMRN003", + ]) { expect(wire.includes(s)).toBe(false); } }); From a1851a179ed298ffbb6bef31fc4bd96c4d40999a Mon Sep 17 00:00:00 2001 From: Noah Schatz Date: Tue, 11 Aug 2026 15:08:25 +0000 Subject: [PATCH 5/6] fix(deid): close the three INTRODUCED findings from conformance-refuter pass 1 Pass 1 returned VERDICT: REFUTED with three INTRODUCED findings inside the bar. All three are addressed here; the remedy diff is what pass 2 grades. F2 (blocker, criteria 2 and 3) -- the 164.514(e)(2) ground was ASSERTED, not checked. PV1-19 was hard-coded to the (R) catch-all without ever reading CX-5, although the adapter already ships categoryForIdentifierType() and uses it for PID-3. 164.514(e)(2)(vii) NAMES medical record numbers and (ix) NAMES account numbers, so for a routine `MR`-typed visit number the slice's own justification was false -- and under limited-data-set the pass republished in the clear the very MRN it had just pseudonymized at PID-3, in the same message, while the determiner's inventory labelled it OTHER_UNIQUE_ID. - PV1-19 now carries routeByTypeCode, resolving its category per repetition from CX-5 exactly as PID-3 does. An MR-typed visit number is pseudonymized to the SAME surrogate as PID-3; AN/SS/MA are transformed likewise; only a VN-typed or untyped value is the encounter identifier and retainable. - LIMITED_DATA_SET_DIRECT_IDENTIFIERS encodes the sixteen direct identifiers as data, so the citation is enforced rather than restated in prose. Exactly two of the eighteen categories are retainable: DATES and (R). - The refusal is defence in depth: the extractor will not mark such a locus, AND the engine refuses it anyway via isRetainableCategory(). F3 (major, criterion 7) -- the engine retained on the locus marker alone and never read options.retainedLoci, so DeidOptions.retainedLoci's own JSDoc ("absent or empty retains nothing") was false at the layer it was declared on, and the class contract was unenforceable because the marker was a class-less boolean. GenericLocus.retainedByPolicy?: boolean becomes retention?: RetainedLocusClass, and handleLocus now needs THREE keys to line up: an adapter proposal, the configured options naming that class, and a retainable category. F4 (major) -- a hand-built bag pairing the reserved `safe-harbor` label with a retention set returned all seven loci while the report printed `Policy: safe-harbor`. assertRetentionContract() makes that a fatal DEID_POLICY_INVALID, the retention analogue of the existing date-shift label guard, on the one route no profile-level check can see. F1 (major, criterion 8) -- the corrected honesty page replaced a true broad limitation with a false narrow enumeration: it named SPM-17 and PV1-7/8/OBR-16 as though complete, when a routine message carries 15 more unrecorded full-precision timestamps (EVN, PV2, PR1, RXA, RXD, FT1, TXA, SPM). Criterion 8 tests exactly this, and a new false claim is worse than the old one. The enumeration is DELETED and replaced by the true class statement in limitations.md, README.md, guides-hl7.md, src/hl7/index.ts, src/hl7/retain.ts and the changeset. F5 (minor) -- the inventory printed a kept year and a kept 14-digit timestamp identically. RetainedQuasiIdentifier now carries its transform, and the rendering says "coarse residual" or "whole value kept". F6 (minor) -- that the retention classes are HL7-only is now stated in the retention module and the changeset rather than left to be discovered. F7 and F8 are PRE-EXISTING (they reproduce on f8b7f56) and are reported as backlog lines, not fixed here. verify.sh green: 621 tests, typecheck, lint, format:check, check, phi-scan, check-no-internal-refs, coverage, build, attw, smoke. --- .changeset/safe-harbor-encounter-loci.md | 12 ++- README.md | 8 +- docs-content/guides-hl7.md | 8 +- docs-content/limitations.md | 11 +-- scripts/phi-allow-list.txt | 3 + src/deidentify.ts | 43 +++++++-- src/hl7/extract.ts | 45 +++++++++- src/hl7/index.ts | 10 ++- src/hl7/retain.ts | 36 ++++++-- src/index.ts | 3 + src/locus.ts | 18 ++-- src/report.ts | 22 ++++- src/retention.ts | 107 +++++++++++++++++++++++ test/corpus/leak-corpus.test.ts | 4 +- test/hl7/deidentify-hl7.test.ts | 102 ++++++++++++++++++++- test/profile.test.ts | 103 ++++++++++++++++++++-- test/report.test.ts | 39 +++++++++ 17 files changed, 513 insertions(+), 61 deletions(-) diff --git a/.changeset/safe-harbor-encounter-loci.md b/.changeset/safe-harbor-encounter-loci.md index 3778418..dc8f133 100644 --- a/.changeset/safe-harbor-encounter-loci.md +++ b/.changeset/safe-harbor-encounter-loci.md @@ -6,10 +6,14 @@ The `safe-harbor` policy now actually removes the encounter loci Safe Harbor req 45 CFR 164.514(b)(2)(i)(C) requires removal of all elements of dates except year that are directly related to an individual, and names admission and discharge dates in the regulation text itself; a visit or order number is a unique identifying code the (R) catch-all reaches. A policy named `safe-harbor` that returned them was a trap for anyone who trusted the name, so this is a deliberate breaking change while the package is pre-alpha. -**What changes.** Under `SAFE_HARBOR_PROFILE` the four dates now generalize to their year and the five identifier loci are blocked as category (R). Under `LIMITED_DATA_SET_PROFILE` all seven are kept unchanged, because 164.514(e)(2)'s limited-data-set exclusion list enumerates sixteen direct identifiers, contains no date, and has no catch-all. The split is expressed by two named retention classes, `encounter-dates` and `encounter-identifiers`, on the new `retainedLoci` field of a profile. +**What changes.** Under `SAFE_HARBOR_PROFILE` the four dates now generalize to their year and the five identifier loci are removed as category (R). Under `LIMITED_DATA_SET_PROFILE` all seven are kept unchanged, because 164.514(e)(2)'s limited-data-set exclusion list enumerates sixteen direct identifiers, contains no date, and has no catch-all. The split is expressed by two named retention classes, `encounter-dates` and `encounter-identifiers`, on the new `retainedLoci` field of a profile. -**Anything still retained is now recorded.** A kept locus emits a manifest entry with disposition `retained`, transform `retain`, and code `DEID_RESIDUAL_RETAINED`, so it reaches the Expert-Determination support report's residual inventory rather than being invisible in both artifacts. `DeidManifestEntry["disposition"]` and `ReportDisposition` gain `"retained"`, and `DispositionSummary` gains a `retained` count. +**Retention takes three independent keys, and a missing one always means the transform runs.** The adapter must propose a class for the locus; the configured options must list that class, so an adapter can never retain anything by itself and an options bag that omits `retainedLoci` keeps nothing; and the resolved category must be one a limited data set may carry at all. That last key is the one that matters most in practice: `PV1-19` is a CX list, and a visit-number field routinely carries a medical record or account number typed as such by the standard's own CX-5 identifier-type code. Both are named by 164.514(e)(2), so both are now routed through the identifier-type code and transformed, and an `MR`-typed visit number gets the _same_ keyed surrogate as the matching PID-3 entry rather than being republished in the clear beside it. `LIMITED_DATA_SET_DIRECT_IDENTIFIERS` and `isRetainableCategory()` are exported so the rule is inspectable: exactly two of the eighteen categories are retainable. -**`defineDeidProfile()`'s widen-never-narrow contract now covers retention, and it reads the opposite way round from a transform override:** dropping a retained class removes more and is allowed; adding one keeps more and is a fatal `DEID_PROFILE_INVALID`. It is a subset test, not a rank comparison. Retention is opt-in at the call site too: only `profileOptions()` carries it, so an options bag built by hand from a profile's `policy` keeps nothing. +**A policy carrying the reserved `safe-harbor` label may not retain at all**, whatever the options bag says. That is a fatal `DEID_POLICY_INVALID`, the retention analogue of the guard that stops a date-shifting policy wearing the same label, and it closes the hand-built-options route no profile-level check can see. -**Corrected claims.** The published limitations page claimed that loci absent from the parser models fail closed, which read as the opposite of the truth for a retained segment; that claim is deleted. In its place the page names what is retained, and names what is still passed through and recorded nowhere: the specimen collection date (SPM-17) and the provider names (PV1-7/8, OBR-16). Retaining a segment is not auditing every field in it. +**Anything still retained is now recorded.** A kept locus emits a manifest entry with disposition `retained`, transform `retain`, and code `DEID_RESIDUAL_RETAINED`, so it reaches the Expert-Determination support report's residual inventory rather than being invisible in both artifacts. `DeidManifestEntry["disposition"]` and `ReportDisposition` gain `"retained"`, `DispositionSummary` gains a `retained` count, and each inventory row now carries its `transform` so a kept year is distinguishable from a kept full-precision timestamp. + +**`defineDeidProfile()`'s widen-never-narrow contract now covers retention, and it reads the opposite way round from a transform override:** dropping a retained class removes more and is allowed; adding one keeps more and is a fatal `DEID_PROFILE_INVALID`. It is a subset test, not a rank comparison. + +**Corrected claims.** The published limitations page claimed that loci absent from the parser models fail closed, which read as the opposite of the truth for a retained segment; that claim is deleted. In its place the page states the true class: every field of a retained segment that the carve-out does not name is still passed through and recorded nowhere, still including full-precision timestamps in EVN, PV2, PR1, RXA, RXD, FT1, TXA and SPM and the provider names in PV1-7/8 and OBR-16. The carve-out narrows that class; it does not close it. Note also that the retention classes are read by the HL7 v2 adapter only: passing a retention set to the other five adapters changes nothing there. diff --git a/README.md b/README.md index 909483b..c0d753e 100644 --- a/README.md +++ b/README.md @@ -120,10 +120,10 @@ are **blocked** as the (R) catch-all. A profile that names their retention class preset does, keeps them **unchanged and recorded**. **Known limitations (this release).** Free text is block-by-default (no built-in scrub; opt-in BYO -redaction: see [Free text](#free-text-block-by-default--byo-redaction)); a field of a retained segment -that the carve-out does not name is **not** de-identified and is recorded nowhere (the specimen -collection date SPM-17, and the _provider_ names PV1-7/8 and OBR-16); the address generalization keeps -only the Safe Harbor 3-digit ZIP. +redaction: see [Free text](#free-text-block-by-default--byo-redaction)); **every** field of a retained +segment that the carve-out does not name is **not** de-identified and is recorded nowhere, which still +includes full-precision timestamps in EVN, PV2, PR1, RXA, RXD, FT1, TXA and SPM and the _provider_ names +in PV1-7/8 and OBR-16, among others; the address generalization keeps only the Safe Harbor 3-digit ZIP. ## De-identify a C-CDA document diff --git a/docs-content/guides-hl7.md b/docs-content/guides-hl7.md index f182db4..3105587 100644 --- a/docs-content/guides-hl7.md +++ b/docs-content/guides-hl7.md @@ -71,9 +71,11 @@ handled differently, structurally, from the parser's typing. ## Known limitations (this release) - Free text is **block-only**: there is no built-in NLP scrub. -- Within **retained** clinical / visit segments, a field the carve-out above does not name is **not** - de-identified and is **not recorded**: the specimen collection date (SPM-17) and the **provider** - names (PV1-7/8, OBR-16). Retaining a segment is not auditing every field in it. +- Within **retained** clinical / visit segments, **every** field the carve-out above does not name is + **not** de-identified and is **not recorded**. That still includes full-precision timestamps in EVN, + PV2, PR1, RXA, RXD, FT1, TXA and SPM, and the **provider** names in PV1-7/8 and OBR-16, among others. + Retaining a segment is not auditing every field in it, and the carve-out narrows this class rather + than closing it. - The address generalization keeps only the Safe Harbor 3-digit ZIP (the permitted state is also dropped, conservative, never a leak). diff --git a/docs-content/limitations.md b/docs-content/limitations.md index 39df865..bdac21f 100644 --- a/docs-content/limitations.md +++ b/docs-content/limitations.md @@ -65,11 +65,12 @@ Two things follow, and the second is the one that surprises people: (DG1-5) dates are reduced to their **year**, and the visit number (PV1-19) with the placer and filler order numbers (OBR-2/3, ORC-2/3) are **blocked** as the (R) catch-all. §164.514(b)(2)(i)(C) names admission and discharge dates in the regulation text itself. -- **A field of a retained structure that no locus map names is still passed through, and is recorded - nowhere.** Named, because a limitation you cannot enumerate is not a limitation: the specimen - collection date (SPM-17) and the attending / referring **provider** names (PV1-7/8, OBR-16). They are - not in the manifest, so they are not in the support report either. If your threat model includes - them, filter them yourself. +- **Every other field of a retained structure is still passed through untouched, and recorded + nowhere.** The carve-out above narrows this class; it does not close it. Full-precision timestamps + survive in EVN, PV2, PR1, RXA, RXD, FT1, TXA and SPM, and the attending / referring **provider** + names survive in PV1-7/8 and OBR-16, among others. None of them is in the manifest, so none is in the + support report either. **Do not read the named loci above as the complete set of what a retained + structure can carry.** If your threat model includes these, filter them yourself. Vendor-proprietary loci absent from public specs are deferred, **not invented**: a quirk is encoded only when a real de-identified document grounds it. diff --git a/scripts/phi-allow-list.txt b/scripts/phi-allow-list.txt index 00570a6..6978733 100644 --- a/scripts/phi-allow-list.txt +++ b/scripts/phi-allow-list.txt @@ -221,6 +221,9 @@ ID ZZPLACER700 ID ZZFILLER700 ID ZZACCT300 ID 5550000020 +# The PV1-19 identifier-type-routing cases (test/hl7/deidentify-hl7.test.ts), written inline. +ID ZZMRN500 +ID ZZVISIT500 # Inline HL7 identifiers in test modules / scripts/smoke.mjs: the five identifier-type placeholders # of the PID-3 repetition test, the surviving MRN of the MRG merge case, and the smoke MRN. ID I1 diff --git a/src/deidentify.ts b/src/deidentify.ts index 1612c74..ac60146 100644 --- a/src/deidentify.ts +++ b/src/deidentify.ts @@ -27,7 +27,12 @@ import type { DeidDocument, GenericLocus, TransformedLocus } from "./locus.js"; import { ManifestBuilder, type DeidManifestEntry, type DeidResult } from "./manifest.js"; import { resolvePolicy, type DeidPolicy, type TransformName } from "./policy.js"; import type { FreeTextRedactor } from "./redactor.js"; -import type { RetainedLocusClass } from "./retention.js"; +import { + assertRetentionContract, + isRetainableCategory, + retains, + type RetainedLocusClass, +} from "./retention.js"; import { dateShift, generalizeAge, @@ -300,6 +305,7 @@ function handleLocus( policy: DeidPolicy, context: DeidContext | undefined, redactor: FreeTextRedactor | undefined, + retainedLoci: readonly RetainedLocusClass[] | undefined, ): LocusOutcome { // Over-scrub guard: a clinical value is not an identifier, retain it untouched. if (locus.kind === "clinical") { @@ -318,10 +324,23 @@ function handleLocus( DEID_DISPOSITION_CODES.DEID_LOCUS_BLOCKED, ); } - // The configured profile's retention set deliberately keeps this class of identifying locus. It is - // reached only AFTER the three fail-closed guards above, so free text and unrecognized structure can - // never be retained by this route however an adapter marks them; and it is always recorded. - if (locus.retainedByPolicy === true) { + // Retention needs THREE independent keys to line up, and any one of them missing means the policy + // transform runs instead: + // 1. the adapter proposed a class for this locus; + // 2. the CONFIGURED OPTIONS list that class (an adapter cannot retain anything by itself, and an + // options bag that never mentions retention keeps nothing); + // 3. the resolved category is one a limited data set may carry at all: never one of the sixteen + // direct identifiers §164.514(e)(2) enumerates. A visit-number field routinely carries a + // medical record or account number, typed as such by the standard's own identifier-type code, + // and keeping THAT would republish in the clear the very identifier the pass pseudonymized + // elsewhere in the same document. + // It is also reached only AFTER the three fail-closed guards above, so free text and unrecognized + // structure can never be retained however an adapter marks them; and a kept locus is always recorded. + if ( + locus.retention !== undefined && + retains(retainedLoci, locus.retention) && + isRetainableCategory(locus.category) + ) { return retainedResidual(locus, locus.category); } return applyTransform(policy.transforms[locus.category], locus, locus.category, context); @@ -338,7 +357,8 @@ function handleLocus( * @param options - The policy and (for keyed transforms) the key context. * @returns The frozen {@link DeidResult}: transformed document + value-free manifest. * @throws {@link DeidError} `EMPTY_INPUT` if the model is null or carries no locus list; `DEID_NO_KEY` - * if a keyed transform is required but no key context was supplied. + * if a keyed transform is required but no key context was supplied; `DEID_POLICY_INVALID` if a + * `safe-harbor`-labelled policy is asked to retain an identifying locus. * @example * ```ts * import { deidentify, SAFE_HARBOR_CATEGORIES } from "@cosyte/deid"; @@ -361,11 +381,20 @@ export function deidentify( throw new DeidError(FATAL_CODES.EMPTY_INPUT, "de-identify requires a model with a loci array"); } const policy = resolvePolicy(options.policy); + // Fail closed on the label: a "safe-harbor"-labelled policy may not retain, however the options bag + // was built. This is the one route no profile-level check can see. + assertRetentionContract(policy.name, options.retainedLoci); const builder = new ManifestBuilder(); const loci: TransformedLocus[] = []; for (const locus of inputLoci) { - const outcome = handleLocus(locus, policy, options.context, options.redactor); + const outcome = handleLocus( + locus, + policy, + options.context, + options.redactor, + options.retainedLoci, + ); loci.push( Object.freeze({ path: locus.path, diff --git a/src/hl7/extract.ts b/src/hl7/extract.ts index 1a6f3de..d192d12 100644 --- a/src/hl7/extract.ts +++ b/src/hl7/extract.ts @@ -23,9 +23,9 @@ import { type Hl7Message, type Segment } from "@cosyte/hl7"; import { SAFE_HARBOR_CATEGORIES } from "../categories.js"; import { safeLocusToken } from "../derived-token.js"; import type { GenericLocus } from "../locus.js"; -import { retains, type RetainedLocusClass } from "../retention.js"; +import { isRetainableCategory, retains, type RetainedLocusClass } from "../retention.js"; import { HL7_LOCUS_MAP, categoryForIdentifierType, type Hl7FieldRule } from "./locus-map.js"; -import { RETAIN_SEGMENTS, RETAINED_LOCUS_RULES } from "./retain.js"; +import { RETAIN_SEGMENTS, RETAINED_LOCUS_RULES, type Hl7RetainedFieldRule } from "./retain.js"; /** * How the applier writes a transformed locus back onto the cloned raw tree. `none` writes **nothing**: @@ -234,16 +234,53 @@ function extractRetainedLoci( ): void { const rules = RETAINED_LOCUS_RULES[type]; if (rules === undefined) return; + const classEnabled = (rule: Hl7RetainedFieldRule): boolean => + retains(retainedLoci, rule.retention); + for (const rule of rules) { if (!hasContent(seg, rule.field)) continue; - const kept = retains(retainedLoci, rule.retention); + + if (rule.routeByTypeCode === true) { + // A CX list: one locus per repetition, category read from the CX-5 identifier-type code, so an + // `MR`/`AN`/`SS`-typed value in a visit-number field is transformed like the identifier it is + // (and gets the SAME keyed surrogate as the matching PID-3 entry) instead of being retained. + const reps = seg.field(rule.field).repetitions.length; + for (let rep = 0; rep < reps; rep += 1) { + const idNumber = componentValue(seg, rule.field, rep, 1); // CX.1 + if (idNumber.length === 0) continue; + const category = categoryForIdentifierType( + componentValue(seg, rule.field, rep, 5), // CX.5 + rule.category, + ); + const kept = classEnabled(rule) && isRetainableCategory(category); + push( + out, + { + path: fieldPath(type, occ, rule.field, rep), + kind: rule.kind, + category, + ...(kept ? { retention: rule.retention } : {}), + value: idNumber, + }, + { + segIndex: seg.absoluteIndex, + field: rule.field, + rep, + edit: kept ? "none" : "id-number", + }, + ); + } + continue; + } + + const kept = classEnabled(rule) && isRetainableCategory(rule.category); push( out, { path: fieldPath(type, occ, rule.field), kind: rule.kind, category: rule.category, - ...(kept ? { retainedByPolicy: true } : {}), + ...(kept ? { retention: rule.retention } : {}), value: seg.field(rule.field).value, }, { diff --git a/src/hl7/index.ts b/src/hl7/index.ts index ec54dc7..825bf6c 100644 --- a/src/hl7/index.ts +++ b/src/hl7/index.ts @@ -26,10 +26,12 @@ * the visit number (PV1-19) and the placer / filler order numbers (OBR-2/3, ORC-2/3) are **blocked** as * the (R) catch-all. A profile that names their retention class keeps them **unchanged and recorded**. * - * **Known limitations.** Free text is block-only (no scrub); a field of a retained segment that the - * carve-out does not name is still passed through untouched and unrecorded (the specimen collection - * date SPM-17, and the provider names PV1-7/8 and OBR-16); the address generalization keeps only the - * Safe Harbor 3-digit ZIP and conservatively drops the (permitted) state as well. + * **Known limitations.** Free text is block-only (no scrub); **every** field of a retained segment that + * the carve-out does not name is still passed through untouched and unrecorded, which continues to + * include full-precision timestamps in EVN, PV2, PR1, RXA, RXD, FT1, TXA and SPM and the provider names + * in PV1-7/8 and OBR-16, among others: the carve-out narrows this class, it does not close it. The + * address generalization keeps only the Safe Harbor 3-digit ZIP and conservatively drops the + * (permitted) state as well. * * @packageDocumentation */ diff --git a/src/hl7/retain.ts b/src/hl7/retain.ts index 2173442..31e5d40 100644 --- a/src/hl7/retain.ts +++ b/src/hl7/retain.ts @@ -20,10 +20,11 @@ * their retention class, and even then they are **recorded**. * * **Documented limitation, and it is narrower than it was but real.** A field inside a retained segment - * that is on **neither** list is still passed through untouched and is **not** recorded anywhere: the - * specimen collection date (SPM-17), the attending / referring *provider* names (PV1-7/8, OBR-16), and - * every other unmapped position. Forgetting a clinical segment here fails **safe**: it is blocked, not - * leaked. Forgetting a *field* of a retained segment does not. + * that is on **neither** list is still passed through untouched and is **not** recorded anywhere. That + * remains a large class, not a short list: full-precision timestamps in EVN, PV2, PR1, RXA, RXD, FT1, + * TXA and SPM, the attending / referring *provider* names (PV1-7/8, OBR-16), and every other unmapped + * position. Forgetting a clinical segment here fails **safe**: it is blocked, not leaked. Forgetting a + * *field* of a retained segment does not. * * @packageDocumentation */ @@ -146,10 +147,24 @@ export interface Hl7RetainedFieldRule { readonly field: number; /** The retention class a profile must name for this locus to survive. */ readonly retention: RetainedLocusClass; - /** The Safe Harbor category the locus carries when the policy acts on it. */ + /** + * The Safe Harbor category the locus carries when the policy acts on it. For a `routeByTypeCode` + * field this is only the **fallback**: the real category is read from the identifier-type code. + */ readonly category: SafeHarborCategory; /** `date` generalizes to year under Safe Harbor; `identifier` is blocked as the (R) catch-all. */ readonly kind: "date" | "identifier"; + /** + * For a **CX** field: resolve the category **per repetition** from the CX-5 identifier-type code + * (HL7 Table 0203), exactly as PID-3 does, and write back only the id-number component. + * + * **This is a safety requirement, not a nicety.** PV1-19 is a CX, and a visit-number field routinely + * carries an `MR` medical record number or an `AN` account number: both are named by + * §164.514(e)(2), so both must be transformed rather than retained even under a limited-data-set + * preset. Reading the code is what stops the pass republishing, in the clear, the very identifier it + * pseudonymized at PID-3 in the same message. + */ + readonly routeByTypeCode?: boolean; } const R = RETAINED_LOCUS_CLASSES; @@ -176,8 +191,15 @@ const C_OTHER = SAFE_HARBOR_CATEGORIES.OTHER_UNIQUE_ID; export const RETAINED_LOCUS_RULES: Readonly> = Object.freeze({ PV1: [ - // PV1-19 Visit Number (CX): the encounter identifier. - { field: 19, retention: R.ENCOUNTER_IDENTIFIERS, category: C_OTHER, kind: "identifier" }, + // PV1-19 Visit Number (CX): the encounter identifier, but only when the CX-5 type code does not + // say it is really an MRN / account / SSN / beneficiary number. + { + field: 19, + retention: R.ENCOUNTER_IDENTIFIERS, + category: C_OTHER, + kind: "identifier", + routeByTypeCode: true, + }, // PV1-44 Admit Date/Time (TS) and PV1-45 Discharge Date/Time (TS). { field: 44, retention: R.ENCOUNTER_DATES, category: C_DATES, kind: "date" }, { field: 45, retention: R.ENCOUNTER_DATES, category: C_DATES, kind: "date" }, diff --git a/src/index.ts b/src/index.ts index 94ca4b1..1b2053f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -91,6 +91,9 @@ export { export { RETAINED_LOCUS_CLASSES, NO_RETAINED_LOCI, + LIMITED_DATA_SET_DIRECT_IDENTIFIERS, + isRetainableCategory, + assertRetentionContract, retains, type RetainedLocusClass, } from "./retention.js"; diff --git a/src/locus.ts b/src/locus.ts index 9e15bd4..36d5fd3 100644 --- a/src/locus.ts +++ b/src/locus.ts @@ -11,6 +11,7 @@ */ import type { SafeHarborCategory } from "./categories.js"; +import type { RetainedLocusClass } from "./retention.js"; /** * The kind of value at a locus: drives which generalization applies and whether the engine must @@ -50,17 +51,16 @@ export interface GenericLocus { /** The Safe Harbor category, when known. Omit to force fail-closed handling as category (R). */ readonly category?: SafeHarborCategory; /** - * Set by an adapter when the **configured policy's retention set** names the class this locus - * belongs to: the value is passed through **unchanged** and recorded as a residual, rather than - * transformed. It is how a limited-data-set preset keeps an admission date or an encounter number - * that Safe Harbor removes. + * The **retention class** this locus belongs to, when an adapter can name one. It is a *proposal*, + * never a decision: the engine keeps the value only if the configured options **also** list this + * class, **and** the resolved category is one a limited data set may carry at all. Both keys are + * required, so an adapter cannot retain anything on its own and a stale marker cannot leak a value. * - * **This is the one flag that makes the engine emit an identifying value.** It is opt-in and - * fails closed: absent (the default) the locus takes its normal policy transform. It is **not** the - * over-scrub guard: a `clinical` locus is not an identifier and is retained without a manifest row, - * whereas a locus marked here **is** identifying and is always recorded. + * Absent (the default) the locus takes its normal policy transform. This is **not** the over-scrub + * guard: a `clinical` locus is not an identifier and is retained without a manifest row, whereas a + * locus marked here **is** identifying and is always recorded when it is kept. */ - readonly retainedByPolicy?: boolean; + readonly retention?: RetainedLocusClass; /** The value at the locus. Consumed by the engine, never copied into the manifest. */ readonly value: string; } diff --git a/src/report.ts b/src/report.ts index 49b6388..ddc5230 100644 --- a/src/report.ts +++ b/src/report.ts @@ -132,6 +132,12 @@ export interface RetainedQuasiIdentifier { readonly category: SafeHarborCategory; /** How many values at this locus retained a residual. */ readonly count: number; + /** + * How the residual arose, so the two kinds are not indistinguishable in the inventory: `generalize` + * left a coarse residual (a year, a safe 3-digit ZIP prefix, an age ≤ 89), while `retain` kept the + * **whole unreduced value**. A determiner reasons about those very differently. + */ + readonly transform: TransformName; } /** @@ -468,7 +474,14 @@ export function buildExpertDeterminationSupportReport( const retainedQuasiIdentifiers = entries .filter((e) => e.code === DEID_DISPOSITION_CODES.DEID_RESIDUAL_RETAINED) - .map((e) => Object.freeze({ locus: e.locus, category: e.category, count: e.count })); + .map((e) => + Object.freeze({ + locus: e.locus, + category: e.category, + count: e.count, + transform: e.transform, + }), + ); const policyName = options.policy === undefined @@ -547,7 +560,7 @@ export function formatExpertDeterminationSupportReport( ); } lines.push(""); - lines.push("## Retained quasi-identifiers (coarse residuals recorded as retained)"); + lines.push("## Retained quasi-identifiers (identifying residuals recorded as retained)"); lines.push(""); if (report.retainedQuasiIdentifiers.length === 0) { lines.push( @@ -573,7 +586,10 @@ export function formatExpertDeterminationSupportReport( lines.push("(§164.514(b)(2)(ii)) consideration for the determiner:"); lines.push(""); for (const r of report.retainedQuasiIdentifiers) { - lines.push(`- ${r.locus}: ${r.category} (×${String(r.count)})`); + // `retain` means the WHOLE value survived; `generalize` means only a coarse residual did. Naming + // it is the difference between "a year is present" and "a full timestamp is present". + const kind = r.transform === "retain" ? "whole value kept" : "coarse residual"; + lines.push(`- ${r.locus}: ${r.category} (×${String(r.count)}, ${kind})`); } } const qi = report.quasiIdentifierStatistics; diff --git a/src/retention.ts b/src/retention.ts index 15cd8ef..3772781 100644 --- a/src/retention.ts +++ b/src/retention.ts @@ -28,9 +28,18 @@ * **The default is retain-nothing.** Every entry point defaults to an empty retention set, so an * options bag that never mentions retention gets the strict treatment (fail closed). * + * **Scope, stated rather than implied: only the HL7 v2 adapter reads these classes today.** Passing a + * retention set to the C-CDA, FHIR, X12, NCPDP or DICOM adapter changes nothing there, so one profile + * means something narrower for those five formats than it does for HL7 v2. The direction is the safe + * one (they stay stricter, never looser) and there is no diagnostic for it, which is why it is written + * here. + * * @packageDocumentation */ +import { SAFE_HARBOR_CATEGORIES, type SafeHarborCategory } from "./categories.js"; +import { DeidError, FATAL_CODES } from "./codes.js"; + /** * The stable registry of retention classes. `key === value` so the full set survives an * `Object.values(...)` snapshot into a stability tripwire. These are part of the public contract: @@ -74,6 +83,104 @@ export const RETAINED_LOCUS_CLASSES = { export type RetainedLocusClass = (typeof RETAINED_LOCUS_CLASSES)[keyof typeof RETAINED_LOCUS_CLASSES]; +/** + * The **sixteen direct identifiers §164.514(e)(2) excludes from a limited data set**, mapped onto this + * library's category model: (i) names, (ii) postal address other than town/city/State/ZIP, + * (iii) telephone, (iv) fax, (v) email, (vi) social security, (vii) medical record, (viii) health plan + * beneficiary, (ix) account, (x) certificate/licence, (xi) vehicle, (xii) device, (xiii) URL, (xiv) IP, + * (xv) biometric, (xvi) full-face image. + * + * **This is the guard that makes the retention citation true rather than merely asserted.** The + * argument for keeping an encounter or order number is that this list has no catch-all; the argument + * for keeping a service date is that it has no date. **Neither argument survives if the value at the + * locus turns out to be one of the sixteen** — and a visit number field routinely carries a medical + * record or account number, typed as such by the standard's own identifier-type code. So retention is + * **refused** whenever the resolved category is on this list, whatever an adapter asked for, in both + * the adapter and the engine. + * + * Exactly two of the eighteen Safe Harbor categories are absent from it: `DATES` and the (R) catch-all + * `OTHER_UNIQUE_ID`. Those two, and only those two, are retainable. + * + * @example + * ```ts + * import { LIMITED_DATA_SET_DIRECT_IDENTIFIERS, SAFE_HARBOR_CATEGORIES } from "@cosyte/deid"; + * + * LIMITED_DATA_SET_DIRECT_IDENTIFIERS.has(SAFE_HARBOR_CATEGORIES.MRN); // => true (never retainable) + * LIMITED_DATA_SET_DIRECT_IDENTIFIERS.has(SAFE_HARBOR_CATEGORIES.DATES); // => false (retainable) + * ``` + */ +export const LIMITED_DATA_SET_DIRECT_IDENTIFIERS: ReadonlySet = new Set([ + SAFE_HARBOR_CATEGORIES.NAMES, + SAFE_HARBOR_CATEGORIES.GEOGRAPHIC, + SAFE_HARBOR_CATEGORIES.PHONE, + SAFE_HARBOR_CATEGORIES.FAX, + SAFE_HARBOR_CATEGORIES.EMAIL, + SAFE_HARBOR_CATEGORIES.SSN, + SAFE_HARBOR_CATEGORIES.MRN, + SAFE_HARBOR_CATEGORIES.HEALTH_PLAN_BENEFICIARY, + SAFE_HARBOR_CATEGORIES.ACCOUNT, + SAFE_HARBOR_CATEGORIES.CERTIFICATE_LICENSE, + SAFE_HARBOR_CATEGORIES.VEHICLE, + SAFE_HARBOR_CATEGORIES.DEVICE, + SAFE_HARBOR_CATEGORIES.URL, + SAFE_HARBOR_CATEGORIES.IP_ADDRESS, + SAFE_HARBOR_CATEGORIES.BIOMETRIC, + SAFE_HARBOR_CATEGORIES.FULL_FACE_PHOTO, +]); + +/** + * Whether a locus of this category may be retained at all. `false` for every one of the sixteen direct + * identifiers {@link LIMITED_DATA_SET_DIRECT_IDENTIFIERS} names, whatever retention class an adapter + * attached to it. + * + * @param category - The resolved Safe Harbor category of the locus. + * @returns `true` only for `DATES` and the (R) catch-all. + * @example + * ```ts + * import { isRetainableCategory, SAFE_HARBOR_CATEGORIES } from "@cosyte/deid"; + * + * isRetainableCategory(SAFE_HARBOR_CATEGORIES.MRN); // => false + * isRetainableCategory(SAFE_HARBOR_CATEGORIES.DATES); // => true + * ``` + */ +export function isRetainableCategory(category: SafeHarborCategory): boolean { + return !LIMITED_DATA_SET_DIRECT_IDENTIFIERS.has(category); +} + +/** + * Enforce the **label contract on retention**, failing closed: a policy carrying the reserved + * `safe-harbor` label may not run with a non-empty retention set. Retaining an admission date or an + * encounter number is strictly weaker than the transform the label promises, so allowing it would let + * an options bag emit a Safe-Harbor-labelled result that is not Safe Harbor. This is the retention + * analogue of the guard that stops a date-shifting policy wearing the same label, and it closes the + * hand-built-options route that no profile check can see. + * + * @param policyName - The name of the resolved policy. + * @param retainedLoci - The retention classes the options bag carries, if any. + * @throws {@link DeidError} `DEID_POLICY_INVALID` when a `safe-harbor`-labelled policy is asked to retain. + * @example + * ```ts + * import { assertRetentionContract } from "@cosyte/deid"; + * + * assertRetentionContract("limited-data-set", ["encounter-dates"]); // ok + * assertRetentionContract("safe-harbor", []); // ok (retains nothing) + * ``` + */ +export function assertRetentionContract( + policyName: string, + retainedLoci: readonly RetainedLocusClass[] | undefined, +): void { + if (policyName !== "safe-harbor" || retainedLoci === undefined || retainedLoci.length === 0) { + return; + } + throw new DeidError( + FATAL_CODES.DEID_POLICY_INVALID, + 'a policy carrying the reserved "safe-harbor" label must not retain any identifying locus: ' + + `retention of ${retainedLoci.map((c) => `"${c}"`).join(", ")} keeps elements Safe Harbor ` + + "removes. Name the policy distinctly.", + ); +} + /** The empty retention set: the fail-closed default every entry point uses. */ export const NO_RETAINED_LOCI: readonly RetainedLocusClass[] = Object.freeze([]); diff --git a/test/corpus/leak-corpus.test.ts b/test/corpus/leak-corpus.test.ts index 5b42c24..b1aeed4 100644 --- a/test/corpus/leak-corpus.test.ts +++ b/test/corpus/leak-corpus.test.ts @@ -416,7 +416,7 @@ describe("encounter loci positive control, the limited-data-set profile still ca "OBR-7", "ORC-2", "ORC-3", - "PV1-19", + "PV1-19[0]", "PV1-44", "PV1-45", ]); @@ -429,7 +429,7 @@ describe("encounter loci positive control, the limited-data-set profile still ca }); const inventoried = new Set(report.retainedQuasiIdentifiers.map((r) => r.locus)); for (const locus of [ - "PV1-19", + "PV1-19[0]", "PV1-44", "PV1-45", "OBR-2", diff --git a/test/hl7/deidentify-hl7.test.ts b/test/hl7/deidentify-hl7.test.ts index 1b7071b..db23432 100644 --- a/test/hl7/deidentify-hl7.test.ts +++ b/test/hl7/deidentify-hl7.test.ts @@ -451,11 +451,14 @@ describe("deidentifyHl7, the encounter loci inside retained segments (§164.514( expect(entry?.disposition).toBe("transformed"); expect(entry?.code).toBe(D.DEID_RESIDUAL_RETAINED); } - // Identifiers: the whole field is gone, assigning authority and all, recorded as blocked (R). - for (const path of ["PV1.19.1", "ORC.2.1", "ORC.3.1", "OBR.2.1", "OBR.3.1"]) { + // The EI order numbers: the whole field is gone. PV1-19 is a CX list, handled per repetition like + // PID-3, so its id-number component is cleared and the assigning authority / type code remain. + for (const path of ["ORC.2.1", "ORC.3.1", "OBR.2.1", "OBR.3.1"]) { expect(document.get(path)).toBeUndefined(); } - for (const locus of ["PV1-19", "ORC-2", "ORC-3", "OBR-2", "OBR-3"]) { + expect(document.get("PV1.19.1")).toBe(""); + expect(document.get("PV1.19.5")).toBe("VN"); // type code retained, the value is what had to go + for (const locus of ["PV1-19[0]", "ORC-2", "ORC-3", "OBR-2", "OBR-3"]) { const entry = manifest.find((m) => m.locus === locus); expect(entry?.category).toBe(C.OTHER_UNIQUE_ID); expect(entry?.disposition).toBe("blocked"); @@ -479,7 +482,7 @@ describe("deidentifyHl7, the encounter loci inside retained segments (§164.514( } // Recorded, every one: a kept identifier that no artifact names is invisible twice over. for (const locus of [ - "PV1-19", + "PV1-19[0]", "PV1-44", "PV1-45", "ORC-2", @@ -545,3 +548,94 @@ describe("deidentifyHl7, the encounter loci inside retained segments (§164.514( } }); }); + +describe("a visit-number field carrying a REAL direct identifier is never retained", () => { + // §164.514(e)(2) enumerates sixteen direct identifiers, and (vii) NAMES medical record numbers while + // (ix) NAMES account numbers. The whole argument for keeping a visit number in a limited data set is + // that the list has no catch-all: that argument evaporates the moment the field actually carries one + // of the sixteen, which PV1-19 routinely does. The standard types it for us at CX-5 (Table 0203). + + /** Build a PV1 whose 19th field is exactly `visitNumber`, counted rather than eyeballed. */ + function pv1(visitNumber: string): string { + const fields = new Array(19).fill(""); + fields[0] = "1"; + fields[1] = "I"; + fields[2] = "W^R^B"; + fields[18] = visitNumber; // PV1-19 + return `PV1|${fields.join("|")}`; + } + + const wire = (visitNumber: string): string => + [ + "MSH|^~\\&|A|B|C|D|20200101||ADT^A03|M1|P|2.5", + "PID|1||ZZMRN500^^^HOSP^MR||ZZFAM^ZZGIV||19850302", + pv1(visitNumber), + ].join("\r"); + + it("the fixture builder really puts the value at PV1-19 (pre-condition)", () => { + // A test that silently seeded PV1-20 would assert nothing at all. + expect(parseHL7(wire("ZZMRN500^^^HOSP^VN")).get("PV1.19.1")).toBe("ZZMRN500"); + }); + + it("an MR-typed visit number is pseudonymized, not retained, EVEN under limited-data-set", () => { + const ctx = createDeidContext({ key: "pv19-key", patientId: "p1" }); + const { document, manifest } = deidentifyHl7( + parseHL7(wire("ZZMRN500^^^HOSP^MR")), + profileOptions(LIMITED_DATA_SET_PROFILE, ctx), + ); + expect(document.toString().includes("ZZMRN500")).toBe(false); + const entry = manifest.find((m) => m.locus === "PV1-19[0]"); + expect(entry?.category).toBe(C.MRN); + expect(entry?.disposition).toBe("transformed"); + // And the surrogate is the SAME one PID-3 got, so a pass can never republish in the clear the + // identifier it just pseudonymized elsewhere in the very same message. + expect(document.get("PV1.19.1")).toBe(document.get("PID.3[0].1")); + }); + + it("AN / SS / MA typed visit numbers are likewise transformed, never retained", () => { + const ctx = createDeidContext({ key: "pv19-key", patientId: "p1" }); + for (const [typeCode, category] of [ + ["AN", C.ACCOUNT], + ["SS", C.SSN], + ["MA", C.HEALTH_PLAN_BENEFICIARY], + ] as const) { + const { document, manifest } = deidentifyHl7( + parseHL7(wire(`ZZMRN500^^^HOSP^${typeCode}`)), + profileOptions(LIMITED_DATA_SET_PROFILE, ctx), + ); + expect(document.toString().includes("ZZMRN500")).toBe(false); + const entry = manifest.find((m) => m.locus === "PV1-19[0]"); + expect(entry?.category).toBe(category); + expect(entry?.disposition).not.toBe("retained"); + } + }); + + it("an untyped or VN-typed visit number IS the encounter identifier, and is retained under LDS", () => { + const ctx = createDeidContext({ key: "pv19-key", patientId: "p1" }); + for (const visitNumber of ["ZZVISIT500^^^HOSP^VN", "ZZVISIT500"]) { + const { document, manifest } = deidentifyHl7( + parseHL7(wire(visitNumber)), + profileOptions(LIMITED_DATA_SET_PROFILE, ctx), + ); + const entry = manifest.find((m) => m.locus === "PV1-19[0]"); + expect(entry?.category).toBe(C.OTHER_UNIQUE_ID); + expect(entry?.disposition).toBe("retained"); + expect(document.get("PV1.19.1")).toBe("ZZVISIT500"); + } + }); + + it("a mixed PV1-19 list routes each repetition on its own type code", () => { + const ctx = createDeidContext({ key: "pv19-key", patientId: "p1" }); + const msg = parseHL7(wire("ZZVISIT500^^^H^VN~ZZMRN500^^^H^MR")); + const { document, manifest } = deidentifyHl7( + msg, + profileOptions(LIMITED_DATA_SET_PROFILE, ctx), + ); + expect(manifest.find((m) => m.locus === "PV1-19[0]")?.disposition).toBe("retained"); + expect(manifest.find((m) => m.locus === "PV1-19[1]")?.category).toBe(C.MRN); + expect(manifest.find((m) => m.locus === "PV1-19[1]")?.disposition).toBe("transformed"); + const out = document.toString(); + expect(out.includes("ZZVISIT500")).toBe(true); // the real encounter identifier survives + expect(out.includes("ZZMRN500")).toBe(false); // the medical record number does not + }); +}); diff --git a/test/profile.test.ts b/test/profile.test.ts index c49ed43..f0a5b8d 100644 --- a/test/profile.test.ts +++ b/test/profile.test.ts @@ -13,6 +13,8 @@ import { FATAL_CODES, LIMITED_DATA_SET_PROFILE, profileOptions, + isRetainableCategory, + LIMITED_DATA_SET_DIRECT_IDENTIFIERS, RETAINED_LOCUS_CLASSES, retains, SAFE_HARBOR_CATEGORIES, @@ -218,7 +220,7 @@ describe("policy-scoped retention, and what widen-never-narrow means for it", () path: "PV1-44", kind: "date", category: C.DATES, - retainedByPolicy: true, + retention: R.ENCOUNTER_DATES, value: "20200103040500", }, ], @@ -232,7 +234,7 @@ describe("policy-scoped retention, and what widen-never-narrow means for it", () expect(manifest[0]?.code).toBe("DEID_RESIDUAL_RETAINED"); }); - it("the retention flag can never keep free text or unrecognized structure (guard order)", () => { + it("the retention marker can never keep free text or unrecognized structure (guard order)", () => { const ctx = createDeidContext({ key: "retain-key", patientId: "p1" }); const { document, manifest } = deidentify( { @@ -241,17 +243,22 @@ describe("policy-scoped retention, and what widen-never-narrow means for it", () path: "NTE-3", kind: "freetext", category: C.OTHER_UNIQUE_ID, - retainedByPolicy: true, + retention: R.ENCOUNTER_DATES, value: "ZZPROSE", }, { path: "ZPI-1", kind: "unknown", category: C.OTHER_UNIQUE_ID, - retainedByPolicy: true, + retention: R.ENCOUNTER_DATES, value: "ZZUNKNOWN", }, - { path: "PID-5", kind: "identifier", retainedByPolicy: true, value: "ZZUNCLASSIFIED" }, + { + path: "PID-5", + kind: "identifier", + retention: R.ENCOUNTER_DATES, + value: "ZZUNCLASSIFIED", + }, ], }, profileOptions(LIMITED_DATA_SET_PROFILE, ctx), @@ -267,3 +274,89 @@ describe("policy-scoped retention, and what widen-never-narrow means for it", () ).toThrow(DeidError); }); }); + +describe("retention needs all three keys, and any one missing means the transform runs", () => { + const R = RETAINED_LOCUS_CLASSES; + const ctx = createDeidContext({ key: "three-keys", patientId: "p1" }); + const dateLocus = { + path: "PV1-44", + kind: "date", + category: C.DATES, + retention: R.ENCOUNTER_DATES, + value: "20200103040500", + } as const; + + it("key 2: an adapter marker alone does NOT retain when the options bag is bare", () => { + // The failure this pins: an engine that trusts the locus marker on its own turns every options + // bag into a retaining one, which is the opposite of the documented fail-closed default. + const { document, manifest } = deidentify({ loci: [dateLocus] }, {}); + expect(document.loci[0]?.value).toBe("2020"); + expect(document.loci[0]?.disposition).toBe("transformed"); + expect(manifest[0]?.transform).toBe("generalize"); + }); + + it("key 2: an explicitly EMPTY retention set does not retain either", () => { + const { document } = deidentify( + { loci: [dateLocus] }, + { policy: LIMITED_DATA_SET_PROFILE.policy, retainedLoci: [], context: ctx }, + ); + expect(document.loci[0]?.value).not.toBe("20200103040500"); + }); + + it("key 2: a set naming a DIFFERENT class does not retain", () => { + const { document } = deidentify( + { loci: [dateLocus] }, + { + policy: LIMITED_DATA_SET_PROFILE.policy, + retainedLoci: [R.ENCOUNTER_IDENTIFIERS], + context: ctx, + }, + ); + expect(document.loci[0]?.value).not.toBe("20200103040500"); + }); + + it("key 3: a category §164.514(e)(2) NAMES is never retainable, whatever is asked for", () => { + // The sixteen direct identifiers of a limited data set. Each one carries a retention marker and a + // matching enabled class, and each one must still be transformed. + for (const category of [C.MRN, C.ACCOUNT, C.SSN, C.HEALTH_PLAN_BENEFICIARY, C.NAMES, C.PHONE]) { + const { document, manifest } = deidentify( + { + loci: [ + { + path: "PV1-19[0]", + kind: "identifier", + category, + retention: R.ENCOUNTER_IDENTIFIERS, + value: "ZZDIRECTID", + }, + ], + }, + profileOptions(LIMITED_DATA_SET_PROFILE, ctx), + ); + expect(document.loci[0]?.value).not.toBe("ZZDIRECTID"); + expect(manifest[0]?.disposition).not.toBe("retained"); + } + // And exactly two of the eighteen are retainable: DATES and the (R) catch-all. + expect(isRetainableCategory(C.DATES)).toBe(true); + expect(isRetainableCategory(C.OTHER_UNIQUE_ID)).toBe(true); + expect(LIMITED_DATA_SET_DIRECT_IDENTIFIERS.size).toBe(16); + }); + + it("the reserved safe-harbor label refuses retention however the options bag was built", () => { + // The route no profile check can see: a hand-built bag pairing the reserved label with a + // retention set. It must be fatal, not a Safe-Harbor-labelled result that is not Safe Harbor. + try { + deidentify( + { loci: [dateLocus] }, + { policy: "safe-harbor", retainedLoci: [R.ENCOUNTER_DATES], context: ctx }, + ); + expect.unreachable("a safe-harbor-labelled policy must not retain"); + } catch (err) { + expect((err as DeidError).code).toBe(FATAL_CODES.DEID_POLICY_INVALID); + } + // The same bag with an empty set is fine: it is the retention, not the option, that is refused. + expect(() => + deidentify({ loci: [dateLocus] }, { policy: "safe-harbor", retainedLoci: [] }), + ).not.toThrow(); + }); +}); diff --git a/test/report.test.ts b/test/report.test.ts index b9c8f62..d738c52 100644 --- a/test/report.test.ts +++ b/test/report.test.ts @@ -414,3 +414,42 @@ describe("formatExpertDeterminationSupportReport, human-readable rendering", () expect(md).toContain("_None recorded._"); }); }); + +describe("the residual inventory distinguishes a kept year from a kept whole value", () => { + const manifest = [ + { + category: C.DATES, + transform: "generalize" as const, + locus: "PID-7", + count: 1, + disposition: "transformed" as const, + code: CODES.DEID_RESIDUAL_RETAINED, + }, + { + category: C.DATES, + transform: "retain" as const, + locus: "PV1-44", + count: 1, + disposition: "retained" as const, + code: CODES.DEID_RESIDUAL_RETAINED, + }, + ]; + + it("carries the transform on every inventory row", () => { + const report = buildExpertDeterminationSupportReport(manifest); + const byLocus = new Map(report.retainedQuasiIdentifiers.map((r) => [r.locus, r.transform])); + expect(byLocus.get("PID-7")).toBe("generalize"); + expect(byLocus.get("PV1-44")).toBe("retain"); + }); + + it("counts a retained disposition, and renders the two kinds differently", () => { + const report = buildExpertDeterminationSupportReport(manifest); + expect(report.dispositionSummary.retained).toBe(1); + expect(report.dispositionSummary.transformed).toBe(1); + const md = formatExpertDeterminationSupportReport(report); + // Without this the two rows are indistinguishable, and a full timestamp reads like a kept year. + expect(md).toContain("PID-7: DATES (×1, coarse residual)"); + expect(md).toContain("PV1-44: DATES (×1, whole value kept)"); + expect(md).toContain("retained: 1"); + }); +}); From a756fb882e83b1cb064e28366a03b06ab6f46946 Mon Sep 17 00:00:00 2001 From: Noah Schatz Date: Tue, 11 Aug 2026 15:18:48 +0000 Subject: [PATCH 6/6] docs: say that PV1-19 is routed by its CX-5 type code, not simply blocked conformance-refuter pass 2 returned VERDICT: NOT REFUTED, with one INTRODUCED minor (N2) it flagged as worth fixing before merge: four published lines still said PV1-19 is "blocked as the (R) catch-all" and none mentioned the type-code routing the remedy added. After that remedy an MR/AN/SS-typed PV1-19 is PSEUDONYMIZED or REDACTED as the identifier it really is, under BOTH profiles, and the limited-data-set line "keeps the visit number unchanged" lacked the caveat that this is only ever true of a VN-typed or untyped one. The defect under-states protection rather than over-stating it, so it is not the class of claim defect that gated pass 1 -- but a published surface that describes a mechanism the code no longer has is a claim defect either way. Corrected in docs-content/limitations.md, docs-content/guides-hl7.md, README.md and src/hl7/index.ts. The changeset already described the routing in full and is unchanged. Not fixed here, held as backlog lines because they reproduce on f8b7f56: - N1: the id-number write-back clears CX.1 only, so a fully-populated PV1-19 keeps CX.6 assigning facility and the CX.7/CX.8 effective/expiration dates while the manifest calls the row `blocked`. Same shape as the existing PID-3 behaviour for an SS-typed repetition. - F2 residual: an UNTYPED PV1-19 carrying an MRN is still retainable under limited-data-set. The wire gives no signal, and no surface claims otherwise. - F7: ~15 unrecorded full-precision timestamps across EVN, PV2, PR1, RXA, RXD, FT1, TXA, SPM, plus a DOB riding through as OBX-5 with a DTM type. - F8: under limited-data-set, PID-7 is date-shifted while the retained encounter dates are not, so intervals are not preserved across that line. verify.sh green. Refuter passes spent: 2 of 4. --- README.md | 6 ++++-- docs-content/guides-hl7.md | 2 +- docs-content/limitations.md | 16 +++++++++++----- src/hl7/index.ts | 10 ++++++++-- 4 files changed, 24 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index c0d753e..51761c8 100644 --- a/README.md +++ b/README.md @@ -116,8 +116,10 @@ as do **NTE-3** comments. Structured clinical values, units, codes, and statuses **Inside a retained segment**, the identifying loci are carved back out: under a Safe-Harbor-labelled policy the admit (PV1-44), discharge (PV1-45), observation (OBR-7) and diagnosis (DG1-5) dates keep only their **year**, and the visit number (PV1-19) with the placer and filler order numbers (OBR-2/3, ORC-2/3) -are **blocked** as the (R) catch-all. A profile that names their retention class, as the limited-data-set -preset does, keeps them **unchanged and recorded**. +are **removed**. A profile that names their retention class, as the limited-data-set preset does, keeps +them **unchanged and recorded**. PV1-19 is a CX list routed by its CX-5 identifier-type code, like PID-3: +only a `VN`-typed or untyped visit number is the encounter identifier, while an `MR`/`AN`/`SS`-typed one +is transformed as the medical record / account / social security number it is, under **both** profiles. **Known limitations (this release).** Free text is block-by-default (no built-in scrub; opt-in BYO redaction: see [Free text](#free-text-block-by-default--byo-redaction)); **every** field of a retained diff --git a/docs-content/guides-hl7.md b/docs-content/guides-hl7.md index 3105587..b5f3aef 100644 --- a/docs-content/guides-hl7.md +++ b/docs-content/guides-hl7.md @@ -47,7 +47,7 @@ surrogate. | **OBX-5, NTE-3** | narrative / ambiguous free text (OBX-5 unless OBX-2 types it structured) | **fail closed**: blocked, never regex-scrubbed | | **MRG / ACC / FAM / PEO / PDA** | known patient-identity / relative / geographic segments absent from the map | **fail closed**: blocked (e.g. a merge message's prior name + MRN) | | **Z-segments / unknown structure** | every populated field | **fail closed**: blocked | -| **PV1-19, OBR-2/3, ORC-2/3** | visit number, placer + filler order numbers, inside retained segments | **blocked** as the (R) catch-all (kept, and recorded, only under a profile that names the class) | +| **PV1-19, OBR-2/3, ORC-2/3** | visit number, placer + filler order numbers, inside retained segments | **removed**. PV1-19 is routed by its CX-5 type code like PID-3: `VN`/untyped is the encounter identifier, removed as (R) and retainable under a profile that names the class; `MR`/`AN`/`SS` is transformed as that identifier under **both** profiles, never retained | | **PV1-44/45, OBR-7, DG1-5** | admit, discharge, observation and diagnosis dates, inside retained segments | → **year** (§164.514(b)(2)(i)(C) names admission and discharge); kept whole, and recorded, only under a profile that names the class | | Retained clinical/administrative segments (an explicit allow-list: OBR, ORC, AL1, DG1, PV1, RX\*, …) | every field except the two rows above | **retained untouched** (the over-scrub guard) | diff --git a/docs-content/limitations.md b/docs-content/limitations.md index bdac21f..755a637 100644 --- a/docs-content/limitations.md +++ b/docs-content/limitations.md @@ -63,8 +63,13 @@ Two things follow, and the second is the one that surprises people: - **The identifying loci inside retained HL7 v2 structures are carved back out and acted on.** Under a Safe-Harbor-labelled policy the admit (PV1-44), discharge (PV1-45), observation (OBR-7) and diagnosis (DG1-5) dates are reduced to their **year**, and the visit number (PV1-19) with the placer and filler - order numbers (OBR-2/3, ORC-2/3) are **blocked** as the (R) catch-all. §164.514(b)(2)(i)(C) names - admission and discharge dates in the regulation text itself. + order numbers (OBR-2/3, ORC-2/3) are **removed**. §164.514(b)(2)(i)(C) names admission and discharge + dates in the regulation text itself. +- **PV1-19 is routed by its CX-5 identifier-type code**, like PID-3. A `VN`-typed or untyped visit + number is the encounter identifier and is removed as the (R) catch-all; an `MR`/`AN`/`SS`-typed one is + handled as the medical record / account / social security number it actually is, so it is + **transformed under both profiles and is never retained** (§164.514(e)(2) names all three). A kept + visit number is therefore only ever one the wire did not type as something stronger. - **Every other field of a retained structure is still passed through untouched, and recorded nowhere.** The carve-out above narrows this class; it does not close it. Full-precision timestamps survive in EVN, PV2, PR1, RXA, RXD, FT1, TXA and SPM, and the attending / referring **provider** @@ -92,9 +97,10 @@ only when a real de-identified document grounds it. It also **keeps unchanged** the two classes §164.514(e)(2) permits and Safe Harbor does not: the **encounter dates** (admission / discharge / service / diagnosis) and the **encounter and order - identifiers** (visit number, placer and filler order numbers). That list of sixteen direct - identifiers names no date and has no catch-all, which is exactly why these survive here and are - removed under Safe Harbor. Every one is still **recorded** as a `DEID_RESIDUAL_RETAINED` residual and + identifiers** (a `VN`-typed or untyped visit number, and the placer and filler order numbers). That + list of sixteen direct identifiers names no date and has no catch-all, which is exactly why these + survive here and are removed under Safe Harbor. It **does** name medical record, account and social + security numbers, so a PV1-19 typed as one of those is transformed here too. Every one is still **recorded** as a `DEID_RESIDUAL_RETAINED` residual and appears in the support report's inventory, so nothing is kept silently. `defineDeidProfile()`'s widen-never-narrow contract covers retention too, and it reads the opposite way diff --git a/src/hl7/index.ts b/src/hl7/index.ts index 825bf6c..d8636ff 100644 --- a/src/hl7/index.ts +++ b/src/hl7/index.ts @@ -23,8 +23,14 @@ * **Inside a retained segment**, the identifying dates and the encounter / order numbers are carved * back out ({@link RETAINED_LOCUS_RULES}): under a Safe-Harbor-labelled policy the admit (PV1-44), * discharge (PV1-45), observation (OBR-7) and diagnosis (DG1-5) dates generalize to their **year**, and - * the visit number (PV1-19) and the placer / filler order numbers (OBR-2/3, ORC-2/3) are **blocked** as - * the (R) catch-all. A profile that names their retention class keeps them **unchanged and recorded**. + * the visit number (PV1-19) and the placer / filler order numbers (OBR-2/3, ORC-2/3) are **removed**. A + * profile that names their retention class keeps them **unchanged and recorded**. + * + * **PV1-19 is a CX list routed by its CX-5 identifier-type code**, exactly like PID-3: a `VN`-typed or + * untyped value is the encounter identifier (removed as the (R) catch-all, retainable), while an + * `MR`/`AN`/`SS`-typed one is handled as the identifier it really is and is **transformed under both + * profiles, never retained** — §164.514(e)(2) names all three, so keeping one would republish in the + * clear the identifier the pass pseudonymized at PID-3 in the same message. * * **Known limitations.** Free text is block-only (no scrub); **every** field of a retained segment that * the carve-out does not name is still passed through untouched and unrecorded, which continues to