From 39992d84018a31b3bcb3b265e6a18f20a6ef87dc Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Thu, 16 Apr 2026 10:20:17 -0400 Subject: [PATCH 01/67] Introduce a filtering step to select columns from specified tables, transfer them to a new dataset table, and apply an initial hardcoded filter via a CTE. --- core/transformations.py | 13 ++- core/utils.py | 157 ++++++++++++++++++++++++++++++++++- reference/filter_config.json | 74 +++++++++++++++++ 3 files changed, 241 insertions(+), 3 deletions(-) create mode 100644 reference/filter_config.json diff --git a/core/transformations.py b/core/transformations.py index 955d5a4..245f49c 100644 --- a/core/transformations.py +++ b/core/transformations.py @@ -928,4 +928,15 @@ def create_standardized_mapping_table( #process_rows(source_table=source_table, destination_table=destination_table) #create_sensitive_tier(source_table=source_table, destination_table=destination_table) - create_standardized_mapping_table("CleanConnect", "nih-nci-dceg-connect-stg-5519.pr2_mvp.mvp") + #create_standardized_mapping_table("CleanConnect", "nih-nci-dceg-connect-stg-5519.pr2_mvp.mvp") + client = bigquery.Client() + filter_config = utils.load_filter_config("reference/filter_config.json") + #print(filter_config) + table_schemas = utils.build_table_schemas(client, filter_config) + #print(table_schemas) + queries = utils.build_filtered_table_query(client, filter_config, table_schemas, "SensitiveTier") + for i, query in enumerate(queries, 1): + print(f"\n{'='*80}") + print(f"Query {i}") + print(f"{'='*80}") + print(query) diff --git a/core/utils.py b/core/utils.py index 3d9cefa..7fbeeff 100644 --- a/core/utils.py +++ b/core/utils.py @@ -6,8 +6,9 @@ import json import logging from collections import defaultdict -from typing import Optional +from typing import Any, Optional from datetime import datetime, timezone +from xmlrpc import client from google.cloud import bigquery, storage import pandas as pd #TODO Try to avoid using pandas @@ -1011,4 +1012,156 @@ def create_missing_columns_json( # Serialize the report to JSON and upload to GCS blob.upload_from_string(json.dumps(report, indent=2, - ensure_ascii=False), content_type="application/json") \ No newline at end of file + ensure_ascii=False), content_type="application/json") + +def load_filter_config(config_file_path: str) -> dict: + """ + Load the filter configuration JSON file that defines destination tables and their source tables. + + Args: + config_file_path (str): The file path to the filter configuration JSON. + Returns: + dict: The loaded filter configuration as a dictionary. + """ + try: + with open(config_file_path, 'r') as f: + config = json.load(f) + return config + except Exception as e: + utils.logger.error(f"Error loading filter configuration from {config_file_path}: {e}") + return {} + +def build_table_schemas(client: bigquery.Client, filter_config: dict) -> dict[tuple[str, str], list[str]]: + """ + Build a lookup of (dataset, table) -> column names using get_column_names. + """ + table_schemas = {} + + for destination_table in filter_config.get("destination_tables", []): + for source in destination_table.get("source_tables", []): + + source_dataset = source["dataset"] + source_table = source["table"] + + key = (source_dataset, source_table) + + # Skip if already fetched + if key in table_schemas: + continue + + # Build fully qualified table name + fq_table = f"{client.project}.{source_dataset}.{source_table}" + + # 🔹 Use your existing function here + columns = get_column_names(client, fq_table) + + table_schemas[key] = columns + + return table_schemas + +def build_filtered_table_query( + client: bigquery.Client, + filter_config: dict[str, Any], + table_schemas: dict[tuple[str, str], list[str]], + target_dataset: str +) -> list[str]: + """ + Generate SQL queries to build destination tables for a specific dataset (e.g., SensitiveTier). + + This function reads a configuration JSON (`filter_config`) that defines destination tables + and their source tables. It filters the configuration to only include destination tables + that match the specified `target_dataset`. + + For each applicable destination table, the function: + - Creates a filtered participants CTE (based on business rules) + - Selects specified columns from the participants table + - Joins additional source tables using Connect_ID + - Includes columns from those source tables (based on explicit columns or select_all) + - Skips joins for tables that do not contribute any columns + + Args: + filter_config (dict): The full configuration containing all destination table definitions. + table_schemas (dict): A mapping of (dataset, table) → list of column names, + used when select_all is True. + target_dataset (str): The destination dataset to build queries for + (e.g., "SensitiveTier"). + """ + project = client.project + queries = [] + + for destination_table in filter_config.get("destination_tables", []): + # Only process tables that belong to the target dataset + if destination_table["dataset"] != target_dataset: + continue + + dest_dataset = destination_table["dataset"] + dest_table = destination_table["table"] + source_tables = destination_table.get("source_tables", []) + + ctes = [] + select_parts = [] + join_parts = [] + + # Always use filtered participants as base + ctes.append(f""" +filtered_participants AS ( + SELECT * + FROM `{project}.CleanConnect.participants` + WHERE d_821247024 = '197316935' --Verified = Yes + AND d_831041022 = '104430631' --Destroy data = No + AND d_747006172 = '104430631' --Withdraw consent = No + AND d_773707518 = '104430631' --Revoke HIPAA = No +) +""".strip()) + + # find participants config + participants_cfg = next( + (t for t in source_tables if t["table"] == "participants"), + None + ) + + # participant columns + if participants_cfg: + for col in participants_cfg.get("columns", []): + select_parts.append(f"participants.{col}") + + # process other tables + for source in source_tables: + if source["table"] == "participants": + continue + + alias = source["table"] + source_ref = f"`{project}.{source['dataset']}.{source['table']}`" + + if source.get("select_all"): + cols = table_schemas.get((source["dataset"], source["table"]), []) + else: + cols = source.get("columns", []) + + if not cols: + continue + + join_parts.append( + f"LEFT JOIN {source_ref} {alias} " + f"ON {alias}.Connect_ID = participants.Connect_ID" + ) + + for col in cols: + select_parts.append(f"{alias}.{col}") + + if not select_parts: + continue + + sql = f""" +CREATE OR REPLACE TABLE `{project}.{dest_dataset}.{dest_table}` AS +WITH +{",\n".join(ctes)} +SELECT + {",\n ".join(select_parts)} +FROM filtered_participants participants +{"\n".join(join_parts)} +""".strip() + + queries.append(sql) + + return queries \ No newline at end of file diff --git a/reference/filter_config.json b/reference/filter_config.json new file mode 100644 index 0000000..dcbe8df --- /dev/null +++ b/reference/filter_config.json @@ -0,0 +1,74 @@ +{ + "destination_tables": [ + { + "dataset": "SensitiveTier", + "table": "mvp", + "source_tables": [ + { + "dataset": "pr2_mvp", + "table": "mvp", + "select_all": true, + "columns": [] + }, + { + "dataset": "CleanConnect", + "table": "participants", + "select_all": false, + "columns": [ + "d_130371375_d_303552867_d_297462035", + "d_130371375_d_303552867_d_320023644", + "d_130371375_d_303552867_d_438636757" + ] + }, + { + "dataset": "CleanConnect", + "table": "module1", + "select_all": false, + "columns": [ + "d_103397024_d_206625031" + ] + } + ] + }, + { + "dataset": "SensitiveTier", + "table": "module1", + "source_tables": [ + { + "dataset": "CleanConnect", + "table": "module1", + "select_all": false, + "columns": [ + "d_103397024_d_206625031" + ] + } + ] + }, + { + "dataset": "FakeTier", + "table": "module1", + "source_tables": [ + { + "dataset": "CleanConnect", + "table": "module1", + "select_all": false, + "columns": [ + "d_103397024_d_206625031" + ] + } + ] + }, + { + "dataset": "SensitiveTier", + "table": "module2", + "source_tables": [ + { + "dataset": "CleanConnect", + "table": "module2", + "select_all": true, + "columns": [] + } + ] + } + ] +} \ No newline at end of file From c25365a197cdaf25514a7e395c3003216c001187 Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Fri, 24 Apr 2026 15:37:40 -0400 Subject: [PATCH 02/67] Rename filter_config.json to subset_config.json and update config structure --- reference/filter_config.json | 74 ---------------------------- reference/subset_config.json | 94 ++++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 74 deletions(-) delete mode 100644 reference/filter_config.json create mode 100644 reference/subset_config.json diff --git a/reference/filter_config.json b/reference/filter_config.json deleted file mode 100644 index dcbe8df..0000000 --- a/reference/filter_config.json +++ /dev/null @@ -1,74 +0,0 @@ -{ - "destination_tables": [ - { - "dataset": "SensitiveTier", - "table": "mvp", - "source_tables": [ - { - "dataset": "pr2_mvp", - "table": "mvp", - "select_all": true, - "columns": [] - }, - { - "dataset": "CleanConnect", - "table": "participants", - "select_all": false, - "columns": [ - "d_130371375_d_303552867_d_297462035", - "d_130371375_d_303552867_d_320023644", - "d_130371375_d_303552867_d_438636757" - ] - }, - { - "dataset": "CleanConnect", - "table": "module1", - "select_all": false, - "columns": [ - "d_103397024_d_206625031" - ] - } - ] - }, - { - "dataset": "SensitiveTier", - "table": "module1", - "source_tables": [ - { - "dataset": "CleanConnect", - "table": "module1", - "select_all": false, - "columns": [ - "d_103397024_d_206625031" - ] - } - ] - }, - { - "dataset": "FakeTier", - "table": "module1", - "source_tables": [ - { - "dataset": "CleanConnect", - "table": "module1", - "select_all": false, - "columns": [ - "d_103397024_d_206625031" - ] - } - ] - }, - { - "dataset": "SensitiveTier", - "table": "module2", - "source_tables": [ - { - "dataset": "CleanConnect", - "table": "module2", - "select_all": true, - "columns": [] - } - ] - } - ] -} \ No newline at end of file diff --git a/reference/subset_config.json b/reference/subset_config.json new file mode 100644 index 0000000..f4d1c6b --- /dev/null +++ b/reference/subset_config.json @@ -0,0 +1,94 @@ +{ + "filter_profiles": { + "default_participants": [ + { "column": "d_821247024", "value": "197316935", "comment": "Verified = Yes" }, + { "column": "d_831041022", "value": "104430631", "comment": "Destroy data = No" }, + { "column": "d_747006172", "value": "104430631", "comment": "Withdraw consent = No" }, + { "column": "d_773707518", "value": "104430631", "comment": "Revoke HIPAA = No"} + ], + "no_filters": [] + }, + "destination_tables": [ + { + "dataset": "SensitiveTier", + "table": "mvp", + "join_key": "Connect_ID", + "filter_profile": "default_participants", + "base_table": { + "dataset": "CleanConnect", + "table": "participants", + "columns": [ + "d_130371375_d_303552867_d_297462035", + "d_130371375_d_303552867_d_320023644", + "d_130371375_d_303552867_d_438636757" + ] + }, + "join_tables": [ + { + "dataset": "pr2_mvp", + "table": "mvp", + "columns": "*" + }, + { + "dataset": "CleanConnect", + "table": "module1", + "columns": [ + "d_103397024_d_206625031" + ] + } + ] + }, + { + "dataset": "SensitiveTier", + "table": "module1", + "join_key": "Connect_ID", + "filter_profile": "default_participants", + "base_table": { + "dataset": "CleanConnect", + "table": "participants", + "columns": [] + }, + "join_tables": [ + { + "dataset": "CleanConnect", + "table": "module1", + "columns": [ + "d_103397024_d_206625031" + ] + } + ] + }, + { + "dataset": "FakeTier", + "table": "module1", + "join_key": "Connect_ID", + "filter_profile": "default_participants", + "base_table": { + "dataset": "CleanConnect", + "table": "participants", + "columns": [] + }, + "join_tables": [ + { + "dataset": "CleanConnect", + "table": "module1", + "columns": [ + "d_103397024_d_206625031" + ] + } + ] + }, + { + "dataset": "SensitiveTier", + "table": "module2", + "join_key": "Connect_ID", + "filter_profile": "no_filters", + "base_table": { + "dataset": "CleanConnect", + "table": "module1", + "columns": "*" + }, + "join_tables": [] + } + ] +} \ No newline at end of file From 989fe3ca66b50d55fd5dab40477b4095841d81a3 Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Fri, 24 Apr 2026 15:39:26 -0400 Subject: [PATCH 03/67] Refactor subset step to be config-driven with dynamic SQL generation - Generate SQL programmatically using CTE-based filtering - Replace hardcoded filters with configurable filter_profiles - Support dynamic base and join table selection via JSON config - Enforce non-null Connect_ID sourced from base table - Add schema validation and missing column reporting - Export generated SQL and reports to GCS - Improve logging and error handling across the pipeline --- core/transformations.py | 152 +++++++++++- core/utils.py | 500 ++++++++++++++++++++++++++++++++-------- 2 files changed, 543 insertions(+), 109 deletions(-) diff --git a/core/transformations.py b/core/transformations.py index 245f49c..b3e489e 100644 --- a/core/transformations.py +++ b/core/transformations.py @@ -922,6 +922,142 @@ def create_standardized_mapping_table( utils.logger.error(f"Exception args: {e.args}") raise e +######################################################################## +############# Subset Table Construction ############################### +######################################################################## + +def create_subset_table( + config_path: str, + destination_dataset: str, + destination_table: str, + missing_report_base_path: str = "gs://pr2-pipeline-artifacts-stg/missing_columns_report/" # constants.MISSING_COLUMNS_REPORT_PATH +) -> dict: + """ + Create a subset table based on configuration. + + This function: + 1. Loads subset configuration + 2. Retrieves the target destination table config + 3. Builds table schemas + 4. Identifies missing columns + 5. Generates SQL + 6. Saves SQL to GCS + 7. Executes query + 8. Writes missing column report JSON + + Args: + config_path (str): Path to subset config JSON. + destination_dataset (str): Target dataset. + destination_table (str): Target table. + missing_report_base_path (str): Base GCS path for missing column reports. + + Returns: + dict: + - status + - submitted_sql_path + - missing_columns_report_path + + Raises: + ValueError: If configuration is invalid. + Exception: If query execution fails. + """ + client = bigquery.Client() + + # Load subset configuration + utils.logger.info("Loading subset configuration...") + subset_config = utils.load_subset_config(config_path) + + # Retrieve the target destination table config + utils.logger.info("Retrieving destination table configuration...") + destination_table_config = utils.get_destination_table_config( + subset_config, + destination_dataset, + destination_table + ) + + if not destination_table_config: + raise ValueError( + f"No config found for {destination_dataset}.{destination_table}" + ) + + # Build table schemas + table_schemas, schema_issues = utils.build_table_schemas( + client, + destination_table_config + ) + + if schema_issues: + utils.logger.warning(f"Schema issues detected: {schema_issues}") + + # Identify missing columns + missing_cleaned_cols, missing_cleaned_loop_vars = utils.find_missing_columns( + destination_table_config, + table_schemas + ) + + # Generate SQL + result = utils.build_subset_query( + client=client, + full_config=subset_config, + destination_table_config=destination_table_config, + table_schemas=table_schemas + ) + + sql = result["sql"] + fq_destination_table = result["destination_table"] + + # Save the SQL to GCS for audit purposes + try: + utils.logger.info("Saving SQL to GCS...") + gcs_client = storage.Client() + gcs_path = f"{constants.OUTPUT_SQL_PATH}{fq_destination_table}.sql" + utils.save_sql_string(sql=sql, path=gcs_path, storage_client=gcs_client) + utils.logger.info(f"SQL saved to GCS at {gcs_path}") + except Exception as e: + utils.logger.exception(f"Error saving SQL to GCS: {e}") + raise e + + # Execute the SQL + try: + utils.logger.info("Executing SQL query...") + query_job = client.query(sql) + utils.logger.info(f"Query job created with ID: {query_job.job_id}") + query_job.result() + utils.logger.info("Query execution completed successfully") + status = f"Table {fq_destination_table} successfully created" + except Exception as e: + utils.logger.exception(f"Error executing SQL: {e}") + # Log more details about the exception + utils.logger.error(f"Exception type: {type(e).__name__}") + utils.logger.error(f"Exception args: {e.args}") + raise e + + # Write missing column report JSON + try: + report_path = f"{missing_report_base_path}{fq_destination_table}_missing_columns.json" + + utils.create_subset_missing_columns_json( + output_path=report_path, + destination_table=fq_destination_table, + missing_cleaned_cols=missing_cleaned_cols, + missing_cleaned_loop_vars=missing_cleaned_loop_vars, + client=gcs_client + ) + + utils.logger.info(f"Missing column report saved to {report_path}") + + except Exception as e: + utils.logger.exception(f"Error writing missing column report for {fq_destination_table}: {e}") + raise e + + # Return result (matches pattern) + return { + "status": status, + "submitted_sql_path": constants.OUTPUT_SQL_PATH, + "missing_columns_report_path": report_path + } + + if __name__ == "__main__": #source_table = "nih-nci-dceg-connect-prod-6d04.ForTestingOnly.module1_v1_with_cleaned_columns" #destination_table = "nih-nci-dceg-connect-prod-6d04.CleanConnect.module1_fixed_binary_and_false_arrays" @@ -930,13 +1066,9 @@ def create_standardized_mapping_table( #create_standardized_mapping_table("CleanConnect", "nih-nci-dceg-connect-stg-5519.pr2_mvp.mvp") client = bigquery.Client() - filter_config = utils.load_filter_config("reference/filter_config.json") - #print(filter_config) - table_schemas = utils.build_table_schemas(client, filter_config) - #print(table_schemas) - queries = utils.build_filtered_table_query(client, filter_config, table_schemas, "SensitiveTier") - for i, query in enumerate(queries, 1): - print(f"\n{'='*80}") - print(f"Query {i}") - print(f"{'='*80}") - print(query) + config_path = "reference/subset_config.json" + create_subset_table( + config_path=config_path, + destination_dataset="SensitiveTier", + destination_table="module2" + ) diff --git a/core/utils.py b/core/utils.py index 7fbeeff..98cfe47 100644 --- a/core/utils.py +++ b/core/utils.py @@ -1014,154 +1014,456 @@ def create_missing_columns_json( blob.upload_from_string(json.dumps(report, indent=2, ensure_ascii=False), content_type="application/json") -def load_filter_config(config_file_path: str) -> dict: +def load_subset_config(config_file_path: str) -> dict: """ - Load the filter configuration JSON file that defines destination tables and their source tables. + Load the subset configuration JSON file. + + This configuration defines filter profiles and destination table mappings + used to generate SQL queries. Args: - config_file_path (str): The file path to the filter configuration JSON. + config_file_path (str): Path to the subset configuration JSON file. + Returns: - dict: The loaded filter configuration as a dictionary. + dict: Parsed configuration dictionary. Returns an empty dict if loading fails. """ try: with open(config_file_path, 'r') as f: - config = json.load(f) - return config + return json.load(f) except Exception as e: - utils.logger.error(f"Error loading filter configuration from {config_file_path}: {e}") + utils.logger.error(f"Error loading subset configuration from {config_file_path}: {e}") return {} -def build_table_schemas(client: bigquery.Client, filter_config: dict) -> dict[tuple[str, str], list[str]]: +def get_destination_table_config( + subset_config: dict[str, Any], + destination_dataset: str, + destination_table: str, +) -> dict[str, Any] | None: """ - Build a lookup of (dataset, table) -> column names using get_column_names. + Retrieve the configuration for a specific destination table. + + Searches the configuration for a destination table matching the given dataset + and table name. + + Args: + subset_config (dict): Full subset configuration. + destination_dataset (str): Target dataset name. + destination_table (str): Target table name. + + Returns: + dict | None: Matching destination table configuration or None if not found. """ - table_schemas = {} + # Loop through each destination table defined in the config + for destination in subset_config.get("destination_tables", []): + dataset_name = destination.get("dataset") + table_name = destination.get("table") - for destination_table in filter_config.get("destination_tables", []): - for source in destination_table.get("source_tables", []): + # Check if both dataset and table match the requested ones + if dataset_name == destination_dataset and table_name == destination_table: + # Return the matching configuration immediately + return destination - source_dataset = source["dataset"] - source_table = source["table"] + # If no match was found after checking all entries, return None + return None - key = (source_dataset, source_table) +def get_fq_base_table( + client: bigquery.Client, + destination_table_config: dict[str, Any] +) -> str: + """ + Construct the fully qualified BigQuery base table name. - # Skip if already fetched - if key in table_schemas: - continue + Uses the base_table configuration and the current project to build + a fully qualified table identifier. + + Args: + client (bigquery.Client): BigQuery client with project context. + destination_table_config (dict): Destination table configuration. + + Returns: + str: Fully qualified table name in the format project.dataset.table. + + Raises: + ValueError: If base_table config is missing or malformed. + """ + project = client.project + + # Base table config must exist and be a dict + base_table_config = destination_table_config["base_table"] - # Build fully qualified table name - fq_table = f"{client.project}.{source_dataset}.{source_table}" + if not isinstance(base_table_config, dict): + raise ValueError( + f"'base_table' must be an object for " + f"{destination_table_config['dataset']}.{destination_table_config['table']}" + ) - # 🔹 Use your existing function here - columns = get_column_names(client, fq_table) + # Validate required keys exist + if "dataset" not in base_table_config or "table" not in base_table_config: + raise ValueError( + f"'base_table' must include 'dataset' and 'table' for " + f"{destination_table_config['dataset']}.{destination_table_config['table']}" + ) - table_schemas[key] = columns + # Extract dataset and table + dataset = base_table_config["dataset"] + table = base_table_config["table"] - return table_schemas + # Return fully qualified table name + return f"{project}.{dataset}.{table}" -def build_filtered_table_query( +def build_table_schemas( client: bigquery.Client, - filter_config: dict[str, Any], - table_schemas: dict[tuple[str, str], list[str]], - target_dataset: str -) -> list[str]: + destination_table_config: dict[str, Any], +) -> tuple[dict[tuple[str, str], list[str]], list[dict[str, Any]]]: + """ + Fetch column schemas for all tables used in a destination configuration. + + Retrieves column names for the base table and all join tables, storing them + in a lookup dictionary for later validation and SQL generation. + + Args: + client (bigquery.Client): BigQuery client. + destination_table_config (dict): Destination table configuration. + + Returns: + tuple: + - dict[(dataset, table), list[str]]: Table schemas. + - list[dict]: Schema retrieval issues (if any). """ - Generate SQL queries to build destination tables for a specific dataset (e.g., SensitiveTier). + # Stores schemas keyed by (dataset, table) + table_schemas: dict[tuple[str, str], list[str]] = {} - This function reads a configuration JSON (`filter_config`) that defines destination tables - and their source tables. It filters the configuration to only include destination tables - that match the specified `target_dataset`. + # Collect schema fetch failures for reporting + schema_issues: list[dict[str, Any]] = [] - For each applicable destination table, the function: - - Creates a filtered participants CTE (based on business rules) - - Selects specified columns from the participants table - - Joins additional source tables using Connect_ID - - Includes columns from those source tables (based on explicit columns or select_all) - - Skips joins for tables that do not contribute any columns + def _fetch_schema(dataset: str, table: str): + key = (dataset, table) + # Skip if already fetched (avoid duplicate queries) + if key in table_schemas: + return + + fq_table = f"{client.project}.{dataset}.{table}" + try: + # Retrieve column names from BigQuery + table_schemas[key] = get_column_names(client, fq_table) + except Exception as e: + utils.logger.error(f"Error fetching schema for {fq_table}: {e}") + # Store empty schema so downstream logic does not break + table_schemas[key] = [] + # Track issue for reporting + schema_issues.append({ + "dataset": dataset, + "table": table, + "error": str(e), + }) + + # Always fetch base table schema first + base_table_config = destination_table_config["base_table"] + _fetch_schema(base_table_config["dataset"], base_table_config["table"]) + + # Fetch schemas for all join tables + for join_table in destination_table_config.get("join_tables", []): + _fetch_schema(join_table["dataset"], join_table["table"]) + + return table_schemas, schema_issues + +def find_missing_columns( + destination_table_config: dict[str, Any], + table_schemas: dict[tuple[str, str], list[str]] +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + """ + Identify columns specified in config that do not exist in source tables. + + Checks both base table and join tables against fetched schemas. Args: - filter_config (dict): The full configuration containing all destination table definitions. - table_schemas (dict): A mapping of (dataset, table) → list of column names, - used when select_all is True. - target_dataset (str): The destination dataset to build queries for - (e.g., "SensitiveTier"). + destination_table_config (dict): Destination table configuration. + table_schemas (dict): Mapping of (dataset, table) to column names. + + Returns: + tuple: + - list[dict]: Missing standard columns. + - list[dict]: Missing loop variable columns. """ - project = client.project - queries = [] + missing_cleaned_cols = [] + missing_cleaned_loop_vars = [] - for destination_table in filter_config.get("destination_tables", []): - # Only process tables that belong to the target dataset - if destination_table["dataset"] != target_dataset: - continue + def _check_cols(config): + dataset = config["dataset"] + table = config["table"] + schema = set(table_schemas.get((dataset, table), [])) + cols = config.get("columns", []) + + # Skip wildcard or empty (means select all or nothing specified) + if cols == "*" or not cols: + return + + if not isinstance(cols, list): + raise ValueError(f"Invalid columns for {dataset}.{table}") + + for col in cols: + # Skip valid columns + if col in schema: + continue + + item = {"dataset": dataset, "table": table, "source_col": col} + + # Separate loop variables versus standard columns + if is_cleaned_loop_variable(col): + missing_cleaned_loop_vars.append(item) + else: + missing_cleaned_cols.append(item) + + # Check base table + _check_cols(destination_table_config["base_table"]) + + # Check join tables + for join_table in destination_table_config.get("join_tables", []): + _check_cols(join_table) + + return missing_cleaned_cols, missing_cleaned_loop_vars - dest_dataset = destination_table["dataset"] - dest_table = destination_table["table"] - source_tables = destination_table.get("source_tables", []) +def build_filter_sql( + filter_profiles: dict[str, Any], + destination_table_config: dict[str, Any] +) -> str: + """ + Generate SQL WHERE clause based on filter profile in the subset config. + + Applies the selected filter profile from the configuration and constructs + SQL conditions. Logs a warning if no filters are applied. + + Args: + filter_profiles (dict): Mapping of filter profile names to filter rules. + destination_table_config (dict): Destination table configuration. + + Returns: + str: SQL WHERE clause string or empty string if no filters are applied. + + Raises: + ValueError: If filter_profile is missing or invalid. + """ + # Get selected filter profile + filter_profile = destination_table_config.get("filter_profile") + if not filter_profile: + raise ValueError( + f"Missing filter_profile for " + f"{destination_table_config['dataset']}.{destination_table_config['table']}" + ) + # Identify filters for the selected table + filters = filter_profiles.get(filter_profile) + + # Catch invalid filter profile names + if filters is None: + raise ValueError(f"Unknown filter_profile: {filter_profile}") + + # Handle "no_filters" case (empty list) + if not filters: + utils.logger.warning( + f"Creating table " + f"{destination_table_config['dataset']}.{destination_table_config['table']} " + f"WITHOUT filters applied (filter_profile={filter_profile})" + ) + return "" - ctes = [] - select_parts = [] - join_parts = [] + clauses = [] - # Always use filtered participants as base - ctes.append(f""" -filtered_participants AS ( + # Build SQL conditions for each filter + for f in filters: + col = f["column"] + val = f["value"] + + # Default to "=" operator unless specified + op = f.get("operator", "=") + + clause = f"{col} {op} '{val}'" + + # Append optional comment + if "comment" in f: + clause += f" -- {f['comment']}" + + clauses.append(clause) + + # Join conditions with AND + return "WHERE " + "\n AND ".join(clauses) + +def build_subset_query( + client: bigquery.Client, + full_config: dict[str, Any], + destination_table_config: dict[str, Any], + table_schemas: dict[tuple[str, str], list[str]], +) -> dict[str, Any]: + """ + Build a CREATE OR REPLACE TABLE SQL query based on subset configuration. + + Combines base table, optional filters, and join tables into a single SQL query. + + Args: + client (bigquery.Client): BigQuery client. + full_config (dict): Full configuration including filter profiles. + destination_table_config (dict): Destination table configuration. + table_schemas (dict): Table schema lookup. + + Returns: + dict: Contains destination table name and generated SQL string. + + Raises: + ValueError: If no columns are selected. + """ + project = client.project + destination_dataset = destination_table_config["dataset"] + destination_table = destination_table_config["table"] + # Default join key if not specified + join_key = destination_table_config.get("join_key", "Connect_ID") + + # Build WHERE clause from filter profile + filter_sql = build_filter_sql( + full_config.get("filter_profiles", {}), + destination_table_config + ) + + # Base table + base_config = destination_table_config["base_table"] + fq_base_table = get_fq_base_table(client, destination_table_config) + base_table = base_config["table"] + + # CTE name and alias + cte_name = f"filtered_{base_table}" + base_alias = base_table + + # Combine base constraint and filter_sql + base_condition = f"{join_key} IS NOT NULL" + + if filter_sql: + #Remove leading WHERE from filter_sql safely + filter_body = filter_sql.replace("WHERE", "", 1).strip() + + final_where = f""" +WHERE {base_condition} + AND {filter_body} +""".strip() + + else: + final_where = f"WHERE {base_condition}" + + # Build CTE with optional filtering + cte = f""" +{cte_name} AS ( SELECT * - FROM `{project}.CleanConnect.participants` - WHERE d_821247024 = '197316935' --Verified = Yes - AND d_831041022 = '104430631' --Destroy data = No - AND d_747006172 = '104430631' --Withdraw consent = No - AND d_773707518 = '104430631' --Revoke HIPAA = No + FROM `{fq_base_table}` + {final_where} ) -""".strip()) +""".strip() - # find participants config - participants_cfg = next( - (t for t in source_tables if t["table"] == "participants"), - None - ) + select_parts = [] + join_parts = [] - # participant columns - if participants_cfg: - for col in participants_cfg.get("columns", []): - select_parts.append(f"participants.{col}") + # Always include join key first from base table + select_parts.append(f"{base_alias}.{join_key}") - # process other tables - for source in source_tables: - if source["table"] == "participants": - continue + # Base table columns + base_cols = base_config.get("columns", []) - alias = source["table"] - source_ref = f"`{project}.{source['dataset']}.{source['table']}`" + # Lookup schema for validation + base_schema = set(table_schemas.get((base_config["dataset"], base_config["table"]), [])) - if source.get("select_all"): - cols = table_schemas.get((source["dataset"], source["table"]), []) - else: - cols = source.get("columns", []) + if base_cols == "*": + base_cols = list(base_schema) - if not cols: - continue + for col in base_cols: + if col in base_schema: + if col == join_key: + continue # Skip join key since it is already included + select_parts.append(f"{base_alias}.{col}") - join_parts.append( - f"LEFT JOIN {source_ref} {alias} " - f"ON {alias}.Connect_ID = participants.Connect_ID" - ) + # Join tables + for join_table in destination_table_config.get("join_tables", []): + ds = join_table["dataset"] + tbl = join_table["table"] + + join_schema = set(table_schemas.get((ds, tbl), [])) + cols = join_table.get("columns", []) - for col in cols: - select_parts.append(f"{alias}.{col}") + if cols == "*": + cols = list(join_schema) - if not select_parts: + if not cols: continue - sql = f""" -CREATE OR REPLACE TABLE `{project}.{dest_dataset}.{dest_table}` AS + # Add LEFT JOIN + join_parts.append( + f"LEFT JOIN `{project}.{ds}.{tbl}` {tbl} " + f"ON {tbl}.{join_key} = {base_alias}.{join_key}" + ) + + # Add selected columns + for col in cols: + if col in join_schema: + if col == join_key: + continue # Skip join key since it is already included + select_parts.append(f"{tbl}.{col}") + + # Ensure at least one column is selected + if not select_parts: + raise ValueError(f"No columns selected for {destination_table}") + + sql = f""" +CREATE OR REPLACE TABLE `{project}.{destination_dataset}.{destination_table}` AS WITH -{",\n".join(ctes)} +{cte} SELECT {",\n ".join(select_parts)} -FROM filtered_participants participants +FROM {cte_name} {base_alias} {"\n".join(join_parts)} """.strip() - queries.append(sql) + return { + "destination_table": f"{project}.{destination_dataset}.{destination_table}", + "sql": sql, + } + +def create_subset_missing_columns_json( + output_path: str, + destination_table: str, + missing_cleaned_cols: list[dict[str, Any]], + missing_cleaned_loop_vars: list[dict[str, Any]], + client: storage.Client, +) -> None: + """ + Write a JSON report of missing columns to a GCS location. + + Args: + output_path (str): GCS path (gs://...) to write the report. + destination_table (str): Destination table name. + missing_cleaned_cols (list): Missing standard columns. + missing_cleaned_loop_vars (list): Missing loop variable columns. + client (storage.Client): GCS client. + """ + # Build report structure + report = { + "_metadata": { + "generated_at": datetime.now(timezone.utc).isoformat(), + "source": "pr2-transformation pipeline", + "destination_table": destination_table, + "description": "Report of columns requested by the subset config that were not found in their respective source tables", + "structure": { + "missing_cleaned_cols": "Columns that were not found and are NOT cleaned loop variables — these are unexpected and should be investigated", + "missing_cleaned_loop_vars": "Columns that were not found but ARE cleaned loop variables — these may be expected if the loop variable does not exist for a given table" + } + }, + "missing_cleaned_cols": missing_cleaned_cols, + "missing_cleaned_loop_vars": missing_cleaned_loop_vars + } + + # Parse GCS path + path = output_path.removeprefix("gs://") + bucket_name, blob_path = path.split("/", 1) + + bucket = client.bucket(bucket_name) + blob = bucket.blob(blob_path) - return queries \ No newline at end of file + # Upload JSON report + blob.upload_from_string( + json.dumps(report, indent=2, ensure_ascii=False), + content_type="application/json", + ) \ No newline at end of file From 257127dfbfa29586c469cf0bfa3c2358008d8e63 Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Thu, 21 May 2026 16:40:06 -0400 Subject: [PATCH 04/67] Add duplicate column check when creating the subset table --- core/transformations.py | 6 ++- core/utils.py | 79 ++++++++++++++++++++++++++++-------- reference/subset_config.json | 4 ++ 3 files changed, 70 insertions(+), 19 deletions(-) diff --git a/core/transformations.py b/core/transformations.py index b3e489e..8a5e0c0 100644 --- a/core/transformations.py +++ b/core/transformations.py @@ -1005,6 +1005,7 @@ def create_subset_table( sql = result["sql"] fq_destination_table = result["destination_table"] + duplicate_columns = result.get("duplicate_columns", []) # Save the SQL to GCS for audit purposes try: @@ -1037,11 +1038,12 @@ def create_subset_table( report_path = f"{missing_report_base_path}{fq_destination_table}_missing_columns.json" utils.create_subset_missing_columns_json( + client=gcs_client, output_path=report_path, destination_table=fq_destination_table, missing_cleaned_cols=missing_cleaned_cols, missing_cleaned_loop_vars=missing_cleaned_loop_vars, - client=gcs_client + duplicate_columns=duplicate_columns ) utils.logger.info(f"Missing column report saved to {report_path}") @@ -1070,5 +1072,5 @@ def create_subset_table( create_subset_table( config_path=config_path, destination_dataset="SensitiveTier", - destination_table="module2" + destination_table="mvp" ) diff --git a/core/utils.py b/core/utils.py index 98cfe47..6eb8b96 100644 --- a/core/utils.py +++ b/core/utils.py @@ -1185,8 +1185,8 @@ def find_missing_columns( - list[dict]: Missing standard columns. - list[dict]: Missing loop variable columns. """ - missing_cleaned_cols = [] - missing_cleaned_loop_vars = [] + missing_cleaned_cols = {} + missing_cleaned_loop_vars = {} def _check_cols(config): dataset = config["dataset"] @@ -1206,13 +1206,13 @@ def _check_cols(config): if col in schema: continue - item = {"dataset": dataset, "table": table, "source_col": col} + key = (dataset, table, col) # Separate loop variables versus standard columns if is_cleaned_loop_variable(col): - missing_cleaned_loop_vars.append(item) + missing_cleaned_loop_vars[key] = missing_cleaned_loop_vars.get(key, 0) + 1 else: - missing_cleaned_cols.append(item) + missing_cleaned_cols[key] = missing_cleaned_cols.get(key, 0) + 1 # Check base table _check_cols(destination_table_config["base_table"]) @@ -1221,7 +1221,18 @@ def _check_cols(config): for join_table in destination_table_config.get("join_tables", []): _check_cols(join_table) - return missing_cleaned_cols, missing_cleaned_loop_vars + def _format_output(missing_columns): + return [ + { + "dataset": dataset, + "table": table, + "source_col": source_col, + "occurrence_count": occurrence_count + } + for (dataset, table, source_col), occurrence_count in missing_columns.items() + ] + + return _format_output(missing_cleaned_cols), _format_output(missing_cleaned_loop_vars) def build_filter_sql( filter_profiles: dict[str, Any], @@ -1335,7 +1346,7 @@ def build_subset_query( base_condition = f"{join_key} IS NOT NULL" if filter_sql: - #Remove leading WHERE from filter_sql safely + # Remove leading WHERE from filter_sql safely filter_body = filter_sql.replace("WHERE", "", 1).strip() final_where = f""" @@ -1358,8 +1369,12 @@ def build_subset_query( select_parts = [] join_parts = [] + selected_column_names = set() + duplicate_columns = [] + # Always include join key first from base table select_parts.append(f"{base_alias}.{join_key}") + selected_column_names.add(join_key) # Base table columns base_cols = base_config.get("columns", []) @@ -1374,14 +1389,22 @@ def build_subset_query( if col in base_schema: if col == join_key: continue # Skip join key since it is already included + + # If column has already been selected (e.g. from a previous table), + # track it as a duplicate and skip to avoid ambiguity + if col in selected_column_names: + duplicate_columns.append(col) + continue + select_parts.append(f"{base_alias}.{col}") + selected_column_names.add(col) # Join tables for join_table in destination_table_config.get("join_tables", []): - ds = join_table["dataset"] - tbl = join_table["table"] + dataset = join_table["dataset"] + table = join_table["table"] - join_schema = set(table_schemas.get((ds, tbl), [])) + join_schema = set(table_schemas.get((dataset, table), [])) cols = join_table.get("columns", []) if cols == "*": @@ -1392,8 +1415,8 @@ def build_subset_query( # Add LEFT JOIN join_parts.append( - f"LEFT JOIN `{project}.{ds}.{tbl}` {tbl} " - f"ON {tbl}.{join_key} = {base_alias}.{join_key}" + f"LEFT JOIN `{project}.{dataset}.{table}` {table} " + f"ON {table}.{join_key} = {base_alias}.{join_key}" ) # Add selected columns @@ -1401,7 +1424,23 @@ def build_subset_query( if col in join_schema: if col == join_key: continue # Skip join key since it is already included - select_parts.append(f"{tbl}.{col}") + + # If column has already been selected (e.g. from a previous table), + # track it as a duplicate and skip to avoid ambiguity + if col in selected_column_names: + duplicate_columns.append(f"{dataset}.{table}.{col}") + continue + + select_parts.append(f"{table}.{col}") + selected_column_names.add(col) + + # Log duplicate columns (if any) + if duplicate_columns: + unique_dupes = sorted(set(duplicate_columns)) + utils.logger.warning( + f"[{destination_table}] Duplicate columns detected and skipped: " + f"{', '.join(unique_dupes)}. Using first occurrence only." + ) # Ensure at least one column is selected if not select_parts: @@ -1420,24 +1459,27 @@ def build_subset_query( return { "destination_table": f"{project}.{destination_dataset}.{destination_table}", "sql": sql, + "duplicate_columns": sorted(set(duplicate_columns)) } def create_subset_missing_columns_json( + client: storage.Client, output_path: str, destination_table: str, missing_cleaned_cols: list[dict[str, Any]], missing_cleaned_loop_vars: list[dict[str, Any]], - client: storage.Client, + duplicate_columns: list[str] ) -> None: """ Write a JSON report of missing columns to a GCS location. Args: + client (storage.Client): GCS client. output_path (str): GCS path (gs://...) to write the report. destination_table (str): Destination table name. missing_cleaned_cols (list): Missing standard columns. missing_cleaned_loop_vars (list): Missing loop variable columns. - client (storage.Client): GCS client. + duplicate_columns (list): List of duplicate column names. """ # Build report structure report = { @@ -1448,11 +1490,14 @@ def create_subset_missing_columns_json( "description": "Report of columns requested by the subset config that were not found in their respective source tables", "structure": { "missing_cleaned_cols": "Columns that were not found and are NOT cleaned loop variables — these are unexpected and should be investigated", - "missing_cleaned_loop_vars": "Columns that were not found but ARE cleaned loop variables — these may be expected if the loop variable does not exist for a given table" + "missing_cleaned_loop_vars": "Columns that were not found but ARE cleaned loop variables — these may be expected if the loop variable does not exist for a given table", + "duplicate_columns": "Columns skipped due to duplicate names in SELECT", + "occurrence_count": "Number of times this column appeared in the config AND was missing" } }, "missing_cleaned_cols": missing_cleaned_cols, - "missing_cleaned_loop_vars": missing_cleaned_loop_vars + "missing_cleaned_loop_vars": missing_cleaned_loop_vars, + "duplicate_columns": duplicate_columns } # Parse GCS path diff --git a/reference/subset_config.json b/reference/subset_config.json index f4d1c6b..24d47ee 100644 --- a/reference/subset_config.json +++ b/reference/subset_config.json @@ -19,7 +19,11 @@ "table": "participants", "columns": [ "d_130371375_d_303552867_d_297462035", + "fakecolumn", + "d_130371375_d_303552867_20", + "d_130371375_d_303552867_20", "d_130371375_d_303552867_d_320023644", + "d_130371375_d_303552867_d_438636757", "d_130371375_d_303552867_d_438636757" ] }, From 8a285df88401333d574e1686053496c73f25886c Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Fri, 22 May 2026 10:13:41 -0400 Subject: [PATCH 05/67] Precompute SQL join/select strings to avoid f-string syntax error --- core/utils.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/core/utils.py b/core/utils.py index 6eb8b96..1b34bec 100644 --- a/core/utils.py +++ b/core/utils.py @@ -1446,14 +1446,17 @@ def build_subset_query( if not select_parts: raise ValueError(f"No columns selected for {destination_table}") + select_sql = ",\n ".join(select_parts) + join_sql = "\n".join(join_parts) + sql = f""" CREATE OR REPLACE TABLE `{project}.{destination_dataset}.{destination_table}` AS WITH {cte} SELECT - {",\n ".join(select_parts)} + {select_sql} FROM {cte_name} {base_alias} -{"\n".join(join_parts)} +{join_sql} """.strip() return { From d25de667513837cbbc2de6d4f5071377f292a9b2 Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Fri, 22 May 2026 16:20:18 -0400 Subject: [PATCH 06/67] Add participant status parquet reader and initial classification logic --- core/utils.py | 183 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 183 insertions(+) diff --git a/core/utils.py b/core/utils.py index 1b34bec..64cd592 100644 --- a/core/utils.py +++ b/core/utils.py @@ -1514,4 +1514,187 @@ def create_subset_missing_columns_json( blob.upload_from_string( json.dumps(report, indent=2, ensure_ascii=False), content_type="application/json", + ) + +def read_parquet_to_dataframe(parquet_url: str) -> pd.DataFrame: + df = pd.read_parquet(parquet_url, engine="pyarrow") + + # Safety filter: keep verified participants only + # Change this to your actual verified indicator if needed + df = df.loc[df["verified_status"].eq("Verified")].copy() + + # Normalize timestamp columns + # Black strings, nulls, and invalid timestamps become NaT + ts_cols = ["consent_withdrawn_ts", "hipaa_revoked_ts", "data_destruction_ts", + "module1_complete_ts", "module2_complete_ts", "module3_complete_ts", + "module4_complete_ts", "bio_complete_ts", "clinicalbio_complete_ts", + "mouthwash_complete_ts", "menstrual_complete_ts", "covid19_complete_ts", + "experience2024_complete_ts"] + for col in ts_cols: + df[col] = pd.to_datetime(df[col], errors="coerce", utc=True) + + # Remove print statement later + print(df) + + return df + +def classify_participants(df: pd.DataFrame) -> pd.DataFrame: + df = df.copy() + + # ========================================================================= + # Boolean masks for conditions + # Each variable becomes a True/False Series across the dataframe + # ========================================================================= + + # Boolean masks for "Yes" flag values + destroy_yes = df["data_destruction_requested"].eq("Yes") + withdraw_yes = df["consent_withdrawn"].eq("Yes") + revoke_yes = df["hipaa_revoked"].eq("Yes") + + # Boolean masks for "No" flag values + destroy_no = df["data_destruction_requested"].eq("No") + withdraw_no = df["consent_withdrawn"].eq("No") + revoke_no = df["hipaa_revoked"].eq("No") + + # Boolean masks for missing timestamps + destroy_ts_missing = df["d_destroy_ts"].isna() + withdraw_ts_missing = df["d_withdraw_ts"].isna() + revoke_ts_missing = df["d_revoke_ts"].isna() + + # ========================================================================= + # Timestamp ordering validation for case 3B + # ========================================================================= + + # revoke_after_withdraw: True when revoke timestamp exists AND is later than withdraw timestamp + # This is considered an anomaly because EHR cutoff cannot be reliably determined + revoke_after_withdraw = ( + df["d_revoke_ts"].notna() + & df["d_withdraw_ts"].notna() + & (df["d_revoke_ts"] > df["d_withdraw_ts"]) + ) + + # revoke_before_withdraw: True when revoke timestamp exists AND occurs before withdraw timestamp + # This is the expected ordering (valid) case 3B scenario + revoke_before_withdraw = ( + df["d_revoke_ts"].notna() + & df["d_withdraw_ts"].notna() + & (df["d_revoke_ts"] < df["d_withdraw_ts"]) + ) + + # ========================================================================= + # Validation / exception masks + # These identify records that violate expected data rules, such as having a + # "Yes" flag but missing the corresponding timestamp. + # These participants are excluded from downstream processing and flagged + # fo rdata quality review. + # ========================================================================= + + # Rule 1: Withdraw Consent = Yes but withdraw timestamp is NULL + # Rule 2: Revoke HIPAA = Yes but revoke timestamp is NULL AND participant did NOT withdraw consent + exception_mask = ( + (withdraw_yes & withdraw_ts_missing) + | (revoke_yes & revoke_ts_missing & withdraw_no) + ) + # ========================================================================= + # Anomaly validation rules + # These participants are still processed downstream but are flagged because + # their data is internally inconsistent or violates expected temporal logic. + # ========================================================================= + + # Rule: Destory Data = Yes but destroy timestamp is NULL + # Participant is still classified as Case 4 and excluded downstream, + # but the missing timestamp is recorded as an anomaly. + anomaly_destroy_missing_ts = destroy_yes & destroy_ts_missing + + # Rule: Revoke timestamp occurs AFTER withdraw timestamp + # This creates ambiguity because EHR cutoff cannot be reliably determined. + # Participant is still processed as Case 3A using withdraw timestamp for + # survey/bio cutoffs, but anomaly is recorded. + anomaly_revoke_after_withdraw = withdraw_yes & revoke_yes & revoke_after_withdraw + + # ============================================================================ + # Case classification masks + # These masks determine which business-rule case each participant belongs to. + # ============================================================================ + + # ============================================================================ + # Case 1 + # - Destroy Data = No + # - Withdraw Consent = No + # - Revoke HIPAA = No + # + # No restrictions apply. + # Participant and all their data can be included without any cutoffs. + # ============================================================================ + case_1 = ( + destroy_no + & withdraw_no + & revoke_no + ) + + # ============================================================================ + # Case 2 + # - Destroy Data = No + # - Withdraw Consent = No + # - Revoke HIPAA = Yes (timestamp present) + # + # EHR cutoff = Revoke timestamp + # Survey cutoff = None + # Biospecimen = None + # ============================================================================ + case_2 = ( + destroy_no + & withdraw_no + & revoke_yes + & df["hipaa_revoked_ts"].notna() + ) + + # ============================================================================ + # Case 3A + # - Destroy Data = No + # - Withdraw Consent = Yes (timestamp present) + # - Revoke HIPAA = Yes + # + # EHR cutoff = Revoke timestamp + # Survey cutoff = Withdraw timestamp + # Biospecimen cutoff = Withdraw timestamp + # ============================================================================ + case_3A = ( + destroy_no + & withdraw_yes + & revoke_yes + & df["consent_withdrawn_ts"].notna() + & df["hipaa_revoked_ts"].isna() + ) + + # ============================================================================ + # Case 3B + # - Destroy Data = No + # - Withdraw Consent = Yes (timestamp present and AFTER revoke timestamp) + # - Revoke HIPAA = Yes (timestamp present and occurs BEFORE withdraw timestamp) + # + # EHR cutoff = Revoke timestamp + # Survey cutoff = Withdraw timestamp + # Biospecimen cutoff = Withdraw timestamp + # ============================================================================ + case_3B = ( + destroy_no + & withdraw_yes + & revoke_yes + & df["consent_withdrawn_ts"].notna() + & revoke_before_withdraw + ) + + # ============================================================================ + # Case 4 + # - Destroy Data = Yes + # - Withdraw Consent = Yes + # - Revoke HIPAA = Yes + # + # The participant and all their data is excluded from downstream processing + # ============================================================================ + case_4 = ( + destroy_yes + & withdraw_yes + & revoke_yes ) \ No newline at end of file From d4b0d531d2e296816fb58171229960eb61e06a17 Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Thu, 18 Jun 2026 17:01:47 -0400 Subject: [PATCH 07/67] Implement participant classification, exclusion, anomaly, and cutoff logic --- core/utils.py | 138 +++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 115 insertions(+), 23 deletions(-) diff --git a/core/utils.py b/core/utils.py index 64cd592..0b0e367 100644 --- a/core/utils.py +++ b/core/utils.py @@ -12,6 +12,7 @@ from google.cloud import bigquery, storage import pandas as pd #TODO Try to avoid using pandas +import numpy as np if __name__ == "__main__": # Add parent directory to Python path when running as script @@ -1557,9 +1558,9 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: revoke_no = df["hipaa_revoked"].eq("No") # Boolean masks for missing timestamps - destroy_ts_missing = df["d_destroy_ts"].isna() - withdraw_ts_missing = df["d_withdraw_ts"].isna() - revoke_ts_missing = df["d_revoke_ts"].isna() + destroy_ts_missing = df["data_destruction_ts"].isna() + withdraw_ts_missing = df["consent_withdrawn_ts"].isna() + revoke_ts_missing = df["hipaa_revoked_ts"].isna() # ========================================================================= # Timestamp ordering validation for case 3B @@ -1568,32 +1569,32 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: # revoke_after_withdraw: True when revoke timestamp exists AND is later than withdraw timestamp # This is considered an anomaly because EHR cutoff cannot be reliably determined revoke_after_withdraw = ( - df["d_revoke_ts"].notna() - & df["d_withdraw_ts"].notna() - & (df["d_revoke_ts"] > df["d_withdraw_ts"]) + df["hipaa_revoked_ts"].notna() + & df["consent_withdrawn_ts"].notna() + & (df["hipaa_revoked_ts"] > df["consent_withdrawn_ts"]) ) # revoke_before_withdraw: True when revoke timestamp exists AND occurs before withdraw timestamp # This is the expected ordering (valid) case 3B scenario revoke_before_withdraw = ( - df["d_revoke_ts"].notna() - & df["d_withdraw_ts"].notna() - & (df["d_revoke_ts"] < df["d_withdraw_ts"]) + df["hipaa_revoked_ts"].notna() + & df["consent_withdrawn_ts"].notna() + & (df["hipaa_revoked_ts"] < df["consent_withdrawn_ts"]) ) # ========================================================================= - # Validation / exception masks + # Validation / exclusion masks # These identify records that violate expected data rules, such as having a # "Yes" flag but missing the corresponding timestamp. # These participants are excluded from downstream processing and flagged - # fo rdata quality review. + # for data quality review. # ========================================================================= # Rule 1: Withdraw Consent = Yes but withdraw timestamp is NULL # Rule 2: Revoke HIPAA = Yes but revoke timestamp is NULL AND participant did NOT withdraw consent - exception_mask = ( - (withdraw_yes & withdraw_ts_missing) - | (revoke_yes & revoke_ts_missing & withdraw_no) + exclusion_mask = ( + (destroy_no & withdraw_yes & withdraw_ts_missing) + | (destroy_no &revoke_yes & revoke_ts_missing & withdraw_no) ) # ========================================================================= # Anomaly validation rules @@ -1604,13 +1605,13 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: # Rule: Destory Data = Yes but destroy timestamp is NULL # Participant is still classified as Case 4 and excluded downstream, # but the missing timestamp is recorded as an anomaly. - anomaly_destroy_missing_ts = destroy_yes & destroy_ts_missing + anomaly_destroy_missing_ts = destroy_yes & withdraw_yes & revoke_yes & destroy_ts_missing # Rule: Revoke timestamp occurs AFTER withdraw timestamp # This creates ambiguity because EHR cutoff cannot be reliably determined. # Participant is still processed as Case 3A using withdraw timestamp for # survey/bio cutoffs, but anomaly is recorded. - anomaly_revoke_after_withdraw = withdraw_yes & revoke_yes & revoke_after_withdraw + anomaly_revoke_after_withdraw = destroy_no & withdraw_yes & revoke_yes & revoke_after_withdraw # ============================================================================ # Case classification masks @@ -1653,31 +1654,35 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: # Case 3A # - Destroy Data = No # - Withdraw Consent = Yes (timestamp present) - # - Revoke HIPAA = Yes + # - Revoke HIPAA = Yes (timestamp is either missing or occurs after withdraw timestamp) # - # EHR cutoff = Revoke timestamp + # EHR cutoff = Withdraw timestamp for normal 3A cases + # EHR cutoff = NULL for anomalous 3A where revoke timestamp occurs after withdraw timestamp # Survey cutoff = Withdraw timestamp # Biospecimen cutoff = Withdraw timestamp # ============================================================================ - case_3A = ( + case_3a = ( destroy_no & withdraw_yes & revoke_yes & df["consent_withdrawn_ts"].notna() - & df["hipaa_revoked_ts"].isna() + & ( + df["hipaa_revoked_ts"].isna() + | revoke_after_withdraw + ) ) # ============================================================================ # Case 3B # - Destroy Data = No - # - Withdraw Consent = Yes (timestamp present and AFTER revoke timestamp) + # - Withdraw Consent = Yes (timestamp present and occurs AFTER revoke timestamp) # - Revoke HIPAA = Yes (timestamp present and occurs BEFORE withdraw timestamp) # # EHR cutoff = Revoke timestamp # Survey cutoff = Withdraw timestamp # Biospecimen cutoff = Withdraw timestamp # ============================================================================ - case_3B = ( + case_3b = ( destroy_no & withdraw_yes & revoke_yes @@ -1697,4 +1702,91 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: destroy_yes & withdraw_yes & revoke_yes - ) \ No newline at end of file + ) + + # Build output classification column based on the above cases and rules + df["case"] = np.select( + [case_1, case_2, case_3a, case_3b, case_4, exclusion_mask], + ["1", "2", "3A", "3B", "4", "EXCLUSION"], + default="UNDEFINED" + ) + + # Build exclusion reason column to specify which rule was violated for + # participants classified as "EXCLUSION" + df["exclusion_reason"] = np.select( + [ + destroy_no & withdraw_yes & withdraw_ts_missing, + destroy_no & revoke_yes & revoke_ts_missing & withdraw_no, + ], + [ + "Withdraw Consent = 'Yes' but timestamp is NULL", + "Revoke HIPAA = 'Yes' but timestamp is NULL", + ], + default=pd.NA + ) + + # Assign reason for unmatched participants + df.loc[ + df["case"] == "UNDEFINED", + "exclusion_reason" + ] = "Participant flag combination did not match any defined case" + + # Build anomaly column to specify which rule was violated for + # participants classified as "1", "2", or "3a" but have internal + # inconsistencies in their data + df["anomaly"] = np.select( + [ + anomaly_destroy_missing_ts, + anomaly_revoke_after_withdraw + ], + [ + "Destroy Data = 'Yes' but timestamp is NULL", + "Revoke HIPAA timestamp occurs AFTER Withdraw Consent timestamp - EHR cutoff could not be determined reliably" + ], + default=pd.NA + ) + + # Cutoff logic + df["ehr_cutoff"] = pd.NaT + df["survey_cutoff"] = pd.NaT + df["bio_cutoff"] = pd.NaT + + # Case 1: No cutoffs, so leave all as NaT; participant is included downstream without restrictions + + # Case 2: EHR cutoff = Revoke timestamp, Survey/Bio cutoff = None + df.loc[case_2, "ehr_cutoff"] = df.loc[case_2, "hipaa_revoked_ts"] + + # Case 3A: EHR/Survey/Bio cutoff = Withdraw timestamp + normal_3a = case_3a & df["hipaa_revoked_ts"].isna() + + df.loc[normal_3a, "ehr_cutoff"] = df.loc[normal_3a, "consent_withdrawn_ts"] + df.loc[normal_3a, "survey_cutoff"] = df.loc[normal_3a, "consent_withdrawn_ts"] + df.loc[normal_3a, "bio_cutoff"] = df.loc[normal_3a, "consent_withdrawn_ts"] + + # Case 3A anomaly + # If revoke_ts is later than withdraw_ts, keep as anomaly and leave ehr_cutoff blank + anomalous_3a = case_3a & revoke_after_withdraw + + df.loc[anomalous_3a, "survey_cutoff"] = df.loc[anomalous_3a, "consent_withdrawn_ts"] + df.loc[anomalous_3a, "bio_cutoff"] = df.loc[anomalous_3a, "consent_withdrawn_ts"] + + # Case 3B: EHR cutoff = Revoke timestamp, Survey/Bio cutoff = Withdraw timestamp + df.loc[case_3b, "ehr_cutoff"] = df.loc[case_3b, "hipaa_revoked_ts"] + df.loc[case_3b, "survey_cutoff"] = df.loc[case_3b, "consent_withdrawn_ts"] + df.loc[case_3b, "bio_cutoff"] = df.loc[case_3b, "consent_withdrawn_ts"] + + # Case 4: No cutoffs, so leave all as NaT; participant is excluded downstream + + df = df.sort_values(by=["exclusion_reason", "anomaly", "case", "Connect_ID"]) + + # Sort dataframe in order of: + # 1. Undefined cases + # 2. Exclusion cases + # 3. Cases with anomalies + # 4. Normal cases + df["sort_order"] = np.select( + [df["case"].eq("UNDEFINED"), df["case"].eq("EXCLUSION"), df["anomaly"].notna()], + [1, 2, 3], default=4 + ) + + return df \ No newline at end of file From 3327edbb4d76ceb68f4a3a9fe834b1d835b00a0d Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Wed, 24 Jun 2026 14:42:41 -0400 Subject: [PATCH 08/67] Cast timestamps to datetime64[us, UTC] and add BQ write function for the classification data --- core/utils.py | 73 +++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 68 insertions(+), 5 deletions(-) diff --git a/core/utils.py b/core/utils.py index 0b0e367..53126c7 100644 --- a/core/utils.py +++ b/core/utils.py @@ -1532,7 +1532,7 @@ def read_parquet_to_dataframe(parquet_url: str) -> pd.DataFrame: "mouthwash_complete_ts", "menstrual_complete_ts", "covid19_complete_ts", "experience2024_complete_ts"] for col in ts_cols: - df[col] = pd.to_datetime(df[col], errors="coerce", utc=True) + df[col] = pd.to_datetime(df[col], errors="coerce", utc=True).astype("datetime64[us, UTC]") # Remove print statement later print(df) @@ -1540,6 +1540,7 @@ def read_parquet_to_dataframe(parquet_url: str) -> pd.DataFrame: return df def classify_participants(df: pd.DataFrame) -> pd.DataFrame: + df = df.copy() # ========================================================================= @@ -1747,9 +1748,9 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: ) # Cutoff logic - df["ehr_cutoff"] = pd.NaT - df["survey_cutoff"] = pd.NaT - df["bio_cutoff"] = pd.NaT + df["ehr_cutoff"] = pd.Series(pd.NaT, index=df.index, dtype="datetime64[us, UTC]") + df["survey_cutoff"] = pd.Series(pd.NaT, index=df.index, dtype="datetime64[us, UTC]") + df["bio_cutoff"] = pd.Series(pd.NaT, index=df.index, dtype="datetime64[us, UTC]") # Case 1: No cutoffs, so leave all as NaT; participant is included downstream without restrictions @@ -1765,6 +1766,7 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: # Case 3A anomaly # If revoke_ts is later than withdraw_ts, keep as anomaly and leave ehr_cutoff blank + # This will still be processed as a 3A case anomalous_3a = case_3a & revoke_after_withdraw df.loc[anomalous_3a, "survey_cutoff"] = df.loc[anomalous_3a, "consent_withdrawn_ts"] @@ -1789,4 +1791,65 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: [1, 2, 3], default=4 ) - return df \ No newline at end of file + return df + +def write_classification_to_bq(classification_df, bq_table_name, client) -> None: + """ + Write the classification DataFrame to a BigQuery table + Args: + classification_df (pd.DataFrame): DataFrame containing participant classifications. + bq_table_name (str): Fully qualified BigQuery table name (project.dataset.table). + client (bigquery.Client): BigQuery client for writing data. + """ + job_config = bigquery.LoadJobConfig( + schema=[ + # String columns + bigquery.SchemaField("Connect_ID", "STRING"), + bigquery.SchemaField("verified_status", "STRING"), + bigquery.SchemaField("verified_status_concept_id", "STRING"), + bigquery.SchemaField("consent_withdrawn", "STRING"), + bigquery.SchemaField("consent_withdrawn_concept_id", "STRING"), + bigquery.SchemaField("hipaa_revoked", "STRING"), + bigquery.SchemaField("hipaa_revoked_concept_id", "STRING"), + bigquery.SchemaField("data_destruction_requested", "STRING"), + bigquery.SchemaField("data_destruction_requested_concept_id", "STRING"), + + # Timestamp columns + bigquery.SchemaField("consent_withdrawn_ts", "TIMESTAMP"), + bigquery.SchemaField("hipaa_revoked_ts", "TIMESTAMP"), + bigquery.SchemaField("data_destruction_ts", "TIMESTAMP"), + bigquery.SchemaField("module1_complete_ts", "TIMESTAMP"), + bigquery.SchemaField("module2_complete_ts", "TIMESTAMP"), + bigquery.SchemaField("module3_complete_ts", "TIMESTAMP"), + bigquery.SchemaField("module4_complete_ts", "TIMESTAMP"), + bigquery.SchemaField("bio_complete_ts", "TIMESTAMP"), + bigquery.SchemaField("clinicalbio_complete_ts", "TIMESTAMP"), + bigquery.SchemaField("mouthwash_complete_ts", "TIMESTAMP"), + bigquery.SchemaField("menstrual_complete_ts", "TIMESTAMP"), + bigquery.SchemaField("covid19_complete_ts", "TIMESTAMP"), + bigquery.SchemaField("experience2024_complete_ts", "TIMESTAMP"), + + # Classification fields + bigquery.SchemaField("case", "STRING"), + bigquery.SchemaField("exclusion_reason", "STRING"), + bigquery.SchemaField("anomaly", "STRING"), + + # Derived cutoffs + bigquery.SchemaField("ehr_cutoff", "TIMESTAMP"), + bigquery.SchemaField("survey_cutoff", "TIMESTAMP"), + bigquery.SchemaField("bio_cutoff", "TIMESTAMP"), + + # Helper field + bigquery.SchemaField("sort_order", "INTEGER"), + ], + write_disposition=bigquery.WriteDisposition.WRITE_TRUNCATE + ) + + # Load DataFrame to BigQuery + job = client.load_table_from_dataframe( + classification_df, + bq_table_name, + job_config=job_config + ) + job.result() # Wait for the job to complete + utils.logger.info(f"Classification data written to {bq_table_name} successfully.") \ No newline at end of file From b939787fccd6bf7e57c02a39eff1f4b32b30c47f Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Wed, 1 Jul 2026 14:09:31 -0400 Subject: [PATCH 09/67] Add support for optional classification_filter in subset query generation and update subset config for testing --- core/utils.py | 53 ++++++++++++++++++++++++++---------- reference/subset_config.json | 15 ++++++++-- 2 files changed, 51 insertions(+), 17 deletions(-) diff --git a/core/utils.py b/core/utils.py index 53126c7..66538cd 100644 --- a/core/utils.py +++ b/core/utils.py @@ -1304,23 +1304,27 @@ def build_subset_query( full_config: dict[str, Any], destination_table_config: dict[str, Any], table_schemas: dict[tuple[str, str], list[str]], + classification_table: Optional[str] = None ) -> dict[str, Any]: """ Build a CREATE OR REPLACE TABLE SQL query based on subset configuration. - Combines base table, optional filters, and join tables into a single SQL query. + Combines base table, optional filters, optional classification filter, + and join tables into a single SQL query. Args: client (bigquery.Client): BigQuery client. full_config (dict): Full configuration including filter profiles. destination_table_config (dict): Destination table configuration. table_schemas (dict): Table schema lookup. + classification_table (str, optional): Fully qualified classification table name Returns: dict: Contains destination table name and generated SQL string. Raises: - ValueError: If no columns are selected. + ValueError: If no columns are selected or if classification_table is missing + when allowed_cases is populated. """ project = client.project destination_dataset = destination_table_config["dataset"] @@ -1328,6 +1332,10 @@ def build_subset_query( # Default join key if not specified join_key = destination_table_config.get("join_key", "Connect_ID") + # Check for classification filter + classification_filter = destination_table_config.get("classification_filter") + allowed_cases = classification_filter.get("allowed_cases", []) if classification_filter else [] + # Build WHERE clause from filter profile filter_sql = build_filter_sql( full_config.get("filter_profiles", {}), @@ -1343,29 +1351,46 @@ def build_subset_query( cte_name = f"filtered_{base_table}" base_alias = base_table - # Combine base constraint and filter_sql - base_condition = f"{join_key} IS NOT NULL" + # Build CTE WHERE clause + # Combines base constraint, filter profile, and classification filter + + # Add classification filter condition if present + if allowed_cases: + where_conditions = [f"p.{join_key} IS NOT NULL"] # Base constraint to ensure join key is not null + cases_str = ", ".join([f"'{c}'" for c in allowed_cases]) + where_conditions.append(f"c.`case` IN ({cases_str})") + else: + where_conditions = [f"{join_key} IS NOT NULL"] + # Add raw column filter conditions if present if filter_sql: # Remove leading WHERE from filter_sql safely filter_body = filter_sql.replace("WHERE", "", 1).strip() + where_conditions.append(filter_body) - final_where = f""" -WHERE {base_condition} - AND {filter_body} -""".strip() + final_where = "WHERE " + "\n AND ".join(where_conditions) + # Build CTE with or without classification join + if allowed_cases: + if not classification_table: + raise ValueError( + "classification_table must be provided when allowed_cases is populated." + ) + cte = f""" +{cte_name} AS ( + SELECT p.* + FROM `{fq_base_table}` p + INNER JOIN `{classification_table}` c + ON p.{join_key} = c.{join_key} + {final_where} +)""".strip() else: - final_where = f"WHERE {base_condition}" - - # Build CTE with optional filtering - cte = f""" + cte = f""" {cte_name} AS ( SELECT * FROM `{fq_base_table}` {final_where} -) -""".strip() +)""".strip() select_parts = [] join_parts = [] diff --git a/reference/subset_config.json b/reference/subset_config.json index 24d47ee..9d31752 100644 --- a/reference/subset_config.json +++ b/reference/subset_config.json @@ -10,10 +10,13 @@ }, "destination_tables": [ { - "dataset": "SensitiveTier", - "table": "mvp", + "dataset": "pr2_mvp", + "table": "mvp_case_1_2", "join_key": "Connect_ID", - "filter_profile": "default_participants", + "filter_profile": "no_filters", + "classification_filter": { + "allowed_cases": ["1", "2"] + }, "base_table": { "dataset": "CleanConnect", "table": "participants", @@ -47,6 +50,9 @@ "table": "module1", "join_key": "Connect_ID", "filter_profile": "default_participants", + "classification_filter": { + "allowed_cases": [] + }, "base_table": { "dataset": "CleanConnect", "table": "participants", @@ -67,6 +73,9 @@ "table": "module1", "join_key": "Connect_ID", "filter_profile": "default_participants", + "classification_filter": { + "allowed_cases": [] + }, "base_table": { "dataset": "CleanConnect", "table": "participants", From 15e236468b1c35191b9b99dbfced15dc6d1b13ea Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Wed, 1 Jul 2026 14:43:29 -0400 Subject: [PATCH 10/67] Filter to verified participants while reading the parquet file instead of after loading it --- core/utils.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/core/utils.py b/core/utils.py index 66538cd..e8030cf 100644 --- a/core/utils.py +++ b/core/utils.py @@ -1543,11 +1543,10 @@ def create_subset_missing_columns_json( ) def read_parquet_to_dataframe(parquet_url: str) -> pd.DataFrame: - df = pd.read_parquet(parquet_url, engine="pyarrow") - - # Safety filter: keep verified participants only - # Change this to your actual verified indicator if needed - df = df.loc[df["verified_status"].eq("Verified")].copy() + # Only keep verified participants + df = pd.read_parquet(parquet_url, + engine="pyarrow", + filters=[("verified_status", "==", "Verified")]) # Normalize timestamp columns # Black strings, nulls, and invalid timestamps become NaT @@ -1758,7 +1757,7 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: ] = "Participant flag combination did not match any defined case" # Build anomaly column to specify which rule was violated for - # participants classified as "1", "2", or "3a" but have internal + # participants classified as cases 4 or 3A but have internal # inconsistencies in their data df["anomaly"] = np.select( [ From 0b8c021a8ed309828277791e28111462e443c010 Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Wed, 1 Jul 2026 14:56:11 -0400 Subject: [PATCH 11/67] Remove unnecessary print() and copy() statements --- core/utils.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/core/utils.py b/core/utils.py index e8030cf..30b8802 100644 --- a/core/utils.py +++ b/core/utils.py @@ -1558,15 +1558,9 @@ def read_parquet_to_dataframe(parquet_url: str) -> pd.DataFrame: for col in ts_cols: df[col] = pd.to_datetime(df[col], errors="coerce", utc=True).astype("datetime64[us, UTC]") - # Remove print statement later - print(df) - return df def classify_participants(df: pd.DataFrame) -> pd.DataFrame: - - df = df.copy() - # ========================================================================= # Boolean masks for conditions # Each variable becomes a True/False Series across the dataframe @@ -1803,6 +1797,7 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: # Case 4: No cutoffs, so leave all as NaT; participant is excluded downstream + # Sort the dataframe for code development and debugging purposes. This can be removed in the future. df = df.sort_values(by=["exclusion_reason", "anomaly", "case", "Connect_ID"]) # Sort dataframe in order of: From 070f13a38ee338b971f2b426a1cded96bf261229 Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Thu, 2 Jul 2026 16:52:29 -0400 Subject: [PATCH 12/67] Add subset column censoring based on participant classification and module completion --- core/constants.py | 94 +++++++++++++++++++++++++++++++++++++++++++++++ core/utils.py | 65 ++++++++++++++++++++++++++++++-- 2 files changed, 155 insertions(+), 4 deletions(-) diff --git a/core/constants.py b/core/constants.py index 7ff823a..6025b97 100644 --- a/core/constants.py +++ b/core/constants.py @@ -185,3 +185,97 @@ } ] } + +# Secondary source concept IDs for each module/survey +MODULE_CENSOR_RULES = { + # Module 1: Background and Overall Health + "726699695": { + "source_table": "module1", + "status_col": "module1_status", + "completion_ts": "module1_complete_ts", + "cutoff": "survey_cutoff", + }, + # Module 2: Medications, Reproductive Health, Exercise, and Sleep + "745268907": { + "source_table": "module2", + "status_col": "module2_status", + "completion_ts": "module2_complete_ts", + "cutoff": "survey_cutoff", + }, + # Module 3: Smoking, Alcohol, and Sun Exposure + "965707586": { + "source_table": "module3", + "status_col": "module3_status", + "completion_ts": "module3_complete_ts", + "cutoff": "survey_cutoff", + }, + # Module 4: Where You Live and Work + "716117817": { + "source_table": "module4", + "status_col": "module4_status", + "completion_ts": "module4_complete_ts", + "cutoff": "survey_cutoff", + }, + # Blood/Urine/Mouthwash + "299215535": { + "source_table": "bioSurvey", + "status_col": "bio_status", + "completion_ts": "bio_complete_ts", + "cutoff": "survey_cutoff", + }, + # Blood/Urine + "826163434": { + "source_table": "clinicalBioSurvey", + "status_col": "clinicalbio_status", + "completion_ts": "clinicalbio_complete_ts", + "cutoff": "survey_cutoff", + }, + # Mouthwash + "390351864": { + "source_table": "mouthwash", + "status_col": "mouthwash_status", + "completion_ts": "mouthwash_complete_ts", + "cutoff": "survey_cutoff", + }, + # Menstrual Cycle (contains two secondary source concept IDs) + "912367929": { + "source_table": "menstrualSurvey", + "status_col": "menstrual_status", + "completion_ts": "menstrual_complete_ts", + "cutoff": "survey_cutoff", + }, + # Menstrual Cycle (contains two secondary source concept IDs) + "232438133": { + "source_table": "menstrualSurvey", + "status_col": "menstrual_status", + "completion_ts": "menstrual_complete_ts", + "cutoff": "survey_cutoff", + }, + # COVID-19 + "793330426": { + "source_table": "covid19Survey", + "status_col": "covid19_status", + "completion_ts": "covid19_complete_ts", + "cutoff": "survey_cutoff", + }, + # 2024 Connect Experience Survey + "506648060": { + "source_table": "experience2024", + "status_col": "experience2024_status", + "completion_ts": "experience2024_complete_ts", + "cutoff": "survey_cutoff", + }, +} + +NON_CENSORED_SECONDARY_SOURCE_CIDS = { + "214456996": {"source_table": "participants"}, # Eligibility Screener + "332759827": {"source_table": "participants"}, # User Profile + "218595434": {"source_table": "participants"}, # Verification + "883203566": {"source_table": "participants"}, # Sign in + "273437590": {"source_table": "participants"}, # Consent + "104913069": {"source_table": "participants"}, # Research- Finalization and shipping +} + +ALWAYS_INCLUDE_NON_CID_COLUMNS = { + "token", +} \ No newline at end of file diff --git a/core/utils.py b/core/utils.py index 30b8802..3d0ac51 100644 --- a/core/utils.py +++ b/core/utils.py @@ -1350,6 +1350,7 @@ def build_subset_query( # CTE name and alias cte_name = f"filtered_{base_table}" base_alias = base_table + classification_alias = base_alias # Build CTE WHERE clause # Combines base constraint, filter profile, and classification filter @@ -1378,7 +1379,9 @@ def build_subset_query( ) cte = f""" {cte_name} AS ( - SELECT p.* + SELECT + p.*, + c.* EXCEPT({join_key}) FROM `{fq_base_table}` p INNER JOIN `{classification_table}` c ON p.{join_key} = c.{join_key} @@ -1422,7 +1425,13 @@ def build_subset_query( duplicate_columns.append(col) continue - select_parts.append(f"{base_alias}.{col}") + select_parts.append( + utils.render_subset_expression( + source_alias=base_alias, + col_name=col, + classification_alias=classification_alias, + ) +) selected_column_names.add(col) # Join tables @@ -1457,7 +1466,13 @@ def build_subset_query( duplicate_columns.append(f"{dataset}.{table}.{col}") continue - select_parts.append(f"{table}.{col}") + select_parts.append( + utils.render_subset_expression( + source_alias=table, + col_name=col, + classification_alias=classification_alias, + ) +) selected_column_names.add(col) # Log duplicate columns (if any) @@ -1871,4 +1886,46 @@ def write_classification_to_bq(classification_df, bq_table_name, client) -> None job_config=job_config ) job.result() # Wait for the job to complete - utils.logger.info(f"Classification data written to {bq_table_name} successfully.") \ No newline at end of file + utils.logger.info(f"Classification data written to {bq_table_name} successfully.") + +def get_first_cid(column_name: str) -> str | None: + cids = extract_ordered_concept_ids(column_name) + return cids[0] if cids else None + +def render_subset_expression( + source_alias: str, + col_name: str, + classification_alias: str = "c", +) -> str: + first_cid = get_first_cid(col_name) + + if first_cid is None: + if col_name in constants.ALWAYS_INCLUDE_NON_CID_COLUMNS: + return f"{source_alias}.{col_name}" + return f"NULL AS {col_name}" + + if first_cid in constants.NON_CENSORED_SECONDARY_SOURCE_CIDS: + return f"{source_alias}.{col_name}" + + # Known module columns: keep only if module status is "Submitted" + # and for cases 3A/3B before cutoff + rule = constants.MODULE_CENSOR_RULES.get(first_cid) + if rule is not None: + return f""" + CASE + WHEN {classification_alias}.{rule["status_col"]} = 'Submitted' + AND ( + {classification_alias}.`case` NOT IN ('3A', '3B') + OR ( + {classification_alias}.{rule["completion_ts"]} IS NOT NULL + AND {classification_alias}.{rule["cutoff"]} IS NOT NULL + AND {classification_alias}.{rule["completion_ts"]} < {classification_alias}.{rule["cutoff"]} + ) + ) + THEN {source_alias}.{col_name} + ELSE NULL + END AS {col_name} + """.strip() + + # Unknown first CID: fail closed + return f"NULL AS {col_name}" From 2299782846ab1bf4258b3c0eca814ce02e3441d8 Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Tue, 7 Jul 2026 13:17:58 -0400 Subject: [PATCH 13/67] Comment out code related to bioSurvey, clinicalBioSurvey, and Mouthwash surveys --- core/constants.py | 36 ++++++++++++++++++------------------ core/utils.py | 10 +++++----- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/core/constants.py b/core/constants.py index 6025b97..afd1ff9 100644 --- a/core/constants.py +++ b/core/constants.py @@ -217,26 +217,26 @@ "cutoff": "survey_cutoff", }, # Blood/Urine/Mouthwash - "299215535": { - "source_table": "bioSurvey", - "status_col": "bio_status", - "completion_ts": "bio_complete_ts", - "cutoff": "survey_cutoff", - }, + #"299215535": { + # "source_table": "bioSurvey", + # "status_col": "bio_status", + # "completion_ts": "bio_complete_ts", + # "cutoff": "survey_cutoff", + #}, # Blood/Urine - "826163434": { - "source_table": "clinicalBioSurvey", - "status_col": "clinicalbio_status", - "completion_ts": "clinicalbio_complete_ts", - "cutoff": "survey_cutoff", - }, + #"826163434": { + # "source_table": "clinicalBioSurvey", + # "status_col": "clinicalbio_status", + # "completion_ts": "clinicalbio_complete_ts", + # "cutoff": "survey_cutoff", + #}, # Mouthwash - "390351864": { - "source_table": "mouthwash", - "status_col": "mouthwash_status", - "completion_ts": "mouthwash_complete_ts", - "cutoff": "survey_cutoff", - }, + #"390351864": { + # "source_table": "mouthwash", + # "status_col": "mouthwash_status", + # "completion_ts": "mouthwash_complete_ts", + # "cutoff": "survey_cutoff", + #}, # Menstrual Cycle (contains two secondary source concept IDs) "912367929": { "source_table": "menstrualSurvey", diff --git a/core/utils.py b/core/utils.py index 3d0ac51..9dff67f 100644 --- a/core/utils.py +++ b/core/utils.py @@ -1567,8 +1567,8 @@ def read_parquet_to_dataframe(parquet_url: str) -> pd.DataFrame: # Black strings, nulls, and invalid timestamps become NaT ts_cols = ["consent_withdrawn_ts", "hipaa_revoked_ts", "data_destruction_ts", "module1_complete_ts", "module2_complete_ts", "module3_complete_ts", - "module4_complete_ts", "bio_complete_ts", "clinicalbio_complete_ts", - "mouthwash_complete_ts", "menstrual_complete_ts", "covid19_complete_ts", + "module4_complete_ts", #"bio_complete_ts", "clinicalbio_complete_ts", "mouthwash_complete_ts", + "menstrual_complete_ts", "covid19_complete_ts", "experience2024_complete_ts"] for col in ts_cols: df[col] = pd.to_datetime(df[col], errors="coerce", utc=True).astype("datetime64[us, UTC]") @@ -1856,9 +1856,9 @@ def write_classification_to_bq(classification_df, bq_table_name, client) -> None bigquery.SchemaField("module2_complete_ts", "TIMESTAMP"), bigquery.SchemaField("module3_complete_ts", "TIMESTAMP"), bigquery.SchemaField("module4_complete_ts", "TIMESTAMP"), - bigquery.SchemaField("bio_complete_ts", "TIMESTAMP"), - bigquery.SchemaField("clinicalbio_complete_ts", "TIMESTAMP"), - bigquery.SchemaField("mouthwash_complete_ts", "TIMESTAMP"), + #bigquery.SchemaField("bio_complete_ts", "TIMESTAMP"), + #bigquery.SchemaField("clinicalbio_complete_ts", "TIMESTAMP"), + #bigquery.SchemaField("mouthwash_complete_ts", "TIMESTAMP"), bigquery.SchemaField("menstrual_complete_ts", "TIMESTAMP"), bigquery.SchemaField("covid19_complete_ts", "TIMESTAMP"), bigquery.SchemaField("experience2024_complete_ts", "TIMESTAMP"), From b7f5dccec49fb91a0594a9bb9e0bf2d958647e95 Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Thu, 9 Jul 2026 15:25:14 -0400 Subject: [PATCH 14/67] Change case 'EXCLUSION' to 'DATA_QUALITY_EXCLUSION' --- core/utils.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/core/utils.py b/core/utils.py index 9dff67f..7d11fa7 100644 --- a/core/utils.py +++ b/core/utils.py @@ -1626,7 +1626,7 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: # Rule 1: Withdraw Consent = Yes but withdraw timestamp is NULL # Rule 2: Revoke HIPAA = Yes but revoke timestamp is NULL AND participant did NOT withdraw consent - exclusion_mask = ( + data_quality_mask = ( (destroy_no & withdraw_yes & withdraw_ts_missing) | (destroy_no &revoke_yes & revoke_ts_missing & withdraw_no) ) @@ -1740,13 +1740,13 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: # Build output classification column based on the above cases and rules df["case"] = np.select( - [case_1, case_2, case_3a, case_3b, case_4, exclusion_mask], - ["1", "2", "3A", "3B", "4", "EXCLUSION"], + [case_1, case_2, case_3a, case_3b, case_4, data_quality_mask], + ["1", "2", "3A", "3B", "4", "DATA_QUALITY_EXCLUSION"], default="UNDEFINED" ) # Build exclusion reason column to specify which rule was violated for - # participants classified as "EXCLUSION" + # participants classified as "DATA_QUALITY_EXCLUSION" df["exclusion_reason"] = np.select( [ destroy_no & withdraw_yes & withdraw_ts_missing, @@ -1817,11 +1817,11 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: # Sort dataframe in order of: # 1. Undefined cases - # 2. Exclusion cases + # 2. Data quality exclusion cases # 3. Cases with anomalies # 4. Normal cases df["sort_order"] = np.select( - [df["case"].eq("UNDEFINED"), df["case"].eq("EXCLUSION"), df["anomaly"].notna()], + [df["case"].eq("UNDEFINED"), df["case"].eq("DATA_QUALITY_EXCLUSION"), df["anomaly"].notna()], [1, 2, 3], default=4 ) From 9a67a4994d6157d77ba4c27f818eed3d36166b6d Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Thu, 9 Jul 2026 15:54:20 -0400 Subject: [PATCH 15/67] Add the classification table as an optional argument for create_subset_table() --- core/transformations.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/core/transformations.py b/core/transformations.py index 8a5e0c0..afb701c 100644 --- a/core/transformations.py +++ b/core/transformations.py @@ -3,6 +3,7 @@ import re import os import sys +from typing import Optional from flask import json from google.cloud import bigquery from google.cloud import storage @@ -930,7 +931,8 @@ def create_subset_table( config_path: str, destination_dataset: str, destination_table: str, - missing_report_base_path: str = "gs://pr2-pipeline-artifacts-stg/missing_columns_report/" # constants.MISSING_COLUMNS_REPORT_PATH + missing_report_base_path: str = "gs://pr2-pipeline-artifacts-stg/missing_columns_report/", # constants.MISSING_COLUMNS_REPORT_PATH + classification_table: Optional[str] = None ) -> dict: """ Create a subset table based on configuration. @@ -950,6 +952,7 @@ def create_subset_table( destination_dataset (str): Target dataset. destination_table (str): Target table. missing_report_base_path (str): Base GCS path for missing column reports. + classification_table (Optional[str]): Fully qualified BigQuery classification table name. Returns: dict: @@ -1000,7 +1003,8 @@ def create_subset_table( client=client, full_config=subset_config, destination_table_config=destination_table_config, - table_schemas=table_schemas + table_schemas=table_schemas, + classification_table=classification_table ) sql = result["sql"] From c8287f0202f0a740cb2db91b595f3089827b3a17 Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Thu, 9 Jul 2026 16:01:58 -0400 Subject: [PATCH 16/67] Add synthetic table generator for censorship testing --- core/generate_synthetic_bq_table.py | 215 ++++++++++++++++++++++++++++ core/transformations.py | 66 +++++++++ core/utils.py | 157 ++++++++++++++++++++ 3 files changed, 438 insertions(+) create mode 100644 core/generate_synthetic_bq_table.py diff --git a/core/generate_synthetic_bq_table.py b/core/generate_synthetic_bq_table.py new file mode 100644 index 0000000..d1a0c98 --- /dev/null +++ b/core/generate_synthetic_bq_table.py @@ -0,0 +1,215 @@ +''' +Generates a synthetic "mvp" base table to be used in testing. This synthetic +table is designed to be 1:1 with the participant_status_synthetic.parquet +generated by the generate_synthetic_parquet.py. + +Strategy: + - Every column gets a distinct filler value specific to the Connect_ID + and column name. + - A handful of Connect_IDs get deliberate NULLs sprinkled into specific + columns to test that "originally NULL" and "censored down to NULL" + are distinguishable and that pass-through columns correctly preserve + NULLs rather than accidentally filling them. + - Column identity (which CID governs which column) is derived from the + constants.MODULE_CENSOR_RULES, constants.NON_CENSORED_SECONDARY_SOURCE_CIDS, + and the known "uncovered" CIDs (130371375, 111111111), so this file stays + correct if constants.py changes. +''' + +import os +import sys + +if __name__ == "__main__": + sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import pandas as pd +from core import constants, utils + +MVP_COLUMNS = [ + "d_130371375_d_303552867_d_297462035", # uncovered CID (unknown), no censoring rule + "d_111111111_d_222222222", # uncovered CID (unknown), no censoring rule + "d_883203566_d_253532712", # non-censored secondary source CID + "d_214456996_d_142654897_d_196856782", # non-censored secondary source CID + "d_273437590_d_811353546", # non-censored secondary source CID + "token", + "d_726699695_d_783167257", + "d_726699695_d_384191091_d_706998638", + "d_726699695_d_384191091_d_458435048", + "d_745268907_d_536735468", + "d_745268907_d_384191091", + "d_745268907_d_745268907", + "d_965707586_d_976570371", + "d_965707586_d_947205597_d_712653855", + "d_965707586_d_763164658", + "d_716117817_d_663265240", + "d_716117817_d_111111111", + "d_716117817_d_222222222", + "d_912367929_d_111111111", + "d_912367929_d_222222222", + "d_912367929_d_333333333", + "d_232438133_d_111111111", + "d_232438133_d_222222222", + "d_232438133_d_333333333", + "d_793330426_d_111111111", + "d_793330426_d_222222222", + "d_793330426_d_333333333", + "d_506648060_d_111111111", + "d_506648060_d_222222222", + "d_506648060_d_333333333" +] + +# Connect_IDs from generate_synthetic_parquet.py -- reused here so +# the base table's Connect_IDs line up 1:1 with the classification table. +TEST_CONNECT_IDS = [ + "TEST_UNVERIFIED", + "TEST_CASE1_ALL_SUBMITTED", + "TEST_CASE1_MOD1_NOT_SUBMITTED", + "TEST_CASE1_MOD_STATUS_NULL", + "TEST_CASE1_MOD_MENSTRUAL_NOT_SUBMITTED", + "TEST_CASE2_BASIC", + "TEST_CASE2_MOD2_IN_PROGRESS", + "TEST_CASE2_MOD3_STATUS_NULL", + "TEST_CASE3A_MOD1_BEFORE_CUTOFF", + "TEST_CASE3A_MOD1_AFTER_CUTOFF", + "TEST_CASE3A_MOD1_EXACT_CUTOFF", + "TEST_CASE3A_ANOMALY_REVOKE_AFTER_WITHDRAW", + "TEST_CASE3A_SUBMITTED_BUT_NO_TS", + "TEST_CASE3A_ALL_FOUR_MODULES_MIXED", + "TEST_CASE3B_MOD2_BEFORE_CUTOFF", + "TEST_CASE3B_MOD2_AFTER_CUTOFF", + "TEST_CASE4_BASIC", + "TEST_CASE4_ANOMALY_MISSING_DESTROY_TS", + "TEST_EXCLUSION_WITHDRAW_NO_TS", + "TEST_EXCLUSION_REVOKE_NO_TS", + "TEST_UNDEFINED_DESTROY_ONLY", + "TEST_BLANK_MODULE_STATUS", + "TEST_UNKNOWN_MODULE_STATUS", + "TEST_UNKNOWN_STATUS_CASE3A", +] + +# Connect_IDs that get deliberate NULLs sprinkled in. +# These are chosen to hit each category: a module_censor column, a +# non_censored column, and an uncovered-CID column, so "originally NULL" +# behaves correctly no matter which branch of render_subset_expression +# handles that column. +NULL_INJECTION_PLAN = { + "TEST_CASE1_ALL_SUBMITTED": [ + "d_726699695_d_783167257", # module_censor (module1) -- originally NULL, should STAY NULL (case 1, no cutoff issue, but source was empty) + "d_883203566_d_253532712", # non_censored -- originally NULL, should pass through as NULL + ], + "TEST_CASE2_BASIC": [ + "d_130371375_d_303552867_d_297462035", # uncovered CID -- irrelevant, always NULL regardless of source value + "d_965707586_d_976570371", # module_censor (module3) -- originally NULL + ], + "TEST_CASE3A_MOD1_BEFORE_CUTOFF": [ + "d_726699695_d_783167257", # module_censor (module1), before cutoff -- originally NULL, should STAY NULL (nothing to censor, but also nothing to show) + ] +} + +def build_mvp_df() -> pd.DataFrame: + """ + Returns a DataFrame matching the real "mvp" table schema, with every + Connect_ID from TEST_CONNECT_IDS present, every column filled with a + UNIQUE, traceable value by default, and deliberate NULLs injected per + NULL_INJECTION_PLAN. + + Each cell holds "{Connect_ID}_{column_name}" as a value, so misplaced + or leaked values are traceable on sight — no lookup table needed. + """ + rows = [] + for connect_id in TEST_CONNECT_IDS: + row = {"Connect_ID": connect_id} + for col in MVP_COLUMNS: + if col == "token": + row[col] = f"TOKEN_{connect_id}" + else: + row[col] = f"{connect_id}_{col}" + rows.append(row) + + df = pd.DataFrame(rows) + + # Apply deliberate NULL injections + for connect_id, cols_to_null in NULL_INJECTION_PLAN.items(): + for col in cols_to_null: + df.loc[df["Connect_ID"] == connect_id, col] = None + + return df + + +def classify_columns_by_rule() -> dict: + """ + Diagnostic helper: cross-references MVP_COLUMNS against constants.py + to show which category each column falls into. Useful for confirming + NULL_INJECTION_PLAN actually covers all three categories and for + catching any column whose first CID is not recognized anywhere. + """ + categories = { + "module_censor": [], + "non_censored": [], + "uncovered": [], + "no_cid": [], + } + + for col in MVP_COLUMNS: + first_cid = utils.get_first_cid(col) + if first_cid is None: + categories["no_cid"].append(col) + elif first_cid in constants.MODULE_CENSOR_RULES: + categories["module_censor"].append(col) + elif first_cid in constants.NON_CENSORED_SECONDARY_SOURCE_CIDS: + categories["non_censored"].append(col) + else: + categories["uncovered"].append(col) + + return categories + + +def write_mvp_to_bq(mvp_df: pd.DataFrame, bq_table_name: str, client) -> None: + """ + Writes the synthetic mvp DataFrame to a BigQuery table. + + All data columns are loaded as STRING, matching the real mvp table's + schema (these are flattened survey response columns, not typed data). + Uses WRITE_TRUNCATE so reruns overwrite cleanly rather than appending + duplicate test rows. + + Args: + mvp_df: DataFrame from build_mvp_df(). + bq_table_name: Fully qualified destination table + (e.g. "project.dataset.mvp_synthetic"). + client: bigquery.Client instance. + """ + schema = [bigquery.SchemaField("Connect_ID", "STRING")] + schema += [ + bigquery.SchemaField(col, "STRING") + for col in MVP_COLUMNS + ] + + job_config = bigquery.LoadJobConfig( + schema=schema, + write_disposition=bigquery.WriteDisposition.WRITE_TRUNCATE, + ) + + job = client.load_table_from_dataframe(mvp_df, bq_table_name, job_config=job_config) + job.result() # Wait for the job to complete + utils.logger.info(f"Synthetic data written to {bq_table_name} successfully.") + + +if __name__ == "__main__": + from google.cloud import bigquery + + df = build_mvp_df() + print(df.to_string(index=False)) + print(f"\nGenerated {len(df)} synthetic mvp rows across {len(MVP_COLUMNS)} columns.") + + categories = classify_columns_by_rule() + print("\nColumn classification:") + for category, cols in categories.items(): + print(f" {category}: {len(cols)} columns") + if category == "uncovered": + print(f" -> {cols}") + + # Write to BigQuery + client = bigquery.Client() + destination_table = "nih-nci-dceg-connect-stg-5519.pr2_mvp.mvp_synthetic" + write_mvp_to_bq(df, destination_table, client) \ No newline at end of file diff --git a/core/transformations.py b/core/transformations.py index afb701c..ad6eab3 100644 --- a/core/transformations.py +++ b/core/transformations.py @@ -1063,6 +1063,72 @@ def create_subset_table( "missing_columns_report_path": report_path } +def create_censorship_summary_table( + output_table: str, + classification_table: str, + destination_table: str, +) -> dict: + """ + Generates the censorship summary query, saves it to GCS for audit + purposes (consistent with other transform functions in this codebase), + executes it, and writes the result to destination_table. + + Args: + output_table: Fully qualified final subset/output table (e.g. the + mvp_case_1_2_3A_3B table). Used only to determine which + columns exist and are therefore eligible for censorship + classification. + classification_table: Fully qualified table with case/status/timestamp + columns for every participant. + destination_table: Fully qualified table to create with the summary. + + Returns: + dict: Contains "status" and "submitted_sql_path". + + Raises: + Exception: Propagates any errors encountered during query execution. + """ + client = bigquery.Client() + + output_columns = utils.get_column_names(client, output_table) + column_rule_map = utils.build_column_rule_map(output_columns) + + utils.logger.info( + f"[{destination_table}] Found {len(column_rule_map)} censorable columns out of " + f"{len(output_columns)} total columns in {output_table}" + ) + + sql = utils.build_censorship_summary_sql( + column_rule_map=column_rule_map, + classification_table=classification_table, + ) + + final_sql = f"CREATE OR REPLACE TABLE `{destination_table}` AS ({sql})" + + # Save the SQL to GCS for audit purposes, matching existing pipeline convention + try: + gcs_client = storage.Client() + gcs_path = f"{constants.OUTPUT_SQL_PATH}{destination_table}.sql" + utils.save_sql_string(sql=final_sql, path=gcs_path, storage_client=gcs_client) + except Exception as e: + utils.logger.exception(f"[{destination_table}] Error saving censorship summary SQL to {gcs_path}") + raise e + + # Execute the SQL + try: + utils.logger.info(f"[{destination_table}] Executing censorship summary query...") + query_job = client.query(final_sql) + query_job.result() + status = f"[{destination_table}] Table successfully created with censorship summary." + utils.logger.info(status) + except Exception as e: + utils.logger.exception(f"[{destination_table}] Error executing censorship summary SQL: {e}") + raise e + + return { + "status": status, + "submitted_sql_path": constants.OUTPUT_SQL_PATH, + } if __name__ == "__main__": #source_table = "nih-nci-dceg-connect-prod-6d04.ForTestingOnly.module1_v1_with_cleaned_columns" diff --git a/core/utils.py b/core/utils.py index 7d11fa7..8e5741f 100644 --- a/core/utils.py +++ b/core/utils.py @@ -1929,3 +1929,160 @@ def render_subset_expression( # Unknown first CID: fail closed return f"NULL AS {col_name}" + +def build_column_rule_map(output_columns: list[str]) -> dict[str, dict]: + """ + Maps each output column to its MODULE_CENSOR_RULES entry based on the + column's first concept ID. Columns whose first CID has no entry in + MODULE_CENSOR_RULES are excluded (they are not module_censor columns). + + Args: + output_columns: List of column names from the destination table + (e.g. via utils.get_column_names). + + Returns: + dict: {column_name: rule_dict} for every column governed by a + module censorship rule. + """ + column_rule_map = {} + for col in output_columns: + first_cid = utils.get_first_cid(col) + if first_cid is None: + continue + rule = constants.MODULE_CENSOR_RULES.get(first_cid) + if rule is not None: + column_rule_map[col] = rule + return column_rule_map + + +def _build_censorship_case_block( + column_name: str, + rule: dict, + classification_alias: str = "c", +) -> str: + """ + Builds a single SELECT block reporting whether `column_name` was + censored for each Connect_ID and why. + + Args: + column_name: The output column governed by this rule. + rule: One entry from constants.MODULE_CENSOR_RULES + (keys: status_col, completion_ts, cutoff). + classification_alias: Alias of the classification table. + + Returns: + str: A SELECT statement returning Connect_ID, case, column_name, + reason for rows where this column was censored. + """ + status_col = rule["status_col"] + completion_ts = rule["completion_ts"] + cutoff = rule["cutoff"] + + return f""" + SELECT + {classification_alias}.Connect_ID, + {classification_alias}.`case`, + '{column_name}' AS column_name, + CASE + WHEN {classification_alias}.{status_col} IS DISTINCT FROM 'Submitted' + THEN CONCAT( + '{status_col} = ', + IFNULL({classification_alias}.{status_col}, 'NULL') + ) + WHEN {classification_alias}.`case` IN ('3A', '3B') + AND ( + {classification_alias}.{completion_ts} IS NULL + OR {classification_alias}.{cutoff} IS NULL + OR {classification_alias}.{completion_ts} >= {classification_alias}.{cutoff} + ) + THEN CONCAT( + '{completion_ts} = ', + IFNULL(CAST({classification_alias}.{completion_ts} AS STRING), 'NULL'), + ', on or after ', '{cutoff}', ' = ', + IFNULL(CAST({classification_alias}.{cutoff} AS STRING), 'NULL') + ) + ELSE NULL + END AS reason + FROM `{{classification_table}}` {classification_alias} + """.strip() + + +def build_censorship_summary_sql( + column_rule_map: dict[str, dict], + classification_table: str, + classification_alias: str = "c", +) -> str: + """ + Builds the full UNION ALL query across all censorable columns. + + Args: + column_rule_map: {column_name: rule_dict}, as produced by + build_column_rule_map. + classification_table: Fully qualified table containing case, status, + completion_ts, and cutoff columns for every + participant (e.g. classification_2026_07_07). + classification_alias: Alias used for that table in the generated SQL. + + Returns: + str: A complete SQL query returning Connect_ID, case, column_name, + reason for every censorship event, filtered to reason IS NOT NULL. + + Raises: + ValueError: If column_rule_map is empty. + """ + if not column_rule_map: + raise ValueError("column_rule_map is empty; nothing to summarize.") + + blocks = [] + for column_name, rule in column_rule_map.items(): + block = _build_censorship_case_block( + column_name=column_name, + rule=rule, + classification_alias=classification_alias, + ).replace("{classification_table}", classification_table) + blocks.append(block) + + unioned = "\nUNION ALL\n".join(blocks) + + final_sql = f""" + SELECT * + FROM ( + {unioned} + ) + WHERE reason IS NOT NULL + ORDER BY Connect_ID, column_name + """.strip() + + return final_sql + +def get_censorship_rollup(destination_table: str) -> dict: + """ + Queries the censorship summary table and returns two rollups: + 1. per-column counts (how many participants were censored, per column) + 2. per-case counts (how many censorship events occurred, per case) + + Args: + destination_table: The table created by create_censorship_summary_table. + + Returns: + dict: {"by_column": [...], "by_case": [...]} + """ + client = bigquery.Client() + + by_column_sql = f""" + SELECT column_name, COUNT(*) AS censored_count + FROM `{destination_table}` + GROUP BY column_name + ORDER BY censored_count DESC + """ + by_case_sql = f""" + SELECT `case`, COUNT(*) AS censorship_events + FROM `{destination_table}` + GROUP BY `case` + ORDER BY `case` + """ + + by_column = [dict(row) for row in client.query(by_column_sql).result()] + by_case = [dict(row) for row in client.query(by_case_sql).result()] + + return {"by_column": by_column, "by_case": by_case} From c2b6aff6aa14efd3dca73ff734cca603116815bb Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Thu, 9 Jul 2026 16:03:23 -0400 Subject: [PATCH 17/67] Add synthetic participant_status.parquet generator for classify_participants testing --- core/generate_synthetic_parquet.py | 322 +++++++++++++++++++++++++++++ 1 file changed, 322 insertions(+) create mode 100644 core/generate_synthetic_parquet.py diff --git a/core/generate_synthetic_parquet.py b/core/generate_synthetic_parquet.py new file mode 100644 index 0000000..7e89fb0 --- /dev/null +++ b/core/generate_synthetic_parquet.py @@ -0,0 +1,322 @@ +''' +Generates a synthetic participant_status.parquet file matching the real +schema (Connect_ID, verified_status, consent/revoke/destroy flags + ts, +module1-4 + menstrual/covid19/experience2024 status + ts), with one row +per test case / edge case, for feeding into read_parquet_to_dataframe() +and classify_participants() end-to-end. + +''' + +import pandas as pd + +# Placeholder concept IDs -- arbitrary, only used to populate the +# *_status_concept_id companion columns so the schema shape matches. +# "UNKNOWN" and None are explicitly mapped to None: an UNKNOWN status +# means no concept ID was ever assigned, so its companion column should +# be NULL, same as an outright missing status. +STATUS_CID_MAP = { + "Submitted": "231311385", + "Started": "615768760", + "Not Started": "972455046", + "In Progress": "615768760", + "UNKNOWN": None, + None: None, +} +YES_CID = "353358909" +NO_CID = "104430631" + + +def _flag_cid(value): + if value == "Yes": + return YES_CID + if value == "No": + return NO_CID + return None + + +def _status_cid(value): + # Any status not explicitly mapped (including "UNKNOWN") returns None, + # matching real data where UNKNOWN/unrecognized statuses have no CID. + return STATUS_CID_MAP.get(value, None) + + +def build_participant_status_df() -> pd.DataFrame: + """ + Returns a DataFrame of synthetic participant_status rows, one per + test case / edge case, matching the real parquet schema. + + NOTE: verified_status is always "Verified" for every row except one + deliberate "unverified" row, since read_parquet_to_dataframe() filters + to verified_status == "Verified" at read time -- rows with any other + value should disappear before classify_participants() ever sees them. + """ + rows = [] + + def add( + connect_id, test_label, + verified_status="Verified", + destroy=None, withdraw=None, revoke=None, + destroy_ts=None, withdraw_ts=None, revoke_ts=None, + module1_status="Submitted", module1_ts=None, + module2_status="Submitted", module2_ts=None, + module3_status="Submitted", module3_ts=None, + module4_status="Submitted", module4_ts=None, + menstrual_status="Submitted", menstrual_ts=None, + covid19_status="Submitted", covid19_ts=None, + experience2024_status="Submitted", experience2024_ts=None, + ): + # Default flags to "No" unless explicitly overridden (mirrors + # realistic participant_status data -- flags are rarely NULL + # except in the deliberate UNDEFINED edge case below). + destroy = destroy if destroy is not None else "No" + withdraw = withdraw if withdraw is not None else "No" + revoke = revoke if revoke is not None else "No" + + rows.append({ + "Connect_ID": connect_id, + "test_label": test_label, # extra column for reference; drop before real use if needed + "verified_status": verified_status, + "verified_status_concept_id": "197316935" if verified_status == "Verified" else "875007964", + "consent_withdrawn": withdraw, + "consent_withdrawn_concept_id": _flag_cid(withdraw), + "hipaa_revoked": revoke, + "hipaa_revoked_concept_id": _flag_cid(revoke), + "data_destruction_requested": destroy, + "data_destruction_requested_concept_id": _flag_cid(destroy), + "module1_status": module1_status, + "module1_status_concept_id": _status_cid(module1_status), + "module2_status": module2_status, + "module2_status_concept_id": _status_cid(module2_status), + "module3_status": module3_status, + "module3_status_concept_id": _status_cid(module3_status), + "module4_status": module4_status, + "module4_status_concept_id": _status_cid(module4_status), + "menstrual_status": menstrual_status, + "menstrual_status_concept_id": _status_cid(menstrual_status), + "covid19_status": covid19_status, + "covid19_status_concept_id": _status_cid(covid19_status), + "experience2024_status": experience2024_status, + "experience2024_status_concept_id": _status_cid(experience2024_status), + "consent_withdrawn_ts": withdraw_ts, + "hipaa_revoked_ts": revoke_ts, + "data_destruction_ts": destroy_ts, + "module1_complete_ts": module1_ts, + "module2_complete_ts": module2_ts, + "module3_complete_ts": module3_ts, + "module4_complete_ts": module4_ts, + "menstrual_complete_ts": menstrual_ts, + "covid19_complete_ts": covid19_ts, + "experience2024_complete_ts": experience2024_ts, + }) + + # ------------------------------------------------------------------ + # UNVERIFIED: should be filtered out entirely by + # read_parquet_to_dataframe()'s pyarrow filter, before classification + # ever runs. If this Connect_ID shows up downstream, the filter broke. + # ------------------------------------------------------------------ + add( + "TEST_UNVERIFIED", "not verified -- must be dropped at parquet read time", + verified_status="Unverified", + ) + + # ------------------------------------------------------------------ + # CASE 1: No restrictions. + # ------------------------------------------------------------------ + add("TEST_CASE1_ALL_SUBMITTED", "case 1, everything submitted") + + add( + "TEST_CASE1_MOD1_NOT_SUBMITTED", "case 1, module1 not submitted", + module1_status="Not Started", + ) + + add( + "TEST_CASE1_MOD_STATUS_NULL", "case 1, module2 status is NULL", + module2_status=None, + ) + + add( + "TEST_CASE1_MOD_MENSTRUAL_NOT_SUBMITTED", "case 1, all modules submitted but menstrual", + menstrual_status="Not Started", + module1_ts="2025-03-04 09:18:42+00:00", + module2_ts="2025-03-11 14:37:05+00:00", + module3_ts="2025-03-19 16:52:18+00:00", + module4_ts="2025-03-28 11:09:56+00:00", + covid19_ts="2025-04-06 08:43:21+00:00", + experience2024_ts="2025-04-17 19:26:47+00:00", + ) + + # ------------------------------------------------------------------ + # CASE 2: Destroy=No, Withdraw=No, Revoke=Yes (ts present). + # ------------------------------------------------------------------ + add( + "TEST_CASE2_BASIC", "case 2, revoke present, all modules submitted", + revoke="Yes", revoke_ts="2025-03-01 00:00:00+00:00", + ) + + add( + "TEST_CASE2_MOD2_IN_PROGRESS", "case 2, module2 status = In Progress", + revoke="Yes", revoke_ts="2025-03-01 00:00:00+00:00", + module2_status="In Progress", + ) + + add( + "TEST_CASE2_MOD3_STATUS_NULL", "case 2, module3 status is NULL", + revoke="Yes", revoke_ts="2025-03-01 00:00:00+00:00", + module3_status=None, + ) + + # ------------------------------------------------------------------ + # CASE 3A: Withdraw=Yes (ts present), Revoke=Yes, + # revoke_ts missing OR occurs AFTER withdraw_ts. + # ------------------------------------------------------------------ + add( + "TEST_CASE3A_MOD1_BEFORE_CUTOFF", "case 3A, module1 completed before cutoff", + withdraw="Yes", withdraw_ts="2025-04-01 00:00:00+00:00", + revoke="Yes", revoke_ts=None, + module1_ts="2025-03-15 00:00:00+00:00", + ) + + add( + "TEST_CASE3A_MOD1_AFTER_CUTOFF", "case 3A, module1 completed after cutoff", + withdraw="Yes", withdraw_ts="2025-06-28 00:00:00+00:00", + revoke="Yes", revoke_ts=None, + module1_ts="2025-06-28 00:00:01+00:00", + ) + + add( + "TEST_CASE3A_MOD1_EXACT_CUTOFF", "case 3A, module1 completed exactly at cutoff (boundary test)", + withdraw="Yes", withdraw_ts="2025-12-31 23:59:59+00:00", + revoke="Yes", revoke_ts=None, + module1_ts="2025-12-31 23:59:59+00:00", + ) + + add( + "TEST_CASE3A_ANOMALY_REVOKE_AFTER_WITHDRAW", + "case 3A anomaly, revoke_ts after withdraw_ts -- ehr_cutoff should be NULL", + withdraw="Yes", withdraw_ts="2025-04-01 00:00:00+00:00", + revoke="Yes", revoke_ts="2025-05-01 00:00:00+00:00", + module1_ts="2025-03-15 00:00:00+00:00", + ) + + add( + "TEST_CASE3A_SUBMITTED_BUT_NO_TS", + "case 3A, module1 status Submitted but completion_ts NULL (data quality issue)", + withdraw="Yes", withdraw_ts="2025-04-01 00:00:00+00:00", + revoke="Yes", revoke_ts=None, + module1_status="Submitted", module1_ts=None, + ) + + add( + "TEST_CASE3A_ALL_FOUR_MODULES_MIXED", + "case 3A, all 4 modules -- 2 before cutoff, 2 after, to check independence", + withdraw="Yes", withdraw_ts="2025-04-01 00:00:00+00:00", + revoke="Yes", revoke_ts=None, + module1_ts="2025-03-01 00:00:00+00:00", # before -> not censored + module2_ts="2025-05-01 00:00:00+00:00", # after -> censored + module3_ts="2025-03-20 00:00:00+00:00", # before -> not censored + module4_ts="2025-06-01 00:00:00+00:00", # after -> censored + ) + + + # ------------------------------------------------------------------ + # CASE 3B: Withdraw=Yes (ts present), Revoke=Yes (ts present), + # revoke_ts occurs BEFORE withdraw_ts. + # ------------------------------------------------------------------ + add( + "TEST_CASE3B_MOD2_BEFORE_CUTOFF", "case 3B, module2 completed before withdraw cutoff", + withdraw="Yes", withdraw_ts="2025-05-01 00:00:00+00:00", + revoke="Yes", revoke_ts="2025-03-01 00:00:00+00:00", + module2_ts="2025-04-01 00:00:00+00:00", + ) + + add( + "TEST_CASE3B_MOD2_AFTER_CUTOFF", "case 3B, module2 completed after withdraw cutoff", + withdraw="Yes", withdraw_ts="2025-05-01 00:00:00+00:00", + revoke="Yes", revoke_ts="2025-03-01 00:00:00+00:00", + module2_ts="2025-06-01 00:00:00+00:00", + ) + + # ------------------------------------------------------------------ + # CASE 4: Destroy=Yes, Withdraw=Yes, Revoke=Yes -- fully excluded. + # ------------------------------------------------------------------ + add( + "TEST_CASE4_BASIC", "case 4, full destruction request", + destroy="Yes", withdraw="Yes", revoke="Yes", + destroy_ts="2025-06-01 00:00:00+00:00", + withdraw_ts="2025-03-01 00:00:00+00:00", + revoke_ts="2025-03-01 00:00:00+00:00", + ) + + add( + "TEST_CASE4_ANOMALY_MISSING_DESTROY_TS", "case 4, destroy=Yes but destroy_ts NULL", + destroy="Yes", withdraw="Yes", revoke="Yes", + destroy_ts=None, + withdraw_ts="2025-03-01 00:00:00+00:00", + revoke_ts="2025-03-01 00:00:00+00:00", + ) + + # ------------------------------------------------------------------ + # EXCLUSION: data quality violations + # ------------------------------------------------------------------ + add( + "TEST_EXCLUSION_WITHDRAW_NO_TS", "withdraw=Yes but withdraw_ts NULL", + withdraw="Yes", withdraw_ts=None, + ) + + add( + "TEST_EXCLUSION_REVOKE_NO_TS", "revoke=Yes but revoke_ts NULL, withdraw=No", + revoke="Yes", revoke_ts=None, withdraw="No", + ) + + # ------------------------------------------------------------------ + # UNDEFINED: flag combination matches nothing -- destroy=Yes but + # withdraw/revoke=No, which matches no defined case and is not an + # exclusion rule either) + # ------------------------------------------------------------------ + add( + "TEST_UNDEFINED_DESTROY_ONLY", + "destroy=Yes but withdraw=No, revoke=No -- matches no defined case", + destroy="Yes", withdraw="No", revoke="No", + ) + + # ------------------------------------------------------------------ + # BLANK STRING edge case -- empty string rather than NULL + # ------------------------------------------------------------------ + add( + "TEST_BLANK_MODULE_STATUS", "case 1, module4 status is blank string", + module4_status="", + ) + + # ------------------------------------------------------------------ + # UNKNOWN status edge case + # (e.g. module1_status = "UNKNOWN"). Since "Submitted" is the only + # value MODULE_CENSOR_RULES treats as passing, UNKNOWN should be + # treated the same as any other non-Submitted value: censored. + # Its companion concept_id column should be NULL (see STATUS_CID_MAP). + # ------------------------------------------------------------------ + add( + "TEST_UNKNOWN_MODULE_STATUS", "case 1, module3 status is UNKNOWN -- should be censored, CID NULL", + module3_status="UNKNOWN", + ) + + add( + "TEST_UNKNOWN_STATUS_CASE3A", + "case 3A, module1 status UNKNOWN -- censored regardless of cutoff since not Submitted", + withdraw="Yes", withdraw_ts="2025-04-01 00:00:00+00:00", + revoke="Yes", revoke_ts=None, + module1_status="UNKNOWN", module1_ts="2025-03-01 00:00:00+00:00", # ts before cutoff, but status is not Submitted + module2_status="Submitted", module2_ts="2025-03-01 00:00:00+00:00", # ts before cutoff, status Submitted + ) + + df = pd.DataFrame(rows) + + return df + +if __name__ == "__main__": + df = build_participant_status_df() + print(df[["Connect_ID", "test_label", "verified_status", + "data_destruction_requested", "consent_withdrawn", "hipaa_revoked"]].to_string(index=False)) + print(f"\nGenerated {len(df)} synthetic participant_status rows.") + output_path = "/Users/trivittge/Desktop/participant_status_synthetic.parquet" + df.to_parquet(output_path, engine="pyarrow", index=False) + print(f"Written to {output_path}") From b7aab8bafb57bd342f9778888dde598c9dc24896 Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Thu, 9 Jul 2026 17:03:16 -0400 Subject: [PATCH 18/67] Fix data_quality_mask: require revoke=Yes in withdraw-missing-ts clause Prevents destroy=No/withdraw=Yes/Revoke=No rows from being misclassified as DATA_QUALITY_EXCLUSION when they should be UNDEFINED. Updates synthetic participant TEST_EXCLUSION_WITHDRAW_NO_TS to include revoke=Yes so the test actually covers this clause. --- core/generate_synthetic_parquet.py | 2 +- core/utils.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/generate_synthetic_parquet.py b/core/generate_synthetic_parquet.py index 7e89fb0..e52686f 100644 --- a/core/generate_synthetic_parquet.py +++ b/core/generate_synthetic_parquet.py @@ -260,7 +260,7 @@ def add( # ------------------------------------------------------------------ add( "TEST_EXCLUSION_WITHDRAW_NO_TS", "withdraw=Yes but withdraw_ts NULL", - withdraw="Yes", withdraw_ts=None, + withdraw="Yes", revoke="Yes", withdraw_ts=None, ) add( diff --git a/core/utils.py b/core/utils.py index 8e5741f..5f153d4 100644 --- a/core/utils.py +++ b/core/utils.py @@ -1627,8 +1627,8 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: # Rule 1: Withdraw Consent = Yes but withdraw timestamp is NULL # Rule 2: Revoke HIPAA = Yes but revoke timestamp is NULL AND participant did NOT withdraw consent data_quality_mask = ( - (destroy_no & withdraw_yes & withdraw_ts_missing) - | (destroy_no &revoke_yes & revoke_ts_missing & withdraw_no) + (destroy_no & withdraw_yes & revoke_yes & withdraw_ts_missing) + | (destroy_no & withdraw_no & revoke_yes & revoke_ts_missing) ) # ========================================================================= # Anomaly validation rules From be78f07030a03e32450babd76be090d20f5bea01 Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Fri, 10 Jul 2026 13:13:31 -0400 Subject: [PATCH 19/67] Precompute module eligibility flags once per gate instead of per column Previously, render_subset_expression emitted a full CASE/WHEN block for every censored output column, repeating the same status/timestamp/ cutoff condition once per column that shared a gate (e.g. the same module2 condition was duplicated 3x, menstrual 6x, etc.). This made the generated SQL large and repetitive, and meant BigQuery had to re-evaluate identical boolean logic multiple times per row. This commit moves that logic into the CTE as a set of precomputed boolean flags (one per unique status_col), built by the new build_unique_eligibility_rules() helper. Each censored column now just references its flag via IF(flag, col, NULL) instead of restating the full condition. build_unique_eligibility_rules() dedupes MODULE_CENSOR_RULES by status_col (since some CIDs, e.g. the two menstrual survey CIDs, legitimately share one gate) and raises ValueError if two CIDs share a status_col but disagree on completion_ts/cutoff, to catch config mistakes that would otherwise silently miscompute eligibility for consent-sensitive columns. Verified equivalent output against the synthetic dataset via bidirectional EXCEPT DISTINCT between the old and new generated tables (zero rows either direction). --- core/utils.py | 87 ++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 69 insertions(+), 18 deletions(-) diff --git a/core/utils.py b/core/utils.py index 5f153d4..09cc6a6 100644 --- a/core/utils.py +++ b/core/utils.py @@ -1299,6 +1299,47 @@ def build_filter_sql( # Join conditions with AND return "WHERE " + "\n AND ".join(clauses) +def build_unique_eligibility_rules(module_censor_rules: dict[str, dict]) -> dict[str, dict]: + """ + Deduplicates MODULE_CENSOR_RULES by status_col, so that CIDs which share + the same gating condition (e.g. the two menstrual survey CIDs, which both + gate on menstrual_status/menstrual_complete_ts/survey_cutoff) collapse to + a single rule instead of being treated as independent conditions. + + Raises if two different CIDs claim the same status_col but disagree on + completion_ts or cutoff — that would mean two columns silently computing + different "eligibility" under the same flag name, which is a data + integrity risk given this logic gates consent/censorship-sensitive data. + + Args: + module_censor_rules: constants.MODULE_CENSOR_RULES + + Returns: + dict: {status_col: rule} with one entry per unique gating condition. + + Raises: + ValueError: If two CIDs share a status_col but have conflicting + completion_ts or cutoff values. + """ + unique_rules: dict[str, dict] = {} + signatures: dict[str, tuple] = {} + + for cid, rule in module_censor_rules.items(): + status_col = rule["status_col"] + sig = (rule["status_col"], rule["completion_ts"], rule["cutoff"]) + + if status_col in signatures and signatures[status_col] != sig: + raise ValueError( + f"Conflicting MODULE_CENSOR_RULES entries share status_col=" + f"{status_col!r} but disagree on completion_ts/cutoff: " + f"{signatures[status_col]} vs {sig} (conflict introduced by CID {cid})" + ) + + signatures[status_col] = sig + unique_rules[status_col] = rule + + return unique_rules + def build_subset_query( client: bigquery.Client, full_config: dict[str, Any], @@ -1371,6 +1412,29 @@ def build_subset_query( final_where = "WHERE " + "\n AND ".join(where_conditions) + # Build eligibility flag columns for the CTE + # One boolean per unique gating rule (deduped by status_col) + eligibility_flags_sql = "" + if allowed_cases: + unique_rules = build_unique_eligibility_rules(constants.MODULE_CENSOR_RULES) + if unique_rules: + flag_lines = [] + for status_col, rule in unique_rules.items(): + completion_ts = rule["completion_ts"] + cutoff = rule["cutoff"] + flag_lines.append(f""" ( + c.{status_col} = 'Submitted' + AND ( + c.`case` NOT IN ('3A', '3B') + OR ( + c.{completion_ts} IS NOT NULL + AND c.{cutoff} IS NOT NULL + AND c.{completion_ts} < c.{cutoff} + ) + ) + ) AS {status_col}_eligible""") + eligibility_flags_sql = ",\n" + ",\n".join(flag_lines) + # Build CTE with or without classification join if allowed_cases: if not classification_table: @@ -1381,7 +1445,7 @@ def build_subset_query( {cte_name} AS ( SELECT p.*, - c.* EXCEPT({join_key}) + c.* EXCEPT({join_key}){eligibility_flags_sql} FROM `{fq_base_table}` p INNER JOIN `{classification_table}` c ON p.{join_key} = c.{join_key} @@ -1430,8 +1494,8 @@ def build_subset_query( source_alias=base_alias, col_name=col, classification_alias=classification_alias, + ) ) -) selected_column_names.add(col) # Join tables @@ -1908,24 +1972,11 @@ def render_subset_expression( return f"{source_alias}.{col_name}" # Known module columns: keep only if module status is "Submitted" - # and for cases 3A/3B before cutoff + # and for cases 3A/3B before cutoff. rule = constants.MODULE_CENSOR_RULES.get(first_cid) if rule is not None: - return f""" - CASE - WHEN {classification_alias}.{rule["status_col"]} = 'Submitted' - AND ( - {classification_alias}.`case` NOT IN ('3A', '3B') - OR ( - {classification_alias}.{rule["completion_ts"]} IS NOT NULL - AND {classification_alias}.{rule["cutoff"]} IS NOT NULL - AND {classification_alias}.{rule["completion_ts"]} < {classification_alias}.{rule["cutoff"]} - ) - ) - THEN {source_alias}.{col_name} - ELSE NULL - END AS {col_name} - """.strip() + flag_name = f"{rule['status_col']}_eligible" + return f"IF({classification_alias}.{flag_name}, {source_alias}.{col_name}, NULL) AS {col_name}" # Unknown first CID: fail closed return f"NULL AS {col_name}" From bb697a53d9e53040a60c59fb7045044b107d934f Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Fri, 10 Jul 2026 13:18:32 -0400 Subject: [PATCH 20/67] Cast NULL placeholder columns to STRING instead of letting BigQuery infer INT64, so all columns in the generated table have a consistent type --- core/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/utils.py b/core/utils.py index 09cc6a6..8d54651 100644 --- a/core/utils.py +++ b/core/utils.py @@ -1966,7 +1966,7 @@ def render_subset_expression( if first_cid is None: if col_name in constants.ALWAYS_INCLUDE_NON_CID_COLUMNS: return f"{source_alias}.{col_name}" - return f"NULL AS {col_name}" + return f"CAST(NULL AS STRING) AS {col_name}" if first_cid in constants.NON_CENSORED_SECONDARY_SOURCE_CIDS: return f"{source_alias}.{col_name}" @@ -1979,7 +1979,7 @@ def render_subset_expression( return f"IF({classification_alias}.{flag_name}, {source_alias}.{col_name}, NULL) AS {col_name}" # Unknown first CID: fail closed - return f"NULL AS {col_name}" + return f"CAST(NULL AS STRING) AS {col_name}" def build_column_rule_map(output_columns: list[str]) -> dict[str, dict]: """ From 7d258533ddce01adeddded1ead7ddf774ccdb1ce Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Fri, 10 Jul 2026 15:16:28 -0400 Subject: [PATCH 21/67] Add synthetic test case where revoke_ts == withdraw_ts (currently classified as UNDEFINED) --- core/generate_synthetic_bq_table.py | 1 + core/generate_synthetic_parquet.py | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/core/generate_synthetic_bq_table.py b/core/generate_synthetic_bq_table.py index d1a0c98..3ecff52 100644 --- a/core/generate_synthetic_bq_table.py +++ b/core/generate_synthetic_bq_table.py @@ -85,6 +85,7 @@ "TEST_BLANK_MODULE_STATUS", "TEST_UNKNOWN_MODULE_STATUS", "TEST_UNKNOWN_STATUS_CASE3A", + "TEST_REVOKE_EQUALS_WITHDRAW", ] # Connect_IDs that get deliberate NULLs sprinkled in. diff --git a/core/generate_synthetic_parquet.py b/core/generate_synthetic_parquet.py index e52686f..d27b350 100644 --- a/core/generate_synthetic_parquet.py +++ b/core/generate_synthetic_parquet.py @@ -308,6 +308,26 @@ def add( module2_status="Submitted", module2_ts="2025-03-01 00:00:00+00:00", # ts before cutoff, status Submitted ) + # ------------------------------------------------------------------ + # EDGE CASE: revoke_ts and withdraw_ts are EXACTLY equal. + # + # Currently UNDEFINED: revoke_after_withdraw uses strict ">" and + # revoke_before_withdraw uses strict "<", so equal timestamps make + # both False, and case_3a / case_3b both fail to match. This + # participant is expected to surface as UNDEFINED until/unless + # classify_participants() is updated to handle the tie explicitly + # (e.g., by changing revoke_after_withdraw to ">=" so ties resolve + # to 3A). Included here specifically to make that gap visible in + # test output rather than discovering it against real data. + # ------------------------------------------------------------------ + add( + "TEST_REVOKE_EQUALS_WITHDRAW", + "revoke_ts == withdraw_ts exactly -- currently falls through to UNDEFINED", + withdraw="Yes", withdraw_ts="2025-04-01 22:22:22+00:00", + revoke="Yes", revoke_ts="2025-04-01 22:22:22+00:00", + module1_ts="2025-03-30 08:12:15+00:00", + ) + df = pd.DataFrame(rows) return df From 5b2c6daf0e27d2e0b6dc0fe404fe93258fd4d956 Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Fri, 10 Jul 2026 16:12:18 -0400 Subject: [PATCH 22/67] Track columns with unrecognized CIDs in the missing-columns report --- core/transformations.py | 4 ++- core/utils.py | 73 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 73 insertions(+), 4 deletions(-) diff --git a/core/transformations.py b/core/transformations.py index ad6eab3..ab358b1 100644 --- a/core/transformations.py +++ b/core/transformations.py @@ -1010,6 +1010,7 @@ def create_subset_table( sql = result["sql"] fq_destination_table = result["destination_table"] duplicate_columns = result.get("duplicate_columns", []) + unrecognized_cid_columns = result.get("unrecognized_cid_columns", []) # Save the SQL to GCS for audit purposes try: @@ -1047,7 +1048,8 @@ def create_subset_table( destination_table=fq_destination_table, missing_cleaned_cols=missing_cleaned_cols, missing_cleaned_loop_vars=missing_cleaned_loop_vars, - duplicate_columns=duplicate_columns + duplicate_columns=duplicate_columns, + unrecognized_cid_columns=unrecognized_cid_columns ) utils.logger.info(f"Missing column report saved to {report_path}") diff --git a/core/utils.py b/core/utils.py index 8d54651..93f312a 100644 --- a/core/utils.py +++ b/core/utils.py @@ -1464,6 +1464,7 @@ def build_subset_query( selected_column_names = set() duplicate_columns = [] + unrecognized_cid_columns = [] # Always include join key first from base table select_parts.append(f"{base_alias}.{join_key}") @@ -1489,6 +1490,14 @@ def build_subset_query( duplicate_columns.append(col) continue + # Flag unrecognized CIDs before/alongside rendering + if is_unrecognized_censorship_cid(col): + unrecognized_cid_columns.append({ + "dataset": base_config["dataset"], + "table": base_config["table"], + "column": col + }) + select_parts.append( utils.render_subset_expression( source_alias=base_alias, @@ -1530,6 +1539,14 @@ def build_subset_query( duplicate_columns.append(f"{dataset}.{table}.{col}") continue + # Flag unrecognized CIDs for join-table columns too + if is_unrecognized_censorship_cid(col): + unrecognized_cid_columns.append({ + "dataset": dataset, + "table": table, + "column": col + }) + select_parts.append( utils.render_subset_expression( source_alias=table, @@ -1547,6 +1564,15 @@ def build_subset_query( f"{', '.join(unique_dupes)}. Using first occurrence only." ) + # Log unrecognized CID columns (if any) - same pattern as duplicate_columns logging + if unrecognized_cid_columns: + unrecognized_names = sorted({c["column"] for c in unrecognized_cid_columns}) + utils.logger.warning( + f"[{destination_table}] Columns with unrecognized CIDs detected " + f"(will be set to NULL by fail-closed default): {', '.join(unrecognized_names)}. " + f"Please verify these CIDs are valid and update MODULE_CENSOR_RULES if necessary." + ) + # Ensure at least one column is selected if not select_parts: raise ValueError(f"No columns selected for {destination_table}") @@ -1567,7 +1593,8 @@ def build_subset_query( return { "destination_table": f"{project}.{destination_dataset}.{destination_table}", "sql": sql, - "duplicate_columns": sorted(set(duplicate_columns)) + "duplicate_columns": sorted(set(duplicate_columns)), + "unrecognized_cid_columns": unrecognized_cid_columns } def create_subset_missing_columns_json( @@ -1576,7 +1603,8 @@ def create_subset_missing_columns_json( destination_table: str, missing_cleaned_cols: list[dict[str, Any]], missing_cleaned_loop_vars: list[dict[str, Any]], - duplicate_columns: list[str] + duplicate_columns: list[str], + unrecognized_cid_columns: list[dict[str, Any]] ) -> None: """ Write a JSON report of missing columns to a GCS location. @@ -1588,6 +1616,7 @@ def create_subset_missing_columns_json( missing_cleaned_cols (list): Missing standard columns. missing_cleaned_loop_vars (list): Missing loop variable columns. duplicate_columns (list): List of duplicate column names. + unrecognized_cid_columns (list): List of columns with unrecognized CIDs. """ # Build report structure report = { @@ -1600,12 +1629,14 @@ def create_subset_missing_columns_json( "missing_cleaned_cols": "Columns that were not found and are NOT cleaned loop variables — these are unexpected and should be investigated", "missing_cleaned_loop_vars": "Columns that were not found but ARE cleaned loop variables — these may be expected if the loop variable does not exist for a given table", "duplicate_columns": "Columns skipped due to duplicate names in SELECT", + "unrecognized_cid_columns": "Columns whose first CID matched neither MODULE_CENSOR_RULES nor NON_CENSORED_SECONDARY_SOURCE_CIDS. These were set to NULL by the fail-closed default and likely indicate a naming mistake, a missing MODULE_CENSOR_RULES entry, or a column that does not belong in this table", "occurrence_count": "Number of times this column appeared in the config AND was missing" } }, "missing_cleaned_cols": missing_cleaned_cols, "missing_cleaned_loop_vars": missing_cleaned_loop_vars, - "duplicate_columns": duplicate_columns + "duplicate_columns": duplicate_columns, + "unrecognized_cid_columns": unrecognized_cid_columns } # Parse GCS path @@ -1956,6 +1987,42 @@ def get_first_cid(column_name: str) -> str | None: cids = extract_ordered_concept_ids(column_name) return cids[0] if cids else None +def is_unrecognized_censorship_cid(col_name: str) -> bool: + """ + Returns True if col_name's first CID does not match any known pattern + (i.e., it is not in MODULE_CENSOR_RULES), is not in + NON_CENSORED_SECONDARY_SOURCE_CIDS, and is not an ALWAYS_INCLUDE_NON_CID + column. These columns fall through to the fail-closed NULL branch in + render_subset_expression, which is safe (no data leaks) but usually + signals a naming mistake, a missing MODULE_CENSOR_RULES entry, or a + column that does not belong in this table at all. + + Columns with no CID at all (e.g. "token") are NOT flagged here since + the absence of a CID is a different, already-handled case + (ALWAYS_INCLUDE_NON_CID_COLUMNS) rather than an unrecognized one. + + Args: + col_name: The column name to check. + + Returns: + bool: True if this column's first CID is unrecognized and would + be set to NULL by render_subset_expression's fail-closed path. + """ + first_cid = get_first_cid(col_name) + + if first_cid is None: + # No CID present at all - handled separately by + # ALWAYS_INCLUDE_NON_CID_COLUMNS, not a "censorship pattern" issue. + return False + + if first_cid in constants.NON_CENSORED_SECONDARY_SOURCE_CIDS: + return False + + if first_cid in constants.MODULE_CENSOR_RULES: + return False + + return True + def render_subset_expression( source_alias: str, col_name: str, From b51057c2d55efa3753226a2d3064c13216cfa1db Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Fri, 10 Jul 2026 16:19:24 -0400 Subject: [PATCH 23/67] Use sorted() instead of list() when expanding wildcard column selections, so generated SQL has deterministic column order across runs --- core/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/utils.py b/core/utils.py index 93f312a..ab81fb3 100644 --- a/core/utils.py +++ b/core/utils.py @@ -1477,7 +1477,7 @@ def build_subset_query( base_schema = set(table_schemas.get((base_config["dataset"], base_config["table"]), [])) if base_cols == "*": - base_cols = list(base_schema) + base_cols = sorted(base_schema) for col in base_cols: if col in base_schema: @@ -1516,7 +1516,7 @@ def build_subset_query( cols = join_table.get("columns", []) if cols == "*": - cols = list(join_schema) + cols = sorted(join_schema) if not cols: continue From 67e633628a8978e3e1e90a8a31eb2ea2007e902f Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Mon, 13 Jul 2026 15:25:54 -0400 Subject: [PATCH 24/67] Add a separate 3A anomaly for revoke_ts == withdraw_ts (previously UNDEFINED), setting ehr_cutoff to withdraw_ts since there is no ambiguity window when the two timestamps are equal --- core/utils.py | 55 ++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 44 insertions(+), 11 deletions(-) diff --git a/core/utils.py b/core/utils.py index ab81fb3..f4ca325 100644 --- a/core/utils.py +++ b/core/utils.py @@ -1692,7 +1692,7 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: revoke_ts_missing = df["hipaa_revoked_ts"].isna() # ========================================================================= - # Timestamp ordering validation for case 3B + # Timestamp validation for case 3 # ========================================================================= # revoke_after_withdraw: True when revoke timestamp exists AND is later than withdraw timestamp @@ -1703,6 +1703,16 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: & (df["hipaa_revoked_ts"] > df["consent_withdrawn_ts"]) ) + # revoke_equals_withdraw: True when revoke timestamp exists AND is equal to withdraw timestamp + # Treated as its own, separate 3A anomaly from revoke_after_withdraw. Since there is no + # ambiguity window, both events happened at the same instant, so withdraw_ts can safely be used + # for ehr_cutoff too. + revoke_equals_withdraw = ( + df["hipaa_revoked_ts"].notna() + & df["consent_withdrawn_ts"].notna() + & (df["hipaa_revoked_ts"] == df["consent_withdrawn_ts"]) + ) + # revoke_before_withdraw: True when revoke timestamp exists AND occurs before withdraw timestamp # This is the expected ordering (valid) case 3B scenario revoke_before_withdraw = ( @@ -1742,6 +1752,14 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: # survey/bio cutoffs, but anomaly is recorded. anomaly_revoke_after_withdraw = destroy_no & withdraw_yes & revoke_yes & revoke_after_withdraw + # Rule: Revoke timestamp is EQUAL to withdraw timestamp + # A separate, distinct anomaly from anomaly_revoke_after_withdraw. The + # ehr_cutoff CAN be reliably set since revoke and withdraw happened at + # the same instant (withdraw_ts = revoke_ts, so using either is + # equivalent). Still flagged as an anomaly for visibility/review but + # processed with a real ehr_cutoff rather than leaving it blank. + anomaly_revoke_equals_withdraw = destroy_no & withdraw_yes & revoke_yes & revoke_equals_withdraw + # ============================================================================ # Case classification masks # These masks determine which business-rule case each participant belongs to. @@ -1783,10 +1801,13 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: # Case 3A # - Destroy Data = No # - Withdraw Consent = Yes (timestamp present) - # - Revoke HIPAA = Yes (timestamp is either missing or occurs after withdraw timestamp) + # - Revoke HIPAA = Yes (timestamp is either missing, occurs strictly + # after withdraw timestamp, or occurs at exactly the same time as + # withdraw timestamp) # # EHR cutoff = Withdraw timestamp for normal 3A cases - # EHR cutoff = NULL for anomalous 3A where revoke timestamp occurs after withdraw timestamp + # EHR cutoff = NULL for the revoke-after-withdraw anomaly + # EHR cutoff = Withdraw timestamp for the revoke-equals-withdraw anomaly # Survey cutoff = Withdraw timestamp # Biospecimen cutoff = Withdraw timestamp # ============================================================================ @@ -1798,6 +1819,7 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: & ( df["hipaa_revoked_ts"].isna() | revoke_after_withdraw + | revoke_equals_withdraw ) ) @@ -1866,11 +1888,13 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: df["anomaly"] = np.select( [ anomaly_destroy_missing_ts, - anomaly_revoke_after_withdraw + anomaly_revoke_after_withdraw, + anomaly_revoke_equals_withdraw ], [ "Destroy Data = 'Yes' but timestamp is NULL", - "Revoke HIPAA timestamp occurs AFTER Withdraw Consent timestamp - EHR cutoff could not be determined reliably" + "Revoke HIPAA timestamp occurs AFTER Withdraw Consent timestamp - EHR cutoff could not be determined reliably", + "Revoke HIPAA timestamp is EQUAL to Withdraw Consent timestamp - EHR cutoff set to Withdraw Consent timestamp" ], default=pd.NA ) @@ -1892,13 +1916,22 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: df.loc[normal_3a, "survey_cutoff"] = df.loc[normal_3a, "consent_withdrawn_ts"] df.loc[normal_3a, "bio_cutoff"] = df.loc[normal_3a, "consent_withdrawn_ts"] - # Case 3A anomaly - # If revoke_ts is later than withdraw_ts, keep as anomaly and leave ehr_cutoff blank - # This will still be processed as a 3A case - anomalous_3a = case_3a & revoke_after_withdraw + # Case 3A anomaly - revoke AFTER withdraw: ehr_cutoff stays NULL. + # If revoke_ts is later than withdraw_ts, keep as anomaly and leave + # ehr_cutoff blank. This will still be processed as a 3A case. + anomalous_3a_revoke_after = case_3a & revoke_after_withdraw + + df.loc[anomalous_3a_revoke_after, "survey_cutoff"] = df.loc[anomalous_3a_revoke_after, "consent_withdrawn_ts"] + df.loc[anomalous_3a_revoke_after, "bio_cutoff"] = df.loc[anomalous_3a_revoke_after, "consent_withdrawn_ts"] + + # Case 3A anomaly - revoke EQUALS withdraw: ehr_cutoff CAN be reliably + # set since revoke and withdraw happened at the same instant. Use + # withdraw_ts for all three cutoffs. This will be processed as a 3A case. + anomalous_3a_revoke_equals = case_3a & revoke_equals_withdraw - df.loc[anomalous_3a, "survey_cutoff"] = df.loc[anomalous_3a, "consent_withdrawn_ts"] - df.loc[anomalous_3a, "bio_cutoff"] = df.loc[anomalous_3a, "consent_withdrawn_ts"] + df.loc[anomalous_3a_revoke_equals, "ehr_cutoff"] = df.loc[anomalous_3a_revoke_equals, "consent_withdrawn_ts"] + df.loc[anomalous_3a_revoke_equals, "survey_cutoff"] = df.loc[anomalous_3a_revoke_equals, "consent_withdrawn_ts"] + df.loc[anomalous_3a_revoke_equals, "bio_cutoff"] = df.loc[anomalous_3a_revoke_equals, "consent_withdrawn_ts"] # Case 3B: EHR cutoff = Revoke timestamp, Survey/Bio cutoff = Withdraw timestamp df.loc[case_3b, "ehr_cutoff"] = df.loc[case_3b, "hipaa_revoked_ts"] From a738b8b02d061e9a6693c59ceb733e26441a50ee Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Mon, 13 Jul 2026 15:29:03 -0400 Subject: [PATCH 25/67] Update TEST_REVOKE_EQUALS_WITHDRAW to reflect resolved 3A anomaly behavior and exercise the cutoff boundary across four modules --- core/generate_synthetic_parquet.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/core/generate_synthetic_parquet.py b/core/generate_synthetic_parquet.py index d27b350..3bbcd43 100644 --- a/core/generate_synthetic_parquet.py +++ b/core/generate_synthetic_parquet.py @@ -309,23 +309,23 @@ def add( ) # ------------------------------------------------------------------ - # EDGE CASE: revoke_ts and withdraw_ts are EXACTLY equal. + # Case 3A anomaly: revoke_ts and withdraw_ts are equal. # - # Currently UNDEFINED: revoke_after_withdraw uses strict ">" and - # revoke_before_withdraw uses strict "<", so equal timestamps make - # both False, and case_3a / case_3b both fail to match. This - # participant is expected to surface as UNDEFINED until/unless - # classify_participants() is updated to handle the tie explicitly - # (e.g., by changing revoke_after_withdraw to ">=" so ties resolve - # to 3A). Included here specifically to make that gap visible in - # test output rather than discovering it against real data. + # Resolves to case 3A, flagged via anomaly_revoke_equals_withdraw. + # Unlike the revoke-after-withdraw anomaly (where ehr_cutoff is left + # NULL because the true cutoff cannot be determined), the ehr_cutoff + # is set to withdraw_ts since revoke and withdraw happened at + # the same instant, so there is no ambiguity window between them. # ------------------------------------------------------------------ add( "TEST_REVOKE_EQUALS_WITHDRAW", - "revoke_ts == withdraw_ts exactly -- currently falls through to UNDEFINED", + "case 3A anomaly, revoke_ts == withdraw_ts -- ehr_cutoff set to withdraw_ts)", withdraw="Yes", withdraw_ts="2025-04-01 22:22:22+00:00", revoke="Yes", revoke_ts="2025-04-01 22:22:22+00:00", - module1_ts="2025-03-30 08:12:15+00:00", + module2_ts="2025-04-01 22:22:22+00:00", # same -> censored + module3_ts="2025-04-01 22:22:23+00:00", # after -> censored + covid19_ts="2025-03-30 08:12:15+00:00", # before -> not censored + experience2024_ts="2025-04-01 22:22:21+00:00", # before -> not censored ) df = pd.DataFrame(rows) From eb285410509133bc1ca350298d8d1f23c0f462b6 Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Mon, 13 Jul 2026 15:38:57 -0400 Subject: [PATCH 26/67] Clarify comments for cases 1 and 4 --- core/utils.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/core/utils.py b/core/utils.py index f4ca325..294f193 100644 --- a/core/utils.py +++ b/core/utils.py @@ -1904,7 +1904,9 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: df["survey_cutoff"] = pd.Series(pd.NaT, index=df.index, dtype="datetime64[us, UTC]") df["bio_cutoff"] = pd.Series(pd.NaT, index=df.index, dtype="datetime64[us, UTC]") - # Case 1: No cutoffs, so leave all as NaT; participant is included downstream without restrictions + # Case 1: No cutoffs, so leave all columns as NaT (as initialized previously). + # Participant is included downstream without restrictions. No further + # code is needed here for this case. # Case 2: EHR cutoff = Revoke timestamp, Survey/Bio cutoff = None df.loc[case_2, "ehr_cutoff"] = df.loc[case_2, "hipaa_revoked_ts"] @@ -1938,7 +1940,8 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: df.loc[case_3b, "survey_cutoff"] = df.loc[case_3b, "consent_withdrawn_ts"] df.loc[case_3b, "bio_cutoff"] = df.loc[case_3b, "consent_withdrawn_ts"] - # Case 4: No cutoffs, so leave all as NaT; participant is excluded downstream + # Case 4: No cutoffs, so leave all columns as NaT (as initialized previously). + # Participant is excluded downstream. No further code is needed here for this case. # Sort the dataframe for code development and debugging purposes. This can be removed in the future. df = df.sort_values(by=["exclusion_reason", "anomaly", "case", "Connect_ID"]) From 8e5cb4f9ef3129632eddef49c1846fafbf21c8e4 Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Mon, 13 Jul 2026 16:11:46 -0400 Subject: [PATCH 27/67] Update logs to include the applicable table name at the beginning of logging statement --- core/transformations.py | 32 ++++++++++++++++---------------- core/utils.py | 5 +++-- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/core/transformations.py b/core/transformations.py index ab358b1..d4fc2f6 100644 --- a/core/transformations.py +++ b/core/transformations.py @@ -967,11 +967,11 @@ def create_subset_table( client = bigquery.Client() # Load subset configuration - utils.logger.info("Loading subset configuration...") + utils.logger.info(f"[{destination_table}] Loading subset configuration...") subset_config = utils.load_subset_config(config_path) # Retrieve the target destination table config - utils.logger.info("Retrieving destination table configuration...") + utils.logger.info(f"[{destination_table}] Retrieving destination table configuration...") destination_table_config = utils.get_destination_table_config( subset_config, destination_dataset, @@ -980,7 +980,7 @@ def create_subset_table( if not destination_table_config: raise ValueError( - f"No config found for {destination_dataset}.{destination_table}" + f"[{destination_table}] No config found for {destination_dataset}.{destination_table}" ) # Build table schemas @@ -990,7 +990,7 @@ def create_subset_table( ) if schema_issues: - utils.logger.warning(f"Schema issues detected: {schema_issues}") + utils.logger.warning(f"[{destination_table}] Schema issues detected: {schema_issues}") # Identify missing columns missing_cleaned_cols, missing_cleaned_loop_vars = utils.find_missing_columns( @@ -1014,28 +1014,28 @@ def create_subset_table( # Save the SQL to GCS for audit purposes try: - utils.logger.info("Saving SQL to GCS...") + utils.logger.info(f"[{destination_table}] Saving SQL to GCS...") gcs_client = storage.Client() gcs_path = f"{constants.OUTPUT_SQL_PATH}{fq_destination_table}.sql" utils.save_sql_string(sql=sql, path=gcs_path, storage_client=gcs_client) - utils.logger.info(f"SQL saved to GCS at {gcs_path}") + utils.logger.info(f"[{destination_table}] SQL saved to GCS at {gcs_path}") except Exception as e: - utils.logger.exception(f"Error saving SQL to GCS: {e}") + utils.logger.exception(f"[{destination_table}] Error saving SQL to GCS: {e}") raise e # Execute the SQL try: - utils.logger.info("Executing SQL query...") + utils.logger.info(f"[{destination_table}] Executing SQL query...") query_job = client.query(sql) - utils.logger.info(f"Query job created with ID: {query_job.job_id}") + utils.logger.info(f"[{destination_table}] Query job created with ID: {query_job.job_id}") query_job.result() - utils.logger.info("Query execution completed successfully") - status = f"Table {fq_destination_table} successfully created" + utils.logger.info(f"[{destination_table}] Query execution completed successfully") + status = f"[{destination_table}] Table {fq_destination_table} successfully created" except Exception as e: - utils.logger.exception(f"Error executing SQL: {e}") + utils.logger.exception(f"[{destination_table}] Error executing SQL: {e}") # Log more details about the exception - utils.logger.error(f"Exception type: {type(e).__name__}") - utils.logger.error(f"Exception args: {e.args}") + utils.logger.error(f"[{destination_table}] Exception type: {type(e).__name__}") + utils.logger.error(f"[{destination_table}] Exception args: {e.args}") raise e # Write missing column report JSON @@ -1052,10 +1052,10 @@ def create_subset_table( unrecognized_cid_columns=unrecognized_cid_columns ) - utils.logger.info(f"Missing column report saved to {report_path}") + utils.logger.info(f"[{destination_table}] Missing column report saved to {report_path}") except Exception as e: - utils.logger.exception(f"Error writing missing column report for {fq_destination_table}: {e}") + utils.logger.exception(f"[{destination_table}] Error writing missing column report for {fq_destination_table}: {e}") raise e # Return result (matches pattern) diff --git a/core/utils.py b/core/utils.py index 294f193..8c6d177 100644 --- a/core/utils.py +++ b/core/utils.py @@ -1272,7 +1272,7 @@ def build_filter_sql( # Handle "no_filters" case (empty list) if not filters: utils.logger.warning( - f"Creating table " + f"[{destination_table_config['table']}] Creating table " f"{destination_table_config['dataset']}.{destination_table_config['table']} " f"WITHOUT filters applied (filter_profile={filter_profile})" ) @@ -2017,7 +2017,8 @@ def write_classification_to_bq(classification_df, bq_table_name, client) -> None job_config=job_config ) job.result() # Wait for the job to complete - utils.logger.info(f"Classification data written to {bq_table_name} successfully.") + _, _, table_short_name = utils.parse_fq_table(bq_table_name) + utils.logger.info(f"[{table_short_name}] Classification data written to {bq_table_name} successfully") def get_first_cid(column_name: str) -> str | None: cids = extract_ordered_concept_ids(column_name) From b38037c84144d397cf2f1dc7c807a03ba521019b Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Wed, 15 Jul 2026 11:40:58 -0400 Subject: [PATCH 28/67] Fix censorship summary: dedup rules sharing a status_col - Rename _build_censorship_case_block -> _build_reason_cte_block; the block now keys on status_col instead of column_name, so CIDs that share a status_col (e.g. the two menstrual survey CIDs) collapse to a single reason block instead of generating identical CASE logic twice. - Join deduped reasons back to every output column via a new column_status_map CTE, so each column (including duplicate CIDs) still gets its own row in the final result. - Pass classification_table as a real parameter to _build_reason_cte_block instead of templating it via a placeholder + string.replace(). - Reword reason messages: quote status values, add "(empty string)" annotation, distinguish NULL vs "too late" as separate sub-reasons. --- core/utils.py | 139 +++++++++++++++++++++++++++++++++++--------------- 1 file changed, 98 insertions(+), 41 deletions(-) diff --git a/core/utils.py b/core/utils.py index 8c6d177..e27a9da 100644 --- a/core/utils.py +++ b/core/utils.py @@ -2110,26 +2110,28 @@ def build_column_rule_map(output_columns: list[str]) -> dict[str, dict]: return column_rule_map -def _build_censorship_case_block( - column_name: str, +def _build_reason_cte_block( + status_col: str, rule: dict, + classification_table: str, classification_alias: str = "c", ) -> str: """ - Builds a single SELECT block reporting whether `column_name` was + Builds a single SELECT block reporting whether column_name was censored for each Connect_ID and why. Args: - column_name: The output column governed by this rule. - rule: One entry from constants.MODULE_CENSOR_RULES - (keys: status_col, completion_ts, cutoff). + status_col: The deduped status column name (e.g. "module2_status"). + rule: The rule dict for this status_col — one value from the + {status_col: rule} mapping built by build_censorship_summary_sql's + dedup step (keys: status_col, completion_ts, cutoff). + classification_table: Fully qualified classification table. classification_alias: Alias of the classification table. Returns: - str: A SELECT statement returning Connect_ID, case, column_name, - reason for rows where this column was censored. + str: A SELECT statement returning Connect_ID, case, status_col, + and reason. """ - status_col = rule["status_col"] completion_ts = rule["completion_ts"] cutoff = rule["cutoff"] @@ -2137,28 +2139,48 @@ def _build_censorship_case_block( SELECT {classification_alias}.Connect_ID, {classification_alias}.`case`, - '{column_name}' AS column_name, + '{status_col}' AS status_col, + -- Reason 1: Status is not "Submitted" at all (NULL, empty, or any + -- other value). This alone is disqualifying regardless of case. CASE WHEN {classification_alias}.{status_col} IS DISTINCT FROM 'Submitted' - THEN CONCAT( - '{status_col} = ', - IFNULL({classification_alias}.{status_col}, 'NULL') - ) + THEN CASE + WHEN {classification_alias}.{status_col} IS NULL + THEN '{status_col} = NULL' + ELSE CONCAT( + '{status_col} = "', {classification_alias}.{status_col}, '"', + IF({classification_alias}.{status_col} = '', ' (empty string)', '') + ) + END + -- Reason 2: Status was "Submitted", but for cases 3A/3B the + -- completion has to land strictly before the cutoff. Missing + -- timestamps or a completion on/after cutoff both count as + -- censored. Treat NULL and "too late" as distinct sub-reasons + -- for a clearer message. WHEN {classification_alias}.`case` IN ('3A', '3B') AND ( {classification_alias}.{completion_ts} IS NULL OR {classification_alias}.{cutoff} IS NULL OR {classification_alias}.{completion_ts} >= {classification_alias}.{cutoff} ) - THEN CONCAT( - '{completion_ts} = ', - IFNULL(CAST({classification_alias}.{completion_ts} AS STRING), 'NULL'), - ', on or after ', '{cutoff}', ' = ', - IFNULL(CAST({classification_alias}.{cutoff} AS STRING), 'NULL') - ) + THEN CASE + WHEN {classification_alias}.{completion_ts} IS NULL + THEN '{completion_ts} = NULL' + WHEN {classification_alias}.{cutoff} IS NULL + THEN '{cutoff} = NULL' + ELSE CONCAT( + '{completion_ts} = ', + CAST({classification_alias}.{completion_ts} AS STRING), + ' is on or after ', '{cutoff}', ' = ', + CAST({classification_alias}.{cutoff} AS STRING) + ) + END + -- Otherwise: Status was Submitted, and either this is not a 3A/3B case + -- (no cutoff applies) or completion was safely before cutoff - this + -- participant is not censored for this column. ELSE NULL END AS reason - FROM `{{classification_table}}` {classification_alias} + FROM `{classification_table}` {classification_alias} """.strip() @@ -2175,37 +2197,72 @@ def build_censorship_summary_sql( build_column_rule_map. classification_table: Fully qualified table containing case, status, completion_ts, and cutoff columns for every - participant (e.g. classification_2026_07_07). + participant. classification_alias: Alias used for that table in the generated SQL. Returns: str: A complete SQL query returning Connect_ID, case, column_name, - reason for every censorship event, filtered to reason IS NOT NULL. + and reason for every censorship event, filtered to reason IS NOT NULL. Raises: ValueError: If column_rule_map is empty. """ - if not column_rule_map: - raise ValueError("column_rule_map is empty; nothing to summarize.") - - blocks = [] - for column_name, rule in column_rule_map.items(): - block = _build_censorship_case_block( - column_name=column_name, + # Multiple output columns can point at the same status_col — e.g., both + # menstrual-survey CIDs (912367929, 232438133) drive menstrual_status. + # Collapse to one rule per status_col, so _build_reason_cte_block is not + # generating identical CASE logic twice for the same underlying column. + # + # Note: Unlike build_unique_eligibility_rules (used for the *_eligible + # flags in build_subset_query), this dedup does NOT check for + # conflicting completion_ts/cutoff values across CIDs sharing a + # status_col; it assumes column_rule_map is already consistent since + # every entry in it traces back to the same MODULE_CENSOR_RULES that + # build_unique_eligibility_rules validates elsewhere in the pipeline. + status_col_to_rule: dict[str, dict] = {} + for rule in column_rule_map.values(): + # Values are guaranteed identical for a shared status_col; see note above + status_col_to_rule[rule["status_col"]] = rule + + # Build one reason-block per unique status_col then UNION them together + reason_blocks = [ + _build_reason_cte_block( + status_col=status_col, rule=rule, + classification_table=classification_table, classification_alias=classification_alias, - ).replace("{classification_table}", classification_table) - blocks.append(block) - - unioned = "\nUNION ALL\n".join(blocks) + ) + for status_col, rule in status_col_to_rule.items() + ] + reasons_cte = "\nUNION ALL\n".join(reason_blocks) + + # Map every output column back to its status_col as a VALUES/STRUCT + # literal, so each column (including duplicates like the two menstrual + # CIDs) joins onto its single shared reasons row below. + column_map_rows = ",\n ".join( + f"('{col}', '{rule['status_col']}')" + for col, rule in column_rule_map.items() + ) final_sql = f""" - SELECT * - FROM ( - {unioned} + WITH reasons AS ( + {reasons_cte} + ), + column_status_map AS ( + SELECT * FROM UNNEST([ + STRUCT + {column_map_rows} + ]) ) - WHERE reason IS NOT NULL - ORDER BY Connect_ID, column_name + SELECT + reasons.Connect_ID, + reasons.`case`, + column_status_map.column_name, + reasons.reason + FROM reasons + JOIN column_status_map + ON reasons.status_col = column_status_map.status_col + WHERE reasons.reason IS NOT NULL + ORDER BY reasons.Connect_ID, column_status_map.column_name """.strip() return final_sql @@ -2213,8 +2270,8 @@ def build_censorship_summary_sql( def get_censorship_rollup(destination_table: str) -> dict: """ Queries the censorship summary table and returns two rollups: - 1. per-column counts (how many participants were censored, per column) - 2. per-case counts (how many censorship events occurred, per case) + 1. Per-column counts (how many participants were censored, per column) + 2. Per-case counts (how many censorship events occurred, per case) Args: destination_table: The table created by create_censorship_summary_table. From 50ed5afc392465db7c4db901a563e264153b406c Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Wed, 15 Jul 2026 15:01:20 -0400 Subject: [PATCH 29/67] Share eligibility condition between subset table and censorship summary Extract render_eligibility_condition() as the single source of truth for "is {status_col} eligible" used both to build the *_eligible flags in build_subset_query's CTE and to gate the censored/not-censored branch in _build_reason_cte_block. Previously these were two hand-written copies of the same condition that could silently drift apart; now a change to one always propagates to the other. --- core/utils.py | 124 +++++++++++++++++++++++++++++++------------------- 1 file changed, 78 insertions(+), 46 deletions(-) diff --git a/core/utils.py b/core/utils.py index e27a9da..72b6607 100644 --- a/core/utils.py +++ b/core/utils.py @@ -1340,6 +1340,41 @@ def build_unique_eligibility_rules(module_censor_rules: dict[str, dict]) -> dict return unique_rules +def render_eligibility_condition( + status_col: str, + completion_ts: str, + cutoff: str, + alias: str = "c", +) -> str: + """ + The single source of truth for "is {status_col} eligible" — status is + Submitted, and for 3A/3B cases, completion happened strictly before + cutoff. Used both to build the *_eligible flags in build_subset_query's + CTE and to determine the not-censored/censored branch in + _build_reason_cte_block, so the real table and the censorship summary + can never compute this condition differently from each other. + + Args: + status_col: The status column name (e.g. "module2_status"). + completion_ts: The completion timestamp column name. + cutoff: The cutoff timestamp column name. + alias: Table alias to qualify all column references with. + + Returns: + str: A parenthesized boolean SQL expression (no AS alias). + """ + return f"""( + {alias}.{status_col} = 'Submitted' + AND ( + {alias}.`case` NOT IN ('3A', '3B') + OR ( + {alias}.{completion_ts} IS NOT NULL + AND {alias}.{cutoff} IS NOT NULL + AND {alias}.{completion_ts} < {alias}.{cutoff} + ) + ) + )""" + def build_subset_query( client: bigquery.Client, full_config: dict[str, Any], @@ -1422,17 +1457,10 @@ def build_subset_query( for status_col, rule in unique_rules.items(): completion_ts = rule["completion_ts"] cutoff = rule["cutoff"] - flag_lines.append(f""" ( - c.{status_col} = 'Submitted' - AND ( - c.`case` NOT IN ('3A', '3B') - OR ( - c.{completion_ts} IS NOT NULL - AND c.{cutoff} IS NOT NULL - AND c.{completion_ts} < c.{cutoff} - ) - ) - ) AS {status_col}_eligible""") + eligible_expr = render_eligibility_condition( + status_col, completion_ts, cutoff, alias="c" + ) + flag_lines.append(f" {eligible_expr} AS {status_col}_eligible") eligibility_flags_sql = ",\n" + ",\n".join(flag_lines) # Build CTE with or without classification join @@ -2134,50 +2162,54 @@ def _build_reason_cte_block( """ completion_ts = rule["completion_ts"] cutoff = rule["cutoff"] + eligible_expr = render_eligibility_condition( + status_col, completion_ts, cutoff, alias=classification_alias + ) return f""" SELECT {classification_alias}.Connect_ID, {classification_alias}.`case`, '{status_col}' AS status_col, - -- Reason 1: Status is not "Submitted" at all (NULL, empty, or any - -- other value). This alone is disqualifying regardless of case. CASE - WHEN {classification_alias}.{status_col} IS DISTINCT FROM 'Submitted' + -- The eligibility decision itself is delegated to + -- render_eligibility_condition — the same expression used to + -- build {status_col}_eligible in build_subset_query. Everything + -- below only decides which human-readable message to show once + -- it is known the participant is NOT eligible. + WHEN {eligible_expr} IS NOT TRUE THEN CASE - WHEN {classification_alias}.{status_col} IS NULL - THEN '{status_col} = NULL' - ELSE CONCAT( - '{status_col} = "', {classification_alias}.{status_col}, '"', - IF({classification_alias}.{status_col} = '', ' (empty string)', '') - ) - END - -- Reason 2: Status was "Submitted", but for cases 3A/3B the - -- completion has to land strictly before the cutoff. Missing - -- timestamps or a completion on/after cutoff both count as - -- censored. Treat NULL and "too late" as distinct sub-reasons - -- for a clearer message. - WHEN {classification_alias}.`case` IN ('3A', '3B') - AND ( - {classification_alias}.{completion_ts} IS NULL - OR {classification_alias}.{cutoff} IS NULL - OR {classification_alias}.{completion_ts} >= {classification_alias}.{cutoff} - ) - THEN CASE - WHEN {classification_alias}.{completion_ts} IS NULL - THEN '{completion_ts} = NULL' - WHEN {classification_alias}.{cutoff} IS NULL - THEN '{cutoff} = NULL' - ELSE CONCAT( - '{completion_ts} = ', - CAST({classification_alias}.{completion_ts} AS STRING), - ' is on or after ', '{cutoff}', ' = ', - CAST({classification_alias}.{cutoff} AS STRING) - ) + -- Reason 1: Status is not "Submitted" (NULL, empty, + -- or any other value). This alone is disqualifying + -- regardless of case. + WHEN {classification_alias}.{status_col} IS DISTINCT FROM 'Submitted' + THEN CASE + WHEN {classification_alias}.{status_col} IS NULL + THEN '{status_col} = NULL' + ELSE CONCAT( + '{status_col} = "', {classification_alias}.{status_col}, '"', + IF({classification_alias}.{status_col} = '', ' (empty string)', '') + ) + END + -- Reason 2: Status was "Submitted", but for cases 3A/3B the + -- completion has to land strictly before the cutoff. Missing + -- timestamps or a completion on/after cutoff both count as + -- censored. Treat NULL and "too late" as distinct + -- sub-reasons for a clearer message. + ELSE CASE + WHEN {classification_alias}.{completion_ts} IS NULL + THEN '{completion_ts} = NULL' + WHEN {classification_alias}.{cutoff} IS NULL + THEN '{cutoff} = NULL' + ELSE CONCAT( + '{completion_ts} = ', + CAST({classification_alias}.{completion_ts} AS STRING), + ' is on or after ', '{cutoff}', ' = ', + CAST({classification_alias}.{cutoff} AS STRING) + ) + END END - -- Otherwise: Status was Submitted, and either this is not a 3A/3B case - -- (no cutoff applies) or completion was safely before cutoff - this - -- participant is not censored for this column. + -- Eligible: Not censored for this column. ELSE NULL END AS reason FROM `{classification_table}` {classification_alias} From bc26142f488638ace62bf13f3a48667455aeee9b Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Wed, 15 Jul 2026 15:11:07 -0400 Subject: [PATCH 30/67] Fix indentation in eligibility SQL builders --- core/utils.py | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/core/utils.py b/core/utils.py index 72b6607..e80ecb8 100644 --- a/core/utils.py +++ b/core/utils.py @@ -1458,7 +1458,7 @@ def build_subset_query( completion_ts = rule["completion_ts"] cutoff = rule["cutoff"] eligible_expr = render_eligibility_condition( - status_col, completion_ts, cutoff, alias="c" + status_col, completion_ts, cutoff, alias="c" ) flag_lines.append(f" {eligible_expr} AS {status_col}_eligible") eligibility_flags_sql = ",\n" + ",\n".join(flag_lines) @@ -2164,7 +2164,7 @@ def _build_reason_cte_block( cutoff = rule["cutoff"] eligible_expr = render_eligibility_condition( status_col, completion_ts, cutoff, alias=classification_alias - ) + ) return f""" SELECT @@ -2276,26 +2276,26 @@ def build_censorship_summary_sql( ) final_sql = f""" - WITH reasons AS ( +WITH reasons AS ( {reasons_cte} - ), - column_status_map AS ( - SELECT * FROM UNNEST([ - STRUCT - {column_map_rows} - ]) - ) - SELECT - reasons.Connect_ID, - reasons.`case`, - column_status_map.column_name, - reasons.reason - FROM reasons - JOIN column_status_map - ON reasons.status_col = column_status_map.status_col - WHERE reasons.reason IS NOT NULL - ORDER BY reasons.Connect_ID, column_status_map.column_name - """.strip() +), +column_status_map AS ( + SELECT * FROM UNNEST([ + STRUCT + {column_map_rows} + ]) +) +SELECT + reasons.Connect_ID, + reasons.`case`, + column_status_map.column_name, + reasons.reason +FROM reasons +JOIN column_status_map + ON reasons.status_col = column_status_map.status_col +WHERE reasons.reason IS NOT NULL +ORDER BY reasons.Connect_ID, column_status_map.column_name +""".strip() return final_sql From e6a4255649ff90e217e8a246908847847f4e84ef Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Wed, 15 Jul 2026 15:40:58 -0400 Subject: [PATCH 31/67] Generate censorship summary rollup JSON --- core/transformations.py | 26 ++++++++++++++++++++-- core/utils.py | 49 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 2 deletions(-) diff --git a/core/transformations.py b/core/transformations.py index d4fc2f6..f4177a9 100644 --- a/core/transformations.py +++ b/core/transformations.py @@ -1069,11 +1069,13 @@ def create_censorship_summary_table( output_table: str, classification_table: str, destination_table: str, + rollup_report_base_path: str ) -> dict: """ Generates the censorship summary query, saves it to GCS for audit purposes (consistent with other transform functions in this codebase), - executes it, and writes the result to destination_table. + executes it, writes the result to destination_table, and creates a + JSON rollup report of censorship counts by column and by case. Args: output_table: Fully qualified final subset/output table (e.g. the @@ -1083,9 +1085,10 @@ def create_censorship_summary_table( classification_table: Fully qualified table with case/status/timestamp columns for every participant. destination_table: Fully qualified table to create with the summary. + rollup_report_base_path: Base GCS path for the censorship rollup report. Returns: - dict: Contains "status" and "submitted_sql_path". + dict: Contains "status", "submitted_sql_path", and "rollup_report_path". Raises: Exception: Propagates any errors encountered during query execution. @@ -1127,9 +1130,28 @@ def create_censorship_summary_table( utils.logger.exception(f"[{destination_table}] Error executing censorship summary SQL: {e}") raise e + # Compute and write the rollup report + try: + rollup = utils.get_censorship_rollup(destination_table) + report_path = f"{rollup_report_base_path}{destination_table}_rollup.json" + + utils.create_censorship_rollup_json( + client=gcs_client, + output_path=report_path, + destination_table=destination_table, + by_column=rollup["by_column"], + by_case=rollup["by_case"], + ) + + utils.logger.info(f"[{destination_table}] Censorship rollup report saved to {report_path}") + except Exception as e: + utils.logger.exception(f"[{destination_table}] Error writing censorship rollup report: {e}") + raise e + return { "status": status, "submitted_sql_path": constants.OUTPUT_SQL_PATH, + "rollup_report_path": report_path, } if __name__ == "__main__": diff --git a/core/utils.py b/core/utils.py index e80ecb8..4e4d9ca 100644 --- a/core/utils.py +++ b/core/utils.py @@ -2330,3 +2330,52 @@ def get_censorship_rollup(destination_table: str) -> dict: by_case = [dict(row) for row in client.query(by_case_sql).result()] return {"by_column": by_column, "by_case": by_case} + +def create_censorship_rollup_json( + client: storage.Client, + output_path: str, + destination_table: str, + by_column: list[dict[str, Any]], + by_case: list[dict[str, Any]], +) -> None: + """ + Write a JSON report of censorship rollup counts to a GCS location. + + Args: + client (storage.Client): GCS client. + output_path (str): GCS path (gs://...) to write the report. + destination_table (str): Censorship summary table name (the table + get_censorship_rollup was queried against). + by_column (list): Per-column censored counts, as returned by + get_censorship_rollup()["by_column"]. + by_case (list): Per-case censorship event counts, as returned by + get_censorship_rollup()["by_case"]. + """ + # Build report structure + report = { + "_metadata": { + "generated_at": datetime.now(timezone.utc).isoformat(), + "source": "pr2-transformation pipeline", + "destination_table": destination_table, + "description": "Rollup of censorship events from the censorship summary table, broken down by column and by case", + "structure": { + "by_column": "Per-column counts of how many participants were censored for that column, ordered by censored_count descending", + "by_case": "Per-case counts of how many censorship events occurred, ordered by case" + } + }, + "by_column": by_column, + "by_case": by_case, + } + + # Parse GCS path + path = output_path.removeprefix("gs://") + bucket_name, blob_path = path.split("/", 1) + + bucket = client.bucket(bucket_name) + blob = bucket.blob(blob_path) + + # Upload JSON report + blob.upload_from_string( + json.dumps(report, indent=2, ensure_ascii=False), + content_type="application/json", + ) \ No newline at end of file From cfdd37bad607519fe7c271605df374facf484231 Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Wed, 15 Jul 2026 16:51:19 -0400 Subject: [PATCH 32/67] Rename subset-named functions to match destination/censorship terminology build_filter_sql -> build_row_filter_sql (disambiguates from classification-case filtering elsewhere in the same function) build_subset_query -> build_destination_table_query (this builds the destination table's full query, which includes column selection, row filtering, and censorship) render_subset_expression -> render_censored_column_expression (this renders per-column censorship logic, not column selection; "subset" was misleading here since selection already happened by this point) Updates docstrings and comments referencing the old names. No logic changes. --- core/utils.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/core/utils.py b/core/utils.py index 4e4d9ca..11b7cfb 100644 --- a/core/utils.py +++ b/core/utils.py @@ -1235,7 +1235,7 @@ def _format_output(missing_columns): return _format_output(missing_cleaned_cols), _format_output(missing_cleaned_loop_vars) -def build_filter_sql( +def build_row_filter_sql( filter_profiles: dict[str, Any], destination_table_config: dict[str, Any] ) -> str: @@ -1375,7 +1375,7 @@ def render_eligibility_condition( ) )""" -def build_subset_query( +def build_destination_table_query( client: bigquery.Client, full_config: dict[str, Any], destination_table_config: dict[str, Any], @@ -1413,7 +1413,7 @@ def build_subset_query( allowed_cases = classification_filter.get("allowed_cases", []) if classification_filter else [] # Build WHERE clause from filter profile - filter_sql = build_filter_sql( + filter_sql = build_row_filter_sql( full_config.get("filter_profiles", {}), destination_table_config ) @@ -1527,7 +1527,7 @@ def build_subset_query( }) select_parts.append( - utils.render_subset_expression( + utils.render_censored_column_expression( source_alias=base_alias, col_name=col, classification_alias=classification_alias, @@ -1576,7 +1576,7 @@ def build_subset_query( }) select_parts.append( - utils.render_subset_expression( + utils.render_censored_column_expression( source_alias=table, col_name=col, classification_alias=classification_alias, @@ -2058,9 +2058,9 @@ def is_unrecognized_censorship_cid(col_name: str) -> bool: (i.e., it is not in MODULE_CENSOR_RULES), is not in NON_CENSORED_SECONDARY_SOURCE_CIDS, and is not an ALWAYS_INCLUDE_NON_CID column. These columns fall through to the fail-closed NULL branch in - render_subset_expression, which is safe (no data leaks) but usually - signals a naming mistake, a missing MODULE_CENSOR_RULES entry, or a - column that does not belong in this table at all. + render_censored_column_expression, which is safe (no data leaks) but + usually signals a naming mistake, a missing MODULE_CENSOR_RULES entry, + or a column that does not belong in this table at all. Columns with no CID at all (e.g. "token") are NOT flagged here since the absence of a CID is a different, already-handled case @@ -2071,7 +2071,8 @@ def is_unrecognized_censorship_cid(col_name: str) -> bool: Returns: bool: True if this column's first CID is unrecognized and would - be set to NULL by render_subset_expression's fail-closed path. + be set to NULL by render_censored_column_expression's + fail-closed path. """ first_cid = get_first_cid(col_name) @@ -2088,7 +2089,7 @@ def is_unrecognized_censorship_cid(col_name: str) -> bool: return True -def render_subset_expression( +def render_censored_column_expression( source_alias: str, col_name: str, classification_alias: str = "c", @@ -2174,9 +2175,9 @@ def _build_reason_cte_block( CASE -- The eligibility decision itself is delegated to -- render_eligibility_condition — the same expression used to - -- build {status_col}_eligible in build_subset_query. Everything - -- below only decides which human-readable message to show once - -- it is known the participant is NOT eligible. + -- build {status_col}_eligible in build_destination_table_query. + -- Everything below only decides which human-readable message to + -- show once it is known the participant is NOT eligible. WHEN {eligible_expr} IS NOT TRUE THEN CASE -- Reason 1: Status is not "Submitted" (NULL, empty, @@ -2245,7 +2246,7 @@ def build_censorship_summary_sql( # generating identical CASE logic twice for the same underlying column. # # Note: Unlike build_unique_eligibility_rules (used for the *_eligible - # flags in build_subset_query), this dedup does NOT check for + # flags in build_destination_table_query), this dedup does NOT check for # conflicting completion_ts/cutoff values across CIDs sharing a # status_col; it assumes column_rule_map is already consistent since # every entry in it traces back to the same MODULE_CENSOR_RULES that From d3f457d129d8a5c45d1a58a86dcc593c9822ce1a Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Wed, 15 Jul 2026 16:55:34 -0400 Subject: [PATCH 33/67] Rename create_subset_table to create_destination_table Follows the utils.py rename (build_subset_query -> build_destination_table_query). Updates the call site and section header comment. No logic changes. --- core/transformations.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/transformations.py b/core/transformations.py index f4177a9..da09f25 100644 --- a/core/transformations.py +++ b/core/transformations.py @@ -927,7 +927,7 @@ def create_standardized_mapping_table( ############# Subset Table Construction ############################### ######################################################################## -def create_subset_table( +def create_destination_table( config_path: str, destination_dataset: str, destination_table: str, @@ -999,7 +999,7 @@ def create_subset_table( ) # Generate SQL - result = utils.build_subset_query( + result = utils.build_destination_table_query( client=client, full_config=subset_config, destination_table_config=destination_table_config, From 3927361a42829caa895eac50976b033bc04e9e29 Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Thu, 16 Jul 2026 11:14:44 -0400 Subject: [PATCH 34/67] Finish renaming subset-named functions and references to destination/censorship terminology --- core/transformations.py | 25 +++++++------- core/utils.py | 33 ++++++++++--------- ...et_config.json => destination_config.json} | 0 3 files changed, 29 insertions(+), 29 deletions(-) rename reference/{subset_config.json => destination_config.json} (100%) diff --git a/core/transformations.py b/core/transformations.py index da09f25..58a48d7 100644 --- a/core/transformations.py +++ b/core/transformations.py @@ -935,10 +935,10 @@ def create_destination_table( classification_table: Optional[str] = None ) -> dict: """ - Create a subset table based on configuration. + Create the destination table based on configuration. This function: - 1. Loads subset configuration + 1. Loads the destination table configuration 2. Retrieves the target destination table config 3. Builds table schemas 4. Identifies missing columns @@ -948,7 +948,7 @@ def create_destination_table( 8. Writes missing column report JSON Args: - config_path (str): Path to subset config JSON. + config_path (str): Path to destination config JSON. destination_dataset (str): Target dataset. destination_table (str): Target table. missing_report_base_path (str): Base GCS path for missing column reports. @@ -966,14 +966,14 @@ def create_destination_table( """ client = bigquery.Client() - # Load subset configuration - utils.logger.info(f"[{destination_table}] Loading subset configuration...") - subset_config = utils.load_subset_config(config_path) + # Load destination table configuration + utils.logger.info(f"[{destination_table}] Loading destination table configuration...") + destination_config = utils.load_destination_config(config_path) # Retrieve the target destination table config utils.logger.info(f"[{destination_table}] Retrieving destination table configuration...") destination_table_config = utils.get_destination_table_config( - subset_config, + destination_config, destination_dataset, destination_table ) @@ -1001,7 +1001,7 @@ def create_destination_table( # Generate SQL result = utils.build_destination_table_query( client=client, - full_config=subset_config, + full_config=destination_config, destination_table_config=destination_table_config, table_schemas=table_schemas, classification_table=classification_table @@ -1042,7 +1042,7 @@ def create_destination_table( try: report_path = f"{missing_report_base_path}{fq_destination_table}_missing_columns.json" - utils.create_subset_missing_columns_json( + utils.create_destination_missing_columns_json( client=gcs_client, output_path=report_path, destination_table=fq_destination_table, @@ -1078,10 +1078,9 @@ def create_censorship_summary_table( JSON rollup report of censorship counts by column and by case. Args: - output_table: Fully qualified final subset/output table (e.g. the - mvp_case_1_2_3A_3B table). Used only to determine which - columns exist and are therefore eligible for censorship - classification. + output_table: Fully qualified final destination table. Used only + to determine which columns exist and are therefore + eligible for censorship classification. classification_table: Fully qualified table with case/status/timestamp columns for every participant. destination_table: Fully qualified table to create with the summary. diff --git a/core/utils.py b/core/utils.py index 11b7cfb..f3cde0d 100644 --- a/core/utils.py +++ b/core/utils.py @@ -1015,15 +1015,15 @@ def create_missing_columns_json( blob.upload_from_string(json.dumps(report, indent=2, ensure_ascii=False), content_type="application/json") -def load_subset_config(config_file_path: str) -> dict: +def load_destination_config(config_file_path: str) -> dict: """ - Load the subset configuration JSON file. + Load the destination configuration JSON file. - This configuration defines filter profiles and destination table mappings + This configuration defines destination table mappings and related settings used to generate SQL queries. Args: - config_file_path (str): Path to the subset configuration JSON file. + config_file_path (str): Path to the destination configuration JSON file. Returns: dict: Parsed configuration dictionary. Returns an empty dict if loading fails. @@ -1032,11 +1032,11 @@ def load_subset_config(config_file_path: str) -> dict: with open(config_file_path, 'r') as f: return json.load(f) except Exception as e: - utils.logger.error(f"Error loading subset configuration from {config_file_path}: {e}") + utils.logger.error(f"Error loading destination configuration from {config_file_path}: {e}") return {} def get_destination_table_config( - subset_config: dict[str, Any], + destination_config: dict[str, Any], destination_dataset: str, destination_table: str, ) -> dict[str, Any] | None: @@ -1047,7 +1047,7 @@ def get_destination_table_config( and table name. Args: - subset_config (dict): Full subset configuration. + destination_config (dict): Full destination table configuration. destination_dataset (str): Target dataset name. destination_table (str): Target table name. @@ -1055,7 +1055,7 @@ def get_destination_table_config( dict | None: Matching destination table configuration or None if not found. """ # Loop through each destination table defined in the config - for destination in subset_config.get("destination_tables", []): + for destination in destination_config.get("destination_tables", []): dataset_name = destination.get("dataset") table_name = destination.get("table") @@ -1240,7 +1240,7 @@ def build_row_filter_sql( destination_table_config: dict[str, Any] ) -> str: """ - Generate SQL WHERE clause based on filter profile in the subset config. + Generate SQL WHERE clause based on filter profile in the destination config. Applies the selected filter profile from the configuration and constructs SQL conditions. Logs a warning if no filters are applied. @@ -1349,10 +1349,11 @@ def render_eligibility_condition( """ The single source of truth for "is {status_col} eligible" — status is Submitted, and for 3A/3B cases, completion happened strictly before - cutoff. Used both to build the *_eligible flags in build_subset_query's - CTE and to determine the not-censored/censored branch in - _build_reason_cte_block, so the real table and the censorship summary - can never compute this condition differently from each other. + cutoff. Used both to build the *_eligible flags in + build_destination_table_query's CTE and to determine the + not-censored/censored branch in _build_reason_cte_block, so the real + table and the censorship summary can never compute this condition + differently from each other. Args: status_col: The status column name (e.g. "module2_status"). @@ -1383,7 +1384,7 @@ def build_destination_table_query( classification_table: Optional[str] = None ) -> dict[str, Any]: """ - Build a CREATE OR REPLACE TABLE SQL query based on subset configuration. + Build a CREATE OR REPLACE TABLE SQL query based on destination configuration. Combines base table, optional filters, optional classification filter, and join tables into a single SQL query. @@ -1625,7 +1626,7 @@ def build_destination_table_query( "unrecognized_cid_columns": unrecognized_cid_columns } -def create_subset_missing_columns_json( +def create_destination_missing_columns_json( client: storage.Client, output_path: str, destination_table: str, @@ -1652,7 +1653,7 @@ def create_subset_missing_columns_json( "generated_at": datetime.now(timezone.utc).isoformat(), "source": "pr2-transformation pipeline", "destination_table": destination_table, - "description": "Report of columns requested by the subset config that were not found in their respective source tables", + "description": "Report of columns requested by the destination config that were not found in their respective source tables", "structure": { "missing_cleaned_cols": "Columns that were not found and are NOT cleaned loop variables — these are unexpected and should be investigated", "missing_cleaned_loop_vars": "Columns that were not found but ARE cleaned loop variables — these may be expected if the loop variable does not exist for a given table", diff --git a/reference/subset_config.json b/reference/destination_config.json similarity index 100% rename from reference/subset_config.json rename to reference/destination_config.json From 1cfe265781640632d2704ff1db1f9219e741487b Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Fri, 17 Jul 2026 12:55:10 -0400 Subject: [PATCH 35/67] Add create_classification_table orchestration function Wraps the parquet-read -> classify -> optional local CSV -> BigQuery-write sequence, matching the logging/error-handling pattern of create_destination_table and create_censorship_summary_table. --- core/transformations.py | 74 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/core/transformations.py b/core/transformations.py index 58a48d7..e5f1a74 100644 --- a/core/transformations.py +++ b/core/transformations.py @@ -926,6 +926,80 @@ def create_standardized_mapping_table( ######################################################################## ############# Subset Table Construction ############################### ######################################################################## +def create_classification_table( + parquet_url: str, + classification_table: str, + local_csv_path: Optional[str] = None, +) -> dict: + """ + Reads participant status data from parquet, classifies each participant + into a case (1, 2, 3A, 3B, 4, or an exclusion/anomaly), and writes the + result to a BigQuery classification table. + + This function: + 1. Reads and filters the parquet file to verified participants + 2. Classifies participants into cases based on consent/revoke/destroy flags + 3. Optionally writes a local CSV for inspection/debugging + 4. Writes the classification DataFrame to a BigQuery table + + Args: + parquet_url: GCS path to the participant_status parquet file. + classification_table: Fully qualified BigQuery table to write the + classification results to. + local_csv_path: Optional local file path to save the classification + DataFrame as CSV for inspection. + + Returns: + dict: Contains "status" and "classification_table". + + Raises: + Exception: Propagates any errors encountered during read, classification, + CSV write, or BigQuery write. + """ + client = bigquery.Client() + _, _, table_short_name = utils.parse_fq_table(classification_table) + + # Read the parquet file into a DataFrame + try: + utils.logger.info(f"[{table_short_name}] Reading parquet from {parquet_url}") + classification_df = utils.read_parquet_to_dataframe(parquet_url) + utils.logger.info(f"[{table_short_name}] Read {len(classification_df)} verified participant rows") + except Exception as e: + utils.logger.exception(f"[{table_short_name}] Error reading parquet from {parquet_url}: {e}") + raise e + + # Classify participants into cases + try: + utils.logger.info(f"[{table_short_name}] Classifying participants...") + classification_df = utils.classify_participants(classification_df) + utils.logger.info(f"[{table_short_name}] Classification complete") + except Exception as e: + utils.logger.exception(f"[{table_short_name}] Error classifying participants: {e}") + raise e + + # Optionally write the classification DataFrame to a local CSV for inspection + if local_csv_path: + try: + classification_df.to_csv(local_csv_path, index=False) + utils.logger.info(f"[{table_short_name}] Local CSV saved to {local_csv_path}") + except Exception as e: + utils.logger.exception(f"[{table_short_name}] Error saving local CSV to {local_csv_path}: {e}") + raise e + + # Write the classification DataFrame to BigQuery + try: + utils.logger.info(f"[{table_short_name}] Writing classification to BigQuery...") + utils.write_classification_to_bq(classification_df, classification_table, client) + status = f"[{table_short_name}] Classification table successfully created at {classification_table}" + utils.logger.info(status) + except Exception as e: + utils.logger.exception(f"[{table_short_name}] Error writing classification to BigQuery: {e}") + raise e + + return { + "status": status, + "classification_table": classification_table, + } def create_destination_table( config_path: str, From 29ca57934fc0cc54bf6e69f582bfb33b25e87642 Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Fri, 17 Jul 2026 12:59:13 -0400 Subject: [PATCH 36/67] Use short table names in create_destination_table/create_censorship_summary_table logs Bracketed log prefixes now use the short table name (via parse_fq_table) instead of the full fully-qualified name, for readability. Full names still appear in "successfully created at {table}" status messages. --- core/transformations.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/core/transformations.py b/core/transformations.py index e5f1a74..d53c03b 100644 --- a/core/transformations.py +++ b/core/transformations.py @@ -1104,7 +1104,7 @@ def create_destination_table( utils.logger.info(f"[{destination_table}] Query job created with ID: {query_job.job_id}") query_job.result() utils.logger.info(f"[{destination_table}] Query execution completed successfully") - status = f"[{destination_table}] Table {fq_destination_table} successfully created" + status = f"[{destination_table}] Table successfully created at {fq_destination_table}" except Exception as e: utils.logger.exception(f"[{destination_table}] Error executing SQL: {e}") # Log more details about the exception @@ -1156,7 +1156,7 @@ def create_censorship_summary_table( to determine which columns exist and are therefore eligible for censorship classification. classification_table: Fully qualified table with case/status/timestamp - columns for every participant. + columns for every participant. destination_table: Fully qualified table to create with the summary. rollup_report_base_path: Base GCS path for the censorship rollup report. @@ -1167,12 +1167,13 @@ def create_censorship_summary_table( Exception: Propagates any errors encountered during query execution. """ client = bigquery.Client() + _, _, table_short_name = utils.parse_fq_table(destination_table) output_columns = utils.get_column_names(client, output_table) column_rule_map = utils.build_column_rule_map(output_columns) utils.logger.info( - f"[{destination_table}] Found {len(column_rule_map)} censorable columns out of " + f"[{table_short_name}] Found {len(column_rule_map)} censorable columns out of " f"{len(output_columns)} total columns in {output_table}" ) @@ -1189,18 +1190,19 @@ def create_censorship_summary_table( gcs_path = f"{constants.OUTPUT_SQL_PATH}{destination_table}.sql" utils.save_sql_string(sql=final_sql, path=gcs_path, storage_client=gcs_client) except Exception as e: - utils.logger.exception(f"[{destination_table}] Error saving censorship summary SQL to {gcs_path}") + utils.logger.exception(f"[{table_short_name}] Error saving censorship summary SQL to {gcs_path}") raise e # Execute the SQL try: - utils.logger.info(f"[{destination_table}] Executing censorship summary query...") + utils.logger.info(f"[{table_short_name}] Executing censorship summary query...") query_job = client.query(final_sql) + utils.logger.info(f"[{table_short_name}] Query job created with ID: {query_job.job_id}") query_job.result() - status = f"[{destination_table}] Table successfully created with censorship summary." + status = f"[{table_short_name}] Censorship summary table successfully created at {destination_table}" utils.logger.info(status) except Exception as e: - utils.logger.exception(f"[{destination_table}] Error executing censorship summary SQL: {e}") + utils.logger.exception(f"[{table_short_name}] Error executing censorship summary SQL: {e}") raise e # Compute and write the rollup report @@ -1216,9 +1218,9 @@ def create_censorship_summary_table( by_case=rollup["by_case"], ) - utils.logger.info(f"[{destination_table}] Censorship rollup report saved to {report_path}") + utils.logger.info(f"[{table_short_name}] Censorship rollup report saved to {report_path}") except Exception as e: - utils.logger.exception(f"[{destination_table}] Error writing censorship rollup report: {e}") + utils.logger.exception(f"[{table_short_name}] Error writing censorship rollup report: {e}") raise e return { From 3bcc5cdffe49989ec45f6e42e2d409cc9a83c4d5 Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Fri, 17 Jul 2026 13:09:15 -0400 Subject: [PATCH 37/67] Clarify build_row_filter_sql: no filter_profile conditions != no row restriction join-key and classification-case filters are added elsewhere in build_destination_table_query, so a table using filter_profile "no_filters" can still end up row-restricted. Downgrade the no-filters log from warning to info, since using "no_filters" is often intentional. --- core/utils.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/core/utils.py b/core/utils.py index f3cde0d..918741c 100644 --- a/core/utils.py +++ b/core/utils.py @@ -1240,20 +1240,28 @@ def build_row_filter_sql( destination_table_config: dict[str, Any] ) -> str: """ - Generate SQL WHERE clause based on filter profile in the destination config. + Generate a SQL WHERE clause from the destination table's filter_profile. - Applies the selected filter profile from the configuration and constructs - SQL conditions. Logs a warning if no filters are applied. + Looks up the filter_profile named in destination_table_config, resolves + it against filter_profiles, and builds a WHERE clause from its column/ + value conditions. This is only one of two ways a table's rows can end + up restricted. build_destination_table_query separately adds a + join-key-not-null condition and (if allowed_cases is set) a + classification-case filter. A table using the "no_filters" profile can + still be row-restricted by those other conditions; this function only + reports on the conditions it itself contributes. Args: filter_profiles (dict): Mapping of filter profile names to filter rules. destination_table_config (dict): Destination table configuration. Returns: - str: SQL WHERE clause string or empty string if no filters are applied. + str: SQL WHERE clause string or an empty string if the resolved + filter_profile has no conditions. Raises: - ValueError: If filter_profile is missing or invalid. + ValueError: If filter_profile is missing from destination_table_config + or if the named profile is not found in filter_profiles. """ # Get selected filter profile filter_profile = destination_table_config.get("filter_profile") @@ -1271,10 +1279,10 @@ def build_row_filter_sql( # Handle "no_filters" case (empty list) if not filters: - utils.logger.warning( - f"[{destination_table_config['table']}] Creating table " - f"{destination_table_config['dataset']}.{destination_table_config['table']} " - f"WITHOUT filters applied (filter_profile={filter_profile})" + utils.logger.info( + f"[{destination_table_config['table']}] " + f"filter_profile '{filter_profile}' has no conditions; " + f"no row filter SQL generated here" ) return "" From 214ad501d12bbc16d58cee5f979273e9b8ab0fc3 Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Fri, 17 Jul 2026 13:15:25 -0400 Subject: [PATCH 38/67] Remove redundant logging statement for the classification table --- core/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/core/utils.py b/core/utils.py index 918741c..791ed89 100644 --- a/core/utils.py +++ b/core/utils.py @@ -2055,7 +2055,6 @@ def write_classification_to_bq(classification_df, bq_table_name, client) -> None ) job.result() # Wait for the job to complete _, _, table_short_name = utils.parse_fq_table(bq_table_name) - utils.logger.info(f"[{table_short_name}] Classification data written to {bq_table_name} successfully") def get_first_cid(column_name: str) -> str | None: cids = extract_ordered_concept_ids(column_name) From a4b0d0f82e1beee3e5baab43517696711e707fda Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Fri, 17 Jul 2026 14:07:01 -0400 Subject: [PATCH 39/67] Refactor utils.py by moving participant classification into its own module --- core/classification.py | 372 +++++++++++++++++++++++++++++++++++++++++ core/utils.py | 367 ---------------------------------------- 2 files changed, 372 insertions(+), 367 deletions(-) create mode 100644 core/classification.py diff --git a/core/classification.py b/core/classification.py new file mode 100644 index 0000000..9628ac5 --- /dev/null +++ b/core/classification.py @@ -0,0 +1,372 @@ +"""Participant classification and cutoff calculation utilities.""" + +import pandas as pd +import numpy as np + +from google.cloud import bigquery + +def read_parquet_to_dataframe(parquet_url: str) -> pd.DataFrame: + # Only keep verified participants + df = pd.read_parquet(parquet_url, + engine="pyarrow", + filters=[("verified_status", "==", "Verified")]) + + # Normalize timestamp columns + # Black strings, nulls, and invalid timestamps become NaT + ts_cols = ["consent_withdrawn_ts", "hipaa_revoked_ts", "data_destruction_ts", + "module1_complete_ts", "module2_complete_ts", "module3_complete_ts", + "module4_complete_ts", #"bio_complete_ts", "clinicalbio_complete_ts", "mouthwash_complete_ts", + "menstrual_complete_ts", "covid19_complete_ts", + "experience2024_complete_ts"] + for col in ts_cols: + df[col] = pd.to_datetime(df[col], errors="coerce", utc=True).astype("datetime64[us, UTC]") + + return df + +def classify_participants(df: pd.DataFrame) -> pd.DataFrame: + # ========================================================================= + # Boolean masks for conditions + # Each variable becomes a True/False Series across the dataframe + # ========================================================================= + + # Boolean masks for "Yes" flag values + destroy_yes = df["data_destruction_requested"].eq("Yes") + withdraw_yes = df["consent_withdrawn"].eq("Yes") + revoke_yes = df["hipaa_revoked"].eq("Yes") + + # Boolean masks for "No" flag values + destroy_no = df["data_destruction_requested"].eq("No") + withdraw_no = df["consent_withdrawn"].eq("No") + revoke_no = df["hipaa_revoked"].eq("No") + + # Boolean masks for missing timestamps + destroy_ts_missing = df["data_destruction_ts"].isna() + withdraw_ts_missing = df["consent_withdrawn_ts"].isna() + revoke_ts_missing = df["hipaa_revoked_ts"].isna() + + # ========================================================================= + # Timestamp validation for case 3 + # ========================================================================= + + # revoke_after_withdraw: True when revoke timestamp exists AND is later than withdraw timestamp + # This is considered an anomaly because EHR cutoff cannot be reliably determined + revoke_after_withdraw = ( + df["hipaa_revoked_ts"].notna() + & df["consent_withdrawn_ts"].notna() + & (df["hipaa_revoked_ts"] > df["consent_withdrawn_ts"]) + ) + + # revoke_equals_withdraw: True when revoke timestamp exists AND is equal to withdraw timestamp + # Treated as its own, separate 3A anomaly from revoke_after_withdraw. Since there is no + # ambiguity window, both events happened at the same instant, so withdraw_ts can safely be used + # for ehr_cutoff too. + revoke_equals_withdraw = ( + df["hipaa_revoked_ts"].notna() + & df["consent_withdrawn_ts"].notna() + & (df["hipaa_revoked_ts"] == df["consent_withdrawn_ts"]) + ) + + # revoke_before_withdraw: True when revoke timestamp exists AND occurs before withdraw timestamp + # This is the expected ordering (valid) case 3B scenario + revoke_before_withdraw = ( + df["hipaa_revoked_ts"].notna() + & df["consent_withdrawn_ts"].notna() + & (df["hipaa_revoked_ts"] < df["consent_withdrawn_ts"]) + ) + + # ========================================================================= + # Validation / exclusion masks + # These identify records that violate expected data rules, such as having a + # "Yes" flag but missing the corresponding timestamp. + # These participants are excluded from downstream processing and flagged + # for data quality review. + # ========================================================================= + + # Rule 1: Withdraw Consent = Yes but withdraw timestamp is NULL + # Rule 2: Revoke HIPAA = Yes but revoke timestamp is NULL AND participant did NOT withdraw consent + data_quality_mask = ( + (destroy_no & withdraw_yes & revoke_yes & withdraw_ts_missing) + | (destroy_no & withdraw_no & revoke_yes & revoke_ts_missing) + ) + # ========================================================================= + # Anomaly validation rules + # These participants are still processed downstream but are flagged because + # their data is internally inconsistent or violates expected temporal logic. + # ========================================================================= + + # Rule: Destory Data = Yes but destroy timestamp is NULL + # Participant is still classified as Case 4 and excluded downstream, + # but the missing timestamp is recorded as an anomaly. + anomaly_destroy_missing_ts = destroy_yes & withdraw_yes & revoke_yes & destroy_ts_missing + + # Rule: Revoke timestamp occurs AFTER withdraw timestamp + # This creates ambiguity because EHR cutoff cannot be reliably determined. + # Participant is still processed as Case 3A using withdraw timestamp for + # survey/bio cutoffs, but anomaly is recorded. + anomaly_revoke_after_withdraw = destroy_no & withdraw_yes & revoke_yes & revoke_after_withdraw + + # Rule: Revoke timestamp is EQUAL to withdraw timestamp + # A separate, distinct anomaly from anomaly_revoke_after_withdraw. The + # ehr_cutoff CAN be reliably set since revoke and withdraw happened at + # the same instant (withdraw_ts = revoke_ts, so using either is + # equivalent). Still flagged as an anomaly for visibility/review but + # processed with a real ehr_cutoff rather than leaving it blank. + anomaly_revoke_equals_withdraw = destroy_no & withdraw_yes & revoke_yes & revoke_equals_withdraw + + # ============================================================================ + # Case classification masks + # These masks determine which business-rule case each participant belongs to. + # ============================================================================ + + # ============================================================================ + # Case 1 + # - Destroy Data = No + # - Withdraw Consent = No + # - Revoke HIPAA = No + # + # No restrictions apply. + # Participant and all their data can be included without any cutoffs. + # ============================================================================ + case_1 = ( + destroy_no + & withdraw_no + & revoke_no + ) + + # ============================================================================ + # Case 2 + # - Destroy Data = No + # - Withdraw Consent = No + # - Revoke HIPAA = Yes (timestamp present) + # + # EHR cutoff = Revoke timestamp + # Survey cutoff = None + # Biospecimen = None + # ============================================================================ + case_2 = ( + destroy_no + & withdraw_no + & revoke_yes + & df["hipaa_revoked_ts"].notna() + ) + + # ============================================================================ + # Case 3A + # - Destroy Data = No + # - Withdraw Consent = Yes (timestamp present) + # - Revoke HIPAA = Yes (timestamp is either missing, occurs strictly + # after withdraw timestamp, or occurs at exactly the same time as + # withdraw timestamp) + # + # EHR cutoff = Withdraw timestamp for normal 3A cases + # EHR cutoff = NULL for the revoke-after-withdraw anomaly + # EHR cutoff = Withdraw timestamp for the revoke-equals-withdraw anomaly + # Survey cutoff = Withdraw timestamp + # Biospecimen cutoff = Withdraw timestamp + # ============================================================================ + case_3a = ( + destroy_no + & withdraw_yes + & revoke_yes + & df["consent_withdrawn_ts"].notna() + & ( + df["hipaa_revoked_ts"].isna() + | revoke_after_withdraw + | revoke_equals_withdraw + ) + ) + + # ============================================================================ + # Case 3B + # - Destroy Data = No + # - Withdraw Consent = Yes (timestamp present and occurs AFTER revoke timestamp) + # - Revoke HIPAA = Yes (timestamp present and occurs BEFORE withdraw timestamp) + # + # EHR cutoff = Revoke timestamp + # Survey cutoff = Withdraw timestamp + # Biospecimen cutoff = Withdraw timestamp + # ============================================================================ + case_3b = ( + destroy_no + & withdraw_yes + & revoke_yes + & df["consent_withdrawn_ts"].notna() + & revoke_before_withdraw + ) + + # ============================================================================ + # Case 4 + # - Destroy Data = Yes + # - Withdraw Consent = Yes + # - Revoke HIPAA = Yes + # + # The participant and all their data is excluded from downstream processing + # ============================================================================ + case_4 = ( + destroy_yes + & withdraw_yes + & revoke_yes + ) + + # Build output classification column based on the above cases and rules + df["case"] = np.select( + [case_1, case_2, case_3a, case_3b, case_4, data_quality_mask], + ["1", "2", "3A", "3B", "4", "DATA_QUALITY_EXCLUSION"], + default="UNDEFINED" + ) + + # Build exclusion reason column to specify which rule was violated for + # participants classified as "DATA_QUALITY_EXCLUSION" + df["exclusion_reason"] = np.select( + [ + destroy_no & withdraw_yes & withdraw_ts_missing, + destroy_no & revoke_yes & revoke_ts_missing & withdraw_no, + ], + [ + "Withdraw Consent = 'Yes' but timestamp is NULL", + "Revoke HIPAA = 'Yes' but timestamp is NULL", + ], + default=pd.NA + ) + + # Assign reason for unmatched participants + df.loc[ + df["case"] == "UNDEFINED", + "exclusion_reason" + ] = "Participant flag combination did not match any defined case" + + # Build anomaly column to specify which rule was violated for + # participants classified as cases 4 or 3A but have internal + # inconsistencies in their data + df["anomaly"] = np.select( + [ + anomaly_destroy_missing_ts, + anomaly_revoke_after_withdraw, + anomaly_revoke_equals_withdraw + ], + [ + "Destroy Data = 'Yes' but timestamp is NULL", + "Revoke HIPAA timestamp occurs AFTER Withdraw Consent timestamp - EHR cutoff could not be determined reliably", + "Revoke HIPAA timestamp is EQUAL to Withdraw Consent timestamp - EHR cutoff set to Withdraw Consent timestamp" + ], + default=pd.NA + ) + + # Cutoff logic + df["ehr_cutoff"] = pd.Series(pd.NaT, index=df.index, dtype="datetime64[us, UTC]") + df["survey_cutoff"] = pd.Series(pd.NaT, index=df.index, dtype="datetime64[us, UTC]") + df["bio_cutoff"] = pd.Series(pd.NaT, index=df.index, dtype="datetime64[us, UTC]") + + # Case 1: No cutoffs, so leave all columns as NaT (as initialized previously). + # Participant is included downstream without restrictions. No further + # code is needed here for this case. + + # Case 2: EHR cutoff = Revoke timestamp, Survey/Bio cutoff = None + df.loc[case_2, "ehr_cutoff"] = df.loc[case_2, "hipaa_revoked_ts"] + + # Case 3A: EHR/Survey/Bio cutoff = Withdraw timestamp + normal_3a = case_3a & df["hipaa_revoked_ts"].isna() + + df.loc[normal_3a, "ehr_cutoff"] = df.loc[normal_3a, "consent_withdrawn_ts"] + df.loc[normal_3a, "survey_cutoff"] = df.loc[normal_3a, "consent_withdrawn_ts"] + df.loc[normal_3a, "bio_cutoff"] = df.loc[normal_3a, "consent_withdrawn_ts"] + + # Case 3A anomaly - revoke AFTER withdraw: ehr_cutoff stays NULL. + # If revoke_ts is later than withdraw_ts, keep as anomaly and leave + # ehr_cutoff blank. This will still be processed as a 3A case. + anomalous_3a_revoke_after = case_3a & revoke_after_withdraw + + df.loc[anomalous_3a_revoke_after, "survey_cutoff"] = df.loc[anomalous_3a_revoke_after, "consent_withdrawn_ts"] + df.loc[anomalous_3a_revoke_after, "bio_cutoff"] = df.loc[anomalous_3a_revoke_after, "consent_withdrawn_ts"] + + # Case 3A anomaly - revoke EQUALS withdraw: ehr_cutoff CAN be reliably + # set since revoke and withdraw happened at the same instant. Use + # withdraw_ts for all three cutoffs. This will be processed as a 3A case. + anomalous_3a_revoke_equals = case_3a & revoke_equals_withdraw + + df.loc[anomalous_3a_revoke_equals, "ehr_cutoff"] = df.loc[anomalous_3a_revoke_equals, "consent_withdrawn_ts"] + df.loc[anomalous_3a_revoke_equals, "survey_cutoff"] = df.loc[anomalous_3a_revoke_equals, "consent_withdrawn_ts"] + df.loc[anomalous_3a_revoke_equals, "bio_cutoff"] = df.loc[anomalous_3a_revoke_equals, "consent_withdrawn_ts"] + + # Case 3B: EHR cutoff = Revoke timestamp, Survey/Bio cutoff = Withdraw timestamp + df.loc[case_3b, "ehr_cutoff"] = df.loc[case_3b, "hipaa_revoked_ts"] + df.loc[case_3b, "survey_cutoff"] = df.loc[case_3b, "consent_withdrawn_ts"] + df.loc[case_3b, "bio_cutoff"] = df.loc[case_3b, "consent_withdrawn_ts"] + + # Case 4: No cutoffs, so leave all columns as NaT (as initialized previously). + # Participant is excluded downstream. No further code is needed here for this case. + + # Sort the dataframe for code development and debugging purposes. This can be removed in the future. + df = df.sort_values(by=["exclusion_reason", "anomaly", "case", "Connect_ID"]) + + # Sort dataframe in order of: + # 1. Undefined cases + # 2. Data quality exclusion cases + # 3. Cases with anomalies + # 4. Normal cases + df["sort_order"] = np.select( + [df["case"].eq("UNDEFINED"), df["case"].eq("DATA_QUALITY_EXCLUSION"), df["anomaly"].notna()], + [1, 2, 3], default=4 + ) + + return df + +def write_classification_to_bq(classification_df, bq_table_name, client) -> None: + """ + Write the classification DataFrame to a BigQuery table + Args: + classification_df (pd.DataFrame): DataFrame containing participant classifications. + bq_table_name (str): Fully qualified BigQuery table name (project.dataset.table). + client (bigquery.Client): BigQuery client for writing data. + """ + job_config = bigquery.LoadJobConfig( + schema=[ + # String columns + bigquery.SchemaField("Connect_ID", "STRING"), + bigquery.SchemaField("verified_status", "STRING"), + bigquery.SchemaField("verified_status_concept_id", "STRING"), + bigquery.SchemaField("consent_withdrawn", "STRING"), + bigquery.SchemaField("consent_withdrawn_concept_id", "STRING"), + bigquery.SchemaField("hipaa_revoked", "STRING"), + bigquery.SchemaField("hipaa_revoked_concept_id", "STRING"), + bigquery.SchemaField("data_destruction_requested", "STRING"), + bigquery.SchemaField("data_destruction_requested_concept_id", "STRING"), + + # Timestamp columns + bigquery.SchemaField("consent_withdrawn_ts", "TIMESTAMP"), + bigquery.SchemaField("hipaa_revoked_ts", "TIMESTAMP"), + bigquery.SchemaField("data_destruction_ts", "TIMESTAMP"), + bigquery.SchemaField("module1_complete_ts", "TIMESTAMP"), + bigquery.SchemaField("module2_complete_ts", "TIMESTAMP"), + bigquery.SchemaField("module3_complete_ts", "TIMESTAMP"), + bigquery.SchemaField("module4_complete_ts", "TIMESTAMP"), + #bigquery.SchemaField("bio_complete_ts", "TIMESTAMP"), + #bigquery.SchemaField("clinicalbio_complete_ts", "TIMESTAMP"), + #bigquery.SchemaField("mouthwash_complete_ts", "TIMESTAMP"), + bigquery.SchemaField("menstrual_complete_ts", "TIMESTAMP"), + bigquery.SchemaField("covid19_complete_ts", "TIMESTAMP"), + bigquery.SchemaField("experience2024_complete_ts", "TIMESTAMP"), + + # Classification fields + bigquery.SchemaField("case", "STRING"), + bigquery.SchemaField("exclusion_reason", "STRING"), + bigquery.SchemaField("anomaly", "STRING"), + + # Derived cutoffs + bigquery.SchemaField("ehr_cutoff", "TIMESTAMP"), + bigquery.SchemaField("survey_cutoff", "TIMESTAMP"), + bigquery.SchemaField("bio_cutoff", "TIMESTAMP"), + + # Helper field + bigquery.SchemaField("sort_order", "INTEGER"), + ], + write_disposition=bigquery.WriteDisposition.WRITE_TRUNCATE + ) + + # Load DataFrame to BigQuery + job = client.load_table_from_dataframe( + classification_df, + bq_table_name, + job_config=job_config + ) + job.result() # Wait for the job to complete \ No newline at end of file diff --git a/core/utils.py b/core/utils.py index 791ed89..8cb3726 100644 --- a/core/utils.py +++ b/core/utils.py @@ -1689,373 +1689,6 @@ def create_destination_missing_columns_json( content_type="application/json", ) -def read_parquet_to_dataframe(parquet_url: str) -> pd.DataFrame: - # Only keep verified participants - df = pd.read_parquet(parquet_url, - engine="pyarrow", - filters=[("verified_status", "==", "Verified")]) - - # Normalize timestamp columns - # Black strings, nulls, and invalid timestamps become NaT - ts_cols = ["consent_withdrawn_ts", "hipaa_revoked_ts", "data_destruction_ts", - "module1_complete_ts", "module2_complete_ts", "module3_complete_ts", - "module4_complete_ts", #"bio_complete_ts", "clinicalbio_complete_ts", "mouthwash_complete_ts", - "menstrual_complete_ts", "covid19_complete_ts", - "experience2024_complete_ts"] - for col in ts_cols: - df[col] = pd.to_datetime(df[col], errors="coerce", utc=True).astype("datetime64[us, UTC]") - - return df - -def classify_participants(df: pd.DataFrame) -> pd.DataFrame: - # ========================================================================= - # Boolean masks for conditions - # Each variable becomes a True/False Series across the dataframe - # ========================================================================= - - # Boolean masks for "Yes" flag values - destroy_yes = df["data_destruction_requested"].eq("Yes") - withdraw_yes = df["consent_withdrawn"].eq("Yes") - revoke_yes = df["hipaa_revoked"].eq("Yes") - - # Boolean masks for "No" flag values - destroy_no = df["data_destruction_requested"].eq("No") - withdraw_no = df["consent_withdrawn"].eq("No") - revoke_no = df["hipaa_revoked"].eq("No") - - # Boolean masks for missing timestamps - destroy_ts_missing = df["data_destruction_ts"].isna() - withdraw_ts_missing = df["consent_withdrawn_ts"].isna() - revoke_ts_missing = df["hipaa_revoked_ts"].isna() - - # ========================================================================= - # Timestamp validation for case 3 - # ========================================================================= - - # revoke_after_withdraw: True when revoke timestamp exists AND is later than withdraw timestamp - # This is considered an anomaly because EHR cutoff cannot be reliably determined - revoke_after_withdraw = ( - df["hipaa_revoked_ts"].notna() - & df["consent_withdrawn_ts"].notna() - & (df["hipaa_revoked_ts"] > df["consent_withdrawn_ts"]) - ) - - # revoke_equals_withdraw: True when revoke timestamp exists AND is equal to withdraw timestamp - # Treated as its own, separate 3A anomaly from revoke_after_withdraw. Since there is no - # ambiguity window, both events happened at the same instant, so withdraw_ts can safely be used - # for ehr_cutoff too. - revoke_equals_withdraw = ( - df["hipaa_revoked_ts"].notna() - & df["consent_withdrawn_ts"].notna() - & (df["hipaa_revoked_ts"] == df["consent_withdrawn_ts"]) - ) - - # revoke_before_withdraw: True when revoke timestamp exists AND occurs before withdraw timestamp - # This is the expected ordering (valid) case 3B scenario - revoke_before_withdraw = ( - df["hipaa_revoked_ts"].notna() - & df["consent_withdrawn_ts"].notna() - & (df["hipaa_revoked_ts"] < df["consent_withdrawn_ts"]) - ) - - # ========================================================================= - # Validation / exclusion masks - # These identify records that violate expected data rules, such as having a - # "Yes" flag but missing the corresponding timestamp. - # These participants are excluded from downstream processing and flagged - # for data quality review. - # ========================================================================= - - # Rule 1: Withdraw Consent = Yes but withdraw timestamp is NULL - # Rule 2: Revoke HIPAA = Yes but revoke timestamp is NULL AND participant did NOT withdraw consent - data_quality_mask = ( - (destroy_no & withdraw_yes & revoke_yes & withdraw_ts_missing) - | (destroy_no & withdraw_no & revoke_yes & revoke_ts_missing) - ) - # ========================================================================= - # Anomaly validation rules - # These participants are still processed downstream but are flagged because - # their data is internally inconsistent or violates expected temporal logic. - # ========================================================================= - - # Rule: Destory Data = Yes but destroy timestamp is NULL - # Participant is still classified as Case 4 and excluded downstream, - # but the missing timestamp is recorded as an anomaly. - anomaly_destroy_missing_ts = destroy_yes & withdraw_yes & revoke_yes & destroy_ts_missing - - # Rule: Revoke timestamp occurs AFTER withdraw timestamp - # This creates ambiguity because EHR cutoff cannot be reliably determined. - # Participant is still processed as Case 3A using withdraw timestamp for - # survey/bio cutoffs, but anomaly is recorded. - anomaly_revoke_after_withdraw = destroy_no & withdraw_yes & revoke_yes & revoke_after_withdraw - - # Rule: Revoke timestamp is EQUAL to withdraw timestamp - # A separate, distinct anomaly from anomaly_revoke_after_withdraw. The - # ehr_cutoff CAN be reliably set since revoke and withdraw happened at - # the same instant (withdraw_ts = revoke_ts, so using either is - # equivalent). Still flagged as an anomaly for visibility/review but - # processed with a real ehr_cutoff rather than leaving it blank. - anomaly_revoke_equals_withdraw = destroy_no & withdraw_yes & revoke_yes & revoke_equals_withdraw - - # ============================================================================ - # Case classification masks - # These masks determine which business-rule case each participant belongs to. - # ============================================================================ - - # ============================================================================ - # Case 1 - # - Destroy Data = No - # - Withdraw Consent = No - # - Revoke HIPAA = No - # - # No restrictions apply. - # Participant and all their data can be included without any cutoffs. - # ============================================================================ - case_1 = ( - destroy_no - & withdraw_no - & revoke_no - ) - - # ============================================================================ - # Case 2 - # - Destroy Data = No - # - Withdraw Consent = No - # - Revoke HIPAA = Yes (timestamp present) - # - # EHR cutoff = Revoke timestamp - # Survey cutoff = None - # Biospecimen = None - # ============================================================================ - case_2 = ( - destroy_no - & withdraw_no - & revoke_yes - & df["hipaa_revoked_ts"].notna() - ) - - # ============================================================================ - # Case 3A - # - Destroy Data = No - # - Withdraw Consent = Yes (timestamp present) - # - Revoke HIPAA = Yes (timestamp is either missing, occurs strictly - # after withdraw timestamp, or occurs at exactly the same time as - # withdraw timestamp) - # - # EHR cutoff = Withdraw timestamp for normal 3A cases - # EHR cutoff = NULL for the revoke-after-withdraw anomaly - # EHR cutoff = Withdraw timestamp for the revoke-equals-withdraw anomaly - # Survey cutoff = Withdraw timestamp - # Biospecimen cutoff = Withdraw timestamp - # ============================================================================ - case_3a = ( - destroy_no - & withdraw_yes - & revoke_yes - & df["consent_withdrawn_ts"].notna() - & ( - df["hipaa_revoked_ts"].isna() - | revoke_after_withdraw - | revoke_equals_withdraw - ) - ) - - # ============================================================================ - # Case 3B - # - Destroy Data = No - # - Withdraw Consent = Yes (timestamp present and occurs AFTER revoke timestamp) - # - Revoke HIPAA = Yes (timestamp present and occurs BEFORE withdraw timestamp) - # - # EHR cutoff = Revoke timestamp - # Survey cutoff = Withdraw timestamp - # Biospecimen cutoff = Withdraw timestamp - # ============================================================================ - case_3b = ( - destroy_no - & withdraw_yes - & revoke_yes - & df["consent_withdrawn_ts"].notna() - & revoke_before_withdraw - ) - - # ============================================================================ - # Case 4 - # - Destroy Data = Yes - # - Withdraw Consent = Yes - # - Revoke HIPAA = Yes - # - # The participant and all their data is excluded from downstream processing - # ============================================================================ - case_4 = ( - destroy_yes - & withdraw_yes - & revoke_yes - ) - - # Build output classification column based on the above cases and rules - df["case"] = np.select( - [case_1, case_2, case_3a, case_3b, case_4, data_quality_mask], - ["1", "2", "3A", "3B", "4", "DATA_QUALITY_EXCLUSION"], - default="UNDEFINED" - ) - - # Build exclusion reason column to specify which rule was violated for - # participants classified as "DATA_QUALITY_EXCLUSION" - df["exclusion_reason"] = np.select( - [ - destroy_no & withdraw_yes & withdraw_ts_missing, - destroy_no & revoke_yes & revoke_ts_missing & withdraw_no, - ], - [ - "Withdraw Consent = 'Yes' but timestamp is NULL", - "Revoke HIPAA = 'Yes' but timestamp is NULL", - ], - default=pd.NA - ) - - # Assign reason for unmatched participants - df.loc[ - df["case"] == "UNDEFINED", - "exclusion_reason" - ] = "Participant flag combination did not match any defined case" - - # Build anomaly column to specify which rule was violated for - # participants classified as cases 4 or 3A but have internal - # inconsistencies in their data - df["anomaly"] = np.select( - [ - anomaly_destroy_missing_ts, - anomaly_revoke_after_withdraw, - anomaly_revoke_equals_withdraw - ], - [ - "Destroy Data = 'Yes' but timestamp is NULL", - "Revoke HIPAA timestamp occurs AFTER Withdraw Consent timestamp - EHR cutoff could not be determined reliably", - "Revoke HIPAA timestamp is EQUAL to Withdraw Consent timestamp - EHR cutoff set to Withdraw Consent timestamp" - ], - default=pd.NA - ) - - # Cutoff logic - df["ehr_cutoff"] = pd.Series(pd.NaT, index=df.index, dtype="datetime64[us, UTC]") - df["survey_cutoff"] = pd.Series(pd.NaT, index=df.index, dtype="datetime64[us, UTC]") - df["bio_cutoff"] = pd.Series(pd.NaT, index=df.index, dtype="datetime64[us, UTC]") - - # Case 1: No cutoffs, so leave all columns as NaT (as initialized previously). - # Participant is included downstream without restrictions. No further - # code is needed here for this case. - - # Case 2: EHR cutoff = Revoke timestamp, Survey/Bio cutoff = None - df.loc[case_2, "ehr_cutoff"] = df.loc[case_2, "hipaa_revoked_ts"] - - # Case 3A: EHR/Survey/Bio cutoff = Withdraw timestamp - normal_3a = case_3a & df["hipaa_revoked_ts"].isna() - - df.loc[normal_3a, "ehr_cutoff"] = df.loc[normal_3a, "consent_withdrawn_ts"] - df.loc[normal_3a, "survey_cutoff"] = df.loc[normal_3a, "consent_withdrawn_ts"] - df.loc[normal_3a, "bio_cutoff"] = df.loc[normal_3a, "consent_withdrawn_ts"] - - # Case 3A anomaly - revoke AFTER withdraw: ehr_cutoff stays NULL. - # If revoke_ts is later than withdraw_ts, keep as anomaly and leave - # ehr_cutoff blank. This will still be processed as a 3A case. - anomalous_3a_revoke_after = case_3a & revoke_after_withdraw - - df.loc[anomalous_3a_revoke_after, "survey_cutoff"] = df.loc[anomalous_3a_revoke_after, "consent_withdrawn_ts"] - df.loc[anomalous_3a_revoke_after, "bio_cutoff"] = df.loc[anomalous_3a_revoke_after, "consent_withdrawn_ts"] - - # Case 3A anomaly - revoke EQUALS withdraw: ehr_cutoff CAN be reliably - # set since revoke and withdraw happened at the same instant. Use - # withdraw_ts for all three cutoffs. This will be processed as a 3A case. - anomalous_3a_revoke_equals = case_3a & revoke_equals_withdraw - - df.loc[anomalous_3a_revoke_equals, "ehr_cutoff"] = df.loc[anomalous_3a_revoke_equals, "consent_withdrawn_ts"] - df.loc[anomalous_3a_revoke_equals, "survey_cutoff"] = df.loc[anomalous_3a_revoke_equals, "consent_withdrawn_ts"] - df.loc[anomalous_3a_revoke_equals, "bio_cutoff"] = df.loc[anomalous_3a_revoke_equals, "consent_withdrawn_ts"] - - # Case 3B: EHR cutoff = Revoke timestamp, Survey/Bio cutoff = Withdraw timestamp - df.loc[case_3b, "ehr_cutoff"] = df.loc[case_3b, "hipaa_revoked_ts"] - df.loc[case_3b, "survey_cutoff"] = df.loc[case_3b, "consent_withdrawn_ts"] - df.loc[case_3b, "bio_cutoff"] = df.loc[case_3b, "consent_withdrawn_ts"] - - # Case 4: No cutoffs, so leave all columns as NaT (as initialized previously). - # Participant is excluded downstream. No further code is needed here for this case. - - # Sort the dataframe for code development and debugging purposes. This can be removed in the future. - df = df.sort_values(by=["exclusion_reason", "anomaly", "case", "Connect_ID"]) - - # Sort dataframe in order of: - # 1. Undefined cases - # 2. Data quality exclusion cases - # 3. Cases with anomalies - # 4. Normal cases - df["sort_order"] = np.select( - [df["case"].eq("UNDEFINED"), df["case"].eq("DATA_QUALITY_EXCLUSION"), df["anomaly"].notna()], - [1, 2, 3], default=4 - ) - - return df - -def write_classification_to_bq(classification_df, bq_table_name, client) -> None: - """ - Write the classification DataFrame to a BigQuery table - Args: - classification_df (pd.DataFrame): DataFrame containing participant classifications. - bq_table_name (str): Fully qualified BigQuery table name (project.dataset.table). - client (bigquery.Client): BigQuery client for writing data. - """ - job_config = bigquery.LoadJobConfig( - schema=[ - # String columns - bigquery.SchemaField("Connect_ID", "STRING"), - bigquery.SchemaField("verified_status", "STRING"), - bigquery.SchemaField("verified_status_concept_id", "STRING"), - bigquery.SchemaField("consent_withdrawn", "STRING"), - bigquery.SchemaField("consent_withdrawn_concept_id", "STRING"), - bigquery.SchemaField("hipaa_revoked", "STRING"), - bigquery.SchemaField("hipaa_revoked_concept_id", "STRING"), - bigquery.SchemaField("data_destruction_requested", "STRING"), - bigquery.SchemaField("data_destruction_requested_concept_id", "STRING"), - - # Timestamp columns - bigquery.SchemaField("consent_withdrawn_ts", "TIMESTAMP"), - bigquery.SchemaField("hipaa_revoked_ts", "TIMESTAMP"), - bigquery.SchemaField("data_destruction_ts", "TIMESTAMP"), - bigquery.SchemaField("module1_complete_ts", "TIMESTAMP"), - bigquery.SchemaField("module2_complete_ts", "TIMESTAMP"), - bigquery.SchemaField("module3_complete_ts", "TIMESTAMP"), - bigquery.SchemaField("module4_complete_ts", "TIMESTAMP"), - #bigquery.SchemaField("bio_complete_ts", "TIMESTAMP"), - #bigquery.SchemaField("clinicalbio_complete_ts", "TIMESTAMP"), - #bigquery.SchemaField("mouthwash_complete_ts", "TIMESTAMP"), - bigquery.SchemaField("menstrual_complete_ts", "TIMESTAMP"), - bigquery.SchemaField("covid19_complete_ts", "TIMESTAMP"), - bigquery.SchemaField("experience2024_complete_ts", "TIMESTAMP"), - - # Classification fields - bigquery.SchemaField("case", "STRING"), - bigquery.SchemaField("exclusion_reason", "STRING"), - bigquery.SchemaField("anomaly", "STRING"), - - # Derived cutoffs - bigquery.SchemaField("ehr_cutoff", "TIMESTAMP"), - bigquery.SchemaField("survey_cutoff", "TIMESTAMP"), - bigquery.SchemaField("bio_cutoff", "TIMESTAMP"), - - # Helper field - bigquery.SchemaField("sort_order", "INTEGER"), - ], - write_disposition=bigquery.WriteDisposition.WRITE_TRUNCATE - ) - - # Load DataFrame to BigQuery - job = client.load_table_from_dataframe( - classification_df, - bq_table_name, - job_config=job_config - ) - job.result() # Wait for the job to complete - _, _, table_short_name = utils.parse_fq_table(bq_table_name) - def get_first_cid(column_name: str) -> str | None: cids = extract_ordered_concept_ids(column_name) return cids[0] if cids else None From 368751d0ef9fdd3210455036e6f0c9e6a7d28375 Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Fri, 17 Jul 2026 14:30:23 -0400 Subject: [PATCH 40/67] Add docstrings to classification module --- core/classification.py | 60 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 55 insertions(+), 5 deletions(-) diff --git a/core/classification.py b/core/classification.py index 9628ac5..a2e824c 100644 --- a/core/classification.py +++ b/core/classification.py @@ -6,6 +6,23 @@ from google.cloud import bigquery def read_parquet_to_dataframe(parquet_url: str) -> pd.DataFrame: + """ + Read participant status data from a parquet file and prepare it for + classification. + + The parquet file is filtered to include only verified participants. + Timestamp fields used during participant classification are normalized + to timezone-aware UTC datetimes with invalid or missing values coerced + to NaT. + + Args: + parquet_url (str): GCS or local path to the participant status + parquet file. + + Returns: + pd.DataFrame: DataFrame containing verified participants with + normalized timestamp columns. + """ # Only keep verified participants df = pd.read_parquet(parquet_url, engine="pyarrow", @@ -24,6 +41,26 @@ def read_parquet_to_dataframe(parquet_url: str) -> pd.DataFrame: return df def classify_participants(df: pd.DataFrame) -> pd.DataFrame: + """ + Classify participants into consent and data-retention cases. + + Evaluates each participant's HIPAA revocation, consent withdrawn, + and data destruction status to assign one of the supported business + rule cases (1, 2, 3A, 3B, or 4). Participants with invalid flag or + timestamp combinations are marked as data-quality exclusions or + undefined. + + The function also computes the appropriate EHR, survey, and + biospecimen cutoff timestamps along with any anomaly or exclusion + reason needed for downstream processing. + + Args: + df (pd.DataFrame): Participant status DataFrame. + + Returns: + pd.DataFrame: The input DataFrame with classification, cutoff, + anomaly, exclusion, and helper columns added. + """ # ========================================================================= # Boolean masks for conditions # Each variable becomes a True/False Series across the dataframe @@ -311,13 +348,26 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: return df -def write_classification_to_bq(classification_df, bq_table_name, client) -> None: +def write_classification_to_bq( + classification_df: pd.DataFrame, + bq_table_name: str, + client: bigquery.Client, +) -> None: """ - Write the classification DataFrame to a BigQuery table + Write participant classification results to a BigQuery table. + + Creates or replaces the destination table using the expected schema + for participant classification output. + Args: - classification_df (pd.DataFrame): DataFrame containing participant classifications. - bq_table_name (str): Fully qualified BigQuery table name (project.dataset.table). - client (bigquery.Client): BigQuery client for writing data. + classification_df (pd.DataFrame): DataFrame containing participant + classifications and derived cutoff fields. + bq_table_name (str): Fully qualified BigQuery table name + (project.dataset.table). + client (bigquery.Client): BigQuery client used to load the table. + + Returns: + None """ job_config = bigquery.LoadJobConfig( schema=[ From 8f2c49990e39f3d8da1e17a526d2a634a2719cfa Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Fri, 17 Jul 2026 15:15:24 -0400 Subject: [PATCH 41/67] Route classification calls through classification module --- core/transformations.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/transformations.py b/core/transformations.py index d53c03b..fe9d95c 100644 --- a/core/transformations.py +++ b/core/transformations.py @@ -12,7 +12,7 @@ if __name__ == "__main__": sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from core import constants, utils, transform_renderer +from core import constants, utils, transform_renderer, classification ######################################################################## ############# Table-level Transformations ############################# @@ -962,7 +962,7 @@ def create_classification_table( # Read the parquet file into a DataFrame try: utils.logger.info(f"[{table_short_name}] Reading parquet from {parquet_url}") - classification_df = utils.read_parquet_to_dataframe(parquet_url) + classification_df = classification.read_parquet_to_dataframe(parquet_url) utils.logger.info(f"[{table_short_name}] Read {len(classification_df)} verified participant rows") except Exception as e: utils.logger.exception(f"[{table_short_name}] Error reading parquet from {parquet_url}: {e}") @@ -971,7 +971,7 @@ def create_classification_table( # Classify participants into cases try: utils.logger.info(f"[{table_short_name}] Classifying participants...") - classification_df = utils.classify_participants(classification_df) + classification_df = classification.classify_participants(classification_df) utils.logger.info(f"[{table_short_name}] Classification complete") except Exception as e: utils.logger.exception(f"[{table_short_name}] Error classifying participants: {e}") @@ -989,7 +989,7 @@ def create_classification_table( # Write the classification DataFrame to BigQuery try: utils.logger.info(f"[{table_short_name}] Writing classification to BigQuery...") - utils.write_classification_to_bq(classification_df, classification_table, client) + classification.write_classification_to_bq(classification_df, classification_table, client) status = f"[{table_short_name}] Classification table successfully created at {classification_table}" utils.logger.info(status) except Exception as e: From 97d4fbeb05f4ce43881ab217927800eab2aa5164 Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Fri, 17 Jul 2026 16:02:52 -0400 Subject: [PATCH 42/67] Move destination table building from utils.py to destination_table_builder and update callers --- core/destination_table_builder.py | 684 ++++++++++++++++++++++++++++++ core/transformations.py | 14 +- 2 files changed, 691 insertions(+), 7 deletions(-) create mode 100644 core/destination_table_builder.py diff --git a/core/destination_table_builder.py b/core/destination_table_builder.py new file mode 100644 index 0000000..b441aae --- /dev/null +++ b/core/destination_table_builder.py @@ -0,0 +1,684 @@ +"""Utilities for building destination tables from configuration.""" + +import json +from core import constants, utils +from typing import Any, Optional +from datetime import datetime, timezone + +from google.cloud import bigquery, storage + + + +def load_destination_config(config_file_path: str) -> dict: + """ + Load the destination configuration JSON file. + + This configuration defines destination table mappings and related settings + used to generate SQL queries. + + Args: + config_file_path (str): Path to the destination configuration JSON file. + + Returns: + dict: Parsed configuration dictionary. Returns an empty dict if loading fails. + """ + try: + with open(config_file_path, 'r') as f: + return json.load(f) + except Exception as e: + utils.logger.error(f"Error loading destination configuration from {config_file_path}: {e}") + return {} + +def get_destination_table_config( + destination_config: dict[str, Any], + destination_dataset: str, + destination_table: str, +) -> dict[str, Any] | None: + """ + Retrieve the configuration for a specific destination table. + + Searches the configuration for a destination table matching the given dataset + and table name. + + Args: + destination_config (dict): Full destination table configuration. + destination_dataset (str): Target dataset name. + destination_table (str): Target table name. + + Returns: + dict | None: Matching destination table configuration or None if not found. + """ + # Loop through each destination table defined in the config + for destination in destination_config.get("destination_tables", []): + dataset_name = destination.get("dataset") + table_name = destination.get("table") + + # Check if both dataset and table match the requested ones + if dataset_name == destination_dataset and table_name == destination_table: + # Return the matching configuration immediately + return destination + + # If no match was found after checking all entries, return None + return None + +def get_fq_base_table( + client: bigquery.Client, + destination_table_config: dict[str, Any] +) -> str: + """ + Construct the fully qualified BigQuery base table name. + + Uses the base_table configuration and the current project to build + a fully qualified table identifier. + + Args: + client (bigquery.Client): BigQuery client with project context. + destination_table_config (dict): Destination table configuration. + + Returns: + str: Fully qualified table name in the format project.dataset.table. + + Raises: + ValueError: If base_table config is missing or malformed. + """ + project = client.project + + # Base table config must exist and be a dict + base_table_config = destination_table_config["base_table"] + + if not isinstance(base_table_config, dict): + raise ValueError( + f"'base_table' must be an object for " + f"{destination_table_config['dataset']}.{destination_table_config['table']}" + ) + + # Validate required keys exist + if "dataset" not in base_table_config or "table" not in base_table_config: + raise ValueError( + f"'base_table' must include 'dataset' and 'table' for " + f"{destination_table_config['dataset']}.{destination_table_config['table']}" + ) + + # Extract dataset and table + dataset = base_table_config["dataset"] + table = base_table_config["table"] + + # Return fully qualified table name + return f"{project}.{dataset}.{table}" + +def build_table_schemas( + client: bigquery.Client, + destination_table_config: dict[str, Any], +) -> tuple[dict[tuple[str, str], list[str]], list[dict[str, Any]]]: + """ + Fetch column schemas for all tables used in a destination configuration. + + Retrieves column names for the base table and all join tables, storing them + in a lookup dictionary for later validation and SQL generation. + + Args: + client (bigquery.Client): BigQuery client. + destination_table_config (dict): Destination table configuration. + + Returns: + tuple: + - dict[(dataset, table), list[str]]: Table schemas. + - list[dict]: Schema retrieval issues (if any). + """ + # Stores schemas keyed by (dataset, table) + table_schemas: dict[tuple[str, str], list[str]] = {} + + # Collect schema fetch failures for reporting + schema_issues: list[dict[str, Any]] = [] + + def _fetch_schema(dataset: str, table: str): + key = (dataset, table) + # Skip if already fetched (avoid duplicate queries) + if key in table_schemas: + return + + fq_table = f"{client.project}.{dataset}.{table}" + try: + # Retrieve column names from BigQuery + table_schemas[key] = utils.get_column_names(client, fq_table) + except Exception as e: + utils.logger.error(f"Error fetching schema for {fq_table}: {e}") + # Store empty schema so downstream logic does not break + table_schemas[key] = [] + # Track issue for reporting + schema_issues.append({ + "dataset": dataset, + "table": table, + "error": str(e), + }) + + # Always fetch base table schema first + base_table_config = destination_table_config["base_table"] + _fetch_schema(base_table_config["dataset"], base_table_config["table"]) + + # Fetch schemas for all join tables + for join_table in destination_table_config.get("join_tables", []): + _fetch_schema(join_table["dataset"], join_table["table"]) + + return table_schemas, schema_issues + +def find_missing_columns( + destination_table_config: dict[str, Any], + table_schemas: dict[tuple[str, str], list[str]] +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + """ + Identify columns specified in config that do not exist in source tables. + + Checks both base table and join tables against fetched schemas. + + Args: + destination_table_config (dict): Destination table configuration. + table_schemas (dict): Mapping of (dataset, table) to column names. + + Returns: + tuple: + - list[dict]: Missing standard columns. + - list[dict]: Missing loop variable columns. + """ + missing_cleaned_cols = {} + missing_cleaned_loop_vars = {} + + def _check_cols(config): + dataset = config["dataset"] + table = config["table"] + schema = set(table_schemas.get((dataset, table), [])) + cols = config.get("columns", []) + + # Skip wildcard or empty (means select all or nothing specified) + if cols == "*" or not cols: + return + + if not isinstance(cols, list): + raise ValueError(f"Invalid columns for {dataset}.{table}") + + for col in cols: + # Skip valid columns + if col in schema: + continue + + key = (dataset, table, col) + + # Separate loop variables versus standard columns + if is_cleaned_loop_variable(col): + missing_cleaned_loop_vars[key] = missing_cleaned_loop_vars.get(key, 0) + 1 + else: + missing_cleaned_cols[key] = missing_cleaned_cols.get(key, 0) + 1 + + # Check base table + _check_cols(destination_table_config["base_table"]) + + # Check join tables + for join_table in destination_table_config.get("join_tables", []): + _check_cols(join_table) + + def _format_output(missing_columns): + return [ + { + "dataset": dataset, + "table": table, + "source_col": source_col, + "occurrence_count": occurrence_count + } + for (dataset, table, source_col), occurrence_count in missing_columns.items() + ] + + return _format_output(missing_cleaned_cols), _format_output(missing_cleaned_loop_vars) + +def build_row_filter_sql( + filter_profiles: dict[str, Any], + destination_table_config: dict[str, Any] +) -> str: + """ + Generate a SQL WHERE clause from the destination table's filter_profile. + + Looks up the filter_profile named in destination_table_config, resolves + it against filter_profiles, and builds a WHERE clause from its column/ + value conditions. This is only one of two ways a table's rows can end + up restricted. build_destination_table_query separately adds a + join-key-not-null condition and (if allowed_cases is set) a + classification-case filter. A table using the "no_filters" profile can + still be row-restricted by those other conditions; this function only + reports on the conditions it itself contributes. + + Args: + filter_profiles (dict): Mapping of filter profile names to filter rules. + destination_table_config (dict): Destination table configuration. + + Returns: + str: SQL WHERE clause string or an empty string if the resolved + filter_profile has no conditions. + + Raises: + ValueError: If filter_profile is missing from destination_table_config + or if the named profile is not found in filter_profiles. + """ + # Get selected filter profile + filter_profile = destination_table_config.get("filter_profile") + if not filter_profile: + raise ValueError( + f"Missing filter_profile for " + f"{destination_table_config['dataset']}.{destination_table_config['table']}" + ) + # Identify filters for the selected table + filters = filter_profiles.get(filter_profile) + + # Catch invalid filter profile names + if filters is None: + raise ValueError(f"Unknown filter_profile: {filter_profile}") + + # Handle "no_filters" case (empty list) + if not filters: + utils.logger.info( + f"[{destination_table_config['table']}] " + f"filter_profile '{filter_profile}' has no conditions; " + f"no row filter SQL generated here" + ) + return "" + + clauses = [] + + # Build SQL conditions for each filter + for f in filters: + col = f["column"] + val = f["value"] + + # Default to "=" operator unless specified + op = f.get("operator", "=") + + clause = f"{col} {op} '{val}'" + + # Append optional comment + if "comment" in f: + clause += f" -- {f['comment']}" + + clauses.append(clause) + + # Join conditions with AND + return "WHERE " + "\n AND ".join(clauses) + +def build_unique_eligibility_rules(module_censor_rules: dict[str, dict]) -> dict[str, dict]: + """ + Deduplicates MODULE_CENSOR_RULES by status_col, so that CIDs which share + the same gating condition (e.g. the two menstrual survey CIDs, which both + gate on menstrual_status/menstrual_complete_ts/survey_cutoff) collapse to + a single rule instead of being treated as independent conditions. + + Raises if two different CIDs claim the same status_col but disagree on + completion_ts or cutoff — that would mean two columns silently computing + different "eligibility" under the same flag name, which is a data + integrity risk given this logic gates consent/censorship-sensitive data. + + Args: + module_censor_rules: constants.MODULE_CENSOR_RULES + + Returns: + dict: {status_col: rule} with one entry per unique gating condition. + + Raises: + ValueError: If two CIDs share a status_col but have conflicting + completion_ts or cutoff values. + """ + unique_rules: dict[str, dict] = {} + signatures: dict[str, tuple] = {} + + for cid, rule in module_censor_rules.items(): + status_col = rule["status_col"] + sig = (rule["status_col"], rule["completion_ts"], rule["cutoff"]) + + if status_col in signatures and signatures[status_col] != sig: + raise ValueError( + f"Conflicting MODULE_CENSOR_RULES entries share status_col=" + f"{status_col!r} but disagree on completion_ts/cutoff: " + f"{signatures[status_col]} vs {sig} (conflict introduced by CID {cid})" + ) + + signatures[status_col] = sig + unique_rules[status_col] = rule + + return unique_rules + +def render_eligibility_condition( + status_col: str, + completion_ts: str, + cutoff: str, + alias: str = "c", +) -> str: + """ + The single source of truth for "is {status_col} eligible" — status is + Submitted, and for 3A/3B cases, completion happened strictly before + cutoff. Used both to build the *_eligible flags in + build_destination_table_query's CTE and to determine the + not-censored/censored branch in _build_reason_cte_block, so the real + table and the censorship summary can never compute this condition + differently from each other. + + Args: + status_col: The status column name (e.g. "module2_status"). + completion_ts: The completion timestamp column name. + cutoff: The cutoff timestamp column name. + alias: Table alias to qualify all column references with. + + Returns: + str: A parenthesized boolean SQL expression (no AS alias). + """ + return f"""( + {alias}.{status_col} = 'Submitted' + AND ( + {alias}.`case` NOT IN ('3A', '3B') + OR ( + {alias}.{completion_ts} IS NOT NULL + AND {alias}.{cutoff} IS NOT NULL + AND {alias}.{completion_ts} < {alias}.{cutoff} + ) + ) + )""" + +def build_destination_table_query( + client: bigquery.Client, + full_config: dict[str, Any], + destination_table_config: dict[str, Any], + table_schemas: dict[tuple[str, str], list[str]], + classification_table: Optional[str] = None +) -> dict[str, Any]: + """ + Build a CREATE OR REPLACE TABLE SQL query based on destination configuration. + + Combines base table, optional filters, optional classification filter, + and join tables into a single SQL query. + + Args: + client (bigquery.Client): BigQuery client. + full_config (dict): Full configuration including filter profiles. + destination_table_config (dict): Destination table configuration. + table_schemas (dict): Table schema lookup. + classification_table (str, optional): Fully qualified classification table name + + Returns: + dict: Contains destination table name and generated SQL string. + + Raises: + ValueError: If no columns are selected or if classification_table is missing + when allowed_cases is populated. + """ + project = client.project + destination_dataset = destination_table_config["dataset"] + destination_table = destination_table_config["table"] + # Default join key if not specified + join_key = destination_table_config.get("join_key", "Connect_ID") + + # Check for classification filter + classification_filter = destination_table_config.get("classification_filter") + allowed_cases = classification_filter.get("allowed_cases", []) if classification_filter else [] + + # Build WHERE clause from filter profile + filter_sql = build_row_filter_sql( + full_config.get("filter_profiles", {}), + destination_table_config + ) + + # Base table + base_config = destination_table_config["base_table"] + fq_base_table = get_fq_base_table(client, destination_table_config) + base_table = base_config["table"] + + # CTE name and alias + cte_name = f"filtered_{base_table}" + base_alias = base_table + classification_alias = base_alias + + # Build CTE WHERE clause + # Combines base constraint, filter profile, and classification filter + + # Add classification filter condition if present + if allowed_cases: + where_conditions = [f"p.{join_key} IS NOT NULL"] # Base constraint to ensure join key is not null + cases_str = ", ".join([f"'{c}'" for c in allowed_cases]) + where_conditions.append(f"c.`case` IN ({cases_str})") + else: + where_conditions = [f"{join_key} IS NOT NULL"] + + # Add raw column filter conditions if present + if filter_sql: + # Remove leading WHERE from filter_sql safely + filter_body = filter_sql.replace("WHERE", "", 1).strip() + where_conditions.append(filter_body) + + final_where = "WHERE " + "\n AND ".join(where_conditions) + + # Build eligibility flag columns for the CTE + # One boolean per unique gating rule (deduped by status_col) + eligibility_flags_sql = "" + if allowed_cases: + unique_rules = build_unique_eligibility_rules(constants.MODULE_CENSOR_RULES) + if unique_rules: + flag_lines = [] + for status_col, rule in unique_rules.items(): + completion_ts = rule["completion_ts"] + cutoff = rule["cutoff"] + eligible_expr = render_eligibility_condition( + status_col, completion_ts, cutoff, alias="c" + ) + flag_lines.append(f" {eligible_expr} AS {status_col}_eligible") + eligibility_flags_sql = ",\n" + ",\n".join(flag_lines) + + # Build CTE with or without classification join + if allowed_cases: + if not classification_table: + raise ValueError( + "classification_table must be provided when allowed_cases is populated." + ) + cte = f""" +{cte_name} AS ( + SELECT + p.*, + c.* EXCEPT({join_key}){eligibility_flags_sql} + FROM `{fq_base_table}` p + INNER JOIN `{classification_table}` c + ON p.{join_key} = c.{join_key} + {final_where} +)""".strip() + else: + cte = f""" +{cte_name} AS ( + SELECT * + FROM `{fq_base_table}` + {final_where} +)""".strip() + + select_parts = [] + join_parts = [] + + selected_column_names = set() + duplicate_columns = [] + unrecognized_cid_columns = [] + + # Always include join key first from base table + select_parts.append(f"{base_alias}.{join_key}") + selected_column_names.add(join_key) + + # Base table columns + base_cols = base_config.get("columns", []) + + # Lookup schema for validation + base_schema = set(table_schemas.get((base_config["dataset"], base_config["table"]), [])) + + if base_cols == "*": + base_cols = sorted(base_schema) + + for col in base_cols: + if col in base_schema: + if col == join_key: + continue # Skip join key since it is already included + + # If column has already been selected (e.g. from a previous table), + # track it as a duplicate and skip to avoid ambiguity + if col in selected_column_names: + duplicate_columns.append(col) + continue + + # Flag unrecognized CIDs before/alongside rendering + if utils.is_unrecognized_censorship_cid(col): + unrecognized_cid_columns.append({ + "dataset": base_config["dataset"], + "table": base_config["table"], + "column": col + }) + + select_parts.append( + utils.render_censored_column_expression( + source_alias=base_alias, + col_name=col, + classification_alias=classification_alias, + ) + ) + selected_column_names.add(col) + + # Join tables + for join_table in destination_table_config.get("join_tables", []): + dataset = join_table["dataset"] + table = join_table["table"] + + join_schema = set(table_schemas.get((dataset, table), [])) + cols = join_table.get("columns", []) + + if cols == "*": + cols = sorted(join_schema) + + if not cols: + continue + + # Add LEFT JOIN + join_parts.append( + f"LEFT JOIN `{project}.{dataset}.{table}` {table} " + f"ON {table}.{join_key} = {base_alias}.{join_key}" + ) + + # Add selected columns + for col in cols: + if col in join_schema: + if col == join_key: + continue # Skip join key since it is already included + + # If column has already been selected (e.g. from a previous table), + # track it as a duplicate and skip to avoid ambiguity + if col in selected_column_names: + duplicate_columns.append(f"{dataset}.{table}.{col}") + continue + + # Flag unrecognized CIDs for join-table columns too + if is_unrecognized_censorship_cid(col): + unrecognized_cid_columns.append({ + "dataset": dataset, + "table": table, + "column": col + }) + + select_parts.append( + utils.render_censored_column_expression( + source_alias=table, + col_name=col, + classification_alias=classification_alias, + ) +) + selected_column_names.add(col) + + # Log duplicate columns (if any) + if duplicate_columns: + unique_dupes = sorted(set(duplicate_columns)) + utils.logger.warning( + f"[{destination_table}] Duplicate columns detected and skipped: " + f"{', '.join(unique_dupes)}. Using first occurrence only." + ) + + # Log unrecognized CID columns (if any) - same pattern as duplicate_columns logging + if unrecognized_cid_columns: + unrecognized_names = sorted({c["column"] for c in unrecognized_cid_columns}) + utils.logger.warning( + f"[{destination_table}] Columns with unrecognized CIDs detected " + f"(will be set to NULL by fail-closed default): {', '.join(unrecognized_names)}. " + f"Please verify these CIDs are valid and update MODULE_CENSOR_RULES if necessary." + ) + + # Ensure at least one column is selected + if not select_parts: + raise ValueError(f"No columns selected for {destination_table}") + + select_sql = ",\n ".join(select_parts) + join_sql = "\n".join(join_parts) + + sql = f""" +CREATE OR REPLACE TABLE `{project}.{destination_dataset}.{destination_table}` AS +WITH +{cte} +SELECT + {select_sql} +FROM {cte_name} {base_alias} +{join_sql} +""".strip() + + return { + "destination_table": f"{project}.{destination_dataset}.{destination_table}", + "sql": sql, + "duplicate_columns": sorted(set(duplicate_columns)), + "unrecognized_cid_columns": unrecognized_cid_columns + } + +def create_destination_missing_columns_json( + client: storage.Client, + output_path: str, + destination_table: str, + missing_cleaned_cols: list[dict[str, Any]], + missing_cleaned_loop_vars: list[dict[str, Any]], + duplicate_columns: list[str], + unrecognized_cid_columns: list[dict[str, Any]] +) -> None: + """ + Write a JSON report of missing columns to a GCS location. + + Args: + client (storage.Client): GCS client. + output_path (str): GCS path (gs://...) to write the report. + destination_table (str): Destination table name. + missing_cleaned_cols (list): Missing standard columns. + missing_cleaned_loop_vars (list): Missing loop variable columns. + duplicate_columns (list): List of duplicate column names. + unrecognized_cid_columns (list): List of columns with unrecognized CIDs. + """ + # Build report structure + report = { + "_metadata": { + "generated_at": datetime.now(timezone.utc).isoformat(), + "source": "pr2-transformation pipeline", + "destination_table": destination_table, + "description": "Report of columns requested by the destination config that were not found in their respective source tables", + "structure": { + "missing_cleaned_cols": "Columns that were not found and are NOT cleaned loop variables — these are unexpected and should be investigated", + "missing_cleaned_loop_vars": "Columns that were not found but ARE cleaned loop variables — these may be expected if the loop variable does not exist for a given table", + "duplicate_columns": "Columns skipped due to duplicate names in SELECT", + "unrecognized_cid_columns": "Columns whose first CID matched neither MODULE_CENSOR_RULES nor NON_CENSORED_SECONDARY_SOURCE_CIDS. These were set to NULL by the fail-closed default and likely indicate a naming mistake, a missing MODULE_CENSOR_RULES entry, or a column that does not belong in this table", + "occurrence_count": "Number of times this column appeared in the config AND was missing" + } + }, + "missing_cleaned_cols": missing_cleaned_cols, + "missing_cleaned_loop_vars": missing_cleaned_loop_vars, + "duplicate_columns": duplicate_columns, + "unrecognized_cid_columns": unrecognized_cid_columns + } + + # Parse GCS path + path = output_path.removeprefix("gs://") + bucket_name, blob_path = path.split("/", 1) + + bucket = client.bucket(bucket_name) + blob = bucket.blob(blob_path) + + # Upload JSON report + blob.upload_from_string( + json.dumps(report, indent=2, ensure_ascii=False), + content_type="application/json", + ) \ No newline at end of file diff --git a/core/transformations.py b/core/transformations.py index fe9d95c..bfe2442 100644 --- a/core/transformations.py +++ b/core/transformations.py @@ -12,7 +12,7 @@ if __name__ == "__main__": sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from core import constants, utils, transform_renderer, classification +from core import constants, utils, transform_renderer, classification, destination_table_builder ######################################################################## ############# Table-level Transformations ############################# @@ -1042,11 +1042,11 @@ def create_destination_table( # Load destination table configuration utils.logger.info(f"[{destination_table}] Loading destination table configuration...") - destination_config = utils.load_destination_config(config_path) + destination_config = destination_table_builder.load_destination_config(config_path) # Retrieve the target destination table config utils.logger.info(f"[{destination_table}] Retrieving destination table configuration...") - destination_table_config = utils.get_destination_table_config( + destination_table_config = destination_table_builder.get_destination_table_config( destination_config, destination_dataset, destination_table @@ -1058,7 +1058,7 @@ def create_destination_table( ) # Build table schemas - table_schemas, schema_issues = utils.build_table_schemas( + table_schemas, schema_issues = destination_table_builder.build_table_schemas( client, destination_table_config ) @@ -1067,13 +1067,13 @@ def create_destination_table( utils.logger.warning(f"[{destination_table}] Schema issues detected: {schema_issues}") # Identify missing columns - missing_cleaned_cols, missing_cleaned_loop_vars = utils.find_missing_columns( + missing_cleaned_cols, missing_cleaned_loop_vars = destination_table_builder.find_missing_columns( destination_table_config, table_schemas ) # Generate SQL - result = utils.build_destination_table_query( + result = destination_table_builder.build_destination_table_query( client=client, full_config=destination_config, destination_table_config=destination_table_config, @@ -1116,7 +1116,7 @@ def create_destination_table( try: report_path = f"{missing_report_base_path}{fq_destination_table}_missing_columns.json" - utils.create_destination_missing_columns_json( + destination_table_builder.create_destination_missing_columns_json( client=gcs_client, output_path=report_path, destination_table=fq_destination_table, From a59ecfa88fb5b4e11c67eafbb98e7d6ab88531ad Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Fri, 17 Jul 2026 16:54:13 -0400 Subject: [PATCH 43/67] Move functions into respective modules (destination_table_builder.py, censorship.py) --- core/utils.py | 1009 +------------------------------------------------ 1 file changed, 1 insertion(+), 1008 deletions(-) diff --git a/core/utils.py b/core/utils.py index 8cb3726..659e585 100644 --- a/core/utils.py +++ b/core/utils.py @@ -1013,1011 +1013,4 @@ def create_missing_columns_json( # Serialize the report to JSON and upload to GCS blob.upload_from_string(json.dumps(report, indent=2, - ensure_ascii=False), content_type="application/json") - -def load_destination_config(config_file_path: str) -> dict: - """ - Load the destination configuration JSON file. - - This configuration defines destination table mappings and related settings - used to generate SQL queries. - - Args: - config_file_path (str): Path to the destination configuration JSON file. - - Returns: - dict: Parsed configuration dictionary. Returns an empty dict if loading fails. - """ - try: - with open(config_file_path, 'r') as f: - return json.load(f) - except Exception as e: - utils.logger.error(f"Error loading destination configuration from {config_file_path}: {e}") - return {} - -def get_destination_table_config( - destination_config: dict[str, Any], - destination_dataset: str, - destination_table: str, -) -> dict[str, Any] | None: - """ - Retrieve the configuration for a specific destination table. - - Searches the configuration for a destination table matching the given dataset - and table name. - - Args: - destination_config (dict): Full destination table configuration. - destination_dataset (str): Target dataset name. - destination_table (str): Target table name. - - Returns: - dict | None: Matching destination table configuration or None if not found. - """ - # Loop through each destination table defined in the config - for destination in destination_config.get("destination_tables", []): - dataset_name = destination.get("dataset") - table_name = destination.get("table") - - # Check if both dataset and table match the requested ones - if dataset_name == destination_dataset and table_name == destination_table: - # Return the matching configuration immediately - return destination - - # If no match was found after checking all entries, return None - return None - -def get_fq_base_table( - client: bigquery.Client, - destination_table_config: dict[str, Any] -) -> str: - """ - Construct the fully qualified BigQuery base table name. - - Uses the base_table configuration and the current project to build - a fully qualified table identifier. - - Args: - client (bigquery.Client): BigQuery client with project context. - destination_table_config (dict): Destination table configuration. - - Returns: - str: Fully qualified table name in the format project.dataset.table. - - Raises: - ValueError: If base_table config is missing or malformed. - """ - project = client.project - - # Base table config must exist and be a dict - base_table_config = destination_table_config["base_table"] - - if not isinstance(base_table_config, dict): - raise ValueError( - f"'base_table' must be an object for " - f"{destination_table_config['dataset']}.{destination_table_config['table']}" - ) - - # Validate required keys exist - if "dataset" not in base_table_config or "table" not in base_table_config: - raise ValueError( - f"'base_table' must include 'dataset' and 'table' for " - f"{destination_table_config['dataset']}.{destination_table_config['table']}" - ) - - # Extract dataset and table - dataset = base_table_config["dataset"] - table = base_table_config["table"] - - # Return fully qualified table name - return f"{project}.{dataset}.{table}" - -def build_table_schemas( - client: bigquery.Client, - destination_table_config: dict[str, Any], -) -> tuple[dict[tuple[str, str], list[str]], list[dict[str, Any]]]: - """ - Fetch column schemas for all tables used in a destination configuration. - - Retrieves column names for the base table and all join tables, storing them - in a lookup dictionary for later validation and SQL generation. - - Args: - client (bigquery.Client): BigQuery client. - destination_table_config (dict): Destination table configuration. - - Returns: - tuple: - - dict[(dataset, table), list[str]]: Table schemas. - - list[dict]: Schema retrieval issues (if any). - """ - # Stores schemas keyed by (dataset, table) - table_schemas: dict[tuple[str, str], list[str]] = {} - - # Collect schema fetch failures for reporting - schema_issues: list[dict[str, Any]] = [] - - def _fetch_schema(dataset: str, table: str): - key = (dataset, table) - # Skip if already fetched (avoid duplicate queries) - if key in table_schemas: - return - - fq_table = f"{client.project}.{dataset}.{table}" - try: - # Retrieve column names from BigQuery - table_schemas[key] = get_column_names(client, fq_table) - except Exception as e: - utils.logger.error(f"Error fetching schema for {fq_table}: {e}") - # Store empty schema so downstream logic does not break - table_schemas[key] = [] - # Track issue for reporting - schema_issues.append({ - "dataset": dataset, - "table": table, - "error": str(e), - }) - - # Always fetch base table schema first - base_table_config = destination_table_config["base_table"] - _fetch_schema(base_table_config["dataset"], base_table_config["table"]) - - # Fetch schemas for all join tables - for join_table in destination_table_config.get("join_tables", []): - _fetch_schema(join_table["dataset"], join_table["table"]) - - return table_schemas, schema_issues - -def find_missing_columns( - destination_table_config: dict[str, Any], - table_schemas: dict[tuple[str, str], list[str]] -) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: - """ - Identify columns specified in config that do not exist in source tables. - - Checks both base table and join tables against fetched schemas. - - Args: - destination_table_config (dict): Destination table configuration. - table_schemas (dict): Mapping of (dataset, table) to column names. - - Returns: - tuple: - - list[dict]: Missing standard columns. - - list[dict]: Missing loop variable columns. - """ - missing_cleaned_cols = {} - missing_cleaned_loop_vars = {} - - def _check_cols(config): - dataset = config["dataset"] - table = config["table"] - schema = set(table_schemas.get((dataset, table), [])) - cols = config.get("columns", []) - - # Skip wildcard or empty (means select all or nothing specified) - if cols == "*" or not cols: - return - - if not isinstance(cols, list): - raise ValueError(f"Invalid columns for {dataset}.{table}") - - for col in cols: - # Skip valid columns - if col in schema: - continue - - key = (dataset, table, col) - - # Separate loop variables versus standard columns - if is_cleaned_loop_variable(col): - missing_cleaned_loop_vars[key] = missing_cleaned_loop_vars.get(key, 0) + 1 - else: - missing_cleaned_cols[key] = missing_cleaned_cols.get(key, 0) + 1 - - # Check base table - _check_cols(destination_table_config["base_table"]) - - # Check join tables - for join_table in destination_table_config.get("join_tables", []): - _check_cols(join_table) - - def _format_output(missing_columns): - return [ - { - "dataset": dataset, - "table": table, - "source_col": source_col, - "occurrence_count": occurrence_count - } - for (dataset, table, source_col), occurrence_count in missing_columns.items() - ] - - return _format_output(missing_cleaned_cols), _format_output(missing_cleaned_loop_vars) - -def build_row_filter_sql( - filter_profiles: dict[str, Any], - destination_table_config: dict[str, Any] -) -> str: - """ - Generate a SQL WHERE clause from the destination table's filter_profile. - - Looks up the filter_profile named in destination_table_config, resolves - it against filter_profiles, and builds a WHERE clause from its column/ - value conditions. This is only one of two ways a table's rows can end - up restricted. build_destination_table_query separately adds a - join-key-not-null condition and (if allowed_cases is set) a - classification-case filter. A table using the "no_filters" profile can - still be row-restricted by those other conditions; this function only - reports on the conditions it itself contributes. - - Args: - filter_profiles (dict): Mapping of filter profile names to filter rules. - destination_table_config (dict): Destination table configuration. - - Returns: - str: SQL WHERE clause string or an empty string if the resolved - filter_profile has no conditions. - - Raises: - ValueError: If filter_profile is missing from destination_table_config - or if the named profile is not found in filter_profiles. - """ - # Get selected filter profile - filter_profile = destination_table_config.get("filter_profile") - if not filter_profile: - raise ValueError( - f"Missing filter_profile for " - f"{destination_table_config['dataset']}.{destination_table_config['table']}" - ) - # Identify filters for the selected table - filters = filter_profiles.get(filter_profile) - - # Catch invalid filter profile names - if filters is None: - raise ValueError(f"Unknown filter_profile: {filter_profile}") - - # Handle "no_filters" case (empty list) - if not filters: - utils.logger.info( - f"[{destination_table_config['table']}] " - f"filter_profile '{filter_profile}' has no conditions; " - f"no row filter SQL generated here" - ) - return "" - - clauses = [] - - # Build SQL conditions for each filter - for f in filters: - col = f["column"] - val = f["value"] - - # Default to "=" operator unless specified - op = f.get("operator", "=") - - clause = f"{col} {op} '{val}'" - - # Append optional comment - if "comment" in f: - clause += f" -- {f['comment']}" - - clauses.append(clause) - - # Join conditions with AND - return "WHERE " + "\n AND ".join(clauses) - -def build_unique_eligibility_rules(module_censor_rules: dict[str, dict]) -> dict[str, dict]: - """ - Deduplicates MODULE_CENSOR_RULES by status_col, so that CIDs which share - the same gating condition (e.g. the two menstrual survey CIDs, which both - gate on menstrual_status/menstrual_complete_ts/survey_cutoff) collapse to - a single rule instead of being treated as independent conditions. - - Raises if two different CIDs claim the same status_col but disagree on - completion_ts or cutoff — that would mean two columns silently computing - different "eligibility" under the same flag name, which is a data - integrity risk given this logic gates consent/censorship-sensitive data. - - Args: - module_censor_rules: constants.MODULE_CENSOR_RULES - - Returns: - dict: {status_col: rule} with one entry per unique gating condition. - - Raises: - ValueError: If two CIDs share a status_col but have conflicting - completion_ts or cutoff values. - """ - unique_rules: dict[str, dict] = {} - signatures: dict[str, tuple] = {} - - for cid, rule in module_censor_rules.items(): - status_col = rule["status_col"] - sig = (rule["status_col"], rule["completion_ts"], rule["cutoff"]) - - if status_col in signatures and signatures[status_col] != sig: - raise ValueError( - f"Conflicting MODULE_CENSOR_RULES entries share status_col=" - f"{status_col!r} but disagree on completion_ts/cutoff: " - f"{signatures[status_col]} vs {sig} (conflict introduced by CID {cid})" - ) - - signatures[status_col] = sig - unique_rules[status_col] = rule - - return unique_rules - -def render_eligibility_condition( - status_col: str, - completion_ts: str, - cutoff: str, - alias: str = "c", -) -> str: - """ - The single source of truth for "is {status_col} eligible" — status is - Submitted, and for 3A/3B cases, completion happened strictly before - cutoff. Used both to build the *_eligible flags in - build_destination_table_query's CTE and to determine the - not-censored/censored branch in _build_reason_cte_block, so the real - table and the censorship summary can never compute this condition - differently from each other. - - Args: - status_col: The status column name (e.g. "module2_status"). - completion_ts: The completion timestamp column name. - cutoff: The cutoff timestamp column name. - alias: Table alias to qualify all column references with. - - Returns: - str: A parenthesized boolean SQL expression (no AS alias). - """ - return f"""( - {alias}.{status_col} = 'Submitted' - AND ( - {alias}.`case` NOT IN ('3A', '3B') - OR ( - {alias}.{completion_ts} IS NOT NULL - AND {alias}.{cutoff} IS NOT NULL - AND {alias}.{completion_ts} < {alias}.{cutoff} - ) - ) - )""" - -def build_destination_table_query( - client: bigquery.Client, - full_config: dict[str, Any], - destination_table_config: dict[str, Any], - table_schemas: dict[tuple[str, str], list[str]], - classification_table: Optional[str] = None -) -> dict[str, Any]: - """ - Build a CREATE OR REPLACE TABLE SQL query based on destination configuration. - - Combines base table, optional filters, optional classification filter, - and join tables into a single SQL query. - - Args: - client (bigquery.Client): BigQuery client. - full_config (dict): Full configuration including filter profiles. - destination_table_config (dict): Destination table configuration. - table_schemas (dict): Table schema lookup. - classification_table (str, optional): Fully qualified classification table name - - Returns: - dict: Contains destination table name and generated SQL string. - - Raises: - ValueError: If no columns are selected or if classification_table is missing - when allowed_cases is populated. - """ - project = client.project - destination_dataset = destination_table_config["dataset"] - destination_table = destination_table_config["table"] - # Default join key if not specified - join_key = destination_table_config.get("join_key", "Connect_ID") - - # Check for classification filter - classification_filter = destination_table_config.get("classification_filter") - allowed_cases = classification_filter.get("allowed_cases", []) if classification_filter else [] - - # Build WHERE clause from filter profile - filter_sql = build_row_filter_sql( - full_config.get("filter_profiles", {}), - destination_table_config - ) - - # Base table - base_config = destination_table_config["base_table"] - fq_base_table = get_fq_base_table(client, destination_table_config) - base_table = base_config["table"] - - # CTE name and alias - cte_name = f"filtered_{base_table}" - base_alias = base_table - classification_alias = base_alias - - # Build CTE WHERE clause - # Combines base constraint, filter profile, and classification filter - - # Add classification filter condition if present - if allowed_cases: - where_conditions = [f"p.{join_key} IS NOT NULL"] # Base constraint to ensure join key is not null - cases_str = ", ".join([f"'{c}'" for c in allowed_cases]) - where_conditions.append(f"c.`case` IN ({cases_str})") - else: - where_conditions = [f"{join_key} IS NOT NULL"] - - # Add raw column filter conditions if present - if filter_sql: - # Remove leading WHERE from filter_sql safely - filter_body = filter_sql.replace("WHERE", "", 1).strip() - where_conditions.append(filter_body) - - final_where = "WHERE " + "\n AND ".join(where_conditions) - - # Build eligibility flag columns for the CTE - # One boolean per unique gating rule (deduped by status_col) - eligibility_flags_sql = "" - if allowed_cases: - unique_rules = build_unique_eligibility_rules(constants.MODULE_CENSOR_RULES) - if unique_rules: - flag_lines = [] - for status_col, rule in unique_rules.items(): - completion_ts = rule["completion_ts"] - cutoff = rule["cutoff"] - eligible_expr = render_eligibility_condition( - status_col, completion_ts, cutoff, alias="c" - ) - flag_lines.append(f" {eligible_expr} AS {status_col}_eligible") - eligibility_flags_sql = ",\n" + ",\n".join(flag_lines) - - # Build CTE with or without classification join - if allowed_cases: - if not classification_table: - raise ValueError( - "classification_table must be provided when allowed_cases is populated." - ) - cte = f""" -{cte_name} AS ( - SELECT - p.*, - c.* EXCEPT({join_key}){eligibility_flags_sql} - FROM `{fq_base_table}` p - INNER JOIN `{classification_table}` c - ON p.{join_key} = c.{join_key} - {final_where} -)""".strip() - else: - cte = f""" -{cte_name} AS ( - SELECT * - FROM `{fq_base_table}` - {final_where} -)""".strip() - - select_parts = [] - join_parts = [] - - selected_column_names = set() - duplicate_columns = [] - unrecognized_cid_columns = [] - - # Always include join key first from base table - select_parts.append(f"{base_alias}.{join_key}") - selected_column_names.add(join_key) - - # Base table columns - base_cols = base_config.get("columns", []) - - # Lookup schema for validation - base_schema = set(table_schemas.get((base_config["dataset"], base_config["table"]), [])) - - if base_cols == "*": - base_cols = sorted(base_schema) - - for col in base_cols: - if col in base_schema: - if col == join_key: - continue # Skip join key since it is already included - - # If column has already been selected (e.g. from a previous table), - # track it as a duplicate and skip to avoid ambiguity - if col in selected_column_names: - duplicate_columns.append(col) - continue - - # Flag unrecognized CIDs before/alongside rendering - if is_unrecognized_censorship_cid(col): - unrecognized_cid_columns.append({ - "dataset": base_config["dataset"], - "table": base_config["table"], - "column": col - }) - - select_parts.append( - utils.render_censored_column_expression( - source_alias=base_alias, - col_name=col, - classification_alias=classification_alias, - ) - ) - selected_column_names.add(col) - - # Join tables - for join_table in destination_table_config.get("join_tables", []): - dataset = join_table["dataset"] - table = join_table["table"] - - join_schema = set(table_schemas.get((dataset, table), [])) - cols = join_table.get("columns", []) - - if cols == "*": - cols = sorted(join_schema) - - if not cols: - continue - - # Add LEFT JOIN - join_parts.append( - f"LEFT JOIN `{project}.{dataset}.{table}` {table} " - f"ON {table}.{join_key} = {base_alias}.{join_key}" - ) - - # Add selected columns - for col in cols: - if col in join_schema: - if col == join_key: - continue # Skip join key since it is already included - - # If column has already been selected (e.g. from a previous table), - # track it as a duplicate and skip to avoid ambiguity - if col in selected_column_names: - duplicate_columns.append(f"{dataset}.{table}.{col}") - continue - - # Flag unrecognized CIDs for join-table columns too - if is_unrecognized_censorship_cid(col): - unrecognized_cid_columns.append({ - "dataset": dataset, - "table": table, - "column": col - }) - - select_parts.append( - utils.render_censored_column_expression( - source_alias=table, - col_name=col, - classification_alias=classification_alias, - ) -) - selected_column_names.add(col) - - # Log duplicate columns (if any) - if duplicate_columns: - unique_dupes = sorted(set(duplicate_columns)) - utils.logger.warning( - f"[{destination_table}] Duplicate columns detected and skipped: " - f"{', '.join(unique_dupes)}. Using first occurrence only." - ) - - # Log unrecognized CID columns (if any) - same pattern as duplicate_columns logging - if unrecognized_cid_columns: - unrecognized_names = sorted({c["column"] for c in unrecognized_cid_columns}) - utils.logger.warning( - f"[{destination_table}] Columns with unrecognized CIDs detected " - f"(will be set to NULL by fail-closed default): {', '.join(unrecognized_names)}. " - f"Please verify these CIDs are valid and update MODULE_CENSOR_RULES if necessary." - ) - - # Ensure at least one column is selected - if not select_parts: - raise ValueError(f"No columns selected for {destination_table}") - - select_sql = ",\n ".join(select_parts) - join_sql = "\n".join(join_parts) - - sql = f""" -CREATE OR REPLACE TABLE `{project}.{destination_dataset}.{destination_table}` AS -WITH -{cte} -SELECT - {select_sql} -FROM {cte_name} {base_alias} -{join_sql} -""".strip() - - return { - "destination_table": f"{project}.{destination_dataset}.{destination_table}", - "sql": sql, - "duplicate_columns": sorted(set(duplicate_columns)), - "unrecognized_cid_columns": unrecognized_cid_columns - } - -def create_destination_missing_columns_json( - client: storage.Client, - output_path: str, - destination_table: str, - missing_cleaned_cols: list[dict[str, Any]], - missing_cleaned_loop_vars: list[dict[str, Any]], - duplicate_columns: list[str], - unrecognized_cid_columns: list[dict[str, Any]] -) -> None: - """ - Write a JSON report of missing columns to a GCS location. - - Args: - client (storage.Client): GCS client. - output_path (str): GCS path (gs://...) to write the report. - destination_table (str): Destination table name. - missing_cleaned_cols (list): Missing standard columns. - missing_cleaned_loop_vars (list): Missing loop variable columns. - duplicate_columns (list): List of duplicate column names. - unrecognized_cid_columns (list): List of columns with unrecognized CIDs. - """ - # Build report structure - report = { - "_metadata": { - "generated_at": datetime.now(timezone.utc).isoformat(), - "source": "pr2-transformation pipeline", - "destination_table": destination_table, - "description": "Report of columns requested by the destination config that were not found in their respective source tables", - "structure": { - "missing_cleaned_cols": "Columns that were not found and are NOT cleaned loop variables — these are unexpected and should be investigated", - "missing_cleaned_loop_vars": "Columns that were not found but ARE cleaned loop variables — these may be expected if the loop variable does not exist for a given table", - "duplicate_columns": "Columns skipped due to duplicate names in SELECT", - "unrecognized_cid_columns": "Columns whose first CID matched neither MODULE_CENSOR_RULES nor NON_CENSORED_SECONDARY_SOURCE_CIDS. These were set to NULL by the fail-closed default and likely indicate a naming mistake, a missing MODULE_CENSOR_RULES entry, or a column that does not belong in this table", - "occurrence_count": "Number of times this column appeared in the config AND was missing" - } - }, - "missing_cleaned_cols": missing_cleaned_cols, - "missing_cleaned_loop_vars": missing_cleaned_loop_vars, - "duplicate_columns": duplicate_columns, - "unrecognized_cid_columns": unrecognized_cid_columns - } - - # Parse GCS path - path = output_path.removeprefix("gs://") - bucket_name, blob_path = path.split("/", 1) - - bucket = client.bucket(bucket_name) - blob = bucket.blob(blob_path) - - # Upload JSON report - blob.upload_from_string( - json.dumps(report, indent=2, ensure_ascii=False), - content_type="application/json", - ) - -def get_first_cid(column_name: str) -> str | None: - cids = extract_ordered_concept_ids(column_name) - return cids[0] if cids else None - -def is_unrecognized_censorship_cid(col_name: str) -> bool: - """ - Returns True if col_name's first CID does not match any known pattern - (i.e., it is not in MODULE_CENSOR_RULES), is not in - NON_CENSORED_SECONDARY_SOURCE_CIDS, and is not an ALWAYS_INCLUDE_NON_CID - column. These columns fall through to the fail-closed NULL branch in - render_censored_column_expression, which is safe (no data leaks) but - usually signals a naming mistake, a missing MODULE_CENSOR_RULES entry, - or a column that does not belong in this table at all. - - Columns with no CID at all (e.g. "token") are NOT flagged here since - the absence of a CID is a different, already-handled case - (ALWAYS_INCLUDE_NON_CID_COLUMNS) rather than an unrecognized one. - - Args: - col_name: The column name to check. - - Returns: - bool: True if this column's first CID is unrecognized and would - be set to NULL by render_censored_column_expression's - fail-closed path. - """ - first_cid = get_first_cid(col_name) - - if first_cid is None: - # No CID present at all - handled separately by - # ALWAYS_INCLUDE_NON_CID_COLUMNS, not a "censorship pattern" issue. - return False - - if first_cid in constants.NON_CENSORED_SECONDARY_SOURCE_CIDS: - return False - - if first_cid in constants.MODULE_CENSOR_RULES: - return False - - return True - -def render_censored_column_expression( - source_alias: str, - col_name: str, - classification_alias: str = "c", -) -> str: - first_cid = get_first_cid(col_name) - - if first_cid is None: - if col_name in constants.ALWAYS_INCLUDE_NON_CID_COLUMNS: - return f"{source_alias}.{col_name}" - return f"CAST(NULL AS STRING) AS {col_name}" - - if first_cid in constants.NON_CENSORED_SECONDARY_SOURCE_CIDS: - return f"{source_alias}.{col_name}" - - # Known module columns: keep only if module status is "Submitted" - # and for cases 3A/3B before cutoff. - rule = constants.MODULE_CENSOR_RULES.get(first_cid) - if rule is not None: - flag_name = f"{rule['status_col']}_eligible" - return f"IF({classification_alias}.{flag_name}, {source_alias}.{col_name}, NULL) AS {col_name}" - - # Unknown first CID: fail closed - return f"CAST(NULL AS STRING) AS {col_name}" - -def build_column_rule_map(output_columns: list[str]) -> dict[str, dict]: - """ - Maps each output column to its MODULE_CENSOR_RULES entry based on the - column's first concept ID. Columns whose first CID has no entry in - MODULE_CENSOR_RULES are excluded (they are not module_censor columns). - - Args: - output_columns: List of column names from the destination table - (e.g. via utils.get_column_names). - - Returns: - dict: {column_name: rule_dict} for every column governed by a - module censorship rule. - """ - column_rule_map = {} - for col in output_columns: - first_cid = utils.get_first_cid(col) - if first_cid is None: - continue - rule = constants.MODULE_CENSOR_RULES.get(first_cid) - if rule is not None: - column_rule_map[col] = rule - return column_rule_map - - -def _build_reason_cte_block( - status_col: str, - rule: dict, - classification_table: str, - classification_alias: str = "c", -) -> str: - """ - Builds a single SELECT block reporting whether column_name was - censored for each Connect_ID and why. - - Args: - status_col: The deduped status column name (e.g. "module2_status"). - rule: The rule dict for this status_col — one value from the - {status_col: rule} mapping built by build_censorship_summary_sql's - dedup step (keys: status_col, completion_ts, cutoff). - classification_table: Fully qualified classification table. - classification_alias: Alias of the classification table. - - Returns: - str: A SELECT statement returning Connect_ID, case, status_col, - and reason. - """ - completion_ts = rule["completion_ts"] - cutoff = rule["cutoff"] - eligible_expr = render_eligibility_condition( - status_col, completion_ts, cutoff, alias=classification_alias - ) - - return f""" - SELECT - {classification_alias}.Connect_ID, - {classification_alias}.`case`, - '{status_col}' AS status_col, - CASE - -- The eligibility decision itself is delegated to - -- render_eligibility_condition — the same expression used to - -- build {status_col}_eligible in build_destination_table_query. - -- Everything below only decides which human-readable message to - -- show once it is known the participant is NOT eligible. - WHEN {eligible_expr} IS NOT TRUE - THEN CASE - -- Reason 1: Status is not "Submitted" (NULL, empty, - -- or any other value). This alone is disqualifying - -- regardless of case. - WHEN {classification_alias}.{status_col} IS DISTINCT FROM 'Submitted' - THEN CASE - WHEN {classification_alias}.{status_col} IS NULL - THEN '{status_col} = NULL' - ELSE CONCAT( - '{status_col} = "', {classification_alias}.{status_col}, '"', - IF({classification_alias}.{status_col} = '', ' (empty string)', '') - ) - END - -- Reason 2: Status was "Submitted", but for cases 3A/3B the - -- completion has to land strictly before the cutoff. Missing - -- timestamps or a completion on/after cutoff both count as - -- censored. Treat NULL and "too late" as distinct - -- sub-reasons for a clearer message. - ELSE CASE - WHEN {classification_alias}.{completion_ts} IS NULL - THEN '{completion_ts} = NULL' - WHEN {classification_alias}.{cutoff} IS NULL - THEN '{cutoff} = NULL' - ELSE CONCAT( - '{completion_ts} = ', - CAST({classification_alias}.{completion_ts} AS STRING), - ' is on or after ', '{cutoff}', ' = ', - CAST({classification_alias}.{cutoff} AS STRING) - ) - END - END - -- Eligible: Not censored for this column. - ELSE NULL - END AS reason - FROM `{classification_table}` {classification_alias} - """.strip() - - -def build_censorship_summary_sql( - column_rule_map: dict[str, dict], - classification_table: str, - classification_alias: str = "c", -) -> str: - """ - Builds the full UNION ALL query across all censorable columns. - - Args: - column_rule_map: {column_name: rule_dict}, as produced by - build_column_rule_map. - classification_table: Fully qualified table containing case, status, - completion_ts, and cutoff columns for every - participant. - classification_alias: Alias used for that table in the generated SQL. - - Returns: - str: A complete SQL query returning Connect_ID, case, column_name, - and reason for every censorship event, filtered to reason IS NOT NULL. - - Raises: - ValueError: If column_rule_map is empty. - """ - # Multiple output columns can point at the same status_col — e.g., both - # menstrual-survey CIDs (912367929, 232438133) drive menstrual_status. - # Collapse to one rule per status_col, so _build_reason_cte_block is not - # generating identical CASE logic twice for the same underlying column. - # - # Note: Unlike build_unique_eligibility_rules (used for the *_eligible - # flags in build_destination_table_query), this dedup does NOT check for - # conflicting completion_ts/cutoff values across CIDs sharing a - # status_col; it assumes column_rule_map is already consistent since - # every entry in it traces back to the same MODULE_CENSOR_RULES that - # build_unique_eligibility_rules validates elsewhere in the pipeline. - status_col_to_rule: dict[str, dict] = {} - for rule in column_rule_map.values(): - # Values are guaranteed identical for a shared status_col; see note above - status_col_to_rule[rule["status_col"]] = rule - - # Build one reason-block per unique status_col then UNION them together - reason_blocks = [ - _build_reason_cte_block( - status_col=status_col, - rule=rule, - classification_table=classification_table, - classification_alias=classification_alias, - ) - for status_col, rule in status_col_to_rule.items() - ] - reasons_cte = "\nUNION ALL\n".join(reason_blocks) - - # Map every output column back to its status_col as a VALUES/STRUCT - # literal, so each column (including duplicates like the two menstrual - # CIDs) joins onto its single shared reasons row below. - column_map_rows = ",\n ".join( - f"('{col}', '{rule['status_col']}')" - for col, rule in column_rule_map.items() - ) - - final_sql = f""" -WITH reasons AS ( - {reasons_cte} -), -column_status_map AS ( - SELECT * FROM UNNEST([ - STRUCT - {column_map_rows} - ]) -) -SELECT - reasons.Connect_ID, - reasons.`case`, - column_status_map.column_name, - reasons.reason -FROM reasons -JOIN column_status_map - ON reasons.status_col = column_status_map.status_col -WHERE reasons.reason IS NOT NULL -ORDER BY reasons.Connect_ID, column_status_map.column_name -""".strip() - - return final_sql - -def get_censorship_rollup(destination_table: str) -> dict: - """ - Queries the censorship summary table and returns two rollups: - 1. Per-column counts (how many participants were censored, per column) - 2. Per-case counts (how many censorship events occurred, per case) - - Args: - destination_table: The table created by create_censorship_summary_table. - - Returns: - dict: {"by_column": [...], "by_case": [...]} - """ - client = bigquery.Client() - - by_column_sql = f""" - SELECT column_name, COUNT(*) AS censored_count - FROM `{destination_table}` - GROUP BY column_name - ORDER BY censored_count DESC - """ - by_case_sql = f""" - SELECT `case`, COUNT(*) AS censorship_events - FROM `{destination_table}` - GROUP BY `case` - ORDER BY `case` - """ - - by_column = [dict(row) for row in client.query(by_column_sql).result()] - by_case = [dict(row) for row in client.query(by_case_sql).result()] - - return {"by_column": by_column, "by_case": by_case} - -def create_censorship_rollup_json( - client: storage.Client, - output_path: str, - destination_table: str, - by_column: list[dict[str, Any]], - by_case: list[dict[str, Any]], -) -> None: - """ - Write a JSON report of censorship rollup counts to a GCS location. - - Args: - client (storage.Client): GCS client. - output_path (str): GCS path (gs://...) to write the report. - destination_table (str): Censorship summary table name (the table - get_censorship_rollup was queried against). - by_column (list): Per-column censored counts, as returned by - get_censorship_rollup()["by_column"]. - by_case (list): Per-case censorship event counts, as returned by - get_censorship_rollup()["by_case"]. - """ - # Build report structure - report = { - "_metadata": { - "generated_at": datetime.now(timezone.utc).isoformat(), - "source": "pr2-transformation pipeline", - "destination_table": destination_table, - "description": "Rollup of censorship events from the censorship summary table, broken down by column and by case", - "structure": { - "by_column": "Per-column counts of how many participants were censored for that column, ordered by censored_count descending", - "by_case": "Per-case counts of how many censorship events occurred, ordered by case" - } - }, - "by_column": by_column, - "by_case": by_case, - } - - # Parse GCS path - path = output_path.removeprefix("gs://") - bucket_name, blob_path = path.split("/", 1) - - bucket = client.bucket(bucket_name) - blob = bucket.blob(blob_path) - - # Upload JSON report - blob.upload_from_string( - json.dumps(report, indent=2, ensure_ascii=False), - content_type="application/json", - ) \ No newline at end of file + ensure_ascii=False), content_type="application/json") \ No newline at end of file From c706f48acdfdf6d43b1546ef4e32b02304205293 Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Fri, 17 Jul 2026 16:59:56 -0400 Subject: [PATCH 44/67] Move censorship functions into its own module and route applicable calls through the censorship module --- core/censorship.py | 342 ++++++++++++++++++++++++++++++ core/destination_table_builder.py | 13 +- core/transformations.py | 10 +- 3 files changed, 353 insertions(+), 12 deletions(-) create mode 100644 core/censorship.py diff --git a/core/censorship.py b/core/censorship.py new file mode 100644 index 0000000..ab7a742 --- /dev/null +++ b/core/censorship.py @@ -0,0 +1,342 @@ +"""Utilities for applying censorship rules and building censorship summaries.""" + +import json +from typing import Any +from datetime import datetime, timezone + +from google.cloud import bigquery, storage + +from core import constants, utils, destination_table_builder + +def get_first_cid(column_name: str) -> str | None: + cids = utils.extract_ordered_concept_ids(column_name) + return cids[0] if cids else None + +def is_unrecognized_censorship_cid(col_name: str) -> bool: + """ + Returns True if col_name's first CID does not match any known pattern + (i.e., it is not in MODULE_CENSOR_RULES), is not in + NON_CENSORED_SECONDARY_SOURCE_CIDS, and is not an ALWAYS_INCLUDE_NON_CID + column. These columns fall through to the fail-closed NULL branch in + render_censored_column_expression, which is safe (no data leaks) but + usually signals a naming mistake, a missing MODULE_CENSOR_RULES entry, + or a column that does not belong in this table at all. + + Columns with no CID at all (e.g. "token") are NOT flagged here since + the absence of a CID is a different, already-handled case + (ALWAYS_INCLUDE_NON_CID_COLUMNS) rather than an unrecognized one. + + Args: + col_name: The column name to check. + + Returns: + bool: True if this column's first CID is unrecognized and would + be set to NULL by render_censored_column_expression's + fail-closed path. + """ + first_cid = get_first_cid(col_name) + + if first_cid is None: + # No CID present at all - handled separately by + # ALWAYS_INCLUDE_NON_CID_COLUMNS, not a "censorship pattern" issue. + return False + + if first_cid in constants.NON_CENSORED_SECONDARY_SOURCE_CIDS: + return False + + if first_cid in constants.MODULE_CENSOR_RULES: + return False + + return True + +def render_censored_column_expression( + source_alias: str, + col_name: str, + classification_alias: str = "c", +) -> str: + first_cid = get_first_cid(col_name) + + if first_cid is None: + if col_name in constants.ALWAYS_INCLUDE_NON_CID_COLUMNS: + return f"{source_alias}.{col_name}" + return f"CAST(NULL AS STRING) AS {col_name}" + + if first_cid in constants.NON_CENSORED_SECONDARY_SOURCE_CIDS: + return f"{source_alias}.{col_name}" + + # Known module columns: keep only if module status is "Submitted" + # and for cases 3A/3B before cutoff. + rule = constants.MODULE_CENSOR_RULES.get(first_cid) + if rule is not None: + flag_name = f"{rule['status_col']}_eligible" + return f"IF({classification_alias}.{flag_name}, {source_alias}.{col_name}, NULL) AS {col_name}" + + # Unknown first CID: fail closed + return f"CAST(NULL AS STRING) AS {col_name}" + +def build_column_rule_map(output_columns: list[str]) -> dict[str, dict]: + """ + Maps each output column to its MODULE_CENSOR_RULES entry based on the + column's first concept ID. Columns whose first CID has no entry in + MODULE_CENSOR_RULES are excluded (they are not module_censor columns). + + Args: + output_columns: List of column names from the destination table + (e.g. via utils.get_column_names). + + Returns: + dict: {column_name: rule_dict} for every column governed by a + module censorship rule. + """ + column_rule_map = {} + for col in output_columns: + first_cid = get_first_cid(col) + if first_cid is None: + continue + rule = constants.MODULE_CENSOR_RULES.get(first_cid) + if rule is not None: + column_rule_map[col] = rule + return column_rule_map + + +def _build_reason_cte_block( + status_col: str, + rule: dict, + classification_table: str, + classification_alias: str = "c", +) -> str: + """ + Builds a single SELECT block reporting whether column_name was + censored for each Connect_ID and why. + + Args: + status_col: The deduped status column name (e.g. "module2_status"). + rule: The rule dict for this status_col — one value from the + {status_col: rule} mapping built by build_censorship_summary_sql's + dedup step (keys: status_col, completion_ts, cutoff). + classification_table: Fully qualified classification table. + classification_alias: Alias of the classification table. + + Returns: + str: A SELECT statement returning Connect_ID, case, status_col, + and reason. + """ + completion_ts = rule["completion_ts"] + cutoff = rule["cutoff"] + eligible_expr = destination_table_builder.render_eligibility_condition( + status_col, completion_ts, cutoff, alias=classification_alias + ) + + return f""" + SELECT + {classification_alias}.Connect_ID, + {classification_alias}.`case`, + '{status_col}' AS status_col, + CASE + -- The eligibility decision itself is delegated to + -- render_eligibility_condition — the same expression used to + -- build {status_col}_eligible in build_destination_table_query. + -- Everything below only decides which human-readable message to + -- show once it is known the participant is NOT eligible. + WHEN {eligible_expr} IS NOT TRUE + THEN CASE + -- Reason 1: Status is not "Submitted" (NULL, empty, + -- or any other value). This alone is disqualifying + -- regardless of case. + WHEN {classification_alias}.{status_col} IS DISTINCT FROM 'Submitted' + THEN CASE + WHEN {classification_alias}.{status_col} IS NULL + THEN '{status_col} = NULL' + ELSE CONCAT( + '{status_col} = "', {classification_alias}.{status_col}, '"', + IF({classification_alias}.{status_col} = '', ' (empty string)', '') + ) + END + -- Reason 2: Status was "Submitted", but for cases 3A/3B the + -- completion has to land strictly before the cutoff. Missing + -- timestamps or a completion on/after cutoff both count as + -- censored. Treat NULL and "too late" as distinct + -- sub-reasons for a clearer message. + ELSE CASE + WHEN {classification_alias}.{completion_ts} IS NULL + THEN '{completion_ts} = NULL' + WHEN {classification_alias}.{cutoff} IS NULL + THEN '{cutoff} = NULL' + ELSE CONCAT( + '{completion_ts} = ', + CAST({classification_alias}.{completion_ts} AS STRING), + ' is on or after ', '{cutoff}', ' = ', + CAST({classification_alias}.{cutoff} AS STRING) + ) + END + END + -- Eligible: Not censored for this column. + ELSE NULL + END AS reason + FROM `{classification_table}` {classification_alias} + """.strip() + + +def build_censorship_summary_sql( + column_rule_map: dict[str, dict], + classification_table: str, + classification_alias: str = "c", +) -> str: + """ + Builds the full UNION ALL query across all censorable columns. + + Args: + column_rule_map: {column_name: rule_dict}, as produced by + build_column_rule_map. + classification_table: Fully qualified table containing case, status, + completion_ts, and cutoff columns for every + participant. + classification_alias: Alias used for that table in the generated SQL. + + Returns: + str: A complete SQL query returning Connect_ID, case, column_name, + and reason for every censorship event, filtered to reason IS NOT NULL. + + Raises: + ValueError: If column_rule_map is empty. + """ + # Multiple output columns can point at the same status_col — e.g., both + # menstrual-survey CIDs (912367929, 232438133) drive menstrual_status. + # Collapse to one rule per status_col, so _build_reason_cte_block is not + # generating identical CASE logic twice for the same underlying column. + # + # Note: Unlike build_unique_eligibility_rules (used for the *_eligible + # flags in build_destination_table_query), this dedup does NOT check for + # conflicting completion_ts/cutoff values across CIDs sharing a + # status_col; it assumes column_rule_map is already consistent since + # every entry in it traces back to the same MODULE_CENSOR_RULES that + # build_unique_eligibility_rules validates elsewhere in the pipeline. + status_col_to_rule: dict[str, dict] = {} + for rule in column_rule_map.values(): + # Values are guaranteed identical for a shared status_col; see note above + status_col_to_rule[rule["status_col"]] = rule + + # Build one reason-block per unique status_col then UNION them together + reason_blocks = [ + _build_reason_cte_block( + status_col=status_col, + rule=rule, + classification_table=classification_table, + classification_alias=classification_alias, + ) + for status_col, rule in status_col_to_rule.items() + ] + reasons_cte = "\nUNION ALL\n".join(reason_blocks) + + # Map every output column back to its status_col as a VALUES/STRUCT + # literal, so each column (including duplicates like the two menstrual + # CIDs) joins onto its single shared reasons row below. + column_map_rows = ",\n ".join( + f"('{col}', '{rule['status_col']}')" + for col, rule in column_rule_map.items() + ) + + final_sql = f""" +WITH reasons AS ( + {reasons_cte} +), +column_status_map AS ( + SELECT * FROM UNNEST([ + STRUCT + {column_map_rows} + ]) +) +SELECT + reasons.Connect_ID, + reasons.`case`, + column_status_map.column_name, + reasons.reason +FROM reasons +JOIN column_status_map + ON reasons.status_col = column_status_map.status_col +WHERE reasons.reason IS NOT NULL +ORDER BY reasons.Connect_ID, column_status_map.column_name +""".strip() + + return final_sql + +def get_censorship_rollup(destination_table: str) -> dict: + """ + Queries the censorship summary table and returns two rollups: + 1. Per-column counts (how many participants were censored, per column) + 2. Per-case counts (how many censorship events occurred, per case) + + Args: + destination_table: The table created by create_censorship_summary_table. + + Returns: + dict: {"by_column": [...], "by_case": [...]} + """ + client = bigquery.Client() + + by_column_sql = f""" + SELECT column_name, COUNT(*) AS censored_count + FROM `{destination_table}` + GROUP BY column_name + ORDER BY censored_count DESC + """ + by_case_sql = f""" + SELECT `case`, COUNT(*) AS censorship_events + FROM `{destination_table}` + GROUP BY `case` + ORDER BY `case` + """ + + by_column = [dict(row) for row in client.query(by_column_sql).result()] + by_case = [dict(row) for row in client.query(by_case_sql).result()] + + return {"by_column": by_column, "by_case": by_case} + +def create_censorship_rollup_json( + client: storage.Client, + output_path: str, + destination_table: str, + by_column: list[dict[str, Any]], + by_case: list[dict[str, Any]], +) -> None: + """ + Write a JSON report of censorship rollup counts to a GCS location. + + Args: + client (storage.Client): GCS client. + output_path (str): GCS path (gs://...) to write the report. + destination_table (str): Censorship summary table name (the table + get_censorship_rollup was queried against). + by_column (list): Per-column censored counts, as returned by + get_censorship_rollup()["by_column"]. + by_case (list): Per-case censorship event counts, as returned by + get_censorship_rollup()["by_case"]. + """ + # Build report structure + report = { + "_metadata": { + "generated_at": datetime.now(timezone.utc).isoformat(), + "source": "pr2-transformation pipeline", + "destination_table": destination_table, + "description": "Rollup of censorship events from the censorship summary table, broken down by column and by case", + "structure": { + "by_column": "Per-column counts of how many participants were censored for that column, ordered by censored_count descending", + "by_case": "Per-case counts of how many censorship events occurred, ordered by case" + } + }, + "by_column": by_column, + "by_case": by_case, + } + + # Parse GCS path + path = output_path.removeprefix("gs://") + bucket_name, blob_path = path.split("/", 1) + + bucket = client.bucket(bucket_name) + blob = bucket.blob(blob_path) + + # Upload JSON report + blob.upload_from_string( + json.dumps(report, indent=2, ensure_ascii=False), + content_type="application/json", + ) \ No newline at end of file diff --git a/core/destination_table_builder.py b/core/destination_table_builder.py index b441aae..8253d5b 100644 --- a/core/destination_table_builder.py +++ b/core/destination_table_builder.py @@ -1,13 +1,12 @@ """Utilities for building destination tables from configuration.""" import json -from core import constants, utils from typing import Any, Optional from datetime import datetime, timezone from google.cloud import bigquery, storage - +from core import constants, utils, censorship def load_destination_config(config_file_path: str) -> dict: """ @@ -204,7 +203,7 @@ def _check_cols(config): key = (dataset, table, col) # Separate loop variables versus standard columns - if is_cleaned_loop_variable(col): + if utils.is_cleaned_loop_variable(col): missing_cleaned_loop_vars[key] = missing_cleaned_loop_vars.get(key, 0) + 1 else: missing_cleaned_cols[key] = missing_cleaned_cols.get(key, 0) + 1 @@ -522,7 +521,7 @@ def build_destination_table_query( continue # Flag unrecognized CIDs before/alongside rendering - if utils.is_unrecognized_censorship_cid(col): + if censorship.is_unrecognized_censorship_cid(col): unrecognized_cid_columns.append({ "dataset": base_config["dataset"], "table": base_config["table"], @@ -530,7 +529,7 @@ def build_destination_table_query( }) select_parts.append( - utils.render_censored_column_expression( + censorship.render_censored_column_expression( source_alias=base_alias, col_name=col, classification_alias=classification_alias, @@ -571,7 +570,7 @@ def build_destination_table_query( continue # Flag unrecognized CIDs for join-table columns too - if is_unrecognized_censorship_cid(col): + if censorship.is_unrecognized_censorship_cid(col): unrecognized_cid_columns.append({ "dataset": dataset, "table": table, @@ -579,7 +578,7 @@ def build_destination_table_query( }) select_parts.append( - utils.render_censored_column_expression( + censorship.render_censored_column_expression( source_alias=table, col_name=col, classification_alias=classification_alias, diff --git a/core/transformations.py b/core/transformations.py index bfe2442..05add89 100644 --- a/core/transformations.py +++ b/core/transformations.py @@ -12,7 +12,7 @@ if __name__ == "__main__": sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from core import constants, utils, transform_renderer, classification, destination_table_builder +from core import constants, utils, transform_renderer, classification, destination_table_builder, censorship ######################################################################## ############# Table-level Transformations ############################# @@ -1170,14 +1170,14 @@ def create_censorship_summary_table( _, _, table_short_name = utils.parse_fq_table(destination_table) output_columns = utils.get_column_names(client, output_table) - column_rule_map = utils.build_column_rule_map(output_columns) + column_rule_map = censorship.build_column_rule_map(output_columns) utils.logger.info( f"[{table_short_name}] Found {len(column_rule_map)} censorable columns out of " f"{len(output_columns)} total columns in {output_table}" ) - sql = utils.build_censorship_summary_sql( + sql = censorship.build_censorship_summary_sql( column_rule_map=column_rule_map, classification_table=classification_table, ) @@ -1207,10 +1207,10 @@ def create_censorship_summary_table( # Compute and write the rollup report try: - rollup = utils.get_censorship_rollup(destination_table) + rollup = censorship.get_censorship_rollup(destination_table) report_path = f"{rollup_report_base_path}{destination_table}_rollup.json" - utils.create_censorship_rollup_json( + censorship.create_censorship_rollup_json( client=gcs_client, output_path=report_path, destination_table=destination_table, From 95bb336d462b5acb3dbb88a6187034cd6943dc4f Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Tue, 21 Jul 2026 14:08:48 -0400 Subject: [PATCH 45/67] Add comment to SQL query output file --- core/destination_table_builder.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/destination_table_builder.py b/core/destination_table_builder.py index 8253d5b..a31521e 100644 --- a/core/destination_table_builder.py +++ b/core/destination_table_builder.py @@ -611,6 +611,9 @@ def build_destination_table_query( join_sql = "\n".join(join_parts) sql = f""" +/* Combined transformation query to build {project}.{destination_dataset}.{destination_table} from the destination configuration. + Applies participant classification, configured row filters, and censorship rules if configured. */ + CREATE OR REPLACE TABLE `{project}.{destination_dataset}.{destination_table}` AS WITH {cte} From 033286437d138f5e92bef89ad54cd34e240e08a6 Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Tue, 21 Jul 2026 15:00:57 -0400 Subject: [PATCH 46/67] Move censorship summary CREATE TABLE wrapping into censorship.py - build_censorship_summary_sql: own the full CREATE OR REPLACE TABLE (...) statement, including header comment, instead of returning a bare SELECT for transforms.py to wrap - build_censorship_summary_sql: add output_table param, use it in "/* Censorship summary query for {output_table} -> {destination_table} */" header comment - create_censorship_summary_table (transforms.py): drop the local CREATE TABLE wrapper and pass output_table/destination_table through to build_censorship_summary_sql directly, matching the pattern used by destination_table_builder.build_destination_table_query --- core/censorship.py | 11 +++++++++++ core/transformations.py | 6 +++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/core/censorship.py b/core/censorship.py index ab7a742..194ca41 100644 --- a/core/censorship.py +++ b/core/censorship.py @@ -178,6 +178,8 @@ def _build_reason_cte_block( def build_censorship_summary_sql( + output_table: str, + destination_table: str, column_rule_map: dict[str, dict], classification_table: str, classification_alias: str = "c", @@ -186,6 +188,11 @@ def build_censorship_summary_sql( Builds the full UNION ALL query across all censorable columns. Args: + output_table: Fully qualified final destination table that + column_rule_map's columns were derived from. Used + only for the descriptive header comment in the + generated SQL. + destination_table: Fully qualified table to write the censorship summary for. column_rule_map: {column_name: rule_dict}, as produced by build_column_rule_map. classification_table: Fully qualified table containing case, status, @@ -237,6 +244,9 @@ def build_censorship_summary_sql( ) final_sql = f""" +/* Censorship summary query for {output_table} -> {destination_table} */ + +CREATE OR REPLACE TABLE `{destination_table}` AS ( WITH reasons AS ( {reasons_cte} ), @@ -256,6 +266,7 @@ def build_censorship_summary_sql( ON reasons.status_col = column_status_map.status_col WHERE reasons.reason IS NOT NULL ORDER BY reasons.Connect_ID, column_status_map.column_name +) """.strip() return final_sql diff --git a/core/transformations.py b/core/transformations.py index 05add89..16b325c 100644 --- a/core/transformations.py +++ b/core/transformations.py @@ -1177,13 +1177,13 @@ def create_censorship_summary_table( f"{len(output_columns)} total columns in {output_table}" ) - sql = censorship.build_censorship_summary_sql( + final_sql = censorship.build_censorship_summary_sql( + output_table=output_table, + destination_table=destination_table, column_rule_map=column_rule_map, classification_table=classification_table, ) - final_sql = f"CREATE OR REPLACE TABLE `{destination_table}` AS ({sql})" - # Save the SQL to GCS for audit purposes, matching existing pipeline convention try: gcs_client = storage.Client() From 7d830ef5a091e1ab7b35a681cc369011867b63d8 Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Wed, 22 Jul 2026 10:41:26 -0400 Subject: [PATCH 47/67] Remove columns from bioSurvey and add education variable --- reference/pr2_mvp_column_mapping.json | 43 ++++++--------------------- 1 file changed, 9 insertions(+), 34 deletions(-) diff --git a/reference/pr2_mvp_column_mapping.json b/reference/pr2_mvp_column_mapping.json index 74c2d5e..6814344 100644 --- a/reference/pr2_mvp_column_mapping.json +++ b/reference/pr2_mvp_column_mapping.json @@ -1,6 +1,6 @@ { "_metadata": { - "generated_at": "2026-04-03T18:16:06.301340+00:00", + "generated_at": "2026-06-30T14:31:31.102738+00:00", "source": "CSV conversion script", "description": "Column mapping of concept IDs grouped by table", "structure": { @@ -19,33 +19,10 @@ "cleanconnect_cid": "Concept ID from CleanConnect", "cleanconnect_cid_updated": "CleanConnect concept ID with secondary source concept ID prepended" }, - "total_tables": 4, + "total_tables": 3, "total_rows": 67, - "missing_loop_variables": { - "bioSurvey": [ - { - "flatconnect_cid": "", - "cleanconnect_cid": "d_715581797_10_v2", - "cleanconnect_cid_updated": "" - } - ] - }, - "missing_columns": { - "participants": [ - { - "flatconnect_cid": "", - "cleanconnect_cid": "test1", - "cleanconnect_cid_updated": "" - } - ], - "module1": [ - { - "flatconnect_cid": "", - "cleanconnect_cid": "test2", - "cleanconnect_cid_updated": "test2" - } - ] - }, + "missing_loop_variables": {}, + "missing_columns": {}, "duplicate_rows": {} }, "tables": { @@ -356,6 +333,11 @@ "flatconnect_cid": "D_746012894", "cleanconnect_cid": "d_746012894", "cleanconnect_cid_updated": "d_726699695_d_746012894" + }, + { + "flatconnect_cid": "D_367803647_D_367803647", + "cleanconnect_cid": "d_367803647_d_367803647", + "cleanconnect_cid_updated": "d_726699695_d_367803647_d_367803647" } ], "module3": [ @@ -384,13 +366,6 @@ "cleanconnect_cid": "d_304657762", "cleanconnect_cid_updated": "d_965707586_d_304657762" } - ], - "bioSurvey": [ - { - "flatconnect_cid": "", - "cleanconnect_cid": "d_715581797_10_v2", - "cleanconnect_cid_updated": "" - } ] } } \ No newline at end of file From 17db89f7d5f7fcf176c91da86750c6e9508fd55a Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Wed, 22 Jul 2026 11:59:27 -0400 Subject: [PATCH 48/67] Update test data in destination_config.json --- reference/destination_config.json | 58 ++++++++----------------------- 1 file changed, 15 insertions(+), 43 deletions(-) diff --git a/reference/destination_config.json b/reference/destination_config.json index 9d31752..8320ab3 100644 --- a/reference/destination_config.json +++ b/reference/destination_config.json @@ -11,11 +11,11 @@ "destination_tables": [ { "dataset": "pr2_mvp", - "table": "mvp_case_1_2", + "table": "mvp_case_1_2_3A_3B_2026_07_07", "join_key": "Connect_ID", "filter_profile": "no_filters", "classification_filter": { - "allowed_cases": ["1", "2"] + "allowed_cases": ["1", "2", "3A", "3B"] }, "base_table": { "dataset": "CleanConnect", @@ -47,58 +47,30 @@ }, { "dataset": "SensitiveTier", - "table": "module1", + "table": "mvp_0_1", "join_key": "Connect_ID", - "filter_profile": "default_participants", + "filter_profile": "no_filters", "classification_filter": { - "allowed_cases": [] + "allowed_cases": ["1", "2", "3A", "3B"] }, "base_table": { - "dataset": "CleanConnect", - "table": "participants", - "columns": [] + "dataset": "pr2_mvp", + "table": "mvp", + "columns": "*" }, - "join_tables": [ - { - "dataset": "CleanConnect", - "table": "module1", - "columns": [ - "d_103397024_d_206625031" - ] - } - ] + "join_tables": [] }, { - "dataset": "FakeTier", - "table": "module1", + "dataset": "pr2_mvp", + "table": "mvp_synthetic_2026_07_15_case_1_2_3A_3B", "join_key": "Connect_ID", - "filter_profile": "default_participants", + "filter_profile": "no_filters", "classification_filter": { - "allowed_cases": [] + "allowed_cases": ["1", "2", "3A", "3B"] }, "base_table": { - "dataset": "CleanConnect", - "table": "participants", - "columns": [] - }, - "join_tables": [ - { - "dataset": "CleanConnect", - "table": "module1", - "columns": [ - "d_103397024_d_206625031" - ] - } - ] - }, - { - "dataset": "SensitiveTier", - "table": "module2", - "join_key": "Connect_ID", - "filter_profile": "no_filters", - "base_table": { - "dataset": "CleanConnect", - "table": "module1", + "dataset": "pr2_mvp", + "table": "mvp_synthetic", "columns": "*" }, "join_tables": [] From 8ab433d523209e24c0ef330d7732a8a38d98c06d Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Wed, 22 Jul 2026 15:14:19 -0400 Subject: [PATCH 49/67] Create endpoints for classification, destination table creation, and censorship --- core/endpoints.py | 69 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/core/endpoints.py b/core/endpoints.py index 7a4135b..a7daaf0 100644 --- a/core/endpoints.py +++ b/core/endpoints.py @@ -112,4 +112,73 @@ def create_standardized_mapping_table(): 'message': str(e) }), 500 +@app.route('/create_classification_table', methods=['POST']) +def create_classification_table(): + mapping: dict[str, any] = request.get_json() or {} + parquet_url = mapping.get("parquet_url") + classification_table = mapping.get("classification_table") + + try: + utils.logger.info(f"create_classification_table endpoint called. Building {classification_table} from {parquet_url}.") + status = transformations.create_classification_table( + parquet_url=parquet_url, + classification_table=classification_table + ) + return jsonify({ + 'status': status, + 'timestamp': datetime.utcnow().isoformat(), + 'service': constants.SERVICE_NAME + }), 200 + except Exception as e: + utils.logger.exception("An error occurred in create_classification_table endpoint.") + return jsonify({'error': 'Internal Server Error', 'message': str(e)}), 500 + + +@app.route('/create_destination_table', methods=['POST']) +def create_destination_table(): + mapping: dict[str, any] = request.get_json() or {} + config_path = mapping.get("config_path") + destination_dataset = mapping.get("destination_dataset") + destination_table = mapping.get("destination_table") + classification_table = mapping.get("classification_table") + try: + utils.logger.info(f"create_destination_table endpoint called. Creating {destination_table} in {destination_dataset}.") + status = transformations.create_destination_table( + config_path=config_path, + destination_dataset=destination_dataset, + destination_table=destination_table, + classification_table=classification_table + ) + return jsonify({ + 'status': status, + 'timestamp': datetime.utcnow().isoformat(), + 'service': constants.SERVICE_NAME + }), 200 + except Exception as e: + utils.logger.exception("An error occurred in create_destination_table endpoint.") + return jsonify({'error': 'Internal Server Error', 'message': str(e)}), 500 + + +@app.route('/create_censorship_summary_table', methods=['POST']) +def create_censorship_summary_table(): + mapping: dict[str, any] = request.get_json() or {} + output_table = mapping.get("output_table") + classification_table = mapping.get("classification_table") + destination_table = mapping.get("destination_table") + + try: + utils.logger.info(f"create_censorship_summary_table endpoint called. Creating {destination_table}.") + status = transformations.create_censorship_summary_table( + output_table=output_table, + classification_table=classification_table, + destination_table=destination_table + ) + return jsonify({ + 'status': status, + 'timestamp': datetime.utcnow().isoformat(), + 'service': constants.SERVICE_NAME + }), 200 + except Exception as e: + utils.logger.exception("An error occurred in create_censorship_summary_table endpoint.") + return jsonify({'error': 'Internal Server Error', 'message': str(e)}), 500 \ No newline at end of file From d4ecf39f631072ddc444a4b74934b5c55a65604f Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Wed, 22 Jul 2026 15:48:51 -0400 Subject: [PATCH 50/67] Use a constant for the missing report directory path --- core/transformations.py | 48 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 43 insertions(+), 5 deletions(-) diff --git a/core/transformations.py b/core/transformations.py index 16b325c..8224af9 100644 --- a/core/transformations.py +++ b/core/transformations.py @@ -1005,7 +1005,7 @@ def create_destination_table( config_path: str, destination_dataset: str, destination_table: str, - missing_report_base_path: str = "gs://pr2-pipeline-artifacts-stg/missing_columns_report/", # constants.MISSING_COLUMNS_REPORT_PATH + missing_report_base_path: str = constants.MISSING_COLUMNS_REPORT_PATH, classification_table: Optional[str] = None ) -> dict: """ @@ -1143,7 +1143,7 @@ def create_censorship_summary_table( output_table: str, classification_table: str, destination_table: str, - rollup_report_base_path: str + rollup_report_base_path: str = constants.MISSING_COLUMNS_REPORT_PATH ) -> dict: """ Generates the censorship summary query, saves it to GCS for audit @@ -1237,9 +1237,47 @@ def create_censorship_summary_table( #create_standardized_mapping_table("CleanConnect", "nih-nci-dceg-connect-stg-5519.pr2_mvp.mvp") client = bigquery.Client() - config_path = "reference/subset_config.json" - create_subset_table( + config_path = "reference/destination_config.json" + + create_classification_table( + parquet_url="gs://ehr_pipeline_tmp_stg/participant_status.parquet", + classification_table="nih-nci-dceg-connect-stg-5519.pr2_mvp.classification", + local_csv_path=r"/Users/trivittge/Desktop/classification_df.csv", + ) + + print("######################") + create_destination_table( config_path=config_path, destination_dataset="SensitiveTier", - destination_table="mvp" + destination_table="mvp_0_1", + classification_table="nih-nci-dceg-connect-stg-5519.pr2_mvp.classification" ) + print("######################") + create_censorship_summary_table( + output_table="nih-nci-dceg-connect-stg-5519.SensitiveTier.mvp_0_1", + classification_table="nih-nci-dceg-connect-stg-5519.pr2_mvp.classification", + destination_table="nih-nci-dceg-connect-stg-5519.pr2_mvp.mvp_0_1_censorship_summary", + rollup_report_base_path="gs://pr2-pipeline-artifacts-stg/missing_columns_report/" + ) + + # Synthetic data +# create_classification_table( +# parquet_url="gs://ehr_pipeline_tmp_stg/participant_status_synthetic.parquet", +# classification_table="nih-nci-dceg-connect-stg-5519.pr2_mvp.classification_synthetic_2026_07_15", +# local_csv_path=r"/Users/trivittge/Desktop/classification_df_synthetic_2026_07_15.csv", +# ) +# +# print("######################") +# create_destination_table( +# config_path=config_path, +# destination_dataset="pr2_mvp", +# destination_table="mvp_synthetic_2026_07_15_case_1_2_3A_3B", +# classification_table="nih-nci-dceg-connect-stg-5519.pr2_mvp.classification_synthetic_2026_07_15" +# ) +# print("######################") +# create_censorship_summary_table( +# output_table="nih-nci-dceg-connect-stg-5519.pr2_mvp.mvp_synthetic_2026_07_15_case_1_2_3A_3B", +# classification_table="nih-nci-dceg-connect-stg-5519.pr2_mvp.classification_synthetic_2026_07_15", +# destination_table="nih-nci-dceg-connect-stg-5519.pr2_mvp.censorship_summary_synthetic_2026_07_15", +# rollup_report_base_path="gs://pr2-pipeline-artifacts-stg/missing_columns_report/" +# ) \ No newline at end of file From 8b073ff64e865d854f3c4876e9147e7dd9e51be1 Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Thu, 23 Jul 2026 10:50:38 -0400 Subject: [PATCH 51/67] Add test data to destination_config.json --- reference/destination_config.json | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/reference/destination_config.json b/reference/destination_config.json index 8320ab3..c850f71 100644 --- a/reference/destination_config.json +++ b/reference/destination_config.json @@ -10,8 +10,8 @@ }, "destination_tables": [ { - "dataset": "pr2_mvp", - "table": "mvp_case_1_2_3A_3B_2026_07_07", + "dataset": "SensitiveTier", + "table": "mvp_test", "join_key": "Connect_ID", "filter_profile": "no_filters", "classification_filter": { @@ -34,7 +34,12 @@ { "dataset": "pr2_mvp", "table": "mvp", - "columns": "*" + "columns": [ + "d_726699695_d_384191091_d_746038746", + "d_726699695_d_384191091_d_807835037", + "d_965707586_d_947205597_d_712653855", + "d_965707586_d_976570371" + ] }, { "dataset": "CleanConnect", From ef1e417834865348c0f0817a35c63a597e90282b Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Fri, 31 Jul 2026 16:28:39 -0400 Subject: [PATCH 52/67] Keep only the allowable cases in the censorship summary table --- core/censorship.py | 15 ++++++++++++--- core/transformations.py | 5 +++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/core/censorship.py b/core/censorship.py index 194ca41..98a1b65 100644 --- a/core/censorship.py +++ b/core/censorship.py @@ -103,7 +103,8 @@ def _build_reason_cte_block( status_col: str, rule: dict, classification_table: str, - classification_alias: str = "c", + allowed_cases: list[str], + classification_alias: str = "c" ) -> str: """ Builds a single SELECT block reporting whether column_name was @@ -115,6 +116,8 @@ def _build_reason_cte_block( {status_col: rule} mapping built by build_censorship_summary_sql's dedup step (keys: status_col, completion_ts, cutoff). classification_table: Fully qualified classification table. + allowed_cases: Cases that pass output_table's classification_filter + (e.g. ["1", "2", "3A", "3B"]). classification_alias: Alias of the classification table. Returns: @@ -127,6 +130,8 @@ def _build_reason_cte_block( status_col, completion_ts, cutoff, alias=classification_alias ) + allowed_cases_sql = ", ".join(f"'{c}'" for c in allowed_cases) + return f""" SELECT {classification_alias}.Connect_ID, @@ -174,15 +179,16 @@ def _build_reason_cte_block( ELSE NULL END AS reason FROM `{classification_table}` {classification_alias} + WHERE {classification_alias}.`case` IN ({allowed_cases_sql}) """.strip() - def build_censorship_summary_sql( output_table: str, destination_table: str, column_rule_map: dict[str, dict], classification_table: str, - classification_alias: str = "c", + allowed_cases: list[str], + classification_alias: str = "c" ) -> str: """ Builds the full UNION ALL query across all censorable columns. @@ -198,6 +204,8 @@ def build_censorship_summary_sql( classification_table: Fully qualified table containing case, status, completion_ts, and cutoff columns for every participant. + allowed_cases: Cases that pass output_table's classification_filter + (e.g. ["1", "2", "3A", "3B"]). classification_alias: Alias used for that table in the generated SQL. Returns: @@ -230,6 +238,7 @@ def build_censorship_summary_sql( rule=rule, classification_table=classification_table, classification_alias=classification_alias, + allowed_cases=allowed_cases, ) for status_col, rule in status_col_to_rule.items() ] diff --git a/core/transformations.py b/core/transformations.py index 8224af9..8b51120 100644 --- a/core/transformations.py +++ b/core/transformations.py @@ -1143,6 +1143,7 @@ def create_censorship_summary_table( output_table: str, classification_table: str, destination_table: str, + allowed_cases: list[str], rollup_report_base_path: str = constants.MISSING_COLUMNS_REPORT_PATH ) -> dict: """ @@ -1158,6 +1159,8 @@ def create_censorship_summary_table( classification_table: Fully qualified table with case/status/timestamp columns for every participant. destination_table: Fully qualified table to create with the summary. + allowed_cases: Cases that pass output_table's classification_filter + (e.g. ["1", "2", "3A", "3B"]). rollup_report_base_path: Base GCS path for the censorship rollup report. Returns: @@ -1182,6 +1185,7 @@ def create_censorship_summary_table( destination_table=destination_table, column_rule_map=column_rule_map, classification_table=classification_table, + allowed_cases=allowed_cases ) # Save the SQL to GCS for audit purposes, matching existing pipeline convention @@ -1257,6 +1261,7 @@ def create_censorship_summary_table( output_table="nih-nci-dceg-connect-stg-5519.SensitiveTier.mvp_0_1", classification_table="nih-nci-dceg-connect-stg-5519.pr2_mvp.classification", destination_table="nih-nci-dceg-connect-stg-5519.pr2_mvp.mvp_0_1_censorship_summary", + allowed_cases=["1", "2", "3A", "3B"], rollup_report_base_path="gs://pr2-pipeline-artifacts-stg/missing_columns_report/" ) From 85fbd44f822a1236116034c9b02a32abf2af8fa1 Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Fri, 31 Jul 2026 17:08:33 -0400 Subject: [PATCH 53/67] Look up the allowed_cases in the censorship summary endpoint --- core/endpoints.py | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/core/endpoints.py b/core/endpoints.py index a7daaf0..11e7401 100644 --- a/core/endpoints.py +++ b/core/endpoints.py @@ -4,7 +4,7 @@ from flask import Flask, jsonify, request # type: ignore -from core import constants, transformations, utils, request_helpers +from core import constants, transformations, utils, request_helpers, destination_table_builder app = Flask(__name__) @@ -166,13 +166,34 @@ def create_censorship_summary_table(): output_table = mapping.get("output_table") classification_table = mapping.get("classification_table") destination_table = mapping.get("destination_table") + config_path = mapping.get("config_path") + destination_dataset = mapping.get("destination_dataset") + censorship_table = mapping.get("censorship_table") try: - utils.logger.info(f"create_censorship_summary_table endpoint called. Creating {destination_table}.") + utils.logger.info(f"create_censorship_summary_table endpoint called. Creating {censorship_table}.") + destination_config = destination_table_builder.load_destination_config(config_path) + destination_table_config = destination_table_builder.get_destination_table_config( + destination_config, + destination_dataset, + censorship_table + ) + if not destination_table_config: + raise ValueError( + f"No config found for {destination_dataset}.{censorship_table}" + ) + allowed_cases = destination_table_config.get("classification_filter", {}).get("allowed_cases") + if not allowed_cases: + raise ValueError( + f"No classification_filter.allowed_cases found for " + f"{destination_dataset}.{censorship_table}" + ) + status = transformations.create_censorship_summary_table( output_table=output_table, classification_table=classification_table, - destination_table=destination_table + destination_table=destination_table, + allowed_cases=allowed_cases, ) return jsonify({ 'status': status, From e535aa15f4fc782cad259b6efe1dcb2e13e8ef44 Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Wed, 5 Aug 2026 15:41:23 -0400 Subject: [PATCH 54/67] Add a check to see if a destination table uses the classification_filter (allowed_cases is not empty) --- core/destination_table_builder.py | 23 ++++++++++++++++++++- core/endpoints.py | 34 ++++++++++++++++++++++++++++--- 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/core/destination_table_builder.py b/core/destination_table_builder.py index a31521e..81102bb 100644 --- a/core/destination_table_builder.py +++ b/core/destination_table_builder.py @@ -683,4 +683,25 @@ def create_destination_missing_columns_json( blob.upload_from_string( json.dumps(report, indent=2, ensure_ascii=False), content_type="application/json", - ) \ No newline at end of file + ) + +def table_has_classification_filter( + destination_config: dict[str, Any], + destination_dataset: str, + destination_table: str, +) -> bool: + """ + Checks whether a given destination table's config defines a + classification_filter with non-empty allowed_cases. Mirrors the + exact same allowed_cases resolution used inside + build_destination_table_query, so this can never disagree with + what that function actually does. + """ + destination_table_config = get_destination_table_config( + destination_config, destination_dataset, destination_table + ) + if not destination_table_config: + return False + classification_filter = destination_table_config.get("classification_filter") + allowed_cases = classification_filter.get("allowed_cases", []) if classification_filter else [] + return bool(allowed_cases) \ No newline at end of file diff --git a/core/endpoints.py b/core/endpoints.py index 11e7401..39a1d33 100644 --- a/core/endpoints.py +++ b/core/endpoints.py @@ -159,6 +159,26 @@ def create_destination_table(): utils.logger.exception("An error occurred in create_destination_table endpoint.") return jsonify({'error': 'Internal Server Error', 'message': str(e)}), 500 +@app.route('/check_has_classification_filter', methods=['POST']) +def check_has_classification_filter(): + mapping: dict[str, any] = request.get_json() or {} + config_path = mapping.get("config_path") + destination_dataset = mapping.get("destination_dataset") + destination_table = mapping.get("destination_table") + + try: + destination_config = destination_table_builder.load_destination_config(config_path) + has_filter = destination_table_builder.table_has_classification_filter( + destination_config, destination_dataset, destination_table + ) + return jsonify({ + 'has_classification_filter': has_filter, + 'timestamp': datetime.utcnow().isoformat(), + 'service': constants.SERVICE_NAME + }), 200 + except Exception as e: + utils.logger.exception("An error occurred in check_has_classification_filter endpoint.") + return jsonify({'error': 'Internal Server Error', 'message': str(e)}), 500 @app.route('/create_censorship_summary_table', methods=['POST']) def create_censorship_summary_table(): @@ -184,10 +204,18 @@ def create_censorship_summary_table(): ) allowed_cases = destination_table_config.get("classification_filter", {}).get("allowed_cases") if not allowed_cases: - raise ValueError( - f"No classification_filter.allowed_cases found for " - f"{destination_dataset}.{censorship_table}" + msg = ( + f"[{censorship_table}] Skipped: no classification_filter.allowed_cases " + f"configured. This table is not classification-gated, so no " + f"censorship summary applies" ) + utils.logger.info(msg) + return jsonify({ + 'status': msg, + 'skipped': True, + 'timestamp': datetime.utcnow().isoformat(), + 'service': constants.SERVICE_NAME + }), 200 status = transformations.create_censorship_summary_table( output_table=output_table, From 98e0a931cd6115867850342917c59171c9af44d5 Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Wed, 5 Aug 2026 15:44:45 -0400 Subject: [PATCH 55/67] Update destionation_config.json to include two test tables that do not use the classification_filter --- reference/destination_config.json | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/reference/destination_config.json b/reference/destination_config.json index c850f71..a480d96 100644 --- a/reference/destination_config.json +++ b/reference/destination_config.json @@ -67,12 +67,24 @@ }, { "dataset": "pr2_mvp", - "table": "mvp_synthetic_2026_07_15_case_1_2_3A_3B", + "table": "NO_CLASSIFICATION_EMPTY_LIST", "join_key": "Connect_ID", "filter_profile": "no_filters", "classification_filter": { - "allowed_cases": ["1", "2", "3A", "3B"] + "allowed_cases": [] + }, + "base_table": { + "dataset": "pr2_mvp", + "table": "mvp_synthetic", + "columns": "*" }, + "join_tables": [] + }, + { + "dataset": "pr2_mvp", + "table": "NO_CLASSIFICATION", + "join_key": "Connect_ID", + "filter_profile": "no_filters", "base_table": { "dataset": "pr2_mvp", "table": "mvp_synthetic", From fc43b9d32b1a0490879331f7d040dca073704c09 Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Wed, 5 Aug 2026 15:58:55 -0400 Subject: [PATCH 56/67] Update test destination tables in destination_config.json --- reference/destination_config.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/reference/destination_config.json b/reference/destination_config.json index a480d96..a2ffbed 100644 --- a/reference/destination_config.json +++ b/reference/destination_config.json @@ -66,7 +66,7 @@ "join_tables": [] }, { - "dataset": "pr2_mvp", + "dataset": "SensitiveTier", "table": "NO_CLASSIFICATION_EMPTY_LIST", "join_key": "Connect_ID", "filter_profile": "no_filters", @@ -75,19 +75,19 @@ }, "base_table": { "dataset": "pr2_mvp", - "table": "mvp_synthetic", + "table": "mvp", "columns": "*" }, "join_tables": [] }, { - "dataset": "pr2_mvp", + "dataset": "SensitiveTier", "table": "NO_CLASSIFICATION", "join_key": "Connect_ID", "filter_profile": "no_filters", "base_table": { "dataset": "pr2_mvp", - "table": "mvp_synthetic", + "table": "mvp", "columns": "*" }, "join_tables": [] From a27dcbb73830ae677f48ea43653f0cc7e7fa2362 Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Thu, 6 Aug 2026 13:45:22 -0400 Subject: [PATCH 57/67] Skip eligibility gating on module-censored columns for destination tables without a classification_filter, instead of generating invalid SQL --- core/censorship.py | 29 +++++++++++++++++++++++++++++ core/destination_table_builder.py | 6 +++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/core/censorship.py b/core/censorship.py index 98a1b65..5e80e02 100644 --- a/core/censorship.py +++ b/core/censorship.py @@ -53,7 +53,30 @@ def render_censored_column_expression( source_alias: str, col_name: str, classification_alias: str = "c", + has_classification: bool = True ) -> str: + """ + Renders the SELECT expression for a single column, applying + censorship rules only when classification is actually in effect + for this destination table. + + Args: + source_alias: Alias of the table this column is selected from. + col_name: The column name to render. + classification_alias: Alias of the classification table, only + meaningful when has_classification=True. + has_classification: Whether this destination table's query + actually joined a classification table + (i.e. allowed_cases was non-empty). When + False, module-censored columns are passed + through as plain SELECT expressions instead + of being wrapped in an eligibility IF(...), + since no {status_col}_eligible column exists + to reference in that case. + + Returns: + str: A SQL SELECT expression for this column, aliased to col_name. + """ first_cid = get_first_cid(col_name) if first_cid is None: @@ -68,6 +91,12 @@ def render_censored_column_expression( # and for cases 3A/3B before cutoff. rule = constants.MODULE_CENSOR_RULES.get(first_cid) if rule is not None: + if not has_classification: + # No classification join exists in this table's query, so + # there is no {status_col}_eligible column to reference. + # Pass the column through unfiltered rather than referencing + # a column that does not exist. + return f"{source_alias}.{col_name}" flag_name = f"{rule['status_col']}_eligible" return f"IF({classification_alias}.{flag_name}, {source_alias}.{col_name}, NULL) AS {col_name}" diff --git a/core/destination_table_builder.py b/core/destination_table_builder.py index 81102bb..fbaef86 100644 --- a/core/destination_table_builder.py +++ b/core/destination_table_builder.py @@ -414,6 +414,8 @@ def build_destination_table_query( classification_filter = destination_table_config.get("classification_filter") allowed_cases = classification_filter.get("allowed_cases", []) if classification_filter else [] + has_classification = bool(allowed_cases) + # Build WHERE clause from filter profile filter_sql = build_row_filter_sql( full_config.get("filter_profiles", {}), @@ -533,6 +535,7 @@ def build_destination_table_query( source_alias=base_alias, col_name=col, classification_alias=classification_alias, + has_classification=has_classification ) ) selected_column_names.add(col) @@ -582,6 +585,7 @@ def build_destination_table_query( source_alias=table, col_name=col, classification_alias=classification_alias, + has_classification=has_classification ) ) selected_column_names.add(col) @@ -612,7 +616,7 @@ def build_destination_table_query( sql = f""" /* Combined transformation query to build {project}.{destination_dataset}.{destination_table} from the destination configuration. - Applies participant classification, configured row filters, and censorship rules if configured. */ + Applies participant classification, row filters, and censorship rules if configured. */ CREATE OR REPLACE TABLE `{project}.{destination_dataset}.{destination_table}` AS WITH From 89620ee8ec00dde262f81fc360be64e915a487bb Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Fri, 7 Aug 2026 13:47:37 -0400 Subject: [PATCH 58/67] Add logging to Cloud Run service when a table does not have the classification_filter configured --- core/endpoints.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/core/endpoints.py b/core/endpoints.py index 39a1d33..8aa6638 100644 --- a/core/endpoints.py +++ b/core/endpoints.py @@ -166,6 +166,16 @@ def check_has_classification_filter(): destination_dataset = mapping.get("destination_dataset") destination_table = mapping.get("destination_table") + if has_filter: + utils.logger.info( + f"[{destination_table}] This table has classification_filter configured" + ) + else: + utils.logger.info( + f"[{destination_table}] No classification_filter configured; " + f"censorship summary will be skipped for this table" + ) + try: destination_config = destination_table_builder.load_destination_config(config_path) has_filter = destination_table_builder.table_has_classification_filter( From b6c20d285a076d0e14eaa20e1d9f37120f00797e Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Fri, 7 Aug 2026 14:06:59 -0400 Subject: [PATCH 59/67] Shift if statement into the try block --- core/endpoints.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/core/endpoints.py b/core/endpoints.py index 8aa6638..574937f 100644 --- a/core/endpoints.py +++ b/core/endpoints.py @@ -166,21 +166,22 @@ def check_has_classification_filter(): destination_dataset = mapping.get("destination_dataset") destination_table = mapping.get("destination_table") - if has_filter: - utils.logger.info( - f"[{destination_table}] This table has classification_filter configured" - ) - else: - utils.logger.info( - f"[{destination_table}] No classification_filter configured; " - f"censorship summary will be skipped for this table" - ) - try: destination_config = destination_table_builder.load_destination_config(config_path) has_filter = destination_table_builder.table_has_classification_filter( destination_config, destination_dataset, destination_table ) + + if has_filter: + utils.logger.info( + f"[{destination_table}] This table has classification_filter configured" + ) + else: + utils.logger.info( + f"[{destination_table}] No classification_filter configured; " + f"censorship summary will be skipped for this table" + ) + return jsonify({ 'has_classification_filter': has_filter, 'timestamp': datetime.utcnow().isoformat(), From dba8ab3c4da3152e35a15a5eea84321697ed7530 Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Fri, 7 Aug 2026 14:37:52 -0400 Subject: [PATCH 60/67] Remove 'token' as a column to include --- core/constants.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/core/constants.py b/core/constants.py index afd1ff9..3bcce65 100644 --- a/core/constants.py +++ b/core/constants.py @@ -276,6 +276,4 @@ "104913069": {"source_table": "participants"}, # Research- Finalization and shipping } -ALWAYS_INCLUDE_NON_CID_COLUMNS = { - "token", -} \ No newline at end of file +ALWAYS_INCLUDE_NON_CID_COLUMNS = {} \ No newline at end of file From d6560f9ace550ace385d185cd37bc2f00a47573b Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Mon, 10 Aug 2026 16:09:42 -0400 Subject: [PATCH 61/67] Add test table that is using the filter_profile and classification_filter --- reference/destination_config.json | 41 +++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/reference/destination_config.json b/reference/destination_config.json index a2ffbed..3c96e8e 100644 --- a/reference/destination_config.json +++ b/reference/destination_config.json @@ -91,6 +91,47 @@ "columns": "*" }, "join_tables": [] + }, + { + "dataset": "SensitiveTier", + "table": "mvp_test_filters", + "join_key": "Connect_ID", + "filter_profile": "default_participants", + "classification_filter": { + "allowed_cases": ["1", "2", "3A", "3B"] + }, + "base_table": { + "dataset": "CleanConnect", + "table": "participants", + "columns": [ + "d_130371375_d_303552867_d_297462035", + "fakecolumn", + "d_130371375_d_303552867_20", + "d_130371375_d_303552867_20", + "d_130371375_d_303552867_d_320023644", + "d_130371375_d_303552867_d_438636757", + "d_130371375_d_303552867_d_438636757" + ] + }, + "join_tables": [ + { + "dataset": "pr2_mvp", + "table": "mvp", + "columns": [ + "d_726699695_d_384191091_d_746038746", + "d_726699695_d_384191091_d_807835037", + "d_965707586_d_947205597_d_712653855", + "d_965707586_d_976570371" + ] + }, + { + "dataset": "CleanConnect", + "table": "module1", + "columns": [ + "d_103397024_d_206625031" + ] + } + ] } ] } \ No newline at end of file From fd805ff1bf98c69a5195c4d620d9c329073507db Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Thu, 13 Aug 2026 10:54:08 -0400 Subject: [PATCH 62/67] Add test table where no classification and row filtering are present but unrecognized CIDs are --- reference/destination_config.json | 41 +++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/reference/destination_config.json b/reference/destination_config.json index 3c96e8e..d1fc5ca 100644 --- a/reference/destination_config.json +++ b/reference/destination_config.json @@ -132,6 +132,47 @@ ] } ] + }, + { + "dataset": "SensitiveTier", + "table": "mvp_test_NO_FILTERS", + "join_key": "Connect_ID", + "filter_profile": "no_filters", + "classification_filter": { + "allowed_cases": [] + }, + "base_table": { + "dataset": "CleanConnect", + "table": "participants", + "columns": [ + "d_130371375_d_303552867_d_297462035", + "fakecolumn", + "d_130371375_d_303552867_20", + "d_130371375_d_303552867_20", + "d_130371375_d_303552867_d_320023644", + "d_130371375_d_303552867_d_438636757", + "d_130371375_d_303552867_d_438636757" + ] + }, + "join_tables": [ + { + "dataset": "pr2_mvp", + "table": "mvp", + "columns": [ + "d_726699695_d_384191091_d_746038746", + "d_726699695_d_384191091_d_807835037", + "d_965707586_d_947205597_d_712653855", + "d_965707586_d_976570371" + ] + }, + { + "dataset": "CleanConnect", + "table": "module1", + "columns": [ + "d_103397024_d_206625031" + ] + } + ] } ] } \ No newline at end of file From b1c8cc021c94f50742cf4056fb7f7f3e45819c01 Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Fri, 14 Aug 2026 13:26:12 -0400 Subject: [PATCH 63/67] Remove bio_cutoff code since it is currently not in use --- core/classification.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/core/classification.py b/core/classification.py index a2e824c..c037ec7 100644 --- a/core/classification.py +++ b/core/classification.py @@ -139,7 +139,7 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: # Rule: Revoke timestamp occurs AFTER withdraw timestamp # This creates ambiguity because EHR cutoff cannot be reliably determined. # Participant is still processed as Case 3A using withdraw timestamp for - # survey/bio cutoffs, but anomaly is recorded. + # survey cutoff, but anomaly is recorded. anomaly_revoke_after_withdraw = destroy_no & withdraw_yes & revoke_yes & revoke_after_withdraw # Rule: Revoke timestamp is EQUAL to withdraw timestamp @@ -292,21 +292,19 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: # Cutoff logic df["ehr_cutoff"] = pd.Series(pd.NaT, index=df.index, dtype="datetime64[us, UTC]") df["survey_cutoff"] = pd.Series(pd.NaT, index=df.index, dtype="datetime64[us, UTC]") - df["bio_cutoff"] = pd.Series(pd.NaT, index=df.index, dtype="datetime64[us, UTC]") # Case 1: No cutoffs, so leave all columns as NaT (as initialized previously). # Participant is included downstream without restrictions. No further # code is needed here for this case. - # Case 2: EHR cutoff = Revoke timestamp, Survey/Bio cutoff = None + # Case 2: EHR cutoff = Revoke timestamp, Survey cutoff = None df.loc[case_2, "ehr_cutoff"] = df.loc[case_2, "hipaa_revoked_ts"] - # Case 3A: EHR/Survey/Bio cutoff = Withdraw timestamp + # Case 3A: EHR/Survey cutoff = Withdraw timestamp normal_3a = case_3a & df["hipaa_revoked_ts"].isna() df.loc[normal_3a, "ehr_cutoff"] = df.loc[normal_3a, "consent_withdrawn_ts"] df.loc[normal_3a, "survey_cutoff"] = df.loc[normal_3a, "consent_withdrawn_ts"] - df.loc[normal_3a, "bio_cutoff"] = df.loc[normal_3a, "consent_withdrawn_ts"] # Case 3A anomaly - revoke AFTER withdraw: ehr_cutoff stays NULL. # If revoke_ts is later than withdraw_ts, keep as anomaly and leave @@ -314,7 +312,6 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: anomalous_3a_revoke_after = case_3a & revoke_after_withdraw df.loc[anomalous_3a_revoke_after, "survey_cutoff"] = df.loc[anomalous_3a_revoke_after, "consent_withdrawn_ts"] - df.loc[anomalous_3a_revoke_after, "bio_cutoff"] = df.loc[anomalous_3a_revoke_after, "consent_withdrawn_ts"] # Case 3A anomaly - revoke EQUALS withdraw: ehr_cutoff CAN be reliably # set since revoke and withdraw happened at the same instant. Use @@ -323,12 +320,10 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: df.loc[anomalous_3a_revoke_equals, "ehr_cutoff"] = df.loc[anomalous_3a_revoke_equals, "consent_withdrawn_ts"] df.loc[anomalous_3a_revoke_equals, "survey_cutoff"] = df.loc[anomalous_3a_revoke_equals, "consent_withdrawn_ts"] - df.loc[anomalous_3a_revoke_equals, "bio_cutoff"] = df.loc[anomalous_3a_revoke_equals, "consent_withdrawn_ts"] - # Case 3B: EHR cutoff = Revoke timestamp, Survey/Bio cutoff = Withdraw timestamp + # Case 3B: EHR cutoff = Revoke timestamp, Survey cutoff = Withdraw timestamp df.loc[case_3b, "ehr_cutoff"] = df.loc[case_3b, "hipaa_revoked_ts"] df.loc[case_3b, "survey_cutoff"] = df.loc[case_3b, "consent_withdrawn_ts"] - df.loc[case_3b, "bio_cutoff"] = df.loc[case_3b, "consent_withdrawn_ts"] # Case 4: No cutoffs, so leave all columns as NaT (as initialized previously). # Participant is excluded downstream. No further code is needed here for this case. @@ -405,7 +400,6 @@ def write_classification_to_bq( # Derived cutoffs bigquery.SchemaField("ehr_cutoff", "TIMESTAMP"), bigquery.SchemaField("survey_cutoff", "TIMESTAMP"), - bigquery.SchemaField("bio_cutoff", "TIMESTAMP"), # Helper field bigquery.SchemaField("sort_order", "INTEGER"), From 4ce692a281c62e90aa49b5c589c85b50880c4485 Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Fri, 14 Aug 2026 14:04:10 -0400 Subject: [PATCH 64/67] Fix comment typographical error --- core/classification.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/classification.py b/core/classification.py index c037ec7..6e4be53 100644 --- a/core/classification.py +++ b/core/classification.py @@ -28,8 +28,8 @@ def read_parquet_to_dataframe(parquet_url: str) -> pd.DataFrame: engine="pyarrow", filters=[("verified_status", "==", "Verified")]) - # Normalize timestamp columns - # Black strings, nulls, and invalid timestamps become NaT + # Normalize timestamp columns. + # Blank strings, nulls, and invalid timestamps become NaT. ts_cols = ["consent_withdrawn_ts", "hipaa_revoked_ts", "data_destruction_ts", "module1_complete_ts", "module2_complete_ts", "module3_complete_ts", "module4_complete_ts", #"bio_complete_ts", "clinicalbio_complete_ts", "mouthwash_complete_ts", From 5639519234c992764eecf08c22cbada4085d4ab4 Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Fri, 14 Aug 2026 14:08:42 -0400 Subject: [PATCH 65/67] Add a comment in constants.py and classification.py explaining why BUM, BU, and Mouthwash modules and timestamps are commented out --- core/classification.py | 6 ++++++ core/constants.py | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/core/classification.py b/core/classification.py index 6e4be53..cfc65f9 100644 --- a/core/classification.py +++ b/core/classification.py @@ -32,6 +32,9 @@ def read_parquet_to_dataframe(parquet_url: str) -> pd.DataFrame: # Blank strings, nulls, and invalid timestamps become NaT. ts_cols = ["consent_withdrawn_ts", "hipaa_revoked_ts", "data_destruction_ts", "module1_complete_ts", "module2_complete_ts", "module3_complete_ts", + # Blood/Urine/Mouthwash, Blood/Urine, and Mouthwash are commented out + # because it is unclear how the follow-up specimens will be handled. + # For now, these modules with thier timestamps are excluded. "module4_complete_ts", #"bio_complete_ts", "clinicalbio_complete_ts", "mouthwash_complete_ts", "menstrual_complete_ts", "covid19_complete_ts", "experience2024_complete_ts"] @@ -385,6 +388,9 @@ def write_classification_to_bq( bigquery.SchemaField("module2_complete_ts", "TIMESTAMP"), bigquery.SchemaField("module3_complete_ts", "TIMESTAMP"), bigquery.SchemaField("module4_complete_ts", "TIMESTAMP"), + # Blood/Urine/Mouthwash, Blood/Urine, and Mouthwash are commented out + # because it is unclear how the follow-up specimens will be handled. + # For now, these modules with thier timestamps are excluded. #bigquery.SchemaField("bio_complete_ts", "TIMESTAMP"), #bigquery.SchemaField("clinicalbio_complete_ts", "TIMESTAMP"), #bigquery.SchemaField("mouthwash_complete_ts", "TIMESTAMP"), diff --git a/core/constants.py b/core/constants.py index 3bcce65..7b55943 100644 --- a/core/constants.py +++ b/core/constants.py @@ -216,6 +216,10 @@ "completion_ts": "module4_complete_ts", "cutoff": "survey_cutoff", }, + # Blood/Urine/Mouthwash, Blood/Urine, and Mouthwash are commented out + # because it is unclear how the follow-up specimens will be handled. + # For now, these modules and their timestamps are excluded. + # # Blood/Urine/Mouthwash #"299215535": { # "source_table": "bioSurvey", From b2603ef3387d6d4848cfe928b7fbb4547baae451 Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Fri, 14 Aug 2026 14:53:37 -0400 Subject: [PATCH 66/67] Remove references to biospecimen cutoff in docstrings and comments --- core/classification.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/core/classification.py b/core/classification.py index cfc65f9..d54230f 100644 --- a/core/classification.py +++ b/core/classification.py @@ -53,9 +53,9 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: timestamp combinations are marked as data-quality exclusions or undefined. - The function also computes the appropriate EHR, survey, and - biospecimen cutoff timestamps along with any anomaly or exclusion - reason needed for downstream processing. + The function also determines the appropriate EHR and survey cutoff + timestamps along with any anomaly or exclusion reason needed for + downstream processing. Args: df (pd.DataFrame): Participant status DataFrame. @@ -181,7 +181,6 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: # # EHR cutoff = Revoke timestamp # Survey cutoff = None - # Biospecimen = None # ============================================================================ case_2 = ( destroy_no @@ -202,7 +201,6 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: # EHR cutoff = NULL for the revoke-after-withdraw anomaly # EHR cutoff = Withdraw timestamp for the revoke-equals-withdraw anomaly # Survey cutoff = Withdraw timestamp - # Biospecimen cutoff = Withdraw timestamp # ============================================================================ case_3a = ( destroy_no @@ -224,7 +222,6 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: # # EHR cutoff = Revoke timestamp # Survey cutoff = Withdraw timestamp - # Biospecimen cutoff = Withdraw timestamp # ============================================================================ case_3b = ( destroy_no From fd92d4d7807c4bbded8b3c136941edaae385e073 Mon Sep 17 00:00:00 2001 From: gloria-trivitt Date: Tue, 18 Aug 2026 13:58:53 -0400 Subject: [PATCH 67/67] Rename the 'case' column/variable/parameter name to 'consent_group' across classify_participants, the destination table query builder, the censorship summary SQL, Flask endpoints, and destination_config.json (allowed_cases -> allowed_consent_groups) --- core/censorship.py | 72 ++++++++++----------- core/classification.py | 103 +++++++++++++++--------------- core/destination_table_builder.py | 38 +++++------ core/endpoints.py | 8 +-- core/transformations.py | 23 +++---- reference/destination_config.json | 10 +-- 6 files changed, 127 insertions(+), 127 deletions(-) diff --git a/core/censorship.py b/core/censorship.py index 5e80e02..74cbf7f 100644 --- a/core/censorship.py +++ b/core/censorship.py @@ -67,8 +67,8 @@ def render_censored_column_expression( meaningful when has_classification=True. has_classification: Whether this destination table's query actually joined a classification table - (i.e. allowed_cases was non-empty). When - False, module-censored columns are passed + (i.e. allowed_consent_groups was non-empty). + When False, module-censored columns are passed through as plain SELECT expressions instead of being wrapped in an eligibility IF(...), since no {status_col}_eligible column exists @@ -88,7 +88,7 @@ def render_censored_column_expression( return f"{source_alias}.{col_name}" # Known module columns: keep only if module status is "Submitted" - # and for cases 3A/3B before cutoff. + # and for Consent Groups 3A/3B before cutoff. rule = constants.MODULE_CENSOR_RULES.get(first_cid) if rule is not None: if not has_classification: @@ -132,7 +132,7 @@ def _build_reason_cte_block( status_col: str, rule: dict, classification_table: str, - allowed_cases: list[str], + allowed_consent_groups: list[str], classification_alias: str = "c" ) -> str: """ @@ -145,12 +145,12 @@ def _build_reason_cte_block( {status_col: rule} mapping built by build_censorship_summary_sql's dedup step (keys: status_col, completion_ts, cutoff). classification_table: Fully qualified classification table. - allowed_cases: Cases that pass output_table's classification_filter - (e.g. ["1", "2", "3A", "3B"]). + allowed_consent_groups: Consent groups that pass output_table's + classification_filter (e.g. ["1", "2", "3A", "3B"]). classification_alias: Alias of the classification table. Returns: - str: A SELECT statement returning Connect_ID, case, status_col, + str: A SELECT statement returning Connect_ID, consent_group, status_col, and reason. """ completion_ts = rule["completion_ts"] @@ -159,12 +159,12 @@ def _build_reason_cte_block( status_col, completion_ts, cutoff, alias=classification_alias ) - allowed_cases_sql = ", ".join(f"'{c}'" for c in allowed_cases) + allowed_consent_groups_sql = ", ".join(f"'{c}'" for c in allowed_consent_groups) return f""" SELECT {classification_alias}.Connect_ID, - {classification_alias}.`case`, + {classification_alias}.consent_group, '{status_col}' AS status_col, CASE -- The eligibility decision itself is delegated to @@ -176,7 +176,7 @@ def _build_reason_cte_block( THEN CASE -- Reason 1: Status is not "Submitted" (NULL, empty, -- or any other value). This alone is disqualifying - -- regardless of case. + -- regardless of consent_group. WHEN {classification_alias}.{status_col} IS DISTINCT FROM 'Submitted' THEN CASE WHEN {classification_alias}.{status_col} IS NULL @@ -186,10 +186,10 @@ def _build_reason_cte_block( IF({classification_alias}.{status_col} = '', ' (empty string)', '') ) END - -- Reason 2: Status was "Submitted", but for cases 3A/3B the - -- completion has to land strictly before the cutoff. Missing - -- timestamps or a completion on/after cutoff both count as - -- censored. Treat NULL and "too late" as distinct + -- Reason 2: Status was "Submitted", but for Consent Groups 3A/3B + -- the completion has to land strictly before the cutoff. + -- Missing timestamps or a completion on/after cutoff both count + -- as censored. Treat NULL and "too late" as distinct -- sub-reasons for a clearer message. ELSE CASE WHEN {classification_alias}.{completion_ts} IS NULL @@ -208,7 +208,7 @@ def _build_reason_cte_block( ELSE NULL END AS reason FROM `{classification_table}` {classification_alias} - WHERE {classification_alias}.`case` IN ({allowed_cases_sql}) + WHERE {classification_alias}.consent_group IN ({allowed_consent_groups_sql}) """.strip() def build_censorship_summary_sql( @@ -216,7 +216,7 @@ def build_censorship_summary_sql( destination_table: str, column_rule_map: dict[str, dict], classification_table: str, - allowed_cases: list[str], + allowed_consent_groups: list[str], classification_alias: str = "c" ) -> str: """ @@ -230,15 +230,15 @@ def build_censorship_summary_sql( destination_table: Fully qualified table to write the censorship summary for. column_rule_map: {column_name: rule_dict}, as produced by build_column_rule_map. - classification_table: Fully qualified table containing case, status, + classification_table: Fully qualified table containing consent_group, status, completion_ts, and cutoff columns for every participant. - allowed_cases: Cases that pass output_table's classification_filter - (e.g. ["1", "2", "3A", "3B"]). + allowed_consent_groups: Consent groups that pass output_table's classification_filter + (e.g. ["1", "2", "3A", "3B"]). classification_alias: Alias used for that table in the generated SQL. Returns: - str: A complete SQL query returning Connect_ID, case, column_name, + str: A complete SQL query returning Connect_ID, consent_group, column_name, and reason for every censorship event, filtered to reason IS NOT NULL. Raises: @@ -267,7 +267,7 @@ def build_censorship_summary_sql( rule=rule, classification_table=classification_table, classification_alias=classification_alias, - allowed_cases=allowed_cases, + allowed_consent_groups=allowed_consent_groups, ) for status_col, rule in status_col_to_rule.items() ] @@ -296,7 +296,7 @@ def build_censorship_summary_sql( ) SELECT reasons.Connect_ID, - reasons.`case`, + reasons.consent_group, column_status_map.column_name, reasons.reason FROM reasons @@ -313,13 +313,13 @@ def get_censorship_rollup(destination_table: str) -> dict: """ Queries the censorship summary table and returns two rollups: 1. Per-column counts (how many participants were censored, per column) - 2. Per-case counts (how many censorship events occurred, per case) + 2. Per-consent-group counts (how many censorship events occurred, per consent group) Args: destination_table: The table created by create_censorship_summary_table. Returns: - dict: {"by_column": [...], "by_case": [...]} + dict: {"by_column": [...], "by_consent_group": [...]} """ client = bigquery.Client() @@ -329,24 +329,24 @@ def get_censorship_rollup(destination_table: str) -> dict: GROUP BY column_name ORDER BY censored_count DESC """ - by_case_sql = f""" - SELECT `case`, COUNT(*) AS censorship_events + by_consent_group_sql = f""" + SELECT consent_group, COUNT(*) AS censorship_events FROM `{destination_table}` - GROUP BY `case` - ORDER BY `case` + GROUP BY consent_group + ORDER BY consent_group """ by_column = [dict(row) for row in client.query(by_column_sql).result()] - by_case = [dict(row) for row in client.query(by_case_sql).result()] + by_consent_group = [dict(row) for row in client.query(by_consent_group_sql).result()] - return {"by_column": by_column, "by_case": by_case} + return {"by_column": by_column, "by_consent_group": by_consent_group} def create_censorship_rollup_json( client: storage.Client, output_path: str, destination_table: str, by_column: list[dict[str, Any]], - by_case: list[dict[str, Any]], + by_consent_group: list[dict[str, Any]], ) -> None: """ Write a JSON report of censorship rollup counts to a GCS location. @@ -358,8 +358,8 @@ def create_censorship_rollup_json( get_censorship_rollup was queried against). by_column (list): Per-column censored counts, as returned by get_censorship_rollup()["by_column"]. - by_case (list): Per-case censorship event counts, as returned by - get_censorship_rollup()["by_case"]. + by_consent_group (list): Per-consent-group censorship event counts, as returned by + get_censorship_rollup()["by_consent_group"]. """ # Build report structure report = { @@ -367,14 +367,14 @@ def create_censorship_rollup_json( "generated_at": datetime.now(timezone.utc).isoformat(), "source": "pr2-transformation pipeline", "destination_table": destination_table, - "description": "Rollup of censorship events from the censorship summary table, broken down by column and by case", + "description": "Rollup of censorship events from the censorship summary table, broken down by column and by consent group", "structure": { "by_column": "Per-column counts of how many participants were censored for that column, ordered by censored_count descending", - "by_case": "Per-case counts of how many censorship events occurred, ordered by case" + "by_consent_group": "Per-consent-group counts of how many censorship events occurred, ordered by consent_group" } }, "by_column": by_column, - "by_case": by_case, + "by_consent_group": by_consent_group, } # Parse GCS path diff --git a/core/classification.py b/core/classification.py index d54230f..bb4fba7 100644 --- a/core/classification.py +++ b/core/classification.py @@ -45,13 +45,12 @@ def read_parquet_to_dataframe(parquet_url: str) -> pd.DataFrame: def classify_participants(df: pd.DataFrame) -> pd.DataFrame: """ - Classify participants into consent and data-retention cases. + Classify participants into consent and data-retention groups. Evaluates each participant's HIPAA revocation, consent withdrawn, - and data destruction status to assign one of the supported business - rule cases (1, 2, 3A, 3B, or 4). Participants with invalid flag or - timestamp combinations are marked as data-quality exclusions or - undefined. + and data destruction status to assign one of the consent groups + (1, 2, 3A, 3B, or 4). Participants with invalid flag or timestamp + combinations are marked as data-quality exclusions or undefined. The function also determines the appropriate EHR and survey cutoff timestamps along with any anomaly or exclusion reason needed for @@ -85,7 +84,7 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: revoke_ts_missing = df["hipaa_revoked_ts"].isna() # ========================================================================= - # Timestamp validation for case 3 + # Timestamp validation for Consent Group 3 # ========================================================================= # revoke_after_withdraw: True when revoke timestamp exists AND is later than withdraw timestamp @@ -107,7 +106,7 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: ) # revoke_before_withdraw: True when revoke timestamp exists AND occurs before withdraw timestamp - # This is the expected ordering (valid) case 3B scenario + # This is the expected ordering (valid) Consent Group 3B scenario revoke_before_withdraw = ( df["hipaa_revoked_ts"].notna() & df["consent_withdrawn_ts"].notna() @@ -135,13 +134,13 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: # ========================================================================= # Rule: Destory Data = Yes but destroy timestamp is NULL - # Participant is still classified as Case 4 and excluded downstream, + # Participant is still classified as Consent Group 4 and excluded downstream, # but the missing timestamp is recorded as an anomaly. anomaly_destroy_missing_ts = destroy_yes & withdraw_yes & revoke_yes & destroy_ts_missing # Rule: Revoke timestamp occurs AFTER withdraw timestamp # This creates ambiguity because EHR cutoff cannot be reliably determined. - # Participant is still processed as Case 3A using withdraw timestamp for + # Participant is still processed as Consent Group 3A using withdraw timestamp for # survey cutoff, but anomaly is recorded. anomaly_revoke_after_withdraw = destroy_no & withdraw_yes & revoke_yes & revoke_after_withdraw @@ -154,12 +153,12 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: anomaly_revoke_equals_withdraw = destroy_no & withdraw_yes & revoke_yes & revoke_equals_withdraw # ============================================================================ - # Case classification masks - # These masks determine which business-rule case each participant belongs to. + # Consent group classification masks + # These masks determine which consent group each participant belongs to. # ============================================================================ # ============================================================================ - # Case 1 + # Consent Group 1 # - Destroy Data = No # - Withdraw Consent = No # - Revoke HIPAA = No @@ -167,14 +166,14 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: # No restrictions apply. # Participant and all their data can be included without any cutoffs. # ============================================================================ - case_1 = ( + consent_group_1 = ( destroy_no & withdraw_no & revoke_no ) # ============================================================================ - # Case 2 + # Consent Group 2 # - Destroy Data = No # - Withdraw Consent = No # - Revoke HIPAA = Yes (timestamp present) @@ -182,7 +181,7 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: # EHR cutoff = Revoke timestamp # Survey cutoff = None # ============================================================================ - case_2 = ( + consent_group_2 = ( destroy_no & withdraw_no & revoke_yes @@ -190,19 +189,19 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: ) # ============================================================================ - # Case 3A + # Consent Group 3A # - Destroy Data = No # - Withdraw Consent = Yes (timestamp present) # - Revoke HIPAA = Yes (timestamp is either missing, occurs strictly # after withdraw timestamp, or occurs at exactly the same time as # withdraw timestamp) # - # EHR cutoff = Withdraw timestamp for normal 3A cases + # EHR cutoff = Withdraw timestamp for normal Consent Group 3A # EHR cutoff = NULL for the revoke-after-withdraw anomaly # EHR cutoff = Withdraw timestamp for the revoke-equals-withdraw anomaly # Survey cutoff = Withdraw timestamp # ============================================================================ - case_3a = ( + consent_group_3a = ( destroy_no & withdraw_yes & revoke_yes @@ -215,7 +214,7 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: ) # ============================================================================ - # Case 3B + # Consent Group 3B # - Destroy Data = No # - Withdraw Consent = Yes (timestamp present and occurs AFTER revoke timestamp) # - Revoke HIPAA = Yes (timestamp present and occurs BEFORE withdraw timestamp) @@ -223,7 +222,7 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: # EHR cutoff = Revoke timestamp # Survey cutoff = Withdraw timestamp # ============================================================================ - case_3b = ( + consent_group_3b = ( destroy_no & withdraw_yes & revoke_yes @@ -232,22 +231,22 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: ) # ============================================================================ - # Case 4 + # Consent Group 4 # - Destroy Data = Yes # - Withdraw Consent = Yes # - Revoke HIPAA = Yes # # The participant and all their data is excluded from downstream processing # ============================================================================ - case_4 = ( + consent_group_4 = ( destroy_yes & withdraw_yes & revoke_yes ) - # Build output classification column based on the above cases and rules - df["case"] = np.select( - [case_1, case_2, case_3a, case_3b, case_4, data_quality_mask], + # Build output classification column based on the above consent groups and rules + df["consent_group"] = np.select( + [consent_group_1, consent_group_2, consent_group_3a, consent_group_3b, consent_group_4, data_quality_mask], ["1", "2", "3A", "3B", "4", "DATA_QUALITY_EXCLUSION"], default="UNDEFINED" ) @@ -268,12 +267,12 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: # Assign reason for unmatched participants df.loc[ - df["case"] == "UNDEFINED", + df["consent_group"] == "UNDEFINED", "exclusion_reason" - ] = "Participant flag combination did not match any defined case" + ] = "Participant flag combination did not match any defined consent group" # Build anomaly column to specify which rule was violated for - # participants classified as cases 4 or 3A but have internal + # participants classified as Consent Groups 4 or 3A but have internal # inconsistencies in their data df["anomaly"] = np.select( [ @@ -293,51 +292,51 @@ def classify_participants(df: pd.DataFrame) -> pd.DataFrame: df["ehr_cutoff"] = pd.Series(pd.NaT, index=df.index, dtype="datetime64[us, UTC]") df["survey_cutoff"] = pd.Series(pd.NaT, index=df.index, dtype="datetime64[us, UTC]") - # Case 1: No cutoffs, so leave all columns as NaT (as initialized previously). + # Consent Group 1: No cutoffs, so leave all columns as NaT (as initialized previously). # Participant is included downstream without restrictions. No further - # code is needed here for this case. + # code is needed here for this consent group. - # Case 2: EHR cutoff = Revoke timestamp, Survey cutoff = None - df.loc[case_2, "ehr_cutoff"] = df.loc[case_2, "hipaa_revoked_ts"] + # Consent Group 2: EHR cutoff = Revoke timestamp, Survey cutoff = None + df.loc[consent_group_2, "ehr_cutoff"] = df.loc[consent_group_2, "hipaa_revoked_ts"] - # Case 3A: EHR/Survey cutoff = Withdraw timestamp - normal_3a = case_3a & df["hipaa_revoked_ts"].isna() + # Consent Group 3A: EHR/Survey cutoff = Withdraw timestamp + normal_3a = consent_group_3a & df["hipaa_revoked_ts"].isna() df.loc[normal_3a, "ehr_cutoff"] = df.loc[normal_3a, "consent_withdrawn_ts"] df.loc[normal_3a, "survey_cutoff"] = df.loc[normal_3a, "consent_withdrawn_ts"] - # Case 3A anomaly - revoke AFTER withdraw: ehr_cutoff stays NULL. + # Consent Group 3A anomaly - revoke AFTER withdraw: ehr_cutoff stays NULL. # If revoke_ts is later than withdraw_ts, keep as anomaly and leave - # ehr_cutoff blank. This will still be processed as a 3A case. - anomalous_3a_revoke_after = case_3a & revoke_after_withdraw + # ehr_cutoff blank. This will still be processed as Consent Group 3A. + anomalous_3a_revoke_after = consent_group_3a & revoke_after_withdraw df.loc[anomalous_3a_revoke_after, "survey_cutoff"] = df.loc[anomalous_3a_revoke_after, "consent_withdrawn_ts"] - # Case 3A anomaly - revoke EQUALS withdraw: ehr_cutoff CAN be reliably + # Consent Group 3A anomaly - revoke EQUALS withdraw: ehr_cutoff CAN be reliably # set since revoke and withdraw happened at the same instant. Use - # withdraw_ts for all three cutoffs. This will be processed as a 3A case. - anomalous_3a_revoke_equals = case_3a & revoke_equals_withdraw + # withdraw_ts for all cutoffs. This will be processed as Consent Group 3A. + anomalous_3a_revoke_equals = consent_group_3a & revoke_equals_withdraw df.loc[anomalous_3a_revoke_equals, "ehr_cutoff"] = df.loc[anomalous_3a_revoke_equals, "consent_withdrawn_ts"] df.loc[anomalous_3a_revoke_equals, "survey_cutoff"] = df.loc[anomalous_3a_revoke_equals, "consent_withdrawn_ts"] - # Case 3B: EHR cutoff = Revoke timestamp, Survey cutoff = Withdraw timestamp - df.loc[case_3b, "ehr_cutoff"] = df.loc[case_3b, "hipaa_revoked_ts"] - df.loc[case_3b, "survey_cutoff"] = df.loc[case_3b, "consent_withdrawn_ts"] + # Consent Group 3B: EHR cutoff = Revoke timestamp, Survey cutoff = Withdraw timestamp + df.loc[consent_group_3b, "ehr_cutoff"] = df.loc[consent_group_3b, "hipaa_revoked_ts"] + df.loc[consent_group_3b, "survey_cutoff"] = df.loc[consent_group_3b, "consent_withdrawn_ts"] - # Case 4: No cutoffs, so leave all columns as NaT (as initialized previously). - # Participant is excluded downstream. No further code is needed here for this case. + # Consent Group 4: No cutoffs, so leave all columns as NaT (as initialized previously). + # Participant is excluded downstream. No further code is needed here for this consent group. # Sort the dataframe for code development and debugging purposes. This can be removed in the future. - df = df.sort_values(by=["exclusion_reason", "anomaly", "case", "Connect_ID"]) + df = df.sort_values(by=["exclusion_reason", "anomaly", "consent_group", "Connect_ID"]) # Sort dataframe in order of: - # 1. Undefined cases - # 2. Data quality exclusion cases - # 3. Cases with anomalies - # 4. Normal cases + # 1. Undefined + # 2. Consent groups with data quality exclusion + # 3. Consent groups with anomalies + # 4. Normal consent groups df["sort_order"] = np.select( - [df["case"].eq("UNDEFINED"), df["case"].eq("DATA_QUALITY_EXCLUSION"), df["anomaly"].notna()], + [df["consent_group"].eq("UNDEFINED"), df["consent_group"].eq("DATA_QUALITY_EXCLUSION"), df["anomaly"].notna()], [1, 2, 3], default=4 ) @@ -396,7 +395,7 @@ def write_classification_to_bq( bigquery.SchemaField("experience2024_complete_ts", "TIMESTAMP"), # Classification fields - bigquery.SchemaField("case", "STRING"), + bigquery.SchemaField("consent_group", "STRING"), bigquery.SchemaField("exclusion_reason", "STRING"), bigquery.SchemaField("anomaly", "STRING"), diff --git a/core/destination_table_builder.py b/core/destination_table_builder.py index fbaef86..6bbe957 100644 --- a/core/destination_table_builder.py +++ b/core/destination_table_builder.py @@ -239,8 +239,8 @@ def build_row_filter_sql( it against filter_profiles, and builds a WHERE clause from its column/ value conditions. This is only one of two ways a table's rows can end up restricted. build_destination_table_query separately adds a - join-key-not-null condition and (if allowed_cases is set) a - classification-case filter. A table using the "no_filters" profile can + join-key-not-null condition and (if allowed_consent_groups is set) a + classification filter. A table using the "no_filters" profile can still be row-restricted by those other conditions; this function only reports on the conditions it itself contributes. @@ -349,7 +349,7 @@ def render_eligibility_condition( ) -> str: """ The single source of truth for "is {status_col} eligible" — status is - Submitted, and for 3A/3B cases, completion happened strictly before + Submitted, and for Consent Groups 3A/3B, completion happened strictly before cutoff. Used both to build the *_eligible flags in build_destination_table_query's CTE and to determine the not-censored/censored branch in _build_reason_cte_block, so the real @@ -368,7 +368,7 @@ def render_eligibility_condition( return f"""( {alias}.{status_col} = 'Submitted' AND ( - {alias}.`case` NOT IN ('3A', '3B') + {alias}.consent_group NOT IN ('3A', '3B') OR ( {alias}.{completion_ts} IS NOT NULL AND {alias}.{cutoff} IS NOT NULL @@ -402,7 +402,7 @@ def build_destination_table_query( Raises: ValueError: If no columns are selected or if classification_table is missing - when allowed_cases is populated. + when allowed_consent_groups is populated. """ project = client.project destination_dataset = destination_table_config["dataset"] @@ -412,9 +412,9 @@ def build_destination_table_query( # Check for classification filter classification_filter = destination_table_config.get("classification_filter") - allowed_cases = classification_filter.get("allowed_cases", []) if classification_filter else [] + allowed_consent_groups = classification_filter.get("allowed_consent_groups", []) if classification_filter else [] - has_classification = bool(allowed_cases) + has_classification = bool(allowed_consent_groups) # Build WHERE clause from filter profile filter_sql = build_row_filter_sql( @@ -436,10 +436,10 @@ def build_destination_table_query( # Combines base constraint, filter profile, and classification filter # Add classification filter condition if present - if allowed_cases: + if allowed_consent_groups: where_conditions = [f"p.{join_key} IS NOT NULL"] # Base constraint to ensure join key is not null - cases_str = ", ".join([f"'{c}'" for c in allowed_cases]) - where_conditions.append(f"c.`case` IN ({cases_str})") + consent_groups_str = ", ".join([f"'{c}'" for c in allowed_consent_groups]) + where_conditions.append(f"c.consent_group IN ({consent_groups_str})") else: where_conditions = [f"{join_key} IS NOT NULL"] @@ -454,7 +454,7 @@ def build_destination_table_query( # Build eligibility flag columns for the CTE # One boolean per unique gating rule (deduped by status_col) eligibility_flags_sql = "" - if allowed_cases: + if allowed_consent_groups: unique_rules = build_unique_eligibility_rules(constants.MODULE_CENSOR_RULES) if unique_rules: flag_lines = [] @@ -468,10 +468,10 @@ def build_destination_table_query( eligibility_flags_sql = ",\n" + ",\n".join(flag_lines) # Build CTE with or without classification join - if allowed_cases: + if allowed_consent_groups: if not classification_table: raise ValueError( - "classification_table must be provided when allowed_cases is populated." + "classification_table must be provided when allowed_consent_groups is populated." ) cte = f""" {cte_name} AS ( @@ -696,10 +696,10 @@ def table_has_classification_filter( ) -> bool: """ Checks whether a given destination table's config defines a - classification_filter with non-empty allowed_cases. Mirrors the - exact same allowed_cases resolution used inside - build_destination_table_query, so this can never disagree with - what that function actually does. + classification_filter with non-empty allowed_consent_groups. + Mirrors the exact same allowed_consent_groups resolution used + inside build_destination_table_query, so this can never disagree + with what that function actually does. """ destination_table_config = get_destination_table_config( destination_config, destination_dataset, destination_table @@ -707,5 +707,5 @@ def table_has_classification_filter( if not destination_table_config: return False classification_filter = destination_table_config.get("classification_filter") - allowed_cases = classification_filter.get("allowed_cases", []) if classification_filter else [] - return bool(allowed_cases) \ No newline at end of file + allowed_consent_groups = classification_filter.get("allowed_consent_groups", []) if classification_filter else [] + return bool(allowed_consent_groups) \ No newline at end of file diff --git a/core/endpoints.py b/core/endpoints.py index 574937f..6228a49 100644 --- a/core/endpoints.py +++ b/core/endpoints.py @@ -213,10 +213,10 @@ def create_censorship_summary_table(): raise ValueError( f"No config found for {destination_dataset}.{censorship_table}" ) - allowed_cases = destination_table_config.get("classification_filter", {}).get("allowed_cases") - if not allowed_cases: + allowed_consent_groups = destination_table_config.get("classification_filter", {}).get("allowed_consent_groups") + if not allowed_consent_groups: msg = ( - f"[{censorship_table}] Skipped: no classification_filter.allowed_cases " + f"[{censorship_table}] Skipped: no classification_filter.allowed_consent_groups " f"configured. This table is not classification-gated, so no " f"censorship summary applies" ) @@ -232,7 +232,7 @@ def create_censorship_summary_table(): output_table=output_table, classification_table=classification_table, destination_table=destination_table, - allowed_cases=allowed_cases, + allowed_consent_groups=allowed_consent_groups, ) return jsonify({ 'status': status, diff --git a/core/transformations.py b/core/transformations.py index 8b51120..425af57 100644 --- a/core/transformations.py +++ b/core/transformations.py @@ -933,12 +933,13 @@ def create_classification_table( ) -> dict: """ Reads participant status data from parquet, classifies each participant - into a case (1, 2, 3A, 3B, 4, or an exclusion/anomaly), and writes the - result to a BigQuery classification table. + into a consent group (1, 2, 3A, 3B, 4, or an exclusion/anomaly), and + writes the result to a BigQuery classification table. This function: 1. Reads and filters the parquet file to verified participants - 2. Classifies participants into cases based on consent/revoke/destroy flags + 2. Classifies participants into consent groups based on + consent/revoke/destroy flags 3. Optionally writes a local CSV for inspection/debugging 4. Writes the classification DataFrame to a BigQuery table @@ -968,7 +969,7 @@ def create_classification_table( utils.logger.exception(f"[{table_short_name}] Error reading parquet from {parquet_url}: {e}") raise e - # Classify participants into cases + # Classify participants into consent groups try: utils.logger.info(f"[{table_short_name}] Classifying participants...") classification_df = classification.classify_participants(classification_df) @@ -1143,24 +1144,24 @@ def create_censorship_summary_table( output_table: str, classification_table: str, destination_table: str, - allowed_cases: list[str], + allowed_consent_groups: list[str], rollup_report_base_path: str = constants.MISSING_COLUMNS_REPORT_PATH ) -> dict: """ Generates the censorship summary query, saves it to GCS for audit purposes (consistent with other transform functions in this codebase), executes it, writes the result to destination_table, and creates a - JSON rollup report of censorship counts by column and by case. + JSON rollup report of censorship counts by column and by consent_group. Args: output_table: Fully qualified final destination table. Used only to determine which columns exist and are therefore eligible for censorship classification. - classification_table: Fully qualified table with case/status/timestamp + classification_table: Fully qualified table with consent_group/status/timestamp columns for every participant. destination_table: Fully qualified table to create with the summary. - allowed_cases: Cases that pass output_table's classification_filter - (e.g. ["1", "2", "3A", "3B"]). + allowed_consent_groups: Consent groups that are allowed to be included + in the censorship summary. rollup_report_base_path: Base GCS path for the censorship rollup report. Returns: @@ -1185,7 +1186,7 @@ def create_censorship_summary_table( destination_table=destination_table, column_rule_map=column_rule_map, classification_table=classification_table, - allowed_cases=allowed_cases + allowed_consent_groups=allowed_consent_groups ) # Save the SQL to GCS for audit purposes, matching existing pipeline convention @@ -1219,7 +1220,7 @@ def create_censorship_summary_table( output_path=report_path, destination_table=destination_table, by_column=rollup["by_column"], - by_case=rollup["by_case"], + by_consent_group=rollup["by_consent_group"], ) utils.logger.info(f"[{table_short_name}] Censorship rollup report saved to {report_path}") diff --git a/reference/destination_config.json b/reference/destination_config.json index d1fc5ca..47b2bc3 100644 --- a/reference/destination_config.json +++ b/reference/destination_config.json @@ -15,7 +15,7 @@ "join_key": "Connect_ID", "filter_profile": "no_filters", "classification_filter": { - "allowed_cases": ["1", "2", "3A", "3B"] + "allowed_consent_groups": ["1", "2", "3A", "3B"] }, "base_table": { "dataset": "CleanConnect", @@ -56,7 +56,7 @@ "join_key": "Connect_ID", "filter_profile": "no_filters", "classification_filter": { - "allowed_cases": ["1", "2", "3A", "3B"] + "allowed_consent_groups": ["1", "2", "3A", "3B"] }, "base_table": { "dataset": "pr2_mvp", @@ -71,7 +71,7 @@ "join_key": "Connect_ID", "filter_profile": "no_filters", "classification_filter": { - "allowed_cases": [] + "allowed_consent_groups": [] }, "base_table": { "dataset": "pr2_mvp", @@ -98,7 +98,7 @@ "join_key": "Connect_ID", "filter_profile": "default_participants", "classification_filter": { - "allowed_cases": ["1", "2", "3A", "3B"] + "allowed_consent_groups": ["1", "2", "3A", "3B"] }, "base_table": { "dataset": "CleanConnect", @@ -139,7 +139,7 @@ "join_key": "Connect_ID", "filter_profile": "no_filters", "classification_filter": { - "allowed_cases": [] + "allowed_consent_groups": [] }, "base_table": { "dataset": "CleanConnect",