Add Sensitive Tier destination table pipeline with configurable participant eligibility filtering and suppression

core/constants.py

@@ -185,3 +185,99 @@
         }
     ]
 }
+
+# Secondary source concept IDs for each module/survey
+MODULE_CENSOR_RULES = {
+    # Module 1: Background and Overall Health
+    "726699695": {
+        "source_table": "module1",
+        "status_col": "module1_status",
+        "completion_ts": "module1_complete_ts",
+        "cutoff": "survey_cutoff",
+    },
+    # Module 2: Medications, Reproductive Health, Exercise, and Sleep
+    "745268907": {
+        "source_table": "module2",
+        "status_col": "module2_status",
+        "completion_ts": "module2_complete_ts",
+        "cutoff": "survey_cutoff",
+    },
+    # Module 3: Smoking, Alcohol, and Sun Exposure
+    "965707586": {
+        "source_table": "module3",
+        "status_col": "module3_status",
+        "completion_ts": "module3_complete_ts",
+        "cutoff": "survey_cutoff",
+    },
+    # Module 4: Where You Live and Work
+    "716117817": {
+        "source_table": "module4",
+        "status_col": "module4_status",
+        "completion_ts": "module4_complete_ts",
+        "cutoff": "survey_cutoff",
+    },
+    # Blood/Urine/Mouthwash, Blood/Urine, and Mouthwash are commented out
+    # because it is unclear how the follow-up specimens will be handled.
+    # For now, these modules and their timestamps are excluded.
+    #
+    # Blood/Urine/Mouthwash
+    #"299215535": {
+    #    "source_table": "bioSurvey",
+    #    "status_col": "bio_status",
+    #    "completion_ts": "bio_complete_ts",
+    #    "cutoff": "survey_cutoff",
+    #},
+    # Blood/Urine
+    #"826163434": {
+    #    "source_table": "clinicalBioSurvey",
+    #    "status_col": "clinicalbio_status",
+    #    "completion_ts": "clinicalbio_complete_ts",
+    #    "cutoff": "survey_cutoff",
+    #},
+    # Mouthwash
+    #"390351864": {
+    #    "source_table": "mouthwash",
+    #    "status_col": "mouthwash_status",
+    #    "completion_ts": "mouthwash_complete_ts",
+    #    "cutoff": "survey_cutoff",
+    #},
+    # Menstrual Cycle (contains two secondary source concept IDs)
+    "912367929": {
+        "source_table": "menstrualSurvey",
+        "status_col": "menstrual_status",
+        "completion_ts": "menstrual_complete_ts",
+        "cutoff": "survey_cutoff",
+    },
+    # Menstrual Cycle (contains two secondary source concept IDs)
+    "232438133": {
+        "source_table": "menstrualSurvey",
+        "status_col": "menstrual_status",
+        "completion_ts": "menstrual_complete_ts",
+        "cutoff": "survey_cutoff",
+    },
+    # COVID-19
+    "793330426": {
+        "source_table": "covid19Survey",
+        "status_col": "covid19_status",
+        "completion_ts": "covid19_complete_ts",
+        "cutoff": "survey_cutoff",
+    },
+    # 2024 Connect Experience Survey
+    "506648060": {
+        "source_table": "experience2024",
+        "status_col": "experience2024_status",
+        "completion_ts": "experience2024_complete_ts",
+        "cutoff": "survey_cutoff",
+    },
+}
+
+NON_CENSORED_SECONDARY_SOURCE_CIDS = {
+    "214456996": {"source_table": "participants"}, # Eligibility Screener
+    "332759827": {"source_table": "participants"}, # User Profile
+    "218595434": {"source_table": "participants"}, # Verification
+    "883203566": {"source_table": "participants"}, # Sign in
+    "273437590": {"source_table": "participants"}, # Consent
+    "104913069": {"source_table": "participants"}, # Research- Finalization and shipping
+}
+
+ALWAYS_INCLUDE_NON_CID_COLUMNS = {}