From 8de01ba40fd51a287c519f71772d373a62e9d25d Mon Sep 17 00:00:00 2001 From: Delun Gong Date: Wed, 26 Aug 2026 08:08:07 +0800 Subject: [PATCH] fix: harden scientific units, provenance, and BL19B2 resume Rescue local workbench follow-up WIP after the PR #6 squash-merge. Guard uncertainty overflow, keep undefined high-q diagnostics as None, validate CalibrationContext provenance sequences, parse semicolon/decimal-comma and canSAS unit records more strictly, and pin BL19B2 output collisions plus reintegration contracts. Workbench session restore, theme, and queue contracts follow the same scientific gates. --- SASAbs.py | 594 +++++++++++++++---- src/saxsabs/cli.py | 31 +- src/saxsabs/core/buffer_subtraction.py | 61 +- src/saxsabs/core/calibration.py | 25 +- src/saxsabs/core/calibration_context.py | 137 +++-- src/saxsabs/core/detector_reduction.py | 16 +- src/saxsabs/core/fluorescence_subtraction.py | 119 +++- src/saxsabs/core/intensity_state.py | 49 +- src/saxsabs/core/normalization.py | 6 +- src/saxsabs/core/uncertainty.py | 36 +- src/saxsabs/io/parsers.py | 497 +++++++++++++--- src/saxsabs/io/writers.py | 148 +++-- src/saxsabs/workflows/bl19b2_abs2d.py | 235 ++++++-- src/saxsabs/workflows/bl19b2_integrate1d.py | 266 +++++++-- tests/test_bl19b2_abs2d.py | 28 + tests/test_bl19b2_integrate1d.py | 180 +++++- tests/test_buffer_subtraction.py | 50 ++ tests/test_calibration.py | 16 + tests/test_calibration_context.py | 107 +++- tests/test_cli.py | 84 +++ tests/test_detector_reduction.py | 30 + tests/test_fluorescence_subtraction.py | 96 +++ tests/test_intensity_state.py | 17 + tests/test_io_formats.py | 154 ++++- tests/test_normalization.py | 22 + tests/test_parsers.py | 167 ++++++ tests/test_uncertainty.py | 44 ++ tests/test_version_metadata.py | 40 +- tests/test_workbench_output_paths.py | 2 +- tests/test_workbench_scientific.py | 183 +++++- 30 files changed, 3008 insertions(+), 432 deletions(-) diff --git a/SASAbs.py b/SASAbs.py index 9c3a74d..490070d 100644 --- a/SASAbs.py +++ b/SASAbs.py @@ -22,6 +22,7 @@ import math import pandas as pd import datetime +import importlib.metadata as importlib_metadata from io import StringIO import re import unicodedata @@ -40,9 +41,23 @@ def _read_package_version() -> str: try: text = version_file.read_text(encoding="utf-8") except OSError: - return "2.0.0" + text = "" match = re.search(r'^__version__\s*=\s*"([^"]+)"', text, re.MULTILINE) - return match.group(1) if match else "2.0.0" + if match: + return match.group(1) + try: + metadata_version = importlib_metadata.version("saxsabs") + if metadata_version: + return metadata_version + except Exception: + pass + try: + import saxsabs + + package_version = getattr(saxsabs, "__version__", None) + except Exception: + package_version = None + return str(package_version) if package_version else "unknown" APP_VERSION = _read_package_version() @@ -65,6 +80,7 @@ def _read_package_version() -> str: "app_title": f"{APP_NAME} v{APP_VERSION}", "header_title": f"{APP_NAME} | Absolute Intensity Calibration", "theme_toggle": "🌓 Theme", + "theme_unavailable": "unavailable", "lang_toggle_to_zh": "中文", "lang_toggle_to_en": "English", "tab1": "\U0001f4d0 1. K-Factor Calibration", @@ -495,6 +511,7 @@ def _read_package_version() -> str: "app_title": f"{APP_NAME} v{APP_VERSION}", "header_title": f"{APP_NAME}｜绝对强度校正", "theme_toggle": "🌓 切换深色/浅色模式", + "theme_unavailable": "不可用", "lang_toggle_to_zh": "中文", "lang_toggle_to_en": "English", "tab1": "\U0001f4d0 1. K 因子标定", @@ -1115,6 +1132,10 @@ def _read_package_version() -> str: try: from saxs_ui_kit import apply_ios_theme, promote_primary_buttons, toggle_theme, ToolTip + + def theme_backend_available(): + return True + except Exception: # ---- sv_ttk Sun-Valley theme (lightweight Win11-style) ---- try: @@ -1122,20 +1143,42 @@ def _read_package_version() -> str: except ImportError: _sv_ttk = None + def theme_backend_available(): + return _sv_ttk is not None + def apply_ios_theme(root): if _sv_ttk is not None: _sv_ttk.set_theme("light") + return True + if root is not None: + try: + root._saxsabs_theme_unavailable = True + except Exception: + pass + return False def promote_primary_buttons(root): return None # sv_ttk handles Accent.TButton natively def toggle_theme(root): - if _sv_ttk is not None: - _sv_ttk.toggle_theme() - # update native tk widgets after theme switch - app = getattr(root, '_app_ref', None) + if _sv_ttk is None: + app = getattr(root, "_app_ref", None) if app is not None: - app._sync_native_widget_colors() + callback = getattr(app, "_report_theme_unavailable", None) + if callback is not None: + callback() + else: + try: + root._saxsabs_theme_unavailable = True + except Exception: + pass + return False + _sv_ttk.toggle_theme() + # update native tk widgets after theme switch + app = getattr(root, '_app_ref', None) + if app is not None: + app._sync_native_widget_colors() + return True class ToolTip: """Improved cross-platform tooltip with smarter positioning and i18n support.""" @@ -1220,7 +1263,8 @@ def update_text(self, new_text): except Exception: def load_session(path): with open(path, "r", encoding="utf-8") as f: - return json.load(f) + payload = json.load(f) + return _validate_session_payload(payload) def session_geometry(session_payload): if not isinstance(session_payload, dict): @@ -1228,6 +1272,67 @@ def session_geometry(session_payload): geom = session_payload.get("geometry", {}) return geom if isinstance(geom, dict) else {} + +def _validate_session_payload(payload): + """Validate session fields before they can mutate Workbench state. + + Session files are user-authored provenance inputs. Missing optional fields + remain backward compatible, while present schema, numeric, and geometry + values are rejected explicitly instead of being coerced to NaN or silently + interpreted relative to the process CWD. + """ + if not isinstance(payload, dict): + raise ValueError("session payload must be a JSON object") + schema = payload.get("schema") + if schema is not None and str(schema).strip() not in { + "saxsabs.session.v1", + "saxsabs.workbench.session.v1", + }: + raise ValueError(f"unsupported session schema: {schema!r}") + geometry = payload.get("geometry") + if geometry is not None: + if not isinstance(geometry, dict): + raise ValueError("session geometry must be an object") + for key in ("px_mm", "wl_A", "dist_mm"): + if key not in geometry or geometry[key] is None: + continue + try: + value = float(geometry[key]) + except (TypeError, ValueError) as exc: + raise ValueError(f"session geometry {key} must be numeric") from exc + if not np.isfinite(value) or value <= 0: + raise ValueError(f"session geometry {key} must be finite and > 0") + calibration = payload.get("calibration", {}) + if calibration is not None and not isinstance(calibration, dict): + raise ValueError("session calibration must be an object") + calibration = calibration if isinstance(calibration, dict) else {} + k_value = calibration.get("k_factor", payload.get("k_factor")) + if k_value is not None: + try: + k_numeric = float(k_value) + except (TypeError, ValueError) as exc: + raise ValueError("session k_factor must be numeric") from exc + if not np.isfinite(k_numeric) or k_numeric <= 0: + raise ValueError("session k_factor must be finite and > 0") + path_fields = { + "data_path", + "poni_path", + "bg_path", + "dark_path", + "std_path", + "calibration_record_path", + "record_path", + } + for key in path_fields: + value = payload.get(key) + if value is not None and not isinstance(value, (str, Path)): + raise ValueError(f"session {key} must be a path string") + for key in path_fields: + value = calibration.get(key) + if value is not None and not isinstance(value, (str, Path)): + raise ValueError(f"session calibration {key} must be a path string") + return payload + try: import saxs_mpl_style except Exception: @@ -1859,7 +1964,7 @@ def __init__(self, root, language="en"): # Apply shared scientific plot defaults globally. saxs_mpl_style.apply_nature_style("raw_inspection") - self.set_style() + self.set_style(initialize_theme=True) self._tooltips = [] self._output_format_combos = [] @@ -1872,6 +1977,11 @@ def __init__(self, root, language="en"): self.btn_theme = ttk.Button(top_bar, text=self.tr("theme_toggle"), command=lambda: toggle_theme(self.root)) self.btn_theme.grid(row=0, column=1, sticky="e", padx=(8, 0)) + if not theme_backend_available(): + self.btn_theme.configure( + state="disabled", + text=f"{self.tr('theme_toggle')} ({self.tr('theme_unavailable')})", + ) self.btn_lang = ttk.Button(top_bar, text=self._lang_button_text(), width=10, command=self.toggle_language) self.btn_lang.grid(row=0, column=2, sticky="e", padx=(8, 0)) @@ -2648,9 +2758,11 @@ def show_warning(self, title_key, message): def confirm_action(self, message_key): return messagebox.askyesno(self.tr("confirm_clear_title"), self.tr(message_key)) - def set_style(self): - # Apply Sun-Valley theme first (light by default) - apply_ios_theme(self.root) + def set_style(self, *, initialize_theme=False): + # Apply the light default only during initialization. Re-applying it + # during a user toggle would immediately erase the selected dark theme. + if initialize_theme: + apply_ios_theme(self.root) style = ttk.Style() # Only fall back to clam if sv_ttk is not active current = style.theme_use() @@ -2766,6 +2878,19 @@ def set_style(self): self._scroll_canvases: list = [] self.root._app_ref = self # allow toggle_theme callback to reach us + def _report_theme_unavailable(self): + message = "Theme control unavailable: sv_ttk is not installed." + if hasattr(self, "_status_var"): + self._status_var.set(message) + if hasattr(self, "btn_theme"): + try: + self.btn_theme.configure( + state="disabled", + text=f"{self.tr('theme_toggle')} ({self.tr('theme_unavailable')})", + ) + except Exception: + pass + def _register_native_widget(self, widget): """Track a tk.Text or tk.Listbox so its colours follow the theme.""" self._native_widgets.append(widget) @@ -7289,7 +7414,7 @@ def dry_run_external_1d(self): return rows = [] - files = list(dict.fromkeys(self.t3_files)) + files, _queue_changed = self.normalize_t3_queue() failed_files = 0 risky_files = 0 pipeline_mode = self.t3_pipeline_mode.get().strip().lower() @@ -7578,21 +7703,16 @@ def dry_run_external_1d(self): def run_external_1d_batch(self): try: - self._require_current_workbench_preflight("t3") + files, _queue_changed = self.normalize_t3_queue() if bool(self.t3_resume_enabled.get()): raise ValueError( "Tab3 formal output does not permit legacy exists-only resume." ) - if not self.t3_files: + if not files: raise ValueError("队列为空：请先添加外部1D文件。") - - files = list(dict.fromkeys(self.t3_files)) - if len(files) < len(self.t3_files): - self.t3_files = files - self.lb_ext1d.delete(0, tk.END) - for f in self.t3_files: - self.lb_ext1d.insert(tk.END, Path(f).name) - self.refresh_external_1d_status() + # Approval must bind the final normalized queue. Duplicate removal + # invalidates any approval that was made for the old queue. + self._require_current_workbench_preflight("t3") k = float(self.global_vars["k_factor"].get()) if not np.isfinite(k) or k <= 0: @@ -8714,9 +8834,6 @@ def run_calibration(self): robust_min_points=3, robust_zero_mad_relative_tolerance=1e-12, ) - self.global_vars["k_factor"].set(k_val) - self.global_vars["k_solid_angle"].set("on" if apply_solid_angle else "off") - # Report self.report("-" * 30) self.report(self.tr("rpt_calib_ok")) @@ -8809,11 +8926,35 @@ def run_calibration(self): "CalibrationContextFingerprint": calibration_context.fingerprint(), }) df.to_csv(save_path, index=False) + # Read back the scientific check artifact before it can become the + # active calibration state. A successful write call alone does not + # prove that the expected columns and values reached disk. + check_readback = pd.read_csv(save_path) + required_check_columns = { + "Q", + "I_Abs", + "Error_Statistical", + "Error_Partial_K_Included", + "CalibrationContextFingerprint", + } + if not required_check_columns.issubset(check_readback.columns): + raise ValueError("calibration check readback is missing required columns") + if check_readback.empty or not np.all(np.isfinite(check_readback["Q"].to_numpy())): + raise ValueError("calibration check readback is empty or non-finite") + if set(check_readback["CalibrationContextFingerprint"].astype(str)) != { + calibration_context.fingerprint() + }: + raise ValueError("calibration check readback context fingerprint mismatch") context_path = calibration_output_dir / f"calibration_context_{run_id}.json" context_path.write_text( json.dumps(calibration_context.to_dict(), indent=2, ensure_ascii=False), encoding="utf-8", ) + context_readback = json.loads(context_path.read_text(encoding="utf-8")) + if not isinstance(context_readback, dict): + raise ValueError("calibration context readback is not an object") + if context_readback != calibration_context.to_dict(): + raise ValueError("calibration context readback mismatch") record_path = calibration_output_dir / f"calibration_record_{run_id}.json" self.save_calibration_record( record_path, @@ -8835,16 +8976,6 @@ def run_calibration(self): reference_i=(i_ref if std_key not in {"SRM3600", "Water_20C"} else None), ) validated_record = _core_read_calibration_record(record_path) - self.calibration_context = validated_record.calibration_context - self.calibration_record_provenance_complete = bool( - validated_record.provenance_complete - ) - self.calibration_record_source_files_verified = bool( - validated_record.provenance_complete - ) - self.calibration_k_value = float(k_val) - self.calibration_uncertainty = calibration_uncertainty - self.calibration_record_path = validated_record.record_path self.report(f"Saved profile: {save_path.name}") self.report(f"Saved calibration context: {context_path.name}") provenance_missing = validated_record.provenance_missing @@ -8874,6 +9005,24 @@ def run_calibration(self): calibration_record_path=record_path, calibration_uncertainty=calibration_uncertainty, ) + # Commit the newly calibrated state only after every scientific + # artifact has been written, read back, and the history transaction + # has succeeded. Any exception above therefore leaves the prior K + # and CalibrationContext untouched. + self.global_vars["k_factor"].set(k_val) + self.global_vars["k_solid_angle"].set( + "on" if apply_solid_angle else "off" + ) + self.calibration_context = validated_record.calibration_context + self.calibration_record_provenance_complete = bool( + validated_record.provenance_complete + ) + self.calibration_record_source_files_verified = bool( + validated_record.provenance_complete + ) + self.calibration_k_value = float(k_val) + self.calibration_uncertainty = calibration_uncertainty + self.calibration_record_path = validated_record.record_path self.report("K history updated.") except Exception as e: @@ -9162,6 +9311,92 @@ def get_selected_modes(self): modes.append("radial_chi") return modes + def normalize_t2_queue(self): + """Deduplicate the Tab2 queue before any approval or execution check.""" + original = list(getattr(self, "t2_files", []) or []) + normalized = [] + seen = set() + for value in original: + text = str(value).strip() + if not text: + continue + try: + key = str(Path(text).expanduser().resolve()).casefold() + except OSError: + key = text.casefold() + if key in seen: + continue + seen.add(key) + normalized.append(text) + changed = normalized != original + if changed: + self.t2_files = normalized + listbox = getattr(self, "lb_batch", None) + if listbox is not None: + try: + listbox.delete(0, tk.END) + for value in normalized: + listbox.insert(tk.END, Path(value).name) + except Exception: + pass + self.refresh_queue_status() + self._invalidate_workbench_preflight("t2") + return normalized, changed + + def normalize_t3_queue(self): + """Deduplicate the Tab3 queue before preflight approval or execution.""" + original = list(getattr(self, "t3_files", []) or []) + normalized = [] + seen = set() + for value in original: + text = str(value).strip() + if not text: + continue + try: + key = str(Path(text).expanduser().resolve()).casefold() + except OSError: + key = text.casefold() + if key in seen: + continue + seen.add(key) + normalized.append(text) + changed = normalized != original + if changed: + self.t3_files = normalized + listbox = getattr(self, "lb_ext1d", None) + if listbox is not None: + try: + listbox.delete(0, tk.END) + for value in normalized: + listbox.insert(tk.END, Path(value).name) + except Exception: + pass + self.refresh_external_1d_status() + self._invalidate_workbench_preflight("t3") + return normalized, changed + + def validate_t2_mode_contract(self, selected_modes, fluorescence_enabled): + """Reject fluorescence for radial-chi, whose output is not corrected.""" + if "radial_chi" in set(selected_modes or ()) and bool(fluorescence_enabled): + raise ValueError( + "radial_chi output does not apply fluorescence correction; disable " + "fluorescence or select a corrected 1D Q output before continuing." + ) + + @staticmethod + def validate_fixed_reference_shape(sample_path, reference_payload): + """Use the execution loader to verify a fixed reference/sample shape pair.""" + loaded_sample = _workbench_load_detector_image(sample_path, dtype=None) + sample_shape = tuple(np.asarray(loaded_sample.data).shape) + reference_shape = tuple( + np.asarray(reference_payload["fixed_dark_data"]).shape + ) + if sample_shape != reference_shape: + raise ValueError( + f"sample/reference shape mismatch {sample_shape} vs {reference_shape}" + ) + return sample_shape + def add_bg_library_files(self): fs = filedialog.askopenfilenames(filetypes=[("Image", "*.tif *.tiff *.edf *.cbf")]) initial_count = len(self.t2_bg_candidates) @@ -9279,6 +9514,11 @@ def load_data(path): reason = "" try: + fluorescence_payload = context.get("fluorescence") or {} + self.validate_t2_mode_contract( + context.get("selected_modes", ()), + fluorescence_payload.get("enabled", False), + ) run_policy = context.get("run_policy") if run_policy is None: run_policy = SimpleNamespace( @@ -10041,7 +10281,12 @@ def prepare_batch_references(self, *, ref_mode, bg_path, dark_path, monitor_mode } def run_batch(self): try: - self._require_current_workbench_preflight("t2") + original_queue_count = len(getattr(self, "t2_files", []) or []) + files, queue_changed = self.normalize_t2_queue() + if queue_changed: + self.log( + f"[提示] 队列去重：移除重复文件 {original_queue_count - len(files)} 个" + ) if str(self.t2_calc_mode.get()).strip().lower() != "fixed": raise ValueError( "Tab2 formal output requires fixed thickness; legacy per-frame " @@ -10051,8 +10296,11 @@ def run_batch(self): raise ValueError( "Tab2 formal output does not permit legacy exists-only resume." ) - if not self.t2_files: + if not files: raise ValueError("队列为空：请先添加样品文件。") + # Approval must cover the normalized queue and every setting used + # below; a stale approval is rejected after duplicate removal. + self._require_current_workbench_preflight("t2") k = float(self.global_vars["k_factor"].get()) bg_p = self.global_vars["bg_path"].get() dk_p = self.global_vars["dark_path"].get() @@ -10069,16 +10317,11 @@ def run_batch(self): self.log(f"[配置] I0 归一化模式: {monitor_mode} (norm={self.monitor_norm_formula(monitor_mode)})") self.log(f"[配置] SolidAngle 修正: {'ON' if bool(self.t2_apply_solid_angle.get()) else 'OFF'}") - files = list(dict.fromkeys(self.t2_files)) - if len(files) < len(self.t2_files): - self.log(f"[提示] 队列去重：移除重复文件 {len(self.t2_files) - len(files)} 个") - self.t2_files = files - self.lb_batch.delete(0, tk.END) - for f in self.t2_files: - self.lb_batch.insert(tk.END, Path(f).name) - self.refresh_queue_status() - selected_modes = self.get_selected_modes() + fluorescence_enabled = bool( + getattr(getattr(self, "t2_fluo_enabled", None), "get", lambda: False)() + ) + self.validate_t2_mode_contract(selected_modes, fluorescence_enabled) output_format = str( getattr(self, "t2_output_format", None).get() if getattr(self, "t2_output_format", None) is not None @@ -10930,7 +11173,12 @@ def _preflight_label_text(self, gate): def dry_run(self): if not self.t2_files: return - files = list(dict.fromkeys(self.t2_files)) + original_queue_count = len(self.t2_files) + files, queue_changed = self.normalize_t2_queue() + if queue_changed: + self.log( + f"[提示] 队列去重：移除重复文件 {original_queue_count - len(files)} 个" + ) rows = [] failed_files = 0 risky_files = 0 @@ -10941,6 +11189,15 @@ def dry_run(self): calibration_gate_error = None active_calibration_context = None formal_config_errors = [] + try: + self.validate_t2_mode_contract( + selected_modes, + bool( + getattr(getattr(self, "t2_fluo_enabled", None), "get", lambda: False)() + ), + ) + except ValueError as exc: + formal_config_errors.append(str(exc)) if str(mode).strip().lower() != "fixed": formal_config_errors.append( "Formal Tab2 output requires fixed thickness; per-frame " @@ -11021,12 +11278,39 @@ def dry_run(self): mu = thickness_config["mu_cm_inv"] inst_issues = [] sample_norms = [] - bg_norm = self.compute_norm_factor( - self.global_vars["bg_exp"].get(), - self.global_vars["bg_i0"].get(), - 1.0, - monitor_mode, - ) + bg_norm = np.nan + fixed_reference = None + if str(self.t2_ref_mode.get()).strip().lower() == "fixed": + try: + bg_var = self.global_vars.get("bg_path") + dark_var = self.global_vars.get("dark_path") + if bg_var is None or dark_var is None: + raise ValueError("fixed BG/Dark reference variables are unavailable") + fixed_reference = self.prepare_batch_references( + ref_mode="fixed", + bg_path=bg_var.get(), + dark_path=dark_var.get(), + monitor_mode=monitor_mode, + ) + bg_norm = float(fixed_reference["fixed_bg_norm"]) + except (OSError, RuntimeError, TypeError, ValueError) as exc: + formal_config_errors.append( + f"Fixed BG/Dark reference validation failed: {exc}" + ) + warnings.append( + f"Fixed BG/Dark reference validation failed: {exc}" + ) + else: + try: + bg_norm = self.compute_norm_factor( + self.global_vars["bg_exp"].get(), + self.global_vars["bg_i0"].get(), + 1.0, + monitor_mode, + ) + except (TypeError, ValueError) as exc: + formal_config_errors.append(f"BG normalization invalid: {exc}") + warnings.append(f"BG normalization invalid: {exc}") export_cal2d = bool( getattr(self, "t2_export_cal2d", None).get() @@ -11136,6 +11420,14 @@ def dry_run(self): else: d_mm = float(thickness_config["fixed_thickness_cm"]) * 10.0 + if self.t2_ref_mode.get() == "fixed" and fixed_reference is not None: + try: + self.validate_fixed_reference_shape(fp, fixed_reference) + except ValueError as exc: + stat = f"Error: {exc}" + except Exception as exc: + stat = f"Error: sample image unreadable: {exc}" + if self.t2_ref_mode.get() == "auto": try: loaded = _workbench_load_detector_image(fp, dtype=None) @@ -12103,7 +12395,22 @@ def _get_std_reference_data(self): raise ValueError("请选择标准参考曲线文件。") from saxsabs.io.parsers import read_external_1d_profile prof = read_external_1d_profile(ref_path) - q_user = prof["x"] + # Reuse the same strict axis boundary as external 1D input. + # Reference curves are physical Q data for K fitting; accepting + # raw x values here would silently mix nm^-1, chi, or 2theta. + prepared = self.prepare_external_profile_axis( + ref_path, + prof, + mode="auto", + ) + if prepared.get("x_label") != "Q_A^-1" or prepared.get( + "x_conversion" + ) not in {"none", "q_nm^-1_to_q_a^-1"}: + raise ValueError( + "标准参考曲线必须明确标记为 Q 轴（A^-1 或 nm^-1）；" + "缺失、chi 或 2theta 轴语义均不允许用于 K 标定。" + ) + q_user = prepared["x"] i_user = _profile_intensity(prof) return get_reference_data(key, q_user=q_user, i_user=i_user) elif key == "Water_20C": @@ -12228,49 +12535,117 @@ def _ts_extractor(p): # For now just update the info label (visual grouped list comes in follow-up) self.refresh_queue_status() + def _clear_calibration_state(self): + """Clear all cached K/context fields after rejected session provenance.""" + self.calibration_context = None + self.calibration_k_value = None + self.calibration_uncertainty = None + self.calibration_record_path = None + self.calibration_record_provenance_complete = False + self.calibration_record_source_files_verified = False + global_vars = getattr(self, "global_vars", {}) + k_var = global_vars.get("k_factor") if isinstance(global_vars, dict) else None + if k_var is not None: + try: + k_var.set("") + except Exception: + pass + + @staticmethod + def _resolve_session_path(session_dir, value): + text = str(value or "").strip() + if not text: + return "" + path = Path(text).expanduser() + return str((path if path.is_absolute() else Path(session_dir) / path).resolve()) + def apply_session(self, session_path: str): + session_file = Path(session_path).expanduser().resolve() try: - sess = load_session(session_path) - except Exception as e: - self.show_error("session_error_title", self.tr("session_error_body").format(err=e)) + sess = _validate_session_payload(load_session(session_file)) + except Exception as exc: + self._clear_calibration_state() + self.show_error( + "session_error_title", + self.tr("session_error_body").format(err=exc), + ) return notes = [] - geom = session_geometry(sess) - if geom: - px_mm = geom.get("px_mm") - wl_a = geom.get("wl_A") - dist_mm = geom.get("dist_mm") - self.session_geometry_fallback = { - "wavelength_a": float(wl_a) if wl_a is not None else None, - "distance_m": (float(dist_mm) / 1000.0) if dist_mm is not None else None, - "pixel1_m": (float(px_mm) / 1000.0) if px_mm is not None else None, - "pixel2_m": (float(px_mm) / 1000.0) if px_mm is not None else None, - "energy_kev": (HC_KEV_A / float(wl_a)) if (wl_a is not None and float(wl_a) > 0) else None, - } - notes.append("Session geometry loaded (used as consistency fallback when headers are missing).") + session_dir = session_file.parent + self.session_geometry_fallback = {} + try: + geom = session_geometry(sess) + if geom: + _validate_session_payload({"geometry": geom}) + px_mm = geom.get("px_mm") + wl_a = geom.get("wl_A") + dist_mm = geom.get("dist_mm") + self.session_geometry_fallback = { + "wavelength_a": float(wl_a) if wl_a is not None else None, + "distance_m": (float(dist_mm) / 1000.0) if dist_mm is not None else None, + "pixel1_m": (float(px_mm) / 1000.0) if px_mm is not None else None, + "pixel2_m": (float(px_mm) / 1000.0) if px_mm is not None else None, + "energy_kev": (HC_KEV_A / float(wl_a)) if wl_a is not None else None, + } + notes.append( + "Session geometry loaded (used as consistency fallback when headers are missing)." + ) + except Exception as exc: + self._clear_calibration_state() + self.show_error( + "session_error_title", + self.tr("session_error_body").format(err=exc), + ) + return - # Optional calibration paths from session payload (forward-compatible) - cal = sess.get("calibration", {}) if isinstance(sess.get("calibration", {}), dict) else {} + cal = sess.get("calibration", {}) if isinstance(sess.get("calibration"), dict) else {} + candidate_values = { + "poni": cal.get("poni_path", sess.get("poni_path", "")), + "bg": cal.get("bg_path", sess.get("bg_path", "")), + "dark": cal.get("dark_path", sess.get("dark_path", "")), + "std": cal.get("std_path", sess.get("std_path", "")), + } candidate_paths = { - "poni": str(cal.get("poni_path", sess.get("poni_path", ""))).strip(), - "bg": str(cal.get("bg_path", sess.get("bg_path", ""))).strip(), - "dark": str(cal.get("dark_path", sess.get("dark_path", ""))).strip(), - "std": str(cal.get("std_path", sess.get("std_path", ""))).strip(), + key: self._resolve_session_path(session_dir, value) + for key, value in candidate_values.items() } - if candidate_paths["poni"] and Path(candidate_paths["poni"]).is_file(): - self.global_vars["poni_path"].set(candidate_paths["poni"]) - notes.append(f"PONI loaded from session: {Path(candidate_paths['poni']).name}") - if candidate_paths["bg"] and Path(candidate_paths["bg"]).is_file(): - self.global_vars["bg_path"].set(candidate_paths["bg"]) - notes.append(f"Background loaded from session: {Path(candidate_paths['bg']).name}") - if candidate_paths["dark"] and Path(candidate_paths["dark"]).is_file(): - self.global_vars["dark_path"].set(candidate_paths["dark"]) - notes.append(f"Dark loaded from session: {Path(candidate_paths['dark']).name}") - if candidate_paths["std"] and Path(candidate_paths["std"]).is_file(): - self.t1_files["std"].set(candidate_paths["std"]) - self.on_load_std_t1(candidate_paths["std"]) - notes.append(f"Std image loaded from session std_path: {Path(candidate_paths['std']).name}") + + def load_file_setting(key, variable, label, callback=None): + value = candidate_paths[key] + if not value: + return + path = Path(value) + if not path.is_file(): + try: + variable.set("") + except Exception: + pass + notes.append(f"{label} path not found: {path}") + return + variable.set(str(path)) + notes.append(f"{label} loaded from session: {path.name}") + if callback is not None: + try: + callback(str(path)) + except Exception as exc: + notes.append(f"Session callback load failed for {label}: {exc}") + + for key, variable_key, label in ( + ("poni", "poni_path", "PONI"), + ("bg", "bg_path", "Background"), + ("dark", "dark_path", "Dark"), + ): + variable = self.global_vars.get(variable_key) + if variable is not None: + load_file_setting(key, variable, label) + if hasattr(self, "t1_files") and "std" in self.t1_files: + load_file_setting( + "std", + self.t1_files["std"], + "Std image", + callback=self.on_load_std_t1, + ) record_value = cal.get( "calibration_record_path", @@ -12278,38 +12653,35 @@ def apply_session(self, session_path: str): ) record_text = str(record_value or "").strip() if record_text: - record_path = Path(record_text).expanduser() - if not record_path.is_absolute(): - record_path = Path(session_path).resolve().parent / record_path + record_path = Path(self._resolve_session_path(session_dir, record_text)) try: self.load_calibration_record(record_path) notes.append(f"Complete calibration record loaded: {record_path.name}") except Exception as exc: - self.calibration_context = None - self.calibration_k_value = None - self.calibration_uncertainty = None - self.calibration_record_path = None + self._clear_calibration_state() notes.append(f"Calibration record rejected: {exc}") elif "k_factor" in cal or "k_factor" in sess: - self.calibration_context = None - self.calibration_k_value = None - self.calibration_uncertainty = None - self.calibration_record_path = None + self._clear_calibration_state() notes.append( "Legacy/manual K ignored because the session has no complete CalibrationContext record." ) - data_path = str(sess.get("data_path", "")).strip() - if data_path: - p = Path(data_path) - if p.is_file() and p.suffix.lower() in (".tif", ".tiff"): - self.t1_files["std"].set(str(p)) - self.on_load_std_t1(str(p)) - notes.append(f"Std image loaded from session: {p.name}") - elif p.is_file(): - notes.append(f"Session data is not TIFF, skipped for Std: {p.name}") + data_value = sess.get("data_path", "") + data_text = self._resolve_session_path(session_dir, data_value) + if data_text: + path = Path(data_text) + if path.is_file() and path.suffix.lower() in (".tif", ".tiff"): + if hasattr(self, "t1_files") and "std" in self.t1_files: + self.t1_files["std"].set(str(path)) + try: + self.on_load_std_t1(str(path)) + except Exception as exc: + notes.append(f"Session callback load failed for data: {exc}") + notes.append(f"Std image loaded from session: {path.name}") + elif path.is_file(): + notes.append(f"Session data is not TIFF, skipped for Std: {path.name}") else: - notes.append(f"Session data path not found: {data_path}") + notes.append(f"Session data path not found: {path}") if not notes: notes.append("Session loaded.") diff --git a/src/saxsabs/cli.py b/src/saxsabs/cli.py index 9e5f4d3..2b5ea80 100644 --- a/src/saxsabs/cli.py +++ b/src/saxsabs/cli.py @@ -46,14 +46,19 @@ def _die(message: str) -> None: raise SystemExit(1) +_EXPECTED_INPUT_ERRORS = (OSError, UnicodeError, ValueError, TypeError, ImportError) + + def _clean_column_name(name: object) -> str: return "".join(ch for ch in str(name).strip().lower() if ch.isalnum()) def _column_score(name: str, role: str) -> int: + raw_name = str(name) + name = _clean_column_name(name) if role == "q": exact = {"q", "chi", "radial", "2theta", "twotheta", "s", "x"} - prefixes = ("q", "chi", "radial", "twotheta") + prefixes = ("chi", "radial", "twotheta") suffixes = ("q",) else: exact = {"i", "intensity", "irel", "iabs", "signal", "count", "counts", "y"} @@ -62,6 +67,8 @@ def _column_score(name: str, role: str) -> int: if name in exact: return 300 + if role == "q" and q_axis_kind(raw_name) == "q": + return 200 if any(name.startswith(prefix) and len(name) > len(prefix) for prefix in prefixes): return 200 if any(name.endswith(suffix) and len(name) > len(suffix) for suffix in suffixes): @@ -133,7 +140,7 @@ def _resolve_column( best_col = None best_score = 0 for col in columns: - score = _column_score(_clean_column_name(col), role) + score = _column_score(col, role) if score > best_score: best_col = col best_score = score @@ -775,13 +782,21 @@ def main() -> None: return if args.command == "parse-header": - header = json.loads(args.header_json.read_text(encoding="utf-8")) - exp, mon, trans = parse_header_values(header) + try: + header = json.loads(args.header_json.read_text(encoding="utf-8-sig")) + if not isinstance(header, dict): + raise ValueError("header JSON top level must be an object") + exp, mon, trans = parse_header_values(header) + except _EXPECTED_INPUT_ERRORS as exc: + _die(f"parse-header failed: {exc}") print(json.dumps({"exp_s": exp, "i0": mon, "trans": trans}, ensure_ascii=False)) return if args.command == "parse-external1d": - result = read_external_1d_profile(args.input) + try: + result = read_external_1d_profile(args.input) + except _EXPECTED_INPUT_ERRORS as exc: + _die(f"parse-external1d failed: {exc}") print( json.dumps( { @@ -846,7 +861,7 @@ def main() -> None: i_ref=profile_intensity(reference), q_window=(args.qmin, args.qmax), ) - except ValueError as exc: + except _EXPECTED_INPUT_ERRORS as exc: _die(f"estimate-k failed: {exc}") print( json.dumps( @@ -890,7 +905,7 @@ def main() -> None: sample_profile=sample, buffer_profile=buffer_profile, ) - except ValueError as exc: + except _EXPECTED_INPUT_ERRORS as exc: _die(f"subtract-buffer failed: {exc}") print( json.dumps( @@ -942,7 +957,7 @@ def main() -> None: err_fluorescence=err_fluo, fluorescence_profile=fluo_profile, ) - except ValueError as exc: + except _EXPECTED_INPUT_ERRORS as exc: _die(f"subtract-fluorescence failed: {exc}") print( json.dumps( diff --git a/src/saxsabs/core/buffer_subtraction.py b/src/saxsabs/core/buffer_subtraction.py index 621d4dd..abfdd5b 100644 --- a/src/saxsabs/core/buffer_subtraction.py +++ b/src/saxsabs/core/buffer_subtraction.py @@ -38,10 +38,12 @@ class BufferSubtractionResult: Propagated uncertainty. alpha : float Scaling factor applied to the buffer curve. - high_q_residual_mean : float - Mean intensity in the high-*q* diagnostic window (should be ≈0). - high_q_check_passed : bool - *True* if |mean| < 3 × σ in the diagnostic window. + high_q_residual_mean : float | None + Mean intensity in the high-*q* diagnostic window (should be ≈0), or + ``None`` when fewer than three points make the diagnostic undefined. + high_q_check_passed : bool | None + *True* if |mean| < 3 × σ in the diagnostic window, *False* when it + fails, or ``None`` when the diagnostic was not performed. err_statistical : np.ndarray Statistical component, excluding the uncertainty contribution from alpha. alpha_uncertainty : float | None @@ -52,8 +54,8 @@ class BufferSubtractionResult: i_subtracted: np.ndarray err_subtracted: np.ndarray alpha: float - high_q_residual_mean: float = 0.0 - high_q_check_passed: bool = True + high_q_residual_mean: float | None = None + high_q_check_passed: bool | None = None alpha_uncertainty: float | None = None err_statistical: np.ndarray | None = None @@ -69,6 +71,16 @@ def _as_1d_float_array(name: str, values: np.ndarray | None, *, require_finite: return arr +def _square_uncertainty(name: str, values: np.ndarray) -> np.ndarray: + """Square uncertainty values while rejecting finite overflow.""" + + with np.errstate(over="ignore", invalid="ignore"): + squared = np.square(values) + if np.any(np.isinf(squared)): + raise ValueError(f"{name} uncertainty propagation overflowed") + return squared + + def _prepare_source_grid( q_source: np.ndarray, y_source: np.ndarray, @@ -123,7 +135,7 @@ def _prepare_variance_grid( """ order = np.argsort(q_source) q_sorted = q_source[order] - variance_sorted = np.square(sigma_source[order]) + variance_sorted = _square_uncertainty(label, sigma_source[order]) uq, inv = np.unique(q_sorted, return_inverse=True) if uq.size < 2: raise ValueError(f"{label} q grid must contain at least 2 unique points") @@ -134,7 +146,11 @@ def _prepare_variance_grid( for group in range(uq.size): group_variance = variance_sorted[inv == group] if np.all(np.isfinite(group_variance)): - variance_of_mean[group] = float(group_variance.sum() / group_variance.size**2) + with np.errstate(over="ignore", invalid="ignore", divide="ignore"): + value = group_variance.sum() / group_variance.size**2 + if not np.isfinite(value): + raise ValueError(f"{label} uncertainty propagation overflowed") + variance_of_mean[group] = float(value) return uq, variance_of_mean @@ -293,19 +309,36 @@ def subtract_buffer( q_s, q_b, e_b, label="buffer uncertainty" ) else: - buffer_variance = np.square(e_b) + buffer_variance = _square_uncertainty("err_buffer", e_b) # Subtraction - i_sub = i_s - alpha * i_b + with np.errstate(over="ignore", invalid="ignore"): + i_sub = i_s - alpha * i_b + if not np.all(np.isfinite(i_sub)): + raise ValueError("buffer subtraction produced non-finite intensity") # Unknown input errors intentionally yield NaN, never an optimistic partial budget. - variance_statistical = np.square(e_s) + alpha**2 * buffer_variance + with np.errstate(over="ignore", invalid="ignore"): + alpha_squared = np.square(alpha) + if np.isinf(alpha_squared): + raise ValueError("buffer uncertainty propagation overflowed") + variance_sample = _square_uncertainty("err_sample", e_s) + with np.errstate(over="ignore", invalid="ignore"): + variance_statistical = variance_sample + alpha_squared * buffer_variance + if np.any(np.isinf(variance_statistical)): + raise ValueError("buffer uncertainty propagation overflowed") err_statistical = np.sqrt(variance_statistical) variance_sub = variance_statistical.copy() if alpha_uncertainty is None: variance_sub = variance_sub + np.full_like(i_b, np.nan) else: - variance_sub = variance_sub + np.square(i_b * alpha_uncertainty) + with np.errstate(over="ignore", invalid="ignore"): + alpha_term = i_b * alpha_uncertainty + alpha_variance = _square_uncertainty("alpha", alpha_term) + with np.errstate(over="ignore", invalid="ignore"): + variance_sub = variance_sub + alpha_variance + if np.any(np.isinf(variance_sub)): + raise ValueError("buffer uncertainty propagation overflowed") err_sub = np.sqrt(variance_sub) # High-q diagnostic @@ -315,8 +348,8 @@ def subtract_buffer( residual_std = float(np.std(i_sub[mask])) check_ok = abs(residual_mean) < 3.0 * max(residual_std, 1e-30) else: - residual_mean = 0.0 - check_ok = True # not enough points for diagnostic + residual_mean = None + check_ok = None # not enough points for diagnostic return BufferSubtractionResult( q=q_s, diff --git a/src/saxsabs/core/calibration.py b/src/saxsabs/core/calibration.py index e46eb9a..e5ebdb8 100644 --- a/src/saxsabs/core/calibration.py +++ b/src/saxsabs/core/calibration.py @@ -304,7 +304,30 @@ def estimate_k_factor_robust( raise ValueError("q overlap with reference is insufficient") i_meas_interp = np.interp(q_ref_used, q_m, i_m) - valid = np.isfinite(i_meas_interp) & (i_meas_interp > positive_floor) + # ``np.interp`` linearly bridges a bad measured point. That can turn a + # segment such as (1.0, -0.1, 1.0) into apparently positive values and + # invent a plausible K. A reference point between measured samples is + # usable only when both bracketing measured endpoints pass the floor. An + # exact source point needs only that source value itself. + raw_upper = np.searchsorted(q_m, q_ref_used, side="left") + exact_source = (raw_upper < q_m.size) & np.isclose( + q_m[np.clip(raw_upper, 0, q_m.size - 1)], + q_ref_used, + rtol=0.0, + atol=1e-14, + ) + upper = np.clip(raw_upper, 1, q_m.size - 1) + lower = upper - 1 + exact_indices = np.clip(raw_upper, 0, q_m.size - 1) + lower[exact_source] = exact_indices[exact_source] + upper[exact_source] = exact_indices[exact_source] + segment_valid = ( + np.isfinite(i_m[lower]) + & (i_m[lower] > positive_floor) + & np.isfinite(i_m[upper]) + & (i_m[upper] > positive_floor) + ) + valid = segment_valid & np.isfinite(i_meas_interp) & (i_meas_interp > positive_floor) if int(valid.sum()) < min_points: raise ValueError("measured signal too weak or non-positive in overlap region") diff --git a/src/saxsabs/core/calibration_context.py b/src/saxsabs/core/calibration_context.py index f370f0b..5af814a 100644 --- a/src/saxsabs/core/calibration_context.py +++ b/src/saxsabs/core/calibration_context.py @@ -139,12 +139,50 @@ def builtin_reference_identity( raise ValueError(f"standard {key!r} does not have a built-in reference model") -def _validate_optional_sha256(value: str | None, *, field_name: str) -> None: +def _validate_optional_sha256(value: str | None, *, field_name: str) -> str | None: if value is None: - return + return None text = str(value).strip().lower() if len(text) != 64 or any(character not in "0123456789abcdef" for character in text): raise ValueError(f"{field_name} must be a 64-character SHA-256 digest or null") + return text + + +def _validate_required_sha256(value: object, *, field_name: str) -> str: + """Validate and canonicalize a non-null SHA-256 sequence entry.""" + + normalized = _validate_optional_sha256(value, field_name=field_name) + if normalized is None: + raise ValueError(f"{field_name} entries must be non-null SHA-256 digests") + return normalized + + +_ORDERED_PROVENANCE_SEQUENCE_FIELDS = ( + "background_data_sha256", + "dark_data_sha256", + "background_monitors", + "background_transmissions", + "background_exposure_s", + "dark_exposure_s", +) + + +def _require_ordered_sequence( + value: object, + *, + field_name: str, + allow_none: bool = False, +) -> tuple[object, ...] | None: + """Accept only deterministic list/tuple provenance containers.""" + + if value is None and allow_none: + return None + if not isinstance(value, (list, tuple)): + raise ValueError( + f"{field_name} must be an ordered list or tuple" + + (" or null" if allow_none else "") + ) + return tuple(value) def _validate_positive(value: float | None, *, field_name: str) -> None: @@ -228,8 +266,13 @@ def __post_init__(self) -> None: raise ValueError("monitor_mode must be 'rate' or 'integrated'") if not str(self.formula_version).strip(): raise ValueError("formula_version is required") - if not str(self.poni_sha256).strip(): + poni_sha256 = _validate_optional_sha256( + self.poni_sha256, + field_name="poni_sha256", + ) + if poni_sha256 is None: raise ValueError("poni_sha256 is required") + object.__setattr__(self, "poni_sha256", poni_sha256) standard_key = normalize_standard_key(self.standard_key) object.__setattr__(self, "standard_key", standard_key) thickness = float(self.standard_thickness_cm) @@ -250,37 +293,46 @@ def __post_init__(self) -> None: if not math.isfinite(factor) or factor < -1 or factor > 1: raise ValueError("polarization_factor must be finite and between -1 and 1") - sequence_fields = ( - "background_data_sha256", - "dark_data_sha256", - "background_monitors", - "background_transmissions", - "background_exposure_s", - "dark_exposure_s", + for field_name in _ORDERED_PROVENANCE_SEQUENCE_FIELDS: + object.__setattr__( + self, + field_name, + _require_ordered_sequence( + getattr(self, field_name), + field_name=field_name, + ), + ) + object.__setattr__( + self, + "q_window", + _require_ordered_sequence( + self.q_window, + field_name="q_window", + allow_none=True, + ), ) - for field_name in sequence_fields: - object.__setattr__(self, field_name, tuple(getattr(self, field_name))) - if self.q_window is not None: - object.__setattr__(self, "q_window", tuple(self.q_window)) - _validate_optional_sha256( - self.standard_data_sha256, - field_name="standard_data_sha256", - ) - _validate_optional_sha256( - self.reference_curve_sha256, - field_name="reference_curve_sha256", - ) - _validate_optional_sha256( - self.reference_canonical_sha256, - field_name="reference_canonical_sha256", - ) - for field_name, values in ( - ("background_data_sha256", self.background_data_sha256), - ("dark_data_sha256", self.dark_data_sha256), + for field_name in ( + "mask_sha256", + "flat_sha256", + "standard_data_sha256", + "reference_curve_sha256", + "reference_canonical_sha256", ): - for value in values: - _validate_optional_sha256(value, field_name=field_name) + object.__setattr__( + self, + field_name, + _validate_optional_sha256( + getattr(self, field_name), + field_name=field_name, + ), + ) + for field_name in ("background_data_sha256", "dark_data_sha256"): + raw_values = getattr(self, field_name) + normalized_values = tuple( + _validate_required_sha256(value, field_name=field_name) for value in raw_values + ) + object.__setattr__(self, field_name, normalized_values) _validate_positive(self.standard_monitor, field_name="standard_monitor") _validate_positive(self.standard_exposure_s, field_name="standard_exposure_s") @@ -389,17 +441,18 @@ def to_dict(self) -> dict[str, Any]: @classmethod def from_dict(cls, value: dict[str, Any]) -> "CalibrationContext": normalized = dict(value) - for field_name in ( - "background_data_sha256", - "dark_data_sha256", - "background_monitors", - "background_transmissions", - "background_exposure_s", - "dark_exposure_s", - "q_window", - ): - if field_name in normalized and normalized[field_name] is not None: - normalized[field_name] = tuple(normalized[field_name]) + for field_name in _ORDERED_PROVENANCE_SEQUENCE_FIELDS: + if field_name in normalized: + _require_ordered_sequence( + normalized[field_name], + field_name=field_name, + ) + if "q_window" in normalized: + _require_ordered_sequence( + normalized["q_window"], + field_name="q_window", + allow_none=True, + ) return cls(**normalized) def fingerprint(self) -> str: diff --git a/src/saxsabs/core/detector_reduction.py b/src/saxsabs/core/detector_reduction.py index bef361a..5185040 100644 --- a/src/saxsabs/core/detector_reduction.py +++ b/src/saxsabs/core/detector_reduction.py @@ -89,13 +89,19 @@ def normalize_detector_frame( image_exp = _positive_finite("image_exposure_s", image_exposure_s) dark_exp = _positive_finite("dark_exposure_s", dark_exposure_s) - dark_scale = image_exp / dark_exp norm = compute_norm_factor(image_exp, monitor, transmission, monitor_mode) if not math.isfinite(norm) or norm <= 0: raise ValueError("detector normalization factor must be finite and > 0") + with np.errstate(over="ignore", invalid="ignore", divide="ignore"): + dark_scale = image_exp / dark_exp + normalized_image = (image_arr - dark_arr * dark_scale) / norm + if not math.isfinite(dark_scale): + raise ValueError("detector dark scale must be finite") + if not np.all(np.isfinite(normalized_image)): + raise ValueError("normalized detector image contains non-finite values") return NormalizedDetectorFrame( - image=(image_arr - dark_arr * dark_scale) / norm, + image=normalized_image, normalization_factor=float(norm), dark_scale=float(dark_scale), ) @@ -140,8 +146,12 @@ def build_nist_net_image( transmission=1.0, monitor_mode=monitor_mode, ) + with np.errstate(over="ignore", invalid="ignore"): + net_image = sample_frame.image - alpha_value * background_frame.image + if not np.all(np.isfinite(net_image)): + raise ValueError("net detector image contains non-finite values") return NetDetectorImage( - image=sample_frame.image - alpha_value * background_frame.image, + image=net_image, norm_sample=sample_frame.normalization_factor, norm_background=background_frame.normalization_factor, dark_scale_sample=sample_frame.dark_scale, diff --git a/src/saxsabs/core/fluorescence_subtraction.py b/src/saxsabs/core/fluorescence_subtraction.py index a090105..49cbb42 100644 --- a/src/saxsabs/core/fluorescence_subtraction.py +++ b/src/saxsabs/core/fluorescence_subtraction.py @@ -47,8 +47,8 @@ class FluorescenceSubtractionResult: beta: float f0: float f_profile: np.ndarray - high_q_residual_mean: float = 0.0 - high_q_check_passed: bool = True + high_q_residual_mean: float | None = None + high_q_check_passed: bool | None = None high_q_window: tuple[float, float] | None = None high_q_points: int = 0 negative_fraction: float = 0.0 @@ -101,6 +101,16 @@ def _optional_nonnegative_uncertainty(name: str, value: float | None) -> float | return out +def _square_uncertainty(name: str, values: np.ndarray | float) -> np.ndarray: + """Square uncertainty values without exposing floating-point overflow.""" + + with np.errstate(over="ignore", invalid="ignore"): + squared = np.square(values) + if np.any(np.isinf(squared)): + raise ValueError(f"{name} uncertainty propagation overflowed") + return np.asarray(squared, dtype=np.float64) + + def _validate_beta(beta: float) -> float: value = float(beta) if not np.isfinite(value) or value <= 0: @@ -180,7 +190,7 @@ def _prepare_variance_grid( ) -> tuple[np.ndarray, np.ndarray]: order = np.argsort(q_source) q_sorted = q_source[order] - variance_sorted = np.square(sigma_source[order]) + variance_sorted = _square_uncertainty(label, sigma_source[order]) uq, inv = np.unique(q_sorted, return_inverse=True) if uq.size < 2: raise ValueError(f"{label} q grid must contain at least 2 unique points") @@ -191,9 +201,11 @@ def _prepare_variance_grid( for group in range(uq.size): group_variance = variance_sorted[inv == group] if np.all(np.isfinite(group_variance)): - variance_of_mean[group] = float( - group_variance.sum() / group_variance.size**2 - ) + with np.errstate(over="ignore", invalid="ignore", divide="ignore"): + value = group_variance.sum() / group_variance.size**2 + if not np.isfinite(value): + raise ValueError(f"{label} uncertainty propagation overflowed") + variance_of_mean[group] = float(value) return uq, variance_of_mean @@ -264,7 +276,7 @@ def _residual_diagnostics( q: np.ndarray, i_corr: np.ndarray, window: tuple[float, float], -) -> tuple[float, bool, int]: +) -> tuple[float | None, bool | None, int]: mask = _window_mask(q, i_corr, window) n_points = int(mask.sum()) if n_points >= 3: @@ -272,7 +284,7 @@ def _residual_diagnostics( residual_std = float(np.std(i_corr[mask])) check_ok = abs(residual_mean) < 3.0 * max(residual_std, 1e-30) return residual_mean, check_ok, n_points - return 0.0, True, n_points + return None, None, n_points def subtract_fluorescence( @@ -366,7 +378,10 @@ def subtract_fluorescence( if f0_uncertainty is None: f_variance = np.full_like(i_s, np.nan) else: - f_variance = np.full_like(i_s, f0_uncertainty**2) + f_variance = np.full_like( + i_s, + _square_uncertainty("f0", f0_uncertainty), + ) elif parsed_method in { FluorescenceMethod.HIGH_Q_MEAN, FluorescenceMethod.HIGH_Q_MEDIAN, @@ -387,7 +402,10 @@ def subtract_fluorescence( if f0_uncertainty is None: f_variance = np.full_like(i_s, np.nan) else: - f_variance = np.full_like(i_s, f0_uncertainty**2) + f_variance = np.full_like( + i_s, + _square_uncertainty("f0", f0_uncertainty), + ) else: if f0 is not None: raise ValueError("measured_profile refuses a scalar f0") @@ -429,28 +447,46 @@ def subtract_fluorescence( ) else: f_profile = i_f - f_variance = np.square(e_f) + f_variance = _square_uncertainty("err_fluorescence", e_f) finite_f = f_profile[np.isfinite(f_profile)] f0_value = float(np.mean(finite_f)) if finite_f.size else float("nan") if not np.isfinite(f0_value) or f0_value < 0: raise ValueError("measured fluorescence intensity must be finite and >= 0") f0_uncertainty = None - subtracted_term = beta_value * f_profile + with np.errstate(over="ignore", invalid="ignore"): + subtracted_term = beta_value * f_profile + if np.any(np.isinf(subtracted_term)): + raise ValueError("fluorescence subtraction produced non-finite intensity") i_corr = i_s - subtracted_term - - variance_statistical = np.square(e_s) + (beta_value**2) * f_variance + if np.any(np.isinf(i_corr)): + raise ValueError("fluorescence subtraction produced non-finite intensity") + + beta_squared = _square_uncertainty("beta", beta_value) + sample_variance = _square_uncertainty("err_abs", e_s) + with np.errstate(over="ignore", invalid="ignore"): + variance_statistical = sample_variance + beta_squared * f_variance + if np.any(np.isinf(variance_statistical)): + raise ValueError("fluorescence uncertainty propagation overflowed") if parsed_method is not FluorescenceMethod.MEASURED_PROFILE and f0_uncertainty is None: # Unknown u(F0) is a combined-budget gap, not a missing sample error. - variance_statistical = np.square(e_s) + variance_statistical = sample_variance err_statistical = np.sqrt(variance_statistical) if beta_uncertainty is None or ( parsed_method is not FluorescenceMethod.MEASURED_PROFILE and f0_uncertainty is None ): variance_combined = np.full_like(i_s, np.nan) else: - variance_combined = variance_statistical + np.square(f_profile * beta_uncertainty) + with np.errstate(over="ignore", invalid="ignore"): + beta_term = f_profile * beta_uncertainty + beta_variance = _square_uncertainty("beta", beta_term) + with np.errstate(over="ignore", invalid="ignore"): + variance_combined = variance_statistical + beta_variance + if np.any(np.isinf(variance_combined)): + raise ValueError("fluorescence uncertainty propagation overflowed") err_combined = np.sqrt(variance_combined) + if np.any(np.isinf(err_statistical)) or np.any(np.isinf(err_combined)): + raise ValueError("fluorescence uncertainty propagation overflowed") diag_window = residual_window if diag_window is None: @@ -495,15 +531,54 @@ def combine_sequential_standard_uncertainties( """ stat = np.asarray(next_statistical, dtype=np.float64) nxt = np.asarray(next_combined, dtype=np.float64) + prev_stat = np.asarray(previous_statistical, dtype=np.float64) + prev_comb = ( + None + if previous_combined is None + else np.asarray(previous_combined, dtype=np.float64) + ) + if ( + stat.shape != nxt.shape + or prev_stat.shape != stat.shape + or (prev_comb is not None and prev_comb.shape != prev_stat.shape) + ): + raise ValueError("all uncertainty arrays must have exactly equal shapes") + supplied_arrays = (prev_stat, stat, nxt) + if prev_comb is not None: + supplied_arrays += (prev_comb,) + if any(np.any(np.isinf(values)) for values in supplied_arrays): + raise ValueError("sequential uncertainty inputs must not contain infinities") if previous_combined is None: return stat, nxt - prev_stat = np.asarray(previous_statistical, dtype=np.float64) - prev_comb = np.asarray(previous_combined, dtype=np.float64) - extra_prev = np.square(prev_comb) - np.square(prev_stat) - extra_next = np.square(nxt) - np.square(stat) + assert prev_comb is not None + with np.errstate(over="ignore", invalid="ignore"): + prev_comb_squared = np.square(prev_comb) + prev_stat_squared = np.square(prev_stat) + next_comb_squared = np.square(nxt) + next_stat_squared = np.square(stat) + if any( + np.any(np.isinf(values)) + for values in ( + prev_comb_squared, + prev_stat_squared, + next_comb_squared, + next_stat_squared, + ) + ): + raise ValueError("sequential uncertainty propagation overflowed") + extra_prev = prev_comb_squared - prev_stat_squared + extra_next = next_comb_squared - next_stat_squared extra_prev = np.clip(extra_prev, 0.0, None) extra_next = np.clip(extra_next, 0.0, None) - combined = np.sqrt(np.square(stat) + extra_prev + extra_next) - unknown = ~np.isfinite(prev_comb) | ~np.isfinite(nxt) + with np.errstate(over="ignore", invalid="ignore"): + combined = np.sqrt(next_stat_squared + extra_prev + extra_next) + if np.any(np.isinf(combined)): + raise ValueError("sequential uncertainty propagation overflowed") + unknown = ( + ~np.isfinite(prev_comb) + | ~np.isfinite(nxt) + | ~np.isfinite(prev_stat) + | ~np.isfinite(stat) + ) combined = np.where(unknown, np.nan, combined) return stat, combined diff --git a/src/saxsabs/core/intensity_state.py b/src/saxsabs/core/intensity_state.py index 5fcbb5e..0285caa 100644 --- a/src/saxsabs/core/intensity_state.py +++ b/src/saxsabs/core/intensity_state.py @@ -34,6 +34,48 @@ def is_cm_inv_intensity_unit(value: object) -> bool: return _normalized_token(value) in CM_INV_UNIT_TOKENS +def _column_declares_absolute_cm_inv(value: object) -> bool: + """Recognize only explicit ``I`` columns carrying a reciprocal-cm unit. + + Column names are not units by themselves. In particular, names such as + ``image``/``index`` must not become absolute just because they begin with + the letter ``I``. The base field is therefore restricted to ``I`` or + ``I_abs`` and the suffix must explicitly contain either ``1/cm``, ``/cm`` + or a ``cm^-1`` spelling. + """ + + text = str(value or "").strip().lower().translate( + str.maketrans({"⁻": "-", "−": "-", "–": "-", "—": "-"}) + ) + match = re.match(r"^i(?:_?abs)?(?=$|[\s_:/([{])", text) + if match is None: + return False + suffix = text[match.end() :] + stripped_suffix = suffix.strip().strip("()[]{}").strip() + if is_cm_inv_intensity_unit(stripped_suffix): + return True + if re.fullmatch(r"/\s*cm", stripped_suffix): + return True + return False + + +def _column_declares_absolute_intensity(value: object) -> bool: + """Recognize explicit absolute-intensity naming without prefix guessing.""" + + text = str(value or "").strip() + normalized = _normalized_token(text) + if normalized == "iabs": + return True + if _column_declares_absolute_cm_inv(text): + return True + return normalized in { + "absolute", + "absoluteintensity", + "absolutecm1", + "absolute1cm", + } + + KNOWN_CORRECTIONS = frozenset( { "dark", @@ -226,10 +268,11 @@ def assess_intensity_state(profile: Mapping[str, object]) -> IntensityStateAsses evidence.append("conflicting_correction_ledgers") semantic_states: set[IntensityState] = set() - i_col = _normalized_token(profile.get("i_col", "")) - if i_col.startswith("iabs") or "absolut" in i_col or "cm1" in i_col: + raw_i_col = profile.get("i_col", "") + i_col = _normalized_token(raw_i_col) + if _column_declares_absolute_intensity(raw_i_col): semantic_states.add(IntensityState.ABSOLUTE_CM_INV) - evidence.append(f"column:{profile.get('i_col')}") + evidence.append(f"column:{raw_i_col}") elif i_col.startswith("irel") or "relative" in i_col: semantic_states.add(IntensityState.RELATIVE) evidence.append(f"column:{profile.get('i_col')}") diff --git a/src/saxsabs/core/normalization.py b/src/saxsabs/core/normalization.py index 0617f97..7eaeb7f 100644 --- a/src/saxsabs/core/normalization.py +++ b/src/saxsabs/core/normalization.py @@ -80,7 +80,9 @@ def compute_norm_factor(exp: float | None, mon: float | None, trans: float | Non return math.nan if not math.isfinite(exp_v) or exp_v <= 0: return math.nan - return exp_v * mon_v * trans_v + product = exp_v * mon_v * trans_v + return product if math.isfinite(product) and product > 0 else math.nan if mode_n == "integrated": - return mon_v * trans_v + product = mon_v * trans_v + return product if math.isfinite(product) and product > 0 else math.nan diff --git a/src/saxsabs/core/uncertainty.py b/src/saxsabs/core/uncertainty.py index a5f2f9f..1afab6a 100644 --- a/src/saxsabs/core/uncertainty.py +++ b/src/saxsabs/core/uncertainty.py @@ -110,7 +110,11 @@ def relative_component( ) if np.any(np.isnan(relative)): unknown.append(name) - return magnitude * relative + with np.errstate(over="ignore", invalid="ignore"): + component = magnitude * relative + if np.any(np.isinf(component)): + raise ValueError(f"{name} uncertainty propagation overflowed") + return component statistical = absolute_component("statistical", statistical_standard_uncertainty) k_component = relative_component("k", k_relative_standard_uncertainty) @@ -145,7 +149,10 @@ def relative_component( ) from exc if not np.all(np.isfinite(buffer_arr)): raise ValueError("buffer_intensity must contain only finite values") - alpha = np.abs(buffer_arr) * alpha_u + with np.errstate(over="ignore", invalid="ignore"): + alpha = np.abs(buffer_arr) * alpha_u + if np.any(np.isinf(alpha)): + raise ValueError("alpha uncertainty propagation overflowed") else: alpha = alpha_u.copy() @@ -159,7 +166,20 @@ def relative_component( mu, alpha, ) - combined = np.sqrt(np.sum([np.square(component) for component in components], axis=0)) + component_squares: list[np.ndarray] = [] + for component in components: + with np.errstate(over="ignore", invalid="ignore"): + squared = np.square(component) + if np.any(np.isinf(squared)): + raise ValueError("combined standard uncertainty propagation overflowed") + component_squares.append(squared) + with np.errstate(over="ignore", invalid="ignore"): + combined = np.sqrt(np.sum(component_squares, axis=0)) + known_square_sum = np.nansum(component_squares, axis=0) + if np.any(np.isinf(known_square_sum)): + raise ValueError("combined standard uncertainty propagation overflowed") + if np.any(np.isinf(combined)): + raise ValueError("combined standard uncertainty propagation overflowed") if coverage_factor is None: expanded = np.full(shape, np.nan, dtype=np.float64) @@ -167,7 +187,15 @@ def relative_component( coverage_factor = float(coverage_factor) if not np.isfinite(coverage_factor) or coverage_factor <= 0: raise ValueError("coverage_factor must be finite and > 0") - expanded = combined * coverage_factor + for component in components: + with np.errstate(over="ignore", invalid="ignore"): + expanded_component = component * coverage_factor + if np.any(np.isinf(expanded_component)): + raise ValueError("expanded uncertainty propagation overflowed") + with np.errstate(over="ignore", invalid="ignore"): + expanded = combined * coverage_factor + if np.any(np.isinf(expanded)): + raise ValueError("expanded uncertainty propagation overflowed") status = ( "complete" diff --git a/src/saxsabs/io/parsers.py b/src/saxsabs/io/parsers.py index 64ce365..5c32ceb 100644 --- a/src/saxsabs/io/parsers.py +++ b/src/saxsabs/io/parsers.py @@ -30,7 +30,11 @@ import numpy as np import pandas as pd -from saxsabs.core.intensity_state import IntensityState, assess_intensity_state +from saxsabs.core.intensity_state import ( + IntensityState, + assess_intensity_state, + is_cm_inv_intensity_unit, +) FLOAT_PATTERN = re.compile(r"[-+]?\d*\.?\d+(?:[eE][-+]?\d+)?") @@ -240,12 +244,32 @@ def _unit_delimiters_are_balanced(text: str) -> bool: def q_axis_kind(name: object) -> str: """Classify a source X-column as Q, chi, two-theta, or unknown.""" - token = _unit_token(name) - if "2theta" in token or "twotheta" in token: + text = _normalise_unit_text(name).strip() + if re.search( + r"(? int: return 2 +def _column_declares_absolute_cm_inv_header(value: object) -> bool: + """Return whether an I/I_abs header explicitly carries cm^-1.""" + + text = _normalise_unit_text(value).strip() + match = re.match(r"^i(?:_?abs)?(?=$|[\s_:/([{])", text) + if match is None: + return False + suffix = text[match.end() :].strip().strip("()[]{}").strip() + return is_cm_inv_intensity_unit(suffix) or bool( + re.fullmatch(r"/\s*cm", suffix) + ) + + def _match_column_score( name: str, *, @@ -348,17 +385,48 @@ def _match_column_score( return 0 +def _q_column_score(name: Any) -> int: + """Score Q-like headers without accepting arbitrary q-prefixed words.""" + + clean = _clean_column_name(name) + if clean in {"q", "chi", "radial", "2theta", "twotheta", "s", "x"}: + return 300 + if q_axis_kind(name) == "q": + return 200 + text = _normalise_unit_text(name).strip() + if re.search(r"(?:[_:\-\s])q$", text, flags=re.IGNORECASE): + return 150 + if clean.startswith(("chi", "radial", "twotheta")): + return 200 + return 0 + + def _intensity_column_score(name: Any) -> int: """Score intensity headers without treating every ``i...`` name as I.""" + clean = _clean_column_name(name) score = _match_column_score( - _clean_column_name(name), - exact={"i", "intensity", "irel", "iabs", "signal", "count", "counts", "y"}, - prefixes=("intensity", "signal", "count", "irel", "iabs"), + clean, + exact={ + "i", + "intensity", + "irel", + "iref", + "imeas", + "iabs", + "signal", + "count", + "counts", + "y", + }, + prefixes=("intensity", "signal", "count", "irel"), suffixes=("intensity",), ) - text = _normalise_unit_text(name) - if re.match(r"^i(?=$|[\s_:/\-([{])", text): + if clean.startswith("iabs") and ( + clean == "iabs" or _column_declares_absolute_cm_inv_header(name) + ): + score = max(score, 200) + if _column_declares_absolute_cm_inv_header(name): score = max(score, 200) return score @@ -376,7 +444,20 @@ def _pick_named_column( for col in cols: if col in used: continue - if exact == {"i", "intensity", "irel", "iabs", "signal", "count", "counts", "y"}: + if exact == {"q", "chi", "radial", "2theta", "twotheta", "s", "x"}: + score = _q_column_score(col) + elif exact == { + "i", + "intensity", + "irel", + "iref", + "imeas", + "iabs", + "signal", + "count", + "counts", + "y", + }: score = _intensity_column_score(col) else: score = _match_column_score( @@ -395,12 +476,7 @@ def _comment_header_score(tokens: list[str]) -> int: score = 0 for token in tokens: name = _clean_column_name(token) - score += _match_column_score( - name, - exact={"q", "chi", "radial", "2theta", "twotheta", "s", "x"}, - prefixes=("q", "chi", "radial", "twotheta"), - suffixes=("q",), - ) + score += _q_column_score(token) score += _intensity_column_score(token) score += _match_column_score( name, @@ -527,6 +603,8 @@ def _has_malformed_unit_header_tokens(tokens: list[str]) -> bool: if q_axis_kind(token) != "q": continue next_token = tokens[index + 1] + if q_axis_kind(next_token) == "q": + return True opener = next_token[:1] expected = opening_to_closing.get(opener) if expected is None: @@ -765,6 +843,147 @@ def extract_float(raw: Any) -> float | None: return None +def _parse_semicolon_decimal_comma_token(value: str) -> float: + """Parse one value when semicolon is the field delimiter. + + A comma inside a semicolon-delimited field can be a decimal mark or a + thousands separator. We accept only decimal-comma spellings that are + unambiguous from the token itself; ambiguous three-digit groups fail + closed instead of being silently split into extra columns. + """ + + token = value.strip().strip('"') + if not token: + raise ValueError("empty value in semicolon/decimal-comma profile") + if token.lower() in {"nan", "+nan", "-nan"}: + return float("nan") + if token.lower() in {"inf", "+inf", "-inf", "infinity", "+infinity", "-infinity"}: + raise ValueError(f"non-finite value {value!r}") + if "," not in token: + try: + number = float(token) + except (TypeError, ValueError) as exc: + raise ValueError(f"invalid numeric value {value!r}") from exc + else: + if "." in token or token.count(",") != 1: + raise ValueError( + f"ambiguous decimal-comma value {value!r}; use a single decimal comma" + ) + match = re.fullmatch(r"[-+]?\d+,\d+", token) + if match is None: + raise ValueError(f"invalid decimal-comma value {value!r}") + integer, fraction = token.rsplit(",", 1) + # ``1,234`` is equally plausible as 1.234 or 1234. A zero-leading + # decimal (0,234) is also ambiguous with a grouped value in a generic + # text file, so require a non-three-digit fractional width here. + if ( + len(fraction) == 3 + and len(integer.lstrip("+-")) <= 3 + and int(integer) != 0 + ): + raise ValueError(f"ambiguous decimal-comma value {value!r}") + number = float(f"{integer}.{fraction}") + if np.isinf(number): + raise ValueError(f"non-finite value {value!r}") + return number + + +def _read_semicolon_decimal_comma_dataframe(path: str | Path) -> pd.DataFrame | None: + """Read an unambiguous semicolon/decimal-comma table, if present. + + The generic pandas delimiter inference treats both delimiters as field + separators and can turn ``0,10;100,0`` into four columns. This parser is + selected only when the data rows clearly use semicolon as the outer + delimiter, and validates every data value before returning. + """ + + lines = Path(path).read_text(encoding="utf-8-sig", errors="strict").splitlines() + meaningful: list[tuple[int, str]] = [] + comment_header_tokens: list[str] | None = None + for index, line in enumerate(lines): + raw_stripped = line.strip() + if raw_stripped.startswith("#"): + candidate = raw_stripped.lstrip("#").strip() + if ";" in candidate: + candidate_tokens = [ + field.strip() + for field in next(csv.reader([candidate], delimiter=";")) + ] + if ( + len(candidate_tokens) >= 2 + and _comment_header_score(candidate_tokens) > 0 + and any(FLOAT_PATTERN.fullmatch(token) is None for token in candidate_tokens) + ): + comment_header_tokens = candidate_tokens + continue + stripped = _strip_inline_comment(line).strip() + if stripped: + meaningful.append((index, stripped)) + if not meaningful: + return None + + first_line = meaningful[0][1] + if ";" not in first_line: + return None + first_tokens = [ + field.strip() + for field in next(csv.reader([first_line], delimiter=";")) + ] + numeric_markers = { + "nan", "+nan", "-nan", "inf", "+inf", "-inf", + "infinity", "+infinity", "-infinity", + } + + def is_numeric_token(token: str) -> bool: + if token.strip().lower() in numeric_markers: + return True + try: + _parse_semicolon_decimal_comma_token(token) + except ValueError: + return False + return True + + first_is_numeric = len(first_tokens) >= 2 and all( + is_numeric_token(token) for token in first_tokens + ) + header_tokens = comment_header_tokens if first_is_numeric else first_tokens + data_start = 0 if first_is_numeric else 1 + data_rows = meaningful[data_start:] + if not data_rows: + return None + + # A semicolon-only table belongs to the normal parser; this special route + # is needed only when at least one data field contains a comma. + if not any("," in text for _, text in data_rows): + return None + + parsed_rows: list[list[float]] = [] + expected_width: int | None = None + for _, text in data_rows: + try: + fields = next(csv.reader([text], delimiter=";")) + except csv.Error as exc: + raise ValueError("cannot parse semicolon/decimal-comma profile") from exc + if expected_width is None: + expected_width = len(fields) + if len(fields) != expected_width or len(fields) < 2: + raise ValueError("data rows have inconsistent semicolon field widths") + parsed_rows.append([_parse_semicolon_decimal_comma_token(field) for field in fields]) + + if header_tokens is not None and len(header_tokens) != expected_width: + raise ValueError("header and data widths disagree in semicolon profile") + frame = pd.DataFrame( + parsed_rows, + columns=header_tokens if header_tokens is not None else None, + ) + # Full-file assertion: the special route must never hand a partially + # parsed table to the heuristic column selector. + if frame.empty or frame.shape[1] < 2 or np.isinf(frame.to_numpy(dtype=float)).any(): + raise ValueError("semicolon/decimal-comma profile contains invalid data") + frame.attrs["saxsabs_semicolon_decimal_comma"] = True + return frame + + def normalize_transmission(trans: float | None, raw: Any = None, key: Any = None) -> float | None: if trans is None: return None @@ -869,7 +1088,7 @@ def read_external_1d_profile( if ext == ".xml": try: return read_cansas1d_xml(p) - except Exception as exc: + except (ET.ParseError, OSError, UnicodeError, ValueError) as exc: raise ValueError(f"Cannot parse canSAS XML file: {p.name}") from exc elif ext in (".h5", ".hdf5", ".hdf", ".nxs"): return read_nxcansas_h5(p) @@ -877,12 +1096,22 @@ def read_external_1d_profile( dfs: list[pd.DataFrame] = [] errs: list[str] = [] - comment_header_df = _read_comment_header_dataframe(p) - has_comment_header = comment_header_df is not None - plain_header_tokens = _read_plain_header_tokens(p) - physical_data_width = _physical_data_width(p) - if comment_header_df is not None: - dfs.append(comment_header_df) + # Detect this grammar before any generic pandas trial: splitting both + # comma and semicolon would otherwise produce a plausible but wrong Q/I + # pair and the later heuristic cannot recover the lost decimal marks. + semicolon_decimal_df = _read_semicolon_decimal_comma_dataframe(p) + if semicolon_decimal_df is not None: + dfs.append(semicolon_decimal_df) + has_comment_header = False + plain_header_tokens = None + physical_data_width = semicolon_decimal_df.shape[1] + else: + comment_header_df = _read_comment_header_dataframe(p) + has_comment_header = comment_header_df is not None + plain_header_tokens = _read_plain_header_tokens(p) + physical_data_width = _physical_data_width(p) + if comment_header_df is not None: + dfs.append(comment_header_df) read_trials: list[dict[str, Any]] = [ {"sep": None, "engine": "python", "comment": "#"}, @@ -891,7 +1120,7 @@ def read_external_1d_profile( ] malformed_header_detected = False - for kw in read_trials: + for kw in ([] if semicolon_decimal_df is not None else read_trials): try: df = pd.read_csv(path, encoding="utf-8-sig", **kw) if kw.get("header") is None and df is not None and not df.empty: @@ -964,7 +1193,18 @@ def read_external_1d_profile( i_col, i_named = _pick_named_column( cols, {x_col}, - exact={"i", "intensity", "irel", "iabs", "signal", "count", "counts", "y"}, + exact={ + "i", + "intensity", + "irel", + "iref", + "imeas", + "iabs", + "signal", + "count", + "counts", + "y", + }, prefixes=("intensity", "signal", "count", "irel", "iabs", "i"), suffixes=("intensity",), ) @@ -992,6 +1232,10 @@ def read_external_1d_profile( x = pd.to_numeric(df[x_col], errors="coerce").to_numpy(dtype=np.float64, na_value=np.nan) intensity = pd.to_numeric(df[i_col], errors="coerce").to_numpy(dtype=np.float64, na_value=np.nan) + if df.attrs.get("saxsabs_semicolon_decimal_comma") and ( + not np.all(np.isfinite(x)) or not np.all(np.isfinite(intensity)) + ): + raise ValueError("semicolon/decimal-comma Q and I must be finite in every row") mask = np.isfinite(x) & np.isfinite(intensity) if int(mask.sum()) < 3: continue @@ -1046,6 +1290,63 @@ def read_external_1d_profile( _CANSAS_NS = "urn:cansas1d:1.1" +def _intensity_unit_semantics(value: object) -> str | None: + """Return a stable semantic token for an explicit intensity unit.""" + + text = str(value or "").strip() + if not text: + return None + if is_cm_inv_intensity_unit(text): + return "cm^-1" + token = _unit_token(text) + return f"unit:{token}" if token else None + + +def _validate_intensity_unit_records( + records: list[str], + *, + label: str, +) -> tuple[str | None, str]: + """Validate per-point unit declarations and return semantic/raw units.""" + + present = [bool(str(value).strip()) for value in records] + if any(present) and not all(present): + raise ValueError(f"canSAS XML contains mixed/missing {label} units") + semantics = {_intensity_unit_semantics(value) for value in records if str(value).strip()} + if len(semantics) > 1: + raise ValueError(f"canSAS XML contains inconsistent {label} units") + semantic = next(iter(semantics), None) + raw = next((str(value).strip() for value in records if str(value).strip()), "") + return semantic, raw + + +def _validate_i_dev_units( + i_semantics: str | None, + i_dev_records: list[str], + *, + has_i_dev: bool, +) -> None: + if not has_i_dev: + return + present = [bool(str(value).strip()) for value in i_dev_records] + if any(present) and not all(present): + raise ValueError("canSAS XML contains mixed/missing Idev units") + if not any(present): + if i_semantics is not None: + raise ValueError("canSAS XML Idev units are missing while I has units") + return + if any(present): + if i_semantics is None: + raise ValueError("canSAS XML Idev units conflict with missing I units") + dev_semantics = { + _intensity_unit_semantics(value) + for value in i_dev_records + if str(value).strip() + } + if dev_semantics != {i_semantics}: + raise ValueError("canSAS XML Idev units do not match I units") + + def read_cansas1d_xml(path: str | Path) -> dict[str, Any]: """Read a canSAS 1D XML file and return a profile dict. @@ -1066,36 +1367,70 @@ def read_cansas1d_xml(path: str | Path) -> dict[str, Any]: q_vals: list[float] = [] i_vals: list[float] = [] e_vals: list[float] = [] - intensity_unit = "" + i_unit_records: list[str] = [] + i_dev_unit_records: list[str] = [] + has_i_dev = False q_unit_records: list[tuple[str, str | None]] = [] - for idata in root.iter(f"{ns}Idata"): + idata_elements = list(root.iter(f"{ns}Idata")) + if not idata_elements: + raise ValueError(f"canSAS XML contains no Idata points: {p.name}") + for idata in idata_elements: q_el = idata.find(f"{ns}Q") i_el = idata.find(f"{ns}I") if q_el is None or i_el is None: - continue + raise ValueError(f"canSAS XML Idata point is missing Q or I: {p.name}") try: - q_value = float(q_el.text) - i_value = float(i_el.text) - except (TypeError, ValueError): - continue + q_value = float((q_el.text or "").strip()) + i_value = float((i_el.text or "").strip()) + except (TypeError, ValueError) as exc: + raise ValueError(f"canSAS XML contains non-numeric Q or I: {p.name}") from exc q_vals.append(q_value) i_vals.append(i_value) raw_q_unit = str(q_el.attrib.get("unit", "") or "").strip() q_unit_records.append((raw_q_unit, canonicalize_q_unit(raw_q_unit))) - if not intensity_unit: - intensity_unit = str(i_el.attrib.get("unit", "") or "").strip() + i_unit_records.append(str(i_el.attrib.get("unit", "") or "").strip()) e_el = idata.find(f"{ns}Idev") - if e_el is not None and e_el.text: + if e_el is not None: + has_i_dev = True + i_dev_unit_records.append(str(e_el.attrib.get("unit", "") or "").strip()) try: - e_vals.append(float(e_el.text)) - except (TypeError, ValueError): - e_vals.append(np.nan) + e_value = ( + np.nan + if not (e_el.text or "").strip() + else float((e_el.text or "").strip()) + ) + except (TypeError, ValueError) as exc: + raise ValueError(f"canSAS XML contains non-numeric Idev: {p.name}") from exc + e_vals.append(e_value) else: e_vals.append(np.nan) if len(q_vals) < 2: raise ValueError(f"canSAS XML contains too few data points: {p.name}") + x = np.asarray(q_vals, dtype=np.float64) + intensity = np.asarray(i_vals, dtype=np.float64) + if x.ndim != 1 or intensity.ndim != 1 or x.shape != intensity.shape: + raise ValueError(f"canSAS XML Q and I must be matching 1-D arrays: {p.name}") + if not np.all(np.isfinite(x)) or not np.all(np.isfinite(intensity)): + raise ValueError(f"canSAS XML Q and I must contain only finite values: {p.name}") + err = np.asarray(e_vals, dtype=np.float64) + if err.shape != x.shape: + raise ValueError(f"canSAS XML Idev shape does not match Q/I: {p.name}") + if np.any(np.isinf(err)) or np.any(np.isfinite(err) & (err < 0)): + raise ValueError( + f"canSAS XML Idev must be NaN or finite and non-negative: {p.name}" + ) + + intensity_semantics, intensity_unit = _validate_intensity_unit_records( + i_unit_records, + label="I", + ) + _validate_i_dev_units( + intensity_semantics, + i_dev_unit_records, + has_i_dev=has_i_dev, + ) canonical_by_point = [canonical for _, canonical in q_unit_records] known_q_units = {unit for unit in canonical_by_point if unit is not None} @@ -1114,10 +1449,6 @@ def read_cansas1d_xml(path: str | Path) -> dict[str, Any]: "", ) - x = np.asarray(q_vals, dtype=np.float64) - intensity = np.asarray(i_vals, dtype=np.float64) - err = np.asarray(e_vals, dtype=np.float64) if e_vals else np.full_like(x, np.nan) - operator_provenance: dict[str, str] = {} for process in root.iter(f"{ns}SASprocess"): for term in process.iter(f"{ns}term"): @@ -1132,7 +1463,7 @@ def read_cansas1d_xml(path: str | Path) -> dict[str, Any]: "x_unit": q_unit, "x_unit_raw": q_unit_raw, "i_col": "I", - "err_col": "Idev", + "err_col": "Idev" if has_i_dev else "", "intensity_unit": intensity_unit, "operator_provenance": operator_provenance, }, @@ -1167,11 +1498,12 @@ def read_nxcansas_h5(path: str | Path) -> dict[str, Any]: i_ds = None e_ds = None intensity_unit = "" + i_dev_unit = "" q_unit: str | None = None q_unit_raw = "" def _find_sasdata(group: Any) -> bool: - nonlocal q_ds, i_ds, e_ds, intensity_unit, q_unit, q_unit_raw + nonlocal q_ds, i_ds, e_ds, intensity_unit, i_dev_unit, q_unit, q_unit_raw cls = group.attrs.get("canSAS_class", "") if isinstance(cls, bytes): cls = cls.decode() @@ -1192,6 +1524,12 @@ def _find_sasdata(group: Any) -> bool: intensity_unit = str(raw_unit or "").strip() if "Idev" in group: e_ds = group["Idev"][()] + raw_i_dev_unit = group["Idev"].attrs.get("units", "") + if isinstance(raw_i_dev_unit, bytes): + raw_i_dev_unit = raw_i_dev_unit.decode( + "utf-8", errors="replace" + ) + i_dev_unit = str(raw_i_dev_unit or "").strip() return True for key in group: item = group[key] @@ -1228,24 +1566,44 @@ def _collect_operator_provenance(_name: str, item: Any) -> None: if q_ds is None or i_ds is None: raise ValueError(f"Cannot find SASdata/Q,I datasets in {p.name}") - x = np.asarray(q_ds, dtype=np.float64).ravel() - intensity = np.asarray(i_ds, dtype=np.float64).ravel() - err = ( - np.asarray(e_ds, dtype=np.float64).ravel() - if e_ds is not None - else np.full_like(x, np.nan) - ) - + x = np.asarray(q_ds, dtype=np.float64) + intensity = np.asarray(i_ds, dtype=np.float64) + if x.ndim != 1 or intensity.ndim != 1: + raise ValueError(f"NXcanSAS Q and I must be 1-D arrays in {p.name}") if x.shape != intensity.shape: raise ValueError( f"NXcanSAS dataset length mismatch in {p.name}: " f"Q has {x.size} points, I has {intensity.size}" ) + if x.size < 2: + raise ValueError(f"NXcanSAS contains too few data points: {p.name}") + if not np.all(np.isfinite(x)) or not np.all(np.isfinite(intensity)): + raise ValueError(f"NXcanSAS Q and I must contain only finite values: {p.name}") + err = ( + np.asarray(e_ds, dtype=np.float64) + if e_ds is not None + else np.full_like(x, np.nan) + ) + if err.shape != x.shape: raise ValueError( f"NXcanSAS dataset length mismatch in {p.name}: " f"Q has {x.size} points, Idev has {err.size}" ) + if np.any(np.isinf(err)) or np.any(np.isfinite(err) & (err < 0)): + raise ValueError( + f"NXcanSAS Idev must be NaN or finite and non-negative in {p.name}" + ) + intensity_semantics = _intensity_unit_semantics(intensity_unit) + if e_ds is not None: + if i_dev_unit: + if ( + intensity_semantics is None + or _intensity_unit_semantics(i_dev_unit) != intensity_semantics + ): + raise ValueError(f"NXcanSAS Idev units do not match I units in {p.name}") + elif intensity_semantics is not None: + raise ValueError(f"NXcanSAS Idev unit is missing while I has units in {p.name}") order = np.argsort(x) return _attach_intensity_arrays( @@ -1287,17 +1645,24 @@ def _collect_operator_provenance(_name: str, item: Any) -> None: def _try_parse_datetime(value: Any) -> float | None: """Best-effort conversion of many date/time header formats to unix timestamp.""" + + def parse_numeric_epoch(number: float) -> float | None: + if not np.isfinite(number): + return None + # Choose the unit whose conversion lands in a plausible Unix epoch + # range. Testing the converted value, rather than only the raw + # magnitude, distinguishes seconds, milliseconds, microseconds and + # nanoseconds for both native numerics and numeric strings. + for scale in (1.0, 1.0e3, 1.0e6, 1.0e9): + seconds = number / scale + if 1.0e9 <= seconds <= 5.0e9: + return seconds + return None + if value is None: return None if isinstance(value, (int, float, np.number)): - v = float(value) - # Heuristic: if it looks like seconds since epoch (2001-01-01 .. 2100) - if 1e9 < v < 4e9: - return v - # If it looks like milliseconds - if 1e12 < v < 4e15: - return v / 1000.0 - return None + return parse_numeric_epoch(float(value)) s = str(value).strip() if not s or s.lower() in ("none", "null", "nan"): @@ -1305,12 +1670,10 @@ def _try_parse_datetime(value: Any) -> float | None: # Try common numeric unix cases first try: - v = float(s) - if 1e9 < v < 4e9: - return v - if 1e12 < v < 4e15: - return v / 1000.0 - except Exception: + parsed = parse_numeric_epoch(float(s)) + if parsed is not None: + return parsed + except (TypeError, ValueError): pass # Try python's datetime + dateutil if present diff --git a/src/saxsabs/io/writers.py b/src/saxsabs/io/writers.py index 2148ff2..236b8f8 100644 --- a/src/saxsabs/io/writers.py +++ b/src/saxsabs/io/writers.py @@ -8,6 +8,8 @@ from __future__ import annotations +import os +import tempfile import xml.etree.ElementTree as ET from pathlib import Path from typing import Any @@ -109,10 +111,14 @@ def _prepare_profile_arrays( i_abs: np.ndarray, err: np.ndarray | None = None, ) -> tuple[np.ndarray, np.ndarray, np.ndarray | None]: - q_arr = np.asarray(q, dtype=np.float64).ravel() - i_arr = np.asarray(i_abs, dtype=np.float64).ravel() + q_arr = np.asarray(q, dtype=np.float64) + i_arr = np.asarray(i_abs, dtype=np.float64) + if q_arr.ndim != 1 or i_arr.ndim != 1: + raise ValueError("q and intensity must be 1-D arrays") if q_arr.shape != i_arr.shape: raise ValueError("q and intensity must have the same shape") + if q_arr.size == 0: + raise ValueError("q and intensity must not be empty") if not np.all(np.isfinite(q_arr)): raise ValueError("q must contain only finite values") if not np.all(np.isfinite(i_arr)): @@ -120,9 +126,15 @@ def _prepare_profile_arrays( e_arr = None if err is not None: - e_arr = np.asarray(err, dtype=np.float64).ravel() + e_arr = np.asarray(err, dtype=np.float64) + if e_arr.ndim != 1: + raise ValueError("uncertainty must be a 1-D array") if e_arr.shape != q_arr.shape: raise ValueError("q and uncertainty must have the same shape") + if np.any(np.isinf(e_arr)): + raise ValueError("uncertainty must not contain infinite values") + if np.any(np.isfinite(e_arr) & (e_arr < 0)): + raise ValueError("uncertainty must be non-negative or NaN") return q_arr, i_arr, e_arr @@ -263,60 +275,80 @@ def write_nxcansas_h5( out.parent.mkdir(parents=True, exist_ok=True) q_arr, i_arr, e_arr = _prepare_profile_arrays(q, i_abs, err) - with h5py.File(str(out), "w") as f: - entry = f.create_group("sasentry01") - entry.attrs["NX_class"] = "NXentry" - entry.attrs["canSAS_class"] = "SASentry" - entry.attrs["version"] = "1.1" - entry["definition"] = "NXcanSAS" - entry["title"] = meta.get("title", "SAXS profile") - entry["run"] = meta.get("run", "001") - - # SASdata - data = entry.create_group("sasdata01") - data.attrs["NX_class"] = "NXdata" - data.attrs["canSAS_class"] = "SASdata" - data.attrs["signal"] = "I" - data.attrs["I_axes"] = "Q" - data.attrs["Q_indices"] = 0 - - ds_q = data.create_dataset("Q", data=q_arr) - ds_q.attrs["units"] = "1/angstrom" - - ds_i = data.create_dataset("I", data=i_arr) - ds_i.attrs["units"] = intensity_unit - - if e_arr is not None: - ds_e = data.create_dataset("Idev", data=e_arr) - ds_e.attrs["units"] = intensity_unit - - # SASinstrument (minimal) - inst = entry.create_group("sasinstrument01") - inst.attrs["NX_class"] = "NXinstrument" - inst.attrs["canSAS_class"] = "SASinstrument" - if meta.get("instrument_name"): - inst["name"] = meta["instrument_name"] - - src = inst.create_group("source01") - src.attrs["NX_class"] = "NXsource" - src["radiation"] = "x-ray" - if "wavelength_A" in meta: - ds_wl = src.create_dataset( - "incident_wavelength", data=meta["wavelength_A"] - ) - ds_wl.attrs["units"] = "angstrom" - - # SASprocess - process = entry.create_group("sasprocess01") - process.attrs["NX_class"] = "NXprocess" - process.attrs["canSAS_class"] = "SASprocess" - process["name"] = meta.get("process_name", "SAXSAbs absolute calibration") - for key, value in _operator_provenance_from_metadata(meta).items(): - process[key] = value - # SASsample - sample = entry.create_group("sassample01") - sample.attrs["NX_class"] = "NXsample" - sample.attrs["canSAS_class"] = "SASsample" - sample["name"] = meta.get("sample_name", "unknown") + # h5py creates/truncates its target before validating every metadata + # assignment. Build beside the destination and replace only after the + # complete file has closed successfully, so a failed write cannot destroy + # a previously valid result. + fd, temporary_name = tempfile.mkstemp( + prefix=f".{out.name}.", suffix=".tmp", dir=str(out.parent) + ) + os.close(fd) + temporary = Path(temporary_name) + committed = False + try: + with h5py.File(str(temporary), "w") as f: + entry = f.create_group("sasentry01") + entry.attrs["NX_class"] = "NXentry" + entry.attrs["canSAS_class"] = "SASentry" + entry.attrs["version"] = "1.1" + entry["definition"] = "NXcanSAS" + entry["title"] = meta.get("title", "SAXS profile") + entry["run"] = meta.get("run", "001") + + # SASdata + data = entry.create_group("sasdata01") + data.attrs["NX_class"] = "NXdata" + data.attrs["canSAS_class"] = "SASdata" + data.attrs["signal"] = "I" + data.attrs["I_axes"] = "Q" + data.attrs["Q_indices"] = 0 + + ds_q = data.create_dataset("Q", data=q_arr) + ds_q.attrs["units"] = "1/angstrom" + + ds_i = data.create_dataset("I", data=i_arr) + ds_i.attrs["units"] = intensity_unit + + if e_arr is not None: + ds_e = data.create_dataset("Idev", data=e_arr) + ds_e.attrs["units"] = intensity_unit + + # SASinstrument (minimal) + inst = entry.create_group("sasinstrument01") + inst.attrs["NX_class"] = "NXinstrument" + inst.attrs["canSAS_class"] = "SASinstrument" + if meta.get("instrument_name"): + inst["name"] = meta["instrument_name"] + + src = inst.create_group("source01") + src.attrs["NX_class"] = "NXsource" + src["radiation"] = "x-ray" + if "wavelength_A" in meta: + ds_wl = src.create_dataset( + "incident_wavelength", data=meta["wavelength_A"] + ) + ds_wl.attrs["units"] = "angstrom" + + # SASprocess + process = entry.create_group("sasprocess01") + process.attrs["NX_class"] = "NXprocess" + process.attrs["canSAS_class"] = "SASprocess" + process["name"] = meta.get("process_name", "SAXSAbs absolute calibration") + for key, value in _operator_provenance_from_metadata(meta).items(): + process[key] = value + # SASsample + sample = entry.create_group("sassample01") + sample.attrs["NX_class"] = "NXsample" + sample.attrs["canSAS_class"] = "SASsample" + sample["name"] = meta.get("sample_name", "unknown") + + os.replace(str(temporary), str(out)) + committed = True + finally: + if not committed: + try: + temporary.unlink() + except FileNotFoundError: + pass return out diff --git a/src/saxsabs/workflows/bl19b2_abs2d.py b/src/saxsabs/workflows/bl19b2_abs2d.py index 72b1190..0ef195c 100644 --- a/src/saxsabs/workflows/bl19b2_abs2d.py +++ b/src/saxsabs/workflows/bl19b2_abs2d.py @@ -1629,6 +1629,46 @@ def validate_instrument_consistency( return tuple(warnings) +def validate_output_collisions( + sample_paths: list[Path], + *, + input_root: str | Path, + output_root: str | Path, +) -> None: + """Reject distinct inputs which would overwrite one generated output set. + + TIFF and TIFF-with-extra-suffix are both accepted by the BL19B2 scanner, but + ``Path.stem`` maps ``frame.tif`` and ``frame.tiff`` to the same generated + names. This check is intentionally performed before any output directory or + provenance artifact is created. + """ + collisions: dict[Path, list[Path]] = {} + for source in sample_paths: + paths = build_output_paths( + source, + input_root=input_root, + output_root=output_root, + ) + for target in (paths.h5, paths.edf, paths.metadata, paths.preview): + collisions.setdefault(target.resolve(), []).append(Path(source)) + duplicate_targets = { + target: sources + for target, sources in collisions.items() + if len({source.resolve() for source in sources}) > 1 + } + if not duplicate_targets: + return + details = [] + for target, sources in sorted(duplicate_targets.items(), key=lambda item: str(item[0])): + names = ", ".join(str(source) for source in sources) + details.append(f"{target}: {names}") + raise ValueError( + "generated output collision: different input names map to the same output " + "stem; rename one input or choose a different input layout before writing. " + + " | ".join(details) + ) + + def natural_key(path: str | Path) -> list[Any]: text = str(path) parts = re.split(r"(\d+)", text) @@ -2893,6 +2933,11 @@ def write_provenance_package( ) -> ProvenancePaths: out_root = config.resolved_output_root() paths = _provenance_paths(out_root) + package_root = out_root.resolve() + + def generated(path: Path) -> str: + return path.resolve().relative_to(package_root).as_posix() + paths.run_command.parent.mkdir(parents=True, exist_ok=True) if control_inputs is None: control_inputs = _load_run_control_inputs(config) @@ -2915,7 +2960,7 @@ def write_provenance_package( "output_root": str(out_root), "source_poni_path": _optional_path_text(config.poni_path), "pydidas_cali_yaml": _optional_path_text(config.pydidas_cali_yaml), - "safe_poni_path": str(safe_poni_path), + "safe_poni_path": generated(safe_poni_path), "references": { "dark": str(reference_paths.dark), "background": str(reference_paths.background), @@ -2924,8 +2969,8 @@ def write_provenance_package( "user_mask": str(reference_paths.mask or ""), }, "mask": { - "npy": str(mask_info.npy_path), - "edf": str(mask_info.edf_path), + "npy": generated(mask_info.npy_path), + "edf": generated(mask_info.edf_path), "checksum_sha256": mask_info.checksum_sha256, "user_mask_pixels": mask_info.user_mask_pixels, "detector_mask_pixels": mask_info.detector_mask_pixels, @@ -2954,10 +2999,10 @@ def write_provenance_package( "details": str(paths.code_state), }, "files": { - "run_command": str(paths.run_command), - "processing_environment": str(paths.processing_environment), - "code_state": str(paths.code_state), - "provenance_summary": str(paths.provenance_summary), + "run_command": generated(paths.run_command), + "processing_environment": generated(paths.processing_environment), + "code_state": generated(paths.code_state), + "provenance_summary": generated(paths.provenance_summary), }, } _write_json(paths.provenance_summary, summary) @@ -3378,6 +3423,19 @@ def calibrate_standard( label="standard", ) ) + # Background pixels participate in the K calibration and must therefore be + # bound to the same detector geometry as the standard. Checking only the + # standard lets a same-shaped but different-energy/beam-centre blank silently + # contaminate the calibration curve. + warnings.extend( + validate_instrument_consistency( + bg_header, + image_shape=background.shape, + integrator=ai, + label="background", + reference_header=std_header, + ) + ) kwargs: dict[str, Any] = { "unit": "q_A^-1", "correctSolidAngle": bool(config.correct_solid_angle_for_k), @@ -3726,15 +3784,23 @@ def _pydidas_index_row( paths: OutputPaths, safe_poni_path: Path, mask_info: MaskInfo, + output_root: Path | None = None, ) -> dict[str, Any]: + package_root = Path(output_root).resolve() if output_root is not None else None + + def generated(path: Path) -> str: + if package_root is None: + return str(path) + return path.resolve().relative_to(package_root).as_posix() + return { "raw_sample": str(source), - "edf": str(paths.edf), - "hdf5": str(paths.h5), - "poni": str(safe_poni_path), - "mask": str(mask_info.npy_path), - "mask_edf": str(mask_info.edf_path), - "metadata": str(paths.metadata), + "edf": generated(paths.edf), + "hdf5": generated(paths.h5), + "poni": generated(safe_poni_path), + "mask": generated(mask_info.npy_path), + "mask_edf": generated(mask_info.edf_path), + "metadata": generated(paths.metadata), "normalization_factor": 1.0, "dark": "", "flat": "", @@ -3746,6 +3812,7 @@ def _validate_resumed_array( *, metadata: dict[str, Any], label: str, + package_root: Path | None = None, ) -> None: arr = np.asarray(image) image_meta = metadata.get("output_image", {}) @@ -3761,7 +3828,12 @@ def _validate_resumed_array( finite = np.isfinite(arr) if np.all(finite): return - mask_path = Path(str(metadata.get("mask", {}).get("npy", ""))) + raw_mask_path = Path(str(metadata.get("mask", {}).get("npy", ""))) + mask_path = ( + raw_mask_path + if raw_mask_path.is_absolute() or package_root is None + else package_root / raw_mask_path + ) try: mask = np.load(mask_path, allow_pickle=False) except (OSError, ValueError) as exc: @@ -3770,7 +3842,43 @@ def _validate_resumed_array( raise ValueError(f"existing {label} contains non-finite unmasked detector values") +def _package_root_from_output_paths(paths: OutputPaths) -> Path: + """Infer package root without confusing an input folder named ``metadata``.""" + metadata_path = paths.metadata.resolve() + h5_path = paths.h5.resolve() + edf_path = paths.edf.resolve() + for candidate in (paths.metadata.parent, *paths.metadata.parents): + if candidate.name.casefold() == "metadata": + root = candidate.parent.resolve() + try: + metadata_path.relative_to(root / "metadata") + h5_path.relative_to(root / "images_h5") + edf_path.relative_to(root / "images_edf") + except ValueError: + continue + return root + raise ValueError(f"cannot infer BL19B2 package root from metadata path: {paths.metadata}") + + +def _resolve_generated_output_path( + value: str, + *, + package_root: Path, + expected: Path, + label: str, +) -> Path: + """Resolve a generated output pointer while accepting old absolute packages.""" + recorded = Path(value) + resolved = ( + recorded if recorded.is_absolute() else package_root / recorded + ).resolve() + if resolved != expected.resolve(): + raise ValueError(f"existing output path mismatch for {label}: {value!r}") + return resolved + + def _validate_existing_outputs(paths: OutputPaths, metadata: dict[str, Any]) -> None: + package_root = _package_root_from_output_paths(paths) outputs = metadata.get("outputs", {}) external_k_contract = _k_calibration_contract( metadata.get("absolute_calibration", {}), @@ -3782,8 +3890,14 @@ def _validate_existing_outputs(paths: OutputPaths, metadata: dict[str, Any]) -> ("metadata", paths.metadata), ): recorded = str(outputs.get(key, "")) - if not recorded or Path(recorded).resolve() != expected_path.resolve(): + if not recorded: raise ValueError(f"existing output path mismatch for {key}: {recorded!r}") + _resolve_generated_output_path( + recorded, + package_root=package_root, + expected=expected_path, + label=key, + ) for label, path, key in ( ("HDF5", paths.h5, "hdf5_sha256"), ("EDF", paths.edf, "edf_sha256"), @@ -3884,7 +3998,12 @@ def _validate_existing_outputs(paths: OutputPaths, metadata: dict[str, Any]) -> ) except (OSError, KeyError, TypeError, json.JSONDecodeError) as exc: raise ValueError(f"existing HDF5 output is unreadable or incomplete: {paths.h5}") from exc - _validate_resumed_array(h5_image, metadata=metadata, label="HDF5") + _validate_resumed_array( + h5_image, + metadata=metadata, + label="HDF5", + package_root=package_root, + ) try: loaded_edf = load_detector_image(paths.edf, dtype=None) @@ -3936,13 +4055,25 @@ def _validate_existing_outputs(paths: OutputPaths, metadata: dict[str, Any]) -> ) if header.get("ExpandedUStatus") != expected_expanded_status: raise ValueError("existing EDF expanded uncertainty status mismatch") - if Path(str(header.get("UncertaintyHDF5", ""))).resolve() != paths.h5.resolve(): + uncertainty_pointer = Path(str(header.get("UncertaintyHDF5", ""))) + if ( + uncertainty_pointer.is_absolute() + and uncertainty_pointer.resolve() != paths.h5.resolve() + ) or ( + not uncertainty_pointer.is_absolute() + and (package_root / uncertainty_pointer).resolve() != paths.h5.resolve() + ): raise ValueError("existing EDF uncertainty HDF5 pointer mismatch") except (OSError, KeyError, TypeError, ValueError) as exc: if isinstance(exc, ValueError) and "existing EDF" in str(exc): raise raise ValueError(f"existing EDF output is unreadable or incomplete: {paths.edf}") from exc - _validate_resumed_array(edf_image, metadata=metadata, label="EDF") + _validate_resumed_array( + edf_image, + metadata=metadata, + label="EDF", + package_root=package_root, + ) def _validate_metadata_processing_signature(metadata: dict[str, Any]) -> None: @@ -4029,6 +4160,7 @@ def _frame_qc_row_from_metadata( if metadata.get("frame_signature") != expected_frame_signature: raise ValueError(f"existing BL19B2 output frame signature mismatch for {rel}") _validate_existing_outputs(paths, metadata) + package_root = _package_root_from_output_paths(paths) outputs = metadata.get("outputs", {}) qc = metadata.get("qc", {}) normalization = metadata.get("normalization", {}) @@ -4037,15 +4169,24 @@ def _frame_qc_row_from_metadata( mask = metadata.get("mask", {}) warnings = metadata.get("warnings", []) warning_text = " | ".join(str(item) for item in warnings) if isinstance(warnings, list) else str(warnings) - preview = str(outputs.get("preview") or paths.preview) - if not Path(preview).exists(): + raw_preview = str(outputs.get("preview") or paths.preview) + preview_path = Path(raw_preview) + if not preview_path.is_absolute(): + preview_path = package_root / preview_path + preview = str(preview_path) + if not preview_path.exists(): preview = "" + def output_path(key: str, expected: Path) -> str: + raw = str(outputs.get(key) or expected) + path = Path(raw) + return str(path if path.is_absolute() else package_root / path) + return { "relative_path": str(rel), "status": "success_existing", - "hdf5": str(outputs.get("hdf5") or paths.h5), - "edf": str(outputs.get("edf") or paths.edf), - "metadata": str(outputs.get("metadata") or paths.metadata), + "hdf5": output_path("hdf5", paths.h5), + "edf": output_path("edf", paths.edf), + "metadata": output_path("metadata", paths.metadata), "preview": preview, "processing_signature": metadata.get("processing_signature", ""), "mask": str(mask.get("npy", "")), @@ -4094,6 +4235,11 @@ def _frame_metadata( if control_inputs is None: control_inputs = _load_run_control_inputs(config) control_provenance = _control_inputs_provenance_payload(control_inputs) + package_root = config.resolved_output_root().resolve() + + def generated(path: Path) -> str: + return path.resolve().relative_to(package_root).as_posix() + metadata = { 'sample_selection': control_provenance['include_manifest'], "schema": SCHEMA_VERSION, @@ -4104,10 +4250,10 @@ def _frame_metadata( "frame_signature": _frame_signature(processing_signature, source_identity), "raw_sample": str(source), "outputs": { - "hdf5": str(paths.h5), - "edf": str(paths.edf), - "metadata": str(paths.metadata), - "preview": str(paths.preview), + "hdf5": generated(paths.h5), + "edf": generated(paths.edf), + "metadata": generated(paths.metadata), + "preview": generated(paths.preview), }, "intensity_unit": INTENSITY_UNIT, "output_image": { @@ -4165,8 +4311,8 @@ def _frame_metadata( "transmission_policy": "QC only; normalized with T_bg=1 under NIST convention", }, "mask": { - "npy": str(mask_info.npy_path), - "edf": str(mask_info.edf_path), + "npy": generated(mask_info.npy_path), + "edf": generated(mask_info.edf_path), "checksum_sha256": mask_info.checksum_sha256, "convention": "pyFAI: 0=valid, 1=masked", "sources": { @@ -4182,7 +4328,7 @@ def _frame_metadata( }, }, "geometry": { - "poni": str(safe_poni_path), + "poni": generated(safe_poni_path), "source_poni_path": _optional_path_text(config.poni_path), "pydidas_cali_yaml": _optional_path_text(config.pydidas_cali_yaml), "energy_kev": header.energy_kev, @@ -4210,14 +4356,14 @@ def _frame_metadata( "polarization": False, }, "corrections_deferred_to_integration": { - "mask": str(mask_info.npy_path), + "mask": generated(mask_info.npy_path), "solid_angle": bool(config.correct_solid_angle_for_k), "polarization_factor": config.polarization_factor, }, "recommended_reintegration": { "dark": None, "flat": None, - "mask": str(mask_info.npy_path), + "mask": generated(mask_info.npy_path), "normalization_factor": 1.0, "do_not_repeat": ["dark", "background", "transmission", "monitor", "thickness", "K"], "correctSolidAngle": bool(config.correct_solid_angle_for_k), @@ -4309,6 +4455,13 @@ def run_bl19b2_abs2d(config: BL19B2Abs2DConfig) -> dict[str, Any]: inventory_rows, sample_paths = scan_inputs( config, include_manifest=control_inputs.include_manifest ) + # Validate the complete selected input set before creating any package + # directory or copying generated configuration artifacts. + validate_output_collisions( + sample_paths, + input_root=input_root, + output_root=out_root, + ) reference_sources: dict[str, DetectorSourceSnapshot] | None = None if not config.dry_run: @@ -4496,6 +4649,12 @@ def run_bl19b2_abs2d(config: BL19B2Abs2DConfig) -> dict[str, Any]: software_versions=software_versions, code_state=code_state, ) + for output_key in ("hdf5", "edf", "metadata", "preview"): + value = row.get(output_key) + if value: + output_path = Path(str(value)) + if output_path.is_absolute(): + row[output_key] = output_path.resolve().relative_to(out_root.resolve()).as_posix() skipped += 1 frame_qc_rows.append(row) manifest_row = {"raw_sample": str(source), **row} @@ -4507,6 +4666,7 @@ def run_bl19b2_abs2d(config: BL19B2Abs2DConfig) -> dict[str, Any]: paths=paths, safe_poni_path=safe_poni, mask_info=mask_info, + output_root=out_root, ) ) if row.get("warnings"): @@ -4674,10 +4834,12 @@ def run_bl19b2_abs2d(config: BL19B2Abs2DConfig) -> dict[str, Any]: row = { "relative_path": str(rel), "status": "success", - "hdf5": str(paths.h5), - "edf": str(paths.edf), - "metadata": str(paths.metadata), - "preview": str(paths.preview) if preview_written else "", + "hdf5": paths.h5.relative_to(out_root).as_posix(), + "edf": paths.edf.relative_to(out_root).as_posix(), + "metadata": paths.metadata.relative_to(out_root).as_posix(), + "preview": paths.preview.relative_to(out_root).as_posix() + if preview_written + else "", "processing_signature": processing_signature, "mask": str(mask_info.npy_path), "k_factor": calibration.k_factor, @@ -4696,6 +4858,7 @@ def run_bl19b2_abs2d(config: BL19B2Abs2DConfig) -> dict[str, Any]: paths=paths, safe_poni_path=safe_poni, mask_info=mask_info, + output_root=out_root, ) ) if warnings: diff --git a/src/saxsabs/workflows/bl19b2_integrate1d.py b/src/saxsabs/workflows/bl19b2_integrate1d.py index afed10d..29cbd8e 100644 --- a/src/saxsabs/workflows/bl19b2_integrate1d.py +++ b/src/saxsabs/workflows/bl19b2_integrate1d.py @@ -14,7 +14,7 @@ import io import json import re -from dataclasses import dataclass +from dataclasses import dataclass, replace from pathlib import Path, PurePosixPath from typing import Any @@ -104,6 +104,13 @@ def _metadata_scientific_sha256(metadata: dict[str, Any]) -> str: return _canonical_hash(scientific) +def _processing_signature_digest(payload: dict[str, Any]) -> str: + """Recompute the canonical BL19B2 2D processing-signature digest.""" + if not isinstance(payload, dict): + raise ValueError("2D processing_signature_payload must be an object") + return _canonical_hash(payload) + + def _package_file(path: Path, root: Path, label: str) -> Path: root_resolved = root.resolve(strict=True) resolved = path.resolve(strict=True) @@ -129,11 +136,24 @@ def _relative_input_path(value: str) -> Path: return Path(*pure.parts) -def _resolve_manifest_output(row: dict[str, str], names: tuple[str, ...], label: str) -> Path: +def _resolve_manifest_output( + row: dict[str, str], + names: tuple[str, ...], + label: str, + *, + package_root: Path | None = None, +) -> Path: values = [row.get(name, "").strip() for name in names if row.get(name, "").strip()] if not values: raise ValueError(f"successful 2D manifest row is missing {label}") - resolved = [Path(value).resolve(strict=True) for value in values] + resolved = [ + ( + Path(value) + if Path(value).is_absolute() or package_root is None + else package_root / Path(value) + ).resolve(strict=True) + for value in values + ] if any(path != resolved[0] for path in resolved[1:]): raise ValueError(f"conflicting {label} columns in 2D manifest") return resolved[0] @@ -177,8 +197,18 @@ def _read_manifest(config: Integrate1DConfig) -> tuple[Path, bytes, list[_InputR if rel_key in seen_rel: raise ValueError(f"duplicate relative_path in 2D manifest: {rel}") seen_rel.add(rel_key) - edf = _resolve_manifest_output(row, ("edf", "output_edf"), "EDF path") - metadata = _resolve_manifest_output(row, ("metadata", "output_metadata"), "metadata path") + edf = _resolve_manifest_output( + row, + ("edf", "output_edf"), + "EDF path", + package_root=package, + ) + metadata = _resolve_manifest_output( + row, + ("metadata", "output_metadata"), + "metadata path", + package_root=package, + ) expected_edf = (image_root / rel.parent / f"{rel.stem}_abs2d_cm-1.edf").resolve() expected_meta = (metadata_root / rel.parent / f"{rel.stem}_abs2d.json").resolve() if edf != expected_edf or metadata != expected_meta: @@ -331,6 +361,114 @@ def _read_profile(path: Path, npt: int) -> tuple[np.ndarray, np.ndarray]: return data[:, 0], data[:, 1] +def _resolve_package_path(value: Any, package_root: Path, *, label: str) -> Path: + """Resolve a generated package pointer, accepting legacy absolute paths.""" + text = str(value or "").strip() + if not text: + raise ValueError(f"2D metadata is missing {label}") + path = Path(text) + resolved = (path if path.is_absolute() else package_root / path).resolve() + return resolved + + +def _reintegration_contract(metadata: dict[str, Any], item: _InputRow) -> dict[str, Any]: + recommended = metadata.get("recommended_reintegration", {}) + if not isinstance(recommended, dict): + raise ValueError(f"2D reintegration contract is missing for {item.relative_path}") + if "correctSolidAngle" not in recommended: + raise ValueError( + f"2D reintegration correctSolidAngle is missing for {item.relative_path}" + ) + correct_solid_angle = recommended["correctSolidAngle"] + if not isinstance(correct_solid_angle, bool): + raise ValueError( + f"2D reintegration correctSolidAngle must be boolean for {item.relative_path}" + ) + def _polarization_value(value: Any, label: str) -> float | None: + if value is None: + return None + if isinstance(value, bool) or not isinstance( + value, (int, float, np.integer, np.floating) + ): + raise ValueError( + f"2D reintegration {label} must be numeric or null for {item.relative_path}" + ) + number = float(value) + if not np.isfinite(number) or number < -1.0 or number > 1.0: + raise ValueError( + f"2D reintegration {label} must be in [-1, 1] for {item.relative_path}" + ) + return number + + polarization = _polarization_value( + recommended.get("polarization_factor"), "polarization_factor" + ) + deferred = metadata.get("corrections_deferred_to_integration") + if isinstance(deferred, dict): + deferred_solid = deferred.get("solid_angle") + if deferred_solid is not None and not isinstance(deferred_solid, bool): + raise ValueError( + f"2D reintegration solid-angle contract must be boolean for {item.relative_path}" + ) + if deferred_solid is not None and deferred_solid != correct_solid_angle: + raise ValueError( + f"2D reintegration solid-angle contract is inconsistent for {item.relative_path}" + ) + deferred_pol = deferred.get("polarization_factor") + deferred_pol = _polarization_value(deferred_pol, "polarization_factor") + if deferred_pol != polarization: + raise ValueError( + f"2D reintegration polarization contract is inconsistent for {item.relative_path}" + ) + signed_payload = metadata.get("processing_signature_payload", {}) + if isinstance(signed_payload, dict): + signed_fields = { + "correct_solid_angle_for_k", + "polarization_factor", + } + signed_fields_present = signed_fields & signed_payload.keys() + if signed_fields_present and signed_fields_present != signed_fields: + raise ValueError( + f"2D reintegration signed settings are incomplete for {item.relative_path}" + ) + if signed_fields_present == signed_fields: + signed_solid = signed_payload["correct_solid_angle_for_k"] + if not isinstance(signed_solid, bool): + raise ValueError( + "2D reintegration signed correct_solid_angle_for_k must be " + f"boolean for {item.relative_path}" + ) + signed_pol = _polarization_value( + signed_payload["polarization_factor"], "signed polarization_factor" + ) + if signed_solid != correct_solid_angle: + raise ValueError( + f"2D reintegration correctSolidAngle must match signed package settings " + f"for {item.relative_path}" + ) + if signed_pol != polarization: + raise ValueError( + f"2D reintegration polarization_factor must match signed package settings " + f"for {item.relative_path}" + ) + # Legacy hand-built packages did not sign the deferred correction policy. + # Keep their historical true/null contract strict so a post-hoc metadata + # edit cannot silently change the numerical operator. + if not signed_fields_present: + if correct_solid_angle is not True: + raise ValueError( + f"2D reintegration correctSolidAngle must be true for {item.relative_path}" + ) + if polarization is not None: + raise ValueError( + f"2D reintegration polarization_factor must be null for {item.relative_path}" + ) + return { + "correctSolidAngle": correct_solid_angle, + "polarization_factor": polarization, + } + + def _validate_metadata( item: _InputRow, mask_path: Path, @@ -338,7 +476,8 @@ def _validate_metadata( mask_checksum_sha256: str, poni_path: Path, poni_sha256: str, -) -> tuple[dict[str, Any], str]: + package_root: Path, +) -> tuple[dict[str, Any], str, dict[str, Any]]: raw = item.metadata.read_bytes() try: metadata = json.loads(raw.decode("utf-8")) @@ -346,18 +485,46 @@ def _validate_metadata( raise ValueError(f"unreadable 2D metadata: {item.metadata}") from exc if metadata.get("processing_signature") != item.processing_signature: raise ValueError(f"2D processing signature mismatch for {item.relative_path}") + try: + recomputed_signature = _processing_signature_digest( + metadata.get("processing_signature_payload") + ) + except ValueError as exc: + raise ValueError( + f"2D processing signature payload is invalid for {item.relative_path}" + ) from exc + if recomputed_signature != metadata.get("processing_signature"): + raise ValueError( + f"2D processing signature payload digest mismatch for {item.relative_path}" + ) + if recomputed_signature != item.processing_signature: + raise ValueError( + f"2D processing signature does not match manifest for {item.relative_path}" + ) if metadata.get("intensity_unit") != "cm^-1" or not metadata.get("frame_signature"): raise ValueError(f"2D metadata lacks absolute-intensity/frame provenance for {item.relative_path}") - recorded_edf = Path(str(metadata.get("outputs", {}).get("edf", ""))).resolve() + recorded_edf = _resolve_package_path( + metadata.get("outputs", {}).get("edf"), + package_root, + label="EDF output", + ) if recorded_edf != item.edf: raise ValueError(f"2D metadata EDF pointer mismatch for {item.relative_path}") - recorded_mask = Path(str(metadata.get("mask", {}).get("npy", ""))).resolve() + recorded_mask = _resolve_package_path( + metadata.get("mask", {}).get("npy"), + package_root, + label="mask path", + ) if recorded_mask != mask_path: raise ValueError(f"2D metadata mask pointer mismatch for {item.relative_path}") recorded_mask_checksum = str(metadata.get("mask", {}).get("checksum_sha256", "")) if not recorded_mask_checksum or recorded_mask_checksum != mask_checksum_sha256: raise ValueError(f"2D metadata mask array checksum mismatch for {item.relative_path}") - recorded_poni = Path(str(metadata.get("geometry", {}).get("poni", ""))).resolve() + recorded_poni = _resolve_package_path( + metadata.get("geometry", {}).get("poni"), + package_root, + label="PONI path", + ) if recorded_poni != poni_path: raise ValueError(f"2D metadata PONI pointer mismatch for {item.relative_path}") recorded_poni_sha = str( @@ -389,18 +556,8 @@ def _validate_metadata( normalization = recommended.get("normalization_factor") if isinstance(normalization, bool) or normalization != 1.0: raise ValueError(f"2D reintegration normalization must be 1 for {item.relative_path}") - if recommended.get("correctSolidAngle") is not True: - raise ValueError( - f"2D reintegration correctSolidAngle must be true for {item.relative_path}" - ) - if ( - "polarization_factor" not in recommended - or recommended["polarization_factor"] is not None - ): - raise ValueError( - f"2D reintegration polarization_factor must be null for {item.relative_path}" - ) - return metadata, _metadata_scientific_sha256(metadata) + contract = _reintegration_contract(metadata, item) + return metadata, _metadata_scientific_sha256(metadata), contract def _load_validate_edf(item: _InputRow, metadata: dict[str, Any], mask: np.ndarray) -> np.ndarray: @@ -433,8 +590,8 @@ def _integrate(ai: Any, image: np.ndarray, mask: np.ndarray, config: Integrate1D unit=config.unit, method=config.method, mask=mask, - correctSolidAngle=True, - polarization_factor=None, + correctSolidAngle=bool(config.correct_solid_angle), + polarization_factor=config.polarization_factor, dark=None, flat=None, normalization_factor=1.0, @@ -463,8 +620,6 @@ def run_bl19b2_integrate1d(config: Integrate1DConfig) -> dict[str, Any]: package = Path(config.package_root).resolve(strict=True) if config.npt != 5500 or config.unit != "q_A^-1" or config.method.casefold() != "csr": raise ValueError("BL19B2 production integration requires 5500 q_A^-1 points with CSR") - if config.correct_solid_angle is not True or config.polarization_factor is not None: - raise ValueError("BL19B2 production integration requires solid-angle=True and no polarization") manifest, manifest_bytes, items = _read_manifest(config) poni = _package_file( Path(config.poni_path or _find_single(package / "config" / "geometry", "*.poni", "PONI")), @@ -478,16 +633,25 @@ def run_bl19b2_integrate1d(config: Integrate1DConfig) -> dict[str, Any]: mask = _load_mask(mask_path) poni_sha256 = _file_sha256(poni) mask_array_sha256 = _mask_checksum(mask) - validated_inputs: dict[Path, tuple[dict[str, Any], str, str]] = {} + validated_inputs: dict[Path, tuple[dict[str, Any], str, str, dict[str, Any]]] = {} selection_rows: list[dict[str, str]] = [] + reintegration_contract: dict[str, Any] | None = None for item in items: - metadata, metadata_scientific_sha = _validate_metadata( + metadata, metadata_scientific_sha, item_contract = _validate_metadata( item, mask_path, mask_checksum_sha256=mask_array_sha256, poni_path=poni, poni_sha256=poni_sha256, + package_root=package, ) + if reintegration_contract is None: + reintegration_contract = item_contract + elif item_contract != reintegration_contract: + raise ValueError( + "selected 2D frames do not share one solid-angle/polarization " + f"reintegration contract; mismatch at {item.relative_path}" + ) edf_sha = _file_sha256(item.edf) recorded_edf_sha = str(metadata.get("outputs", {}).get("edf_sha256", "")) if not recorded_edf_sha or edf_sha != recorded_edf_sha: @@ -496,6 +660,7 @@ def run_bl19b2_integrate1d(config: Integrate1DConfig) -> dict[str, Any]: metadata, metadata_scientific_sha, edf_sha, + item_contract, ) selection_rows.append( { @@ -507,6 +672,13 @@ def run_bl19b2_integrate1d(config: Integrate1DConfig) -> dict[str, Any]: "metadata_scientific_sha256": metadata_scientific_sha, } ) + if reintegration_contract is None: # pragma: no cover - _read_manifest guarantees rows + raise ValueError("2D manifest contains no reintegration contract") + effective_config = replace( + config, + correct_solid_angle=bool(reintegration_contract["correctSolidAngle"]), + polarization_factor=reintegration_contract["polarization_factor"], + ) selection_doc = { "schema": "saxsabs.bl19b2_integrate1d.input_selection.v1", "frames": selection_rows, @@ -521,14 +693,14 @@ def run_bl19b2_integrate1d(config: Integrate1DConfig) -> dict[str, Any]: "npt": config.npt, "unit": config.unit, "method": "csr", - "correctSolidAngle": True, - "polarization_factor": None, + "correctSolidAngle": effective_config.correct_solid_angle, + "polarization_factor": effective_config.polarization_factor, "dark": None, "flat": None, "normalization_factor": 1.0, "do_not_repeat": sorted(DO_NOT_REPEAT), "pyFAI_version": _version("pyFAI"), - **_fluorescence_config_payload(config), + **_fluorescence_config_payload(effective_config), } run_signature = _canonical_hash(signature_payload) out = config.output_root() @@ -570,8 +742,10 @@ def run_bl19b2_integrate1d(config: Integrate1DConfig) -> dict[str, Any]: readme = ( "# BL19B2 absolute 1D integration\n\n" "The EDF inputs were already corrected for dark, background, monitor, transmission, " - "thickness, and K. This step applies only the package mask and one solid-angle correction " - "during pyFAI CSR integration. No polarization correction is applied. " + "thickness, and K. This step applies only the package mask and the 2D package's " + f"solid-angle contract (correctSolidAngle={effective_config.correct_solid_angle}) " + "during pyFAI CSR integration, with " + f"polarization_factor={effective_config.polarization_factor!r}. " "Optional fluorescence subtraction, if configured, is applied to the absolute 1D curve.\n" ).encode("utf-8") _write_new_or_verify(out / "README.md", readme) @@ -584,7 +758,7 @@ def run_bl19b2_integrate1d(config: Integrate1DConfig) -> dict[str, Any]: profiles: dict[Path, list[tuple[_InputRow, np.ndarray, np.ndarray, dict[str, Any]]]] = {} manifest_rows: list[dict[str, Any]] = [] for item in items: - metadata, metadata_scientific_sha, edf_sha = validated_inputs[item.relative_path] + metadata, metadata_scientific_sha, edf_sha, _item_contract = validated_inputs[item.relative_path] frame_payload = { "run_signature": run_signature, "relative_path": item.relative_path.as_posix(), @@ -604,29 +778,41 @@ def run_bl19b2_integrate1d(config: Integrate1DConfig) -> dict[str, Any]: raise ValueError(f"1D resume frame signature mismatch for {item.relative_path}") if side_doc.get("profile_sha256") != _file_sha256(profile): raise ValueError(f"1D resume profile checksum mismatch for {item.relative_path}") + side_policy = side_doc.get("integration_policy", {}) + if ( + side_policy.get("correctSolidAngle") + != effective_config.correct_solid_angle + or side_policy.get("polarization_factor") + != effective_config.polarization_factor + ): + raise ValueError( + f"1D resume reintegration contract mismatch for {item.relative_path}" + ) q, intensity = _read_profile(profile, config.npt) skipped += 1 else: image = _load_validate_edf(item, metadata, mask) - q, intensity = _integrate(ai, image, mask, config) - intensity, fluorescence_diag = _apply_optional_fluorescence(q, intensity, config) + q, intensity = _integrate(ai, image, mask, effective_config) + intensity, fluorescence_diag = _apply_optional_fluorescence( + q, intensity, effective_config + ) profile_data = _profile_bytes(q, intensity) _write_new_or_verify(profile, profile_data) side_doc = { "schema": SCHEMA_VERSION, "frame_signature": frame_signature, "frame_signature_payload": frame_payload, - "profile": str(profile), + "profile": profile.relative_to(out).as_posix(), "profile_sha256": _sha256_bytes(profile_data), - "q_points": config.npt, + "q_points": effective_config.npt, "q_min_A^-1": float(q[0]), "q_max_A^-1": float(q[-1]), "integration_policy": { "unit": "q_A^-1", "method": "csr", - "mask": str(mask_path), - "correctSolidAngle": True, - "polarization_factor": None, + "mask": mask_path.relative_to(package).as_posix(), + "correctSolidAngle": effective_config.correct_solid_angle, + "polarization_factor": effective_config.polarization_factor, "dark": None, "flat": None, "normalization_factor": 1.0, diff --git a/tests/test_bl19b2_abs2d.py b/tests/test_bl19b2_abs2d.py index c9b1ab5..de2846e 100644 --- a/tests/test_bl19b2_abs2d.py +++ b/tests/test_bl19b2_abs2d.py @@ -413,6 +413,20 @@ def test_build_output_paths_preserves_relative_folder_and_uses_abs2d_suffix(tmp_ assert paths.preview.name == "frame_001_preview.png" +def test_validate_output_collisions_rejects_tif_and_tiff_same_stem(tmp_path: Path): + root = tmp_path / "dat001" + sample_dir = root / "3#_sample" + sample_dir.mkdir(parents=True) + sources = [sample_dir / "frame.tif", sample_dir / "frame.tiff"] + + with pytest.raises(ValueError, match="generated output collision"): + bl19b2.validate_output_collisions( + sources, + input_root=root, + output_root=tmp_path / "out", + ) + + def test_parse_pydidas_cali_yaml_converts_geometry_units(tmp_path: Path): mask = tmp_path / "Mask.edf" cali = tmp_path / "Cali.yaml" @@ -686,6 +700,20 @@ def test_frame_qc_row_from_metadata_restores_existing_resume_summary( assert "BG ABS" in row["warnings"] +def test_resume_package_root_handles_input_directory_named_metadata(tmp_path: Path): + input_root = tmp_path / "input" + source = input_root / "metadata" / "frame.tif" + output_root = tmp_path / "out" + paths = build_output_paths( + source, + input_root=input_root, + output_root=output_root, + ) + + assert paths.metadata == output_root / "metadata" / "metadata" / "frame_abs2d.json" + assert bl19b2._package_root_from_output_paths(paths) == output_root.resolve() + + def test_subtract_dark_scales_dark_to_sample_exposure(): image = np.array([[12.0, 22.0]]) dark = np.array([[1.0, 2.0]]) diff --git a/tests/test_bl19b2_integrate1d.py b/tests/test_bl19b2_integrate1d.py index 9f91639..ede55a4 100644 --- a/tests/test_bl19b2_integrate1d.py +++ b/tests/test_bl19b2_integrate1d.py @@ -3,6 +3,7 @@ import csv import hashlib import json +import shutil from pathlib import Path from types import SimpleNamespace @@ -16,7 +17,13 @@ def _sha(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() -def _build_package(tmp_path: Path) -> tuple[Path, Path]: +def _build_package( + tmp_path: Path, + *, + correct_solid_angle: bool = True, + polarization_factor: float | None = None, + signed_contract: bool = False, +) -> tuple[Path, Path]: package = tmp_path / "package" rel = Path("problem") / "sample_00001.tif" edf = package / "images_edf" / rel.parent / "sample_00001_abs2d_cm-1.edf" @@ -29,14 +36,21 @@ def _build_package(tmp_path: Path) -> tuple[Path, Path]: edf.write_bytes(b"stable synthetic EDF") np.save(mask, np.zeros((2, 2), dtype=np.uint8)) poni.write_text("synthetic poni\n", encoding="utf-8") + processing_payload = {"safe_poni_checksum_sha256": _sha(poni)} + if signed_contract: + processing_payload.update( + { + "correct_solid_angle_for_k": correct_solid_angle, + "polarization_factor": polarization_factor, + } + ) + processing_signature = integration._processing_signature_digest(processing_payload) metadata.write_text( json.dumps( { "schema": "saxsabs.bl19b2_abs2d.v4", - "processing_signature": "2d-signature", - "processing_signature_payload": { - "safe_poni_checksum_sha256": _sha(poni) - }, + "processing_signature": processing_signature, + "processing_signature_payload": processing_payload, "frame_signature": "2d-frame-signature", "intensity_unit": "cm^-1", "outputs": {"edf": str(edf), "edf_sha256": _sha(edf)}, @@ -63,8 +77,8 @@ def _build_package(tmp_path: Path) -> tuple[Path, Path]: "dark": None, "flat": None, "normalization_factor": 1.0, - "correctSolidAngle": True, - "polarization_factor": None, + "correctSolidAngle": correct_solid_angle, + "polarization_factor": polarization_factor, "do_not_repeat": [ "dark", "background", @@ -90,7 +104,7 @@ def _build_package(tmp_path: Path) -> tuple[Path, Path]: "status": "processed", "edf": edf, "metadata": metadata, - "processing_signature": "2d-signature", + "processing_signature": processing_signature, } ) return package, manifest @@ -142,6 +156,92 @@ def integrate1d(self, image, npt, **kwargs): assert len(calls) == 1 +def test_integration_honors_signed_2d_solid_angle_and_polarization_contract( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + package, _manifest = _build_package( + tmp_path, + correct_solid_angle=False, + polarization_factor=0.95, + signed_contract=True, + ) + calls: list[dict[str, object]] = [] + + class FakeIntegrator: + def integrate1d(self, image, npt, **kwargs): + calls.append(kwargs) + return SimpleNamespace( + radial=np.linspace(0.001, 1.0, npt), + intensity=np.linspace(2.0, 3.0, npt), + ) + + monkeypatch.setattr(integration, "_load_integrator", lambda _path: FakeIntegrator()) + monkeypatch.setattr( + integration, + "_load_validate_edf", + lambda _item, _metadata, _mask: np.zeros((2, 2), dtype=np.float32), + ) + monkeypatch.setattr(integration, "_version", lambda _name: "test-pyfai") + + result = integration.run_bl19b2_integrate1d(integration.Integrate1DConfig(package)) + + assert result["processed"] == 1 + assert calls[0]["correctSolidAngle"] is False + assert calls[0]["polarization_factor"] == pytest.approx(0.95) + signature = json.loads( + (package / "integration" / "config" / "run_signature.json").read_text( + encoding="utf-8" + ) + ) + assert signature["payload"]["correctSolidAngle"] is False + assert signature["payload"]["polarization_factor"] == pytest.approx(0.95) + readme = (package / "integration" / "README.md").read_text(encoding="utf-8") + assert "correctSolidAngle=False" in readme + assert "polarization_factor=0.95" in readme + + +def test_integration_relocates_package_with_relative_2d_manifest_paths( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + package, manifest = _build_package(tmp_path, signed_contract=True) + metadata_path = next((package / "metadata").rglob("*.json")) + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + metadata["outputs"]["edf"] = "images_edf/problem/sample_00001_abs2d_cm-1.edf" + metadata["mask"]["npy"] = "masks/bl19b2_mask.npy" + metadata["geometry"]["poni"] = "config/geometry/geometry.poni" + metadata_path.write_text(json.dumps(metadata), encoding="utf-8") + with manifest.open(encoding="utf-8", newline="") as stream: + rows = list(csv.DictReader(stream)) + rows[0]["edf"] = "images_edf/problem/sample_00001_abs2d_cm-1.edf" + rows[0]["metadata"] = "metadata/problem/sample_00001_abs2d.json" + with manifest.open("w", newline="", encoding="utf-8") as stream: + writer = csv.DictWriter(stream, fieldnames=list(rows[0])) + writer.writeheader() + writer.writerows(rows) + moved = tmp_path / "moved-package" + shutil.copytree(package, moved) + + class FakeIntegrator: + def integrate1d(self, image, npt, **kwargs): + return SimpleNamespace( + radial=np.linspace(0.001, 1.0, npt), + intensity=np.linspace(2.0, 3.0, npt), + ) + + monkeypatch.setattr(integration, "_load_integrator", lambda _path: FakeIntegrator()) + monkeypatch.setattr( + integration, + "_load_validate_edf", + lambda _item, _metadata, _mask: np.zeros((2, 2), dtype=np.float32), + ) + monkeypatch.setattr(integration, "_version", lambda _name: "test-pyfai") + + result = integration.run_bl19b2_integrate1d(integration.Integrate1DConfig(moved)) + + assert result["processed"] == 1 + assert (moved / "integration" / "profiles" / "problem").is_dir() + + def test_integration_optional_constant_fluorescence_changes_1d_and_signature( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ): @@ -462,12 +562,74 @@ def test_integration_rejects_tampered_reintegration_contract( integration.run_bl19b2_integrate1d(integration.Integrate1DConfig(package)) +def test_signed_null_polarization_cannot_override_recommended_value( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + package, _manifest = _build_package( + tmp_path, + correct_solid_angle=True, + polarization_factor=0.95, + signed_contract=True, + ) + metadata_path = next((package / "metadata").rglob("*.json")) + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + metadata["recommended_reintegration"]["polarization_factor"] = None + metadata_path.write_text(json.dumps(metadata), encoding="utf-8") + monkeypatch.setattr(integration, "_load_integrator", lambda _path: object()) + + with pytest.raises(ValueError, match="match signed package settings"): + integration.run_bl19b2_integrate1d(integration.Integrate1DConfig(package)) + + +@pytest.mark.parametrize( + "field", + ["recommended_reintegration", "processing_signature_payload"], +) +def test_reintegration_polarization_contract_rejects_wrong_types( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + field: str, +): + package, manifest = _build_package( + tmp_path, + correct_solid_angle=True, + polarization_factor=0.95, + signed_contract=True, + ) + metadata_path = next((package / "metadata").rglob("*.json")) + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + metadata[field]["polarization_factor"] = "0.95" + if field != "recommended_reintegration": + signature = integration._processing_signature_digest( + metadata["processing_signature_payload"] + ) + metadata["processing_signature"] = signature + with manifest.open("r", newline="", encoding="utf-8") as stream: + reader = csv.DictReader(stream) + rows = list(reader) + fieldnames = reader.fieldnames + assert fieldnames is not None + rows[0]["processing_signature"] = signature + with manifest.open("w", newline="", encoding="utf-8") as stream: + writer = csv.DictWriter( + stream, + fieldnames=fieldnames, + ) + writer.writeheader() + writer.writerows(rows) + metadata_path.write_text(json.dumps(metadata), encoding="utf-8") + monkeypatch.setattr(integration, "_load_integrator", lambda _path: object()) + + with pytest.raises(ValueError, match="numeric or null"): + integration.run_bl19b2_integrate1d(integration.Integrate1DConfig(package)) + + @pytest.mark.parametrize( ("field", "error_match"), [ ("mask_checksum", "mask array checksum mismatch"), ("poni_pointer", "PONI pointer mismatch"), - ("poni_checksum", "PONI checksum mismatch"), + ("poni_checksum", "payload digest mismatch"), ], ) def test_integration_rejects_tampered_2d_input_binding_metadata( diff --git a/tests/test_buffer_subtraction.py b/tests/test_buffer_subtraction.py index ed71953..418ad85 100644 --- a/tests/test_buffer_subtraction.py +++ b/tests/test_buffer_subtraction.py @@ -248,6 +248,23 @@ def test_subtract_buffer_preserves_legacy_positional_high_q_window(): assert result.high_q_residual_mean == pytest.approx(2.0) +def test_subtract_buffer_marks_undefined_high_q_diagnostic_as_none(): + q = np.array([0.10, 0.20, 0.30]) + result = _sub( + q, + np.full(3, 5.0), + np.full(3, 0.1), + q, + np.full(3, 2.0), + np.full(3, 0.1), + high_q_diag=(0.09, 0.21), + alpha_uncertainty=0.0, + ) + + assert result.high_q_residual_mean is None + assert result.high_q_check_passed is None + + @pytest.mark.parametrize("field", ["err_sample", "err_buffer"]) def test_subtract_buffer_rejects_infinite_uncertainty(field): q = np.array([0.01, 0.02, 0.03], dtype=float) @@ -265,6 +282,39 @@ def test_subtract_buffer_rejects_infinite_uncertainty(field): _sub(**kwargs) +@pytest.mark.parametrize("field", ["err_sample", "err_buffer"]) +def test_subtract_buffer_rejects_finite_uncertainty_overflow(field): + q = np.array([0.01, 0.02, 0.03], dtype=float) + kwargs = { + "q_sample": q, + "i_sample": np.ones(3), + "err_sample": np.full(3, 1.0e200), + "q_buffer": q, + "i_buffer": np.ones(3), + "err_buffer": np.full(3, 1.0e200), + "alpha_uncertainty": 0.0, + } + + kwargs[field] = np.full(3, 1.0e200) + with pytest.raises(ValueError, match="overflowed"): + _sub(**kwargs) + + +def test_subtract_buffer_rejects_extreme_finite_alpha_as_controlled_error(): + q = np.array([0.01, 0.02, 0.03], dtype=float) + with pytest.raises(ValueError, match="overflowed"): + _sub( + q, + np.ones(3), + np.zeros(3), + q, + np.ones(3), + np.zeros(3), + alpha=1.0e200, + alpha_uncertainty=0.0, + ) + + def test_subtract_buffer_refuses_unlabeled_profiles(): q = np.array([0.01, 0.02, 0.03]) with pytest.raises(ValueError, match="sample_profile and buffer_profile"): diff --git a/tests/test_calibration.py b/tests/test_calibration.py index a037cf7..c47eb45 100644 --- a/tests/test_calibration.py +++ b/tests/test_calibration.py @@ -25,6 +25,22 @@ def test_estimate_k_factor_robust_basic(): assert out.points_used >= 3 +def test_estimate_k_factor_rejects_interpolation_across_nonpositive_source_point(): + q_meas = np.array([0.01, 0.03, 0.05]) + i_meas = np.array([1.0, -0.1, 1.0]) + q_ref = np.array([0.01, 0.02, 0.03, 0.04, 0.05]) + i_ref = np.ones_like(q_ref) + + with pytest.raises(ValueError, match="signal too weak|valid ratio"): + estimate_k_factor_robust( + q_meas, + i_meas, + q_ref=q_ref, + i_ref=i_ref, + q_window=(0.01, 0.05), + ) + + def test_default_nist_calibration_reports_certificate_aware_k_uncertainty(): true_k = 2.5 q = NIST_SRM3600_DATA[:, 0] diff --git a/tests/test_calibration_context.py b/tests/test_calibration_context.py index ebca2aa..3eb02f0 100644 --- a/tests/test_calibration_context.py +++ b/tests/test_calibration_context.py @@ -14,8 +14,8 @@ def _context(**overrides): values = { "formula_version": "v3_nist_blank", "monitor_mode": "rate", - "poni_sha256": "poni-sha", - "mask_sha256": "mask-sha", + "poni_sha256": "a" * 64, + "mask_sha256": "b" * 64, "flat_sha256": None, "correct_solid_angle": True, "polarization_factor": None, @@ -36,9 +36,9 @@ def test_calibration_context_fingerprint_includes_standard_provenance(): ("field", "value"), [ ("monitor_mode", "integrated"), - ("poni_sha256", "other-poni"), - ("mask_sha256", "other-mask"), - ("flat_sha256", "flat-sha"), + ("poni_sha256", "c" * 64), + ("mask_sha256", "d" * 64), + ("flat_sha256", "e" * 64), ("correct_solid_angle", False), ("polarization_factor", 0.95), ], @@ -195,6 +195,103 @@ def test_calibration_context_from_dict_rejects_alias_with_wrong_srm3600_thicknes CalibrationContext.from_dict(payload) +def test_calibration_context_canonicalizes_trimmed_uppercase_sha256_fields(): + context = _context( + poni_sha256=" " + "A" * 64 + " ", + mask_sha256=" " + "B" * 64 + " ", + standard_data_sha256=" " + "C" * 64 + " ", + background_data_sha256=(" " + "D" * 64 + " ",), + ) + + assert context.poni_sha256 == "a" * 64 + assert context.mask_sha256 == "b" * 64 + assert context.standard_data_sha256 == "c" * 64 + assert context.background_data_sha256 == ("d" * 64,) + assert CalibrationContext.from_dict(context.to_dict()) == context + + +@pytest.mark.parametrize("field", ["poni_sha256", "mask_sha256", "flat_sha256"]) +def test_calibration_context_rejects_malformed_sha256_fields(field): + with pytest.raises(ValueError, match=field): + _context(**{field: "not-a-digest"}) + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("background_data_sha256", (None,)), + ("background_data_sha256", ("",)), + ("background_data_sha256", ("not-a-digest",)), + ("dark_data_sha256", (None,)), + ("dark_data_sha256", ("",)), + ("dark_data_sha256", ("not-a-digest",)), + ], +) +def test_calibration_context_rejects_invalid_hash_sequence_entries(field, value): + with pytest.raises(ValueError, match=field): + _context(**{field: value}) + + +@pytest.mark.parametrize("field", ["background_data_sha256", "dark_data_sha256"]) +def test_calibration_context_rejects_none_hash_sequence(field): + with pytest.raises(ValueError, match=field): + _context(**{field: None}) + + +@pytest.mark.parametrize("field", ["background_data_sha256", "dark_data_sha256"]) +def test_calibration_context_rejects_scalar_hash_sequence(field): + with pytest.raises(ValueError, match=field): + _context(**{field: ""}) + + +@pytest.mark.parametrize( + "value", + [ + {"b" * 64: "tampered"}, + {"b" * 64}, + frozenset({"b" * 64}), + ], +) +def test_calibration_context_rejects_unordered_hash_sequences(value): + with pytest.raises(ValueError, match="ordered list or tuple"): + _context(background_data_sha256=value) + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("background_data_sha256", {"b" * 64: "tampered"}), + ("dark_data_sha256", {"c" * 64}), + ("background_monitors", {900.0: "tampered"}), + ("background_transmissions", {0.91}), + ("background_exposure_s", {10.0: "tampered"}), + ("dark_exposure_s", frozenset({10.0})), + ("q_window", {"qmin": 0.01, "qmax": 0.2}), + ], +) +def test_calibration_context_rejects_unordered_provenance_sequences(field, value): + with pytest.raises(ValueError, match="ordered list or tuple"): + _context(**{field: value}) + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("background_data_sha256", {"b" * 64: "tampered"}), + ("background_monitors", {900.0: "tampered"}), + ("q_window", {"qmin": 0.01, "qmax": 0.2}), + ], +) +def test_calibration_context_from_dict_rejects_unordered_provenance_sequences( + field, value +): + payload = _context().to_dict() + payload[field] = value + + with pytest.raises(ValueError, match="ordered list or tuple"): + CalibrationContext.from_dict(payload) + + def test_calibration_context_aliases_have_one_canonical_fingerprint(): direct = _context(standard_key="SRM3600") alias = _context(standard_key="nist srm 3600") diff --git a/tests/test_cli.py b/tests/test_cli.py index f598d0c..655c0cf 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -20,6 +20,90 @@ def test_cli_norm_factor(capsys: pytest.CaptureFixture[str], monkeypatch: pytest assert out == "5.0" +def test_cli_parse_header_accepts_bom_json_object(tmp_path: Path, capsys, monkeypatch): + path = tmp_path / "header.json" + path.write_text('{"exposure": "2 s", "monitor": 10, "transmission": 0.5}', encoding="utf-8-sig") + monkeypatch.setattr( + sys, + "argv", + ["saxsabs", "parse-header", "--header-json", str(path)], + ) + + main() + + assert json.loads(capsys.readouterr().out)["i0"] == 10.0 + + +def test_cli_parse_header_rejects_non_mapping_without_traceback( + tmp_path: Path, capsys, monkeypatch +): + path = tmp_path / "header.json" + path.write_text("[1, 2, 3]", encoding="utf-8") + monkeypatch.setattr( + sys, + "argv", + ["saxsabs", "parse-header", "--header-json", str(path)], + ) + + with pytest.raises(SystemExit): + main() + error = capsys.readouterr().err + assert "parse-header failed" in error + assert "Traceback" not in error + + +def test_cli_parse_external1d_reports_missing_file_without_traceback( + tmp_path: Path, capsys, monkeypatch +): + missing = tmp_path / "missing.dat" + monkeypatch.setattr( + sys, + "argv", + ["saxsabs", "parse-external1d", "--input", str(missing)], + ) + + with pytest.raises(SystemExit): + main() + error = capsys.readouterr().err + assert "parse-external1d failed" in error + assert "Traceback" not in error + + +@pytest.mark.parametrize( + "command", + [ + ["estimate-k", "--meas"], + ["subtract-buffer", "--sample"], + ["subtract-fluorescence", "--sample"], + ], +) +def test_cli_expected_missing_input_errors_are_concise( + tmp_path: Path, capsys, monkeypatch, command +): + missing = tmp_path / "missing.dat" + if command[0] == "estimate-k": + argv = ["saxsabs", *command, str(missing)] + elif command[0] == "subtract-buffer": + argv = ["saxsabs", *command, str(missing), "--buffer", str(missing)] + else: + argv = [ + "saxsabs", + *command, + str(missing), + "--method", + "constant", + "--f0", + "0", + ] + monkeypatch.setattr(sys, "argv", argv) + + with pytest.raises(SystemExit): + main() + error = capsys.readouterr().err + assert f"{command[0]} failed" in error + assert "Traceback" not in error + + def test_cli_norm_factor_rejects_non_finite_result( capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch, diff --git a/tests/test_detector_reduction.py b/tests/test_detector_reduction.py index 10cb0a0..e3caf5c 100644 --- a/tests/test_detector_reduction.py +++ b/tests/test_detector_reduction.py @@ -27,6 +27,36 @@ def test_normalize_detector_frame_scales_integrated_dark_by_exposure(): np.testing.assert_allclose(result.image, [[10.0]]) +def test_normalize_detector_frame_rejects_nonfinite_dark_scale(): + with pytest.raises(ValueError, match="dark scale"): + normalize_detector_frame( + np.array([[1.0]]), + np.array([[0.0]]), + image_exposure_s=1.0e308, + dark_exposure_s=1.0e-308, + monitor=1.0, + transmission=1.0, + monitor_mode="integrated", + ) + + +def test_build_nist_net_image_rejects_nonfinite_final_image(): + with pytest.raises(ValueError, match="net detector image"): + build_nist_net_image( + np.array([[0.0]]), + np.array([[1.0e308]]), + np.array([[0.0]]), + sample_exposure_s=1.0, + background_exposure_s=1.0, + dark_exposure_s=1.0, + sample_monitor=1.0, + background_monitor=1.0, + sample_transmission=1.0, + monitor_mode="integrated", + alpha=1.0e308, + ) + + def test_build_nist_net_image_does_not_divide_blank_by_blank_transmission(): sample = np.array([[70.0]]) blank = np.array([[30.0]]) diff --git a/tests/test_fluorescence_subtraction.py b/tests/test_fluorescence_subtraction.py index cd2044d..ed111c1 100644 --- a/tests/test_fluorescence_subtraction.py +++ b/tests/test_fluorescence_subtraction.py @@ -56,6 +56,22 @@ def test_constant_subtraction_and_error(): assert out.f0 == pytest.approx(2.0) +def test_fluorescence_residual_diagnostic_is_tri_state_for_too_few_points(): + out = _sub( + np.array([0.01, 0.02, 0.20]), + np.array([12.0, 11.0, 6.0]), + np.full(3, 0.1), + method="constant", + f0=2.0, + f0_uncertainty=0.0, + beta=1.0, + beta_uncertainty=0.0, + ) + + assert out.high_q_residual_mean is None + assert out.high_q_check_passed is None + + def test_beta_scales_constant_and_propagates_beta_uncertainty(): q = np.array([0.01, 0.02, 0.03]) out = _sub( @@ -415,3 +431,83 @@ def test_combine_sequential_adds_independent_extras(): ) np.testing.assert_allclose(stat, next_stat) np.testing.assert_allclose(comb, np.sqrt(0.1**2 + 0.2**2 + 0.3**2 + 0.4**2)) + + +def test_combine_sequential_rejects_broadcastable_but_unequal_shapes(): + with pytest.raises(ValueError, match="exactly equal shapes"): + combine_sequential_standard_uncertainties( + np.ones(2), + np.ones(2), + np.ones(1), + np.ones(1), + ) + + +def test_combine_sequential_validates_previous_statistical_shape_without_combined(): + with pytest.raises(ValueError, match="exactly equal shapes"): + combine_sequential_standard_uncertainties( + np.ones(2), + None, + np.ones(1), + np.ones(1), + ) + + +def test_combine_sequential_rejects_finite_overflow_without_warning(): + with pytest.raises(ValueError, match="overflowed"): + combine_sequential_standard_uncertainties( + np.full(2, 1.0e200), + np.full(2, 2.0e200), + np.full(2, 1.0e200), + np.full(2, 2.0e200), + ) + + +def test_combine_sequential_rejects_infinite_next_combined_before_none_early_return(): + with pytest.raises(ValueError, match="must not contain infinities"): + combine_sequential_standard_uncertainties( + np.array([1.0]), + None, + np.array([1.0]), + np.array([np.inf]), + ) + + +@pytest.mark.parametrize("argument", ["previous_statistical", "next_statistical"]) +def test_combine_sequential_rejects_infinite_statistical_inputs(argument): + kwargs = { + "previous_statistical": np.array([1.0]), + "previous_combined": None, + "next_statistical": np.array([1.0]), + "next_combined": np.array([1.0]), + } + kwargs[argument] = np.array([np.inf]) + + with pytest.raises(ValueError, match="must not contain infinities"): + combine_sequential_standard_uncertainties(**kwargs) + + +def test_combine_sequential_preserves_nan_unknown_with_none_previous_combined(): + stat, combined = combine_sequential_standard_uncertainties( + np.array([1.0]), + None, + np.array([1.0]), + np.array([np.nan]), + ) + + np.testing.assert_allclose(stat, [1.0]) + assert np.isnan(combined[0]) + + +def test_fluorescence_rejects_extreme_finite_beta_as_controlled_error(): + with pytest.raises(ValueError, match="overflowed"): + _sub( + np.array([0.01, 0.02, 0.03]), + np.full(3, 10.0), + np.zeros(3), + method="constant", + f0=1.0, + f0_uncertainty=0.0, + beta=1.0e200, + beta_uncertainty=0.0, + ) diff --git a/tests/test_intensity_state.py b/tests/test_intensity_state.py index e0b4d24..dd758c5 100644 --- a/tests/test_intensity_state.py +++ b/tests/test_intensity_state.py @@ -30,6 +30,23 @@ def test_absolute_column_is_rejected_before_k_or_thickness_is_reapplied(): require_relative_input_for_absolute_scaling(profile, profile_name="sample.dat") +@pytest.mark.parametrize("column", ["I/cm", "I (1/cm)", "I (cm^-1)", "I_abs (cm^-1)"]) +def test_explicit_absolute_intensity_headers_are_semantic_and_machine_readable(column): + assessment = assess_intensity_state({"i_col": column}) + + assert assessment.state is IntensityState.ABSOLUTE_CM_INV + + +@pytest.mark.parametrize( + "column", + ["imagecm1", "indexcm1", "I_absorbance", "iabsorption", "iabsent"], +) +def test_unrelated_i_prefixed_headers_remain_ambiguous(column): + assessment = assess_intensity_state({"i_col": column}) + + assert assessment.state is IntensityState.AMBIGUOUS + + def test_unitless_absolute_metadata_is_ambiguous(): assessment = assess_intensity_state( {"i_col": "I", "operator_provenance": {"intensity_state": "absolute"}} diff --git a/tests/test_io_formats.py b/tests/test_io_formats.py index c10fc84..78952d5 100644 --- a/tests/test_io_formats.py +++ b/tests/test_io_formats.py @@ -172,15 +172,101 @@ def test_write_rejects_nonfinite_intensity(self, tmp_path, bad_value): metadata=ABS_META, ) + @pytest.mark.parametrize( + "q,i,err", + [ + (np.array([[0.1, 0.2]]), np.array([[10.0, 9.0]]), None), + (np.array([]), np.array([]), None), + (np.array([0.1, 0.2]), np.array([10.0, 9.0]), np.array([1.0, -1.0])), + (np.array([0.1, 0.2]), np.array([10.0, 9.0]), np.array([1.0, np.inf])), + ], + ) + def test_write_rejects_invalid_1d_or_uncertainty_inputs(self, tmp_path, q, i, err): + with pytest.raises(ValueError): + write_cansas1d_xml(tmp_path / "invalid-input.xml", q, i, err, metadata=ABS_META) + + def test_reader_tracks_profile_without_i_dev(self, tmp_path): + q, i_abs, _ = self._make_data(4) + result = read_cansas1d_xml( + write_cansas1d_xml(tmp_path / "no-idev.xml", q, i_abs, metadata=ABS_META) + ) + + assert result["err_col"] == "" + assert np.all(np.isnan(result["uncertainty"])) + + def test_reader_accepts_equivalent_i_units_but_rejects_missing_or_conflicting_units( + self, tmp_path + ): + q, i_abs, err = self._make_data(4) + xml_path = tmp_path / "units.xml" + write_cansas1d_xml(xml_path, q, i_abs, err, metadata=ABS_META) + tree = ET.parse(xml_path) + namespace = "{urn:cansas1d:1.1}" + i_elements = list(tree.getroot().iter(f"{namespace}I")) + i_elements[1].set("unit", "cm^-1") + tree.write(xml_path, encoding="utf-8", xml_declaration=True) + assert read_cansas1d_xml(xml_path)["intensity_unit"] == "1/cm" + + tree = ET.parse(xml_path) + list(tree.getroot().iter(f"{namespace}I"))[1].attrib.pop("unit") + tree.write(xml_path, encoding="utf-8", xml_declaration=True) + with pytest.raises(ValueError, match="mixed/missing I units"): + read_cansas1d_xml(xml_path) + + write_cansas1d_xml(xml_path, q, i_abs, err, metadata=ABS_META) + tree = ET.parse(xml_path) + next(tree.getroot().iter(f"{namespace}Idev")).set("unit", "1/nm") + tree.write(xml_path, encoding="utf-8", xml_declaration=True) + with pytest.raises(ValueError, match="Idev units"): + read_cansas1d_xml(xml_path) + + write_cansas1d_xml(xml_path, q, i_abs, err, metadata=ABS_META) + tree = ET.parse(xml_path) + for element in tree.getroot().iter(f"{namespace}Idev"): + element.attrib.pop("unit") + tree.write(xml_path, encoding="utf-8", xml_declaration=True) + with pytest.raises(ValueError, match="Idev units are missing"): + read_cansas1d_xml(xml_path) + + @pytest.mark.parametrize("bad_value", ["nan", "inf", "-inf"]) + def test_reader_rejects_nonfinite_xml_q_or_i(self, tmp_path, bad_value): + q, i_abs, _ = self._make_data(3) + xml_path = tmp_path / "bad-xml.xml" + write_cansas1d_xml(xml_path, q, i_abs, metadata=ABS_META) + tree = ET.parse(xml_path) + first_q = next(tree.getroot().iter("{urn:cansas1d:1.1}Q")) + first_q.text = bad_value + tree.write(xml_path, encoding="utf-8", xml_declaration=True) + with pytest.raises(ValueError, match="finite"): + read_cansas1d_xml(xml_path) + + @pytest.mark.parametrize("bad_value", ["inf", "-inf", "-0.1"]) + def test_reader_rejects_invalid_xml_i_dev(self, tmp_path, bad_value): + q, i_abs, err = self._make_data(3) + xml_path = tmp_path / "bad-idev.xml" + write_cansas1d_xml(xml_path, q, i_abs, err, metadata=ABS_META) + tree = ET.parse(xml_path) + next(tree.getroot().iter("{urn:cansas1d:1.1}Idev")).text = bad_value + tree.write(xml_path, encoding="utf-8", xml_declaration=True) + with pytest.raises(ValueError, match="Idev"): + read_cansas1d_xml(xml_path) + # --------------------------------------------------------------------------- -# NXcanSAS HDF5 round-trip (skip if h5py unavailable) +# NXcanSAS HDF5 round-trip (skip only these tests if h5py is unavailable) # --------------------------------------------------------------------------- -h5py = pytest.importorskip("h5py") +try: + import h5py +except ImportError: # pragma: no cover - exercised in minimal installations + h5py = None from saxsabs.io.parsers import read_nxcansas_h5 # noqa: E402 class TestNXcanSASHDF5: + @pytest.fixture(autouse=True) + def _require_h5py(self): + pytest.importorskip("h5py") + def _make_data(self, n=50): q = np.linspace(0.01, 0.30, n) i_abs = 100.0 / q @@ -306,3 +392,67 @@ def test_reader_rejects_malformed_dataset_lengths(self, tmp_path): assert "length mismatch" in str(exc) else: raise AssertionError("Expected ValueError for malformed NXcanSAS file") + + @pytest.mark.parametrize( + "q,i,err", + [ + (np.array([[0.1, 0.2]]), np.array([[10.0, 9.0]]), None), + (np.array([]), np.array([]), None), + (np.array([0.1, 0.2]), np.array([10.0, 9.0]), np.array([1.0, -1.0])), + (np.array([0.1, 0.2]), np.array([10.0, 9.0]), np.array([1.0, np.inf])), + ], + ) + def test_write_rejects_invalid_1d_or_uncertainty_inputs(self, tmp_path, q, i, err): + with pytest.raises(ValueError): + write_nxcansas_h5(tmp_path / "invalid-input.h5", q, i, err, metadata=ABS_META) + + def test_writer_is_atomic_when_metadata_assignment_fails(self, tmp_path): + q, i_abs, err = self._make_data(4) + target = tmp_path / "atomic.h5" + write_nxcansas_h5(target, q, i_abs, err, metadata=ABS_META) + original = target.read_bytes() + + with pytest.raises((TypeError, ValueError)): + write_nxcansas_h5( + target, + q, + i_abs, + err, + metadata=_meta(title=None), + ) + + assert target.read_bytes() == original + assert not list(tmp_path.glob(f".{target.name}.*.tmp")) + + @pytest.mark.parametrize("bad_value", [np.nan, np.inf, -np.inf]) + def test_reader_rejects_nonfinite_hdf_q_or_i(self, tmp_path, bad_value): + path = tmp_path / "bad-values.h5" + with h5py.File(path, "w") as f: + data = f.create_group("sasdata01") + data.attrs["canSAS_class"] = "SASdata" + data.create_dataset("Q", data=np.array([0.1, bad_value, 0.3])) + data.create_dataset("I", data=np.array([10.0, 9.0, 8.0])) + with pytest.raises(ValueError, match="finite"): + read_nxcansas_h5(path) + + def test_reader_rejects_non_1d_hdf_datasets(self, tmp_path): + path = tmp_path / "two-dimensional.h5" + with h5py.File(path, "w") as f: + data = f.create_group("sasdata01") + data.attrs["canSAS_class"] = "SASdata" + data.create_dataset("Q", data=np.ones((2, 2))) + data.create_dataset("I", data=np.ones((2, 2))) + with pytest.raises(ValueError, match="1-D"): + read_nxcansas_h5(path) + + @pytest.mark.parametrize("bad_value", [np.inf, -1.0]) + def test_reader_rejects_invalid_hdf_i_dev(self, tmp_path, bad_value): + path = tmp_path / "bad-idev.h5" + with h5py.File(path, "w") as f: + data = f.create_group("sasdata01") + data.attrs["canSAS_class"] = "SASdata" + data.create_dataset("Q", data=np.array([0.1, 0.2, 0.3])) + data.create_dataset("I", data=np.array([10.0, 9.0, 8.0])) + data.create_dataset("Idev", data=np.array([0.1, bad_value, 0.1])) + with pytest.raises(ValueError, match="Idev"): + read_nxcansas_h5(path) diff --git a/tests/test_normalization.py b/tests/test_normalization.py index 034b023..b8cd88c 100644 --- a/tests/test_normalization.py +++ b/tests/test_normalization.py @@ -31,6 +31,28 @@ def test_compute_norm_factor_invalid_inputs_return_nan(): assert math.isnan(out4) +def test_compute_norm_factor_overflow_returns_nan(): + out = compute_norm_factor( + exp=1.0e308, + mon=1.0e308, + trans=1.0, + mode="rate", + ) + + assert math.isnan(out) + + +@pytest.mark.parametrize( + ("exp", "mon", "trans", "mode"), + [ + (1.0e-200, 1.0e-200, 1.0e-200, "rate"), + (None, 1.0e-200, 1.0e-200, "integrated"), + ], +) +def test_compute_norm_factor_underflow_returns_nan(exp, mon, trans, mode): + assert math.isnan(compute_norm_factor(exp, mon, trans, mode)) + + def test_compute_norm_factor_unknown_mode_raises_before_missing_inputs(): with pytest.raises(ValueError, match="Unknown I0 normalization mode"): compute_norm_factor(exp=None, mon=None, trans=None, mode="unsupported") diff --git a/tests/test_parsers.py b/tests/test_parsers.py index 10282b8..e4462cd 100644 --- a/tests/test_parsers.py +++ b/tests/test_parsers.py @@ -9,7 +9,9 @@ infer_q_unit_from_column, normalize_transmission, parse_header_values, + q_axis_kind, read_external_1d_profile, + _try_parse_datetime, ) @@ -168,6 +170,171 @@ def test_read_external_1d_profile_csv(tmp_path: Path): assert "i_rel" not in out +def test_read_external_1d_profile_supports_unambiguous_semicolon_decimal_comma( + tmp_path: Path, +): + profile = tmp_path / "decimal-comma.dat" + profile.write_text( + "Q;I\n" + "0,10;100,0\n" + "0,20;90,0\n" + "0,30;80,0\n", + encoding="utf-8", + ) + + result = read_external_1d_profile(profile) + + np.testing.assert_allclose(result["x"], [0.10, 0.20, 0.30]) + np.testing.assert_allclose(result["intensity"], [100.0, 90.0, 80.0]) + + +def test_decimal_comma_zero_leading_three_digit_fraction_is_not_thousands_grouping( + tmp_path: Path, +): + profile = tmp_path / "decimal-comma-zero-leading.dat" + profile.write_text( + "Q;I\n" + "0,100;-0,100\n" + "0,200;0,200\n" + "0,300;0,300\n", + encoding="utf-8", + ) + + result = read_external_1d_profile(profile) + + np.testing.assert_allclose(result["x"], [0.1, 0.2, 0.3]) + np.testing.assert_allclose(result["intensity"], [-0.1, 0.2, 0.3]) + + +def test_read_external_1d_profile_supports_comment_semicolon_decimal_comma_header( + tmp_path: Path, +): + profile = tmp_path / "comment-decimal-comma.dat" + profile.write_text( + "# Q;I\n" + "0,10;100,0\n" + "0,20;90,0\n" + "0,30;80,0\n", + encoding="utf-8", + ) + + result = read_external_1d_profile(profile) + + assert result["x_col"] == "Q" + assert result["i_col"] == "I" + + +@pytest.mark.parametrize("column", ["I/cm", "I (1/cm)", "I (cm^-1)", "I_abs (cm^-1)"]) +def test_explicit_absolute_intensity_headers_expose_i_abs(tmp_path: Path, column: str): + profile = tmp_path / "absolute-header.csv" + profile.write_text( + f"Q,{column}\n0.1,100\n0.2,90\n0.3,80\n", + encoding="utf-8", + ) + + result = read_external_1d_profile(profile) + + assert result["intensity_state"] == "absolute_cm^-1" + np.testing.assert_allclose(result["i_abs"], [100.0, 90.0, 80.0]) + + +@pytest.mark.parametrize("column", ["I_ref", "I_meas"]) +def test_external_profile_accepts_exact_reference_or_measured_intensity_header( + tmp_path: Path, column: str +): + profile = tmp_path / "semantic-intensity-header.csv" + profile.write_text( + f"q_ref,{column}\n0.1,100\n0.2,90\n0.3,80\n", + encoding="utf-8", + ) + + result = read_external_1d_profile(profile) + + assert result["i_col"] == column + np.testing.assert_allclose(result["intensity"], [100.0, 90.0, 80.0]) + + +@pytest.mark.parametrize( + "column", + ["imagecm1", "indexcm1", "I_absorbance", "iabsorption", "iabsent"], +) +def test_external_profile_rejects_unrelated_i_prefixed_header(tmp_path: Path, column: str): + profile = tmp_path / "false-intensity-header.csv" + profile.write_text( + f"Q,{column}\n0.1,100\n0.2,90\n0.3,80\n", + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="numeric columns"): + read_external_1d_profile(profile) + + +def test_read_external_1d_profile_rejects_ambiguous_semicolon_comma_values( + tmp_path: Path, +): + profile = tmp_path / "ambiguous-comma.dat" + profile.write_text( + "Q;I\n" + "1,234;5,678\n" + "2,345;6,789\n" + "3,456;7,890\n", + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="ambiguous decimal-comma"): + read_external_1d_profile(profile) + + +def test_read_external_1d_profile_keeps_nan_uncertainty_in_decimal_comma_table( + tmp_path: Path, +): + profile = tmp_path / "decimal-comma-error.dat" + profile.write_text( + "Q;I;Error\n" + "0,10;100,0;1,0\n" + "0,20;90,0;NaN\n" + "0,30;80,0;1,0\n", + encoding="utf-8", + ) + + result = read_external_1d_profile(profile) + + assert np.isnan(result["uncertainty"][1]) + + +@pytest.mark.parametrize("name", ["quality", "query", "qwerty"]) +def test_q_axis_kind_does_not_classify_q_prefixed_words_as_q(name): + assert q_axis_kind(name) == "unknown" + + +@pytest.mark.parametrize("name", ["machine", "mychi", "not2theta"]) +def test_q_axis_kind_requires_chi_and_two_theta_token_boundaries(name): + assert q_axis_kind(name) == "unknown" + + +@pytest.mark.parametrize("name", ["q", "q_ref", "Q_A^-1", "Q(nm^-1)", "q1"]) +def test_q_axis_kind_accepts_unambiguous_q_forms(name): + assert q_axis_kind(name) == "q" + + +def test_q_axis_kind_accepts_compact_angstrom_inverse_header(): + assert q_axis_kind("qA^-1") == "q" + assert infer_q_unit_from_column("qA^-1") == "A^-1" + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + (1_700_000_000, 1_700_000_000.0), + (1_700_000_000_000, 1_700_000_000.0), + (1_700_000_000_000_000, 1_700_000_000.0), + ("1700000000000000000", 1_700_000_000.0), + ], +) +def test_try_parse_datetime_distinguishes_epoch_units(value, expected): + assert _try_parse_datetime(value) == pytest.approx(expected) + + def test_read_external_1d_profile_space_delimited(tmp_path: Path): f = tmp_path / "profile.dat" f.write_text( diff --git a/tests/test_uncertainty.py b/tests/test_uncertainty.py index 6a75a31..c501541 100644 --- a/tests/test_uncertainty.py +++ b/tests/test_uncertainty.py @@ -122,6 +122,50 @@ def test_system_coverage_factor_is_reported_separately_from_combined_status(): np.testing.assert_allclose(budget.expanded_uncertainty, [2.0]) +def test_uncertainty_combination_overflow_raises_instead_of_partial_status(): + kwargs = { + "intensity": np.array([1.0e308]), + "statistical_standard_uncertainty": 1.0e308, + "k_relative_standard_uncertainty": 0.0, + "standard_relative_standard_uncertainty": 0.0, + "transmission_relative_standard_uncertainty": 0.0, + "monitor_relative_standard_uncertainty": 0.0, + "thickness_relative_standard_uncertainty": 0.0, + "mu_relative_standard_uncertainty": 0.0, + "alpha_standard_uncertainty": 0.0, + } + + with pytest.raises(ValueError, match="combined standard uncertainty"): + propagate_absolute_uncertainty(**kwargs) + + +def test_uncertainty_expanded_overflow_raises(): + kwargs = { + "intensity": np.array([1.0]), + "statistical_standard_uncertainty": 1.0e154, + "k_relative_standard_uncertainty": 0.0, + "standard_relative_standard_uncertainty": 0.0, + "transmission_relative_standard_uncertainty": 0.0, + "monitor_relative_standard_uncertainty": 0.0, + "thickness_relative_standard_uncertainty": 0.0, + "mu_relative_standard_uncertainty": 0.0, + "alpha_standard_uncertainty": 0.0, + "coverage_factor": 1.0e308, + } + + with pytest.raises(ValueError, match="expanded uncertainty"): + propagate_absolute_uncertainty(**kwargs) + + +def test_uncertainty_overflow_is_not_masked_by_unknown_components(): + with pytest.raises(ValueError, match="expanded uncertainty"): + propagate_absolute_uncertainty( + intensity=np.array([1.0]), + statistical_standard_uncertainty=1.0e154, + coverage_factor=1.0e308, + ) + + @pytest.mark.parametrize( ("keyword", "value"), [ diff --git a/tests/test_version_metadata.py b/tests/test_version_metadata.py index 0893a4d..298cdc7 100644 --- a/tests/test_version_metadata.py +++ b/tests/test_version_metadata.py @@ -1,9 +1,11 @@ from __future__ import annotations +import importlib.util import json from datetime import date from pathlib import Path import re +import sys from saxsabs import __version__ @@ -11,12 +13,11 @@ ROOT = Path(__file__).resolve().parents[1] -def test_release_version_metadata_is_consistent(): +def test_release_version_metadata_is_consistent(monkeypatch): pyproject = (ROOT / "pyproject.toml").read_text(encoding="utf-8") citation = (ROOT / "CITATION.cff").read_text(encoding="utf-8") codemeta = json.loads((ROOT / "codemeta.json").read_text(encoding="utf-8")) zenodo = json.loads((ROOT / ".zenodo.json").read_text(encoding="utf-8")) - workbench = (ROOT / "SASAbs.py").read_text(encoding="utf-8") changelog = (ROOT / "CHANGELOG.md").read_text(encoding="utf-8") paper = (ROOT / "paper" / "paper.md").read_text(encoding="utf-8") @@ -28,7 +29,40 @@ def test_release_version_metadata_is_consistent(): assert re.search(rf'^version: "{re.escape(__version__)}"$', citation, re.MULTILINE) assert codemeta["version"] == __version__ assert zenodo["version"] == __version__ - assert workbench.count(f'"{__version__}"') >= 2 + spec = importlib.util.spec_from_file_location( + "saxsabs_workbench_version_metadata_test", + ROOT / "SASAbs.py", + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + assert module.APP_VERSION == __version__ + + class MissingSourceVersionPath: + def __init__(self, *_parts): + pass + + def resolve(self): + return self + + @property + def parent(self): + return self + + def __truediv__(self, _part): + return self + + def read_text(self, **_kwargs): + raise OSError("source tree unavailable") + + monkeypatch.setattr(module, "Path", MissingSourceVersionPath) + monkeypatch.setattr( + module.importlib_metadata, + "version", + lambda distribution: __version__ if distribution == "saxsabs" else "", + ) + assert module._read_package_version() == __version__ changelog_heading = re.search( rf"(?m)^## \[{re.escape(__version__)}\] - (?:Unreleased|\d{{4}}-\d{{2}}-\d{{2}})$", changelog, diff --git a/tests/test_workbench_output_paths.py b/tests/test_workbench_output_paths.py index e80f51e..2722d8f 100644 --- a/tests/test_workbench_output_paths.py +++ b/tests/test_workbench_output_paths.py @@ -16,7 +16,7 @@ def _labeled_export_context(): return CalibrationContext( formula_version="v3_nist_blank", monitor_mode="rate", - poni_sha256="poni-sha", + poni_sha256="0" * 64, mask_sha256=None, flat_sha256=None, correct_solid_angle=True, diff --git a/tests/test_workbench_scientific.py b/tests/test_workbench_scientific.py index a390855..0d14c1f 100644 --- a/tests/test_workbench_scientific.py +++ b/tests/test_workbench_scientific.py @@ -134,6 +134,63 @@ def test_tab3_two_theta_requires_wavelength_and_converts_to_q(): assert conversion == "two_theta_deg_to_q_a^-1" +def test_workbench_custom_reference_converts_nm_q_and_rejects_two_theta( + monkeypatch: pytest.MonkeyPatch, +): + module = _load_workbench_module() + app = module.SAXSAbsWorkbenchApp.__new__(module.SAXSAbsWorkbenchApp) + app.t1_std_type = _Var("Custom") + app.t1_std_ref_path = _Var("reference.dat") + profiles = { + "q": { + "x": np.array([0.1, 0.2, 0.3]), + "x_col": "q", + "x_unit": "nm^-1", + "x_unit_raw": "nm^-1", + "intensity": np.array([1.0, 2.0, 3.0]), + }, + "two_theta": { + "x": np.array([1.0, 2.0, 3.0]), + "x_col": "2theta", + "intensity": np.array([1.0, 2.0, 3.0]), + }, + } + captured = {} + monkeypatch.setattr( + "saxsabs.io.parsers.read_external_1d_profile", + lambda _path: profiles["q"], + ) + monkeypatch.setattr( + module, + "get_reference_data", + lambda _key, q_user, i_user: (captured.setdefault("q", q_user), i_user), + ) + + app._get_std_reference_data() + np.testing.assert_allclose(captured["q"], np.array([0.01, 0.02, 0.03])) + + monkeypatch.setattr( + "saxsabs.io.parsers.read_external_1d_profile", + lambda _path: profiles["two_theta"], + ) + with pytest.raises(ValueError, match="2theta|标准参考曲线"): + app._get_std_reference_data() + + +def test_theme_control_reports_unavailable_without_sv_ttk(monkeypatch: pytest.MonkeyPatch): + module = _load_workbench_module() + if not hasattr(module, "_sv_ttk"): + pytest.skip("shared saxs_ui_kit theme backend is active") + module._sv_ttk = None + root = SimpleNamespace() + status = [] + app = SimpleNamespace(_report_theme_unavailable=lambda: status.append("unavailable")) + root._app_ref = app + + assert module.toggle_theme(root) is False + assert status == ["unavailable"] + + @pytest.mark.parametrize( ("x_col", "mode", "wavelength"), [ @@ -761,6 +818,75 @@ def test_tab2_fixed_references_still_fail_closed_when_paths_are_missing(): monitor_mode="rate", ) + +def test_tab2_fixed_reference_dry_check_rejects_sample_shape_mismatch( + monkeypatch: pytest.MonkeyPatch, +): + module = _load_workbench_module() + app = module.SAXSAbsWorkbenchApp.__new__(module.SAXSAbsWorkbenchApp) + monkeypatch.setattr( + module, + "_workbench_load_detector_image", + lambda _path, dtype=None: SimpleNamespace(data=np.zeros((3, 4))), + ) + payload = {"fixed_dark_data": np.zeros((2, 2))} + + with pytest.raises(ValueError, match="sample/reference shape mismatch"): + app.validate_fixed_reference_shape("sample.tif", payload) + + +def test_tab2_radial_chi_with_fluorescence_fails_closed(): + module = _load_workbench_module() + app = module.SAXSAbsWorkbenchApp.__new__(module.SAXSAbsWorkbenchApp) + + with pytest.raises(ValueError, match="radial_chi.*fluorescence"): + app.validate_t2_mode_contract(["radial_chi"], True) + + app.validate_t2_mode_contract(["radial_chi"], False) + + +def test_tab2_queue_normalization_removes_duplicate_paths_before_approval(): + module = _load_workbench_module() + app = module.SAXSAbsWorkbenchApp.__new__(module.SAXSAbsWorkbenchApp) + app.t2_files = ["sample.tif", "sample.tif", "other.tif"] + app.refresh_queue_status = lambda: None + invalidated = [] + app._invalidate_workbench_preflight = invalidated.append + + normalized, changed = app.normalize_t2_queue() + + assert changed is True + assert normalized == [str(Path("sample.tif")), str(Path("other.tif"))] + assert app.t2_files == normalized + assert invalidated == ["t2"] + + +def test_tab3_queue_change_invalidates_old_approval_before_execution(): + module = _load_workbench_module() + app = module.SAXSAbsWorkbenchApp.__new__(module.SAXSAbsWorkbenchApp) + app.language = "en" + app.t3_files = ["sample.dat", "sample.dat"] + app.t3_resume_enabled = _Var(False) + app.refresh_external_1d_status = lambda: None + invalidated = [] + app._invalidate_workbench_preflight = invalidated.append + observed = {} + + def require(tab): + observed["tab"] = tab + observed["files"] = list(app.t3_files) + raise RuntimeError("stale preflight approval") + + app._require_current_workbench_preflight = require + errors = [] + app.show_error = lambda _title, message: errors.append(message) + + app.run_external_1d_batch() + + assert observed == {"tab": "t3", "files": ["sample.dat"]} + assert invalidated == ["t3"] + assert errors and "stale preflight" in errors[0] + def test_tab2_dry_run_marks_calibration_gate_failure_as_blocked(): module = _load_workbench_module() app = module.SAXSAbsWorkbenchApp.__new__(module.SAXSAbsWorkbenchApp) @@ -979,7 +1105,9 @@ def capture_gate(**kwargs): @pytest.mark.parametrize( ("export_cal2d", "expected_failed_files"), - [(False, 1), (True, 0)], + # Cal2D execution still consumes fixed BG/Dark references; missing or + # unreadable references must block every formal output mode. + [(False, 1), (True, 1)], ) def test_tab2_dry_run_empty_modes_blocks_unless_cal2d_export_is_enabled( export_cal2d, expected_failed_files @@ -2012,6 +2140,7 @@ def test_workbench_calibration_record_rejects_missing_or_tampered_context( def test_apply_session_invalidates_legacy_k_without_context(tmp_path): module = _load_workbench_module() app = _record_app(module) + app.language = "en" app.calibration_context = object() app.calibration_k_value = 2.5 app.session_geometry_fallback = {} @@ -2063,6 +2192,58 @@ def test_apply_session_safely_loads_complete_relative_calibration_record(tmp_pat assert any("record loaded" in message.lower() for message in shown) +def test_apply_session_resolves_relative_paths_from_session_directory_and_reports_callback_error( + tmp_path: Path, +): + module = _load_workbench_module() + app = _record_app(module) + inputs = tmp_path / "inputs" + inputs.mkdir() + standard = inputs / "standard.tif" + standard.write_bytes(b"placeholder") + app.t1_files = {"std": _Var("")} + app.on_load_std_t1 = lambda path: (_ for _ in ()).throw(RuntimeError("header callback failed")) + shown = [] + app.show_error = lambda *_args, **_kwargs: None + app.show_info = lambda _title, message: shown.append(message) + session_path = tmp_path / "session.json" + session_path.write_text( + json.dumps( + { + "schema": "saxsabs.session.v1", + "calibration": {"std_path": "inputs/standard.tif"}, + } + ), + encoding="utf-8", + ) + + app.apply_session(str(session_path)) + + assert app.t1_files["std"].get() == str(standard.resolve()) + assert any("callback load failed" in message.lower() for message in shown) + + +def test_apply_session_rejects_invalid_geometry_and_clears_stale_k(tmp_path: Path): + module = _load_workbench_module() + app = _record_app(module) + app.language = "en" + app.calibration_context = object() + app.calibration_k_value = 3.0 + shown = [] + app.show_error = lambda _title, message: shown.append(message) + session_path = tmp_path / "invalid-session.json" + session_path.write_text( + json.dumps({"geometry": {"wl_A": 0.0}}), + encoding="utf-8", + ) + + app.apply_session(str(session_path)) + + assert app.calibration_context is None + assert app.calibration_k_value is None + assert shown and "geometry" in shown[0].lower() + + def test_calibrated_2d_reintegration_matches_direct_absolute_1d_with_same_policy(): pytest.importorskip("pyFAI") from pyFAI.integrator.azimuthal import AzimuthalIntegrator