feat(cadecon): noise-constrained sparsity for low-SNR deconvolution

apps/cadecon/src/App.tsx

@@ -20,6 +20,7 @@ import { ImportOverlay } from './components/layout/ImportOverlay.tsx';
 import { RasterOverview } from './components/raster/RasterOverview.tsx';
 import { SubsetConfig } from './components/controls/SubsetConfig.tsx';
 import { AlgorithmSettings } from './components/controls/AlgorithmSettings.tsx';
+import { DisplaySettings } from './components/controls/DisplaySettings.tsx';
 import { RunControls } from './components/controls/RunControls.tsx';
 import { ProgressBar } from './components/controls/ProgressBar.tsx';
 import { ConvergencePanel } from './components/charts/ConvergencePanel.tsx';
@@ -163,6 +164,15 @@ const App: Component = () => {
                   <AlgorithmSettings />
                 </DashboardPanel>
 
+                <DashboardPanel
+                  id="display-options"
+                  variant="controls"
+                  data-tutorial="display-options"
+                >
+                  <p class="panel-label">Display Options</p>
+                  <DisplaySettings />
+                </DashboardPanel>
+
                 <DashboardPanel id="run-controls" variant="controls" data-tutorial="run-controls">
                   <p class="panel-label">Run Controls</p>
                   <RunControls />

apps/cadecon/src/components/controls/AlgorithmSettings.tsx

@@ -9,6 +9,8 @@ import {
   setHpFilterEnabled,
   lpFilterEnabled,
   setLpFilterEnabled,
+  noiseConstrained,
+  setNoiseConstrained,
   maxIterations,
   setMaxIterations,
   convergenceTol,
@@ -87,6 +89,14 @@ export function AlgorithmSettings(): JSX.Element {
           onChange={setLpFilterEnabled}
           disabled={isRunLocked()}
         />
+
+        <ToggleSwitch
+          label="Noise-Constrained Sparsity"
+          description="Stops adding spikes once the fit reaches the noise floor (suppresses spurious low-SNR spikes; below SNR≈1 it may trim real signal — toggle off)"
+          checked={noiseConstrained()}
+          onChange={setNoiseConstrained}
+          disabled={isRunLocked()}
+        />
       </div>
     </div>
   );

apps/cadecon/src/components/controls/DisplaySettings.tsx

@@ -0,0 +1,26 @@
+import type { JSX } from 'solid-js';
+import { ToggleSwitch } from './ToggleSwitch.tsx';
+import { sparsityCompareEnabled, setSparsityCompareEnabled } from '../../lib/algorithm-store.ts';
+import { isRunLocked } from '../../lib/iteration-store.ts';
+
+/**
+ * Display / diagnostic options. These do not change the deconvolution result —
+ * they add extra material for visual inspection. The sparsity-comparison overlay
+ * must be enabled BEFORE a run because it is computed during the run, so it lives
+ * with the pre-run controls rather than being a purely post-run view toggle.
+ */
+export function DisplaySettings(): JSX.Element {
+  return (
+    <div class="param-panel">
+      <div class="param-panel__sliders">
+        <ToggleSwitch
+          label="Sparsity Comparison Overlay"
+          description="Visual inspection only — solves every trace with BOTH sparsity settings each iteration so the Trace Inspector can overlay them. Does not change results, and roughly doubles run time."
+          checked={sparsityCompareEnabled()}
+          onChange={setSparsityCompareEnabled}
+          disabled={isRunLocked()}
+        />
+      </div>
+    </div>
+  );
+}

apps/cadecon/src/components/traces/TraceInspector.tsx

@@ -53,9 +53,11 @@ import {
   setShowGTCalcium,
   showGTSpikes,
   setShowGTSpikes,
+  showSparsityCompare,
+  setShowSparsityCompare,
   viewedIteration,
 } from '../../lib/viz-store.ts';
-import { upsampleFactor } from '../../lib/algorithm-store.ts';
+import { upsampleFactor, noiseConstrained } from '../../lib/algorithm-store.ts';
 import { subsetRectangles, selectedSubsetIdx } from '../../lib/subset-store.ts';
 import {
   createGroundTruthCalciumSeries,
@@ -72,6 +74,8 @@ const RESID_GAP_FRAC = 0.05;
 const RESID_SCALE = 0.25;
 const TRANSIENT_TAU_MULTIPLIER = 2;
 const TRACE_INSPECTOR_ZOOM_WINDOW_S = 60;
+// Distinct dashed color for the sparsity-comparison overlay (the OPPOSITE setting).
+const COMPARE_COLOR = '#e040fb';
 
 interface BandLayout {
   deconvTop: number;
@@ -211,6 +215,19 @@ export function TraceInspector(): JSX.Element {
     (): Float32Array | null => effectiveResult()?.filteredTrace ?? null,
   );
 
+  // Sparsity-comparison overlay: the opposite-setting spike counts, precomputed
+  // per iteration during the run (when the comparison option is enabled) and
+  // stored on the result. Read-only here — no solving on the browse path.
+  const comparisonDeconv = createMemo(
+    (): Float32Array | null => effectiveResult()?.comparisonSCounts ?? null,
+  );
+  const hasComparison = createMemo(() => comparisonDeconv() != null);
+
+  // Labels for the two deconv traces, by their actual setting.
+  const settingShort = (nc: boolean) => (nc ? 'noise-constr.' : 'standard');
+  const deconvLabel = () => `Deconv (${settingShort(noiseConstrained())})`;
+  const compareLabel = () => `Deconv (${settingShort(!noiseConstrained())})`;
+
   // Zoom window state
   const totalDuration = createMemo(() => {
     const raw = fullRawTrace();
@@ -311,7 +328,7 @@ export function TraceInspector(): JSX.Element {
     });
   };
 
-  const EMPTY_DATA: [number[], ...number[][]] = [[], [], [], [], [], [], [], []];
+  const EMPTY_DATA: [number[], ...number[][]] = [[], [], [], [], [], [], [], [], []];
   const DOWNSAMPLE_BUCKETS = 600;
 
   const zoomData = createMemo<[number[], ...number[][]]>(() => {
@@ -378,6 +395,17 @@ export function TraceInspector(): JSX.Element {
       dsDeconv = new Array(dsX.length).fill(null) as number[];
     }
 
+    // Sparsity comparison — opposite-setting deconv, same deconv band
+    const comp = comparisonDeconv();
+    let dsCompare: number[];
+    if (comp && comp.length >= endSample) {
+      const compSlice = comp.subarray(startSample, endSample);
+      const [, dsCompRaw] = downsampleMinMax(x, compSlice, DOWNSAMPLE_BUCKETS);
+      dsCompare = scaleToDeconvBand(dsCompRaw, rawMin, rawMax);
+    } else {
+      dsCompare = new Array(dsX.length).fill(null) as number[];
+    }
+
     // Residual — compute against the working trace (what the solver actually fit)
     const residSource = isFiltered ? (dsFiltered as number[]) : dsRaw;
     const dsResid = computeResiduals(residSource, dsFit, rawMin, rawMax, dsX.length);
@@ -409,6 +437,7 @@ export function TraceInspector(): JSX.Element {
       dsResid,
       dsGTCalcium,
       dsGTSpikes,
+      dsCompare,
     ];
   });
 
@@ -424,10 +453,17 @@ export function TraceInspector(): JSX.Element {
       { label: 'Raw', stroke: TRACE_COLORS.raw, width: 1, show: showRaw() },
       { label: 'Filtered', stroke: TRACE_COLORS.filtered, width: 1.5, show: showFiltered() },
       { label: 'Fit', stroke: TRACE_COLORS.fit, width: 1.5, show: showFit() },
-      { label: 'Deconv', stroke: TRACE_COLORS.deconv, width: 1, show: showDeconv() },
+      { label: deconvLabel(), stroke: TRACE_COLORS.deconv, width: 1, show: showDeconv() },
       { label: 'Residual', stroke: TRACE_COLORS.resid, width: 1, show: showResidual() },
       gtCaSeries,
       gtSpkSeries,
+      {
+        label: compareLabel(),
+        stroke: COMPARE_COLOR,
+        width: 1,
+        dash: [4, 3],
+        show: showSparsityCompare() && hasComparison(),
+      },
     ];
   });
 
@@ -458,7 +494,7 @@ export function TraceInspector(): JSX.Element {
       {
         key: 'deconv',
         color: TRACE_COLORS.deconv,
-        label: 'Deconv',
+        label: deconvLabel(),
         visible: showDeconv,
         setVisible: setShowDeconv,
       },
@@ -470,6 +506,16 @@ export function TraceInspector(): JSX.Element {
         setVisible: setShowResidual,
       },
     ];
+    if (hasComparison()) {
+      items.push({
+        key: 'compare',
+        color: COMPARE_COLOR,
+        label: compareLabel(),
+        visible: showSparsityCompare,
+        setVisible: setShowSparsityCompare,
+        dashed: true,
+      });
+    }
     if (gtVisible()) {
       items.push(
         {
@@ -502,6 +548,10 @@ export function TraceInspector(): JSX.Element {
     const r = effectiveResult();
     return r ? r.sCounts.reduce((s, v) => s + v, 0).toFixed(0) : '--';
   };
+  const comparisonSpikeCount = () => {
+    const c = comparisonDeconv();
+    return c ? c.reduce((s, v) => s + v, 0).toFixed(0) : null;
+  };
 
   // Subset highlight zones for the minimap — show which time regions
   // the algorithm operates on for the currently selected cell.
@@ -545,7 +595,15 @@ export function TraceInspector(): JSX.Element {
         <div class="trace-inspector__stats">
           <span>alpha: {alpha()}</span>
           <span>PVE: {pve()}</span>
-          <span>spikes: {spikeCount()}</span>
+          <Show
+            when={hasComparison() && comparisonSpikeCount() != null}
+            fallback={<span>spikes: {spikeCount()}</span>}
+          >
+            <span>
+              spikes: {spikeCount()} ({settingShort(noiseConstrained())}) · {comparisonSpikeCount()}{' '}
+              ({settingShort(!noiseConstrained())})
+            </span>
+          </Show>
         </div>
       </div>
 

apps/cadecon/src/lib/algorithm-store.ts

@@ -7,6 +7,14 @@ import { samplingRate } from './data-store.ts';
 const [upsampleTarget, setUpsampleTarget] = createSignal(300);
 const [hpFilterEnabled, setHpFilterEnabled] = createSignal(true);
 const [lpFilterEnabled, setLpFilterEnabled] = createSignal(true);
+// Noise-constrained sparsity: pick the sparsest spike support whose residual
+// still reaches the data-derived noise floor (no tuning knob). Suppresses noise
+// fit as spurious spikes; benefit concentrates at low SNR. Off = current max-PVE.
+const [noiseConstrained, setNoiseConstrained] = createSignal(true);
+// Opt-in: during each iteration, also solve every trace with the OPPOSITE
+// noise-constrained setting and store it, so the Trace Inspector can overlay the
+// two instantly (no live solve). Doubles inference cost — off by default.
+const [sparsityCompareEnabled, setSparsityCompareEnabled] = createSignal(false);
 const [maxIterations, setMaxIterations] = createSignal(20);
 
 // Convergence is tested in kernel SHAPE space (peak time + FWHM). convergenceTol
@@ -53,6 +61,10 @@ export {
   setHpFilterEnabled,
   lpFilterEnabled,
   setLpFilterEnabled,
+  noiseConstrained,
+  setNoiseConstrained,
+  sparsityCompareEnabled,
+  setSparsityCompareEnabled,
   maxIterations,
   setMaxIterations,
   convergenceTol,

apps/cadecon/src/lib/cadecon-pool.ts

@@ -26,6 +26,8 @@ interface TraceJobFields {
   hpEnabled: boolean;
   lpEnabled: boolean;
   lambda: number;
+  noiseConstrained: boolean;
+  computeComparison: boolean;
   warmCounts?: Float32Array;
   onComplete(result: TraceResult): void;
 }
@@ -120,6 +122,8 @@ const caDeconRouter: MessageRouter<CaDeconPoolJob, CaDeconWorkerOutbound> = {
           hpEnabled: job.hpEnabled,
           lpEnabled: job.lpEnabled,
           lambda: job.lambda,
+          noiseConstrained: job.noiseConstrained,
+          computeComparison: job.computeComparison,
           warmCounts: warmCopy,
         },
         transfers,

apps/cadecon/src/lib/iteration-manager.ts

@@ -43,6 +43,8 @@ import {
   finalSelectionWindow,
   hpFilterEnabled,
   lpFilterEnabled,
+  noiseConstrained,
+  sparsityCompareEnabled,
   traceFistaMaxIters,
   traceFistaTol,
   kernelFistaMaxIters,
@@ -164,6 +166,8 @@ function dispatchTraceJobs(
   hpEnabled: boolean,
   lpEnabled: boolean,
   lambda: number,
+  noiseConstrained: boolean,
+  computeComparison: boolean,
   prevResults?: Map<number, Float32Array>,
 ): Promise<Array<Map<number, TraceResult>>> {
   return new Promise((resolve) => {
@@ -211,6 +215,8 @@ function dispatchTraceJobs(
         hpEnabled,
         lpEnabled,
         lambda,
+        noiseConstrained,
+        computeComparison,
         warmCounts,
         onComplete(result: TraceResult) {
           results[subsetIdx].set(cell, result);
@@ -447,6 +453,8 @@ export async function startRun(): Promise<void> {
   const hpOn = hpFilterEnabled();
   const lpOn = lpFilterEnabled();
   const sparsityLambda = 0.0;
+  const noiseConstrainedOn = noiseConstrained();
+  const computeComparison = sparsityCompareEnabled();
 
   // Create pool
   pool = createCaDeconWorkerPool();
@@ -583,6 +591,8 @@ export async function startRun(): Promise<void> {
       hpOn,
       lpOn,
       sparsityLambda,
+      noiseConstrainedOn,
+      computeComparison,
       prevTraceCounts,
     );
 
@@ -601,6 +611,8 @@ export async function startRun(): Promise<void> {
     >();
     // Map cell → full-length filtered trace (stitched from subset windows)
     const cellFiltered = new Map<number, Float32Array>();
+    // Map cell → full-length opposite-setting counts (comparison overlay)
+    const cellComparison = new Map<number, Float32Array>();
     const batchEntries: Record<string, import('./iteration-store.ts').TraceResultEntry> = {};
     for (let si = 0; si < rects.length; si++) {
       const rect = rects[si];
@@ -621,6 +633,15 @@ export async function startRun(): Promise<void> {
           }
           fullFilt.set(result.filteredTrace, rect.tStart);
         }
+        // Stitch comparison counts subset windows into full-length arrays
+        if (result.comparisonSCounts) {
+          let fullCmp = cellComparison.get(cell);
+          if (!fullCmp) {
+            fullCmp = new Float32Array(nTp);
+            cellComparison.set(cell, fullCmp);
+          }
+          fullCmp.set(result.comparisonSCounts, rect.tStart);
+        }
         cellScalars.set(cell, {
           alpha: result.alpha,
           baseline: result.baseline,
@@ -638,6 +659,7 @@ export async function startRun(): Promise<void> {
           baseline: result.baseline,
           threshold: result.threshold,
           pve: result.pve,
+          comparisonSCounts: result.comparisonSCounts,
         };
       }
     }
@@ -656,6 +678,7 @@ export async function startRun(): Promise<void> {
         baseline: scalars.baseline,
         threshold: scalars.threshold,
         pve: scalars.pve,
+        comparisonSCounts: cellComparison.get(cell),
       };
     }
 
@@ -907,6 +930,8 @@ export async function startRun(): Promise<void> {
           hpEnabled: hpOn,
           lpEnabled: lpOn,
           lambda: sparsityLambda,
+          noiseConstrained: noiseConstrainedOn,
+          computeComparison,
           warmCounts,
           onComplete(result: TraceResult) {
             batch(() => {
@@ -919,6 +944,7 @@ export async function startRun(): Promise<void> {
                 baseline: result.baseline,
                 threshold: result.threshold,
                 pve: result.pve,
+                comparisonSCounts: result.comparisonSCounts,
               });
               finCompleted++;
               setCompletedSubsetTraceJobs(finCompleted);

apps/cadecon/src/lib/iteration-store.ts

@@ -59,6 +59,8 @@ export interface TraceResultEntry {
   baseline: number;
   threshold: number;
   pve: number;
+  /** Spike counts from the opposite noise-constrained setting (comparison overlay). */
+  comparisonSCounts?: Float32Array;
 }
 
 function cellSubsetKey(cellIndex: number, subsetIdx: number): string {

apps/cadecon/src/lib/tutorial/content/04-interpreting.ts

@@ -51,7 +51,17 @@ export const interpretingTutorial: Tutorial = {
       side: 'top',
       popoverClass: 'driver-popover--wide',
     },
-    // Step 6: When to change algorithm settings
+    // Step 6: Sparsity comparison overlay (display-only diagnostic)
+    {
+      element: '[data-tutorial="display-options"]',
+      title: 'Seeing the Sparsity’s Impact',
+      description:
+        'CaDecon runs <b>noise-constrained sparsity</b> by default: it keeps only the spikes needed to explain the trace down to its noise floor, which cleans up spurious activity — the effect is largest on low-SNR cells. To <i>see</i> how much it is doing, enable <b>Sparsity Comparison Overlay</b> here <b>before</b> a run. The Trace Inspector then overlays the deconvolution computed both with and without the sparsity, and shows both spike counts, so you can judge the impact per cell.<br><br>' +
+        'This is a <b>visual-inspection aid only</b> — it does not change the result. Because it solves every trace twice, it roughly <b>doubles run time</b>, so it is off by default. Leave it off for normal runs.',
+      side: 'right',
+      popoverClass: 'driver-popover--wide',
+    },
+    // Step 7: When to change algorithm settings
     {
       element: '[data-tutorial="algorithm-settings"]',
       title: 'When to Adjust Settings',