Auto gain plan failed

[yannachen]

Error:

Traceback (most recent call last):
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky_queueserver/manager/worker.py", line 296, in _execute_plan
    result = func()
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky_queueserver/manager/worker.py", line 486, in start_plan_func
    return self._RE(plan_func(*plan_args, **plan_kwargs), {"all": [self._run_reg_cb]}, **plan_meta)
           ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/run_engine.py", line 976, in __call__
    plan_return = self._resume_task(init_func=_build_task)
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/run_engine.py", line 1121, in _resume_task
    raise exc
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/run_engine.py", line 1756, in _run
    raise err
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/run_engine.py", line 1610, in _run
    msg = self._plan_stack[-1].send(resp)
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/preprocessors.py", line 1399, in __call__
    return (yield from plan)
            ^^^^^^^^^^^^^^^
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/preprocessors.py", line 1250, in baseline_wrapper
    return (yield from plan)
            ^^^^^^^^^^^^^^^
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/preprocessors.py", line 863, in monitor_during_wrapper
    return (yield from plan2)
            ^^^^^^^^^^^^^^^^
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/preprocessors.py", line 188, in plan_mutator
    raise ex
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/preprocessors.py", line 141, in plan_mutator
    msg = plan_stack[-1].send(ret)
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/preprocessors.py", line 188, in plan_mutator
    raise ex
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/preprocessors.py", line 141, in plan_mutator
    msg = plan_stack[-1].send(ret)
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/preprocessors.py", line 925, in fly_during_wrapper
    return (yield from plan2)
            ^^^^^^^^^^^^^^^^
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/preprocessors.py", line 188, in plan_mutator
    raise ex
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/preprocessors.py", line 141, in plan_mutator
    msg = plan_stack[-1].send(ret)
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/preprocessors.py", line 188, in plan_mutator
    raise ex
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/preprocessors.py", line 141, in plan_mutator
    msg = plan_stack[-1].send(ret)
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/preprocessors.py", line 1399, in __call__
    return (yield from plan)
            ^^^^^^^^^^^^^^^
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/preprocessors.py", line 1250, in baseline_wrapper
    return (yield from plan)
            ^^^^^^^^^^^^^^^
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/preprocessors.py", line 863, in monitor_during_wrapper
    return (yield from plan2)
            ^^^^^^^^^^^^^^^^
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/preprocessors.py", line 188, in plan_mutator
    raise ex
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/preprocessors.py", line 141, in plan_mutator
    msg = plan_stack[-1].send(ret)
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/preprocessors.py", line 188, in plan_mutator
    raise ex
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/preprocessors.py", line 141, in plan_mutator
    msg = plan_stack[-1].send(ret)
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/preprocessors.py", line 925, in fly_during_wrapper
    return (yield from plan2)
            ^^^^^^^^^^^^^^^^
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/preprocessors.py", line 188, in plan_mutator
    raise ex
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/preprocessors.py", line 141, in plan_mutator
    msg = plan_stack[-1].send(ret)
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/preprocessors.py", line 188, in plan_mutator
    raise ex
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/preprocessors.py", line 141, in plan_mutator
    msg = plan_stack[-1].send(ret)
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/utils/__init__.py", line 1279, in dec_inner
    return (yield from plan)
            ^^^^^^^^^^^^^^^
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/preprocessors.py", line 548, in finalize_wrapper
    ret = yield from plan
          ^^^^^^^^^^^^^^^
  File "/home/beams0/S25IDCUSER/src/haven/src/haven/preprocessors/aps_suspenders.py", line 122, in install_suspenders
    yield from plan
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/utils/__init__.py", line 1279, in dec_inner
    return (yield from plan)
            ^^^^^^^^^^^^^^^
  File "/home/beams0/S25IDCUSER/src/haven/src/haven/preprocessors/open_shutters.py", line 88, in open_shutters_wrapper
    return_val = yield from new_plan
                 ^^^^^^^^^^^^^^^^^^^
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/preprocessors.py", line 548, in finalize_wrapper
    ret = yield from plan
          ^^^^^^^^^^^^^^^
  File "/home/beams0/S25IDCUSER/src/haven/src/haven/preprocessors/open_shutters.py", line 159, in open_on_kickoff_wrapper
    response = yield msg
               ^^^^^^^^^
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/preprocessors.py", line 96, in plan_mutator
    msg = plan_stack[-1].throw(exception)
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/utils/__init__.py", line 213, in single_gen
    return (yield msg)
            ^^^^^^^^^
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/preprocessors.py", line 213, in plan_mutator
    inner_ret = yield msg
                ^^^^^^^^^
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/preprocessors.py", line 213, in plan_mutator
    inner_ret = yield msg
                ^^^^^^^^^
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/preprocessors.py", line 96, in plan_mutator
    msg = plan_stack[-1].throw(exception)
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/utils/__init__.py", line 213, in single_gen
    return (yield msg)
            ^^^^^^^^^
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/preprocessors.py", line 213, in plan_mutator
    inner_ret = yield msg
                ^^^^^^^^^
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/preprocessors.py", line 213, in plan_mutator
    inner_ret = yield msg
                ^^^^^^^^^
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/preprocessors.py", line 96, in plan_mutator
    msg = plan_stack[-1].throw(exception)
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/utils/__init__.py", line 213, in single_gen
    return (yield msg)
            ^^^^^^^^^
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/preprocessors.py", line 213, in plan_mutator
    inner_ret = yield msg
                ^^^^^^^^^
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/preprocessors.py", line 213, in plan_mutator
    inner_ret = yield msg
                ^^^^^^^^^
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/preprocessors.py", line 96, in plan_mutator
    msg = plan_stack[-1].throw(exception)
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/preprocessors.py", line 843, in new_gen
    yield from ensure_generator(monitor_msgs)
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/utils/__init__.py", line 192, in <genexpr>
    gen = (msg for msg in gen)
           ^^^
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/preprocessors.py", line 213, in plan_mutator
    inner_ret = yield msg
                ^^^^^^^^^
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/preprocessors.py", line 213, in plan_mutator
    inner_ret = yield msg
                ^^^^^^^^^
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/run_engine.py", line 1677, in _run
    new_response = await coro(msg)
                   ^^^^^^^^^^^^^^^
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/run_engine.py", line 2040, in _monitor
    await current_run.monitor(msg)
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/bundlers.py", line 466, in monitor
    obj.subscribe(emit_event, **kwargs)
    ~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/ophyd_async/core/_signal.py", line 254, in subscribe_reading
    self._get_cache().subscribe(function)
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/ophyd_async/core/_signal.py", line 171, in subscribe
    self._notify(function)
    ~~~~~~~~~~~~^^^^^^^^^^
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/ophyd_async/core/_signal.py", line 166, in _notify
    function({self._signal.name: self._ensure_reading()})
    ~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/bundlers.py", line 462, in emit_event
    self.emit_sync(DocumentNames.event, doc)
    ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/run_engine.py", line 2658, in emit_sync
    self.dispatcher.process(name, doc)
    ~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/run_engine.py", line 2681, in process
    exceptions = self.cb_registry.process(name, name.name, doc)
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/utils/__init__.py", line 443, in process
    func(*args, **kwargs)
    ~~~~^^^^^^^^^^^^^^^^^
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky/utils/__init__.py", line 533, in __call__
    return mtd(*args, **kwargs)
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/event_model/__init__.py", line 203, in __call__
    return self._dispatch(name, doc, validate)
           ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/event_model/__init__.py", line 220, in _dispatch
    output_event_page = self.event_page(event_page)
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/event_model/__init__.py", line 1653, in event_page
    callback("event_page", doc)
    ~~~~~~~~^^^^^^^^^^^^^^^^^^^
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky_adaptive/per_event.py", line 78, in callback
    independent, measurement = extract_event_page(independent_keys, dependent_keys, payload=doc["data"])
                               ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky_adaptive/utils/__init__.py", line 96, in extract_event_page
    return tuple(np.atleast_2d(np.asarray([payload[k] for k in key_list])).T for key_list in key_lists)
  File "/home/beams0/S25IDCUSER/src/haven/.pixi/envs/25idc/lib/python3.13/site-packages/bluesky_adaptive/utils/__init__.py", line 96, in <genexpr>
    return tuple(np.atleast_2d(np.asarray([payload[k] for k in key_list])).T for key_list in key_lists)
                                           ~~~~~~~^^^
KeyError: 'I0-preamp-gain_level'

[canismarko]

For some reason the problem goes away if we remove the aps current from the supplemental data preprocessor in the startup file. Not sure what to make of that.