From 719e83788031ef78daff523061cf63e8d3a9eb0f Mon Sep 17 00:00:00 2001 From: Riley Scheid Date: Thu, 13 Aug 2026 22:14:06 +0000 Subject: [PATCH 01/31] Make `is Any` annotation checks legal under `strict_equality` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mypy 1.18 rejects `if x is Any:` as `comparison-overlap` when `x` is declared as a union of concrete types. The comparison is correct at runtime: `get_args` on an annotation such as `dict[str, Any]` hands back the `typing.Any` object itself, so the converters meet it as a value. Only the declared parameter types disagree, since they do not mention `Any`. Route the comparison through `is_annotation_any`, whose parameter is typed `object` — the honest domain of what typing introspection returns. The converters' declared unions stay as they are, and the runtime behavior is identical: the same `is` comparison, one call deeper. Until now this error failed the build of every target whose closure reaches `reboot/api.py` under a fresh mypy run, which is how it was found: Bazel's remote cache had been serving stale mypy results, so CI never re-ran mypy over these files. Co-Authored-By: Claude Fable 5 --- reboot/api.py | 15 ++++++++++++++- reboot/pydantic_schema_to_zod.py | 3 ++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/reboot/api.py b/reboot/api.py index 8987d9f6..40c6d01e 100644 --- a/reboot/api.py +++ b/reboot/api.py @@ -42,6 +42,19 @@ typing.Type[Dict[str, Any]], ] + +def is_annotation_any(annotation: object) -> bool: + """Whether `annotation` is the `typing.Any` special form. + + `get_args` on an annotation such as `dict[str, Any]` hands back the + `typing.Any` object itself, so converters that recurse into + annotations meet it as a value. Comparing through an `object`-typed + parameter keeps the comparison legal under `strict_equality` for + callers whose declared parameter types do not mention `Any`. + """ + return annotation is Any + + # We don't allow passing arbitrary default values, only these empty # defaults, which also matches with Protobuf semantics. ALLOWED_DEFAULT_BY_FIELD_TYPE = { @@ -638,7 +651,7 @@ def _proto_to_pydantic( # 'None' directly. return None - if output_type is Any: + if is_annotation_any(output_type): # Reverse of the `Any` case in `_pydantic_to_proto`: a # `google.protobuf.Value` becomes the JSON value it holds. assert isinstance(input, Value) diff --git a/reboot/pydantic_schema_to_zod.py b/reboot/pydantic_schema_to_zod.py index 9b9c1fb8..6c41220e 100644 --- a/reboot/pydantic_schema_to_zod.py +++ b/reboot/pydantic_schema_to_zod.py @@ -16,6 +16,7 @@ Model, UserPydanticError, get_field_tag, + is_annotation_any, snake_to_camel, ) from reboot.fail import fail @@ -208,7 +209,7 @@ def pydantic_to_zod( # Currently only used for methods with no response. return 'z.void()' - if input is Any: + if is_annotation_any(input): # `Any` — e.g. a `dict[str, Any]` map value — accepts any JSON # value. `z.json()` is the schema our `zod-to-proto` converts # to a `google.protobuf.Value`. From ccccb71513633198c5380360d508f238b83f4529 Mon Sep 17 00:00:00 2001 From: Riley Scheid Date: Tue, 11 Aug 2026 03:23:51 +0000 Subject: [PATCH 02/31] Allow all methods, not just MCP methods, to have a description A `description=` was only set on `McpMethodOptions.description`, so a Reader, Writer, Transaction or Workflow that was not also an MCP tool didn't have it. It is now set on `MethodOptions.description` for every method, and MCP tool and resource descriptions read from there. `McpMethodOptions.description` is deprecated but still read as a fallback, so protos that already set it keep their descriptions. Co-Authored-By: Claude Opus 5 (1M context) --- rbt/v1alpha1/options.proto | 8 +++- reboot/protoc_gen_reboot_generic.py | 18 ++++++-- reboot/pydantic_schema_to_proto.py | 17 ++++--- reboot/templates/reboot.py.j2 | 4 +- tests/reboot/protoc_gen_reboot_tests.py | 61 +++++++++++++++++++++++++ 5 files changed, 94 insertions(+), 14 deletions(-) diff --git a/rbt/v1alpha1/options.proto b/rbt/v1alpha1/options.proto index 394b9cc8..9c8227ca 100644 --- a/rbt/v1alpha1/options.proto +++ b/rbt/v1alpha1/options.proto @@ -53,8 +53,9 @@ message McpMethodOptions { optional bool resource = 2; // Custom name for the tool/resource (defaults to method name). optional string name = 3; - // Description for the tool/resource. - optional string description = 4; + // Deprecated: write `MethodOptions.description` instead. Read as a + // fallback when it is the only description present. + optional string description = 4 [deprecated = true]; // Display title for the tool/resource. optional string title = 5; } @@ -102,6 +103,9 @@ message MethodOptions { // MCP options for exposing this method as a tool or resource. optional McpMethodOptions mcp = 9; + + // What this method does, in the author's own words. + optional string description = 10; } extend google.protobuf.MethodOptions { diff --git a/reboot/protoc_gen_reboot_generic.py b/reboot/protoc_gen_reboot_generic.py index e41efc71..1e3a4f3b 100644 --- a/reboot/protoc_gen_reboot_generic.py +++ b/reboot/protoc_gen_reboot_generic.py @@ -157,7 +157,6 @@ class ProtoMcpOptions: tool: bool resource: bool name: Optional[str] - description: Optional[str] title: Optional[str] @@ -167,6 +166,7 @@ class ProtoMethodOptions: constructor: bool state_streaming: bool has_errors: bool + description: Optional[str] mcp: Optional[ProtoMcpOptions] @@ -898,16 +898,28 @@ def _proto_method_options( tool=mcp.tool, resource=mcp.resource, name=mcp.name if mcp.HasField('name') else None, - description=mcp.description - if mcp.HasField('description') else None, title=mcp.title if mcp.HasField('title') else None, ) + description: Optional[str] = None + if method_options.HasField('description'): + description = method_options.description + elif ( + method_options.HasField('mcp') and + method_options.mcp.HasField('description') + ): + # An application that was created before + # `MethodOptions.description` will have the deprecated + # `mcp` description, which is permitted for backward + # compatibility. + description = method_options.mcp.description + return ProtoMethodOptions( kind=kind, constructor=self._is_method_constructor(method), state_streaming=state_streaming, has_errors=len(method_options.errors) > 0, + description=description, mcp=mcp_options, ) diff --git a/reboot/pydantic_schema_to_proto.py b/reboot/pydantic_schema_to_proto.py index f37ee679..2f774a1c 100644 --- a/reboot/pydantic_schema_to_proto.py +++ b/reboot/pydantic_schema_to_proto.py @@ -921,6 +921,16 @@ async def generate_proto_file_from_api( f" errors: [\"{type_name}{to_pascal_case(method_name)}Errors\"],\n" ) + # What the author said the method does, written + # whether or not the method is exposed to MCP. + if method_spec.description is not None: + # The description can contain `\` character, so we + # need to escape it for proto string literal. + await proto.write( + " description: " + f'"{_escape_string_for_proto(method_spec.description)}",\n' + ) + # MCP options for exposing method as tool/resource. if method_spec.mcp is not None: mcp = method_spec.mcp @@ -935,13 +945,6 @@ async def generate_proto_file_from_api( mcp_fields.append( f'name: "{_escape_string_for_proto(mcp.name)}"' ) - if method_spec.description is not None: - # The description can contain `\` character, so - # we need to escape it for proto string literal. - mcp_fields.append( - "description: " - f'"{_escape_string_for_proto(method_spec.description)}"' - ) if mcp.title is not None: # The title can contain `\` character, so we need # to escape it for proto string literal. diff --git a/reboot/templates/reboot.py.j2 b/reboot/templates/reboot.py.j2 index 2acaeb46..66db0879 100644 --- a/reboot/templates/reboot.py.j2 +++ b/reboot/templates/reboot.py.j2 @@ -2692,7 +2692,7 @@ class {{ state.proto.name }}BaseServicer(IMPORT_reboot_aio_servicers.Servicer): {% set tool_name = method.options.proto.mcp.name if method.options.proto.mcp.name else mcp_name_prefix + (method.proto.name | to_snake) %} {% set tool_title = method.options.proto.mcp.title if method.options.proto.mcp.title else method.proto.name %} {% set tool_description_suffix = "" if state.proto.auto_construct != AUTO_CONSTRUCT_UNSPECIFIED else " on " + state.proto.name %} -{% set tool_description = method.options.proto.mcp.description if method.options.proto.mcp.description else "Invoke " + method.proto.name + tool_description_suffix + "." %} +{% set tool_description = method.options.proto.description if method.options.proto.description else "Invoke " + method.proto.name + tool_description_suffix + "." %} {% set request_type = state.proto.name + "." + method.proto.name + "Request" %} # Tool for '{{ method.proto.full_name }}'. @@ -2741,7 +2741,7 @@ class {{ state.proto.name }}BaseServicer(IMPORT_reboot_aio_servicers.Servicer): {% set method_path = method.proto.name | to_snake %} {% set resource_name = method.options.proto.mcp.name if method.options.proto.mcp.name else mcp_name_prefix + method_path %} {% set resource_title = method.options.proto.mcp.title if method.options.proto.mcp.title else method.proto.name %} -{% set resource_description = method.options.proto.mcp.description if method.options.proto.mcp.description else state.proto.name + " state." %} +{% set resource_description = method.options.proto.description if method.options.proto.description else state.proto.name + " state." %} {% set request_type = state.proto.name + "." + method.proto.name + "Request" %} # Resource for '{{ method.proto.full_name }}'. diff --git a/tests/reboot/protoc_gen_reboot_tests.py b/tests/reboot/protoc_gen_reboot_tests.py index 964c4a79..c1a28ca5 100644 --- a/tests/reboot/protoc_gen_reboot_tests.py +++ b/tests/reboot/protoc_gen_reboot_tests.py @@ -4,6 +4,7 @@ from google.protobuf.compiler import plugin_pb2 from google.protobuf.descriptor_pb2 import FileDescriptorSet from google.protobuf.descriptor_pool import DescriptorPool +from rbt.v1alpha1 import options_pb2 from reboot.protoc_gen_reboot_generic import ( BaseFile, ProtocPlugin, @@ -173,6 +174,66 @@ def test_map_type(self) -> None: map_field_type = fields['metadata'] self.assertEqual(map_field_type, "dict[str, str]") + def _greet_method_options(self): + test_plugin(self.plugin, self.descriptor_set) + + template_data = self.plugin.proto_to_template_data[ + 'tests/reboot/greeter.proto'] + methods = { + method.proto.name: method + for method in template_data.clients[0].services[0].methods + } + return methods['Greet'].options.proto + + def _greet_method_descriptor(self): + file = self.descriptor_set.file[-1] + self.assertEqual(file.name, 'tests/reboot/greeter.proto') + + service = file.service[0] + self.assertEqual(service.name, 'GreeterMethods') + + methods = {method.name: method for method in service.method} + return methods['Greet'] + + def test_method_description_reaches_the_template(self) -> None: + method = self._greet_method_descriptor() + method.options.Extensions[options_pb2.method + ].description = 'Greet someone.' + + self.assertEqual( + self._greet_method_options().description, + 'Greet someone.', + ) + + def test_deprecated_mcp_description_still_reaches_the_template( + self + ) -> None: + # An application that was created before + # `MethodOptions.description` will have the deprecated `mcp` + # description, which is permitted for backward compatibility. + method = self._greet_method_descriptor() + method.options.Extensions[options_pb2.method + ].mcp.description = 'Greet someone.' + + self.assertEqual( + self._greet_method_options().description, + 'Greet someone.', + ) + + def test_method_description_wins_over_the_deprecated_one(self) -> None: + method = self._greet_method_descriptor() + options = method.options.Extensions[options_pb2.method] + options.description = 'What the author wrote.' + options.mcp.description = 'The deprecated spelling.' + + self.assertEqual( + self._greet_method_options().description, + 'What the author wrote.', + ) + + def test_method_without_a_description_has_none(self) -> None: + self.assertIsNone(self._greet_method_options().description) + class ToLowerCamelTest(unittest.TestCase): """Test the `to_lower_camel` Jinja filter override. From 88616079a4810764c7f233b870269f70de7594ee Mon Sep 17 00:00:00 2001 From: Riley Scheid Date: Tue, 11 Aug 2026 03:25:02 +0000 Subject: [PATCH 03/31] Add a dev-mode companion application that serves a dashboard The first of several changes, split so that each can be reviewed on its own and so the dashboard can be tried out while the rest is written. It is not a documented feature yet: nothing opens a page by itself, so seeing one means passing `--open-dashboard` or visiting the URL. `rbt dev run` now starts a second Reboot application alongside the developer's, with its own state store, holding what the dashboard needs. The companion watches the developer's `api/` directory and records what those files declare, so the dashboard can describe an application. It also serves the page itself. The page reads that schema reactively and renders one section per state type: its fields, and each method's kind, whether it constructs, whether it is reachable over MCP, its signature and the errors it raises. Dashboard state, such as which detail views are open and which are closed, is saved in Reboot state, so it survives a hot reload and an `rbt dev run` restart. Auto-open is complete but off. `_AUTO_OPEN_DASHBOARD` is False, so only `--open-dashboard` opens a page. We don't reopen the dashboard if the developer already has it open, and we use the `Presence` library to determine whether they do. Note that presence does not drain through a DevPod workstation's port forward, which is filed separately. This will eventually supersede the inspect dashboard at `/__/inspect`, which lists state instances and their values. It does not replace it yet and both exist meanwhile: this describes an application's API, its state types, their fields and their methods, and cannot yet show the data behind them. Co-Authored-By: Claude Opus 5 (1M context) --- rbt/dashboard/v1/BUILD.bazel | 67 ++ rbt/dashboard/v1/dashboard.proto | 173 +++++ rbt/dashboard/v1/package.json | 3 + rbt/std/presence/v1/presence.proto | 16 + rbt/v1alpha1/inspect/inspect.proto | 51 ++ reboot/BUILD.bazel | 1 + reboot/cli/commands/BUILD.bazel | 26 + reboot/cli/commands/dashboard.py | 245 +++++++ reboot/cli/commands/dev.py | 207 +++++- reboot/cli/common/BUILD.bazel | 1 + reboot/cli/common/cli.py | 12 +- reboot/dashboard/BUILD.bazel | 120 +++ reboot/dashboard/api_reader.py | 202 ++++++ reboot/dashboard/api_watcher.py | 190 +++++ reboot/dashboard/constants.py | 46 ++ reboot/dashboard/frontend/BUILD.bazel | 51 ++ reboot/dashboard/frontend/dashboard.css | 681 ++++++++++++++++++ reboot/dashboard/frontend/index.html | 18 + reboot/dashboard/frontend/src/constants.ts | 5 + reboot/dashboard/frontend/src/main.tsx | 527 ++++++++++++++ reboot/dashboard/frontend/tsconfig.json | 16 + reboot/dashboard/main.py | 84 +++ reboot/dashboard/servicers.py | 131 ++++ reboot/inspect/BUILD.bazel | 13 + reboot/inspect/describe_state_type.py | 141 ++++ reboot/std/presence/v1/presence.py | 9 + tests/reboot/cli/BUILD.bazel | 14 + tests/reboot/cli/dashboard_tests.py | 127 ++++ tests/reboot/cli/dev_tests.py | 16 + tests/reboot/dashboard/BUILD.bazel | 83 +++ tests/reboot/dashboard/api/shop/v1/helper.py | 3 + tests/reboot/dashboard/api/shop/v1/shop.py | 48 ++ tests/reboot/dashboard/api_reader_tests.py | 127 ++++ tests/reboot/dashboard/api_watcher_tests.py | 111 +++ tests/reboot/dashboard/application_tests.py | 102 +++ tests/reboot/dashboard/dashboard_tests.py | 418 +++++++++++ .../reboot/dashboard/open_dashboard_tests.py | 219 ++++++ tests/reboot/dashboard/preferences_tests.py | 166 +++++ 38 files changed, 4466 insertions(+), 4 deletions(-) create mode 100644 rbt/dashboard/v1/BUILD.bazel create mode 100644 rbt/dashboard/v1/dashboard.proto create mode 100644 rbt/dashboard/v1/package.json create mode 100644 reboot/cli/commands/dashboard.py create mode 100644 reboot/dashboard/BUILD.bazel create mode 100644 reboot/dashboard/api_reader.py create mode 100644 reboot/dashboard/api_watcher.py create mode 100644 reboot/dashboard/constants.py create mode 100644 reboot/dashboard/frontend/BUILD.bazel create mode 100644 reboot/dashboard/frontend/dashboard.css create mode 100644 reboot/dashboard/frontend/index.html create mode 100644 reboot/dashboard/frontend/src/constants.ts create mode 100644 reboot/dashboard/frontend/src/main.tsx create mode 100644 reboot/dashboard/frontend/tsconfig.json create mode 100644 reboot/dashboard/main.py create mode 100644 reboot/dashboard/servicers.py create mode 100644 reboot/inspect/describe_state_type.py create mode 100644 tests/reboot/cli/dashboard_tests.py create mode 100644 tests/reboot/dashboard/BUILD.bazel create mode 100644 tests/reboot/dashboard/api/shop/v1/helper.py create mode 100644 tests/reboot/dashboard/api/shop/v1/shop.py create mode 100644 tests/reboot/dashboard/api_reader_tests.py create mode 100644 tests/reboot/dashboard/api_watcher_tests.py create mode 100644 tests/reboot/dashboard/application_tests.py create mode 100644 tests/reboot/dashboard/dashboard_tests.py create mode 100644 tests/reboot/dashboard/open_dashboard_tests.py create mode 100644 tests/reboot/dashboard/preferences_tests.py diff --git a/rbt/dashboard/v1/BUILD.bazel b/rbt/dashboard/v1/BUILD.bazel new file mode 100644 index 00000000..84478778 --- /dev/null +++ b/rbt/dashboard/v1/BUILD.bazel @@ -0,0 +1,67 @@ +load( + "@com_github_reboot_dev_reboot//reboot:rules.bzl", + "js_proto_library", + "js_reboot_library", + "js_reboot_react_library", + "py_reboot_library", +) +load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library") + +proto_library( + name = "dashboard_proto", + srcs = [ + ":dashboard.proto", + ], + visibility = ["//visibility:public"], + deps = [ + "@com_github_reboot_dev_reboot//rbt/v1alpha1:options_proto", + ], +) + +py_reboot_library( + name = "dashboard_py_reboot", + proto = "dashboard.proto", + proto_library = ":dashboard_proto", + visibility = ["//visibility:public"], +) + +js_proto_library( + name = "dashboard_js_proto", + package_json = ":package.json", + proto = "dashboard.proto", + proto_deps = [ + ":dashboard_proto", + # ISSUE(https://github.com/reboot-dev/mono/issues/3218): Until we can + # use `create_protoc_plugin_rule` we need to repeat the dependencies of + # the `proto_libraries` here. + "@com_github_reboot_dev_reboot//rbt/v1alpha1:options_proto", + "@com_google_protobuf//:descriptor_proto", + ], + visibility = ["//visibility:public"], +) + +js_reboot_library( + name = "dashboard_js_reboot", + srcs = [ + ":dashboard_proto", + ], + proto = "dashboard.proto", + visibility = ["//visibility:public"], + deps = [ + ":dashboard_js_proto", + ], +) + +js_reboot_react_library( + name = "dashboard_js_reboot_react", + srcs = [ + ":dashboard_js_proto", + ], + proto = "dashboard.proto", + proto_deps = [ + ":dashboard_proto", + "@com_github_reboot_dev_reboot//rbt/v1alpha1:options_proto", + "@com_google_protobuf//:descriptor_proto", + ], + visibility = ["//visibility:public"], +) diff --git a/rbt/dashboard/v1/dashboard.proto b/rbt/dashboard/v1/dashboard.proto new file mode 100644 index 00000000..38368499 --- /dev/null +++ b/rbt/dashboard/v1/dashboard.proto @@ -0,0 +1,173 @@ +syntax = "proto3"; + +package rbt.dashboard.v1; + +import "rbt/v1alpha1/options.proto"; + +//////////////////////////////////////////////////////////////////////// + +// The information the developer dashboard needs to display data +// about the Reboot application being developed. + +message FieldInfo { + string name = 1; + string type = 2; +} + +message MethodInfo { + string name = 1; + string kind = 2; + repeated FieldInfo arguments = 3; + repeated FieldInfo returns = 4; + repeated string errors = 5; + optional string description = 6; + bool factory = 7; + bool mcp = 8; +} + +message StateTypeInfo { + string name = 1; + + // The file the developer declared it in, e.g. + // "bank/v1/account.py". + string file = 2; + repeated FieldInfo fields = 3; + repeated MethodInfo methods = 4; +} + +//////////////////////////////////////////////////////////////////////// + +// What the dashboard application has read of the developer's API +// files, so that a browser can read it without reaching the +// application itself. +message API { + option (rbt.v1alpha1.state) = { + }; + + // The state types the developer's API files declare, which exist + // before the application does. + repeated StateTypeInfo state_types = 1; + + // Why the API files could not be read, if they could not be. A + // half-written file is the normal case while someone is typing, and + // saying so beats showing nothing. + string error = 2; +} + +message APIGetRequest {} + +message APIGetResponse { + repeated StateTypeInfo state_types = 1; + string error = 2; +} + +message APIUpdateRequest { + repeated StateTypeInfo state_types = 1; + string error = 2; +} + +message APIUpdateResponse {} + +message APIWatchRequest {} + +message APIWatchResponse {} + +//////////////////////////////////////////////////////////////////////// + +// What the developer has told the dashboard about opening dashboards. +// +// Kept here rather than in a file under their project because it is a +// fact about their machine and their browser, not about their +// application, and nothing about it belongs in their repository. It +// survives a hot reload and an `rbt dev run` restart because the +// dashboard application's state store does. +message Preferences { + option (rbt.v1alpha1.state) = { + }; + + // Whether the developer has asked `rbt dev run` to stop opening a + // dashboard by itself. Recorded as the exception rather than as + // the rule, so that somebody who has never chosen gets a dashboard + // opened when nobody is looking at one. + // + // `--open-dashboard` opens one whatever this says. + bool suppress_open_on_restart = 1; + + // The state types whose methods the developer has opened on the + // dashboard, by fully qualified name such as `bank.v1.Account`. + // Held as the set that is open rather than the set that is closed, + // so that somebody who has clicked nothing gets a page where every + // state type shows its methods but none of their detail. + // + // Kept sorted, so that clicking two state types open in one order + // and then the other stores the same thing both times. + repeated string expanded_state_types = 2; +} + +message PreferencesGetRequest {} + +message PreferencesGetResponse { + bool suppress_open_on_restart = 1; + repeated string expanded_state_types = 2; +} + +message PreferencesSetSuppressOpenOnRestartRequest { + bool suppress_open_on_restart = 1; +} + +message PreferencesSetSuppressOpenOnRestartResponse {} + +message PreferencesSetExpandedRequest { + // The fully qualified name of one state type, spelled the way + // `StateTypeInfo` spells it. + string state_type = 1; + + bool expanded = 2; +} + +message PreferencesSetExpandedResponse {} + +//////////////////////////////////////////////////////////////////////// + +service APIMethods { + rpc Get(APIGetRequest) returns (APIGetResponse) { + option (rbt.v1alpha1.method).reader = { + }; + } + + rpc Update(APIUpdateRequest) returns (APIUpdateResponse) { + option (rbt.v1alpha1.method).writer = { + }; + } + + // Watches the developer's API files for as long as the dashboard + // application runs, reading each one that changes. + rpc Watch(APIWatchRequest) returns (APIWatchResponse) { + option (rbt.v1alpha1.method).workflow = { + }; + } +} + +//////////////////////////////////////////////////////////////////////// + +service PreferencesMethods { + rpc Get(PreferencesGetRequest) returns (PreferencesGetResponse) { + option (rbt.v1alpha1.method).reader = { + }; + } + + rpc SetSuppressOpenOnRestart(PreferencesSetSuppressOpenOnRestartRequest) + returns (PreferencesSetSuppressOpenOnRestartResponse) { + option (rbt.v1alpha1.method).writer = { + }; + } + + // One state type per call rather than the whole set, so that two + // tabs opening two different state types at the same moment do not + // each write back a set that predates the other's click. + rpc SetExpanded(PreferencesSetExpandedRequest) + returns (PreferencesSetExpandedResponse) { + option (rbt.v1alpha1.method).writer = { + }; + } +} diff --git a/rbt/dashboard/v1/package.json b/rbt/dashboard/v1/package.json new file mode 100644 index 00000000..3dbc1ca5 --- /dev/null +++ b/rbt/dashboard/v1/package.json @@ -0,0 +1,3 @@ +{ + "type": "module" +} diff --git a/rbt/std/presence/v1/presence.proto b/rbt/std/presence/v1/presence.proto index 38ac3238..23e0528b 100644 --- a/rbt/std/presence/v1/presence.proto +++ b/rbt/std/presence/v1/presence.proto @@ -24,6 +24,15 @@ message Presence { * `Subscriber` using the API described there. */ service PresenceMethods { + /** + * Constructs the `Presence` instance, empty, so that `List` + * answers before the first `Subscribe`. + */ + rpc Create(CreateRequest) returns (CreateResponse) { + option (rbt.v1alpha1.method).writer = { + }; + } + /** * Registers a subscriber as present. * @@ -54,6 +63,13 @@ service PresenceMethods { //////////////////////////////////////////////////////////////////////// +// See `Create`. +message CreateRequest {} + +message CreateResponse {} + +//////////////////////////////////////////////////////////////////////// + // See `Subscribe`. message SubscribeRequest { // The subscriber to register as present. diff --git a/rbt/v1alpha1/inspect/inspect.proto b/rbt/v1alpha1/inspect/inspect.proto index 5031d26c..478a0e92 100644 --- a/rbt/v1alpha1/inspect/inspect.proto +++ b/rbt/v1alpha1/inspect/inspect.proto @@ -66,6 +66,57 @@ message GetStateResponse { //////////////////////////////////////////////////////////////////////// +// A field of a state, or an argument of a method. +message FieldInfo { + string name = 1; + + // Rendered for a reader, e.g. "str", "float", "AccountState". + string type = 2; +} + +message MethodInfo { + string name = 1; + + // One of "reader", "writer", "transaction" or "workflow". + string kind = 2; + + // The request message's fields, flattened. A method taking no + // request has none. + repeated FieldInfo arguments = 3; + + // The response's fields; empty when the method returns + // nothing. + repeated FieldInfo returns = 4; + + // Names of the error types the method declares it may raise. + repeated string errors = 5; + + // The description its author wrote, when there is one. + optional string description = 6; + + // Whether this method is a factory, constructing the state + // rather than requiring it to already exist. + bool factory = 7; + + // Whether the method is exposed as an MCP tool or resource. + bool mcp = 8; +} + +message StateTypeInfo { + // Fully qualified, e.g. "bank.v1.Account". + string name = 1; + + // The file the developer declared it in, e.g. + // "bank/v1/account.py". + string file = 2; + + repeated FieldInfo fields = 3; + + repeated MethodInfo methods = 4; +} + +//////////////////////////////////////////////////////////////////////// + service Inspect { // The list of state types in an application is static, however, we // make this a streaming RPC so that the client can hear when it diff --git a/reboot/BUILD.bazel b/reboot/BUILD.bazel index 2bb7b4cd..c7d6d83d 100644 --- a/reboot/BUILD.bazel +++ b/reboot/BUILD.bazel @@ -516,6 +516,7 @@ py_library( ":python_thirdparty", "//reboot/aio:python", "//reboot/cli:main_py", + "//reboot/dashboard:main_py", "//reboot/mcp:python", "//reboot/nodejs:python", "@com_github_reboot_dev_reboot//protoc_gen_mypy_plugin:protoc-gen-mypy", diff --git a/reboot/cli/commands/BUILD.bazel b/reboot/cli/commands/BUILD.bazel index 6f059898..763242c8 100644 --- a/reboot/cli/commands/BUILD.bazel +++ b/reboot/cli/commands/BUILD.bazel @@ -1,6 +1,27 @@ load("@rbt_pypi//:requirements.bzl", "requirement") load("@rules_python//python:defs.bzl", "py_library") +py_library( + name = "dashboard_py", + srcs = ["dashboard.py"], + srcs_version = "PY3", + visibility = ["//visibility:public"], + deps = [ + ":dev_py", + "//reboot:version_py", + "//reboot/cli/common:directories_py", + "//reboot/cli/common:rc_py", + "//reboot/cli/common:subprocesses_py", + "//reboot/cli/common:terminal_py", + "//reboot/dashboard:constants_py", + # The application this command spawns as a subprocess, so + # that it is importable from the CLI's own interpreter. + "//reboot/dashboard:main_py", + "@com_github_reboot_dev_reboot//reboot:settings_py", + "@com_github_reboot_dev_reboot//reboot/aio/backoff:python", + ], +) + py_library( name = "dev_py", srcs = ["dev.py"], @@ -12,6 +33,10 @@ py_library( requirement("cryptography"), requirement("python-dotenv"), ":generate_py", + "//rbt/dashboard/v1:dashboard_py_reboot", + "//rbt/std/presence/v1:presence_py_reboot", + "//reboot/aio:aborted_py", + "//reboot/aio:external_py", "//reboot/cli/common:directories_py", "//reboot/cli/common:frontend_py", "//reboot/cli/common:monkeys_py", @@ -20,6 +45,7 @@ py_library( "//reboot/cli/common:transpile_py", "//reboot/cli/common:watch_py", "//reboot/controller:plan_makers_py", + "//reboot/dashboard:constants_py", "//reboot/server:local_envoy_factory_py", "@com_github_reboot_dev_reboot//reboot:settings_py", "@com_github_reboot_dev_reboot//reboot/aio:exceptions_py", diff --git a/reboot/cli/commands/dashboard.py b/reboot/cli/commands/dashboard.py new file mode 100644 index 00000000..35f51ea0 --- /dev/null +++ b/reboot/cli/commands/dashboard.py @@ -0,0 +1,245 @@ +"""The `rbt dashboard` command, which runs the developer dashboard.""" +import argparse +import asyncio +import os +import secrets +import shutil +import sys +from pathlib import Path +from reboot.aio.backoff import Backoff +from reboot.cli.commands.dev import ( + check_local_envoy_mode, + try_and_become_child_subreaper_on_linux, +) +from reboot.cli.common import terminal +from reboot.cli.common.directories import ( + add_working_directory_options, + dot_rbt_directory, + use_working_directory, +) +from reboot.cli.common.rc import ArgumentParser +from reboot.cli.common.subprocesses import Subprocesses +from reboot.dashboard.constants import ( + DASHBOARD_PATH, + DEFAULT_DASHBOARD_PORT, + ENVVAR_RBT_API_DIRECTORY, +) +from reboot.settings import ( + ENVVAR_RBT_DEV, + ENVVAR_RBT_FRONTEND_DIST_PATH, + ENVVAR_RBT_FRONTEND_HOST, + ENVVAR_RBT_FRONTEND_ROOT_PATH, + ENVVAR_RBT_NAME, + ENVVAR_RBT_NODEJS, + ENVVAR_RBT_SERVERS, + ENVVAR_RBT_STATE_DIRECTORY, + ENVVAR_REBOOT_CRYPTO_ROOT_KEYS, + ENVVAR_REBOOT_EXPECTED_VERSION, + ENVVAR_REBOOT_LOCAL_ENVOY, + ENVVAR_REBOOT_LOCAL_ENVOY_PORT, + ENVVAR_REBOOT_OAUTH_SIGNING_SECRET, +) +from reboot.version import REBOOT_VERSION +from typing import Optional + +# The dashboard application's name, which names its state directory +# under `.rbt/`. A sibling of `.rbt/dev/` rather than inside it, so +# that it can never collide with a developer's application, whose +# state lives at `.rbt/dev//`. +DASHBOARD_STATE_DIRECTORY_NAME = 'dashboard' + + +def dashboard_subcommands() -> list[str]: + return ['dashboard'] + + +def register_dashboard(parser: ArgumentParser): + add_working_directory_options(parser.subcommand('dashboard')) + + parser.subcommand('dashboard').add_argument( + '--api-directory', + type=str, + required=True, + help='directory containing the API files the dashboard watches', + ) + + parser.subcommand('dashboard').add_argument( + '--port', + type=int, + help='port on which the dashboard will serve traffic; defaults to ' + f'{DEFAULT_DASHBOARD_PORT}', + ) + + +def _dashboard_env( + args, + parser: ArgumentParser, + *, + port: int, + api_directory: str, +) -> dict[str, str]: + """The environment for the dashboard application. + + Built from the ambient environment rather than from any + application environment, so that nothing naming a developer's + application, such as its name, state directory, port, launcher or + frontend, reaches an application that shares none of it. + """ + composed = os.environ.copy() + + # Every other application-flavored variable is overwritten below; + # these four have no dashboard value to overwrite them with, so a + # developer's shell export would leak through and make the + # dashboard a Node.js application or serve their frontend. + for name in ( + ENVVAR_RBT_NODEJS, + ENVVAR_RBT_FRONTEND_HOST, + ENVVAR_RBT_FRONTEND_DIST_PATH, + ENVVAR_RBT_FRONTEND_ROOT_PATH, + ): + composed.pop(name, None) + + composed[ENVVAR_RBT_DEV] = 'true' + composed[ENVVAR_REBOOT_EXPECTED_VERSION] = REBOOT_VERSION + composed[ENVVAR_REBOOT_LOCAL_ENVOY] = 'true' + composed[ENVVAR_REBOOT_LOCAL_ENVOY_PORT] = str(port) + + # A single server, so that a subscriber's `Connect` and + # `Toggle` always land on the same process; presence tracks its + # connections in memory there. `ENVVAR_REBOOT_LOCAL_ENVOY` is + # set above because one server otherwise turns Envoy off, and + # the browser has to reach this application. + composed[ENVVAR_RBT_SERVERS] = '1' + + # Where the developer's API files are, which the dashboard can + # read whether or not anything is running. Passed the way the + # developer spelled it, so that a file can be shown as + # `api/bank/v1/account.py`; the dashboard runs in this working + # directory, where that spelling resolves. + composed[ENVVAR_RBT_API_DIRECTORY] = api_directory + + composed[ENVVAR_RBT_NAME] = DASHBOARD_STATE_DIRECTORY_NAME + + state_directory = ( + dot_rbt_directory(args, parser) / DASHBOARD_STATE_DIRECTORY_NAME + ) + composed[ENVVAR_RBT_STATE_DIRECTORY] = str(state_directory) + + root_keys_path = state_directory / 'crypto-root-keys' + if root_keys_path.exists(): + composed[ENVVAR_REBOOT_CRYPTO_ROOT_KEYS] = root_keys_path.read_text() + else: + root_keys = f'v1:{secrets.token_urlsafe(32)}' + root_keys_path.parent.mkdir(parents=True, exist_ok=True) + root_keys_path.write_text(root_keys) + composed[ENVVAR_REBOOT_CRYPTO_ROOT_KEYS] = root_keys + + composed[ENVVAR_REBOOT_OAUTH_SIGNING_SECRET] = composed[ + ENVVAR_REBOOT_CRYPTO_ROOT_KEYS] + + return composed + + +async def _run_dashboard( + *, + env: dict[str, str], + state_directory: Path, + subprocesses: Subprocesses, +) -> None: + """Runs the dashboard application, restarting it if it exits. + + The dashboard's schema changes whenever Reboot's does, so the + expected reason for it to fail at startup is a backwards + incompatibility after an upgrade. Its state is ours and is + disposable, so the first failure deletes it and tries again + without asking. A second failure is something else, and gets + reported once rather than silently retried forever. + """ + backoff = Backoff() + failures = 0 + reported = False + + while True: + async with subprocesses.exec( + sys.executable, + '-m', + 'reboot.dashboard.main', + env=env, + # The application's own startup output would drown the one + # line this command prints; anything it writes to stderr + # still reaches the terminal. + stdout=asyncio.subprocess.DEVNULL, + ) as process: + await process.wait() + failed = process.returncode != 0 + + if not failed: + failures = 0 + else: + failures += 1 + + if failures == 1: + await asyncio.to_thread( + shutil.rmtree, state_directory, ignore_errors=True + ) + elif not reported: + reported = True + terminal.warn( + 'The dashboard application keeps failing to start; ' + 'still trying.' + ) + + await backoff() + + +async def dashboard( + args, + parser: ArgumentParser, +) -> int: + """Implementation of the 'dashboard' subcommand.""" + with use_working_directory(args, parser): + # If on Linux try to become a child subreaper so that we can + # properly clean up all processes descendant from us! Envoy in + # particular is a grandchild, and one that outlives the + # application would keep answering on the dashboard's port. + try_and_become_child_subreaper_on_linux() + + subprocesses = Subprocesses() + + # Pick the mode in which we'll run a local Envoy proxy and + # check that the mode is usable, e.g. that Docker is running + # and can access the Envoy proxy image, or that the `envoy` + # executable runs. Fail otherwise. + await check_local_envoy_mode(subprocesses) + + port = args.port or DEFAULT_DASHBOARD_PORT + + env = _dashboard_env( + args, + parser, + port=port, + api_directory=args.api_directory, + ) + + terminal.info( + 'Your dashboard is at ' + f'http://127.0.0.1:{port}{DASHBOARD_PATH}/\n' + ) + + await _run_dashboard( + env=env, + state_directory=Path(env[ENVVAR_RBT_STATE_DIRECTORY]), + subprocesses=subprocesses, + ) + + return 0 + + +async def handle_dashboard_subcommand( + args: argparse.Namespace, + *, + parser: ArgumentParser, +) -> Optional[int]: + if args.subcommand == 'dashboard': + return await dashboard(args, parser) + return None diff --git a/reboot/cli/commands/dev.py b/reboot/cli/commands/dev.py index 00f10e6a..45c1e2fc 100644 --- a/reboot/cli/commands/dev.py +++ b/reboot/cli/commands/dev.py @@ -12,6 +12,7 @@ import sys import termios import tty +import webbrowser from colorama import Fore from cryptography import x509 from cryptography.hazmat.backends import default_backend @@ -23,9 +24,12 @@ OTEL_EXPORTER_OTLP_TRACES_INSECURE, ) from pathlib import Path +from rbt.dashboard.v1.dashboard_rbt import Preferences +from rbt.std.presence.v1.presence_rbt import Presence from reboot.aio.backoff import Backoff from reboot.aio.contexts import EffectValidation from reboot.aio.exceptions import InputError +from reboot.aio.external import ExternalContext from reboot.cli.commands.generate import generate_direct # We import the whole `terminal` module (as opposed to the methods it contains) # to allow us to mock these methods out in tests. @@ -52,6 +56,12 @@ ) from reboot.cli.common.watch import FileWatcher, file_watcher from reboot.controller.plan_makers import validate_num_servers +from reboot.dashboard.constants import ( + DASHBOARD_PATH, + DEFAULT_DASHBOARD_PORT, + PREFERENCES_ID, + PRESENCE_ID, +) from reboot.server.local_envoy_factory import LocalEnvoyFactory from reboot.settings import ( DEFAULT_SECURE_PORT, @@ -233,6 +243,28 @@ def _register_dev_run(parser: ArgumentParser): f'{DEFAULT_LOCAL_ENVOY_PORT}', ) + parser.subcommand('dev run').add_argument( + '--open-dashboard', + type=bool, + # Three states, two of which currently agree: + # '--open-dashboard' opens one; unset and + # '--no-open-dashboard' both leave the browser alone while + # `_AUTO_OPEN_DASHBOARD` is off. Unset regains a meaning of + # its own once that is turned on: open one unless somebody + # is already looking at one, or the banner said not to. + default=None, + help='open a dashboard in your browser once your application ' + 'is serving', + ) + + parser.subcommand('dev run').add_argument( + '--dashboard-port', + type=int, + help='port on which the developer dashboard, started separately ' + f'with `rbt dashboard`, is serving; defaults to ' + f'{DEFAULT_DASHBOARD_PORT}', + ) + parser.subcommand('dev run').add_argument( '--watch', type=str, @@ -420,6 +452,138 @@ async def _run( application_started_event.clear() +async def _viewers(dashboard_url: str) -> list[str]: + """The subscriber ids of everyone looking at a dashboard. + + The dashboard constructs the `Presence` instance, empty, when it + initializes, so there is an answer from the moment it is up. + """ + context = ExternalContext(name="dev-run-open-dashboard", url=dashboard_url) + response = await Presence.ref(PRESENCE_ID).List(context) + return list(response.subscriber_ids) + + +async def _open_on_restart(dashboard_url: str) -> bool: + """Whether the developer still wants a dashboard opened for them. + + The dashboard's banner writes this when they click it, and it + outlives the `rbt dev run` they clicked it in. The dashboard + writes the default when it initializes, so nobody ever clicking + means a dashboard opens. + """ + context = ExternalContext(name="dev-run-open-dashboard", url=dashboard_url) + response = await Preferences.ref(PREFERENCES_ID).Get(context) + return not response.suppress_open_on_restart + + +# Whether `rbt dev run` may open a dashboard nobody asked it for. +# +# False while the dashboard is still being built, so that the only way +# to see one is `--open-dashboard`. Everything that decides when to +# open one by itself, the `Presence` subscribers and the banner's +# choice, is covered by `open_dashboard_tests`; turning auto-open on +# is this constant and nothing else. +_AUTO_OPEN_DASHBOARD = False + + +async def _open_dashboard_once( + *, + dashboard_url: str, + forced: bool, +) -> None: + """Opens a dashboard, unless the developer would rather it didn't. + + They would rather it didn't in two cases. One is that somebody is + already looking at one: the page subscribes to `Presence` for as + long as it is open, so a tab left up from an earlier run keeps a + second one from appearing, and a tab they closed is replaced. The + other is that they clicked the dashboard's "Don't reopen this + dashboard on restart" banner, which is remembered until they click + the banner that undoes it. + + `Presence` learns that a viewer has gone from the cancellation of + the page's `Connect` RPC, and nothing else. A proxy that holds its + server-side socket open after the browser goes away therefore + leaves a viewer listed who is not there, and the effect is that no + dashboard opens, which `--open-dashboard` overrides, as it + overrides the banner. + """ + if not forced: + if not await _open_on_restart(dashboard_url): + terminal.info( + 'You asked for this dashboard not to be reopened; run ' + 'with `--open-dashboard` to see it anyway, or visit ' + f'{dashboard_url}{DASHBOARD_PATH}/' + ) + return + + if len(await _viewers(dashboard_url)) > 0: + # Say why nothing opened, since a run that opens nothing + # and explains nothing is indistinguishable from a broken + # one, particularly when the tab being counted is behind + # another window, or on another screen. + terminal.info( + 'A dashboard is already open for this application; run ' + 'with `--open-dashboard` for another, or visit ' + f'{dashboard_url}{DASHBOARD_PATH}/' + ) + return + + # `webbrowser` honors `$BROWSER`, which is what makes this work in + # Codespaces and devcontainers, and returns `False` rather than + # raising when there is no browser to open. + page_url = f'{dashboard_url}{DASHBOARD_PATH}/' + + if not await asyncio.to_thread(webbrowser.open, page_url): + terminal.warn( + f"Could not open a browser; your dashboard is at {page_url}" + ) + + +async def _dashboard_reachable(port: int) -> bool: + """Whether something is accepting connections on the dashboard's + port.""" + try: + _, writer = await asyncio.wait_for( + asyncio.open_connection('127.0.0.1', port), + timeout=2.0, + ) + except (OSError, asyncio.TimeoutError): + return False + writer.close() + try: + await writer.wait_closed() + except OSError: + pass + return True + + +async def _open_dashboard( + *, + dashboard_url: str, + forced: bool, + application_serving_event: asyncio.Event, +) -> None: + """Opens a dashboard once the developer's application is serving + traffic, so that nothing the developer waits on is ever waiting on + this.""" + await application_serving_event.wait() + + try: + await _open_dashboard_once( + dashboard_url=dashboard_url, + forced=forced, + ) + except Exception as e: + # Never let this take down `rbt dev run`; the developer's + # application is unaffected and the dashboard is still + # reachable by hand. + terminal.warn( + f"Could not open a dashboard ({e}); it is at " + f"{dashboard_url}{DASHBOARD_PATH}/" + ) + + def try_and_become_child_subreaper_on_linux(): if sys.platform == 'linux': # The 'pyprctl' module is available on Linux only. @@ -542,6 +706,7 @@ async def _check_local_envoy_status( port: int, terminate_after_health_check: bool, application_started_event: asyncio.Event, + application_serving_event: Optional[asyncio.Event], tls_certificate: Optional[str], root_certificate: Optional[str], tracing: Tracing, @@ -650,6 +815,9 @@ def create_channel( was_application_serving = is_application_serving if is_application_serving: + if application_serving_event is not None: + application_serving_event.set() + terminal.info("Application is serving traffic ...\n") # MCP server and endpoint is not supported for Nodejs currently. mcp_line = ( @@ -1358,12 +1526,32 @@ async def __dev_run( await check_local_envoy_mode(subprocesses) env[ENVVAR_REBOOT_LOCAL_ENVOY] = 'true' + # The developer dashboard runs separately, started by + # `rbt dashboard`; this run only decides whether to open a window + # on it. + dashboard_port = args.dashboard_port or DEFAULT_DASHBOARD_PORT + + open_dashboard = ( + args.open_dashboard is True or + (_AUTO_OPEN_DASHBOARD and args.open_dashboard is not False) + ) + + if open_dashboard and not await _dashboard_reachable(dashboard_port): + terminal.fail( + 'You asked for a dashboard, but no developer dashboard is ' + f'serving on port {dashboard_port}. Start one with ' + '`rbt dashboard`, then run this again.' + ) + + application_serving_event = (asyncio.Event() if open_dashboard else None) + health_check_task = asyncio.create_task( _check_local_envoy_status( port=args.port or DEFAULT_LOCAL_ENVOY_PORT, terminate_after_health_check=args.terminate_after_health_check or False, application_started_event=application_started_event, + application_serving_event=application_serving_event, tls_certificate=args.tls_certificate, root_certificate=args.tls_root_certificate, tracing=tracing, @@ -1419,9 +1607,9 @@ def crypto_root_keys() -> str: protect, so that a JWT signed with them dies with the state it refers to (pushing clients back through the OAuth flow, whose fresh mint re-constructs per-user state): a named application - persists a random value in its state directory — stable across - restarts, deleted by `rbt dev expunge` and by the in-run `x` - expunge — while an anonymous application, whose state doesn't + persists a random value in its state directory, stable across + restarts and deleted by `rbt dev expunge` and by the in-run `x` + expunge, while an anonymous application, whose state doesn't survive a restart, gets fresh random keys on every (re)start. """ if args.application_name is None: @@ -1437,6 +1625,19 @@ def crypto_root_keys() -> str: root_keys_path.write_text(root_keys) return root_keys + if open_dashboard: + assert application_serving_event is not None + background_command_tasks.append( + asyncio.create_task( + _open_dashboard( + dashboard_url=f'http://127.0.0.1:{dashboard_port}', + forced=args.open_dashboard is True, + application_serving_event=application_serving_event, + ), + name=f'_open_dashboard(...) in {__name__}', + ) + ) + if tracing == Tracing.JAEGER: # TODO: dynamic port. See comment in `_run_jaeger()`. env[OTEL_EXPORTER_OTLP_TRACES_ENDPOINT] = "localhost:4317" diff --git a/reboot/cli/common/BUILD.bazel b/reboot/cli/common/BUILD.bazel index de53cec0..c15aae2b 100644 --- a/reboot/cli/common/BUILD.bazel +++ b/reboot/cli/common/BUILD.bazel @@ -145,6 +145,7 @@ py_library( deps = [ ":rc_py", ":update_check_py", + "//reboot/cli/commands:dashboard_py", "//reboot/cli/commands:dev_py", "//reboot/cli/commands:export_import_py", "//reboot/cli/commands:generate_py", diff --git a/reboot/cli/common/cli.py b/reboot/cli/common/cli.py index 5279e3b9..9d4dd7ab 100644 --- a/reboot/cli/common/cli.py +++ b/reboot/cli/common/cli.py @@ -7,6 +7,11 @@ handle_cloud_subcommand, register_cloud, ) +from reboot.cli.commands.dashboard import ( + dashboard_subcommands, + handle_dashboard_subcommand, + register_dashboard, +) from reboot.cli.commands.dev import ( dev_subcommands, handle_dev_subcommand, @@ -71,7 +76,7 @@ def create_parser( program='rbt', filename='.rbtrc', subcommands=( - cloud_subcommands() + dev_subcommands() + + cloud_subcommands() + dashboard_subcommands() + dev_subcommands() + export_and_import_subcommands() + generate_subcommands() + init_subcommands() + inspect_subcommands() + serve_subcommands() + task_subcommands() @@ -83,6 +88,7 @@ def create_parser( add_global_options(parser) register_cloud(parser) + register_dashboard(parser) register_dev(parser) register_export_and_import(parser) register_generate(parser) @@ -124,6 +130,10 @@ async def cli() -> int: if (result := await handle_cloud_subcommand(args)) is not None: return result + elif ( + result := await handle_dashboard_subcommand(args, parser=parser) + ) is not None: + return result elif ( result := await handle_dev_subcommand( args, diff --git a/reboot/dashboard/BUILD.bazel b/reboot/dashboard/BUILD.bazel new file mode 100644 index 00000000..4a7a1d0c --- /dev/null +++ b/reboot/dashboard/BUILD.bazel @@ -0,0 +1,120 @@ +load("@rbt_pypi//:requirements.bzl", "requirement") +load("@rules_python//python:defs.bzl", "py_binary", "py_library") + +py_library( + name = "constants_py", + srcs = ["constants.py"], + srcs_version = "PY3", + visibility = ["//visibility:public"], +) + +py_library( + name = "api_reader_py", + srcs = ["api_reader.py"], + srcs_version = "PY3", + visibility = ["//visibility:public"], + deps = [ + "//rbt/dashboard/v1:dashboard_py_reboot", + "//reboot:api_py", + ], +) + +py_library( + name = "api_watcher_py", + srcs = ["api_watcher.py"], + srcs_version = "PY3", + visibility = ["//visibility:public"], + deps = [ + ":api_reader_py", + ":constants_py", + "//rbt/dashboard/v1:dashboard_py_reboot", + "//reboot/aio:external_py", + "//reboot/cli/common:watch_py", + ], +) + +py_library( + name = "servicers_py", + srcs = ["servicers.py"], + srcs_version = "PY3", + visibility = ["//visibility:public"], + deps = [ + ":api_watcher_py", + ":constants_py", + "//rbt/dashboard/v1:dashboard_py_reboot", + "//reboot/aio:servicers_py", + "//reboot/std/presence/v1:presence_py", + ], +) + +# The page, its stylesheet and its bundle, served as three files by +# the `StaticFiles` mount in `main.py`. Only the script's entry point +# is rewritten: the one in the template, `./src/main.tsx`, is +# what Vite serves in development and does not exist in a built page, +# where `esbuild` produced `dashboard_bundle.js` instead. The +# stylesheet is copied as it is, because `./dashboard.css` resolves +# the same way in both. Every output must be *this* package's, because +# the application serves them from beside its own module rather than +# from a project root, and a `genrule` can only write within its own +# package. +genrule( + name = "dashboard_dist", + srcs = [ + "//reboot/dashboard/frontend:dashboard.css", + "//reboot/dashboard/frontend:dashboard_bundle.js", + "//reboot/dashboard/frontend:index.html", + ], + outs = [ + "dashboard/index.html", + "dashboard/dashboard.css", + "dashboard/dashboard_bundle.js", + ], + cmd = "sed 's|\\./src/main\\.tsx|./dashboard_bundle.js|' " + + "$(location //reboot/dashboard/frontend:index.html) " + + "> $(location dashboard/index.html) && " + + "cp $(location //reboot/dashboard/frontend:dashboard.css) " + + "$(location dashboard/dashboard.css) && " + + "cp $(location //reboot/dashboard/frontend:dashboard_bundle.js) " + + "$(location dashboard/dashboard_bundle.js)", +) + +py_library( + name = "main_py", + srcs = ["main.py"], + data = [ + ":dashboard_dist", + ], + srcs_version = "PY3", + visibility = ["//visibility:public"], + deps = [ + ":api_watcher_py", + ":constants_py", + ":servicers_py", + "//rbt/dashboard/v1:dashboard_py_reboot", + "//reboot/aio:applications_py", + "//reboot/aio:external_py", + "//reboot/aio:headers_py", + "//reboot/aio/backoff:python", + requirement("starlette"), + ], +) + +py_binary( + name = "main", + srcs = ["main.py"], + data = [ + ":dashboard_dist", + ], + main = "main.py", + visibility = ["//visibility:public"], + deps = [ + ":constants_py", + ":servicers_py", + "//rbt/dashboard/v1:dashboard_py_reboot", + "//reboot/aio:applications_py", + "//reboot/aio:external_py", + "//reboot/aio:headers_py", + "//reboot/aio/backoff:python", + requirement("starlette"), + ], +) diff --git a/reboot/dashboard/api_reader.py b/reboot/dashboard/api_reader.py new file mode 100644 index 00000000..dbf55816 --- /dev/null +++ b/reboot/dashboard/api_reader.py @@ -0,0 +1,202 @@ +"""Describes one of the developer's API files. + +Run as a subprocess: + + python -m reboot.dashboard.api_reader \ + + +and it writes a JSON list of `StateTypeInfo` to stdout, or a message +to stderr and a non-zero exit if the file cannot be read. + +A subprocess for two reasons. Reading a Pydantic API means importing +it, so doing it in the dashboard would accumulate stale modules across +edits. And it derives a module path from a relative filename, so it +needs a working directory and `sys.path` that the dashboard should not +adopt. + +The description comes from walking the imported `API` object itself, +so everything is spelled the way its author spelled it: method names +as `names_like_this`, field types as `int` or `Optional[str]`, and +errors by the names of the declared models. +""" +import asyncio +import importlib +import json +import os +import sys +import types +from google.protobuf.json_format import MessageToDict +from rbt.dashboard.v1.dashboard_pb2 import FieldInfo, MethodInfo, StateTypeInfo +from reboot.api import API, MethodModel, Model +from typing import Any, Literal, Optional, Union, get_args, get_origin + + +def _type_string(annotation) -> str: + """The source spelling of `annotation`, e.g. `Optional[str]`.""" + if annotation is type(None): + return 'None' + if annotation is Any: + return 'Any' + + origin = get_origin(annotation) + + if origin is Union or origin is types.UnionType: + arguments = get_args(annotation) + others = [a for a in arguments if a is not type(None)] + spelled = ', '.join(_type_string(a) for a in others) + if len(others) == len(arguments): + return f'Union[{spelled}]' + if len(others) == 1: + return f'Optional[{spelled}]' + return f'Optional[Union[{spelled}]]' + if origin is Literal: + return str(annotation).replace('typing.', '') + if origin is list: + (item,) = get_args(annotation) + return f'list[{_type_string(item)}]' + if origin is dict: + key, value = get_args(annotation) + return f'dict[{_type_string(key)}, {_type_string(value)}]' + if origin is None and isinstance(annotation, type): + return annotation.__name__ + return str(annotation).replace('typing.', '') + + +def _fields_of(model: type[Model]) -> list[FieldInfo]: + return [ + FieldInfo(name=name, type=_type_string(field.annotation)) + for name, field in model.model_fields.items() + ] + + +def _describe_method(method_name: str, spec: MethodModel) -> MethodInfo: + info = MethodInfo( + name=method_name, + kind=spec.kind.value, + factory=spec.factory, + mcp=spec.mcp is not None, + errors=[error.__name__ for error in spec.errors], + ) + + if spec.request is not None: + info.arguments.extend(_fields_of(spec.request)) + if spec.response is not None: + info.returns.extend(_fields_of(spec.response)) + if spec.description is not None: + info.description = spec.description + + return info + + +def describe(api_directory: str, filename: str) -> list[dict]: + """Describes the state types declared in one API file. + + State type names are qualified by the file's directory, the way + the generated code qualifies them: `shop/v1/shop.py` declaring + `Shop` yields `shop.v1.Shop`. + """ + # The path as the developer spelled it, joined before anything + # resolves it away: with `--api-directory=api` the file shows as + # `api/bank/v1/account.py`, the path they would open. + file = os.path.join(api_directory, filename) + + directory = os.path.abspath(api_directory) + os.chdir(directory) + sys.path.insert(0, directory) + + module = importlib.import_module( + filename.rsplit('.py', 1)[0].replace(os.sep, '.') + ) + + api = getattr(module, 'api', None) + if not isinstance(api, API): + # Not every file in the directory declares an API; one + # holding shared code simply has nothing to describe. + return [] + + package = os.path.dirname(filename).replace(os.sep, '.') + + described = [] + for type_name, type_obj in api.get_types().items(): + info = StateTypeInfo( + name=f'{package}.{type_name}', + file=file, + fields=_fields_of(type_obj.state), + ) + if type_obj.description is not None: + info.description = type_obj.description + + for method_name, spec in type_obj.methods.items(): + # A `UI` method has no RPC to call, so there is nothing + # to put in a method row for it. + if isinstance(spec, MethodModel): + info.methods.append(_describe_method(method_name, spec)) + + described.append(MessageToDict(info, preserving_proto_field_name=True)) + + return described + + +async def read( + api_directory: str, + filename: str, +) -> tuple[list[dict], Optional[str]]: + """Describes one API file in a subprocess. + + Returns the state types it declares, and a message when it could + not be read. A half-written file is the normal case while someone + is typing, and is worth showing rather than hiding. + """ + process = await asyncio.create_subprocess_exec( + sys.executable, + '-m', + # Not `__name__`, which is `__main__` when this module is the + # one being run. + 'reboot.dashboard.api_reader', + api_directory, + filename, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + # Reading an API file imports it, and an import writes + # `__pycache__` beside the source, inside the directory + # being watched, so the write is itself a change, and every + # edit costs a second pass over every file. It also leaves + # bytecode in the developer's tree that nothing else put + # there. + env={ + **os.environ, 'PYTHONDONTWRITEBYTECODE': '1' + }, + ) + out, errors = await process.communicate() + + if process.returncode != 0: + return [], errors.decode().strip() + + try: + return json.loads(out), None + except json.JSONDecodeError as e: + return [], f'Could not read the description: {e}' + + +def main() -> int: + if len(sys.argv) != 3: + print(f'usage: {sys.argv[0]} ', file=sys.stderr) + return 2 + + try: + print(json.dumps(describe(sys.argv[1], sys.argv[2]))) + except SystemExit: + # A malformed API can reach `fail()` inside `reboot.api`, + # which raises this after printing why. Being a subprocess, + # that is a message for the dashboard rather than the end of + # it. + return 1 + except Exception as e: + print(f'{type(e).__name__}: {e}', file=sys.stderr) + return 1 + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/reboot/dashboard/api_watcher.py b/reboot/dashboard/api_watcher.py new file mode 100644 index 00000000..7646ce2a --- /dev/null +++ b/reboot/dashboard/api_watcher.py @@ -0,0 +1,190 @@ +"""Watches the developer's API files and updates what they declare. + +The dashboard may start before the application exists. In an agentic +flow the API files are written first, then generated code, then +servicers, then a build, then a running process, so asking the +application would say nothing for minutes. Reading the files says +something immediately, and says more with each file that lands. + +Per file, so that state types appear as they are written rather than +all at once at the end, and so that one file which does not parse, +the normal case while someone is typing, costs only its own types. +""" +from google.protobuf.json_format import ParseDict +from log.log import get_logger +from pathlib import Path +from rbt.dashboard.v1.dashboard_pb2 import StateTypeInfo +from rbt.dashboard.v1.dashboard_rbt import API +from reboot.aio.contexts import WorkflowContext +from reboot.cli.common.watch import file_watcher +from reboot.dashboard.api_reader import read +from reboot.dashboard.constants import API_ID +from typing import Optional +from watchdog.events import FileSystemEvent + +logger = get_logger(__name__) + +# Only Pydantic APIs can be read so far. `.proto` and `.ts` are the +# other two forms `rbt generate` accepts; both are static parses and +# neither is written yet. +API_GLOB = '**/*.py' + +# Suffixes of the files `rbt generate` writes, which it therefore +# skips on the way back in. The same three appear in +# `reboot/cli/commands/generate.py`, which decides what to generate +# from, and in `reboot/cli/commands/dev.py`, which decides what to +# watch. Keep the three lists in step. +GENERATED_SUFFIXES = ('_rbt.py', '_pb2.py', '_pb2_grpc.py') + + +def _files(api_directory: Path) -> list[str]: + """Every candidate API file, relative to `api_directory`. + + Every `.py` the developer wrote, which is the rule `rbt generate` + uses. Whether one of them declares an API is answered by reading + it: an API is a Python object, built when the module executes, so + no amount of looking at the text settles it. A file that declares + none costs one subprocess and describes nothing, and `rbt dev run` + is already importing all of these on every save to regenerate. + """ + return sorted( + str(path.relative_to(api_directory)) + for path in api_directory.glob(API_GLOB) + if not path.name.endswith(GENERATED_SUFFIXES) + ) + + +class _Descriptions: + """What each file last declared, and what went wrong reading it. + + Keyed by file so that a file which stops parsing keeps the types + it last had: blanking the dashboard on every keystroke would make + it unreadable exactly while it is being used. + """ + + def __init__(self) -> None: + self._state_types: dict[str, list[dict]] = {} + self._errors: dict[str, str] = {} + + def update( + self, + filename: str, + state_types: list[dict], + error: Optional[str], + ) -> None: + if error is None: + self._state_types[filename] = state_types + self._errors.pop(filename, None) + else: + self._errors[filename] = error + + def retain(self, filenames: set[str]) -> None: + """Forgets files that are no longer there.""" + for stored in list(self._state_types): + if stored not in filenames: + del self._state_types[stored] + for stored in list(self._errors): + if stored not in filenames: + del self._errors[stored] + + def state_types(self) -> list[StateTypeInfo]: + described = [] + for filename in sorted(self._state_types): + for state_type in self._state_types[filename]: + described.append(ParseDict(state_type, StateTypeInfo())) + return described + + def error(self) -> str: + return '\n'.join( + f'{filename}: {self._errors[filename]}' + for filename in sorted(self._errors) + ) + + +def _event_filenames(event: FileSystemEvent, directory: Path) -> set[str]: + """The filenames an event names, relative to `directory`. + + Both of its paths, because a rename reports where the file went as + well as where it was. A path that is not under the directory is + left out, and an event that names nothing under it is the caller's + signal that it could not place the event at all. + """ + filenames = set() + for path in (event.src_path, event.dest_path): + if not path: + continue + try: + filenames.add(str(Path(path).relative_to(directory))) + except ValueError: + continue + return filenames + + +async def watch(context: WorkflowContext, *, api_directory: str) -> None: + """Updates what the API files declare, for as long as this runs.""" + directory = Path(api_directory).resolve() + descriptions = _Descriptions() + updated: Optional[tuple] = None + + async def update_if_changed(alias: str) -> None: + nonlocal updated + + current = (descriptions.state_types(), descriptions.error()) + if current != updated: + updated = current + # Every write from a workflow needs an identity, and this + # one writes once per file that changed, on every + # iteration. + await API.ref(API_ID).per_iteration(alias).Update( + context, + state_types=descriptions.state_types(), + error=descriptions.error(), + ) + + # Everything, once: the developer may have written the whole API + # before the dashboard started. After this only what changes is + # read again. + previous_listing = set(_files(directory)) + pending = set(previous_listing) + + with file_watcher() as watcher: + async for iteration in context.loop('read what changed'): + # The watch is armed before anything is read, so a save + # made during a read is not missed: it resolves `event` + # rather than arriving while nothing is listening. A watch + # is consumed by one event, so it is re-entered for each, + # the same shape `rbt dev run` uses. + async with watcher.watch( + [API_GLOB], + root_dir=str(directory), + ) as event: + # Updating after each file rather than after the + # batch is what makes the types appear as they are + # written. + for filename in sorted(pending): + state_types, error = await read(api_directory, filename) + descriptions.update(filename, state_types, error) + await update_if_changed(f'read {filename}') + + changed = await event + + # A listing is a glob and no file reads, so it is taken on + # every change: it is what notices a file added or deleted, + # which an event naming one path cannot. + filenames = set(_files(directory)) + event_filenames = _event_filenames(changed, directory) + pending = ( + (filenames - previous_listing) | (event_filenames & filenames) + ) + previous_listing = filenames + + if not event_filenames: + # The glob only matches `.py` under this directory, so + # an event that names nothing under it means its paths + # did not resolve the way this one did. Read every file + # rather than let the page go quietly stale on a + # mismatch this cannot see. + pending = filenames + + descriptions.retain(filenames) + await update_if_changed('retain') diff --git a/reboot/dashboard/constants.py b/reboot/dashboard/constants.py new file mode 100644 index 00000000..5a16d3b9 --- /dev/null +++ b/reboot/dashboard/constants.py @@ -0,0 +1,46 @@ +"""Values shared between the dashboard application and the CLI. + +Kept apart from `main.py` so that reading them does not drag in the +application and everything it serves with. +""" + +# Where the dashboard application serves its page, relative to its +# own address. +DASHBOARD_PATH = '/dashboard' + +# The dashboard application's port. Deliberately not adjacent to +# `rbt dev run`'s default port of 9991: VS Code forwards a port +# upward when the one it wants is already taken on the developer's +# machine, so a second dev container serving on 9991 arrives on +# 9992. A dashboard sitting there could be reached in place of +# somebody else's backend, which half-works and is far more +# confusing than not working at all. 9871 is below 9991 so upward +# forwarding never reaches it, outside the 999x band (9990 k3d and +# WildFly, 9993 ZeroTier, 9997 Splunk), clear of +# 9000/9090/9200/9222/9229, hard to confuse with 9991 when reading +# logs, and outside the Linux ephemeral range. +DEFAULT_DASHBOARD_PORT = 9871 + +# The `API` state holding the shape the developer's API files +# declare, as the dashboard application last read them. +API_ID = 'api' + +# The `Preferences` state holding what the developer has said about +# opening dashboards: the dashboard's banner writes it and `rbt dev +# run` reads it. +PREFERENCES_ID = 'preferences' + +# The directory the developer's API files are in, which +# `rbt dashboard` takes as `--api-directory`. Separate from the +# application's URL because the files are there long before anything +# is serving, and the dashboard is meant to be startable that early. +ENVVAR_RBT_API_DIRECTORY = 'RBT_API_DIRECTORY' + +# The `Presence` state the dashboard page subscribes to, recording who +# is looking at a dashboard right now. `rbt dev run` reads it to decide +# whether to open one. +# +# The page names this, `API_ID` and `PREFERENCES_ID` independently, +# in `frontend/src/constants.ts`, since TypeScript cannot read them +# from here. Keep the two in step. +PRESENCE_ID = 'dashboard' diff --git a/reboot/dashboard/frontend/BUILD.bazel b/reboot/dashboard/frontend/BUILD.bazel new file mode 100644 index 00000000..6e5b62ed --- /dev/null +++ b/reboot/dashboard/frontend/BUILD.bazel @@ -0,0 +1,51 @@ +load("@aspect_rules_esbuild//esbuild:defs.bzl", "esbuild") +load("@aspect_rules_ts//ts:defs.bzl", "ts_config", "ts_project") + +ts_config( + name = "tsconfig", + src = "tsconfig.json", +) + +ts_project( + name = "dashboard_ts", + srcs = [ + "src/constants.ts", + "src/main.tsx", + ], + tsconfig = ":tsconfig", + deps = [ + "//:node_modules/@reboot-dev/reboot-react", + "//:node_modules/@reboot-dev/reboot-std", + "//:node_modules/@reboot-dev/reboot-std-api", + "//:node_modules/@reboot-dev/reboot-std-react", + "//:node_modules/@reboot-dev/reboot-web", + "//:node_modules/react", + "//:node_modules/react-dom", + "//:node_modules/uuid", + "//rbt/dashboard/v1:dashboard_js_reboot_react", + ], +) + +# The `tsconfig` is passed so the bundle uses the automatic JSX +# runtime; without it the bundle crashes with `React is not defined`. +esbuild( + name = "dashboard_bundle", + srcs = [ + ":dashboard_ts", + ], + bazel_sandbox_plugin = False, + entry_point = "src/main.js", + format = "esm", + output = "dashboard_bundle.js", + platform = "browser", + tsconfig = "tsconfig.json", + visibility = ["//reboot/dashboard:__pkg__"], +) + +exports_files( + [ + "dashboard.css", + "index.html", + ], + visibility = ["//reboot/dashboard:__pkg__"], +) diff --git a/reboot/dashboard/frontend/dashboard.css b/reboot/dashboard/frontend/dashboard.css new file mode 100644 index 00000000..f6dfb208 --- /dev/null +++ b/reboot/dashboard/frontend/dashboard.css @@ -0,0 +1,681 @@ +/* Tokens from the Reboot Cloud design system the mockup imports + (`_ds/reboot-cloud-design-system-.../_ds_bundle.css`), as HSL + triplets so they compose with `hsl(var(--token) / alpha)` the + way the source does. Light only: the system has `.dark:` + utilities but the design never opts in. */ +:root { + --background: 47 36% 95%; + --foreground: 211.1 71.7% 22.2%; + --card: 0 0% 100%; + --primary: 211 72% 22%; + --primary-foreground: 355.7 100% 97.3%; + --secondary: 115 49% 76%; + --muted: 240 4.8% 95.9%; + --muted-foreground: 240 3.8% 46.1%; + --accent: 166 47% 61%; + --destructive: 0 84.2% 60.2%; + --border: 240 5.9% 90%; + --radius: 0.5rem; + + /* Values the design uses directly, alongside the tokens. */ + --sidebar: 47 30% 92%; + --border-strong: 240 5.9% 86%; + --border-soft: 240 5.9% 94%; + --surface-sunken: 240 4.8% 97%; + --prose: 211 40% 30%; + --returns: 166 47% 33%; + --errors: 0 62% 45%; +} + +* { + box-sizing: border-box; +} + +html, +body, +#root { + height: 100%; + margin: 0; +} + +body { + font-family: ui-sans-serif, system-ui, sans-serif; + background: hsl(var(--background)); + color: hsl(var(--foreground)); + font-size: 15px; + line-height: 1.5; + -webkit-font-smoothing: antialiased; +} + +a { + color: hsl(var(--primary)); + text-decoration: none; +} +a:hover { + color: hsl(166 47% 41%); +} +::selection { + background: hsl(var(--accent) / 0.35); +} + +main { + max-width: 840px; + margin: 0 auto; + padding: 40px 48px; +} + +.muted { + color: hsl(var(--muted-foreground)); +} + +.error { + color: hsl(var(--errors)); + background: hsl(var(--destructive) / 0.08); + border: 1px solid hsl(var(--destructive) / 0.3); + border-radius: var(--radius); + padding: 12px 14px; + /* The reader subprocess's stderr, so line breaks are meaningful. */ + white-space: pre-wrap; + font-family: ui-monospace, Menlo, monospace; + font-size: 12px; +} + +/* The banner sits above whatever the page is showing, and the + rest of the page takes what is left. `min-height: 0` so the + sidebar and the document pane scroll themselves instead of + growing the page. */ +.app { + display: flex; + flex-direction: column; + height: 100%; +} + +.shell { + display: grid; + grid-template-columns: 250px 1fr; + flex: 1; + min-height: 0; + overflow: hidden; +} + +/* --- Banner --- */ + +.banner { + display: flex; + justify-content: center; + padding: 5px 20px; + background: hsl(var(--card)); + border-bottom: 1px solid hsl(var(--border)); +} + +/* A button, because it acts rather than navigates, dressed as a + link so it asks for as little of the page as it is worth. */ +.banner-link { + padding: 0; + border: none; + background: none; + font: inherit; + font-size: 11.5px; + color: hsl(var(--muted-foreground)); + text-decoration: underline; + text-underline-offset: 2px; + cursor: pointer; +} + +.banner-link:hover { + color: hsl(var(--foreground)); +} + +/* --- Sidebar --- */ + +nav { + background: hsl(var(--sidebar)); + border-right: 1px solid hsl(var(--border)); + overflow-y: auto; + padding: 14px 8px; +} + +.eyebrow { + font-family: ui-monospace, Menlo, monospace; + font-size: 10.5px; + letter-spacing: 0.08em; + text-transform: uppercase; + color: hsl(var(--muted-foreground)); +} + +nav > .eyebrow { + padding: 0 8px 8px; +} + +.namespace-head { + display: grid; + grid-template-columns: 10px 1fr auto; + align-items: center; + gap: 0 7px; + width: 100%; + padding: 6px 10px; + border: none; + border-radius: 5px; + background: transparent; + color: inherit; + font: inherit; + text-align: left; + cursor: pointer; +} + +.namespace-head:hover { + background: hsl(var(--card) / 0.6); +} + +.caret { + font-family: ui-monospace, Menlo, monospace; + font-size: 10px; + color: hsl(240 3.8% 55%); +} + +.namespace-name { + font-family: ui-monospace, Menlo, monospace; + font-size: 11.5px; + font-weight: 600; +} + +nav a { + display: grid; + grid-template-columns: 1fr auto; + align-items: center; + gap: 0 8px; + padding: 5px 10px 5px 34px; + border-radius: 5px; + color: inherit; +} + +nav a:hover { + background: hsl(var(--card) / 0.6); + color: inherit; +} + +.nav-name { + font-weight: 600; + font-size: 12.5px; +} + +/* The counts name what they count, since a namespace's number and a + state type's number sit in the same column and mean different + things. `nowrap` because the sidebar is narrow enough that "12 + state types" would otherwise break across two lines. */ +.nav-count { + font-family: ui-monospace, Menlo, monospace; + font-size: 10px; + color: hsl(var(--muted-foreground)); + white-space: nowrap; +} + +/* --- Document pane --- */ + +.pane { + overflow-y: auto; + background: hsl(var(--background)); +} + +header { + max-width: 840px; + margin: 0 auto; + padding: 40px 48px 28px; +} + +header h1 { + font-size: 34px; + font-weight: 650; + letter-spacing: -0.02em; + margin: 6px 0 0; +} + +.state-type { + max-width: 840px; + margin: 0 auto; + padding: 0 48px 48px; +} + +.state-type > .eyebrow:first-child { + display: block; +} + +.state-type { + border-top: 1px solid hsl(var(--border-strong)); + padding-top: 36px; +} + +.state-type h2 { + font-size: 28px; + font-weight: 650; + letter-spacing: -0.02em; + margin: 6px 0 4px; +} + +.file { + font-family: ui-monospace, Menlo, monospace; + font-size: 11.5px; + color: hsl(var(--muted-foreground)); +} + +.eyebrow.section { + margin: 26px 0 10px; +} + +/* --- Expanding a state type --- */ + +/* The heading and its count on the left, the button pushed to + the right edge. Aligned on the baseline rather than centred, + so the button's label sits on the same line as the 28px + heading beside it. When the row is too narrow to hold both, + the button wraps to its own line and `space-between` has + nothing left to spread. */ +.state-type-head { + display: flex; + align-items: baseline; + justify-content: space-between; + gap: 8px 20px; + flex-wrap: wrap; +} + +.state-type-heading { + display: flex; + align-items: baseline; + gap: 12px; + flex-wrap: wrap; + min-width: 0; +} + +/* Solid while closed and outlined while open, so the one that + has something left to offer is the one that draws the eye. */ +.expand-button { + display: inline-flex; + align-items: center; + flex: none; + gap: 8px; + padding: 9px 18px; + border: 1px solid hsl(var(--primary)); + border-radius: calc(var(--radius) - 2px); + background: hsl(var(--primary)); + color: hsl(var(--primary-foreground)); + box-shadow: 0 1px 2px hsl(240 10% 40% / 0.2); + font: inherit; + font-size: 13px; + font-weight: 600; + cursor: pointer; + user-select: none; +} + +.expand-button:hover { + opacity: 0.88; +} + +.expand-button:active { + transform: translateY(1px); +} + +.state-type.is-expanded .expand-button { + border-color: hsl(240 5.9% 78%); + background: hsl(var(--muted)); + color: hsl(var(--primary)); + box-shadow: none; +} + +.expand-button .caret { + font-family: ui-monospace, Menlo, monospace; + font-size: 11px; +} + +.summary-line { + font-family: ui-monospace, Menlo, monospace; + font-size: 11px; + color: hsl(240 3.8% 55%); +} + +.empty { + border: 1px dashed hsl(var(--border-strong)); + border-radius: calc(var(--radius) - 2px); + padding: 12px 14px; + font-size: 12.5px; + color: hsl(var(--muted-foreground)); + line-height: 1.5; +} + +.fields { + display: flex; + flex-direction: column; + gap: 8px; +} + +.field { + display: flex; + align-items: baseline; + gap: 10px; + border: 1px solid hsl(240 5.9% 92%); + border-radius: calc(var(--radius) - 2px); + padding: 10px 14px; + background: hsl(var(--card) / 0.6); +} + +.field-name { + font-family: ui-monospace, Menlo, monospace; + font-size: 12.5px; + font-weight: 600; +} + +.field-type { + font-family: ui-monospace, Menlo, monospace; + font-size: 11.5px; + color: hsl(var(--muted-foreground)); +} + +.methods { + display: flex; + flex-direction: column; + gap: 14px; +} + +.method { + border: 1px solid hsl(var(--border)); + border-radius: var(--radius); + background: hsl(var(--card)); + overflow: hidden; +} + +.method-head { + padding: 14px 18px 12px; +} + +.method-title { + display: flex; + align-items: center; + gap: 8px; + flex-wrap: wrap; +} + +/* Closed, the methods of one state type share a single set of + columns, so every kind pill starts at the same x and so does every + tag: the eye can run down the list and see which methods write and + which are reachable over MCP without reading any of them. Open, + this all reverts to the flex row above, where each row packs its + pills against its own name and the reader is looking at one method + rather than comparing several. + + `subgrid` passes the columns down from `.methods` through the card + and its head, so the tracks are sized once across every row rather + than per row. Each element is pinned to a column by name, because + `factory` and `mcp` are optional: auto-placement would slide a + method's `mcp` tag into the empty `factory` column and lose the + alignment that is the whole point. + + Behind `@supports` because a browser without subgrid drops that one + declaration and keeps the rest, which would leave the cards as + plain grids whose columns are sized per row: worse than the flex + row it replaced. Chrome has had subgrid since 117; anything older + keeps the flex row and simply does not line up. */ +@supports (grid-template-columns: subgrid) { + .state-type:not(.is-expanded) .methods { + display: grid; + grid-template-columns: max-content max-content 1fr; + /* The 14px is the gap between methods; the columns want the 8px + that `.method-title` uses when it is laid out as flex. */ + row-gap: 14px; + column-gap: 8px; + } + + .state-type:not(.is-expanded) .method, + .state-type:not(.is-expanded) .method-head, + .state-type:not(.is-expanded) .method-title { + display: grid; + grid-column: 1 / -1; + grid-template-columns: subgrid; + align-items: center; + } + + /* A scroll container cannot be a subgrid, so the columns would stop + propagating at the card. `.method` is one only because of the + `overflow: hidden` that clips the detail to the card's rounded + corners, and a closed method has no detail drawn to clip. */ + .state-type:not(.is-expanded) .method { + overflow: visible; + } + + .state-type:not(.is-expanded) .method-detail { + grid-column: 1 / -1; + } + + .state-type:not(.is-expanded) .method-name { + grid-column: 1; + } + + .state-type:not(.is-expanded) .kind { + grid-column: 2; + } + + .state-type:not(.is-expanded) .tags { + grid-column: 3; + } +} + +/* Whatever tags a method has, side by side. Taken out of the + layout when it has none, so an open section has no gap to an + empty box in its title's flex row. */ +.tags { + display: flex; + align-items: center; + gap: 8px; +} + +.tags:empty { + display: none; +} + +/* Two families of pill. A `kind` is what the method does to + state, tinted along a scale from a read that changes nothing + to a workflow that runs over time; every method has exactly + one. A `tag` is something else that is true of the method, + and is solid rather than tinted so that it never reads as a + fifth kind. Both have a 1px border so the two families sit + at the same height beside each other. + + The neutral base below is what a `saga`, or a method whose kind + is unset, gets, since only the four kinds that exist today have a + colour. */ +/* Both pills are one width with the label centred in it, so a + column of them reads as a column rather than as ragged text. + `min-width` rather than `width`: a label longer than any we + have today widens its own pill instead of being clipped. 92px + fits the longest, `transaction`. */ +.kind, +.tag { + display: inline-flex; + align-items: center; + justify-content: center; + min-width: 92px; +} + +.kind { + font-family: ui-monospace, Menlo, monospace; + font-size: 10px; + padding: 2px 8px; + border-radius: 999px; + background: hsl(var(--muted)); + border: 1px solid hsl(240 5.9% 84%); + color: hsl(var(--muted-foreground)); +} + +/* Teal: reads state and changes nothing. */ +.kind-reader { + background: hsl(166 47% 61% / 0.18); + border-color: hsl(166 40% 55% / 0.45); + color: hsl(166 55% 27%); +} + +/* Blue: writes the one state it is called on. */ +.kind-writer { + background: hsl(211 72% 45% / 0.14); + border-color: hsl(211 60% 50% / 0.4); + color: hsl(211 72% 32%); +} + +/* Violet: writes across states, atomically. */ +.kind-transaction { + background: hsl(275 55% 55% / 0.14); + border-color: hsl(275 45% 55% / 0.4); + color: hsl(275 50% 40%); +} + +/* Amber: runs past the call that started it. */ +.kind-workflow { + background: hsl(32 90% 55% / 0.18); + border-color: hsl(32 75% 50% / 0.42); + color: hsl(28 80% 33%); +} + +.tag { + font-family: ui-monospace, Menlo, monospace; + font-size: 9px; + font-weight: 600; + padding: 2px 7px; + border: 1px solid transparent; + border-radius: 999px; + color: hsl(0 0% 100%); +} + +/* Navy: brings the state into existence. */ +.tag-factory { + background: hsl(211 72% 26%); +} + +/* Magenta: reachable by an agent over MCP. */ +.tag-mcp { + background: hsl(318 55% 38%); +} + +/* A pill that carries a definition. The mark is the invitation to + hover; the tooltip is the answer. `cursor: help` says the same + thing the mark does, for whoever reads cursors first. */ +.defined { + position: relative; + cursor: help; +} + +/* Small and translucent so it reads as an aside rather than as part + of the label. */ +.define-mark { + margin-left: 4px; + font-size: 8px; + opacity: 0.55; +} + +/* Above the pill, centred, and inert to the mouse so that moving + toward it never flickers it away. The pill styles it sits in are + undone piece by piece: pills are bold, tight and centred, and a + sentence is none of those. */ +.definition { + position: absolute; + bottom: calc(100% + 8px); + left: 50%; + transform: translateX(-50%); + width: max-content; + max-width: 260px; + padding: 8px 10px; + border-radius: 6px; + background: hsl(var(--primary)); + color: hsl(var(--primary-foreground)); + font-family: ui-sans-serif, system-ui, sans-serif; + font-size: 11.5px; + font-weight: 400; + line-height: 1.45; + letter-spacing: normal; + text-align: left; + text-transform: none; + white-space: normal; + pointer-events: none; + opacity: 0; + visibility: hidden; + transition: opacity 120ms ease; + z-index: 10; +} + +.defined:hover .definition { + opacity: 1; + visibility: visible; +} + +/* The section eyebrow sits at the pane's left edge, and the pane + clips whatever leaves it, so a centred tooltip loses its left + half. Open rightward from the label instead. */ +.eyebrow .definition { + left: 0; + transform: none; +} + +/* The height animation. A grid row interpolates from `0fr` to + `1fr`, which is how the detail can grow to whatever height it + happens to need without anything measuring it first, since there + is no height to read while it is closed. The inner element + owns `overflow: hidden`, because a grid row can only shrink + below its content when the content itself is willing to be + clipped. + + Opacity is on a shorter, later curve than the height so that + text fades in against a box that has already begun to open, + rather than appearing at full strength in a 1px slot. */ +.method-detail { + display: grid; + grid-template-rows: 0fr; + opacity: 0; + transition: grid-template-rows 240ms cubic-bezier(0.32, 0.72, 0, 1), + opacity 120ms ease-out; +} + +.method-detail-inner { + overflow: hidden; +} + +.state-type.is-expanded .method-detail { + grid-template-rows: 1fr; + opacity: 1; + transition: grid-template-rows 240ms cubic-bezier(0.32, 0.72, 0, 1), + opacity 180ms ease-in 60ms; +} + +@media (prefers-reduced-motion: reduce) { + .method-detail, + .state-type.is-expanded .method-detail { + transition-duration: 1ms; + } +} + +.method-description { + font-size: 13px; + line-height: 1.55; + color: hsl(var(--prose)); + margin: 0; + padding: 0 18px 12px; + text-wrap: pretty; +} + +.method-signature { + display: flex; + align-items: baseline; + gap: 12px; + flex-wrap: wrap; + padding: 10px 18px; + background: hsl(var(--surface-sunken)); + border-top: 1px solid hsl(var(--border-soft)); + font-family: ui-monospace, Menlo, monospace; + font-size: 11.5px; + color: hsl(var(--muted-foreground)); +} + +.arrow { + color: hsl(240 3.8% 65%); +} +.returns { + color: hsl(var(--returns)); +} +.errors { + color: hsl(var(--errors)); +} diff --git a/reboot/dashboard/frontend/index.html b/reboot/dashboard/frontend/index.html new file mode 100644 index 00000000..cb6c81d1 --- /dev/null +++ b/reboot/dashboard/frontend/index.html @@ -0,0 +1,18 @@ + + + + + + + Reboot dashboard + + + +
+ + + diff --git a/reboot/dashboard/frontend/src/constants.ts b/reboot/dashboard/frontend/src/constants.ts new file mode 100644 index 00000000..c8f515b8 --- /dev/null +++ b/reboot/dashboard/frontend/src/constants.ts @@ -0,0 +1,5 @@ +// Mirrors `reboot/dashboard/constants.py`, which +// TypeScript cannot read. Keep the two in step. +export const PRESENCE_ID = "dashboard"; +export const API_ID = "api"; +export const PREFERENCES_ID = "preferences"; diff --git a/reboot/dashboard/frontend/src/main.tsx b/reboot/dashboard/frontend/src/main.tsx new file mode 100644 index 00000000..b6fa0ecf --- /dev/null +++ b/reboot/dashboard/frontend/src/main.tsx @@ -0,0 +1,527 @@ +import type { MethodInfo, StateTypeInfo } from "@dashboard/dashboard_pb"; +import { useAPI, usePreferences } from "@dashboard/dashboard_rbt_react"; +import { RebootClientProvider } from "@reboot-dev/reboot-react"; +import { Presence } from "@reboot-dev/reboot-std-react/presence"; +import { + FC, + StrictMode, + useCallback, + useEffect, + useLayoutEffect, + useMemo, + useRef, + useState, +} from "react"; +import { createRoot } from "react-dom/client"; +import { v4 as uuidv4 } from "uuid"; +import { API_ID, PREFERENCES_ID, PRESENCE_ID } from "./constants"; + +// One subscriber per tab, for as long as the tab is open. +const SUBSCRIBER_ID = uuidv4(); + +// What each pill means, for somebody meeting Reboot for the first +// time. A pill whose word is not here, such as a kind this page +// does not know, simply gets no mark and no tooltip. +const DEFINITIONS: Record = { + reader: + "Reads state without changing it, so any number can safely " + + "execute concurrently. A reactive caller keeps receiving fresh " + + "results as the state changes.", + writer: + "Changes this state. Writers on one state run one at a time, " + + "each seeing the result of the one before it.", + transaction: + "Changes state, and can call methods on other states with " + + "every change landing together or none of them landing at all.", + workflow: + "A durable background task. It can loop and wait for as long " + + "as it needs, and after a restart it resumes where it was.", + factory: + "Brings a state into existence: it is called with a new id " + + "rather than on a state that already exists.", + mcp: "Callable by AI agents as a tool, over the Model Context " + "Protocol.", + "state type": + "A durable data type. Each instance, named by an id, has fields " + + "that Reboot persists for you. Methods are the way to read and " + + "change them. You can have as many of these as you want.", +}; + +// A pill, with its definition a hover away when it has one. The +// small mark is what says there is something to hover. +const Pill: FC<{ className: string; label: string; meaning?: string }> = ({ + className, + label, + meaning, +}) => + meaning === undefined ? ( + {label} + ) : ( + + {label} + + + {meaning} + + + ); + +const Kind: FC<{ kind: string }> = ({ kind }) => ( + +); + +// A state type's namespace is its proto package: `bank.v1.Account` +// lives in `bank.v1`, which is the developer's `api/bank/v1/`. +const namespaceOf = (name: string): string => + name.slice(0, name.lastIndexOf(".")); + +const typeNameOf = (name: string): string => + name.slice(name.lastIndexOf(".") + 1); + +// Standard-library types an application uses are real and worth being +// able to inspect, but they aren't what the developer wrote, so they +// start collapsed. +const isStandardLibrary = (namespace: string): boolean => + namespace.startsWith("rbt."); + +const Namespace: FC<{ namespace: string; types: StateTypeInfo[] }> = ({ + namespace, + types, +}) => { + const [open, setOpen] = useState(!isStandardLibrary(namespace)); + + return ( +
+ + {open && ( + + )} +
+ ); +}; + +const Method: FC<{ method: MethodInfo }> = ({ method }) => { + const args = method.arguments + .map((argument) => `${argument.name}: ${argument.type}`) + .join(", "); + + // The response's keys and value types, spelled the way a Python + // reader would write them. + const returns = method.returns + .map((field) => `${field.name}: ${field.type}`) + .join(", "); + + return ( +
+
+
+ {method.name} + {/* The kind first, and always: every method has one, so it + lands in the same place in every row and the eye can run + down the column. The tags after it are the exceptions. */} + + {/* One cell for whichever tags a method has, rather than a + column each: both are optional, so a column each would + make every method with neither hold that width open + as dead space. A method that is both a factory and + an MCP tool draws both, side by side. */} + + {method.factory && ( + + )} + {method.mcp && ( + + )} + +
+
+ {/* What the method's own row grows to show. Kept mounted while + the section is closed, because the animation that opens it + is a CSS transition on this element rather than a mount. */} +
+
+ {method.description !== undefined && ( +

{method.description}

+ )} +
+ + ({args}) →{" "} + + {method.returns.length > 0 ? `{${returns}}` : "None"} + + + {method.errors.length > 0 && ( + raises {method.errors.join(", ")} + )} +
+
+
+
+ ); +}; + +const countOf = (n: number, noun: string): string => + `${n} ${n === 1 ? noun : `${noun}s`}`; + +// Horizontal only, and deliberately. A pill's sideways move, between +// the column it shares while closed and its own row while open, is a +// layout change that CSS cannot transition, so it is animated here: +// measure where each pill was, let the layout happen, animate it from +// there. Its vertical move is not ours to animate. The detail growing +// is what pushes the methods below it down, and that already animates +// over the same 240ms, so translating them as well would move them +// twice and they would appear to fly in from above or below. +// +// `offsetLeft` rather than `getBoundingClientRect()` because it is a +// layout position and ignores transforms: a render that lands while a +// pill is mid-slide reads where it is going rather than where it +// momentarily is, so the next toggle starts from the truth. +const SLIDE_MS = 240; +const SLIDE_EASING = "cubic-bezier(0.32, 0.72, 0, 1)"; + +const useSlidingPills = (expanded: boolean) => { + const section = useRef(null); + const before = useRef(new WeakMap()); + const wasExpanded = useRef(expanded); + + // No dependency list: every render re-measures, so the positions + // this animates from are the ones on screen rather than the ones + // from the last toggle, which a window resize would have moved. + useLayoutEffect(() => { + const pills = section.current?.querySelectorAll(".kind, .tag"); + if (pills === undefined) { + return; + } + + // Only opening or closing moves them; other renders just leave + // fresh measurements behind for the next one that does. + const toggled = wasExpanded.current !== expanded; + wasExpanded.current = expanded; + + const still = window.matchMedia("(prefers-reduced-motion: reduce)").matches; + + // `forEach` rather than `for...of`: a `NodeList`'s iterator is + // typed as `Node`, which has no box to measure, while its + // `forEach` keeps the element type the selector asked for. + pills.forEach((pill) => { + const was = before.current.get(pill); + const now = pill.offsetLeft; + before.current.set(pill, now); + + if (!toggled || still || was === undefined || was === now) { + return; + } + + pill.animate( + [{ transform: `translateX(${was - now}px)` }, { transform: "none" }], + { duration: SLIDE_MS, easing: SLIDE_EASING } + ); + }); + }); + + return section; +}; + +const StateType: FC<{ + stateType: StateTypeInfo; + expanded: boolean; + onToggle: () => void; +}> = ({ stateType, expanded, onToggle }) => { + const section = useSlidingPills(expanded); + + return ( + // Every method's detail opens and closes off this one class, so a + // section is one transition rather than one per method. +
+
+ +
+
+
+

{typeNameOf(stateType.name)}

+ + {countOf(stateType.fields.length, "field")} ·{" "} + {countOf(stateType.methods.length, "method")} + +
+ +
+
{stateType.file}
+ +
state
+ {stateType.fields.length === 0 ? ( +
+ No state fields. The key is the whole state. +
+ ) : ( +
+ {stateType.fields.map((field) => ( +
+ {field.name} + {field.type} +
+ ))} +
+ )} + +
methods
+
+ {stateType.methods.map((method) => ( + + ))} +
+
+ ); +}; + +// Whether `rbt dev run` may open a dashboard by itself, and the one +// click that changes the answer. +const Banner: FC<{ suppressed: boolean; onToggle: () => void }> = ({ + suppressed, + onToggle, +}) => ( +
+ +
+); + +const Overview: FC<{ + isExpanded: (name: string) => boolean; + onToggle: (name: string) => void; +}> = ({ isExpanded, onToggle }) => { + // The dashboard's own state: what it read of the developer's API + // files. Nothing here reaches the application, so the application + // does not have to exist. + const { useGet } = useAPI({ id: API_ID }); + const { response, isLoading } = useGet(); + + // What the developer's API files declare. Those exist before the + // application is generated, built or started, which is why they are + // what this page shows. + const read = response?.stateTypes; + + // Restarting `rbt dashboard` closes this page's connection for a + // few seconds. Keep the last shape that was read so the page stays + // readable across that. + const seen = useRef([]); + + if (read !== undefined && read.length > 0) { + seen.current = read; + } + + const stateTypes: StateTypeInfo[] = read?.length ? read : seen.current; + + // Why the API files could not be read, shown beside the last shape + // that was: a half-written file is the normal case while someone is + // typing, and saying so beats showing nothing. + const error = response?.error ?? ""; + + const namespaces = useMemo(() => { + const byNamespace = new Map(); + for (const stateType of stateTypes) { + const namespace = namespaceOf(stateType.name); + const types = byNamespace.get(namespace); + if (types === undefined) { + byNamespace.set(namespace, [stateType]); + } else { + types.push(stateType); + } + } + // The developer's own namespaces first; the standard library is + // theirs to use but not theirs to read. + return [...byNamespace.entries()] + .map(([namespace, types]) => ({ namespace, types })) + .sort((a, b) => { + const standard = + Number(isStandardLibrary(a.namespace)) - + Number(isStandardLibrary(b.namespace)); + return standard !== 0 + ? standard + : a.namespace.localeCompare(b.namespace); + }); + }, [stateTypes]); + + // Only before anything has ever been read; afterwards the last + // shape is shown instead. + if (isLoading && stateTypes.length === 0) { + return ( +
+

Reboot application

+

Reading your API…

+
+ ); + } + + if (stateTypes.length === 0) { + return ( +
+

Reboot application

+

+ Waiting for your API. Nothing in your API directory declares state + types yet. +

+ {error &&
{error}
} +
+ ); + } + + return ( +
+ +
+
+
application domain
+

+ {stateTypes.length} state types in {namespaces.length}{" "} + {namespaces.length === 1 ? "namespace" : "namespaces"} +

+
+ {error &&
{error}
} + {stateTypes.map((stateType) => ( + onToggle(stateType.name)} + key={stateType.name} + /> + ))} +
+
+ ); +}; + +// Everything the developer has told this dashboard, in one place. +// Both choices are the dashboard application's state rather than +// this page's, so they survive the tab, the hot reload and the +// `rbt dev run` they were made in. +const App: FC = () => { + const { useGet, setSuppressOpenOnRestart, setExpanded } = usePreferences({ + id: PREFERENCES_ID, + }); + const { response } = useGet(); + + // Until the read lands, say what the CLI does when nothing has been + // written, which is the same thing it does on a false field. + const suppressed = response?.suppressOpenOnRestart ?? false; + + const stored = useMemo( + () => new Set(response?.expandedStateTypes ?? []), + [response?.expandedStateTypes] + ); + + // A click that has not yet come back from the application, standing + // in for the read until it does. Without it a section would sit + // still for a whole round trip after being clicked, which reads as + // a dead button rather than as a slow one. + const [clicked, setClicked] = useState(new Map()); + + // Drop each stand-in once the read agrees with it, so that a later + // change from another tab is followed rather than held off forever. + useEffect(() => { + setClicked((clicked) => { + const waiting = new Map( + [...clicked].filter(([name, expanded]) => stored.has(name) !== expanded) + ); + return waiting.size === clicked.size ? clicked : waiting; + }); + }, [stored]); + + const isExpanded = useCallback( + (name: string): boolean => clicked.get(name) ?? stored.has(name), + [clicked, stored] + ); + + const onToggle = useCallback( + (name: string): void => { + const expanded = !isExpanded(name); + setClicked((clicked) => new Map(clicked).set(name, expanded)); + setExpanded({ stateType: name, expanded }); + }, + [isExpanded, setExpanded] + ); + + return ( +
+ + setSuppressOpenOnRestart({ suppressOpenOnRestart: !suppressed }) + } + /> + +
+ ); +}; + +const root = document.getElementById("root"); + +if (root !== null) { + createRoot(root).render( + + {/* No `url`: the page and its presence are served by the same + application, so the client uses this page's origin. */} + + + + + + + ); +} diff --git a/reboot/dashboard/frontend/tsconfig.json b/reboot/dashboard/frontend/tsconfig.json new file mode 100644 index 00000000..3446f50f --- /dev/null +++ b/reboot/dashboard/frontend/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "es2018", + "module": "esnext", + "jsx": "react-jsx", + "moduleResolution": "bundler", + "skipLibCheck": true, + "verbatimModuleSyntax": true, + "baseUrl": ".", + "paths": { + // The generated bindings for the dashboard's own state, which + // `dashboard_js_reboot_react` emits beside its proto. + "@dashboard/*": ["../../../rbt/dashboard/v1/*"] + } + } +} diff --git a/reboot/dashboard/main.py b/reboot/dashboard/main.py new file mode 100644 index 00000000..fdfc9e1b --- /dev/null +++ b/reboot/dashboard/main.py @@ -0,0 +1,84 @@ +"""The developer dashboard application. + +A Reboot application owned by the framework, holding the state the +dashboard needs but which must not be written into the +application under development, and serving the dashboard's page. It +is not part of the Reboot API and nothing imports it; it runs as its +own process, with its own state store, alongside the application +being developed. +""" +import asyncio +from pathlib import Path +from rbt.dashboard.v1.dashboard_rbt import API, Preferences +from rbt.std.presence.v1.presence_rbt import Presence +from reboot.aio.applications import Application +from reboot.aio.external import InitializeContext +from reboot.dashboard.constants import ( + API_ID, + DASHBOARD_PATH, + PREFERENCES_ID, + PRESENCE_ID, +) +from reboot.dashboard.servicers import servicers +from starlette.staticfiles import StaticFiles + +# The built page, beside this module, which is the same arrangement +# `InspectServicer` uses for its own assets. Mounting it directly +# avoids `RBT_FRONTEND_DIST_PATH`, which resolves against a project +# root discovered by walking up from a servicer's file; the servicers +# here come from `reboot.std.presence`, so no such root exists above +# them. +_DASHBOARD_DIRECTORY = Path(__file__).parent / 'dashboard' + + +def application() -> Application: + """The dashboard application, with its page mounted.""" + application = Application( + servicers=servicers(), + initialize=initialize, + ) + + application.http.mount( + DASHBOARD_PATH, + app=StaticFiles( + directory=str(_DASHBOARD_DIRECTORY), + # `html=True` so the directory URL serves `index.html`. + # `check_dir=False` so a not-yet-built page doesn't stop + # the application starting. `follow_symlink=True` because + # under Bazel runfiles the built page is a symlink into + # `bazel-out`, which Starlette's default `realpath` check + # rejects as escaping the served directory. + html=True, + check_dir=False, + follow_symlink=True, + ), + ) + + return application + + +async def initialize(context: InitializeContext) -> None: + """Gives `Preferences` the answer somebody who has never clicked + its banner should get.""" + await Preferences.ref(PREFERENCES_ID).SetSuppressOpenOnRestart( + context, + suppress_open_on_restart=False, + ) + + # Construct the `Presence` instance, empty, so that a read of + # who is looking at a dashboard has an answer from the moment the + # dashboard is up. + await Presence.ref(PRESENCE_ID).Create(context) + + # Idempotently, so that a restart of a named application finds + # the `Watch` it already spawned rather than starting a second + # watcher. + _ = await API.ref(API_ID).idempotently('watch').spawn().Watch(context) + + +async def main(): + await application().run() + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/reboot/dashboard/servicers.py b/reboot/dashboard/servicers.py new file mode 100644 index 00000000..dec9cd41 --- /dev/null +++ b/reboot/dashboard/servicers.py @@ -0,0 +1,131 @@ +"""Servicers for the developer dashboard application.""" +import os +import reboot.std.presence.v1.presence +from rbt.dashboard.v1.dashboard_pb2 import ( + APIGetRequest, + APIGetResponse, + APIUpdateRequest, + APIUpdateResponse, + PreferencesGetRequest, + PreferencesGetResponse, + PreferencesSetExpandedRequest, + PreferencesSetExpandedResponse, + PreferencesSetSuppressOpenOnRestartRequest, + PreferencesSetSuppressOpenOnRestartResponse, +) +from rbt.dashboard.v1.dashboard_rbt import API, Preferences +from reboot.aio.auth.authorizers import allow +from reboot.aio.contexts import ReaderContext, WorkflowContext, WriterContext +from reboot.aio.servicers import Servicer +from reboot.dashboard.api_watcher import watch +from reboot.dashboard.constants import ENVVAR_RBT_API_DIRECTORY + + +class APIServicer(API.Servicer): + """Holds the shape the developer's API files declare.""" + + def authorizer(self): + return allow() + + async def Get( + self, + context: ReaderContext, + request: APIGetRequest, + ) -> APIGetResponse: + return APIGetResponse( + state_types=self.state.state_types, + error=self.state.error, + ) + + @classmethod + async def Watch( + cls, + context: WorkflowContext, + request: API.WatchRequest, + ) -> API.WatchResponse: + """Reads the developer's API files when they change. + + The directory comes from the environment each time this runs, + so that an `rbt dashboard` restarted against a different one + reads the new directory. Taking it from the request would keep + whichever directory the run that started watching had. + """ + api_directory = os.environ[ENVVAR_RBT_API_DIRECTORY] + + await watch(context, api_directory=api_directory) + + return API.WatchResponse() + + async def Update( + self, + context: WriterContext, + request: APIUpdateRequest, + ) -> APIUpdateResponse: + del self.state.state_types[:] + self.state.state_types.extend(request.state_types) + self.state.error = request.error + return APIUpdateResponse() + + +class PreferencesServicer(Preferences.Servicer): + """Holds what the developer has said about their dashboard. + + Two unrelated choices share one state because both are facts about + this machine's dashboard rather than about the application, and + each has its own writer so that recording one never overwrites the + other. + """ + + def authorizer(self): + return allow() + + async def Get( + self, + context: ReaderContext, + request: PreferencesGetRequest, + ) -> PreferencesGetResponse: + return PreferencesGetResponse( + suppress_open_on_restart=self.state.suppress_open_on_restart, + expanded_state_types=self.state.expanded_state_types, + ) + + async def SetSuppressOpenOnRestart( + self, + context: WriterContext, + request: PreferencesSetSuppressOpenOnRestartRequest, + ) -> PreferencesSetSuppressOpenOnRestartResponse: + self.state.suppress_open_on_restart = request.suppress_open_on_restart + return PreferencesSetSuppressOpenOnRestartResponse() + + async def SetExpanded( + self, + context: WriterContext, + request: PreferencesSetExpandedRequest, + ) -> PreferencesSetExpandedResponse: + expanded = set(self.state.expanded_state_types) + + if request.expanded: + expanded.add(request.state_type) + else: + expanded.discard(request.state_type) + + self.state.expanded_state_types[:] = sorted(expanded) + + return PreferencesSetExpandedResponse() + + +def servicers() -> list[type[Servicer]]: + """The servicers that back the dashboard's own state. + + This state belongs to the dashboard rather than to the application + being developed, so it lives in its own application and its own + state store. + + This is a library rather than something built into the application + below it, so that what these servicers are stays separate from what + ends up hosting them. + """ + return [ + APIServicer, + PreferencesServicer, + ] + reboot.std.presence.v1.presence.servicers() diff --git a/reboot/inspect/BUILD.bazel b/reboot/inspect/BUILD.bazel index 007e4e00..5083da6d 100644 --- a/reboot/inspect/BUILD.bazel +++ b/reboot/inspect/BUILD.bazel @@ -49,6 +49,19 @@ esbuild( platform = "browser", ) +py_library( + name = "describe_state_type_py", + srcs = ["describe_state_type.py"], + srcs_version = "PY3", + visibility = ["//visibility:public"], + deps = [ + "//log:log_py", + "//rbt/v1alpha1:options_py_proto", + "//rbt/v1alpha1/inspect:inspect_py_proto", + "//reboot/aio:types_py", + ], +) + py_library( name = "servicer_py", srcs = ["servicer.py"], diff --git a/reboot/inspect/describe_state_type.py b/reboot/inspect/describe_state_type.py new file mode 100644 index 00000000..62bbe43e --- /dev/null +++ b/reboot/inspect/describe_state_type.py @@ -0,0 +1,141 @@ +"""Describes Reboot state types from protobuf descriptors. + +Everything comes from the descriptor pool it is handed: a state type's +fields, and the methods of the services that supply them. Field and +method types are rendered as the Python names a person reads on a +page, such as `int` and `list[str]`, rather than as their protobuf +spelling. +""" +from google.protobuf import descriptor_pool +from google.protobuf.descriptor import FieldDescriptor +from log.log import get_logger +from rbt.v1alpha1 import options_pb2 +from rbt.v1alpha1.inspect.inspect_pb2 import ( + FieldInfo, + MethodInfo, + StateTypeInfo, +) +from reboot.aio.types import StateTypeName +from typing import Iterable, Optional + +logger = get_logger(__name__) + +_TYPE_NAMES = { + FieldDescriptor.TYPE_DOUBLE: 'float', + FieldDescriptor.TYPE_FLOAT: 'float', + FieldDescriptor.TYPE_INT64: 'int', + FieldDescriptor.TYPE_UINT64: 'int', + FieldDescriptor.TYPE_INT32: 'int', + FieldDescriptor.TYPE_FIXED64: 'int', + FieldDescriptor.TYPE_FIXED32: 'int', + FieldDescriptor.TYPE_BOOL: 'bool', + FieldDescriptor.TYPE_STRING: 'str', + FieldDescriptor.TYPE_BYTES: 'bytes', + FieldDescriptor.TYPE_UINT32: 'int', + FieldDescriptor.TYPE_SFIXED32: 'int', + FieldDescriptor.TYPE_SFIXED64: 'int', + FieldDescriptor.TYPE_SINT32: 'int', + FieldDescriptor.TYPE_SINT64: 'int', +} + + +def _type_name(field) -> str: + """How to render `field`'s type.""" + if field.type in ( + FieldDescriptor.TYPE_MESSAGE, FieldDescriptor.TYPE_GROUP + ): + name = field.message_type.name + elif field.type == FieldDescriptor.TYPE_ENUM: + name = field.enum_type.name + else: + name = _TYPE_NAMES.get(field.type, 'unknown') + + if field.label == FieldDescriptor.LABEL_REPEATED: + return f'list[{name}]' + return name + + +def _fields_of(message) -> list[FieldInfo]: + return [ + FieldInfo(name=field.name, type=_type_name(field)) + for field in message.fields + ] + + +def _describe_method(method) -> MethodInfo: + options = method.GetOptions().Extensions[options_pb2.method] + kind = options.WhichOneof('kind') or '' + + info = MethodInfo( + name=method.name, + kind=kind, + arguments=_fields_of(method.input_type), + errors=list(options.errors), + mcp=options.HasField('mcp'), + ) + + # An empty response means the method returns nothing; saying + # "Empty" would be an implementation detail leaking out. The + # response's fields rather than its name, because a synthesized + # name such as `ShopRemainingResponse` says nothing the fields + # don't. + if method.output_type.full_name != 'google.protobuf.Empty': + info.returns.extend(_fields_of(method.output_type)) + + # An application that was created before `MethodOptions.description` + # will have the deprecated `mcp` description, which is permitted + # for backward compatibility. + if options.description: + info.description = options.description + elif options.HasField('mcp') and options.mcp.description: + info.description = options.mcp.description + + # Only writers and transactions can construct. + if kind in ('writer', 'transaction'): + info.factory = getattr(options, kind).HasField('constructor') + + return info + + +def describe_state_type( + pool: descriptor_pool.DescriptorPool, + state_type_name: StateTypeName, + service_names: Iterable[str], + file: str, +) -> Optional[StateTypeInfo]: + """Describes one state type, or `None` when its descriptors can't + be found, since a state type we can't describe shouldn't stop us + describing the rest. + + `pool` holds the state type's descriptors and those of the + services named in `service_names`, which supply its methods. + + `file` is the file the developer declared the state type in, which + is reported as it is. A Pydantic API is described from a `.proto` + synthesized from it, so the descriptors name a file that only + exists inside the build. + """ + try: + state = pool.FindMessageTypeByName(state_type_name) + except KeyError: + logger.warning( + f"No descriptor for state type '{state_type_name}'; " + "omitting it from the schema" + ) + return None + + info = StateTypeInfo( + name=state_type_name, + file=file, + fields=_fields_of(state), + ) + + for service_name in service_names: + try: + service = pool.FindServiceByName(service_name) + except KeyError: + continue + for method in service.methods: + info.methods.append(_describe_method(method)) + + return info diff --git a/reboot/std/presence/v1/presence.py b/reboot/std/presence/v1/presence.py index 69025d14..86740dc1 100644 --- a/reboot/std/presence/v1/presence.py +++ b/reboot/std/presence/v1/presence.py @@ -19,6 +19,7 @@ WaitForDisconnectRequest, WaitForDisconnectResponse, ) +from rbt.std.presence.v1 import presence_rbt from rbt.std.presence.v1.presence_rbt import ( ListRequest, ListResponse, @@ -67,6 +68,14 @@ class PresenceServicer(Presence.singleton.Servicer): def authorizer(self): return allow() + async def Create( + self, + context: WriterContext, + state: Presence.State, + request: presence_rbt.CreateRequest, + ) -> presence_rbt.CreateResponse: + return presence_rbt.CreateResponse() + async def Subscribe( self, context: WriterContext, diff --git a/tests/reboot/cli/BUILD.bazel b/tests/reboot/cli/BUILD.bazel index 0c03443d..fcbac08f 100644 --- a/tests/reboot/cli/BUILD.bazel +++ b/tests/reboot/cli/BUILD.bazel @@ -33,6 +33,20 @@ py_test( ], ) +py_test( + name = "dashboard_tests_py", + srcs = [ + "dashboard_tests.py", + ], + main = "dashboard_tests.py", + deps = [ + ":mock_exit_py", + "//reboot/cli/common:cli_py", + "//reboot/cli/common:rc_py", + "//reboot/dashboard:constants_py", + ], +) + py_test( name = "dev_tests_py", srcs = [ diff --git a/tests/reboot/cli/dashboard_tests.py b/tests/reboot/cli/dashboard_tests.py new file mode 100644 index 00000000..f5bea4ff --- /dev/null +++ b/tests/reboot/cli/dashboard_tests.py @@ -0,0 +1,127 @@ +import os +import tempfile +import unittest +from reboot.cli.commands import dashboard +from reboot.cli.common import cli +from reboot.cli.common.directories import dot_rbt_directory +from reboot.cli.common.rc import ArgumentParser +from reboot.dashboard.constants import DEFAULT_DASHBOARD_PORT +from tests.reboot.cli.mock_exit import ( + MockExitException, + mock_raise_instead_of_exit, +) +from unittest.mock import patch + + +@patch('argparse.ArgumentParser.exit', mock_raise_instead_of_exit) +class RbtDashboardTestCase(unittest.IsolatedAsyncioTestCase): + + def _parse(self, state_directory: str): + parser: ArgumentParser = cli.create_parser( + argv=[ + 'rbt', + f'--state-directory={state_directory}', + 'dashboard', + '--api-directory=api', + ] + ) + args, _ = parser.parse_args() + return args, parser + + async def test_api_directory_is_required(self) -> None: + parser: ArgumentParser = cli.create_parser(argv=['rbt', 'dashboard']) + with self.assertRaises(MockExitException): + parser.parse_args() + + async def test_env_is_isolated_from_any_application(self) -> None: + with tempfile.TemporaryDirectory() as state_directory: + args, parser = self._parse(state_directory) + + # Values naming a developer's application must not survive + # into the dashboard's environment; if any did, the + # dashboard would collide with their state directory or + # port. + with patch.dict( + os.environ, + { + 'RBT_NAME': 'app', + 'RBT_STATE_DIRECTORY': '/somewhere/app', + 'RBT_NODEJS': 'true', + 'REBOOT_LOCAL_ENVOY_PORT': '9991', + }, + ): + env = dashboard._dashboard_env( + args, + parser, + port=DEFAULT_DASHBOARD_PORT, + api_directory=args.api_directory, + ) + + self.assertEqual(env['RBT_NAME'], 'dashboard') + self.assertNotIn('RBT_NODEJS', env) + self.assertEqual( + env['REBOOT_LOCAL_ENVOY_PORT'], + str(DEFAULT_DASHBOARD_PORT), + ) + + # One server, and Envoy explicitly on: one server would + # otherwise turn Envoy off, and the browser has to reach + # the dashboard. + self.assertEqual(env['RBT_SERVERS'], '1') + self.assertEqual(env['REBOOT_LOCAL_ENVOY'], 'true') + + # A sibling of `.rbt/dev/`, so that it can never collide + # with an application's state at `.rbt/dev//`. + self.assertEqual( + env['RBT_STATE_DIRECTORY'], + str(dot_rbt_directory(args, parser) / 'dashboard'), + ) + + async def test_keys_differ_from_any_application(self) -> None: + with tempfile.TemporaryDirectory() as state_directory: + args, parser = self._parse(state_directory) + + with patch.dict( + os.environ, {'REBOOT_CRYPTO_ROOT_KEYS': 'v1:theirs'} + ): + env = dashboard._dashboard_env( + args, + parser, + port=DEFAULT_DASHBOARD_PORT, + api_directory=args.api_directory, + ) + + self.assertNotEqual(env['REBOOT_CRYPTO_ROOT_KEYS'], 'v1:theirs') + + # Stable across restarts, so tokens the dashboard mints + # stay valid until its state is deleted. + again = dashboard._dashboard_env( + args, + parser, + port=DEFAULT_DASHBOARD_PORT, + api_directory=args.api_directory, + ) + self.assertEqual( + env['REBOOT_CRYPTO_ROOT_KEYS'], + again['REBOOT_CRYPTO_ROOT_KEYS'], + ) + + async def test_is_told_where_the_api_files_are(self) -> None: + with tempfile.TemporaryDirectory() as state_directory: + args, parser = self._parse(state_directory) + + env = dashboard._dashboard_env( + args, + parser, + port=DEFAULT_DASHBOARD_PORT, + api_directory=args.api_directory, + ) + + # As the developer spelled it, so files can be shown as + # `api/bank/v1/account.py`; the dashboard runs in the + # working directory where that spelling resolves. + self.assertEqual(env['RBT_API_DIRECTORY'], 'api') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/reboot/cli/dev_tests.py b/tests/reboot/cli/dev_tests.py index 80785e37..1993842a 100644 --- a/tests/reboot/cli/dev_tests.py +++ b/tests/reboot/cli/dev_tests.py @@ -119,6 +119,22 @@ async def test_environment_variable(self) -> None: [['E1', 'V1'], ['E2', 'V2'], ['E3', 'V3']], ) + async def test_open_dashboard_requires_a_reachable_dashboard(self) -> None: + # `rbt dev run --open-dashboard` refuses to start when nothing + # is serving on the dashboard's port, telling the developer to + # run `rbt dashboard`. + server = await asyncio.start_server( + lambda reader, writer: writer.close(), '127.0.0.1', 0 + ) + port = server.sockets[0].getsockname()[1] + + self.assertTrue(await dev._dashboard_reachable(port)) + + server.close() + await server.wait_closed() + + self.assertFalse(await dev._dashboard_reachable(port)) + async def test_dev_expunge_requires_name(self) -> None: parser: ArgumentParser = cli.create_parser( argv=[ diff --git a/tests/reboot/dashboard/BUILD.bazel b/tests/reboot/dashboard/BUILD.bazel new file mode 100644 index 00000000..c9956e23 --- /dev/null +++ b/tests/reboot/dashboard/BUILD.bazel @@ -0,0 +1,83 @@ +load("@rules_python//python:defs.bzl", "py_test") +load("//tests/reboot/react:py_web_test_suite_env.bzl", "py_web_test_suite_env") + +py_test( + name = "api_reader_tests_py", + srcs = ["api_reader_tests.py"], + data = glob(["api/**"]), + main = "api_reader_tests.py", + deps = [ + "//reboot/dashboard:api_reader_py", + ], +) + +py_test( + name = "api_watcher_tests_py", + srcs = ["api_watcher_tests.py"], + main = "api_watcher_tests.py", + deps = [ + "//reboot/aio:tests_py", + "//reboot/dashboard:api_watcher_py", + "//reboot/dashboard:main_py", + ], +) + +py_test( + name = "application_tests_py", + srcs = [":application_tests.py"], + main = "application_tests.py", + deps = [ + "//reboot/aio:tests_py", + "//reboot/dashboard:servicers_py", + "//reboot/std/presence/v1:presence_py", + ], +) + +py_web_test_suite_env( + name = "dashboard_tests_py", + srcs = ["dashboard_tests.py"], + browsers = [ + "@io_bazel_rules_webtesting//browsers:chromium-local", + ], + main = "dashboard_tests.py", + py_test_tags = [ + "macos_not_supported", + "requires-linux-x86", + ], + tags = [ + "macos_not_supported", + "requires-linux-x86", + ], + deps = [ + "//reboot/aio:tests_py", + "//reboot/dashboard:api_watcher_py", + "//reboot/dashboard:main_py", + "//reboot/std/presence/v1:presence_py", + "@io_bazel_rules_webtesting//testing/web", + ], +) + +py_test( + name = "preferences_tests_py", + srcs = ["preferences_tests.py"], + main = "preferences_tests.py", + deps = [ + "//rbt/dashboard/v1:dashboard_py_reboot", + "//reboot/aio:tests_py", + "//reboot/dashboard:main_py", + ], +) + +py_test( + name = "open_dashboard_tests_py", + srcs = ["open_dashboard_tests.py"], + main = "open_dashboard_tests.py", + deps = [ + "//rbt/dashboard/v1:dashboard_py_reboot", + "//rbt/std/presence/subscriber/v1:subscriber_py_reboot", + "//rbt/std/presence/v1:presence_py_reboot", + "//reboot/aio:tests_py", + "//reboot/cli/commands:dev_py", + "//reboot/dashboard:main_py", + ], +) diff --git a/tests/reboot/dashboard/api/shop/v1/helper.py b/tests/reboot/dashboard/api/shop/v1/helper.py new file mode 100644 index 00000000..0c75c3a7 --- /dev/null +++ b/tests/reboot/dashboard/api/shop/v1/helper.py @@ -0,0 +1,3 @@ +"""Shared code beside an API file, with no `api` of its own.""" + +TAX = 0.1 diff --git a/tests/reboot/dashboard/api/shop/v1/shop.py b/tests/reboot/dashboard/api/shop/v1/shop.py new file mode 100644 index 00000000..f49b264d --- /dev/null +++ b/tests/reboot/dashboard/api/shop/v1/shop.py @@ -0,0 +1,48 @@ +from reboot.api import ( + API, + Field, + Methods, + Model, + Reader, + Tool, + Transaction, + Type, +) + + +class ShopState(Model): + name: str = Field(tag=1) + open: bool = Field(tag=2) + + +class StockRequest(Model): + item: str = Field(tag=1) + quantity: int = Field(tag=2) + + +class StockResponse(Model): + remaining: int = Field(tag=1) + + +class OutOfStockError(Model): + item: str = Field(tag=1) + + +ShopMethods = Methods( + create=Transaction(request=None, response=None, factory=True, mcp=None), + stock=Transaction( + request=StockRequest, + response=None, + description="Add stock of an item.", + mcp=None, + ), + remaining=Reader( + request=StockRequest, + response=StockResponse, + errors=[OutOfStockError], + description="How much of an item is left.", + mcp=Tool(), + ), +) + +api = API(Shop=Type(state=ShopState, methods=ShopMethods)) diff --git a/tests/reboot/dashboard/api_reader_tests.py b/tests/reboot/dashboard/api_reader_tests.py new file mode 100644 index 00000000..b5d3d23e --- /dev/null +++ b/tests/reboot/dashboard/api_reader_tests.py @@ -0,0 +1,127 @@ +"""The dashboard describes an API file without the application. + +This is what lets the dashboard show state types before anything has +been built: `rbt generate` has not run, no servicer exists, and there +is no process to ask. Only the file the developer wrote. +""" +import os +import tempfile +import unittest +from pathlib import Path +from reboot.dashboard.api_reader import read + +API_DIRECTORY = str(Path(__file__).parent / 'api') + + +def _by_name(state_types: list[dict]) -> dict[str, dict]: + return {state_type['name']: state_type for state_type in state_types} + + +def _method(state_type: dict, name: str) -> dict: + for method in state_type['methods']: + if method['name'] == name: + return method + raise AssertionError(f"No method '{name}' in {state_type['name']}") + + +class APIReaderTest(unittest.IsolatedAsyncioTestCase): + + async def test_describes_a_state_type_and_its_methods(self) -> None: + state_types, error = await read(API_DIRECTORY, 'shop/v1/shop.py') + + self.assertIsNone(error) + + shop = _by_name(state_types)['shop.v1.Shop'] + + # The file the developer wrote, spelled from where the + # dashboard was started: the API directory as given, then the + # path inside it. + self.assertEqual( + shop['file'], + os.path.join(API_DIRECTORY, 'shop/v1/shop.py'), + ) + + self.assertEqual( + [field['name'] for field in shop['fields']], + ['name', 'open'], + ) + + # The methods come from the file, with the names and kinds + # their author wrote. + stock = _method(shop, 'stock') + self.assertEqual(stock['kind'], 'transaction') + self.assertEqual( + [argument['name'] for argument in stock['arguments']], + ['item', 'quantity'], + ) + + # `stock` has a `description` and is not an MCP tool: prose + # reaches the page whether or not its author also exposed the + # method to MCP. + self.assertEqual(stock['description'], 'Add stock of an item.') + self.assertNotIn('mcp', stock) + + remaining = _method(shop, 'remaining') + self.assertEqual(remaining['kind'], 'reader') + self.assertEqual( + remaining['returns'], + [{ + 'name': 'remaining', + 'type': 'int', + }], + ) + self.assertTrue(remaining['mcp']) + + # The errors a method declares, by the names of the declared + # models. + self.assertEqual(remaining['errors'], ['OutOfStockError']) + self.assertEqual( + remaining['description'], + 'How much of an item is left.', + ) + + # A factory constructs the state, and returns nothing. + create = _method(shop, 'create') + self.assertTrue(create['factory']) + self.assertNotIn('returns', create) + + async def test_a_file_with_no_api_describes_nothing(self) -> None: + # A directory holds shared code as well as APIs, and reading a + # module that declares no `api` is not an error. + state_types, error = await read(API_DIRECTORY, 'shop/v1/helper.py') + + self.assertIsNone(error) + self.assertEqual(state_types, []) + + async def test_a_file_that_does_not_parse_reports_why(self) -> None: + # Half-written files are the normal case while someone is + # typing. The reader has to survive them and say what is + # wrong, because that message is what the developer needs. + with tempfile.TemporaryDirectory() as directory: + os.makedirs(os.path.join(directory, 'shop', 'v1')) + Path(os.path.join(directory, 'shop', 'v1', 'shop.py') + ).write_text('from reboot.api import API\napi = API(\n') + + state_types, error = await read(directory, 'shop/v1/shop.py') + + self.assertEqual(state_types, []) + assert error is not None + self.assertIn('SyntaxError', error) + + async def test_reading_does_not_write_to_the_developer_s_tree( + self + ) -> None: + # Reading walks the API object in memory and leaves the + # developer's tree exactly as it was. + before = sorted(os.listdir(os.path.join(API_DIRECTORY, 'shop', 'v1'))) + + await read(API_DIRECTORY, 'shop/v1/shop.py') + + self.assertEqual( + before, + sorted(os.listdir(os.path.join(API_DIRECTORY, 'shop', 'v1'))), + ) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/reboot/dashboard/api_watcher_tests.py b/tests/reboot/dashboard/api_watcher_tests.py new file mode 100644 index 00000000..e5fa6b84 --- /dev/null +++ b/tests/reboot/dashboard/api_watcher_tests.py @@ -0,0 +1,111 @@ +"""State types appear as the developer writes their API files. + +The dashboard is up before the application exists, so this is the +first thing a dashboard can show: not what is running, but what has +been written so far. +""" +import asyncio +import os +import tempfile +import unittest +from pathlib import Path +from rbt.dashboard.v1.dashboard_rbt import API +from reboot.aio.tests import Reboot +from reboot.dashboard.constants import API_ID, ENVVAR_RBT_API_DIRECTORY +from reboot.dashboard.main import application +from typing import Optional +from unittest.mock import patch + +SHOP = ''' +from reboot.api import API, Field, Methods, Model, Reader, Type + + +class {state}State(Model): + name: str = Field(tag=1) + + +class LookRequest(Model): + item: str = Field(tag=1) + + +class LookResponse(Model): + found: bool = Field(tag=1) + + +{state}Methods = Methods( + look=Reader( + request=LookRequest, + response=LookResponse, + description=None, + mcp=None, + ), +) + +api = API({state}=Type(state={state}State, methods={state}Methods)) +''' + + +class APIWatcherTest(unittest.IsolatedAsyncioTestCase): + + watcher: Optional[asyncio.Task] = None + + async def asyncSetUp(self) -> None: + # The workflow reads the directory when the application comes + # up, so it has to exist and be named first. + self._directory = tempfile.TemporaryDirectory() + self.directory = Path(self._directory.name) + self._environment = patch.dict( + os.environ, + {ENVVAR_RBT_API_DIRECTORY: str(self.directory)}, + ) + self._environment.start() + + self.rbt = Reboot() + await self.rbt.start() + await self.rbt.up(application(), local_envoy=True) + self.url = f'http://127.0.0.1:{self.rbt.envoy_port()}' + + async def asyncTearDown(self) -> None: + await self.rbt.stop() + self._environment.stop() + self._directory.cleanup() + + def _write(self, directory: Path, name: str, state: str) -> None: + path = directory / 'shop' / 'v1' / f'{name}.py' + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(SHOP.format(state=state)) + + async def _wait_for(self, satisfied): + while True: + context = self.rbt.create_external_context(name=self.id()) + try: + response = await API.ref(API_ID).Get(context) + if satisfied(response): + return response + except Exception: + pass + await asyncio.sleep(0.1) + + async def test_types_appear_as_files_are_written(self) -> None: + # The workflow is already watching: it was scheduled when the + # application came up. + self._write(self.directory, 'shop', 'Shop') + + response = await self._wait_for(lambda api: len(api.state_types) == 1) + self.assertEqual( + [state.name for state in response.state_types], + ['shop.v1.Shop'], + ) + self.assertEqual(response.error, '') + + self._write(self.directory, 'depot', 'Depot') + + response = await self._wait_for(lambda api: len(api.state_types) == 2) + self.assertEqual( + sorted(state.name for state in response.state_types), + ['shop.v1.Depot', 'shop.v1.Shop'], + ) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/reboot/dashboard/application_tests.py b/tests/reboot/dashboard/application_tests.py new file mode 100644 index 00000000..64ca0c13 --- /dev/null +++ b/tests/reboot/dashboard/application_tests.py @@ -0,0 +1,102 @@ +import asyncio +import unittest +from rbt.v1alpha1.errors_pb2 import NotFound, StateNotConstructed +from reboot.aio.applications import Application +from reboot.aio.external import ExternalContext +from reboot.aio.tests import Reboot +from reboot.dashboard.servicers import servicers +from reboot.std.presence.v1.presence import Presence, Subscriber + + +class TestDashboardApplication(unittest.IsolatedAsyncioTestCase): + """Checks that the dashboard application stands up on its own and + that presence works against it, which is the whole reason it + exists.""" + + async def asyncSetUp(self) -> None: + self.rbt = Reboot() + await self.rbt.start() + await self.rbt.up(Application(servicers=servicers())) + + async def asyncTearDown(self) -> None: + await self.rbt.stop() + + async def make_connection( + self, + presence_ref: Presence.WeakReference, + subscriber_ref: Subscriber.WeakReference, + context: ExternalContext, + nonce: str, + ) -> asyncio.Task: + """Connects `subscriber_ref` and subscribes it to `presence_ref`. + + `Toggle` is retried because it races `Connect`, which is what + registers the connection; until that has happened `Toggle` + reports `NotFound`. Returns the task running `Connect`, which + stays pending for as long as the subscriber is present. + """ + await subscriber_ref.idempotently().Create(context) + + connect_failed = False + + async def connect(): + nonlocal connect_failed + try: + await subscriber_ref.Connect(context, nonce=nonce) + except: + connect_failed = True + + connect_task = asyncio.create_task(connect()) + + attempt = 0 + while not connect_failed: + try: + await subscriber_ref.idempotently( + f"Attempt {attempt}", + ).Toggle(context, nonce=nonce) + except Subscriber.ToggleAborted as aborted: + if isinstance(aborted.error, NotFound): + attempt += 1 + continue + raise + + await presence_ref.Subscribe( + context, subscriber_id=subscriber_ref.state_id + ) + break + + return connect_task + + async def test_presence_reports_a_connected_subscriber(self) -> None: + context = self.rbt.create_external_context(name=f"test-{self.id()}") + + presence = Presence.ref("dashboard") + subscriber = Subscriber.ref("a-dashboard-tab") + + # Until a writer constructs the `Presence` state, `List` + # aborts rather than reporting an empty list. The dashboard's + # `initialize` calls `Create` for exactly this reason, so its + # readers always have an instance to read; this application has + # no `initialize`, which is what lets this case be seen. + with self.assertRaises(Presence.ListAborted) as aborted: + await presence.List(context) + self.assertIsInstance(aborted.exception.error, StateNotConstructed) + + connect_task = await self.make_connection( + presence, subscriber, context, nonce="nonce" + ) + + response = await presence.List(context) + self.assertEqual(list(response.subscriber_ids), [subscriber.state_id]) + + # Cancelling `Connect` is what a closing browser tab does, and + # the subscriber must drain back out again. + connect_task.cancel() + + async for response in presence.reactively().List(context): + if list(response.subscriber_ids) == []: + break + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/reboot/dashboard/dashboard_tests.py b/tests/reboot/dashboard/dashboard_tests.py new file mode 100644 index 00000000..67b1f6c7 --- /dev/null +++ b/tests/reboot/dashboard/dashboard_tests.py @@ -0,0 +1,418 @@ +"""The dashboard application serves its page, and that page describes +the application under development. + +It is itself a Reboot application, so it can describe itself: pointing +the page at its own address exercises the whole path (config route, +API read, and rendering) in one process. +""" +import asyncio +import socket +import unittest +from rbt.dashboard.v1.dashboard_pb2 import FieldInfo, MethodInfo, StateTypeInfo +from rbt.dashboard.v1.dashboard_rbt import API, Preferences +from reboot.aio.tests import Reboot +from reboot.dashboard.constants import ( + API_ID, + DASHBOARD_PATH, + PREFERENCES_ID, + PRESENCE_ID, +) +from reboot.dashboard.main import application +from reboot.std.presence.v1.presence import Presence +from selenium.webdriver.common.by import By +from selenium.webdriver.support import expected_conditions +from selenium.webdriver.support.wait import WebDriverWait +from testing.web import webtest + + +def _driver(): + return webtest.new_webdriver_session( + capabilities={ + 'goog:chromeOptions': + { + 'args': + [ + '--headless', + '--no-sandbox', + '--disable-dev-shm-usage', + ], + }, + 'goog:loggingPrefs': { + 'browser': 'ALL', + }, + } + ) + + +class DashboardTest(unittest.IsolatedAsyncioTestCase): + + async def asyncSetUp(self) -> None: + # The page is served by the dashboard application, so the + # test needs its + # address to point a browser at it. + with socket.socket() as probe: + probe.bind(('127.0.0.1', 0)) + port = probe.getsockname()[1] + + self.url = f'http://127.0.0.1:{port}' + + self.rbt = Reboot() + await self.rbt.start() + await self.rbt.up( + application(), + local_envoy=True, + local_envoy_port=port, + ) + + async def asyncTearDown(self) -> None: + await self.rbt.stop() + + async def _wait_for_viewers(self, satisfied, driver=None) -> None: + """Polls until presence satisfies `satisfied`. + + Polling rather than reading reactively because `List` aborts + with `StateNotConstructed` until somebody has subscribed at + least once, which is where a fresh application starts. + + Reports what it sees as it goes, including anything the page + logged. This wait has no deadline, so when it does not finish + the test is killed with its `finally` unrun. Printing only at + the end would mean printing nothing in the one case worth + explaining. + """ + polls = 0 + while True: + context = self.rbt.create_external_context(name=self.id()) + # A fresh reference per context; one cannot be shared. + presence = Presence.ref(PRESENCE_ID) + viewers: list[str] = [] + try: + response = await presence.List(context) + viewers = list(response.subscriber_ids) + if satisfied(viewers): + return + except Presence.ListAborted: + if satisfied([]): + return + + polls += 1 + if polls % 10 == 0: + print(f'##### still waiting, {polls} polls, viewers={viewers}') + if driver is not None: + for entry in await asyncio.to_thread( + driver.get_log, 'browser' + ): + print(f'##### page: {entry}') + text = await asyncio.to_thread( + lambda: driver.find_element(By.TAG_NAME, 'body').text + ) + print(f'##### page text: {text[:300]!r}') + + await asyncio.sleep(0.5) + + async def _record_state_types(self) -> None: + """Puts what an API file would yield into the application. + + The reading of files is covered by `api_reader_tests` and + `api_watcher_tests`. What is left to show here is that the + page renders whatever the application holds, so this writes + that directly, and the test keeps no watcher, no observer + thread and no subprocess alive alongside a browser. + """ + context = self.rbt.create_external_context(name=self.id()) + await API.ref(API_ID).Update( + context, + state_types=[ + StateTypeInfo( + name='shop.v1.Shop', + file='api/shop/v1/shop.py', + fields=[FieldInfo(name='name', type='str')], + methods=[ + MethodInfo( + name='look', + kind='reader', + arguments=[FieldInfo(name='item', type='str')], + returns=[FieldInfo(name='found', type='bool')], + ), + ], + ), + ], + error='', + ) + + def _run(self, body): + driver = _driver() + try: + return body(driver) + finally: + print("##### Browser logs #####") + for entry in driver.get_log('browser'): + print(entry) + print("##### End of browser logs #####") + driver.quit() + + async def test_describes_what_the_api_files_declare(self) -> None: + # Nothing here is generated, built or serving: the page shows + # a state type because a file on disk declares one. + def body(driver): + driver.get(f'{self.url}{DASHBOARD_PATH}/') + WebDriverWait(driver, 60).until( + expected_conditions.presence_of_element_located( + (By.ID, 'shop.v1.Shop') + ) + ) + return driver.page_source + + await self._record_state_types() + + page = await asyncio.to_thread(self._run, body) + + # A method, its kind, and its source file all come from the + # file rather than from anything the page knew in advance, + # spelled the way its author spelled them. + self.assertIn('look', page) + self.assertIn('reader', page) + self.assertIn('shop/v1/shop.py', page) + + # State types are grouped by their proto package, which is the + # directory the developer wrote them in. + self.assertIn('shop.v1', page) + + # The sidebar's two counts sit in the same column and count + # different things, so each says what it counts. The fixture + # declares one of each, which also covers the singular. + self.assertIn('1 state type', page) + self.assertIn('1 method', page) + + async def test_says_why_a_file_could_not_be_read(self) -> None: + # A half-written file is the normal case while someone is + # typing, so the page says what went wrong while keeping the + # shape it last read beside it. + def body(driver): + driver.get(f'{self.url}{DASHBOARD_PATH}/') + WebDriverWait(driver, 60).until( + expected_conditions.presence_of_element_located( + (By.CLASS_NAME, 'error') + ) + ) + return driver.page_source + + context = self.rbt.create_external_context(name=self.id()) + await API.ref(API_ID).Update( + context, + state_types=[ + StateTypeInfo( + name='shop.v1.Shop', + file='api/shop/v1/shop.py', + fields=[FieldInfo(name='name', type='str')], + ), + ], + error='shop.py: SyntaxError: invalid syntax', + ) + + page = await asyncio.to_thread(self._run, body) + + self.assertIn('shop.py: SyntaxError: invalid syntax', page) + + # The error does not blank the page: what was last read is + # still there to work against. + self.assertIn('shop.v1.Shop', page) + + # The two labels the banner's one link shows, which are also the + # two things it does. + _TURN_OFF = "Don't reopen this dashboard on restart" + _TURN_ON = 'Open this dashboard on every restart' + + def _click_the_banner(self, driver, showing: str, becomes: str) -> None: + """Clicks the banner's link once it reads `showing`. + + Waits for `becomes` afterwards rather than returning as soon as + the click lands: the new label comes from the reactive read of + `Preferences`, so seeing it is how the test knows the choice + reached the application and came back. + """ + button = (By.CLASS_NAME, 'banner-link') + + WebDriverWait(driver, 60).until( + expected_conditions.text_to_be_present_in_element(button, showing) + ) + driver.find_element(*button).click() + WebDriverWait(driver, 60).until( + expected_conditions.text_to_be_present_in_element(button, becomes) + ) + + async def _suppress_open_on_restart(self) -> bool: + context = self.rbt.create_external_context(name=self.id()) + response = await Preferences.ref(PREFERENCES_ID).Get(context) + return response.suppress_open_on_restart + + async def test_the_banner_turns_reopening_off(self) -> None: + # What the banner writes is exactly what `rbt dev run` reads + # before deciding whether to open a dashboard, which is what + # `open_dashboard_tests` covers from the other side. + + def body(driver): + driver.get(f'{self.url}{DASHBOARD_PATH}/') + self._click_the_banner( + driver, + showing=self._TURN_OFF, + becomes=self._TURN_ON, + ) + + await asyncio.to_thread(self._run, body) + + self.assertTrue(await self._suppress_open_on_restart()) + + async def test_the_banner_turns_reopening_back_on(self) -> None: + # A developer who clicked once is not stuck with it: the page + # they load next offers the choice the other way round. + + context = self.rbt.create_external_context(name=self.id()) + await Preferences.ref(PREFERENCES_ID).SetSuppressOpenOnRestart( + context, + suppress_open_on_restart=True, + ) + + def body(driver): + driver.get(f'{self.url}{DASHBOARD_PATH}/') + self._click_the_banner( + driver, + showing=self._TURN_ON, + becomes=self._TURN_OFF, + ) + + await asyncio.to_thread(self._run, body) + + self.assertFalse(await self._suppress_open_on_restart()) + + # The two labels a state type's one button shows. + _EXPAND = 'Expand details' + _HIDE = 'Hide details' + + def _click_to_expand(self, driver, showing: str, becomes: str) -> None: + """Clicks a state type's button once it reads `showing`. + + Waits for `becomes` afterwards, which is how the test knows + the click was taken, since the label comes from the same state + the detail's height does. + """ + button = (By.CLASS_NAME, 'expand-button') + + WebDriverWait(driver, 60).until( + expected_conditions.text_to_be_present_in_element(button, showing) + ) + driver.find_element(*button).click() + WebDriverWait(driver, 60).until( + expected_conditions.text_to_be_present_in_element(button, becomes) + ) + + @staticmethod + def _detail_height(driver) -> float: + """How tall the first method's detail is drawn. + + Measured rather than asked of `is_displayed()`, because the + detail stays in the document whether or not its state type is + open: what closing does is collapse the grid row it sits in to + nothing, which is what makes the height animate at all. + """ + return driver.execute_script( + 'const detail = document.querySelector(".method-detail-inner");' + 'return detail === null' + ' ? -1' + ' : detail.getBoundingClientRect().height;' + ) + + def _wait_for_detail(self, driver, opened: bool) -> None: + """Waits out the animation, rather than sleeping its duration.""" + WebDriverWait( + driver, 60 + ).until(lambda driver: (self._detail_height(driver) > 0) == opened) + + async def _expanded_state_types(self) -> list[str]: + context = self.rbt.create_external_context(name=self.id()) + response = await Preferences.ref(PREFERENCES_ID).Get(context) + return list(response.expanded_state_types) + + async def test_expanding_a_state_type_opens_its_method_detail( + self + ) -> None: + await self._record_state_types() + + def body(driver): + driver.get(f'{self.url}{DASHBOARD_PATH}/') + self._click_to_expand( + driver, + showing=self._EXPAND, + becomes=self._HIDE, + ) + self._wait_for_detail(driver, opened=True) + + # The height is transitioned rather than switched. Read + # off the property list rather than by sampling a height + # part-way through, which would be a race against the + # animation this is checking for. + self.assertIn( + 'grid-template-rows', + driver.execute_script( + 'const detail =' + ' document.querySelector(".method-detail");' + 'return getComputedStyle(detail).transitionProperty;' + ), + ) + + await asyncio.to_thread(self._run, body) + + # And the click reached the application, which is what makes + # it + # outlast the tab it was made in. + self.assertEqual(await self._expanded_state_types(), ['shop.v1.Shop']) + + async def test_a_state_type_expanded_earlier_is_open_on_load(self) -> None: + # The state a previous `rbt dev run` left behind, which is the + # whole reason the choice lives in the dashboard application. + await self._record_state_types() + + context = self.rbt.create_external_context(name=self.id()) + await Preferences.ref(PREFERENCES_ID).SetExpanded( + context, + state_type='shop.v1.Shop', + expanded=True, + ) + + def body(driver): + driver.get(f'{self.url}{DASHBOARD_PATH}/') + WebDriverWait(driver, 60).until( + expected_conditions.text_to_be_present_in_element( + (By.CLASS_NAME, 'expand-button'), + self._HIDE, + ) + ) + self._wait_for_detail(driver, opened=True) + + await asyncio.to_thread(self._run, body) + + async def test_the_page_holds_presence(self) -> None: + # `rbt dev run` decides whether to open a dashboard by asking + # who is looking at one, so the page being counted while it is + # up and dropped once it is gone is what that decision rests + # on. + driver = await asyncio.to_thread(_driver) + try: + await asyncio.to_thread(driver.get, f'{self.url}{DASHBOARD_PATH}/') + + # Wait for the viewer to register, rather than assuming a + # page load is enough. + await self._wait_for_viewers( + lambda viewers: viewers != [], + driver=driver, + ) + finally: + await asyncio.to_thread(driver.quit) + + # With the browser gone the viewer must drain, which is what + # makes presence usable as a liveness signal at all. + await self._wait_for_viewers(lambda viewers: viewers == []) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/reboot/dashboard/open_dashboard_tests.py b/tests/reboot/dashboard/open_dashboard_tests.py new file mode 100644 index 00000000..cead0245 --- /dev/null +++ b/tests/reboot/dashboard/open_dashboard_tests.py @@ -0,0 +1,219 @@ +"""`rbt dev run` opens a dashboard when nobody is looking at one. + +The dashboard page subscribes to `Presence` for as long as it is open, +so the question the CLI asks is who is looking right now, which +reopens a dashboard the developer closed and never puts a second tab in +front of one they left up. It asks a second question first: whether the +developer clicked the dashboard's "Don't reopen this dashboard on +restart" banner, which is remembered in `Preferences`. + +The page's own subscription and its banner are exercised in +`dashboard_tests`; here the subscriber and the choice are made +directly, so these tests need no browser. +""" +import asyncio +import unittest +from rbt.dashboard.v1.dashboard_rbt import Preferences +from rbt.std.presence.subscriber.v1.subscriber_rbt import Subscriber +from rbt.std.presence.v1.presence_rbt import Presence +from rbt.v1alpha1.errors_pb2 import NotFound +from reboot.aio.tests import Reboot +from reboot.cli.commands.dev import _open_dashboard_once +from reboot.dashboard.constants import ( + DASHBOARD_PATH, + PREFERENCES_ID, + PRESENCE_ID, +) +from reboot.dashboard.main import application +from unittest.mock import patch + + +class OpenDashboardTest(unittest.IsolatedAsyncioTestCase): + + async def asyncSetUp(self) -> None: + self.rbt = Reboot() + await self.rbt.start() + await self.rbt.up(application(), local_envoy=True) + self.url = f'http://127.0.0.1:{self.rbt.envoy_port()}' + self.dashboard_url = f'{self.url}{DASHBOARD_PATH}/' + self._connections: list[asyncio.Task] = [] + + async def asyncTearDown(self) -> None: + for connection in self._connections: + connection.cancel() + await self.rbt.stop() + + async def _view(self, subscriber_id: str) -> None: + """Subscribes as a page would, and stays subscribed. + + `Connect` never returns, and `Toggle` has to land after it, so + the two run concurrently and `Toggle` is retried until the + connection it depends on exists, the same handshake + `reboot/std/react/presence` performs in the browser. + """ + context = self.rbt.create_external_context(name=self.id()) + subscriber = Subscriber.ref(subscriber_id) + nonce = subscriber_id + + await subscriber.idempotently().Create(context) + + self._connections.append( + asyncio.create_task(subscriber.Connect(context, nonce=nonce)) + ) + + attempt = 0 + while True: + try: + await subscriber.idempotently( + f'Attempt {attempt}', + ).Toggle(context, nonce=nonce) + break + except Subscriber.ToggleAborted as aborted: + if not isinstance(aborted.error, NotFound): + raise + attempt += 1 + + await Presence.ref(PRESENCE_ID).Subscribe( + context, + subscriber_id=subscriber_id, + ) + + async def _suppress_reopening(self, suppress: bool) -> None: + """Makes the choice the dashboard's banner makes.""" + context = self.rbt.create_external_context(name=self.id()) + await Preferences.ref(PREFERENCES_ID).SetSuppressOpenOnRestart( + context, + suppress_open_on_restart=suppress, + ) + + async def _viewers(self) -> list[str]: + context = self.rbt.create_external_context(name=self.id()) + try: + response = await Presence.ref(PRESENCE_ID).List(context) + return list(response.subscriber_ids) + except Presence.ListAborted: + return [] + + async def test_opens_when_nobody_is_looking(self) -> None: + self.assertEqual(await self._viewers(), []) + + with patch('webbrowser.open', return_value=True) as browser: + await _open_dashboard_once(dashboard_url=self.url, forced=False) + + # The browser gets the dashboard's path; `ExternalContext` only + # ever sees the origin, which is all it accepts. + browser.assert_called_once_with(self.dashboard_url) + + async def test_does_not_open_when_somebody_is_looking(self) -> None: + await self._view('a-tab-that-is-open') + self.assertEqual(await self._viewers(), ['a-tab-that-is-open']) + + with patch('webbrowser.open', return_value=True) as browser: + with patch('reboot.cli.common.terminal.info') as told: + await _open_dashboard_once( + dashboard_url=self.url, + forced=False, + ) + + browser.assert_not_called() + + # And it must say so: the tab being counted may be behind + # another window, so a run that opens nothing and explains + # nothing is indistinguishable from a broken one. + told.assert_called_once() + self.assertIn('--open-dashboard', told.call_args.args[0]) + self.assertIn(self.dashboard_url, told.call_args.args[0]) + + async def test_opens_again_once_the_last_viewer_has_gone(self) -> None: + await self._view('a-tab-that-closes') + + for connection in self._connections: + connection.cancel() + self._connections = [] + + # Cancelling `Connect` is the only signal presence has, and it + # reaches the subscriber list by way of `WaitForDisconnect` + # untoggling and `Watch` then dropping the subscriber, so wait + # for the list rather than assuming the cancellation was + # enough. + while await self._viewers() != []: + await asyncio.sleep(0.1) + + with patch('webbrowser.open', return_value=True) as browser: + await _open_dashboard_once(dashboard_url=self.url, forced=False) + + browser.assert_called_once_with(self.dashboard_url) + + async def test_forcing_opens_even_though_somebody_is_looking(self) -> None: + await self._view('a-tab-that-is-open') + + with patch('webbrowser.open', return_value=True) as browser: + await _open_dashboard_once(dashboard_url=self.url, forced=True) + + browser.assert_called_once_with(self.dashboard_url) + + async def test_does_not_open_when_the_developer_asked_it_not_to( + self + ) -> None: + # Nobody is looking at a dashboard, so the only thing keeping + # one from opening is the choice the banner recorded. + await self._suppress_reopening(True) + + with patch('webbrowser.open', return_value=True) as browser: + with patch('reboot.cli.common.terminal.info') as told: + await _open_dashboard_once( + dashboard_url=self.url, + forced=False, + ) + + browser.assert_not_called() + + # And it must say how to get one anyway, since a choice made + # in an earlier `rbt dev run` is not something the developer + # is looking at now. + told.assert_called_once() + self.assertIn('--open-dashboard', told.call_args.args[0]) + self.assertIn(self.dashboard_url, told.call_args.args[0]) + + async def test_forcing_opens_even_though_the_developer_asked_it_not_to( + self + ) -> None: + await self._suppress_reopening(True) + + with patch('webbrowser.open', return_value=True) as browser: + await _open_dashboard_once(dashboard_url=self.url, forced=True) + + browser.assert_called_once_with(self.dashboard_url) + + async def test_opens_again_once_the_developer_has_asked_for_it_back( + self + ) -> None: + # The second banner undoes the first, so a developer who + # clicked once is not stuck with it. + await self._suppress_reopening(True) + await self._suppress_reopening(False) + + with patch('webbrowser.open', return_value=True) as browser: + await _open_dashboard_once(dashboard_url=self.url, forced=False) + + browser.assert_called_once_with(self.dashboard_url) + + async def test_says_where_the_dashboard_is_when_none_could_be_opened( + self + ) -> None: + # `webbrowser.open` returns `False` rather than raising when + # there is nothing to open, which is the headless case. Nothing + # was shown, so the developer is told the address instead. + with patch('webbrowser.open', return_value=False): + with patch('reboot.cli.common.terminal.warn') as warned: + await _open_dashboard_once( + dashboard_url=self.url, + forced=False, + ) + + warned.assert_called_once() + self.assertIn(self.dashboard_url, warned.call_args.args[0]) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/reboot/dashboard/preferences_tests.py b/tests/reboot/dashboard/preferences_tests.py new file mode 100644 index 00000000..48be12fc --- /dev/null +++ b/tests/reboot/dashboard/preferences_tests.py @@ -0,0 +1,166 @@ +"""`Preferences` has a value before anybody has chosen one. + +The dashboard's banner renders from a reactive read of `Preferences`, +and a reader aborts with `StateNotConstructed` until something has +written, so the dashboard application writes the defaults at +startup. That write must not undo a choice the developer already +made, because it runs on every start of `rbt dashboard`, which is +exactly when a click from an earlier run has to survive. + +The banner that does the clicking is exercised in `dashboard_tests`, +and what `rbt dev run` does with the answer in `open_dashboard_tests`. +""" +import unittest +import uuid +from rbt.dashboard.v1.dashboard_rbt import Preferences +from reboot.aio.external import InitializeContext +from reboot.aio.tests import Reboot +from reboot.dashboard.constants import PREFERENCES_ID +from reboot.dashboard.main import application, initialize + + +class PreferencesTest(unittest.IsolatedAsyncioTestCase): + + async def asyncSetUp(self) -> None: + self.rbt = Reboot() + await self.rbt.start() + await self.rbt.up(application(), local_envoy=True) + self.url = f'http://127.0.0.1:{self.rbt.envoy_port()}' + + async def asyncTearDown(self) -> None: + await self.rbt.stop() + + def _initialize_context(self) -> InitializeContext: + """A restart's context, seeded as + `Reboot.create_initialize_context` seeds it.""" + return InitializeContext( + name=self.id(), + url=self.url, + idempotency_seed=uuid.uuid5( + uuid.NAMESPACE_DNS, 'anonymous.rbt.dev' + ), + ) + + async def _get(self) -> bool: + context = self.rbt.create_external_context(name=self.id()) + response = await Preferences.ref(PREFERENCES_ID).Get(context) + return response.suppress_open_on_restart + + async def _set_suppress(self, suppress: bool) -> None: + """Makes the choice the dashboard's banner makes.""" + context = self.rbt.create_external_context(name=self.id()) + await Preferences.ref(PREFERENCES_ID).SetSuppressOpenOnRestart( + context, + suppress_open_on_restart=suppress, + ) + + async def _expanded(self) -> list[str]: + context = self.rbt.create_external_context(name=self.id()) + response = await Preferences.ref(PREFERENCES_ID).Get(context) + return list(response.expanded_state_types) + + async def _set_expanded(self, state_type: str, expanded: bool) -> None: + """Makes the choice a state type's `Expand details` makes.""" + context = self.rbt.create_external_context(name=self.id()) + await Preferences.ref(PREFERENCES_ID).SetExpanded( + context, + state_type=state_type, + expanded=expanded, + ) + + async def test_starting_writes_a_default_that_can_be_read(self) -> None: + # The application's `initialize` constructed `Preferences` + # when it came up; a reader would otherwise abort with + # `StateNotConstructed`, and a page that loaded first would + # have nothing to render its banner from. + # + # False, so that somebody who has never clicked the banner gets + # a dashboard opened for them. + self.assertFalse(await self._get()) + + async def test_constructing_leaves_a_choice_already_made_alone( + self + ) -> None: + await self._set_suppress(True) + + await initialize(self._initialize_context()) + + self.assertTrue(await self._get()) + + async def test_constructing_twice_leaves_a_later_choice_alone( + self + ) -> None: + # The restart case: the dashboard constructs on every + # `rbt dashboard`, and the click it must not undo was made + # after the first of those. + await initialize(self._initialize_context()) + await self._set_suppress(True) + + await initialize(self._initialize_context()) + + self.assertTrue(await self._get()) + + async def test_collapsing_removes_the_state_type(self) -> None: + await self._set_expanded('bank.v1.Account', True) + await self._set_expanded('bank.v1.Bank', True) + + await self._set_expanded('bank.v1.Account', False) + + self.assertEqual(await self._expanded(), ['bank.v1.Bank']) + + async def test_expanding_twice_records_the_state_type_once(self) -> None: + # Two tabs can each send the same click, and a page that + # reconnects can send one it already sent. + await self._set_expanded('bank.v1.Account', True) + await self._set_expanded('bank.v1.Account', True) + + self.assertEqual(await self._expanded(), ['bank.v1.Account']) + + async def test_collapsing_what_was_never_expanded_is_no_error( + self + ) -> None: + await self._set_expanded('bank.v1.Account', False) + + self.assertEqual(await self._expanded(), []) + + async def test_the_order_clicked_in_does_not_change_what_is_stored( + self + ) -> None: + await self._set_expanded('bank.v1.Customer', True) + await self._set_expanded('bank.v1.Account', True) + + # Sorted, so that the reactive read does not push a change to + # every open page when the only difference is the order two + # clicks happened to arrive in. + self.assertEqual( + await self._expanded(), + ['bank.v1.Account', 'bank.v1.Customer'], + ) + + async def test_expanding_leaves_the_reopening_choice_alone(self) -> None: + # The reason there are two writers rather than one that takes + # both fields: a page that expands a state type must not write + # back a stale answer to a question it was not asked. + await self._set_suppress(True) + + await self._set_expanded('bank.v1.Account', True) + + self.assertTrue(await self._get()) + + async def test_the_reopening_choice_leaves_expansions_alone(self) -> None: + await self._set_expanded('bank.v1.Account', True) + + await self._set_suppress(True) + + self.assertEqual(await self._expanded(), ['bank.v1.Account']) + + async def test_constructing_leaves_expansions_alone(self) -> None: + await self._set_expanded('bank.v1.Account', True) + + await initialize(self._initialize_context()) + + self.assertEqual(await self._expanded(), ['bank.v1.Account']) + + +if __name__ == '__main__': + unittest.main() From 62dbe40f94825a31248b229e4fd1ba4316ca0716 Mon Sep 17 00:00:00 2001 From: Riley Scheid Date: Wed, 12 Aug 2026 04:50:45 +0000 Subject: [PATCH 04/31] Allow a state type to have a description A method can say what it does, but a state type is the sum of its state and its methods, and its name alone does not say what it is for. `Type` now takes a description, which the dashboard shows beside the state type's name and file. Co-Authored-By: Claude Opus 5 (1M context) --- rbt/dashboard/v1/dashboard.proto | 3 + rbt/v1alpha1/inspect/inspect.proto | 51 -------- rbt/v1alpha1/options.proto | 6 + reboot/api.py | 3 + reboot/dashboard/frontend/dashboard.css | 26 ++++ reboot/dashboard/frontend/src/main.tsx | 35 ++++- reboot/inspect/BUILD.bazel | 13 -- reboot/inspect/describe_state_type.py | 141 --------------------- reboot/pydantic_schema_to_proto.py | 16 ++- tests/reboot/dashboard/api/shop/v1/shop.py | 8 +- tests/reboot/dashboard/api_reader_tests.py | 5 + 11 files changed, 97 insertions(+), 210 deletions(-) delete mode 100644 reboot/inspect/describe_state_type.py diff --git a/rbt/dashboard/v1/dashboard.proto b/rbt/dashboard/v1/dashboard.proto index 38368499..3c5a8f01 100644 --- a/rbt/dashboard/v1/dashboard.proto +++ b/rbt/dashboard/v1/dashboard.proto @@ -33,6 +33,9 @@ message StateTypeInfo { string file = 2; repeated FieldInfo fields = 3; repeated MethodInfo methods = 4; + + // What the state type does, in the author's own words. + optional string description = 5; } //////////////////////////////////////////////////////////////////////// diff --git a/rbt/v1alpha1/inspect/inspect.proto b/rbt/v1alpha1/inspect/inspect.proto index 478a0e92..5031d26c 100644 --- a/rbt/v1alpha1/inspect/inspect.proto +++ b/rbt/v1alpha1/inspect/inspect.proto @@ -66,57 +66,6 @@ message GetStateResponse { //////////////////////////////////////////////////////////////////////// -// A field of a state, or an argument of a method. -message FieldInfo { - string name = 1; - - // Rendered for a reader, e.g. "str", "float", "AccountState". - string type = 2; -} - -message MethodInfo { - string name = 1; - - // One of "reader", "writer", "transaction" or "workflow". - string kind = 2; - - // The request message's fields, flattened. A method taking no - // request has none. - repeated FieldInfo arguments = 3; - - // The response's fields; empty when the method returns - // nothing. - repeated FieldInfo returns = 4; - - // Names of the error types the method declares it may raise. - repeated string errors = 5; - - // The description its author wrote, when there is one. - optional string description = 6; - - // Whether this method is a factory, constructing the state - // rather than requiring it to already exist. - bool factory = 7; - - // Whether the method is exposed as an MCP tool or resource. - bool mcp = 8; -} - -message StateTypeInfo { - // Fully qualified, e.g. "bank.v1.Account". - string name = 1; - - // The file the developer declared it in, e.g. - // "bank/v1/account.py". - string file = 2; - - repeated FieldInfo fields = 3; - - repeated MethodInfo methods = 4; -} - -//////////////////////////////////////////////////////////////////////// - service Inspect { // The list of state types in an application is static, however, we // make this a streaming RPC so that the client can hear when it diff --git a/rbt/v1alpha1/options.proto b/rbt/v1alpha1/options.proto index 9c8227ca..c6c1b261 100644 --- a/rbt/v1alpha1/options.proto +++ b/rbt/v1alpha1/options.proto @@ -163,6 +163,12 @@ message StateOptions { // `RBT_VALIDATE_TRUSTED_EFFECTS` is set. See // https://github.com/reboot-dev/mono/issues/4499. bool trusted_effects = 4; + + // What the state type does, in the author's own words. Worth + // writing about the parts a reader cannot derive, such as what it + // is the consistency boundary for, rather than restating the + // methods listed beside it. + optional string description = 5; } extend google.protobuf.MessageOptions { diff --git a/reboot/api.py b/reboot/api.py index 40c6d01e..0eda580b 100644 --- a/reboot/api.py +++ b/reboot/api.py @@ -1027,12 +1027,14 @@ class Type(pydantic.BaseModel): state: typing.Type[Model] methods: Methods + description: Optional[str] = None def __init__( self, *, state: typing.Type[Model], methods: Methods, + description: Optional[str] = None, ): def validate_all_fields_are_reboot_base_classes( @@ -1187,6 +1189,7 @@ def validate_all_fields_are_reboot_base_classes( super().__init__( state=state, methods=methods, + description=description, ) diff --git a/reboot/dashboard/frontend/dashboard.css b/reboot/dashboard/frontend/dashboard.css index f6dfb208..99ce79fa 100644 --- a/reboot/dashboard/frontend/dashboard.css +++ b/reboot/dashboard/frontend/dashboard.css @@ -252,6 +252,32 @@ header h1 { margin: 6px 0 4px; } +/* What the author says the state type is, under its name and + file. Wider measure and no padding of its own: it belongs to + the section's heading rather than to a card, the way + `.method-description` belongs to a method. */ +.state-type-description { + max-width: 62ch; + margin: 12px 0 0; + font-size: 14px; + line-height: 1.55; + color: hsl(var(--prose)); + text-wrap: pretty; +} + +/* Code inside a description, written as `backticks` by its author. + `0.9em` rather than a pixel size, so it follows whichever + description it sits in. */ +.state-type-description code, +.method-description code { + font-family: ui-monospace, Menlo, monospace; + font-size: 0.9em; + background: hsl(var(--surface-sunken)); + border: 1px solid hsl(var(--border-soft)); + border-radius: 4px; + padding: 0 4px; +} + .file { font-family: ui-monospace, Menlo, monospace; font-size: 11.5px; diff --git a/reboot/dashboard/frontend/src/main.tsx b/reboot/dashboard/frontend/src/main.tsx index b6fa0ecf..5c8a883f 100644 --- a/reboot/dashboard/frontend/src/main.tsx +++ b/reboot/dashboard/frontend/src/main.tsx @@ -75,6 +75,30 @@ const Kind: FC<{ kind: string }> = ({ kind }) => ( /> ); +// A description, with the spans its author wrote in `backticks` +// rendered as code rather than shown with their backticks. An +// unpaired backtick is kept as text, since it opens nothing. +const Description: FC<{ className: string; text: string }> = ({ + className, + text, +}) => { + const parts = text.split("`"); + return ( +

+ {parts.map((part, index) => { + // `split` alternates text and code, so odd indexes are code, + // except a last part at an odd index, whose backtick was + // never closed. + const unclosed = index === parts.length - 1 && parts.length % 2 === 0; + if (index % 2 === 1 && !unclosed) { + return {part}; + } + return {unclosed ? "`" + part : part}; + })} +

+ ); +}; + // A state type's namespace is its proto package: `bank.v1.Account` // lives in `bank.v1`, which is the developer's `api/bank/v1/`. const namespaceOf = (name: string): string => @@ -171,7 +195,10 @@ const Method: FC<{ method: MethodInfo }> = ({ method }) => {
{method.description !== undefined && ( -

{method.description}

+ )}
@@ -295,6 +322,12 @@ const StateType: FC<{
{stateType.file}
+ {stateType.description !== undefined && ( + + )}
state
{stateType.fields.length === 0 ? ( diff --git a/reboot/inspect/BUILD.bazel b/reboot/inspect/BUILD.bazel index 5083da6d..007e4e00 100644 --- a/reboot/inspect/BUILD.bazel +++ b/reboot/inspect/BUILD.bazel @@ -49,19 +49,6 @@ esbuild( platform = "browser", ) -py_library( - name = "describe_state_type_py", - srcs = ["describe_state_type.py"], - srcs_version = "PY3", - visibility = ["//visibility:public"], - deps = [ - "//log:log_py", - "//rbt/v1alpha1:options_py_proto", - "//rbt/v1alpha1/inspect:inspect_py_proto", - "//reboot/aio:types_py", - ], -) - py_library( name = "servicer_py", srcs = ["servicer.py"], diff --git a/reboot/inspect/describe_state_type.py b/reboot/inspect/describe_state_type.py deleted file mode 100644 index 62bbe43e..00000000 --- a/reboot/inspect/describe_state_type.py +++ /dev/null @@ -1,141 +0,0 @@ -"""Describes Reboot state types from protobuf descriptors. - -Everything comes from the descriptor pool it is handed: a state type's -fields, and the methods of the services that supply them. Field and -method types are rendered as the Python names a person reads on a -page, such as `int` and `list[str]`, rather than as their protobuf -spelling. -""" -from google.protobuf import descriptor_pool -from google.protobuf.descriptor import FieldDescriptor -from log.log import get_logger -from rbt.v1alpha1 import options_pb2 -from rbt.v1alpha1.inspect.inspect_pb2 import ( - FieldInfo, - MethodInfo, - StateTypeInfo, -) -from reboot.aio.types import StateTypeName -from typing import Iterable, Optional - -logger = get_logger(__name__) - -_TYPE_NAMES = { - FieldDescriptor.TYPE_DOUBLE: 'float', - FieldDescriptor.TYPE_FLOAT: 'float', - FieldDescriptor.TYPE_INT64: 'int', - FieldDescriptor.TYPE_UINT64: 'int', - FieldDescriptor.TYPE_INT32: 'int', - FieldDescriptor.TYPE_FIXED64: 'int', - FieldDescriptor.TYPE_FIXED32: 'int', - FieldDescriptor.TYPE_BOOL: 'bool', - FieldDescriptor.TYPE_STRING: 'str', - FieldDescriptor.TYPE_BYTES: 'bytes', - FieldDescriptor.TYPE_UINT32: 'int', - FieldDescriptor.TYPE_SFIXED32: 'int', - FieldDescriptor.TYPE_SFIXED64: 'int', - FieldDescriptor.TYPE_SINT32: 'int', - FieldDescriptor.TYPE_SINT64: 'int', -} - - -def _type_name(field) -> str: - """How to render `field`'s type.""" - if field.type in ( - FieldDescriptor.TYPE_MESSAGE, FieldDescriptor.TYPE_GROUP - ): - name = field.message_type.name - elif field.type == FieldDescriptor.TYPE_ENUM: - name = field.enum_type.name - else: - name = _TYPE_NAMES.get(field.type, 'unknown') - - if field.label == FieldDescriptor.LABEL_REPEATED: - return f'list[{name}]' - return name - - -def _fields_of(message) -> list[FieldInfo]: - return [ - FieldInfo(name=field.name, type=_type_name(field)) - for field in message.fields - ] - - -def _describe_method(method) -> MethodInfo: - options = method.GetOptions().Extensions[options_pb2.method] - kind = options.WhichOneof('kind') or '' - - info = MethodInfo( - name=method.name, - kind=kind, - arguments=_fields_of(method.input_type), - errors=list(options.errors), - mcp=options.HasField('mcp'), - ) - - # An empty response means the method returns nothing; saying - # "Empty" would be an implementation detail leaking out. The - # response's fields rather than its name, because a synthesized - # name such as `ShopRemainingResponse` says nothing the fields - # don't. - if method.output_type.full_name != 'google.protobuf.Empty': - info.returns.extend(_fields_of(method.output_type)) - - # An application that was created before `MethodOptions.description` - # will have the deprecated `mcp` description, which is permitted - # for backward compatibility. - if options.description: - info.description = options.description - elif options.HasField('mcp') and options.mcp.description: - info.description = options.mcp.description - - # Only writers and transactions can construct. - if kind in ('writer', 'transaction'): - info.factory = getattr(options, kind).HasField('constructor') - - return info - - -def describe_state_type( - pool: descriptor_pool.DescriptorPool, - state_type_name: StateTypeName, - service_names: Iterable[str], - file: str, -) -> Optional[StateTypeInfo]: - """Describes one state type, or `None` when its descriptors can't - be found, since a state type we can't describe shouldn't stop us - describing the rest. - - `pool` holds the state type's descriptors and those of the - services named in `service_names`, which supply its methods. - - `file` is the file the developer declared the state type in, which - is reported as it is. A Pydantic API is described from a `.proto` - synthesized from it, so the descriptors name a file that only - exists inside the build. - """ - try: - state = pool.FindMessageTypeByName(state_type_name) - except KeyError: - logger.warning( - f"No descriptor for state type '{state_type_name}'; " - "omitting it from the schema" - ) - return None - - info = StateTypeInfo( - name=state_type_name, - file=file, - fields=_fields_of(state), - ) - - for service_name in service_names: - try: - service = pool.FindServiceByName(service_name) - except KeyError: - continue - for method in service.methods: - info.methods.append(_describe_method(method)) - - return info diff --git a/reboot/pydantic_schema_to_proto.py b/reboot/pydantic_schema_to_proto.py index 2f774a1c..e42a6d32 100644 --- a/reboot/pydantic_schema_to_proto.py +++ b/reboot/pydantic_schema_to_proto.py @@ -173,6 +173,8 @@ async def generate( # Auto-construct enum value name for this state type, # or None for non-auto-constructed types. auto_construct: Optional[str] = None, + # What the state type does, in the author's own words. + description: Optional[str] = None, ): origin = get_origin(schema) args = get_args(schema) @@ -183,12 +185,19 @@ async def generate( await proto.write(f"message {name} {{\n") if state: - if uis or auto_construct: - # Generate state option with UIs and/or - # auto-construct annotation. Proto text + if uis or auto_construct or description is not None: + # Generate state option with UIs, a description + # and/or auto-construct annotation. Proto text # format uses repeated field names, not # array syntax. await proto.write(" option (rbt.v1alpha1.state) = {\n") + if description is not None: + # The description can contain `\` character, so we + # need to escape it for proto string literal. + await proto.write( + " description: " + f'"{_escape_string_for_proto(description)}"\n' + ) if auto_construct is not None: await proto.write( f" auto_construct: " @@ -811,6 +820,7 @@ async def generate_proto_file_from_api( uis=uis if uis else None, auto_construct=_PER_USER_ID if type_name == AUTO_CONSTRUCT_STATE_TYPE else None, + description=type_obj.description, ) await proto.write('\n') diff --git a/tests/reboot/dashboard/api/shop/v1/shop.py b/tests/reboot/dashboard/api/shop/v1/shop.py index f49b264d..efa1ca60 100644 --- a/tests/reboot/dashboard/api/shop/v1/shop.py +++ b/tests/reboot/dashboard/api/shop/v1/shop.py @@ -45,4 +45,10 @@ class OutOfStockError(Model): ), ) -api = API(Shop=Type(state=ShopState, methods=ShopMethods)) +api = API( + Shop=Type( + state=ShopState, + methods=ShopMethods, + description="A shop, and the stock it has to sell.", + ) +) diff --git a/tests/reboot/dashboard/api_reader_tests.py b/tests/reboot/dashboard/api_reader_tests.py index b5d3d23e..9cf31d72 100644 --- a/tests/reboot/dashboard/api_reader_tests.py +++ b/tests/reboot/dashboard/api_reader_tests.py @@ -41,6 +41,11 @@ async def test_describes_a_state_type_and_its_methods(self) -> None: os.path.join(API_DIRECTORY, 'shop/v1/shop.py'), ) + self.assertEqual( + shop['description'], + 'A shop, and the stock it has to sell.', + ) + self.assertEqual( [field['name'] for field in shop['fields']], ['name', 'open'], From 2d349ceaf08a5d9bf137cb3f4511c7916bac5257 Mon Sep 17 00:00:00 2001 From: Riley Scheid Date: Thu, 13 Aug 2026 06:23:03 +0000 Subject: [PATCH 05/31] Give the manylinux builder a Python that ships `libpython` The manylinux images build every CPython with `--disable-shared` and delete even the static `libpython` archives. The `reboot-dev-reboot` genrule links `reboot_native.node` with `-lpython3.10`, a flag emitted by `python3.10-config --ldflags --embed`. That link has never been able to succeed inside these images. CI stayed green only while Bazel's remote cache served the genrule's outputs. The first cache miss made every platform fail deterministically. On x86_64 that miss came from a runner hardware swap: it changed the `lscpu` portion of `the_environment.txt`, and with it the whole cache scope. Point `python`/`python3` at a python-build-standalone CPython 3.10, which ships `libpython3.10.so`. It is the same build `reboot/nodejs/prepare_environment.sh` downloads. `pip`/`pip3` stay on the manylinux interpreter, whose layout `auditwheel` and the wheel builds expect. `python3` and `pip` therefore deliberately name different installations. Co-Authored-By: Claude Fable 5 --- Dockerfile | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index e587b446..d698e26c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -797,11 +797,25 @@ RUN dnf install -y clang gcc-c++ && dnf clean all \ && mkdir -p /usr/lib/llvm-${CLANG_VERSION}/lib \ && ln -sf /usr/lib/clang /usr/lib/llvm-${CLANG_VERSION}/lib/clang -# The `manylinux` image comes with multiple Python versions. We'll use -# 3.10 to match our main build environment. Set up symlinks so `python` -# and `python3` point to Python 3.10. -RUN ln -sf /opt/python/cp310-cp310/bin/python /usr/local/bin/python \ - && ln -sf /opt/python/cp310-cp310/bin/python /usr/local/bin/python3 \ +# The `manylinux` image comes with multiple Python versions, but all +# of them are built `--disable-shared` with no `libpython` installed, +# and embedding Python — which building `reboot_native.node` does — +# needs `-lpython3.10` at link time. Point `python` and `python3` at a +# python-build-standalone CPython 3.10 instead, which ships +# `libpython3.10.so`; it is the same build +# `reboot/nodejs/prepare_environment.sh` uses. `pip` and `pip3` stay +# on the manylinux 3.10, whose layout `auditwheel` and the wheel +# builds expect, so `python3` and `pip` deliberately name different +# installations. +RUN set -e; \ + if [ "${TARGETARCH}" = "amd64" ]; then ARCH=x86_64; else ARCH=aarch64; fi; \ + mkdir /tmp/python-build-standalone; \ + wget -qO- "https://github.com/indygreg/python-build-standalone/releases/download/20240814/cpython-3.10.14+20240814-${ARCH}-unknown-linux-gnu-install_only.tar.gz" \ + | tar -xzf - -C /tmp/python-build-standalone \ + && mv /tmp/python-build-standalone/python /opt/reboot-python \ + && rmdir /tmp/python-build-standalone \ + && ln -sf /opt/reboot-python/bin/python3 /usr/local/bin/python \ + && ln -sf /opt/reboot-python/bin/python3 /usr/local/bin/python3 \ && ln -sf /opt/python/cp310-cp310/bin/pip /usr/local/bin/pip \ && ln -sf /opt/python/cp310-cp310/bin/pip /usr/local/bin/pip3 From f318226cc1e8ffadebe6bf2f65292a4393b5531b Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Mon, 17 Aug 2026 03:13:21 +0000 Subject: [PATCH 06/31] Take the dashboard's API directory from the `.rbtrc` `rbt dashboard` needed `--api-directory`, naming a directory the `.rbtrc` already names for `rbt generate`. Two places to say the same thing is two places to change it, and nothing tells you when only one of them moves -- the dashboard just watches a directory the rest of the tooling has stopped using. So it reads what `rbt generate` was told instead, through a new `ArgumentParser.dot_rc_arguments`, which returns what the `.rbtrc` gives any subcommand. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01QPEVMhyDRxZEuH8eiykv99 --- reboot/cli/commands/dashboard.py | 32 ++++++++++++---- reboot/cli/common/rc.py | 16 ++++++++ reboot/dashboard/api_reader.py | 2 +- reboot/dashboard/constants.py | 8 ++-- tests/reboot/cli/dashboard_tests.py | 58 +++++++++++++++++++++-------- 5 files changed, 87 insertions(+), 29 deletions(-) diff --git a/reboot/cli/commands/dashboard.py b/reboot/cli/commands/dashboard.py index 35f51ea0..43bb6d52 100644 --- a/reboot/cli/commands/dashboard.py +++ b/reboot/cli/commands/dashboard.py @@ -56,13 +56,6 @@ def dashboard_subcommands() -> list[str]: def register_dashboard(parser: ArgumentParser): add_working_directory_options(parser.subcommand('dashboard')) - parser.subcommand('dashboard').add_argument( - '--api-directory', - type=str, - required=True, - help='directory containing the API files the dashboard watches', - ) - parser.subcommand('dashboard').add_argument( '--port', type=int, @@ -71,6 +64,29 @@ def register_dashboard(parser: ArgumentParser): ) +def _api_directory(parser: ArgumentParser) -> str: + """Returns the directory holding the developer's API files, which + is the directory they tell `rbt generate` to read them from. + + Taken from there rather than named again here, so that moving the + API files is one edit and the dashboard cannot end up watching a + directory the rest of the tooling has stopped using. + """ + for argument in parser.dot_rc_arguments('generate'): + # The directory is the one thing `rbt generate` takes that is + # not a flag; its flags say where to put what it generates. + if not argument.startswith('-'): + return argument + + terminal.fail( + 'Could not tell where your API files are. `rbt dashboard` reads ' + f'that from the same place `rbt generate` does, so name a ' + f'directory for it in your {parser.dot_rc_filename}:\n' + '\n' + ' generate api/\n' + ) + + def _dashboard_env( args, parser: ArgumentParser, @@ -218,7 +234,7 @@ async def dashboard( args, parser, port=port, - api_directory=args.api_directory, + api_directory=_api_directory(parser), ) terminal.info( diff --git a/reboot/cli/common/rc.py b/reboot/cli/common/rc.py index 918dd91e..b72e5f94 100644 --- a/reboot/cli/common/rc.py +++ b/reboot/cli/common/rc.py @@ -739,6 +739,22 @@ def subcommand(self, subcommand: str) -> SubcommandParser: raise ValueError(f"Invalid subcommand '{subcommand}'") return self._subcommand_parsers[subcommand] + def dot_rc_arguments(self, subcommand: str) -> list[str]: + """Returns the arguments the '.rc' file gives a subcommand, + spelled as they are written there and in the order they appear, + and nothing at all when there is no such file. + + Only the lines that always apply: one written for a config, + such as `dev run:hmr --frontend-host=...`, is left out, because + whether that config was asked for is not known here. + """ + if self.dot_rc is None: + return [] + + flags = self._read_flags_from_dot_rc(self.dot_rc_filename, self.dot_rc) + + return flags[(subcommand, None)] + def parse_args(self) -> tuple[argparse.Namespace, list[str]]: """Pass through to top-level parser with the expanded arguments after first validating that all flags include '=' between them and their value.""" diff --git a/reboot/dashboard/api_reader.py b/reboot/dashboard/api_reader.py index dbf55816..5c677ff3 100644 --- a/reboot/dashboard/api_reader.py +++ b/reboot/dashboard/api_reader.py @@ -96,7 +96,7 @@ def describe(api_directory: str, filename: str) -> list[dict]: `Shop` yields `shop.v1.Shop`. """ # The path as the developer spelled it, joined before anything - # resolves it away: with `--api-directory=api` the file shows as + # resolves it away: with `generate api/` the file shows as # `api/bank/v1/account.py`, the path they would open. file = os.path.join(api_directory, filename) diff --git a/reboot/dashboard/constants.py b/reboot/dashboard/constants.py index 5a16d3b9..2c1b5a05 100644 --- a/reboot/dashboard/constants.py +++ b/reboot/dashboard/constants.py @@ -30,10 +30,10 @@ # run` reads it. PREFERENCES_ID = 'preferences' -# The directory the developer's API files are in, which -# `rbt dashboard` takes as `--api-directory`. Separate from the -# application's URL because the files are there long before anything -# is serving, and the dashboard is meant to be startable that early. +# The directory the developer's API files are in, as the `.rbtrc` +# spells it for `rbt generate`. Separate from the application's URL +# because the files are there long before anything is serving, and +# the dashboard is meant to be startable that early. ENVVAR_RBT_API_DIRECTORY = 'RBT_API_DIRECTORY' # The `Presence` state the dashboard page subscribes to, recording who diff --git a/tests/reboot/cli/dashboard_tests.py b/tests/reboot/cli/dashboard_tests.py index f5bea4ff..9371b03d 100644 --- a/tests/reboot/cli/dashboard_tests.py +++ b/tests/reboot/cli/dashboard_tests.py @@ -6,32 +6,58 @@ from reboot.cli.common.directories import dot_rbt_directory from reboot.cli.common.rc import ArgumentParser from reboot.dashboard.constants import DEFAULT_DASHBOARD_PORT -from tests.reboot.cli.mock_exit import ( - MockExitException, - mock_raise_instead_of_exit, -) +from tests.reboot.cli.mock_exit import mock_raise_instead_of_exit from unittest.mock import patch @patch('argparse.ArgumentParser.exit', mock_raise_instead_of_exit) class RbtDashboardTestCase(unittest.IsolatedAsyncioTestCase): - def _parse(self, state_directory: str): + def _parse(self, state_directory: str, *, rbtrc: str = 'generate api/'): + rc_file = os.path.join(state_directory, '.rbtrc') + with open(rc_file, 'w') as file: + file.write(rbtrc + '\n') + parser: ArgumentParser = cli.create_parser( + rc_file=rc_file, argv=[ 'rbt', f'--state-directory={state_directory}', 'dashboard', - '--api-directory=api', - ] + ], ) args, _ = parser.parse_args() return args, parser - async def test_api_directory_is_required(self) -> None: - parser: ArgumentParser = cli.create_parser(argv=['rbt', 'dashboard']) - with self.assertRaises(MockExitException): - parser.parse_args() + async def test_the_api_directory_comes_from_generate(self) -> None: + """Naming it twice is how the two come to disagree, so it is + named once, where `rbt generate` already needs it.""" + with tempfile.TemporaryDirectory() as state_directory: + _, parser = self._parse( + state_directory, + rbtrc=( + '# Find the API files in `api/`.\n' + 'generate api/\n' + '\n' + 'generate --python=backend/api\n' + 'generate --react=frontend/api\n' + '\n' + 'dev run --application=backend/src/main.py\n' + 'dev run:hmr --frontend-host=http://localhost:4444' + ), + ) + + self.assertEqual(dashboard._api_directory(parser), 'api/') + + async def test_an_rbtrc_that_says_nothing_about_generate(self) -> None: + with tempfile.TemporaryDirectory() as state_directory: + _, parser = self._parse( + state_directory, + rbtrc='dev run --application=backend/src/main.py', + ) + + with self.assertRaises(SystemExit): + dashboard._api_directory(parser) async def test_env_is_isolated_from_any_application(self) -> None: with tempfile.TemporaryDirectory() as state_directory: @@ -54,7 +80,7 @@ async def test_env_is_isolated_from_any_application(self) -> None: args, parser, port=DEFAULT_DASHBOARD_PORT, - api_directory=args.api_directory, + api_directory=dashboard._api_directory(parser), ) self.assertEqual(env['RBT_NAME'], 'dashboard') @@ -88,7 +114,7 @@ async def test_keys_differ_from_any_application(self) -> None: args, parser, port=DEFAULT_DASHBOARD_PORT, - api_directory=args.api_directory, + api_directory=dashboard._api_directory(parser), ) self.assertNotEqual(env['REBOOT_CRYPTO_ROOT_KEYS'], 'v1:theirs') @@ -99,7 +125,7 @@ async def test_keys_differ_from_any_application(self) -> None: args, parser, port=DEFAULT_DASHBOARD_PORT, - api_directory=args.api_directory, + api_directory=dashboard._api_directory(parser), ) self.assertEqual( env['REBOOT_CRYPTO_ROOT_KEYS'], @@ -114,13 +140,13 @@ async def test_is_told_where_the_api_files_are(self) -> None: args, parser, port=DEFAULT_DASHBOARD_PORT, - api_directory=args.api_directory, + api_directory=dashboard._api_directory(parser), ) # As the developer spelled it, so files can be shown as # `api/bank/v1/account.py`; the dashboard runs in the # working directory where that spelling resolves. - self.assertEqual(env['RBT_API_DIRECTORY'], 'api') + self.assertEqual(env['RBT_API_DIRECTORY'], 'api/') if __name__ == '__main__': From 9af6c7743eff8e0ca68ec38a29ef3f6ba4ef5bbb Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Mon, 17 Aug 2026 01:17:46 +0000 Subject: [PATCH 07/31] Add `cooperatively`, for work that never waits Work that waits on nothing -- parsing, hashing, encoding -- never gives the event loop a chance of its own, so a servicer doing it over a collection holds its process for as long as the whole collection takes, and everything else it serves waits that long. `concurrently` is the wrong tool, because there is nothing to overlap. Measured over twelve parses of a 45KB file, it left the loop unable to answer for 24ms at a stretch -- 15ms even limited to one at a time, since its tasks are scheduled together and the loop drains several before looking at anything else -- and cost 30% more wall-clock in task machinery. An `asyncio.sleep(0)` in the loop measures best, at 6ms, but invites the question of why it is there and not somewhere else. This answers it: the yield falls out of how the work was grouped, which is a decision the caller has to make anyway, and the collection bounds it the way `concurrently`'s does. It takes elements rather than awaitables, because nothing is being run: the work stays in the caller's body, where it can go on mutating whatever it likes. Note an `async for` alone will not do -- `await` on something that resolves without suspending never reaches the event loop at all, which is why the yield has to live in here. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01QPEVMhyDRxZEuH8eiykv99 --- reboot/aio/BUILD.bazel | 8 ++++++++ reboot/aio/cooperatively.py | 39 +++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 reboot/aio/cooperatively.py diff --git a/reboot/aio/BUILD.bazel b/reboot/aio/BUILD.bazel index 54ee332d..1a103f95 100644 --- a/reboot/aio/BUILD.bazel +++ b/reboot/aio/BUILD.bazel @@ -127,6 +127,13 @@ py_library( ], ) +py_library( + name = "cooperatively_py", + srcs = ["cooperatively.py"], + srcs_version = "PY3", + visibility = ["//visibility:public"], +) + py_library( name = "directories_py", srcs = ["directories.py"], @@ -567,6 +574,7 @@ py_library( ":caller_id_py", ":concurrently_py", ":contexts_py", + ":cooperatively_py", ":directories_py", ":exceptions_py", ":external_py", diff --git a/reboot/aio/cooperatively.py b/reboot/aio/cooperatively.py new file mode 100644 index 00000000..3722877c --- /dev/null +++ b/reboot/aio/cooperatively.py @@ -0,0 +1,39 @@ +import asyncio +from typing import AsyncIterator, Iterable, TypeVar + +ElementT = TypeVar("ElementT") + + +async def cooperatively( + elements: Iterable[ElementT] +) -> AsyncIterator[ElementT]: + """Returns an iterator over `elements` that leaves the event loop + free between them. + + For work that is *not* waiting on anything -- parsing, hashing, + encoding -- and so never gives the event loop a chance of its own. + A servicer doing such work over a collection holds its process for + as long as the whole collection takes, and everything else it + serves waits that long. + + `concurrently` is the wrong tool for that: it exists to overlap + work that waits, and its tasks are scheduled together, so the loop + drains several of them before it looks at anything else. Measured + over twelve parses of a 45KB file, `concurrently` left the loop + unable to answer for 24ms at a stretch (15ms even limited to one + at a time), against 6ms here -- and cost 30% more wall-clock in + task machinery for work that has no waiting to overlap. + + async for path in cooperatively(paths): + index(path) # holds the interpreter; nothing waits + + The chunk is the unit of work handed in, so the loop is left free + exactly as often as there are elements. Make them small enough + that one of them is a delay nobody minds. + """ + for element in elements: + # The whole point: `await` on something that resolves without + # suspending never reaches the event loop, so this has to be + # something that does. + await asyncio.sleep(0) + yield element From f2cfb37b4b4ff7c3522681be14efe8df81db5b0e Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Sun, 16 Aug 2026 22:28:12 +0000 Subject: [PATCH 08/31] Work out where each state type is implemented The API files say which state types exist. They say nothing about which file implements one, and the name does not say either -- `servicers.py` may implement several state types while being named after none of them. What does say is the application: Application(servicers=[AccountServicer, BankServicer, ...]) so this reads the entry point, resolves each registered servicer back to the file defining it, and asks that class what it services. Read rather than imported. Importing an application means having its generated code, its dependencies and its `sys.path`, and the dashboard is meant to work before any of that exists -- the same reason the API files are read the way they are. Driven by the API rather than by the filesystem: the API is what says which state types there are to look for, so a state type appearing or disappearing is what sets this going. `until_changes` suspends the workflow in between, so it wakes when the declarations move rather than on a timer. Recorded one state per state type, so that working out one state type's implementation neither waits on nor overwrites another's. A state type the application registers no servicer for is recorded as such rather than left looking unanalyzed. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01QPEVMhyDRxZEuH8eiykv99 --- rbt/dashboard/v1/dashboard.proto | 59 ++ reboot/cli/commands/dashboard.py | 28 + reboot/dashboard/BUILD.bazel | 15 + reboot/dashboard/constants.py | 12 + reboot/dashboard/implementation_watcher.py | 276 +++++++++ reboot/dashboard/main.py | 16 +- reboot/dashboard/servicers.py | 64 ++- tests/reboot/cli/dashboard_tests.py | 41 ++ tests/reboot/dashboard/BUILD.bazel | 11 + .../dashboard/implementation_watcher_tests.py | 532 ++++++++++++++++++ 10 files changed, 1043 insertions(+), 11 deletions(-) create mode 100644 reboot/dashboard/implementation_watcher.py create mode 100644 tests/reboot/dashboard/implementation_watcher_tests.py diff --git a/rbt/dashboard/v1/dashboard.proto b/rbt/dashboard/v1/dashboard.proto index 3c5a8f01..ce1d7c68 100644 --- a/rbt/dashboard/v1/dashboard.proto +++ b/rbt/dashboard/v1/dashboard.proto @@ -77,6 +77,48 @@ message APIWatchResponse {} //////////////////////////////////////////////////////////////////////// +// One servicer found in the developer's application, and the state +// type it services. +message ServicerInfo { + // The state type it services, spelled as `StateTypeInfo.name`. + string state_type = 1; + + // The file it is written in, spelled the way the developer would + // open it, e.g. "backend/src/account_servicer.py". + string file = 2; +} + +// What the dashboard application has read of the developer's +// application: which of their files implements each state type. +// +// Held apart from `API` because the two are read from different +// places, at different moments, and neither says anything about the +// other. An API file says a state type exists; only the application +// says where it is implemented. Whoever wants both asks for both. +message Implementation { + option (rbt.v1alpha1.state) = { + }; + + // Every servicer found, sorted. A state type with no entry here is + // one no servicer was found for -- which, before this has ever been + // written, is all of them. A state type with two is two classes + // servicing it, which is for whoever reads this to make of what + // they will. + repeated ServicerInfo servicers = 1; +} + +message ImplementationGetRequest {} + +message ImplementationGetResponse { + repeated ServicerInfo servicers = 1; +} + +message ImplementationWatchRequest {} + +message ImplementationWatchResponse {} + +//////////////////////////////////////////////////////////////////////// + // What the developer has told the dashboard about opening dashboards. // // Kept here rather than in a file under their project because it is a @@ -153,6 +195,23 @@ service APIMethods { //////////////////////////////////////////////////////////////////////// +service ImplementationMethods { + rpc Get(ImplementationGetRequest) returns (ImplementationGetResponse) { + option (rbt.v1alpha1.method).reader = { + }; + } + + // Watches the developer's application for as long as the dashboard + // application runs, working out which of their files implements + // each state type whenever one of those files changes. + rpc Watch(ImplementationWatchRequest) returns (ImplementationWatchResponse) { + option (rbt.v1alpha1.method).workflow = { + }; + } +} + +//////////////////////////////////////////////////////////////////////// + service PreferencesMethods { rpc Get(PreferencesGetRequest) returns (PreferencesGetResponse) { option (rbt.v1alpha1.method).reader = { diff --git a/reboot/cli/commands/dashboard.py b/reboot/cli/commands/dashboard.py index 43bb6d52..93eb2ede 100644 --- a/reboot/cli/commands/dashboard.py +++ b/reboot/cli/commands/dashboard.py @@ -23,6 +23,7 @@ DASHBOARD_PATH, DEFAULT_DASHBOARD_PORT, ENVVAR_RBT_API_DIRECTORY, + ENVVAR_RBT_APPLICATION, ) from reboot.settings import ( ENVVAR_RBT_DEV, @@ -87,12 +88,30 @@ def _api_directory(parser: ArgumentParser) -> str: ) +def _application(parser: ArgumentParser) -> Optional[str]: + """Returns the developer's application, which is the one they tell + `rbt dev run` to run, and `None` when they tell it none. + + Read rather than asked for again, so that moving the application + is one edit; a second place to name it is a second place to forget + to change. `None` is what an application this cannot read looks + like -- a Node.js one names no Python for the servicers to be in. + """ + for argument in parser.dot_rc_arguments('dev run'): + name, separator, value = argument.partition('=') + if name == '--application' and separator == '=': + return value + + return None + + def _dashboard_env( args, parser: ArgumentParser, *, port: int, api_directory: str, + application: Optional[str], ) -> dict[str, str]: """The environment for the dashboard application. @@ -134,6 +153,14 @@ def _dashboard_env( # directory, where that spelling resolves. composed[ENVVAR_RBT_API_DIRECTORY] = api_directory + # Where the developer's servicers are, spelled the same way and + # for the same reason. Left out of the environment entirely when + # the developer named no application, which is what tells the + # dashboard there is nothing to look for. + composed.pop(ENVVAR_RBT_APPLICATION, None) + if application is not None: + composed[ENVVAR_RBT_APPLICATION] = application + composed[ENVVAR_RBT_NAME] = DASHBOARD_STATE_DIRECTORY_NAME state_directory = ( @@ -235,6 +262,7 @@ async def dashboard( parser, port=port, api_directory=_api_directory(parser), + application=_application(parser), ) terminal.info( diff --git a/reboot/dashboard/BUILD.bazel b/reboot/dashboard/BUILD.bazel index 4a7a1d0c..8e4a976b 100644 --- a/reboot/dashboard/BUILD.bazel +++ b/reboot/dashboard/BUILD.bazel @@ -33,6 +33,20 @@ py_library( ], ) +py_library( + name = "implementation_watcher_py", + srcs = ["implementation_watcher.py"], + srcs_version = "PY3", + visibility = ["//visibility:public"], + deps = [ + ":constants_py", + "//rbt/dashboard/v1:dashboard_py_reboot", + "//reboot/aio:cooperatively_py", + "//reboot/aio:external_py", + "//reboot/cli/common:watch_py", + ], +) + py_library( name = "servicers_py", srcs = ["servicers.py"], @@ -41,6 +55,7 @@ py_library( deps = [ ":api_watcher_py", ":constants_py", + ":implementation_watcher_py", "//rbt/dashboard/v1:dashboard_py_reboot", "//reboot/aio:servicers_py", "//reboot/std/presence/v1:presence_py", diff --git a/reboot/dashboard/constants.py b/reboot/dashboard/constants.py index 2c1b5a05..545bcccd 100644 --- a/reboot/dashboard/constants.py +++ b/reboot/dashboard/constants.py @@ -25,6 +25,11 @@ # declare, as the dashboard application last read them. API_ID = 'api' +# The `Implementation` state holding which of the developer's files +# implements each state type, as the dashboard application last read +# their application. +IMPLEMENTATION_ID = 'implementation' + # The `Preferences` state holding what the developer has said about # opening dashboards: the dashboard's banner writes it and `rbt dev # run` reads it. @@ -36,6 +41,13 @@ # the dashboard is meant to be startable that early. ENVVAR_RBT_API_DIRECTORY = 'RBT_API_DIRECTORY' +# The developer's application entry point, as the `.rbtrc` spells it +# for `rbt dev run`. What the API files declare says nothing about +# which file implements a state type; the application, which registers +# the servicers, is what says. Unset when the developer named none, in +# which case no implementation is looked for. +ENVVAR_RBT_APPLICATION = 'RBT_APPLICATION' + # The `Presence` state the dashboard page subscribes to, recording who # is looking at a dashboard right now. `rbt dev run` reads it to decide # whether to open one. diff --git a/reboot/dashboard/implementation_watcher.py b/reboot/dashboard/implementation_watcher.py new file mode 100644 index 00000000..c43ccd9a --- /dev/null +++ b/reboot/dashboard/implementation_watcher.py @@ -0,0 +1,276 @@ +"""How each of the developer's state types is implemented, kept up +to date for as long as the dashboard runs. + +Their API files say which state types exist and nothing about where +one is implemented; the name does not say either, since `servicers.py` +may implement several and is named after none of them. What says is +the application, where the servicers are registered -- so this starts +at its entry point and follows its imports, collecting every class +that says what it services: + + class AccountServicer(Account.Servicer): + +Following the imports rather than reading the list handed to +`Application`, because a servicer reaches it by any number of routes +-- `servicers=servicers()`, a list built elsewhere, a name rebound +behind a conditional import -- which have one thing in common: the +file defining the servicer had to be imported for any of them to run. + +Where the walk stops is what makes this the developer's code rather +than somebody else's. A module resolves only if a root holds it, so +an import of an installed package leads nowhere, and no state type of +theirs is waiting on one: their API files declare none of those. + +Read rather than imported, because importing an application means +having its generated code, its dependencies and its `sys.path`, and +the dashboard is meant to work before any of that exists. A file at a +time through `cooperatively`, so that the dashboard goes on answering +while a large one is walked. + +And driven by the filesystem, because that is what it is a function +of: where a state type is implemented can only move when the +developer's source moves, so an edit under the roots is what wakes +this, and nothing else does. +""" +import ast +import os +from rbt.dashboard.v1.dashboard_pb2 import ServicerInfo +from rbt.dashboard.v1.dashboard_rbt import Implementation +from reboot.aio.contexts import WorkflowContext +from reboot.aio.cooperatively import cooperatively +from reboot.cli.common.watch import file_watcher +from typing import Optional + +GENERATED_SUFFIX = '_rbt' + +# Every file the developer might have written a servicer in, which is +# the rule `rbt generate` and `rbt dev run` both use for source. +SOURCE_GLOB = '**/*.py' + + +def _roots(application: str) -> list[str]: + """Returns the directories the developer's modules are found + under, which is what running the application puts first on its + path.""" + return [os.path.dirname(application)] + + +def _parse(filename: str) -> ast.Module: + with open(filename) as file: + return ast.parse(file.read()) + + +def _imports( + module: ast.Module +) -> tuple[dict[str, tuple[str, str]], list[str]]: + """Returns every symbol a file imported -- keyed by the name the + file calls it, valued by the module it came from and its name + there, so `Account` -> (`bank.v1.account_rbt`, `Account`) -- and + beside it every module the file has imported. + + Symbols come from every import, wherever it is written. One inside + an `if` or a `try` binds its name just as one at the top of the + file does, and guarding an import is common enough to be worth + reading. `ast.walk` yields the shallowest first, so a name bound + at the top of the file wins over one bound inside something. + + Since we cannot tell whether `y` in `from x import y` is a module + of its own or a name defined in `x`, `x.y` is listed as a module + too. Adding that guess is safe because a module only becomes a + file if a root holds one by that name, so `x.y` naming something + that is not a module resolves to nothing and is dropped. + """ + symbols: dict[str, tuple[str, str]] = {} + imported_modules: list[str] = [] + + for node in ast.walk(module): + # A relative import names no module of its own, and + # `from . import x` has nowhere to be read from here. + match node: + case ast.Import(names=names): + imported_modules.extend(alias.name for alias in names) + case ast.ImportFrom( + module=str(imported_module), level=0, names=names + ): + imported_modules.append(imported_module) + for alias in names: + symbols.setdefault( + alias.asname or alias.name, + (imported_module, alias.name), + ) + imported_modules.append(f'{imported_module}.{alias.name}') + + return symbols, imported_modules + + +def _resolve(imported_module: str, *, roots: list[str]) -> Optional[str]: + """Returns the file a module names if one of `roots` contains it, + and `None` otherwise. + + `None` is not a failure: the standard library and installed + packages live outside every root, so `import asyncio` resolves to + nothing and there is nothing to read. + """ + relative = imported_module.replace('.', os.sep) + + for root in roots: + for candidate in ( + os.path.join(root, relative + '.py'), + os.path.join(root, relative, '__init__.py'), + ): + if os.path.isfile(candidate): + return candidate + + return None + + +def _state_type_if_servicer( + class_definition: ast.ClassDef, *, symbols: dict[str, tuple[str, str]] +) -> Optional[str]: + """Returns the state type a class services, if it says so. + + A servicer says it by what it inherits: `Account.Servicer`, or + `Account.singleton.Servicer` for a singleton. + """ + for base in class_definition.bases: + match base: + case ( + ast.Attribute(value=ast.Name(id=name), attr='Servicer') | + ast.Attribute( + value=ast. + Attribute(value=ast.Name(id=name), attr='singleton'), + attr='Servicer', + ) + ): + match symbols.get(name): + case (str(imported_module), str(attribute) + ) if imported_module.endswith(GENERATED_SUFFIX): + package = imported_module.rsplit('.', 1)[0] + return f'{package}.{attribute}' + + return None + + +async def servicer_files( + *, + application: str, + roots: Optional[list[str]] = None, +) -> list[tuple[str, str]]: + """Returns every servicer found in the developer's application as + the state type it services and the file it is written in, sorted, + with no state type appearing twice for one file. + + A state type appearing twice is two classes servicing it, which is + for whoever reads this to make of what they will; a state type not + appearing at all is one no servicer was found for, which a file + that would not parse looks like too. The files are spelled the way + the developer would open them. + + `roots` are the directories a module may be found under, which is + both how a module name becomes a file and where the developer's + code is taken to end. It defaults to the application's own + directory, which is what running the application puts first on its + path. + """ + if roots is None: + roots = _roots(application) + + servicers: set[tuple[str, str]] = set() + read: set[str] = set() + + pending = [application] + + while len(pending) > 0: + # What the last round of imports led to. + current, pending = pending, [] + + # Parsing holds on to the interpreter for as long as it takes, + # so a file at a time leaves the dashboard free to answer. + async for filename in cooperatively(current): + if filename in read: + continue + read.add(filename) + + try: + module: ast.Module = _parse(filename) + except (SyntaxError, OSError): + # Which state type this file would have serviced is + # precisely what went unread, so there is nothing to + # say about it and it falls through to no servicer + # having been found. + continue + + # First, and on its own: `ast.walk` is breadth-first, so a + # top-level class comes out before an import nested in a + # `try`, and every name must be known before any class is + # resolved. + symbols, imported_modules = _imports(module) + + for node in ast.walk(module): + match node: + case ast.ClassDef(): + state_type = _state_type_if_servicer( + node, symbols=symbols + ) + if state_type is not None: + servicers.add((state_type, filename)) + + for imported_module in imported_modules: + resolved = _resolve(imported_module, roots=roots) + if resolved is not None: + pending.append(resolved) + + return sorted(servicers) + + +async def watch(context: WorkflowContext, *, application: str) -> None: + """Returns only when the dashboard stops, recording the servicers + in the developer's application for as long as it runs.""" + roots = _roots(application) + globs = [os.path.join(root, SOURCE_GLOB) for root in roots] + + recorded: Optional[list[tuple[str, str]]] = None + + with file_watcher() as watcher: + async for iteration in context.loop('Watch the application'): + # The watch is armed before anything is read, so a save + # made during the walk resolves `event` rather than + # arriving while nothing is listening. A watch is consumed + # by one event, so it is re-entered for each. + async with watcher.watch(globs) as event: + servicers = await servicer_files( + application=application, roots=roots + ) + + # Most edits move no servicer, and a write wakes every + # browser reading `Get`, so one is only worth making + # when the answer is different. + if servicers != recorded: + + async def record(state) -> None: + del state.servicers[:] + for state_type, file in servicers: + state.servicers.append( + ServicerInfo( + state_type=state_type, + file=file, + ) + ) + + # Written inline rather than through a method of + # its own: the workflow runs on this very state. + await Implementation.ref().per_iteration( + 'Record the servicers' + ).write(context, record) + + # After the write, so that what is remembered is + # what was recorded and not what was about to be. + recorded = servicers + + # Which save wakes this, and when, is not + # deterministic, and a replay may wait on a different + # one. Nothing depends on that: an iteration reads the + # application from scratch and writes what it finds, + # so one that runs at a different moment writes the + # same answer or a newer one. + await event diff --git a/reboot/dashboard/main.py b/reboot/dashboard/main.py index fdfc9e1b..7c24d3af 100644 --- a/reboot/dashboard/main.py +++ b/reboot/dashboard/main.py @@ -9,13 +9,14 @@ """ import asyncio from pathlib import Path -from rbt.dashboard.v1.dashboard_rbt import API, Preferences +from rbt.dashboard.v1.dashboard_rbt import API, Implementation, Preferences from rbt.std.presence.v1.presence_rbt import Presence from reboot.aio.applications import Application from reboot.aio.external import InitializeContext from reboot.dashboard.constants import ( API_ID, DASHBOARD_PATH, + IMPLEMENTATION_ID, PREFERENCES_ID, PRESENCE_ID, ) @@ -70,10 +71,15 @@ async def initialize(context: InitializeContext) -> None: # dashboard is up. await Presence.ref(PRESENCE_ID).Create(context) - # Idempotently, so that a restart of a named application finds - # the `Watch` it already spawned rather than starting a second - # watcher. - _ = await API.ref(API_ID).idempotently('watch').spawn().Watch(context) + # Idempotency is required of every mutation from `initialize`, + # and needs no alias: the key is derived from the method, the + # state id and `initialize`'s seed, which is itself derived from + # the application. So a restart finds the watchers it already + # spawned rather than starting more. + _ = await API.ref(API_ID).idempotently().spawn().Watch(context) + + _ = await Implementation.ref(IMPLEMENTATION_ID + ).idempotently().spawn().Watch(context) async def main(): diff --git a/reboot/dashboard/servicers.py b/reboot/dashboard/servicers.py index dec9cd41..5a8804e2 100644 --- a/reboot/dashboard/servicers.py +++ b/reboot/dashboard/servicers.py @@ -13,18 +13,24 @@ PreferencesSetSuppressOpenOnRestartRequest, PreferencesSetSuppressOpenOnRestartResponse, ) -from rbt.dashboard.v1.dashboard_rbt import API, Preferences +from rbt.dashboard.v1.dashboard_rbt import API, Implementation, Preferences from reboot.aio.auth.authorizers import allow from reboot.aio.contexts import ReaderContext, WorkflowContext, WriterContext from reboot.aio.servicers import Servicer -from reboot.dashboard.api_watcher import watch -from reboot.dashboard.constants import ENVVAR_RBT_API_DIRECTORY +from reboot.dashboard import api_watcher, implementation_watcher +from reboot.dashboard.constants import ( + ENVVAR_RBT_API_DIRECTORY, + ENVVAR_RBT_APPLICATION, +) class APIServicer(API.Servicer): """Holds the shape the developer's API files declare.""" def authorizer(self): + # Anyone who can reach this can already read the files it + # describes: it holds nothing but the shape of the developer's + # own API files, and only ever runs under `rbt dashboard`. return allow() async def Get( @@ -43,7 +49,8 @@ async def Watch( context: WorkflowContext, request: API.WatchRequest, ) -> API.WatchResponse: - """Reads the developer's API files when they change. + """Returns only when the dashboard stops, reading the + developer's API files whenever they change. The directory comes from the environment each time this runs, so that an `rbt dashboard` restarted against a different one @@ -52,7 +59,7 @@ async def Watch( """ api_directory = os.environ[ENVVAR_RBT_API_DIRECTORY] - await watch(context, api_directory=api_directory) + await api_watcher.watch(context, api_directory=api_directory) return API.WatchResponse() @@ -67,6 +74,47 @@ async def Update( return APIUpdateResponse() +class ImplementationServicer(Implementation.Servicer): + """Holds where each state type the developer declared is + implemented.""" + + def authorizer(self): + # Anyone who can reach this can already read the files it + # names: it holds nothing but paths into the developer's own + # checkout, and only ever runs under `rbt dashboard`. + return allow() + + async def Get( + self, + context: ReaderContext, + request: Implementation.GetRequest, + ) -> Implementation.GetResponse: + return Implementation.GetResponse(servicers=self.state.servicers) + + @classmethod + async def Watch( + cls, + context: WorkflowContext, + request: Implementation.WatchRequest, + ) -> Implementation.WatchResponse: + """Returns only when the dashboard stops, working out which of + the developer's files implements each state type. + + The application comes from the environment each time this + runs, for the same reason the API directory does. A developer + who named none gets nothing looked for, which is the normal + case for a Node.js application. + """ + application = os.environ.get(ENVVAR_RBT_APPLICATION) + + if application is not None: + await implementation_watcher.watch( + context, application=application + ) + + return Implementation.WatchResponse() + + class PreferencesServicer(Preferences.Servicer): """Holds what the developer has said about their dashboard. @@ -77,6 +125,9 @@ class PreferencesServicer(Preferences.Servicer): """ def authorizer(self): + # Nothing here is worth keeping from anyone who can reach it: + # it holds what this machine's own browser was told about + # opening dashboards, and only ever runs under `rbt dashboard`. return allow() async def Get( @@ -115,7 +166,7 @@ async def SetExpanded( def servicers() -> list[type[Servicer]]: - """The servicers that back the dashboard's own state. + """Returns the servicers that back the dashboard's own state. This state belongs to the dashboard rather than to the application being developed, so it lives in its own application and its own @@ -127,5 +178,6 @@ def servicers() -> list[type[Servicer]]: """ return [ APIServicer, + ImplementationServicer, PreferencesServicer, ] + reboot.std.presence.v1.presence.servicers() diff --git a/tests/reboot/cli/dashboard_tests.py b/tests/reboot/cli/dashboard_tests.py index 9371b03d..68b06986 100644 --- a/tests/reboot/cli/dashboard_tests.py +++ b/tests/reboot/cli/dashboard_tests.py @@ -49,6 +49,43 @@ async def test_the_api_directory_comes_from_generate(self) -> None: self.assertEqual(dashboard._api_directory(parser), 'api/') + async def test_the_application_comes_from_dev_run(self) -> None: + """Named once, where `rbt dev run` already needs it.""" + with tempfile.TemporaryDirectory() as state_directory: + args, parser = self._parse( + state_directory, + rbtrc=( + 'generate api/\n' + 'dev run --application=backend/src/main.py' + ), + ) + + env = dashboard._dashboard_env( + args, + parser, + port=DEFAULT_DASHBOARD_PORT, + api_directory=dashboard._api_directory(parser), + application=dashboard._application(parser), + ) + + self.assertEqual(env['RBT_APPLICATION'], 'backend/src/main.py') + + async def test_an_rbtrc_that_names_no_application(self) -> None: + """Somebody who names none gets a dashboard that looks for no + implementations, rather than an error.""" + with tempfile.TemporaryDirectory() as state_directory: + args, parser = self._parse(state_directory, rbtrc='generate api/') + + env = dashboard._dashboard_env( + args, + parser, + port=DEFAULT_DASHBOARD_PORT, + api_directory=dashboard._api_directory(parser), + application=dashboard._application(parser), + ) + + self.assertNotIn('RBT_APPLICATION', env) + async def test_an_rbtrc_that_says_nothing_about_generate(self) -> None: with tempfile.TemporaryDirectory() as state_directory: _, parser = self._parse( @@ -81,6 +118,7 @@ async def test_env_is_isolated_from_any_application(self) -> None: parser, port=DEFAULT_DASHBOARD_PORT, api_directory=dashboard._api_directory(parser), + application=dashboard._application(parser), ) self.assertEqual(env['RBT_NAME'], 'dashboard') @@ -115,6 +153,7 @@ async def test_keys_differ_from_any_application(self) -> None: parser, port=DEFAULT_DASHBOARD_PORT, api_directory=dashboard._api_directory(parser), + application=dashboard._application(parser), ) self.assertNotEqual(env['REBOOT_CRYPTO_ROOT_KEYS'], 'v1:theirs') @@ -126,6 +165,7 @@ async def test_keys_differ_from_any_application(self) -> None: parser, port=DEFAULT_DASHBOARD_PORT, api_directory=dashboard._api_directory(parser), + application=dashboard._application(parser), ) self.assertEqual( env['REBOOT_CRYPTO_ROOT_KEYS'], @@ -141,6 +181,7 @@ async def test_is_told_where_the_api_files_are(self) -> None: parser, port=DEFAULT_DASHBOARD_PORT, api_directory=dashboard._api_directory(parser), + application=dashboard._application(parser), ) # As the developer spelled it, so files can be shown as diff --git a/tests/reboot/dashboard/BUILD.bazel b/tests/reboot/dashboard/BUILD.bazel index c9956e23..39fa43f4 100644 --- a/tests/reboot/dashboard/BUILD.bazel +++ b/tests/reboot/dashboard/BUILD.bazel @@ -22,6 +22,17 @@ py_test( ], ) +py_test( + name = "implementation_watcher_tests_py", + srcs = ["implementation_watcher_tests.py"], + main = "implementation_watcher_tests.py", + deps = [ + "//reboot/aio:tests_py", + "//reboot/dashboard:implementation_watcher_py", + "//reboot/dashboard:main_py", + ], +) + py_test( name = "application_tests_py", srcs = [":application_tests.py"], diff --git a/tests/reboot/dashboard/implementation_watcher_tests.py b/tests/reboot/dashboard/implementation_watcher_tests.py new file mode 100644 index 00000000..85f5dde4 --- /dev/null +++ b/tests/reboot/dashboard/implementation_watcher_tests.py @@ -0,0 +1,532 @@ +"""Where a state type is implemented follows what the API declares. + +The API files say which state types exist, so a state type appearing +is what sets the dashboard looking for the file that implements it. +""" +import os +import tempfile +import unittest +from pathlib import Path +from rbt.dashboard.v1.dashboard_rbt import API, Implementation +from reboot.aio.tests import Reboot +from reboot.dashboard.constants import ( + API_ID, + ENVVAR_RBT_API_DIRECTORY, + ENVVAR_RBT_APPLICATION, + IMPLEMENTATION_ID, +) +from reboot.dashboard.implementation_watcher import servicer_files +from reboot.dashboard.main import application +from unittest.mock import patch + +API_FILE = ''' +from reboot.api import API, Field, Methods, Model, Reader, Type + + +class {state}State(Model): + name: str = Field(tag=1) + + +class LookRequest(Model): + item: str = Field(tag=1) + + +class LookResponse(Model): + found: bool = Field(tag=1) + + +{state}Methods = Methods( + look=Reader( + request=LookRequest, + response=LookResponse, + description=None, + mcp=None, + ), +) + +api = API( + {state}=Type( + state={state}State, + methods={state}Methods, + description={description}, + ) +) +''' + +SERVICER = ''' +from shop.v1.{module}_rbt import {state} + + +class {state}Servicer({state}.Servicer): + + async def look(self, context, request): + pass +''' + +SINGLETON = ''' +from shop.v1.{module}_rbt import {state} + + +class {state}Servicer({state}.singleton.Servicer): + + async def stock(self, context, request): + pass +''' + +# The two the file-finding tests use, spelled out once. +SHOP = SERVICER.format(state='Shop', module='shop') +DEPOT = SINGLETON.format(state='Depot', module='depot') + +APPLICATION = ''' +from shop_servicer import ShopServicer +from reboot.aio.applications import Application + + +async def main(): + await Application(servicers=[ShopServicer]).run() +''' + + +class ImplementationWatcherTest(unittest.IsolatedAsyncioTestCase): + + async def asyncSetUp(self) -> None: + # Both are read when the application comes up, so they have to + # exist and be named first. + self._api = tempfile.TemporaryDirectory() + self._source = tempfile.TemporaryDirectory() + self.api = Path(self._api.name) + self.source = Path(self._source.name) + + (self.source / 'shop_servicer.py').write_text( + SERVICER.format(state='Shop', module='shop') + ) + (self.source / 'main.py').write_text(APPLICATION) + + self._environment = patch.dict( + os.environ, + { + ENVVAR_RBT_API_DIRECTORY: str(self.api), + ENVVAR_RBT_APPLICATION: str(self.source / 'main.py'), + }, + ) + self._environment.start() + + self.rbt = Reboot() + await self.rbt.start() + await self.rbt.up(application(), local_envoy=True) + + async def asyncTearDown(self) -> None: + await self.rbt.stop() + self._environment.stop() + self._source.cleanup() + self._api.cleanup() + + def _declare( + self, + name: str, + *, + state: str, + description: str = 'None', + ) -> None: + path = self.api / 'shop' / 'v1' / f'{name}.py' + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + API_FILE.format( + state=state, + description='None' + if description == 'None' else repr(description), + ) + ) + + async def _servicers(self, *, satisfied): + """Returns the files recorded against each state type once + they satisfy, reading again whenever they change. + + A list per state type, because two classes servicing one is + two entries rather than anything the recording adjudicates. + """ + context = self.rbt.create_external_context(name=self.id()) + + async for response in Implementation.ref(IMPLEMENTATION_ID + ).reactively().Get(context): + servicers: dict[str, list[str]] = {} + + for servicer in response.servicers: + servicers.setdefault(servicer.state_type, + []).append(servicer.file) + + if satisfied(servicers): + return servicers + + raise AssertionError('never satisfied') + + async def test_records_the_servicer_it_finds(self) -> None: + servicers = await self._servicers( + satisfied=lambda found: 'shop.v1.Shop' in found + ) + + self.assertEqual( + servicers['shop.v1.Shop'], + [str(self.source / 'shop_servicer.py')], + ) + + async def test_a_state_type_nothing_services(self) -> None: + """A state type nothing services is one with no entry, which + is how a reader tells it apart from one that was placed.""" + self._declare('depot', state='Depot') + + servicers = await self._servicers( + satisfied=lambda found: 'shop.v1.Shop' in found + ) + + self.assertNotIn('shop.v1.Depot', servicers) + + async def test_a_state_type_two_classes_service(self) -> None: + """Both files are recorded against it, rather than one being + chosen between them.""" + (self.source / 'other_servicer.py').write_text(SHOP) + (self.source / 'main.py').write_text( + APPLICATION.replace( + 'from shop_servicer import ShopServicer', + 'from other_servicer import ShopServicer as Other\n' + 'from shop_servicer import ShopServicer', + ) + ) + + servicers = await self._servicers( + satisfied=lambda found: len(found.get('shop.v1.Shop', [])) == 2 + ) + + self.assertEqual( + servicers['shop.v1.Shop'], [ + str(self.source / 'other_servicer.py'), + str(self.source / 'shop_servicer.py'), + ] + ) + + async def test_a_servicer_written_after_the_dashboard_started( + self + ) -> None: + """The application is watched, so a servicer written while the + dashboard runs is found without a restart.""" + await self._servicers(satisfied=lambda found: 'shop.v1.Shop' in found) + + (self.source / 'depot_servicer.py').write_text(DEPOT) + (self.source / 'main.py').write_text( + APPLICATION.replace( + 'from shop_servicer import ShopServicer', + 'from depot_servicer import DepotServicer\n' + 'from shop_servicer import ShopServicer', + ) + ) + + servicers = await self._servicers( + satisfied=lambda found: 'shop.v1.Depot' in found + ) + + self.assertEqual( + servicers['shop.v1.Depot'], + [str(self.source / 'depot_servicer.py')], + ) + + async def test_what_is_declared_and_what_implements_it_are_separate( + self + ) -> None: + """Read from different places by different workflows, and so + recorded without either waiting on the other.""" + self._declare('shop', state='Shop') + + servicers = await self._servicers( + satisfied=lambda found: 'shop.v1.Shop' in found + ) + self.assertEqual( + servicers['shop.v1.Shop'], + [str(self.source / 'shop_servicer.py')], + ) + + context = self.rbt.create_external_context(name=self.id()) + + async for response in API.ref(API_ID).reactively().Get(context): + if any( + state_type.name == 'shop.v1.Shop' + for state_type in response.state_types + ): + return + + raise AssertionError("'shop.v1.Shop' was never declared") + + +class ServicerFilesTest(unittest.IsolatedAsyncioTestCase): + """Which file implements which state type.""" + + def setUp(self) -> None: + self._directory = tempfile.TemporaryDirectory() + self.directory = Path(self._directory.name) + + def tearDown(self) -> None: + self._directory.cleanup() + + def _write(self, name: str, *, source: str) -> str: + path = self.directory / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(source) + return str(path) + + async def test_finds_the_file_a_state_type_is_implemented_in(self) -> None: + self._write('shop_servicer.py', source=SHOP) + application = self._write('main.py', source=APPLICATION) + + servicers = await servicer_files(application=application) + + self.assertEqual( + servicers, + [('shop.v1.Shop', str(self.directory / 'shop_servicer.py'))], + ) + + async def test_a_singleton_says_what_it_services_the_same_way( + self + ) -> None: + self._write('depot_servicer.py', source=DEPOT) + application = self._write( + 'main.py', + source=APPLICATION.replace('shop_servicer', + 'depot_servicer').replace( + 'ShopServicer', 'DepotServicer' + ), + ) + + servicers = await servicer_files(application=application) + + self.assertEqual( + servicers, + [('shop.v1.Depot', str(self.directory / 'depot_servicer.py'))], + ) + + async def test_several_state_types_in_one_file(self) -> None: + """A file is named after at most one of the state types it + implements, which is why the application is what says.""" + self._write('servicers.py', source=SHOP + DEPOT) + application = self._write( + 'main.py', source=''' +from servicers import DepotServicer, ShopServicer +from reboot.aio.applications import Application + + +async def main(): + await Application(servicers=[ShopServicer, DepotServicer]).run() +''' + ) + + servicers = await servicer_files(application=application) + + self.assertEqual( + servicers, [ + ('shop.v1.Depot', str(self.directory / 'servicers.py')), + ('shop.v1.Shop', str(self.directory / 'servicers.py')), + ] + ) + + ################################################################### + # Reached however the application reaches it. + + async def test_an_application_that_does_not_spell_out_its_servicers( + self + ) -> None: + """Only running it would say what `servicers()` returns -- but + its module had to be imported for it to be callable at all, + which is enough.""" + self._write('servicers.py', source=SHOP) + application = self._write( + 'main.py', source=''' +from servicers import servicers +from reboot.aio.applications import Application + + +async def main(): + await Application(servicers=servicers()).run() +''' + ) + + servicers = await servicer_files(application=application) + + self.assertEqual( + servicers, + [('shop.v1.Shop', str(self.directory / 'servicers.py'))], + ) + + async def test_a_servicer_reached_through_another_module(self) -> None: + """Imported by something imported by the application.""" + self._write('shop_servicer.py', source=SHOP) + self._write( + 'servicers.py', source='from shop_servicer import ShopServicer\n' + ) + application = self._write( + 'main.py', source=''' +from servicers import ShopServicer +from reboot.aio.applications import Application + + +async def main(): + await Application(servicers=[ShopServicer]).run() +''' + ) + + servicers = await servicer_files(application=application) + + self.assertEqual( + servicers, + [('shop.v1.Shop', str(self.directory / 'shop_servicer.py'))], + ) + + async def test_an_import_that_is_not_at_the_top_of_the_file(self) -> None: + """Guarding an import is common; it binds its name just the + same.""" + self._write('shop_servicer.py', source=SHOP) + application = self._write( + 'main.py', source=''' +import os +from reboot.aio.applications import Application + +if os.environ.get('LEGACY'): + from legacy_servicer import ShopServicer +else: + from shop_servicer import ShopServicer + + +async def main(): + await Application(servicers=[ShopServicer]).run() +''' + ) + + servicers = await servicer_files(application=application) + + self.assertEqual( + servicers, + [('shop.v1.Shop', str(self.directory / 'shop_servicer.py'))], + ) + + async def test_a_servicer_in_a_package(self) -> None: + self._write('servicers/__init__.py', source='') + self._write('servicers/shop.py', source=SHOP) + application = self._write( + 'main.py', source=''' +from servicers.shop import ShopServicer +from reboot.aio.applications import Application + + +async def main(): + await Application(servicers=[ShopServicer]).run() +''' + ) + + servicers = await servicer_files(application=application) + + self.assertEqual( + servicers, + [('shop.v1.Shop', str(self.directory / 'servicers' / 'shop.py'))], + ) + + ################################################################### + # Where the walk stops. + + async def test_an_import_of_somebody_elses_package_leads_nowhere( + self + ) -> None: + """A module no root holds is not the developer's code. Reading + it is not the dashboard's business, and no state type of theirs + is waiting for it.""" + elsewhere = tempfile.TemporaryDirectory() + try: + (Path(elsewhere.name) / 'library.py').write_text(SHOP) + + application = self._write( + 'main.py', source=''' +from library import ShopServicer +from reboot.aio.applications import Application + + +async def main(): + await Application(servicers=[ShopServicer]).run() +''' + ) + + servicers = await servicer_files(application=application) + + self.assertEqual(servicers, []) + + # Named as a root, the very same import leads there. + servicers = await servicer_files( + application=application, + roots=[str(self.directory), elsewhere.name], + ) + self.assertEqual( + servicers, + [('shop.v1.Shop', str(Path(elsewhere.name) / 'library.py'))], + ) + finally: + elsewhere.cleanup() + + ################################################################### + # What it cannot place. + + async def test_a_file_that_will_not_parse(self) -> None: + """Its servicers go unfound, because which state types they + service is precisely what went unread.""" + self._write('shop_servicer.py', source='class ShopServicer(') + application = self._write('main.py', source=APPLICATION) + + servicers = await servicer_files(application=application) + + self.assertEqual(servicers, []) + + async def test_an_application_that_is_not_there(self) -> None: + servicers = await servicer_files( + application=str(self.directory / 'nowhere.py') + ) + + self.assertEqual(servicers, []) + + async def test_two_classes_servicing_the_same_state_type(self) -> None: + """Both are recorded, rather than one being chosen between + them: which one runs is not something this can see.""" + self._write('shop_servicer.py', source=SHOP) + self._write('other_servicer.py', source=SHOP) + application = self._write( + 'main.py', source=''' +from other_servicer import ShopServicer as Other +from shop_servicer import ShopServicer +from reboot.aio.applications import Application + + +async def main(): + await Application(servicers=[ShopServicer]).run() +''' + ) + + servicers = await servicer_files(application=application) + + self.assertEqual( + servicers, [ + ('shop.v1.Shop', str(self.directory / 'other_servicer.py')), + ('shop.v1.Shop', str(self.directory / 'shop_servicer.py')), + ] + ) + + async def test_a_class_that_services_nothing(self) -> None: + """A class whose base names no generated module is not a + servicer this can place.""" + self._write( + 'shop_servicer.py', source=''' +class ShopServicer(SomethingElse): + pass +''' + ) + application = self._write('main.py', source=APPLICATION) + + servicers = await servicer_files(application=application) + + self.assertEqual(servicers, []) + + +if __name__ == '__main__': + unittest.main() From 2fc04d44b2a52707bfd44afdd889175ef38affaa Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Mon, 17 Aug 2026 06:06:37 +0000 Subject: [PATCH 09/31] Parse again only the files that changed The walk parsed every file the application reaches on every save, and an edit changes one of them. It now records what each file was found to hold along with a digest of the bytes it held, and parses one again only when those bytes differ. A digest and not `st_mtime_ns`, which is only as fine as the kernel's coarse clock: measured here, 163 of 200 consecutive rewrites of a file shared an mtime, so a save landing in the same tick as a read would have left that file looking untouched for good. Reading and hashing 50 files costs 1.6ms against 74ms to parse them, so asking exactly is still nearly all of the saving. Reachability is still worked out from the application every time, but over what is already held rather than by parsing: a file that stops being imported drops out however recently it changed, and one that starts being imported is parsed for the first time. A file that will not parse is left unrecorded, so it is tried again on the next save. `File` is where the analysis will attach: it says what a file holds, and asking whether that is still true is the question a hash of each method will answer one level finer. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01QPEVMhyDRxZEuH8eiykv99 --- reboot/dashboard/implementation_watcher.py | 170 +++++++++++----- .../dashboard/implementation_watcher_tests.py | 189 +++++++++++++----- 2 files changed, 262 insertions(+), 97 deletions(-) diff --git a/reboot/dashboard/implementation_watcher.py b/reboot/dashboard/implementation_watcher.py index c43ccd9a..e7ccce55 100644 --- a/reboot/dashboard/implementation_watcher.py +++ b/reboot/dashboard/implementation_watcher.py @@ -28,12 +28,14 @@ class AccountServicer(Account.Servicer): while a large one is walked. And driven by the filesystem, because that is what it is a function -of: where a state type is implemented can only move when the -developer's source moves, so an edit under the roots is what wakes +of: where a state type is implemented can only change when the +developer's source changes, so an edit under the roots is what wakes this, and nothing else does. """ import ast +import hashlib import os +from dataclasses import dataclass from rbt.dashboard.v1.dashboard_pb2 import ServicerInfo from rbt.dashboard.v1.dashboard_rbt import Implementation from reboot.aio.contexts import WorkflowContext @@ -55,9 +57,9 @@ def _roots(application: str) -> list[str]: return [os.path.dirname(application)] -def _parse(filename: str) -> ast.Module: - with open(filename) as file: - return ast.parse(file.read()) +def _read(filename: str) -> bytes: + with open(filename, 'rb') as file: + return file.read() def _imports( @@ -151,20 +153,87 @@ def _state_type_if_servicer( return None -async def servicer_files( - *, - application: str, - roots: Optional[list[str]] = None, -) -> list[tuple[str, str]]: - """Returns every servicer found in the developer's application as - the state type it services and the file it is written in, sorted, - with no state type appearing twice for one file. +@dataclass(frozen=True, kw_only=True) +class File: + """What one of the developer's files was found to hold. + + `digest` is of the bytes it held, and is what says whether + parsing it again would say anything new. Not `st_mtime_ns`, which is only + as fine as the kernel's coarse clock -- around ten milliseconds -- + so a save landing in the same tick as a read leaves an mtime that + says nothing happened. + """ + digest: bytes + imported_modules: list[str] + state_types: list[str] + + +def _parse(source: bytes, *, digest: bytes) -> Optional[File]: + """Returns what a file holds, and `None` when it will not parse. + + A file that will not parse is left unrecorded rather than recorded + as empty, so that the next round parses it again: half-written + is the normal state of a file somebody is typing into. + """ + try: + module: ast.Module = ast.parse(source) + except SyntaxError: + return None + + # First, and on its own: `ast.walk` is breadth-first, so a + # top-level class comes out before an import nested in a `try`, + # and every name must be known before any class is resolved. + symbols, imported_modules = _imports(module) + + state_types: set[str] = set() + + for node in ast.walk(module): + match node: + case ast.ClassDef(): + state_type = _state_type_if_servicer(node, symbols=symbols) + if state_type is not None: + state_types.add(state_type) + + return File( + digest=digest, + imported_modules=imported_modules, + state_types=sorted(state_types), + ) + + +def servicers(files: dict[str, File]) -> list[tuple[str, str]]: + """Returns every servicer found, as the state type it services and + the file it is written in, sorted. A state type appearing twice is two classes servicing it, which is for whoever reads this to make of what they will; a state type not appearing at all is one no servicer was found for, which a file - that would not parse looks like too. The files are spelled the way - the developer would open them. + that would not parse looks like too. + """ + return sorted( + (state_type, filename) + for filename, file in files.items() + for state_type in file.state_types + ) + + +async def files( + *, + application: str, + roots: Optional[list[str]] = None, + known: Optional[dict[str, File]] = None, +) -> dict[str, File]: + """Returns what each file the developer's application reaches + holds, keyed by the file, spelled the way they would open it. + + Only what it reaches: a file that has stopped being imported is + absent, however recently it changed, and one that has started + being imported is parsed for the first time. + + `known` is what a previous call returned, and spares this one from + parsing a file whose bytes have not changed since. Parsing is the + expensive part -- around fifty times what reading and hashing the + bytes costs -- and an edit changes one file. `roots` are the directories a module may be found under, which is both how a module name becomes a file and where the developer's @@ -175,8 +244,10 @@ async def servicer_files( if roots is None: roots = _roots(application) - servicers: set[tuple[str, str]] = set() - read: set[str] = set() + if known is None: + known = {} + + reachable: dict[str, File] = {} pending = [application] @@ -187,40 +258,32 @@ async def servicer_files( # Parsing holds on to the interpreter for as long as it takes, # so a file at a time leaves the dashboard free to answer. async for filename in cooperatively(current): - if filename in read: + if filename in reachable: continue - read.add(filename) try: - module: ast.Module = _parse(filename) - except (SyntaxError, OSError): - # Which state type this file would have serviced is - # precisely what went unread, so there is nothing to - # say about it and it falls through to no servicer - # having been found. + source: bytes = _read(filename) + except OSError: + continue + + digest = hashlib.sha256(source).digest() + + file = known.get(filename) + + if file is None or file.digest != digest: + file = _parse(source, digest=digest) + + if file is None: continue - # First, and on its own: `ast.walk` is breadth-first, so a - # top-level class comes out before an import nested in a - # `try`, and every name must be known before any class is - # resolved. - symbols, imported_modules = _imports(module) - - for node in ast.walk(module): - match node: - case ast.ClassDef(): - state_type = _state_type_if_servicer( - node, symbols=symbols - ) - if state_type is not None: - servicers.add((state_type, filename)) - - for imported_module in imported_modules: + reachable[filename] = file + + for imported_module in file.imported_modules: resolved = _resolve(imported_module, roots=roots) if resolved is not None: pending.append(resolved) - return sorted(servicers) + return reachable async def watch(context: WorkflowContext, *, application: str) -> None: @@ -230,6 +293,7 @@ async def watch(context: WorkflowContext, *, application: str) -> None: globs = [os.path.join(root, SOURCE_GLOB) for root in roots] recorded: Optional[list[tuple[str, str]]] = None + known: dict[str, File] = {} with file_watcher() as watcher: async for iteration in context.loop('Watch the application'): @@ -238,18 +302,20 @@ async def watch(context: WorkflowContext, *, application: str) -> None: # arriving while nothing is listening. A watch is consumed # by one event, so it is re-entered for each. async with watcher.watch(globs) as event: - servicers = await servicer_files( - application=application, roots=roots + known = await files( + application=application, roots=roots, known=known ) - # Most edits move no servicer, and a write wakes every + found = servicers(known) + + # Most edits change no servicer, and a write wakes every # browser reading `Get`, so one is only worth making # when the answer is different. - if servicers != recorded: + if found != recorded: async def record(state) -> None: del state.servicers[:] - for state_type, file in servicers: + for state_type, file in found: state.servicers.append( ServicerInfo( state_type=state_type, @@ -265,12 +331,12 @@ async def record(state) -> None: # After the write, so that what is remembered is # what was recorded and not what was about to be. - recorded = servicers + recorded = found # Which save wakes this, and when, is not # deterministic, and a replay may wait on a different - # one. Nothing depends on that: an iteration reads the - # application from scratch and writes what it finds, - # so one that runs at a different moment writes the - # same answer or a newer one. + # one. Nothing depends on that: an iteration parses + # whatever has changed since the last and writes what + # it finds, so one that runs at a different moment + # writes the same answer or a newer one. await event diff --git a/tests/reboot/dashboard/implementation_watcher_tests.py b/tests/reboot/dashboard/implementation_watcher_tests.py index 85f5dde4..53213863 100644 --- a/tests/reboot/dashboard/implementation_watcher_tests.py +++ b/tests/reboot/dashboard/implementation_watcher_tests.py @@ -9,13 +9,14 @@ from pathlib import Path from rbt.dashboard.v1.dashboard_rbt import API, Implementation from reboot.aio.tests import Reboot +from reboot.dashboard import implementation_watcher from reboot.dashboard.constants import ( API_ID, ENVVAR_RBT_API_DIRECTORY, ENVVAR_RBT_APPLICATION, IMPLEMENTATION_ID, ) -from reboot.dashboard.implementation_watcher import servicer_files +from reboot.dashboard.implementation_watcher import files, servicers from reboot.dashboard.main import application from unittest.mock import patch @@ -149,24 +150,23 @@ async def _servicers(self, *, satisfied): async for response in Implementation.ref(IMPLEMENTATION_ID ).reactively().Get(context): - servicers: dict[str, list[str]] = {} + found: dict[str, list[str]] = {} for servicer in response.servicers: - servicers.setdefault(servicer.state_type, - []).append(servicer.file) + found.setdefault(servicer.state_type, []).append(servicer.file) - if satisfied(servicers): - return servicers + if satisfied(found): + return found raise AssertionError('never satisfied') async def test_records_the_servicer_it_finds(self) -> None: - servicers = await self._servicers( + found = await self._servicers( satisfied=lambda found: 'shop.v1.Shop' in found ) self.assertEqual( - servicers['shop.v1.Shop'], + found['shop.v1.Shop'], [str(self.source / 'shop_servicer.py')], ) @@ -175,11 +175,11 @@ async def test_a_state_type_nothing_services(self) -> None: is how a reader tells it apart from one that was placed.""" self._declare('depot', state='Depot') - servicers = await self._servicers( + found = await self._servicers( satisfied=lambda found: 'shop.v1.Shop' in found ) - self.assertNotIn('shop.v1.Depot', servicers) + self.assertNotIn('shop.v1.Depot', found) async def test_a_state_type_two_classes_service(self) -> None: """Both files are recorded against it, rather than one being @@ -193,12 +193,12 @@ async def test_a_state_type_two_classes_service(self) -> None: ) ) - servicers = await self._servicers( + found = await self._servicers( satisfied=lambda found: len(found.get('shop.v1.Shop', [])) == 2 ) self.assertEqual( - servicers['shop.v1.Shop'], [ + found['shop.v1.Shop'], [ str(self.source / 'other_servicer.py'), str(self.source / 'shop_servicer.py'), ] @@ -220,12 +220,12 @@ async def test_a_servicer_written_after_the_dashboard_started( ) ) - servicers = await self._servicers( + found = await self._servicers( satisfied=lambda found: 'shop.v1.Depot' in found ) self.assertEqual( - servicers['shop.v1.Depot'], + found['shop.v1.Depot'], [str(self.source / 'depot_servicer.py')], ) @@ -236,11 +236,11 @@ async def test_what_is_declared_and_what_implements_it_are_separate( recorded without either waiting on the other.""" self._declare('shop', state='Shop') - servicers = await self._servicers( + found = await self._servicers( satisfied=lambda found: 'shop.v1.Shop' in found ) self.assertEqual( - servicers['shop.v1.Shop'], + found['shop.v1.Shop'], [str(self.source / 'shop_servicer.py')], ) @@ -276,10 +276,10 @@ async def test_finds_the_file_a_state_type_is_implemented_in(self) -> None: self._write('shop_servicer.py', source=SHOP) application = self._write('main.py', source=APPLICATION) - servicers = await servicer_files(application=application) + found = servicers(await files(application=application)) self.assertEqual( - servicers, + found, [('shop.v1.Shop', str(self.directory / 'shop_servicer.py'))], ) @@ -295,10 +295,10 @@ async def test_a_singleton_says_what_it_services_the_same_way( ), ) - servicers = await servicer_files(application=application) + found = servicers(await files(application=application)) self.assertEqual( - servicers, + found, [('shop.v1.Depot', str(self.directory / 'depot_servicer.py'))], ) @@ -317,10 +317,10 @@ async def main(): ''' ) - servicers = await servicer_files(application=application) + found = servicers(await files(application=application)) self.assertEqual( - servicers, [ + found, [ ('shop.v1.Depot', str(self.directory / 'servicers.py')), ('shop.v1.Shop', str(self.directory / 'servicers.py')), ] @@ -347,10 +347,10 @@ async def main(): ''' ) - servicers = await servicer_files(application=application) + found = servicers(await files(application=application)) self.assertEqual( - servicers, + found, [('shop.v1.Shop', str(self.directory / 'servicers.py'))], ) @@ -371,10 +371,10 @@ async def main(): ''' ) - servicers = await servicer_files(application=application) + found = servicers(await files(application=application)) self.assertEqual( - servicers, + found, [('shop.v1.Shop', str(self.directory / 'shop_servicer.py'))], ) @@ -398,10 +398,10 @@ async def main(): ''' ) - servicers = await servicer_files(application=application) + found = servicers(await files(application=application)) self.assertEqual( - servicers, + found, [('shop.v1.Shop', str(self.directory / 'shop_servicer.py'))], ) @@ -419,10 +419,10 @@ async def main(): ''' ) - servicers = await servicer_files(application=application) + found = servicers(await files(application=application)) self.assertEqual( - servicers, + found, [('shop.v1.Shop', str(self.directory / 'servicers' / 'shop.py'))], ) @@ -450,22 +450,121 @@ async def main(): ''' ) - servicers = await servicer_files(application=application) + found = servicers(await files(application=application)) - self.assertEqual(servicers, []) + self.assertEqual(found, []) # Named as a root, the very same import leads there. - servicers = await servicer_files( - application=application, - roots=[str(self.directory), elsewhere.name], + found = servicers( + await files( + application=application, + roots=[str(self.directory), elsewhere.name], + ) ) self.assertEqual( - servicers, + found, [('shop.v1.Shop', str(Path(elsewhere.name) / 'library.py'))], ) finally: elsewhere.cleanup() + ################################################################### + # Parse again only what has changed. + + async def test_a_file_written_with_the_same_bytes_is_not_parsed_again( + self + ) -> None: + """Identical bytes are nothing to parse, which is only + observable as the parsing not happening.""" + servicer = self._write('shop_servicer.py', source=SHOP) + application = self._write('main.py', source=APPLICATION) + + known = await files(application=application) + + Path(servicer).write_text(SHOP) + + with patch.object( + implementation_watcher, + '_parse', + wraps=implementation_watcher._parse, + ) as parse: + await files(application=application, known=known) + + parse.assert_not_called() + + async def test_a_file_written_with_other_bytes_is_parsed_again( + self + ) -> None: + """Even with the mtime it was read with put back, which is + what a save landing in the same clock tick leaves behind.""" + servicer = self._write('shop_servicer.py', source=SHOP) + application = self._write('main.py', source=APPLICATION) + + modified = os.stat(servicer).st_mtime_ns + known = await files(application=application) + + Path(servicer).write_text(DEPOT) + os.utime(servicer, ns=(modified, modified)) + + found = servicers(await files(application=application, known=known)) + + self.assertEqual(found, [('shop.v1.Depot', servicer)]) + + async def test_a_servicer_that_stops_being_imported_is_dropped( + self + ) -> None: + """However recently it changed: what the application reaches + is what it registers.""" + self._write('shop_servicer.py', source=SHOP) + self._write('depot_servicer.py', source=DEPOT) + application = self._write( + 'main.py', + source=APPLICATION.replace( + 'from shop_servicer import ShopServicer', + 'from depot_servicer import DepotServicer\n' + 'from shop_servicer import ShopServicer', + ), + ) + + known = await files(application=application) + self.assertEqual(len(servicers(known)), 2) + + self._write('main.py', source=APPLICATION) + + found = servicers(await files(application=application, known=known)) + + self.assertEqual( + found, + [('shop.v1.Shop', str(self.directory / 'shop_servicer.py'))], + ) + + async def test_a_servicer_that_starts_being_imported_is_found( + self + ) -> None: + self._write('shop_servicer.py', source=SHOP) + application = self._write('main.py', source=APPLICATION) + + known = await files(application=application) + + self._write('depot_servicer.py', source=DEPOT) + self._write( + 'main.py', + source=APPLICATION.replace( + 'from shop_servicer import ShopServicer', + 'from depot_servicer import DepotServicer\n' + 'from shop_servicer import ShopServicer', + ), + ) + + found = servicers(await files(application=application, known=known)) + + self.assertEqual( + found, [ + ('shop.v1.Depot', str(self.directory / 'depot_servicer.py')), + ('shop.v1.Shop', str(self.directory / 'shop_servicer.py')), + ] + ) + ################################################################### # What it cannot place. @@ -475,16 +574,16 @@ async def test_a_file_that_will_not_parse(self) -> None: self._write('shop_servicer.py', source='class ShopServicer(') application = self._write('main.py', source=APPLICATION) - servicers = await servicer_files(application=application) + found = servicers(await files(application=application)) - self.assertEqual(servicers, []) + self.assertEqual(found, []) async def test_an_application_that_is_not_there(self) -> None: - servicers = await servicer_files( - application=str(self.directory / 'nowhere.py') + found = servicers( + await files(application=str(self.directory / 'nowhere.py')) ) - self.assertEqual(servicers, []) + self.assertEqual(found, []) async def test_two_classes_servicing_the_same_state_type(self) -> None: """Both are recorded, rather than one being chosen between @@ -503,10 +602,10 @@ async def main(): ''' ) - servicers = await servicer_files(application=application) + found = servicers(await files(application=application)) self.assertEqual( - servicers, [ + found, [ ('shop.v1.Shop', str(self.directory / 'other_servicer.py')), ('shop.v1.Shop', str(self.directory / 'shop_servicer.py')), ] @@ -523,9 +622,9 @@ class ShopServicer(SomethingElse): ) application = self._write('main.py', source=APPLICATION) - servicers = await servicer_files(application=application) + found = servicers(await files(application=application)) - self.assertEqual(servicers, []) + self.assertEqual(found, []) if __name__ == '__main__': From 98150b83a6afc5bf6c22fe22c6cd87f2405214b4 Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Mon, 17 Aug 2026 06:58:38 +0000 Subject: [PATCH 10/31] Record the methods each servicer defines Scaffolding for analyzing them: what a servicer is made of, and a digest of each method that says whether analyzing it again would say anything new. Nothing is analyzed yet -- a `Method` is a name and a digest, and what it calls is the field that follows. The digest is over `ast.dump` without attributes, so it is of what the method says rather than how it is laid out: reformatting it, writing a comment in it, or pushing it down the file with an edit above leave it alone. That is what will keep an application of a thousand state types from re-analyzing everything on every save, one level finer than the file digest already does for parsing. Methods are recorded in the order they are written, which is the order somebody reading the file meets them. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01QPEVMhyDRxZEuH8eiykv99 --- rbt/dashboard/v1/dashboard.proto | 20 +++ reboot/dashboard/implementation_watcher.py | 89 +++++++---- .../dashboard/implementation_watcher_tests.py | 150 ++++++++++++++---- 3 files changed, 201 insertions(+), 58 deletions(-) diff --git a/rbt/dashboard/v1/dashboard.proto b/rbt/dashboard/v1/dashboard.proto index ce1d7c68..4f16d961 100644 --- a/rbt/dashboard/v1/dashboard.proto +++ b/rbt/dashboard/v1/dashboard.proto @@ -80,12 +80,32 @@ message APIWatchResponse {} // One servicer found in the developer's application, and the state // type it services. message ServicerInfo { + // Represents one method a servicer defines, and what analyzing it + // found. + message Method { + // Represents the name of the method as the developer wrote it, + // e.g. "deposit". A method the API files declare is spelled the + // same way there. + string name = 1; + + // Represents a hash digest of the method's syntax with the lines + // and columns left out, so that a comment added above the method + // or arguments rewrapped across lines do not change it. It says + // whether the method does anything different, not whether it was + // written down differently. + bytes digest = 2; + } + // The state type it services, spelled as `StateTypeInfo.name`. string state_type = 1; // The file it is written in, spelled the way the developer would // open it, e.g. "backend/src/account_servicer.py". string file = 2; + + // Represents every method it defines, in the order they are + // written, which is the order somebody reading the file meets them. + repeated Method methods = 3; } // What the dashboard application has read of the developer's diff --git a/reboot/dashboard/implementation_watcher.py b/reboot/dashboard/implementation_watcher.py index e7ccce55..23422c88 100644 --- a/reboot/dashboard/implementation_watcher.py +++ b/reboot/dashboard/implementation_watcher.py @@ -155,25 +155,57 @@ def _state_type_if_servicer( @dataclass(frozen=True, kw_only=True) class File: - """What one of the developer's files was found to hold. + """What one of the developer's files was found to hold.""" - `digest` is of the bytes it held, and is what says whether - parsing it again would say anything new. Not `st_mtime_ns`, which is only - as fine as the kernel's coarse clock -- around ten milliseconds -- - so a save landing in the same tick as a read leaves an mtime that - says nothing happened. - """ + # Of the bytes the file held, saying whether parsing it again + # would say anything new. Not `st_mtime_ns`, which is only as fine + # as the kernel's coarse clock -- around ten milliseconds -- so a + # save landing in the same tick as a read leaves an mtime that + # says nothing happened. digest: bytes + + # Every module the file names, to be followed if a root holds it. imported_modules: list[str] - state_types: list[str] + # Every servicer the file defines. + servicers: list[ServicerInfo] + + +def _digest(node: ast.AST) -> bytes: + """Returns a digest of what a piece of syntax says. + + The digest is computed using `ast.dump` without attributes so that + the lines and columns are left out, and thus a comment added above + a method or arguments rewrapped across lines do not change the + digest. + """ + return hashlib.sha256(ast.dump(node, + include_attributes=False).encode()).digest() + + +def _methods(class_definition: ast.ClassDef) -> list[ServicerInfo.Method]: + """Returns the methods a class defines, in the order written.""" + methods = [] -def _parse(source: bytes, *, digest: bytes) -> Optional[File]: + for node in class_definition.body: + match node: + case ( + ast.FunctionDef(name=str(name)) | + ast.AsyncFunctionDef(name=str(name)) + ): + methods.append( + ServicerInfo.Method(name=name, digest=_digest(node)) + ) + + return methods + + +def _parse(source: bytes, *, digest: bytes, filename: str) -> Optional[File]: """Returns what a file holds, and `None` when it will not parse. A file that will not parse is left unrecorded rather than recorded - as empty, so that the next round parses it again: half-written - is the normal state of a file somebody is typing into. + as empty, so that the next round parses it again: half-written is + the normal state of a file somebody is typing into. """ try: module: ast.Module = ast.parse(source) @@ -185,25 +217,31 @@ def _parse(source: bytes, *, digest: bytes) -> Optional[File]: # and every name must be known before any class is resolved. symbols, imported_modules = _imports(module) - state_types: set[str] = set() + servicers = [] for node in ast.walk(module): match node: case ast.ClassDef(): state_type = _state_type_if_servicer(node, symbols=symbols) if state_type is not None: - state_types.add(state_type) + servicers.append( + ServicerInfo( + state_type=state_type, + file=filename, + methods=_methods(node), + ) + ) return File( digest=digest, imported_modules=imported_modules, - state_types=sorted(state_types), + servicers=servicers, ) -def servicers(files: dict[str, File]) -> list[tuple[str, str]]: - """Returns every servicer found, as the state type it services and - the file it is written in, sorted. +def servicers(files: dict[str, File]) -> list[ServicerInfo]: + """Returns every servicer found, sorted by the state type it + services and the file it is written in. A state type appearing twice is two classes servicing it, which is for whoever reads this to make of what they will; a state type not @@ -211,9 +249,8 @@ def servicers(files: dict[str, File]) -> list[tuple[str, str]]: that would not parse looks like too. """ return sorted( - (state_type, filename) - for filename, file in files.items() - for state_type in file.state_types + (servicer for file in files.values() for servicer in file.servicers), + key=lambda servicer: (servicer.state_type, servicer.file), ) @@ -271,7 +308,7 @@ async def files( file = known.get(filename) if file is None or file.digest != digest: - file = _parse(source, digest=digest) + file = _parse(source, digest=digest, filename=filename) if file is None: continue @@ -292,7 +329,7 @@ async def watch(context: WorkflowContext, *, application: str) -> None: roots = _roots(application) globs = [os.path.join(root, SOURCE_GLOB) for root in roots] - recorded: Optional[list[tuple[str, str]]] = None + recorded: Optional[list[ServicerInfo]] = None known: dict[str, File] = {} with file_watcher() as watcher: @@ -315,13 +352,7 @@ async def watch(context: WorkflowContext, *, application: str) -> None: async def record(state) -> None: del state.servicers[:] - for state_type, file in found: - state.servicers.append( - ServicerInfo( - state_type=state_type, - file=file, - ) - ) + state.servicers.extend(found) # Written inline rather than through a method of # its own: the workflow runs on this very state. diff --git a/tests/reboot/dashboard/implementation_watcher_tests.py b/tests/reboot/dashboard/implementation_watcher_tests.py index 53213863..8e8ad71a 100644 --- a/tests/reboot/dashboard/implementation_watcher_tests.py +++ b/tests/reboot/dashboard/implementation_watcher_tests.py @@ -7,6 +7,7 @@ import tempfile import unittest from pathlib import Path +from rbt.dashboard.v1.dashboard_pb2 import ServicerInfo from rbt.dashboard.v1.dashboard_rbt import API, Implementation from reboot.aio.tests import Reboot from reboot.dashboard import implementation_watcher @@ -16,7 +17,7 @@ ENVVAR_RBT_APPLICATION, IMPLEMENTATION_ID, ) -from reboot.dashboard.implementation_watcher import files, servicers +from reboot.dashboard.implementation_watcher import File, files, servicers from reboot.dashboard.main import application from unittest.mock import patch @@ -88,6 +89,14 @@ async def main(): ''' +def _state_types_and_files(files: dict[str, File]) -> list[tuple[str, str]]: + """Returns every servicer as the state type it services and the + file it is written in.""" + return [ + (servicer.state_type, servicer.file) for servicer in servicers(files) + ] + + class ImplementationWatcherTest(unittest.IsolatedAsyncioTestCase): async def asyncSetUp(self) -> None: @@ -140,8 +149,8 @@ def _declare( ) async def _servicers(self, *, satisfied): - """Returns the files recorded against each state type once - they satisfy, reading again whenever they change. + """Returns the servicers recorded against each state type + once they satisfy, reading again whenever they change. A list per state type, because two classes servicing one is two entries rather than anything the recording adjudicates. @@ -150,10 +159,10 @@ async def _servicers(self, *, satisfied): async for response in Implementation.ref(IMPLEMENTATION_ID ).reactively().Get(context): - found: dict[str, list[str]] = {} + found: dict[str, list[ServicerInfo]] = {} for servicer in response.servicers: - found.setdefault(servicer.state_type, []).append(servicer.file) + found.setdefault(servicer.state_type, []).append(servicer) if satisfied(found): return found @@ -166,13 +175,26 @@ async def test_records_the_servicer_it_finds(self) -> None: ) self.assertEqual( - found['shop.v1.Shop'], + [servicer.file for servicer in found['shop.v1.Shop']], [str(self.source / 'shop_servicer.py')], ) + async def test_the_methods_reach_the_state(self) -> None: + """What the browser will join against what the API files say + each state type declares.""" + found = await self._servicers( + satisfied=lambda found: 'shop.v1.Shop' in found + ) + + self.assertEqual( + [method.name for method in found['shop.v1.Shop'][0].methods], + ['look'], + ) + async def test_a_state_type_nothing_services(self) -> None: """A state type nothing services is one with no entry, which - is how a reader tells it apart from one that was placed.""" + is how a reader tells it apart from one a servicer was found + for.""" self._declare('depot', state='Depot') found = await self._servicers( @@ -198,7 +220,7 @@ async def test_a_state_type_two_classes_service(self) -> None: ) self.assertEqual( - found['shop.v1.Shop'], [ + [servicer.file for servicer in found['shop.v1.Shop']], [ str(self.source / 'other_servicer.py'), str(self.source / 'shop_servicer.py'), ] @@ -225,7 +247,7 @@ async def test_a_servicer_written_after_the_dashboard_started( ) self.assertEqual( - found['shop.v1.Depot'], + [servicer.file for servicer in found['shop.v1.Depot']], [str(self.source / 'depot_servicer.py')], ) @@ -240,7 +262,7 @@ async def test_what_is_declared_and_what_implements_it_are_separate( satisfied=lambda found: 'shop.v1.Shop' in found ) self.assertEqual( - found['shop.v1.Shop'], + [servicer.file for servicer in found['shop.v1.Shop']], [str(self.source / 'shop_servicer.py')], ) @@ -276,7 +298,7 @@ async def test_finds_the_file_a_state_type_is_implemented_in(self) -> None: self._write('shop_servicer.py', source=SHOP) application = self._write('main.py', source=APPLICATION) - found = servicers(await files(application=application)) + found = _state_types_and_files(await files(application=application)) self.assertEqual( found, @@ -295,7 +317,7 @@ async def test_a_singleton_says_what_it_services_the_same_way( ), ) - found = servicers(await files(application=application)) + found = _state_types_and_files(await files(application=application)) self.assertEqual( found, @@ -317,7 +339,7 @@ async def main(): ''' ) - found = servicers(await files(application=application)) + found = _state_types_and_files(await files(application=application)) self.assertEqual( found, [ @@ -347,7 +369,7 @@ async def main(): ''' ) - found = servicers(await files(application=application)) + found = _state_types_and_files(await files(application=application)) self.assertEqual( found, @@ -371,7 +393,7 @@ async def main(): ''' ) - found = servicers(await files(application=application)) + found = _state_types_and_files(await files(application=application)) self.assertEqual( found, @@ -398,7 +420,7 @@ async def main(): ''' ) - found = servicers(await files(application=application)) + found = _state_types_and_files(await files(application=application)) self.assertEqual( found, @@ -419,7 +441,7 @@ async def main(): ''' ) - found = servicers(await files(application=application)) + found = _state_types_and_files(await files(application=application)) self.assertEqual( found, @@ -450,12 +472,14 @@ async def main(): ''' ) - found = servicers(await files(application=application)) + found = _state_types_and_files( + await files(application=application) + ) self.assertEqual(found, []) # Named as a root, the very same import leads there. - found = servicers( + found = _state_types_and_files( await files( application=application, roots=[str(self.directory), elsewhere.name], @@ -468,6 +492,68 @@ async def main(): finally: elsewhere.cleanup() + ################################################################### + # The methods each servicer defines. + + async def test_records_the_methods_a_servicer_defines(self) -> None: + self._write('shop_servicer.py', source=SHOP) + application = self._write('main.py', source=APPLICATION) + + found = servicers(await files(application=application)) + + self.assertEqual( + [method.name for method in found[0].methods], ['look'] + ) + + async def test_a_method_reformatted_digests_the_same(self) -> None: + """The digest is over what the method says, so laying it out + differently or writing a comment in it is not a change.""" + self._write('shop_servicer.py', source=SHOP) + application = self._write('main.py', source=APPLICATION) + + before = servicers(await files(application=application)) + + self._write( + 'shop_servicer.py', + source=SHOP.replace( + 'async def look(self, context, request):\n pass', + 'async def look(\n' + ' self,\n' + ' context,\n' + ' request,\n' + ' ):\n' + ' # Nothing to look up yet.\n' + ' pass', + ), + ) + + after = servicers(await files(application=application)) + + self.assertEqual( + [method.digest for method in after[0].methods], + [method.digest for method in before[0].methods], + ) + + async def test_a_method_whose_body_changes_digests_differently( + self + ) -> None: + self._write('shop_servicer.py', source=SHOP) + application = self._write('main.py', source=APPLICATION) + + before = servicers(await files(application=application)) + + self._write( + 'shop_servicer.py', + source=SHOP.replace(' pass', ' return None'), + ) + + after = servicers(await files(application=application)) + + self.assertNotEqual( + after[0].methods[0].digest, + before[0].methods[0].digest, + ) + ################################################################### # Parse again only what has changed. @@ -506,7 +592,9 @@ async def test_a_file_written_with_other_bytes_is_parsed_again( Path(servicer).write_text(DEPOT) os.utime(servicer, ns=(modified, modified)) - found = servicers(await files(application=application, known=known)) + found = _state_types_and_files( + await files(application=application, known=known) + ) self.assertEqual(found, [('shop.v1.Depot', servicer)]) @@ -531,7 +619,9 @@ async def test_a_servicer_that_stops_being_imported_is_dropped( self._write('main.py', source=APPLICATION) - found = servicers(await files(application=application, known=known)) + found = _state_types_and_files( + await files(application=application, known=known) + ) self.assertEqual( found, @@ -556,7 +646,9 @@ async def test_a_servicer_that_starts_being_imported_is_found( ), ) - found = servicers(await files(application=application, known=known)) + found = _state_types_and_files( + await files(application=application, known=known) + ) self.assertEqual( found, [ @@ -566,7 +658,7 @@ async def test_a_servicer_that_starts_being_imported_is_found( ) ################################################################### - # What it cannot place. + # What it finds no servicer for. async def test_a_file_that_will_not_parse(self) -> None: """Its servicers go unfound, because which state types they @@ -574,12 +666,12 @@ async def test_a_file_that_will_not_parse(self) -> None: self._write('shop_servicer.py', source='class ShopServicer(') application = self._write('main.py', source=APPLICATION) - found = servicers(await files(application=application)) + found = _state_types_and_files(await files(application=application)) self.assertEqual(found, []) async def test_an_application_that_is_not_there(self) -> None: - found = servicers( + found = _state_types_and_files( await files(application=str(self.directory / 'nowhere.py')) ) @@ -602,7 +694,7 @@ async def main(): ''' ) - found = servicers(await files(application=application)) + found = _state_types_and_files(await files(application=application)) self.assertEqual( found, [ @@ -612,8 +704,8 @@ async def main(): ) async def test_a_class_that_services_nothing(self) -> None: - """A class whose base names no generated module is not a - servicer this can place.""" + """A class whose base names no generated module is not one + this recognizes as a servicer.""" self._write( 'shop_servicer.py', source=''' class ShopServicer(SomethingElse): @@ -622,7 +714,7 @@ class ShopServicer(SomethingElse): ) application = self._write('main.py', source=APPLICATION) - found = servicers(await files(application=application)) + found = _state_types_and_files(await files(application=application)) self.assertEqual(found, []) From 44919911a6deeb198c5e058326b1455264262d77 Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Mon, 17 Aug 2026 08:07:20 +0000 Subject: [PATCH 11/31] Track what a method's names hold The first half of analyzing a method: before anything can say that `account.deposit(...)` calls `bank.v1.Account.deposit`, something has to say what `account` holds. A name holds a reference to a state type, or the context the method was called with, or nothing this can say -- in which case it is not recorded at all. References are what a call is made through; the context is what marks a helper as worth following into, since a function handed one may make Reboot calls of its own. Names are taken together rather than by scope, so one bound inside an `if` or a comprehension is held the same way, which is what lets a reference captured by a nested function be seen. Statements are visited in the order written, not through `ast.walk`, which is breadth first and would resolve `b = a` against an `a` assigned later. Nothing calls this yet. What it holds is what the next cut reads to say which state type a call is made on. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01QPEVMhyDRxZEuH8eiykv99 --- reboot/dashboard/implementation_watcher.py | 324 +++++++++++++++++- .../dashboard/implementation_watcher_tests.py | 173 ++++++++++ 2 files changed, 490 insertions(+), 7 deletions(-) diff --git a/reboot/dashboard/implementation_watcher.py b/reboot/dashboard/implementation_watcher.py index 23422c88..12fb85e8 100644 --- a/reboot/dashboard/implementation_watcher.py +++ b/reboot/dashboard/implementation_watcher.py @@ -35,13 +35,14 @@ class AccountServicer(Account.Servicer): import ast import hashlib import os -from dataclasses import dataclass +from dataclasses import dataclass, replace from rbt.dashboard.v1.dashboard_pb2 import ServicerInfo from rbt.dashboard.v1.dashboard_rbt import Implementation from reboot.aio.contexts import WorkflowContext from reboot.aio.cooperatively import cooperatively from reboot.cli.common.watch import file_watcher -from typing import Optional +from types import MappingProxyType +from typing import Iterator, Mapping, Optional GENERATED_SUFFIX = '_rbt' @@ -126,6 +127,25 @@ def _resolve(imported_module: str, *, roots: list[str]) -> Optional[str]: return None +def _state_type_if_imported( + name: str, *, symbols: Mapping[str, tuple[str, str]] +) -> Optional[str]: + """Returns the state type a name holds, if it was imported from + generated code. + + A state type reaches the developer's code one way, as a name + imported from the module generated for its API file: `Account` + from `bank.v1.account_rbt` is `bank.v1.Account`. + """ + match symbols.get(name): + case (str(imported_module), + str(attribute)) if imported_module.endswith(GENERATED_SUFFIX): + package = imported_module.rsplit('.', 1)[0] + return f'{package}.{attribute}' + + return None + + def _state_type_if_servicer( class_definition: ast.ClassDef, *, symbols: dict[str, tuple[str, str]] ) -> Optional[str]: @@ -144,15 +164,305 @@ def _state_type_if_servicer( attr='Servicer', ) ): - match symbols.get(name): - case (str(imported_module), str(attribute) - ) if imported_module.endswith(GENERATED_SUFFIX): - package = imported_module.rsplit('.', 1)[0] - return f'{package}.{attribute}' + state_type = _state_type_if_imported(name, symbols=symbols) + if state_type is not None: + return state_type return None +@dataclass(frozen=True, kw_only=True) +class Reference: + """A name holding a reference to one of the developer's state + types, such as `Account.ref(id)` was assigned to.""" + + # The state type referred to, spelled as `StateTypeInfo.name`. + state_type: str + + +@dataclass(frozen=True, kw_only=True) +class Context: + """A name holding the context a method was called with.""" + + +# What a name in a method body was found to hold. A name holding +# anything else is not here at all: what it holds is not something +# this can say. +Local = Reference | Context + + +def _statements(node: ast.AST) -> Iterator[ast.stmt]: + """Returns every statement under a node, in the order written. + + Depth first, unlike `ast.walk`, which is breadth first and so + would yield an assignment at the top of a method after one nested + inside an `if` further down. + """ + for child in ast.iter_child_nodes(node): + if isinstance(child, ast.stmt): + yield child + yield from _statements(child) + + +@dataclass(frozen=True, kw_only=True) +class Call: + """One Reboot call a method's body was found to make.""" + + # The state type the call is made on, spelled as + # `StateTypeInfo.name`. + state_type: str + + # The method called, as the developer wrote it. + method: str + + +@dataclass(frozen=True, kw_only=True) +class Analysis: + """What analyzing one method's body has found so far. + + Immutable: carrying the analysis forward means holding the one a + method below returned, so what anybody else holds is never + changed under them. + """ + + # The state type the servicer this method is written in services, + # which is what `self.ref()` refers to. + state_type: str + + # What each imported name refers to, from `_imports`. The same + # for the whole method, so carried untouched. + symbols: Mapping[str, tuple[str, str]] + + # The Reboot calls found, in the order met. What the analysis is + # for. + calls: tuple[Call, ...] + + # What was met that the analysis does not follow, spelled the way + # it was written, so whoever reads the calls can be told they are + # likely incomplete rather than left to trust them. + unsupported: tuple[str, ...] + + # What each name holds at the point the analysis has reached. A + # `MappingProxyType`, so writing into it raises rather than + # quietly leaking. + locals: Mapping[str, Local] + + def bind(self, name: str, local: Optional[Local]) -> 'Analysis': + """Returns this analysis with `name` holding `local`, or + holding nothing when `local` is `None` -- which is how a name + assigned something the analysis cannot say stops being + held.""" + locals = dict(self.locals) + + if local is None: + locals.pop(name, None) + else: + locals[name] = local + + return replace(self, locals=MappingProxyType(locals)) + + def flag(self, unsupported: ast.AST) -> 'Analysis': + """Returns this analysis with a piece of syntax recorded as + unsupported, spelled the way it was written.""" + return replace( + self, + unsupported=self.unsupported + (ast.unparse(unsupported),), + ) + + +def _analysis( + *, state_type: str, symbols: Mapping[str, tuple[str, str]] +) -> Analysis: + """Returns the analysis everything starts from: nothing found, + nothing held.""" + return Analysis( + state_type=state_type, + symbols=symbols, + calls=(), + unsupported=(), + locals=MappingProxyType({}), + ) + + +def _reboot_related(expression: ast.expr, *, analysis: Analysis) -> bool: + """Returns whether anything in an expression touches something + Reboot related: a `.ref` however it is reached, a name holding a + reference or the context, or a name imported as a state type. + + What ordinary Python does is never Reboot related, however little + of it the analysis follows; this is what keeps the unsupported + list evidence rather than noise. + """ + for node in ast.walk(expression): + match node: + case ast.Attribute(attr='ref'): + return True + case ast.Name(id=str(name)): + if name in analysis.locals: + return True + if _state_type_if_imported( + name, symbols=analysis.symbols + ) is not None: + return True + + return False + + +def _evaluate(expression: ast.expr, *, + analysis: Analysis) -> tuple[Optional[Local], Analysis]: + """Returns what an expression evaluates to, in the only terms the + analysis knows -- a reference to a state type, or the context -- + and the analysis carried forward. `None` for the value, which is + most expressions, means neither, so nothing can be said about a + name it is assigned to or a call made on it. + """ + match expression: + case ast.Await(value=value): + # Awaiting evaluates to what was awaited. + return _evaluate(value, analysis=analysis) + + case ast.Call( + func=ast.Attribute(value=ast.Name(id='self'), attr='ref') + ): + # A servicer reaching the state it is servicing. Matched + # before `Account.ref(id)` below, which would otherwise + # match this too and find that `self` names no state + # type. + return Reference(state_type=analysis.state_type), analysis + + case ast.Call(func=ast.Attribute(value=ast.Name(id=name), attr='ref')): + # `Account.ref(id)`, the one way to name an existing + # state -- unless `name` is no state type, which falls + # through to be flagged below: a `.ref` on something + # unresolvable is almost certainly a reference being + # lost. + state_type = _state_type_if_imported( + name, symbols=analysis.symbols + ) + if state_type is not None: + return Reference(state_type=state_type), analysis + + case ast.Name(id=str(name)) if name in analysis.locals: + # `another = account`, holding whatever `account` holds. + return analysis.locals[name], analysis + + # Nothing this evaluates. Said out loud when the expression + # touches something Reboot related -- whatever it does with it is + # not followed, so the calls are likely incomplete -- and left + # alone when it is ordinary Python, which was never claimed. + if _reboot_related(expression, analysis=analysis): + analysis = analysis.flag(expression) + + return None, analysis + + +def _assign( + assign: ast.Assign | ast.AnnAssign | ast.AugAssign, + *, + analysis: Analysis, +) -> Analysis: + """Returns the analysis carried past an assignment: each plain + name bound, and an assignment with a target this cannot bind + recorded as unsupported. + + A plain name is bound to what the value evaluates to, or stops + being held when that is nothing -- which is what every augmented + assignment does, since `x += y` makes `x` hold something no name + was ever bound to. A bare annotation binds nothing. Any other + target -- unpacked into a tuple, stored on an attribute or a + subscript -- is not followed; every name inside one stops being + held, since whatever it held before is no longer what it holds. + """ + targets: list[ast.expr] + value: Optional[ast.expr] + + match assign: + case ast.Assign(): + targets = assign.targets + value = assign.value + case ast.AnnAssign(): + if assign.value is None: + # A bare annotation binds nothing. + return analysis + targets = [assign.target] + value = assign.value + case ast.AugAssign(): + targets = [assign.target] + value = None + + local: Optional[Local] = None + + if value is not None: + local, analysis = _evaluate(value, analysis=analysis) + + for target in targets: + match target: + case ast.Name(id=str(name)): + analysis = analysis.bind(name, local) + case _: + # A target this cannot bind: unpacked into a tuple, + # stored on an attribute or a subscript. The names + # inside it were assigned something all the same, so + # each stops holding whatever it held before. + for node in ast.walk(target): + match node: + case ast.Name(id=str(name)): + # `account` in `account, _ = ...`. + analysis = analysis.bind(name, None) + analysis = analysis.flag(assign) + + return analysis + + +def _analyze( + method: ast.FunctionDef | ast.AsyncFunctionDef, + *, + state_type: str, + symbols: dict[str, tuple[str, str]], +) -> Analysis: + """Returns the Reboot calls a method's body makes, in the order + met. + + Statements are visited in the order written, saying what each + name holds before moving to the next, so that when a call is met + the name it is made through resolves to whatever it held at that + point in the body. Tracking the names is not a result of its own; + it is what makes a call through `account` mean a call on + `bank.v1.Account`. + + The method's context is held first, from the parameter after + `self`. A name assigned twice holds what it was assigned last, + and one assigned something this cannot follow stops being held at + all. + + A name is not a variable in any one scope: a comprehension or a + nested function binds names of its own, and they are all taken + together. Reboot code does not usually reuse a name for a + reference and something else, and taking them together is what + lets a reference reach a nested function that uses it. + """ + analysis = _analysis(state_type=state_type, symbols=symbols) + + # `self` first, then the context; a `@classmethod` takes `cls` + # in its place, and a workflow or a task takes the context the + # same way. + arguments = method.args.posonlyargs + method.args.args + + if len(arguments) > 1 and arguments[0].arg in ('self', 'cls'): + analysis = analysis.bind(arguments[1].arg, Context()) + + for statement in _statements(method): + # TODO: Record the calls this statement makes, through what + # each name holds right now -- before what it assigns is + # bound, since a statement's value runs before its target. + + match statement: + case ast.Assign() | ast.AnnAssign() | ast.AugAssign(): + analysis = _assign(statement, analysis=analysis) + + return analysis + + @dataclass(frozen=True, kw_only=True) class File: """What one of the developer's files was found to hold.""" diff --git a/tests/reboot/dashboard/implementation_watcher_tests.py b/tests/reboot/dashboard/implementation_watcher_tests.py index 8e8ad71a..2488e320 100644 --- a/tests/reboot/dashboard/implementation_watcher_tests.py +++ b/tests/reboot/dashboard/implementation_watcher_tests.py @@ -3,6 +3,7 @@ The API files say which state types exist, so a state type appearing is what sets the dashboard looking for the file that implements it. """ +import ast import os import tempfile import unittest @@ -19,6 +20,7 @@ ) from reboot.dashboard.implementation_watcher import File, files, servicers from reboot.dashboard.main import application +from typing import Mapping from unittest.mock import patch API_FILE = ''' @@ -97,6 +99,177 @@ def _state_types_and_files(files: dict[str, File]) -> list[tuple[str, str]]: ] +class AnalyzeTest(unittest.TestCase): + """What analyzing a method's body found. + + Nothing records calls yet, so these observe the analysis through + what each name held when the body ended.""" + + def _analyze(self, body: str) -> implementation_watcher.Analysis: + module = ast.parse( + SERVICER.format(state='Shop', module='shop').replace( + ' async def look(self, context, request):\n pass', + ' async def look(self, context, request):\n' + body, + ) + ) + symbols, _ = implementation_watcher._imports(module) + + for node in ast.walk(module): + match node: + case ast.AsyncFunctionDef(name='look'): + return implementation_watcher._analyze( + node, + state_type='shop.v1.Shop', + symbols=symbols, + ) + + raise AssertionError('no method to analyze') + + def _locals(self, body: str) -> Mapping[str, implementation_watcher.Local]: + return self._analyze(body).locals + + def test_the_context_is_the_parameter_after_self(self) -> None: + self.assertEqual( + self._locals(' pass'), + {'context': implementation_watcher.Context()}, + ) + + def test_a_reference_to_another_state_type(self) -> None: + names = self._locals( + ' from shop.v1.depot_rbt import Depot\n' + ' depot = Depot.ref(request.depot)' + ) + + self.assertEqual( + names['depot'], + implementation_watcher.Reference(state_type='shop.v1.Depot'), + ) + + def test_a_reference_to_the_state_being_serviced(self) -> None: + names = self._locals(' shop = self.ref()') + + self.assertEqual( + names['shop'], + implementation_watcher.Reference(state_type='shop.v1.Shop'), + ) + + def test_a_name_holding_what_another_holds(self) -> None: + names = self._locals( + ' shop = self.ref()\n' + ' same = shop' + ) + + self.assertEqual(names['same'], names['shop']) + + def test_a_name_assigned_something_it_cannot_follow(self) -> None: + """Held until it is not: assigning over a reference with + something unreadable stops it being a reference.""" + names = self._locals( + ' shop = self.ref()\n' + ' shop = whatever()' + ) + + self.assertNotIn('shop', names) + + def test_a_name_bound_inside_a_block(self) -> None: + """Names are taken together rather than by scope, so one bound + inside an `if` is held the same way.""" + names = self._locals( + ' if request.wanted:\n' + ' shop = self.ref()' + ) + + self.assertEqual( + names['shop'], + implementation_watcher.Reference(state_type='shop.v1.Shop'), + ) + + def test_a_name_holding_nothing_it_can_say(self) -> None: + names = self._locals(' total = request.a + request.b') + + self.assertNotIn('total', names) + + def test_an_assignment_it_cannot_bind_is_unsupported(self) -> None: + """Said as written, so a reader of the calls can be told they + are likely incomplete and find why. Twice here: the context + flows into a call that is not followed, and the unpacking is a + target that cannot be bound.""" + analysis = self._analyze(' account, _ = whatever(context)') + + self.assertEqual( + analysis.unsupported, ( + 'whatever(context)', + '(account, _) = whatever(context)', + ) + ) + + def test_an_unbindable_target_stops_its_names_being_held(self) -> None: + """`account` no longer holds the reference: whatever the + unpacking gave it is not something this followed.""" + analysis = self._analyze( + ' account = self.ref()\n' + ' account, _ = whatever(context)' + ) + + self.assertNotIn('account', analysis.locals) + + def test_an_annotated_assignment_binds_the_same_way(self) -> None: + names = self._analyze(' shop: object = self.ref()').locals + + self.assertEqual( + names['shop'], + implementation_watcher.Reference(state_type='shop.v1.Shop'), + ) + + def test_a_bare_annotation_binds_nothing(self) -> None: + analysis = self._analyze( + ' shop = self.ref()\n' + ' shop: object' + ) + + self.assertEqual( + analysis.locals['shop'], + implementation_watcher.Reference(state_type='shop.v1.Shop'), + ) + self.assertEqual(analysis.unsupported, ()) + + def test_an_augmented_assignment_stops_a_name_being_held(self) -> None: + """`x += y` makes `x` hold something no name was ever bound + to, whatever the two held.""" + analysis = self._analyze( + ' shop = self.ref()\n' + ' shop += request.a' + ) + + self.assertNotIn('shop', analysis.locals) + self.assertEqual(analysis.unsupported, ()) + + def test_a_ref_on_something_unresolvable_is_unsupported(self) -> None: + """Almost certainly a reference being lost, however it was + reached.""" + analysis = self._analyze(" shop = stores.Shop.ref('a')") + + self.assertEqual(analysis.unsupported, ("stores.Shop.ref('a')",)) + self.assertNotIn('shop', analysis.locals) + + def test_a_state_type_used_some_other_way_is_unsupported(self) -> None: + analysis = self._analyze(' shop = Shop.open(context)') + + self.assertEqual(analysis.unsupported, ('Shop.open(context)',)) + + def test_the_context_reaching_an_unfollowed_call_is_unsupported( + self + ) -> None: + analysis = self._analyze(' result = self._helper(context)') + + self.assertEqual(analysis.unsupported, ('self._helper(context)',)) + + def test_an_ordinary_assignment_is_not_unsupported(self) -> None: + analysis = self._analyze(' total = request.a + request.b') + + self.assertEqual(analysis.unsupported, ()) + + class ImplementationWatcherTest(unittest.IsolatedAsyncioTestCase): async def asyncSetUp(self) -> None: From 5e430df55a5d072215147f54542525795a20c263 Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Wed, 19 Aug 2026 06:07:27 +0000 Subject: [PATCH 12/31] Resolve a name through what a file's names refer to Scaffolding for recognizing a state type however it is reached, with no new spelling recognized yet. What a file's imports bound each of its names to becomes `Imports`, whose `try_resolve_module` answers which module a dotted name refers into and what it refers to there; the modules those imports may have Python load are its `may_load`, which is what following the imports follows. Resolution asks about a dotted name (`_path` turns `rbt.Shop` into one) rather than a bare one, and a servicer's base is matched by shape -- `X.Servicer`, or `X.singleton.Servicer` -- with `X` any dotted name. Only the one spelling that already resolved does: a name imported from a `_rbt` module directly. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QPEVMhyDRxZEuH8eiykv99 --- reboot/dashboard/implementation_watcher.py | 357 ++++++++++-------- .../dashboard/implementation_watcher_tests.py | 60 +-- 2 files changed, 235 insertions(+), 182 deletions(-) diff --git a/reboot/dashboard/implementation_watcher.py b/reboot/dashboard/implementation_watcher.py index 12fb85e8..ff27315a 100644 --- a/reboot/dashboard/implementation_watcher.py +++ b/reboot/dashboard/implementation_watcher.py @@ -16,16 +16,16 @@ class AccountServicer(Account.Servicer): behind a conditional import -- which have one thing in common: the file defining the servicer had to be imported for any of them to run. -Where the walk stops is what makes this the developer's code rather -than somebody else's. A module resolves only if a root holds it, so -an import of an installed package leads nowhere, and no state type of -theirs is waiting on one: their API files declare none of those. +Where the walk of the imports stops is what makes this the +developer's code rather than somebody else's. A module resolves to a +file only if a root holds it, so an import of an installed package +leads nowhere. Read rather than imported, because importing an application means having its generated code, its dependencies and its `sys.path`, and the dashboard is meant to work before any of that exists. A file at a time through `cooperatively`, so that the dashboard goes on answering -while a large one is walked. +while a large application is read. And driven by the filesystem, because that is what it is a function of: where a state type is implemented can only change when the @@ -42,7 +42,11 @@ class AccountServicer(Account.Servicer): from reboot.aio.cooperatively import cooperatively from reboot.cli.common.watch import file_watcher from types import MappingProxyType -from typing import Iterator, Mapping, Optional +from typing import Iterator, Mapping, Optional, Sequence + +# A SHA-256 digest -- of a file's bytes, or of a method's syntax -- +# saying whether what was digested has changed. +Digest = bytes GENERATED_SUFFIX = '_rbt' @@ -63,50 +67,106 @@ def _read(filename: str) -> bytes: return file.read() -def _imports( - module: ast.Module -) -> tuple[dict[str, tuple[str, str]], list[str]]: - """Returns every symbol a file imported -- keyed by the name the - file calls it, valued by the module it came from and its name - there, so `Account` -> (`bank.v1.account_rbt`, `Account`) -- and - beside it every module the file has imported. +@dataclass(frozen=True, kw_only=True) +class Imports: + """What a file's imports bound each of its names to.""" + + @dataclass(frozen=True, kw_only=True) + class Symbol: + """What `from x import y [as z]` binds a name to: `y` of + module `x` -- which may itself turn out to be a module.""" + + # The module it was imported from. + module: str + + # Its name there, which the local name may differ from. + name: str + + # By the name the file calls it. + bindings: Mapping[str, 'Import'] + + # Every module these imports may have Python load: `import a.b.c` + # loads `a.b.c`; `from x import y` loads `x`, and `x.y` too when + # `y` is a module of its own; a star import loads its module. + # Each is tried as a file under the roots, which is how the rest + # of the application's files are found. + may_load: tuple[str, ...] + + def try_resolve_module(self, + path: Sequence[str]) -> Optional[tuple[str, str]]: + """Returns the module a dotted name refers into and the name + it refers to there, and `None` when no binding answers.""" + head, *rest = path + + match self.bindings.get(head): + case Imports.Symbol(module=module, name=name) if not rest: + return module, name + + return None + + def try_resolve_state_type(self, path: Sequence[str]) -> Optional[str]: + """Returns the state type a dotted name refers to, if where + it was bound from is generated code.""" + resolved = self.try_resolve_module(path) + + if resolved is None: + return None + + module, attribute = resolved + + # The `_rbt` module name alone says what a name from one is: + # `Account` from `bank.v1.account_rbt` is `bank.v1.Account`. + if not module.endswith(GENERATED_SUFFIX): + return None + + return f"{module.rsplit('.', 1)[0]}.{attribute}" + - Symbols come from every import, wherever it is written. One inside +# What one import bound a name to. +Import = Imports.Symbol + + +def _imports(module: ast.Module) -> Imports: + """Returns what a file's imports bound each of its names to. + + Names come from every import, wherever it is written. One inside an `if` or a `try` binds its name just as one at the top of the file does, and guarding an import is common enough to be worth reading. `ast.walk` yields the shallowest first, so a name bound at the top of the file wins over one bound inside something. - - Since we cannot tell whether `y` in `from x import y` is a module - of its own or a name defined in `x`, `x.y` is listed as a module - too. Adding that guess is safe because a module only becomes a - file if a root holds one by that name, so `x.y` naming something - that is not a module resolves to nothing and is dropped. """ - symbols: dict[str, tuple[str, str]] = {} - imported_modules: list[str] = [] + bindings: dict[str, Import] = {} + may_load: list[str] = [] for node in ast.walk(module): - # A relative import names no module of its own, and - # `from . import x` has nowhere to be read from here. match node: case ast.Import(names=names): - imported_modules.extend(alias.name for alias in names) - case ast.ImportFrom( - module=str(imported_module), level=0, names=names - ): - imported_modules.append(imported_module) + may_load.extend(alias.name for alias in names) + + case ast.ImportFrom(module=str(from_module), level=0, names=names): + may_load.append(from_module) + for alias in names: - symbols.setdefault( + bindings.setdefault( alias.asname or alias.name, - (imported_module, alias.name), + Imports.Symbol(module=from_module, name=alias.name), ) - imported_modules.append(f'{imported_module}.{alias.name}') - - return symbols, imported_modules + # Since we cannot tell whether `y` in + # `from x import y` is a module of its own or a + # name defined in `x`, `x.y` may be loaded too. + # The guess is safe: a module only becomes a file + # if a root holds one by that name, so `x.y` + # naming something that is not a module resolves + # to nothing and is dropped. + may_load.append(_join(from_module, alias.name)) + + return Imports( + bindings=MappingProxyType(bindings), + may_load=tuple(may_load), + ) -def _resolve(imported_module: str, *, roots: list[str]) -> Optional[str]: +def _try_find_file_of(module: str, *, roots: Sequence[str]) -> Optional[str]: """Returns the file a module names if one of `roots` contains it, and `None` otherwise. @@ -114,7 +174,7 @@ def _resolve(imported_module: str, *, roots: list[str]) -> Optional[str]: packages live outside every root, so `import asyncio` resolves to nothing and there is nothing to read. """ - relative = imported_module.replace('.', os.sep) + relative = module.replace('.', os.sep) for root in roots: for candidate in ( @@ -127,44 +187,59 @@ def _resolve(imported_module: str, *, roots: list[str]) -> Optional[str]: return None -def _state_type_if_imported( - name: str, *, symbols: Mapping[str, tuple[str, str]] -) -> Optional[str]: - """Returns the state type a name holds, if it was imported from - generated code. - - A state type reaches the developer's code one way, as a name - imported from the module generated for its API file: `Account` - from `bank.v1.account_rbt` is `bank.v1.Account`. - """ - match symbols.get(name): - case (str(imported_module), - str(attribute)) if imported_module.endswith(GENERATED_SUFFIX): - package = imported_module.rsplit('.', 1)[0] - return f'{package}.{attribute}' +def _path(expression: ast.expr) -> Optional[list[str]]: + """Returns the dotted name an expression spells -- `rbt.Shop` as + `['rbt', 'Shop']` -- and `None` when it does not spell one.""" + match expression: + case ast.Name(id=str(name)): + return [name] + case ast.Attribute(value=value, attr=str(attribute)): + prefix = _path(value) + if prefix is not None: + return prefix + [attribute] return None +def _join(base: str, *parts: str) -> str: + """Returns a module extended by more components.""" + return '.'.join([base, *parts]) + + +@dataclass(frozen=True, kw_only=True) +class File: + """What one of the developer's files was found to hold.""" + + # Of the bytes the file held, saying whether parsing it again + # would say anything new. + digest: Digest + + # Every module the file's imports may have Python load, to be + # followed if a root holds it. + may_load: tuple[str, ...] + + # Every servicer the file defines. + servicers: tuple[ServicerInfo, ...] + + def _state_type_if_servicer( - class_definition: ast.ClassDef, *, symbols: dict[str, tuple[str, str]] + class_definition: ast.ClassDef, *, imports: Imports ) -> Optional[str]: """Returns the state type a class services, if it says so. A servicer says it by what it inherits: `Account.Servicer`, or - `Account.singleton.Servicer` for a singleton. + `Account.singleton.Servicer` for a singleton, with `Account` + reached by any dotted name that refers to a state type. """ for base in class_definition.bases: match base: - case ( - ast.Attribute(value=ast.Name(id=name), attr='Servicer') | - ast.Attribute( - value=ast. - Attribute(value=ast.Name(id=name), attr='singleton'), - attr='Servicer', - ) - ): - state_type = _state_type_if_imported(name, symbols=symbols) + case ast.Attribute(value=value, attr='Servicer'): + path = _path(value) + if path is None: + continue + if len(path) > 1 and path[-1] == 'singleton': + path = path[:-1] + state_type = imports.try_resolve_state_type(path) if state_type is not None: return state_type @@ -229,9 +304,9 @@ class Analysis: # which is what `self.ref()` refers to. state_type: str - # What each imported name refers to, from `_imports`. The same + # What the file's imports bound each of its names to. The same # for the whole method, so carried untouched. - symbols: Mapping[str, tuple[str, str]] + imports: Imports # The Reboot calls found, in the order met. What the analysis is # for. @@ -247,11 +322,23 @@ class Analysis: # quietly leaking. locals: Mapping[str, Local] - def bind(self, name: str, local: Optional[Local]) -> 'Analysis': - """Returns this analysis with `name` holding `local`, or - holding nothing when `local` is `None` -- which is how a name - assigned something the analysis cannot say stops being - held.""" + @classmethod + def create(cls, *, state_type: str, imports: Imports) -> 'Analysis': + """Returns the analysis everything starts from: nothing found, + nothing bound.""" + return cls( + state_type=state_type, + imports=imports, + calls=(), + unsupported=(), + locals=MappingProxyType({}), + ) + + def with_local(self, name: str, local: Optional[Local]) -> 'Analysis': + """Returns this analysis with `name` bound to `local`, or + bound to nothing when `local` is `None` -- which is how a + name assigned something the analysis cannot say stops being + bound.""" locals = dict(self.locals) if local is None: @@ -261,7 +348,7 @@ def bind(self, name: str, local: Optional[Local]) -> 'Analysis': return replace(self, locals=MappingProxyType(locals)) - def flag(self, unsupported: ast.AST) -> 'Analysis': + def with_unsupported(self, unsupported: ast.AST) -> 'Analysis': """Returns this analysis with a piece of syntax recorded as unsupported, spelled the way it was written.""" return replace( @@ -270,24 +357,10 @@ def flag(self, unsupported: ast.AST) -> 'Analysis': ) -def _analysis( - *, state_type: str, symbols: Mapping[str, tuple[str, str]] -) -> Analysis: - """Returns the analysis everything starts from: nothing found, - nothing held.""" - return Analysis( - state_type=state_type, - symbols=symbols, - calls=(), - unsupported=(), - locals=MappingProxyType({}), - ) - - def _reboot_related(expression: ast.expr, *, analysis: Analysis) -> bool: """Returns whether anything in an expression touches something Reboot related: a `.ref` however it is reached, a name holding a - reference or the context, or a name imported as a state type. + reference or the context, or a name that refers to a state type. What ordinary Python does is never Reboot related, however little of it the analysis follows; this is what keeps the unsupported @@ -300,9 +373,8 @@ def _reboot_related(expression: ast.expr, *, analysis: Analysis) -> bool: case ast.Name(id=str(name)): if name in analysis.locals: return True - if _state_type_if_imported( - name, symbols=analysis.symbols - ) is not None: + state_type = analysis.imports.try_resolve_state_type([name]) + if state_type is not None: return True return False @@ -326,21 +398,21 @@ def _evaluate(expression: ast.expr, *, ): # A servicer reaching the state it is servicing. Matched # before `Account.ref(id)` below, which would otherwise - # match this too and find that `self` names no state + # match this too and find that `self` refers to no state # type. return Reference(state_type=analysis.state_type), analysis - case ast.Call(func=ast.Attribute(value=ast.Name(id=name), attr='ref')): - # `Account.ref(id)`, the one way to name an existing - # state -- unless `name` is no state type, which falls - # through to be flagged below: a `.ref` on something - # unresolvable is almost certainly a reference being - # lost. - state_type = _state_type_if_imported( - name, symbols=analysis.symbols - ) - if state_type is not None: - return Reference(state_type=state_type), analysis + case ast.Call(func=ast.Attribute(value=receiver, attr='ref')): + # `Account.ref(id)`, or `rbt.Shop.ref(id)` through a + # module, the one way to name an existing state -- unless + # the name refers to no state type, which falls through + # to be flagged below: a `.ref` on something unresolvable + # is almost certainly a reference being lost. + path = _path(receiver) + if path is not None: + state_type = analysis.imports.try_resolve_state_type(path) + if state_type is not None: + return Reference(state_type=state_type), analysis case ast.Name(id=str(name)) if name in analysis.locals: # `another = account`, holding whatever `account` holds. @@ -351,7 +423,7 @@ def _evaluate(expression: ast.expr, *, # not followed, so the calls are likely incomplete -- and left # alone when it is ordinary Python, which was never claimed. if _reboot_related(expression, analysis=analysis): - analysis = analysis.flag(expression) + analysis = analysis.with_unsupported(expression) return None, analysis @@ -398,7 +470,7 @@ def _assign( for target in targets: match target: case ast.Name(id=str(name)): - analysis = analysis.bind(name, local) + analysis = analysis.with_local(name, local) case _: # A target this cannot bind: unpacked into a tuple, # stored on an attribute or a subscript. The names @@ -408,17 +480,17 @@ def _assign( match node: case ast.Name(id=str(name)): # `account` in `account, _ = ...`. - analysis = analysis.bind(name, None) - analysis = analysis.flag(assign) + analysis = analysis.with_local(name, None) + analysis = analysis.with_unsupported(assign) return analysis -def _analyze( +def _analyze_method( method: ast.FunctionDef | ast.AsyncFunctionDef, *, state_type: str, - symbols: dict[str, tuple[str, str]], + imports: Imports, ) -> Analysis: """Returns the Reboot calls a method's body makes, in the order met. @@ -441,7 +513,7 @@ def _analyze( reference and something else, and taking them together is what lets a reference reach a nested function that uses it. """ - analysis = _analysis(state_type=state_type, symbols=symbols) + analysis = Analysis.create(state_type=state_type, imports=imports) # `self` first, then the context; a `@classmethod` takes `cls` # in its place, and a workflow or a task takes the context the @@ -449,7 +521,7 @@ def _analyze( arguments = method.args.posonlyargs + method.args.args if len(arguments) > 1 and arguments[0].arg in ('self', 'cls'): - analysis = analysis.bind(arguments[1].arg, Context()) + analysis = analysis.with_local(arguments[1].arg, Context()) for statement in _statements(method): # TODO: Record the calls this statement makes, through what @@ -463,25 +535,7 @@ def _analyze( return analysis -@dataclass(frozen=True, kw_only=True) -class File: - """What one of the developer's files was found to hold.""" - - # Of the bytes the file held, saying whether parsing it again - # would say anything new. Not `st_mtime_ns`, which is only as fine - # as the kernel's coarse clock -- around ten milliseconds -- so a - # save landing in the same tick as a read leaves an mtime that - # says nothing happened. - digest: bytes - - # Every module the file names, to be followed if a root holds it. - imported_modules: list[str] - - # Every servicer the file defines. - servicers: list[ServicerInfo] - - -def _digest(node: ast.AST) -> bytes: +def _digest(node: ast.AST) -> Digest: """Returns a digest of what a piece of syntax says. The digest is computed using `ast.dump` without attributes so that @@ -510,29 +564,26 @@ def _methods(class_definition: ast.ClassDef) -> list[ServicerInfo.Method]: return methods -def _parse(source: bytes, *, digest: bytes, filename: str) -> Optional[File]: +def _parse(source: bytes, *, digest: Digest, filename: str) -> Optional[File]: """Returns what a file holds, and `None` when it will not parse. A file that will not parse is left unrecorded rather than recorded - as empty, so that the next round parses it again: half-written is - the normal state of a file somebody is typing into. + as empty, so that the next iteration parses it again: half-written + is the normal state of a file somebody is typing into. """ try: module: ast.Module = ast.parse(source) except SyntaxError: return None - # First, and on its own: `ast.walk` is breadth-first, so a - # top-level class comes out before an import nested in a `try`, - # and every name must be known before any class is resolved. - symbols, imported_modules = _imports(module) + imports = _imports(module) servicers = [] for node in ast.walk(module): match node: case ast.ClassDef(): - state_type = _state_type_if_servicer(node, symbols=symbols) + state_type = _state_type_if_servicer(node, imports=imports) if state_type is not None: servicers.append( ServicerInfo( @@ -544,12 +595,12 @@ def _parse(source: bytes, *, digest: bytes, filename: str) -> Optional[File]: return File( digest=digest, - imported_modules=imported_modules, - servicers=servicers, + may_load=imports.may_load, + servicers=tuple(servicers), ) -def servicers(files: dict[str, File]) -> list[ServicerInfo]: +def servicers(files: Mapping[str, File]) -> list[ServicerInfo]: """Returns every servicer found, sorted by the state type it services and the file it is written in. @@ -564,11 +615,11 @@ def servicers(files: dict[str, File]) -> list[ServicerInfo]: ) -async def files( +async def analyze( *, application: str, - roots: Optional[list[str]] = None, - known: Optional[dict[str, File]] = None, + roots: Optional[Sequence[str]] = None, + known: Optional[Mapping[str, File]] = None, ) -> dict[str, File]: """Returns what each file the developer's application reaches holds, keyed by the file, spelled the way they would open it. @@ -625,10 +676,12 @@ async def files( reachable[filename] = file - for imported_module in file.imported_modules: - resolved = _resolve(imported_module, roots=roots) - if resolved is not None: - pending.append(resolved) + pending.extend( + filename for filename in ( + _try_find_file_of(module, roots=roots) + for module in file.may_load + ) if filename is not None + ) return reachable @@ -645,19 +698,19 @@ async def watch(context: WorkflowContext, *, application: str) -> None: with file_watcher() as watcher: async for iteration in context.loop('Watch the application'): # The watch is armed before anything is read, so a save - # made during the walk resolves `event` rather than - # arriving while nothing is listening. A watch is consumed - # by one event, so it is re-entered for each. + # made during an iteration resolves `event` rather than + # arriving while nothing is listening. A watch is + # consumed by one event, so it is re-entered for each. async with watcher.watch(globs) as event: - known = await files( + known = await analyze( application=application, roots=roots, known=known ) found = servicers(known) - # Most edits change no servicer, and a write wakes every - # browser reading `Get`, so one is only worth making - # when the answer is different. + # Most edits change no servicer, and a write wakes + # every browser reading `Get`, so one is only worth + # making when the answer is different. if found != recorded: async def record(state) -> None: diff --git a/tests/reboot/dashboard/implementation_watcher_tests.py b/tests/reboot/dashboard/implementation_watcher_tests.py index 2488e320..2852f3fd 100644 --- a/tests/reboot/dashboard/implementation_watcher_tests.py +++ b/tests/reboot/dashboard/implementation_watcher_tests.py @@ -18,7 +18,7 @@ ENVVAR_RBT_APPLICATION, IMPLEMENTATION_ID, ) -from reboot.dashboard.implementation_watcher import File, files, servicers +from reboot.dashboard.implementation_watcher import File, analyze, servicers from reboot.dashboard.main import application from typing import Mapping from unittest.mock import patch @@ -112,15 +112,15 @@ def _analyze(self, body: str) -> implementation_watcher.Analysis: ' async def look(self, context, request):\n' + body, ) ) - symbols, _ = implementation_watcher._imports(module) + imports = implementation_watcher._imports(module) for node in ast.walk(module): match node: case ast.AsyncFunctionDef(name='look'): - return implementation_watcher._analyze( + return implementation_watcher._analyze_method( node, state_type='shop.v1.Shop', - symbols=symbols, + imports=imports, ) raise AssertionError('no method to analyze') @@ -471,7 +471,7 @@ async def test_finds_the_file_a_state_type_is_implemented_in(self) -> None: self._write('shop_servicer.py', source=SHOP) application = self._write('main.py', source=APPLICATION) - found = _state_types_and_files(await files(application=application)) + found = _state_types_and_files(await analyze(application=application)) self.assertEqual( found, @@ -490,7 +490,7 @@ async def test_a_singleton_says_what_it_services_the_same_way( ), ) - found = _state_types_and_files(await files(application=application)) + found = _state_types_and_files(await analyze(application=application)) self.assertEqual( found, @@ -512,7 +512,7 @@ async def main(): ''' ) - found = _state_types_and_files(await files(application=application)) + found = _state_types_and_files(await analyze(application=application)) self.assertEqual( found, [ @@ -542,7 +542,7 @@ async def main(): ''' ) - found = _state_types_and_files(await files(application=application)) + found = _state_types_and_files(await analyze(application=application)) self.assertEqual( found, @@ -566,7 +566,7 @@ async def main(): ''' ) - found = _state_types_and_files(await files(application=application)) + found = _state_types_and_files(await analyze(application=application)) self.assertEqual( found, @@ -593,7 +593,7 @@ async def main(): ''' ) - found = _state_types_and_files(await files(application=application)) + found = _state_types_and_files(await analyze(application=application)) self.assertEqual( found, @@ -614,7 +614,7 @@ async def main(): ''' ) - found = _state_types_and_files(await files(application=application)) + found = _state_types_and_files(await analyze(application=application)) self.assertEqual( found, @@ -646,14 +646,14 @@ async def main(): ) found = _state_types_and_files( - await files(application=application) + await analyze(application=application) ) self.assertEqual(found, []) # Named as a root, the very same import leads there. found = _state_types_and_files( - await files( + await analyze( application=application, roots=[str(self.directory), elsewhere.name], ) @@ -672,7 +672,7 @@ async def test_records_the_methods_a_servicer_defines(self) -> None: self._write('shop_servicer.py', source=SHOP) application = self._write('main.py', source=APPLICATION) - found = servicers(await files(application=application)) + found = servicers(await analyze(application=application)) self.assertEqual( [method.name for method in found[0].methods], ['look'] @@ -684,7 +684,7 @@ async def test_a_method_reformatted_digests_the_same(self) -> None: self._write('shop_servicer.py', source=SHOP) application = self._write('main.py', source=APPLICATION) - before = servicers(await files(application=application)) + before = servicers(await analyze(application=application)) self._write( 'shop_servicer.py', @@ -700,7 +700,7 @@ async def test_a_method_reformatted_digests_the_same(self) -> None: ), ) - after = servicers(await files(application=application)) + after = servicers(await analyze(application=application)) self.assertEqual( [method.digest for method in after[0].methods], @@ -713,14 +713,14 @@ async def test_a_method_whose_body_changes_digests_differently( self._write('shop_servicer.py', source=SHOP) application = self._write('main.py', source=APPLICATION) - before = servicers(await files(application=application)) + before = servicers(await analyze(application=application)) self._write( 'shop_servicer.py', source=SHOP.replace(' pass', ' return None'), ) - after = servicers(await files(application=application)) + after = servicers(await analyze(application=application)) self.assertNotEqual( after[0].methods[0].digest, @@ -738,7 +738,7 @@ async def test_a_file_written_with_the_same_bytes_is_not_parsed_again( servicer = self._write('shop_servicer.py', source=SHOP) application = self._write('main.py', source=APPLICATION) - known = await files(application=application) + known = await analyze(application=application) Path(servicer).write_text(SHOP) @@ -747,7 +747,7 @@ async def test_a_file_written_with_the_same_bytes_is_not_parsed_again( '_parse', wraps=implementation_watcher._parse, ) as parse: - await files(application=application, known=known) + await analyze(application=application, known=known) parse.assert_not_called() @@ -760,13 +760,13 @@ async def test_a_file_written_with_other_bytes_is_parsed_again( application = self._write('main.py', source=APPLICATION) modified = os.stat(servicer).st_mtime_ns - known = await files(application=application) + known = await analyze(application=application) Path(servicer).write_text(DEPOT) os.utime(servicer, ns=(modified, modified)) found = _state_types_and_files( - await files(application=application, known=known) + await analyze(application=application, known=known) ) self.assertEqual(found, [('shop.v1.Depot', servicer)]) @@ -787,13 +787,13 @@ async def test_a_servicer_that_stops_being_imported_is_dropped( ), ) - known = await files(application=application) + known = await analyze(application=application) self.assertEqual(len(servicers(known)), 2) self._write('main.py', source=APPLICATION) found = _state_types_and_files( - await files(application=application, known=known) + await analyze(application=application, known=known) ) self.assertEqual( @@ -807,7 +807,7 @@ async def test_a_servicer_that_starts_being_imported_is_found( self._write('shop_servicer.py', source=SHOP) application = self._write('main.py', source=APPLICATION) - known = await files(application=application) + known = await analyze(application=application) self._write('depot_servicer.py', source=DEPOT) self._write( @@ -820,7 +820,7 @@ async def test_a_servicer_that_starts_being_imported_is_found( ) found = _state_types_and_files( - await files(application=application, known=known) + await analyze(application=application, known=known) ) self.assertEqual( @@ -839,13 +839,13 @@ async def test_a_file_that_will_not_parse(self) -> None: self._write('shop_servicer.py', source='class ShopServicer(') application = self._write('main.py', source=APPLICATION) - found = _state_types_and_files(await files(application=application)) + found = _state_types_and_files(await analyze(application=application)) self.assertEqual(found, []) async def test_an_application_that_is_not_there(self) -> None: found = _state_types_and_files( - await files(application=str(self.directory / 'nowhere.py')) + await analyze(application=str(self.directory / 'nowhere.py')) ) self.assertEqual(found, []) @@ -867,7 +867,7 @@ async def main(): ''' ) - found = _state_types_and_files(await files(application=application)) + found = _state_types_and_files(await analyze(application=application)) self.assertEqual( found, [ @@ -887,7 +887,7 @@ class ShopServicer(SomethingElse): ) application = self._write('main.py', source=APPLICATION) - found = _state_types_and_files(await files(application=application)) + found = _state_types_and_files(await analyze(application=application)) self.assertEqual(found, []) From 55c7e2d1c371a270aac8b58b606f2537a3addc67 Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Wed, 19 Aug 2026 06:07:50 +0000 Subject: [PATCH 13/31] Follow a re-export to the state type it refers to A name imported from a module that is not generated is followed into that module's own file, and on, until the chain ends in a `_rbt` module -- so a state type re-exported through another of the developer's files resolves. A circular import terminates: a file already being followed is not followed again. Following made a file's answer depend on other files' contents, so an iteration now carries a `Files` value through everything it does: `known` is what the previous iteration analyzed, `parsed` what this one has parsed and not yet analyzed, `analyzed` what it has finished, and `pending` the frontier: every file reached, parsed or kept, whose imports are not yet followed -- entering once and leaving once, handled. Each `File` records the files it depends on, by the digest each had when it was read -- recorded even when resolution answered "no", since a "no" depends on them just the same -- and a known file is kept only while its own digest and every dependency's still match; otherwise it is parsed and analyzed again. A file read mid-chain is parsed once into `pending`, and being parsed this iteration it can never serve a stale answer. Analyzing is one thing, not a "processing" step and an "analysis" step: `_analyze_file` carries one `Analysis` -- immutable, like `Files` -- through every class and method in a file, resolving what each class services, analyzing each method with `_analyze_method`, and recording each method under its servicer with the method reset for the next. The dependencies the analysis accumulates while resolving names are the file's, harvested when the file finishes into `Files`. Resolution reads other files through the `Analysis` too; a chain of imports is followed by asking each followed file's own imports, so whose imports a name resolves against is always the file that bound it. A chain is never followed into generated code by reading it: the `_rbt` module name alone says what a name from one is. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QPEVMhyDRxZEuH8eiykv99 --- reboot/dashboard/implementation_watcher.py | 681 ++++++++++++++---- .../dashboard/implementation_watcher_tests.py | 100 ++- 2 files changed, 619 insertions(+), 162 deletions(-) diff --git a/reboot/dashboard/implementation_watcher.py b/reboot/dashboard/implementation_watcher.py index ff27315a..f9efeea4 100644 --- a/reboot/dashboard/implementation_watcher.py +++ b/reboot/dashboard/implementation_watcher.py @@ -16,10 +16,9 @@ class AccountServicer(Account.Servicer): behind a conditional import -- which have one thing in common: the file defining the servicer had to be imported for any of them to run. -Where the walk of the imports stops is what makes this the -developer's code rather than somebody else's. A module resolves to a -file only if a root holds it, so an import of an installed package -leads nowhere. +Where following stops is what makes this the developer's code rather +than somebody else's. A module resolves to a file only if a root +holds it, so an import of an installed package leads nowhere. Read rather than imported, because importing an application means having its generated code, its dependencies and its `sys.path`, and @@ -50,6 +49,11 @@ class AccountServicer(Account.Servicer): GENERATED_SUFFIX = '_rbt' +# Suffixes of the files `rbt generate` writes. Named so that a chain +# of imports is never followed into generated code by reading it -- +# the `_rbt` module name alone says what a name from one is. +GENERATED_SUFFIXES = ('_rbt.py', '_pb2.py', '_pb2_grpc.py') + # Every file the developer might have written a servicer in, which is # the rule `rbt generate` and `rbt dev run` both use for source. SOURCE_GLOB = '**/*.py' @@ -104,22 +108,51 @@ def try_resolve_module(self, return None - def try_resolve_state_type(self, path: Sequence[str]) -> Optional[str]: - """Returns the state type a dotted name refers to, if where - it was bound from is generated code.""" + def try_resolve_state_type( + self, + path: Sequence[str], + *, + analysis: 'Analysis', + visiting: frozenset[str] = frozenset(), + ) -> tuple[Optional[str], 'Analysis']: + """Returns the state type a dotted name refers to, if + following where these imports bound it from ends in generated + code, and the analysis carried forward, since answering may + read files. + + Every file read becomes a dependency of the analysis whether + or not a state type was found: a "no" that read it depends on + it just the same, since a change to it may turn the "no" into + a "yes". `visiting` is the files already being read through, + which is what stops a circular import. + """ resolved = self.try_resolve_module(path) if resolved is None: - return None + return None, analysis module, attribute = resolved # The `_rbt` module name alone says what a name from one is: # `Account` from `bank.v1.account_rbt` is `bank.v1.Account`. - if not module.endswith(GENERATED_SUFFIX): - return None + if module.endswith(GENERATED_SUFFIX): + return f"{module.rsplit('.', 1)[0]}.{attribute}", analysis + + found, files = analysis.files.lookup_or_parse_module( + module, visiting=visiting + ) + if found is None: + return None, analysis - return f"{module.rsplit('.', 1)[0]}.{attribute}" + analysis = analysis.with_dependency(found, files=files) + + # The rest of the chain resolves against the followed file's + # own imports: theirs to answer. + return found.imports.try_resolve_state_type( + [attribute], + analysis=analysis, + visiting=visiting | {found.filename}, + ) # What one import bound a name to. @@ -206,44 +239,314 @@ def _join(base: str, *parts: str) -> str: return '.'.join([base, *parts]) +@dataclass(frozen=True, kw_only=True) +class ParsedFile: + """What parsing one file said, before any name in it is + resolved.""" + + # The file this is, spelled the way the developer would open it. + filename: str + + # Of the bytes the file held, saying whether parsing it again + # would say anything new. + digest: Digest + + # What the file's imports bound each of its names to. + imports: Imports + + # The syntax itself, so that a file parsed while resolving names + # is not parsed again when its own servicers are looked for. + module: ast.Module + + @dataclass(frozen=True, kw_only=True) class File: - """What one of the developer's files was found to hold.""" + """What analyzing one of the developer's files found.""" + + # The file this is, spelled the way the developer would open it. + filename: str # Of the bytes the file held, saying whether parsing it again # would say anything new. digest: Digest - # Every module the file's imports may have Python load, to be - # followed if a root holds it. - may_load: tuple[str, ...] + # What the file's imports bound each of its names to. + imports: Imports + + # The files this file depends on, by the digest each had when it + # was read. If any of them changes, this file needs reanalyzing. + dependencies: Mapping[str, Digest] # Every servicer the file defines. servicers: tuple[ServicerInfo, ...] -def _state_type_if_servicer( - class_definition: ast.ClassDef, *, imports: Imports -) -> Optional[str]: - """Returns the state type a class services, if it says so. +@dataclass(frozen=True, kw_only=True) +class Files: + """The developer's files, as far as one iteration of the watch + has taken them. - A servicer says it by what it inherits: `Account.Servicer`, or - `Account.singleton.Servicer` for a singleton, with `Account` - reached by any dotted name that refers to a state type. + An iteration is one call to `files()`, which `watch` makes each + time a save wakes it. + + Immutable: carrying the iteration forward means holding the one a + method below returned. """ - for base in class_definition.bases: - match base: - case ast.Attribute(value=value, attr='Servicer'): - path = _path(value) - if path is None: - continue - if len(path) > 1 and path[-1] == 'singleton': - path = path[:-1] - state_type = imports.try_resolve_state_type(path) - if state_type is not None: - return state_type - return None + # Where the developer's modules are found: the application's own + # directory. What an iteration is allowed to analyze. + roots: tuple[str, ...] + + # What the previous iteration analyzed, so a file unchanged + # since is neither parsed nor analyzed again. + known: Mapping[str, File] + + # Parsed this iteration, not yet analyzed. + parsed: Mapping[str, ParsedFile] + + # Analyzed: this iteration, each servicer and method resolved + # against bytes as they are on disk right now -- or a previous + # one, verified unchanged. What an iteration returns. + analyzed: Mapping[str, File] + + # Reached this iteration, imports not yet followed. The loop's + # frontier: every file enters once, when it is parsed or kept, + # and leaves once, handled. + pending: frozenset[str] + + # The digest of every file read this iteration, by filename -- + # including files that joined no other map. What spares a second + # read of the same bytes. The bytes themselves are not kept: they + # are needed again only when a file changed and must be parsed, + # which a save makes rare, and the parse path reads its own. + digests: Mapping[str, Digest] + + @classmethod + def create( + cls, *, roots: Sequence[str], known: Mapping[str, File] + ) -> 'Files': + """Returns the files an iteration starts from: nothing + parsed, nothing analyzed.""" + return cls( + roots=tuple(roots), + known=MappingProxyType(dict(known)), + parsed=MappingProxyType({}), + analyzed=MappingProxyType({}), + pending=frozenset(), + digests=MappingProxyType({}), + ) + + def lookup(self, filename: str) -> Optional[ParsedFile | File]: + """Returns what this iteration read for a file, wherever it + lives -- `analyzed` or `parsed` -- and `None` when + it has not read the file.""" + for entries in (self.analyzed, self.parsed): + entry = entries.get(filename) + if entry is not None: + return entry + + return None + + def with_parsed_file(self, parsed: ParsedFile) -> 'Files': + """Returns this with one more file parsed, into `parsed` and + onto the frontier.""" + return replace( + self, + parsed=MappingProxyType({ + **self.parsed, parsed.filename: parsed + }), + pending=self.pending | {parsed.filename}, + ) + + def with_reused_known_file(self, file: File) -> 'Files': + """Returns this with a file the previous iteration analyzed + kept: verified unchanged, so analyzed as it was -- and onto + the frontier, its imports still to follow.""" + return replace( + self, + analyzed=MappingProxyType({ + **self.analyzed, file.filename: file + }), + pending=self.pending | {file.filename}, + ) + + def with_analyzed_file( + self, + filename: str, + *, + dependencies: Mapping[str, Digest], + servicers: Sequence[ServicerInfo], + ) -> 'Files': + """Returns this with a file analyzed: a `File` built from + its `parsed` entry and what analyzing it found, moved to + `analyzed`.""" + parsed = self.parsed[filename] + + file = File( + filename=parsed.filename, + digest=parsed.digest, + imports=parsed.imports, + dependencies=MappingProxyType(dict(dependencies)), + servicers=tuple(servicers), + ) + + parsed_files = dict(self.parsed) + del parsed_files[filename] + + return replace( + self, + parsed=MappingProxyType(parsed_files), + analyzed=MappingProxyType({ + **self.analyzed, filename: file + }), + ) + + def without_pending_file(self, filename: str) -> 'Files': + """Returns this with a file taken off the frontier: analyzed, + and its imports followed.""" + return replace(self, pending=self.pending - {filename}) + + def with_digest(self, filename: str, digest: Digest) -> 'Files': + """Returns this with the digest of one read file recorded, so + a later question about the same bytes is answered without the + disk.""" + return replace( + self, + digests=MappingProxyType({ + **self.digests, filename: digest + }), + ) + + def needs_reanalyzing(self, file: File, *, + digest: Digest) -> tuple[bool, 'Files']: + """Returns whether a file the previous iteration analyzed + cannot be reused -- its bytes changed, or a file it depends + on did, including one that can no longer be read at all -- + and this with every digest read to answer recorded. + + A dependency already read this iteration, reached or merely + digested, is checked against the digest recorded for it, so + answering never reads the same bytes twice. + """ + if digest != file.digest: + return True, self + + files = self + + for filename, previous_digest in file.dependencies.items(): + found = files.lookup(filename) + if found is not None: + if found.digest != previous_digest: + return True, files + continue + + current = files.digests.get(filename) + if current is None: + try: + current = hashlib.sha256(_read(filename)).digest() + except OSError: + return True, files + files = files.with_digest(filename, current) + + if current != previous_digest: + return True, files + + return False, files + + def lookup_or_parse_filename( + self, filename: str + ) -> tuple[Optional[ParsedFile | File], 'Files']: + """Returns a file as this iteration has it -- looked up among + what is already read, verified against the previous iteration + when unchanged, and otherwise read and parsed, joining + `parsed` or `outside` by who contains it. `None` when it + cannot be read or will not parse. + """ + found = self.lookup(filename) + if found is not None: + return found, self + + files = self + source: Optional[bytes] = None + + digest = files.digests.get(filename) + if digest is None: + try: + source = _read(filename) + except OSError: + return None, files + digest = hashlib.sha256(source).digest() + files = files.with_digest(filename, digest) + + known = files.known.get(filename) + if known is not None: + reanalyze, files = files.needs_reanalyzing(known, digest=digest) + if not reanalyze: + return known, files.with_reused_known_file(known) + + # Parsing needs the bytes, which a check that recorded only + # the digest did not keep. + if source is None: + try: + source = _read(filename) + except OSError: + return None, files + digest = hashlib.sha256(source).digest() + files = files.with_digest(filename, digest) + + parsed = _parse(source, filename=filename, digest=digest) + if parsed is None: + return None, files + + return parsed, files.with_parsed_file(parsed) + + def lookup_or_parse_module( + self, module: str, *, visiting: frozenset[str] = frozenset() + ) -> tuple[Optional[ParsedFile | File], 'Files']: + """Returns the file a module names as this iteration has it, + and `None` when no searched root contains the module, the + module is generated -- its name alone says what a name from + one is -- or reading it would circle back to a file already + being read through. When it returns `None`, the files come + back as they were: nothing found means nothing joined. + """ + filename = _try_find_file_of(module, roots=self.roots) + + if filename is None or filename.endswith(GENERATED_SUFFIXES): + return None, self + + if filename in visiting: + return None, self + + return self.lookup_or_parse_filename(filename) + + def imports(self, filename: str) -> Optional[Imports]: + """Returns what a file's imports bound, wherever this + iteration read the file.""" + found = self.lookup(filename) + return found.imports if found is not None else None + + +def _parse(source: bytes, *, filename: str, + digest: Digest) -> Optional[ParsedFile]: + """Returns what parsing one file said, and `None` when it will + not parse. + + A file that will not parse is left unrecorded rather than recorded + as empty, so that the next iteration parses it again: half-written + is the normal state of a file somebody is typing into. + """ + try: + module: ast.Module = ast.parse(source) + except SyntaxError: + return None + + return ParsedFile( + filename=filename, + digest=digest, + imports=_imports(module), + module=module, + ) @dataclass(frozen=True, kw_only=True) @@ -293,23 +596,37 @@ class Call: @dataclass(frozen=True, kw_only=True) class Analysis: - """What analyzing one method's body has found so far. + """One file's analysis in flight: what it has found, what it has + read, and the method it is thinking in. - Immutable: carrying the analysis forward means holding the one a - method below returned, so what anybody else holds is never - changed under them. + Born when a file's analysis starts, finished into `Files` when + the file is done. Immutable: carrying the analysis forward means + keeping the one a method below returned, so nobody else's copy is + ever changed under them. """ - # The state type the servicer this method is written in services, - # which is what `self.ref()` refers to. - state_type: str + # The file being analyzed, whose imports are what names resolve + # against. Moves while a chain of imports is followed, since the + # rest of a chain resolves against each followed file's own + # imports, and comes back when the chain answers. + filename: str - # What the file's imports bound each of its names to. The same - # for the whole method, so carried untouched. - imports: Imports + # The developer's files, read and grown as the analysis resolves + # names, and handed back to the iteration when the file is done. + files: Files + + # The files the analysis's answers depend on, as read so far, by + # the digest each had. If any of them changes, the file needs + # analyzing again. + dependencies: Mapping[str, Digest] - # The Reboot calls found, in the order met. What the analysis is - # for. + # The state type the class being analyzed services, which is what + # `self.ref()` refers to. `None` outside a servicer. + state_type: Optional[str] + + # The Reboot calls the method being analyzed makes, in the order + # met. Folded into the method's entry when the method is + # recorded. calls: tuple[Call, ...] # What was met that the analysis does not follow, spelled the way @@ -317,18 +634,60 @@ class Analysis: # likely incomplete rather than left to trust them. unsupported: tuple[str, ...] - # What each name holds at the point the analysis has reached. A + # What each of the method's names is bound to at the point the + # analysis has reached. The working state the analysis thinks + # with, dropped when the method is recorded. A # `MappingProxyType`, so writing into it raises rather than # quietly leaking. locals: Mapping[str, Local] @classmethod - def create(cls, *, state_type: str, imports: Imports) -> 'Analysis': - """Returns the analysis everything starts from: nothing found, - nothing bound.""" + def create(cls, *, filename: str, files: Files) -> 'Analysis': + """Returns the analysis a file starts from: nothing found, + nothing read, no method begun.""" return cls( - state_type=state_type, - imports=imports, + filename=filename, + files=files, + dependencies=MappingProxyType({}), + state_type=None, + calls=(), + unsupported=(), + locals=MappingProxyType({}), + ) + + def imports(self) -> Imports: + """Returns what the method's file's imports bound.""" + imports = self.files.imports(self.filename) + assert imports is not None + return imports + + def with_dependency( + self, found: ParsedFile | File, *, files: Files + ) -> 'Analysis': + """Returns this analysis depending on one more file: the + digest it was read with recorded, and the files grown by + reading it carried forward.""" + return replace( + self, + dependencies=MappingProxyType( + { + **self.dependencies, found.filename: found.digest + } + ), + files=files, + ) + + def with_state_type(self, state_type: str) -> 'Analysis': + """Returns this analysis inside a class servicing a state + type: what `self.ref()` refers to until the class ends.""" + return replace(self, state_type=state_type) + + def with_method_reset(self) -> 'Analysis': + """Returns this analysis with the method reset -- its calls, + unsupported and locals cleared -- ready for the next + method.""" + return replace( + self, calls=(), unsupported=(), locals=MappingProxyType({}), @@ -357,10 +716,41 @@ def with_unsupported(self, unsupported: ast.AST) -> 'Analysis': ) -def _reboot_related(expression: ast.expr, *, analysis: Analysis) -> bool: +def _state_type_if_servicer( + class_definition: ast.ClassDef, + *, + analysis: Analysis, +) -> tuple[Optional[str], Analysis]: + """Returns the state type a class services, if it says so, and + the analysis carried forward, since answering may read files. + + A servicer says it by what it inherits: `Account.Servicer`, or + `Account.singleton.Servicer` for a singleton, with `Account` + reached by any dotted name that refers to a state type. + """ + for base in class_definition.bases: + match base: + case ast.Attribute(value=value, attr='Servicer'): + path = _path(value) + if path is None: + continue + if len(path) > 1 and path[-1] == 'singleton': + path = path[:-1] + state_type, analysis = analysis.imports( + ).try_resolve_state_type(path, analysis=analysis) + if state_type is not None: + return state_type, analysis + + return None, analysis + + +def _reboot_related(expression: ast.expr, *, + analysis: Analysis) -> tuple[bool, Analysis]: """Returns whether anything in an expression touches something - Reboot related: a `.ref` however it is reached, a name holding a - reference or the context, or a name that refers to a state type. + Reboot related -- a `.ref` however it is reached, a name holding + a reference or the context, or a name that refers to a state type + -- and the analysis carried forward, since answering may read + files. What ordinary Python does is never Reboot related, however little of it the analysis follows; this is what keeps the unsupported @@ -369,15 +759,16 @@ def _reboot_related(expression: ast.expr, *, analysis: Analysis) -> bool: for node in ast.walk(expression): match node: case ast.Attribute(attr='ref'): - return True + return True, analysis case ast.Name(id=str(name)): if name in analysis.locals: - return True - state_type = analysis.imports.try_resolve_state_type([name]) + return True, analysis + state_type, analysis = analysis.imports( + ).try_resolve_state_type([name], analysis=analysis) if state_type is not None: - return True + return True, analysis - return False + return False, analysis def _evaluate(expression: ast.expr, *, @@ -395,7 +786,7 @@ def _evaluate(expression: ast.expr, *, case ast.Call( func=ast.Attribute(value=ast.Name(id='self'), attr='ref') - ): + ) if analysis.state_type is not None: # A servicer reaching the state it is servicing. Matched # before `Account.ref(id)` below, which would otherwise # match this too and find that `self` refers to no state @@ -410,7 +801,8 @@ def _evaluate(expression: ast.expr, *, # is almost certainly a reference being lost. path = _path(receiver) if path is not None: - state_type = analysis.imports.try_resolve_state_type(path) + state_type, analysis = analysis.imports( + ).try_resolve_state_type(path, analysis=analysis) if state_type is not None: return Reference(state_type=state_type), analysis @@ -422,7 +814,8 @@ def _evaluate(expression: ast.expr, *, # touches something Reboot related -- whatever it does with it is # not followed, so the calls are likely incomplete -- and left # alone when it is ordinary Python, which was never claimed. - if _reboot_related(expression, analysis=analysis): + related, analysis = _reboot_related(expression, analysis=analysis) + if related: analysis = analysis.with_unsupported(expression) return None, analysis @@ -489,11 +882,10 @@ def _assign( def _analyze_method( method: ast.FunctionDef | ast.AsyncFunctionDef, *, - state_type: str, - imports: Imports, + analysis: Analysis, ) -> Analysis: - """Returns the Reboot calls a method's body makes, in the order - met. + """Returns the analysis carried through one method's body, the + Reboot calls it makes found in the order met. Statements are visited in the order written, saying what each name holds before moving to the next, so that when a call is met @@ -513,8 +905,6 @@ def _analyze_method( reference and something else, and taking them together is what lets a reference reach a nested function that uses it. """ - analysis = Analysis.create(state_type=state_type, imports=imports) - # `self` first, then the context; a `@classmethod` takes `cls` # in its place, and a workflow or a task takes the context the # same way. @@ -547,56 +937,56 @@ def _digest(node: ast.AST) -> Digest: include_attributes=False).encode()).digest() -def _methods(class_definition: ast.ClassDef) -> list[ServicerInfo.Method]: - """Returns the methods a class defines, in the order written.""" - methods = [] +def _analyze_file(filename: str, files: Files) -> Files: + """Returns the iteration carried past analyzing one `pending` + file: every class saying what it services recorded, every method + those classes define analyzed and recorded under them, and the + file finished into `analyzed` with the files the analysis read + depended on. - for node in class_definition.body: - match node: - case ( - ast.FunctionDef(name=str(name)) | - ast.AsyncFunctionDef(name=str(name)) - ): - methods.append( - ServicerInfo.Method(name=name, digest=_digest(node)) - ) - - return methods - - -def _parse(source: bytes, *, digest: Digest, filename: str) -> Optional[File]: - """Returns what a file holds, and `None` when it will not parse. - - A file that will not parse is left unrecorded rather than recorded - as empty, so that the next iteration parses it again: half-written - is the normal state of a file somebody is typing into. + One `Analysis` is carried through the whole file: the + dependencies it accumulates are the file's, while the method + resets as each one is recorded. """ - try: - module: ast.Module = ast.parse(source) - except SyntaxError: - return None + parsed = files.parsed[filename] - imports = _imports(module) + analysis = Analysis.create(filename=filename, files=files) - servicers = [] + servicers: list[ServicerInfo] = [] - for node in ast.walk(module): + for node in ast.walk(parsed.module): match node: case ast.ClassDef(): - state_type = _state_type_if_servicer(node, imports=imports) - if state_type is not None: - servicers.append( - ServicerInfo( - state_type=state_type, - file=filename, - methods=_methods(node), - ) - ) - - return File( - digest=digest, - may_load=imports.may_load, - servicers=tuple(servicers), + state_type, analysis = _state_type_if_servicer( + node, analysis=analysis + ) + if state_type is None: + continue + servicer = ServicerInfo(state_type=state_type, file=filename) + analysis = analysis.with_state_type(state_type) + for statement in node.body: + match statement: + case ( + ast.FunctionDef(name=str(name)) | + ast.AsyncFunctionDef(name=str(name)) + ): + analysis = _analyze_method( + statement, analysis=analysis + ) + servicer.methods.append( + ServicerInfo.Method( + name=name, digest=_digest(statement) + ) + ) + # Calls and unsupported fold in here once + # the proto has fields for them. + analysis = analysis.with_method_reset() + servicers.append(servicer) + + return analysis.files.with_analyzed_file( + filename, + dependencies=dict(analysis.dependencies), + servicers=servicers, ) @@ -628,10 +1018,9 @@ async def analyze( absent, however recently it changed, and one that has started being imported is parsed for the first time. - `known` is what a previous call returned, and spares this one from - parsing a file whose bytes have not changed since. Parsing is the - expensive part -- around fifty times what reading and hashing the - bytes costs -- and an edit changes one file. + `known` is what a previous call returned, and spares this one + from parsing and analyzing a file that has not changed since -- + neither in its own bytes nor in any file it depends on. `roots` are the directories a module may be found under, which is both how a module name becomes a file and where the developer's @@ -642,48 +1031,36 @@ async def analyze( if roots is None: roots = _roots(application) - if known is None: - known = {} + files = Files.create(roots=roots, known=known or {}) - reachable: dict[str, File] = {} + _, files = files.lookup_or_parse_filename(application) - pending = [application] + while len(files.pending) > 0: + # Parsing and analyzing hold on to the interpreter for as + # long as they take, so a file at a time leaves the dashboard + # free to answer. + async for filename in cooperatively(files.pending): + match files.parsed.get(filename): + case ParsedFile(): + files = _analyze_file(filename, files) + case None: + # Nothing to analyze: the file was kept because + # neither its bytes nor any of its dependencies + # changed, so the analysis it got when it last + # changed still stands, in `analyzed` already. + pass - while len(pending) > 0: - # What the last round of imports led to. - current, pending = pending, [] - - # Parsing holds on to the interpreter for as long as it takes, - # so a file at a time leaves the dashboard free to answer. - async for filename in cooperatively(current): - if filename in reachable: - continue - - try: - source: bytes = _read(filename) - except OSError: - continue - - digest = hashlib.sha256(source).digest() - - file = known.get(filename) - - if file is None or file.digest != digest: - file = _parse(source, digest=digest, filename=filename) - - if file is None: - continue + files = files.without_pending_file(filename) - reachable[filename] = file + # Following the file's imports is how the rest of the + # application is reached. + imports = files.imports(filename) + assert imports is not None - pending.extend( - filename for filename in ( - _try_find_file_of(module, roots=roots) - for module in file.may_load - ) if filename is not None - ) + for module in imports.may_load: + _, files = files.lookup_or_parse_module(module) - return reachable + return dict(files.analyzed) async def watch(context: WorkflowContext, *, application: str) -> None: diff --git a/tests/reboot/dashboard/implementation_watcher_tests.py b/tests/reboot/dashboard/implementation_watcher_tests.py index 2852f3fd..2362cd48 100644 --- a/tests/reboot/dashboard/implementation_watcher_tests.py +++ b/tests/reboot/dashboard/implementation_watcher_tests.py @@ -106,21 +106,28 @@ class AnalyzeTest(unittest.TestCase): what each name held when the body ended.""" def _analyze(self, body: str) -> implementation_watcher.Analysis: - module = ast.parse( - SERVICER.format(state='Shop', module='shop').replace( - ' async def look(self, context, request):\n pass', - ' async def look(self, context, request):\n' + body, - ) + source = SERVICER.format(state='Shop', module='shop').replace( + ' async def look(self, context, request):\n pass', + ' async def look(self, context, request):\n' + body, + ).encode() + + parsed = implementation_watcher._parse( + source, filename='servicer.py', digest=b'' ) - imports = implementation_watcher._imports(module) + assert parsed is not None + + files = implementation_watcher.Files.create(roots=[], known={} + ).with_parsed_file(parsed) + + analysis = implementation_watcher.Analysis.create( + filename='servicer.py', files=files + ).with_state_type('shop.v1.Shop') - for node in ast.walk(module): + for node in ast.walk(parsed.module): match node: case ast.AsyncFunctionDef(name='look'): return implementation_watcher._analyze_method( - node, - state_type='shop.v1.Shop', - imports=imports, + node, analysis=analysis ) raise AssertionError('no method to analyze') @@ -830,6 +837,79 @@ async def test_a_servicer_that_starts_being_imported_is_found( ] ) + ################################################################### + # Followed to the state type however it is reached. + + async def test_a_state_type_reexported_through_another_file(self) -> None: + self._write('exports.py', source='from shop.v1.shop_rbt import Shop\n') + self._write( + 'shop_servicer.py', + source=SHOP.replace( + 'from shop.v1.shop_rbt import Shop', + 'from exports import Shop', + ), + ) + application = self._write('main.py', source=APPLICATION) + + found = _state_types_and_files(await analyze(application=application)) + + self.assertEqual( + found, + [('shop.v1.Shop', str(self.directory / 'shop_servicer.py'))], + ) + + async def test_a_changed_reexport_reresolves_its_dependents(self) -> None: + """The staleness the `followed` digests exist to close: what a + file's servicers say depends on the files resolving them read, + however unchanged its own bytes are.""" + self._write('exports.py', source='from shop.v1.shop_rbt import Shop\n') + self._write( + 'shop_servicer.py', + source=SHOP.replace( + 'from shop.v1.shop_rbt import Shop', + 'from exports import Shop', + ), + ) + application = self._write('main.py', source=APPLICATION) + + known = await analyze(application=application) + self.assertEqual( + _state_types_and_files(known), + [('shop.v1.Shop', str(self.directory / 'shop_servicer.py'))], + ) + + # `shop_servicer.py` is untouched; only what `Shop` refers to + # changes. + self._write( + 'exports.py', + source='from shop.v1.depot_rbt import Depot as Shop\n', + ) + + found = _state_types_and_files( + await analyze(application=application, known=known) + ) + + self.assertEqual( + found, + [('shop.v1.Depot', str(self.directory / 'shop_servicer.py'))], + ) + + async def test_a_circular_import_resolves_to_nothing(self) -> None: + self._write('back.py', source='from forth import Shop\n') + self._write('forth.py', source='from back import Shop\n') + self._write( + 'shop_servicer.py', + source=SHOP.replace( + 'from shop.v1.shop_rbt import Shop', + 'from back import Shop', + ), + ) + application = self._write('main.py', source=APPLICATION) + + found = _state_types_and_files(await analyze(application=application)) + + self.assertEqual(found, []) + ################################################################### # What it finds no servicer for. From 31f40f6e86a8d3c06159c2e2a6ef8e7624615330 Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Wed, 19 Aug 2026 06:08:13 +0000 Subject: [PATCH 14/31] Resolve a state type aliased at top level A top-level `Alias = Name` binds the alias to whatever the name was imported as, in written order, so an alias of an alias resolves too. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QPEVMhyDRxZEuH8eiykv99 --- reboot/dashboard/implementation_watcher.py | 15 ++++++++++++++- .../dashboard/implementation_watcher_tests.py | 17 +++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/reboot/dashboard/implementation_watcher.py b/reboot/dashboard/implementation_watcher.py index f9efeea4..6730e37b 100644 --- a/reboot/dashboard/implementation_watcher.py +++ b/reboot/dashboard/implementation_watcher.py @@ -86,7 +86,8 @@ class Symbol: # Its name there, which the local name may differ from. name: str - # By the name the file calls it. + # By the name the file calls it. A top-level `Alias = Name` binds + # the alias to whatever `Name` was bound to. bindings: Mapping[str, 'Import'] # Every module these imports may have Python load: `import a.b.c` @@ -193,6 +194,18 @@ def _imports(module: ast.Module) -> Imports: # to nothing and is dropped. may_load.append(_join(from_module, alias.name)) + # A top-level `Alias = Name` binds `Alias` to whatever `Name` was + # bound to. In written order, so an alias of an alias resolves + # too. After the imports, whose bindings are what an alias copies. + for statement in module.body: + match statement: + case ast.Assign( + targets=[ast.Name(id=str(target))], + value=ast.Name(id=str(value)), + ): + if value in bindings: + bindings[target] = bindings[value] + return Imports( bindings=MappingProxyType(bindings), may_load=tuple(may_load), diff --git a/tests/reboot/dashboard/implementation_watcher_tests.py b/tests/reboot/dashboard/implementation_watcher_tests.py index 2362cd48..b2359c06 100644 --- a/tests/reboot/dashboard/implementation_watcher_tests.py +++ b/tests/reboot/dashboard/implementation_watcher_tests.py @@ -858,6 +858,23 @@ async def test_a_state_type_reexported_through_another_file(self) -> None: [('shop.v1.Shop', str(self.directory / 'shop_servicer.py'))], ) + async def test_a_state_type_aliased_at_top_level(self) -> None: + self._write( + 'shop_servicer.py', + source=SHOP.replace( + 'class ShopServicer(Shop.Servicer):', + 'Store = Shop\n\n\nclass ShopServicer(Store.Servicer):', + ), + ) + application = self._write('main.py', source=APPLICATION) + + found = _state_types_and_files(await analyze(application=application)) + + self.assertEqual( + found, + [('shop.v1.Shop', str(self.directory / 'shop_servicer.py'))], + ) + async def test_a_changed_reexport_reresolves_its_dependents(self) -> None: """The staleness the `followed` digests exist to close: what a file's servicers say depends on the files resolving them read, From 83f4c52a6caea3943f5642024bb6b661823efc36 Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Wed, 19 Aug 2026 06:08:36 +0000 Subject: [PATCH 15/31] Resolve a state type reached through a module alias `import shop.v1.shop_rbt as rbt` binds `rbt` to the module, and `rbt.Shop` reaches the state type through it -- as does `import a.b`, which binds `a`, a component at a time. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QPEVMhyDRxZEuH8eiykv99 --- reboot/dashboard/implementation_watcher.py | 30 +++++++++++++++++-- .../dashboard/implementation_watcher_tests.py | 17 +++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/reboot/dashboard/implementation_watcher.py b/reboot/dashboard/implementation_watcher.py index 6730e37b..7ef1e7e0 100644 --- a/reboot/dashboard/implementation_watcher.py +++ b/reboot/dashboard/implementation_watcher.py @@ -86,6 +86,13 @@ class Symbol: # Its name there, which the local name may differ from. name: str + @dataclass(frozen=True, kw_only=True) + class Module: + """What `import x [as y]` binds a name to: module `x` + itself.""" + + module: str + # By the name the file calls it. A top-level `Alias = Name` binds # the alias to whatever `Name` was bound to. bindings: Mapping[str, 'Import'] @@ -107,6 +114,12 @@ def try_resolve_module(self, case Imports.Symbol(module=module, name=name) if not rest: return module, name + case Imports.Module(module=module) if rest: + # A module alone is never more than a module, so a + # name bound to one refers to something only with + # more components. + return _join(module, *rest[:-1]), rest[-1] + return None def try_resolve_state_type( @@ -157,7 +170,7 @@ def try_resolve_state_type( # What one import bound a name to. -Import = Imports.Symbol +Import = Imports.Symbol | Imports.Module def _imports(module: ast.Module) -> Imports: @@ -175,7 +188,20 @@ def _imports(module: ast.Module) -> Imports: for node in ast.walk(module): match node: case ast.Import(names=names): - may_load.extend(alias.name for alias in names) + for alias in names: + may_load.append(alias.name) + if alias.asname is not None: + bindings.setdefault( + alias.asname, + Imports.Module(module=alias.name), + ) + else: + # `import a.b` binds `a`, and `a.b.Shop` is + # reached from it a component at a time. + first = alias.name.split('.', 1)[0] + bindings.setdefault( + first, Imports.Module(module=first) + ) case ast.ImportFrom(module=str(from_module), level=0, names=names): may_load.append(from_module) diff --git a/tests/reboot/dashboard/implementation_watcher_tests.py b/tests/reboot/dashboard/implementation_watcher_tests.py index b2359c06..d13619c4 100644 --- a/tests/reboot/dashboard/implementation_watcher_tests.py +++ b/tests/reboot/dashboard/implementation_watcher_tests.py @@ -875,6 +875,23 @@ async def test_a_state_type_aliased_at_top_level(self) -> None: [('shop.v1.Shop', str(self.directory / 'shop_servicer.py'))], ) + async def test_a_state_type_imported_through_a_module_alias(self) -> None: + self._write( + 'shop_servicer.py', + source=SHOP.replace( + 'from shop.v1.shop_rbt import Shop', + 'import shop.v1.shop_rbt as generated', + ).replace('(Shop.Servicer)', '(generated.Shop.Servicer)'), + ) + application = self._write('main.py', source=APPLICATION) + + found = _state_types_and_files(await analyze(application=application)) + + self.assertEqual( + found, + [('shop.v1.Shop', str(self.directory / 'shop_servicer.py'))], + ) + async def test_a_changed_reexport_reresolves_its_dependents(self) -> None: """The staleness the `followed` digests exist to close: what a file's servicers say depends on the files resolving them read, From be811b3123b128cb1e951ddaeae43589df262de7 Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Wed, 19 Aug 2026 06:09:01 +0000 Subject: [PATCH 16/31] Resolve a state type reached through an imported submodule `from shop.v1 import shop_rbt` imports a module as easily as a name, so a dotted name whose head was imported that way is tried as a module of where it came from: `shop_rbt.Shop` is `shop.v1.shop_rbt`'s `Shop`. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QPEVMhyDRxZEuH8eiykv99 --- reboot/dashboard/implementation_watcher.py | 6 +++++- .../dashboard/implementation_watcher_tests.py | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/reboot/dashboard/implementation_watcher.py b/reboot/dashboard/implementation_watcher.py index 7ef1e7e0..0df7064c 100644 --- a/reboot/dashboard/implementation_watcher.py +++ b/reboot/dashboard/implementation_watcher.py @@ -111,7 +111,11 @@ def try_resolve_module(self, head, *rest = path match self.bindings.get(head): - case Imports.Symbol(module=module, name=name) if not rest: + case Imports.Symbol(module=module, name=name): + if rest: + # Used dotted: the code treats it as a module, + # and is believed. + return _join(module, name, *rest[:-1]), rest[-1] return module, name case Imports.Module(module=module) if rest: diff --git a/tests/reboot/dashboard/implementation_watcher_tests.py b/tests/reboot/dashboard/implementation_watcher_tests.py index d13619c4..838ec62e 100644 --- a/tests/reboot/dashboard/implementation_watcher_tests.py +++ b/tests/reboot/dashboard/implementation_watcher_tests.py @@ -892,6 +892,25 @@ async def test_a_state_type_imported_through_a_module_alias(self) -> None: [('shop.v1.Shop', str(self.directory / 'shop_servicer.py'))], ) + async def test_a_state_type_reached_through_an_imported_submodule( + self + ) -> None: + self._write( + 'shop_servicer.py', + source=SHOP.replace( + 'from shop.v1.shop_rbt import Shop', + 'from shop.v1 import shop_rbt', + ).replace('(Shop.Servicer)', '(shop_rbt.Shop.Servicer)'), + ) + application = self._write('main.py', source=APPLICATION) + + found = _state_types_and_files(await analyze(application=application)) + + self.assertEqual( + found, + [('shop.v1.Shop', str(self.directory / 'shop_servicer.py'))], + ) + async def test_a_changed_reexport_reresolves_its_dependents(self) -> None: """The staleness the `followed` digests exist to close: what a file's servicers say depends on the files resolving them read, From 88e7eba072f314a81e8c1e7c636622afe8d904c2 Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Wed, 19 Aug 2026 06:09:25 +0000 Subject: [PATCH 17/31] Resolve state types through relative imports A relative import is resolved to a path where it is collected, since that is the one place the importing file's own directory is known; from there it is followed like any other module, spelled as a path, with `os.sep` telling the two spellings apart. A name from a generated module reached relatively is spelled back as dotted from the root that holds it, so the same state type is named whichever way it was imported. An iteration follows them to their files too, so a servicer reachable only through one is found -- with files deduped by their absolute path, since a file can now be reached under two spellings. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QPEVMhyDRxZEuH8eiykv99 --- reboot/dashboard/implementation_watcher.py | 146 +++++++++++++++--- .../dashboard/implementation_watcher_tests.py | 62 ++++++++ 2 files changed, 188 insertions(+), 20 deletions(-) diff --git a/reboot/dashboard/implementation_watcher.py b/reboot/dashboard/implementation_watcher.py index 0df7064c..2501c3fe 100644 --- a/reboot/dashboard/implementation_watcher.py +++ b/reboot/dashboard/implementation_watcher.py @@ -35,6 +35,7 @@ class AccountServicer(Account.Servicer): import hashlib import os from dataclasses import dataclass, replace +from pathlib import Path from rbt.dashboard.v1.dashboard_pb2 import ServicerInfo from rbt.dashboard.v1.dashboard_rbt import Implementation from reboot.aio.contexts import WorkflowContext @@ -154,6 +155,34 @@ def try_resolve_state_type( # The `_rbt` module name alone says what a name from one is: # `Account` from `bank.v1.account_rbt` is `bank.v1.Account`. if module.endswith(GENERATED_SUFFIX): + # A relative import spells its module as a path, but a + # state type's name is dotted. In a file + # `backend/src/shop/cart/servicer.py`: + # + # from ..v1 import shop_rbt + # module -> `backend/src/shop/v1/shop_rbt` + # + # `Shop` from that module must still be named + # `shop.v1.Shop`, the same as if it had been imported + # as `shop.v1.shop_rbt`. The dots come back by cutting + # the root off the front of the path: cutting + # `backend/src` leaves `shop/v1/shop_rbt`, and joining + # its parts with dots gives `shop.v1.shop_rbt`. If no + # root contains the path, there is no way to spell the + # name, so there is no answer. + if os.sep in module: + module_path = Path(module).absolute() + for root in list(analysis.files.roots): + directory = Path(root).absolute() + if module_path.is_relative_to(directory): + dotted = '.'.join( + module_path.relative_to(directory).parts + ) + return ( + f"{dotted.rsplit('.', 1)[0]}.{attribute}", + analysis, + ) + return None, analysis return f"{module.rsplit('.', 1)[0]}.{attribute}", analysis found, files = analysis.files.lookup_or_parse_module( @@ -177,7 +206,9 @@ def try_resolve_state_type( Import = Imports.Symbol | Imports.Module -def _imports(module: ast.Module) -> Imports: +def _imports( + module: ast.Module, *, directory: Optional[str] = None +) -> Imports: """Returns what a file's imports bound each of its names to. Names come from every import, wherever it is written. One inside @@ -185,6 +216,9 @@ def _imports(module: ast.Module) -> Imports: file does, and guarding an import is common enough to be worth reading. `ast.walk` yields the shallowest first, so a name bound at the top of the file wins over one bound inside something. + + `directory` is where the file itself is, which is what a relative + import is relative to; without it relative imports bind nothing. """ bindings: dict[str, Import] = {} may_load: list[str] = [] @@ -207,13 +241,51 @@ def _imports(module: ast.Module) -> Imports: first, Imports.Module(module=first) ) - case ast.ImportFrom(module=str(from_module), level=0, names=names): - may_load.append(from_module) + case ast.ImportFrom( + module=from_module, level=int(level), names=names + ): + # Having a `level` means we have to determine what + # file Python might load. For example, in a file + # `shop/cart/servicer.py`: + # + # from shop.cart.types import Item + # level 0 -> `shop.cart.types` + # from .types import Item + # level 1 -> `shop/cart/types` + # from ..api import Item + # level 2 -> `shop/api` + # from .. import api + # level 2, no module -> `shop` + # + # `level` counts the leading dots. No dots: keep the + # module as written. One dot: start from the file's + # own directory. Each extra dot: climb one parent. + # If a module comes after the dots, it goes under + # that directory: `..api` is `shop` plus `api`, so + # `shop/api`. A relative import can only be spelled + # as a path, and without a `directory` the dots + # point at nothing, so the import binds nothing. + if level == 0: + if from_module is None: + continue + base = from_module + elif directory is not None: + climbed = directory + for _ in range(level - 1): + climbed = os.path.dirname(climbed) + if from_module is None: + base = climbed + else: + base = os.path.join(climbed, *from_module.split('.')) + else: + continue + + may_load.append(base) for alias in names: bindings.setdefault( alias.asname or alias.name, - Imports.Symbol(module=from_module, name=alias.name), + Imports.Symbol(module=base, name=alias.name), ) # Since we cannot tell whether `y` in # `from x import y` is a module of its own or a @@ -222,7 +294,7 @@ def _imports(module: ast.Module) -> Imports: # if a root holds one by that name, so `x.y` # naming something that is not a module resolves # to nothing and is dropped. - may_load.append(_join(from_module, alias.name)) + may_load.append(_join(base, alias.name)) # A top-level `Alias = Name` binds `Alias` to whatever `Name` was # bound to. In written order, so an alias of an alias resolves @@ -246,19 +318,28 @@ def _try_find_file_of(module: str, *, roots: Sequence[str]) -> Optional[str]: """Returns the file a module names if one of `roots` contains it, and `None` otherwise. - `None` is not a failure: the standard library and installed - packages live outside every root, so `import asyncio` resolves to - nothing and there is nothing to read. + A module spelled as a path -- what a relative import resolved to + -- is looked for where it already points. `None` is not a failure: + the standard library and installed packages live outside every + root, so `import asyncio` resolves to nothing and there is nothing + to read. """ - relative = module.replace('.', os.sep) - - for root in roots: - for candidate in ( - os.path.join(root, relative + '.py'), - os.path.join(root, relative, '__init__.py'), - ): - if os.path.isfile(candidate): - return candidate + if os.sep in module: + candidates = [ + module + '.py', + os.path.join(module, '__init__.py'), + ] + else: + relative = module.replace('.', os.sep) + candidates = [ + os.path.join(root, relative + suffix) + for root in roots + for suffix in ('.py', os.sep + '__init__.py') + ] + + for candidate in candidates: + if os.path.isfile(candidate): + return candidate return None @@ -278,7 +359,11 @@ def _path(expression: ast.expr) -> Optional[list[str]]: def _join(base: str, *parts: str) -> str: - """Returns a module extended by more components.""" + """Returns a module extended by more components, joined the way + the module is spelled: with dots for a dotted name, as a path for + a path.""" + if os.sep in base: + return os.path.join(base, *parts) return '.'.join([base, *parts]) @@ -382,12 +467,22 @@ def create( def lookup(self, filename: str) -> Optional[ParsedFile | File]: """Returns what this iteration read for a file, wherever it lives -- `analyzed` or `parsed` -- and `None` when - it has not read the file.""" + it has not read the file. + + By the file's absolute path, since a relative import spells + a file one way and an absolute import another. + """ for entries in (self.analyzed, self.parsed): entry = entries.get(filename) if entry is not None: return entry + absolute = Path(filename).absolute() + for entries in (self.analyzed, self.parsed): + for entry in entries.values(): + if Path(entry.filename).absolute() == absolute: + return entry + return None def with_parsed_file(self, parsed: ParsedFile) -> 'Files': @@ -522,6 +617,15 @@ def lookup_or_parse_filename( files = files.with_digest(filename, digest) known = files.known.get(filename) + if known is None: + # A relative import spells a file one way and an absolute + # import another, so the previous iteration is searched + # by absolute path too. + absolute = Path(filename).absolute() + for file in files.known.values(): + if Path(file.filename).absolute() == absolute: + known = file + break if known is not None: reanalyze, files = files.needs_reanalyzing(known, digest=digest) if not reanalyze: @@ -587,7 +691,9 @@ def _parse(source: bytes, *, filename: str, return ParsedFile( filename=filename, digest=digest, - imports=_imports(module), + imports=_imports( + module, directory=os.path.dirname(os.path.abspath(filename)) + ), module=module, ) diff --git a/tests/reboot/dashboard/implementation_watcher_tests.py b/tests/reboot/dashboard/implementation_watcher_tests.py index 838ec62e..d643676a 100644 --- a/tests/reboot/dashboard/implementation_watcher_tests.py +++ b/tests/reboot/dashboard/implementation_watcher_tests.py @@ -911,6 +911,68 @@ async def test_a_state_type_reached_through_an_imported_submodule( [('shop.v1.Shop', str(self.directory / 'shop_servicer.py'))], ) + async def test_a_servicer_reached_through_a_relative_import(self) -> None: + self._write('package/__init__.py', source='') + self._write('package/shop_servicer.py', source=SHOP) + self._write( + 'package/servicers.py', + source='from .shop_servicer import ShopServicer\n', + ) + application = self._write( + 'main.py', + source=APPLICATION.replace( + 'from shop_servicer import ShopServicer', + 'from package.servicers import ShopServicer', + ), + ) + + found = _state_types_and_files(await analyze(application=application)) + + self.assertEqual( + found, + [ + ( + 'shop.v1.Shop', + str(self.directory / 'package' / 'shop_servicer.py'), + ) + ], + ) + + async def test_a_state_type_reexported_through_a_relative_import( + self + ) -> None: + self._write('package/__init__.py', source='') + self._write( + 'package/exports.py', + source='from shop.v1.shop_rbt import Shop\n', + ) + self._write( + 'package/shop_servicer.py', + source=SHOP.replace( + 'from shop.v1.shop_rbt import Shop', + 'from .exports import Shop', + ), + ) + application = self._write( + 'main.py', + source=APPLICATION.replace( + 'from shop_servicer import ShopServicer', + 'from package.shop_servicer import ShopServicer', + ), + ) + + found = _state_types_and_files(await analyze(application=application)) + + self.assertEqual( + found, + [ + ( + 'shop.v1.Shop', + str(self.directory / 'package' / 'shop_servicer.py'), + ) + ], + ) + async def test_a_changed_reexport_reresolves_its_dependents(self) -> None: """The staleness the `followed` digests exist to close: what a file's servicers say depends on the files resolving them read, From 9a4d89e67bbe18dafbbedeced2c681d50736eefb Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Wed, 19 Aug 2026 06:09:49 +0000 Subject: [PATCH 18/31] Resolve a state type that was star imported A name bound by no import may be any star-imported module's. The last star import wins in Python, so they are searched last to first. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QPEVMhyDRxZEuH8eiykv99 --- reboot/dashboard/implementation_watcher.py | 31 ++++++++++++++++++- .../dashboard/implementation_watcher_tests.py | 18 +++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/reboot/dashboard/implementation_watcher.py b/reboot/dashboard/implementation_watcher.py index 2501c3fe..9ef4af2d 100644 --- a/reboot/dashboard/implementation_watcher.py +++ b/reboot/dashboard/implementation_watcher.py @@ -98,6 +98,10 @@ class Module: # the alias to whatever `Name` was bound to. bindings: Mapping[str, 'Import'] + # The modules star-imported -- which bind no name, and so are + # searched, last to first, when no binding answers. + stars: tuple[str, ...] + # Every module these imports may have Python load: `import a.b.c` # loads `a.b.c`; `from x import y` loads `x`, and `x.y` too when # `y` is a module of its own; a star import loads its module. @@ -108,7 +112,9 @@ class Module: def try_resolve_module(self, path: Sequence[str]) -> Optional[tuple[str, str]]: """Returns the module a dotted name refers into and the name - it refers to there, and `None` when no binding answers.""" + it refers to there, and `None` when no binding answers -- + which leaves the star imports, whose files this cannot read. + """ head, *rest = path match self.bindings.get(head): @@ -148,6 +154,24 @@ def try_resolve_state_type( resolved = self.try_resolve_module(path) if resolved is None: + # The name may be any star-imported module's. The last + # star import wins in Python, so they are searched last + # to first -- each file's own imports answering for the + # names it may define. + for star in reversed(self.stars): + found, files = analysis.files.lookup_or_parse_module( + star, visiting=visiting + ) + if found is None: + continue + analysis = analysis.with_dependency(found, files=files) + state_type, analysis = found.imports.try_resolve_state_type( + path, + analysis=analysis, + visiting=visiting | {found.filename}, + ) + if state_type is not None: + return state_type, analysis return None, analysis module, attribute = resolved @@ -221,6 +245,7 @@ def _imports( import is relative to; without it relative imports bind nothing. """ bindings: dict[str, Import] = {} + stars: list[str] = [] may_load: list[str] = [] for node in ast.walk(module): @@ -283,6 +308,9 @@ def _imports( may_load.append(base) for alias in names: + if alias.name == '*': + stars.append(base) + continue bindings.setdefault( alias.asname or alias.name, Imports.Symbol(module=base, name=alias.name), @@ -310,6 +338,7 @@ def _imports( return Imports( bindings=MappingProxyType(bindings), + stars=tuple(stars), may_load=tuple(may_load), ) diff --git a/tests/reboot/dashboard/implementation_watcher_tests.py b/tests/reboot/dashboard/implementation_watcher_tests.py index d643676a..c4856ac0 100644 --- a/tests/reboot/dashboard/implementation_watcher_tests.py +++ b/tests/reboot/dashboard/implementation_watcher_tests.py @@ -973,6 +973,24 @@ async def test_a_state_type_reexported_through_a_relative_import( ], ) + async def test_a_state_type_star_imported(self) -> None: + self._write('exports.py', source='from shop.v1.shop_rbt import Shop\n') + self._write( + 'shop_servicer.py', + source=SHOP.replace( + 'from shop.v1.shop_rbt import Shop', + 'from exports import *', + ), + ) + application = self._write('main.py', source=APPLICATION) + + found = _state_types_and_files(await analyze(application=application)) + + self.assertEqual( + found, + [('shop.v1.Shop', str(self.directory / 'shop_servicer.py'))], + ) + async def test_a_changed_reexport_reresolves_its_dependents(self) -> None: """The staleness the `followed` digests exist to close: what a file's servicers say depends on the files resolving them read, From 2812187097d11e13ccb07a488dd6d80c532a1926 Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Wed, 19 Aug 2026 06:10:11 +0000 Subject: [PATCH 19/31] Resolve state types re-exported by installed packages The std library offers its state types this way: `from reboot.std.collections.v1.sorted_map import SortedMap` is a plain module re-exporting from generated code. A name now resolves through the directories on `sys.path` as well -- the developer's own, since the dashboard runs in their environment. Installed files are read for what their names refer to and nothing else: never analyzed, never watched. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QPEVMhyDRxZEuH8eiykv99 --- reboot/dashboard/implementation_watcher.py | 113 ++++++++++++++---- .../dashboard/implementation_watcher_tests.py | 71 ++++++++++- 2 files changed, 159 insertions(+), 25 deletions(-) diff --git a/reboot/dashboard/implementation_watcher.py b/reboot/dashboard/implementation_watcher.py index 9ef4af2d..e37374c8 100644 --- a/reboot/dashboard/implementation_watcher.py +++ b/reboot/dashboard/implementation_watcher.py @@ -17,8 +17,9 @@ class AccountServicer(Account.Servicer): file defining the servicer had to be imported for any of them to run. Where following stops is what makes this the developer's code rather -than somebody else's. A module resolves to a file only if a root -holds it, so an import of an installed package leads nowhere. +than somebody else's. A module resolves to a file only if a root or a +package holds it, and a file a package holds is read for what its +imports bound, and nothing else. Read rather than imported, because importing an application means having its generated code, its dependencies and its `sys.path`, and @@ -34,6 +35,7 @@ class AccountServicer(Account.Servicer): import ast import hashlib import os +import sys from dataclasses import dataclass, replace from pathlib import Path from rbt.dashboard.v1.dashboard_pb2 import ServicerInfo @@ -72,6 +74,13 @@ def _read(filename: str) -> bytes: return file.read() +def _is_file_within_directory(*, filename: str, directory: str) -> bool: + """Returns whether a file is inside a directory, by their + absolute paths, so the answer does not depend on how either was + spelled.""" + return Path(filename).absolute().is_relative_to(Path(directory).absolute()) + + @dataclass(frozen=True, kw_only=True) class Imports: """What a file's imports bound each of its names to.""" @@ -160,7 +169,7 @@ def try_resolve_state_type( # names it may define. for star in reversed(self.stars): found, files = analysis.files.lookup_or_parse_module( - star, visiting=visiting + star, include_packages_in_roots=True, visiting=visiting ) if found is None: continue @@ -196,7 +205,7 @@ def try_resolve_state_type( # name, so there is no answer. if os.sep in module: module_path = Path(module).absolute() - for root in list(analysis.files.roots): + for root in [*analysis.files.roots, *analysis.files.packages]: directory = Path(root).absolute() if module_path.is_relative_to(directory): dotted = '.'.join( @@ -210,7 +219,7 @@ def try_resolve_state_type( return f"{module.rsplit('.', 1)[0]}.{attribute}", analysis found, files = analysis.files.lookup_or_parse_module( - module, visiting=visiting + module, include_packages_in_roots=True, visiting=visiting ) if found is None: return None, analysis @@ -454,6 +463,10 @@ class Files: # directory. What an iteration is allowed to analyze. roots: tuple[str, ...] + # Where installed code is found: `sys.path`. Followed for what a + # name refers to, never analyzed. + packages: tuple[str, ...] + # What the previous iteration analyzed, so a file unchanged # since is neither parsed nor analyzed again. known: Mapping[str, File] @@ -461,6 +474,10 @@ class Files: # Parsed this iteration, not yet analyzed. parsed: Mapping[str, ParsedFile] + # Parsed this iteration from the packages: read for what their + # imports bound, and nothing else -- never analyzed. + parsed_from_packages: Mapping[str, ParsedFile] + # Analyzed: this iteration, each servicer and method resolved # against bytes as they are on disk right now -- or a previous # one, verified unchanged. What an iteration returns. @@ -480,14 +497,20 @@ class Files: @classmethod def create( - cls, *, roots: Sequence[str], known: Mapping[str, File] + cls, + *, + roots: Sequence[str], + packages: Sequence[str], + known: Mapping[str, File], ) -> 'Files': """Returns the files an iteration starts from: nothing parsed, nothing analyzed.""" return cls( roots=tuple(roots), + packages=tuple(packages), known=MappingProxyType(dict(known)), parsed=MappingProxyType({}), + parsed_from_packages=MappingProxyType({}), analyzed=MappingProxyType({}), pending=frozenset(), digests=MappingProxyType({}), @@ -495,19 +518,19 @@ def create( def lookup(self, filename: str) -> Optional[ParsedFile | File]: """Returns what this iteration read for a file, wherever it - lives -- `analyzed` or `parsed` -- and `None` when - it has not read the file. + lives -- `analyzed`, `parsed` or `parsed_from_packages` -- + and `None` when it has not read the file. By the file's absolute path, since a relative import spells a file one way and an absolute import another. """ - for entries in (self.analyzed, self.parsed): + for entries in (self.analyzed, self.parsed, self.parsed_from_packages): entry = entries.get(filename) if entry is not None: return entry absolute = Path(filename).absolute() - for entries in (self.analyzed, self.parsed): + for entries in (self.analyzed, self.parsed, self.parsed_from_packages): for entry in entries.values(): if Path(entry.filename).absolute() == absolute: return entry @@ -515,14 +538,31 @@ def lookup(self, filename: str) -> Optional[ParsedFile | File]: return None def with_parsed_file(self, parsed: ParsedFile) -> 'Files': - """Returns this with one more file parsed, into `parsed` and - onto the frontier.""" + """Returns this with one more file parsed -- into `parsed` + and onto the frontier when a root holds it, into + `parsed_from_packages` otherwise.""" + if any( + _is_file_within_directory( + filename=parsed.filename, directory=root + ) for root in self.roots + ): + return replace( + self, + parsed=MappingProxyType( + { + **self.parsed, parsed.filename: parsed + } + ), + pending=self.pending | {parsed.filename}, + ) + return replace( self, - parsed=MappingProxyType({ - **self.parsed, parsed.filename: parsed - }), - pending=self.pending | {parsed.filename}, + parsed_from_packages=MappingProxyType( + { + **self.parsed_from_packages, parsed.filename: parsed + } + ), ) def with_reused_known_file(self, file: File) -> 'Files': @@ -626,8 +666,8 @@ def lookup_or_parse_filename( """Returns a file as this iteration has it -- looked up among what is already read, verified against the previous iteration when unchanged, and otherwise read and parsed, joining - `parsed` or `outside` by who contains it. `None` when it - cannot be read or will not parse. + `parsed` or `parsed_from_packages` by who contains it. + `None` when it cannot be read or will not parse. """ found = self.lookup(filename) if found is not None: @@ -677,7 +717,11 @@ def lookup_or_parse_filename( return parsed, files.with_parsed_file(parsed) def lookup_or_parse_module( - self, module: str, *, visiting: frozenset[str] = frozenset() + self, + module: str, + *, + include_packages_in_roots: bool = False, + visiting: frozenset[str] = frozenset(), ) -> tuple[Optional[ParsedFile | File], 'Files']: """Returns the file a module names as this iteration has it, and `None` when no searched root contains the module, the @@ -685,8 +729,16 @@ def lookup_or_parse_module( one is -- or reading it would circle back to a file already being read through. When it returns `None`, the files come back as they were: nothing found means nothing joined. + + With `include_packages_in_roots`, the packages are searched + after the roots, which is how a name is followed into + installed code. """ - filename = _try_find_file_of(module, roots=self.roots) + roots: Sequence[str] = self.roots + if include_packages_in_roots: + roots = [*roots, *self.packages] + + filename = _try_find_file_of(module, roots=roots) if filename is None or filename.endswith(GENERATED_SUFFIXES): return None, self @@ -1187,6 +1239,7 @@ async def analyze( *, application: str, roots: Optional[Sequence[str]] = None, + packages: Optional[Sequence[str]] = None, known: Optional[Mapping[str, File]] = None, ) -> dict[str, File]: """Returns what each file the developer's application reaches @@ -1204,12 +1257,16 @@ async def analyze( both how a module name becomes a file and where the developer's code is taken to end. It defaults to the application's own directory, which is what running the application puts first on its - path. + path. `packages` are directories installed code may be found + under; a name is followed through them to the state type it + refers to, but they are never analyzed. """ if roots is None: roots = _roots(application) - files = Files.create(roots=roots, known=known or {}) + files = Files.create( + roots=roots, packages=packages or [], known=known or {} + ) _, files = files.lookup_or_parse_filename(application) @@ -1247,6 +1304,13 @@ async def watch(context: WorkflowContext, *, application: str) -> None: roots = _roots(application) globs = [os.path.join(root, SOURCE_GLOB) for root in roots] + # Where installed code may be found: this process runs in the + # developer's environment, so its own path is theirs. A name is + # followed through these to the state type it refers to -- the + # std library re-exports its state types from plain modules -- + # but they are never analyzed or watched. + packages = [path for path in sys.path if path and os.path.isdir(path)] + recorded: Optional[list[ServicerInfo]] = None known: dict[str, File] = {} @@ -1258,7 +1322,10 @@ async def watch(context: WorkflowContext, *, application: str) -> None: # consumed by one event, so it is re-entered for each. async with watcher.watch(globs) as event: known = await analyze( - application=application, roots=roots, known=known + application=application, + roots=roots, + packages=packages, + known=known, ) found = servicers(known) diff --git a/tests/reboot/dashboard/implementation_watcher_tests.py b/tests/reboot/dashboard/implementation_watcher_tests.py index c4856ac0..8d67990d 100644 --- a/tests/reboot/dashboard/implementation_watcher_tests.py +++ b/tests/reboot/dashboard/implementation_watcher_tests.py @@ -116,8 +116,9 @@ def _analyze(self, body: str) -> implementation_watcher.Analysis: ) assert parsed is not None - files = implementation_watcher.Files.create(roots=[], known={} - ).with_parsed_file(parsed) + files = implementation_watcher.Files.create( + roots=[], packages=[], known={} + ).with_parsed_file(parsed) analysis = implementation_watcher.Analysis.create( filename='servicer.py', files=files @@ -875,6 +876,72 @@ async def test_a_state_type_aliased_at_top_level(self) -> None: [('shop.v1.Shop', str(self.directory / 'shop_servicer.py'))], ) + async def test_a_state_type_reexported_by_an_installed_package( + self + ) -> None: + """The way the std library offers its state types: a plain + module re-exporting them from generated code.""" + installed = tempfile.TemporaryDirectory() + try: + (Path(installed.name) / 'maps.py').write_text( + 'from rbt.std.collections.v1.sorted_map_rbt import ' + 'SortedMap\n' + ) + + self._write( + 'shop_servicer.py', + source=SHOP.replace( + 'from shop.v1.shop_rbt import Shop', + 'from maps import SortedMap as Shop', + ), + ) + application = self._write('main.py', source=APPLICATION) + + found = _state_types_and_files( + await analyze( + application=application, + packages=[installed.name], + ) + ) + + self.assertEqual( + found, + [ + ( + 'rbt.std.collections.v1.SortedMap', + str(self.directory / 'shop_servicer.py'), + ) + ], + ) + finally: + installed.cleanup() + + async def test_an_installed_package_is_never_scanned_for_servicers( + self + ) -> None: + """Followed for what a name refers to, and nothing else: a + servicer defined in one belongs to somebody else.""" + installed = tempfile.TemporaryDirectory() + try: + (Path(installed.name) / 'maps.py').write_text(SHOP) + + self._write( + 'shop_servicer.py', + source='from maps import ShopServicer\n', + ) + application = self._write('main.py', source=APPLICATION) + + found = _state_types_and_files( + await analyze( + application=application, + packages=[installed.name], + ) + ) + + self.assertEqual(found, []) + finally: + installed.cleanup() + async def test_a_state_type_imported_through_a_module_alias(self) -> None: self._write( 'shop_servicer.py', From 9445bd65cd88f04ca1e0cd938d4367ce592ef586 Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Wed, 19 Aug 2026 06:10:36 +0000 Subject: [PATCH 20/31] Read and stat files off the event loop `_read` and `_try_find_file_of` make OS calls, and they were made with the event loop held, so a slow disk stalled every dashboard request for as long as the disk took. Both now go through `aiofiles`, which runs the call in a thread -- the way file operations are done everywhere else in the repo -- and everything between `files()` and the two of them becomes `async` to carry the `await` down. Parsing still holds the interpreter: `ast.parse` is CPU-bound, so no thread frees the loop from it, and `cooperatively` already bounds it to a file at a time. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QPEVMhyDRxZEuH8eiykv99 --- reboot/dashboard/BUILD.bazel | 1 + reboot/dashboard/implementation_watcher.py | 85 +++++++++--------- .../dashboard/implementation_watcher_tests.py | 87 +++++++++++-------- 3 files changed, 97 insertions(+), 76 deletions(-) diff --git a/reboot/dashboard/BUILD.bazel b/reboot/dashboard/BUILD.bazel index 8e4a976b..993134a9 100644 --- a/reboot/dashboard/BUILD.bazel +++ b/reboot/dashboard/BUILD.bazel @@ -39,6 +39,7 @@ py_library( srcs_version = "PY3", visibility = ["//visibility:public"], deps = [ + requirement("aiofiles"), ":constants_py", "//rbt/dashboard/v1:dashboard_py_reboot", "//reboot/aio:cooperatively_py", diff --git a/reboot/dashboard/implementation_watcher.py b/reboot/dashboard/implementation_watcher.py index e37374c8..f32a3104 100644 --- a/reboot/dashboard/implementation_watcher.py +++ b/reboot/dashboard/implementation_watcher.py @@ -32,6 +32,8 @@ class AccountServicer(Account.Servicer): developer's source changes, so an edit under the roots is what wakes this, and nothing else does. """ +import aiofiles +import aiofiles.os import ast import hashlib import os @@ -69,9 +71,9 @@ def _roots(application: str) -> list[str]: return [os.path.dirname(application)] -def _read(filename: str) -> bytes: - with open(filename, 'rb') as file: - return file.read() +async def _read(filename: str) -> bytes: + async with aiofiles.open(filename, 'rb') as file: + return await file.read() def _is_file_within_directory(*, filename: str, directory: str) -> bool: @@ -142,7 +144,7 @@ def try_resolve_module(self, return None - def try_resolve_state_type( + async def try_resolve_state_type( self, path: Sequence[str], *, @@ -168,13 +170,13 @@ def try_resolve_state_type( # to first -- each file's own imports answering for the # names it may define. for star in reversed(self.stars): - found, files = analysis.files.lookup_or_parse_module( + found, files = await analysis.files.lookup_or_parse_module( star, include_packages_in_roots=True, visiting=visiting ) if found is None: continue analysis = analysis.with_dependency(found, files=files) - state_type, analysis = found.imports.try_resolve_state_type( + state_type, analysis = await found.imports.try_resolve_state_type( path, analysis=analysis, visiting=visiting | {found.filename}, @@ -218,7 +220,7 @@ def try_resolve_state_type( return None, analysis return f"{module.rsplit('.', 1)[0]}.{attribute}", analysis - found, files = analysis.files.lookup_or_parse_module( + found, files = await analysis.files.lookup_or_parse_module( module, include_packages_in_roots=True, visiting=visiting ) if found is None: @@ -228,7 +230,7 @@ def try_resolve_state_type( # The rest of the chain resolves against the followed file's # own imports: theirs to answer. - return found.imports.try_resolve_state_type( + return await found.imports.try_resolve_state_type( [attribute], analysis=analysis, visiting=visiting | {found.filename}, @@ -352,7 +354,8 @@ def _imports( ) -def _try_find_file_of(module: str, *, roots: Sequence[str]) -> Optional[str]: +async def _try_find_file_of(module: str, *, + roots: Sequence[str]) -> Optional[str]: """Returns the file a module names if one of `roots` contains it, and `None` otherwise. @@ -376,7 +379,7 @@ def _try_find_file_of(module: str, *, roots: Sequence[str]) -> Optional[str]: ] for candidate in candidates: - if os.path.isfile(candidate): + if await aiofiles.os.path.isfile(candidate): return candidate return None @@ -624,8 +627,8 @@ def with_digest(self, filename: str, digest: Digest) -> 'Files': }), ) - def needs_reanalyzing(self, file: File, *, - digest: Digest) -> tuple[bool, 'Files']: + async def needs_reanalyzing(self, file: File, *, + digest: Digest) -> tuple[bool, 'Files']: """Returns whether a file the previous iteration analyzed cannot be reused -- its bytes changed, or a file it depends on did, including one that can no longer be read at all -- @@ -650,7 +653,7 @@ def needs_reanalyzing(self, file: File, *, current = files.digests.get(filename) if current is None: try: - current = hashlib.sha256(_read(filename)).digest() + current = hashlib.sha256(await _read(filename)).digest() except OSError: return True, files files = files.with_digest(filename, current) @@ -660,7 +663,7 @@ def needs_reanalyzing(self, file: File, *, return False, files - def lookup_or_parse_filename( + async def lookup_or_parse_filename( self, filename: str ) -> tuple[Optional[ParsedFile | File], 'Files']: """Returns a file as this iteration has it -- looked up among @@ -679,7 +682,7 @@ def lookup_or_parse_filename( digest = files.digests.get(filename) if digest is None: try: - source = _read(filename) + source = await _read(filename) except OSError: return None, files digest = hashlib.sha256(source).digest() @@ -696,7 +699,9 @@ def lookup_or_parse_filename( known = file break if known is not None: - reanalyze, files = files.needs_reanalyzing(known, digest=digest) + reanalyze, files = await files.needs_reanalyzing( + known, digest=digest + ) if not reanalyze: return known, files.with_reused_known_file(known) @@ -704,7 +709,7 @@ def lookup_or_parse_filename( # the digest did not keep. if source is None: try: - source = _read(filename) + source = await _read(filename) except OSError: return None, files digest = hashlib.sha256(source).digest() @@ -716,7 +721,7 @@ def lookup_or_parse_filename( return parsed, files.with_parsed_file(parsed) - def lookup_or_parse_module( + async def lookup_or_parse_module( self, module: str, *, @@ -738,7 +743,7 @@ def lookup_or_parse_module( if include_packages_in_roots: roots = [*roots, *self.packages] - filename = _try_find_file_of(module, roots=roots) + filename = await _try_find_file_of(module, roots=roots) if filename is None or filename.endswith(GENERATED_SUFFIXES): return None, self @@ -746,7 +751,7 @@ def lookup_or_parse_module( if filename in visiting: return None, self - return self.lookup_or_parse_filename(filename) + return await self.lookup_or_parse_filename(filename) def imports(self, filename: str) -> Optional[Imports]: """Returns what a file's imports bound, wherever this @@ -946,7 +951,7 @@ def with_unsupported(self, unsupported: ast.AST) -> 'Analysis': ) -def _state_type_if_servicer( +async def _state_type_if_servicer( class_definition: ast.ClassDef, *, analysis: Analysis, @@ -966,7 +971,7 @@ def _state_type_if_servicer( continue if len(path) > 1 and path[-1] == 'singleton': path = path[:-1] - state_type, analysis = analysis.imports( + state_type, analysis = await analysis.imports( ).try_resolve_state_type(path, analysis=analysis) if state_type is not None: return state_type, analysis @@ -974,8 +979,8 @@ def _state_type_if_servicer( return None, analysis -def _reboot_related(expression: ast.expr, *, - analysis: Analysis) -> tuple[bool, Analysis]: +async def _reboot_related(expression: ast.expr, *, + analysis: Analysis) -> tuple[bool, Analysis]: """Returns whether anything in an expression touches something Reboot related -- a `.ref` however it is reached, a name holding a reference or the context, or a name that refers to a state type @@ -993,7 +998,7 @@ def _reboot_related(expression: ast.expr, *, case ast.Name(id=str(name)): if name in analysis.locals: return True, analysis - state_type, analysis = analysis.imports( + state_type, analysis = await analysis.imports( ).try_resolve_state_type([name], analysis=analysis) if state_type is not None: return True, analysis @@ -1001,8 +1006,8 @@ def _reboot_related(expression: ast.expr, *, return False, analysis -def _evaluate(expression: ast.expr, *, - analysis: Analysis) -> tuple[Optional[Local], Analysis]: +async def _evaluate(expression: ast.expr, *, + analysis: Analysis) -> tuple[Optional[Local], Analysis]: """Returns what an expression evaluates to, in the only terms the analysis knows -- a reference to a state type, or the context -- and the analysis carried forward. `None` for the value, which is @@ -1012,7 +1017,7 @@ def _evaluate(expression: ast.expr, *, match expression: case ast.Await(value=value): # Awaiting evaluates to what was awaited. - return _evaluate(value, analysis=analysis) + return await _evaluate(value, analysis=analysis) case ast.Call( func=ast.Attribute(value=ast.Name(id='self'), attr='ref') @@ -1031,7 +1036,7 @@ def _evaluate(expression: ast.expr, *, # is almost certainly a reference being lost. path = _path(receiver) if path is not None: - state_type, analysis = analysis.imports( + state_type, analysis = await analysis.imports( ).try_resolve_state_type(path, analysis=analysis) if state_type is not None: return Reference(state_type=state_type), analysis @@ -1044,14 +1049,14 @@ def _evaluate(expression: ast.expr, *, # touches something Reboot related -- whatever it does with it is # not followed, so the calls are likely incomplete -- and left # alone when it is ordinary Python, which was never claimed. - related, analysis = _reboot_related(expression, analysis=analysis) + related, analysis = await _reboot_related(expression, analysis=analysis) if related: analysis = analysis.with_unsupported(expression) return None, analysis -def _assign( +async def _assign( assign: ast.Assign | ast.AnnAssign | ast.AugAssign, *, analysis: Analysis, @@ -1088,7 +1093,7 @@ def _assign( local: Optional[Local] = None if value is not None: - local, analysis = _evaluate(value, analysis=analysis) + local, analysis = await _evaluate(value, analysis=analysis) for target in targets: match target: @@ -1109,7 +1114,7 @@ def _assign( return analysis -def _analyze_method( +async def _analyze_method( method: ast.FunctionDef | ast.AsyncFunctionDef, *, analysis: Analysis, @@ -1150,7 +1155,7 @@ def _analyze_method( match statement: case ast.Assign() | ast.AnnAssign() | ast.AugAssign(): - analysis = _assign(statement, analysis=analysis) + analysis = await _assign(statement, analysis=analysis) return analysis @@ -1167,7 +1172,7 @@ def _digest(node: ast.AST) -> Digest: include_attributes=False).encode()).digest() -def _analyze_file(filename: str, files: Files) -> Files: +async def _analyze_file(filename: str, files: Files) -> Files: """Returns the iteration carried past analyzing one `pending` file: every class saying what it services recorded, every method those classes define analyzed and recorded under them, and the @@ -1187,7 +1192,7 @@ def _analyze_file(filename: str, files: Files) -> Files: for node in ast.walk(parsed.module): match node: case ast.ClassDef(): - state_type, analysis = _state_type_if_servicer( + state_type, analysis = await _state_type_if_servicer( node, analysis=analysis ) if state_type is None: @@ -1200,7 +1205,7 @@ def _analyze_file(filename: str, files: Files) -> Files: ast.FunctionDef(name=str(name)) | ast.AsyncFunctionDef(name=str(name)) ): - analysis = _analyze_method( + analysis = await _analyze_method( statement, analysis=analysis ) servicer.methods.append( @@ -1268,7 +1273,7 @@ async def analyze( roots=roots, packages=packages or [], known=known or {} ) - _, files = files.lookup_or_parse_filename(application) + _, files = await files.lookup_or_parse_filename(application) while len(files.pending) > 0: # Parsing and analyzing hold on to the interpreter for as @@ -1277,7 +1282,7 @@ async def analyze( async for filename in cooperatively(files.pending): match files.parsed.get(filename): case ParsedFile(): - files = _analyze_file(filename, files) + files = await _analyze_file(filename, files) case None: # Nothing to analyze: the file was kept because # neither its bytes nor any of its dependencies @@ -1293,7 +1298,7 @@ async def analyze( assert imports is not None for module in imports.may_load: - _, files = files.lookup_or_parse_module(module) + _, files = await files.lookup_or_parse_module(module) return dict(files.analyzed) diff --git a/tests/reboot/dashboard/implementation_watcher_tests.py b/tests/reboot/dashboard/implementation_watcher_tests.py index 8d67990d..f32bbfce 100644 --- a/tests/reboot/dashboard/implementation_watcher_tests.py +++ b/tests/reboot/dashboard/implementation_watcher_tests.py @@ -105,7 +105,7 @@ class AnalyzeTest(unittest.TestCase): Nothing records calls yet, so these observe the analysis through what each name held when the body ended.""" - def _analyze(self, body: str) -> implementation_watcher.Analysis: + async def _analyze(self, body: str) -> implementation_watcher.Analysis: source = SERVICER.format(state='Shop', module='shop').replace( ' async def look(self, context, request):\n pass', ' async def look(self, context, request):\n' + body, @@ -127,23 +127,24 @@ def _analyze(self, body: str) -> implementation_watcher.Analysis: for node in ast.walk(parsed.module): match node: case ast.AsyncFunctionDef(name='look'): - return implementation_watcher._analyze_method( + return await implementation_watcher._analyze_method( node, analysis=analysis ) raise AssertionError('no method to analyze') - def _locals(self, body: str) -> Mapping[str, implementation_watcher.Local]: - return self._analyze(body).locals + async def _locals(self, + body: str) -> Mapping[str, implementation_watcher.Local]: + return (await self._analyze(body)).locals - def test_the_context_is_the_parameter_after_self(self) -> None: + async def test_the_context_is_the_parameter_after_self(self) -> None: self.assertEqual( - self._locals(' pass'), + await self._locals(' pass'), {'context': implementation_watcher.Context()}, ) - def test_a_reference_to_another_state_type(self) -> None: - names = self._locals( + async def test_a_reference_to_another_state_type(self) -> None: + names = await self._locals( ' from shop.v1.depot_rbt import Depot\n' ' depot = Depot.ref(request.depot)' ) @@ -153,36 +154,36 @@ def test_a_reference_to_another_state_type(self) -> None: implementation_watcher.Reference(state_type='shop.v1.Depot'), ) - def test_a_reference_to_the_state_being_serviced(self) -> None: - names = self._locals(' shop = self.ref()') + async def test_a_reference_to_the_state_being_serviced(self) -> None: + names = await self._locals(' shop = self.ref()') self.assertEqual( names['shop'], implementation_watcher.Reference(state_type='shop.v1.Shop'), ) - def test_a_name_holding_what_another_holds(self) -> None: - names = self._locals( + async def test_a_name_holding_what_another_holds(self) -> None: + names = await self._locals( ' shop = self.ref()\n' ' same = shop' ) self.assertEqual(names['same'], names['shop']) - def test_a_name_assigned_something_it_cannot_follow(self) -> None: + async def test_a_name_assigned_something_it_cannot_follow(self) -> None: """Held until it is not: assigning over a reference with something unreadable stops it being a reference.""" - names = self._locals( + names = await self._locals( ' shop = self.ref()\n' ' shop = whatever()' ) self.assertNotIn('shop', names) - def test_a_name_bound_inside_a_block(self) -> None: + async def test_a_name_bound_inside_a_block(self) -> None: """Names are taken together rather than by scope, so one bound inside an `if` is held the same way.""" - names = self._locals( + names = await self._locals( ' if request.wanted:\n' ' shop = self.ref()' ) @@ -192,17 +193,19 @@ def test_a_name_bound_inside_a_block(self) -> None: implementation_watcher.Reference(state_type='shop.v1.Shop'), ) - def test_a_name_holding_nothing_it_can_say(self) -> None: - names = self._locals(' total = request.a + request.b') + async def test_a_name_holding_nothing_it_can_say(self) -> None: + names = await self._locals(' total = request.a + request.b') self.assertNotIn('total', names) - def test_an_assignment_it_cannot_bind_is_unsupported(self) -> None: + async def test_an_assignment_it_cannot_bind_is_unsupported(self) -> None: """Said as written, so a reader of the calls can be told they are likely incomplete and find why. Twice here: the context flows into a call that is not followed, and the unpacking is a target that cannot be bound.""" - analysis = self._analyze(' account, _ = whatever(context)') + analysis = await self._analyze( + ' account, _ = whatever(context)' + ) self.assertEqual( analysis.unsupported, ( @@ -211,26 +214,30 @@ def test_an_assignment_it_cannot_bind_is_unsupported(self) -> None: ) ) - def test_an_unbindable_target_stops_its_names_being_held(self) -> None: + async def test_an_unbindable_target_stops_its_names_being_held( + self + ) -> None: """`account` no longer holds the reference: whatever the unpacking gave it is not something this followed.""" - analysis = self._analyze( + analysis = await self._analyze( ' account = self.ref()\n' ' account, _ = whatever(context)' ) self.assertNotIn('account', analysis.locals) - def test_an_annotated_assignment_binds_the_same_way(self) -> None: - names = self._analyze(' shop: object = self.ref()').locals + async def test_an_annotated_assignment_binds_the_same_way(self) -> None: + names = ( + await self._analyze(' shop: object = self.ref()') + ).locals self.assertEqual( names['shop'], implementation_watcher.Reference(state_type='shop.v1.Shop'), ) - def test_a_bare_annotation_binds_nothing(self) -> None: - analysis = self._analyze( + async def test_a_bare_annotation_binds_nothing(self) -> None: + analysis = await self._analyze( ' shop = self.ref()\n' ' shop: object' ) @@ -241,10 +248,12 @@ def test_a_bare_annotation_binds_nothing(self) -> None: ) self.assertEqual(analysis.unsupported, ()) - def test_an_augmented_assignment_stops_a_name_being_held(self) -> None: + async def test_an_augmented_assignment_stops_a_name_being_held( + self + ) -> None: """`x += y` makes `x` hold something no name was ever bound to, whatever the two held.""" - analysis = self._analyze( + analysis = await self._analyze( ' shop = self.ref()\n' ' shop += request.a' ) @@ -252,28 +261,34 @@ def test_an_augmented_assignment_stops_a_name_being_held(self) -> None: self.assertNotIn('shop', analysis.locals) self.assertEqual(analysis.unsupported, ()) - def test_a_ref_on_something_unresolvable_is_unsupported(self) -> None: + async def test_a_ref_on_something_unresolvable_is_unsupported( + self + ) -> None: """Almost certainly a reference being lost, however it was reached.""" - analysis = self._analyze(" shop = stores.Shop.ref('a')") + analysis = await self._analyze(" shop = stores.Shop.ref('a')") self.assertEqual(analysis.unsupported, ("stores.Shop.ref('a')",)) self.assertNotIn('shop', analysis.locals) - def test_a_state_type_used_some_other_way_is_unsupported(self) -> None: - analysis = self._analyze(' shop = Shop.open(context)') + async def test_a_state_type_used_some_other_way_is_unsupported( + self + ) -> None: + analysis = await self._analyze(' shop = Shop.open(context)') self.assertEqual(analysis.unsupported, ('Shop.open(context)',)) - def test_the_context_reaching_an_unfollowed_call_is_unsupported( + async def test_the_context_reaching_an_unfollowed_call_is_unsupported( self ) -> None: - analysis = self._analyze(' result = self._helper(context)') + analysis = await self._analyze( + ' result = self._helper(context)' + ) self.assertEqual(analysis.unsupported, ('self._helper(context)',)) - def test_an_ordinary_assignment_is_not_unsupported(self) -> None: - analysis = self._analyze(' total = request.a + request.b') + async def test_an_ordinary_assignment_is_not_unsupported(self) -> None: + analysis = await self._analyze(' total = request.a + request.b') self.assertEqual(analysis.unsupported, ()) From 473bad008c8ad3269d215d5e6561a2246d9698d9 Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Wed, 19 Aug 2026 06:11:00 +0000 Subject: [PATCH 21/31] Say why a save during an iteration is never missed A save landing while an iteration reads produces a torn snapshot: one file read before the save, another after. The watch is armed before anything is read, so the save's event is already waiting when the iteration finishes and the next one begins at once -- where a file kept against a stale dependency digest fails its check and is analyzed again. The digests recorded per dependency are what make the tear detectable. Installed packages are the exception: they are read but not watched, so a `pip install` made while the dashboard runs is only noticed on the next save under the roots. Watching the packages too may be worth doing someday. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QPEVMhyDRxZEuH8eiykv99 --- reboot/dashboard/implementation_watcher.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/reboot/dashboard/implementation_watcher.py b/reboot/dashboard/implementation_watcher.py index f32a3104..ea4dae20 100644 --- a/reboot/dashboard/implementation_watcher.py +++ b/reboot/dashboard/implementation_watcher.py @@ -1313,7 +1313,12 @@ async def watch(context: WorkflowContext, *, application: str) -> None: # developer's environment, so its own path is theirs. A name is # followed through these to the state type it refers to -- the # std library re-exports its state types from plain modules -- - # but they are never analyzed or watched. + # but they are never analyzed or watched. A change to installed + # code, such as a `pip install` while the dashboard runs, is + # noticed on the next save under the roots, when the digests + # recorded for package files no longer match; watching the + # packages themselves would notice it immediately, and may be + # worth doing someday. packages = [path for path in sys.path if path and os.path.isdir(path)] recorded: Optional[list[ServicerInfo]] = None @@ -1325,6 +1330,14 @@ async def watch(context: WorkflowContext, *, application: str) -> None: # made during an iteration resolves `event` rather than # arriving while nothing is listening. A watch is # consumed by one event, so it is re-entered for each. + # + # The arming is also why a save landing mid-iteration is + # safe. The iteration may record a torn snapshot -- one + # file read before the save and another after -- but the + # save's event is already waiting, so the next iteration + # begins at once, and any file kept against a stale + # dependency digest fails its check there and is analyzed + # again. async with watcher.watch(globs) as event: known = await analyze( application=application, From ab8b4f33618ecbb1b0c74e9ee3bb0366be9981c8 Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Wed, 19 Aug 2026 07:23:51 +0000 Subject: [PATCH 22/31] Record the calls a method makes through a reference A call on anything that evaluates to a reference is a Reboot call. The reference says which state type is called, the attribute names the method, and the context handed as the first argument is what makes it a call, so a call without the context is unsupported rather than recorded. A call is a `ServicerInfo.Method.Call`, recorded on the analysis in the order met and folded into the method's entry under its servicer. A statement that is just an expression is now evaluated too, since that is how most calls are written. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QPEVMhyDRxZEuH8eiykv99 --- rbt/dashboard/v1/dashboard.proto | 42 +++++++++ reboot/dashboard/implementation_watcher.py | 86 ++++++++++++++----- .../dashboard/implementation_watcher_tests.py | 65 ++++++++++++++ 3 files changed, 173 insertions(+), 20 deletions(-) diff --git a/rbt/dashboard/v1/dashboard.proto b/rbt/dashboard/v1/dashboard.proto index 4f16d961..bbba21a7 100644 --- a/rbt/dashboard/v1/dashboard.proto +++ b/rbt/dashboard/v1/dashboard.proto @@ -94,6 +94,48 @@ message ServicerInfo { // whether the method does anything different, not whether it was // written down differently. bytes digest = 2; + + // Represents one Reboot call the method makes. + message Call { + // Represents how the call is reached. + enum How { + UNKNOWN = 0; + + // A plain method call on a reference. + CALL = 1; + + // Scheduled to run later with `.schedule(when=...)`. + SCHEDULE = 2; + + // Run as a task with `.spawn()`. + SPAWN = 3; + + // A constructor. It is a method called on the state type + // itself and it makes the state. + CONSTRUCT = 4; + + // A workflow reading the state with `.read(context)`. + READ = 5; + + // A workflow writing the state with `.write(context, ...)`. + WRITE = 6; + } + + // Represents the state type the call is made on, spelled as + // `StateTypeInfo.name`. + string state_type = 1; + + // Represents the method called as the developer wrote it. + // Empty for a workflow `read` or `write` because those name + // no method. + string method = 2; + + How how = 3; + } + + // Represents every Reboot call the method makes, in the order + // met. + repeated Call calls = 3; } // The state type it services, spelled as `StateTypeInfo.name`. diff --git a/reboot/dashboard/implementation_watcher.py b/reboot/dashboard/implementation_watcher.py index ea4dae20..83075324 100644 --- a/reboot/dashboard/implementation_watcher.py +++ b/reboot/dashboard/implementation_watcher.py @@ -792,6 +792,12 @@ class Reference: # The state type referred to, spelled as `StateTypeInfo.name`. state_type: str + # How a call made through this reference is reached. Plain by + # default. + how: 'ServicerInfo.Method.Call.How.ValueType' = ( + ServicerInfo.Method.Call.How.CALL + ) + @dataclass(frozen=True, kw_only=True) class Context: @@ -817,18 +823,6 @@ def _statements(node: ast.AST) -> Iterator[ast.stmt]: yield from _statements(child) -@dataclass(frozen=True, kw_only=True) -class Call: - """One Reboot call a method's body was found to make.""" - - # The state type the call is made on, spelled as - # `StateTypeInfo.name`. - state_type: str - - # The method called, as the developer wrote it. - method: str - - @dataclass(frozen=True, kw_only=True) class Analysis: """One file's analysis in flight: what it has found, what it has @@ -862,7 +856,7 @@ class Analysis: # The Reboot calls the method being analyzed makes, in the order # met. Folded into the method's entry when the method is # recorded. - calls: tuple[Call, ...] + calls: tuple[ServicerInfo.Method.Call, ...] # What was met that the analysis does not follow, spelled the way # it was written, so whoever reads the calls can be told they are @@ -912,6 +906,11 @@ def with_dependency( files=files, ) + def with_call(self, call: ServicerInfo.Method.Call) -> 'Analysis': + """Returns this analysis with one more Reboot call recorded, + in the order met.""" + return replace(self, calls=(*self.calls, call)) + def with_state_type(self, state_type: str) -> 'Analysis': """Returns this analysis inside a class servicing a state type: what `self.ref()` refers to until the class ends.""" @@ -1006,6 +1005,23 @@ async def _reboot_related(expression: ast.expr, *, return False, analysis +async def _first_argument_is_the_context( + arguments: Sequence[ast.expr], *, analysis: Analysis +) -> tuple[bool, Analysis]: + """Returns whether a call hands the context as its first + argument, the way every Reboot call does.""" + if len(arguments) == 0: + return False, analysis + + first, analysis = await _evaluate(arguments[0], analysis=analysis) + + match first: + case Context(): + return True, analysis + + return False, analysis + + async def _evaluate(expression: ast.expr, *, analysis: Analysis) -> tuple[Optional[Local], Analysis]: """Returns what an expression evaluates to, in the only terms the @@ -1041,6 +1057,32 @@ async def _evaluate(expression: ast.expr, *, if state_type is not None: return Reference(state_type=state_type), analysis + case ast.Call( + func=ast.Attribute(value=receiver, attr=str(attribute)), + args=arguments, + ): + # A call on a reference is a Reboot call. The reference + # says which state type is called and the attribute + # names the method. + local, analysis = await _evaluate(receiver, analysis=analysis) + match local: + case Reference(state_type=state_type, how=how): + handed, analysis = await _first_argument_is_the_context( + arguments, analysis=analysis + ) + # Every Reboot call hands the context first. A + # call without it is not one and falls through + # to be flagged, since it touches a reference. + if handed: + analysis = analysis.with_call( + ServicerInfo.Method.Call( + state_type=state_type, + method=attribute, + how=how, + ) + ) + return None, analysis + case ast.Name(id=str(name)) if name in analysis.locals: # `another = account`, holding whatever `account` holds. return analysis.locals[name], analysis @@ -1149,14 +1191,16 @@ async def _analyze_method( analysis = analysis.with_local(arguments[1].arg, Context()) for statement in _statements(method): - # TODO: Record the calls this statement makes, through what - # each name holds right now -- before what it assigns is - # bound, since a statement's value runs before its target. - match statement: case ast.Assign() | ast.AnnAssign() | ast.AugAssign(): analysis = await _assign(statement, analysis=analysis) + case ast.Expr(value=value): + # A statement that is just an expression. Most + # calls are written this way, like + # `await shop.restock(context)` on its own line. + _, analysis = await _evaluate(value, analysis=analysis) + return analysis @@ -1210,11 +1254,13 @@ async def _analyze_file(filename: str, files: Files) -> Files: ) servicer.methods.append( ServicerInfo.Method( - name=name, digest=_digest(statement) + name=name, + digest=_digest(statement), + calls=analysis.calls, ) ) - # Calls and unsupported fold in here once - # the proto has fields for them. + # Unsupported folds in here once the + # proto has a field for it. analysis = analysis.with_method_reset() servicers.append(servicer) diff --git a/tests/reboot/dashboard/implementation_watcher_tests.py b/tests/reboot/dashboard/implementation_watcher_tests.py index f32bbfce..ede931b3 100644 --- a/tests/reboot/dashboard/implementation_watcher_tests.py +++ b/tests/reboot/dashboard/implementation_watcher_tests.py @@ -287,6 +287,71 @@ async def test_the_context_reaching_an_unfollowed_call_is_unsupported( self.assertEqual(analysis.unsupported, ('self._helper(context)',)) + async def test_a_call_through_a_reference(self) -> None: + analysis = await self._analyze( + " await Shop.ref('a').restock(context)" + ) + self.assertEqual( + analysis.calls, + ( + ServicerInfo.Method.Call( + state_type='shop.v1.Shop', + method='restock', + how=ServicerInfo.Method.Call.How.CALL, + ), + ), + ) + + async def test_a_call_through_a_name(self) -> None: + analysis = await self._analyze( + " shop = Shop.ref('a')\n" + ' await shop.restock(context)' + ) + self.assertEqual( + analysis.calls, + ( + ServicerInfo.Method.Call( + state_type='shop.v1.Shop', + method='restock', + how=ServicerInfo.Method.Call.How.CALL, + ), + ), + ) + + async def test_a_call_through_self(self) -> None: + analysis = await self._analyze( + ' await self.ref().restock(context)' + ) + self.assertEqual( + analysis.calls, + ( + ServicerInfo.Method.Call( + state_type='shop.v1.Shop', + method='restock', + how=ServicerInfo.Method.Call.How.CALL, + ), + ), + ) + + async def test_calls_are_recorded_in_the_order_met(self) -> None: + analysis = await self._analyze( + " await Shop.ref('a').restock(context)\n" + " await Shop.ref('b').close(context)" + ) + self.assertEqual( + [call.method for call in analysis.calls], + ['restock', 'close'], + ) + + async def test_a_call_without_the_context_is_unsupported(self) -> None: + analysis = await self._analyze( + " await Shop.ref('a').restock(request)" + ) + self.assertEqual(analysis.calls, ()) + self.assertEqual( + analysis.unsupported, ("Shop.ref('a').restock(request)",) + ) + async def test_an_ordinary_assignment_is_not_unsupported(self) -> None: analysis = await self._analyze(' total = request.a + request.b') From b3e54eddfa91a0bdadd2ab6c395fda06981d878a Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Wed, 19 Aug 2026 07:24:17 +0000 Subject: [PATCH 23/31] See through the modifiers that change nothing `idempotently`, `per_workflow`, `per_iteration`, `always`, `reactively` and `until` each return something the method can still be called on, so a chain passes through them and what comes after one of them names the method. Without this, `ref.idempotently('x').deposit(context)` would record a call to a method named `idempotently`. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QPEVMhyDRxZEuH8eiykv99 --- reboot/dashboard/implementation_watcher.py | 25 ++++++++++++++++--- .../dashboard/implementation_watcher_tests.py | 19 ++++++++++++++ 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/reboot/dashboard/implementation_watcher.py b/reboot/dashboard/implementation_watcher.py index 83075324..24880bb6 100644 --- a/reboot/dashboard/implementation_watcher.py +++ b/reboot/dashboard/implementation_watcher.py @@ -784,6 +784,21 @@ def _parse(source: bytes, *, filename: str, ) +# Modifier methods return something the method can still be called +# on. A call chain passes through them and what comes after one of +# them names the method. +MODIFIERS = frozenset( + { + 'idempotently', + 'per_workflow', + 'per_iteration', + 'always', + 'reactively', + 'until', + } +) + + @dataclass(frozen=True, kw_only=True) class Reference: """A name holding a reference to one of the developer's state @@ -1066,7 +1081,11 @@ async def _evaluate(expression: ast.expr, *, # names the method. local, analysis = await _evaluate(receiver, analysis=analysis) match local: - case Reference(state_type=state_type, how=how): + case Reference() as reference: + if attribute in MODIFIERS: + # A modifier. The chain passes through it + # and what comes after names the method. + return reference, analysis handed, analysis = await _first_argument_is_the_context( arguments, analysis=analysis ) @@ -1076,9 +1095,9 @@ async def _evaluate(expression: ast.expr, *, if handed: analysis = analysis.with_call( ServicerInfo.Method.Call( - state_type=state_type, + state_type=reference.state_type, method=attribute, - how=how, + how=reference.how, ) ) return None, analysis diff --git a/tests/reboot/dashboard/implementation_watcher_tests.py b/tests/reboot/dashboard/implementation_watcher_tests.py index ede931b3..98fb1165 100644 --- a/tests/reboot/dashboard/implementation_watcher_tests.py +++ b/tests/reboot/dashboard/implementation_watcher_tests.py @@ -343,6 +343,25 @@ async def test_calls_are_recorded_in_the_order_met(self) -> None: ['restock', 'close'], ) + async def test_a_modifier_passes_the_call_through(self) -> None: + analysis = await self._analyze( + " await Shop.ref('a').idempotently('x').restock(context)" + ) + self.assertEqual( + [(call.method, call.how) for call in analysis.calls], + [('restock', ServicerInfo.Method.Call.How.CALL)], + ) + + async def test_stacked_modifiers_pass_the_call_through(self) -> None: + analysis = await self._analyze( + " shop = Shop.ref('a').per_workflow('x').always()\n" + ' await shop.reactively().restock(context)' + ) + self.assertEqual( + [call.method for call in analysis.calls], + ['restock'], + ) + async def test_a_call_without_the_context_is_unsupported(self) -> None: analysis = await self._analyze( " await Shop.ref('a').restock(request)" From d8edb0adb57aa29b2e2a76c9714ab2246736abf5 Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Wed, 19 Aug 2026 07:24:42 +0000 Subject: [PATCH 24/31] Record a scheduled or spawned call as such `.schedule(when=...)` runs the method later and `.spawn()` runs it as a task, so a chain remembers meeting one of them and the call at its end is recorded as reached that way. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QPEVMhyDRxZEuH8eiykv99 --- reboot/dashboard/implementation_watcher.py | 15 +++++++++++++++ .../dashboard/implementation_watcher_tests.py | 18 ++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/reboot/dashboard/implementation_watcher.py b/reboot/dashboard/implementation_watcher.py index 24880bb6..400ea377 100644 --- a/reboot/dashboard/implementation_watcher.py +++ b/reboot/dashboard/implementation_watcher.py @@ -798,6 +798,16 @@ def _parse(source: bytes, *, filename: str, } ) +# The modifiers that change how the call after them is reached. +HOW_MODIFIERS: Mapping[str, 'ServicerInfo.Method.Call.How.ValueType'] = ( + MappingProxyType( + { + 'schedule': ServicerInfo.Method.Call.How.SCHEDULE, + 'spawn': ServicerInfo.Method.Call.How.SPAWN, + } + ) +) + @dataclass(frozen=True, kw_only=True) class Reference: @@ -1086,6 +1096,11 @@ async def _evaluate(expression: ast.expr, *, # A modifier. The chain passes through it # and what comes after names the method. return reference, analysis + how = HOW_MODIFIERS.get(attribute) + if how is not None: + # The chain passes through and remembers + # how the call at the end is reached. + return replace(reference, how=how), analysis handed, analysis = await _first_argument_is_the_context( arguments, analysis=analysis ) diff --git a/tests/reboot/dashboard/implementation_watcher_tests.py b/tests/reboot/dashboard/implementation_watcher_tests.py index 98fb1165..5e020fa0 100644 --- a/tests/reboot/dashboard/implementation_watcher_tests.py +++ b/tests/reboot/dashboard/implementation_watcher_tests.py @@ -362,6 +362,24 @@ async def test_stacked_modifiers_pass_the_call_through(self) -> None: ['restock'], ) + async def test_a_scheduled_call(self) -> None: + analysis = await self._analyze( + ' await self.ref().schedule().restock(context)' + ) + self.assertEqual( + [(call.method, call.how) for call in analysis.calls], + [('restock', ServicerInfo.Method.Call.How.SCHEDULE)], + ) + + async def test_a_spawned_call(self) -> None: + analysis = await self._analyze( + " await Shop.ref('a').spawn().restock(context)" + ) + self.assertEqual( + [(call.method, call.how) for call in analysis.calls], + [('restock', ServicerInfo.Method.Call.How.SPAWN)], + ) + async def test_a_call_without_the_context_is_unsupported(self) -> None: analysis = await self._analyze( " await Shop.ref('a').restock(request)" From ebb846eb3922f59028082754ce286191ec00d88e Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Wed, 19 Aug 2026 07:25:06 +0000 Subject: [PATCH 25/31] Record a constructor A method called on the state type itself, handed the context, is a constructor: it makes the state and returns a reference to it with the response. The call is recorded as such, and unpacking the pair, as in `shop, _ = await Shop.open(context, 'a')`, binds the first name to a reference to the state made, so calls through it resolve. A state type reached some other way is still unsupported, since whatever the code does with it is not followed. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QPEVMhyDRxZEuH8eiykv99 --- reboot/dashboard/implementation_watcher.py | 73 +++++++++++++++++-- .../dashboard/implementation_watcher_tests.py | 25 ++++++- 2 files changed, 88 insertions(+), 10 deletions(-) diff --git a/reboot/dashboard/implementation_watcher.py b/reboot/dashboard/implementation_watcher.py index 400ea377..8b21d745 100644 --- a/reboot/dashboard/implementation_watcher.py +++ b/reboot/dashboard/implementation_watcher.py @@ -835,6 +835,15 @@ class Context: Local = Reference | Context +@dataclass(frozen=True, kw_only=True) +class Constructed: + """What a constructor call evaluates to. It is a pair of a + reference to the new state and the method's response.""" + + # The state type made, spelled as `StateTypeInfo.name`. + state_type: str + + def _statements(node: ast.AST) -> Iterator[ast.stmt]: """Returns every statement under a node, in the order written. @@ -1047,13 +1056,15 @@ async def _first_argument_is_the_context( return False, analysis -async def _evaluate(expression: ast.expr, *, - analysis: Analysis) -> tuple[Optional[Local], Analysis]: +async def _evaluate( + expression: ast.expr, *, analysis: Analysis +) -> tuple[Optional[Local | Constructed], Analysis]: """Returns what an expression evaluates to, in the only terms the - analysis knows -- a reference to a state type, or the context -- - and the analysis carried forward. `None` for the value, which is - most expressions, means neither, so nothing can be said about a - name it is assigned to or a call made on it. + analysis knows. Those are a reference to a state type, the + context, and what a constructor call makes. `None` for the + value, which is most expressions, means none of those, so + nothing can be said about a name it is assigned to or a call + made on it. """ match expression: case ast.Await(value=value): @@ -1086,6 +1097,31 @@ async def _evaluate(expression: ast.expr, *, func=ast.Attribute(value=receiver, attr=str(attribute)), args=arguments, ): + # A constructor is a method called on the state type + # itself with the context as the first argument. It + # makes the state and returns a reference to it along + # with the response. + path = _path(receiver) + if path is not None and len(arguments) > 0: + state_type, analysis = await analysis.imports( + ).try_resolve_state_type(path, analysis=analysis) + if state_type is not None: + handed, analysis = await _first_argument_is_the_context( + arguments, analysis=analysis + ) + if handed: + analysis = analysis.with_call( + ServicerInfo.Method.Call( + state_type=state_type, + method=attribute, + how=ServicerInfo.Method.Call.How.CONSTRUCT, + ) + ) + return ( + Constructed(state_type=state_type), + analysis, + ) + # A call on a reference is a Reboot call. The reference # says which state type is called and the attribute # names the method. @@ -1166,10 +1202,31 @@ async def _assign( targets = [assign.target] value = None - local: Optional[Local] = None + evaluated: Optional[Local | Constructed] = None if value is not None: - local, analysis = await _evaluate(value, analysis=analysis) + evaluated, analysis = await _evaluate(value, analysis=analysis) + + match targets, evaluated: + case [ast.Tuple(elts=[ast.Name(id=str(first)), *rest]) + ], Constructed(state_type=state_type): + # `shop, response = await Shop.open(context, 'a')` + # binds the first name to a reference to the new state. + # The other names get the response and are not tracked. + analysis = analysis.with_local( + first, Reference(state_type=state_type) + ) + for element in rest: + for node in ast.walk(element): + match node: + case ast.Name(id=str(name)): + analysis = analysis.with_local(name, None) + return analysis + + # A name cannot be bound to the pair itself. + local: Optional[Local] = ( + None if isinstance(evaluated, Constructed) else evaluated + ) for target in targets: match target: diff --git a/tests/reboot/dashboard/implementation_watcher_tests.py b/tests/reboot/dashboard/implementation_watcher_tests.py index 5e020fa0..25bb6043 100644 --- a/tests/reboot/dashboard/implementation_watcher_tests.py +++ b/tests/reboot/dashboard/implementation_watcher_tests.py @@ -274,9 +274,9 @@ async def test_a_ref_on_something_unresolvable_is_unsupported( async def test_a_state_type_used_some_other_way_is_unsupported( self ) -> None: - analysis = await self._analyze(' shop = Shop.open(context)') + analysis = await self._analyze(' shop = Shop.open(request)') - self.assertEqual(analysis.unsupported, ('Shop.open(context)',)) + self.assertEqual(analysis.unsupported, ('Shop.open(request)',)) async def test_the_context_reaching_an_unfollowed_call_is_unsupported( self @@ -380,6 +380,27 @@ async def test_a_spawned_call(self) -> None: [('restock', ServicerInfo.Method.Call.How.SPAWN)], ) + async def test_a_constructor(self) -> None: + analysis = await self._analyze(" await Shop.open(context, 'a')") + self.assertEqual( + [(call.method, call.how) for call in analysis.calls], + [('open', ServicerInfo.Method.Call.How.CONSTRUCT)], + ) + + async def test_a_constructor_unpacked_binds_a_reference(self) -> None: + analysis = await self._analyze( + " shop, _ = await Shop.open(context, 'a')\n" + ' await shop.restock(context)' + ) + self.assertEqual( + [(call.method, call.how) for call in analysis.calls], + [ + ('open', ServicerInfo.Method.Call.How.CONSTRUCT), + ('restock', ServicerInfo.Method.Call.How.CALL), + ], + ) + self.assertEqual(analysis.unsupported, ()) + async def test_a_call_without_the_context_is_unsupported(self) -> None: analysis = await self._analyze( " await Shop.ref('a').restock(request)" From c52c8dfe15e17b3ce2939a4dbc061a7215a7f286 Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Wed, 19 Aug 2026 07:25:30 +0000 Subject: [PATCH 26/31] Record a call on every state `forall` reaches `Account.forall(ids)` names many existing states the way `ref` names one, so what it evaluates to is a reference and a call through it is recorded like any other. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QPEVMhyDRxZEuH8eiykv99 --- reboot/dashboard/implementation_watcher.py | 7 ++++++- tests/reboot/dashboard/implementation_watcher_tests.py | 9 +++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/reboot/dashboard/implementation_watcher.py b/reboot/dashboard/implementation_watcher.py index 8b21d745..788777ec 100644 --- a/reboot/dashboard/implementation_watcher.py +++ b/reboot/dashboard/implementation_watcher.py @@ -1080,12 +1080,17 @@ async def _evaluate( # type. return Reference(state_type=analysis.state_type), analysis - case ast.Call(func=ast.Attribute(value=receiver, attr='ref')): + case ast.Call( + func=ast.Attribute(value=receiver, attr='ref' | 'forall') + ): # `Account.ref(id)`, or `rbt.Shop.ref(id)` through a # module, the one way to name an existing state -- unless # the name refers to no state type, which falls through # to be flagged below: a `.ref` on something unresolvable # is almost certainly a reference being lost. + # `Account.forall(ids)` names many existing states the + # way `ref` names one, so a call through it is a call on + # each of them. path = _path(receiver) if path is not None: state_type, analysis = await analysis.imports( diff --git a/tests/reboot/dashboard/implementation_watcher_tests.py b/tests/reboot/dashboard/implementation_watcher_tests.py index 25bb6043..d602b84c 100644 --- a/tests/reboot/dashboard/implementation_watcher_tests.py +++ b/tests/reboot/dashboard/implementation_watcher_tests.py @@ -401,6 +401,15 @@ async def test_a_constructor_unpacked_binds_a_reference(self) -> None: ) self.assertEqual(analysis.unsupported, ()) + async def test_a_call_through_forall(self) -> None: + analysis = await self._analyze( + ' await Shop.forall(ids).restock(context)' + ) + self.assertEqual( + [(call.method, call.how) for call in analysis.calls], + [('restock', ServicerInfo.Method.Call.How.CALL)], + ) + async def test_a_call_without_the_context_is_unsupported(self) -> None: analysis = await self._analyze( " await Shop.ref('a').restock(request)" From 09aaad91b8327105520ae7c2931f82ad6ccd96f3 Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Wed, 19 Aug 2026 07:25:55 +0000 Subject: [PATCH 27/31] Record a workflow's reads and writes `.read(context)` and `.write(context, ...)` on a reference reach the state directly and name no method, so each is recorded as what it is, with an empty method name. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QPEVMhyDRxZEuH8eiykv99 --- reboot/dashboard/implementation_watcher.py | 15 +++++++++++++++ .../dashboard/implementation_watcher_tests.py | 18 ++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/reboot/dashboard/implementation_watcher.py b/reboot/dashboard/implementation_watcher.py index 788777ec..f7b0376c 100644 --- a/reboot/dashboard/implementation_watcher.py +++ b/reboot/dashboard/implementation_watcher.py @@ -1149,6 +1149,21 @@ async def _evaluate( # call without it is not one and falls through # to be flagged, since it touches a reference. if handed: + if attribute in ('read', 'write'): + # A workflow reading or writing the + # state. It names no method. + analysis = analysis.with_call( + ServicerInfo.Method.Call( + state_type=reference.state_type, + method='', + how=( + ServicerInfo.Method.Call.How.READ + if attribute == 'read' else + ServicerInfo.Method.Call.How.WRITE + ), + ) + ) + return None, analysis analysis = analysis.with_call( ServicerInfo.Method.Call( state_type=reference.state_type, diff --git a/tests/reboot/dashboard/implementation_watcher_tests.py b/tests/reboot/dashboard/implementation_watcher_tests.py index d602b84c..728fa9c0 100644 --- a/tests/reboot/dashboard/implementation_watcher_tests.py +++ b/tests/reboot/dashboard/implementation_watcher_tests.py @@ -410,6 +410,24 @@ async def test_a_call_through_forall(self) -> None: [('restock', ServicerInfo.Method.Call.How.CALL)], ) + async def test_a_workflow_read(self) -> None: + analysis = await self._analyze( + ' state = await self.ref().read(context)' + ) + self.assertEqual( + [(call.method, call.how) for call in analysis.calls], + [('', ServicerInfo.Method.Call.How.READ)], + ) + + async def test_a_workflow_write(self) -> None: + analysis = await self._analyze( + ' await self.ref().write(context, mutate)' + ) + self.assertEqual( + [(call.method, call.how) for call in analysis.calls], + [('', ServicerInfo.Method.Call.How.WRITE)], + ) + async def test_a_call_without_the_context_is_unsupported(self) -> None: analysis = await self._analyze( " await Shop.ref('a').restock(request)" From 28e2c361beda94fba5f8b2e0a39b172e9d1bcc90 Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Wed, 19 Aug 2026 07:30:20 +0000 Subject: [PATCH 28/31] Record what a method does that the analysis could not follow The method's proto entry gains `unsupported`, so whoever reads the calls can be told they may be incomplete instead of being left to trust them. With that, everything a method's analysis finds now has a home under its servicer, and the calls are proven to reach it all the way through `analyze`. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QPEVMhyDRxZEuH8eiykv99 --- rbt/dashboard/v1/dashboard.proto | 5 +++ reboot/dashboard/implementation_watcher.py | 3 +- .../dashboard/implementation_watcher_tests.py | 42 +++++++++++++++++++ 3 files changed, 48 insertions(+), 2 deletions(-) diff --git a/rbt/dashboard/v1/dashboard.proto b/rbt/dashboard/v1/dashboard.proto index bbba21a7..a86e4568 100644 --- a/rbt/dashboard/v1/dashboard.proto +++ b/rbt/dashboard/v1/dashboard.proto @@ -136,6 +136,11 @@ message ServicerInfo { // Represents every Reboot call the method makes, in the order // met. repeated Call calls = 3; + + // Represents what the method does that the analysis could not + // follow, spelled the way the developer wrote it. When this is + // not empty the calls may be incomplete. + repeated string unsupported = 4; } // The state type it services, spelled as `StateTypeInfo.name`. diff --git a/reboot/dashboard/implementation_watcher.py b/reboot/dashboard/implementation_watcher.py index f7b0376c..410fa079 100644 --- a/reboot/dashboard/implementation_watcher.py +++ b/reboot/dashboard/implementation_watcher.py @@ -1368,10 +1368,9 @@ async def _analyze_file(filename: str, files: Files) -> Files: name=name, digest=_digest(statement), calls=analysis.calls, + unsupported=analysis.unsupported, ) ) - # Unsupported folds in here once the - # proto has a field for it. analysis = analysis.with_method_reset() servicers.append(servicer) diff --git a/tests/reboot/dashboard/implementation_watcher_tests.py b/tests/reboot/dashboard/implementation_watcher_tests.py index 728fa9c0..a5c3266b 100644 --- a/tests/reboot/dashboard/implementation_watcher_tests.py +++ b/tests/reboot/dashboard/implementation_watcher_tests.py @@ -841,6 +841,48 @@ async def main(): ################################################################### # The methods each servicer defines. + async def test_the_calls_a_method_makes_reach_its_entry(self) -> None: + self._write( + 'shop_servicer.py', + source=SHOP.replace( + ' async def look(self, context, request):\n pass', + ' async def look(self, context, request):\n' + " await Shop.ref('a').restock(context)", + ), + ) + application = self._write('main.py', source=APPLICATION) + + found = servicers(await analyze(application=application)) + + self.assertEqual( + list(found[0].methods[0].calls), + [ + ServicerInfo.Method.Call( + state_type='shop.v1.Shop', + method='restock', + how=ServicerInfo.Method.Call.How.CALL, + ) + ], + ) + + async def test_the_unsupported_reaches_its_entry(self) -> None: + self._write( + 'shop_servicer.py', + source=SHOP.replace( + ' async def look(self, context, request):\n pass', + ' async def look(self, context, request):\n' + ' result = self.helper(context)', + ), + ) + application = self._write('main.py', source=APPLICATION) + + found = servicers(await analyze(application=application)) + + self.assertEqual( + list(found[0].methods[0].unsupported), + ['self.helper(context)'], + ) + async def test_records_the_methods_a_servicer_defines(self) -> None: self._write('shop_servicer.py', source=SHOP) application = self._write('main.py', source=APPLICATION) From 801ec5c04e3016502f0379a0cbb1b0e29767d1f0 Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Wed, 19 Aug 2026 07:40:47 +0000 Subject: [PATCH 29/31] Follow a helper handed the context A call that hands the context to a function defined in the same file is followed into it. The helper's body is analyzed with the receiving parameter bound to the context and with its names kept apart from the caller's, so the calls the helper makes are recorded for the calling method. A helper is followed at most once for each method, which is what stops a helper that reaches itself and keeps its calls from being recorded more than once. A helper the file does not define, or defines more than once, is still unsupported. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QPEVMhyDRxZEuH8eiykv99 --- reboot/dashboard/implementation_watcher.py | 182 +++++++++++++++++- .../dashboard/implementation_watcher_tests.py | 68 +++++++ 2 files changed, 247 insertions(+), 3 deletions(-) diff --git a/reboot/dashboard/implementation_watcher.py b/reboot/dashboard/implementation_watcher.py index 410fa079..583546c8 100644 --- a/reboot/dashboard/implementation_watcher.py +++ b/reboot/dashboard/implementation_watcher.py @@ -897,6 +897,12 @@ class Analysis: # likely incomplete rather than left to trust them. unsupported: tuple[str, ...] + # The helpers this method's analysis has followed, by name. A + # helper is followed at most once, so the calls it makes are + # recorded once. The name is enough to tell helpers apart since + # one is only followed when its file defines the name once. + helpers: frozenset[str] + # What each of the method's names is bound to at the point the # analysis has reached. The working state the analysis thinks # with, dropped when the method is recorded. A @@ -913,6 +919,7 @@ def create(cls, *, filename: str, files: Files) -> 'Analysis': files=files, dependencies=MappingProxyType({}), state_type=None, + helpers=frozenset(), calls=(), unsupported=(), locals=MappingProxyType({}), @@ -940,6 +947,21 @@ def with_dependency( files=files, ) + def with_locals(self, locals: Mapping[str, Local]) -> 'Analysis': + """Returns this analysis with the names bound wholesale, for + entering and leaving a followed helper.""" + return replace(self, locals=MappingProxyType(dict(locals))) + + def with_helper(self, name: str) -> 'Analysis': + """Returns this analysis with one more helper followed.""" + return replace(self, helpers=self.helpers | {name}) + + def module(self) -> ast.Module: + """Returns the syntax of the file being analyzed.""" + found = self.files.lookup(self.filename) + assert isinstance(found, ParsedFile) + return found.module + def with_call(self, call: ServicerInfo.Method.Call) -> 'Analysis': """Returns this analysis with one more Reboot call recorded, in the order met.""" @@ -951,11 +973,12 @@ def with_state_type(self, state_type: str) -> 'Analysis': return replace(self, state_type=state_type) def with_method_reset(self) -> 'Analysis': - """Returns this analysis with the method reset -- its calls, - unsupported and locals cleared -- ready for the next + """Returns this analysis with the method reset, its calls, + unsupported, locals and helpers cleared, ready for the next method.""" return replace( self, + helpers=frozenset(), calls=(), unsupported=(), locals=MappingProxyType({}), @@ -1039,6 +1062,95 @@ async def _reboot_related(expression: ast.expr, *, return False, analysis +def _function_named( + name: str, *, module: ast.Module +) -> Optional[ast.FunctionDef | ast.AsyncFunctionDef]: + """Returns the one function in a file with a name. `None` when + there is none or more than one, since a name spelled twice is + not worth guessing about.""" + found = [ + node for node in ast.walk(module) + if isinstance(node, (ast.FunctionDef, + ast.AsyncFunctionDef)) and node.name == name + ] + + if len(found) == 1: + return found[0] + + return None + + +async def _context_argument( + arguments: Sequence[ast.expr], + keywords: Sequence[ast.keyword], + *, + analysis: Analysis, +) -> tuple[Optional[int | str], 'Analysis']: + """Returns where a call hands the context over. That is the + position of the first argument that is the context, or the name + of the first keyword that is, or `None`.""" + for position, argument in enumerate(arguments): + value, analysis = await _evaluate(argument, analysis=analysis) + match value: + case Context(): + return position, analysis + + for keyword in keywords: + if keyword.arg is None: + continue + value, analysis = await _evaluate(keyword.value, analysis=analysis) + match value: + case Context(): + return keyword.arg, analysis + + return None, analysis + + +def _parameter_for_the_context( + function: ast.FunctionDef | ast.AsyncFunctionDef, + handed: int | str, + *, + method: bool, +) -> Optional[str]: + """Returns the name of the helper's parameter that receives the + context, and `None` when the helper has no such parameter. A + method's first parameter is `self`, so positions shift by one.""" + parameters = function.args.posonlyargs + function.args.args + + if isinstance(handed, str): + for parameter in parameters + function.args.kwonlyargs: + if parameter.arg == handed: + return handed + return None + + position = handed + 1 if method else handed + if position < len(parameters): + return parameters[position].arg + + return None + + +async def _follow_helper( + function: ast.FunctionDef | ast.AsyncFunctionDef, + *, + context_parameter: str, + analysis: Analysis, +) -> Analysis: + """Returns the analysis carried through a helper's body. The + helper's parameter that receives the context is bound to it and + the helper's names are kept apart from the caller's, so the + calls the helper makes are recorded for the calling method.""" + saved_locals = analysis.locals + + analysis = analysis.with_helper(function.name).with_locals( + {context_parameter: Context()} + ) + + analysis = await _analyze_statements(function, analysis=analysis) + + return analysis.with_locals(saved_locals) + + async def _first_argument_is_the_context( arguments: Sequence[ast.expr], *, analysis: Analysis ) -> tuple[bool, Analysis]: @@ -1098,9 +1210,36 @@ async def _evaluate( if state_type is not None: return Reference(state_type=state_type), analysis + case ast.Call( + func=ast.Name(id=str(function_name)), + args=arguments, + keywords=keywords, + ): + # A helper in the same file, handed the context, is + # followed so the calls it makes are recorded too. + handed, analysis = await _context_argument( + arguments, keywords, analysis=analysis + ) + if handed is not None and function_name not in analysis.helpers: + function = _function_named( + function_name, module=analysis.module() + ) + if function is not None: + parameter = _parameter_for_the_context( + function, handed, method=False + ) + if parameter is not None: + analysis = await _follow_helper( + function, + context_parameter=parameter, + analysis=analysis, + ) + return None, analysis + case ast.Call( func=ast.Attribute(value=receiver, attr=str(attribute)), args=arguments, + keywords=keywords, ): # A constructor is a method called on the state type # itself with the context as the first argument. It @@ -1173,6 +1312,33 @@ async def _evaluate( ) return None, analysis + match receiver: + case ast.Name(id='self'): + # A helper method in the same file, handed the + # context, is followed so the calls it makes are + # recorded too. + handed, analysis = await _context_argument( + arguments, keywords, analysis=analysis + ) + if ( + handed is not None and + attribute not in analysis.helpers + ): + function = _function_named( + attribute, module=analysis.module() + ) + if function is not None: + parameter = _parameter_for_the_context( + function, handed, method=True + ) + if parameter is not None: + analysis = await _follow_helper( + function, + context_parameter=parameter, + analysis=analysis, + ) + return None, analysis + case ast.Name(id=str(name)) if name in analysis.locals: # `another = account`, holding whatever `account` holds. return analysis.locals[name], analysis @@ -1301,7 +1467,17 @@ async def _analyze_method( if len(arguments) > 1 and arguments[0].arg in ('self', 'cls'): analysis = analysis.with_local(arguments[1].arg, Context()) - for statement in _statements(method): + return await _analyze_statements(method, analysis=analysis) + + +async def _analyze_statements( + function: ast.FunctionDef | ast.AsyncFunctionDef, + *, + analysis: Analysis, +) -> Analysis: + """Returns the analysis carried through every statement under a + function, in the order written.""" + for statement in _statements(function): match statement: case ast.Assign() | ast.AnnAssign() | ast.AugAssign(): analysis = await _assign(statement, analysis=analysis) diff --git a/tests/reboot/dashboard/implementation_watcher_tests.py b/tests/reboot/dashboard/implementation_watcher_tests.py index a5c3266b..bfc47ca6 100644 --- a/tests/reboot/dashboard/implementation_watcher_tests.py +++ b/tests/reboot/dashboard/implementation_watcher_tests.py @@ -883,6 +883,74 @@ async def test_the_unsupported_reaches_its_entry(self) -> None: ['self.helper(context)'], ) + async def test_a_helper_method_handed_the_context_is_followed( + self + ) -> None: + self._write( + 'shop_servicer.py', + source=SHOP.replace( + ' async def look(self, context, request):\n pass', + ' async def look(self, context, request):\n' + ' await self.restock_everything(context)\n' + '\n' + ' async def restock_everything(self, context):\n' + " await Shop.ref('a').restock(context)", + ), + ) + application = self._write('main.py', source=APPLICATION) + + found = servicers(await analyze(application=application)) + + self.assertEqual( + [call.method for call in found[0].methods[0].calls], + ['restock'], + ) + + async def test_a_bare_helper_handed_the_context_is_followed(self) -> None: + self._write( + 'shop_servicer.py', + source=SHOP.replace( + ' async def look(self, context, request):\n pass', + ' async def look(self, context, request):\n' + ' restock_everything(context)', + ) + ( + '\n' + '\n' + 'def restock_everything(context):\n' + " Shop.ref('a').restock(context)\n" + ), + ) + application = self._write('main.py', source=APPLICATION) + + found = servicers(await analyze(application=application)) + + self.assertEqual( + [call.method for call in found[0].methods[0].calls], + ['restock'], + ) + + async def test_a_helper_reaching_itself_still_terminates(self) -> None: + self._write( + 'shop_servicer.py', + source=SHOP.replace( + ' async def look(self, context, request):\n pass', + ' async def look(self, context, request):\n' + ' await self.restock_everything(context)\n' + '\n' + ' async def restock_everything(self, context):\n' + ' await self.restock_everything(context)\n' + " await Shop.ref('a').restock(context)", + ), + ) + application = self._write('main.py', source=APPLICATION) + + found = servicers(await analyze(application=application)) + + self.assertEqual( + [call.method for call in found[0].methods[0].calls], + ['restock'], + ) + async def test_records_the_methods_a_servicer_defines(self) -> None: self._write('shop_servicer.py', source=SHOP) application = self._write('main.py', source=APPLICATION) From 3bf7daa9827cf4f01a10aaae04677992235cbe22 Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Wed, 19 Aug 2026 07:41:12 +0000 Subject: [PATCH 30/31] Follow a helper for what it returns A helper in the same file is now followed even when it is not handed the context, since what it returns may be a reference or the context, and a name assigned the result is bound to it. A helper returning a tuple is unpacked element by element, the way a constructor's pair already was, and returns that disagree bind nothing. A tuple expression itself now evaluates the same way, so a reference in a tuple that nothing unpacks is still unsupported. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QPEVMhyDRxZEuH8eiykv99 --- reboot/dashboard/implementation_watcher.py | 148 ++++++++++++++---- .../dashboard/implementation_watcher_tests.py | 66 ++++++++ 2 files changed, 182 insertions(+), 32 deletions(-) diff --git a/reboot/dashboard/implementation_watcher.py b/reboot/dashboard/implementation_watcher.py index 583546c8..6bfa7272 100644 --- a/reboot/dashboard/implementation_watcher.py +++ b/reboot/dashboard/implementation_watcher.py @@ -844,6 +844,16 @@ class Constructed: state_type: str +@dataclass(frozen=True, kw_only=True) +class Tupled: + """A tuple an expression evaluates to, element by element, as + far as the analysis can say.""" + + # What each element is, in order. `None` for an element the + # analysis cannot say anything about. + values: tuple[Optional[Local], ...] + + def _statements(node: ast.AST) -> Iterator[ast.stmt]: """Returns every statement under a node, in the order written. @@ -1062,6 +1072,25 @@ async def _reboot_related(expression: ast.expr, *, return False, analysis +def _returns_of( + function: ast.FunctionDef | ast.AsyncFunctionDef +) -> Iterator[ast.Return]: + """Returns every `return` written in a function itself. Nested + functions keep their own.""" + + def walk(node: ast.AST) -> Iterator[ast.Return]: + for child in ast.iter_child_nodes(node): + match child: + case ast.Return(): + yield child + case ast.FunctionDef() | ast.AsyncFunctionDef() | ast.Lambda(): + continue + case _: + yield from walk(child) + + yield from walk(function) + + def _function_named( name: str, *, module: ast.Module ) -> Optional[ast.FunctionDef | ast.AsyncFunctionDef]: @@ -1133,22 +1162,43 @@ def _parameter_for_the_context( async def _follow_helper( function: ast.FunctionDef | ast.AsyncFunctionDef, *, - context_parameter: str, + context_parameter: Optional[str], analysis: Analysis, -) -> Analysis: - """Returns the analysis carried through a helper's body. The - helper's parameter that receives the context is bound to it and - the helper's names are kept apart from the caller's, so the - calls the helper makes are recorded for the calling method.""" +) -> tuple[Optional[Local | Constructed | Tupled], Analysis]: + """Returns what a helper returns and the analysis carried + through its body. The helper's parameter that receives the + context, when one does, is bound to it, and the helper's names + are kept apart from the caller's, so the calls the helper makes + are recorded for the calling method. + + The returned value is what every `return` in the helper + evaluates to. Returns that disagree return nothing, since the + analysis cannot say which one a caller gets.""" saved_locals = analysis.locals analysis = analysis.with_helper(function.name).with_locals( - {context_parameter: Context()} + {} if context_parameter is None else {context_parameter: Context()} ) analysis = await _analyze_statements(function, analysis=analysis) - return analysis.with_locals(saved_locals) + value: Optional[Local | Constructed | Tupled] = None + first = True + + for returned in _returns_of(function): + if returned.value is None: + returned_value: Optional[Local | Constructed | Tupled] = None + else: + returned_value, analysis = await _evaluate( + returned.value, analysis=analysis + ) + if first: + value = returned_value + first = False + elif value != returned_value: + value = None + + return value, analysis.with_locals(saved_locals) async def _first_argument_is_the_context( @@ -1170,7 +1220,7 @@ async def _first_argument_is_the_context( async def _evaluate( expression: ast.expr, *, analysis: Analysis -) -> tuple[Optional[Local | Constructed], Analysis]: +) -> tuple[Optional[Local | Constructed | Tupled], Analysis]: """Returns what an expression evaluates to, in the only terms the analysis knows. Those are a reference to a state type, the context, and what a constructor call makes. `None` for the @@ -1215,26 +1265,28 @@ async def _evaluate( args=arguments, keywords=keywords, ): - # A helper in the same file, handed the context, is - # followed so the calls it makes are recorded too. + # A helper in the same file is followed, so the calls + # it makes are recorded too and what it returns can be + # bound to a name. handed, analysis = await _context_argument( arguments, keywords, analysis=analysis ) - if handed is not None and function_name not in analysis.helpers: + if function_name not in analysis.helpers: function = _function_named( function_name, module=analysis.module() ) if function is not None: - parameter = _parameter_for_the_context( - function, handed, method=False + parameter = ( + None if handed is None else _parameter_for_the_context( + function, handed, method=False + ) ) - if parameter is not None: - analysis = await _follow_helper( + if handed is None or parameter is not None: + return await _follow_helper( function, context_parameter=parameter, analysis=analysis, ) - return None, analysis case ast.Call( func=ast.Attribute(value=receiver, attr=str(attribute)), @@ -1314,35 +1366,48 @@ async def _evaluate( match receiver: case ast.Name(id='self'): - # A helper method in the same file, handed the - # context, is followed so the calls it makes are - # recorded too. + # A helper method in the same file is followed, + # so the calls it makes are recorded too and + # what it returns can be bound to a name. handed, analysis = await _context_argument( arguments, keywords, analysis=analysis ) - if ( - handed is not None and - attribute not in analysis.helpers - ): + if attribute not in analysis.helpers: function = _function_named( attribute, module=analysis.module() ) if function is not None: - parameter = _parameter_for_the_context( - function, handed, method=True + parameter = ( + None if handed is None else + _parameter_for_the_context( + function, handed, method=True + ) ) - if parameter is not None: - analysis = await _follow_helper( + if handed is None or parameter is not None: + return await _follow_helper( function, context_parameter=parameter, analysis=analysis, ) - return None, analysis case ast.Name(id=str(name)) if name in analysis.locals: # `another = account`, holding whatever `account` holds. return analysis.locals[name], analysis + case ast.Tuple(elts=elements): + # A tuple, element by element, so an unpacking + # assignment can bind each name. + values: list[Optional[Local]] = [] + for element in elements: + element_value, analysis = await _evaluate( + element, analysis=analysis + ) + values.append( + element_value if + isinstance(element_value, (Reference, Context)) else None + ) + return Tupled(values=tuple(values)), analysis + # Nothing this evaluates. Said out loud when the expression # touches something Reboot related -- whatever it does with it is # not followed, so the calls are likely incomplete -- and left @@ -1388,12 +1453,23 @@ async def _assign( targets = [assign.target] value = None - evaluated: Optional[Local | Constructed] = None + evaluated: Optional[Local | Constructed | Tupled] = None if value is not None: evaluated, analysis = await _evaluate(value, analysis=analysis) match targets, evaluated: + case [ast.Tuple(elts=elements)], Tupled(values=values) if ( + len(elements) == len(values) and + all(isinstance(element, ast.Name) for element in elements) + ): + # `shop, name = self.shop_and_name()` binds each name to + # its element. + for element, element_value in zip(elements, values): + assert isinstance(element, ast.Name) + analysis = analysis.with_local(element.id, element_value) + return analysis + case [ast.Tuple(elts=[ast.Name(id=str(first)), *rest]) ], Constructed(state_type=state_type): # `shop, response = await Shop.open(context, 'a')` @@ -1409,11 +1485,19 @@ async def _assign( analysis = analysis.with_local(name, None) return analysis - # A name cannot be bound to the pair itself. + # A name cannot be bound to a constructor's pair or to a tuple + # itself. local: Optional[Local] = ( - None if isinstance(evaluated, Constructed) else evaluated + None if isinstance(evaluated, (Constructed, Tupled)) else evaluated ) + if isinstance(evaluated, Tupled) and any( + value is not None for value in evaluated.values + ): + # A reference or the context in a tuple nothing unpacks is + # not followed further. + analysis = analysis.with_unsupported(assign) + for target in targets: match target: case ast.Name(id=str(name)): diff --git a/tests/reboot/dashboard/implementation_watcher_tests.py b/tests/reboot/dashboard/implementation_watcher_tests.py index bfc47ca6..19d21ca4 100644 --- a/tests/reboot/dashboard/implementation_watcher_tests.py +++ b/tests/reboot/dashboard/implementation_watcher_tests.py @@ -951,6 +951,72 @@ async def test_a_helper_reaching_itself_still_terminates(self) -> None: ['restock'], ) + async def test_a_helper_returning_a_reference(self) -> None: + self._write( + 'shop_servicer.py', + source=SHOP.replace( + ' async def look(self, context, request):\n pass', + ' async def look(self, context, request):\n' + ' shop = self.shop_of_the_day()\n' + ' await shop.restock(context)\n' + '\n' + ' def shop_of_the_day(self):\n' + " return Shop.ref('a')", + ), + ) + application = self._write('main.py', source=APPLICATION) + + found = servicers(await analyze(application=application)) + + self.assertEqual( + [call.method for call in found[0].methods[0].calls], + ['restock'], + ) + + async def test_a_helper_returning_a_tuple(self) -> None: + self._write( + 'shop_servicer.py', + source=SHOP.replace( + ' async def look(self, context, request):\n pass', + ' async def look(self, context, request):\n' + ' shop, name = self.shop_and_name()\n' + ' await shop.restock(context)\n' + '\n' + ' def shop_and_name(self):\n' + " return Shop.ref('a'), 'a'", + ), + ) + application = self._write('main.py', source=APPLICATION) + + found = servicers(await analyze(application=application)) + + self.assertEqual( + [call.method for call in found[0].methods[0].calls], + ['restock'], + ) + + async def test_a_helper_returning_the_context(self) -> None: + self._write( + 'shop_servicer.py', + source=SHOP.replace( + ' async def look(self, context, request):\n pass', + ' async def look(self, context, request):\n' + ' somehow = self.the_context(context)\n' + " await Shop.ref('a').restock(somehow)\n" + '\n' + ' def the_context(self, context):\n' + ' return context', + ), + ) + application = self._write('main.py', source=APPLICATION) + + found = servicers(await analyze(application=application)) + + self.assertEqual( + [call.method for call in found[0].methods[0].calls], + ['restock'], + ) + async def test_records_the_methods_a_servicer_defines(self) -> None: self._write('shop_servicer.py', source=SHOP) application = self._write('main.py', source=APPLICATION) From 27e16514c6f39c8f581c6581a0034530b5f41e93 Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Wed, 19 Aug 2026 07:43:19 +0000 Subject: [PATCH 31/31] Set aside the calls that only look like Reboot calls A call that hands the context first, on a receiver the analysis cannot identify, is recorded as ambiguous. The method name is kept and the state type is left empty, so the dashboard can say that a method named `restock` is called on something, rather than saying nothing at all. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QPEVMhyDRxZEuH8eiykv99 --- rbt/dashboard/v1/dashboard.proto | 6 +++ reboot/dashboard/implementation_watcher.py | 37 ++++++++++++++++++- .../dashboard/implementation_watcher_tests.py | 29 +++++++++++++++ 3 files changed, 70 insertions(+), 2 deletions(-) diff --git a/rbt/dashboard/v1/dashboard.proto b/rbt/dashboard/v1/dashboard.proto index a86e4568..e3a65fd3 100644 --- a/rbt/dashboard/v1/dashboard.proto +++ b/rbt/dashboard/v1/dashboard.proto @@ -141,6 +141,12 @@ message ServicerInfo { // follow, spelled the way the developer wrote it. When this is // not empty the calls may be incomplete. repeated string unsupported = 4; + + // Represents a call that only looks like a Reboot call. It + // hands the context first but the analysis could not identify + // the receiver, so the method name is recorded and the state + // type is left empty. + repeated Call ambiguous = 5; } // The state type it services, spelled as `StateTypeInfo.name`. diff --git a/reboot/dashboard/implementation_watcher.py b/reboot/dashboard/implementation_watcher.py index 6bfa7272..5d0c4dcc 100644 --- a/reboot/dashboard/implementation_watcher.py +++ b/reboot/dashboard/implementation_watcher.py @@ -902,6 +902,11 @@ class Analysis: # recorded. calls: tuple[ServicerInfo.Method.Call, ...] + # The calls that only look like Reboot calls, made on receivers + # the analysis could not identify. Each records the method name + # with an empty state type. + ambiguous: tuple[ServicerInfo.Method.Call, ...] + # What was met that the analysis does not follow, spelled the way # it was written, so whoever reads the calls can be told they are # likely incomplete rather than left to trust them. @@ -931,6 +936,7 @@ def create(cls, *, filename: str, files: Files) -> 'Analysis': state_type=None, helpers=frozenset(), calls=(), + ambiguous=(), unsupported=(), locals=MappingProxyType({}), ) @@ -962,6 +968,11 @@ def with_locals(self, locals: Mapping[str, Local]) -> 'Analysis': entering and leaving a followed helper.""" return replace(self, locals=MappingProxyType(dict(locals))) + def with_ambiguous(self, call: ServicerInfo.Method.Call) -> 'Analysis': + """Returns this analysis with one more call that only looks + like a Reboot call recorded.""" + return replace(self, ambiguous=(*self.ambiguous, call)) + def with_helper(self, name: str) -> 'Analysis': """Returns this analysis with one more helper followed.""" return replace(self, helpers=self.helpers | {name}) @@ -984,12 +995,13 @@ def with_state_type(self, state_type: str) -> 'Analysis': def with_method_reset(self) -> 'Analysis': """Returns this analysis with the method reset, its calls, - unsupported, locals and helpers cleared, ready for the next - method.""" + ambiguous, unsupported, locals and helpers cleared, ready + for the next method.""" return replace( self, helpers=frozenset(), calls=(), + ambiguous=(), unsupported=(), locals=MappingProxyType({}), ) @@ -1390,6 +1402,26 @@ async def _evaluate( analysis=analysis, ) + if local is None and not ( + isinstance(receiver, ast.Name) and receiver.id == 'self' + ): + handed_first, analysis = await _first_argument_is_the_context( + arguments, analysis=analysis + ) + if handed_first: + # It hands the context first, so it looks like a + # Reboot call, but the receiver cannot be + # identified. The method name is worth keeping + # even without the state type. + analysis = analysis.with_ambiguous( + ServicerInfo.Method.Call( + state_type='', + method=attribute, + how=ServicerInfo.Method.Call.How.CALL, + ) + ) + return None, analysis + case ast.Name(id=str(name)) if name in analysis.locals: # `another = account`, holding whatever `account` holds. return analysis.locals[name], analysis @@ -1629,6 +1661,7 @@ async def _analyze_file(filename: str, files: Files) -> Files: digest=_digest(statement), calls=analysis.calls, unsupported=analysis.unsupported, + ambiguous=analysis.ambiguous, ) ) analysis = analysis.with_method_reset() diff --git a/tests/reboot/dashboard/implementation_watcher_tests.py b/tests/reboot/dashboard/implementation_watcher_tests.py index 19d21ca4..57f0b310 100644 --- a/tests/reboot/dashboard/implementation_watcher_tests.py +++ b/tests/reboot/dashboard/implementation_watcher_tests.py @@ -437,6 +437,17 @@ async def test_a_call_without_the_context_is_unsupported(self) -> None: analysis.unsupported, ("Shop.ref('a').restock(request)",) ) + async def test_an_unidentified_receiver_is_ambiguous(self) -> None: + analysis = await self._analyze( + ' await self.shops.restock(context)' + ) + self.assertEqual( + [call.method for call in analysis.ambiguous], + ['restock'], + ) + self.assertEqual(analysis.calls, ()) + self.assertEqual(analysis.unsupported, ()) + async def test_an_ordinary_assignment_is_not_unsupported(self) -> None: analysis = await self._analyze(' total = request.a + request.b') @@ -1017,6 +1028,24 @@ async def test_a_helper_returning_the_context(self) -> None: ['restock'], ) + async def test_the_ambiguous_calls_reach_their_entry(self) -> None: + self._write( + 'shop_servicer.py', + source=SHOP.replace( + ' async def look(self, context, request):\n pass', + ' async def look(self, context, request):\n' + ' await self.shops.restock(context)', + ), + ) + application = self._write('main.py', source=APPLICATION) + + found = servicers(await analyze(application=application)) + + self.assertEqual( + [call.method for call in found[0].methods[0].ambiguous], + ['restock'], + ) + async def test_records_the_methods_a_servicer_defines(self) -> None: self._write('shop_servicer.py', source=SHOP) application = self._write('main.py', source=APPLICATION)