From 719e83788031ef78daff523061cf63e8d3a9eb0f Mon Sep 17 00:00:00 2001 From: Riley Scheid Date: Thu, 13 Aug 2026 22:14:06 +0000 Subject: [PATCH 01/19] Make `is Any` annotation checks legal under `strict_equality` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mypy 1.18 rejects `if x is Any:` as `comparison-overlap` when `x` is declared as a union of concrete types. The comparison is correct at runtime: `get_args` on an annotation such as `dict[str, Any]` hands back the `typing.Any` object itself, so the converters meet it as a value. Only the declared parameter types disagree, since they do not mention `Any`. Route the comparison through `is_annotation_any`, whose parameter is typed `object` — the honest domain of what typing introspection returns. The converters' declared unions stay as they are, and the runtime behavior is identical: the same `is` comparison, one call deeper. Until now this error failed the build of every target whose closure reaches `reboot/api.py` under a fresh mypy run, which is how it was found: Bazel's remote cache had been serving stale mypy results, so CI never re-ran mypy over these files. Co-Authored-By: Claude Fable 5 --- reboot/api.py | 15 ++++++++++++++- reboot/pydantic_schema_to_zod.py | 3 ++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/reboot/api.py b/reboot/api.py index 8987d9f6..40c6d01e 100644 --- a/reboot/api.py +++ b/reboot/api.py @@ -42,6 +42,19 @@ typing.Type[Dict[str, Any]], ] + +def is_annotation_any(annotation: object) -> bool: + """Whether `annotation` is the `typing.Any` special form. + + `get_args` on an annotation such as `dict[str, Any]` hands back the + `typing.Any` object itself, so converters that recurse into + annotations meet it as a value. Comparing through an `object`-typed + parameter keeps the comparison legal under `strict_equality` for + callers whose declared parameter types do not mention `Any`. + """ + return annotation is Any + + # We don't allow passing arbitrary default values, only these empty # defaults, which also matches with Protobuf semantics. ALLOWED_DEFAULT_BY_FIELD_TYPE = { @@ -638,7 +651,7 @@ def _proto_to_pydantic( # 'None' directly. return None - if output_type is Any: + if is_annotation_any(output_type): # Reverse of the `Any` case in `_pydantic_to_proto`: a # `google.protobuf.Value` becomes the JSON value it holds. assert isinstance(input, Value) diff --git a/reboot/pydantic_schema_to_zod.py b/reboot/pydantic_schema_to_zod.py index 9b9c1fb8..6c41220e 100644 --- a/reboot/pydantic_schema_to_zod.py +++ b/reboot/pydantic_schema_to_zod.py @@ -16,6 +16,7 @@ Model, UserPydanticError, get_field_tag, + is_annotation_any, snake_to_camel, ) from reboot.fail import fail @@ -208,7 +209,7 @@ def pydantic_to_zod( # Currently only used for methods with no response. return 'z.void()' - if input is Any: + if is_annotation_any(input): # `Any` — e.g. a `dict[str, Any]` map value — accepts any JSON # value. `z.json()` is the schema our `zod-to-proto` converts # to a `google.protobuf.Value`. From ccccb71513633198c5380360d508f238b83f4529 Mon Sep 17 00:00:00 2001 From: Riley Scheid Date: Tue, 11 Aug 2026 03:23:51 +0000 Subject: [PATCH 02/19] Allow all methods, not just MCP methods, to have a description A `description=` was only set on `McpMethodOptions.description`, so a Reader, Writer, Transaction or Workflow that was not also an MCP tool didn't have it. It is now set on `MethodOptions.description` for every method, and MCP tool and resource descriptions read from there. `McpMethodOptions.description` is deprecated but still read as a fallback, so protos that already set it keep their descriptions. Co-Authored-By: Claude Opus 5 (1M context) --- rbt/v1alpha1/options.proto | 8 +++- reboot/protoc_gen_reboot_generic.py | 18 ++++++-- reboot/pydantic_schema_to_proto.py | 17 ++++--- reboot/templates/reboot.py.j2 | 4 +- tests/reboot/protoc_gen_reboot_tests.py | 61 +++++++++++++++++++++++++ 5 files changed, 94 insertions(+), 14 deletions(-) diff --git a/rbt/v1alpha1/options.proto b/rbt/v1alpha1/options.proto index 394b9cc8..9c8227ca 100644 --- a/rbt/v1alpha1/options.proto +++ b/rbt/v1alpha1/options.proto @@ -53,8 +53,9 @@ message McpMethodOptions { optional bool resource = 2; // Custom name for the tool/resource (defaults to method name). optional string name = 3; - // Description for the tool/resource. - optional string description = 4; + // Deprecated: write `MethodOptions.description` instead. Read as a + // fallback when it is the only description present. + optional string description = 4 [deprecated = true]; // Display title for the tool/resource. optional string title = 5; } @@ -102,6 +103,9 @@ message MethodOptions { // MCP options for exposing this method as a tool or resource. optional McpMethodOptions mcp = 9; + + // What this method does, in the author's own words. + optional string description = 10; } extend google.protobuf.MethodOptions { diff --git a/reboot/protoc_gen_reboot_generic.py b/reboot/protoc_gen_reboot_generic.py index e41efc71..1e3a4f3b 100644 --- a/reboot/protoc_gen_reboot_generic.py +++ b/reboot/protoc_gen_reboot_generic.py @@ -157,7 +157,6 @@ class ProtoMcpOptions: tool: bool resource: bool name: Optional[str] - description: Optional[str] title: Optional[str] @@ -167,6 +166,7 @@ class ProtoMethodOptions: constructor: bool state_streaming: bool has_errors: bool + description: Optional[str] mcp: Optional[ProtoMcpOptions] @@ -898,16 +898,28 @@ def _proto_method_options( tool=mcp.tool, resource=mcp.resource, name=mcp.name if mcp.HasField('name') else None, - description=mcp.description - if mcp.HasField('description') else None, title=mcp.title if mcp.HasField('title') else None, ) + description: Optional[str] = None + if method_options.HasField('description'): + description = method_options.description + elif ( + method_options.HasField('mcp') and + method_options.mcp.HasField('description') + ): + # An application that was created before + # `MethodOptions.description` will have the deprecated + # `mcp` description, which is permitted for backward + # compatibility. + description = method_options.mcp.description + return ProtoMethodOptions( kind=kind, constructor=self._is_method_constructor(method), state_streaming=state_streaming, has_errors=len(method_options.errors) > 0, + description=description, mcp=mcp_options, ) diff --git a/reboot/pydantic_schema_to_proto.py b/reboot/pydantic_schema_to_proto.py index f37ee679..2f774a1c 100644 --- a/reboot/pydantic_schema_to_proto.py +++ b/reboot/pydantic_schema_to_proto.py @@ -921,6 +921,16 @@ async def generate_proto_file_from_api( f" errors: [\"{type_name}{to_pascal_case(method_name)}Errors\"],\n" ) + # What the author said the method does, written + # whether or not the method is exposed to MCP. + if method_spec.description is not None: + # The description can contain `\` character, so we + # need to escape it for proto string literal. + await proto.write( + " description: " + f'"{_escape_string_for_proto(method_spec.description)}",\n' + ) + # MCP options for exposing method as tool/resource. if method_spec.mcp is not None: mcp = method_spec.mcp @@ -935,13 +945,6 @@ async def generate_proto_file_from_api( mcp_fields.append( f'name: "{_escape_string_for_proto(mcp.name)}"' ) - if method_spec.description is not None: - # The description can contain `\` character, so - # we need to escape it for proto string literal. - mcp_fields.append( - "description: " - f'"{_escape_string_for_proto(method_spec.description)}"' - ) if mcp.title is not None: # The title can contain `\` character, so we need # to escape it for proto string literal. diff --git a/reboot/templates/reboot.py.j2 b/reboot/templates/reboot.py.j2 index 2acaeb46..66db0879 100644 --- a/reboot/templates/reboot.py.j2 +++ b/reboot/templates/reboot.py.j2 @@ -2692,7 +2692,7 @@ class {{ state.proto.name }}BaseServicer(IMPORT_reboot_aio_servicers.Servicer): {% set tool_name = method.options.proto.mcp.name if method.options.proto.mcp.name else mcp_name_prefix + (method.proto.name | to_snake) %} {% set tool_title = method.options.proto.mcp.title if method.options.proto.mcp.title else method.proto.name %} {% set tool_description_suffix = "" if state.proto.auto_construct != AUTO_CONSTRUCT_UNSPECIFIED else " on " + state.proto.name %} -{% set tool_description = method.options.proto.mcp.description if method.options.proto.mcp.description else "Invoke " + method.proto.name + tool_description_suffix + "." %} +{% set tool_description = method.options.proto.description if method.options.proto.description else "Invoke " + method.proto.name + tool_description_suffix + "." %} {% set request_type = state.proto.name + "." + method.proto.name + "Request" %} # Tool for '{{ method.proto.full_name }}'. @@ -2741,7 +2741,7 @@ class {{ state.proto.name }}BaseServicer(IMPORT_reboot_aio_servicers.Servicer): {% set method_path = method.proto.name | to_snake %} {% set resource_name = method.options.proto.mcp.name if method.options.proto.mcp.name else mcp_name_prefix + method_path %} {% set resource_title = method.options.proto.mcp.title if method.options.proto.mcp.title else method.proto.name %} -{% set resource_description = method.options.proto.mcp.description if method.options.proto.mcp.description else state.proto.name + " state." %} +{% set resource_description = method.options.proto.description if method.options.proto.description else state.proto.name + " state." %} {% set request_type = state.proto.name + "." + method.proto.name + "Request" %} # Resource for '{{ method.proto.full_name }}'. diff --git a/tests/reboot/protoc_gen_reboot_tests.py b/tests/reboot/protoc_gen_reboot_tests.py index 964c4a79..c1a28ca5 100644 --- a/tests/reboot/protoc_gen_reboot_tests.py +++ b/tests/reboot/protoc_gen_reboot_tests.py @@ -4,6 +4,7 @@ from google.protobuf.compiler import plugin_pb2 from google.protobuf.descriptor_pb2 import FileDescriptorSet from google.protobuf.descriptor_pool import DescriptorPool +from rbt.v1alpha1 import options_pb2 from reboot.protoc_gen_reboot_generic import ( BaseFile, ProtocPlugin, @@ -173,6 +174,66 @@ def test_map_type(self) -> None: map_field_type = fields['metadata'] self.assertEqual(map_field_type, "dict[str, str]") + def _greet_method_options(self): + test_plugin(self.plugin, self.descriptor_set) + + template_data = self.plugin.proto_to_template_data[ + 'tests/reboot/greeter.proto'] + methods = { + method.proto.name: method + for method in template_data.clients[0].services[0].methods + } + return methods['Greet'].options.proto + + def _greet_method_descriptor(self): + file = self.descriptor_set.file[-1] + self.assertEqual(file.name, 'tests/reboot/greeter.proto') + + service = file.service[0] + self.assertEqual(service.name, 'GreeterMethods') + + methods = {method.name: method for method in service.method} + return methods['Greet'] + + def test_method_description_reaches_the_template(self) -> None: + method = self._greet_method_descriptor() + method.options.Extensions[options_pb2.method + ].description = 'Greet someone.' + + self.assertEqual( + self._greet_method_options().description, + 'Greet someone.', + ) + + def test_deprecated_mcp_description_still_reaches_the_template( + self + ) -> None: + # An application that was created before + # `MethodOptions.description` will have the deprecated `mcp` + # description, which is permitted for backward compatibility. + method = self._greet_method_descriptor() + method.options.Extensions[options_pb2.method + ].mcp.description = 'Greet someone.' + + self.assertEqual( + self._greet_method_options().description, + 'Greet someone.', + ) + + def test_method_description_wins_over_the_deprecated_one(self) -> None: + method = self._greet_method_descriptor() + options = method.options.Extensions[options_pb2.method] + options.description = 'What the author wrote.' + options.mcp.description = 'The deprecated spelling.' + + self.assertEqual( + self._greet_method_options().description, + 'What the author wrote.', + ) + + def test_method_without_a_description_has_none(self) -> None: + self.assertIsNone(self._greet_method_options().description) + class ToLowerCamelTest(unittest.TestCase): """Test the `to_lower_camel` Jinja filter override. From 88616079a4810764c7f233b870269f70de7594ee Mon Sep 17 00:00:00 2001 From: Riley Scheid Date: Tue, 11 Aug 2026 03:25:02 +0000 Subject: [PATCH 03/19] Add a dev-mode companion application that serves a dashboard The first of several changes, split so that each can be reviewed on its own and so the dashboard can be tried out while the rest is written. It is not a documented feature yet: nothing opens a page by itself, so seeing one means passing `--open-dashboard` or visiting the URL. `rbt dev run` now starts a second Reboot application alongside the developer's, with its own state store, holding what the dashboard needs. The companion watches the developer's `api/` directory and records what those files declare, so the dashboard can describe an application. It also serves the page itself. The page reads that schema reactively and renders one section per state type: its fields, and each method's kind, whether it constructs, whether it is reachable over MCP, its signature and the errors it raises. Dashboard state, such as which detail views are open and which are closed, is saved in Reboot state, so it survives a hot reload and an `rbt dev run` restart. Auto-open is complete but off. `_AUTO_OPEN_DASHBOARD` is False, so only `--open-dashboard` opens a page. We don't reopen the dashboard if the developer already has it open, and we use the `Presence` library to determine whether they do. Note that presence does not drain through a DevPod workstation's port forward, which is filed separately. This will eventually supersede the inspect dashboard at `/__/inspect`, which lists state instances and their values. It does not replace it yet and both exist meanwhile: this describes an application's API, its state types, their fields and their methods, and cannot yet show the data behind them. Co-Authored-By: Claude Opus 5 (1M context) --- rbt/dashboard/v1/BUILD.bazel | 67 ++ rbt/dashboard/v1/dashboard.proto | 173 +++++ rbt/dashboard/v1/package.json | 3 + rbt/std/presence/v1/presence.proto | 16 + rbt/v1alpha1/inspect/inspect.proto | 51 ++ reboot/BUILD.bazel | 1 + reboot/cli/commands/BUILD.bazel | 26 + reboot/cli/commands/dashboard.py | 245 +++++++ reboot/cli/commands/dev.py | 207 +++++- reboot/cli/common/BUILD.bazel | 1 + reboot/cli/common/cli.py | 12 +- reboot/dashboard/BUILD.bazel | 120 +++ reboot/dashboard/api_reader.py | 202 ++++++ reboot/dashboard/api_watcher.py | 190 +++++ reboot/dashboard/constants.py | 46 ++ reboot/dashboard/frontend/BUILD.bazel | 51 ++ reboot/dashboard/frontend/dashboard.css | 681 ++++++++++++++++++ reboot/dashboard/frontend/index.html | 18 + reboot/dashboard/frontend/src/constants.ts | 5 + reboot/dashboard/frontend/src/main.tsx | 527 ++++++++++++++ reboot/dashboard/frontend/tsconfig.json | 16 + reboot/dashboard/main.py | 84 +++ reboot/dashboard/servicers.py | 131 ++++ reboot/inspect/BUILD.bazel | 13 + reboot/inspect/describe_state_type.py | 141 ++++ reboot/std/presence/v1/presence.py | 9 + tests/reboot/cli/BUILD.bazel | 14 + tests/reboot/cli/dashboard_tests.py | 127 ++++ tests/reboot/cli/dev_tests.py | 16 + tests/reboot/dashboard/BUILD.bazel | 83 +++ tests/reboot/dashboard/api/shop/v1/helper.py | 3 + tests/reboot/dashboard/api/shop/v1/shop.py | 48 ++ tests/reboot/dashboard/api_reader_tests.py | 127 ++++ tests/reboot/dashboard/api_watcher_tests.py | 111 +++ tests/reboot/dashboard/application_tests.py | 102 +++ tests/reboot/dashboard/dashboard_tests.py | 418 +++++++++++ .../reboot/dashboard/open_dashboard_tests.py | 219 ++++++ tests/reboot/dashboard/preferences_tests.py | 166 +++++ 38 files changed, 4466 insertions(+), 4 deletions(-) create mode 100644 rbt/dashboard/v1/BUILD.bazel create mode 100644 rbt/dashboard/v1/dashboard.proto create mode 100644 rbt/dashboard/v1/package.json create mode 100644 reboot/cli/commands/dashboard.py create mode 100644 reboot/dashboard/BUILD.bazel create mode 100644 reboot/dashboard/api_reader.py create mode 100644 reboot/dashboard/api_watcher.py create mode 100644 reboot/dashboard/constants.py create mode 100644 reboot/dashboard/frontend/BUILD.bazel create mode 100644 reboot/dashboard/frontend/dashboard.css create mode 100644 reboot/dashboard/frontend/index.html create mode 100644 reboot/dashboard/frontend/src/constants.ts create mode 100644 reboot/dashboard/frontend/src/main.tsx create mode 100644 reboot/dashboard/frontend/tsconfig.json create mode 100644 reboot/dashboard/main.py create mode 100644 reboot/dashboard/servicers.py create mode 100644 reboot/inspect/describe_state_type.py create mode 100644 tests/reboot/cli/dashboard_tests.py create mode 100644 tests/reboot/dashboard/BUILD.bazel create mode 100644 tests/reboot/dashboard/api/shop/v1/helper.py create mode 100644 tests/reboot/dashboard/api/shop/v1/shop.py create mode 100644 tests/reboot/dashboard/api_reader_tests.py create mode 100644 tests/reboot/dashboard/api_watcher_tests.py create mode 100644 tests/reboot/dashboard/application_tests.py create mode 100644 tests/reboot/dashboard/dashboard_tests.py create mode 100644 tests/reboot/dashboard/open_dashboard_tests.py create mode 100644 tests/reboot/dashboard/preferences_tests.py diff --git a/rbt/dashboard/v1/BUILD.bazel b/rbt/dashboard/v1/BUILD.bazel new file mode 100644 index 00000000..84478778 --- /dev/null +++ b/rbt/dashboard/v1/BUILD.bazel @@ -0,0 +1,67 @@ +load( + "@com_github_reboot_dev_reboot//reboot:rules.bzl", + "js_proto_library", + "js_reboot_library", + "js_reboot_react_library", + "py_reboot_library", +) +load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library") + +proto_library( + name = "dashboard_proto", + srcs = [ + ":dashboard.proto", + ], + visibility = ["//visibility:public"], + deps = [ + "@com_github_reboot_dev_reboot//rbt/v1alpha1:options_proto", + ], +) + +py_reboot_library( + name = "dashboard_py_reboot", + proto = "dashboard.proto", + proto_library = ":dashboard_proto", + visibility = ["//visibility:public"], +) + +js_proto_library( + name = "dashboard_js_proto", + package_json = ":package.json", + proto = "dashboard.proto", + proto_deps = [ + ":dashboard_proto", + # ISSUE(https://github.com/reboot-dev/mono/issues/3218): Until we can + # use `create_protoc_plugin_rule` we need to repeat the dependencies of + # the `proto_libraries` here. + "@com_github_reboot_dev_reboot//rbt/v1alpha1:options_proto", + "@com_google_protobuf//:descriptor_proto", + ], + visibility = ["//visibility:public"], +) + +js_reboot_library( + name = "dashboard_js_reboot", + srcs = [ + ":dashboard_proto", + ], + proto = "dashboard.proto", + visibility = ["//visibility:public"], + deps = [ + ":dashboard_js_proto", + ], +) + +js_reboot_react_library( + name = "dashboard_js_reboot_react", + srcs = [ + ":dashboard_js_proto", + ], + proto = "dashboard.proto", + proto_deps = [ + ":dashboard_proto", + "@com_github_reboot_dev_reboot//rbt/v1alpha1:options_proto", + "@com_google_protobuf//:descriptor_proto", + ], + visibility = ["//visibility:public"], +) diff --git a/rbt/dashboard/v1/dashboard.proto b/rbt/dashboard/v1/dashboard.proto new file mode 100644 index 00000000..38368499 --- /dev/null +++ b/rbt/dashboard/v1/dashboard.proto @@ -0,0 +1,173 @@ +syntax = "proto3"; + +package rbt.dashboard.v1; + +import "rbt/v1alpha1/options.proto"; + +//////////////////////////////////////////////////////////////////////// + +// The information the developer dashboard needs to display data +// about the Reboot application being developed. + +message FieldInfo { + string name = 1; + string type = 2; +} + +message MethodInfo { + string name = 1; + string kind = 2; + repeated FieldInfo arguments = 3; + repeated FieldInfo returns = 4; + repeated string errors = 5; + optional string description = 6; + bool factory = 7; + bool mcp = 8; +} + +message StateTypeInfo { + string name = 1; + + // The file the developer declared it in, e.g. + // "bank/v1/account.py". + string file = 2; + repeated FieldInfo fields = 3; + repeated MethodInfo methods = 4; +} + +//////////////////////////////////////////////////////////////////////// + +// What the dashboard application has read of the developer's API +// files, so that a browser can read it without reaching the +// application itself. +message API { + option (rbt.v1alpha1.state) = { + }; + + // The state types the developer's API files declare, which exist + // before the application does. + repeated StateTypeInfo state_types = 1; + + // Why the API files could not be read, if they could not be. A + // half-written file is the normal case while someone is typing, and + // saying so beats showing nothing. + string error = 2; +} + +message APIGetRequest {} + +message APIGetResponse { + repeated StateTypeInfo state_types = 1; + string error = 2; +} + +message APIUpdateRequest { + repeated StateTypeInfo state_types = 1; + string error = 2; +} + +message APIUpdateResponse {} + +message APIWatchRequest {} + +message APIWatchResponse {} + +//////////////////////////////////////////////////////////////////////// + +// What the developer has told the dashboard about opening dashboards. +// +// Kept here rather than in a file under their project because it is a +// fact about their machine and their browser, not about their +// application, and nothing about it belongs in their repository. It +// survives a hot reload and an `rbt dev run` restart because the +// dashboard application's state store does. +message Preferences { + option (rbt.v1alpha1.state) = { + }; + + // Whether the developer has asked `rbt dev run` to stop opening a + // dashboard by itself. Recorded as the exception rather than as + // the rule, so that somebody who has never chosen gets a dashboard + // opened when nobody is looking at one. + // + // `--open-dashboard` opens one whatever this says. + bool suppress_open_on_restart = 1; + + // The state types whose methods the developer has opened on the + // dashboard, by fully qualified name such as `bank.v1.Account`. + // Held as the set that is open rather than the set that is closed, + // so that somebody who has clicked nothing gets a page where every + // state type shows its methods but none of their detail. + // + // Kept sorted, so that clicking two state types open in one order + // and then the other stores the same thing both times. + repeated string expanded_state_types = 2; +} + +message PreferencesGetRequest {} + +message PreferencesGetResponse { + bool suppress_open_on_restart = 1; + repeated string expanded_state_types = 2; +} + +message PreferencesSetSuppressOpenOnRestartRequest { + bool suppress_open_on_restart = 1; +} + +message PreferencesSetSuppressOpenOnRestartResponse {} + +message PreferencesSetExpandedRequest { + // The fully qualified name of one state type, spelled the way + // `StateTypeInfo` spells it. + string state_type = 1; + + bool expanded = 2; +} + +message PreferencesSetExpandedResponse {} + +//////////////////////////////////////////////////////////////////////// + +service APIMethods { + rpc Get(APIGetRequest) returns (APIGetResponse) { + option (rbt.v1alpha1.method).reader = { + }; + } + + rpc Update(APIUpdateRequest) returns (APIUpdateResponse) { + option (rbt.v1alpha1.method).writer = { + }; + } + + // Watches the developer's API files for as long as the dashboard + // application runs, reading each one that changes. + rpc Watch(APIWatchRequest) returns (APIWatchResponse) { + option (rbt.v1alpha1.method).workflow = { + }; + } +} + +//////////////////////////////////////////////////////////////////////// + +service PreferencesMethods { + rpc Get(PreferencesGetRequest) returns (PreferencesGetResponse) { + option (rbt.v1alpha1.method).reader = { + }; + } + + rpc SetSuppressOpenOnRestart(PreferencesSetSuppressOpenOnRestartRequest) + returns (PreferencesSetSuppressOpenOnRestartResponse) { + option (rbt.v1alpha1.method).writer = { + }; + } + + // One state type per call rather than the whole set, so that two + // tabs opening two different state types at the same moment do not + // each write back a set that predates the other's click. + rpc SetExpanded(PreferencesSetExpandedRequest) + returns (PreferencesSetExpandedResponse) { + option (rbt.v1alpha1.method).writer = { + }; + } +} diff --git a/rbt/dashboard/v1/package.json b/rbt/dashboard/v1/package.json new file mode 100644 index 00000000..3dbc1ca5 --- /dev/null +++ b/rbt/dashboard/v1/package.json @@ -0,0 +1,3 @@ +{ + "type": "module" +} diff --git a/rbt/std/presence/v1/presence.proto b/rbt/std/presence/v1/presence.proto index 38ac3238..23e0528b 100644 --- a/rbt/std/presence/v1/presence.proto +++ b/rbt/std/presence/v1/presence.proto @@ -24,6 +24,15 @@ message Presence { * `Subscriber` using the API described there. */ service PresenceMethods { + /** + * Constructs the `Presence` instance, empty, so that `List` + * answers before the first `Subscribe`. + */ + rpc Create(CreateRequest) returns (CreateResponse) { + option (rbt.v1alpha1.method).writer = { + }; + } + /** * Registers a subscriber as present. * @@ -54,6 +63,13 @@ service PresenceMethods { //////////////////////////////////////////////////////////////////////// +// See `Create`. +message CreateRequest {} + +message CreateResponse {} + +//////////////////////////////////////////////////////////////////////// + // See `Subscribe`. message SubscribeRequest { // The subscriber to register as present. diff --git a/rbt/v1alpha1/inspect/inspect.proto b/rbt/v1alpha1/inspect/inspect.proto index 5031d26c..478a0e92 100644 --- a/rbt/v1alpha1/inspect/inspect.proto +++ b/rbt/v1alpha1/inspect/inspect.proto @@ -66,6 +66,57 @@ message GetStateResponse { //////////////////////////////////////////////////////////////////////// +// A field of a state, or an argument of a method. +message FieldInfo { + string name = 1; + + // Rendered for a reader, e.g. "str", "float", "AccountState". + string type = 2; +} + +message MethodInfo { + string name = 1; + + // One of "reader", "writer", "transaction" or "workflow". + string kind = 2; + + // The request message's fields, flattened. A method taking no + // request has none. + repeated FieldInfo arguments = 3; + + // The response's fields; empty when the method returns + // nothing. + repeated FieldInfo returns = 4; + + // Names of the error types the method declares it may raise. + repeated string errors = 5; + + // The description its author wrote, when there is one. + optional string description = 6; + + // Whether this method is a factory, constructing the state + // rather than requiring it to already exist. + bool factory = 7; + + // Whether the method is exposed as an MCP tool or resource. + bool mcp = 8; +} + +message StateTypeInfo { + // Fully qualified, e.g. "bank.v1.Account". + string name = 1; + + // The file the developer declared it in, e.g. + // "bank/v1/account.py". + string file = 2; + + repeated FieldInfo fields = 3; + + repeated MethodInfo methods = 4; +} + +//////////////////////////////////////////////////////////////////////// + service Inspect { // The list of state types in an application is static, however, we // make this a streaming RPC so that the client can hear when it diff --git a/reboot/BUILD.bazel b/reboot/BUILD.bazel index 2bb7b4cd..c7d6d83d 100644 --- a/reboot/BUILD.bazel +++ b/reboot/BUILD.bazel @@ -516,6 +516,7 @@ py_library( ":python_thirdparty", "//reboot/aio:python", "//reboot/cli:main_py", + "//reboot/dashboard:main_py", "//reboot/mcp:python", "//reboot/nodejs:python", "@com_github_reboot_dev_reboot//protoc_gen_mypy_plugin:protoc-gen-mypy", diff --git a/reboot/cli/commands/BUILD.bazel b/reboot/cli/commands/BUILD.bazel index 6f059898..763242c8 100644 --- a/reboot/cli/commands/BUILD.bazel +++ b/reboot/cli/commands/BUILD.bazel @@ -1,6 +1,27 @@ load("@rbt_pypi//:requirements.bzl", "requirement") load("@rules_python//python:defs.bzl", "py_library") +py_library( + name = "dashboard_py", + srcs = ["dashboard.py"], + srcs_version = "PY3", + visibility = ["//visibility:public"], + deps = [ + ":dev_py", + "//reboot:version_py", + "//reboot/cli/common:directories_py", + "//reboot/cli/common:rc_py", + "//reboot/cli/common:subprocesses_py", + "//reboot/cli/common:terminal_py", + "//reboot/dashboard:constants_py", + # The application this command spawns as a subprocess, so + # that it is importable from the CLI's own interpreter. + "//reboot/dashboard:main_py", + "@com_github_reboot_dev_reboot//reboot:settings_py", + "@com_github_reboot_dev_reboot//reboot/aio/backoff:python", + ], +) + py_library( name = "dev_py", srcs = ["dev.py"], @@ -12,6 +33,10 @@ py_library( requirement("cryptography"), requirement("python-dotenv"), ":generate_py", + "//rbt/dashboard/v1:dashboard_py_reboot", + "//rbt/std/presence/v1:presence_py_reboot", + "//reboot/aio:aborted_py", + "//reboot/aio:external_py", "//reboot/cli/common:directories_py", "//reboot/cli/common:frontend_py", "//reboot/cli/common:monkeys_py", @@ -20,6 +45,7 @@ py_library( "//reboot/cli/common:transpile_py", "//reboot/cli/common:watch_py", "//reboot/controller:plan_makers_py", + "//reboot/dashboard:constants_py", "//reboot/server:local_envoy_factory_py", "@com_github_reboot_dev_reboot//reboot:settings_py", "@com_github_reboot_dev_reboot//reboot/aio:exceptions_py", diff --git a/reboot/cli/commands/dashboard.py b/reboot/cli/commands/dashboard.py new file mode 100644 index 00000000..35f51ea0 --- /dev/null +++ b/reboot/cli/commands/dashboard.py @@ -0,0 +1,245 @@ +"""The `rbt dashboard` command, which runs the developer dashboard.""" +import argparse +import asyncio +import os +import secrets +import shutil +import sys +from pathlib import Path +from reboot.aio.backoff import Backoff +from reboot.cli.commands.dev import ( + check_local_envoy_mode, + try_and_become_child_subreaper_on_linux, +) +from reboot.cli.common import terminal +from reboot.cli.common.directories import ( + add_working_directory_options, + dot_rbt_directory, + use_working_directory, +) +from reboot.cli.common.rc import ArgumentParser +from reboot.cli.common.subprocesses import Subprocesses +from reboot.dashboard.constants import ( + DASHBOARD_PATH, + DEFAULT_DASHBOARD_PORT, + ENVVAR_RBT_API_DIRECTORY, +) +from reboot.settings import ( + ENVVAR_RBT_DEV, + ENVVAR_RBT_FRONTEND_DIST_PATH, + ENVVAR_RBT_FRONTEND_HOST, + ENVVAR_RBT_FRONTEND_ROOT_PATH, + ENVVAR_RBT_NAME, + ENVVAR_RBT_NODEJS, + ENVVAR_RBT_SERVERS, + ENVVAR_RBT_STATE_DIRECTORY, + ENVVAR_REBOOT_CRYPTO_ROOT_KEYS, + ENVVAR_REBOOT_EXPECTED_VERSION, + ENVVAR_REBOOT_LOCAL_ENVOY, + ENVVAR_REBOOT_LOCAL_ENVOY_PORT, + ENVVAR_REBOOT_OAUTH_SIGNING_SECRET, +) +from reboot.version import REBOOT_VERSION +from typing import Optional + +# The dashboard application's name, which names its state directory +# under `.rbt/`. A sibling of `.rbt/dev/` rather than inside it, so +# that it can never collide with a developer's application, whose +# state lives at `.rbt/dev//`. +DASHBOARD_STATE_DIRECTORY_NAME = 'dashboard' + + +def dashboard_subcommands() -> list[str]: + return ['dashboard'] + + +def register_dashboard(parser: ArgumentParser): + add_working_directory_options(parser.subcommand('dashboard')) + + parser.subcommand('dashboard').add_argument( + '--api-directory', + type=str, + required=True, + help='directory containing the API files the dashboard watches', + ) + + parser.subcommand('dashboard').add_argument( + '--port', + type=int, + help='port on which the dashboard will serve traffic; defaults to ' + f'{DEFAULT_DASHBOARD_PORT}', + ) + + +def _dashboard_env( + args, + parser: ArgumentParser, + *, + port: int, + api_directory: str, +) -> dict[str, str]: + """The environment for the dashboard application. + + Built from the ambient environment rather than from any + application environment, so that nothing naming a developer's + application, such as its name, state directory, port, launcher or + frontend, reaches an application that shares none of it. + """ + composed = os.environ.copy() + + # Every other application-flavored variable is overwritten below; + # these four have no dashboard value to overwrite them with, so a + # developer's shell export would leak through and make the + # dashboard a Node.js application or serve their frontend. + for name in ( + ENVVAR_RBT_NODEJS, + ENVVAR_RBT_FRONTEND_HOST, + ENVVAR_RBT_FRONTEND_DIST_PATH, + ENVVAR_RBT_FRONTEND_ROOT_PATH, + ): + composed.pop(name, None) + + composed[ENVVAR_RBT_DEV] = 'true' + composed[ENVVAR_REBOOT_EXPECTED_VERSION] = REBOOT_VERSION + composed[ENVVAR_REBOOT_LOCAL_ENVOY] = 'true' + composed[ENVVAR_REBOOT_LOCAL_ENVOY_PORT] = str(port) + + # A single server, so that a subscriber's `Connect` and + # `Toggle` always land on the same process; presence tracks its + # connections in memory there. `ENVVAR_REBOOT_LOCAL_ENVOY` is + # set above because one server otherwise turns Envoy off, and + # the browser has to reach this application. + composed[ENVVAR_RBT_SERVERS] = '1' + + # Where the developer's API files are, which the dashboard can + # read whether or not anything is running. Passed the way the + # developer spelled it, so that a file can be shown as + # `api/bank/v1/account.py`; the dashboard runs in this working + # directory, where that spelling resolves. + composed[ENVVAR_RBT_API_DIRECTORY] = api_directory + + composed[ENVVAR_RBT_NAME] = DASHBOARD_STATE_DIRECTORY_NAME + + state_directory = ( + dot_rbt_directory(args, parser) / DASHBOARD_STATE_DIRECTORY_NAME + ) + composed[ENVVAR_RBT_STATE_DIRECTORY] = str(state_directory) + + root_keys_path = state_directory / 'crypto-root-keys' + if root_keys_path.exists(): + composed[ENVVAR_REBOOT_CRYPTO_ROOT_KEYS] = root_keys_path.read_text() + else: + root_keys = f'v1:{secrets.token_urlsafe(32)}' + root_keys_path.parent.mkdir(parents=True, exist_ok=True) + root_keys_path.write_text(root_keys) + composed[ENVVAR_REBOOT_CRYPTO_ROOT_KEYS] = root_keys + + composed[ENVVAR_REBOOT_OAUTH_SIGNING_SECRET] = composed[ + ENVVAR_REBOOT_CRYPTO_ROOT_KEYS] + + return composed + + +async def _run_dashboard( + *, + env: dict[str, str], + state_directory: Path, + subprocesses: Subprocesses, +) -> None: + """Runs the dashboard application, restarting it if it exits. + + The dashboard's schema changes whenever Reboot's does, so the + expected reason for it to fail at startup is a backwards + incompatibility after an upgrade. Its state is ours and is + disposable, so the first failure deletes it and tries again + without asking. A second failure is something else, and gets + reported once rather than silently retried forever. + """ + backoff = Backoff() + failures = 0 + reported = False + + while True: + async with subprocesses.exec( + sys.executable, + '-m', + 'reboot.dashboard.main', + env=env, + # The application's own startup output would drown the one + # line this command prints; anything it writes to stderr + # still reaches the terminal. + stdout=asyncio.subprocess.DEVNULL, + ) as process: + await process.wait() + failed = process.returncode != 0 + + if not failed: + failures = 0 + else: + failures += 1 + + if failures == 1: + await asyncio.to_thread( + shutil.rmtree, state_directory, ignore_errors=True + ) + elif not reported: + reported = True + terminal.warn( + 'The dashboard application keeps failing to start; ' + 'still trying.' + ) + + await backoff() + + +async def dashboard( + args, + parser: ArgumentParser, +) -> int: + """Implementation of the 'dashboard' subcommand.""" + with use_working_directory(args, parser): + # If on Linux try to become a child subreaper so that we can + # properly clean up all processes descendant from us! Envoy in + # particular is a grandchild, and one that outlives the + # application would keep answering on the dashboard's port. + try_and_become_child_subreaper_on_linux() + + subprocesses = Subprocesses() + + # Pick the mode in which we'll run a local Envoy proxy and + # check that the mode is usable, e.g. that Docker is running + # and can access the Envoy proxy image, or that the `envoy` + # executable runs. Fail otherwise. + await check_local_envoy_mode(subprocesses) + + port = args.port or DEFAULT_DASHBOARD_PORT + + env = _dashboard_env( + args, + parser, + port=port, + api_directory=args.api_directory, + ) + + terminal.info( + 'Your dashboard is at ' + f'http://127.0.0.1:{port}{DASHBOARD_PATH}/\n' + ) + + await _run_dashboard( + env=env, + state_directory=Path(env[ENVVAR_RBT_STATE_DIRECTORY]), + subprocesses=subprocesses, + ) + + return 0 + + +async def handle_dashboard_subcommand( + args: argparse.Namespace, + *, + parser: ArgumentParser, +) -> Optional[int]: + if args.subcommand == 'dashboard': + return await dashboard(args, parser) + return None diff --git a/reboot/cli/commands/dev.py b/reboot/cli/commands/dev.py index 00f10e6a..45c1e2fc 100644 --- a/reboot/cli/commands/dev.py +++ b/reboot/cli/commands/dev.py @@ -12,6 +12,7 @@ import sys import termios import tty +import webbrowser from colorama import Fore from cryptography import x509 from cryptography.hazmat.backends import default_backend @@ -23,9 +24,12 @@ OTEL_EXPORTER_OTLP_TRACES_INSECURE, ) from pathlib import Path +from rbt.dashboard.v1.dashboard_rbt import Preferences +from rbt.std.presence.v1.presence_rbt import Presence from reboot.aio.backoff import Backoff from reboot.aio.contexts import EffectValidation from reboot.aio.exceptions import InputError +from reboot.aio.external import ExternalContext from reboot.cli.commands.generate import generate_direct # We import the whole `terminal` module (as opposed to the methods it contains) # to allow us to mock these methods out in tests. @@ -52,6 +56,12 @@ ) from reboot.cli.common.watch import FileWatcher, file_watcher from reboot.controller.plan_makers import validate_num_servers +from reboot.dashboard.constants import ( + DASHBOARD_PATH, + DEFAULT_DASHBOARD_PORT, + PREFERENCES_ID, + PRESENCE_ID, +) from reboot.server.local_envoy_factory import LocalEnvoyFactory from reboot.settings import ( DEFAULT_SECURE_PORT, @@ -233,6 +243,28 @@ def _register_dev_run(parser: ArgumentParser): f'{DEFAULT_LOCAL_ENVOY_PORT}', ) + parser.subcommand('dev run').add_argument( + '--open-dashboard', + type=bool, + # Three states, two of which currently agree: + # '--open-dashboard' opens one; unset and + # '--no-open-dashboard' both leave the browser alone while + # `_AUTO_OPEN_DASHBOARD` is off. Unset regains a meaning of + # its own once that is turned on: open one unless somebody + # is already looking at one, or the banner said not to. + default=None, + help='open a dashboard in your browser once your application ' + 'is serving', + ) + + parser.subcommand('dev run').add_argument( + '--dashboard-port', + type=int, + help='port on which the developer dashboard, started separately ' + f'with `rbt dashboard`, is serving; defaults to ' + f'{DEFAULT_DASHBOARD_PORT}', + ) + parser.subcommand('dev run').add_argument( '--watch', type=str, @@ -420,6 +452,138 @@ async def _run( application_started_event.clear() +async def _viewers(dashboard_url: str) -> list[str]: + """The subscriber ids of everyone looking at a dashboard. + + The dashboard constructs the `Presence` instance, empty, when it + initializes, so there is an answer from the moment it is up. + """ + context = ExternalContext(name="dev-run-open-dashboard", url=dashboard_url) + response = await Presence.ref(PRESENCE_ID).List(context) + return list(response.subscriber_ids) + + +async def _open_on_restart(dashboard_url: str) -> bool: + """Whether the developer still wants a dashboard opened for them. + + The dashboard's banner writes this when they click it, and it + outlives the `rbt dev run` they clicked it in. The dashboard + writes the default when it initializes, so nobody ever clicking + means a dashboard opens. + """ + context = ExternalContext(name="dev-run-open-dashboard", url=dashboard_url) + response = await Preferences.ref(PREFERENCES_ID).Get(context) + return not response.suppress_open_on_restart + + +# Whether `rbt dev run` may open a dashboard nobody asked it for. +# +# False while the dashboard is still being built, so that the only way +# to see one is `--open-dashboard`. Everything that decides when to +# open one by itself, the `Presence` subscribers and the banner's +# choice, is covered by `open_dashboard_tests`; turning auto-open on +# is this constant and nothing else. +_AUTO_OPEN_DASHBOARD = False + + +async def _open_dashboard_once( + *, + dashboard_url: str, + forced: bool, +) -> None: + """Opens a dashboard, unless the developer would rather it didn't. + + They would rather it didn't in two cases. One is that somebody is + already looking at one: the page subscribes to `Presence` for as + long as it is open, so a tab left up from an earlier run keeps a + second one from appearing, and a tab they closed is replaced. The + other is that they clicked the dashboard's "Don't reopen this + dashboard on restart" banner, which is remembered until they click + the banner that undoes it. + + `Presence` learns that a viewer has gone from the cancellation of + the page's `Connect` RPC, and nothing else. A proxy that holds its + server-side socket open after the browser goes away therefore + leaves a viewer listed who is not there, and the effect is that no + dashboard opens, which `--open-dashboard` overrides, as it + overrides the banner. + """ + if not forced: + if not await _open_on_restart(dashboard_url): + terminal.info( + 'You asked for this dashboard not to be reopened; run ' + 'with `--open-dashboard` to see it anyway, or visit ' + f'{dashboard_url}{DASHBOARD_PATH}/' + ) + return + + if len(await _viewers(dashboard_url)) > 0: + # Say why nothing opened, since a run that opens nothing + # and explains nothing is indistinguishable from a broken + # one, particularly when the tab being counted is behind + # another window, or on another screen. + terminal.info( + 'A dashboard is already open for this application; run ' + 'with `--open-dashboard` for another, or visit ' + f'{dashboard_url}{DASHBOARD_PATH}/' + ) + return + + # `webbrowser` honors `$BROWSER`, which is what makes this work in + # Codespaces and devcontainers, and returns `False` rather than + # raising when there is no browser to open. + page_url = f'{dashboard_url}{DASHBOARD_PATH}/' + + if not await asyncio.to_thread(webbrowser.open, page_url): + terminal.warn( + f"Could not open a browser; your dashboard is at {page_url}" + ) + + +async def _dashboard_reachable(port: int) -> bool: + """Whether something is accepting connections on the dashboard's + port.""" + try: + _, writer = await asyncio.wait_for( + asyncio.open_connection('127.0.0.1', port), + timeout=2.0, + ) + except (OSError, asyncio.TimeoutError): + return False + writer.close() + try: + await writer.wait_closed() + except OSError: + pass + return True + + +async def _open_dashboard( + *, + dashboard_url: str, + forced: bool, + application_serving_event: asyncio.Event, +) -> None: + """Opens a dashboard once the developer's application is serving + traffic, so that nothing the developer waits on is ever waiting on + this.""" + await application_serving_event.wait() + + try: + await _open_dashboard_once( + dashboard_url=dashboard_url, + forced=forced, + ) + except Exception as e: + # Never let this take down `rbt dev run`; the developer's + # application is unaffected and the dashboard is still + # reachable by hand. + terminal.warn( + f"Could not open a dashboard ({e}); it is at " + f"{dashboard_url}{DASHBOARD_PATH}/" + ) + + def try_and_become_child_subreaper_on_linux(): if sys.platform == 'linux': # The 'pyprctl' module is available on Linux only. @@ -542,6 +706,7 @@ async def _check_local_envoy_status( port: int, terminate_after_health_check: bool, application_started_event: asyncio.Event, + application_serving_event: Optional[asyncio.Event], tls_certificate: Optional[str], root_certificate: Optional[str], tracing: Tracing, @@ -650,6 +815,9 @@ def create_channel( was_application_serving = is_application_serving if is_application_serving: + if application_serving_event is not None: + application_serving_event.set() + terminal.info("Application is serving traffic ...\n") # MCP server and endpoint is not supported for Nodejs currently. mcp_line = ( @@ -1358,12 +1526,32 @@ async def __dev_run( await check_local_envoy_mode(subprocesses) env[ENVVAR_REBOOT_LOCAL_ENVOY] = 'true' + # The developer dashboard runs separately, started by + # `rbt dashboard`; this run only decides whether to open a window + # on it. + dashboard_port = args.dashboard_port or DEFAULT_DASHBOARD_PORT + + open_dashboard = ( + args.open_dashboard is True or + (_AUTO_OPEN_DASHBOARD and args.open_dashboard is not False) + ) + + if open_dashboard and not await _dashboard_reachable(dashboard_port): + terminal.fail( + 'You asked for a dashboard, but no developer dashboard is ' + f'serving on port {dashboard_port}. Start one with ' + '`rbt dashboard`, then run this again.' + ) + + application_serving_event = (asyncio.Event() if open_dashboard else None) + health_check_task = asyncio.create_task( _check_local_envoy_status( port=args.port or DEFAULT_LOCAL_ENVOY_PORT, terminate_after_health_check=args.terminate_after_health_check or False, application_started_event=application_started_event, + application_serving_event=application_serving_event, tls_certificate=args.tls_certificate, root_certificate=args.tls_root_certificate, tracing=tracing, @@ -1419,9 +1607,9 @@ def crypto_root_keys() -> str: protect, so that a JWT signed with them dies with the state it refers to (pushing clients back through the OAuth flow, whose fresh mint re-constructs per-user state): a named application - persists a random value in its state directory — stable across - restarts, deleted by `rbt dev expunge` and by the in-run `x` - expunge — while an anonymous application, whose state doesn't + persists a random value in its state directory, stable across + restarts and deleted by `rbt dev expunge` and by the in-run `x` + expunge, while an anonymous application, whose state doesn't survive a restart, gets fresh random keys on every (re)start. """ if args.application_name is None: @@ -1437,6 +1625,19 @@ def crypto_root_keys() -> str: root_keys_path.write_text(root_keys) return root_keys + if open_dashboard: + assert application_serving_event is not None + background_command_tasks.append( + asyncio.create_task( + _open_dashboard( + dashboard_url=f'http://127.0.0.1:{dashboard_port}', + forced=args.open_dashboard is True, + application_serving_event=application_serving_event, + ), + name=f'_open_dashboard(...) in {__name__}', + ) + ) + if tracing == Tracing.JAEGER: # TODO: dynamic port. See comment in `_run_jaeger()`. env[OTEL_EXPORTER_OTLP_TRACES_ENDPOINT] = "localhost:4317" diff --git a/reboot/cli/common/BUILD.bazel b/reboot/cli/common/BUILD.bazel index de53cec0..c15aae2b 100644 --- a/reboot/cli/common/BUILD.bazel +++ b/reboot/cli/common/BUILD.bazel @@ -145,6 +145,7 @@ py_library( deps = [ ":rc_py", ":update_check_py", + "//reboot/cli/commands:dashboard_py", "//reboot/cli/commands:dev_py", "//reboot/cli/commands:export_import_py", "//reboot/cli/commands:generate_py", diff --git a/reboot/cli/common/cli.py b/reboot/cli/common/cli.py index 5279e3b9..9d4dd7ab 100644 --- a/reboot/cli/common/cli.py +++ b/reboot/cli/common/cli.py @@ -7,6 +7,11 @@ handle_cloud_subcommand, register_cloud, ) +from reboot.cli.commands.dashboard import ( + dashboard_subcommands, + handle_dashboard_subcommand, + register_dashboard, +) from reboot.cli.commands.dev import ( dev_subcommands, handle_dev_subcommand, @@ -71,7 +76,7 @@ def create_parser( program='rbt', filename='.rbtrc', subcommands=( - cloud_subcommands() + dev_subcommands() + + cloud_subcommands() + dashboard_subcommands() + dev_subcommands() + export_and_import_subcommands() + generate_subcommands() + init_subcommands() + inspect_subcommands() + serve_subcommands() + task_subcommands() @@ -83,6 +88,7 @@ def create_parser( add_global_options(parser) register_cloud(parser) + register_dashboard(parser) register_dev(parser) register_export_and_import(parser) register_generate(parser) @@ -124,6 +130,10 @@ async def cli() -> int: if (result := await handle_cloud_subcommand(args)) is not None: return result + elif ( + result := await handle_dashboard_subcommand(args, parser=parser) + ) is not None: + return result elif ( result := await handle_dev_subcommand( args, diff --git a/reboot/dashboard/BUILD.bazel b/reboot/dashboard/BUILD.bazel new file mode 100644 index 00000000..4a7a1d0c --- /dev/null +++ b/reboot/dashboard/BUILD.bazel @@ -0,0 +1,120 @@ +load("@rbt_pypi//:requirements.bzl", "requirement") +load("@rules_python//python:defs.bzl", "py_binary", "py_library") + +py_library( + name = "constants_py", + srcs = ["constants.py"], + srcs_version = "PY3", + visibility = ["//visibility:public"], +) + +py_library( + name = "api_reader_py", + srcs = ["api_reader.py"], + srcs_version = "PY3", + visibility = ["//visibility:public"], + deps = [ + "//rbt/dashboard/v1:dashboard_py_reboot", + "//reboot:api_py", + ], +) + +py_library( + name = "api_watcher_py", + srcs = ["api_watcher.py"], + srcs_version = "PY3", + visibility = ["//visibility:public"], + deps = [ + ":api_reader_py", + ":constants_py", + "//rbt/dashboard/v1:dashboard_py_reboot", + "//reboot/aio:external_py", + "//reboot/cli/common:watch_py", + ], +) + +py_library( + name = "servicers_py", + srcs = ["servicers.py"], + srcs_version = "PY3", + visibility = ["//visibility:public"], + deps = [ + ":api_watcher_py", + ":constants_py", + "//rbt/dashboard/v1:dashboard_py_reboot", + "//reboot/aio:servicers_py", + "//reboot/std/presence/v1:presence_py", + ], +) + +# The page, its stylesheet and its bundle, served as three files by +# the `StaticFiles` mount in `main.py`. Only the script's entry point +# is rewritten: the one in the template, `./src/main.tsx`, is +# what Vite serves in development and does not exist in a built page, +# where `esbuild` produced `dashboard_bundle.js` instead. The +# stylesheet is copied as it is, because `./dashboard.css` resolves +# the same way in both. Every output must be *this* package's, because +# the application serves them from beside its own module rather than +# from a project root, and a `genrule` can only write within its own +# package. +genrule( + name = "dashboard_dist", + srcs = [ + "//reboot/dashboard/frontend:dashboard.css", + "//reboot/dashboard/frontend:dashboard_bundle.js", + "//reboot/dashboard/frontend:index.html", + ], + outs = [ + "dashboard/index.html", + "dashboard/dashboard.css", + "dashboard/dashboard_bundle.js", + ], + cmd = "sed 's|\\./src/main\\.tsx|./dashboard_bundle.js|' " + + "$(location //reboot/dashboard/frontend:index.html) " + + "> $(location dashboard/index.html) && " + + "cp $(location //reboot/dashboard/frontend:dashboard.css) " + + "$(location dashboard/dashboard.css) && " + + "cp $(location //reboot/dashboard/frontend:dashboard_bundle.js) " + + "$(location dashboard/dashboard_bundle.js)", +) + +py_library( + name = "main_py", + srcs = ["main.py"], + data = [ + ":dashboard_dist", + ], + srcs_version = "PY3", + visibility = ["//visibility:public"], + deps = [ + ":api_watcher_py", + ":constants_py", + ":servicers_py", + "//rbt/dashboard/v1:dashboard_py_reboot", + "//reboot/aio:applications_py", + "//reboot/aio:external_py", + "//reboot/aio:headers_py", + "//reboot/aio/backoff:python", + requirement("starlette"), + ], +) + +py_binary( + name = "main", + srcs = ["main.py"], + data = [ + ":dashboard_dist", + ], + main = "main.py", + visibility = ["//visibility:public"], + deps = [ + ":constants_py", + ":servicers_py", + "//rbt/dashboard/v1:dashboard_py_reboot", + "//reboot/aio:applications_py", + "//reboot/aio:external_py", + "//reboot/aio:headers_py", + "//reboot/aio/backoff:python", + requirement("starlette"), + ], +) diff --git a/reboot/dashboard/api_reader.py b/reboot/dashboard/api_reader.py new file mode 100644 index 00000000..dbf55816 --- /dev/null +++ b/reboot/dashboard/api_reader.py @@ -0,0 +1,202 @@ +"""Describes one of the developer's API files. + +Run as a subprocess: + + python -m reboot.dashboard.api_reader \ + + +and it writes a JSON list of `StateTypeInfo` to stdout, or a message +to stderr and a non-zero exit if the file cannot be read. + +A subprocess for two reasons. Reading a Pydantic API means importing +it, so doing it in the dashboard would accumulate stale modules across +edits. And it derives a module path from a relative filename, so it +needs a working directory and `sys.path` that the dashboard should not +adopt. + +The description comes from walking the imported `API` object itself, +so everything is spelled the way its author spelled it: method names +as `names_like_this`, field types as `int` or `Optional[str]`, and +errors by the names of the declared models. +""" +import asyncio +import importlib +import json +import os +import sys +import types +from google.protobuf.json_format import MessageToDict +from rbt.dashboard.v1.dashboard_pb2 import FieldInfo, MethodInfo, StateTypeInfo +from reboot.api import API, MethodModel, Model +from typing import Any, Literal, Optional, Union, get_args, get_origin + + +def _type_string(annotation) -> str: + """The source spelling of `annotation`, e.g. `Optional[str]`.""" + if annotation is type(None): + return 'None' + if annotation is Any: + return 'Any' + + origin = get_origin(annotation) + + if origin is Union or origin is types.UnionType: + arguments = get_args(annotation) + others = [a for a in arguments if a is not type(None)] + spelled = ', '.join(_type_string(a) for a in others) + if len(others) == len(arguments): + return f'Union[{spelled}]' + if len(others) == 1: + return f'Optional[{spelled}]' + return f'Optional[Union[{spelled}]]' + if origin is Literal: + return str(annotation).replace('typing.', '') + if origin is list: + (item,) = get_args(annotation) + return f'list[{_type_string(item)}]' + if origin is dict: + key, value = get_args(annotation) + return f'dict[{_type_string(key)}, {_type_string(value)}]' + if origin is None and isinstance(annotation, type): + return annotation.__name__ + return str(annotation).replace('typing.', '') + + +def _fields_of(model: type[Model]) -> list[FieldInfo]: + return [ + FieldInfo(name=name, type=_type_string(field.annotation)) + for name, field in model.model_fields.items() + ] + + +def _describe_method(method_name: str, spec: MethodModel) -> MethodInfo: + info = MethodInfo( + name=method_name, + kind=spec.kind.value, + factory=spec.factory, + mcp=spec.mcp is not None, + errors=[error.__name__ for error in spec.errors], + ) + + if spec.request is not None: + info.arguments.extend(_fields_of(spec.request)) + if spec.response is not None: + info.returns.extend(_fields_of(spec.response)) + if spec.description is not None: + info.description = spec.description + + return info + + +def describe(api_directory: str, filename: str) -> list[dict]: + """Describes the state types declared in one API file. + + State type names are qualified by the file's directory, the way + the generated code qualifies them: `shop/v1/shop.py` declaring + `Shop` yields `shop.v1.Shop`. + """ + # The path as the developer spelled it, joined before anything + # resolves it away: with `--api-directory=api` the file shows as + # `api/bank/v1/account.py`, the path they would open. + file = os.path.join(api_directory, filename) + + directory = os.path.abspath(api_directory) + os.chdir(directory) + sys.path.insert(0, directory) + + module = importlib.import_module( + filename.rsplit('.py', 1)[0].replace(os.sep, '.') + ) + + api = getattr(module, 'api', None) + if not isinstance(api, API): + # Not every file in the directory declares an API; one + # holding shared code simply has nothing to describe. + return [] + + package = os.path.dirname(filename).replace(os.sep, '.') + + described = [] + for type_name, type_obj in api.get_types().items(): + info = StateTypeInfo( + name=f'{package}.{type_name}', + file=file, + fields=_fields_of(type_obj.state), + ) + if type_obj.description is not None: + info.description = type_obj.description + + for method_name, spec in type_obj.methods.items(): + # A `UI` method has no RPC to call, so there is nothing + # to put in a method row for it. + if isinstance(spec, MethodModel): + info.methods.append(_describe_method(method_name, spec)) + + described.append(MessageToDict(info, preserving_proto_field_name=True)) + + return described + + +async def read( + api_directory: str, + filename: str, +) -> tuple[list[dict], Optional[str]]: + """Describes one API file in a subprocess. + + Returns the state types it declares, and a message when it could + not be read. A half-written file is the normal case while someone + is typing, and is worth showing rather than hiding. + """ + process = await asyncio.create_subprocess_exec( + sys.executable, + '-m', + # Not `__name__`, which is `__main__` when this module is the + # one being run. + 'reboot.dashboard.api_reader', + api_directory, + filename, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + # Reading an API file imports it, and an import writes + # `__pycache__` beside the source, inside the directory + # being watched, so the write is itself a change, and every + # edit costs a second pass over every file. It also leaves + # bytecode in the developer's tree that nothing else put + # there. + env={ + **os.environ, 'PYTHONDONTWRITEBYTECODE': '1' + }, + ) + out, errors = await process.communicate() + + if process.returncode != 0: + return [], errors.decode().strip() + + try: + return json.loads(out), None + except json.JSONDecodeError as e: + return [], f'Could not read the description: {e}' + + +def main() -> int: + if len(sys.argv) != 3: + print(f'usage: {sys.argv[0]} ', file=sys.stderr) + return 2 + + try: + print(json.dumps(describe(sys.argv[1], sys.argv[2]))) + except SystemExit: + # A malformed API can reach `fail()` inside `reboot.api`, + # which raises this after printing why. Being a subprocess, + # that is a message for the dashboard rather than the end of + # it. + return 1 + except Exception as e: + print(f'{type(e).__name__}: {e}', file=sys.stderr) + return 1 + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/reboot/dashboard/api_watcher.py b/reboot/dashboard/api_watcher.py new file mode 100644 index 00000000..7646ce2a --- /dev/null +++ b/reboot/dashboard/api_watcher.py @@ -0,0 +1,190 @@ +"""Watches the developer's API files and updates what they declare. + +The dashboard may start before the application exists. In an agentic +flow the API files are written first, then generated code, then +servicers, then a build, then a running process, so asking the +application would say nothing for minutes. Reading the files says +something immediately, and says more with each file that lands. + +Per file, so that state types appear as they are written rather than +all at once at the end, and so that one file which does not parse, +the normal case while someone is typing, costs only its own types. +""" +from google.protobuf.json_format import ParseDict +from log.log import get_logger +from pathlib import Path +from rbt.dashboard.v1.dashboard_pb2 import StateTypeInfo +from rbt.dashboard.v1.dashboard_rbt import API +from reboot.aio.contexts import WorkflowContext +from reboot.cli.common.watch import file_watcher +from reboot.dashboard.api_reader import read +from reboot.dashboard.constants import API_ID +from typing import Optional +from watchdog.events import FileSystemEvent + +logger = get_logger(__name__) + +# Only Pydantic APIs can be read so far. `.proto` and `.ts` are the +# other two forms `rbt generate` accepts; both are static parses and +# neither is written yet. +API_GLOB = '**/*.py' + +# Suffixes of the files `rbt generate` writes, which it therefore +# skips on the way back in. The same three appear in +# `reboot/cli/commands/generate.py`, which decides what to generate +# from, and in `reboot/cli/commands/dev.py`, which decides what to +# watch. Keep the three lists in step. +GENERATED_SUFFIXES = ('_rbt.py', '_pb2.py', '_pb2_grpc.py') + + +def _files(api_directory: Path) -> list[str]: + """Every candidate API file, relative to `api_directory`. + + Every `.py` the developer wrote, which is the rule `rbt generate` + uses. Whether one of them declares an API is answered by reading + it: an API is a Python object, built when the module executes, so + no amount of looking at the text settles it. A file that declares + none costs one subprocess and describes nothing, and `rbt dev run` + is already importing all of these on every save to regenerate. + """ + return sorted( + str(path.relative_to(api_directory)) + for path in api_directory.glob(API_GLOB) + if not path.name.endswith(GENERATED_SUFFIXES) + ) + + +class _Descriptions: + """What each file last declared, and what went wrong reading it. + + Keyed by file so that a file which stops parsing keeps the types + it last had: blanking the dashboard on every keystroke would make + it unreadable exactly while it is being used. + """ + + def __init__(self) -> None: + self._state_types: dict[str, list[dict]] = {} + self._errors: dict[str, str] = {} + + def update( + self, + filename: str, + state_types: list[dict], + error: Optional[str], + ) -> None: + if error is None: + self._state_types[filename] = state_types + self._errors.pop(filename, None) + else: + self._errors[filename] = error + + def retain(self, filenames: set[str]) -> None: + """Forgets files that are no longer there.""" + for stored in list(self._state_types): + if stored not in filenames: + del self._state_types[stored] + for stored in list(self._errors): + if stored not in filenames: + del self._errors[stored] + + def state_types(self) -> list[StateTypeInfo]: + described = [] + for filename in sorted(self._state_types): + for state_type in self._state_types[filename]: + described.append(ParseDict(state_type, StateTypeInfo())) + return described + + def error(self) -> str: + return '\n'.join( + f'{filename}: {self._errors[filename]}' + for filename in sorted(self._errors) + ) + + +def _event_filenames(event: FileSystemEvent, directory: Path) -> set[str]: + """The filenames an event names, relative to `directory`. + + Both of its paths, because a rename reports where the file went as + well as where it was. A path that is not under the directory is + left out, and an event that names nothing under it is the caller's + signal that it could not place the event at all. + """ + filenames = set() + for path in (event.src_path, event.dest_path): + if not path: + continue + try: + filenames.add(str(Path(path).relative_to(directory))) + except ValueError: + continue + return filenames + + +async def watch(context: WorkflowContext, *, api_directory: str) -> None: + """Updates what the API files declare, for as long as this runs.""" + directory = Path(api_directory).resolve() + descriptions = _Descriptions() + updated: Optional[tuple] = None + + async def update_if_changed(alias: str) -> None: + nonlocal updated + + current = (descriptions.state_types(), descriptions.error()) + if current != updated: + updated = current + # Every write from a workflow needs an identity, and this + # one writes once per file that changed, on every + # iteration. + await API.ref(API_ID).per_iteration(alias).Update( + context, + state_types=descriptions.state_types(), + error=descriptions.error(), + ) + + # Everything, once: the developer may have written the whole API + # before the dashboard started. After this only what changes is + # read again. + previous_listing = set(_files(directory)) + pending = set(previous_listing) + + with file_watcher() as watcher: + async for iteration in context.loop('read what changed'): + # The watch is armed before anything is read, so a save + # made during a read is not missed: it resolves `event` + # rather than arriving while nothing is listening. A watch + # is consumed by one event, so it is re-entered for each, + # the same shape `rbt dev run` uses. + async with watcher.watch( + [API_GLOB], + root_dir=str(directory), + ) as event: + # Updating after each file rather than after the + # batch is what makes the types appear as they are + # written. + for filename in sorted(pending): + state_types, error = await read(api_directory, filename) + descriptions.update(filename, state_types, error) + await update_if_changed(f'read {filename}') + + changed = await event + + # A listing is a glob and no file reads, so it is taken on + # every change: it is what notices a file added or deleted, + # which an event naming one path cannot. + filenames = set(_files(directory)) + event_filenames = _event_filenames(changed, directory) + pending = ( + (filenames - previous_listing) | (event_filenames & filenames) + ) + previous_listing = filenames + + if not event_filenames: + # The glob only matches `.py` under this directory, so + # an event that names nothing under it means its paths + # did not resolve the way this one did. Read every file + # rather than let the page go quietly stale on a + # mismatch this cannot see. + pending = filenames + + descriptions.retain(filenames) + await update_if_changed('retain') diff --git a/reboot/dashboard/constants.py b/reboot/dashboard/constants.py new file mode 100644 index 00000000..5a16d3b9 --- /dev/null +++ b/reboot/dashboard/constants.py @@ -0,0 +1,46 @@ +"""Values shared between the dashboard application and the CLI. + +Kept apart from `main.py` so that reading them does not drag in the +application and everything it serves with. +""" + +# Where the dashboard application serves its page, relative to its +# own address. +DASHBOARD_PATH = '/dashboard' + +# The dashboard application's port. Deliberately not adjacent to +# `rbt dev run`'s default port of 9991: VS Code forwards a port +# upward when the one it wants is already taken on the developer's +# machine, so a second dev container serving on 9991 arrives on +# 9992. A dashboard sitting there could be reached in place of +# somebody else's backend, which half-works and is far more +# confusing than not working at all. 9871 is below 9991 so upward +# forwarding never reaches it, outside the 999x band (9990 k3d and +# WildFly, 9993 ZeroTier, 9997 Splunk), clear of +# 9000/9090/9200/9222/9229, hard to confuse with 9991 when reading +# logs, and outside the Linux ephemeral range. +DEFAULT_DASHBOARD_PORT = 9871 + +# The `API` state holding the shape the developer's API files +# declare, as the dashboard application last read them. +API_ID = 'api' + +# The `Preferences` state holding what the developer has said about +# opening dashboards: the dashboard's banner writes it and `rbt dev +# run` reads it. +PREFERENCES_ID = 'preferences' + +# The directory the developer's API files are in, which +# `rbt dashboard` takes as `--api-directory`. Separate from the +# application's URL because the files are there long before anything +# is serving, and the dashboard is meant to be startable that early. +ENVVAR_RBT_API_DIRECTORY = 'RBT_API_DIRECTORY' + +# The `Presence` state the dashboard page subscribes to, recording who +# is looking at a dashboard right now. `rbt dev run` reads it to decide +# whether to open one. +# +# The page names this, `API_ID` and `PREFERENCES_ID` independently, +# in `frontend/src/constants.ts`, since TypeScript cannot read them +# from here. Keep the two in step. +PRESENCE_ID = 'dashboard' diff --git a/reboot/dashboard/frontend/BUILD.bazel b/reboot/dashboard/frontend/BUILD.bazel new file mode 100644 index 00000000..6e5b62ed --- /dev/null +++ b/reboot/dashboard/frontend/BUILD.bazel @@ -0,0 +1,51 @@ +load("@aspect_rules_esbuild//esbuild:defs.bzl", "esbuild") +load("@aspect_rules_ts//ts:defs.bzl", "ts_config", "ts_project") + +ts_config( + name = "tsconfig", + src = "tsconfig.json", +) + +ts_project( + name = "dashboard_ts", + srcs = [ + "src/constants.ts", + "src/main.tsx", + ], + tsconfig = ":tsconfig", + deps = [ + "//:node_modules/@reboot-dev/reboot-react", + "//:node_modules/@reboot-dev/reboot-std", + "//:node_modules/@reboot-dev/reboot-std-api", + "//:node_modules/@reboot-dev/reboot-std-react", + "//:node_modules/@reboot-dev/reboot-web", + "//:node_modules/react", + "//:node_modules/react-dom", + "//:node_modules/uuid", + "//rbt/dashboard/v1:dashboard_js_reboot_react", + ], +) + +# The `tsconfig` is passed so the bundle uses the automatic JSX +# runtime; without it the bundle crashes with `React is not defined`. +esbuild( + name = "dashboard_bundle", + srcs = [ + ":dashboard_ts", + ], + bazel_sandbox_plugin = False, + entry_point = "src/main.js", + format = "esm", + output = "dashboard_bundle.js", + platform = "browser", + tsconfig = "tsconfig.json", + visibility = ["//reboot/dashboard:__pkg__"], +) + +exports_files( + [ + "dashboard.css", + "index.html", + ], + visibility = ["//reboot/dashboard:__pkg__"], +) diff --git a/reboot/dashboard/frontend/dashboard.css b/reboot/dashboard/frontend/dashboard.css new file mode 100644 index 00000000..f6dfb208 --- /dev/null +++ b/reboot/dashboard/frontend/dashboard.css @@ -0,0 +1,681 @@ +/* Tokens from the Reboot Cloud design system the mockup imports + (`_ds/reboot-cloud-design-system-.../_ds_bundle.css`), as HSL + triplets so they compose with `hsl(var(--token) / alpha)` the + way the source does. Light only: the system has `.dark:` + utilities but the design never opts in. */ +:root { + --background: 47 36% 95%; + --foreground: 211.1 71.7% 22.2%; + --card: 0 0% 100%; + --primary: 211 72% 22%; + --primary-foreground: 355.7 100% 97.3%; + --secondary: 115 49% 76%; + --muted: 240 4.8% 95.9%; + --muted-foreground: 240 3.8% 46.1%; + --accent: 166 47% 61%; + --destructive: 0 84.2% 60.2%; + --border: 240 5.9% 90%; + --radius: 0.5rem; + + /* Values the design uses directly, alongside the tokens. */ + --sidebar: 47 30% 92%; + --border-strong: 240 5.9% 86%; + --border-soft: 240 5.9% 94%; + --surface-sunken: 240 4.8% 97%; + --prose: 211 40% 30%; + --returns: 166 47% 33%; + --errors: 0 62% 45%; +} + +* { + box-sizing: border-box; +} + +html, +body, +#root { + height: 100%; + margin: 0; +} + +body { + font-family: ui-sans-serif, system-ui, sans-serif; + background: hsl(var(--background)); + color: hsl(var(--foreground)); + font-size: 15px; + line-height: 1.5; + -webkit-font-smoothing: antialiased; +} + +a { + color: hsl(var(--primary)); + text-decoration: none; +} +a:hover { + color: hsl(166 47% 41%); +} +::selection { + background: hsl(var(--accent) / 0.35); +} + +main { + max-width: 840px; + margin: 0 auto; + padding: 40px 48px; +} + +.muted { + color: hsl(var(--muted-foreground)); +} + +.error { + color: hsl(var(--errors)); + background: hsl(var(--destructive) / 0.08); + border: 1px solid hsl(var(--destructive) / 0.3); + border-radius: var(--radius); + padding: 12px 14px; + /* The reader subprocess's stderr, so line breaks are meaningful. */ + white-space: pre-wrap; + font-family: ui-monospace, Menlo, monospace; + font-size: 12px; +} + +/* The banner sits above whatever the page is showing, and the + rest of the page takes what is left. `min-height: 0` so the + sidebar and the document pane scroll themselves instead of + growing the page. */ +.app { + display: flex; + flex-direction: column; + height: 100%; +} + +.shell { + display: grid; + grid-template-columns: 250px 1fr; + flex: 1; + min-height: 0; + overflow: hidden; +} + +/* --- Banner --- */ + +.banner { + display: flex; + justify-content: center; + padding: 5px 20px; + background: hsl(var(--card)); + border-bottom: 1px solid hsl(var(--border)); +} + +/* A button, because it acts rather than navigates, dressed as a + link so it asks for as little of the page as it is worth. */ +.banner-link { + padding: 0; + border: none; + background: none; + font: inherit; + font-size: 11.5px; + color: hsl(var(--muted-foreground)); + text-decoration: underline; + text-underline-offset: 2px; + cursor: pointer; +} + +.banner-link:hover { + color: hsl(var(--foreground)); +} + +/* --- Sidebar --- */ + +nav { + background: hsl(var(--sidebar)); + border-right: 1px solid hsl(var(--border)); + overflow-y: auto; + padding: 14px 8px; +} + +.eyebrow { + font-family: ui-monospace, Menlo, monospace; + font-size: 10.5px; + letter-spacing: 0.08em; + text-transform: uppercase; + color: hsl(var(--muted-foreground)); +} + +nav > .eyebrow { + padding: 0 8px 8px; +} + +.namespace-head { + display: grid; + grid-template-columns: 10px 1fr auto; + align-items: center; + gap: 0 7px; + width: 100%; + padding: 6px 10px; + border: none; + border-radius: 5px; + background: transparent; + color: inherit; + font: inherit; + text-align: left; + cursor: pointer; +} + +.namespace-head:hover { + background: hsl(var(--card) / 0.6); +} + +.caret { + font-family: ui-monospace, Menlo, monospace; + font-size: 10px; + color: hsl(240 3.8% 55%); +} + +.namespace-name { + font-family: ui-monospace, Menlo, monospace; + font-size: 11.5px; + font-weight: 600; +} + +nav a { + display: grid; + grid-template-columns: 1fr auto; + align-items: center; + gap: 0 8px; + padding: 5px 10px 5px 34px; + border-radius: 5px; + color: inherit; +} + +nav a:hover { + background: hsl(var(--card) / 0.6); + color: inherit; +} + +.nav-name { + font-weight: 600; + font-size: 12.5px; +} + +/* The counts name what they count, since a namespace's number and a + state type's number sit in the same column and mean different + things. `nowrap` because the sidebar is narrow enough that "12 + state types" would otherwise break across two lines. */ +.nav-count { + font-family: ui-monospace, Menlo, monospace; + font-size: 10px; + color: hsl(var(--muted-foreground)); + white-space: nowrap; +} + +/* --- Document pane --- */ + +.pane { + overflow-y: auto; + background: hsl(var(--background)); +} + +header { + max-width: 840px; + margin: 0 auto; + padding: 40px 48px 28px; +} + +header h1 { + font-size: 34px; + font-weight: 650; + letter-spacing: -0.02em; + margin: 6px 0 0; +} + +.state-type { + max-width: 840px; + margin: 0 auto; + padding: 0 48px 48px; +} + +.state-type > .eyebrow:first-child { + display: block; +} + +.state-type { + border-top: 1px solid hsl(var(--border-strong)); + padding-top: 36px; +} + +.state-type h2 { + font-size: 28px; + font-weight: 650; + letter-spacing: -0.02em; + margin: 6px 0 4px; +} + +.file { + font-family: ui-monospace, Menlo, monospace; + font-size: 11.5px; + color: hsl(var(--muted-foreground)); +} + +.eyebrow.section { + margin: 26px 0 10px; +} + +/* --- Expanding a state type --- */ + +/* The heading and its count on the left, the button pushed to + the right edge. Aligned on the baseline rather than centred, + so the button's label sits on the same line as the 28px + heading beside it. When the row is too narrow to hold both, + the button wraps to its own line and `space-between` has + nothing left to spread. */ +.state-type-head { + display: flex; + align-items: baseline; + justify-content: space-between; + gap: 8px 20px; + flex-wrap: wrap; +} + +.state-type-heading { + display: flex; + align-items: baseline; + gap: 12px; + flex-wrap: wrap; + min-width: 0; +} + +/* Solid while closed and outlined while open, so the one that + has something left to offer is the one that draws the eye. */ +.expand-button { + display: inline-flex; + align-items: center; + flex: none; + gap: 8px; + padding: 9px 18px; + border: 1px solid hsl(var(--primary)); + border-radius: calc(var(--radius) - 2px); + background: hsl(var(--primary)); + color: hsl(var(--primary-foreground)); + box-shadow: 0 1px 2px hsl(240 10% 40% / 0.2); + font: inherit; + font-size: 13px; + font-weight: 600; + cursor: pointer; + user-select: none; +} + +.expand-button:hover { + opacity: 0.88; +} + +.expand-button:active { + transform: translateY(1px); +} + +.state-type.is-expanded .expand-button { + border-color: hsl(240 5.9% 78%); + background: hsl(var(--muted)); + color: hsl(var(--primary)); + box-shadow: none; +} + +.expand-button .caret { + font-family: ui-monospace, Menlo, monospace; + font-size: 11px; +} + +.summary-line { + font-family: ui-monospace, Menlo, monospace; + font-size: 11px; + color: hsl(240 3.8% 55%); +} + +.empty { + border: 1px dashed hsl(var(--border-strong)); + border-radius: calc(var(--radius) - 2px); + padding: 12px 14px; + font-size: 12.5px; + color: hsl(var(--muted-foreground)); + line-height: 1.5; +} + +.fields { + display: flex; + flex-direction: column; + gap: 8px; +} + +.field { + display: flex; + align-items: baseline; + gap: 10px; + border: 1px solid hsl(240 5.9% 92%); + border-radius: calc(var(--radius) - 2px); + padding: 10px 14px; + background: hsl(var(--card) / 0.6); +} + +.field-name { + font-family: ui-monospace, Menlo, monospace; + font-size: 12.5px; + font-weight: 600; +} + +.field-type { + font-family: ui-monospace, Menlo, monospace; + font-size: 11.5px; + color: hsl(var(--muted-foreground)); +} + +.methods { + display: flex; + flex-direction: column; + gap: 14px; +} + +.method { + border: 1px solid hsl(var(--border)); + border-radius: var(--radius); + background: hsl(var(--card)); + overflow: hidden; +} + +.method-head { + padding: 14px 18px 12px; +} + +.method-title { + display: flex; + align-items: center; + gap: 8px; + flex-wrap: wrap; +} + +/* Closed, the methods of one state type share a single set of + columns, so every kind pill starts at the same x and so does every + tag: the eye can run down the list and see which methods write and + which are reachable over MCP without reading any of them. Open, + this all reverts to the flex row above, where each row packs its + pills against its own name and the reader is looking at one method + rather than comparing several. + + `subgrid` passes the columns down from `.methods` through the card + and its head, so the tracks are sized once across every row rather + than per row. Each element is pinned to a column by name, because + `factory` and `mcp` are optional: auto-placement would slide a + method's `mcp` tag into the empty `factory` column and lose the + alignment that is the whole point. + + Behind `@supports` because a browser without subgrid drops that one + declaration and keeps the rest, which would leave the cards as + plain grids whose columns are sized per row: worse than the flex + row it replaced. Chrome has had subgrid since 117; anything older + keeps the flex row and simply does not line up. */ +@supports (grid-template-columns: subgrid) { + .state-type:not(.is-expanded) .methods { + display: grid; + grid-template-columns: max-content max-content 1fr; + /* The 14px is the gap between methods; the columns want the 8px + that `.method-title` uses when it is laid out as flex. */ + row-gap: 14px; + column-gap: 8px; + } + + .state-type:not(.is-expanded) .method, + .state-type:not(.is-expanded) .method-head, + .state-type:not(.is-expanded) .method-title { + display: grid; + grid-column: 1 / -1; + grid-template-columns: subgrid; + align-items: center; + } + + /* A scroll container cannot be a subgrid, so the columns would stop + propagating at the card. `.method` is one only because of the + `overflow: hidden` that clips the detail to the card's rounded + corners, and a closed method has no detail drawn to clip. */ + .state-type:not(.is-expanded) .method { + overflow: visible; + } + + .state-type:not(.is-expanded) .method-detail { + grid-column: 1 / -1; + } + + .state-type:not(.is-expanded) .method-name { + grid-column: 1; + } + + .state-type:not(.is-expanded) .kind { + grid-column: 2; + } + + .state-type:not(.is-expanded) .tags { + grid-column: 3; + } +} + +/* Whatever tags a method has, side by side. Taken out of the + layout when it has none, so an open section has no gap to an + empty box in its title's flex row. */ +.tags { + display: flex; + align-items: center; + gap: 8px; +} + +.tags:empty { + display: none; +} + +/* Two families of pill. A `kind` is what the method does to + state, tinted along a scale from a read that changes nothing + to a workflow that runs over time; every method has exactly + one. A `tag` is something else that is true of the method, + and is solid rather than tinted so that it never reads as a + fifth kind. Both have a 1px border so the two families sit + at the same height beside each other. + + The neutral base below is what a `saga`, or a method whose kind + is unset, gets, since only the four kinds that exist today have a + colour. */ +/* Both pills are one width with the label centred in it, so a + column of them reads as a column rather than as ragged text. + `min-width` rather than `width`: a label longer than any we + have today widens its own pill instead of being clipped. 92px + fits the longest, `transaction`. */ +.kind, +.tag { + display: inline-flex; + align-items: center; + justify-content: center; + min-width: 92px; +} + +.kind { + font-family: ui-monospace, Menlo, monospace; + font-size: 10px; + padding: 2px 8px; + border-radius: 999px; + background: hsl(var(--muted)); + border: 1px solid hsl(240 5.9% 84%); + color: hsl(var(--muted-foreground)); +} + +/* Teal: reads state and changes nothing. */ +.kind-reader { + background: hsl(166 47% 61% / 0.18); + border-color: hsl(166 40% 55% / 0.45); + color: hsl(166 55% 27%); +} + +/* Blue: writes the one state it is called on. */ +.kind-writer { + background: hsl(211 72% 45% / 0.14); + border-color: hsl(211 60% 50% / 0.4); + color: hsl(211 72% 32%); +} + +/* Violet: writes across states, atomically. */ +.kind-transaction { + background: hsl(275 55% 55% / 0.14); + border-color: hsl(275 45% 55% / 0.4); + color: hsl(275 50% 40%); +} + +/* Amber: runs past the call that started it. */ +.kind-workflow { + background: hsl(32 90% 55% / 0.18); + border-color: hsl(32 75% 50% / 0.42); + color: hsl(28 80% 33%); +} + +.tag { + font-family: ui-monospace, Menlo, monospace; + font-size: 9px; + font-weight: 600; + padding: 2px 7px; + border: 1px solid transparent; + border-radius: 999px; + color: hsl(0 0% 100%); +} + +/* Navy: brings the state into existence. */ +.tag-factory { + background: hsl(211 72% 26%); +} + +/* Magenta: reachable by an agent over MCP. */ +.tag-mcp { + background: hsl(318 55% 38%); +} + +/* A pill that carries a definition. The mark is the invitation to + hover; the tooltip is the answer. `cursor: help` says the same + thing the mark does, for whoever reads cursors first. */ +.defined { + position: relative; + cursor: help; +} + +/* Small and translucent so it reads as an aside rather than as part + of the label. */ +.define-mark { + margin-left: 4px; + font-size: 8px; + opacity: 0.55; +} + +/* Above the pill, centred, and inert to the mouse so that moving + toward it never flickers it away. The pill styles it sits in are + undone piece by piece: pills are bold, tight and centred, and a + sentence is none of those. */ +.definition { + position: absolute; + bottom: calc(100% + 8px); + left: 50%; + transform: translateX(-50%); + width: max-content; + max-width: 260px; + padding: 8px 10px; + border-radius: 6px; + background: hsl(var(--primary)); + color: hsl(var(--primary-foreground)); + font-family: ui-sans-serif, system-ui, sans-serif; + font-size: 11.5px; + font-weight: 400; + line-height: 1.45; + letter-spacing: normal; + text-align: left; + text-transform: none; + white-space: normal; + pointer-events: none; + opacity: 0; + visibility: hidden; + transition: opacity 120ms ease; + z-index: 10; +} + +.defined:hover .definition { + opacity: 1; + visibility: visible; +} + +/* The section eyebrow sits at the pane's left edge, and the pane + clips whatever leaves it, so a centred tooltip loses its left + half. Open rightward from the label instead. */ +.eyebrow .definition { + left: 0; + transform: none; +} + +/* The height animation. A grid row interpolates from `0fr` to + `1fr`, which is how the detail can grow to whatever height it + happens to need without anything measuring it first, since there + is no height to read while it is closed. The inner element + owns `overflow: hidden`, because a grid row can only shrink + below its content when the content itself is willing to be + clipped. + + Opacity is on a shorter, later curve than the height so that + text fades in against a box that has already begun to open, + rather than appearing at full strength in a 1px slot. */ +.method-detail { + display: grid; + grid-template-rows: 0fr; + opacity: 0; + transition: grid-template-rows 240ms cubic-bezier(0.32, 0.72, 0, 1), + opacity 120ms ease-out; +} + +.method-detail-inner { + overflow: hidden; +} + +.state-type.is-expanded .method-detail { + grid-template-rows: 1fr; + opacity: 1; + transition: grid-template-rows 240ms cubic-bezier(0.32, 0.72, 0, 1), + opacity 180ms ease-in 60ms; +} + +@media (prefers-reduced-motion: reduce) { + .method-detail, + .state-type.is-expanded .method-detail { + transition-duration: 1ms; + } +} + +.method-description { + font-size: 13px; + line-height: 1.55; + color: hsl(var(--prose)); + margin: 0; + padding: 0 18px 12px; + text-wrap: pretty; +} + +.method-signature { + display: flex; + align-items: baseline; + gap: 12px; + flex-wrap: wrap; + padding: 10px 18px; + background: hsl(var(--surface-sunken)); + border-top: 1px solid hsl(var(--border-soft)); + font-family: ui-monospace, Menlo, monospace; + font-size: 11.5px; + color: hsl(var(--muted-foreground)); +} + +.arrow { + color: hsl(240 3.8% 65%); +} +.returns { + color: hsl(var(--returns)); +} +.errors { + color: hsl(var(--errors)); +} diff --git a/reboot/dashboard/frontend/index.html b/reboot/dashboard/frontend/index.html new file mode 100644 index 00000000..cb6c81d1 --- /dev/null +++ b/reboot/dashboard/frontend/index.html @@ -0,0 +1,18 @@ + + + + + + + Reboot dashboard + + + +
+ + + diff --git a/reboot/dashboard/frontend/src/constants.ts b/reboot/dashboard/frontend/src/constants.ts new file mode 100644 index 00000000..c8f515b8 --- /dev/null +++ b/reboot/dashboard/frontend/src/constants.ts @@ -0,0 +1,5 @@ +// Mirrors `reboot/dashboard/constants.py`, which +// TypeScript cannot read. Keep the two in step. +export const PRESENCE_ID = "dashboard"; +export const API_ID = "api"; +export const PREFERENCES_ID = "preferences"; diff --git a/reboot/dashboard/frontend/src/main.tsx b/reboot/dashboard/frontend/src/main.tsx new file mode 100644 index 00000000..b6fa0ecf --- /dev/null +++ b/reboot/dashboard/frontend/src/main.tsx @@ -0,0 +1,527 @@ +import type { MethodInfo, StateTypeInfo } from "@dashboard/dashboard_pb"; +import { useAPI, usePreferences } from "@dashboard/dashboard_rbt_react"; +import { RebootClientProvider } from "@reboot-dev/reboot-react"; +import { Presence } from "@reboot-dev/reboot-std-react/presence"; +import { + FC, + StrictMode, + useCallback, + useEffect, + useLayoutEffect, + useMemo, + useRef, + useState, +} from "react"; +import { createRoot } from "react-dom/client"; +import { v4 as uuidv4 } from "uuid"; +import { API_ID, PREFERENCES_ID, PRESENCE_ID } from "./constants"; + +// One subscriber per tab, for as long as the tab is open. +const SUBSCRIBER_ID = uuidv4(); + +// What each pill means, for somebody meeting Reboot for the first +// time. A pill whose word is not here, such as a kind this page +// does not know, simply gets no mark and no tooltip. +const DEFINITIONS: Record = { + reader: + "Reads state without changing it, so any number can safely " + + "execute concurrently. A reactive caller keeps receiving fresh " + + "results as the state changes.", + writer: + "Changes this state. Writers on one state run one at a time, " + + "each seeing the result of the one before it.", + transaction: + "Changes state, and can call methods on other states with " + + "every change landing together or none of them landing at all.", + workflow: + "A durable background task. It can loop and wait for as long " + + "as it needs, and after a restart it resumes where it was.", + factory: + "Brings a state into existence: it is called with a new id " + + "rather than on a state that already exists.", + mcp: "Callable by AI agents as a tool, over the Model Context " + "Protocol.", + "state type": + "A durable data type. Each instance, named by an id, has fields " + + "that Reboot persists for you. Methods are the way to read and " + + "change them. You can have as many of these as you want.", +}; + +// A pill, with its definition a hover away when it has one. The +// small mark is what says there is something to hover. +const Pill: FC<{ className: string; label: string; meaning?: string }> = ({ + className, + label, + meaning, +}) => + meaning === undefined ? ( + {label} + ) : ( + + {label} + + + {meaning} + + + ); + +const Kind: FC<{ kind: string }> = ({ kind }) => ( + +); + +// A state type's namespace is its proto package: `bank.v1.Account` +// lives in `bank.v1`, which is the developer's `api/bank/v1/`. +const namespaceOf = (name: string): string => + name.slice(0, name.lastIndexOf(".")); + +const typeNameOf = (name: string): string => + name.slice(name.lastIndexOf(".") + 1); + +// Standard-library types an application uses are real and worth being +// able to inspect, but they aren't what the developer wrote, so they +// start collapsed. +const isStandardLibrary = (namespace: string): boolean => + namespace.startsWith("rbt."); + +const Namespace: FC<{ namespace: string; types: StateTypeInfo[] }> = ({ + namespace, + types, +}) => { + const [open, setOpen] = useState(!isStandardLibrary(namespace)); + + return ( +
+ + {open && ( + + )} +
+ ); +}; + +const Method: FC<{ method: MethodInfo }> = ({ method }) => { + const args = method.arguments + .map((argument) => `${argument.name}: ${argument.type}`) + .join(", "); + + // The response's keys and value types, spelled the way a Python + // reader would write them. + const returns = method.returns + .map((field) => `${field.name}: ${field.type}`) + .join(", "); + + return ( +
+
+
+ {method.name} + {/* The kind first, and always: every method has one, so it + lands in the same place in every row and the eye can run + down the column. The tags after it are the exceptions. */} + + {/* One cell for whichever tags a method has, rather than a + column each: both are optional, so a column each would + make every method with neither hold that width open + as dead space. A method that is both a factory and + an MCP tool draws both, side by side. */} + + {method.factory && ( + + )} + {method.mcp && ( + + )} + +
+
+ {/* What the method's own row grows to show. Kept mounted while + the section is closed, because the animation that opens it + is a CSS transition on this element rather than a mount. */} +
+
+ {method.description !== undefined && ( +

{method.description}

+ )} +
+ + ({args}) →{" "} + + {method.returns.length > 0 ? `{${returns}}` : "None"} + + + {method.errors.length > 0 && ( + raises {method.errors.join(", ")} + )} +
+
+
+
+ ); +}; + +const countOf = (n: number, noun: string): string => + `${n} ${n === 1 ? noun : `${noun}s`}`; + +// Horizontal only, and deliberately. A pill's sideways move, between +// the column it shares while closed and its own row while open, is a +// layout change that CSS cannot transition, so it is animated here: +// measure where each pill was, let the layout happen, animate it from +// there. Its vertical move is not ours to animate. The detail growing +// is what pushes the methods below it down, and that already animates +// over the same 240ms, so translating them as well would move them +// twice and they would appear to fly in from above or below. +// +// `offsetLeft` rather than `getBoundingClientRect()` because it is a +// layout position and ignores transforms: a render that lands while a +// pill is mid-slide reads where it is going rather than where it +// momentarily is, so the next toggle starts from the truth. +const SLIDE_MS = 240; +const SLIDE_EASING = "cubic-bezier(0.32, 0.72, 0, 1)"; + +const useSlidingPills = (expanded: boolean) => { + const section = useRef(null); + const before = useRef(new WeakMap()); + const wasExpanded = useRef(expanded); + + // No dependency list: every render re-measures, so the positions + // this animates from are the ones on screen rather than the ones + // from the last toggle, which a window resize would have moved. + useLayoutEffect(() => { + const pills = section.current?.querySelectorAll(".kind, .tag"); + if (pills === undefined) { + return; + } + + // Only opening or closing moves them; other renders just leave + // fresh measurements behind for the next one that does. + const toggled = wasExpanded.current !== expanded; + wasExpanded.current = expanded; + + const still = window.matchMedia("(prefers-reduced-motion: reduce)").matches; + + // `forEach` rather than `for...of`: a `NodeList`'s iterator is + // typed as `Node`, which has no box to measure, while its + // `forEach` keeps the element type the selector asked for. + pills.forEach((pill) => { + const was = before.current.get(pill); + const now = pill.offsetLeft; + before.current.set(pill, now); + + if (!toggled || still || was === undefined || was === now) { + return; + } + + pill.animate( + [{ transform: `translateX(${was - now}px)` }, { transform: "none" }], + { duration: SLIDE_MS, easing: SLIDE_EASING } + ); + }); + }); + + return section; +}; + +const StateType: FC<{ + stateType: StateTypeInfo; + expanded: boolean; + onToggle: () => void; +}> = ({ stateType, expanded, onToggle }) => { + const section = useSlidingPills(expanded); + + return ( + // Every method's detail opens and closes off this one class, so a + // section is one transition rather than one per method. +
+
+ +
+
+
+

{typeNameOf(stateType.name)}

+ + {countOf(stateType.fields.length, "field")} ·{" "} + {countOf(stateType.methods.length, "method")} + +
+ +
+
{stateType.file}
+ +
state
+ {stateType.fields.length === 0 ? ( +
+ No state fields. The key is the whole state. +
+ ) : ( +
+ {stateType.fields.map((field) => ( +
+ {field.name} + {field.type} +
+ ))} +
+ )} + +
methods
+
+ {stateType.methods.map((method) => ( + + ))} +
+
+ ); +}; + +// Whether `rbt dev run` may open a dashboard by itself, and the one +// click that changes the answer. +const Banner: FC<{ suppressed: boolean; onToggle: () => void }> = ({ + suppressed, + onToggle, +}) => ( +
+ +
+); + +const Overview: FC<{ + isExpanded: (name: string) => boolean; + onToggle: (name: string) => void; +}> = ({ isExpanded, onToggle }) => { + // The dashboard's own state: what it read of the developer's API + // files. Nothing here reaches the application, so the application + // does not have to exist. + const { useGet } = useAPI({ id: API_ID }); + const { response, isLoading } = useGet(); + + // What the developer's API files declare. Those exist before the + // application is generated, built or started, which is why they are + // what this page shows. + const read = response?.stateTypes; + + // Restarting `rbt dashboard` closes this page's connection for a + // few seconds. Keep the last shape that was read so the page stays + // readable across that. + const seen = useRef([]); + + if (read !== undefined && read.length > 0) { + seen.current = read; + } + + const stateTypes: StateTypeInfo[] = read?.length ? read : seen.current; + + // Why the API files could not be read, shown beside the last shape + // that was: a half-written file is the normal case while someone is + // typing, and saying so beats showing nothing. + const error = response?.error ?? ""; + + const namespaces = useMemo(() => { + const byNamespace = new Map(); + for (const stateType of stateTypes) { + const namespace = namespaceOf(stateType.name); + const types = byNamespace.get(namespace); + if (types === undefined) { + byNamespace.set(namespace, [stateType]); + } else { + types.push(stateType); + } + } + // The developer's own namespaces first; the standard library is + // theirs to use but not theirs to read. + return [...byNamespace.entries()] + .map(([namespace, types]) => ({ namespace, types })) + .sort((a, b) => { + const standard = + Number(isStandardLibrary(a.namespace)) - + Number(isStandardLibrary(b.namespace)); + return standard !== 0 + ? standard + : a.namespace.localeCompare(b.namespace); + }); + }, [stateTypes]); + + // Only before anything has ever been read; afterwards the last + // shape is shown instead. + if (isLoading && stateTypes.length === 0) { + return ( +
+

Reboot application

+

Reading your API…

+
+ ); + } + + if (stateTypes.length === 0) { + return ( +
+

Reboot application

+

+ Waiting for your API. Nothing in your API directory declares state + types yet. +

+ {error &&
{error}
} +
+ ); + } + + return ( +
+ +
+
+
application domain
+

+ {stateTypes.length} state types in {namespaces.length}{" "} + {namespaces.length === 1 ? "namespace" : "namespaces"} +

+
+ {error &&
{error}
} + {stateTypes.map((stateType) => ( + onToggle(stateType.name)} + key={stateType.name} + /> + ))} +
+
+ ); +}; + +// Everything the developer has told this dashboard, in one place. +// Both choices are the dashboard application's state rather than +// this page's, so they survive the tab, the hot reload and the +// `rbt dev run` they were made in. +const App: FC = () => { + const { useGet, setSuppressOpenOnRestart, setExpanded } = usePreferences({ + id: PREFERENCES_ID, + }); + const { response } = useGet(); + + // Until the read lands, say what the CLI does when nothing has been + // written, which is the same thing it does on a false field. + const suppressed = response?.suppressOpenOnRestart ?? false; + + const stored = useMemo( + () => new Set(response?.expandedStateTypes ?? []), + [response?.expandedStateTypes] + ); + + // A click that has not yet come back from the application, standing + // in for the read until it does. Without it a section would sit + // still for a whole round trip after being clicked, which reads as + // a dead button rather than as a slow one. + const [clicked, setClicked] = useState(new Map()); + + // Drop each stand-in once the read agrees with it, so that a later + // change from another tab is followed rather than held off forever. + useEffect(() => { + setClicked((clicked) => { + const waiting = new Map( + [...clicked].filter(([name, expanded]) => stored.has(name) !== expanded) + ); + return waiting.size === clicked.size ? clicked : waiting; + }); + }, [stored]); + + const isExpanded = useCallback( + (name: string): boolean => clicked.get(name) ?? stored.has(name), + [clicked, stored] + ); + + const onToggle = useCallback( + (name: string): void => { + const expanded = !isExpanded(name); + setClicked((clicked) => new Map(clicked).set(name, expanded)); + setExpanded({ stateType: name, expanded }); + }, + [isExpanded, setExpanded] + ); + + return ( +
+ + setSuppressOpenOnRestart({ suppressOpenOnRestart: !suppressed }) + } + /> + +
+ ); +}; + +const root = document.getElementById("root"); + +if (root !== null) { + createRoot(root).render( + + {/* No `url`: the page and its presence are served by the same + application, so the client uses this page's origin. */} + + + + + + + ); +} diff --git a/reboot/dashboard/frontend/tsconfig.json b/reboot/dashboard/frontend/tsconfig.json new file mode 100644 index 00000000..3446f50f --- /dev/null +++ b/reboot/dashboard/frontend/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "es2018", + "module": "esnext", + "jsx": "react-jsx", + "moduleResolution": "bundler", + "skipLibCheck": true, + "verbatimModuleSyntax": true, + "baseUrl": ".", + "paths": { + // The generated bindings for the dashboard's own state, which + // `dashboard_js_reboot_react` emits beside its proto. + "@dashboard/*": ["../../../rbt/dashboard/v1/*"] + } + } +} diff --git a/reboot/dashboard/main.py b/reboot/dashboard/main.py new file mode 100644 index 00000000..fdfc9e1b --- /dev/null +++ b/reboot/dashboard/main.py @@ -0,0 +1,84 @@ +"""The developer dashboard application. + +A Reboot application owned by the framework, holding the state the +dashboard needs but which must not be written into the +application under development, and serving the dashboard's page. It +is not part of the Reboot API and nothing imports it; it runs as its +own process, with its own state store, alongside the application +being developed. +""" +import asyncio +from pathlib import Path +from rbt.dashboard.v1.dashboard_rbt import API, Preferences +from rbt.std.presence.v1.presence_rbt import Presence +from reboot.aio.applications import Application +from reboot.aio.external import InitializeContext +from reboot.dashboard.constants import ( + API_ID, + DASHBOARD_PATH, + PREFERENCES_ID, + PRESENCE_ID, +) +from reboot.dashboard.servicers import servicers +from starlette.staticfiles import StaticFiles + +# The built page, beside this module, which is the same arrangement +# `InspectServicer` uses for its own assets. Mounting it directly +# avoids `RBT_FRONTEND_DIST_PATH`, which resolves against a project +# root discovered by walking up from a servicer's file; the servicers +# here come from `reboot.std.presence`, so no such root exists above +# them. +_DASHBOARD_DIRECTORY = Path(__file__).parent / 'dashboard' + + +def application() -> Application: + """The dashboard application, with its page mounted.""" + application = Application( + servicers=servicers(), + initialize=initialize, + ) + + application.http.mount( + DASHBOARD_PATH, + app=StaticFiles( + directory=str(_DASHBOARD_DIRECTORY), + # `html=True` so the directory URL serves `index.html`. + # `check_dir=False` so a not-yet-built page doesn't stop + # the application starting. `follow_symlink=True` because + # under Bazel runfiles the built page is a symlink into + # `bazel-out`, which Starlette's default `realpath` check + # rejects as escaping the served directory. + html=True, + check_dir=False, + follow_symlink=True, + ), + ) + + return application + + +async def initialize(context: InitializeContext) -> None: + """Gives `Preferences` the answer somebody who has never clicked + its banner should get.""" + await Preferences.ref(PREFERENCES_ID).SetSuppressOpenOnRestart( + context, + suppress_open_on_restart=False, + ) + + # Construct the `Presence` instance, empty, so that a read of + # who is looking at a dashboard has an answer from the moment the + # dashboard is up. + await Presence.ref(PRESENCE_ID).Create(context) + + # Idempotently, so that a restart of a named application finds + # the `Watch` it already spawned rather than starting a second + # watcher. + _ = await API.ref(API_ID).idempotently('watch').spawn().Watch(context) + + +async def main(): + await application().run() + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/reboot/dashboard/servicers.py b/reboot/dashboard/servicers.py new file mode 100644 index 00000000..dec9cd41 --- /dev/null +++ b/reboot/dashboard/servicers.py @@ -0,0 +1,131 @@ +"""Servicers for the developer dashboard application.""" +import os +import reboot.std.presence.v1.presence +from rbt.dashboard.v1.dashboard_pb2 import ( + APIGetRequest, + APIGetResponse, + APIUpdateRequest, + APIUpdateResponse, + PreferencesGetRequest, + PreferencesGetResponse, + PreferencesSetExpandedRequest, + PreferencesSetExpandedResponse, + PreferencesSetSuppressOpenOnRestartRequest, + PreferencesSetSuppressOpenOnRestartResponse, +) +from rbt.dashboard.v1.dashboard_rbt import API, Preferences +from reboot.aio.auth.authorizers import allow +from reboot.aio.contexts import ReaderContext, WorkflowContext, WriterContext +from reboot.aio.servicers import Servicer +from reboot.dashboard.api_watcher import watch +from reboot.dashboard.constants import ENVVAR_RBT_API_DIRECTORY + + +class APIServicer(API.Servicer): + """Holds the shape the developer's API files declare.""" + + def authorizer(self): + return allow() + + async def Get( + self, + context: ReaderContext, + request: APIGetRequest, + ) -> APIGetResponse: + return APIGetResponse( + state_types=self.state.state_types, + error=self.state.error, + ) + + @classmethod + async def Watch( + cls, + context: WorkflowContext, + request: API.WatchRequest, + ) -> API.WatchResponse: + """Reads the developer's API files when they change. + + The directory comes from the environment each time this runs, + so that an `rbt dashboard` restarted against a different one + reads the new directory. Taking it from the request would keep + whichever directory the run that started watching had. + """ + api_directory = os.environ[ENVVAR_RBT_API_DIRECTORY] + + await watch(context, api_directory=api_directory) + + return API.WatchResponse() + + async def Update( + self, + context: WriterContext, + request: APIUpdateRequest, + ) -> APIUpdateResponse: + del self.state.state_types[:] + self.state.state_types.extend(request.state_types) + self.state.error = request.error + return APIUpdateResponse() + + +class PreferencesServicer(Preferences.Servicer): + """Holds what the developer has said about their dashboard. + + Two unrelated choices share one state because both are facts about + this machine's dashboard rather than about the application, and + each has its own writer so that recording one never overwrites the + other. + """ + + def authorizer(self): + return allow() + + async def Get( + self, + context: ReaderContext, + request: PreferencesGetRequest, + ) -> PreferencesGetResponse: + return PreferencesGetResponse( + suppress_open_on_restart=self.state.suppress_open_on_restart, + expanded_state_types=self.state.expanded_state_types, + ) + + async def SetSuppressOpenOnRestart( + self, + context: WriterContext, + request: PreferencesSetSuppressOpenOnRestartRequest, + ) -> PreferencesSetSuppressOpenOnRestartResponse: + self.state.suppress_open_on_restart = request.suppress_open_on_restart + return PreferencesSetSuppressOpenOnRestartResponse() + + async def SetExpanded( + self, + context: WriterContext, + request: PreferencesSetExpandedRequest, + ) -> PreferencesSetExpandedResponse: + expanded = set(self.state.expanded_state_types) + + if request.expanded: + expanded.add(request.state_type) + else: + expanded.discard(request.state_type) + + self.state.expanded_state_types[:] = sorted(expanded) + + return PreferencesSetExpandedResponse() + + +def servicers() -> list[type[Servicer]]: + """The servicers that back the dashboard's own state. + + This state belongs to the dashboard rather than to the application + being developed, so it lives in its own application and its own + state store. + + This is a library rather than something built into the application + below it, so that what these servicers are stays separate from what + ends up hosting them. + """ + return [ + APIServicer, + PreferencesServicer, + ] + reboot.std.presence.v1.presence.servicers() diff --git a/reboot/inspect/BUILD.bazel b/reboot/inspect/BUILD.bazel index 007e4e00..5083da6d 100644 --- a/reboot/inspect/BUILD.bazel +++ b/reboot/inspect/BUILD.bazel @@ -49,6 +49,19 @@ esbuild( platform = "browser", ) +py_library( + name = "describe_state_type_py", + srcs = ["describe_state_type.py"], + srcs_version = "PY3", + visibility = ["//visibility:public"], + deps = [ + "//log:log_py", + "//rbt/v1alpha1:options_py_proto", + "//rbt/v1alpha1/inspect:inspect_py_proto", + "//reboot/aio:types_py", + ], +) + py_library( name = "servicer_py", srcs = ["servicer.py"], diff --git a/reboot/inspect/describe_state_type.py b/reboot/inspect/describe_state_type.py new file mode 100644 index 00000000..62bbe43e --- /dev/null +++ b/reboot/inspect/describe_state_type.py @@ -0,0 +1,141 @@ +"""Describes Reboot state types from protobuf descriptors. + +Everything comes from the descriptor pool it is handed: a state type's +fields, and the methods of the services that supply them. Field and +method types are rendered as the Python names a person reads on a +page, such as `int` and `list[str]`, rather than as their protobuf +spelling. +""" +from google.protobuf import descriptor_pool +from google.protobuf.descriptor import FieldDescriptor +from log.log import get_logger +from rbt.v1alpha1 import options_pb2 +from rbt.v1alpha1.inspect.inspect_pb2 import ( + FieldInfo, + MethodInfo, + StateTypeInfo, +) +from reboot.aio.types import StateTypeName +from typing import Iterable, Optional + +logger = get_logger(__name__) + +_TYPE_NAMES = { + FieldDescriptor.TYPE_DOUBLE: 'float', + FieldDescriptor.TYPE_FLOAT: 'float', + FieldDescriptor.TYPE_INT64: 'int', + FieldDescriptor.TYPE_UINT64: 'int', + FieldDescriptor.TYPE_INT32: 'int', + FieldDescriptor.TYPE_FIXED64: 'int', + FieldDescriptor.TYPE_FIXED32: 'int', + FieldDescriptor.TYPE_BOOL: 'bool', + FieldDescriptor.TYPE_STRING: 'str', + FieldDescriptor.TYPE_BYTES: 'bytes', + FieldDescriptor.TYPE_UINT32: 'int', + FieldDescriptor.TYPE_SFIXED32: 'int', + FieldDescriptor.TYPE_SFIXED64: 'int', + FieldDescriptor.TYPE_SINT32: 'int', + FieldDescriptor.TYPE_SINT64: 'int', +} + + +def _type_name(field) -> str: + """How to render `field`'s type.""" + if field.type in ( + FieldDescriptor.TYPE_MESSAGE, FieldDescriptor.TYPE_GROUP + ): + name = field.message_type.name + elif field.type == FieldDescriptor.TYPE_ENUM: + name = field.enum_type.name + else: + name = _TYPE_NAMES.get(field.type, 'unknown') + + if field.label == FieldDescriptor.LABEL_REPEATED: + return f'list[{name}]' + return name + + +def _fields_of(message) -> list[FieldInfo]: + return [ + FieldInfo(name=field.name, type=_type_name(field)) + for field in message.fields + ] + + +def _describe_method(method) -> MethodInfo: + options = method.GetOptions().Extensions[options_pb2.method] + kind = options.WhichOneof('kind') or '' + + info = MethodInfo( + name=method.name, + kind=kind, + arguments=_fields_of(method.input_type), + errors=list(options.errors), + mcp=options.HasField('mcp'), + ) + + # An empty response means the method returns nothing; saying + # "Empty" would be an implementation detail leaking out. The + # response's fields rather than its name, because a synthesized + # name such as `ShopRemainingResponse` says nothing the fields + # don't. + if method.output_type.full_name != 'google.protobuf.Empty': + info.returns.extend(_fields_of(method.output_type)) + + # An application that was created before `MethodOptions.description` + # will have the deprecated `mcp` description, which is permitted + # for backward compatibility. + if options.description: + info.description = options.description + elif options.HasField('mcp') and options.mcp.description: + info.description = options.mcp.description + + # Only writers and transactions can construct. + if kind in ('writer', 'transaction'): + info.factory = getattr(options, kind).HasField('constructor') + + return info + + +def describe_state_type( + pool: descriptor_pool.DescriptorPool, + state_type_name: StateTypeName, + service_names: Iterable[str], + file: str, +) -> Optional[StateTypeInfo]: + """Describes one state type, or `None` when its descriptors can't + be found, since a state type we can't describe shouldn't stop us + describing the rest. + + `pool` holds the state type's descriptors and those of the + services named in `service_names`, which supply its methods. + + `file` is the file the developer declared the state type in, which + is reported as it is. A Pydantic API is described from a `.proto` + synthesized from it, so the descriptors name a file that only + exists inside the build. + """ + try: + state = pool.FindMessageTypeByName(state_type_name) + except KeyError: + logger.warning( + f"No descriptor for state type '{state_type_name}'; " + "omitting it from the schema" + ) + return None + + info = StateTypeInfo( + name=state_type_name, + file=file, + fields=_fields_of(state), + ) + + for service_name in service_names: + try: + service = pool.FindServiceByName(service_name) + except KeyError: + continue + for method in service.methods: + info.methods.append(_describe_method(method)) + + return info diff --git a/reboot/std/presence/v1/presence.py b/reboot/std/presence/v1/presence.py index 69025d14..86740dc1 100644 --- a/reboot/std/presence/v1/presence.py +++ b/reboot/std/presence/v1/presence.py @@ -19,6 +19,7 @@ WaitForDisconnectRequest, WaitForDisconnectResponse, ) +from rbt.std.presence.v1 import presence_rbt from rbt.std.presence.v1.presence_rbt import ( ListRequest, ListResponse, @@ -67,6 +68,14 @@ class PresenceServicer(Presence.singleton.Servicer): def authorizer(self): return allow() + async def Create( + self, + context: WriterContext, + state: Presence.State, + request: presence_rbt.CreateRequest, + ) -> presence_rbt.CreateResponse: + return presence_rbt.CreateResponse() + async def Subscribe( self, context: WriterContext, diff --git a/tests/reboot/cli/BUILD.bazel b/tests/reboot/cli/BUILD.bazel index 0c03443d..fcbac08f 100644 --- a/tests/reboot/cli/BUILD.bazel +++ b/tests/reboot/cli/BUILD.bazel @@ -33,6 +33,20 @@ py_test( ], ) +py_test( + name = "dashboard_tests_py", + srcs = [ + "dashboard_tests.py", + ], + main = "dashboard_tests.py", + deps = [ + ":mock_exit_py", + "//reboot/cli/common:cli_py", + "//reboot/cli/common:rc_py", + "//reboot/dashboard:constants_py", + ], +) + py_test( name = "dev_tests_py", srcs = [ diff --git a/tests/reboot/cli/dashboard_tests.py b/tests/reboot/cli/dashboard_tests.py new file mode 100644 index 00000000..f5bea4ff --- /dev/null +++ b/tests/reboot/cli/dashboard_tests.py @@ -0,0 +1,127 @@ +import os +import tempfile +import unittest +from reboot.cli.commands import dashboard +from reboot.cli.common import cli +from reboot.cli.common.directories import dot_rbt_directory +from reboot.cli.common.rc import ArgumentParser +from reboot.dashboard.constants import DEFAULT_DASHBOARD_PORT +from tests.reboot.cli.mock_exit import ( + MockExitException, + mock_raise_instead_of_exit, +) +from unittest.mock import patch + + +@patch('argparse.ArgumentParser.exit', mock_raise_instead_of_exit) +class RbtDashboardTestCase(unittest.IsolatedAsyncioTestCase): + + def _parse(self, state_directory: str): + parser: ArgumentParser = cli.create_parser( + argv=[ + 'rbt', + f'--state-directory={state_directory}', + 'dashboard', + '--api-directory=api', + ] + ) + args, _ = parser.parse_args() + return args, parser + + async def test_api_directory_is_required(self) -> None: + parser: ArgumentParser = cli.create_parser(argv=['rbt', 'dashboard']) + with self.assertRaises(MockExitException): + parser.parse_args() + + async def test_env_is_isolated_from_any_application(self) -> None: + with tempfile.TemporaryDirectory() as state_directory: + args, parser = self._parse(state_directory) + + # Values naming a developer's application must not survive + # into the dashboard's environment; if any did, the + # dashboard would collide with their state directory or + # port. + with patch.dict( + os.environ, + { + 'RBT_NAME': 'app', + 'RBT_STATE_DIRECTORY': '/somewhere/app', + 'RBT_NODEJS': 'true', + 'REBOOT_LOCAL_ENVOY_PORT': '9991', + }, + ): + env = dashboard._dashboard_env( + args, + parser, + port=DEFAULT_DASHBOARD_PORT, + api_directory=args.api_directory, + ) + + self.assertEqual(env['RBT_NAME'], 'dashboard') + self.assertNotIn('RBT_NODEJS', env) + self.assertEqual( + env['REBOOT_LOCAL_ENVOY_PORT'], + str(DEFAULT_DASHBOARD_PORT), + ) + + # One server, and Envoy explicitly on: one server would + # otherwise turn Envoy off, and the browser has to reach + # the dashboard. + self.assertEqual(env['RBT_SERVERS'], '1') + self.assertEqual(env['REBOOT_LOCAL_ENVOY'], 'true') + + # A sibling of `.rbt/dev/`, so that it can never collide + # with an application's state at `.rbt/dev//`. + self.assertEqual( + env['RBT_STATE_DIRECTORY'], + str(dot_rbt_directory(args, parser) / 'dashboard'), + ) + + async def test_keys_differ_from_any_application(self) -> None: + with tempfile.TemporaryDirectory() as state_directory: + args, parser = self._parse(state_directory) + + with patch.dict( + os.environ, {'REBOOT_CRYPTO_ROOT_KEYS': 'v1:theirs'} + ): + env = dashboard._dashboard_env( + args, + parser, + port=DEFAULT_DASHBOARD_PORT, + api_directory=args.api_directory, + ) + + self.assertNotEqual(env['REBOOT_CRYPTO_ROOT_KEYS'], 'v1:theirs') + + # Stable across restarts, so tokens the dashboard mints + # stay valid until its state is deleted. + again = dashboard._dashboard_env( + args, + parser, + port=DEFAULT_DASHBOARD_PORT, + api_directory=args.api_directory, + ) + self.assertEqual( + env['REBOOT_CRYPTO_ROOT_KEYS'], + again['REBOOT_CRYPTO_ROOT_KEYS'], + ) + + async def test_is_told_where_the_api_files_are(self) -> None: + with tempfile.TemporaryDirectory() as state_directory: + args, parser = self._parse(state_directory) + + env = dashboard._dashboard_env( + args, + parser, + port=DEFAULT_DASHBOARD_PORT, + api_directory=args.api_directory, + ) + + # As the developer spelled it, so files can be shown as + # `api/bank/v1/account.py`; the dashboard runs in the + # working directory where that spelling resolves. + self.assertEqual(env['RBT_API_DIRECTORY'], 'api') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/reboot/cli/dev_tests.py b/tests/reboot/cli/dev_tests.py index 80785e37..1993842a 100644 --- a/tests/reboot/cli/dev_tests.py +++ b/tests/reboot/cli/dev_tests.py @@ -119,6 +119,22 @@ async def test_environment_variable(self) -> None: [['E1', 'V1'], ['E2', 'V2'], ['E3', 'V3']], ) + async def test_open_dashboard_requires_a_reachable_dashboard(self) -> None: + # `rbt dev run --open-dashboard` refuses to start when nothing + # is serving on the dashboard's port, telling the developer to + # run `rbt dashboard`. + server = await asyncio.start_server( + lambda reader, writer: writer.close(), '127.0.0.1', 0 + ) + port = server.sockets[0].getsockname()[1] + + self.assertTrue(await dev._dashboard_reachable(port)) + + server.close() + await server.wait_closed() + + self.assertFalse(await dev._dashboard_reachable(port)) + async def test_dev_expunge_requires_name(self) -> None: parser: ArgumentParser = cli.create_parser( argv=[ diff --git a/tests/reboot/dashboard/BUILD.bazel b/tests/reboot/dashboard/BUILD.bazel new file mode 100644 index 00000000..c9956e23 --- /dev/null +++ b/tests/reboot/dashboard/BUILD.bazel @@ -0,0 +1,83 @@ +load("@rules_python//python:defs.bzl", "py_test") +load("//tests/reboot/react:py_web_test_suite_env.bzl", "py_web_test_suite_env") + +py_test( + name = "api_reader_tests_py", + srcs = ["api_reader_tests.py"], + data = glob(["api/**"]), + main = "api_reader_tests.py", + deps = [ + "//reboot/dashboard:api_reader_py", + ], +) + +py_test( + name = "api_watcher_tests_py", + srcs = ["api_watcher_tests.py"], + main = "api_watcher_tests.py", + deps = [ + "//reboot/aio:tests_py", + "//reboot/dashboard:api_watcher_py", + "//reboot/dashboard:main_py", + ], +) + +py_test( + name = "application_tests_py", + srcs = [":application_tests.py"], + main = "application_tests.py", + deps = [ + "//reboot/aio:tests_py", + "//reboot/dashboard:servicers_py", + "//reboot/std/presence/v1:presence_py", + ], +) + +py_web_test_suite_env( + name = "dashboard_tests_py", + srcs = ["dashboard_tests.py"], + browsers = [ + "@io_bazel_rules_webtesting//browsers:chromium-local", + ], + main = "dashboard_tests.py", + py_test_tags = [ + "macos_not_supported", + "requires-linux-x86", + ], + tags = [ + "macos_not_supported", + "requires-linux-x86", + ], + deps = [ + "//reboot/aio:tests_py", + "//reboot/dashboard:api_watcher_py", + "//reboot/dashboard:main_py", + "//reboot/std/presence/v1:presence_py", + "@io_bazel_rules_webtesting//testing/web", + ], +) + +py_test( + name = "preferences_tests_py", + srcs = ["preferences_tests.py"], + main = "preferences_tests.py", + deps = [ + "//rbt/dashboard/v1:dashboard_py_reboot", + "//reboot/aio:tests_py", + "//reboot/dashboard:main_py", + ], +) + +py_test( + name = "open_dashboard_tests_py", + srcs = ["open_dashboard_tests.py"], + main = "open_dashboard_tests.py", + deps = [ + "//rbt/dashboard/v1:dashboard_py_reboot", + "//rbt/std/presence/subscriber/v1:subscriber_py_reboot", + "//rbt/std/presence/v1:presence_py_reboot", + "//reboot/aio:tests_py", + "//reboot/cli/commands:dev_py", + "//reboot/dashboard:main_py", + ], +) diff --git a/tests/reboot/dashboard/api/shop/v1/helper.py b/tests/reboot/dashboard/api/shop/v1/helper.py new file mode 100644 index 00000000..0c75c3a7 --- /dev/null +++ b/tests/reboot/dashboard/api/shop/v1/helper.py @@ -0,0 +1,3 @@ +"""Shared code beside an API file, with no `api` of its own.""" + +TAX = 0.1 diff --git a/tests/reboot/dashboard/api/shop/v1/shop.py b/tests/reboot/dashboard/api/shop/v1/shop.py new file mode 100644 index 00000000..f49b264d --- /dev/null +++ b/tests/reboot/dashboard/api/shop/v1/shop.py @@ -0,0 +1,48 @@ +from reboot.api import ( + API, + Field, + Methods, + Model, + Reader, + Tool, + Transaction, + Type, +) + + +class ShopState(Model): + name: str = Field(tag=1) + open: bool = Field(tag=2) + + +class StockRequest(Model): + item: str = Field(tag=1) + quantity: int = Field(tag=2) + + +class StockResponse(Model): + remaining: int = Field(tag=1) + + +class OutOfStockError(Model): + item: str = Field(tag=1) + + +ShopMethods = Methods( + create=Transaction(request=None, response=None, factory=True, mcp=None), + stock=Transaction( + request=StockRequest, + response=None, + description="Add stock of an item.", + mcp=None, + ), + remaining=Reader( + request=StockRequest, + response=StockResponse, + errors=[OutOfStockError], + description="How much of an item is left.", + mcp=Tool(), + ), +) + +api = API(Shop=Type(state=ShopState, methods=ShopMethods)) diff --git a/tests/reboot/dashboard/api_reader_tests.py b/tests/reboot/dashboard/api_reader_tests.py new file mode 100644 index 00000000..b5d3d23e --- /dev/null +++ b/tests/reboot/dashboard/api_reader_tests.py @@ -0,0 +1,127 @@ +"""The dashboard describes an API file without the application. + +This is what lets the dashboard show state types before anything has +been built: `rbt generate` has not run, no servicer exists, and there +is no process to ask. Only the file the developer wrote. +""" +import os +import tempfile +import unittest +from pathlib import Path +from reboot.dashboard.api_reader import read + +API_DIRECTORY = str(Path(__file__).parent / 'api') + + +def _by_name(state_types: list[dict]) -> dict[str, dict]: + return {state_type['name']: state_type for state_type in state_types} + + +def _method(state_type: dict, name: str) -> dict: + for method in state_type['methods']: + if method['name'] == name: + return method + raise AssertionError(f"No method '{name}' in {state_type['name']}") + + +class APIReaderTest(unittest.IsolatedAsyncioTestCase): + + async def test_describes_a_state_type_and_its_methods(self) -> None: + state_types, error = await read(API_DIRECTORY, 'shop/v1/shop.py') + + self.assertIsNone(error) + + shop = _by_name(state_types)['shop.v1.Shop'] + + # The file the developer wrote, spelled from where the + # dashboard was started: the API directory as given, then the + # path inside it. + self.assertEqual( + shop['file'], + os.path.join(API_DIRECTORY, 'shop/v1/shop.py'), + ) + + self.assertEqual( + [field['name'] for field in shop['fields']], + ['name', 'open'], + ) + + # The methods come from the file, with the names and kinds + # their author wrote. + stock = _method(shop, 'stock') + self.assertEqual(stock['kind'], 'transaction') + self.assertEqual( + [argument['name'] for argument in stock['arguments']], + ['item', 'quantity'], + ) + + # `stock` has a `description` and is not an MCP tool: prose + # reaches the page whether or not its author also exposed the + # method to MCP. + self.assertEqual(stock['description'], 'Add stock of an item.') + self.assertNotIn('mcp', stock) + + remaining = _method(shop, 'remaining') + self.assertEqual(remaining['kind'], 'reader') + self.assertEqual( + remaining['returns'], + [{ + 'name': 'remaining', + 'type': 'int', + }], + ) + self.assertTrue(remaining['mcp']) + + # The errors a method declares, by the names of the declared + # models. + self.assertEqual(remaining['errors'], ['OutOfStockError']) + self.assertEqual( + remaining['description'], + 'How much of an item is left.', + ) + + # A factory constructs the state, and returns nothing. + create = _method(shop, 'create') + self.assertTrue(create['factory']) + self.assertNotIn('returns', create) + + async def test_a_file_with_no_api_describes_nothing(self) -> None: + # A directory holds shared code as well as APIs, and reading a + # module that declares no `api` is not an error. + state_types, error = await read(API_DIRECTORY, 'shop/v1/helper.py') + + self.assertIsNone(error) + self.assertEqual(state_types, []) + + async def test_a_file_that_does_not_parse_reports_why(self) -> None: + # Half-written files are the normal case while someone is + # typing. The reader has to survive them and say what is + # wrong, because that message is what the developer needs. + with tempfile.TemporaryDirectory() as directory: + os.makedirs(os.path.join(directory, 'shop', 'v1')) + Path(os.path.join(directory, 'shop', 'v1', 'shop.py') + ).write_text('from reboot.api import API\napi = API(\n') + + state_types, error = await read(directory, 'shop/v1/shop.py') + + self.assertEqual(state_types, []) + assert error is not None + self.assertIn('SyntaxError', error) + + async def test_reading_does_not_write_to_the_developer_s_tree( + self + ) -> None: + # Reading walks the API object in memory and leaves the + # developer's tree exactly as it was. + before = sorted(os.listdir(os.path.join(API_DIRECTORY, 'shop', 'v1'))) + + await read(API_DIRECTORY, 'shop/v1/shop.py') + + self.assertEqual( + before, + sorted(os.listdir(os.path.join(API_DIRECTORY, 'shop', 'v1'))), + ) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/reboot/dashboard/api_watcher_tests.py b/tests/reboot/dashboard/api_watcher_tests.py new file mode 100644 index 00000000..e5fa6b84 --- /dev/null +++ b/tests/reboot/dashboard/api_watcher_tests.py @@ -0,0 +1,111 @@ +"""State types appear as the developer writes their API files. + +The dashboard is up before the application exists, so this is the +first thing a dashboard can show: not what is running, but what has +been written so far. +""" +import asyncio +import os +import tempfile +import unittest +from pathlib import Path +from rbt.dashboard.v1.dashboard_rbt import API +from reboot.aio.tests import Reboot +from reboot.dashboard.constants import API_ID, ENVVAR_RBT_API_DIRECTORY +from reboot.dashboard.main import application +from typing import Optional +from unittest.mock import patch + +SHOP = ''' +from reboot.api import API, Field, Methods, Model, Reader, Type + + +class {state}State(Model): + name: str = Field(tag=1) + + +class LookRequest(Model): + item: str = Field(tag=1) + + +class LookResponse(Model): + found: bool = Field(tag=1) + + +{state}Methods = Methods( + look=Reader( + request=LookRequest, + response=LookResponse, + description=None, + mcp=None, + ), +) + +api = API({state}=Type(state={state}State, methods={state}Methods)) +''' + + +class APIWatcherTest(unittest.IsolatedAsyncioTestCase): + + watcher: Optional[asyncio.Task] = None + + async def asyncSetUp(self) -> None: + # The workflow reads the directory when the application comes + # up, so it has to exist and be named first. + self._directory = tempfile.TemporaryDirectory() + self.directory = Path(self._directory.name) + self._environment = patch.dict( + os.environ, + {ENVVAR_RBT_API_DIRECTORY: str(self.directory)}, + ) + self._environment.start() + + self.rbt = Reboot() + await self.rbt.start() + await self.rbt.up(application(), local_envoy=True) + self.url = f'http://127.0.0.1:{self.rbt.envoy_port()}' + + async def asyncTearDown(self) -> None: + await self.rbt.stop() + self._environment.stop() + self._directory.cleanup() + + def _write(self, directory: Path, name: str, state: str) -> None: + path = directory / 'shop' / 'v1' / f'{name}.py' + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(SHOP.format(state=state)) + + async def _wait_for(self, satisfied): + while True: + context = self.rbt.create_external_context(name=self.id()) + try: + response = await API.ref(API_ID).Get(context) + if satisfied(response): + return response + except Exception: + pass + await asyncio.sleep(0.1) + + async def test_types_appear_as_files_are_written(self) -> None: + # The workflow is already watching: it was scheduled when the + # application came up. + self._write(self.directory, 'shop', 'Shop') + + response = await self._wait_for(lambda api: len(api.state_types) == 1) + self.assertEqual( + [state.name for state in response.state_types], + ['shop.v1.Shop'], + ) + self.assertEqual(response.error, '') + + self._write(self.directory, 'depot', 'Depot') + + response = await self._wait_for(lambda api: len(api.state_types) == 2) + self.assertEqual( + sorted(state.name for state in response.state_types), + ['shop.v1.Depot', 'shop.v1.Shop'], + ) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/reboot/dashboard/application_tests.py b/tests/reboot/dashboard/application_tests.py new file mode 100644 index 00000000..64ca0c13 --- /dev/null +++ b/tests/reboot/dashboard/application_tests.py @@ -0,0 +1,102 @@ +import asyncio +import unittest +from rbt.v1alpha1.errors_pb2 import NotFound, StateNotConstructed +from reboot.aio.applications import Application +from reboot.aio.external import ExternalContext +from reboot.aio.tests import Reboot +from reboot.dashboard.servicers import servicers +from reboot.std.presence.v1.presence import Presence, Subscriber + + +class TestDashboardApplication(unittest.IsolatedAsyncioTestCase): + """Checks that the dashboard application stands up on its own and + that presence works against it, which is the whole reason it + exists.""" + + async def asyncSetUp(self) -> None: + self.rbt = Reboot() + await self.rbt.start() + await self.rbt.up(Application(servicers=servicers())) + + async def asyncTearDown(self) -> None: + await self.rbt.stop() + + async def make_connection( + self, + presence_ref: Presence.WeakReference, + subscriber_ref: Subscriber.WeakReference, + context: ExternalContext, + nonce: str, + ) -> asyncio.Task: + """Connects `subscriber_ref` and subscribes it to `presence_ref`. + + `Toggle` is retried because it races `Connect`, which is what + registers the connection; until that has happened `Toggle` + reports `NotFound`. Returns the task running `Connect`, which + stays pending for as long as the subscriber is present. + """ + await subscriber_ref.idempotently().Create(context) + + connect_failed = False + + async def connect(): + nonlocal connect_failed + try: + await subscriber_ref.Connect(context, nonce=nonce) + except: + connect_failed = True + + connect_task = asyncio.create_task(connect()) + + attempt = 0 + while not connect_failed: + try: + await subscriber_ref.idempotently( + f"Attempt {attempt}", + ).Toggle(context, nonce=nonce) + except Subscriber.ToggleAborted as aborted: + if isinstance(aborted.error, NotFound): + attempt += 1 + continue + raise + + await presence_ref.Subscribe( + context, subscriber_id=subscriber_ref.state_id + ) + break + + return connect_task + + async def test_presence_reports_a_connected_subscriber(self) -> None: + context = self.rbt.create_external_context(name=f"test-{self.id()}") + + presence = Presence.ref("dashboard") + subscriber = Subscriber.ref("a-dashboard-tab") + + # Until a writer constructs the `Presence` state, `List` + # aborts rather than reporting an empty list. The dashboard's + # `initialize` calls `Create` for exactly this reason, so its + # readers always have an instance to read; this application has + # no `initialize`, which is what lets this case be seen. + with self.assertRaises(Presence.ListAborted) as aborted: + await presence.List(context) + self.assertIsInstance(aborted.exception.error, StateNotConstructed) + + connect_task = await self.make_connection( + presence, subscriber, context, nonce="nonce" + ) + + response = await presence.List(context) + self.assertEqual(list(response.subscriber_ids), [subscriber.state_id]) + + # Cancelling `Connect` is what a closing browser tab does, and + # the subscriber must drain back out again. + connect_task.cancel() + + async for response in presence.reactively().List(context): + if list(response.subscriber_ids) == []: + break + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/reboot/dashboard/dashboard_tests.py b/tests/reboot/dashboard/dashboard_tests.py new file mode 100644 index 00000000..67b1f6c7 --- /dev/null +++ b/tests/reboot/dashboard/dashboard_tests.py @@ -0,0 +1,418 @@ +"""The dashboard application serves its page, and that page describes +the application under development. + +It is itself a Reboot application, so it can describe itself: pointing +the page at its own address exercises the whole path (config route, +API read, and rendering) in one process. +""" +import asyncio +import socket +import unittest +from rbt.dashboard.v1.dashboard_pb2 import FieldInfo, MethodInfo, StateTypeInfo +from rbt.dashboard.v1.dashboard_rbt import API, Preferences +from reboot.aio.tests import Reboot +from reboot.dashboard.constants import ( + API_ID, + DASHBOARD_PATH, + PREFERENCES_ID, + PRESENCE_ID, +) +from reboot.dashboard.main import application +from reboot.std.presence.v1.presence import Presence +from selenium.webdriver.common.by import By +from selenium.webdriver.support import expected_conditions +from selenium.webdriver.support.wait import WebDriverWait +from testing.web import webtest + + +def _driver(): + return webtest.new_webdriver_session( + capabilities={ + 'goog:chromeOptions': + { + 'args': + [ + '--headless', + '--no-sandbox', + '--disable-dev-shm-usage', + ], + }, + 'goog:loggingPrefs': { + 'browser': 'ALL', + }, + } + ) + + +class DashboardTest(unittest.IsolatedAsyncioTestCase): + + async def asyncSetUp(self) -> None: + # The page is served by the dashboard application, so the + # test needs its + # address to point a browser at it. + with socket.socket() as probe: + probe.bind(('127.0.0.1', 0)) + port = probe.getsockname()[1] + + self.url = f'http://127.0.0.1:{port}' + + self.rbt = Reboot() + await self.rbt.start() + await self.rbt.up( + application(), + local_envoy=True, + local_envoy_port=port, + ) + + async def asyncTearDown(self) -> None: + await self.rbt.stop() + + async def _wait_for_viewers(self, satisfied, driver=None) -> None: + """Polls until presence satisfies `satisfied`. + + Polling rather than reading reactively because `List` aborts + with `StateNotConstructed` until somebody has subscribed at + least once, which is where a fresh application starts. + + Reports what it sees as it goes, including anything the page + logged. This wait has no deadline, so when it does not finish + the test is killed with its `finally` unrun. Printing only at + the end would mean printing nothing in the one case worth + explaining. + """ + polls = 0 + while True: + context = self.rbt.create_external_context(name=self.id()) + # A fresh reference per context; one cannot be shared. + presence = Presence.ref(PRESENCE_ID) + viewers: list[str] = [] + try: + response = await presence.List(context) + viewers = list(response.subscriber_ids) + if satisfied(viewers): + return + except Presence.ListAborted: + if satisfied([]): + return + + polls += 1 + if polls % 10 == 0: + print(f'##### still waiting, {polls} polls, viewers={viewers}') + if driver is not None: + for entry in await asyncio.to_thread( + driver.get_log, 'browser' + ): + print(f'##### page: {entry}') + text = await asyncio.to_thread( + lambda: driver.find_element(By.TAG_NAME, 'body').text + ) + print(f'##### page text: {text[:300]!r}') + + await asyncio.sleep(0.5) + + async def _record_state_types(self) -> None: + """Puts what an API file would yield into the application. + + The reading of files is covered by `api_reader_tests` and + `api_watcher_tests`. What is left to show here is that the + page renders whatever the application holds, so this writes + that directly, and the test keeps no watcher, no observer + thread and no subprocess alive alongside a browser. + """ + context = self.rbt.create_external_context(name=self.id()) + await API.ref(API_ID).Update( + context, + state_types=[ + StateTypeInfo( + name='shop.v1.Shop', + file='api/shop/v1/shop.py', + fields=[FieldInfo(name='name', type='str')], + methods=[ + MethodInfo( + name='look', + kind='reader', + arguments=[FieldInfo(name='item', type='str')], + returns=[FieldInfo(name='found', type='bool')], + ), + ], + ), + ], + error='', + ) + + def _run(self, body): + driver = _driver() + try: + return body(driver) + finally: + print("##### Browser logs #####") + for entry in driver.get_log('browser'): + print(entry) + print("##### End of browser logs #####") + driver.quit() + + async def test_describes_what_the_api_files_declare(self) -> None: + # Nothing here is generated, built or serving: the page shows + # a state type because a file on disk declares one. + def body(driver): + driver.get(f'{self.url}{DASHBOARD_PATH}/') + WebDriverWait(driver, 60).until( + expected_conditions.presence_of_element_located( + (By.ID, 'shop.v1.Shop') + ) + ) + return driver.page_source + + await self._record_state_types() + + page = await asyncio.to_thread(self._run, body) + + # A method, its kind, and its source file all come from the + # file rather than from anything the page knew in advance, + # spelled the way its author spelled them. + self.assertIn('look', page) + self.assertIn('reader', page) + self.assertIn('shop/v1/shop.py', page) + + # State types are grouped by their proto package, which is the + # directory the developer wrote them in. + self.assertIn('shop.v1', page) + + # The sidebar's two counts sit in the same column and count + # different things, so each says what it counts. The fixture + # declares one of each, which also covers the singular. + self.assertIn('1 state type', page) + self.assertIn('1 method', page) + + async def test_says_why_a_file_could_not_be_read(self) -> None: + # A half-written file is the normal case while someone is + # typing, so the page says what went wrong while keeping the + # shape it last read beside it. + def body(driver): + driver.get(f'{self.url}{DASHBOARD_PATH}/') + WebDriverWait(driver, 60).until( + expected_conditions.presence_of_element_located( + (By.CLASS_NAME, 'error') + ) + ) + return driver.page_source + + context = self.rbt.create_external_context(name=self.id()) + await API.ref(API_ID).Update( + context, + state_types=[ + StateTypeInfo( + name='shop.v1.Shop', + file='api/shop/v1/shop.py', + fields=[FieldInfo(name='name', type='str')], + ), + ], + error='shop.py: SyntaxError: invalid syntax', + ) + + page = await asyncio.to_thread(self._run, body) + + self.assertIn('shop.py: SyntaxError: invalid syntax', page) + + # The error does not blank the page: what was last read is + # still there to work against. + self.assertIn('shop.v1.Shop', page) + + # The two labels the banner's one link shows, which are also the + # two things it does. + _TURN_OFF = "Don't reopen this dashboard on restart" + _TURN_ON = 'Open this dashboard on every restart' + + def _click_the_banner(self, driver, showing: str, becomes: str) -> None: + """Clicks the banner's link once it reads `showing`. + + Waits for `becomes` afterwards rather than returning as soon as + the click lands: the new label comes from the reactive read of + `Preferences`, so seeing it is how the test knows the choice + reached the application and came back. + """ + button = (By.CLASS_NAME, 'banner-link') + + WebDriverWait(driver, 60).until( + expected_conditions.text_to_be_present_in_element(button, showing) + ) + driver.find_element(*button).click() + WebDriverWait(driver, 60).until( + expected_conditions.text_to_be_present_in_element(button, becomes) + ) + + async def _suppress_open_on_restart(self) -> bool: + context = self.rbt.create_external_context(name=self.id()) + response = await Preferences.ref(PREFERENCES_ID).Get(context) + return response.suppress_open_on_restart + + async def test_the_banner_turns_reopening_off(self) -> None: + # What the banner writes is exactly what `rbt dev run` reads + # before deciding whether to open a dashboard, which is what + # `open_dashboard_tests` covers from the other side. + + def body(driver): + driver.get(f'{self.url}{DASHBOARD_PATH}/') + self._click_the_banner( + driver, + showing=self._TURN_OFF, + becomes=self._TURN_ON, + ) + + await asyncio.to_thread(self._run, body) + + self.assertTrue(await self._suppress_open_on_restart()) + + async def test_the_banner_turns_reopening_back_on(self) -> None: + # A developer who clicked once is not stuck with it: the page + # they load next offers the choice the other way round. + + context = self.rbt.create_external_context(name=self.id()) + await Preferences.ref(PREFERENCES_ID).SetSuppressOpenOnRestart( + context, + suppress_open_on_restart=True, + ) + + def body(driver): + driver.get(f'{self.url}{DASHBOARD_PATH}/') + self._click_the_banner( + driver, + showing=self._TURN_ON, + becomes=self._TURN_OFF, + ) + + await asyncio.to_thread(self._run, body) + + self.assertFalse(await self._suppress_open_on_restart()) + + # The two labels a state type's one button shows. + _EXPAND = 'Expand details' + _HIDE = 'Hide details' + + def _click_to_expand(self, driver, showing: str, becomes: str) -> None: + """Clicks a state type's button once it reads `showing`. + + Waits for `becomes` afterwards, which is how the test knows + the click was taken, since the label comes from the same state + the detail's height does. + """ + button = (By.CLASS_NAME, 'expand-button') + + WebDriverWait(driver, 60).until( + expected_conditions.text_to_be_present_in_element(button, showing) + ) + driver.find_element(*button).click() + WebDriverWait(driver, 60).until( + expected_conditions.text_to_be_present_in_element(button, becomes) + ) + + @staticmethod + def _detail_height(driver) -> float: + """How tall the first method's detail is drawn. + + Measured rather than asked of `is_displayed()`, because the + detail stays in the document whether or not its state type is + open: what closing does is collapse the grid row it sits in to + nothing, which is what makes the height animate at all. + """ + return driver.execute_script( + 'const detail = document.querySelector(".method-detail-inner");' + 'return detail === null' + ' ? -1' + ' : detail.getBoundingClientRect().height;' + ) + + def _wait_for_detail(self, driver, opened: bool) -> None: + """Waits out the animation, rather than sleeping its duration.""" + WebDriverWait( + driver, 60 + ).until(lambda driver: (self._detail_height(driver) > 0) == opened) + + async def _expanded_state_types(self) -> list[str]: + context = self.rbt.create_external_context(name=self.id()) + response = await Preferences.ref(PREFERENCES_ID).Get(context) + return list(response.expanded_state_types) + + async def test_expanding_a_state_type_opens_its_method_detail( + self + ) -> None: + await self._record_state_types() + + def body(driver): + driver.get(f'{self.url}{DASHBOARD_PATH}/') + self._click_to_expand( + driver, + showing=self._EXPAND, + becomes=self._HIDE, + ) + self._wait_for_detail(driver, opened=True) + + # The height is transitioned rather than switched. Read + # off the property list rather than by sampling a height + # part-way through, which would be a race against the + # animation this is checking for. + self.assertIn( + 'grid-template-rows', + driver.execute_script( + 'const detail =' + ' document.querySelector(".method-detail");' + 'return getComputedStyle(detail).transitionProperty;' + ), + ) + + await asyncio.to_thread(self._run, body) + + # And the click reached the application, which is what makes + # it + # outlast the tab it was made in. + self.assertEqual(await self._expanded_state_types(), ['shop.v1.Shop']) + + async def test_a_state_type_expanded_earlier_is_open_on_load(self) -> None: + # The state a previous `rbt dev run` left behind, which is the + # whole reason the choice lives in the dashboard application. + await self._record_state_types() + + context = self.rbt.create_external_context(name=self.id()) + await Preferences.ref(PREFERENCES_ID).SetExpanded( + context, + state_type='shop.v1.Shop', + expanded=True, + ) + + def body(driver): + driver.get(f'{self.url}{DASHBOARD_PATH}/') + WebDriverWait(driver, 60).until( + expected_conditions.text_to_be_present_in_element( + (By.CLASS_NAME, 'expand-button'), + self._HIDE, + ) + ) + self._wait_for_detail(driver, opened=True) + + await asyncio.to_thread(self._run, body) + + async def test_the_page_holds_presence(self) -> None: + # `rbt dev run` decides whether to open a dashboard by asking + # who is looking at one, so the page being counted while it is + # up and dropped once it is gone is what that decision rests + # on. + driver = await asyncio.to_thread(_driver) + try: + await asyncio.to_thread(driver.get, f'{self.url}{DASHBOARD_PATH}/') + + # Wait for the viewer to register, rather than assuming a + # page load is enough. + await self._wait_for_viewers( + lambda viewers: viewers != [], + driver=driver, + ) + finally: + await asyncio.to_thread(driver.quit) + + # With the browser gone the viewer must drain, which is what + # makes presence usable as a liveness signal at all. + await self._wait_for_viewers(lambda viewers: viewers == []) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/reboot/dashboard/open_dashboard_tests.py b/tests/reboot/dashboard/open_dashboard_tests.py new file mode 100644 index 00000000..cead0245 --- /dev/null +++ b/tests/reboot/dashboard/open_dashboard_tests.py @@ -0,0 +1,219 @@ +"""`rbt dev run` opens a dashboard when nobody is looking at one. + +The dashboard page subscribes to `Presence` for as long as it is open, +so the question the CLI asks is who is looking right now, which +reopens a dashboard the developer closed and never puts a second tab in +front of one they left up. It asks a second question first: whether the +developer clicked the dashboard's "Don't reopen this dashboard on +restart" banner, which is remembered in `Preferences`. + +The page's own subscription and its banner are exercised in +`dashboard_tests`; here the subscriber and the choice are made +directly, so these tests need no browser. +""" +import asyncio +import unittest +from rbt.dashboard.v1.dashboard_rbt import Preferences +from rbt.std.presence.subscriber.v1.subscriber_rbt import Subscriber +from rbt.std.presence.v1.presence_rbt import Presence +from rbt.v1alpha1.errors_pb2 import NotFound +from reboot.aio.tests import Reboot +from reboot.cli.commands.dev import _open_dashboard_once +from reboot.dashboard.constants import ( + DASHBOARD_PATH, + PREFERENCES_ID, + PRESENCE_ID, +) +from reboot.dashboard.main import application +from unittest.mock import patch + + +class OpenDashboardTest(unittest.IsolatedAsyncioTestCase): + + async def asyncSetUp(self) -> None: + self.rbt = Reboot() + await self.rbt.start() + await self.rbt.up(application(), local_envoy=True) + self.url = f'http://127.0.0.1:{self.rbt.envoy_port()}' + self.dashboard_url = f'{self.url}{DASHBOARD_PATH}/' + self._connections: list[asyncio.Task] = [] + + async def asyncTearDown(self) -> None: + for connection in self._connections: + connection.cancel() + await self.rbt.stop() + + async def _view(self, subscriber_id: str) -> None: + """Subscribes as a page would, and stays subscribed. + + `Connect` never returns, and `Toggle` has to land after it, so + the two run concurrently and `Toggle` is retried until the + connection it depends on exists, the same handshake + `reboot/std/react/presence` performs in the browser. + """ + context = self.rbt.create_external_context(name=self.id()) + subscriber = Subscriber.ref(subscriber_id) + nonce = subscriber_id + + await subscriber.idempotently().Create(context) + + self._connections.append( + asyncio.create_task(subscriber.Connect(context, nonce=nonce)) + ) + + attempt = 0 + while True: + try: + await subscriber.idempotently( + f'Attempt {attempt}', + ).Toggle(context, nonce=nonce) + break + except Subscriber.ToggleAborted as aborted: + if not isinstance(aborted.error, NotFound): + raise + attempt += 1 + + await Presence.ref(PRESENCE_ID).Subscribe( + context, + subscriber_id=subscriber_id, + ) + + async def _suppress_reopening(self, suppress: bool) -> None: + """Makes the choice the dashboard's banner makes.""" + context = self.rbt.create_external_context(name=self.id()) + await Preferences.ref(PREFERENCES_ID).SetSuppressOpenOnRestart( + context, + suppress_open_on_restart=suppress, + ) + + async def _viewers(self) -> list[str]: + context = self.rbt.create_external_context(name=self.id()) + try: + response = await Presence.ref(PRESENCE_ID).List(context) + return list(response.subscriber_ids) + except Presence.ListAborted: + return [] + + async def test_opens_when_nobody_is_looking(self) -> None: + self.assertEqual(await self._viewers(), []) + + with patch('webbrowser.open', return_value=True) as browser: + await _open_dashboard_once(dashboard_url=self.url, forced=False) + + # The browser gets the dashboard's path; `ExternalContext` only + # ever sees the origin, which is all it accepts. + browser.assert_called_once_with(self.dashboard_url) + + async def test_does_not_open_when_somebody_is_looking(self) -> None: + await self._view('a-tab-that-is-open') + self.assertEqual(await self._viewers(), ['a-tab-that-is-open']) + + with patch('webbrowser.open', return_value=True) as browser: + with patch('reboot.cli.common.terminal.info') as told: + await _open_dashboard_once( + dashboard_url=self.url, + forced=False, + ) + + browser.assert_not_called() + + # And it must say so: the tab being counted may be behind + # another window, so a run that opens nothing and explains + # nothing is indistinguishable from a broken one. + told.assert_called_once() + self.assertIn('--open-dashboard', told.call_args.args[0]) + self.assertIn(self.dashboard_url, told.call_args.args[0]) + + async def test_opens_again_once_the_last_viewer_has_gone(self) -> None: + await self._view('a-tab-that-closes') + + for connection in self._connections: + connection.cancel() + self._connections = [] + + # Cancelling `Connect` is the only signal presence has, and it + # reaches the subscriber list by way of `WaitForDisconnect` + # untoggling and `Watch` then dropping the subscriber, so wait + # for the list rather than assuming the cancellation was + # enough. + while await self._viewers() != []: + await asyncio.sleep(0.1) + + with patch('webbrowser.open', return_value=True) as browser: + await _open_dashboard_once(dashboard_url=self.url, forced=False) + + browser.assert_called_once_with(self.dashboard_url) + + async def test_forcing_opens_even_though_somebody_is_looking(self) -> None: + await self._view('a-tab-that-is-open') + + with patch('webbrowser.open', return_value=True) as browser: + await _open_dashboard_once(dashboard_url=self.url, forced=True) + + browser.assert_called_once_with(self.dashboard_url) + + async def test_does_not_open_when_the_developer_asked_it_not_to( + self + ) -> None: + # Nobody is looking at a dashboard, so the only thing keeping + # one from opening is the choice the banner recorded. + await self._suppress_reopening(True) + + with patch('webbrowser.open', return_value=True) as browser: + with patch('reboot.cli.common.terminal.info') as told: + await _open_dashboard_once( + dashboard_url=self.url, + forced=False, + ) + + browser.assert_not_called() + + # And it must say how to get one anyway, since a choice made + # in an earlier `rbt dev run` is not something the developer + # is looking at now. + told.assert_called_once() + self.assertIn('--open-dashboard', told.call_args.args[0]) + self.assertIn(self.dashboard_url, told.call_args.args[0]) + + async def test_forcing_opens_even_though_the_developer_asked_it_not_to( + self + ) -> None: + await self._suppress_reopening(True) + + with patch('webbrowser.open', return_value=True) as browser: + await _open_dashboard_once(dashboard_url=self.url, forced=True) + + browser.assert_called_once_with(self.dashboard_url) + + async def test_opens_again_once_the_developer_has_asked_for_it_back( + self + ) -> None: + # The second banner undoes the first, so a developer who + # clicked once is not stuck with it. + await self._suppress_reopening(True) + await self._suppress_reopening(False) + + with patch('webbrowser.open', return_value=True) as browser: + await _open_dashboard_once(dashboard_url=self.url, forced=False) + + browser.assert_called_once_with(self.dashboard_url) + + async def test_says_where_the_dashboard_is_when_none_could_be_opened( + self + ) -> None: + # `webbrowser.open` returns `False` rather than raising when + # there is nothing to open, which is the headless case. Nothing + # was shown, so the developer is told the address instead. + with patch('webbrowser.open', return_value=False): + with patch('reboot.cli.common.terminal.warn') as warned: + await _open_dashboard_once( + dashboard_url=self.url, + forced=False, + ) + + warned.assert_called_once() + self.assertIn(self.dashboard_url, warned.call_args.args[0]) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/reboot/dashboard/preferences_tests.py b/tests/reboot/dashboard/preferences_tests.py new file mode 100644 index 00000000..48be12fc --- /dev/null +++ b/tests/reboot/dashboard/preferences_tests.py @@ -0,0 +1,166 @@ +"""`Preferences` has a value before anybody has chosen one. + +The dashboard's banner renders from a reactive read of `Preferences`, +and a reader aborts with `StateNotConstructed` until something has +written, so the dashboard application writes the defaults at +startup. That write must not undo a choice the developer already +made, because it runs on every start of `rbt dashboard`, which is +exactly when a click from an earlier run has to survive. + +The banner that does the clicking is exercised in `dashboard_tests`, +and what `rbt dev run` does with the answer in `open_dashboard_tests`. +""" +import unittest +import uuid +from rbt.dashboard.v1.dashboard_rbt import Preferences +from reboot.aio.external import InitializeContext +from reboot.aio.tests import Reboot +from reboot.dashboard.constants import PREFERENCES_ID +from reboot.dashboard.main import application, initialize + + +class PreferencesTest(unittest.IsolatedAsyncioTestCase): + + async def asyncSetUp(self) -> None: + self.rbt = Reboot() + await self.rbt.start() + await self.rbt.up(application(), local_envoy=True) + self.url = f'http://127.0.0.1:{self.rbt.envoy_port()}' + + async def asyncTearDown(self) -> None: + await self.rbt.stop() + + def _initialize_context(self) -> InitializeContext: + """A restart's context, seeded as + `Reboot.create_initialize_context` seeds it.""" + return InitializeContext( + name=self.id(), + url=self.url, + idempotency_seed=uuid.uuid5( + uuid.NAMESPACE_DNS, 'anonymous.rbt.dev' + ), + ) + + async def _get(self) -> bool: + context = self.rbt.create_external_context(name=self.id()) + response = await Preferences.ref(PREFERENCES_ID).Get(context) + return response.suppress_open_on_restart + + async def _set_suppress(self, suppress: bool) -> None: + """Makes the choice the dashboard's banner makes.""" + context = self.rbt.create_external_context(name=self.id()) + await Preferences.ref(PREFERENCES_ID).SetSuppressOpenOnRestart( + context, + suppress_open_on_restart=suppress, + ) + + async def _expanded(self) -> list[str]: + context = self.rbt.create_external_context(name=self.id()) + response = await Preferences.ref(PREFERENCES_ID).Get(context) + return list(response.expanded_state_types) + + async def _set_expanded(self, state_type: str, expanded: bool) -> None: + """Makes the choice a state type's `Expand details` makes.""" + context = self.rbt.create_external_context(name=self.id()) + await Preferences.ref(PREFERENCES_ID).SetExpanded( + context, + state_type=state_type, + expanded=expanded, + ) + + async def test_starting_writes_a_default_that_can_be_read(self) -> None: + # The application's `initialize` constructed `Preferences` + # when it came up; a reader would otherwise abort with + # `StateNotConstructed`, and a page that loaded first would + # have nothing to render its banner from. + # + # False, so that somebody who has never clicked the banner gets + # a dashboard opened for them. + self.assertFalse(await self._get()) + + async def test_constructing_leaves_a_choice_already_made_alone( + self + ) -> None: + await self._set_suppress(True) + + await initialize(self._initialize_context()) + + self.assertTrue(await self._get()) + + async def test_constructing_twice_leaves_a_later_choice_alone( + self + ) -> None: + # The restart case: the dashboard constructs on every + # `rbt dashboard`, and the click it must not undo was made + # after the first of those. + await initialize(self._initialize_context()) + await self._set_suppress(True) + + await initialize(self._initialize_context()) + + self.assertTrue(await self._get()) + + async def test_collapsing_removes_the_state_type(self) -> None: + await self._set_expanded('bank.v1.Account', True) + await self._set_expanded('bank.v1.Bank', True) + + await self._set_expanded('bank.v1.Account', False) + + self.assertEqual(await self._expanded(), ['bank.v1.Bank']) + + async def test_expanding_twice_records_the_state_type_once(self) -> None: + # Two tabs can each send the same click, and a page that + # reconnects can send one it already sent. + await self._set_expanded('bank.v1.Account', True) + await self._set_expanded('bank.v1.Account', True) + + self.assertEqual(await self._expanded(), ['bank.v1.Account']) + + async def test_collapsing_what_was_never_expanded_is_no_error( + self + ) -> None: + await self._set_expanded('bank.v1.Account', False) + + self.assertEqual(await self._expanded(), []) + + async def test_the_order_clicked_in_does_not_change_what_is_stored( + self + ) -> None: + await self._set_expanded('bank.v1.Customer', True) + await self._set_expanded('bank.v1.Account', True) + + # Sorted, so that the reactive read does not push a change to + # every open page when the only difference is the order two + # clicks happened to arrive in. + self.assertEqual( + await self._expanded(), + ['bank.v1.Account', 'bank.v1.Customer'], + ) + + async def test_expanding_leaves_the_reopening_choice_alone(self) -> None: + # The reason there are two writers rather than one that takes + # both fields: a page that expands a state type must not write + # back a stale answer to a question it was not asked. + await self._set_suppress(True) + + await self._set_expanded('bank.v1.Account', True) + + self.assertTrue(await self._get()) + + async def test_the_reopening_choice_leaves_expansions_alone(self) -> None: + await self._set_expanded('bank.v1.Account', True) + + await self._set_suppress(True) + + self.assertEqual(await self._expanded(), ['bank.v1.Account']) + + async def test_constructing_leaves_expansions_alone(self) -> None: + await self._set_expanded('bank.v1.Account', True) + + await initialize(self._initialize_context()) + + self.assertEqual(await self._expanded(), ['bank.v1.Account']) + + +if __name__ == '__main__': + unittest.main() From 62dbe40f94825a31248b229e4fd1ba4316ca0716 Mon Sep 17 00:00:00 2001 From: Riley Scheid Date: Wed, 12 Aug 2026 04:50:45 +0000 Subject: [PATCH 04/19] Allow a state type to have a description A method can say what it does, but a state type is the sum of its state and its methods, and its name alone does not say what it is for. `Type` now takes a description, which the dashboard shows beside the state type's name and file. Co-Authored-By: Claude Opus 5 (1M context) --- rbt/dashboard/v1/dashboard.proto | 3 + rbt/v1alpha1/inspect/inspect.proto | 51 -------- rbt/v1alpha1/options.proto | 6 + reboot/api.py | 3 + reboot/dashboard/frontend/dashboard.css | 26 ++++ reboot/dashboard/frontend/src/main.tsx | 35 ++++- reboot/inspect/BUILD.bazel | 13 -- reboot/inspect/describe_state_type.py | 141 --------------------- reboot/pydantic_schema_to_proto.py | 16 ++- tests/reboot/dashboard/api/shop/v1/shop.py | 8 +- tests/reboot/dashboard/api_reader_tests.py | 5 + 11 files changed, 97 insertions(+), 210 deletions(-) delete mode 100644 reboot/inspect/describe_state_type.py diff --git a/rbt/dashboard/v1/dashboard.proto b/rbt/dashboard/v1/dashboard.proto index 38368499..3c5a8f01 100644 --- a/rbt/dashboard/v1/dashboard.proto +++ b/rbt/dashboard/v1/dashboard.proto @@ -33,6 +33,9 @@ message StateTypeInfo { string file = 2; repeated FieldInfo fields = 3; repeated MethodInfo methods = 4; + + // What the state type does, in the author's own words. + optional string description = 5; } //////////////////////////////////////////////////////////////////////// diff --git a/rbt/v1alpha1/inspect/inspect.proto b/rbt/v1alpha1/inspect/inspect.proto index 478a0e92..5031d26c 100644 --- a/rbt/v1alpha1/inspect/inspect.proto +++ b/rbt/v1alpha1/inspect/inspect.proto @@ -66,57 +66,6 @@ message GetStateResponse { //////////////////////////////////////////////////////////////////////// -// A field of a state, or an argument of a method. -message FieldInfo { - string name = 1; - - // Rendered for a reader, e.g. "str", "float", "AccountState". - string type = 2; -} - -message MethodInfo { - string name = 1; - - // One of "reader", "writer", "transaction" or "workflow". - string kind = 2; - - // The request message's fields, flattened. A method taking no - // request has none. - repeated FieldInfo arguments = 3; - - // The response's fields; empty when the method returns - // nothing. - repeated FieldInfo returns = 4; - - // Names of the error types the method declares it may raise. - repeated string errors = 5; - - // The description its author wrote, when there is one. - optional string description = 6; - - // Whether this method is a factory, constructing the state - // rather than requiring it to already exist. - bool factory = 7; - - // Whether the method is exposed as an MCP tool or resource. - bool mcp = 8; -} - -message StateTypeInfo { - // Fully qualified, e.g. "bank.v1.Account". - string name = 1; - - // The file the developer declared it in, e.g. - // "bank/v1/account.py". - string file = 2; - - repeated FieldInfo fields = 3; - - repeated MethodInfo methods = 4; -} - -//////////////////////////////////////////////////////////////////////// - service Inspect { // The list of state types in an application is static, however, we // make this a streaming RPC so that the client can hear when it diff --git a/rbt/v1alpha1/options.proto b/rbt/v1alpha1/options.proto index 9c8227ca..c6c1b261 100644 --- a/rbt/v1alpha1/options.proto +++ b/rbt/v1alpha1/options.proto @@ -163,6 +163,12 @@ message StateOptions { // `RBT_VALIDATE_TRUSTED_EFFECTS` is set. See // https://github.com/reboot-dev/mono/issues/4499. bool trusted_effects = 4; + + // What the state type does, in the author's own words. Worth + // writing about the parts a reader cannot derive, such as what it + // is the consistency boundary for, rather than restating the + // methods listed beside it. + optional string description = 5; } extend google.protobuf.MessageOptions { diff --git a/reboot/api.py b/reboot/api.py index 40c6d01e..0eda580b 100644 --- a/reboot/api.py +++ b/reboot/api.py @@ -1027,12 +1027,14 @@ class Type(pydantic.BaseModel): state: typing.Type[Model] methods: Methods + description: Optional[str] = None def __init__( self, *, state: typing.Type[Model], methods: Methods, + description: Optional[str] = None, ): def validate_all_fields_are_reboot_base_classes( @@ -1187,6 +1189,7 @@ def validate_all_fields_are_reboot_base_classes( super().__init__( state=state, methods=methods, + description=description, ) diff --git a/reboot/dashboard/frontend/dashboard.css b/reboot/dashboard/frontend/dashboard.css index f6dfb208..99ce79fa 100644 --- a/reboot/dashboard/frontend/dashboard.css +++ b/reboot/dashboard/frontend/dashboard.css @@ -252,6 +252,32 @@ header h1 { margin: 6px 0 4px; } +/* What the author says the state type is, under its name and + file. Wider measure and no padding of its own: it belongs to + the section's heading rather than to a card, the way + `.method-description` belongs to a method. */ +.state-type-description { + max-width: 62ch; + margin: 12px 0 0; + font-size: 14px; + line-height: 1.55; + color: hsl(var(--prose)); + text-wrap: pretty; +} + +/* Code inside a description, written as `backticks` by its author. + `0.9em` rather than a pixel size, so it follows whichever + description it sits in. */ +.state-type-description code, +.method-description code { + font-family: ui-monospace, Menlo, monospace; + font-size: 0.9em; + background: hsl(var(--surface-sunken)); + border: 1px solid hsl(var(--border-soft)); + border-radius: 4px; + padding: 0 4px; +} + .file { font-family: ui-monospace, Menlo, monospace; font-size: 11.5px; diff --git a/reboot/dashboard/frontend/src/main.tsx b/reboot/dashboard/frontend/src/main.tsx index b6fa0ecf..5c8a883f 100644 --- a/reboot/dashboard/frontend/src/main.tsx +++ b/reboot/dashboard/frontend/src/main.tsx @@ -75,6 +75,30 @@ const Kind: FC<{ kind: string }> = ({ kind }) => ( /> ); +// A description, with the spans its author wrote in `backticks` +// rendered as code rather than shown with their backticks. An +// unpaired backtick is kept as text, since it opens nothing. +const Description: FC<{ className: string; text: string }> = ({ + className, + text, +}) => { + const parts = text.split("`"); + return ( +

+ {parts.map((part, index) => { + // `split` alternates text and code, so odd indexes are code, + // except a last part at an odd index, whose backtick was + // never closed. + const unclosed = index === parts.length - 1 && parts.length % 2 === 0; + if (index % 2 === 1 && !unclosed) { + return {part}; + } + return {unclosed ? "`" + part : part}; + })} +

+ ); +}; + // A state type's namespace is its proto package: `bank.v1.Account` // lives in `bank.v1`, which is the developer's `api/bank/v1/`. const namespaceOf = (name: string): string => @@ -171,7 +195,10 @@ const Method: FC<{ method: MethodInfo }> = ({ method }) => {
{method.description !== undefined && ( -

{method.description}

+ )}
@@ -295,6 +322,12 @@ const StateType: FC<{
{stateType.file}
+ {stateType.description !== undefined && ( + + )}
state
{stateType.fields.length === 0 ? ( diff --git a/reboot/inspect/BUILD.bazel b/reboot/inspect/BUILD.bazel index 5083da6d..007e4e00 100644 --- a/reboot/inspect/BUILD.bazel +++ b/reboot/inspect/BUILD.bazel @@ -49,19 +49,6 @@ esbuild( platform = "browser", ) -py_library( - name = "describe_state_type_py", - srcs = ["describe_state_type.py"], - srcs_version = "PY3", - visibility = ["//visibility:public"], - deps = [ - "//log:log_py", - "//rbt/v1alpha1:options_py_proto", - "//rbt/v1alpha1/inspect:inspect_py_proto", - "//reboot/aio:types_py", - ], -) - py_library( name = "servicer_py", srcs = ["servicer.py"], diff --git a/reboot/inspect/describe_state_type.py b/reboot/inspect/describe_state_type.py deleted file mode 100644 index 62bbe43e..00000000 --- a/reboot/inspect/describe_state_type.py +++ /dev/null @@ -1,141 +0,0 @@ -"""Describes Reboot state types from protobuf descriptors. - -Everything comes from the descriptor pool it is handed: a state type's -fields, and the methods of the services that supply them. Field and -method types are rendered as the Python names a person reads on a -page, such as `int` and `list[str]`, rather than as their protobuf -spelling. -""" -from google.protobuf import descriptor_pool -from google.protobuf.descriptor import FieldDescriptor -from log.log import get_logger -from rbt.v1alpha1 import options_pb2 -from rbt.v1alpha1.inspect.inspect_pb2 import ( - FieldInfo, - MethodInfo, - StateTypeInfo, -) -from reboot.aio.types import StateTypeName -from typing import Iterable, Optional - -logger = get_logger(__name__) - -_TYPE_NAMES = { - FieldDescriptor.TYPE_DOUBLE: 'float', - FieldDescriptor.TYPE_FLOAT: 'float', - FieldDescriptor.TYPE_INT64: 'int', - FieldDescriptor.TYPE_UINT64: 'int', - FieldDescriptor.TYPE_INT32: 'int', - FieldDescriptor.TYPE_FIXED64: 'int', - FieldDescriptor.TYPE_FIXED32: 'int', - FieldDescriptor.TYPE_BOOL: 'bool', - FieldDescriptor.TYPE_STRING: 'str', - FieldDescriptor.TYPE_BYTES: 'bytes', - FieldDescriptor.TYPE_UINT32: 'int', - FieldDescriptor.TYPE_SFIXED32: 'int', - FieldDescriptor.TYPE_SFIXED64: 'int', - FieldDescriptor.TYPE_SINT32: 'int', - FieldDescriptor.TYPE_SINT64: 'int', -} - - -def _type_name(field) -> str: - """How to render `field`'s type.""" - if field.type in ( - FieldDescriptor.TYPE_MESSAGE, FieldDescriptor.TYPE_GROUP - ): - name = field.message_type.name - elif field.type == FieldDescriptor.TYPE_ENUM: - name = field.enum_type.name - else: - name = _TYPE_NAMES.get(field.type, 'unknown') - - if field.label == FieldDescriptor.LABEL_REPEATED: - return f'list[{name}]' - return name - - -def _fields_of(message) -> list[FieldInfo]: - return [ - FieldInfo(name=field.name, type=_type_name(field)) - for field in message.fields - ] - - -def _describe_method(method) -> MethodInfo: - options = method.GetOptions().Extensions[options_pb2.method] - kind = options.WhichOneof('kind') or '' - - info = MethodInfo( - name=method.name, - kind=kind, - arguments=_fields_of(method.input_type), - errors=list(options.errors), - mcp=options.HasField('mcp'), - ) - - # An empty response means the method returns nothing; saying - # "Empty" would be an implementation detail leaking out. The - # response's fields rather than its name, because a synthesized - # name such as `ShopRemainingResponse` says nothing the fields - # don't. - if method.output_type.full_name != 'google.protobuf.Empty': - info.returns.extend(_fields_of(method.output_type)) - - # An application that was created before `MethodOptions.description` - # will have the deprecated `mcp` description, which is permitted - # for backward compatibility. - if options.description: - info.description = options.description - elif options.HasField('mcp') and options.mcp.description: - info.description = options.mcp.description - - # Only writers and transactions can construct. - if kind in ('writer', 'transaction'): - info.factory = getattr(options, kind).HasField('constructor') - - return info - - -def describe_state_type( - pool: descriptor_pool.DescriptorPool, - state_type_name: StateTypeName, - service_names: Iterable[str], - file: str, -) -> Optional[StateTypeInfo]: - """Describes one state type, or `None` when its descriptors can't - be found, since a state type we can't describe shouldn't stop us - describing the rest. - - `pool` holds the state type's descriptors and those of the - services named in `service_names`, which supply its methods. - - `file` is the file the developer declared the state type in, which - is reported as it is. A Pydantic API is described from a `.proto` - synthesized from it, so the descriptors name a file that only - exists inside the build. - """ - try: - state = pool.FindMessageTypeByName(state_type_name) - except KeyError: - logger.warning( - f"No descriptor for state type '{state_type_name}'; " - "omitting it from the schema" - ) - return None - - info = StateTypeInfo( - name=state_type_name, - file=file, - fields=_fields_of(state), - ) - - for service_name in service_names: - try: - service = pool.FindServiceByName(service_name) - except KeyError: - continue - for method in service.methods: - info.methods.append(_describe_method(method)) - - return info diff --git a/reboot/pydantic_schema_to_proto.py b/reboot/pydantic_schema_to_proto.py index 2f774a1c..e42a6d32 100644 --- a/reboot/pydantic_schema_to_proto.py +++ b/reboot/pydantic_schema_to_proto.py @@ -173,6 +173,8 @@ async def generate( # Auto-construct enum value name for this state type, # or None for non-auto-constructed types. auto_construct: Optional[str] = None, + # What the state type does, in the author's own words. + description: Optional[str] = None, ): origin = get_origin(schema) args = get_args(schema) @@ -183,12 +185,19 @@ async def generate( await proto.write(f"message {name} {{\n") if state: - if uis or auto_construct: - # Generate state option with UIs and/or - # auto-construct annotation. Proto text + if uis or auto_construct or description is not None: + # Generate state option with UIs, a description + # and/or auto-construct annotation. Proto text # format uses repeated field names, not # array syntax. await proto.write(" option (rbt.v1alpha1.state) = {\n") + if description is not None: + # The description can contain `\` character, so we + # need to escape it for proto string literal. + await proto.write( + " description: " + f'"{_escape_string_for_proto(description)}"\n' + ) if auto_construct is not None: await proto.write( f" auto_construct: " @@ -811,6 +820,7 @@ async def generate_proto_file_from_api( uis=uis if uis else None, auto_construct=_PER_USER_ID if type_name == AUTO_CONSTRUCT_STATE_TYPE else None, + description=type_obj.description, ) await proto.write('\n') diff --git a/tests/reboot/dashboard/api/shop/v1/shop.py b/tests/reboot/dashboard/api/shop/v1/shop.py index f49b264d..efa1ca60 100644 --- a/tests/reboot/dashboard/api/shop/v1/shop.py +++ b/tests/reboot/dashboard/api/shop/v1/shop.py @@ -45,4 +45,10 @@ class OutOfStockError(Model): ), ) -api = API(Shop=Type(state=ShopState, methods=ShopMethods)) +api = API( + Shop=Type( + state=ShopState, + methods=ShopMethods, + description="A shop, and the stock it has to sell.", + ) +) diff --git a/tests/reboot/dashboard/api_reader_tests.py b/tests/reboot/dashboard/api_reader_tests.py index b5d3d23e..9cf31d72 100644 --- a/tests/reboot/dashboard/api_reader_tests.py +++ b/tests/reboot/dashboard/api_reader_tests.py @@ -41,6 +41,11 @@ async def test_describes_a_state_type_and_its_methods(self) -> None: os.path.join(API_DIRECTORY, 'shop/v1/shop.py'), ) + self.assertEqual( + shop['description'], + 'A shop, and the stock it has to sell.', + ) + self.assertEqual( [field['name'] for field in shop['fields']], ['name', 'open'], From 2d349ceaf08a5d9bf137cb3f4511c7916bac5257 Mon Sep 17 00:00:00 2001 From: Riley Scheid Date: Thu, 13 Aug 2026 06:23:03 +0000 Subject: [PATCH 05/19] Give the manylinux builder a Python that ships `libpython` The manylinux images build every CPython with `--disable-shared` and delete even the static `libpython` archives. The `reboot-dev-reboot` genrule links `reboot_native.node` with `-lpython3.10`, a flag emitted by `python3.10-config --ldflags --embed`. That link has never been able to succeed inside these images. CI stayed green only while Bazel's remote cache served the genrule's outputs. The first cache miss made every platform fail deterministically. On x86_64 that miss came from a runner hardware swap: it changed the `lscpu` portion of `the_environment.txt`, and with it the whole cache scope. Point `python`/`python3` at a python-build-standalone CPython 3.10, which ships `libpython3.10.so`. It is the same build `reboot/nodejs/prepare_environment.sh` downloads. `pip`/`pip3` stay on the manylinux interpreter, whose layout `auditwheel` and the wheel builds expect. `python3` and `pip` therefore deliberately name different installations. Co-Authored-By: Claude Fable 5 --- Dockerfile | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index e587b446..d698e26c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -797,11 +797,25 @@ RUN dnf install -y clang gcc-c++ && dnf clean all \ && mkdir -p /usr/lib/llvm-${CLANG_VERSION}/lib \ && ln -sf /usr/lib/clang /usr/lib/llvm-${CLANG_VERSION}/lib/clang -# The `manylinux` image comes with multiple Python versions. We'll use -# 3.10 to match our main build environment. Set up symlinks so `python` -# and `python3` point to Python 3.10. -RUN ln -sf /opt/python/cp310-cp310/bin/python /usr/local/bin/python \ - && ln -sf /opt/python/cp310-cp310/bin/python /usr/local/bin/python3 \ +# The `manylinux` image comes with multiple Python versions, but all +# of them are built `--disable-shared` with no `libpython` installed, +# and embedding Python — which building `reboot_native.node` does — +# needs `-lpython3.10` at link time. Point `python` and `python3` at a +# python-build-standalone CPython 3.10 instead, which ships +# `libpython3.10.so`; it is the same build +# `reboot/nodejs/prepare_environment.sh` uses. `pip` and `pip3` stay +# on the manylinux 3.10, whose layout `auditwheel` and the wheel +# builds expect, so `python3` and `pip` deliberately name different +# installations. +RUN set -e; \ + if [ "${TARGETARCH}" = "amd64" ]; then ARCH=x86_64; else ARCH=aarch64; fi; \ + mkdir /tmp/python-build-standalone; \ + wget -qO- "https://github.com/indygreg/python-build-standalone/releases/download/20240814/cpython-3.10.14+20240814-${ARCH}-unknown-linux-gnu-install_only.tar.gz" \ + | tar -xzf - -C /tmp/python-build-standalone \ + && mv /tmp/python-build-standalone/python /opt/reboot-python \ + && rmdir /tmp/python-build-standalone \ + && ln -sf /opt/reboot-python/bin/python3 /usr/local/bin/python \ + && ln -sf /opt/reboot-python/bin/python3 /usr/local/bin/python3 \ && ln -sf /opt/python/cp310-cp310/bin/pip /usr/local/bin/pip \ && ln -sf /opt/python/cp310-cp310/bin/pip /usr/local/bin/pip3 From f318226cc1e8ffadebe6bf2f65292a4393b5531b Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Mon, 17 Aug 2026 03:13:21 +0000 Subject: [PATCH 06/19] Take the dashboard's API directory from the `.rbtrc` `rbt dashboard` needed `--api-directory`, naming a directory the `.rbtrc` already names for `rbt generate`. Two places to say the same thing is two places to change it, and nothing tells you when only one of them moves -- the dashboard just watches a directory the rest of the tooling has stopped using. So it reads what `rbt generate` was told instead, through a new `ArgumentParser.dot_rc_arguments`, which returns what the `.rbtrc` gives any subcommand. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01QPEVMhyDRxZEuH8eiykv99 --- reboot/cli/commands/dashboard.py | 32 ++++++++++++---- reboot/cli/common/rc.py | 16 ++++++++ reboot/dashboard/api_reader.py | 2 +- reboot/dashboard/constants.py | 8 ++-- tests/reboot/cli/dashboard_tests.py | 58 +++++++++++++++++++++-------- 5 files changed, 87 insertions(+), 29 deletions(-) diff --git a/reboot/cli/commands/dashboard.py b/reboot/cli/commands/dashboard.py index 35f51ea0..43bb6d52 100644 --- a/reboot/cli/commands/dashboard.py +++ b/reboot/cli/commands/dashboard.py @@ -56,13 +56,6 @@ def dashboard_subcommands() -> list[str]: def register_dashboard(parser: ArgumentParser): add_working_directory_options(parser.subcommand('dashboard')) - parser.subcommand('dashboard').add_argument( - '--api-directory', - type=str, - required=True, - help='directory containing the API files the dashboard watches', - ) - parser.subcommand('dashboard').add_argument( '--port', type=int, @@ -71,6 +64,29 @@ def register_dashboard(parser: ArgumentParser): ) +def _api_directory(parser: ArgumentParser) -> str: + """Returns the directory holding the developer's API files, which + is the directory they tell `rbt generate` to read them from. + + Taken from there rather than named again here, so that moving the + API files is one edit and the dashboard cannot end up watching a + directory the rest of the tooling has stopped using. + """ + for argument in parser.dot_rc_arguments('generate'): + # The directory is the one thing `rbt generate` takes that is + # not a flag; its flags say where to put what it generates. + if not argument.startswith('-'): + return argument + + terminal.fail( + 'Could not tell where your API files are. `rbt dashboard` reads ' + f'that from the same place `rbt generate` does, so name a ' + f'directory for it in your {parser.dot_rc_filename}:\n' + '\n' + ' generate api/\n' + ) + + def _dashboard_env( args, parser: ArgumentParser, @@ -218,7 +234,7 @@ async def dashboard( args, parser, port=port, - api_directory=args.api_directory, + api_directory=_api_directory(parser), ) terminal.info( diff --git a/reboot/cli/common/rc.py b/reboot/cli/common/rc.py index 918dd91e..b72e5f94 100644 --- a/reboot/cli/common/rc.py +++ b/reboot/cli/common/rc.py @@ -739,6 +739,22 @@ def subcommand(self, subcommand: str) -> SubcommandParser: raise ValueError(f"Invalid subcommand '{subcommand}'") return self._subcommand_parsers[subcommand] + def dot_rc_arguments(self, subcommand: str) -> list[str]: + """Returns the arguments the '.rc' file gives a subcommand, + spelled as they are written there and in the order they appear, + and nothing at all when there is no such file. + + Only the lines that always apply: one written for a config, + such as `dev run:hmr --frontend-host=...`, is left out, because + whether that config was asked for is not known here. + """ + if self.dot_rc is None: + return [] + + flags = self._read_flags_from_dot_rc(self.dot_rc_filename, self.dot_rc) + + return flags[(subcommand, None)] + def parse_args(self) -> tuple[argparse.Namespace, list[str]]: """Pass through to top-level parser with the expanded arguments after first validating that all flags include '=' between them and their value.""" diff --git a/reboot/dashboard/api_reader.py b/reboot/dashboard/api_reader.py index dbf55816..5c677ff3 100644 --- a/reboot/dashboard/api_reader.py +++ b/reboot/dashboard/api_reader.py @@ -96,7 +96,7 @@ def describe(api_directory: str, filename: str) -> list[dict]: `Shop` yields `shop.v1.Shop`. """ # The path as the developer spelled it, joined before anything - # resolves it away: with `--api-directory=api` the file shows as + # resolves it away: with `generate api/` the file shows as # `api/bank/v1/account.py`, the path they would open. file = os.path.join(api_directory, filename) diff --git a/reboot/dashboard/constants.py b/reboot/dashboard/constants.py index 5a16d3b9..2c1b5a05 100644 --- a/reboot/dashboard/constants.py +++ b/reboot/dashboard/constants.py @@ -30,10 +30,10 @@ # run` reads it. PREFERENCES_ID = 'preferences' -# The directory the developer's API files are in, which -# `rbt dashboard` takes as `--api-directory`. Separate from the -# application's URL because the files are there long before anything -# is serving, and the dashboard is meant to be startable that early. +# The directory the developer's API files are in, as the `.rbtrc` +# spells it for `rbt generate`. Separate from the application's URL +# because the files are there long before anything is serving, and +# the dashboard is meant to be startable that early. ENVVAR_RBT_API_DIRECTORY = 'RBT_API_DIRECTORY' # The `Presence` state the dashboard page subscribes to, recording who diff --git a/tests/reboot/cli/dashboard_tests.py b/tests/reboot/cli/dashboard_tests.py index f5bea4ff..9371b03d 100644 --- a/tests/reboot/cli/dashboard_tests.py +++ b/tests/reboot/cli/dashboard_tests.py @@ -6,32 +6,58 @@ from reboot.cli.common.directories import dot_rbt_directory from reboot.cli.common.rc import ArgumentParser from reboot.dashboard.constants import DEFAULT_DASHBOARD_PORT -from tests.reboot.cli.mock_exit import ( - MockExitException, - mock_raise_instead_of_exit, -) +from tests.reboot.cli.mock_exit import mock_raise_instead_of_exit from unittest.mock import patch @patch('argparse.ArgumentParser.exit', mock_raise_instead_of_exit) class RbtDashboardTestCase(unittest.IsolatedAsyncioTestCase): - def _parse(self, state_directory: str): + def _parse(self, state_directory: str, *, rbtrc: str = 'generate api/'): + rc_file = os.path.join(state_directory, '.rbtrc') + with open(rc_file, 'w') as file: + file.write(rbtrc + '\n') + parser: ArgumentParser = cli.create_parser( + rc_file=rc_file, argv=[ 'rbt', f'--state-directory={state_directory}', 'dashboard', - '--api-directory=api', - ] + ], ) args, _ = parser.parse_args() return args, parser - async def test_api_directory_is_required(self) -> None: - parser: ArgumentParser = cli.create_parser(argv=['rbt', 'dashboard']) - with self.assertRaises(MockExitException): - parser.parse_args() + async def test_the_api_directory_comes_from_generate(self) -> None: + """Naming it twice is how the two come to disagree, so it is + named once, where `rbt generate` already needs it.""" + with tempfile.TemporaryDirectory() as state_directory: + _, parser = self._parse( + state_directory, + rbtrc=( + '# Find the API files in `api/`.\n' + 'generate api/\n' + '\n' + 'generate --python=backend/api\n' + 'generate --react=frontend/api\n' + '\n' + 'dev run --application=backend/src/main.py\n' + 'dev run:hmr --frontend-host=http://localhost:4444' + ), + ) + + self.assertEqual(dashboard._api_directory(parser), 'api/') + + async def test_an_rbtrc_that_says_nothing_about_generate(self) -> None: + with tempfile.TemporaryDirectory() as state_directory: + _, parser = self._parse( + state_directory, + rbtrc='dev run --application=backend/src/main.py', + ) + + with self.assertRaises(SystemExit): + dashboard._api_directory(parser) async def test_env_is_isolated_from_any_application(self) -> None: with tempfile.TemporaryDirectory() as state_directory: @@ -54,7 +80,7 @@ async def test_env_is_isolated_from_any_application(self) -> None: args, parser, port=DEFAULT_DASHBOARD_PORT, - api_directory=args.api_directory, + api_directory=dashboard._api_directory(parser), ) self.assertEqual(env['RBT_NAME'], 'dashboard') @@ -88,7 +114,7 @@ async def test_keys_differ_from_any_application(self) -> None: args, parser, port=DEFAULT_DASHBOARD_PORT, - api_directory=args.api_directory, + api_directory=dashboard._api_directory(parser), ) self.assertNotEqual(env['REBOOT_CRYPTO_ROOT_KEYS'], 'v1:theirs') @@ -99,7 +125,7 @@ async def test_keys_differ_from_any_application(self) -> None: args, parser, port=DEFAULT_DASHBOARD_PORT, - api_directory=args.api_directory, + api_directory=dashboard._api_directory(parser), ) self.assertEqual( env['REBOOT_CRYPTO_ROOT_KEYS'], @@ -114,13 +140,13 @@ async def test_is_told_where_the_api_files_are(self) -> None: args, parser, port=DEFAULT_DASHBOARD_PORT, - api_directory=args.api_directory, + api_directory=dashboard._api_directory(parser), ) # As the developer spelled it, so files can be shown as # `api/bank/v1/account.py`; the dashboard runs in the # working directory where that spelling resolves. - self.assertEqual(env['RBT_API_DIRECTORY'], 'api') + self.assertEqual(env['RBT_API_DIRECTORY'], 'api/') if __name__ == '__main__': From 9af6c7743eff8e0ca68ec38a29ef3f6ba4ef5bbb Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Mon, 17 Aug 2026 01:17:46 +0000 Subject: [PATCH 07/19] Add `cooperatively`, for work that never waits Work that waits on nothing -- parsing, hashing, encoding -- never gives the event loop a chance of its own, so a servicer doing it over a collection holds its process for as long as the whole collection takes, and everything else it serves waits that long. `concurrently` is the wrong tool, because there is nothing to overlap. Measured over twelve parses of a 45KB file, it left the loop unable to answer for 24ms at a stretch -- 15ms even limited to one at a time, since its tasks are scheduled together and the loop drains several before looking at anything else -- and cost 30% more wall-clock in task machinery. An `asyncio.sleep(0)` in the loop measures best, at 6ms, but invites the question of why it is there and not somewhere else. This answers it: the yield falls out of how the work was grouped, which is a decision the caller has to make anyway, and the collection bounds it the way `concurrently`'s does. It takes elements rather than awaitables, because nothing is being run: the work stays in the caller's body, where it can go on mutating whatever it likes. Note an `async for` alone will not do -- `await` on something that resolves without suspending never reaches the event loop at all, which is why the yield has to live in here. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01QPEVMhyDRxZEuH8eiykv99 --- reboot/aio/BUILD.bazel | 8 ++++++++ reboot/aio/cooperatively.py | 39 +++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 reboot/aio/cooperatively.py diff --git a/reboot/aio/BUILD.bazel b/reboot/aio/BUILD.bazel index 54ee332d..1a103f95 100644 --- a/reboot/aio/BUILD.bazel +++ b/reboot/aio/BUILD.bazel @@ -127,6 +127,13 @@ py_library( ], ) +py_library( + name = "cooperatively_py", + srcs = ["cooperatively.py"], + srcs_version = "PY3", + visibility = ["//visibility:public"], +) + py_library( name = "directories_py", srcs = ["directories.py"], @@ -567,6 +574,7 @@ py_library( ":caller_id_py", ":concurrently_py", ":contexts_py", + ":cooperatively_py", ":directories_py", ":exceptions_py", ":external_py", diff --git a/reboot/aio/cooperatively.py b/reboot/aio/cooperatively.py new file mode 100644 index 00000000..3722877c --- /dev/null +++ b/reboot/aio/cooperatively.py @@ -0,0 +1,39 @@ +import asyncio +from typing import AsyncIterator, Iterable, TypeVar + +ElementT = TypeVar("ElementT") + + +async def cooperatively( + elements: Iterable[ElementT] +) -> AsyncIterator[ElementT]: + """Returns an iterator over `elements` that leaves the event loop + free between them. + + For work that is *not* waiting on anything -- parsing, hashing, + encoding -- and so never gives the event loop a chance of its own. + A servicer doing such work over a collection holds its process for + as long as the whole collection takes, and everything else it + serves waits that long. + + `concurrently` is the wrong tool for that: it exists to overlap + work that waits, and its tasks are scheduled together, so the loop + drains several of them before it looks at anything else. Measured + over twelve parses of a 45KB file, `concurrently` left the loop + unable to answer for 24ms at a stretch (15ms even limited to one + at a time), against 6ms here -- and cost 30% more wall-clock in + task machinery for work that has no waiting to overlap. + + async for path in cooperatively(paths): + index(path) # holds the interpreter; nothing waits + + The chunk is the unit of work handed in, so the loop is left free + exactly as often as there are elements. Make them small enough + that one of them is a delay nobody minds. + """ + for element in elements: + # The whole point: `await` on something that resolves without + # suspending never reaches the event loop, so this has to be + # something that does. + await asyncio.sleep(0) + yield element From f2cfb37b4b4ff7c3522681be14efe8df81db5b0e Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Sun, 16 Aug 2026 22:28:12 +0000 Subject: [PATCH 08/19] Work out where each state type is implemented The API files say which state types exist. They say nothing about which file implements one, and the name does not say either -- `servicers.py` may implement several state types while being named after none of them. What does say is the application: Application(servicers=[AccountServicer, BankServicer, ...]) so this reads the entry point, resolves each registered servicer back to the file defining it, and asks that class what it services. Read rather than imported. Importing an application means having its generated code, its dependencies and its `sys.path`, and the dashboard is meant to work before any of that exists -- the same reason the API files are read the way they are. Driven by the API rather than by the filesystem: the API is what says which state types there are to look for, so a state type appearing or disappearing is what sets this going. `until_changes` suspends the workflow in between, so it wakes when the declarations move rather than on a timer. Recorded one state per state type, so that working out one state type's implementation neither waits on nor overwrites another's. A state type the application registers no servicer for is recorded as such rather than left looking unanalyzed. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01QPEVMhyDRxZEuH8eiykv99 --- rbt/dashboard/v1/dashboard.proto | 59 ++ reboot/cli/commands/dashboard.py | 28 + reboot/dashboard/BUILD.bazel | 15 + reboot/dashboard/constants.py | 12 + reboot/dashboard/implementation_watcher.py | 276 +++++++++ reboot/dashboard/main.py | 16 +- reboot/dashboard/servicers.py | 64 ++- tests/reboot/cli/dashboard_tests.py | 41 ++ tests/reboot/dashboard/BUILD.bazel | 11 + .../dashboard/implementation_watcher_tests.py | 532 ++++++++++++++++++ 10 files changed, 1043 insertions(+), 11 deletions(-) create mode 100644 reboot/dashboard/implementation_watcher.py create mode 100644 tests/reboot/dashboard/implementation_watcher_tests.py diff --git a/rbt/dashboard/v1/dashboard.proto b/rbt/dashboard/v1/dashboard.proto index 3c5a8f01..ce1d7c68 100644 --- a/rbt/dashboard/v1/dashboard.proto +++ b/rbt/dashboard/v1/dashboard.proto @@ -77,6 +77,48 @@ message APIWatchResponse {} //////////////////////////////////////////////////////////////////////// +// One servicer found in the developer's application, and the state +// type it services. +message ServicerInfo { + // The state type it services, spelled as `StateTypeInfo.name`. + string state_type = 1; + + // The file it is written in, spelled the way the developer would + // open it, e.g. "backend/src/account_servicer.py". + string file = 2; +} + +// What the dashboard application has read of the developer's +// application: which of their files implements each state type. +// +// Held apart from `API` because the two are read from different +// places, at different moments, and neither says anything about the +// other. An API file says a state type exists; only the application +// says where it is implemented. Whoever wants both asks for both. +message Implementation { + option (rbt.v1alpha1.state) = { + }; + + // Every servicer found, sorted. A state type with no entry here is + // one no servicer was found for -- which, before this has ever been + // written, is all of them. A state type with two is two classes + // servicing it, which is for whoever reads this to make of what + // they will. + repeated ServicerInfo servicers = 1; +} + +message ImplementationGetRequest {} + +message ImplementationGetResponse { + repeated ServicerInfo servicers = 1; +} + +message ImplementationWatchRequest {} + +message ImplementationWatchResponse {} + +//////////////////////////////////////////////////////////////////////// + // What the developer has told the dashboard about opening dashboards. // // Kept here rather than in a file under their project because it is a @@ -153,6 +195,23 @@ service APIMethods { //////////////////////////////////////////////////////////////////////// +service ImplementationMethods { + rpc Get(ImplementationGetRequest) returns (ImplementationGetResponse) { + option (rbt.v1alpha1.method).reader = { + }; + } + + // Watches the developer's application for as long as the dashboard + // application runs, working out which of their files implements + // each state type whenever one of those files changes. + rpc Watch(ImplementationWatchRequest) returns (ImplementationWatchResponse) { + option (rbt.v1alpha1.method).workflow = { + }; + } +} + +//////////////////////////////////////////////////////////////////////// + service PreferencesMethods { rpc Get(PreferencesGetRequest) returns (PreferencesGetResponse) { option (rbt.v1alpha1.method).reader = { diff --git a/reboot/cli/commands/dashboard.py b/reboot/cli/commands/dashboard.py index 43bb6d52..93eb2ede 100644 --- a/reboot/cli/commands/dashboard.py +++ b/reboot/cli/commands/dashboard.py @@ -23,6 +23,7 @@ DASHBOARD_PATH, DEFAULT_DASHBOARD_PORT, ENVVAR_RBT_API_DIRECTORY, + ENVVAR_RBT_APPLICATION, ) from reboot.settings import ( ENVVAR_RBT_DEV, @@ -87,12 +88,30 @@ def _api_directory(parser: ArgumentParser) -> str: ) +def _application(parser: ArgumentParser) -> Optional[str]: + """Returns the developer's application, which is the one they tell + `rbt dev run` to run, and `None` when they tell it none. + + Read rather than asked for again, so that moving the application + is one edit; a second place to name it is a second place to forget + to change. `None` is what an application this cannot read looks + like -- a Node.js one names no Python for the servicers to be in. + """ + for argument in parser.dot_rc_arguments('dev run'): + name, separator, value = argument.partition('=') + if name == '--application' and separator == '=': + return value + + return None + + def _dashboard_env( args, parser: ArgumentParser, *, port: int, api_directory: str, + application: Optional[str], ) -> dict[str, str]: """The environment for the dashboard application. @@ -134,6 +153,14 @@ def _dashboard_env( # directory, where that spelling resolves. composed[ENVVAR_RBT_API_DIRECTORY] = api_directory + # Where the developer's servicers are, spelled the same way and + # for the same reason. Left out of the environment entirely when + # the developer named no application, which is what tells the + # dashboard there is nothing to look for. + composed.pop(ENVVAR_RBT_APPLICATION, None) + if application is not None: + composed[ENVVAR_RBT_APPLICATION] = application + composed[ENVVAR_RBT_NAME] = DASHBOARD_STATE_DIRECTORY_NAME state_directory = ( @@ -235,6 +262,7 @@ async def dashboard( parser, port=port, api_directory=_api_directory(parser), + application=_application(parser), ) terminal.info( diff --git a/reboot/dashboard/BUILD.bazel b/reboot/dashboard/BUILD.bazel index 4a7a1d0c..8e4a976b 100644 --- a/reboot/dashboard/BUILD.bazel +++ b/reboot/dashboard/BUILD.bazel @@ -33,6 +33,20 @@ py_library( ], ) +py_library( + name = "implementation_watcher_py", + srcs = ["implementation_watcher.py"], + srcs_version = "PY3", + visibility = ["//visibility:public"], + deps = [ + ":constants_py", + "//rbt/dashboard/v1:dashboard_py_reboot", + "//reboot/aio:cooperatively_py", + "//reboot/aio:external_py", + "//reboot/cli/common:watch_py", + ], +) + py_library( name = "servicers_py", srcs = ["servicers.py"], @@ -41,6 +55,7 @@ py_library( deps = [ ":api_watcher_py", ":constants_py", + ":implementation_watcher_py", "//rbt/dashboard/v1:dashboard_py_reboot", "//reboot/aio:servicers_py", "//reboot/std/presence/v1:presence_py", diff --git a/reboot/dashboard/constants.py b/reboot/dashboard/constants.py index 2c1b5a05..545bcccd 100644 --- a/reboot/dashboard/constants.py +++ b/reboot/dashboard/constants.py @@ -25,6 +25,11 @@ # declare, as the dashboard application last read them. API_ID = 'api' +# The `Implementation` state holding which of the developer's files +# implements each state type, as the dashboard application last read +# their application. +IMPLEMENTATION_ID = 'implementation' + # The `Preferences` state holding what the developer has said about # opening dashboards: the dashboard's banner writes it and `rbt dev # run` reads it. @@ -36,6 +41,13 @@ # the dashboard is meant to be startable that early. ENVVAR_RBT_API_DIRECTORY = 'RBT_API_DIRECTORY' +# The developer's application entry point, as the `.rbtrc` spells it +# for `rbt dev run`. What the API files declare says nothing about +# which file implements a state type; the application, which registers +# the servicers, is what says. Unset when the developer named none, in +# which case no implementation is looked for. +ENVVAR_RBT_APPLICATION = 'RBT_APPLICATION' + # The `Presence` state the dashboard page subscribes to, recording who # is looking at a dashboard right now. `rbt dev run` reads it to decide # whether to open one. diff --git a/reboot/dashboard/implementation_watcher.py b/reboot/dashboard/implementation_watcher.py new file mode 100644 index 00000000..c43ccd9a --- /dev/null +++ b/reboot/dashboard/implementation_watcher.py @@ -0,0 +1,276 @@ +"""How each of the developer's state types is implemented, kept up +to date for as long as the dashboard runs. + +Their API files say which state types exist and nothing about where +one is implemented; the name does not say either, since `servicers.py` +may implement several and is named after none of them. What says is +the application, where the servicers are registered -- so this starts +at its entry point and follows its imports, collecting every class +that says what it services: + + class AccountServicer(Account.Servicer): + +Following the imports rather than reading the list handed to +`Application`, because a servicer reaches it by any number of routes +-- `servicers=servicers()`, a list built elsewhere, a name rebound +behind a conditional import -- which have one thing in common: the +file defining the servicer had to be imported for any of them to run. + +Where the walk stops is what makes this the developer's code rather +than somebody else's. A module resolves only if a root holds it, so +an import of an installed package leads nowhere, and no state type of +theirs is waiting on one: their API files declare none of those. + +Read rather than imported, because importing an application means +having its generated code, its dependencies and its `sys.path`, and +the dashboard is meant to work before any of that exists. A file at a +time through `cooperatively`, so that the dashboard goes on answering +while a large one is walked. + +And driven by the filesystem, because that is what it is a function +of: where a state type is implemented can only move when the +developer's source moves, so an edit under the roots is what wakes +this, and nothing else does. +""" +import ast +import os +from rbt.dashboard.v1.dashboard_pb2 import ServicerInfo +from rbt.dashboard.v1.dashboard_rbt import Implementation +from reboot.aio.contexts import WorkflowContext +from reboot.aio.cooperatively import cooperatively +from reboot.cli.common.watch import file_watcher +from typing import Optional + +GENERATED_SUFFIX = '_rbt' + +# Every file the developer might have written a servicer in, which is +# the rule `rbt generate` and `rbt dev run` both use for source. +SOURCE_GLOB = '**/*.py' + + +def _roots(application: str) -> list[str]: + """Returns the directories the developer's modules are found + under, which is what running the application puts first on its + path.""" + return [os.path.dirname(application)] + + +def _parse(filename: str) -> ast.Module: + with open(filename) as file: + return ast.parse(file.read()) + + +def _imports( + module: ast.Module +) -> tuple[dict[str, tuple[str, str]], list[str]]: + """Returns every symbol a file imported -- keyed by the name the + file calls it, valued by the module it came from and its name + there, so `Account` -> (`bank.v1.account_rbt`, `Account`) -- and + beside it every module the file has imported. + + Symbols come from every import, wherever it is written. One inside + an `if` or a `try` binds its name just as one at the top of the + file does, and guarding an import is common enough to be worth + reading. `ast.walk` yields the shallowest first, so a name bound + at the top of the file wins over one bound inside something. + + Since we cannot tell whether `y` in `from x import y` is a module + of its own or a name defined in `x`, `x.y` is listed as a module + too. Adding that guess is safe because a module only becomes a + file if a root holds one by that name, so `x.y` naming something + that is not a module resolves to nothing and is dropped. + """ + symbols: dict[str, tuple[str, str]] = {} + imported_modules: list[str] = [] + + for node in ast.walk(module): + # A relative import names no module of its own, and + # `from . import x` has nowhere to be read from here. + match node: + case ast.Import(names=names): + imported_modules.extend(alias.name for alias in names) + case ast.ImportFrom( + module=str(imported_module), level=0, names=names + ): + imported_modules.append(imported_module) + for alias in names: + symbols.setdefault( + alias.asname or alias.name, + (imported_module, alias.name), + ) + imported_modules.append(f'{imported_module}.{alias.name}') + + return symbols, imported_modules + + +def _resolve(imported_module: str, *, roots: list[str]) -> Optional[str]: + """Returns the file a module names if one of `roots` contains it, + and `None` otherwise. + + `None` is not a failure: the standard library and installed + packages live outside every root, so `import asyncio` resolves to + nothing and there is nothing to read. + """ + relative = imported_module.replace('.', os.sep) + + for root in roots: + for candidate in ( + os.path.join(root, relative + '.py'), + os.path.join(root, relative, '__init__.py'), + ): + if os.path.isfile(candidate): + return candidate + + return None + + +def _state_type_if_servicer( + class_definition: ast.ClassDef, *, symbols: dict[str, tuple[str, str]] +) -> Optional[str]: + """Returns the state type a class services, if it says so. + + A servicer says it by what it inherits: `Account.Servicer`, or + `Account.singleton.Servicer` for a singleton. + """ + for base in class_definition.bases: + match base: + case ( + ast.Attribute(value=ast.Name(id=name), attr='Servicer') | + ast.Attribute( + value=ast. + Attribute(value=ast.Name(id=name), attr='singleton'), + attr='Servicer', + ) + ): + match symbols.get(name): + case (str(imported_module), str(attribute) + ) if imported_module.endswith(GENERATED_SUFFIX): + package = imported_module.rsplit('.', 1)[0] + return f'{package}.{attribute}' + + return None + + +async def servicer_files( + *, + application: str, + roots: Optional[list[str]] = None, +) -> list[tuple[str, str]]: + """Returns every servicer found in the developer's application as + the state type it services and the file it is written in, sorted, + with no state type appearing twice for one file. + + A state type appearing twice is two classes servicing it, which is + for whoever reads this to make of what they will; a state type not + appearing at all is one no servicer was found for, which a file + that would not parse looks like too. The files are spelled the way + the developer would open them. + + `roots` are the directories a module may be found under, which is + both how a module name becomes a file and where the developer's + code is taken to end. It defaults to the application's own + directory, which is what running the application puts first on its + path. + """ + if roots is None: + roots = _roots(application) + + servicers: set[tuple[str, str]] = set() + read: set[str] = set() + + pending = [application] + + while len(pending) > 0: + # What the last round of imports led to. + current, pending = pending, [] + + # Parsing holds on to the interpreter for as long as it takes, + # so a file at a time leaves the dashboard free to answer. + async for filename in cooperatively(current): + if filename in read: + continue + read.add(filename) + + try: + module: ast.Module = _parse(filename) + except (SyntaxError, OSError): + # Which state type this file would have serviced is + # precisely what went unread, so there is nothing to + # say about it and it falls through to no servicer + # having been found. + continue + + # First, and on its own: `ast.walk` is breadth-first, so a + # top-level class comes out before an import nested in a + # `try`, and every name must be known before any class is + # resolved. + symbols, imported_modules = _imports(module) + + for node in ast.walk(module): + match node: + case ast.ClassDef(): + state_type = _state_type_if_servicer( + node, symbols=symbols + ) + if state_type is not None: + servicers.add((state_type, filename)) + + for imported_module in imported_modules: + resolved = _resolve(imported_module, roots=roots) + if resolved is not None: + pending.append(resolved) + + return sorted(servicers) + + +async def watch(context: WorkflowContext, *, application: str) -> None: + """Returns only when the dashboard stops, recording the servicers + in the developer's application for as long as it runs.""" + roots = _roots(application) + globs = [os.path.join(root, SOURCE_GLOB) for root in roots] + + recorded: Optional[list[tuple[str, str]]] = None + + with file_watcher() as watcher: + async for iteration in context.loop('Watch the application'): + # The watch is armed before anything is read, so a save + # made during the walk resolves `event` rather than + # arriving while nothing is listening. A watch is consumed + # by one event, so it is re-entered for each. + async with watcher.watch(globs) as event: + servicers = await servicer_files( + application=application, roots=roots + ) + + # Most edits move no servicer, and a write wakes every + # browser reading `Get`, so one is only worth making + # when the answer is different. + if servicers != recorded: + + async def record(state) -> None: + del state.servicers[:] + for state_type, file in servicers: + state.servicers.append( + ServicerInfo( + state_type=state_type, + file=file, + ) + ) + + # Written inline rather than through a method of + # its own: the workflow runs on this very state. + await Implementation.ref().per_iteration( + 'Record the servicers' + ).write(context, record) + + # After the write, so that what is remembered is + # what was recorded and not what was about to be. + recorded = servicers + + # Which save wakes this, and when, is not + # deterministic, and a replay may wait on a different + # one. Nothing depends on that: an iteration reads the + # application from scratch and writes what it finds, + # so one that runs at a different moment writes the + # same answer or a newer one. + await event diff --git a/reboot/dashboard/main.py b/reboot/dashboard/main.py index fdfc9e1b..7c24d3af 100644 --- a/reboot/dashboard/main.py +++ b/reboot/dashboard/main.py @@ -9,13 +9,14 @@ """ import asyncio from pathlib import Path -from rbt.dashboard.v1.dashboard_rbt import API, Preferences +from rbt.dashboard.v1.dashboard_rbt import API, Implementation, Preferences from rbt.std.presence.v1.presence_rbt import Presence from reboot.aio.applications import Application from reboot.aio.external import InitializeContext from reboot.dashboard.constants import ( API_ID, DASHBOARD_PATH, + IMPLEMENTATION_ID, PREFERENCES_ID, PRESENCE_ID, ) @@ -70,10 +71,15 @@ async def initialize(context: InitializeContext) -> None: # dashboard is up. await Presence.ref(PRESENCE_ID).Create(context) - # Idempotently, so that a restart of a named application finds - # the `Watch` it already spawned rather than starting a second - # watcher. - _ = await API.ref(API_ID).idempotently('watch').spawn().Watch(context) + # Idempotency is required of every mutation from `initialize`, + # and needs no alias: the key is derived from the method, the + # state id and `initialize`'s seed, which is itself derived from + # the application. So a restart finds the watchers it already + # spawned rather than starting more. + _ = await API.ref(API_ID).idempotently().spawn().Watch(context) + + _ = await Implementation.ref(IMPLEMENTATION_ID + ).idempotently().spawn().Watch(context) async def main(): diff --git a/reboot/dashboard/servicers.py b/reboot/dashboard/servicers.py index dec9cd41..5a8804e2 100644 --- a/reboot/dashboard/servicers.py +++ b/reboot/dashboard/servicers.py @@ -13,18 +13,24 @@ PreferencesSetSuppressOpenOnRestartRequest, PreferencesSetSuppressOpenOnRestartResponse, ) -from rbt.dashboard.v1.dashboard_rbt import API, Preferences +from rbt.dashboard.v1.dashboard_rbt import API, Implementation, Preferences from reboot.aio.auth.authorizers import allow from reboot.aio.contexts import ReaderContext, WorkflowContext, WriterContext from reboot.aio.servicers import Servicer -from reboot.dashboard.api_watcher import watch -from reboot.dashboard.constants import ENVVAR_RBT_API_DIRECTORY +from reboot.dashboard import api_watcher, implementation_watcher +from reboot.dashboard.constants import ( + ENVVAR_RBT_API_DIRECTORY, + ENVVAR_RBT_APPLICATION, +) class APIServicer(API.Servicer): """Holds the shape the developer's API files declare.""" def authorizer(self): + # Anyone who can reach this can already read the files it + # describes: it holds nothing but the shape of the developer's + # own API files, and only ever runs under `rbt dashboard`. return allow() async def Get( @@ -43,7 +49,8 @@ async def Watch( context: WorkflowContext, request: API.WatchRequest, ) -> API.WatchResponse: - """Reads the developer's API files when they change. + """Returns only when the dashboard stops, reading the + developer's API files whenever they change. The directory comes from the environment each time this runs, so that an `rbt dashboard` restarted against a different one @@ -52,7 +59,7 @@ async def Watch( """ api_directory = os.environ[ENVVAR_RBT_API_DIRECTORY] - await watch(context, api_directory=api_directory) + await api_watcher.watch(context, api_directory=api_directory) return API.WatchResponse() @@ -67,6 +74,47 @@ async def Update( return APIUpdateResponse() +class ImplementationServicer(Implementation.Servicer): + """Holds where each state type the developer declared is + implemented.""" + + def authorizer(self): + # Anyone who can reach this can already read the files it + # names: it holds nothing but paths into the developer's own + # checkout, and only ever runs under `rbt dashboard`. + return allow() + + async def Get( + self, + context: ReaderContext, + request: Implementation.GetRequest, + ) -> Implementation.GetResponse: + return Implementation.GetResponse(servicers=self.state.servicers) + + @classmethod + async def Watch( + cls, + context: WorkflowContext, + request: Implementation.WatchRequest, + ) -> Implementation.WatchResponse: + """Returns only when the dashboard stops, working out which of + the developer's files implements each state type. + + The application comes from the environment each time this + runs, for the same reason the API directory does. A developer + who named none gets nothing looked for, which is the normal + case for a Node.js application. + """ + application = os.environ.get(ENVVAR_RBT_APPLICATION) + + if application is not None: + await implementation_watcher.watch( + context, application=application + ) + + return Implementation.WatchResponse() + + class PreferencesServicer(Preferences.Servicer): """Holds what the developer has said about their dashboard. @@ -77,6 +125,9 @@ class PreferencesServicer(Preferences.Servicer): """ def authorizer(self): + # Nothing here is worth keeping from anyone who can reach it: + # it holds what this machine's own browser was told about + # opening dashboards, and only ever runs under `rbt dashboard`. return allow() async def Get( @@ -115,7 +166,7 @@ async def SetExpanded( def servicers() -> list[type[Servicer]]: - """The servicers that back the dashboard's own state. + """Returns the servicers that back the dashboard's own state. This state belongs to the dashboard rather than to the application being developed, so it lives in its own application and its own @@ -127,5 +178,6 @@ def servicers() -> list[type[Servicer]]: """ return [ APIServicer, + ImplementationServicer, PreferencesServicer, ] + reboot.std.presence.v1.presence.servicers() diff --git a/tests/reboot/cli/dashboard_tests.py b/tests/reboot/cli/dashboard_tests.py index 9371b03d..68b06986 100644 --- a/tests/reboot/cli/dashboard_tests.py +++ b/tests/reboot/cli/dashboard_tests.py @@ -49,6 +49,43 @@ async def test_the_api_directory_comes_from_generate(self) -> None: self.assertEqual(dashboard._api_directory(parser), 'api/') + async def test_the_application_comes_from_dev_run(self) -> None: + """Named once, where `rbt dev run` already needs it.""" + with tempfile.TemporaryDirectory() as state_directory: + args, parser = self._parse( + state_directory, + rbtrc=( + 'generate api/\n' + 'dev run --application=backend/src/main.py' + ), + ) + + env = dashboard._dashboard_env( + args, + parser, + port=DEFAULT_DASHBOARD_PORT, + api_directory=dashboard._api_directory(parser), + application=dashboard._application(parser), + ) + + self.assertEqual(env['RBT_APPLICATION'], 'backend/src/main.py') + + async def test_an_rbtrc_that_names_no_application(self) -> None: + """Somebody who names none gets a dashboard that looks for no + implementations, rather than an error.""" + with tempfile.TemporaryDirectory() as state_directory: + args, parser = self._parse(state_directory, rbtrc='generate api/') + + env = dashboard._dashboard_env( + args, + parser, + port=DEFAULT_DASHBOARD_PORT, + api_directory=dashboard._api_directory(parser), + application=dashboard._application(parser), + ) + + self.assertNotIn('RBT_APPLICATION', env) + async def test_an_rbtrc_that_says_nothing_about_generate(self) -> None: with tempfile.TemporaryDirectory() as state_directory: _, parser = self._parse( @@ -81,6 +118,7 @@ async def test_env_is_isolated_from_any_application(self) -> None: parser, port=DEFAULT_DASHBOARD_PORT, api_directory=dashboard._api_directory(parser), + application=dashboard._application(parser), ) self.assertEqual(env['RBT_NAME'], 'dashboard') @@ -115,6 +153,7 @@ async def test_keys_differ_from_any_application(self) -> None: parser, port=DEFAULT_DASHBOARD_PORT, api_directory=dashboard._api_directory(parser), + application=dashboard._application(parser), ) self.assertNotEqual(env['REBOOT_CRYPTO_ROOT_KEYS'], 'v1:theirs') @@ -126,6 +165,7 @@ async def test_keys_differ_from_any_application(self) -> None: parser, port=DEFAULT_DASHBOARD_PORT, api_directory=dashboard._api_directory(parser), + application=dashboard._application(parser), ) self.assertEqual( env['REBOOT_CRYPTO_ROOT_KEYS'], @@ -141,6 +181,7 @@ async def test_is_told_where_the_api_files_are(self) -> None: parser, port=DEFAULT_DASHBOARD_PORT, api_directory=dashboard._api_directory(parser), + application=dashboard._application(parser), ) # As the developer spelled it, so files can be shown as diff --git a/tests/reboot/dashboard/BUILD.bazel b/tests/reboot/dashboard/BUILD.bazel index c9956e23..39fa43f4 100644 --- a/tests/reboot/dashboard/BUILD.bazel +++ b/tests/reboot/dashboard/BUILD.bazel @@ -22,6 +22,17 @@ py_test( ], ) +py_test( + name = "implementation_watcher_tests_py", + srcs = ["implementation_watcher_tests.py"], + main = "implementation_watcher_tests.py", + deps = [ + "//reboot/aio:tests_py", + "//reboot/dashboard:implementation_watcher_py", + "//reboot/dashboard:main_py", + ], +) + py_test( name = "application_tests_py", srcs = [":application_tests.py"], diff --git a/tests/reboot/dashboard/implementation_watcher_tests.py b/tests/reboot/dashboard/implementation_watcher_tests.py new file mode 100644 index 00000000..85f5dde4 --- /dev/null +++ b/tests/reboot/dashboard/implementation_watcher_tests.py @@ -0,0 +1,532 @@ +"""Where a state type is implemented follows what the API declares. + +The API files say which state types exist, so a state type appearing +is what sets the dashboard looking for the file that implements it. +""" +import os +import tempfile +import unittest +from pathlib import Path +from rbt.dashboard.v1.dashboard_rbt import API, Implementation +from reboot.aio.tests import Reboot +from reboot.dashboard.constants import ( + API_ID, + ENVVAR_RBT_API_DIRECTORY, + ENVVAR_RBT_APPLICATION, + IMPLEMENTATION_ID, +) +from reboot.dashboard.implementation_watcher import servicer_files +from reboot.dashboard.main import application +from unittest.mock import patch + +API_FILE = ''' +from reboot.api import API, Field, Methods, Model, Reader, Type + + +class {state}State(Model): + name: str = Field(tag=1) + + +class LookRequest(Model): + item: str = Field(tag=1) + + +class LookResponse(Model): + found: bool = Field(tag=1) + + +{state}Methods = Methods( + look=Reader( + request=LookRequest, + response=LookResponse, + description=None, + mcp=None, + ), +) + +api = API( + {state}=Type( + state={state}State, + methods={state}Methods, + description={description}, + ) +) +''' + +SERVICER = ''' +from shop.v1.{module}_rbt import {state} + + +class {state}Servicer({state}.Servicer): + + async def look(self, context, request): + pass +''' + +SINGLETON = ''' +from shop.v1.{module}_rbt import {state} + + +class {state}Servicer({state}.singleton.Servicer): + + async def stock(self, context, request): + pass +''' + +# The two the file-finding tests use, spelled out once. +SHOP = SERVICER.format(state='Shop', module='shop') +DEPOT = SINGLETON.format(state='Depot', module='depot') + +APPLICATION = ''' +from shop_servicer import ShopServicer +from reboot.aio.applications import Application + + +async def main(): + await Application(servicers=[ShopServicer]).run() +''' + + +class ImplementationWatcherTest(unittest.IsolatedAsyncioTestCase): + + async def asyncSetUp(self) -> None: + # Both are read when the application comes up, so they have to + # exist and be named first. + self._api = tempfile.TemporaryDirectory() + self._source = tempfile.TemporaryDirectory() + self.api = Path(self._api.name) + self.source = Path(self._source.name) + + (self.source / 'shop_servicer.py').write_text( + SERVICER.format(state='Shop', module='shop') + ) + (self.source / 'main.py').write_text(APPLICATION) + + self._environment = patch.dict( + os.environ, + { + ENVVAR_RBT_API_DIRECTORY: str(self.api), + ENVVAR_RBT_APPLICATION: str(self.source / 'main.py'), + }, + ) + self._environment.start() + + self.rbt = Reboot() + await self.rbt.start() + await self.rbt.up(application(), local_envoy=True) + + async def asyncTearDown(self) -> None: + await self.rbt.stop() + self._environment.stop() + self._source.cleanup() + self._api.cleanup() + + def _declare( + self, + name: str, + *, + state: str, + description: str = 'None', + ) -> None: + path = self.api / 'shop' / 'v1' / f'{name}.py' + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + API_FILE.format( + state=state, + description='None' + if description == 'None' else repr(description), + ) + ) + + async def _servicers(self, *, satisfied): + """Returns the files recorded against each state type once + they satisfy, reading again whenever they change. + + A list per state type, because two classes servicing one is + two entries rather than anything the recording adjudicates. + """ + context = self.rbt.create_external_context(name=self.id()) + + async for response in Implementation.ref(IMPLEMENTATION_ID + ).reactively().Get(context): + servicers: dict[str, list[str]] = {} + + for servicer in response.servicers: + servicers.setdefault(servicer.state_type, + []).append(servicer.file) + + if satisfied(servicers): + return servicers + + raise AssertionError('never satisfied') + + async def test_records_the_servicer_it_finds(self) -> None: + servicers = await self._servicers( + satisfied=lambda found: 'shop.v1.Shop' in found + ) + + self.assertEqual( + servicers['shop.v1.Shop'], + [str(self.source / 'shop_servicer.py')], + ) + + async def test_a_state_type_nothing_services(self) -> None: + """A state type nothing services is one with no entry, which + is how a reader tells it apart from one that was placed.""" + self._declare('depot', state='Depot') + + servicers = await self._servicers( + satisfied=lambda found: 'shop.v1.Shop' in found + ) + + self.assertNotIn('shop.v1.Depot', servicers) + + async def test_a_state_type_two_classes_service(self) -> None: + """Both files are recorded against it, rather than one being + chosen between them.""" + (self.source / 'other_servicer.py').write_text(SHOP) + (self.source / 'main.py').write_text( + APPLICATION.replace( + 'from shop_servicer import ShopServicer', + 'from other_servicer import ShopServicer as Other\n' + 'from shop_servicer import ShopServicer', + ) + ) + + servicers = await self._servicers( + satisfied=lambda found: len(found.get('shop.v1.Shop', [])) == 2 + ) + + self.assertEqual( + servicers['shop.v1.Shop'], [ + str(self.source / 'other_servicer.py'), + str(self.source / 'shop_servicer.py'), + ] + ) + + async def test_a_servicer_written_after_the_dashboard_started( + self + ) -> None: + """The application is watched, so a servicer written while the + dashboard runs is found without a restart.""" + await self._servicers(satisfied=lambda found: 'shop.v1.Shop' in found) + + (self.source / 'depot_servicer.py').write_text(DEPOT) + (self.source / 'main.py').write_text( + APPLICATION.replace( + 'from shop_servicer import ShopServicer', + 'from depot_servicer import DepotServicer\n' + 'from shop_servicer import ShopServicer', + ) + ) + + servicers = await self._servicers( + satisfied=lambda found: 'shop.v1.Depot' in found + ) + + self.assertEqual( + servicers['shop.v1.Depot'], + [str(self.source / 'depot_servicer.py')], + ) + + async def test_what_is_declared_and_what_implements_it_are_separate( + self + ) -> None: + """Read from different places by different workflows, and so + recorded without either waiting on the other.""" + self._declare('shop', state='Shop') + + servicers = await self._servicers( + satisfied=lambda found: 'shop.v1.Shop' in found + ) + self.assertEqual( + servicers['shop.v1.Shop'], + [str(self.source / 'shop_servicer.py')], + ) + + context = self.rbt.create_external_context(name=self.id()) + + async for response in API.ref(API_ID).reactively().Get(context): + if any( + state_type.name == 'shop.v1.Shop' + for state_type in response.state_types + ): + return + + raise AssertionError("'shop.v1.Shop' was never declared") + + +class ServicerFilesTest(unittest.IsolatedAsyncioTestCase): + """Which file implements which state type.""" + + def setUp(self) -> None: + self._directory = tempfile.TemporaryDirectory() + self.directory = Path(self._directory.name) + + def tearDown(self) -> None: + self._directory.cleanup() + + def _write(self, name: str, *, source: str) -> str: + path = self.directory / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(source) + return str(path) + + async def test_finds_the_file_a_state_type_is_implemented_in(self) -> None: + self._write('shop_servicer.py', source=SHOP) + application = self._write('main.py', source=APPLICATION) + + servicers = await servicer_files(application=application) + + self.assertEqual( + servicers, + [('shop.v1.Shop', str(self.directory / 'shop_servicer.py'))], + ) + + async def test_a_singleton_says_what_it_services_the_same_way( + self + ) -> None: + self._write('depot_servicer.py', source=DEPOT) + application = self._write( + 'main.py', + source=APPLICATION.replace('shop_servicer', + 'depot_servicer').replace( + 'ShopServicer', 'DepotServicer' + ), + ) + + servicers = await servicer_files(application=application) + + self.assertEqual( + servicers, + [('shop.v1.Depot', str(self.directory / 'depot_servicer.py'))], + ) + + async def test_several_state_types_in_one_file(self) -> None: + """A file is named after at most one of the state types it + implements, which is why the application is what says.""" + self._write('servicers.py', source=SHOP + DEPOT) + application = self._write( + 'main.py', source=''' +from servicers import DepotServicer, ShopServicer +from reboot.aio.applications import Application + + +async def main(): + await Application(servicers=[ShopServicer, DepotServicer]).run() +''' + ) + + servicers = await servicer_files(application=application) + + self.assertEqual( + servicers, [ + ('shop.v1.Depot', str(self.directory / 'servicers.py')), + ('shop.v1.Shop', str(self.directory / 'servicers.py')), + ] + ) + + ################################################################### + # Reached however the application reaches it. + + async def test_an_application_that_does_not_spell_out_its_servicers( + self + ) -> None: + """Only running it would say what `servicers()` returns -- but + its module had to be imported for it to be callable at all, + which is enough.""" + self._write('servicers.py', source=SHOP) + application = self._write( + 'main.py', source=''' +from servicers import servicers +from reboot.aio.applications import Application + + +async def main(): + await Application(servicers=servicers()).run() +''' + ) + + servicers = await servicer_files(application=application) + + self.assertEqual( + servicers, + [('shop.v1.Shop', str(self.directory / 'servicers.py'))], + ) + + async def test_a_servicer_reached_through_another_module(self) -> None: + """Imported by something imported by the application.""" + self._write('shop_servicer.py', source=SHOP) + self._write( + 'servicers.py', source='from shop_servicer import ShopServicer\n' + ) + application = self._write( + 'main.py', source=''' +from servicers import ShopServicer +from reboot.aio.applications import Application + + +async def main(): + await Application(servicers=[ShopServicer]).run() +''' + ) + + servicers = await servicer_files(application=application) + + self.assertEqual( + servicers, + [('shop.v1.Shop', str(self.directory / 'shop_servicer.py'))], + ) + + async def test_an_import_that_is_not_at_the_top_of_the_file(self) -> None: + """Guarding an import is common; it binds its name just the + same.""" + self._write('shop_servicer.py', source=SHOP) + application = self._write( + 'main.py', source=''' +import os +from reboot.aio.applications import Application + +if os.environ.get('LEGACY'): + from legacy_servicer import ShopServicer +else: + from shop_servicer import ShopServicer + + +async def main(): + await Application(servicers=[ShopServicer]).run() +''' + ) + + servicers = await servicer_files(application=application) + + self.assertEqual( + servicers, + [('shop.v1.Shop', str(self.directory / 'shop_servicer.py'))], + ) + + async def test_a_servicer_in_a_package(self) -> None: + self._write('servicers/__init__.py', source='') + self._write('servicers/shop.py', source=SHOP) + application = self._write( + 'main.py', source=''' +from servicers.shop import ShopServicer +from reboot.aio.applications import Application + + +async def main(): + await Application(servicers=[ShopServicer]).run() +''' + ) + + servicers = await servicer_files(application=application) + + self.assertEqual( + servicers, + [('shop.v1.Shop', str(self.directory / 'servicers' / 'shop.py'))], + ) + + ################################################################### + # Where the walk stops. + + async def test_an_import_of_somebody_elses_package_leads_nowhere( + self + ) -> None: + """A module no root holds is not the developer's code. Reading + it is not the dashboard's business, and no state type of theirs + is waiting for it.""" + elsewhere = tempfile.TemporaryDirectory() + try: + (Path(elsewhere.name) / 'library.py').write_text(SHOP) + + application = self._write( + 'main.py', source=''' +from library import ShopServicer +from reboot.aio.applications import Application + + +async def main(): + await Application(servicers=[ShopServicer]).run() +''' + ) + + servicers = await servicer_files(application=application) + + self.assertEqual(servicers, []) + + # Named as a root, the very same import leads there. + servicers = await servicer_files( + application=application, + roots=[str(self.directory), elsewhere.name], + ) + self.assertEqual( + servicers, + [('shop.v1.Shop', str(Path(elsewhere.name) / 'library.py'))], + ) + finally: + elsewhere.cleanup() + + ################################################################### + # What it cannot place. + + async def test_a_file_that_will_not_parse(self) -> None: + """Its servicers go unfound, because which state types they + service is precisely what went unread.""" + self._write('shop_servicer.py', source='class ShopServicer(') + application = self._write('main.py', source=APPLICATION) + + servicers = await servicer_files(application=application) + + self.assertEqual(servicers, []) + + async def test_an_application_that_is_not_there(self) -> None: + servicers = await servicer_files( + application=str(self.directory / 'nowhere.py') + ) + + self.assertEqual(servicers, []) + + async def test_two_classes_servicing_the_same_state_type(self) -> None: + """Both are recorded, rather than one being chosen between + them: which one runs is not something this can see.""" + self._write('shop_servicer.py', source=SHOP) + self._write('other_servicer.py', source=SHOP) + application = self._write( + 'main.py', source=''' +from other_servicer import ShopServicer as Other +from shop_servicer import ShopServicer +from reboot.aio.applications import Application + + +async def main(): + await Application(servicers=[ShopServicer]).run() +''' + ) + + servicers = await servicer_files(application=application) + + self.assertEqual( + servicers, [ + ('shop.v1.Shop', str(self.directory / 'other_servicer.py')), + ('shop.v1.Shop', str(self.directory / 'shop_servicer.py')), + ] + ) + + async def test_a_class_that_services_nothing(self) -> None: + """A class whose base names no generated module is not a + servicer this can place.""" + self._write( + 'shop_servicer.py', source=''' +class ShopServicer(SomethingElse): + pass +''' + ) + application = self._write('main.py', source=APPLICATION) + + servicers = await servicer_files(application=application) + + self.assertEqual(servicers, []) + + +if __name__ == '__main__': + unittest.main() From 2fc04d44b2a52707bfd44afdd889175ef38affaa Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Mon, 17 Aug 2026 06:06:37 +0000 Subject: [PATCH 09/19] Parse again only the files that changed The walk parsed every file the application reaches on every save, and an edit changes one of them. It now records what each file was found to hold along with a digest of the bytes it held, and parses one again only when those bytes differ. A digest and not `st_mtime_ns`, which is only as fine as the kernel's coarse clock: measured here, 163 of 200 consecutive rewrites of a file shared an mtime, so a save landing in the same tick as a read would have left that file looking untouched for good. Reading and hashing 50 files costs 1.6ms against 74ms to parse them, so asking exactly is still nearly all of the saving. Reachability is still worked out from the application every time, but over what is already held rather than by parsing: a file that stops being imported drops out however recently it changed, and one that starts being imported is parsed for the first time. A file that will not parse is left unrecorded, so it is tried again on the next save. `File` is where the analysis will attach: it says what a file holds, and asking whether that is still true is the question a hash of each method will answer one level finer. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01QPEVMhyDRxZEuH8eiykv99 --- reboot/dashboard/implementation_watcher.py | 170 +++++++++++----- .../dashboard/implementation_watcher_tests.py | 189 +++++++++++++----- 2 files changed, 262 insertions(+), 97 deletions(-) diff --git a/reboot/dashboard/implementation_watcher.py b/reboot/dashboard/implementation_watcher.py index c43ccd9a..e7ccce55 100644 --- a/reboot/dashboard/implementation_watcher.py +++ b/reboot/dashboard/implementation_watcher.py @@ -28,12 +28,14 @@ class AccountServicer(Account.Servicer): while a large one is walked. And driven by the filesystem, because that is what it is a function -of: where a state type is implemented can only move when the -developer's source moves, so an edit under the roots is what wakes +of: where a state type is implemented can only change when the +developer's source changes, so an edit under the roots is what wakes this, and nothing else does. """ import ast +import hashlib import os +from dataclasses import dataclass from rbt.dashboard.v1.dashboard_pb2 import ServicerInfo from rbt.dashboard.v1.dashboard_rbt import Implementation from reboot.aio.contexts import WorkflowContext @@ -55,9 +57,9 @@ def _roots(application: str) -> list[str]: return [os.path.dirname(application)] -def _parse(filename: str) -> ast.Module: - with open(filename) as file: - return ast.parse(file.read()) +def _read(filename: str) -> bytes: + with open(filename, 'rb') as file: + return file.read() def _imports( @@ -151,20 +153,87 @@ def _state_type_if_servicer( return None -async def servicer_files( - *, - application: str, - roots: Optional[list[str]] = None, -) -> list[tuple[str, str]]: - """Returns every servicer found in the developer's application as - the state type it services and the file it is written in, sorted, - with no state type appearing twice for one file. +@dataclass(frozen=True, kw_only=True) +class File: + """What one of the developer's files was found to hold. + + `digest` is of the bytes it held, and is what says whether + parsing it again would say anything new. Not `st_mtime_ns`, which is only + as fine as the kernel's coarse clock -- around ten milliseconds -- + so a save landing in the same tick as a read leaves an mtime that + says nothing happened. + """ + digest: bytes + imported_modules: list[str] + state_types: list[str] + + +def _parse(source: bytes, *, digest: bytes) -> Optional[File]: + """Returns what a file holds, and `None` when it will not parse. + + A file that will not parse is left unrecorded rather than recorded + as empty, so that the next round parses it again: half-written + is the normal state of a file somebody is typing into. + """ + try: + module: ast.Module = ast.parse(source) + except SyntaxError: + return None + + # First, and on its own: `ast.walk` is breadth-first, so a + # top-level class comes out before an import nested in a `try`, + # and every name must be known before any class is resolved. + symbols, imported_modules = _imports(module) + + state_types: set[str] = set() + + for node in ast.walk(module): + match node: + case ast.ClassDef(): + state_type = _state_type_if_servicer(node, symbols=symbols) + if state_type is not None: + state_types.add(state_type) + + return File( + digest=digest, + imported_modules=imported_modules, + state_types=sorted(state_types), + ) + + +def servicers(files: dict[str, File]) -> list[tuple[str, str]]: + """Returns every servicer found, as the state type it services and + the file it is written in, sorted. A state type appearing twice is two classes servicing it, which is for whoever reads this to make of what they will; a state type not appearing at all is one no servicer was found for, which a file - that would not parse looks like too. The files are spelled the way - the developer would open them. + that would not parse looks like too. + """ + return sorted( + (state_type, filename) + for filename, file in files.items() + for state_type in file.state_types + ) + + +async def files( + *, + application: str, + roots: Optional[list[str]] = None, + known: Optional[dict[str, File]] = None, +) -> dict[str, File]: + """Returns what each file the developer's application reaches + holds, keyed by the file, spelled the way they would open it. + + Only what it reaches: a file that has stopped being imported is + absent, however recently it changed, and one that has started + being imported is parsed for the first time. + + `known` is what a previous call returned, and spares this one from + parsing a file whose bytes have not changed since. Parsing is the + expensive part -- around fifty times what reading and hashing the + bytes costs -- and an edit changes one file. `roots` are the directories a module may be found under, which is both how a module name becomes a file and where the developer's @@ -175,8 +244,10 @@ async def servicer_files( if roots is None: roots = _roots(application) - servicers: set[tuple[str, str]] = set() - read: set[str] = set() + if known is None: + known = {} + + reachable: dict[str, File] = {} pending = [application] @@ -187,40 +258,32 @@ async def servicer_files( # Parsing holds on to the interpreter for as long as it takes, # so a file at a time leaves the dashboard free to answer. async for filename in cooperatively(current): - if filename in read: + if filename in reachable: continue - read.add(filename) try: - module: ast.Module = _parse(filename) - except (SyntaxError, OSError): - # Which state type this file would have serviced is - # precisely what went unread, so there is nothing to - # say about it and it falls through to no servicer - # having been found. + source: bytes = _read(filename) + except OSError: + continue + + digest = hashlib.sha256(source).digest() + + file = known.get(filename) + + if file is None or file.digest != digest: + file = _parse(source, digest=digest) + + if file is None: continue - # First, and on its own: `ast.walk` is breadth-first, so a - # top-level class comes out before an import nested in a - # `try`, and every name must be known before any class is - # resolved. - symbols, imported_modules = _imports(module) - - for node in ast.walk(module): - match node: - case ast.ClassDef(): - state_type = _state_type_if_servicer( - node, symbols=symbols - ) - if state_type is not None: - servicers.add((state_type, filename)) - - for imported_module in imported_modules: + reachable[filename] = file + + for imported_module in file.imported_modules: resolved = _resolve(imported_module, roots=roots) if resolved is not None: pending.append(resolved) - return sorted(servicers) + return reachable async def watch(context: WorkflowContext, *, application: str) -> None: @@ -230,6 +293,7 @@ async def watch(context: WorkflowContext, *, application: str) -> None: globs = [os.path.join(root, SOURCE_GLOB) for root in roots] recorded: Optional[list[tuple[str, str]]] = None + known: dict[str, File] = {} with file_watcher() as watcher: async for iteration in context.loop('Watch the application'): @@ -238,18 +302,20 @@ async def watch(context: WorkflowContext, *, application: str) -> None: # arriving while nothing is listening. A watch is consumed # by one event, so it is re-entered for each. async with watcher.watch(globs) as event: - servicers = await servicer_files( - application=application, roots=roots + known = await files( + application=application, roots=roots, known=known ) - # Most edits move no servicer, and a write wakes every + found = servicers(known) + + # Most edits change no servicer, and a write wakes every # browser reading `Get`, so one is only worth making # when the answer is different. - if servicers != recorded: + if found != recorded: async def record(state) -> None: del state.servicers[:] - for state_type, file in servicers: + for state_type, file in found: state.servicers.append( ServicerInfo( state_type=state_type, @@ -265,12 +331,12 @@ async def record(state) -> None: # After the write, so that what is remembered is # what was recorded and not what was about to be. - recorded = servicers + recorded = found # Which save wakes this, and when, is not # deterministic, and a replay may wait on a different - # one. Nothing depends on that: an iteration reads the - # application from scratch and writes what it finds, - # so one that runs at a different moment writes the - # same answer or a newer one. + # one. Nothing depends on that: an iteration parses + # whatever has changed since the last and writes what + # it finds, so one that runs at a different moment + # writes the same answer or a newer one. await event diff --git a/tests/reboot/dashboard/implementation_watcher_tests.py b/tests/reboot/dashboard/implementation_watcher_tests.py index 85f5dde4..53213863 100644 --- a/tests/reboot/dashboard/implementation_watcher_tests.py +++ b/tests/reboot/dashboard/implementation_watcher_tests.py @@ -9,13 +9,14 @@ from pathlib import Path from rbt.dashboard.v1.dashboard_rbt import API, Implementation from reboot.aio.tests import Reboot +from reboot.dashboard import implementation_watcher from reboot.dashboard.constants import ( API_ID, ENVVAR_RBT_API_DIRECTORY, ENVVAR_RBT_APPLICATION, IMPLEMENTATION_ID, ) -from reboot.dashboard.implementation_watcher import servicer_files +from reboot.dashboard.implementation_watcher import files, servicers from reboot.dashboard.main import application from unittest.mock import patch @@ -149,24 +150,23 @@ async def _servicers(self, *, satisfied): async for response in Implementation.ref(IMPLEMENTATION_ID ).reactively().Get(context): - servicers: dict[str, list[str]] = {} + found: dict[str, list[str]] = {} for servicer in response.servicers: - servicers.setdefault(servicer.state_type, - []).append(servicer.file) + found.setdefault(servicer.state_type, []).append(servicer.file) - if satisfied(servicers): - return servicers + if satisfied(found): + return found raise AssertionError('never satisfied') async def test_records_the_servicer_it_finds(self) -> None: - servicers = await self._servicers( + found = await self._servicers( satisfied=lambda found: 'shop.v1.Shop' in found ) self.assertEqual( - servicers['shop.v1.Shop'], + found['shop.v1.Shop'], [str(self.source / 'shop_servicer.py')], ) @@ -175,11 +175,11 @@ async def test_a_state_type_nothing_services(self) -> None: is how a reader tells it apart from one that was placed.""" self._declare('depot', state='Depot') - servicers = await self._servicers( + found = await self._servicers( satisfied=lambda found: 'shop.v1.Shop' in found ) - self.assertNotIn('shop.v1.Depot', servicers) + self.assertNotIn('shop.v1.Depot', found) async def test_a_state_type_two_classes_service(self) -> None: """Both files are recorded against it, rather than one being @@ -193,12 +193,12 @@ async def test_a_state_type_two_classes_service(self) -> None: ) ) - servicers = await self._servicers( + found = await self._servicers( satisfied=lambda found: len(found.get('shop.v1.Shop', [])) == 2 ) self.assertEqual( - servicers['shop.v1.Shop'], [ + found['shop.v1.Shop'], [ str(self.source / 'other_servicer.py'), str(self.source / 'shop_servicer.py'), ] @@ -220,12 +220,12 @@ async def test_a_servicer_written_after_the_dashboard_started( ) ) - servicers = await self._servicers( + found = await self._servicers( satisfied=lambda found: 'shop.v1.Depot' in found ) self.assertEqual( - servicers['shop.v1.Depot'], + found['shop.v1.Depot'], [str(self.source / 'depot_servicer.py')], ) @@ -236,11 +236,11 @@ async def test_what_is_declared_and_what_implements_it_are_separate( recorded without either waiting on the other.""" self._declare('shop', state='Shop') - servicers = await self._servicers( + found = await self._servicers( satisfied=lambda found: 'shop.v1.Shop' in found ) self.assertEqual( - servicers['shop.v1.Shop'], + found['shop.v1.Shop'], [str(self.source / 'shop_servicer.py')], ) @@ -276,10 +276,10 @@ async def test_finds_the_file_a_state_type_is_implemented_in(self) -> None: self._write('shop_servicer.py', source=SHOP) application = self._write('main.py', source=APPLICATION) - servicers = await servicer_files(application=application) + found = servicers(await files(application=application)) self.assertEqual( - servicers, + found, [('shop.v1.Shop', str(self.directory / 'shop_servicer.py'))], ) @@ -295,10 +295,10 @@ async def test_a_singleton_says_what_it_services_the_same_way( ), ) - servicers = await servicer_files(application=application) + found = servicers(await files(application=application)) self.assertEqual( - servicers, + found, [('shop.v1.Depot', str(self.directory / 'depot_servicer.py'))], ) @@ -317,10 +317,10 @@ async def main(): ''' ) - servicers = await servicer_files(application=application) + found = servicers(await files(application=application)) self.assertEqual( - servicers, [ + found, [ ('shop.v1.Depot', str(self.directory / 'servicers.py')), ('shop.v1.Shop', str(self.directory / 'servicers.py')), ] @@ -347,10 +347,10 @@ async def main(): ''' ) - servicers = await servicer_files(application=application) + found = servicers(await files(application=application)) self.assertEqual( - servicers, + found, [('shop.v1.Shop', str(self.directory / 'servicers.py'))], ) @@ -371,10 +371,10 @@ async def main(): ''' ) - servicers = await servicer_files(application=application) + found = servicers(await files(application=application)) self.assertEqual( - servicers, + found, [('shop.v1.Shop', str(self.directory / 'shop_servicer.py'))], ) @@ -398,10 +398,10 @@ async def main(): ''' ) - servicers = await servicer_files(application=application) + found = servicers(await files(application=application)) self.assertEqual( - servicers, + found, [('shop.v1.Shop', str(self.directory / 'shop_servicer.py'))], ) @@ -419,10 +419,10 @@ async def main(): ''' ) - servicers = await servicer_files(application=application) + found = servicers(await files(application=application)) self.assertEqual( - servicers, + found, [('shop.v1.Shop', str(self.directory / 'servicers' / 'shop.py'))], ) @@ -450,22 +450,121 @@ async def main(): ''' ) - servicers = await servicer_files(application=application) + found = servicers(await files(application=application)) - self.assertEqual(servicers, []) + self.assertEqual(found, []) # Named as a root, the very same import leads there. - servicers = await servicer_files( - application=application, - roots=[str(self.directory), elsewhere.name], + found = servicers( + await files( + application=application, + roots=[str(self.directory), elsewhere.name], + ) ) self.assertEqual( - servicers, + found, [('shop.v1.Shop', str(Path(elsewhere.name) / 'library.py'))], ) finally: elsewhere.cleanup() + ################################################################### + # Parse again only what has changed. + + async def test_a_file_written_with_the_same_bytes_is_not_parsed_again( + self + ) -> None: + """Identical bytes are nothing to parse, which is only + observable as the parsing not happening.""" + servicer = self._write('shop_servicer.py', source=SHOP) + application = self._write('main.py', source=APPLICATION) + + known = await files(application=application) + + Path(servicer).write_text(SHOP) + + with patch.object( + implementation_watcher, + '_parse', + wraps=implementation_watcher._parse, + ) as parse: + await files(application=application, known=known) + + parse.assert_not_called() + + async def test_a_file_written_with_other_bytes_is_parsed_again( + self + ) -> None: + """Even with the mtime it was read with put back, which is + what a save landing in the same clock tick leaves behind.""" + servicer = self._write('shop_servicer.py', source=SHOP) + application = self._write('main.py', source=APPLICATION) + + modified = os.stat(servicer).st_mtime_ns + known = await files(application=application) + + Path(servicer).write_text(DEPOT) + os.utime(servicer, ns=(modified, modified)) + + found = servicers(await files(application=application, known=known)) + + self.assertEqual(found, [('shop.v1.Depot', servicer)]) + + async def test_a_servicer_that_stops_being_imported_is_dropped( + self + ) -> None: + """However recently it changed: what the application reaches + is what it registers.""" + self._write('shop_servicer.py', source=SHOP) + self._write('depot_servicer.py', source=DEPOT) + application = self._write( + 'main.py', + source=APPLICATION.replace( + 'from shop_servicer import ShopServicer', + 'from depot_servicer import DepotServicer\n' + 'from shop_servicer import ShopServicer', + ), + ) + + known = await files(application=application) + self.assertEqual(len(servicers(known)), 2) + + self._write('main.py', source=APPLICATION) + + found = servicers(await files(application=application, known=known)) + + self.assertEqual( + found, + [('shop.v1.Shop', str(self.directory / 'shop_servicer.py'))], + ) + + async def test_a_servicer_that_starts_being_imported_is_found( + self + ) -> None: + self._write('shop_servicer.py', source=SHOP) + application = self._write('main.py', source=APPLICATION) + + known = await files(application=application) + + self._write('depot_servicer.py', source=DEPOT) + self._write( + 'main.py', + source=APPLICATION.replace( + 'from shop_servicer import ShopServicer', + 'from depot_servicer import DepotServicer\n' + 'from shop_servicer import ShopServicer', + ), + ) + + found = servicers(await files(application=application, known=known)) + + self.assertEqual( + found, [ + ('shop.v1.Depot', str(self.directory / 'depot_servicer.py')), + ('shop.v1.Shop', str(self.directory / 'shop_servicer.py')), + ] + ) + ################################################################### # What it cannot place. @@ -475,16 +574,16 @@ async def test_a_file_that_will_not_parse(self) -> None: self._write('shop_servicer.py', source='class ShopServicer(') application = self._write('main.py', source=APPLICATION) - servicers = await servicer_files(application=application) + found = servicers(await files(application=application)) - self.assertEqual(servicers, []) + self.assertEqual(found, []) async def test_an_application_that_is_not_there(self) -> None: - servicers = await servicer_files( - application=str(self.directory / 'nowhere.py') + found = servicers( + await files(application=str(self.directory / 'nowhere.py')) ) - self.assertEqual(servicers, []) + self.assertEqual(found, []) async def test_two_classes_servicing_the_same_state_type(self) -> None: """Both are recorded, rather than one being chosen between @@ -503,10 +602,10 @@ async def main(): ''' ) - servicers = await servicer_files(application=application) + found = servicers(await files(application=application)) self.assertEqual( - servicers, [ + found, [ ('shop.v1.Shop', str(self.directory / 'other_servicer.py')), ('shop.v1.Shop', str(self.directory / 'shop_servicer.py')), ] @@ -523,9 +622,9 @@ class ShopServicer(SomethingElse): ) application = self._write('main.py', source=APPLICATION) - servicers = await servicer_files(application=application) + found = servicers(await files(application=application)) - self.assertEqual(servicers, []) + self.assertEqual(found, []) if __name__ == '__main__': From 98150b83a6afc5bf6c22fe22c6cd87f2405214b4 Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Mon, 17 Aug 2026 06:58:38 +0000 Subject: [PATCH 10/19] Record the methods each servicer defines Scaffolding for analyzing them: what a servicer is made of, and a digest of each method that says whether analyzing it again would say anything new. Nothing is analyzed yet -- a `Method` is a name and a digest, and what it calls is the field that follows. The digest is over `ast.dump` without attributes, so it is of what the method says rather than how it is laid out: reformatting it, writing a comment in it, or pushing it down the file with an edit above leave it alone. That is what will keep an application of a thousand state types from re-analyzing everything on every save, one level finer than the file digest already does for parsing. Methods are recorded in the order they are written, which is the order somebody reading the file meets them. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01QPEVMhyDRxZEuH8eiykv99 --- rbt/dashboard/v1/dashboard.proto | 20 +++ reboot/dashboard/implementation_watcher.py | 89 +++++++---- .../dashboard/implementation_watcher_tests.py | 150 ++++++++++++++---- 3 files changed, 201 insertions(+), 58 deletions(-) diff --git a/rbt/dashboard/v1/dashboard.proto b/rbt/dashboard/v1/dashboard.proto index ce1d7c68..4f16d961 100644 --- a/rbt/dashboard/v1/dashboard.proto +++ b/rbt/dashboard/v1/dashboard.proto @@ -80,12 +80,32 @@ message APIWatchResponse {} // One servicer found in the developer's application, and the state // type it services. message ServicerInfo { + // Represents one method a servicer defines, and what analyzing it + // found. + message Method { + // Represents the name of the method as the developer wrote it, + // e.g. "deposit". A method the API files declare is spelled the + // same way there. + string name = 1; + + // Represents a hash digest of the method's syntax with the lines + // and columns left out, so that a comment added above the method + // or arguments rewrapped across lines do not change it. It says + // whether the method does anything different, not whether it was + // written down differently. + bytes digest = 2; + } + // The state type it services, spelled as `StateTypeInfo.name`. string state_type = 1; // The file it is written in, spelled the way the developer would // open it, e.g. "backend/src/account_servicer.py". string file = 2; + + // Represents every method it defines, in the order they are + // written, which is the order somebody reading the file meets them. + repeated Method methods = 3; } // What the dashboard application has read of the developer's diff --git a/reboot/dashboard/implementation_watcher.py b/reboot/dashboard/implementation_watcher.py index e7ccce55..23422c88 100644 --- a/reboot/dashboard/implementation_watcher.py +++ b/reboot/dashboard/implementation_watcher.py @@ -155,25 +155,57 @@ def _state_type_if_servicer( @dataclass(frozen=True, kw_only=True) class File: - """What one of the developer's files was found to hold. + """What one of the developer's files was found to hold.""" - `digest` is of the bytes it held, and is what says whether - parsing it again would say anything new. Not `st_mtime_ns`, which is only - as fine as the kernel's coarse clock -- around ten milliseconds -- - so a save landing in the same tick as a read leaves an mtime that - says nothing happened. - """ + # Of the bytes the file held, saying whether parsing it again + # would say anything new. Not `st_mtime_ns`, which is only as fine + # as the kernel's coarse clock -- around ten milliseconds -- so a + # save landing in the same tick as a read leaves an mtime that + # says nothing happened. digest: bytes + + # Every module the file names, to be followed if a root holds it. imported_modules: list[str] - state_types: list[str] + # Every servicer the file defines. + servicers: list[ServicerInfo] + + +def _digest(node: ast.AST) -> bytes: + """Returns a digest of what a piece of syntax says. + + The digest is computed using `ast.dump` without attributes so that + the lines and columns are left out, and thus a comment added above + a method or arguments rewrapped across lines do not change the + digest. + """ + return hashlib.sha256(ast.dump(node, + include_attributes=False).encode()).digest() + + +def _methods(class_definition: ast.ClassDef) -> list[ServicerInfo.Method]: + """Returns the methods a class defines, in the order written.""" + methods = [] -def _parse(source: bytes, *, digest: bytes) -> Optional[File]: + for node in class_definition.body: + match node: + case ( + ast.FunctionDef(name=str(name)) | + ast.AsyncFunctionDef(name=str(name)) + ): + methods.append( + ServicerInfo.Method(name=name, digest=_digest(node)) + ) + + return methods + + +def _parse(source: bytes, *, digest: bytes, filename: str) -> Optional[File]: """Returns what a file holds, and `None` when it will not parse. A file that will not parse is left unrecorded rather than recorded - as empty, so that the next round parses it again: half-written - is the normal state of a file somebody is typing into. + as empty, so that the next round parses it again: half-written is + the normal state of a file somebody is typing into. """ try: module: ast.Module = ast.parse(source) @@ -185,25 +217,31 @@ def _parse(source: bytes, *, digest: bytes) -> Optional[File]: # and every name must be known before any class is resolved. symbols, imported_modules = _imports(module) - state_types: set[str] = set() + servicers = [] for node in ast.walk(module): match node: case ast.ClassDef(): state_type = _state_type_if_servicer(node, symbols=symbols) if state_type is not None: - state_types.add(state_type) + servicers.append( + ServicerInfo( + state_type=state_type, + file=filename, + methods=_methods(node), + ) + ) return File( digest=digest, imported_modules=imported_modules, - state_types=sorted(state_types), + servicers=servicers, ) -def servicers(files: dict[str, File]) -> list[tuple[str, str]]: - """Returns every servicer found, as the state type it services and - the file it is written in, sorted. +def servicers(files: dict[str, File]) -> list[ServicerInfo]: + """Returns every servicer found, sorted by the state type it + services and the file it is written in. A state type appearing twice is two classes servicing it, which is for whoever reads this to make of what they will; a state type not @@ -211,9 +249,8 @@ def servicers(files: dict[str, File]) -> list[tuple[str, str]]: that would not parse looks like too. """ return sorted( - (state_type, filename) - for filename, file in files.items() - for state_type in file.state_types + (servicer for file in files.values() for servicer in file.servicers), + key=lambda servicer: (servicer.state_type, servicer.file), ) @@ -271,7 +308,7 @@ async def files( file = known.get(filename) if file is None or file.digest != digest: - file = _parse(source, digest=digest) + file = _parse(source, digest=digest, filename=filename) if file is None: continue @@ -292,7 +329,7 @@ async def watch(context: WorkflowContext, *, application: str) -> None: roots = _roots(application) globs = [os.path.join(root, SOURCE_GLOB) for root in roots] - recorded: Optional[list[tuple[str, str]]] = None + recorded: Optional[list[ServicerInfo]] = None known: dict[str, File] = {} with file_watcher() as watcher: @@ -315,13 +352,7 @@ async def watch(context: WorkflowContext, *, application: str) -> None: async def record(state) -> None: del state.servicers[:] - for state_type, file in found: - state.servicers.append( - ServicerInfo( - state_type=state_type, - file=file, - ) - ) + state.servicers.extend(found) # Written inline rather than through a method of # its own: the workflow runs on this very state. diff --git a/tests/reboot/dashboard/implementation_watcher_tests.py b/tests/reboot/dashboard/implementation_watcher_tests.py index 53213863..8e8ad71a 100644 --- a/tests/reboot/dashboard/implementation_watcher_tests.py +++ b/tests/reboot/dashboard/implementation_watcher_tests.py @@ -7,6 +7,7 @@ import tempfile import unittest from pathlib import Path +from rbt.dashboard.v1.dashboard_pb2 import ServicerInfo from rbt.dashboard.v1.dashboard_rbt import API, Implementation from reboot.aio.tests import Reboot from reboot.dashboard import implementation_watcher @@ -16,7 +17,7 @@ ENVVAR_RBT_APPLICATION, IMPLEMENTATION_ID, ) -from reboot.dashboard.implementation_watcher import files, servicers +from reboot.dashboard.implementation_watcher import File, files, servicers from reboot.dashboard.main import application from unittest.mock import patch @@ -88,6 +89,14 @@ async def main(): ''' +def _state_types_and_files(files: dict[str, File]) -> list[tuple[str, str]]: + """Returns every servicer as the state type it services and the + file it is written in.""" + return [ + (servicer.state_type, servicer.file) for servicer in servicers(files) + ] + + class ImplementationWatcherTest(unittest.IsolatedAsyncioTestCase): async def asyncSetUp(self) -> None: @@ -140,8 +149,8 @@ def _declare( ) async def _servicers(self, *, satisfied): - """Returns the files recorded against each state type once - they satisfy, reading again whenever they change. + """Returns the servicers recorded against each state type + once they satisfy, reading again whenever they change. A list per state type, because two classes servicing one is two entries rather than anything the recording adjudicates. @@ -150,10 +159,10 @@ async def _servicers(self, *, satisfied): async for response in Implementation.ref(IMPLEMENTATION_ID ).reactively().Get(context): - found: dict[str, list[str]] = {} + found: dict[str, list[ServicerInfo]] = {} for servicer in response.servicers: - found.setdefault(servicer.state_type, []).append(servicer.file) + found.setdefault(servicer.state_type, []).append(servicer) if satisfied(found): return found @@ -166,13 +175,26 @@ async def test_records_the_servicer_it_finds(self) -> None: ) self.assertEqual( - found['shop.v1.Shop'], + [servicer.file for servicer in found['shop.v1.Shop']], [str(self.source / 'shop_servicer.py')], ) + async def test_the_methods_reach_the_state(self) -> None: + """What the browser will join against what the API files say + each state type declares.""" + found = await self._servicers( + satisfied=lambda found: 'shop.v1.Shop' in found + ) + + self.assertEqual( + [method.name for method in found['shop.v1.Shop'][0].methods], + ['look'], + ) + async def test_a_state_type_nothing_services(self) -> None: """A state type nothing services is one with no entry, which - is how a reader tells it apart from one that was placed.""" + is how a reader tells it apart from one a servicer was found + for.""" self._declare('depot', state='Depot') found = await self._servicers( @@ -198,7 +220,7 @@ async def test_a_state_type_two_classes_service(self) -> None: ) self.assertEqual( - found['shop.v1.Shop'], [ + [servicer.file for servicer in found['shop.v1.Shop']], [ str(self.source / 'other_servicer.py'), str(self.source / 'shop_servicer.py'), ] @@ -225,7 +247,7 @@ async def test_a_servicer_written_after_the_dashboard_started( ) self.assertEqual( - found['shop.v1.Depot'], + [servicer.file for servicer in found['shop.v1.Depot']], [str(self.source / 'depot_servicer.py')], ) @@ -240,7 +262,7 @@ async def test_what_is_declared_and_what_implements_it_are_separate( satisfied=lambda found: 'shop.v1.Shop' in found ) self.assertEqual( - found['shop.v1.Shop'], + [servicer.file for servicer in found['shop.v1.Shop']], [str(self.source / 'shop_servicer.py')], ) @@ -276,7 +298,7 @@ async def test_finds_the_file_a_state_type_is_implemented_in(self) -> None: self._write('shop_servicer.py', source=SHOP) application = self._write('main.py', source=APPLICATION) - found = servicers(await files(application=application)) + found = _state_types_and_files(await files(application=application)) self.assertEqual( found, @@ -295,7 +317,7 @@ async def test_a_singleton_says_what_it_services_the_same_way( ), ) - found = servicers(await files(application=application)) + found = _state_types_and_files(await files(application=application)) self.assertEqual( found, @@ -317,7 +339,7 @@ async def main(): ''' ) - found = servicers(await files(application=application)) + found = _state_types_and_files(await files(application=application)) self.assertEqual( found, [ @@ -347,7 +369,7 @@ async def main(): ''' ) - found = servicers(await files(application=application)) + found = _state_types_and_files(await files(application=application)) self.assertEqual( found, @@ -371,7 +393,7 @@ async def main(): ''' ) - found = servicers(await files(application=application)) + found = _state_types_and_files(await files(application=application)) self.assertEqual( found, @@ -398,7 +420,7 @@ async def main(): ''' ) - found = servicers(await files(application=application)) + found = _state_types_and_files(await files(application=application)) self.assertEqual( found, @@ -419,7 +441,7 @@ async def main(): ''' ) - found = servicers(await files(application=application)) + found = _state_types_and_files(await files(application=application)) self.assertEqual( found, @@ -450,12 +472,14 @@ async def main(): ''' ) - found = servicers(await files(application=application)) + found = _state_types_and_files( + await files(application=application) + ) self.assertEqual(found, []) # Named as a root, the very same import leads there. - found = servicers( + found = _state_types_and_files( await files( application=application, roots=[str(self.directory), elsewhere.name], @@ -468,6 +492,68 @@ async def main(): finally: elsewhere.cleanup() + ################################################################### + # The methods each servicer defines. + + async def test_records_the_methods_a_servicer_defines(self) -> None: + self._write('shop_servicer.py', source=SHOP) + application = self._write('main.py', source=APPLICATION) + + found = servicers(await files(application=application)) + + self.assertEqual( + [method.name for method in found[0].methods], ['look'] + ) + + async def test_a_method_reformatted_digests_the_same(self) -> None: + """The digest is over what the method says, so laying it out + differently or writing a comment in it is not a change.""" + self._write('shop_servicer.py', source=SHOP) + application = self._write('main.py', source=APPLICATION) + + before = servicers(await files(application=application)) + + self._write( + 'shop_servicer.py', + source=SHOP.replace( + 'async def look(self, context, request):\n pass', + 'async def look(\n' + ' self,\n' + ' context,\n' + ' request,\n' + ' ):\n' + ' # Nothing to look up yet.\n' + ' pass', + ), + ) + + after = servicers(await files(application=application)) + + self.assertEqual( + [method.digest for method in after[0].methods], + [method.digest for method in before[0].methods], + ) + + async def test_a_method_whose_body_changes_digests_differently( + self + ) -> None: + self._write('shop_servicer.py', source=SHOP) + application = self._write('main.py', source=APPLICATION) + + before = servicers(await files(application=application)) + + self._write( + 'shop_servicer.py', + source=SHOP.replace(' pass', ' return None'), + ) + + after = servicers(await files(application=application)) + + self.assertNotEqual( + after[0].methods[0].digest, + before[0].methods[0].digest, + ) + ################################################################### # Parse again only what has changed. @@ -506,7 +592,9 @@ async def test_a_file_written_with_other_bytes_is_parsed_again( Path(servicer).write_text(DEPOT) os.utime(servicer, ns=(modified, modified)) - found = servicers(await files(application=application, known=known)) + found = _state_types_and_files( + await files(application=application, known=known) + ) self.assertEqual(found, [('shop.v1.Depot', servicer)]) @@ -531,7 +619,9 @@ async def test_a_servicer_that_stops_being_imported_is_dropped( self._write('main.py', source=APPLICATION) - found = servicers(await files(application=application, known=known)) + found = _state_types_and_files( + await files(application=application, known=known) + ) self.assertEqual( found, @@ -556,7 +646,9 @@ async def test_a_servicer_that_starts_being_imported_is_found( ), ) - found = servicers(await files(application=application, known=known)) + found = _state_types_and_files( + await files(application=application, known=known) + ) self.assertEqual( found, [ @@ -566,7 +658,7 @@ async def test_a_servicer_that_starts_being_imported_is_found( ) ################################################################### - # What it cannot place. + # What it finds no servicer for. async def test_a_file_that_will_not_parse(self) -> None: """Its servicers go unfound, because which state types they @@ -574,12 +666,12 @@ async def test_a_file_that_will_not_parse(self) -> None: self._write('shop_servicer.py', source='class ShopServicer(') application = self._write('main.py', source=APPLICATION) - found = servicers(await files(application=application)) + found = _state_types_and_files(await files(application=application)) self.assertEqual(found, []) async def test_an_application_that_is_not_there(self) -> None: - found = servicers( + found = _state_types_and_files( await files(application=str(self.directory / 'nowhere.py')) ) @@ -602,7 +694,7 @@ async def main(): ''' ) - found = servicers(await files(application=application)) + found = _state_types_and_files(await files(application=application)) self.assertEqual( found, [ @@ -612,8 +704,8 @@ async def main(): ) async def test_a_class_that_services_nothing(self) -> None: - """A class whose base names no generated module is not a - servicer this can place.""" + """A class whose base names no generated module is not one + this recognizes as a servicer.""" self._write( 'shop_servicer.py', source=''' class ShopServicer(SomethingElse): @@ -622,7 +714,7 @@ class ShopServicer(SomethingElse): ) application = self._write('main.py', source=APPLICATION) - found = servicers(await files(application=application)) + found = _state_types_and_files(await files(application=application)) self.assertEqual(found, []) From 651337a1d81d7e8f26068d87bfb1c93608edeba9 Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Wed, 19 Aug 2026 22:49:17 +0000 Subject: [PATCH 11/19] Consider any class extending `Servicer` a servicer Which state type a servicer services takes type information: the name in front of `.Servicer` may be spelled any way an import can bind a name. So any class extending a dotted name ending in `.Servicer` is a servicer, recorded under the name the developer wrote, and type information will later replace that name with the fully qualified state type. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QPEVMhyDRxZEuH8eiykv99 --- reboot/dashboard/implementation_watcher.py | 79 +++++++++---------- .../dashboard/implementation_watcher_tests.py | 66 +++++++--------- 2 files changed, 67 insertions(+), 78 deletions(-) diff --git a/reboot/dashboard/implementation_watcher.py b/reboot/dashboard/implementation_watcher.py index 23422c88..ed6ae87f 100644 --- a/reboot/dashboard/implementation_watcher.py +++ b/reboot/dashboard/implementation_watcher.py @@ -43,8 +43,6 @@ class AccountServicer(Account.Servicer): from reboot.cli.common.watch import file_watcher from typing import Optional -GENERATED_SUFFIX = '_rbt' - # Every file the developer might have written a servicer in, which is # the rule `rbt generate` and `rbt dev run` both use for source. SOURCE_GLOB = '**/*.py' @@ -62,19 +60,13 @@ def _read(filename: str) -> bytes: return file.read() -def _imports( - module: ast.Module -) -> tuple[dict[str, tuple[str, str]], list[str]]: - """Returns every symbol a file imported -- keyed by the name the - file calls it, valued by the module it came from and its name - there, so `Account` -> (`bank.v1.account_rbt`, `Account`) -- and - beside it every module the file has imported. +def _imports(module: ast.Module) -> list[str]: + """Returns every module a file has imported. - Symbols come from every import, wherever it is written. One inside - an `if` or a `try` binds its name just as one at the top of the - file does, and guarding an import is common enough to be worth - reading. `ast.walk` yields the shallowest first, so a name bound - at the top of the file wins over one bound inside something. + Modules come from every import, wherever it is written. One + inside an `if` or a `try` imports its module just as one at the + top of the file does, and guarding an import is common enough to + be worth reading. Since we cannot tell whether `y` in `from x import y` is a module of its own or a name defined in `x`, `x.y` is listed as a module @@ -82,7 +74,6 @@ def _imports( file if a root holds one by that name, so `x.y` naming something that is not a module resolves to nothing and is dropped. """ - symbols: dict[str, tuple[str, str]] = {} imported_modules: list[str] = [] for node in ast.walk(module): @@ -96,13 +87,9 @@ def _imports( ): imported_modules.append(imported_module) for alias in names: - symbols.setdefault( - alias.asname or alias.name, - (imported_module, alias.name), - ) imported_modules.append(f'{imported_module}.{alias.name}') - return symbols, imported_modules + return imported_modules def _resolve(imported_module: str, *, roots: list[str]) -> Optional[str]: @@ -126,29 +113,41 @@ def _resolve(imported_module: str, *, roots: list[str]) -> Optional[str]: return None +def _path(expression: ast.expr) -> Optional[list[str]]: + """Returns the dotted name an expression spells -- `rbt.Shop` as + `['rbt', 'Shop']` -- and `None` when it does not spell one.""" + match expression: + case ast.Name(id=str(name)): + return [name] + case ast.Attribute(value=value, attr=str(attribute)): + prefix = _path(value) + if prefix is not None: + return prefix + [attribute] + + return None + + def _state_type_if_servicer( - class_definition: ast.ClassDef, *, symbols: dict[str, tuple[str, str]] + class_definition: ast.ClassDef, ) -> Optional[str]: - """Returns the state type a class services, if it says so. - - A servicer says it by what it inherits: `Account.Servicer`, or - `Account.singleton.Servicer` for a singleton. + """Returns the state type a class services, spelled the way the + developer wrote it, and `None` when the class services nothing. + + A servicer says it by what it inherits: any dotted name ending + in `.Servicer`, such as `Account.Servicer`, `rbt.Shop.Servicer`, + or `Account.singleton.Servicer` for a singleton. Which state + type the name in front refers to exactly takes type information, + so it is recorded as written. """ for base in class_definition.bases: match base: - case ( - ast.Attribute(value=ast.Name(id=name), attr='Servicer') | - ast.Attribute( - value=ast. - Attribute(value=ast.Name(id=name), attr='singleton'), - attr='Servicer', - ) - ): - match symbols.get(name): - case (str(imported_module), str(attribute) - ) if imported_module.endswith(GENERATED_SUFFIX): - package = imported_module.rsplit('.', 1)[0] - return f'{package}.{attribute}' + case ast.Attribute(value=value, attr='Servicer'): + path = _path(value) + if path is None: + continue + if len(path) > 1 and path[-1] == 'singleton': + path = path[:-1] + return '.'.join(path) return None @@ -215,14 +214,14 @@ def _parse(source: bytes, *, digest: bytes, filename: str) -> Optional[File]: # First, and on its own: `ast.walk` is breadth-first, so a # top-level class comes out before an import nested in a `try`, # and every name must be known before any class is resolved. - symbols, imported_modules = _imports(module) + imported_modules = _imports(module) servicers = [] for node in ast.walk(module): match node: case ast.ClassDef(): - state_type = _state_type_if_servicer(node, symbols=symbols) + state_type = _state_type_if_servicer(node) if state_type is not None: servicers.append( ServicerInfo( diff --git a/tests/reboot/dashboard/implementation_watcher_tests.py b/tests/reboot/dashboard/implementation_watcher_tests.py index 8e8ad71a..368f2845 100644 --- a/tests/reboot/dashboard/implementation_watcher_tests.py +++ b/tests/reboot/dashboard/implementation_watcher_tests.py @@ -170,24 +170,20 @@ async def _servicers(self, *, satisfied): raise AssertionError('never satisfied') async def test_records_the_servicer_it_finds(self) -> None: - found = await self._servicers( - satisfied=lambda found: 'shop.v1.Shop' in found - ) + found = await self._servicers(satisfied=lambda found: 'Shop' in found) self.assertEqual( - [servicer.file for servicer in found['shop.v1.Shop']], + [servicer.file for servicer in found['Shop']], [str(self.source / 'shop_servicer.py')], ) async def test_the_methods_reach_the_state(self) -> None: """What the browser will join against what the API files say each state type declares.""" - found = await self._servicers( - satisfied=lambda found: 'shop.v1.Shop' in found - ) + found = await self._servicers(satisfied=lambda found: 'Shop' in found) self.assertEqual( - [method.name for method in found['shop.v1.Shop'][0].methods], + [method.name for method in found['Shop'][0].methods], ['look'], ) @@ -197,11 +193,9 @@ async def test_a_state_type_nothing_services(self) -> None: for.""" self._declare('depot', state='Depot') - found = await self._servicers( - satisfied=lambda found: 'shop.v1.Shop' in found - ) + found = await self._servicers(satisfied=lambda found: 'Shop' in found) - self.assertNotIn('shop.v1.Depot', found) + self.assertNotIn('Depot', found) async def test_a_state_type_two_classes_service(self) -> None: """Both files are recorded against it, rather than one being @@ -216,11 +210,11 @@ async def test_a_state_type_two_classes_service(self) -> None: ) found = await self._servicers( - satisfied=lambda found: len(found.get('shop.v1.Shop', [])) == 2 + satisfied=lambda found: len(found.get('Shop', [])) == 2 ) self.assertEqual( - [servicer.file for servicer in found['shop.v1.Shop']], [ + [servicer.file for servicer in found['Shop']], [ str(self.source / 'other_servicer.py'), str(self.source / 'shop_servicer.py'), ] @@ -231,7 +225,7 @@ async def test_a_servicer_written_after_the_dashboard_started( ) -> None: """The application is watched, so a servicer written while the dashboard runs is found without a restart.""" - await self._servicers(satisfied=lambda found: 'shop.v1.Shop' in found) + await self._servicers(satisfied=lambda found: 'Shop' in found) (self.source / 'depot_servicer.py').write_text(DEPOT) (self.source / 'main.py').write_text( @@ -242,12 +236,10 @@ async def test_a_servicer_written_after_the_dashboard_started( ) ) - found = await self._servicers( - satisfied=lambda found: 'shop.v1.Depot' in found - ) + found = await self._servicers(satisfied=lambda found: 'Depot' in found) self.assertEqual( - [servicer.file for servicer in found['shop.v1.Depot']], + [servicer.file for servicer in found['Depot']], [str(self.source / 'depot_servicer.py')], ) @@ -258,11 +250,9 @@ async def test_what_is_declared_and_what_implements_it_are_separate( recorded without either waiting on the other.""" self._declare('shop', state='Shop') - found = await self._servicers( - satisfied=lambda found: 'shop.v1.Shop' in found - ) + found = await self._servicers(satisfied=lambda found: 'Shop' in found) self.assertEqual( - [servicer.file for servicer in found['shop.v1.Shop']], + [servicer.file for servicer in found['Shop']], [str(self.source / 'shop_servicer.py')], ) @@ -302,7 +292,7 @@ async def test_finds_the_file_a_state_type_is_implemented_in(self) -> None: self.assertEqual( found, - [('shop.v1.Shop', str(self.directory / 'shop_servicer.py'))], + [('Shop', str(self.directory / 'shop_servicer.py'))], ) async def test_a_singleton_says_what_it_services_the_same_way( @@ -321,7 +311,7 @@ async def test_a_singleton_says_what_it_services_the_same_way( self.assertEqual( found, - [('shop.v1.Depot', str(self.directory / 'depot_servicer.py'))], + [('Depot', str(self.directory / 'depot_servicer.py'))], ) async def test_several_state_types_in_one_file(self) -> None: @@ -343,8 +333,8 @@ async def main(): self.assertEqual( found, [ - ('shop.v1.Depot', str(self.directory / 'servicers.py')), - ('shop.v1.Shop', str(self.directory / 'servicers.py')), + ('Depot', str(self.directory / 'servicers.py')), + ('Shop', str(self.directory / 'servicers.py')), ] ) @@ -373,7 +363,7 @@ async def main(): self.assertEqual( found, - [('shop.v1.Shop', str(self.directory / 'servicers.py'))], + [('Shop', str(self.directory / 'servicers.py'))], ) async def test_a_servicer_reached_through_another_module(self) -> None: @@ -397,7 +387,7 @@ async def main(): self.assertEqual( found, - [('shop.v1.Shop', str(self.directory / 'shop_servicer.py'))], + [('Shop', str(self.directory / 'shop_servicer.py'))], ) async def test_an_import_that_is_not_at_the_top_of_the_file(self) -> None: @@ -424,7 +414,7 @@ async def main(): self.assertEqual( found, - [('shop.v1.Shop', str(self.directory / 'shop_servicer.py'))], + [('Shop', str(self.directory / 'shop_servicer.py'))], ) async def test_a_servicer_in_a_package(self) -> None: @@ -445,7 +435,7 @@ async def main(): self.assertEqual( found, - [('shop.v1.Shop', str(self.directory / 'servicers' / 'shop.py'))], + [('Shop', str(self.directory / 'servicers' / 'shop.py'))], ) ################################################################### @@ -487,7 +477,7 @@ async def main(): ) self.assertEqual( found, - [('shop.v1.Shop', str(Path(elsewhere.name) / 'library.py'))], + [('Shop', str(Path(elsewhere.name) / 'library.py'))], ) finally: elsewhere.cleanup() @@ -596,7 +586,7 @@ async def test_a_file_written_with_other_bytes_is_parsed_again( await files(application=application, known=known) ) - self.assertEqual(found, [('shop.v1.Depot', servicer)]) + self.assertEqual(found, [('Depot', servicer)]) async def test_a_servicer_that_stops_being_imported_is_dropped( self @@ -625,7 +615,7 @@ async def test_a_servicer_that_stops_being_imported_is_dropped( self.assertEqual( found, - [('shop.v1.Shop', str(self.directory / 'shop_servicer.py'))], + [('Shop', str(self.directory / 'shop_servicer.py'))], ) async def test_a_servicer_that_starts_being_imported_is_found( @@ -652,8 +642,8 @@ async def test_a_servicer_that_starts_being_imported_is_found( self.assertEqual( found, [ - ('shop.v1.Depot', str(self.directory / 'depot_servicer.py')), - ('shop.v1.Shop', str(self.directory / 'shop_servicer.py')), + ('Depot', str(self.directory / 'depot_servicer.py')), + ('Shop', str(self.directory / 'shop_servicer.py')), ] ) @@ -698,8 +688,8 @@ async def main(): self.assertEqual( found, [ - ('shop.v1.Shop', str(self.directory / 'other_servicer.py')), - ('shop.v1.Shop', str(self.directory / 'shop_servicer.py')), + ('Shop', str(self.directory / 'other_servicer.py')), + ('Shop', str(self.directory / 'shop_servicer.py')), ] ) From 8c2a3479b4411ceb502bb9fc623f13a28e6128f3 Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Wed, 19 Aug 2026 22:49:40 +0000 Subject: [PATCH 12/19] Carry an iteration through `Files` An iteration now carries a `Files` value through everything it does, immutable, like the `Analysis` that carries one file through it: `known` is what the previous iteration analyzed, `parsed` what this one has parsed and not yet analyzed, `analyzed` what it has finished, and `pending` the frontier: every file reached, whose imports are not yet followed, entering once and leaving once. A file depends on the file behind every one of its imports: those are the files that can change what this one means. For now every import is taken as used, since tools like `ruff` keep unused imports out of real code; narrowing to the imports whose names are used can come later if this proves too eager. Each `File` records its dependencies by the digest each had when it was read, and a known file is kept only while its own digest and every dependency's still match; otherwise it is parsed and analyzed again. A digest read once is recorded, so nobody reads the same bytes twice in one iteration. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QPEVMhyDRxZEuH8eiykv99 --- reboot/dashboard/implementation_watcher.py | 646 ++++++++++++++---- .../dashboard/implementation_watcher_tests.py | 90 ++- 2 files changed, 571 insertions(+), 165 deletions(-) diff --git a/reboot/dashboard/implementation_watcher.py b/reboot/dashboard/implementation_watcher.py index ed6ae87f..02e7068e 100644 --- a/reboot/dashboard/implementation_watcher.py +++ b/reboot/dashboard/implementation_watcher.py @@ -16,16 +16,15 @@ class AccountServicer(Account.Servicer): behind a conditional import -- which have one thing in common: the file defining the servicer had to be imported for any of them to run. -Where the walk stops is what makes this the developer's code rather -than somebody else's. A module resolves only if a root holds it, so -an import of an installed package leads nowhere, and no state type of -theirs is waiting on one: their API files declare none of those. +Where following stops is what makes this the developer's code rather +than somebody else's. A module resolves to a file only if a root +holds it, so an import of an installed package leads nowhere. Read rather than imported, because importing an application means having its generated code, its dependencies and its `sys.path`, and the dashboard is meant to work before any of that exists. A file at a time through `cooperatively`, so that the dashboard goes on answering -while a large one is walked. +while a large application is read. And driven by the filesystem, because that is what it is a function of: where a state type is implemented can only change when the @@ -35,13 +34,23 @@ class AccountServicer(Account.Servicer): import ast import hashlib import os -from dataclasses import dataclass +from dataclasses import dataclass, replace from rbt.dashboard.v1.dashboard_pb2 import ServicerInfo from rbt.dashboard.v1.dashboard_rbt import Implementation from reboot.aio.contexts import WorkflowContext from reboot.aio.cooperatively import cooperatively from reboot.cli.common.watch import file_watcher -from typing import Optional +from types import MappingProxyType +from typing import Mapping, Optional, Sequence + +# A SHA-256 digest -- of a file's bytes, or of a method's syntax -- +# saying whether what was digested has changed. +Digest = bytes + +# Suffixes of the files `rbt generate` writes. Named so that a chain +# of imports is never followed into generated code by reading it -- +# the `_rbt` module name alone says what a name from one is. +GENERATED_SUFFIXES = ('_rbt.py', '_pb2.py', '_pb2_grpc.py') # Every file the developer might have written a servicer in, which is # the rule `rbt generate` and `rbt dev run` both use for source. @@ -60,39 +69,56 @@ def _read(filename: str) -> bytes: return file.read() -def _imports(module: ast.Module) -> list[str]: - """Returns every module a file has imported. +@dataclass(frozen=True, kw_only=True) +class Imports: + """What a file's imports may make Python load.""" + + # Every module these imports may have Python load: `import a.b.c` + # loads `a.b.c`; `from x import y` loads `x`, and `x.y` too when + # `y` is a module of its own; a star import loads its module. + # Each is tried as a file under the roots, which is how the rest + # of the application's files are found. + may_load: tuple[str, ...] + + +def _imports(module: ast.Module) -> Imports: + """Returns every module a file's imports may make Python load. Modules come from every import, wherever it is written. One - inside an `if` or a `try` imports its module just as one at the + inside an `if` or a `try` loads its module just as one at the top of the file does, and guarding an import is common enough to be worth reading. - - Since we cannot tell whether `y` in `from x import y` is a module - of its own or a name defined in `x`, `x.y` is listed as a module - too. Adding that guess is safe because a module only becomes a - file if a root holds one by that name, so `x.y` naming something - that is not a module resolves to nothing and is dropped. """ - imported_modules: list[str] = [] + may_load: list[str] = [] for node in ast.walk(module): - # A relative import names no module of its own, and - # `from . import x` has nowhere to be read from here. match node: case ast.Import(names=names): - imported_modules.extend(alias.name for alias in names) - case ast.ImportFrom( - module=str(imported_module), level=0, names=names - ): - imported_modules.append(imported_module) - for alias in names: - imported_modules.append(f'{imported_module}.{alias.name}') + may_load.extend(alias.name for alias in names) - return imported_modules + case ast.ImportFrom(module=str(from_module), level=0, names=names): + base = from_module + may_load.append(base) -def _resolve(imported_module: str, *, roots: list[str]) -> Optional[str]: + for alias in names: + if alias.name == '*': + # A star import loads only its module, which + # is already listed. + continue + # Since we cannot tell whether `y` in + # `from x import y` is a module of its own or a + # name defined in `x`, `x.y` may be loaded too. + # The guess is safe: a module only becomes a file + # if a root holds one by that name, so `x.y` + # naming something that is not a module resolves + # to nothing and is dropped. + may_load.append(_join(base, alias.name)) + + return Imports(may_load=tuple(may_load)) + + +def _try_find_file_of(module: str, *, roots: Sequence[str]) -> Optional[str]: """Returns the file a module names if one of `roots` contains it, and `None` otherwise. @@ -100,7 +126,7 @@ def _resolve(imported_module: str, *, roots: list[str]) -> Optional[str]: packages live outside every root, so `import asyncio` resolves to nothing and there is nothing to read. """ - relative = imported_module.replace('.', os.sep) + relative = module.replace('.', os.sep) for root in roots: for candidate in ( @@ -127,6 +153,372 @@ def _path(expression: ast.expr) -> Optional[list[str]]: return None +def _join(base: str, *parts: str) -> str: + """Returns a module extended by more components.""" + return '.'.join([base, *parts]) + + +@dataclass(frozen=True, kw_only=True) +class ParsedFile: + """What parsing one file said, before any name in it is + resolved.""" + + # The file this is, spelled the way the developer would open it. + filename: str + + # Of the bytes the file held, saying whether parsing it again + # would say anything new. + digest: Digest + + # What the file's imports bound each of its names to. + imports: Imports + + # The syntax itself, so that a file parsed while resolving names + # is not parsed again when its own servicers are looked for. + module: ast.Module + + +@dataclass(frozen=True, kw_only=True) +class File: + """What analyzing one of the developer's files found.""" + + # The file this is, spelled the way the developer would open it. + filename: str + + # Of the bytes the file held, saying whether parsing it again + # would say anything new. + digest: Digest + + # What the file's imports bound each of its names to. + imports: Imports + + # The files this file depends on, by the digest each had when it + # was read. If any of them changes, this file needs reanalyzing. + dependencies: Mapping[str, Digest] + + # Every servicer the file defines. + servicers: tuple[ServicerInfo, ...] + + +@dataclass(frozen=True, kw_only=True) +class Files: + """The developer's files, as far as one iteration of the watch + has taken them. + + An iteration is one call to `files()`, which `watch` makes each + time a save wakes it. + + Immutable: carrying the iteration forward means holding the one a + method below returned. + """ + + # Where the developer's modules are found: the application's own + # directory. What an iteration is allowed to analyze. + roots: tuple[str, ...] + + # What the previous iteration analyzed, so a file unchanged + # since is neither parsed nor analyzed again. + known: Mapping[str, File] + + # Parsed this iteration, not yet analyzed. + parsed: Mapping[str, ParsedFile] + + # Analyzed: this iteration, each servicer and method resolved + # against bytes as they are on disk right now -- or a previous + # one, verified unchanged. What an iteration returns. + analyzed: Mapping[str, File] + + # Reached this iteration, imports not yet followed. The loop's + # frontier: every file enters once, when it is parsed or kept, + # and leaves once, handled. + pending: frozenset[str] + + # The digest of every file read this iteration, by filename -- + # including files that joined no other map. What spares a second + # read of the same bytes. The bytes themselves are not kept: they + # are needed again only when a file changed and must be parsed, + # which a save makes rare, and the parse path reads its own. + digests: Mapping[str, Digest] + + @classmethod + def create( + cls, + *, + roots: Sequence[str], + known: Mapping[str, File], + ) -> 'Files': + """Returns the files an iteration starts from: nothing + parsed, nothing analyzed.""" + return cls( + roots=tuple(roots), + known=MappingProxyType(dict(known)), + parsed=MappingProxyType({}), + analyzed=MappingProxyType({}), + pending=frozenset(), + digests=MappingProxyType({}), + ) + + def lookup(self, filename: str) -> Optional[ParsedFile | File]: + """Returns what this iteration read for a file, wherever it + lives -- `analyzed` or `parsed` -- and `None` when it has + not read the file.""" + for entries in (self.analyzed, self.parsed): + entry = entries.get(filename) + if entry is not None: + return entry + + return None + + def with_parsed_file(self, parsed: ParsedFile) -> 'Files': + """Returns this with one more file parsed, into `parsed` and + onto the frontier.""" + return replace( + self, + parsed=MappingProxyType({ + **self.parsed, parsed.filename: parsed + }), + pending=self.pending | {parsed.filename}, + ) + + def with_reused_known_file(self, file: File) -> 'Files': + """Returns this with a file the previous iteration analyzed + kept: verified unchanged, so analyzed as it was -- and onto + the frontier, its imports still to follow.""" + return replace( + self, + analyzed=MappingProxyType({ + **self.analyzed, file.filename: file + }), + pending=self.pending | {file.filename}, + ) + + def with_analyzed_file( + self, + filename: str, + *, + dependencies: Mapping[str, Digest], + servicers: Sequence[ServicerInfo], + ) -> 'Files': + """Returns this with a file analyzed: a `File` built from + its `parsed` entry and what analyzing it found, moved to + `analyzed`.""" + parsed = self.parsed[filename] + + file = File( + filename=parsed.filename, + digest=parsed.digest, + imports=parsed.imports, + dependencies=MappingProxyType(dict(dependencies)), + servicers=tuple(servicers), + ) + + parsed_files = dict(self.parsed) + del parsed_files[filename] + + return replace( + self, + parsed=MappingProxyType(parsed_files), + analyzed=MappingProxyType({ + **self.analyzed, filename: file + }), + ) + + def without_pending_file(self, filename: str) -> 'Files': + """Returns this with a file taken off the frontier: analyzed, + and its imports followed.""" + return replace(self, pending=self.pending - {filename}) + + def with_digest(self, filename: str, digest: Digest) -> 'Files': + """Returns this with the digest of one read file recorded, so + a later question about the same bytes is answered without the + disk.""" + return replace( + self, + digests=MappingProxyType({ + **self.digests, filename: digest + }), + ) + + def needs_reanalyzing(self, file: File, *, + digest: Digest) -> tuple[bool, 'Files']: + """Returns whether a file the previous iteration analyzed + cannot be reused -- its bytes changed, or a file it depends + on did, including one that can no longer be read at all -- + and this with every digest read to answer recorded. + + A dependency already read this iteration, reached or merely + digested, is checked against the digest recorded for it, so + answering never reads the same bytes twice. + """ + if digest != file.digest: + return True, self + + files = self + + for filename, previous_digest in file.dependencies.items(): + found = files.lookup(filename) + if found is not None: + if found.digest != previous_digest: + return True, files + continue + + current = files.digests.get(filename) + if current is None: + try: + current = hashlib.sha256(_read(filename)).digest() + except OSError: + return True, files + files = files.with_digest(filename, current) + + if current != previous_digest: + return True, files + + return False, files + + def lookup_or_parse_filename( + self, filename: str + ) -> tuple[Optional[ParsedFile | File], 'Files']: + """Returns a file as this iteration has it -- looked up among + what is already read, verified against the previous iteration + when unchanged, and otherwise read and parsed, joining + `parsed`. `None` when it cannot be read or will not parse. + """ + found = self.lookup(filename) + if found is not None: + return found, self + + files = self + source: Optional[bytes] = None + + digest = files.digests.get(filename) + if digest is None: + try: + source = _read(filename) + except OSError: + return None, files + digest = hashlib.sha256(source).digest() + files = files.with_digest(filename, digest) + + known = files.known.get(filename) + if known is not None: + reanalyze, files = files.needs_reanalyzing(known, digest=digest) + if not reanalyze: + return known, files.with_reused_known_file(known) + + # Parsing needs the bytes, which a check that recorded only + # the digest did not keep. + if source is None: + try: + source = _read(filename) + except OSError: + return None, files + digest = hashlib.sha256(source).digest() + files = files.with_digest(filename, digest) + + parsed = _parse(source, filename=filename, digest=digest) + if parsed is None: + return None, files + + return parsed, files.with_parsed_file(parsed) + + def lookup_or_parse_module( + self, module: str + ) -> tuple[Optional[ParsedFile | File], 'Files']: + """Returns the file a module names as this iteration has it, + and `None` when no root contains the module or the module is + generated, which its name alone says. When it returns `None`, + the files come back as they were: nothing found means + nothing joined. + """ + filename = _try_find_file_of(module, roots=self.roots) + + if filename is None or filename.endswith(GENERATED_SUFFIXES): + return None, self + + return self.lookup_or_parse_filename(filename) + + def imports(self, filename: str) -> Optional[Imports]: + """Returns what a file's imports bound, wherever this + iteration read the file.""" + found = self.lookup(filename) + return found.imports if found is not None else None + + +def _parse(source: bytes, *, filename: str, + digest: Digest) -> Optional[ParsedFile]: + """Returns what parsing one file said, and `None` when it will + not parse. + + A file that will not parse is left unrecorded rather than recorded + as empty, so that the next iteration parses it again: half-written + is the normal state of a file somebody is typing into. + """ + try: + module: ast.Module = ast.parse(source) + except SyntaxError: + return None + + return ParsedFile( + filename=filename, + digest=digest, + imports=_imports(module), + module=module, + ) + + +@dataclass(frozen=True, kw_only=True) +class Analysis: + """One file's analysis in flight: what it has found, what it has + read, and the calls of the method it is inside. + + Born when a file's analysis starts, finished into `Files` when + the file is done. Immutable: carrying the analysis forward means + keeping the one a method below returned, so nobody else's copy is + ever changed under them. + """ + + # The file being analyzed, whose imports are what names resolve + # against. Moves while a chain of imports is followed, since the + # rest of a chain resolves against each followed file's own + # imports, and comes back when the chain answers. + filename: str + + # The developer's files, read and grown as the analysis resolves + # names, and handed back to the iteration when the file is done. + files: Files + + # The files the analysis's answers depend on, as read so far, by + # the digest each had. If any of them changes, the file needs + # analyzing again. + dependencies: Mapping[str, Digest] + + @classmethod + def create(cls, *, filename: str, files: Files) -> 'Analysis': + """Returns the analysis a file starts from: nothing found, + nothing read, no method begun.""" + return cls( + filename=filename, + files=files, + dependencies=MappingProxyType({}), + ) + + def with_dependency( + self, found: ParsedFile | File, *, files: Files + ) -> 'Analysis': + """Returns this analysis depending on one more file: the + digest it was read with recorded, and the files grown by + reading it carried forward.""" + return replace( + self, + dependencies=MappingProxyType( + { + **self.dependencies, found.filename: found.digest + } + ), + files=files, + ) + + def _state_type_if_servicer( class_definition: ast.ClassDef, ) -> Optional[str]: @@ -152,25 +544,7 @@ def _state_type_if_servicer( return None -@dataclass(frozen=True, kw_only=True) -class File: - """What one of the developer's files was found to hold.""" - - # Of the bytes the file held, saying whether parsing it again - # would say anything new. Not `st_mtime_ns`, which is only as fine - # as the kernel's coarse clock -- around ten milliseconds -- so a - # save landing in the same tick as a read leaves an mtime that - # says nothing happened. - digest: bytes - - # Every module the file names, to be followed if a root holds it. - imported_modules: list[str] - - # Every servicer the file defines. - servicers: list[ServicerInfo] - - -def _digest(node: ast.AST) -> bytes: +def _digest(node: ast.AST) -> Digest: """Returns a digest of what a piece of syntax says. The digest is computed using `ast.dump` without attributes so that @@ -182,63 +556,72 @@ def _digest(node: ast.AST) -> bytes: include_attributes=False).encode()).digest() -def _methods(class_definition: ast.ClassDef) -> list[ServicerInfo.Method]: - """Returns the methods a class defines, in the order written.""" - methods = [] +def _depend_on_imports(parsed: ParsedFile, *, analysis: Analysis) -> Analysis: + """Returns the analysis depending on the file behind every one + of the file's imports: those are the files that can change what + this one means. Each is read one hop, only for the digest + recorded. - for node in class_definition.body: - match node: - case ( - ast.FunctionDef(name=str(name)) | - ast.AsyncFunctionDef(name=str(name)) - ): - methods.append( - ServicerInfo.Method(name=name, digest=_digest(node)) - ) + For now every import is taken as used: tools like `ruff` keep + unused imports out of real code, so checking which names are + used would narrow little, and it can be narrowed later if this + proves too eager.""" + for module in dict.fromkeys(parsed.imports.may_load): + found, files = analysis.files.lookup_or_parse_module(module) + if found is None: + continue + analysis = analysis.with_dependency(found, files=files) - return methods + return analysis -def _parse(source: bytes, *, digest: bytes, filename: str) -> Optional[File]: - """Returns what a file holds, and `None` when it will not parse. +def _analyze_file(filename: str, files: Files) -> Files: + """Returns the iteration carried past analyzing one `pending` + file: every class saying what it services recorded, every method + those classes define analyzed and recorded under them, and the + file finished into `analyzed` with the files the analysis read + depended on. - A file that will not parse is left unrecorded rather than recorded - as empty, so that the next round parses it again: half-written is - the normal state of a file somebody is typing into. + One `Analysis` is carried through the whole file: the + dependencies it accumulates are the file's, while the method + resets as each one is recorded. """ - try: - module: ast.Module = ast.parse(source) - except SyntaxError: - return None + parsed = files.parsed[filename] - # First, and on its own: `ast.walk` is breadth-first, so a - # top-level class comes out before an import nested in a `try`, - # and every name must be known before any class is resolved. - imported_modules = _imports(module) + analysis = Analysis.create(filename=filename, files=files) - servicers = [] + analysis = _depend_on_imports(parsed, analysis=analysis) - for node in ast.walk(module): + servicers: list[ServicerInfo] = [] + + for node in ast.walk(parsed.module): match node: case ast.ClassDef(): state_type = _state_type_if_servicer(node) - if state_type is not None: - servicers.append( - ServicerInfo( - state_type=state_type, - file=filename, - methods=_methods(node), - ) - ) - - return File( - digest=digest, - imported_modules=imported_modules, + if state_type is None: + continue + servicer = ServicerInfo(state_type=state_type, file=filename) + for statement in node.body: + match statement: + case ( + ast.FunctionDef(name=str(name)) | + ast.AsyncFunctionDef(name=str(name)) + ): + servicer.methods.append( + ServicerInfo.Method( + name=name, digest=_digest(statement) + ) + ) + servicers.append(servicer) + + return analysis.files.with_analyzed_file( + filename, + dependencies=dict(analysis.dependencies), servicers=servicers, ) -def servicers(files: dict[str, File]) -> list[ServicerInfo]: +def servicers(files: Mapping[str, File]) -> list[ServicerInfo]: """Returns every servicer found, sorted by the state type it services and the file it is written in. @@ -253,11 +636,11 @@ def servicers(files: dict[str, File]) -> list[ServicerInfo]: ) -async def files( +async def analyze( *, application: str, - roots: Optional[list[str]] = None, - known: Optional[dict[str, File]] = None, + roots: Optional[Sequence[str]] = None, + known: Optional[Mapping[str, File]] = None, ) -> dict[str, File]: """Returns what each file the developer's application reaches holds, keyed by the file, spelled the way they would open it. @@ -266,10 +649,9 @@ async def files( absent, however recently it changed, and one that has started being imported is parsed for the first time. - `known` is what a previous call returned, and spares this one from - parsing a file whose bytes have not changed since. Parsing is the - expensive part -- around fifty times what reading and hashing the - bytes costs -- and an edit changes one file. + `known` is what a previous call returned, and spares this one + from parsing and analyzing a file that has not changed since -- + neither in its own bytes nor in any file it depends on. `roots` are the directories a module may be found under, which is both how a module name becomes a file and where the developer's @@ -280,46 +662,36 @@ async def files( if roots is None: roots = _roots(application) - if known is None: - known = {} - - reachable: dict[str, File] = {} + files = Files.create(roots=roots, known=known or {}) - pending = [application] + _, files = files.lookup_or_parse_filename(application) - while len(pending) > 0: - # What the last round of imports led to. - current, pending = pending, [] - - # Parsing holds on to the interpreter for as long as it takes, - # so a file at a time leaves the dashboard free to answer. - async for filename in cooperatively(current): - if filename in reachable: - continue + while len(files.pending) > 0: + # Parsing and analyzing hold on to the interpreter for as + # long as they take, so a file at a time leaves the dashboard + # free to answer. + async for filename in cooperatively(files.pending): + match files.parsed.get(filename): + case ParsedFile(): + files = _analyze_file(filename, files) + case None: + # Nothing to analyze: the file was kept because + # neither its bytes nor any of its dependencies + # changed, so the analysis it got when it last + # changed still stands, in `analyzed` already. + pass - try: - source: bytes = _read(filename) - except OSError: - continue - - digest = hashlib.sha256(source).digest() - - file = known.get(filename) - - if file is None or file.digest != digest: - file = _parse(source, digest=digest, filename=filename) - - if file is None: - continue + files = files.without_pending_file(filename) - reachable[filename] = file + # Following the file's imports is how the rest of the + # application is reached. + imports = files.imports(filename) + assert imports is not None - for imported_module in file.imported_modules: - resolved = _resolve(imported_module, roots=roots) - if resolved is not None: - pending.append(resolved) + for module in imports.may_load: + _, files = files.lookup_or_parse_module(module) - return reachable + return dict(files.analyzed) async def watch(context: WorkflowContext, *, application: str) -> None: @@ -334,19 +706,19 @@ async def watch(context: WorkflowContext, *, application: str) -> None: with file_watcher() as watcher: async for iteration in context.loop('Watch the application'): # The watch is armed before anything is read, so a save - # made during the walk resolves `event` rather than - # arriving while nothing is listening. A watch is consumed - # by one event, so it is re-entered for each. + # made during an iteration resolves `event` rather than + # arriving while nothing is listening. A watch is + # consumed by one event, so it is re-entered for each. async with watcher.watch(globs) as event: - known = await files( + known = await analyze( application=application, roots=roots, known=known ) found = servicers(known) - # Most edits change no servicer, and a write wakes every - # browser reading `Get`, so one is only worth making - # when the answer is different. + # Most edits change no servicer, and a write wakes + # every browser reading `Get`, so one is only worth + # making when the answer is different. if found != recorded: async def record(state) -> None: diff --git a/tests/reboot/dashboard/implementation_watcher_tests.py b/tests/reboot/dashboard/implementation_watcher_tests.py index 368f2845..2d1ea5b6 100644 --- a/tests/reboot/dashboard/implementation_watcher_tests.py +++ b/tests/reboot/dashboard/implementation_watcher_tests.py @@ -17,7 +17,7 @@ ENVVAR_RBT_APPLICATION, IMPLEMENTATION_ID, ) -from reboot.dashboard.implementation_watcher import File, files, servicers +from reboot.dashboard.implementation_watcher import File, analyze, servicers from reboot.dashboard.main import application from unittest.mock import patch @@ -288,7 +288,7 @@ async def test_finds_the_file_a_state_type_is_implemented_in(self) -> None: self._write('shop_servicer.py', source=SHOP) application = self._write('main.py', source=APPLICATION) - found = _state_types_and_files(await files(application=application)) + found = _state_types_and_files(await analyze(application=application)) self.assertEqual( found, @@ -307,7 +307,7 @@ async def test_a_singleton_says_what_it_services_the_same_way( ), ) - found = _state_types_and_files(await files(application=application)) + found = _state_types_and_files(await analyze(application=application)) self.assertEqual( found, @@ -329,7 +329,7 @@ async def main(): ''' ) - found = _state_types_and_files(await files(application=application)) + found = _state_types_and_files(await analyze(application=application)) self.assertEqual( found, [ @@ -359,7 +359,7 @@ async def main(): ''' ) - found = _state_types_and_files(await files(application=application)) + found = _state_types_and_files(await analyze(application=application)) self.assertEqual( found, @@ -383,7 +383,7 @@ async def main(): ''' ) - found = _state_types_and_files(await files(application=application)) + found = _state_types_and_files(await analyze(application=application)) self.assertEqual( found, @@ -410,7 +410,7 @@ async def main(): ''' ) - found = _state_types_and_files(await files(application=application)) + found = _state_types_and_files(await analyze(application=application)) self.assertEqual( found, @@ -431,7 +431,7 @@ async def main(): ''' ) - found = _state_types_and_files(await files(application=application)) + found = _state_types_and_files(await analyze(application=application)) self.assertEqual( found, @@ -463,14 +463,14 @@ async def main(): ) found = _state_types_and_files( - await files(application=application) + await analyze(application=application) ) self.assertEqual(found, []) # Named as a root, the very same import leads there. found = _state_types_and_files( - await files( + await analyze( application=application, roots=[str(self.directory), elsewhere.name], ) @@ -485,11 +485,45 @@ async def main(): ################################################################### # The methods each servicer defines. + async def test_any_dotted_base_ending_in_servicer_counts(self) -> None: + self._write( + 'shop_servicer.py', + source=SHOP.replace( + 'class ShopServicer(Shop.Servicer):', + 'class ShopServicer(stores.things.Shop.Servicer):', + ), + ) + application = self._write('main.py', source=APPLICATION) + + found = _state_types_and_files(await analyze(application=application)) + + self.assertEqual( + found, + [ + ( + 'stores.things.Shop', + str(self.directory / 'shop_servicer.py'), + ) + ], + ) + + async def test_every_import_is_a_dependency(self) -> None: + helper = self._write('helper.py', source='VALUE = 1\n') + servicer = self._write( + 'shop_servicer.py', + source=SHOP + '\nimport helper\n', + ) + application = self._write('main.py', source=APPLICATION) + + found = await analyze(application=application) + + self.assertIn(str(helper), found[str(servicer)].dependencies) + async def test_records_the_methods_a_servicer_defines(self) -> None: self._write('shop_servicer.py', source=SHOP) application = self._write('main.py', source=APPLICATION) - found = servicers(await files(application=application)) + found = servicers(await analyze(application=application)) self.assertEqual( [method.name for method in found[0].methods], ['look'] @@ -501,7 +535,7 @@ async def test_a_method_reformatted_digests_the_same(self) -> None: self._write('shop_servicer.py', source=SHOP) application = self._write('main.py', source=APPLICATION) - before = servicers(await files(application=application)) + before = servicers(await analyze(application=application)) self._write( 'shop_servicer.py', @@ -517,7 +551,7 @@ async def test_a_method_reformatted_digests_the_same(self) -> None: ), ) - after = servicers(await files(application=application)) + after = servicers(await analyze(application=application)) self.assertEqual( [method.digest for method in after[0].methods], @@ -530,14 +564,14 @@ async def test_a_method_whose_body_changes_digests_differently( self._write('shop_servicer.py', source=SHOP) application = self._write('main.py', source=APPLICATION) - before = servicers(await files(application=application)) + before = servicers(await analyze(application=application)) self._write( 'shop_servicer.py', source=SHOP.replace(' pass', ' return None'), ) - after = servicers(await files(application=application)) + after = servicers(await analyze(application=application)) self.assertNotEqual( after[0].methods[0].digest, @@ -555,7 +589,7 @@ async def test_a_file_written_with_the_same_bytes_is_not_parsed_again( servicer = self._write('shop_servicer.py', source=SHOP) application = self._write('main.py', source=APPLICATION) - known = await files(application=application) + known = await analyze(application=application) Path(servicer).write_text(SHOP) @@ -564,7 +598,7 @@ async def test_a_file_written_with_the_same_bytes_is_not_parsed_again( '_parse', wraps=implementation_watcher._parse, ) as parse: - await files(application=application, known=known) + await analyze(application=application, known=known) parse.assert_not_called() @@ -577,13 +611,13 @@ async def test_a_file_written_with_other_bytes_is_parsed_again( application = self._write('main.py', source=APPLICATION) modified = os.stat(servicer).st_mtime_ns - known = await files(application=application) + known = await analyze(application=application) Path(servicer).write_text(DEPOT) os.utime(servicer, ns=(modified, modified)) found = _state_types_and_files( - await files(application=application, known=known) + await analyze(application=application, known=known) ) self.assertEqual(found, [('Depot', servicer)]) @@ -604,13 +638,13 @@ async def test_a_servicer_that_stops_being_imported_is_dropped( ), ) - known = await files(application=application) + known = await analyze(application=application) self.assertEqual(len(servicers(known)), 2) self._write('main.py', source=APPLICATION) found = _state_types_and_files( - await files(application=application, known=known) + await analyze(application=application, known=known) ) self.assertEqual( @@ -624,7 +658,7 @@ async def test_a_servicer_that_starts_being_imported_is_found( self._write('shop_servicer.py', source=SHOP) application = self._write('main.py', source=APPLICATION) - known = await files(application=application) + known = await analyze(application=application) self._write('depot_servicer.py', source=DEPOT) self._write( @@ -637,7 +671,7 @@ async def test_a_servicer_that_starts_being_imported_is_found( ) found = _state_types_and_files( - await files(application=application, known=known) + await analyze(application=application, known=known) ) self.assertEqual( @@ -648,7 +682,7 @@ async def test_a_servicer_that_starts_being_imported_is_found( ) ################################################################### - # What it finds no servicer for. + # Followed to the state type however it is reached. async def test_a_file_that_will_not_parse(self) -> None: """Its servicers go unfound, because which state types they @@ -656,13 +690,13 @@ async def test_a_file_that_will_not_parse(self) -> None: self._write('shop_servicer.py', source='class ShopServicer(') application = self._write('main.py', source=APPLICATION) - found = _state_types_and_files(await files(application=application)) + found = _state_types_and_files(await analyze(application=application)) self.assertEqual(found, []) async def test_an_application_that_is_not_there(self) -> None: found = _state_types_and_files( - await files(application=str(self.directory / 'nowhere.py')) + await analyze(application=str(self.directory / 'nowhere.py')) ) self.assertEqual(found, []) @@ -684,7 +718,7 @@ async def main(): ''' ) - found = _state_types_and_files(await files(application=application)) + found = _state_types_and_files(await analyze(application=application)) self.assertEqual( found, [ @@ -704,7 +738,7 @@ class ShopServicer(SomethingElse): ) application = self._write('main.py', source=APPLICATION) - found = _state_types_and_files(await files(application=application)) + found = _state_types_and_files(await analyze(application=application)) self.assertEqual(found, []) From 4441298cb1dcfa6965b42230a236ff73c4bf897e Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Wed, 19 Aug 2026 22:50:03 +0000 Subject: [PATCH 13/19] Reach files through relative imports A relative import is resolved to a path where it is collected, since that is the one place the importing file's own directory is known. From there it is followed to its file like any other module, spelled as a path, with `os.sep` telling the two spellings apart, and files are deduped by their absolute path since a file can now be reached under two spellings. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QPEVMhyDRxZEuH8eiykv99 --- reboot/dashboard/implementation_watcher.py | 112 +++++++++++++++--- .../dashboard/implementation_watcher_tests.py | 28 +++++ 2 files changed, 122 insertions(+), 18 deletions(-) diff --git a/reboot/dashboard/implementation_watcher.py b/reboot/dashboard/implementation_watcher.py index 02e7068e..131141f9 100644 --- a/reboot/dashboard/implementation_watcher.py +++ b/reboot/dashboard/implementation_watcher.py @@ -35,6 +35,7 @@ class AccountServicer(Account.Servicer): import hashlib import os from dataclasses import dataclass, replace +from pathlib import Path from rbt.dashboard.v1.dashboard_pb2 import ServicerInfo from rbt.dashboard.v1.dashboard_rbt import Implementation from reboot.aio.contexts import WorkflowContext @@ -81,13 +82,18 @@ class Imports: may_load: tuple[str, ...] -def _imports(module: ast.Module) -> Imports: +def _imports( + module: ast.Module, *, directory: Optional[str] = None +) -> Imports: """Returns every module a file's imports may make Python load. Modules come from every import, wherever it is written. One inside an `if` or a `try` loads its module just as one at the top of the file does, and guarding an import is common enough to be worth reading. + + `directory` is where the file itself is, which is what a relative + import is relative to; without it relative imports load nothing. """ may_load: list[str] = [] @@ -96,8 +102,44 @@ def _imports(module: ast.Module) -> Imports: case ast.Import(names=names): may_load.extend(alias.name for alias in names) - case ast.ImportFrom(module=str(from_module), level=0, names=names): - base = from_module + case ast.ImportFrom( + module=from_module, level=int(level), names=names + ): + # Having a `level` means we have to determine what + # file Python might load. For example, in a file + # `shop/cart/servicer.py`: + # + # from shop.cart.types import Item + # level 0 -> `shop.cart.types` + # from .types import Item + # level 1 -> `shop/cart/types` + # from ..api import Item + # level 2 -> `shop/api` + # from .. import api + # level 2, no module -> `shop` + # + # `level` counts the leading dots. No dots: keep the + # module as written. One dot: start from the file's + # own directory. Each extra dot: climb one parent. + # If a module comes after the dots, it goes under + # that directory: `..api` is `shop` plus `api`, so + # `shop/api`. A relative import can only be spelled + # as a path, and without a `directory` the dots + # point at nothing, so the import binds nothing. + if level == 0: + if from_module is None: + continue + base = from_module + elif directory is not None: + climbed = directory + for _ in range(level - 1): + climbed = os.path.dirname(climbed) + if from_module is None: + base = climbed + else: + base = os.path.join(climbed, *from_module.split('.')) + else: + continue may_load.append(base) @@ -122,19 +164,28 @@ def _try_find_file_of(module: str, *, roots: Sequence[str]) -> Optional[str]: """Returns the file a module names if one of `roots` contains it, and `None` otherwise. - `None` is not a failure: the standard library and installed - packages live outside every root, so `import asyncio` resolves to - nothing and there is nothing to read. + A module spelled as a path -- what a relative import resolved to + -- is looked for where it already points. `None` is not a failure: + the standard library and installed packages live outside every + root, so `import asyncio` resolves to nothing and there is nothing + to read. """ - relative = module.replace('.', os.sep) - - for root in roots: - for candidate in ( - os.path.join(root, relative + '.py'), - os.path.join(root, relative, '__init__.py'), - ): - if os.path.isfile(candidate): - return candidate + if os.sep in module: + candidates = [ + module + '.py', + os.path.join(module, '__init__.py'), + ] + else: + relative = module.replace('.', os.sep) + candidates = [ + os.path.join(root, relative + suffix) + for root in roots + for suffix in ('.py', os.sep + '__init__.py') + ] + + for candidate in candidates: + if os.path.isfile(candidate): + return candidate return None @@ -154,7 +205,11 @@ def _path(expression: ast.expr) -> Optional[list[str]]: def _join(base: str, *parts: str) -> str: - """Returns a module extended by more components.""" + """Returns a module extended by more components, joined the way + the module is spelled: with dots for a dotted name, as a path for + a path.""" + if os.sep in base: + return os.path.join(base, *parts) return '.'.join([base, *parts]) @@ -261,12 +316,22 @@ def create( def lookup(self, filename: str) -> Optional[ParsedFile | File]: """Returns what this iteration read for a file, wherever it lives -- `analyzed` or `parsed` -- and `None` when it has - not read the file.""" + not read the file. + + By the file's absolute path, since a relative import spells + a file one way and an absolute import another. + """ for entries in (self.analyzed, self.parsed): entry = entries.get(filename) if entry is not None: return entry + absolute = Path(filename).absolute() + for entries in (self.analyzed, self.parsed): + for entry in entries.values(): + if Path(entry.filename).absolute() == absolute: + return entry + return None def with_parsed_file(self, parsed: ParsedFile) -> 'Files': @@ -400,6 +465,15 @@ def lookup_or_parse_filename( files = files.with_digest(filename, digest) known = files.known.get(filename) + if known is None: + # A relative import spells a file one way and an absolute + # import another, so the previous iteration is searched + # by absolute path too. + absolute = Path(filename).absolute() + for file in files.known.values(): + if Path(file.filename).absolute() == absolute: + known = file + break if known is not None: reanalyze, files = files.needs_reanalyzing(known, digest=digest) if not reanalyze: @@ -461,7 +535,9 @@ def _parse(source: bytes, *, filename: str, return ParsedFile( filename=filename, digest=digest, - imports=_imports(module), + imports=_imports( + module, directory=os.path.dirname(os.path.abspath(filename)) + ), module=module, ) diff --git a/tests/reboot/dashboard/implementation_watcher_tests.py b/tests/reboot/dashboard/implementation_watcher_tests.py index 2d1ea5b6..bb976200 100644 --- a/tests/reboot/dashboard/implementation_watcher_tests.py +++ b/tests/reboot/dashboard/implementation_watcher_tests.py @@ -684,6 +684,34 @@ async def test_a_servicer_that_starts_being_imported_is_found( ################################################################### # Followed to the state type however it is reached. + async def test_a_servicer_reached_through_a_relative_import(self) -> None: + self._write('package/__init__.py', source='') + self._write('package/shop_servicer.py', source=SHOP) + self._write( + 'package/servicers.py', + source='from .shop_servicer import ShopServicer\n', + ) + application = self._write( + 'main.py', + source=APPLICATION.replace( + 'from shop_servicer import ShopServicer', + 'from package.servicers import ShopServicer', + ), + ) + + found = _state_types_and_files(await analyze(application=application)) + + self.assertEqual( + found, + [( + 'Shop', + str(self.directory / 'package' / 'shop_servicer.py'), + )], + ) + + ################################################################### + # What it finds no servicer for. + async def test_a_file_that_will_not_parse(self) -> None: """Its servicers go unfound, because which state types they service is precisely what went unread.""" From 8a93205effd617dfe370b35f794a53ca26ddba21 Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Wed, 19 Aug 2026 22:50:27 +0000 Subject: [PATCH 14/19] Read and stat files off the event loop `_read` and `_try_find_file_of` make OS calls, and they were made with the event loop held, so a slow disk stalled every dashboard request for as long as the disk took. Both now go through `aiofiles`, which runs the call in a thread -- the way file operations are done everywhere else in the repo -- and everything between `files()` and the two of them becomes `async` to carry the `await` down. Parsing still holds the interpreter: `ast.parse` is CPU-bound, so no thread frees the loop from it, and `cooperatively` already bounds it to a file at a time. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QPEVMhyDRxZEuH8eiykv99 --- reboot/dashboard/BUILD.bazel | 1 + reboot/dashboard/implementation_watcher.py | 51 ++++++++++++---------- 2 files changed, 30 insertions(+), 22 deletions(-) diff --git a/reboot/dashboard/BUILD.bazel b/reboot/dashboard/BUILD.bazel index 8e4a976b..993134a9 100644 --- a/reboot/dashboard/BUILD.bazel +++ b/reboot/dashboard/BUILD.bazel @@ -39,6 +39,7 @@ py_library( srcs_version = "PY3", visibility = ["//visibility:public"], deps = [ + requirement("aiofiles"), ":constants_py", "//rbt/dashboard/v1:dashboard_py_reboot", "//reboot/aio:cooperatively_py", diff --git a/reboot/dashboard/implementation_watcher.py b/reboot/dashboard/implementation_watcher.py index 131141f9..8b5fef85 100644 --- a/reboot/dashboard/implementation_watcher.py +++ b/reboot/dashboard/implementation_watcher.py @@ -31,6 +31,8 @@ class AccountServicer(Account.Servicer): developer's source changes, so an edit under the roots is what wakes this, and nothing else does. """ +import aiofiles +import aiofiles.os import ast import hashlib import os @@ -65,9 +67,9 @@ def _roots(application: str) -> list[str]: return [os.path.dirname(application)] -def _read(filename: str) -> bytes: - with open(filename, 'rb') as file: - return file.read() +async def _read(filename: str) -> bytes: + async with aiofiles.open(filename, 'rb') as file: + return await file.read() @dataclass(frozen=True, kw_only=True) @@ -160,7 +162,8 @@ def _imports( return Imports(may_load=tuple(may_load)) -def _try_find_file_of(module: str, *, roots: Sequence[str]) -> Optional[str]: +async def _try_find_file_of(module: str, *, + roots: Sequence[str]) -> Optional[str]: """Returns the file a module names if one of `roots` contains it, and `None` otherwise. @@ -184,7 +187,7 @@ def _try_find_file_of(module: str, *, roots: Sequence[str]) -> Optional[str]: ] for candidate in candidates: - if os.path.isfile(candidate): + if await aiofiles.os.path.isfile(candidate): return candidate return None @@ -404,8 +407,8 @@ def with_digest(self, filename: str, digest: Digest) -> 'Files': }), ) - def needs_reanalyzing(self, file: File, *, - digest: Digest) -> tuple[bool, 'Files']: + async def needs_reanalyzing(self, file: File, *, + digest: Digest) -> tuple[bool, 'Files']: """Returns whether a file the previous iteration analyzed cannot be reused -- its bytes changed, or a file it depends on did, including one that can no longer be read at all -- @@ -430,7 +433,7 @@ def needs_reanalyzing(self, file: File, *, current = files.digests.get(filename) if current is None: try: - current = hashlib.sha256(_read(filename)).digest() + current = hashlib.sha256(await _read(filename)).digest() except OSError: return True, files files = files.with_digest(filename, current) @@ -440,7 +443,7 @@ def needs_reanalyzing(self, file: File, *, return False, files - def lookup_or_parse_filename( + async def lookup_or_parse_filename( self, filename: str ) -> tuple[Optional[ParsedFile | File], 'Files']: """Returns a file as this iteration has it -- looked up among @@ -458,7 +461,7 @@ def lookup_or_parse_filename( digest = files.digests.get(filename) if digest is None: try: - source = _read(filename) + source = await _read(filename) except OSError: return None, files digest = hashlib.sha256(source).digest() @@ -475,7 +478,9 @@ def lookup_or_parse_filename( known = file break if known is not None: - reanalyze, files = files.needs_reanalyzing(known, digest=digest) + reanalyze, files = await files.needs_reanalyzing( + known, digest=digest + ) if not reanalyze: return known, files.with_reused_known_file(known) @@ -483,7 +488,7 @@ def lookup_or_parse_filename( # the digest did not keep. if source is None: try: - source = _read(filename) + source = await _read(filename) except OSError: return None, files digest = hashlib.sha256(source).digest() @@ -495,7 +500,7 @@ def lookup_or_parse_filename( return parsed, files.with_parsed_file(parsed) - def lookup_or_parse_module( + async def lookup_or_parse_module( self, module: str ) -> tuple[Optional[ParsedFile | File], 'Files']: """Returns the file a module names as this iteration has it, @@ -504,12 +509,12 @@ def lookup_or_parse_module( the files come back as they were: nothing found means nothing joined. """ - filename = _try_find_file_of(module, roots=self.roots) + filename = await _try_find_file_of(module, roots=self.roots) if filename is None or filename.endswith(GENERATED_SUFFIXES): return None, self - return self.lookup_or_parse_filename(filename) + return await self.lookup_or_parse_filename(filename) def imports(self, filename: str) -> Optional[Imports]: """Returns what a file's imports bound, wherever this @@ -632,7 +637,9 @@ def _digest(node: ast.AST) -> Digest: include_attributes=False).encode()).digest() -def _depend_on_imports(parsed: ParsedFile, *, analysis: Analysis) -> Analysis: +async def _depend_on_imports( + parsed: ParsedFile, *, analysis: Analysis +) -> Analysis: """Returns the analysis depending on the file behind every one of the file's imports: those are the files that can change what this one means. Each is read one hop, only for the digest @@ -643,7 +650,7 @@ def _depend_on_imports(parsed: ParsedFile, *, analysis: Analysis) -> Analysis: used would narrow little, and it can be narrowed later if this proves too eager.""" for module in dict.fromkeys(parsed.imports.may_load): - found, files = analysis.files.lookup_or_parse_module(module) + found, files = await analysis.files.lookup_or_parse_module(module) if found is None: continue analysis = analysis.with_dependency(found, files=files) @@ -651,7 +658,7 @@ def _depend_on_imports(parsed: ParsedFile, *, analysis: Analysis) -> Analysis: return analysis -def _analyze_file(filename: str, files: Files) -> Files: +async def _analyze_file(filename: str, files: Files) -> Files: """Returns the iteration carried past analyzing one `pending` file: every class saying what it services recorded, every method those classes define analyzed and recorded under them, and the @@ -666,7 +673,7 @@ def _analyze_file(filename: str, files: Files) -> Files: analysis = Analysis.create(filename=filename, files=files) - analysis = _depend_on_imports(parsed, analysis=analysis) + analysis = await _depend_on_imports(parsed, analysis=analysis) servicers: list[ServicerInfo] = [] @@ -740,7 +747,7 @@ async def analyze( files = Files.create(roots=roots, known=known or {}) - _, files = files.lookup_or_parse_filename(application) + _, files = await files.lookup_or_parse_filename(application) while len(files.pending) > 0: # Parsing and analyzing hold on to the interpreter for as @@ -749,7 +756,7 @@ async def analyze( async for filename in cooperatively(files.pending): match files.parsed.get(filename): case ParsedFile(): - files = _analyze_file(filename, files) + files = await _analyze_file(filename, files) case None: # Nothing to analyze: the file was kept because # neither its bytes nor any of its dependencies @@ -765,7 +772,7 @@ async def analyze( assert imports is not None for module in imports.may_load: - _, files = files.lookup_or_parse_module(module) + _, files = await files.lookup_or_parse_module(module) return dict(files.analyzed) From ded89928427d8543eb5d40ad51c0f127c801e502 Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Wed, 19 Aug 2026 22:50:50 +0000 Subject: [PATCH 15/19] Say why a save during an iteration is never missed A save landing while an iteration reads produces a torn snapshot: one file read before the save, another after. The watch is armed before anything is read, so the save's event is already waiting when the iteration finishes and the next one begins at once -- where a file kept against a stale dependency digest fails its check and is analyzed again. The digests recorded per dependency are what make the tear detectable. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QPEVMhyDRxZEuH8eiykv99 --- reboot/dashboard/implementation_watcher.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/reboot/dashboard/implementation_watcher.py b/reboot/dashboard/implementation_watcher.py index 8b5fef85..5b05f601 100644 --- a/reboot/dashboard/implementation_watcher.py +++ b/reboot/dashboard/implementation_watcher.py @@ -792,6 +792,14 @@ async def watch(context: WorkflowContext, *, application: str) -> None: # made during an iteration resolves `event` rather than # arriving while nothing is listening. A watch is # consumed by one event, so it is re-entered for each. + # + # The arming is also why a save landing mid-iteration is + # safe. The iteration may record a torn snapshot -- one + # file read before the save and another after -- but the + # save's event is already waiting, so the next iteration + # begins at once, and any file kept against a stale + # dependency digest fails its check there and is analyzed + # again. async with watcher.watch(globs) as event: known = await analyze( application=application, roots=roots, known=known From 10cc9d51de414a1832422b41fd1eb22ee88ae646 Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Thu, 20 Aug 2026 03:10:25 +0000 Subject: [PATCH 16/19] Ask pyright which state type each servicer services Which state type a class services is type information: its base may be spelled `Account.Servicer`, assigned to another name first, or imported from another file. So instead of reading spellings, pyright is asked about every base every class extends. A base whose definition leads into a file `rbt generate` wrote makes the class a servicer, and the state type is the name the generator writes into its classes as `__state_type_name__`, followed through the aliases and base classes of the generated module when the definition lands beside the name rather than on it. A `Pyright` speaks to one `pyright-langserver` over the Language Server Protocol on stdio, with backpressure on every write, and answers about each file as it was last handed over. Whoever reads a file is who hands its text over: the analysis hands each file's text right where it reads and parses it, and it reads everything reachable before it asks anything, so every answer is about the very snapshot the analysis parsed, for every file at once, however the disk changes in between. The watch takes back files that stopped existing, which is the one change parsing never meets. The analysis records the answers, so a file is only asked about again when it needs reanalyzing. That is also why a file needs reanalyzing when anything changed in the closure of its dependencies, however many imports away: its recorded answers were made with all of their help. And generated code, which nobody hands over, gets a fresh server when it changes -- that only happens when `rbt generate` ran -- along with a fresh analysis of everything, since every recorded answer may lean on it. A base pyright cannot yet place in generated code, most often because `rbt generate` has not run since the state type was declared, leaves its servicer unrecorded for now, and the new `generated` field is what tells the dashboard to suggest running `rbt generate`. Each servicer is recorded with the line and column its class is written at, so a reader can be taken to it. The generated directory is read from the `.rbtrc` arguments to `rbt generate` (its `--python` flag), the same way the API directory already is. pyright comes in through npm, pinned, so that Bazel tests run a hermetic `pyright-langserver`. The lockfile sync also picks up the importer specifiers the 1.4.0 release left behind. --- package.json | 1 + pnpm-lock.yaml | 45 +- rbt/dashboard/v1/dashboard.proto | 14 + reboot/cli/commands/dashboard.py | 28 + reboot/dashboard/BUILD.bazel | 14 + reboot/dashboard/constants.py | 7 + reboot/dashboard/implementation_watcher.py | 698 +++++++++++++++--- reboot/dashboard/pyright.py | 354 +++++++++ reboot/dashboard/servicers.py | 11 +- tests/reboot/dashboard/BUILD.bazel | 7 + .../dashboard/implementation_watcher_tests.py | 468 +++++++++--- 11 files changed, 1426 insertions(+), 221 deletions(-) create mode 100644 reboot/dashboard/pyright.py diff --git a/package.json b/package.json index 3cd93757..0c33962d 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "jsdom": "^26.1.0", "js-sha1": "0.7.0", "esbuild": "^0.28.0", + "pyright": "1.1.413", "express": "^5.1.0", "lru-cache-idb": "^0.5.2", "react": "19.2.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 247e10c5..38c73c31 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -24,25 +24,25 @@ importers: specifier: 1.29.0 version: 1.29.0(zod@3.25.76) '@reboot-dev/reboot': - specifier: 1.2.1 + specifier: 1.4.0 version: link:reboot/nodejs '@reboot-dev/reboot-api': - specifier: 1.2.1 + specifier: 1.4.0 version: link:rbt/v1alpha1 '@reboot-dev/reboot-react': - specifier: 1.2.1 + specifier: 1.4.0 version: link:reboot/react '@reboot-dev/reboot-std': - specifier: 1.2.1 + specifier: 1.4.0 version: link:reboot/std '@reboot-dev/reboot-std-api': - specifier: 1.2.1 + specifier: 1.4.0 version: link:rbt/std '@reboot-dev/reboot-std-react': - specifier: 1.2.1 + specifier: 1.4.0 version: link:reboot/std/react '@reboot-dev/reboot-web': - specifier: 1.2.1 + specifier: 1.4.0 version: link:reboot/web '@standard-schema/spec': specifier: 1.0.0 @@ -92,6 +92,9 @@ importers: lru-cache-idb: specifier: ^0.5.2 version: 0.5.2 + pyright: + specifier: 1.1.413 + version: 1.1.413 react: specifier: 19.2.1 version: 19.2.1 @@ -187,7 +190,7 @@ importers: specifier: 1.10.1 version: 1.10.1 '@reboot-dev/reboot-api': - specifier: 1.2.1 + specifier: 1.4.0 version: link:../../rbt/v1alpha1 '@scarf/scarf': specifier: 1.4.0 @@ -254,10 +257,10 @@ importers: specifier: 1.29.0 version: 1.29.0(zod@3.25.76) '@reboot-dev/reboot-api': - specifier: 1.2.1 + specifier: 1.4.0 version: link:../../rbt/v1alpha1 '@reboot-dev/reboot-web': - specifier: 1.2.1 + specifier: 1.4.0 version: link:../web '@scarf/scarf': specifier: 1.4.0 @@ -300,10 +303,10 @@ importers: reboot/std: dependencies: '@reboot-dev/reboot': - specifier: 1.2.1 + specifier: 1.4.0 version: link:../nodejs '@reboot-dev/reboot-std-api': - specifier: 1.2.1 + specifier: 1.4.0 version: link:../../rbt/std '@scarf/scarf': specifier: 1.4.0 @@ -316,16 +319,16 @@ importers: reboot/std/react: dependencies: '@reboot-dev/reboot-api': - specifier: 1.2.1 + specifier: 1.4.0 version: link:../../../rbt/v1alpha1 '@reboot-dev/reboot-react': - specifier: 1.2.1 + specifier: 1.4.0 version: link:../../react '@reboot-dev/reboot-std-api': - specifier: 1.2.1 + specifier: 1.4.0 version: link:../../../rbt/std '@reboot-dev/reboot-web': - specifier: 1.2.1 + specifier: 1.4.0 version: link:../../web '@scarf/scarf': specifier: 1.4.0 @@ -341,7 +344,7 @@ importers: specifier: 1.10.1 version: 1.10.1 '@reboot-dev/reboot-api': - specifier: 1.2.1 + specifier: 1.4.0 version: link:../../rbt/v1alpha1 '@scarf/scarf': specifier: 1.4.0 @@ -3333,6 +3336,14 @@ packages: resolution: {integrity: sha512-QFADYnsVoBMw1srW7OVKEYjG+MbIa49s54w1MA1EDY6r2r/sTcKKYqRX1f4GYvnXP7eN/Pe9HFcX+hwzmrXRHA==} dev: false + /pyright@1.1.413: + resolution: {integrity: sha512-1lpxKrh0DHHpfAQOfciZo2ojua2jase3wwO9at8kldc+F/p1PBscxA5CQ3G1qg5lMOMhXo6ZaiMLMXEAqADIAg==} + engines: {node: '>=14.0.0'} + hasBin: true + optionalDependencies: + fsevents: 2.3.3 + dev: false + /qs@6.14.1: resolution: {integrity: sha512-4EK3+xJl8Ts67nLYNwqw/dsFVnCf+qR7RgXSK9jEEm9unao3njwMDdmsdvoKBKHzxd7tCYz5e5M+SnMjdtXGQQ==} engines: {node: '>=0.6'} diff --git a/rbt/dashboard/v1/dashboard.proto b/rbt/dashboard/v1/dashboard.proto index 4f16d961..5936b340 100644 --- a/rbt/dashboard/v1/dashboard.proto +++ b/rbt/dashboard/v1/dashboard.proto @@ -103,6 +103,12 @@ message ServicerInfo { // open it, e.g. "backend/src/account_servicer.py". string file = 2; + // Represents the line its class is written on, counting from one. + uint32 line = 4; + + // Represents the column its class starts at, counting from zero. + uint32 character = 5; + // Represents every method it defines, in the order they are // written, which is the order somebody reading the file meets them. repeated Method methods = 3; @@ -125,12 +131,20 @@ message Implementation { // servicing it, which is for whoever reads this to make of what // they will. repeated ServicerInfo servicers = 1; + + // Whether `rbt generate` has written the application's generated + // code, which is where the state types the servicers service are + // defined. A servicer only joins the list above once its state + // type is defined there, so while this is false the dashboard + // suggests running `rbt generate`. + bool generated = 2; } message ImplementationGetRequest {} message ImplementationGetResponse { repeated ServicerInfo servicers = 1; + bool generated = 2; } message ImplementationWatchRequest {} diff --git a/reboot/cli/commands/dashboard.py b/reboot/cli/commands/dashboard.py index 93eb2ede..9ef1962a 100644 --- a/reboot/cli/commands/dashboard.py +++ b/reboot/cli/commands/dashboard.py @@ -24,6 +24,7 @@ DEFAULT_DASHBOARD_PORT, ENVVAR_RBT_API_DIRECTORY, ENVVAR_RBT_APPLICATION, + ENVVAR_RBT_GENERATED_DIRECTORY, ) from reboot.settings import ( ENVVAR_RBT_DEV, @@ -105,6 +106,23 @@ def _application(parser: ArgumentParser) -> Optional[str]: return None +def _generated_directory(parser: ArgumentParser) -> Optional[str]: + """Returns the directory `rbt generate` writes Python code into, + which is where its `--python=` flag points, and `None` when it + points nowhere. + + Read rather than asked for again, for the same reason as the + application. `None` is what an application with no generated + Python looks like, such as a Node.js one. + """ + for argument in parser.dot_rc_arguments('generate'): + name, separator, value = argument.partition('=') + if name == '--python' and separator == '=': + return value + + return None + + def _dashboard_env( args, parser: ArgumentParser, @@ -112,6 +130,7 @@ def _dashboard_env( port: int, api_directory: str, application: Optional[str], + generated_directory: Optional[str], ) -> dict[str, str]: """The environment for the dashboard application. @@ -161,6 +180,14 @@ def _dashboard_env( if application is not None: composed[ENVVAR_RBT_APPLICATION] = application + # Where the developer's generated Python is, spelled the same way + # and for the same reason. Left out when the developer named no + # `--python` directory, which is what tells the dashboard there is + # nothing to type the implementation with. + composed.pop(ENVVAR_RBT_GENERATED_DIRECTORY, None) + if generated_directory is not None: + composed[ENVVAR_RBT_GENERATED_DIRECTORY] = generated_directory + composed[ENVVAR_RBT_NAME] = DASHBOARD_STATE_DIRECTORY_NAME state_directory = ( @@ -263,6 +290,7 @@ async def dashboard( port=port, api_directory=_api_directory(parser), application=_application(parser), + generated_directory=_generated_directory(parser), ) terminal.info( diff --git a/reboot/dashboard/BUILD.bazel b/reboot/dashboard/BUILD.bazel index 993134a9..d91440c5 100644 --- a/reboot/dashboard/BUILD.bazel +++ b/reboot/dashboard/BUILD.bazel @@ -1,3 +1,4 @@ +load("@npm//:pyright/package_json.bzl", pyright_bin = "bin") load("@rbt_pypi//:requirements.bzl", "requirement") load("@rules_python//python:defs.bzl", "py_binary", "py_library") @@ -33,6 +34,13 @@ py_library( ], ) +py_library( + name = "pyright_py", + srcs = ["pyright.py"], + srcs_version = "PY3", + visibility = ["//visibility:public"], +) + py_library( name = "implementation_watcher_py", srcs = ["implementation_watcher.py"], @@ -41,6 +49,7 @@ py_library( deps = [ requirement("aiofiles"), ":constants_py", + ":pyright_py", "//rbt/dashboard/v1:dashboard_py_reboot", "//reboot/aio:cooperatively_py", "//reboot/aio:external_py", @@ -134,3 +143,8 @@ py_binary( requirement("starlette"), ], ) + +pyright_bin.pyright_langserver_binary( + name = "pyright_langserver", + visibility = ["//tests/reboot/dashboard:__pkg__"], +) diff --git a/reboot/dashboard/constants.py b/reboot/dashboard/constants.py index 545bcccd..9dce8a13 100644 --- a/reboot/dashboard/constants.py +++ b/reboot/dashboard/constants.py @@ -48,6 +48,13 @@ # which case no implementation is looked for. ENVVAR_RBT_APPLICATION = 'RBT_APPLICATION' +# The directory `rbt generate` writes Python code into, as the +# `.rbtrc` spells it with `--python=`. The generated code is where +# the state types are defined, which is what typing the developer's +# implementation needs. Unset when the developer named none, in which +# case nothing can be typed. +ENVVAR_RBT_GENERATED_DIRECTORY = 'RBT_GENERATED_DIRECTORY' + # The `Presence` state the dashboard page subscribes to, recording who # is looking at a dashboard right now. `rbt dev run` reads it to decide # whether to open one. diff --git a/reboot/dashboard/implementation_watcher.py b/reboot/dashboard/implementation_watcher.py index 5b05f601..021604dd 100644 --- a/reboot/dashboard/implementation_watcher.py +++ b/reboot/dashboard/implementation_watcher.py @@ -16,6 +16,16 @@ class AccountServicer(Account.Servicer): behind a conditional import -- which have one thing in common: the file defining the servicer had to be imported for any of them to run. +Which state type a class services is type information -- its base +may be spelled `Account.Servicer`, assigned to another name first, or +imported from anywhere -- so pyright is asked about every base every +class extends: a servicer is a class with a base whose definition +leads into a file `rbt generate` wrote, and the state type is the +name the generator writes there as `__state_type_name__`. Until +`rbt generate` has written it, the base resolves to nothing, and the +servicer waits unrecorded; `generated` is what tells the dashboard to +suggest running it. + Where following stops is what makes this the developer's code rather than somebody else's. A module resolves to a file only if a root holds it, so an import of an installed package leads nowhere. @@ -34,6 +44,7 @@ class AccountServicer(Account.Servicer): import aiofiles import aiofiles.os import ast +import asyncio import hashlib import os from dataclasses import dataclass, replace @@ -43,6 +54,7 @@ class AccountServicer(Account.Servicer): from reboot.aio.contexts import WorkflowContext from reboot.aio.cooperatively import cooperatively from reboot.cli.common.watch import file_watcher +from reboot.dashboard.pyright import Definition, Pyright from types import MappingProxyType from typing import Mapping, Optional, Sequence @@ -50,6 +62,10 @@ class AccountServicer(Account.Servicer): # saying whether what was digested has changed. Digest = bytes +# The syntax of files by filename, `None` for a file that could not +# be read or would not parse. +Syntax = Mapping[str, Optional['ast.Module']] + # Suffixes of the files `rbt generate` writes. Named so that a chain # of imports is never followed into generated code by reading it -- # the `_rbt` module name alone says what a name from one is. @@ -85,7 +101,9 @@ class Imports: def _imports( - module: ast.Module, *, directory: Optional[str] = None + module: ast.Module, + *, + directory: Optional[str] = None, ) -> Imports: """Returns every module a file's imports may make Python load. @@ -162,8 +180,11 @@ def _imports( return Imports(may_load=tuple(may_load)) -async def _try_find_file_of(module: str, *, - roots: Sequence[str]) -> Optional[str]: +async def _try_find_file_of( + module: str, + *, + roots: Sequence[str], +) -> Optional[str]: """Returns the file a module names if one of `roots` contains it, and `None` otherwise. @@ -193,20 +214,6 @@ async def _try_find_file_of(module: str, *, return None -def _path(expression: ast.expr) -> Optional[list[str]]: - """Returns the dotted name an expression spells -- `rbt.Shop` as - `['rbt', 'Shop']` -- and `None` when it does not spell one.""" - match expression: - case ast.Name(id=str(name)): - return [name] - case ast.Attribute(value=value, attr=str(attribute)): - prefix = _path(value) - if prefix is not None: - return prefix + [attribute] - - return None - - def _join(base: str, *parts: str) -> str: """Returns a module extended by more components, joined the way the module is spelled: with dots for a dotted name, as a path for @@ -254,7 +261,8 @@ class File: # was read. If any of them changes, this file needs reanalyzing. dependencies: Mapping[str, Digest] - # Every servicer the file defines. + # Every servicer the file defines, resolved: which state type each + # services and the calls each method makes. servicers: tuple[ServicerInfo, ...] @@ -298,12 +306,24 @@ class Files: # which a save makes rare, and the parse path reads its own. digests: Mapping[str, Digest] + # The syntax of files read only to be looked into -- generated + # code, and a developer's file a name leads back into after the + # iteration analyzed it -- parsed at most once per iteration. + syntax: Syntax + + # Answers what the names written in the files refer to. Handed + # every file's text right where the file is read and parsed, so + # that its answers are about exactly the bytes the iteration + # parsed. + pyright: Pyright + @classmethod def create( cls, *, roots: Sequence[str], known: Mapping[str, File], + pyright: Pyright, ) -> 'Files': """Returns the files an iteration starts from: nothing parsed, nothing analyzed.""" @@ -314,6 +334,38 @@ def create( analyzed=MappingProxyType({}), pending=frozenset(), digests=MappingProxyType({}), + syntax=MappingProxyType({}), + pyright=pyright, + ) + + async def syntax_of( + self, + filename: str, + ) -> tuple[Optional[ast.Module], 'Files']: + """Returns the syntax of any file, parsed at most once per + iteration, and `None` when the file cannot be read or will + not parse.""" + found = self.lookup(filename) + match found: + case ParsedFile(module=parsed_module): + return parsed_module, self + + if filename in self.syntax: + return self.syntax[filename], self + + module: Optional[ast.Module] = None + try: + source = await _read(filename) + module = ast.parse(source) + await self.pyright.hand(filename, source.decode()) + except (OSError, SyntaxError, UnicodeDecodeError): + pass + + return module, replace( + self, + syntax=MappingProxyType({ + **self.syntax, filename: module + }), ) def lookup(self, filename: str) -> Optional[ParsedFile | File]: @@ -407,12 +459,24 @@ def with_digest(self, filename: str, digest: Digest) -> 'Files': }), ) - async def needs_reanalyzing(self, file: File, *, - digest: Digest) -> tuple[bool, 'Files']: + async def needs_reanalyzing( + self, + file: File, + *, + digest: Digest, + ) -> tuple[bool, 'Files']: """Returns whether a file the previous iteration analyzed - cannot be reused -- its bytes changed, or a file it depends - on did, including one that can no longer be read at all -- - and this with every digest read to answer recorded. + cannot be reused -- its bytes changed, or a file its analysis + depended on did, however many dependencies away, including + one that can no longer be read at all -- and this with every + digest read to answer recorded. + + The dependencies are walked to their closure: what was + analyzed with the help of a dependency was analyzed with the + help of everything that dependency was analyzed with, so a + change anywhere down a chain of imports changes this file's + answers too. Each file met is checked against the digest + recorded by whoever depended on it directly. A dependency already read this iteration, reached or merely digested, is checked against the digest recorded for it, so @@ -423,28 +487,43 @@ async def needs_reanalyzing(self, file: File, *, files = self - for filename, previous_digest in file.dependencies.items(): + pending = list(file.dependencies.items()) + checked: set[str] = set() + + while len(pending) > 0: + filename, previous_digest = pending.pop() + if filename in checked: + continue + checked.add(filename) + found = files.lookup(filename) if found is not None: if found.digest != previous_digest: return True, files - continue - - current = files.digests.get(filename) - if current is None: - try: - current = hashlib.sha256(await _read(filename)).digest() - except OSError: + else: + current = files.digests.get(filename) + if current is None: + try: + current = hashlib.sha256(await + _read(filename)).digest() + except OSError: + return True, files + files = files.with_digest(filename, current) + + if current != previous_digest: return True, files - files = files.with_digest(filename, current) - if current != previous_digest: - return True, files + # The dependency's own dependencies, as it was last + # analyzed with them, are this file's too. + dependency = files.known.get(filename) + if dependency is not None: + pending.extend(dependency.dependencies.items()) return False, files async def lookup_or_parse_filename( - self, filename: str + self, + filename: str, ) -> tuple[Optional[ParsedFile | File], 'Files']: """Returns a file as this iteration has it -- looked up among what is already read, verified against the previous iteration @@ -498,10 +577,19 @@ async def lookup_or_parse_filename( if parsed is None: return None, files + try: + await files.pyright.hand(filename, source.decode()) + except UnicodeDecodeError: + # Pyright takes text; a file whose bytes are not UTF-8 + # cannot be handed over, so questions about it get no + # answers. + pass + return parsed, files.with_parsed_file(parsed) async def lookup_or_parse_module( - self, module: str + self, + module: str, ) -> tuple[Optional[ParsedFile | File], 'Files']: """Returns the file a module names as this iteration has it, and `None` when no root contains the module or the module is @@ -523,8 +611,12 @@ def imports(self, filename: str) -> Optional[Imports]: return found.imports if found is not None else None -def _parse(source: bytes, *, filename: str, - digest: Digest) -> Optional[ParsedFile]: +def _parse( + source: bytes, + *, + filename: str, + digest: Digest, +) -> Optional[ParsedFile]: """Returns what parsing one file said, and `None` when it will not parse. @@ -584,7 +676,10 @@ def create(cls, *, filename: str, files: Files) -> 'Analysis': ) def with_dependency( - self, found: ParsedFile | File, *, files: Files + self, + found: ParsedFile | File, + *, + files: Files, ) -> 'Analysis': """Returns this analysis depending on one more file: the digest it was read with recorded, and the files grown by @@ -600,27 +695,16 @@ def with_dependency( ) -def _state_type_if_servicer( - class_definition: ast.ClassDef, -) -> Optional[str]: - """Returns the state type a class services, spelled the way the - developer wrote it, and `None` when the class services nothing. - - A servicer says it by what it inherits: any dotted name ending - in `.Servicer`, such as `Account.Servicer`, `rbt.Shop.Servicer`, - or `Account.singleton.Servicer` for a singleton. Which state - type the name in front refers to exactly takes type information, - so it is recorded as written. - """ - for base in class_definition.bases: - match base: - case ast.Attribute(value=value, attr='Servicer'): - path = _path(value) - if path is None: - continue - if len(path) > 1 and path[-1] == 'singleton': - path = path[:-1] - return '.'.join(path) +def _position_inside(expression: ast.expr) -> Optional[tuple[int, int]]: + """Returns a line counting from one and a column inside the last + part of a name or attribute, and `None` for anything else.""" + match expression: + case ast.Name() | ast.Attribute(): + if ( + expression.end_lineno is not None and + expression.end_col_offset is not None + ): + return expression.end_lineno, expression.end_col_offset - 1 return None @@ -638,7 +722,9 @@ def _digest(node: ast.AST) -> Digest: async def _depend_on_imports( - parsed: ParsedFile, *, analysis: Analysis + parsed: ParsedFile, + *, + analysis: Analysis, ) -> Analysis: """Returns the analysis depending on the file behind every one of the file's imports: those are the files that can change what @@ -658,16 +744,21 @@ async def _depend_on_imports( return analysis -async def _analyze_file(filename: str, files: Files) -> Files: +async def _analyze_file( + filename: str, + files: Files, + *, + generated_directory: str, +) -> Files: """Returns the iteration carried past analyzing one `pending` - file: every class saying what it services recorded, every method - those classes define analyzed and recorded under them, and the - file finished into `analyzed` with the files the analysis read + file: every class one of whose bases pyright places on a state + type's `Servicer` recorded as a servicer, every method such a + class defines analyzed and recorded under it, and the file + finished into `analyzed` with the files the analysis read depended on. One `Analysis` is carried through the whole file: the - dependencies it accumulates are the file's, while the method - resets as each one is recorded. + dependencies it accumulates are the file's. """ parsed = files.parsed[filename] @@ -675,35 +766,315 @@ async def _analyze_file(filename: str, files: Files) -> Files: analysis = await _depend_on_imports(parsed, analysis=analysis) + files = analysis.files + servicers: list[ServicerInfo] = [] for node in ast.walk(parsed.module): match node: case ast.ClassDef(): - state_type = _state_type_if_servicer(node) - if state_type is None: - continue - servicer = ServicerInfo(state_type=state_type, file=filename) - for statement in node.body: - match statement: - case ( - ast.FunctionDef(name=str(name)) | - ast.AsyncFunctionDef(name=str(name)) - ): - servicer.methods.append( - ServicerInfo.Method( - name=name, digest=_digest(statement) - ) - ) - servicers.append(servicer) - - return analysis.files.with_analyzed_file( + servicer, files = await _servicer_of_class( + node, + filename=filename, + files=files, + generated_directory=generated_directory, + ) + if servicer is not None: + servicers.append(servicer) + + return files.with_analyzed_file( filename, dependencies=dict(analysis.dependencies), servicers=servicers, ) +def _enclosing_definitions( + node: ast.AST, + line: int, +) -> list[ast.ClassDef | ast.FunctionDef | ast.AsyncFunctionDef]: + """Returns the classes and functions a line is written inside, + outermost first, ending with the one whose definition starts on + that line when one does.""" + for child in ast.iter_child_nodes(node): + start = getattr(child, 'lineno', None) + end = getattr(child, 'end_lineno', None) + if start is None or end is None or not start <= line <= end: + continue + match child: + case (ast.ClassDef() | ast.FunctionDef() | ast.AsyncFunctionDef()): + return [child, *_enclosing_definitions(child, line)] + return _enclosing_definitions(child, line) + + return [] + + +def _generated_rbt_filename( + definition: Optional[Definition], + *, + generated_directory: str, +) -> Optional[str]: + """Returns the file a definition is in when that file is a `_rbt` + module `rbt generate` wrote, spelled absolutely, and `None` when + the definition is anywhere else.""" + if definition is None: + return None + + directory = os.path.abspath(generated_directory) + filename = os.path.abspath(definition.filename) + + if not filename.endswith('_rbt.py'): + return None + if not filename.startswith(directory + os.sep): + return None + + return filename + + +def _state_type_name(node: ast.ClassDef) -> Optional[str]: + """Returns the state type a class of generated code belongs to, + spelled as `StateTypeInfo.name`, which the generator writes into + the class as `__state_type_name__`, and `None` for a class + without one.""" + for statement in node.body: + match statement: + case ast.Assign( + targets=[ast.Name(id='__state_type_name__')], + value=( + ast.Call(args=[ast.Constant(value=str(name))]) | + ast.Constant(value=str(name)) + ), + ): + return name + + return None + + +def _top_level_class( + module: ast.Module, + name: str, +) -> Optional[ast.ClassDef]: + """Returns the class a module defines at its top by a name.""" + for statement in module.body: + match statement: + case ast.ClassDef(name=str(found)) if found == name: + return statement + + return None + + +def _last_name(expression: ast.expr) -> Optional[str]: + """Returns the last part of a name or an attribute, such as the + `GreeterServicer` of `x.y.GreeterServicer`, and `None` for + anything else.""" + match expression: + case ast.Name(id=str(name)): + return name + case ast.Attribute(attr=str(name)): + return name + + return None + + +def _state_type_of_class( + class_definition: ast.ClassDef, + *, + module: ast.Module, + asked: frozenset[str] = frozenset(), +) -> Optional[str]: + """Returns the state type a class of generated code belongs to: + its own `__state_type_name__`, or the one a base of it defined in + the same module belongs to, the way `GreeterServicer` leads to + `GreeterBaseServicer`.""" + if class_definition.name in asked: + return None + asked = asked | {class_definition.name} + + name = _state_type_name(class_definition) + if name is not None: + return name + + for base in class_definition.bases: + match base: + case ast.Name(id=str(base_name)): + found = _top_level_class(module, base_name) + if found is None: + continue + name = _state_type_of_class( + found, + module=module, + asked=asked, + ) + if name is not None: + return name + + return None + + +def _state_type_in( + module: ast.Module, + line: int, +) -> Optional[str]: + """Returns the state type defined at a line of a `_rbt` module, + and `None` when the line defines something else. + + The line has to be in a class at the top of the module. When the + line itself is an alias to another class of the module, such as + the generator's `Servicer: TypeAlias = GreeterServicer`, the + class the alias names is what says. + """ + match _enclosing_definitions(module, line): + case [ast.ClassDef() as found]: + pass + case _: + return None + + name = _state_type_of_class(found, module=module) + if name is not None: + return name + + for statement in found.body: + if statement.lineno != line: + continue + match statement: + case ( + ast.Assign(value=value) | ast.AnnAssign(value=value) + ) if value is not None: + named = _last_name(value) + if named is None: + continue + aliased = _top_level_class(module, named) + if aliased is None: + continue + return _state_type_of_class(aliased, module=module) + + return None + + +async def _state_type_named_at( + filename: str, + line: int, + character: int, + *, + files: Files, + generated_directory: str, + asked: frozenset[tuple[str, int, int]] = frozenset(), +) -> tuple[Optional[str], Files]: + """Returns the state type the name written at a position refers + to, and `None` when it refers to anything else. + + Pyright says where the name's definition is. In a `_rbt` module, + the definition itself says which state type. In one of the + developer's files, the name may be an alias, such as + `MyServicer = My.Servicer`, so whatever the aliasing assignment + names is asked about in turn. + """ + if (filename, line, character) in asked: + return None, files + asked = asked | {(filename, line, character)} + + definition = await files.pyright.definition_at( + filename, + line, + character, + ) + if definition is None: + return None, files + + generated = _generated_rbt_filename( + definition, + generated_directory=generated_directory, + ) + if generated is not None: + module, files = await files.syntax_of(generated) + if module is None: + return None, files + return _state_type_in(module, definition.line), files + + module, files = await files.syntax_of(definition.filename) + if module is None: + return None, files + + for node in ast.walk(module): + match node: + case ( + ast.Assign(value=value) | ast.AnnAssign(value=value) + ) if node.lineno == definition.line and value is not None: + position = _position_inside(value) + if position is None: + return None, files + value_line, value_character = position + return await _state_type_named_at( + definition.filename, + value_line, + value_character, + files=files, + generated_directory=generated_directory, + asked=asked, + ) + + return None, files + + +async def _servicer_of_class( + class_definition: ast.ClassDef, + *, + filename: str, + files: Files, + generated_directory: str, +) -> tuple[Optional[ServicerInfo], Files]: + """Returns the servicer a class is, and `None` when it is not + one. + + A servicer is a class with a base referring to a state type's + `Servicer`, however the base is spelled: `Account.Servicer`, a + name it was assigned to first, or a name imported from another + file. Which state type that is, pyright answers; a base whose + name it cannot yet place, most often because `rbt generate` has + not run since the state type was declared, services nothing yet, + and waits. + """ + for base in class_definition.bases: + position = _position_inside(base) + if position is None: + continue + line, character = position + + state_type, files = await _state_type_named_at( + filename, + line, + character, + files=files, + generated_directory=generated_directory, + ) + if state_type is None: + continue + + servicer = ServicerInfo( + state_type=state_type, + file=filename, + line=class_definition.lineno, + character=class_definition.col_offset, + ) + + for statement in class_definition.body: + match statement: + case ( + ast.FunctionDef(name=str(name)) | + ast.AsyncFunctionDef(name=str(name)) + ): + servicer.methods.append( + ServicerInfo.Method( + name=name, + digest=_digest(statement), + ) + ) + + return servicer, files + + return None, files + + def servicers(files: Mapping[str, File]) -> list[ServicerInfo]: """Returns every servicer found, sorted by the state type it services and the file it is written in. @@ -719,9 +1090,35 @@ def servicers(files: Mapping[str, File]) -> list[ServicerInfo]: ) +async def _generated_files(directory: str) -> Mapping[str, tuple[int, int]]: + """Returns every file of generated code under a directory, by + when it was last written and how big it is, which is enough to + say whether `rbt generate` wrote since it was last asked.""" + + def scan() -> dict[str, tuple[int, int]]: + found: dict[str, tuple[int, int]] = {} + for parent, _, filenames in os.walk(directory): + for filename in filenames: + if not filename.endswith(GENERATED_SUFFIXES): + continue + path = os.path.join(parent, filename) + try: + status = os.stat(path) + except OSError: + continue + found[path] = (status.st_mtime_ns, status.st_size) + return found + + # Walking and statting wait on the disk, which is not the event + # loop's to wait on. + return await asyncio.to_thread(scan) + + async def analyze( *, application: str, + pyright: Pyright, + generated_directory: str, roots: Optional[Sequence[str]] = None, known: Optional[Mapping[str, File]] = None, ) -> dict[str, File]: @@ -732,9 +1129,20 @@ async def analyze( absent, however recently it changed, and one that has started being imported is parsed for the first time. + `pyright` answers what the names written in a file refer to, and + is handed every file's text right where the file is read and + parsed. Everything reachable is read before anything is asked, so + the answers are about exactly the snapshot this call parsed -- + every file of it, however the files change on disk in between. A + file that stopped existing is the one thing parsing never meets, + so whoever calls has to take those back itself. + `generated_directory` is where `rbt generate` writes Python code, + which is where the answers worth recording lead. + `known` is what a previous call returned, and spares this one from parsing and analyzing a file that has not changed since -- - neither in its own bytes nor in any file it depends on. + neither in its own bytes nor in any file its analysis depended + on, however many dependencies away. `roots` are the directories a module may be found under, which is both how a module name becomes a file and where the developer's @@ -745,25 +1153,25 @@ async def analyze( if roots is None: roots = _roots(application) - files = Files.create(roots=roots, known=known or {}) + files = Files.create( + roots=roots, + known=known or {}, + pyright=pyright, + ) + # Everything reachable is read and parsed first -- and, as it is + # parsed, handed to pyright -- so that once the asking starts, + # every answer is about the very snapshot the analysis parsed, + # for every file at once. The disk may well keep changing while + # this runs; the two views drift together, and the save that + # changed it wakes the next iteration. _, files = await files.lookup_or_parse_filename(application) while len(files.pending) > 0: - # Parsing and analyzing hold on to the interpreter for as - # long as they take, so a file at a time leaves the dashboard - # free to answer. + # Parsing holds on to the interpreter for as long as it + # takes, so a file at a time leaves the dashboard free to + # answer. async for filename in cooperatively(files.pending): - match files.parsed.get(filename): - case ParsedFile(): - files = await _analyze_file(filename, files) - case None: - # Nothing to analyze: the file was kept because - # neither its bytes nor any of its dependencies - # changed, so the analysis it got when it last - # changed still stands, in `analyzed` already. - pass - files = files.without_pending_file(filename) # Following the file's imports is how the rest of the @@ -774,17 +1182,53 @@ async def analyze( for module in imports.may_load: _, files = await files.lookup_or_parse_module(module) + # Then whatever was parsed is analyzed. A file kept from the + # previous iteration keeps its analysis too: neither its bytes + # nor anything in the closure of its dependencies changed, so its + # recorded answers still stand, in `analyzed` already. + async for filename in cooperatively(list(files.parsed)): + files = await _analyze_file( + filename, + files, + generated_directory=generated_directory, + ) + return dict(files.analyzed) -async def watch(context: WorkflowContext, *, application: str) -> None: +async def watch( + context: WorkflowContext, + *, + application: str, + generated_directory: Optional[str], +) -> None: """Returns only when the dashboard stops, recording the servicers - in the developer's application for as long as it runs.""" + in the developer's application for as long as it runs. + + `generated_directory` is where `rbt generate` writes Python code, + which is where the state types the servicers service are defined. + A developer who named none gets one recording saying so, since + the `.rbtrc` naming it cannot change while the dashboard runs. + """ + if generated_directory is None: + + async def record_nothing_generated(state) -> None: + del state.servicers[:] + state.generated = False + + await Implementation.ref().idempotently( + 'Record that nothing is generated' + ).write(context, record_nothing_generated) + return + roots = _roots(application) globs = [os.path.join(root, SOURCE_GLOB) for root in roots] + globs.append(os.path.join(generated_directory, SOURCE_GLOB)) - recorded: Optional[list[ServicerInfo]] = None + recorded: Optional[tuple[list[ServicerInfo], bool]] = None known: dict[str, File] = {} + generated_files: Optional[Mapping[str, tuple[int, int]]] = None + pyright: Optional[Pyright] = None with file_watcher() as watcher: async for iteration in context.loop('Watch the application'): @@ -801,20 +1245,62 @@ async def watch(context: WorkflowContext, *, application: str) -> None: # dependency digest fails its check there and is analyzed # again. async with watcher.watch(globs) as event: + generated_files_now = await _generated_files( + generated_directory + ) + + # The server reads a file nobody hands it, such as + # generated code, once. The developer's own files are + # handed to it below as they change, but generated + # code changes behind its back, so a change there + # gets a fresh server, which reads everything anew. + # Everything analyzed with the old server's answers + # is forgotten with it. + if pyright is None or generated_files_now != generated_files: + if pyright is not None: + await pyright.stop() + pyright = Pyright() + # Rooted where the dashboard runs, which is the + # developer's working directory, the one both the + # application and the generated directory are + # spelled relative to. + await pyright.start( + root=os.getcwd(), + paths=[*roots, generated_directory], + ) + known = {} + + generated_files = generated_files_now + + # A file that stopped existing has to be taken back, + # so the server sees it the way the disk does. Every + # other change reaches the server as the analysis + # reads the changed files. + for filename in known: + if not await aiofiles.os.path.isfile(filename): + await pyright.take_back(filename) + known = await analyze( - application=application, roots=roots, known=known + application=application, + pyright=pyright, + generated_directory=generated_directory, + roots=roots, + known=known, ) found = servicers(known) + answer = (found, len(generated_files_now) > 0) + # Most edits change no servicer, and a write wakes # every browser reading `Get`, so one is only worth # making when the answer is different. - if found != recorded: + if answer != recorded: async def record(state) -> None: del state.servicers[:] state.servicers.extend(found) + state.generated = len(generated_files_now) > 0 # Written inline rather than through a method of # its own: the workflow runs on this very state. @@ -824,7 +1310,7 @@ async def record(state) -> None: # After the write, so that what is remembered is # what was recorded and not what was about to be. - recorded = found + recorded = answer # Which save wakes this, and when, is not # deterministic, and a replay may wait on a different diff --git a/reboot/dashboard/pyright.py b/reboot/dashboard/pyright.py new file mode 100644 index 00000000..322b4912 --- /dev/null +++ b/reboot/dashboard/pyright.py @@ -0,0 +1,354 @@ +"""Asks pyright what the developer's code means. + +Pyright is a type checker built to answer questions about code as it +is being written, which is exactly the dashboard's situation: it +infers types through unannotated helpers, containers, and branches, +and it degrades to `Unknown` rather than guessing when generated +code does not exist yet. + +Spoken to over the Language Server Protocol on stdio. Whoever reads +a file hands its text over, and answers are about each file as it +was last handed. Files nobody hands over, such as generated code, +the server reads from disk once, so a change to one of those takes +a fresh server to be seen. +""" +import asyncio +import hashlib +import itertools +import json +import os +import shutil +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Optional, Sequence + +# Where the `pyright-langserver` executable is, when somebody such as +# a Bazel test has to say exactly which one to run. Without it the +# executable is taken from the `PATH`. +ENVVAR_RBT_PYRIGHT_LANGSERVER = 'RBT_PYRIGHT_LANGSERVER' + + +@dataclass(frozen=True, kw_only=True) +class Definition: + """Where a name is defined.""" + + # The file the definition is in, spelled absolutely. + filename: str + + # The line the definition starts on, counting from one, which is + # how `ast` counts. + line: int + + # The column the definition starts at, counting from zero. + character: int + + +class Pyright: + """One running pyright language server, answering questions. + + `start` it once, `hand` it every file's text as the file is read, + and ask with `type_at` and `definition_at` about the files it was + handed. Positions are given the way `ast` gives them: lines from + one, columns from zero. + """ + + def __init__(self) -> None: + self._process: Optional[asyncio.subprocess.Process] = None + self._ids = itertools.count(1) + self._replies: dict[int, asyncio.Future[Any]] = {} + self._reader: Optional[asyncio.Task[None]] = None + # For every file handed to the server, the version number the + # protocol wants counted up per file, and a digest of the + # text sent, so that handing the text it already has sends + # nothing. + self._sent: dict[str, tuple[int, bytes]] = {} + + async def start( + self, + *, + root: str, + paths: Sequence[str], + ) -> None: + """Starts the server over `root`, resolving imports through + `paths`, such as the directory `rbt generate` writes to.""" + executable = os.environ.get(ENVVAR_RBT_PYRIGHT_LANGSERVER + ) or shutil.which('pyright-langserver') + + if executable is None: + raise FileNotFoundError( + 'Could not find `pyright-langserver`. Install pyright, ' + 'such as with `npm install pyright`, or point ' + f'`{ENVVAR_RBT_PYRIGHT_LANGSERVER}` at it.' + ) + + self._process = await asyncio.create_subprocess_exec( + executable, + '--stdio', + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.DEVNULL, + ) + + self._reader = asyncio.create_task(self._read_replies()) + + await self._request( + 'initialize', + { + 'processId': os.getpid(), + 'rootUri': Path(root).absolute().as_uri(), + 'capabilities': {}, + 'initializationOptions': {}, + }, + ) + await self._notify('initialized', {}) + await self._notify( + 'workspace/didChangeConfiguration', + { + 'settings': + { + 'python': + { + 'analysis': + { + 'extraPaths': list(paths), + 'typeCheckingMode': 'basic', + 'useLibraryCodeForTypes': True, + } + } + } + }, + ) + + async def stop(self) -> None: + """Stops the server. Questions after this raise.""" + if self._reader is not None: + self._reader.cancel() + if self._process is not None: + self._process.terminate() + await self._process.wait() + + async def hand( + self, + filename: str, + text: str, + ) -> None: + """Hands the server one file's text, so that answers about + the file are about exactly this text. Handing the text the + server already has sends nothing.""" + digest = hashlib.sha256(text.encode()).digest() + + sent = self._sent.get(filename) + if sent is None: + self._sent[filename] = (1, digest) + await self._notify( + 'textDocument/didOpen', + { + 'textDocument': + { + 'uri': Path(filename).absolute().as_uri(), + 'languageId': 'python', + 'version': 1, + 'text': text, + } + }, + ) + return + + version, sent_digest = sent + if sent_digest == digest: + return + + version += 1 + self._sent[filename] = (version, digest) + await self._notify( + 'textDocument/didChange', + { + 'textDocument': + { + 'uri': Path(filename).absolute().as_uri(), + 'version': version, + }, + 'contentChanges': [{ + 'text': text + }], + }, + ) + + async def take_back(self, filename: str) -> None: + """Takes a handed file back, after which the server sees it + the way the disk does; for a file that stopped existing, that + is absent.""" + if self._sent.pop(filename, None) is None: + return + + await self._notify( + 'textDocument/didClose', + {'textDocument': { + 'uri': Path(filename).absolute().as_uri(), + }}, + ) + + async def type_at( + self, + filename: str, + line: int, + character: int, + ) -> Optional[str]: + """Returns the type of the expression at a position in the + text the file was last handed as, spelled the way pyright + spells it, and `None` when pyright does not know, including + when the type is `Unknown` because generated code does not + exist yet.""" + reply = await self._request( + 'textDocument/hover', + self._position(filename, line, character), + ) + + if reply is None: + return None + + contents = reply.get('contents') + if isinstance(contents, dict): + contents = contents.get('value', '') + if not isinstance(contents, str): + return None + + # Pyright writes hovers like `(variable) shop: Shop` or + # `(class) Shop`; the type is what follows the last `: ` on + # the first line that has one. + for line_of in contents.splitlines(): + if ': ' in line_of: + spelled = line_of.rsplit(': ', 1)[1].strip('` ') + if spelled and 'Unknown' not in spelled: + return spelled + return None + + return None + + async def definition_at( + self, + filename: str, + line: int, + character: int, + ) -> Optional[Definition]: + """Returns where the name at a position in the text the file + was last handed as is defined, and `None` when pyright does + not know.""" + reply = await self._request( + 'textDocument/definition', + self._position(filename, line, character), + ) + + match reply: + case [ + { + 'uri': str(uri), + 'range': + { + 'start': + { + 'line': int(line_at), + 'character': int(character_at), + } + }, + }, *_ + ] if uri.startswith('file://'): + return Definition( + filename=uri[len('file://'):], + line=line_at + 1, + character=character_at, + ) + + return None + + def _position( + self, + filename: str, + line: int, + character: int, + ) -> dict[str, Any]: + return { + 'textDocument': { + 'uri': Path(filename).absolute().as_uri() + }, + 'position': { + 'line': line - 1, + 'character': character, + }, + } + + async def _request( + self, + method: str, + params: dict[str, Any], + ) -> Any: + assert self._process is not None and self._process.stdin is not None + id = next(self._ids) + future: asyncio.Future[Any] = asyncio.get_running_loop().create_future( + ) + self._replies[id] = future + self._write( + { + 'jsonrpc': '2.0', + 'id': id, + 'method': method, + 'params': params, + } + ) + await self._process.stdin.drain() + return await future + + async def _notify( + self, + method: str, + params: dict[str, Any], + ) -> None: + assert self._process is not None and self._process.stdin is not None + self._write({'jsonrpc': '2.0', 'method': method, 'params': params}) + await self._process.stdin.drain() + + def _write(self, message: dict[str, Any]) -> None: + assert self._process is not None and self._process.stdin is not None + body = json.dumps(message).encode() + self._process.stdin.write( + b'Content-Length: %d\r\n\r\n%s' % (len(body), body) + ) + + async def _read_replies(self) -> None: + assert self._process is not None and self._process.stdout is not None + while True: + length = 0 + ended = False + while True: + header = await self._process.stdout.readline() + if not header: + ended = True + break + if header.startswith(b'Content-Length:'): + length = int(header.split(b':')[1]) + if header == b'\r\n': + break + if ended: + # The server died. Whoever is waiting on it finds + # out now rather than waiting forever. + for future in self._replies.values(): + if not future.done(): + future.set_exception(RuntimeError('pyright exited')) + self._replies.clear() + return + body = json.loads(await self._process.stdout.readexactly(length)) + # Server-to-client requests must be answered or the + # server waits on us the way we wait on it. + if 'id' in body and 'method' in body: + self._write( + { + 'jsonrpc': '2.0', + 'id': body['id'], + 'result': None, + } + ) + continue + if 'id' in body: + waiting = self._replies.pop(body['id'], None) + if waiting is not None and not waiting.done(): + waiting.set_result(body.get('result')) diff --git a/reboot/dashboard/servicers.py b/reboot/dashboard/servicers.py index 5a8804e2..a65c0a08 100644 --- a/reboot/dashboard/servicers.py +++ b/reboot/dashboard/servicers.py @@ -21,6 +21,7 @@ from reboot.dashboard.constants import ( ENVVAR_RBT_API_DIRECTORY, ENVVAR_RBT_APPLICATION, + ENVVAR_RBT_GENERATED_DIRECTORY, ) @@ -89,7 +90,10 @@ async def Get( context: ReaderContext, request: Implementation.GetRequest, ) -> Implementation.GetResponse: - return Implementation.GetResponse(servicers=self.state.servicers) + return Implementation.GetResponse( + servicers=self.state.servicers, + generated=self.state.generated, + ) @classmethod async def Watch( @@ -106,10 +110,13 @@ async def Watch( case for a Node.js application. """ application = os.environ.get(ENVVAR_RBT_APPLICATION) + generated_directory = os.environ.get(ENVVAR_RBT_GENERATED_DIRECTORY) if application is not None: await implementation_watcher.watch( - context, application=application + context, + application=application, + generated_directory=generated_directory, ) return Implementation.WatchResponse() diff --git a/tests/reboot/dashboard/BUILD.bazel b/tests/reboot/dashboard/BUILD.bazel index 39fa43f4..cd3f367b 100644 --- a/tests/reboot/dashboard/BUILD.bazel +++ b/tests/reboot/dashboard/BUILD.bazel @@ -25,6 +25,13 @@ py_test( py_test( name = "implementation_watcher_tests_py", srcs = ["implementation_watcher_tests.py"], + data = ["//reboot/dashboard:pyright_langserver"], + env = { + # What the `js_binary` launcher wants set when it is run + # outside of a build action. + "BAZEL_BINDIR": ".", + "RBT_PYRIGHT_LANGSERVER": "$(rootpath //reboot/dashboard:pyright_langserver)", + }, main = "implementation_watcher_tests.py", deps = [ "//reboot/aio:tests_py", diff --git a/tests/reboot/dashboard/implementation_watcher_tests.py b/tests/reboot/dashboard/implementation_watcher_tests.py index bb976200..80ed0a56 100644 --- a/tests/reboot/dashboard/implementation_watcher_tests.py +++ b/tests/reboot/dashboard/implementation_watcher_tests.py @@ -15,10 +15,13 @@ API_ID, ENVVAR_RBT_API_DIRECTORY, ENVVAR_RBT_APPLICATION, + ENVVAR_RBT_GENERATED_DIRECTORY, IMPLEMENTATION_ID, ) from reboot.dashboard.implementation_watcher import File, analyze, servicers from reboot.dashboard.main import application +from reboot.dashboard.pyright import Pyright +from typing import Optional from unittest.mock import patch API_FILE = ''' @@ -79,6 +82,55 @@ async def stock(self, context, request): SHOP = SERVICER.format(state='Shop', module='shop') DEPOT = SINGLETON.format(state='Depot', module='depot') +# The shape of a generated module, as far as resolving a state type +# needs: the state type's class and its servicer bases carrying the +# state type's name as `__state_type_name__`, and the `Servicer` +# aliases a servicer's base leads through. +GENERATED = ''' +from typing import TypeAlias + + +def StateTypeName(name): + return name + + +class {state}BaseServicer: + + __state_type_name__ = StateTypeName('shop.v1.{state}') + + +class {state}Servicer({state}BaseServicer): + pass + + +class {state}SingletonServicer({state}BaseServicer): + pass + + +class {state}Singleton: + + Servicer: TypeAlias = {state}SingletonServicer + + +class {state}: + + __state_type_name__ = StateTypeName('shop.v1.{state}') + + Servicer: TypeAlias = {state}Servicer + + singleton: TypeAlias = {state}Singleton +''' + + +def _write_generated(directory: Path) -> None: + """Writes what `rbt generate` would: the generated modules the + fixtures import their state types from.""" + for module, state in (('shop', 'Shop'), ('depot', 'Depot')): + path = directory / 'shop' / 'v1' / f'{module}_rbt.py' + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(GENERATED.format(state=state)) + + APPLICATION = ''' from shop_servicer import ShopServicer from reboot.aio.applications import Application @@ -89,14 +141,6 @@ async def main(): ''' -def _state_types_and_files(files: dict[str, File]) -> list[tuple[str, str]]: - """Returns every servicer as the state type it services and the - file it is written in.""" - return [ - (servicer.state_type, servicer.file) for servicer in servicers(files) - ] - - class ImplementationWatcherTest(unittest.IsolatedAsyncioTestCase): async def asyncSetUp(self) -> None: @@ -104,8 +148,10 @@ async def asyncSetUp(self) -> None: # exist and be named first. self._api = tempfile.TemporaryDirectory() self._source = tempfile.TemporaryDirectory() + self._generated = tempfile.TemporaryDirectory() self.api = Path(self._api.name) self.source = Path(self._source.name) + self.generated = Path(self._generated.name) (self.source / 'shop_servicer.py').write_text( SERVICER.format(state='Shop', module='shop') @@ -117,6 +163,7 @@ async def asyncSetUp(self) -> None: { ENVVAR_RBT_API_DIRECTORY: str(self.api), ENVVAR_RBT_APPLICATION: str(self.source / 'main.py'), + ENVVAR_RBT_GENERATED_DIRECTORY: str(self.generated), }, ) self._environment.start() @@ -128,9 +175,15 @@ async def asyncSetUp(self) -> None: async def asyncTearDown(self) -> None: await self.rbt.stop() self._environment.stop() + self._generated.cleanup() self._source.cleanup() self._api.cleanup() + def _generate(self) -> None: + """Writes the generated modules, the way running + `rbt generate` would.""" + _write_generated(self.generated) + def _declare( self, name: str, @@ -169,21 +222,41 @@ async def _servicers(self, *, satisfied): raise AssertionError('never satisfied') + async def _implementation(self, *, satisfied): + """Returns the recorded implementation once it satisfies, + reading again whenever it changes.""" + context = self.rbt.create_external_context(name=self.id()) + + async for response in Implementation.ref(IMPLEMENTATION_ID + ).reactively().Get(context): + if satisfied(response): + return response + + raise AssertionError('never satisfied') + async def test_records_the_servicer_it_finds(self) -> None: - found = await self._servicers(satisfied=lambda found: 'Shop' in found) + self._generate() + + found = await self._servicers( + satisfied=lambda found: 'shop.v1.Shop' in found + ) self.assertEqual( - [servicer.file for servicer in found['Shop']], + [servicer.file for servicer in found['shop.v1.Shop']], [str(self.source / 'shop_servicer.py')], ) async def test_the_methods_reach_the_state(self) -> None: """What the browser will join against what the API files say each state type declares.""" - found = await self._servicers(satisfied=lambda found: 'Shop' in found) + self._generate() + + found = await self._servicers( + satisfied=lambda found: 'shop.v1.Shop' in found + ) self.assertEqual( - [method.name for method in found['Shop'][0].methods], + [method.name for method in found['shop.v1.Shop'][0].methods], ['look'], ) @@ -191,15 +264,19 @@ async def test_a_state_type_nothing_services(self) -> None: """A state type nothing services is one with no entry, which is how a reader tells it apart from one a servicer was found for.""" + self._generate() self._declare('depot', state='Depot') - found = await self._servicers(satisfied=lambda found: 'Shop' in found) + found = await self._servicers( + satisfied=lambda found: 'shop.v1.Shop' in found + ) - self.assertNotIn('Depot', found) + self.assertNotIn('shop.v1.Depot', found) async def test_a_state_type_two_classes_service(self) -> None: """Both files are recorded against it, rather than one being chosen between them.""" + self._generate() (self.source / 'other_servicer.py').write_text(SHOP) (self.source / 'main.py').write_text( APPLICATION.replace( @@ -210,22 +287,23 @@ async def test_a_state_type_two_classes_service(self) -> None: ) found = await self._servicers( - satisfied=lambda found: len(found.get('Shop', [])) == 2 + satisfied=lambda found: len(found.get('shop.v1.Shop', [])) == 2 ) self.assertEqual( - [servicer.file for servicer in found['Shop']], [ + [servicer.file for servicer in found['shop.v1.Shop']], [ str(self.source / 'other_servicer.py'), str(self.source / 'shop_servicer.py'), ] ) async def test_a_servicer_written_after_the_dashboard_started( - self + self, ) -> None: """The application is watched, so a servicer written while the dashboard runs is found without a restart.""" - await self._servicers(satisfied=lambda found: 'Shop' in found) + self._generate() + await self._servicers(satisfied=lambda found: 'shop.v1.Shop' in found) (self.source / 'depot_servicer.py').write_text(DEPOT) (self.source / 'main.py').write_text( @@ -236,23 +314,28 @@ async def test_a_servicer_written_after_the_dashboard_started( ) ) - found = await self._servicers(satisfied=lambda found: 'Depot' in found) + found = await self._servicers( + satisfied=lambda found: 'shop.v1.Depot' in found + ) self.assertEqual( - [servicer.file for servicer in found['Depot']], + [servicer.file for servicer in found['shop.v1.Depot']], [str(self.source / 'depot_servicer.py')], ) async def test_what_is_declared_and_what_implements_it_are_separate( - self + self, ) -> None: """Read from different places by different workflows, and so recorded without either waiting on the other.""" + self._generate() self._declare('shop', state='Shop') - found = await self._servicers(satisfied=lambda found: 'Shop' in found) + found = await self._servicers( + satisfied=lambda found: 'shop.v1.Shop' in found + ) self.assertEqual( - [servicer.file for servicer in found['Shop']], + [servicer.file for servicer in found['shop.v1.Shop']], [str(self.source / 'shop_servicer.py')], ) @@ -267,15 +350,47 @@ async def test_what_is_declared_and_what_implements_it_are_separate( raise AssertionError("'shop.v1.Shop' was never declared") + async def test_generate_running_while_the_dashboard_runs(self) -> None: + """`rbt generate` writing its code is what ties the waiting + servicers to their state types, and `generated` is what tells + the dashboard to suggest running it.""" + response = await self._implementation( + satisfied=lambda response: not response.generated + ) + self.assertEqual(list(response.servicers), []) + + self._generate() + + response = await self._implementation( + satisfied=lambda response: response.generated and + len(response.servicers) > 0 + ) + self.assertEqual( + [servicer.state_type for servicer in response.servicers], + ['shop.v1.Shop'], + ) + class ServicerFilesTest(unittest.IsolatedAsyncioTestCase): """Which file implements which state type.""" - def setUp(self) -> None: + async def asyncSetUp(self) -> None: self._directory = tempfile.TemporaryDirectory() self.directory = Path(self._directory.name) + self._generated_directory = tempfile.TemporaryDirectory() + self.generated = Path(self._generated_directory.name) + _write_generated(self.generated) + + self.pyright = Pyright() + await self.pyright.start( + root=str(self.directory), + paths=[str(self.directory), + str(self.generated)], + ) - def tearDown(self) -> None: + async def asyncTearDown(self) -> None: + await self.pyright.stop() + self._generated_directory.cleanup() self._directory.cleanup() def _write(self, name: str, *, source: str) -> str: @@ -284,19 +399,47 @@ def _write(self, name: str, *, source: str) -> str: path.write_text(source) return str(path) + async def _analyze( + self, + application: str, + known: Optional[dict[str, File]] = None, + roots: Optional[list[str]] = None, + ) -> dict[str, File]: + """Returns the analysis of an application, asked of the one + pyright the test runs; the analysis hands it every file it + parses.""" + return await analyze( + application=application, + pyright=self.pyright, + generated_directory=str(self.generated), + roots=roots, + known=known or {}, + ) + + def _state_types_and_files( + self, + files: dict[str, File], + ) -> list[tuple[str, str]]: + """Returns every servicer as the state type it services and + the file it is written in.""" + return [ + (servicer.state_type, servicer.file) + for servicer in servicers(files) + ] + async def test_finds_the_file_a_state_type_is_implemented_in(self) -> None: self._write('shop_servicer.py', source=SHOP) application = self._write('main.py', source=APPLICATION) - found = _state_types_and_files(await analyze(application=application)) + found = self._state_types_and_files(await self._analyze(application)) self.assertEqual( found, - [('Shop', str(self.directory / 'shop_servicer.py'))], + [('shop.v1.Shop', str(self.directory / 'shop_servicer.py'))], ) async def test_a_singleton_says_what_it_services_the_same_way( - self + self, ) -> None: self._write('depot_servicer.py', source=DEPOT) application = self._write( @@ -307,11 +450,11 @@ async def test_a_singleton_says_what_it_services_the_same_way( ), ) - found = _state_types_and_files(await analyze(application=application)) + found = self._state_types_and_files(await self._analyze(application)) self.assertEqual( found, - [('Depot', str(self.directory / 'depot_servicer.py'))], + [('shop.v1.Depot', str(self.directory / 'depot_servicer.py'))], ) async def test_several_state_types_in_one_file(self) -> None: @@ -329,12 +472,12 @@ async def main(): ''' ) - found = _state_types_and_files(await analyze(application=application)) + found = self._state_types_and_files(await self._analyze(application)) self.assertEqual( found, [ - ('Depot', str(self.directory / 'servicers.py')), - ('Shop', str(self.directory / 'servicers.py')), + ('shop.v1.Depot', str(self.directory / 'servicers.py')), + ('shop.v1.Shop', str(self.directory / 'servicers.py')), ] ) @@ -342,7 +485,7 @@ async def main(): # Reached however the application reaches it. async def test_an_application_that_does_not_spell_out_its_servicers( - self + self, ) -> None: """Only running it would say what `servicers()` returns -- but its module had to be imported for it to be callable at all, @@ -359,11 +502,11 @@ async def main(): ''' ) - found = _state_types_and_files(await analyze(application=application)) + found = self._state_types_and_files(await self._analyze(application)) self.assertEqual( found, - [('Shop', str(self.directory / 'servicers.py'))], + [('shop.v1.Shop', str(self.directory / 'servicers.py'))], ) async def test_a_servicer_reached_through_another_module(self) -> None: @@ -383,11 +526,11 @@ async def main(): ''' ) - found = _state_types_and_files(await analyze(application=application)) + found = self._state_types_and_files(await self._analyze(application)) self.assertEqual( found, - [('Shop', str(self.directory / 'shop_servicer.py'))], + [('shop.v1.Shop', str(self.directory / 'shop_servicer.py'))], ) async def test_an_import_that_is_not_at_the_top_of_the_file(self) -> None: @@ -410,11 +553,11 @@ async def main(): ''' ) - found = _state_types_and_files(await analyze(application=application)) + found = self._state_types_and_files(await self._analyze(application)) self.assertEqual( found, - [('Shop', str(self.directory / 'shop_servicer.py'))], + [('shop.v1.Shop', str(self.directory / 'shop_servicer.py'))], ) async def test_a_servicer_in_a_package(self) -> None: @@ -431,18 +574,18 @@ async def main(): ''' ) - found = _state_types_and_files(await analyze(application=application)) + found = self._state_types_and_files(await self._analyze(application)) self.assertEqual( found, - [('Shop', str(self.directory / 'servicers' / 'shop.py'))], + [('shop.v1.Shop', str(self.directory / 'servicers' / 'shop.py'))], ) ################################################################### # Where the walk stops. async def test_an_import_of_somebody_elses_package_leads_nowhere( - self + self, ) -> None: """A module no root holds is not the developer's code. Reading it is not the dashboard's business, and no state type of theirs @@ -462,22 +605,22 @@ async def main(): ''' ) - found = _state_types_and_files( - await analyze(application=application) + found = self._state_types_and_files( + await self._analyze(application) ) self.assertEqual(found, []) # Named as a root, the very same import leads there. - found = _state_types_and_files( - await analyze( - application=application, + found = self._state_types_and_files( + await self._analyze( + application, roots=[str(self.directory), elsewhere.name], ) ) self.assertEqual( found, - [('Shop', str(Path(elsewhere.name) / 'library.py'))], + [('shop.v1.Shop', str(Path(elsewhere.name) / 'library.py'))], ) finally: elsewhere.cleanup() @@ -485,27 +628,146 @@ async def main(): ################################################################### # The methods each servicer defines. - async def test_any_dotted_base_ending_in_servicer_counts(self) -> None: + async def test_a_state_type_reached_through_its_module(self) -> None: + """However the state type is spelled, what it refers to is + the answer -- here through a module alias.""" + self._write( + 'shop_servicer.py', source=''' +import shop.v1.shop_rbt as rbt + + +class ShopServicer(rbt.Shop.Servicer): + + async def look(self, context, request): + pass +''' + ) + application = self._write('main.py', source=APPLICATION) + + found = self._state_types_and_files(await self._analyze(application)) + + self.assertEqual( + found, + [( + 'shop.v1.Shop', + str(self.directory / 'shop_servicer.py'), + )], + ) + + async def test_an_aliased_state_type_still_resolves(self) -> None: + """`import ... as` respells the name; what it refers to does + not move.""" + self._write( + 'shop_servicer.py', + source=SHOP.replace( + 'from shop.v1.shop_rbt import Shop', + 'from shop.v1.shop_rbt import Shop as S', + ).replace('(Shop.Servicer)', '(S.Servicer)'), + ) + application = self._write('main.py', source=APPLICATION) + + found = self._state_types_and_files(await self._analyze(application)) + + self.assertEqual( + found, + [( + 'shop.v1.Shop', + str(self.directory / 'shop_servicer.py'), + )], + ) + + async def test_a_servicer_through_an_assigned_alias(self) -> None: + """`MyServicer = Shop.Servicer` respells the base; what it + refers to does not move.""" self._write( 'shop_servicer.py', source=SHOP.replace( 'class ShopServicer(Shop.Servicer):', - 'class ShopServicer(stores.things.Shop.Servicer):', + 'MyServicer = Shop.Servicer\n' + '\n' + '\n' + 'class ShopServicer(MyServicer):', ), ) application = self._write('main.py', source=APPLICATION) - found = _state_types_and_files(await analyze(application=application)) + found = self._state_types_and_files(await self._analyze(application)) self.assertEqual( found, - [ - ( - 'stores.things.Shop', - str(self.directory / 'shop_servicer.py'), - ) - ], + [( + 'shop.v1.Shop', + str(self.directory / 'shop_servicer.py'), + )], + ) + + async def test_a_servicer_through_an_alias_in_another_file( + self, + ) -> None: + self._write( + 'aliases.py', + source='from shop.v1.shop_rbt import Shop\n' + '\n' + 'MyServicer = Shop.Servicer\n', + ) + self._write( + 'shop_servicer.py', + source=SHOP.replace( + 'from shop.v1.shop_rbt import Shop', + 'from aliases import MyServicer', + ).replace( + 'class ShopServicer(Shop.Servicer):', + 'class ShopServicer(MyServicer):', + ), ) + application = self._write('main.py', source=APPLICATION) + + found = self._state_types_and_files(await self._analyze(application)) + + self.assertEqual( + found, + [( + 'shop.v1.Shop', + str(self.directory / 'shop_servicer.py'), + )], + ) + + async def test_a_state_type_that_is_not_generated_yet(self) -> None: + """A name pyright cannot resolve services nothing yet: its + generated module is exactly what has not been written.""" + self._write( + 'shop_servicer.py', + source=SHOP.replace( + 'from shop.v1.shop_rbt import Shop', + 'from warehouse.v1.warehouse_rbt import Shop', + ), + ) + application = self._write('main.py', source=APPLICATION) + + found = self._state_types_and_files(await self._analyze(application)) + + self.assertEqual(found, []) + + async def test_a_base_defined_outside_generated_code(self) -> None: + """Resolving is not enough: the state type has to be defined + in code `rbt generate` wrote.""" + self._write( + 'shop_servicer.py', source=''' +class Local: + + class Servicer: + pass + + +class ShopServicer(Local.Servicer): + pass +''' + ) + application = self._write('main.py', source=APPLICATION) + + found = self._state_types_and_files(await self._analyze(application)) + + self.assertEqual(found, []) async def test_every_import_is_a_dependency(self) -> None: helper = self._write('helper.py', source='VALUE = 1\n') @@ -515,7 +777,7 @@ async def test_every_import_is_a_dependency(self) -> None: ) application = self._write('main.py', source=APPLICATION) - found = await analyze(application=application) + found = await self._analyze(application) self.assertIn(str(helper), found[str(servicer)].dependencies) @@ -523,19 +785,31 @@ async def test_records_the_methods_a_servicer_defines(self) -> None: self._write('shop_servicer.py', source=SHOP) application = self._write('main.py', source=APPLICATION) - found = servicers(await analyze(application=application)) + found = servicers(await self._analyze(application)) self.assertEqual( [method.name for method in found[0].methods], ['look'] ) + async def test_where_the_servicer_is_written(self) -> None: + """The line and column of the class, for a reader to be taken + to it.""" + servicer_file = self._write('shop_servicer.py', source=SHOP) + application = self._write('main.py', source=APPLICATION) + + found = servicers(await self._analyze(application)) + + lines = Path(servicer_file).read_text().splitlines() + line = lines.index('class ShopServicer(Shop.Servicer):') + 1 + self.assertEqual((found[0].line, found[0].character), (line, 0)) + async def test_a_method_reformatted_digests_the_same(self) -> None: """The digest is over what the method says, so laying it out differently or writing a comment in it is not a change.""" self._write('shop_servicer.py', source=SHOP) application = self._write('main.py', source=APPLICATION) - before = servicers(await analyze(application=application)) + before = servicers(await self._analyze(application)) self._write( 'shop_servicer.py', @@ -551,7 +825,7 @@ async def test_a_method_reformatted_digests_the_same(self) -> None: ), ) - after = servicers(await analyze(application=application)) + after = servicers(await self._analyze(application)) self.assertEqual( [method.digest for method in after[0].methods], @@ -559,19 +833,19 @@ async def test_a_method_reformatted_digests_the_same(self) -> None: ) async def test_a_method_whose_body_changes_digests_differently( - self + self, ) -> None: self._write('shop_servicer.py', source=SHOP) application = self._write('main.py', source=APPLICATION) - before = servicers(await analyze(application=application)) + before = servicers(await self._analyze(application)) self._write( 'shop_servicer.py', source=SHOP.replace(' pass', ' return None'), ) - after = servicers(await analyze(application=application)) + after = servicers(await self._analyze(application)) self.assertNotEqual( after[0].methods[0].digest, @@ -582,14 +856,14 @@ async def test_a_method_whose_body_changes_digests_differently( # Parse again only what has changed. async def test_a_file_written_with_the_same_bytes_is_not_parsed_again( - self + self, ) -> None: """Identical bytes are nothing to parse, which is only observable as the parsing not happening.""" servicer = self._write('shop_servicer.py', source=SHOP) application = self._write('main.py', source=APPLICATION) - known = await analyze(application=application) + known = await self._analyze(application) Path(servicer).write_text(SHOP) @@ -598,12 +872,12 @@ async def test_a_file_written_with_the_same_bytes_is_not_parsed_again( '_parse', wraps=implementation_watcher._parse, ) as parse: - await analyze(application=application, known=known) + await self._analyze(application, known=known) parse.assert_not_called() async def test_a_file_written_with_other_bytes_is_parsed_again( - self + self, ) -> None: """Even with the mtime it was read with put back, which is what a save landing in the same clock tick leaves behind.""" @@ -611,19 +885,19 @@ async def test_a_file_written_with_other_bytes_is_parsed_again( application = self._write('main.py', source=APPLICATION) modified = os.stat(servicer).st_mtime_ns - known = await analyze(application=application) + known = await self._analyze(application) Path(servicer).write_text(DEPOT) os.utime(servicer, ns=(modified, modified)) - found = _state_types_and_files( - await analyze(application=application, known=known) + found = self._state_types_and_files( + await self._analyze(application, known=known) ) - self.assertEqual(found, [('Depot', servicer)]) + self.assertEqual(found, [('shop.v1.Depot', servicer)]) async def test_a_servicer_that_stops_being_imported_is_dropped( - self + self, ) -> None: """However recently it changed: what the application reaches is what it registers.""" @@ -638,27 +912,27 @@ async def test_a_servicer_that_stops_being_imported_is_dropped( ), ) - known = await analyze(application=application) + known = await self._analyze(application) self.assertEqual(len(servicers(known)), 2) self._write('main.py', source=APPLICATION) - found = _state_types_and_files( - await analyze(application=application, known=known) + found = self._state_types_and_files( + await self._analyze(application, known=known) ) self.assertEqual( found, - [('Shop', str(self.directory / 'shop_servicer.py'))], + [('shop.v1.Shop', str(self.directory / 'shop_servicer.py'))], ) async def test_a_servicer_that_starts_being_imported_is_found( - self + self, ) -> None: self._write('shop_servicer.py', source=SHOP) application = self._write('main.py', source=APPLICATION) - known = await analyze(application=application) + known = await self._analyze(application) self._write('depot_servicer.py', source=DEPOT) self._write( @@ -670,14 +944,14 @@ async def test_a_servicer_that_starts_being_imported_is_found( ), ) - found = _state_types_and_files( - await analyze(application=application, known=known) + found = self._state_types_and_files( + await self._analyze(application, known=known) ) self.assertEqual( found, [ - ('Depot', str(self.directory / 'depot_servicer.py')), - ('Shop', str(self.directory / 'shop_servicer.py')), + ('shop.v1.Depot', str(self.directory / 'depot_servicer.py')), + ('shop.v1.Shop', str(self.directory / 'shop_servicer.py')), ] ) @@ -699,14 +973,16 @@ async def test_a_servicer_reached_through_a_relative_import(self) -> None: ), ) - found = _state_types_and_files(await analyze(application=application)) + found = self._state_types_and_files(await self._analyze(application)) self.assertEqual( found, - [( - 'Shop', - str(self.directory / 'package' / 'shop_servicer.py'), - )], + [ + ( + 'shop.v1.Shop', + str(self.directory / 'package' / 'shop_servicer.py'), + ) + ], ) ################################################################### @@ -718,13 +994,13 @@ async def test_a_file_that_will_not_parse(self) -> None: self._write('shop_servicer.py', source='class ShopServicer(') application = self._write('main.py', source=APPLICATION) - found = _state_types_and_files(await analyze(application=application)) + found = self._state_types_and_files(await self._analyze(application)) self.assertEqual(found, []) async def test_an_application_that_is_not_there(self) -> None: - found = _state_types_and_files( - await analyze(application=str(self.directory / 'nowhere.py')) + found = self._state_types_and_files( + await self._analyze(str(self.directory / 'nowhere.py')) ) self.assertEqual(found, []) @@ -746,12 +1022,12 @@ async def main(): ''' ) - found = _state_types_and_files(await analyze(application=application)) + found = self._state_types_and_files(await self._analyze(application)) self.assertEqual( found, [ - ('Shop', str(self.directory / 'other_servicer.py')), - ('Shop', str(self.directory / 'shop_servicer.py')), + ('shop.v1.Shop', str(self.directory / 'other_servicer.py')), + ('shop.v1.Shop', str(self.directory / 'shop_servicer.py')), ] ) @@ -766,7 +1042,7 @@ class ShopServicer(SomethingElse): ) application = self._write('main.py', source=APPLICATION) - found = _state_types_and_files(await analyze(application=application)) + found = self._state_types_and_files(await self._analyze(application)) self.assertEqual(found, []) From 7ca886d7e9203d97ae979c2a3086007c4fb7f35f Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Thu, 20 Aug 2026 03:12:16 +0000 Subject: [PATCH 17/19] Record the calls each method makes Showing how the developer's state types relate is what the dashboard wants the implementation for: which methods call which. As a method is analyzed, pyright is asked where the definition of everything its body calls is. When it is in a file `rbt generate` wrote, the call is a call on a state type: the state type is the `__state_type_name__` of the class the definition is in, and the method is the function the definition names. Asking about the definition, rather than following references through the source, makes the answer independent of how the call is spelled: on a reference in a variable, returned by an unannotated helper, taken out of a container, or renamed by an import, pyright resolves them all to the same definition. Calls are met however deep they are written, as a statement of their own, as an argument to another call such as `asyncio.gather`, or inside a comprehension or a nested function, and are recorded in the order written. What the generator defines on a state type to modify a call, `ref`, `schedule`, `idempotently` and the rest, is a closed set and is not recorded as calls, and neither is the generator's own underscored plumbing. A call whose receiver pyright cannot type is dropped rather than guessed at, until a later change records it as ambiguous. --- rbt/dashboard/v1/dashboard.proto | 16 ++ reboot/dashboard/implementation_watcher.py | 135 +++++++++++++++-- .../dashboard/implementation_watcher_tests.py | 138 +++++++++++++++++- 3 files changed, 275 insertions(+), 14 deletions(-) diff --git a/rbt/dashboard/v1/dashboard.proto b/rbt/dashboard/v1/dashboard.proto index 5936b340..313cd567 100644 --- a/rbt/dashboard/v1/dashboard.proto +++ b/rbt/dashboard/v1/dashboard.proto @@ -94,6 +94,22 @@ message ServicerInfo { // whether the method does anything different, not whether it was // written down differently. bytes digest = 2; + + // One call the method's body makes: a method the generated code + // of a state type defines, resolved by pyright from where the + // call is written. + message Call { + // The state type called, spelled as `StateTypeInfo.name`. + string state_type = 1; + + // The method called, spelled the way the developer would write + // it, e.g. "deposit". + string method = 2; + } + + // Represents every call the body makes, in the order they are + // written. + repeated Call calls = 3; } // The state type it services, spelled as `StateTypeInfo.name`. diff --git a/reboot/dashboard/implementation_watcher.py b/reboot/dashboard/implementation_watcher.py index 021604dd..135f7d2b 100644 --- a/reboot/dashboard/implementation_watcher.py +++ b/reboot/dashboard/implementation_watcher.py @@ -56,7 +56,7 @@ class extends: a servicer is a class with a base whose definition from reboot.cli.common.watch import file_watcher from reboot.dashboard.pyright import Definition, Pyright from types import MappingProxyType -from typing import Mapping, Optional, Sequence +from typing import Iterator, Mapping, Optional, Sequence # A SHA-256 digest -- of a file's bytes, or of a method's syntax -- # saying whether what was digested has changed. @@ -695,6 +695,14 @@ def with_dependency( ) +def _calls_in(node: ast.AST) -> Iterator[ast.Call]: + """Returns every call under a node, in the order written.""" + for child in ast.iter_child_nodes(node): + if isinstance(child, ast.Call): + yield child + yield from _calls_in(child) + + def _position_inside(expression: ast.expr) -> Optional[tuple[int, int]]: """Returns a line counting from one and a column inside the last part of a name or attribute, and `None` for anything else.""" @@ -709,6 +717,27 @@ def _position_inside(expression: ast.expr) -> Optional[tuple[int, int]]: return None +def _call_name_positions(method: ast.AST) -> tuple[tuple[int, int], ...]: + """Returns where the name of everything a method's body calls is + written -- a line counting from one and a column inside the + name's last part -- in the order the calls are written. Calls are + met however deep they are written: as a statement of their own, + as an argument to another call, or inside a comprehension or a + nested function. + + The last part is the one whose definition says what is being + called: the `restock` of `shop.restock(...)`. + """ + positions: list[tuple[int, int]] = [] + + for call in _calls_in(method): + position = _position_inside(call.func) + if position is not None: + positions.append(position) + + return tuple(positions) + + def _digest(node: ast.AST) -> Digest: """Returns a digest of what a piece of syntax says. @@ -1016,6 +1045,94 @@ async def _state_type_named_at( return None, files +# What generated code defines on a state type to modify how a call is +# made, rather than to be called as a method of the type. A closed +# set: the generator is what writes them. +MODIFIERS = frozenset( + ( + 'ref', + 'forall', + 'idempotently', + 'unidempotently', + 'per_workflow', + 'per_iteration', + 'always', + 'reactively', + 'until', + 'schedule', + 'spawn', + ) +) + + +async def _call_at( + definition: Optional[Definition], + *, + files: Files, + generated_directory: str, +) -> tuple[Optional['ServicerInfo.Method.Call'], Files]: + """Returns the call whose definition this is: a method the + generated code of a state type defines, however the call reaches + it -- on a reference, through modifiers, or on the state type + itself. `None` when the definition is not one, such as a modifier + like `.schedule(...)`, plumbing the generator writes for itself, + or anything outside generated code. + """ + filename = _generated_rbt_filename( + definition, + generated_directory=generated_directory, + ) + if definition is None or filename is None: + return None, files + + module, files = await files.syntax_of(filename) + if module is None: + return None, files + + match _enclosing_definitions(module, definition.line): + case [ + ast.ClassDef() as found, + *_, + ( + ast.FunctionDef(name=str(method)) | + ast.AsyncFunctionDef(name=str(method)) + ), + ] if (method not in MODIFIERS and not method.startswith('_')): + state_type = _state_type_name(found) + if state_type is None: + return None, files + return ServicerInfo.Method.Call( + state_type=state_type, + method=method, + ), files + + return None, files + + +async def _analyze_method( + method: ast.FunctionDef | ast.AsyncFunctionDef, + *, + filename: str, + files: Files, + generated_directory: str, +) -> tuple['ServicerInfo.Method', Files]: + """Returns one method as the recording wants it: its name, a + digest of its syntax, and every call its body makes that pyright + can place on a state type.""" + recorded = ServicerInfo.Method(name=method.name, digest=_digest(method)) + + for line, character in _call_name_positions(method): + call, files = await _call_at( + await files.pyright.definition_at(filename, line, character), + files=files, + generated_directory=generated_directory, + ) + if call is not None: + recorded.calls.append(call) + + return recorded, files + + async def _servicer_of_class( class_definition: ast.ClassDef, *, @@ -1059,16 +1176,14 @@ async def _servicer_of_class( for statement in class_definition.body: match statement: - case ( - ast.FunctionDef(name=str(name)) | - ast.AsyncFunctionDef(name=str(name)) - ): - servicer.methods.append( - ServicerInfo.Method( - name=name, - digest=_digest(statement), - ) + case ast.FunctionDef() | ast.AsyncFunctionDef(): + method, files = await _analyze_method( + statement, + filename=filename, + files=files, + generated_directory=generated_directory, ) + servicer.methods.append(method) return servicer, files diff --git a/tests/reboot/dashboard/implementation_watcher_tests.py b/tests/reboot/dashboard/implementation_watcher_tests.py index 80ed0a56..3aa3414f 100644 --- a/tests/reboot/dashboard/implementation_watcher_tests.py +++ b/tests/reboot/dashboard/implementation_watcher_tests.py @@ -83,11 +83,14 @@ async def stock(self, context, request): DEPOT = SINGLETON.format(state='Depot', module='depot') # The shape of a generated module, as far as resolving a state type -# needs: the state type's class and its servicer bases carrying the -# state type's name as `__state_type_name__`, and the `Servicer` -# aliases a servicer's base leads through. +# and the calls made on it needs: the state type's class and its +# servicer bases carrying the state type's name as +# `__state_type_name__`, the `Servicer` aliases, a reference with the +# state type's methods, and `ref`. GENERATED = ''' -from typing import TypeAlias +from typing import Generic, TypeAlias, TypeVar + +ScheduleTypeVar = TypeVar('ScheduleTypeVar') def StateTypeName(name): @@ -119,6 +122,22 @@ class {state}: Servicer: TypeAlias = {state}Servicer singleton: TypeAlias = {state}Singleton + + class WeakReference(Generic[ScheduleTypeVar]): + + async def look(self, context, request=None): + raise NotImplementedError + + async def stock(self, context, request=None): + raise NotImplementedError + + @classmethod + def ref(cls, id=None) -> '{state}.WeakReference[None]': + raise NotImplementedError + + @classmethod + async def open(cls, context, id=None): + raise NotImplementedError ''' @@ -852,6 +871,117 @@ async def test_a_method_whose_body_changes_digests_differently( before[0].methods[0].digest, ) + ################################################################### + # The calls each method makes. + + async def _method_of(self, source: str) -> ServicerInfo.Method: + """Returns what was recorded for the one method of a servicer + written with `source` as its body, one line per statement.""" + body = '\n'.join( + ' ' + line for line in source.strip().splitlines() + ) + self._write( + 'shop_servicer.py', source=''' +import asyncio + +from shop.v1.depot_rbt import Depot +from shop.v1.shop_rbt import Shop + + +def helper(): + return Shop.ref('helped') + + +class ShopServicer(Shop.Servicer): + + async def look(self, context, request): +''' + body + '\n' + ) + application = self._write('main.py', source=APPLICATION) + + found = servicers(await self._analyze(application)) + + assert len(found) == 1 + return found[0].methods[0] + + async def _calls(self, source: str) -> list[tuple[str, str]]: + """Returns the calls recorded for the one method of a servicer + written with `source` as its body.""" + return [ + (call.state_type, call.method) + for call in (await self._method_of(source)).calls + ] + + async def test_a_call_on_a_reference(self) -> None: + calls = await self._calls("await Shop.ref('other').stock(context)") + + self.assertEqual(calls, [('shop.v1.Shop', 'stock')]) + + async def test_a_call_through_a_helper(self) -> None: + """The reference comes out of an unannotated helper, which is + what takes pyright rather than reading the spelling.""" + calls = await self._calls( + 'shop = helper()\n' + 'await shop.stock(context)' + ) + + self.assertEqual(calls, [('shop.v1.Shop', 'stock')]) + + async def test_a_call_on_a_reference_out_of_a_container(self) -> None: + calls = await self._calls( + "for shop in [Shop.ref('a'), Shop.ref('b')]:\n" + ' await shop.stock(context)' + ) + + self.assertEqual(calls, [('shop.v1.Shop', 'stock')]) + + async def test_a_constructor_is_a_call(self) -> None: + calls = await self._calls("await Shop.open(context, 'other')") + + self.assertEqual(calls, [('shop.v1.Shop', 'open')]) + + async def test_a_call_on_another_state_type(self) -> None: + calls = await self._calls("await Depot.ref('d').stock(context)") + + self.assertEqual(calls, [('shop.v1.Depot', 'stock')]) + + async def test_a_modifier_is_not_a_call(self) -> None: + calls = await self._calls("shop = Shop.ref('other')") + + self.assertEqual(calls, []) + + async def test_somebody_elses_calls_are_not_recorded(self) -> None: + calls = await self._calls("print(len('x'))") + + self.assertEqual(calls, []) + + async def test_a_call_on_something_unresolvable(self) -> None: + """A receiver pyright cannot type makes a call it cannot + place, which is dropped rather than guessed at.""" + calls = await self._calls( + 'shop = mystery()\n' + 'await shop.stock(context)' + ) + + self.assertEqual(calls, []) + + async def test_the_calls_come_in_the_order_written(self) -> None: + calls = await self._calls( + "await Shop.ref('b').stock(context)\n" + 'await asyncio.gather(\n' + " Shop.ref('a').look(context),\n" + " Depot.ref('d').stock(context),\n" + ')' + ) + + self.assertEqual( + calls, [ + ('shop.v1.Shop', 'stock'), + ('shop.v1.Shop', 'look'), + ('shop.v1.Depot', 'stock'), + ] + ) + ################################################################### # Parse again only what has changed. From f7dc8965661b1520137395af99411ad99a200a91 Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Thu, 20 Aug 2026 03:14:25 +0000 Subject: [PATCH 18/19] Say how each call is reached Where in the state type's class the generator defined the called method already says how the call was made: a method on the scheduled variant was scheduled, on the spawned variant spawned, on the reactive and until variants likewise; `read` and `write` are a workflow's, and name no method; and a method on the state type's class itself is a constructor, reached without a reference because it is what makes the state to refer to. Anything else, on the reference itself, on its idempotent variant, or on `forall`, is a plain call. One shape defeats pyright: the generator types `schedule` by a type variable, which pyright 1.1.413 does not solve, so a call on what `.schedule(...)` returned resolves to nothing at all. The `schedule` itself is typed ordinarily though, and its definition says which state type is being scheduled; the method's name is written right there in the call, and is checked against what the state type defines. So when a call resolves to nothing and is made on a modifier, the modifier is asked instead. --- rbt/dashboard/v1/dashboard.proto | 28 ++- reboot/dashboard/implementation_watcher.py | 209 ++++++++++++++++-- .../dashboard/implementation_watcher_tests.py | 186 ++++++++++++++-- 3 files changed, 391 insertions(+), 32 deletions(-) diff --git a/rbt/dashboard/v1/dashboard.proto b/rbt/dashboard/v1/dashboard.proto index 313cd567..43bd6cf8 100644 --- a/rbt/dashboard/v1/dashboard.proto +++ b/rbt/dashboard/v1/dashboard.proto @@ -99,12 +99,38 @@ message ServicerInfo { // of a state type defines, resolved by pyright from where the // call is written. message Call { + // How the call is reached. + enum How { + UNKNOWN = 0; + // A plain method call on a reference. + CALL = 1; + // A constructor, which both makes the state and returns a + // reference to it. + CONSTRUCT = 2; + // Reached through `.schedule(when=...)`. + SCHEDULE = 3; + // Reached through `.spawn()`. + SPAWN = 4; + // Reached through `.reactively()`. + REACTIVELY = 5; + // Reached through `.until(...)`. + UNTIL = 6; + // A workflow's `.read(context)`, which names no method. + READ = 7; + // A workflow's `.write(context, ...)`, which names no + // method. + WRITE = 8; + } + // The state type called, spelled as `StateTypeInfo.name`. string state_type = 1; // The method called, spelled the way the developer would write - // it, e.g. "deposit". + // it, e.g. "deposit". Empty for `READ` and `WRITE`, which name + // no method. string method = 2; + + How how = 3; } // Represents every call the body makes, in the order they are diff --git a/reboot/dashboard/implementation_watcher.py b/reboot/dashboard/implementation_watcher.py index 135f7d2b..34d3e9b2 100644 --- a/reboot/dashboard/implementation_watcher.py +++ b/reboot/dashboard/implementation_watcher.py @@ -223,6 +223,28 @@ def _join(base: str, *parts: str) -> str: return '.'.join([base, *parts]) +@dataclass(frozen=True, kw_only=True) +class WrittenCall: + """One call as a method's body writes it.""" + + # The line the called name's last part is written on, counting + # from one, the way `ast` counts. The last part is the one whose + # definition says what is being called: the `restock` of + # `shop.restock(...)`. + line: int + + # A column inside the last part, counting from zero. + character: int + + # The last part itself. + name: str + + # A position inside the modifier written just in front, when the + # call is made on one: the `schedule` of + # `shop.schedule().restock(...)`. + modifier: Optional[tuple[int, int]] + + @dataclass(frozen=True, kw_only=True) class ParsedFile: """What parsing one file said, before any name in it is @@ -717,25 +739,44 @@ def _position_inside(expression: ast.expr) -> Optional[tuple[int, int]]: return None -def _call_name_positions(method: ast.AST) -> tuple[tuple[int, int], ...]: - """Returns where the name of everything a method's body calls is - written -- a line counting from one and a column inside the - name's last part -- in the order the calls are written. Calls are - met however deep they are written: as a statement of their own, - as an argument to another call, or inside a comprehension or a - nested function. - - The last part is the one whose definition says what is being - called: the `restock` of `shop.restock(...)`. +def _written_calls(method: ast.AST) -> tuple[WrittenCall, ...]: + """Returns every call a method's body writes, in the order + written. Calls are met however deep they are written: as a + statement of their own, as an argument to another call, or inside + a comprehension or a nested function. """ - positions: list[tuple[int, int]] = [] + written: list[WrittenCall] = [] for call in _calls_in(method): position = _position_inside(call.func) - if position is not None: - positions.append(position) + if position is None: + continue + + match call.func: + case ast.Name(id=str(name)) | ast.Attribute(attr=str(name)): + pass + case _: + continue + + # The modifier just in front, when the call is made on one: + # `shop.schedule().restock(...)` calls `restock` on what + # `schedule` returned. + modifier: Optional[tuple[int, int]] = None + match call.func: + case ast.Attribute(value=ast.Call(func=modified)): + modifier = _position_inside(modified) + + line, character = position + written.append( + WrittenCall( + line=line, + character=character, + name=name, + modifier=modifier, + ) + ) - return tuple(positions) + return tuple(written) def _digest(node: ast.AST) -> Digest: @@ -1065,6 +1106,74 @@ async def _state_type_named_at( ) +def _defines_method(node: ast.ClassDef, name: str) -> bool: + """Returns whether a state type's class defines a method by a + name, anywhere under it: on its `WeakReference`, on a scheduled + or spawned variant, or on the class itself.""" + for child in ast.walk(node): + match child: + case ( + ast.FunctionDef(name=str(defined)) | + ast.AsyncFunctionDef(name=str(defined)) + ) if defined == name: + return True + + return False + + +# How a call is reached, by the class the generator defined the +# called method in. A class not named here, such as `WeakReference` +# itself, `_Idempotently` or `_Forall`, modifies nothing about how. +HOW_BY_CLASS: Mapping[str, 'ServicerInfo.Method.Call.How.ValueType'] = ( + MappingProxyType( + { + '_Schedule': ServicerInfo.Method.Call.How.SCHEDULE, + '_WriterSchedule': ServicerInfo.Method.Call.How.SCHEDULE, + '_Spawn': ServicerInfo.Method.Call.How.SPAWN, + '_Reactively': ServicerInfo.Method.Call.How.REACTIVELY, + '_Until': ServicerInfo.Method.Call.How.UNTIL, + '_UntilChangesSatisfies': ServicerInfo.Method.Call.How.UNTIL, + } + ) +) + + +def _call( + state_type: str, + enclosing: list[ast.ClassDef | ast.FunctionDef | ast.AsyncFunctionDef], + method: str, +) -> 'ServicerInfo.Method.Call': + """Returns one call as the recording wants it, worked out from + where in a state type's class the called method is defined: a + method on the class itself is a constructor, `read` and `write` + name no method, and a scheduled, spawned, reactive or until + variant of the class says so.""" + how = ServicerInfo.Method.Call.How.CALL + for node in enclosing[1:-1]: + match node: + case ast.ClassDef(name=str(name)) if name in HOW_BY_CLASS: + how = HOW_BY_CLASS[name] + + if len(enclosing) == 2: + # Defined on the state type's class itself: a constructor, + # which is reached without a reference because it is what + # makes the state to refer to. + how = ServicerInfo.Method.Call.How.CONSTRUCT + + if method in ('read', 'write'): + return ServicerInfo.Method.Call( + state_type=state_type, + how=ServicerInfo.Method.Call.How.READ + if method == 'read' else ServicerInfo.Method.Call.How.WRITE, + ) + + return ServicerInfo.Method.Call( + state_type=state_type, + method=method, + how=how, + ) + + async def _call_at( definition: Optional[Definition], *, @@ -1098,12 +1207,64 @@ async def _call_at( ast.AsyncFunctionDef(name=str(method)) ), ] if (method not in MODIFIERS and not method.startswith('_')): + state_type = _state_type_name(found) + if state_type is None: + return None, files + enclosing = _enclosing_definitions(module, definition.line) + return _call(state_type, enclosing, method), files + + return None, files + + +async def _scheduled_call_at( + definition: Optional[Definition], + *, + method: str, + files: Files, + generated_directory: str, +) -> tuple[Optional['ServicerInfo.Method.Call'], Files]: + """Returns the call made through a `.schedule(...)` whose + definition this is, and `None` when the definition is not a state + type's `schedule` or the state type defines no such method. + + Asked when the call itself resolves to nothing, which is what a + call on what `.schedule(...)` returned looks like: the generator + types `schedule` by a type variable that pyright does not solve, + so the scheduled variant's methods cannot be reached from it. The + `schedule` itself is typed ordinarily, and says which state type + is being scheduled; the method's name, written right there, says + the rest, checked against what the state type defines. + """ + filename = _generated_rbt_filename( + definition, + generated_directory=generated_directory, + ) + if definition is None or filename is None: + return None, files + + module, files = await files.syntax_of(filename) + if module is None: + return None, files + + match _enclosing_definitions(module, definition.line): + case [ + ast.ClassDef() as found, + *_, + ( + ast.FunctionDef(name='schedule') | + ast.AsyncFunctionDef(name='schedule') + ), + ] if ( + _defines_method(found, method) and method not in MODIFIERS and + method not in ('read', 'write') and not method.startswith('_') + ): state_type = _state_type_name(found) if state_type is None: return None, files return ServicerInfo.Method.Call( state_type=state_type, method=method, + how=ServicerInfo.Method.Call.How.SCHEDULE, ), files return None, files @@ -1121,12 +1282,28 @@ async def _analyze_method( can place on a state type.""" recorded = ServicerInfo.Method(name=method.name, digest=_digest(method)) - for line, character in _call_name_positions(method): + for written in _written_calls(method): call, files = await _call_at( - await files.pyright.definition_at(filename, line, character), + await files.pyright.definition_at( + filename, + written.line, + written.character, + ), files=files, generated_directory=generated_directory, ) + if call is None and written.modifier is not None: + modifier_line, modifier_character = written.modifier + call, files = await _scheduled_call_at( + await files.pyright.definition_at( + filename, + modifier_line, + modifier_character, + ), + method=written.name, + files=files, + generated_directory=generated_directory, + ) if call is not None: recorded.calls.append(call) diff --git a/tests/reboot/dashboard/implementation_watcher_tests.py b/tests/reboot/dashboard/implementation_watcher_tests.py index 3aa3414f..f4a2536d 100644 --- a/tests/reboot/dashboard/implementation_watcher_tests.py +++ b/tests/reboot/dashboard/implementation_watcher_tests.py @@ -24,6 +24,9 @@ from typing import Optional from unittest.mock import patch +# How a call is reached, as the recording spells it. +How = ServicerInfo.Method.Call.How + API_FILE = ''' from reboot.api import API, Field, Methods, Model, Reader, Type @@ -83,10 +86,13 @@ async def stock(self, context, request): DEPOT = SINGLETON.format(state='Depot', module='depot') # The shape of a generated module, as far as resolving a state type -# and the calls made on it needs: the state type's class and its -# servicer bases carrying the state type's name as -# `__state_type_name__`, the `Servicer` aliases, a reference with the -# state type's methods, and `ref`. +# and the calls made on it needs: the state type's class carrying its +# name as `__state_type_name__`, its +# `Servicer` with `ref`, a reference with the state type's methods +# and the generator's modifiers, and the scheduled, spawned, reactive +# and until variants the modifiers lead to. `schedule` is typed by a +# type variable, the way the generator types it, which pyright does +# not solve; the analysis works around exactly that. GENERATED = ''' from typing import Generic, TypeAlias, TypeVar @@ -101,6 +107,11 @@ class {state}BaseServicer: __state_type_name__ = StateTypeName('shop.v1.{state}') + def ref( + self, + ) -> '{state}.WeakReference[{state}.WeakReference._WriterSchedule]': + raise NotImplementedError + class {state}Servicer({state}BaseServicer): pass @@ -125,14 +136,84 @@ class {state}: class WeakReference(Generic[ScheduleTypeVar]): + class _Schedule: + + async def look(self, context, request=None): + raise NotImplementedError + + async def stock(self, context, request=None): + raise NotImplementedError + + class _WriterSchedule: + + async def look(self, context, request=None): + raise NotImplementedError + + async def stock(self, context, request=None): + raise NotImplementedError + + class _Spawn: + + async def look(self, context, request=None): + raise NotImplementedError + + async def stock(self, context, request=None): + raise NotImplementedError + + class _Reactively: + + def look(self, context, request=None): + raise NotImplementedError + + class _Until: + + async def look(self, context, request=None): + raise NotImplementedError + + class _Idempotently(Generic[ScheduleTypeVar]): + + def schedule(self, *, when=None) -> ScheduleTypeVar: + raise NotImplementedError + + async def stock(self, context, request=None): + raise NotImplementedError + + async def write(self, context, writer): + raise NotImplementedError + async def look(self, context, request=None): raise NotImplementedError async def stock(self, context, request=None): raise NotImplementedError + async def read(self, context): + raise NotImplementedError + + async def write(self, context, writer): + raise NotImplementedError + + def schedule(self, *, when=None) -> ScheduleTypeVar: + raise NotImplementedError + + def spawn(self) -> '{state}.WeakReference._Spawn': + raise NotImplementedError + + def reactively(self) -> '{state}.WeakReference._Reactively': + raise NotImplementedError + + def until(self, alias) -> '{state}.WeakReference._Until': + raise NotImplementedError + + def idempotently( + self, alias=None + ) -> '{state}.WeakReference._Idempotently[ScheduleTypeVar]': + raise NotImplementedError + @classmethod - def ref(cls, id=None) -> '{state}.WeakReference[None]': + def ref( + cls, id=None + ) -> '{state}.WeakReference[{state}.WeakReference._Schedule]': raise NotImplementedError @classmethod @@ -904,18 +985,18 @@ async def look(self, context, request): assert len(found) == 1 return found[0].methods[0] - async def _calls(self, source: str) -> list[tuple[str, str]]: + async def _calls(self, source: str) -> list[tuple[str, str, int]]: """Returns the calls recorded for the one method of a servicer written with `source` as its body.""" return [ - (call.state_type, call.method) + (call.state_type, call.method, call.how) for call in (await self._method_of(source)).calls ] async def test_a_call_on_a_reference(self) -> None: calls = await self._calls("await Shop.ref('other').stock(context)") - self.assertEqual(calls, [('shop.v1.Shop', 'stock')]) + self.assertEqual(calls, [('shop.v1.Shop', 'stock', How.CALL)]) async def test_a_call_through_a_helper(self) -> None: """The reference comes out of an unannotated helper, which is @@ -925,7 +1006,7 @@ async def test_a_call_through_a_helper(self) -> None: 'await shop.stock(context)' ) - self.assertEqual(calls, [('shop.v1.Shop', 'stock')]) + self.assertEqual(calls, [('shop.v1.Shop', 'stock', How.CALL)]) async def test_a_call_on_a_reference_out_of_a_container(self) -> None: calls = await self._calls( @@ -933,17 +1014,17 @@ async def test_a_call_on_a_reference_out_of_a_container(self) -> None: ' await shop.stock(context)' ) - self.assertEqual(calls, [('shop.v1.Shop', 'stock')]) + self.assertEqual(calls, [('shop.v1.Shop', 'stock', How.CALL)]) async def test_a_constructor_is_a_call(self) -> None: calls = await self._calls("await Shop.open(context, 'other')") - self.assertEqual(calls, [('shop.v1.Shop', 'open')]) + self.assertEqual(calls, [('shop.v1.Shop', 'open', How.CONSTRUCT)]) async def test_a_call_on_another_state_type(self) -> None: calls = await self._calls("await Depot.ref('d').stock(context)") - self.assertEqual(calls, [('shop.v1.Depot', 'stock')]) + self.assertEqual(calls, [('shop.v1.Depot', 'stock', How.CALL)]) async def test_a_modifier_is_not_a_call(self) -> None: calls = await self._calls("shop = Shop.ref('other')") @@ -976,12 +1057,87 @@ async def test_the_calls_come_in_the_order_written(self) -> None: self.assertEqual( calls, [ - ('shop.v1.Shop', 'stock'), - ('shop.v1.Shop', 'look'), - ('shop.v1.Depot', 'stock'), + ('shop.v1.Shop', 'stock', How.CALL), + ('shop.v1.Shop', 'look', How.CALL), + ('shop.v1.Depot', 'stock', How.CALL), ] ) + ################################################################### + # How each call is reached. + + async def test_a_scheduled_call(self) -> None: + calls = await self._calls( + "await Shop.ref('other').schedule(when=None).stock(context)" + ) + + self.assertEqual(calls, [('shop.v1.Shop', 'stock', How.SCHEDULE)]) + + async def test_a_call_scheduled_on_ourselves(self) -> None: + """`self.ref().schedule()` is how a method schedules its own + state type, such as an interest method scheduling itself.""" + calls = await self._calls('await self.ref().schedule().stock(context)') + + self.assertEqual(calls, [('shop.v1.Shop', 'stock', How.SCHEDULE)]) + + async def test_a_schedule_of_no_such_method(self) -> None: + """The state type defines no such method, so there is nothing + to record.""" + calls = await self._calls( + "await Shop.ref('x').schedule().missing(context)" + ) + + self.assertEqual(calls, []) + + async def test_a_spawned_call(self) -> None: + calls = await self._calls( + "await Shop.ref('other').spawn().stock(context)" + ) + + self.assertEqual(calls, [('shop.v1.Shop', 'stock', How.SPAWN)]) + + async def test_a_reactive_call(self) -> None: + calls = await self._calls( + "Shop.ref('other').reactively().look(context)" + ) + + self.assertEqual(calls, [('shop.v1.Shop', 'look', How.REACTIVELY)]) + + async def test_an_until_call(self) -> None: + calls = await self._calls( + "await Shop.ref('other').until('found').look(context)" + ) + + self.assertEqual(calls, [('shop.v1.Shop', 'look', How.UNTIL)]) + + async def test_an_idempotent_call_is_still_a_call(self) -> None: + calls = await self._calls( + "await Shop.ref('other').idempotently('once').stock(context)" + ) + + self.assertEqual(calls, [('shop.v1.Shop', 'stock', How.CALL)]) + + async def test_a_read_names_no_method(self) -> None: + calls = await self._calls("await Shop.ref('w').read(context)") + + self.assertEqual(calls, [('shop.v1.Shop', '', How.READ)]) + + async def test_a_write_names_no_method(self) -> None: + calls = await self._calls( + "await Shop.ref('w').write(context, lambda state: None)" + ) + + self.assertEqual(calls, [('shop.v1.Shop', '', How.WRITE)]) + + async def test_an_idempotent_write(self) -> None: + calls = await self._calls( + "await Shop.ref('w').idempotently('once').write(\n" + ' context, lambda state: None\n' + ')' + ) + + self.assertEqual(calls, [('shop.v1.Shop', '', How.WRITE)]) + ################################################################### # Parse again only what has changed. From b731827e7883e52b5d7bbce12ce409785bb4f255 Mon Sep 17 00:00:00 2001 From: Benjamin Hindman Date: Thu, 20 Aug 2026 03:16:47 +0000 Subject: [PATCH 19/19] Record what hands the context somewhere unseen, as ambiguous A method's body can hand its context to something the analysis cannot see into: a helper function, a method of its own servicer, or a call on a receiver pyright cannot type. Only whoever has the context can reach other state types, so these are exactly the places calls may hide. Each is recorded as ambiguous, with the called name as the developer wrote it, so that what the analysis does not know is visible rather than merely absent. The context parameter is the one the generator puts first after `self`, and handing it over is handing that name as an argument or a keyword argument. A call that already resolved to a state type's method is not ambiguous, however many contexts it hands over: what it reaches is known. --- rbt/dashboard/v1/dashboard.proto | 9 +++ reboot/dashboard/implementation_watcher.py | 64 +++++++++++++++++-- .../dashboard/implementation_watcher_tests.py | 58 +++++++++++++++++ 3 files changed, 127 insertions(+), 4 deletions(-) diff --git a/rbt/dashboard/v1/dashboard.proto b/rbt/dashboard/v1/dashboard.proto index 43bd6cf8..a11804b1 100644 --- a/rbt/dashboard/v1/dashboard.proto +++ b/rbt/dashboard/v1/dashboard.proto @@ -136,6 +136,15 @@ message ServicerInfo { // Represents every call the body makes, in the order they are // written. repeated Call calls = 3; + + // Represents every call that hands the method's context to + // something the analysis cannot see into, such as a helper + // function, in the order they are written. Only whoever has the + // context can reach other state types, so these are where calls + // may hide; `method` is the called name as the developer wrote + // it, the state type is unknown and left empty, and `how` is + // `UNKNOWN`. + repeated Call ambiguous = 4; } // The state type it services, spelled as `StateTypeInfo.name`. diff --git a/reboot/dashboard/implementation_watcher.py b/reboot/dashboard/implementation_watcher.py index 34d3e9b2..6018304c 100644 --- a/reboot/dashboard/implementation_watcher.py +++ b/reboot/dashboard/implementation_watcher.py @@ -244,6 +244,11 @@ class WrittenCall: # `shop.schedule().restock(...)`. modifier: Optional[tuple[int, int]] + # Whether the call hands the method's context over, as an + # argument or a keyword argument. Only whoever has the context + # can reach other state types. + hands_context: bool + @dataclass(frozen=True, kw_only=True) class ParsedFile: @@ -739,11 +744,49 @@ def _position_inside(expression: ast.expr) -> Optional[tuple[int, int]]: return None -def _written_calls(method: ast.AST) -> tuple[WrittenCall, ...]: +def _context_parameter( + method: ast.FunctionDef | ast.AsyncFunctionDef, +) -> Optional[str]: + """Returns the name of a method's context parameter, which the + generator puts first after `self`, and `None` when the method has + too few parameters to have one.""" + parameters = method.args.args + if len(parameters) >= 2: + return parameters[1].arg + return None + + +def _hands_over(call: ast.Call, name: Optional[str]) -> bool: + """Returns whether a call hands a name over, as an argument or a + keyword argument.""" + if name is None: + return False + + for argument in call.args: + match argument: + case ast.Name(id=str(id)) if id == name: + return True + + for keyword in call.keywords: + match keyword.value: + case ast.Name(id=str(id)) if id == name: + return True + + return False + + +def _written_calls( + method: ast.AST, + *, + context: Optional[str], +) -> tuple[WrittenCall, ...]: """Returns every call a method's body writes, in the order written. Calls are met however deep they are written: as a statement of their own, as an argument to another call, or inside a comprehension or a nested function. + + `context` is the name of the method's context parameter, whose + handing over is worth remembering about each call. """ written: list[WrittenCall] = [] @@ -773,6 +816,7 @@ def _written_calls(method: ast.AST) -> tuple[WrittenCall, ...]: character=character, name=name, modifier=modifier, + hands_context=_hands_over(call, context), ) ) @@ -1278,11 +1322,15 @@ async def _analyze_method( generated_directory: str, ) -> tuple['ServicerInfo.Method', Files]: """Returns one method as the recording wants it: its name, a - digest of its syntax, and every call its body makes that pyright - can place on a state type.""" + digest of its syntax, every call its body makes that pyright can + place on a state type, and, as ambiguous, every call that hands + the context somewhere the analysis cannot see into.""" recorded = ServicerInfo.Method(name=method.name, digest=_digest(method)) - for written in _written_calls(method): + for written in _written_calls( + method, + context=_context_parameter(method), + ): call, files = await _call_at( await files.pyright.definition_at( filename, @@ -1306,6 +1354,14 @@ async def _analyze_method( ) if call is not None: recorded.calls.append(call) + elif written.hands_context: + # Nothing says what this call reaches, and it was handed + # the context, which is what reaching other state types + # takes. Recorded so that what the analysis does not know + # is visible rather than merely absent. + recorded.ambiguous.append( + ServicerInfo.Method.Call(method=written.name) + ) return recorded, files diff --git a/tests/reboot/dashboard/implementation_watcher_tests.py b/tests/reboot/dashboard/implementation_watcher_tests.py index f4a2536d..30c63fdf 100644 --- a/tests/reboot/dashboard/implementation_watcher_tests.py +++ b/tests/reboot/dashboard/implementation_watcher_tests.py @@ -973,6 +973,10 @@ def helper(): return Shop.ref('helped') +def audit(context): + pass + + class ShopServicer(Shop.Servicer): async def look(self, context, request): @@ -993,6 +997,14 @@ async def _calls(self, source: str) -> list[tuple[str, str, int]]: for call in (await self._method_of(source)).calls ] + async def _ambiguous(self, source: str) -> list[str]: + """Returns what was recorded as ambiguous for the one method + of a servicer written with `source` as its body: the called + names, in the order written.""" + return [ + call.method for call in (await self._method_of(source)).ambiguous + ] + async def test_a_call_on_a_reference(self) -> None: calls = await self._calls("await Shop.ref('other').stock(context)") @@ -1138,6 +1150,52 @@ async def test_an_idempotent_write(self) -> None: self.assertEqual(calls, [('shop.v1.Shop', '', How.WRITE)]) + ################################################################### + # What hands the context somewhere the analysis cannot see. + + async def test_handing_the_context_to_a_helper_is_ambiguous(self) -> None: + """The helper may reach any state type with it, and its body + is not analyzed, so what it does is recorded as unknown + rather than silently missing.""" + ambiguous = await self._ambiguous('audit(context)') + + self.assertEqual(ambiguous, ['audit']) + + async def test_handing_the_context_as_a_keyword(self) -> None: + ambiguous = await self._ambiguous('audit(context=context)') + + self.assertEqual(ambiguous, ['audit']) + + async def test_handing_the_context_to_our_own_method(self) -> None: + ambiguous = await self._ambiguous( + 'await self.something(context, request)' + ) + + self.assertEqual(ambiguous, ['something']) + + async def test_a_call_handing_no_context_is_not_ambiguous(self) -> None: + ambiguous = await self._ambiguous("print(len('x'))") + + self.assertEqual(ambiguous, []) + + async def test_a_resolved_call_is_not_also_ambiguous(self) -> None: + ambiguous = await self._ambiguous( + "await Shop.ref('other').stock(context)" + ) + + self.assertEqual(ambiguous, []) + + async def test_an_unresolvable_receiver_is_ambiguous(self) -> None: + """The receiver types to nothing, and the call hands the + context, so it lands with the unknown rather than being + dropped.""" + ambiguous = await self._ambiguous( + 'shop = mystery()\n' + 'await shop.stock(context)' + ) + + self.assertEqual(ambiguous, ['stock']) + ################################################################### # Parse again only what has changed.