Skip to content

refactor: centralize grammar ownership #208

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
rlippmann wants to merge 1 commit into
base: main
Choose a base branch
from
Open

refactor: centralize grammar ownership #208

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions src/context_compiler/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
get_policy_items,
get_premise_value,
)
from .grammar import DirectiveKind, is_canonical_directive, render_directive, validate_directive

__version__ = version("context-compiler")

Expand All @@ -44,6 +45,7 @@
"DECISION_CLARIFY",
"DECISION_PASSTHROUGH",
"DECISION_UPDATE",
"DirectiveKind",
"Engine",
"POLICY_PROHIBIT",
"POLICY_USE",
Expand All @@ -61,11 +63,14 @@
"get_policy_items",
"get_step_decision",
"get_step_state",
"is_canonical_directive",
"is_clarify",
"is_passthrough",
"is_update",
"preview",
"preview_would_mutate",
"render_directive",
"state_diff",
"step",
"validate_directive",
]
75 changes: 3 additions & 72 deletions src/context_compiler/engine.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,7 @@
STATE_PREMISE,
STATE_VERSION,
)
from .grammar import DirectiveKind
from .grammar import _parse_directive as _parse_canonical_directive
from .grammar import DirectiveKind, _parse_canonical_directive

PolicyValue = Literal["use", "prohibit"]

Expand Down Expand Up @@ -75,26 +74,6 @@ class Action:
_COMPOUND_DIRECTIVE_PROMPT = (
"Multiple directives are not supported in one input.\nSubmit each directive separately."
)
_CLEAR_PREMISE = "clear premise"
_RESET_POLICIES = "reset policies"
_CLEAR_STATE = "clear state"
_REMOVE_POLICY_BASE = "remove policy"
_CHANGE_PREMISE_PREFIX = "change premise "
_CHANGE_PREMISE_BASE = "change premise to"
_SET_PREMISE_BASE = "set premise"
_USE_PREFIX = "use "
_PROHIBIT_BASE = "prohibit"
_PROHIBIT_PREFIX = "prohibit "
_CANONICAL_DIRECTIVE_STARTS: tuple[tuple[str, bool], ...] = (
(_CHANGE_PREMISE_BASE, True),
(_SET_PREMISE_BASE, True),
(_REMOVE_POLICY_BASE, True),
(_RESET_POLICIES, False),
(_CLEAR_PREMISE, False),
(_CLEAR_STATE, False),
(_PROHIBIT_BASE, True),
("use", True),
)


def create_engine(state: State | None = None) -> "Engine":
Expand Down Expand Up @@ -303,6 +282,8 @@ def _apply_replacement_explicit(self, new_item: str, old_item: str) -> None:


def _parse_directive(user_input: str) -> Action | None:
# Engine semantics intentionally depend on grammar's internal parsed form.
# This keeps syntax authority in grammar without making the parser a public API.
parsed = _parse_canonical_directive(user_input)
if parsed is None:
return None
Expand Down Expand Up @@ -330,56 +311,6 @@ def _parse_directive(user_input: str) -> Action | None:
return Action(kind="clear_state")


def _contains_compound_directive(user_input: str) -> bool:
first_start = _match_canonical_directive_start(user_input, 0)
if first_start is None:
return False

for index in range(first_start, len(user_input)):
next_start = _match_canonical_directive_start(user_input, index)
if next_start is not None:
return True

return False


def _match_canonical_directive_start(user_input: str, start: int) -> int | None:
if start < 0 or start >= len(user_input):
return None

if start > 0 and user_input[start - 1].isalpha():
return None

for token, require_space_or_end in _CANONICAL_DIRECTIVE_STARTS:
if _matches_directive_token(
user_input, start, token, require_space_or_end=require_space_or_end
):
return start + len(token)

return None


def _matches_directive_token(
user_input: str,
start: int,
token: str,
*,
require_space_or_end: bool = False,
) -> bool:
if not user_input.startswith(token, start):
return False

end = start + len(token)
if end == len(user_input):
return True

next_char = user_input[end]
if require_space_or_end:
return next_char == " "

return not next_char.isalpha()


def _initial_state() -> State:
return {
STATE_PREMISE: None,
Expand Down
36 changes: 24 additions & 12 deletions src/context_compiler/grammar.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ class ValidatedDirective:


@dataclass(frozen=True, slots=True)
class _ParsedDirective:
class _GrammarParsedDirective:
text: str
kind: DirectiveKind
operands: MappingProxyType[str, str]
Expand Down Expand Up @@ -243,7 +243,7 @@ def _contains_multiple_canonical_directives(text: str) -> bool:
return False


def _parse_replace_use(trimmed_text: str) -> _ParsedDirective | None:
def _parse_replace_use(trimmed_text: str) -> _GrammarParsedDirective | None:
match = _REPLACE_RE.fullmatch(trimmed_text)
if match is None:
return None
Expand All @@ -258,14 +258,14 @@ def _parse_replace_use(trimmed_text: str) -> _ParsedDirective | None:
normalized_payload = _normalized_for_matching(trimmed_text)
if normalized_payload.count(_INSTEAD_OF_DELIMITER) != 1:
return None
return _ParsedDirective(
return _GrammarParsedDirective(
text=trimmed_text,
kind=DirectiveKind.REPLACE_USE,
operands=MappingProxyType({"new_item": new_item, "old_item": old_item}),
)


def _parse_directive(text: str) -> _ParsedDirective | None:
def _parse_directive(text: str) -> _GrammarParsedDirective | None:
trimmed_text = _trim_ascii_whitespace(text)
if trimmed_text == "":
return None
Expand All @@ -275,17 +275,17 @@ def _parse_directive(text: str) -> _ParsedDirective | None:
normalized = _normalized_for_matching(trimmed_text)

if normalized == "clear premise":
return _ParsedDirective(
return _GrammarParsedDirective(
text=text, kind=DirectiveKind.CLEAR_PREMISE, operands=MappingProxyType({})
)
if normalized == "reset policies":
return _ParsedDirective(
return _GrammarParsedDirective(
text=text,
kind=DirectiveKind.RESET_POLICIES,
operands=MappingProxyType({}),
)
if normalized == "clear state":
return _ParsedDirective(
return _GrammarParsedDirective(
text=text, kind=DirectiveKind.CLEAR_STATE, operands=MappingProxyType({})
)

Expand All @@ -296,7 +296,7 @@ def _parse_directive(text: str) -> _ParsedDirective | None:
value = match.group("value")
if not _operand_has_content(value) or _operand_starts_with_token(value, "to"):
return None
return _ParsedDirective(
return _GrammarParsedDirective(
text=text,
kind=DirectiveKind.SET_PREMISE,
operands=MappingProxyType({"value": value}),
Expand All @@ -309,7 +309,7 @@ def _parse_directive(text: str) -> _ParsedDirective | None:
value = match.group("value")
if not _operand_has_content(value):
return None
return _ParsedDirective(
return _GrammarParsedDirective(
text=text,
kind=DirectiveKind.CHANGE_PREMISE,
operands=MappingProxyType({"value": value}),
Expand All @@ -332,7 +332,7 @@ def _parse_directive(text: str) -> _ParsedDirective | None:
or _INSTEAD_OF_DELIMITER in normalized_item
):
return None
return _ParsedDirective(
return _GrammarParsedDirective(
text=text,
kind=DirectiveKind.USE_ITEM,
operands=MappingProxyType({"item": item}),
Expand All @@ -345,7 +345,7 @@ def _parse_directive(text: str) -> _ParsedDirective | None:
item = match.group("item")
if not _operand_has_content(item):
return None
return _ParsedDirective(
return _GrammarParsedDirective(
text=text,
kind=DirectiveKind.PROHIBIT_ITEM,
operands=MappingProxyType({"item": item}),
Expand All @@ -358,7 +358,7 @@ def _parse_directive(text: str) -> _ParsedDirective | None:
item = match.group("item")
if not _operand_has_content(item):
return None
return _ParsedDirective(
return _GrammarParsedDirective(
text=text,
kind=DirectiveKind.REMOVE_POLICY,
operands=MappingProxyType({"item": item}),
Expand All @@ -374,6 +374,18 @@ def validate_directive(text: str) -> ValidatedDirective | None:
return ValidatedDirective(text=parsed.text, kind=parsed.kind)


def _parse_canonical_directive(text: str) -> _GrammarParsedDirective | None:
return _parse_directive(text)


def match_canonical_directive_start(text: str, start: int) -> int | None:
return _match_canonical_directive_start(text, start)


def contains_multiple_canonical_directives(text: str) -> bool:
return _contains_multiple_canonical_directives(text)


def is_canonical_directive(text: str) -> bool:
return validate_directive(text) is not None

Expand Down
91 changes: 90 additions & 1 deletion tests/fixtures/conformance/api/public-api-v1.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
"DECISION_CLARIFY",
"DECISION_PASSTHROUGH",
"DECISION_UPDATE",
"DirectiveKind",
"Engine",
"POLICY_PROHIBIT",
"POLICY_USE",
Expand All @@ -35,13 +36,16 @@
"get_policy_items",
"get_step_decision",
"get_step_state",
"is_canonical_directive",
"is_clarify",
"is_passthrough",
"is_update",
"preview",
"preview_would_mutate",
"render_directive",
"state_diff",
"step"
"step",
"validate_directive"
],
"members": {
"Decision": {
Expand All @@ -59,6 +63,9 @@
"kind": "constant",
"value": "update"
},
"DirectiveKind": {
"kind": "class"
},
"Engine": {
"kind": "class"
},
Expand Down Expand Up @@ -283,6 +290,29 @@
]
}
},
"is_canonical_directive": {
"kind": "callable",
"signature": {
"params": [
{
"name": "text",
"kind": "POSITIONAL_OR_KEYWORD",
"has_default": false
}
]
},
"shape_probes": [
{
"kwargs": {
"text": "clear state"
},
"return_shape": {
"type": "boolean",
"const": true
}
}
]
},
"is_clarify": {
"kind": "callable",
"signature": {
Expand Down Expand Up @@ -433,6 +463,34 @@
]
}
},
"render_directive": {
"kind": "callable",
"signature": {
"params": [
{
"name": "kind",
"kind": "POSITIONAL_ONLY",
"has_default": false
},
{
"name": "operands",
"kind": "VAR_KEYWORD",
"has_default": false
}
]
},
"shape_probes": [
{
"args": [
"clear_state"
],
"return_shape": {
"type": "string",
"const": "clear state"
}
}
]
},
"state_diff": {
"kind": "callable",
"signature": {
Expand Down Expand Up @@ -527,6 +585,37 @@
}
}
]
},
"validate_directive": {
"kind": "callable",
"signature": {
"params": [
{
"name": "text",
"kind": "POSITIONAL_OR_KEYWORD",
"has_default": false
}
]
},
"shape_probes": [
{
"kwargs": {
"text": "use docker"
},
"return_shape": {
"kind": "validated_directive",
"text": "use docker"
}
},
{
"kwargs": {
"text": "please use docker"
},
"return_shape": {
"type": "null"
}
}
]
}
}
},
Expand Down
Loading
Loading