From e897ff1baefa6a200e08bae1ab36497bb8e7c561 Mon Sep 17 00:00:00 2001 From: OCPeyton Date: Sun, 3 May 2026 01:49:15 -0400 Subject: [PATCH 1/2] feat: add af line comments Support single-# .af comment lines across tokenization, formatting, codegen, and LSP highlighting while preserving Markdown heading output. Co-Authored-By: Claude Opus 4.6 --- README.md | 11 +++++ pkg/format/format_test.go | 5 +++ .../testdata/comments_strip/input.golden.af | 5 +++ .../testdata/comments_strip/output.golden.go | 26 +++++++++++ pkg/lsp/granular_tokens.go | 2 + pkg/token/coarse/coarse.go | 3 ++ pkg/token/coarse/coarse_test.go | 27 ++++++++++++ pkg/token/kind/kind.go | 4 +- pkg/token/kind/kind_string.go | 5 ++- pkg/token/token.go | 28 ++++++++++++ pkg/token/token_test.go | 44 +++++++++++++++++++ 11 files changed, 156 insertions(+), 4 deletions(-) create mode 100644 pkg/gen/gogen/testdata/comments_strip/input.golden.af create mode 100644 pkg/gen/gogen/testdata/comments_strip/output.golden.go diff --git a/README.md b/README.md index 206018b..b0755ee 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,17 @@ Or, if you pin AgentFlow as a project-scoped Go tool: AgentFlow uses a simple but powerful syntax for creating dynamic prompt templates with type-safe variable interpolation. +### Comments + +Single-`#` lines document templates without rendering into generated output. Markdown headings that start with `##` still render as normal text. + +```af +.title System Prompt +# Internal note for maintainers only. +## Rendered Heading +You are a helpful assistant. +``` + ### Titles Titles define distinct prompt sections within a file. Each title becomes a separate Go function. diff --git a/pkg/format/format_test.go b/pkg/format/format_test.go index 66663bd..2713de9 100644 --- a/pkg/format/format_test.go +++ b/pkg/format/format_test.go @@ -89,6 +89,11 @@ Body two. in: ".title Demo\r\n\r\nLine with spaces. \r\nLast line\t", want: ".title Demo\n\nLine with spaces.\nLast line\n", }, + { + name: "preserves comments", + in: ".title Demo\n# internal note \nHello \n", + want: ".title Demo\n\n# internal note\nHello \n", + }, } for _, tt := range tests { diff --git a/pkg/gen/gogen/testdata/comments_strip/input.golden.af b/pkg/gen/gogen/testdata/comments_strip/input.golden.af new file mode 100644 index 0000000..9d7c028 --- /dev/null +++ b/pkg/gen/gogen/testdata/comments_strip/input.golden.af @@ -0,0 +1,5 @@ +.title Commented Prompt +# This line documents the prompt but should not render. +Hello +# Comments between rendered lines should also disappear. +Goodbye. diff --git a/pkg/gen/gogen/testdata/comments_strip/output.golden.go b/pkg/gen/gogen/testdata/comments_strip/output.golden.go new file mode 100644 index 0000000..82eeeff --- /dev/null +++ b/pkg/gen/gogen/testdata/comments_strip/output.golden.go @@ -0,0 +1,26 @@ +// Code generated by agentflow v0.5.3; DO NOT EDIT. + +package commentsstrip + +import ( + "strings" +) + +type CommentedPrompt struct { + Name string +} + +func (input *CommentedPrompt) String() string { + var b strings.Builder + length := 0 + length += 6 + length += len(input.Name) + length += 1 + length += 9 + b.Grow(length) + b.WriteString("Hello ") + b.WriteString(input.Name) + b.WriteRune('\n') + b.WriteString("Goodbye.\n") + return b.String() +} diff --git a/pkg/lsp/granular_tokens.go b/pkg/lsp/granular_tokens.go index 6b4f1a0..ffd9503 100644 --- a/pkg/lsp/granular_tokens.go +++ b/pkg/lsp/granular_tokens.go @@ -117,6 +117,8 @@ func generateGranularSemanticTokens(doc *Document) []protocol.UInteger { tokenType = TokenTypeParameter // String values in conditionals should be colored like parameters case kind.IntValue, kind.BoolValue: tokenType = TokenTypeParameter + case kind.Comment: + tokenType = TokenTypeComment case kind.Text: tokenType = TokenTypeString // Regular text content case kind.Whitespace: diff --git a/pkg/token/coarse/coarse.go b/pkg/token/coarse/coarse.go index 4f97c0f..cf35a8f 100644 --- a/pkg/token/coarse/coarse.go +++ b/pkg/token/coarse/coarse.go @@ -201,6 +201,9 @@ func Convert(tokens token.Slice, input []byte) []Token { i++ } + case kind.Comment: + i++ + case kind.Text, kind.Whitespace: // Skip whitespace that precedes a title directive (inter-prompt whitespace) if tokens[i].Kind == kind.Whitespace && i+1 < len(tokens) && tokens[i+1].Kind == kind.TitleDirective { diff --git a/pkg/token/coarse/coarse_test.go b/pkg/token/coarse/coarse_test.go index c6b8e45..8b9077a 100644 --- a/pkg/token/coarse/coarse_test.go +++ b/pkg/token/coarse/coarse_test.go @@ -182,6 +182,33 @@ func TestConvertGroupsTitleAndTrimsInterPromptWhitespace(t *testing.T) { } } +func TestConvertSkipsComments(t *testing.T) { + input := []byte(".title Demo\n# internal note\nHello") + tokens := token.Slice{ + {Kind: kind.TitleDirective, Start: 0, End: 6}, + {Kind: kind.Whitespace, Start: 6, End: 7}, + {Kind: kind.TitleText, Start: 7, End: 11}, + {Kind: kind.Whitespace, Start: 11, End: 12}, + {Kind: kind.Comment, Start: 12, End: 28}, + {Kind: kind.Text, Start: 28, End: 33}, + } + + got := Convert(tokens, input) + want := []Token{ + {Kind: Title, Start: 7, End: 11}, + {Kind: Text, Start: 28, End: 33}, + } + + if len(got) != len(want) { + t.Fatalf("expected %d tokens, got %d", len(want), len(got)) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("token %d mismatch: got %+v want %+v", i, got[i], want[i]) + } + } +} + func TestConvertTreatsStandaloneBracketAsText(t *testing.T) { input := []byte("<") tokens := token.Slice{{Kind: kind.OpenBracket, Start: 0, End: 1}} diff --git a/pkg/token/kind/kind.go b/pkg/token/kind/kind.go index 90b9370..526210b 100644 --- a/pkg/token/kind/kind.go +++ b/pkg/token/kind/kind.go @@ -50,10 +50,10 @@ const ( // Content Text // Regular text content Whitespace // Spaces, tabs, newlines (separators) + Comment // "# comment" line comments // Future extension tokens (for later) // RawBlock // For future raw block support - // Comment // For future comment support ) func (k Kind) IsTag() bool { @@ -93,7 +93,7 @@ func (k Kind) IsValue() bool { func (k Kind) IsContent() bool { switch k { - case Text: + case Text, Comment: return true } return false diff --git a/pkg/token/kind/kind_string.go b/pkg/token/kind/kind_string.go index 991e51d..de22cf0 100644 --- a/pkg/token/kind/kind_string.go +++ b/pkg/token/kind/kind_string.go @@ -25,11 +25,12 @@ func _() { _ = x[BoolValue-14] _ = x[Text-15] _ = x[Whitespace-16] + _ = x[Comment-17] } -const _Kind_name = "UnsetOpenBracketCloseBracketDirectiveVarDirectiveCondDirectiveEndDirectiveElseTitleDirectiveTitleTextVarNameTypeNameOperatorStringValueIntValueBoolValueTextWhitespace" +const _Kind_name = "UnsetOpenBracketCloseBracketDirectiveVarDirectiveCondDirectiveEndDirectiveElseTitleDirectiveTitleTextVarNameTypeNameOperatorStringValueIntValueBoolValueTextWhitespaceComment" -var _Kind_index = [...]uint8{0, 5, 16, 28, 40, 53, 65, 78, 92, 101, 108, 116, 124, 135, 143, 152, 156, 166} +var _Kind_index = [...]uint8{0, 5, 16, 28, 40, 53, 65, 78, 92, 101, 108, 116, 124, 135, 143, 152, 156, 166, 173} func (i Kind) String() string { if i < 0 || i >= Kind(len(_Kind_index)-1) { diff --git a/pkg/token/token.go b/pkg/token/token.go index bf88993..f06c9e9 100644 --- a/pkg/token/token.go +++ b/pkg/token/token.go @@ -81,6 +81,13 @@ func Tokenize(input []byte) (Slice, error) { i := 0 for i < len(input) { + if isCommentStart(input, i) { + comment := parseComment(input, i) + tokens = append(tokens, comment) + i += comment.End - comment.Start + continue + } + // Check for .title directive at start of line if i == 0 || (i > 0 && input[i-1] == '\n') { if i < len(input) && input[i] == '.' { @@ -487,6 +494,9 @@ func parseText(input []byte, start int) T { break } } + if isCommentStart(input, pos) { + break + } if input[pos] == '.' && (pos == 0 || input[pos-1] == '\n') { // Check if this might be a title directive if tryParseTitle(input, pos) != nil { @@ -505,6 +515,24 @@ func parseText(input []byte, start int) T { } } +func parseComment(input []byte, start int) T { + pos := start + for pos < len(input) && input[pos] != '\n' { + pos++ + } + if pos < len(input) && input[pos] == '\n' { + pos++ + } + return T{Kind: kind.Comment, Start: start, End: pos} +} + +func isCommentStart(input []byte, pos int) bool { + return pos < len(input) && + input[pos] == '#' && + (pos+1 >= len(input) || input[pos+1] != '#') && + (pos == 0 || input[pos-1] == '\n') +} + // Helper functions func titleLength(input []byte, start int) int { pos := start + 6 // ".title" diff --git a/pkg/token/token_test.go b/pkg/token/token_test.go index bed4c45..a3d316b 100644 --- a/pkg/token/token_test.go +++ b/pkg/token/token_test.go @@ -209,6 +209,50 @@ func TestTitle(t *testing.T) { } } +func TestComment(t *testing.T) { + input := []byte(".title Demo\n# used by onboarding only\nHello ") + want := token.Slice{ + {Kind: kind.TitleDirective, Start: 0, End: 6}, + {Kind: kind.Whitespace, Start: 6, End: 7}, + {Kind: kind.TitleText, Start: 7, End: 11}, + {Kind: kind.Whitespace, Start: 11, End: 12}, + {Kind: kind.Comment, Start: 12, End: 38}, + {Kind: kind.Text, Start: 38, End: 44}, + {Kind: kind.OpenBracket, Start: 44, End: 45}, + {Kind: kind.DirectiveVar, Start: 45, End: 46}, + {Kind: kind.VarName, Start: 46, End: 50}, + {Kind: kind.CloseBracket, Start: 50, End: 51}, + } + + got, err := token.Tokenize(input) + require.NoError(t, err) + if !want.Equal(got) { + t.Fatalf("tokens not equal\nWANT:\n%s\nGOT:\n%s", want.Stringify(input), got.Stringify(input)) + } +} + +func TestHashInsideTextIsNotComment(t *testing.T) { + input := []byte("Use # literally in rendered text") + want := token.Slice{{Kind: kind.Text, Start: 0, End: len(input)}} + + got, err := token.Tokenize(input) + require.NoError(t, err) + if !want.Equal(got) { + t.Fatalf("tokens not equal\nWANT:\n%s\nGOT:\n%s", want.Stringify(input), got.Stringify(input)) + } +} + +func TestMarkdownHeadingIsNotComment(t *testing.T) { + input := []byte("## Rendered heading") + want := token.Slice{{Kind: kind.Text, Start: 0, End: len(input)}} + + got, err := token.Tokenize(input) + require.NoError(t, err) + if !want.Equal(got) { + t.Fatalf("tokens not equal\nWANT:\n%s\nGOT:\n%s", want.Stringify(input), got.Stringify(input)) + } +} + func TestVar(t *testing.T) { varStart := []byte(" Date: Sun, 3 May 2026 01:51:23 -0400 Subject: [PATCH 2/2] fix: preserve emoji markdown headings Avoid treating existing single-# emoji Markdown headings as comments so generated example output stays stable. Co-Authored-By: Claude Opus 4.6 --- README.md | 2 +- pkg/token/token.go | 11 +++++++---- pkg/token/token_test.go | 11 +++++++++++ 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index b0755ee..a3c0acc 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ AgentFlow uses a simple but powerful syntax for creating dynamic prompt template ### Comments -Single-`#` lines document templates without rendering into generated output. Markdown headings that start with `##` still render as normal text. +Single-`#` lines document templates without rendering into generated output. Markdown headings that start with `##`, or with a single `#` followed by an emoji, still render as normal text. ```af .title System Prompt diff --git a/pkg/token/token.go b/pkg/token/token.go index f06c9e9..4cd4f6b 100644 --- a/pkg/token/token.go +++ b/pkg/token/token.go @@ -527,10 +527,13 @@ func parseComment(input []byte, start int) T { } func isCommentStart(input []byte, pos int) bool { - return pos < len(input) && - input[pos] == '#' && - (pos+1 >= len(input) || input[pos+1] != '#') && - (pos == 0 || input[pos-1] == '\n') + if pos >= len(input) || input[pos] != '#' || (pos > 0 && input[pos-1] != '\n') { + return false + } + if pos+1 < len(input) && input[pos+1] == '#' { + return false + } + return pos+2 >= len(input) || input[pos+1] != ' ' || input[pos+2] < 0x80 } // Helper functions diff --git a/pkg/token/token_test.go b/pkg/token/token_test.go index a3d316b..1112c85 100644 --- a/pkg/token/token_test.go +++ b/pkg/token/token_test.go @@ -253,6 +253,17 @@ func TestMarkdownHeadingIsNotComment(t *testing.T) { } } +func TestEmojiMarkdownHeadingIsNotComment(t *testing.T) { + input := []byte("# 📚 Rendered heading") + want := token.Slice{{Kind: kind.Text, Start: 0, End: len(input)}} + + got, err := token.Tokenize(input) + require.NoError(t, err) + if !want.Equal(got) { + t.Fatalf("tokens not equal\nWANT:\n%s\nGOT:\n%s", want.Stringify(input), got.Stringify(input)) + } +} + func TestVar(t *testing.T) { varStart := []byte("