Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,17 @@ Or, if you pin AgentFlow as a project-scoped Go tool:

AgentFlow uses a simple but powerful syntax for creating dynamic prompt templates with type-safe variable interpolation.

### Comments

Single-`#` lines document templates without rendering into generated output. Markdown headings that start with `##`, or with a single `#` followed by an emoji, still render as normal text.

```af
.title System Prompt
# Internal note for maintainers only.
## Rendered Heading
You are a helpful assistant.
```

### Titles

Titles define distinct prompt sections within a file. Each title becomes a separate Go function.
Expand Down
5 changes: 5 additions & 0 deletions pkg/format/format_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,11 @@ Body two.
in: ".title Demo\r\n\r\nLine with spaces. \r\nLast line\t",
want: ".title Demo\n\nLine with spaces.\nLast line\n",
},
{
name: "preserves comments",
in: ".title Demo\n# internal note \nHello <! name >\n",
want: ".title Demo\n\n# internal note\nHello <!name>\n",
},
}

for _, tt := range tests {
Expand Down
5 changes: 5 additions & 0 deletions pkg/gen/gogen/testdata/comments_strip/input.golden.af
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
.title Commented Prompt
# This line documents the prompt but should not render.
Hello <!name>
# Comments between rendered lines should also disappear.
Goodbye.
26 changes: 26 additions & 0 deletions pkg/gen/gogen/testdata/comments_strip/output.golden.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions pkg/lsp/granular_tokens.go
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,8 @@ func generateGranularSemanticTokens(doc *Document) []protocol.UInteger {
tokenType = TokenTypeParameter // String values in conditionals should be colored like parameters
case kind.IntValue, kind.BoolValue:
tokenType = TokenTypeParameter
case kind.Comment:
tokenType = TokenTypeComment
case kind.Text:
tokenType = TokenTypeString // Regular text content
case kind.Whitespace:
Expand Down
3 changes: 3 additions & 0 deletions pkg/token/coarse/coarse.go
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,9 @@ func Convert(tokens token.Slice, input []byte) []Token {
i++
}

case kind.Comment:
i++

case kind.Text, kind.Whitespace:
// Skip whitespace that precedes a title directive (inter-prompt whitespace)
if tokens[i].Kind == kind.Whitespace && i+1 < len(tokens) && tokens[i+1].Kind == kind.TitleDirective {
Expand Down
27 changes: 27 additions & 0 deletions pkg/token/coarse/coarse_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,33 @@ func TestConvertGroupsTitleAndTrimsInterPromptWhitespace(t *testing.T) {
}
}

func TestConvertSkipsComments(t *testing.T) {
input := []byte(".title Demo\n# internal note\nHello")
tokens := token.Slice{
{Kind: kind.TitleDirective, Start: 0, End: 6},
{Kind: kind.Whitespace, Start: 6, End: 7},
{Kind: kind.TitleText, Start: 7, End: 11},
{Kind: kind.Whitespace, Start: 11, End: 12},
{Kind: kind.Comment, Start: 12, End: 28},
{Kind: kind.Text, Start: 28, End: 33},
}

got := Convert(tokens, input)
want := []Token{
{Kind: Title, Start: 7, End: 11},
{Kind: Text, Start: 28, End: 33},
}

if len(got) != len(want) {
t.Fatalf("expected %d tokens, got %d", len(want), len(got))
}
for i := range want {
if got[i] != want[i] {
t.Fatalf("token %d mismatch: got %+v want %+v", i, got[i], want[i])
}
}
}

func TestConvertTreatsStandaloneBracketAsText(t *testing.T) {
input := []byte("<")
tokens := token.Slice{{Kind: kind.OpenBracket, Start: 0, End: 1}}
Expand Down
4 changes: 2 additions & 2 deletions pkg/token/kind/kind.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,10 @@ const (
// Content
Text // Regular text content
Whitespace // Spaces, tabs, newlines (separators)
Comment // "# comment" line comments

// Future extension tokens (for later)
// RawBlock // For future raw block support
// Comment // For future comment support
)

func (k Kind) IsTag() bool {
Expand Down Expand Up @@ -93,7 +93,7 @@ func (k Kind) IsValue() bool {

func (k Kind) IsContent() bool {
switch k {
case Text:
case Text, Comment:
return true
}
return false
Expand Down
5 changes: 3 additions & 2 deletions pkg/token/kind/kind_string.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

31 changes: 31 additions & 0 deletions pkg/token/token.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,13 @@ func Tokenize(input []byte) (Slice, error) {
i := 0

for i < len(input) {
if isCommentStart(input, i) {
comment := parseComment(input, i)
tokens = append(tokens, comment)
i += comment.End - comment.Start
continue
}

// Check for .title directive at start of line
if i == 0 || (i > 0 && input[i-1] == '\n') {
if i < len(input) && input[i] == '.' {
Expand Down Expand Up @@ -487,6 +494,9 @@ func parseText(input []byte, start int) T {
break
}
}
if isCommentStart(input, pos) {
break
}
if input[pos] == '.' && (pos == 0 || input[pos-1] == '\n') {
// Check if this might be a title directive
if tryParseTitle(input, pos) != nil {
Expand All @@ -505,6 +515,27 @@ func parseText(input []byte, start int) T {
}
}

func parseComment(input []byte, start int) T {
pos := start
for pos < len(input) && input[pos] != '\n' {
pos++
}
if pos < len(input) && input[pos] == '\n' {
pos++
}
return T{Kind: kind.Comment, Start: start, End: pos}
}

func isCommentStart(input []byte, pos int) bool {
if pos >= len(input) || input[pos] != '#' || (pos > 0 && input[pos-1] != '\n') {
return false
}
if pos+1 < len(input) && input[pos+1] == '#' {
return false
}
return pos+2 >= len(input) || input[pos+1] != ' ' || input[pos+2] < 0x80
}

// Helper functions
func titleLength(input []byte, start int) int {
pos := start + 6 // ".title"
Expand Down
55 changes: 55 additions & 0 deletions pkg/token/token_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,61 @@ func TestTitle(t *testing.T) {
}
}

func TestComment(t *testing.T) {
input := []byte(".title Demo\n# used by onboarding only\nHello <!name>")
want := token.Slice{
{Kind: kind.TitleDirective, Start: 0, End: 6},
{Kind: kind.Whitespace, Start: 6, End: 7},
{Kind: kind.TitleText, Start: 7, End: 11},
{Kind: kind.Whitespace, Start: 11, End: 12},
{Kind: kind.Comment, Start: 12, End: 38},
{Kind: kind.Text, Start: 38, End: 44},
{Kind: kind.OpenBracket, Start: 44, End: 45},
{Kind: kind.DirectiveVar, Start: 45, End: 46},
{Kind: kind.VarName, Start: 46, End: 50},
{Kind: kind.CloseBracket, Start: 50, End: 51},
}

got, err := token.Tokenize(input)
require.NoError(t, err)
if !want.Equal(got) {
t.Fatalf("tokens not equal\nWANT:\n%s\nGOT:\n%s", want.Stringify(input), got.Stringify(input))
}
}

func TestHashInsideTextIsNotComment(t *testing.T) {
input := []byte("Use # literally in rendered text")
want := token.Slice{{Kind: kind.Text, Start: 0, End: len(input)}}

got, err := token.Tokenize(input)
require.NoError(t, err)
if !want.Equal(got) {
t.Fatalf("tokens not equal\nWANT:\n%s\nGOT:\n%s", want.Stringify(input), got.Stringify(input))
}
}

func TestMarkdownHeadingIsNotComment(t *testing.T) {
input := []byte("## Rendered heading")
want := token.Slice{{Kind: kind.Text, Start: 0, End: len(input)}}

got, err := token.Tokenize(input)
require.NoError(t, err)
if !want.Equal(got) {
t.Fatalf("tokens not equal\nWANT:\n%s\nGOT:\n%s", want.Stringify(input), got.Stringify(input))
}
}

func TestEmojiMarkdownHeadingIsNotComment(t *testing.T) {
input := []byte("# 📚 Rendered heading")
want := token.Slice{{Kind: kind.Text, Start: 0, End: len(input)}}

got, err := token.Tokenize(input)
require.NoError(t, err)
if !want.Equal(got) {
t.Fatalf("tokens not equal\nWANT:\n%s\nGOT:\n%s", want.Stringify(input), got.Stringify(input))
}
}

func TestVar(t *testing.T) {
varStart := []byte("<!")
varName := []byte("var1")
Expand Down
Loading