From f6e512348ad519f0607cf7b0e399c72d725145ef Mon Sep 17 00:00:00 2001 From: gf712 Date: Fri, 4 Sep 2026 16:26:30 +0100 Subject: [PATCH 1/3] parser: index the packrat memo by token position --- src/parser/Parser.cpp | 77 ++++++++++++------------------------------ src/parser/Parser.cppm | 41 ++++++++++++---------- 2 files changed, 46 insertions(+), 72 deletions(-) diff --git a/src/parser/Parser.cpp b/src/parser/Parser.cpp index 1802516e..f554743a 100644 --- a/src/parser/Parser.cpp +++ b/src/parser/Parser.cpp @@ -38,42 +38,13 @@ using namespace parser; return {}; \ } while (0) -[[maybe_unused]] static int hits = 0; - -size_t Parser::CacheHash::operator()(const Parser::CacheKey &cache) const +static std::uint16_t next_memo_rule_id() { - size_t seed = cache.rule.hash_code(); - seed ^= std::bit_cast(cache.token.start().pointer_to_program) + 0x9e3779b9 + (seed << 6) - + (seed >> 2); - seed ^= static_cast(cache.token.token_type()) + 0x9e3779b9 + (seed << 6) + (seed >> 2); - return seed; -} - -bool Parser::CacheEqual::operator()(const Parser::CacheKey &lhs, const Parser::CacheKey &rhs) const -{ - if ((lhs.token.start().pointer_to_program != rhs.token.start().pointer_to_program) - || (lhs.token.token_type() != rhs.token.token_type())) { - return false; - } - return lhs.rule == rhs.rule; + static std::uint16_t next = 0; + return next++; } -// template struct Pattern -// { -// virtual ~Pattern() = default; - -// static bool matches(Parser &p) -// { -// const auto start_stack_size = p.stack().size(); -// const auto start_position = p.token_position(); -// const bool is_match = Derived::matches_impl(p); -// if (!is_match) { -// while (p.stack().size() > start_stack_size) { p.pop_back(); } -// p.token_position() = start_position; -// } -// return is_match; -// } -// }; +template inline const std::uint16_t memo_rule_id = next_memo_rule_id(); template struct traits; @@ -129,19 +100,19 @@ template struct PatternV2 { const auto start_position = p.token_position(); if constexpr (::detail::has_type{}) { - const auto token = p.lexer().peek_token(start_position); - ASSERT(token.has_value()); - Parser::CacheKey line{ typeid(Derived), *token }; - Parser::CacheValue value{ false, start_position }; - p.m_cache[line] = value; + // Seed the left-recursion sentinel: if the rule re-enters itself at this same + // position the lookup below flips the bool, and grow_lr takes over. + p.memo_insert(start_position, memo_rule_id) = + Parser::CacheValue{ false, start_position }; } const std::optional result = Derived::matches_impl(p); if constexpr (::detail::has_type{}) { - const auto token = p.lexer().peek_token(start_position); if (result.has_value()) { - Parser::CacheKey line{ typeid(Derived), *token }; - auto &value = p.m_cache.at(line); - ASSERT(value.has_value()); + // Safe to hold across grow_lr: memo entries live in a deque. + auto *slot = p.memo_find(start_position, memo_rule_id); + ASSERT(slot); + ASSERT(slot->has_value()); + auto &value = *slot; if (std::holds_alternative(value->value) && std::get(value->value)) { return grow_lr(p, start_position, *value); } else { @@ -215,22 +186,18 @@ template class PatternMa const auto t = p.lexer().peek_token(original_token_position); if (!t.has_value()) { return {}; } - std::optional line; if constexpr (::detail::has_type{}) { - line.emplace(Parser::CacheKey{ typeid(CurrentType), *t }); - if (auto it = p.m_cache.find(*line); it != p.m_cache.end()) { - hits++; - auto &cache = it->second; - if (!cache.has_value()) { return {}; } - // auto&& [node, position] = *cache; - auto &value = cache->value; - const auto &position = cache->position; - p.token_position() = position; + if (auto *slot = p.memo_find(original_token_position, memo_rule_id)) { + if (!slot->has_value()) { return {}; } + auto &value = (*slot)->value; + p.token_position() = (*slot)->position; if (std::holds_alternative(value)) { std::get(value) = true; return {}; } else { + // The reference handed to advance() stays valid while it parses on, + // because memo entries live in a deque. auto &v = std::get(value); ASSERT(std::holds_alternative(v)); return advance(p, std::get(v)); @@ -241,14 +208,15 @@ template class PatternMa if (auto result = CurrentType::matches(p)) { if constexpr (::detail::has_type{}) { - p.m_cache[*line] = Parser::CacheValue{ *result, p.token_position() }; + p.memo_insert(original_token_position, memo_rule_id) = + Parser::CacheValue{ *result, p.token_position() }; } return advance(p, *result); } else { p.token_position() = original_token_position; if constexpr (::detail::has_type{}) { - p.m_cache[*line] = std::nullopt; + p.memo_insert(original_token_position, memo_rule_id) = std::nullopt; } return std::nullopt; } @@ -7542,7 +7510,6 @@ struct StatementsPattern : PatternV2 using pattern1 = PatternMatchV2; ResultType statements; while (auto result = pattern1::match(p)) { - // p.m_cache.clear(); auto [statement] = *result; statements.insert(statements.end(), statement.begin(), statement.end()); } diff --git a/src/parser/Parser.cppm b/src/parser/Parser.cppm index 6dc89de8..79ed6a16 100644 --- a/src/parser/Parser.cppm +++ b/src/parser/Parser.cppm @@ -14,22 +14,6 @@ class Parser std::size_t m_token_position{ 0 }; public: - struct CacheKey - { - const std::type_info &rule; - Token token; - }; - - struct CacheHash - { - std::size_t operator()(const CacheKey &cache) const; - }; - - struct CacheEqual - { - bool operator()(const CacheKey &lhs, const CacheKey &rhs) const; - }; - struct CacheValue { // AST nodes are owned by the Module's arena; the cache holds non-owning @@ -39,7 +23,30 @@ class Parser std::size_t position; }; - std::unordered_map, CacheHash, CacheEqual> m_cache; + using MemoSlot = std::optional; + + MemoSlot *memo_find(std::size_t position, std::uint16_t rule) + { + if (position >= m_memo_index.size()) { return nullptr; } + for (const auto &[id, slot] : m_memo_index[position]) { + if (id == rule) { return &m_memo_pool[slot]; } + } + return nullptr; + } + + MemoSlot &memo_insert(std::size_t position, std::uint16_t rule) + { + if (auto *existing = memo_find(position, rule)) { return *existing; } + if (position >= m_memo_index.size()) { m_memo_index.resize(position + 1); } + m_memo_pool.emplace_back(); + m_memo_index[position].emplace_back( + rule, static_cast(m_memo_pool.size() - 1)); + return m_memo_pool.back(); + } + + private: + std::deque m_memo_pool; + std::vector>> m_memo_index; public: Parser(Lexer &l) : m_module(std::make_shared(l.filename())), m_lexer(l) From 2d9f548bdf92c07404b7f80648823a852dbe3649 Mon Sep 17 00:00:00 2001 From: gf712 Date: Fri, 4 Sep 2026 16:30:45 +0100 Subject: [PATCH 2/3] parser: parse binary operators by precedence climbing --- src/parser/Parser.cpp | 372 +++++++++--------------------------------- 1 file changed, 75 insertions(+), 297 deletions(-) diff --git a/src/parser/Parser.cpp b/src/parser/Parser.cpp index f554743a..3b8b36a8 100644 --- a/src/parser/Parser.cpp +++ b/src/parser/Parser.cpp @@ -3183,316 +3183,94 @@ struct FactorPattern : PatternV2 } }; -template<> struct traits -{ - using result_type = ASTNode *; -}; - -struct TermPattern : PatternV2 -{ - using ResultType = typename traits::result_type; - - // term: - // | term '*' factor - // | term '/' factor - // | term '//' factor - // | term '%' factor - // | term '@' factor - // | factor - static std::optional matches_impl(Parser &p) - { - DEBUG_LOG("TermPattern"); - - using pattern1 = PatternMatchV2, - FactorPattern>; - if (auto result = pattern1::match(p)) { - DEBUG_LOG("term '*' factor"); - auto [lhs, _, rhs] = *result; - return p.arena().create(BinaryOpType::MULTIPLY, - lhs, - rhs, - SourceLocation{ lhs->source_location().start, rhs->source_location().end }); - } - - using pattern2 = PatternMatchV2, - FactorPattern>; - if (auto result = pattern2::match(p)) { - DEBUG_LOG("term '/' factor"); - auto [lhs, _, rhs] = *result; - return p.arena().create(BinaryOpType::SLASH, - lhs, - rhs, - SourceLocation{ lhs->source_location().start, rhs->source_location().end }); - } - - using pattern3 = PatternMatchV2, - FactorPattern>; - if (auto result = pattern3::match(p)) { - DEBUG_LOG("term '//' factor"); - auto [lhs, _, rhs] = *result; - return p.arena().create(BinaryOpType::FLOORDIV, - lhs, - rhs, - SourceLocation{ lhs->source_location().start, rhs->source_location().end }); - } - - using pattern4 = PatternMatchV2, - FactorPattern>; - if (auto result = pattern4::match(p)) { - DEBUG_LOG("term '%' factor"); - auto [lhs, _, rhs] = *result; - return p.arena().create(BinaryOpType::MODULO, - lhs, - rhs, - SourceLocation{ lhs->source_location().start, rhs->source_location().end }); - } - - using pattern5 = - PatternMatchV2, FactorPattern>; - if (auto result = pattern5::match(p)) { - DEBUG_LOG("term '@' factor"); - auto [lhs, _, rhs] = *result; - return p.arena().create(BinaryOpType::MATMUL, - lhs, - rhs, - SourceLocation{ lhs->source_location().start, rhs->source_location().end }); - } - - // factor - using pattern6 = PatternMatchV2; - if (auto result = pattern6::match(p)) { - auto [factor] = *result; - return factor; - } - +// The grammar spells the binary operator precedence levels as a chain of left-recursive +// rules - bitwise_or -> bitwise_xor -> bitwise_and -> shift_expr -> sum -> term +// Precedence climbing does one descent to the operand and a token peek per +// operator, and takes the left recursion. BitwiseOrPattern keeps its name because ComparisonPattern +// refers to it; nothing outside the chain referred to the five levels below. +// +// Precedences are the Python ones, lowest binding first. Every operator here is +// left-associative; `not` and the comparisons sit above this in ComparisonPattern, and unary +// +-~ and the right-associative ** below it in FactorPattern, so neither is affected. +struct BinaryOperatorInfo +{ + std::uint8_t precedence; + BinaryOpType type; +}; + +static constexpr std::optional binary_operator(Token::TokenType token) +{ + switch (token) { + case Token::TokenType::VBAR: + return BinaryOperatorInfo{ 1, BinaryOpType::OR }; + case Token::TokenType::CIRCUMFLEX: + return BinaryOperatorInfo{ 2, BinaryOpType::XOR }; + case Token::TokenType::AMPER: + return BinaryOperatorInfo{ 3, BinaryOpType::AND }; + case Token::TokenType::LEFTSHIFT: + return BinaryOperatorInfo{ 4, BinaryOpType::LEFTSHIFT }; + case Token::TokenType::RIGHTSHIFT: + return BinaryOperatorInfo{ 4, BinaryOpType::RIGHTSHIFT }; + case Token::TokenType::PLUS: + return BinaryOperatorInfo{ 5, BinaryOpType::PLUS }; + case Token::TokenType::MINUS: + return BinaryOperatorInfo{ 5, BinaryOpType::MINUS }; + case Token::TokenType::STAR: + return BinaryOperatorInfo{ 6, BinaryOpType::MULTIPLY }; + case Token::TokenType::SLASH: + return BinaryOperatorInfo{ 6, BinaryOpType::SLASH }; + case Token::TokenType::DOUBLESLASH: + return BinaryOperatorInfo{ 6, BinaryOpType::FLOORDIV }; + case Token::TokenType::PERCENT: + return BinaryOperatorInfo{ 6, BinaryOpType::MODULO }; + case Token::TokenType::AT: + return BinaryOperatorInfo{ 6, BinaryOpType::MATMUL }; + default: return {}; } -}; - -template<> struct traits -{ - using result_type = ASTNode *; -}; - -struct SumPattern : PatternV2 -{ - using ResultType = typename traits::result_type; - // left recursive - // sum: - // | sum '+' term - // | sum '-' term - // | term - - static std::optional matches_impl(Parser &p) - { - DEBUG_LOG("SumPattern"); - DEBUG_LOG("{}", p.lexer().peek_token(p.token_position())->to_string()); - // sum '+' term - using pattern1 = - PatternMatchV2, TermPattern>; - if (auto result = pattern1::match(p)) { - DEBUG_LOG("sum '+' term"); - auto [lhs, _, rhs] = *result; - return p.arena().create(BinaryOpType::PLUS, - lhs, - rhs, - SourceLocation{ lhs->source_location().start, rhs->source_location().end }); - } - - // sum '-' term - using pattern2 = - PatternMatchV2, TermPattern>; - if (auto result = pattern2::match(p)) { - DEBUG_LOG("sum '-' term"); - auto [lhs, _, rhs] = *result; - return p.arena().create(BinaryOpType::MINUS, - lhs, - rhs, - SourceLocation{ lhs->source_location().start, rhs->source_location().end }); - } - - // term - using pattern3 = PatternMatchV2; - if (auto result = pattern3::match(p)) { - DEBUG_LOG("term"); - auto [term] = *result; - return term; - } - - return {}; - } -}; - - -template<> struct traits -{ - using result_type = ASTNode *; -}; - -struct ShiftExprPattern : PatternV2 -{ - using ResultType = typename traits::result_type; - - // shift_expr: - // | shift_expr '<<' sum - // | shift_expr '>>' sum - // | sum - static std::optional matches_impl(Parser &p) - { - DEBUG_LOG("ShiftExprPattern"); - DEBUG_LOG("{}", p.lexer().peek_token(p.token_position())->to_string()); - using pattern1 = PatternMatchV2, - SumPattern>; - if (auto result = pattern1::match(p)) { - DEBUG_LOG("shift_expr '<<' sum"); - auto [lhs, _, rhs] = *result; - return p.arena().create(BinaryOpType::LEFTSHIFT, - lhs, - rhs, - SourceLocation{ lhs->source_location().start, rhs->source_location().end }); - } - - using pattern2 = PatternMatchV2, - SumPattern>; - if (auto result = pattern2::match(p)) { - DEBUG_LOG("shift_expr '>>' sum"); - auto [lhs, _, rhs] = *result; - return p.arena().create(BinaryOpType::RIGHTSHIFT, - lhs, - rhs, - SourceLocation{ lhs->source_location().start, rhs->source_location().end }); - } - - using pattern3 = PatternMatchV2; - if (auto result = pattern3::match(p)) { - DEBUG_LOG("sum"); - auto [sum] = *result; - return sum; - } - return {}; - } -}; +} -template<> struct traits +struct BitwiseOrPattern : PatternV2 { - using result_type = ASTNode *; -}; + using ResultType = typename traits::result_type; -struct BitwiseAndPattern : PatternV2 -{ - using ResultType = typename traits::result_type; - // bitwise_and: - // | bitwise_and '&' shift_expr - // | shift_expr + // bitwise_or / bitwise_xor / bitwise_and / shift_expr / sum / term static std::optional matches_impl(Parser &p) { - DEBUG_LOG("bitwise_and"); - - // bitwise_and '&' shift_expr - using pattern1 = PatternMatchV2, - ShiftExprPattern>; - if (auto result = pattern1::match(p)) { - DEBUG_LOG("bitwise_and '&' shift_expr"); - auto [lhs, _, rhs] = *result; - return p.arena().create(BinaryOpType::AND, - lhs, - rhs, - SourceLocation{ lhs->source_location().start, rhs->source_location().end }); - } - - using pattern2 = PatternMatchV2; - if (auto result = pattern2::match(p)) { - DEBUG_LOG("shift_expr"); - auto [shift_expr] = *result; - return shift_expr; - } - - return {}; + DEBUG_LOG("BitwiseOrPattern"); + return climb(p, 1); } -}; -template<> struct traits -{ - using result_type = ASTNode *; -}; - -struct BitwiseXorPattern : PatternV2 -{ - using ResultType = typename traits::result_type; - - // bitwise_xor: - // | bitwise_xor '^' bitwise_and - // | bitwise_and - static std::optional matches_impl(Parser &p) + private: + static std::optional climb(Parser &p, std::uint8_t min_precedence) { - DEBUG_LOG("BitwiseXorPattern"); - DEBUG_LOG("{}", p.lexer().peek_token(p.token_position())->to_string()); - // bitwise_xor '^' bitwise_and - using pattern1 = PatternMatchV2, - BitwiseAndPattern>; - if (auto result = pattern1::match(p)) { - DEBUG_LOG("bitwise_xor '^' bitwise_and"); - auto [lhs, _, rhs] = *result; - return p.arena().create(BinaryOpType::XOR, - lhs, - rhs, - SourceLocation{ lhs->source_location().start, rhs->source_location().end }); - } - - // bitwise_and - using pattern2 = PatternMatchV2; - if (auto result = pattern2::match(p)) { - DEBUG_LOG("bitwise_and"); - auto [and_op] = *result; - return and_op; - } - - return {}; - } -}; + auto operand = PatternMatchV2::match(p); + if (!operand.has_value()) { return {}; } + ASTNode *lhs = std::get<0>(*operand); - -struct BitwiseOrPattern : PatternV2 -{ - using ResultType = typename traits::result_type; - - // bitwise_or: - // | bitwise_or '|' bitwise_xor - // | bitwise_xor - static std::optional matches_impl(Parser &p) - { - DEBUG_LOG("BitwiseOrPattern"); - DEBUG_LOG("{}", p.lexer().peek_token(p.token_position())->to_string()); - // bitwise_or '|' bitwise_xor - using pattern1 = PatternMatchV2, - BitwiseXorPattern>; - if (auto result = pattern1::match(p)) { - DEBUG_LOG("bitwise_or '|' bitwise_xor"); - auto [lhs, _, rhs] = *result; - return p.arena().create(BinaryOpType::OR, + while (true) { + const auto token = p.lexer().peek_token(p.token_position()); + if (!token.has_value()) { break; } + const auto op = binary_operator(token->token_type()); + if (!op.has_value() || op->precedence < min_precedence) { break; } + + // Left associativity: the right operand takes only strictly tighter operators, + // so an operator of equal precedence is left for the next turn of this loop. + const auto before_operator = p.token_position(); + p.token_position() += 1; + auto rhs = climb(p, static_cast(op->precedence + 1)); + if (!rhs.has_value()) { + // No right operand: the ladder would have failed this alternative and + // fallen through to the bare operand, so give the operator back. + p.token_position() = before_operator; + break; + } + lhs = p.arena().create(op->type, lhs, - rhs, - SourceLocation{ lhs->source_location().start, rhs->source_location().end }); + *rhs, + SourceLocation{ lhs->source_location().start, (*rhs)->source_location().end }); } - - // bitwise_xor - using pattern2 = PatternMatchV2; - if (auto result = pattern2::match(p)) { - DEBUG_LOG("bitwise_xor"); - auto [bitwise_xor] = *result; - return bitwise_xor; - } - - return {}; + return lhs; } }; From ed8e1b4de9d52c9e021f5ba4c532aaf341360639 Mon Sep 17 00:00:00 2001 From: gf712 Date: Fri, 4 Sep 2026 16:34:20 +0100 Subject: [PATCH 3/3] parser: parse the postfix chain as a loop --- integration/tests/left_recursive_rules.py | 57 ++++++++ src/parser/Parser.cpp | 166 +++++++++++++--------- 2 files changed, 157 insertions(+), 66 deletions(-) create mode 100644 integration/tests/left_recursive_rules.py diff --git a/integration/tests/left_recursive_rules.py b/integration/tests/left_recursive_rules.py new file mode 100644 index 00000000..06fad7d0 --- /dev/null +++ b/integration/tests/left_recursive_rules.py @@ -0,0 +1,57 @@ +# The parser seeds a left-recursion sentinel only for the rules that can re-enter themselves +# at the same position, which after the precedence-climbing and postfix-loop rewrites is just +# two: dotted_name (`import a.b.c`) and t_primary (the target side of an assignment). +# +# Those two are the reason the sentinel and grow_lr still exist, and this file is what proves +# they still work. If a rule is dropped from is_left_recursive it will recurse without +# terminating, and if a newly left-recursive rule is added without an entry the same happens +# there - so a hang here is as much a failure as a wrong answer. + +# dotted_name: dotted_name '.' NAME | NAME +import os.path + +assert os.path.sep == "/" + +import os.path as shortcut + +assert shortcut.sep == "/" + +from os.path import sep + +assert sep == "/" + + +# t_primary: the left-recursive part of an assignment target. The parser matches the longest +# prefix that is still followed by a postfix operator, and the enclosing rule takes the last +# one, so each extra link here exercises another turn of the seed-growing loop. +nested = {"x": {"y": [1, 2]}} +nested["x"]["y"][0] = 9 +assert nested["x"]["y"][0] == 9 +assert nested["x"]["y"][1] == 2 + + +class Holder: + def __init__(self): + self.d = {"k": [0, 0]} + self.child = None + + +h = Holder() +h.d["k"][0] = 5 +assert h.d["k"][0] == 5 + +# attribute target reached through another attribute +h.child = Holder() +h.child.d["k"][1] = 7 +assert h.child.d["k"][1] == 7 + +# a longer chain: attribute, attribute, subscript, subscript +h.child.child = Holder() +h.child.child.d["k"][0] = 11 +assert h.child.child.d["k"][0] == 11 + +# plain attribute targets still work alongside the chained ones +h.child.child.d = {"k": [1]} +assert h.child.child.d["k"][0] == 1 + +print("left_recursive_rules: ok") diff --git a/src/parser/Parser.cpp b/src/parser/Parser.cpp index 3b8b36a8..ffbeb032 100644 --- a/src/parser/Parser.cpp +++ b/src/parser/Parser.cpp @@ -46,6 +46,24 @@ static std::uint16_t next_memo_rule_id() template inline const std::uint16_t memo_rule_id = next_memo_rule_id(); +// Seeding the left-recursion sentinel is only useful for a rule that can re-enter itself at +// the same position: the seed exists so the re-entry finds it, flips it, and hands control to +// grow_lr. +// Keep this in step with the grammar: a rule whose first element can reach the rule itself +// needs an entry here, or it recurses until the stack runs out. Both current entries are +// direct self-references, which is also all grow_lr claims to detect. +template struct is_left_recursive : std::false_type +{ +}; + +template<> struct is_left_recursive : std::true_type +{ +}; + +template<> struct is_left_recursive : std::true_type +{ +}; + template struct traits; namespace detail { @@ -98,15 +116,21 @@ template struct PatternV2 public: static std::optional matches(Parser &p) { + // Memoisation itself happens in PatternMatchV2_::match, which stores the result + // whether or not a sentinel was seeded; this only governs the left-recursion seed. + static constexpr bool seeds_sentinel = + ::detail::has_type{} + && is_left_recursive::value; + const auto start_position = p.token_position(); - if constexpr (::detail::has_type{}) { + if constexpr (seeds_sentinel) { // Seed the left-recursion sentinel: if the rule re-enters itself at this same // position the lookup below flips the bool, and grow_lr takes over. p.memo_insert(start_position, memo_rule_id) = Parser::CacheValue{ false, start_position }; } - const std::optional result = Derived::matches_impl(p); - if constexpr (::detail::has_type{}) { + const auto &result = Derived::matches_impl(p); + if constexpr (seeds_sentinel) { if (result.has_value()) { // Safe to hold across grow_lr: memo entries live in a deque. auto *slot = p.memo_find(start_position, memo_rule_id); @@ -2968,78 +2992,88 @@ struct PrimaryPattern : PatternV2 // | primary '(' [arguments] ')' // | primary '[' slices ']' // | atom + // + // Spelled left-recursively the rule re-parses the whole primary once per postfix + // operator and leans on grow_lr to extend the seed. The postfix forms all start at the + // position after the primary, so a loop over them does the same work without + // re-entering the rule: parse the atom once, then keep appending. The alternatives are + // tried in the order above - genexp before the call form, since a genexp also opens + // with '('. static std::optional matches_impl(Parser &p) { - // primary '.' NAME DEBUG_LOG("PrimaryPattern"); - using pattern2 = PatternMatchV2, - NAMEPattern>; - if (auto result = pattern2::match(p)) { - DEBUG_LOG(" primary '.' NAME"); - auto [value, _, name_token] = *result; - std::string name{ name_token.token.start().pointer_to_program, - name_token.token.end().pointer_to_program }; - return p.arena().create(value, - name, - ContextType::LOAD, - SourceLocation{ value->source_location().start, name_token.token.end() }); - } - // primary genexp - using pattern3 = PatternMatchV2; - if (auto result = pattern3::match(p)) { - DEBUG_LOG("primary genexp"); - auto [function, arg] = *result; - std::vector args{ arg }; - std::vector kwargs; - return p.arena().create(function, - args, - kwargs, - SourceLocation{ function->source_location().start, arg->source_location().end }); - } + auto atom = PatternMatchV2::match(p); + if (!atom.has_value()) { return {}; } + ASTNode *value = std::get<0>(*atom); - // primary '(' [arguments] ')' - using pattern4 = PatternMatchV2, - ZeroOrOnePatternV2, - SingleTokenPatternV2>; - if (auto result = pattern4::match(p)) { - DEBUG_LOG("primary '(' [arguments] ')'"); - std::vector args; - std::vector kwargs; - auto [function, _, arguments, r] = *result; - if (arguments.has_value()) { - auto [args_, kwargs_] = *arguments; - args = std::move(args_); - kwargs = std::move(kwargs_); + while (true) { + // primary '.' NAME + using dot_name = + PatternMatchV2, NAMEPattern>; + if (auto result = dot_name::match(p)) { + DEBUG_LOG(" primary '.' NAME"); + auto [_, name_token] = *result; + std::string name{ name_token.token.start().pointer_to_program, + name_token.token.end().pointer_to_program }; + value = p.arena().create(value, + name, + ContextType::LOAD, + SourceLocation{ value->source_location().start, name_token.token.end() }); + continue; } - return p.arena().create(function, - args, - kwargs, - SourceLocation{ function->source_location().start, r.token.end() }); - } - // primary '[' slices ']' - using pattern5 = PatternMatchV2, - SlicesPattern, - SingleTokenPatternV2>; - if (auto result = pattern5::match(p)) { - DEBUG_LOG("'[' slices ']'"); - auto [value, l, slices, r] = *result; - return p.arena().create(value, - slices, - ContextType::LOAD, - SourceLocation{ value->source_location().start, r.token.end() }); - } + // primary genexp + if (auto result = PatternMatchV2::match(p)) { + DEBUG_LOG("primary genexp"); + auto [arg] = *result; + std::vector args{ arg }; + std::vector kwargs; + value = p.arena().create(value, + args, + kwargs, + SourceLocation{ value->source_location().start, arg->source_location().end }); + continue; + } - using pattern6 = PatternMatchV2; - if (auto result = pattern6::match(p)) { - auto [atom] = *result; - return atom; + // primary '(' [arguments] ')' + using call = PatternMatchV2, + ZeroOrOnePatternV2, + SingleTokenPatternV2>; + if (auto result = call::match(p)) { + DEBUG_LOG("primary '(' [arguments] ')'"); + std::vector args; + std::vector kwargs; + auto [_, arguments, r] = *result; + if (arguments.has_value()) { + auto [args_, kwargs_] = *arguments; + args = std::move(args_); + kwargs = std::move(kwargs_); + } + value = p.arena().create(value, + args, + kwargs, + SourceLocation{ value->source_location().start, r.token.end() }); + continue; + } + + // primary '[' slices ']' + using subscript = PatternMatchV2, + SlicesPattern, + SingleTokenPatternV2>; + if (auto result = subscript::match(p)) { + DEBUG_LOG("'[' slices ']'"); + auto [l, slices, r] = *result; + value = p.arena().create(value, + slices, + ContextType::LOAD, + SourceLocation{ value->source_location().start, r.token.end() }); + continue; + } + + break; } - return {}; + return value; } };