Skip to content

fix: canonicalize keywords and whitespaces #66

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Snehil-Shah wants to merge 2 commits into
base: main
Choose a base branch
from
Open

fix: canonicalize keywords and whitespaces #66

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c0ad6b6
fix: canonicalize keywords
Snehil-Shah Aug 29, 2026
30eaa80
fix: canonicalize whitespaces
Snehil-Shah Aug 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ TEST_SRCS = $(TEST_DIR)/test_main.cpp \
$(TEST_DIR)/test_update.cpp \
$(TEST_DIR)/test_delete.cpp \
$(TEST_DIR)/test_compound.cpp \
$(TEST_DIR)/test_canonicalization.cpp \
$(TEST_DIR)/test_digest.cpp \
$(TEST_DIR)/test_misc_stmts.cpp \
$(TEST_DIR)/test_value.cpp \
Expand Down
4 changes: 4 additions & 0 deletions include/sql_parser/arena.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,10 @@ class Arena {

StringRef allocate_string(const char* src, uint32_t len);

StringRef allocate_upper(StringRef text);

StringRef allocate_lower(StringRef text);

void reset();

size_t bytes_used() const;
Expand Down
21 changes: 17 additions & 4 deletions include/sql_parser/compound_query_parser.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,10 @@ class CompoundQueryParser {
AstNode* lock = make_node(arena_, NodeType::NODE_LOCKING_CLAUSE);
if (lock) {
Token strength = tok_.next_token();
lock->add_child(make_node(arena_, NodeType::NODE_IDENTIFIER, strength.text));
lock->add_child(make_node(arena_, NodeType::NODE_IDENTIFIER,
strength.type == TokenType::TK_UPDATE ? StringRef{"UPDATE", 6}
: strength.type == TokenType::TK_SHARE ? StringRef{"SHARE", 5}
: strength.text));
result->add_child(lock);
}
}
Expand Down Expand Up @@ -124,7 +127,11 @@ class CompoundQueryParser {

// Consume the set operator
tok_.skip();
StringRef op_text = t.text;
StringRef op_text =
t.type == TokenType::TK_UNION ? StringRef{"UNION", 5}
: t.type == TokenType::TK_INTERSECT ? StringRef{"INTERSECT", 9}
: t.type == TokenType::TK_EXCEPT ? StringRef{"EXCEPT", 6}
: t.text;

// Check for optional ALL
uint16_t flags = 0;
Expand Down Expand Up @@ -226,7 +233,11 @@ class CompoundQueryParser {
if (prec == 0 || prec <= min_prec) break;

tok_.skip();
StringRef op_text = t.text;
StringRef op_text =
t.type == TokenType::TK_UNION ? StringRef{"UNION", 5}
: t.type == TokenType::TK_INTERSECT ? StringRef{"INTERSECT", 9}
: t.type == TokenType::TK_EXCEPT ? StringRef{"EXCEPT", 6}
: t.text;

uint16_t flags = 0;
if (tok_.peek().type == TokenType::TK_ALL) {
Expand Down Expand Up @@ -278,7 +289,9 @@ class CompoundQueryParser {
Token dir = tok_.peek();
if (dir.type == TokenType::TK_ASC || dir.type == TokenType::TK_DESC) {
tok_.skip();
item->add_child(make_node(arena_, NodeType::NODE_IDENTIFIER, dir.text));
item->add_child(make_node(arena_, NodeType::NODE_IDENTIFIER,
dir.type == TokenType::TK_ASC ? StringRef{"ASC", 3}
: StringRef{"DESC", 4}));
}

order_by->add_child(item);
Expand Down
4 changes: 3 additions & 1 deletion include/sql_parser/delete_parser.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -294,7 +294,9 @@ class DeleteParser {
Token dir = tok_.peek();
if (dir.type == TokenType::TK_ASC || dir.type == TokenType::TK_DESC) {
tok_.skip();
item->add_child(make_node(arena_, NodeType::NODE_IDENTIFIER, dir.text));
item->add_child(make_node(arena_, NodeType::NODE_IDENTIFIER,
dir.type == TokenType::TK_ASC ? StringRef{"ASC", 3}
: StringRef{"DESC", 4}));
}

order_by->add_child(item);
Expand Down
99 changes: 74 additions & 25 deletions include/sql_parser/expression_parser.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
#include "sql_parser/ast.h"
#include "sql_parser/arena.h"
#include "sql_parser/user_variable.h"
#include <cstring>

namespace sql_parser {

Expand Down Expand Up @@ -41,6 +42,24 @@ using SubqueryParseCallback = AstNode*(*)(Tokenizer<D>&, Arena&);
template <Dialect D>
class ExpressionParser {
public:
// Keyword operators are stored under their canonical spelling.
static StringRef canonical_op(const Token& op) {
switch (op.type) {
case TokenType::TK_AND: return StringRef{"AND", 3};
case TokenType::TK_OR: return StringRef{"OR", 2};
case TokenType::TK_XOR: return StringRef{"XOR", 3};
case TokenType::TK_NOT: return StringRef{"NOT", 3};
case TokenType::TK_IS: return StringRef{"IS", 2};
case TokenType::TK_IN: return StringRef{"IN", 2};
case TokenType::TK_LIKE: return StringRef{"LIKE", 4};
case TokenType::TK_REGEXP: return StringRef{"REGEXP", 6};
case TokenType::TK_DIV: return StringRef{"DIV", 3};
case TokenType::TK_MOD: return StringRef{"MOD", 3};
case TokenType::TK_BETWEEN: return StringRef{"BETWEEN", 7};
default: return op.text;
}
}

ExpressionParser(Tokenizer<D>& tokenizer, Arena& arena)
: tok_(tokenizer), arena_(arena) {}

Expand All @@ -65,6 +84,24 @@ class ExpressionParser {
}

private:
// Build canonicalized "@name", "@@name" or "@@scope.name" in the arena.
StringRef build_at_identifier_(const char* prefix, const Token& name,
const Token* qualified) {
const size_t prefix_len = std::strlen(prefix);
size_t total = prefix_len + name.text.len;
if (qualified) total += 1 + qualified->text.len;
char* buf = static_cast<char*>(arena_.allocate(total));
if (!buf) return StringRef{nullptr, 0};
size_t off = 0;
std::memcpy(buf + off, prefix, prefix_len); off += prefix_len;
std::memcpy(buf + off, name.text.ptr, name.text.len); off += name.text.len;
if (qualified) {
buf[off++] = '.';
std::memcpy(buf + off, qualified->text.ptr, qualified->text.len);
}
return StringRef{buf, static_cast<uint32_t>(total)};
}

Tokenizer<D>& tok_;
Arena& arena_;
SubqueryParseCallback<D> subquery_cb_ = nullptr;
Expand Down Expand Up @@ -120,7 +157,14 @@ class ExpressionParser {
}
case TokenType::TK_NULL: {
tok_.skip();
return make_node_from_token(arena_, NodeType::NODE_LITERAL_NULL, t);
{
// Keep the source span lossless, but store the keyword under
// its canonical spelling.
AstNode* null_node =
make_node_from_token(arena_, NodeType::NODE_LITERAL_NULL, t);
if (null_node) null_node->set_value(StringRef{"NULL", 4});
return null_node;
}
}
case TokenType::TK_TRUE:
case TokenType::TK_FALSE: {
Expand Down Expand Up @@ -153,11 +197,8 @@ class ExpressionParser {
// User variable: @name
tok_.skip();
Token name = tok_.next_token();
// Build @name as a single COLUMN_REF with combined text
// value_ptr points to @ in original input, len covers @name
StringRef full{t.text.ptr,
static_cast<uint32_t>((name.text.ptr + name.text.len) - t.text.ptr)};
return make_node(arena_, NodeType::NODE_COLUMN_REF, full);
return make_node(arena_, NodeType::NODE_COLUMN_REF,
build_at_identifier_("@", name, nullptr));
}
case TokenType::TK_USER_VARIABLE: {
tok_.skip();
Expand All @@ -167,26 +208,23 @@ class ExpressionParser {
// System variable: @@name or @@scope.name
tok_.skip();
Token name = tok_.next_token();
StringRef full{t.text.ptr,
static_cast<uint32_t>((name.text.ptr + name.text.len) - t.text.ptr)};
AstNode* node = make_node(arena_, NodeType::NODE_COLUMN_REF, full);
// Check for @@scope.name
Token qualified;
const Token* qualified_ptr = nullptr;
if (tok_.peek().type == TokenType::TK_DOT) {
tok_.skip();
Token var_name = tok_.next_token();
full = StringRef{t.text.ptr,
static_cast<uint32_t>((var_name.text.ptr + var_name.text.len) - t.text.ptr)};
node->value_ptr = full.ptr;
node->value_len = full.len;
qualified = tok_.next_token();
qualified_ptr = &qualified;
}
return node;
return make_node(arena_, NodeType::NODE_COLUMN_REF,
build_at_identifier_("@@", name, qualified_ptr));
}
case TokenType::TK_MINUS: {
// Unary minus
tok_.skip();
AstNode* operand = parse(Precedence::UNARY);
if (!operand) return nullptr;
AstNode* node = make_node(arena_, NodeType::NODE_UNARY_OP, t.text);
AstNode* node = make_node(arena_, NodeType::NODE_UNARY_OP, canonical_op(t));
set_span_through_node_(node, t.source, operand);
node->add_child(operand);
return node;
Expand All @@ -196,7 +234,7 @@ class ExpressionParser {
tok_.skip();
AstNode* operand = parse(Precedence::UNARY);
if (!operand) return nullptr;
AstNode* node = make_node(arena_, NodeType::NODE_UNARY_OP, t.text);
AstNode* node = make_node(arena_, NodeType::NODE_UNARY_OP, canonical_op(t));
set_span_through_node_(node, t.source, operand);
node->add_child(operand);
return node;
Expand All @@ -205,7 +243,7 @@ class ExpressionParser {
tok_.skip();
AstNode* operand = parse(Precedence::NOT);
if (!operand) return nullptr;
AstNode* node = make_node(arena_, NodeType::NODE_UNARY_OP, t.text);
AstNode* node = make_node(arena_, NodeType::NODE_UNARY_OP, canonical_op(t));
set_span_through_node_(node, t.source, operand);
node->add_child(operand);
return node;
Expand Down Expand Up @@ -326,7 +364,16 @@ class ExpressionParser {
// Check for function call: name(
if (tok_.peek().type == TokenType::TK_LPAREN) {
tok_.skip(); // consume (
AstNode* func = make_node(arena_, NodeType::NODE_FUNCTION_CALL, name_token.text);
// Function names are case-insensitive, so store them under a canonical spelling.
StringRef func_name = name_token.text;
if constexpr (D == Dialect::MySQL) {
// MySQL folds all function names up:
func_name = arena_.allocate_upper(func_name);
} else if (!token_was_delimited_(name_token)) {
// PostgreSQL folds undelimited function names down:
func_name = arena_.allocate_lower(func_name);
}
AstNode* func = make_node(arena_, NodeType::NODE_FUNCTION_CALL, func_name);
// CAST uses `CAST(expr AS type)` rather than a comma-separated
// argument list. Model it as a function call so consumers can
// reject or handle the expression without leaving valid input
Expand Down Expand Up @@ -449,25 +496,25 @@ class ExpressionParser {
tok_.skip();
AstNode* in_node = parse_in(left);
// Wrap in NOT
AstNode* not_node = make_node(arena_, NodeType::NODE_UNARY_OP, op.text);
AstNode* not_node = make_node(arena_, NodeType::NODE_UNARY_OP, canonical_op(op));
not_node->add_child(in_node);
return not_node;
}
if (actual_op.type == TokenType::TK_BETWEEN) {
tok_.skip();
AstNode* between_node = parse_between(left);
AstNode* not_node = make_node(arena_, NodeType::NODE_UNARY_OP, op.text);
AstNode* not_node = make_node(arena_, NodeType::NODE_UNARY_OP, canonical_op(op));
not_node->add_child(between_node);
return not_node;
}
if (actual_op.type == TokenType::TK_LIKE ||
actual_op.type == TokenType::TK_REGEXP) {
tok_.skip();
AstNode* right = parse(prec);
AstNode* like_node = make_node(arena_, NodeType::NODE_BINARY_OP, actual_op.text);
AstNode* like_node = make_node(arena_, NodeType::NODE_BINARY_OP, canonical_op(actual_op));
like_node->add_child(left);
if (right) like_node->add_child(right);
AstNode* not_node = make_node(arena_, NodeType::NODE_UNARY_OP, op.text);
AstNode* not_node = make_node(arena_, NodeType::NODE_UNARY_OP, canonical_op(op));
not_node->add_child(like_node);
return not_node;
}
Expand Down Expand Up @@ -511,7 +558,7 @@ class ExpressionParser {
// Standard binary operator
AstNode* right = parse(prec);
if (!right) return left;
AstNode* node = make_node(arena_, NodeType::NODE_BINARY_OP, op.text);
AstNode* node = make_node(arena_, NodeType::NODE_BINARY_OP, canonical_op(op));
node->add_child(left);
node->add_child(right);
return node;
Expand Down Expand Up @@ -758,7 +805,9 @@ class ExpressionParser {
Token dir = tok_.peek();
if (dir.type == TokenType::TK_ASC || dir.type == TokenType::TK_DESC) {
tok_.skip();
item->add_child(make_node(arena_, NodeType::NODE_IDENTIFIER, dir.text));
item->add_child(make_node(arena_, NodeType::NODE_IDENTIFIER,
dir.type == TokenType::TK_ASC ? StringRef{"ASC", 3}
: StringRef{"DESC", 4}));
}
ord->add_child(item);
if (tok_.peek().type == TokenType::TK_COMMA) tok_.skip();
Expand Down
16 changes: 12 additions & 4 deletions include/sql_parser/select_parser.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,11 +123,14 @@ class SelectParser {
if (t.type == TokenType::TK_DISTINCT || t.type == TokenType::TK_ALL) {
if (!opts) opts = make_node(arena_, NodeType::NODE_SELECT_OPTIONS);
tok_.skip();
opts->add_child(make_node(arena_, NodeType::NODE_IDENTIFIER, t.text));
opts->add_child(make_node(arena_, NodeType::NODE_IDENTIFIER,
t.type == TokenType::TK_DISTINCT ? StringRef{"DISTINCT", 8}
: StringRef{"ALL", 3}));
} else if (t.type == TokenType::TK_SQL_CALC_FOUND_ROWS) {
if (!opts) opts = make_node(arena_, NodeType::NODE_SELECT_OPTIONS);
tok_.skip();
opts->add_child(make_node(arena_, NodeType::NODE_IDENTIFIER, t.text));
opts->add_child(make_node(arena_, NodeType::NODE_IDENTIFIER,
StringRef{"SQL_CALC_FOUND_ROWS", 19}));
} else {
break;
}
Expand Down Expand Up @@ -298,7 +301,9 @@ class SelectParser {
Token dir = tok_.peek();
if (dir.type == TokenType::TK_ASC || dir.type == TokenType::TK_DESC) {
tok_.skip();
item->add_child(make_node(arena_, NodeType::NODE_IDENTIFIER, dir.text));
item->add_child(make_node(arena_, NodeType::NODE_IDENTIFIER,
dir.type == TokenType::TK_ASC ? StringRef{"ASC", 3}
: StringRef{"DESC", 4}));
}

order_by->add_child(item);
Expand Down Expand Up @@ -349,7 +354,10 @@ class SelectParser {

tok_.skip(); // consume FOR
Token strength = tok_.next_token(); // UPDATE or SHARE
lock->add_child(make_node(arena_, NodeType::NODE_IDENTIFIER, strength.text));
lock->add_child(make_node(arena_, NodeType::NODE_IDENTIFIER,
strength.type == TokenType::TK_UPDATE ? StringRef{"UPDATE", 6}
: strength.type == TokenType::TK_SHARE ? StringRef{"SHARE", 5}
: strength.text));

// Optional: OF table_list
if (tok_.peek().type == TokenType::TK_OF) {
Expand Down
30 changes: 23 additions & 7 deletions include/sql_parser/table_ref_parser.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,34 +109,50 @@ class TableRefParser {
}

// Parse a JOIN clause
// Build canonicalized "<A> <B> ..." in the arena.
StringRef build_join_type(const StringRef* parts, int count) {
if (count <= 0) return StringRef{nullptr, 0};
size_t total = static_cast<size_t>(count - 1);
for (int i = 0; i < count; ++i) total += parts[i].len;
char* buf = static_cast<char*>(arena_.allocate(total));
if (!buf) return StringRef{nullptr, 0};
size_t off = 0;
for (int i = 0; i < count; ++i) {
if (i) buf[off++] = ' ';
for (uint32_t j = 0; j < parts[i].len; ++j) {
const char c = parts[i].ptr[j];
buf[off++] = (c >= 'a' && c <= 'z') ? static_cast<char>(c - 32) : c;
}
}
return StringRef{buf, static_cast<uint32_t>(total)};
}

AstNode* parse_join(AstNode* /* left_ref */) {
AstNode* join = make_node(arena_, NodeType::NODE_JOIN_CLAUSE);
if (!join) return nullptr;

// Consume join type tokens
StringRef parts[8];
int part_count = 0;
Token t = tok_.peek();
StringRef join_type_start = t.text;
StringRef join_type_end = t.text;

// Optional: NATURAL, LEFT, RIGHT, FULL, INNER, OUTER, CROSS
while (t.type == TokenType::TK_NATURAL || t.type == TokenType::TK_LEFT ||
t.type == TokenType::TK_RIGHT || t.type == TokenType::TK_FULL ||
t.type == TokenType::TK_INNER || t.type == TokenType::TK_OUTER ||
t.type == TokenType::TK_CROSS) {
tok_.skip();
join_type_end = t.text;
if (part_count < 8) parts[part_count++] = t.text;
t = tok_.peek();
}

// Expect JOIN keyword
if (t.type == TokenType::TK_JOIN) {
join_type_end = t.text;
if (part_count < 8) parts[part_count++] = t.text;
tok_.skip();
}

// Set join type as value (covers the span from first modifier to JOIN)
StringRef join_type{join_type_start.ptr,
static_cast<uint32_t>((join_type_end.ptr + join_type_end.len) - join_type_start.ptr)};
StringRef join_type = build_join_type(parts, part_count);
join->value_ptr = join_type.ptr;
join->value_len = join_type.len;

Expand Down
4 changes: 3 additions & 1 deletion include/sql_parser/update_parser.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -238,7 +238,9 @@ class UpdateParser {
Token dir = tok_.peek();
if (dir.type == TokenType::TK_ASC || dir.type == TokenType::TK_DESC) {
tok_.skip();
item->add_child(make_node(arena_, NodeType::NODE_IDENTIFIER, dir.text));
item->add_child(make_node(arena_, NodeType::NODE_IDENTIFIER,
dir.type == TokenType::TK_ASC ? StringRef{"ASC", 3}
: StringRef{"DESC", 4}));
}

order_by->add_child(item);
Expand Down
Loading