From 3bc63b6d95db0995a639eb28915c815640456903 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 15 Apr 2026 22:24:29 +0300 Subject: [PATCH 01/11] add more sql --- README.md | 27 +++- ormin/queries.nim | 308 +++++++++++++++++++++++++++++----------- tests/model_postgre.sql | 5 + tests/model_sqlite.sql | 5 + tests/tcommon.nim | 39 +++++ tests/tfeature.nim | 54 ++++++- 6 files changed, 353 insertions(+), 85 deletions(-) diff --git a/README.md b/README.md index 1eb438b..454931b 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,6 @@ Features: TODO: -- Add support for UNION, INTERSECT and EXCEPT. -- Transactions. - Better support for complex nested queries. - Write mysql backend. @@ -47,7 +45,7 @@ let db {.global.} = open("localhost", "user", "password", "dbname") ## Query DSL -`query:` blocks are turned into prepared statements at compile time. Placeholders use `?` for Nim values and `%` for JSON values; Ormin chooses JSON instead of an ad-hoc variant type so your data can flow straight from/into `JsonNode` trees. `!!` splices vendor-specific SQL fragments. Typical clauses such as `where`, `join`, `orderby`, `groupby`, `limit` and `offset` are supported, and `returning` captures generated values (e.g. inserted IDs). Referring to columns from related tables can trigger **automatic join generation** based on foreign keys, reducing boilerplate joins. +`query:` blocks are turned into prepared statements at compile time. Placeholders use `?` for Nim values and `%` for JSON values; Ormin chooses JSON instead of an ad-hoc variant type so your data can flow straight from/into `JsonNode` trees. `!!` splices vendor-specific SQL fragments. Typical clauses such as `where`, `join`, `orderby`, `groupby`, `limit`, `offset`, `exists`, `distinct`, `union`/`intersect`/`except` and `returning` are supported. Referring to columns from related tables can trigger **automatic join generation** based on foreign keys, reducing boilerplate joins. Example snippets: @@ -75,6 +73,29 @@ let postsWithAuthors = query: join Author(name) where author.name == ?userName +# DISTINCT queries and COUNT(DISTINCT ...) +let authorIds = query: + `distinct` Post(author) +let authorCount = query: + select Post(count(distinct author)) + +# NULL predicates use `nil` or `null` +let unassigned = query: + select Ticket(id) + where assignee == nil + +# EXISTS / NOT EXISTS subqueries +let peopleWithPosts = query: + select Person(id) + where exists(select Post(id) where author == ?personId) + +# Set operations use function-call syntax +let mergedIds = query: + union( + select Person(id) where id <= 2, + select Person(id) where id >= 4 + ) + # Multiple joins with pagination let page = query: select Post(title) diff --git a/ormin/queries.nim b/ormin/queries.nim index e4e82ff..293c689 100644 --- a/ormin/queries.nim +++ b/ormin/queries.nim @@ -269,6 +269,95 @@ proc fmtTableList(tableNames: openArray[string]): string = if i > 0: result.add ", " result.add t +proc nodeName(n: NimNode): string {.compileTime.} = + case n.kind + of nnkIdent, nnkSym: + result = n.strVal + of nnkAccQuoted: + if n.len == 1: + result = nodeName(n[0]) + else: + result = "" + else: + result = "" + +proc isQueryClause(name: string): bool {.compileTime.} = + case name.toLowerAscii() + of "select", "distinct", "insert", "update", "replace", "delete", + "where", "join", "innerjoin", "outerjoin", "groupby", "orderby", + "having", "limit", "offset", "returning", "produce": + result = true + else: + result = false + +proc isSetOpName(name: string): bool {.compileTime.} = + case name.toLowerAscii() + of "union", "intersect", "except": + result = true + else: + result = false + +proc isSetOpCall(n: NimNode): bool {.compileTime.} = + n.kind == nnkCall and isSetOpName(nodeName(n[0])) + +proc isNullLiteral(n: NimNode): bool {.compileTime.} = + case n.kind + of nnkNilLit: + result = true + of nnkIdent, nnkSym: + result = cmpIgnoreCase(n.strVal, "null") == 0 + else: + result = false + +proc peelTrailingCommand(n: NimNode): tuple[core, tail: NimNode] {.compileTime.} = + if n.kind == nnkCommand and n.len == 2 and n[1].kind == nnkCommand and + nodeName(n[1][0]) == "on": + return (copyNimTree(n), newEmptyNode()) + + if n.kind == nnkCommand and n.len == 2 and n[1].kind == nnkCommand and + not isQueryClause(nodeName(n[0])) and not isSetOpName(nodeName(n[0])): + return (copyNimTree(n[0]), copyNimTree(n[1])) + + if (n.kind == nnkCommand and (isQueryClause(nodeName(n[0])) or isSetOpName(nodeName(n[0])))) or + isSetOpCall(n): + return (copyNimTree(n), newEmptyNode()) + + if n.len > 0: + let idx = n.len - 1 + let peeled = peelTrailingCommand(n[idx]) + if peeled.tail.kind != nnkEmpty: + result.core = copyNimTree(n) + result.core[idx] = peeled.core + result.tail = peeled.tail + return + + result = (copyNimTree(n), newEmptyNode()) + +proc flattenQueryCommands(n: NimNode; parts: var seq[NimNode]) {.compileTime.} = + case n.kind + of nnkStmtList: + for it in n: + flattenQueryCommands(it, parts) + of nnkCommand: + var cmd = copyNimTree(n) + if cmd.len >= 2: + let idx = cmd.len - 1 + let peeled = peelTrailingCommand(cmd[idx]) + cmd[idx] = peeled.core + parts.add cmd + if peeled.tail.kind != nnkEmpty: + flattenQueryCommands(peeled.tail, parts) + else: + parts.add cmd + else: + parts.add copyNimTree(n) + +proc queryh(n: NimNode; q: QueryBuilder) +proc queryAsString(q: QueryBuilder, n: NimNode): string +proc applyQueryNode(n: NimNode; q: QueryBuilder) +proc renderInlineQuery(n: NimNode; params: var Params; + qb: QueryBuilder): tuple[sql: string, typ: DbType] + proc lookupColumnInEnv(n: NimNode; q: var string; params: var Params; expected: DbType, qb: QueryBuilder): DbType = expectKind(n, nnkIdent) @@ -296,6 +385,9 @@ proc cond(n: NimNode; q: var string; params: var Params; if name == "_": q.add "*" result = DbType(kind: dbUnknown) + elif cmpIgnoreCase(name, "null") == 0: + q.add "NULL" + result = expected else: result = lookupColumnInEnv(n, q, params, expected, qb) of nnkDotExpr: @@ -321,6 +413,12 @@ proc cond(n: NimNode; q: var string; params: var Params; checkCompatible(a, b, n[i]) q.add ")" result = DbType(kind: dbSet) + of nnkNilLit: + q.add "NULL" + result = expected + of nnkDistinctTy: + q.add "distinct " + result = cond(n[0], q, params, expected, qb) of nnkStrLit, nnkRStrLit, nnkTripleStrLit: result = expected if result.kind == dbUnknown: @@ -357,20 +455,32 @@ proc cond(n: NimNode; q: var string; params: var Params; let b = cond(n[2], q, params, result, qb) checkBool b, n[2] of "<=", "<", ">=", ">", "==", "!=", "=~": - let env = qb.env - if env.len == 2: - qb.env = @[env[0]] - let a = cond(n[1], q, params, DbType(kind: dbUnknown), qb) - q.add ' ' - if op == "==": q.add equals - elif op == "!=": q.add nequals - elif op == "=~": q.add "like" - else: q.add op - q.add ' ' - if env.len == 2: - qb.env = @[env[1]] - let b = cond(n[2], q, params, a, qb) - checkCompatible a, b, n + if isNullLiteral(n[1]) or isNullLiteral(n[2]): + if op != "==" and op != "!=": + macros.error "NULL comparisons only support == and !=", n + if isNullLiteral(n[1]) and isNullLiteral(n[2]): + macros.error "NULL cannot be compared against NULL", n + let target = if isNullLiteral(n[1]): n[2] else: n[1] + discard cond(target, q, params, DbType(kind: dbUnknown), qb) + if op == "==": + q.add " is NULL" + else: + q.add " is not NULL" + else: + let env = qb.env + if env.len == 2: + qb.env = @[env[0]] + let a = cond(n[1], q, params, DbType(kind: dbUnknown), qb) + q.add ' ' + if op == "==": q.add equals + elif op == "!=": q.add nequals + elif op == "=~": q.add "like" + else: q.add op + q.add ' ' + if env.len == 2: + qb.env = @[env[1]] + let b = cond(n[2], q, params, a, qb) + checkCompatible a, b, n result = DbType(kind: dbBool) of "in", "notin": let a = cond(n[1], q, params, DbType(kind: dbUnknown), qb) @@ -442,7 +552,20 @@ proc cond(n: NimNode; q: var string; params: var Params; q.add ' ' result = cond(n[1], q, params, DbType(kind: dbUnknown), qb) of nnkCall: - let op = $n[0] + let op = nodeName(n[0]) + if isSetOpCall(n): + let subq = renderInlineQuery(n, params, qb) + q.add subq.sql + result = subq.typ + return + if op == "exists": + expectLen n, 2 + let subq = renderInlineQuery(n[1], params, qb) + q.add "exists (" + q.add subq.sql + q.add ")" + result = DbType(kind: dbBool) + return if op == "asc" or op == "desc": expectLen n, 2 result = cond(n[1], q, params, DbType(kind: dbUnknown), qb) @@ -481,65 +604,11 @@ proc cond(n: NimNode; q: var string; params: var Params; else: checkCompatible(result, t, x) q.add "\Lend" of nnkCommand: - # select subquery - if n.len == 2 and $n[0] == "select" and n[1].kind == nnkCall: - let call = n[1] - let tab = $call[0] - let tabIndex = tableNames.lookup(tab) - if tabIndex < 0: - macros.error "unknown table name: " & tab & " from: " & fmtTableList(tableNames), n[1][0] - else: - let alias = qb.getAlias(tabIndex) - var subenv = @[(tabIndex, alias)] - swap(qb.env, subenv) - var subselect = "select " - for i in 1.. 1: subselect.add ", " - discard cond(call[i], subselect, params, DbType(kind: dbUnknown), qb) - subselect.add " from " - escIdent(subselect, tab) - subselect.add " as " & alias - q.add subselect - swap(qb.env, subenv) - elif n.len == 2 and $n[0] == "select" and n[1].kind == nnkCommand: - result = DbType(kind: dbSet) - let cmd = n[1] - var subselect = "select " - var subenv = qb.env - if cmd.len >= 1 and cmd[0].kind == nnkCall: - let call = cmd[0] - let tab = $call[0] - let tabIndex = tableNames.lookup(tab) - if tabIndex < 0: - macros.error "unknown table name: " & tab & " from: " & fmtTableList(tableNames), n[1][0] - else: - let alias = qb.getAlias(tabIndex) - qb.env = @[(tabindex, alias)] - for i in 1.. 1: subselect.add ", " - discard cond(call[i], subselect, params, DbType(kind: dbUnknown), qb) - subselect.add " from " - escIdent(subselect, tab) - subselect.add " as " & alias - if cmd.len == 2: - if cmd[1].kind in nnkCallKinds and $cmd[1][0] == "where": - subselect.add " where " - discard cond(cmd[1][1], subselect, params, DbType(kind: dbBool), qb) - elif cmd[1].kind in nnkCallKinds and $cmd[1][0] == "groupby": - subselect.add " group by " - let hav = cmd[1][1] - if hav.kind in nnkCallKinds and $hav[1][0] == "having": - discard cond(hav[0], subselect, params, DbType(kind: dbBool), qb) - subselect.add " having " - discard cond(hav[1][1], subselect, params, DbType(kind: dbBool), qb) - else: - discard cond(hav, subselect, params, DbType(kind: dbBool), qb) - else: - macros.error "construct not supported in condition: " & treeRepr cmd, cmd - elif cmd.len >= 2: - macros.error "construct not supported in condition: " & treeRepr cmd, cmd - qb.env = subenv - q.add subselect + let head = nodeName(n[0]) + if head == "select" or head == "distinct": + let subq = renderInlineQuery(n, params, qb) + q.add subq.sql + result = subq.typ else: macros.error "construct not supported in condition: " & treeRepr n, n else: @@ -777,13 +846,18 @@ proc tableSel(n: NimNode; q: QueryBuilder) = proc queryh(n: NimNode; q: QueryBuilder) = expectKind n, nnkCommand - let kind = $n[0] + let kind = nodeName(n[0]) case kind of "select": q.kind = qkSelect q.head = "select " expectLen n, 2 tableSel(n[1], q) + of "distinct": + q.kind = qkSelect + q.head = "select distinct " + expectLen n, 2 + tableSel(n[1], q) of "insert": q.kind = qkInsert q.head = "insert into " @@ -952,6 +1026,82 @@ proc queryAsString(q: QueryBuilder, n: NimNode): string = when defined(debugOrminSql): macros.hint("Ormin SQL:\n" & $result, n) +proc sameReturnShape(a, b: NimNode): bool {.compileTime.} = + if a.len != b.len: + return false + for i in 0.. 1: a[i][1] else: a[i] + let bt = if b[i].kind == nnkIdentDefs and b[i].len > 1: b[i][1] else: b[i] + if repr(at) != repr(bt): + return false + result = true + +proc buildSetOpQuery(n: NimNode; q: QueryBuilder) {.compileTime.} = + if q.kind != qkNone or q.head.len > 0 or q.params.len > 0: + macros.error "set operations must form the whole query", n + if n.len < 3: + macros.error "set operations require at least two queries", n + + let op = nodeName(n[0]).toLowerAscii() + q.kind = qkSelect + q.singleRow = false + + for i in 1.. 1: + q.head.add "\L" & op & "\L" + if isSetOpCall(n[i]): + q.head.add "(" + q.head.add queryAsString(branch, n[i]) + q.head.add ")" + else: + q.head.add queryAsString(branch, n[i]) + + q.qmark = branch.qmark + for p in branch.params: + q.params.add p + + if i == 1: + q.retType = branch.retType + q.retNames = branch.retNames + q.retTypeIsJson = branch.retTypeIsJson + elif branch.retTypeIsJson != q.retTypeIsJson or + not sameReturnShape(branch.retType, q.retType): + macros.error "all set operation branches must return the same types", n[i] + +proc applyQueryNode(n: NimNode; q: QueryBuilder) = + if isSetOpCall(n): + buildSetOpQuery(n, q) + return + + var flattened: seq[NimNode] + flattenQueryCommands(n, flattened) + for part in flattened: + if isSetOpCall(part): + buildSetOpQuery(part, q) + elif part.kind == nnkCommand: + queryh(part, q) + else: + macros.error "illformed query", part + +proc renderInlineQuery(n: NimNode; params: var Params; + qb: QueryBuilder): tuple[sql: string, typ: DbType] = + var subq = newQueryBuilder() + subq.qmark = qb.qmark + applyQueryNode(n, subq) + if subq.kind notin {qkSelect, qkJoin}: + macros.error "subqueries require a select-style query", n + qb.qmark = subq.qmark + for p in subq.params: + params.add p + result.sql = queryAsString(subq, n) + result.typ = DbType(kind: dbSet) + proc newGlobalVar(name, typ: NimNode, value: NimNode): NimNode = result = newTree(nnkVarSection, newTree(nnkIdentDefs, newTree(nnkPragmaExpr, name, @@ -969,9 +1119,7 @@ proc queryImpl(q: QueryBuilder; body: NimNode; attempt, produceJson: bool): NimN expectMinLen body, 1 q.retTypeIsJson = produceJson - for b in body: - if b.kind == nnkCommand: queryh(b, q) - else: macros.error "illformed query", b + applyQueryNode(body, q) let sql = queryAsString(q, body) let prepStmt = genSym(nskLet) let res = genSym(nskVar) @@ -1192,9 +1340,7 @@ proc createRoutine(name, query: NimNode; k: NimNodeKind): NimNode = expectMinLen query, 1 var q = newQueryBuilder() - for b in query: - if b.kind == nnkCommand: queryh(b, q) - else: macros.error "illformed query", b + applyQueryNode(query, q) if q.kind notin {qkSelect, qkJoin}: macros.error "query must be a 'select' or 'join'", query let sql = queryAsString(q, query) diff --git a/tests/model_postgre.sql b/tests/model_postgre.sql index ac6c4e9..bef59ba 100644 --- a/tests/model_postgre.sql +++ b/tests/model_postgre.sql @@ -38,3 +38,8 @@ create table if not exists tb_composite_fk( fk2 integer not null, foreign key (fk1, fk2) references tb_composite_pk(pk1, pk2) ); + +create table if not exists tb_nullable( + id integer primary key, + note text null +); diff --git a/tests/model_sqlite.sql b/tests/model_sqlite.sql index 8b4ecee..b6f4c0f 100644 --- a/tests/model_sqlite.sql +++ b/tests/model_sqlite.sql @@ -41,3 +41,8 @@ create table if not exists tb_blob( id integer primary key, typblob blob not null ); + +create table if not exists tb_nullable( + id integer primary key, + note text null +); diff --git a/tests/tcommon.nim b/tests/tcommon.nim index e147a37..629fd35 100644 --- a/tests/tcommon.nim +++ b/tests/tcommon.nim @@ -412,3 +412,42 @@ suite "composite_pk": select tb_composite_pk(pk1, pk2, message) produce json check res == %*cps.mapIt(%*{"pk1": it[0], "pk2": it[1], "message": it[2]}) + + +suite "nullable": + setup: + db.dropTable(sqlFile, "tb_nullable") + db.createTable(sqlFile, "tb_nullable") + query: + insert tb_nullable(id = 1, note = nil) + query: + insert tb_nullable(id = 2, note = "hello") + + test "where_null": + let res = query: + select tb_nullable(id) + where note == nil + check res == @[1] + + test "where_not_null": + let res = query: + select tb_nullable(id) + where note != null + check res == @[2] + + test "update_to_null": + query: + update tb_nullable(note = null) + where id == 2 + let res = query: + select tb_nullable(id) + where note == nil + check res == @[1, 2] + + test "json_null": + let res = query: + select tb_nullable(id, note) + orderby id + produce json + check res[0]["note"].kind == JNull + check res[1]["note"].getStr == "hello" diff --git a/tests/tfeature.nim b/tests/tfeature.nim index b25f72a..25591fa 100644 --- a/tests/tfeature.nim +++ b/tests/tfeature.nim @@ -240,6 +240,18 @@ suite "query": where id notin ?id1 .. ?id2 check res == persondata.filterIt(it.id < id1 or it.id > id2) + test "distinct": + var res = query: + `distinct` post(author) + res.sort() + let expected = postdata.mapIt(it.author).deduplicate().sortedByIt(it) + check res == expected + + test "count_distinct": + let res = query: + select post(count(distinct author)) + check res == @[postdata.mapIt(it.author).deduplicate().len] + test "limit": let id = 1 let res = query: @@ -400,6 +412,47 @@ suite "query": where id in ( select post(author) groupby author having author == ?id) check res == @[id] + + test "exists": + let authorid = postdata[0].author + let res = query: + select person(id) + where exists(select post(id) where author == ?authorid) + check res.sortedByIt(it) == persondata.mapIt(it.id) + + test "not_exists": + let missingAuthor = personcount + postcount + let res = query: + select person(id) + where not exists(select post(id) where author == ?missingAuthor) + check res.sortedByIt(it) == persondata.mapIt(it.id) + + test "union": + var res = query: + union( + select person(id) where id <= 2, + select person(id) where id >= 4 + ) + res.sort() + check res == @[1, 2, 4, 5] + + test "intersect": + var res = query: + intersect( + select person(id) where id <= 3, + select person(id) where id >= 2 + ) + res.sort() + check res == @[2, 3] + + test "except": + var res = query: + `except`( + select person(id) where id <= 4, + select person(id) where id in {2, 3} + ) + res.sort() + check res == @[1, 4] test "complex_query_subquery_having": let @@ -622,4 +675,3 @@ suite "query": check rows[1].id == 6 check rows[1].views == 77 check rows[1].name == "" - From da994f499025267c86ba34d1acbdfbb19c7000f3 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Wed, 15 Apr 2026 22:50:12 +0300 Subject: [PATCH 02/11] select distinct stuff --- README.md | 6 +----- ormin/queries.nim | 13 +++++++++++-- tests/tfeature.nim | 2 +- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 6a8a06f..41a7984 100644 --- a/README.md +++ b/README.md @@ -17,10 +17,6 @@ Features: TODO: -<<<<<<< HEAD -======= -- Add support for UNION, INTERSECT and EXCEPT. ->>>>>>> upstream/master - Better support for complex nested queries. - Write mysql backend. @@ -106,7 +102,7 @@ let postsWithAuthors = query: # DISTINCT queries and COUNT(DISTINCT ...) let authorIds = query: - `distinct` Post(author) + selectDistinct(Post(author)) let authorCount = query: select Post(count(distinct author)) diff --git a/ormin/queries.nim b/ormin/queries.nim index 293c689..d5fdfcc 100644 --- a/ormin/queries.nim +++ b/ormin/queries.nim @@ -283,7 +283,7 @@ proc nodeName(n: NimNode): string {.compileTime.} = proc isQueryClause(name: string): bool {.compileTime.} = case name.toLowerAscii() - of "select", "distinct", "insert", "update", "replace", "delete", + of "select", "selectdistinct", "insert", "update", "replace", "delete", "where", "join", "innerjoin", "outerjoin", "groupby", "orderby", "having", "limit", "offset", "returning", "produce": result = true @@ -338,6 +338,15 @@ proc flattenQueryCommands(n: NimNode; parts: var seq[NimNode]) {.compileTime.} = of nnkStmtList: for it in n: flattenQueryCommands(it, parts) + of nnkCall: + let name = nodeName(n[0]) + if isQueryClause(name): + var cmd = newNimNode(nnkCommand) + for it in n: + cmd.add copyNimTree(it) + parts.add cmd + else: + parts.add copyNimTree(n) of nnkCommand: var cmd = copyNimTree(n) if cmd.len >= 2: @@ -853,7 +862,7 @@ proc queryh(n: NimNode; q: QueryBuilder) = q.head = "select " expectLen n, 2 tableSel(n[1], q) - of "distinct": + of "selectDistinct": q.kind = qkSelect q.head = "select distinct " expectLen n, 2 diff --git a/tests/tfeature.nim b/tests/tfeature.nim index 25591fa..a5388bc 100644 --- a/tests/tfeature.nim +++ b/tests/tfeature.nim @@ -242,7 +242,7 @@ suite "query": test "distinct": var res = query: - `distinct` post(author) + selectDistinct(post(author)) res.sort() let expected = postdata.mapIt(it.author).deduplicate().sortedByIt(it) check res == expected From bcb4243440f80929d79767fbd55032b8895ade49 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 16 Apr 2026 14:45:33 +0300 Subject: [PATCH 03/11] select distinct stuff --- README.md | 2 +- ormin/queries.nim | 11 ++++++++--- tests/tfeature.nim | 2 +- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 41a7984..244509c 100644 --- a/README.md +++ b/README.md @@ -102,7 +102,7 @@ let postsWithAuthors = query: # DISTINCT queries and COUNT(DISTINCT ...) let authorIds = query: - selectDistinct(Post(author)) + select `distinct` Post(author) let authorCount = query: select Post(count(distinct author)) diff --git a/ormin/queries.nim b/ormin/queries.nim index d5fdfcc..3f99d19 100644 --- a/ormin/queries.nim +++ b/ormin/queries.nim @@ -283,7 +283,7 @@ proc nodeName(n: NimNode): string {.compileTime.} = proc isQueryClause(name: string): bool {.compileTime.} = case name.toLowerAscii() - of "select", "selectdistinct", "insert", "update", "replace", "delete", + of "select", "distinct", "insert", "update", "replace", "delete", "where", "join", "innerjoin", "outerjoin", "groupby", "orderby", "having", "limit", "offset", "returning", "produce": result = true @@ -861,8 +861,13 @@ proc queryh(n: NimNode; q: QueryBuilder) = q.kind = qkSelect q.head = "select " expectLen n, 2 - tableSel(n[1], q) - of "selectDistinct": + if n[1].kind == nnkCommand and nodeName(n[1][0]) == "distinct": + expectLen n[1], 2 + q.head = "select distinct " + tableSel(n[1][1], q) + else: + tableSel(n[1], q) + of "distinct": q.kind = qkSelect q.head = "select distinct " expectLen n, 2 diff --git a/tests/tfeature.nim b/tests/tfeature.nim index a5388bc..fdb1ca2 100644 --- a/tests/tfeature.nim +++ b/tests/tfeature.nim @@ -242,7 +242,7 @@ suite "query": test "distinct": var res = query: - selectDistinct(post(author)) + select `distinct` post(author) res.sort() let expected = postdata.mapIt(it.author).deduplicate().sortedByIt(it) check res == expected From 8c42bc69cac26f90f49451177269e0563710ed41 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 16 Apr 2026 14:53:23 +0300 Subject: [PATCH 04/11] select distinct stuff --- README.md | 9 +++--- ormin/queries.nim | 73 +++++++++++++++++++++++++++++++++++++--------- tests/tfeature.nim | 21 +++++++++++++ 3 files changed, 84 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 244509c..b19edbd 100644 --- a/README.md +++ b/README.md @@ -116,12 +116,11 @@ let peopleWithPosts = query: select Person(id) where exists(select Post(id) where author == ?personId) -# Set operations use function-call syntax +# Set operations can be written inline between select queries let mergedIds = query: - union( - select Person(id) where id <= 2, - select Person(id) where id >= 4 - ) + select Person(id) where id <= 2 + union + select Person(id) where id >= 4 # Multiple joins with pagination let page = query: diff --git a/ormin/queries.nim b/ormin/queries.nim index 3f99d19..4ccdbad 100644 --- a/ormin/queries.nim +++ b/ormin/queries.nim @@ -1050,43 +1050,52 @@ proc sameReturnShape(a, b: NimNode): bool {.compileTime.} = return false result = true -proc buildSetOpQuery(n: NimNode; q: QueryBuilder) {.compileTime.} = +proc buildSetOpQueryParts(op: string; branches: openArray[NimNode]; + q: QueryBuilder; lineInfo: NimNode) {.compileTime.} = if q.kind != qkNone or q.head.len > 0 or q.params.len > 0: - macros.error "set operations must form the whole query", n - if n.len < 3: - macros.error "set operations require at least two queries", n + macros.error "set operations must form the whole query", lineInfo + if branches.len < 2: + macros.error "set operations require at least two queries", lineInfo - let op = nodeName(n[0]).toLowerAscii() q.kind = qkSelect q.singleRow = false - for i in 1.. 1: + if i > 0: q.head.add "\L" & op & "\L" - if isSetOpCall(n[i]): + if isSetOpCall(branchNode): q.head.add "(" - q.head.add queryAsString(branch, n[i]) + q.head.add queryAsString(branch, branchNode) q.head.add ")" else: - q.head.add queryAsString(branch, n[i]) + q.head.add queryAsString(branch, branchNode) q.qmark = branch.qmark for p in branch.params: q.params.add p - if i == 1: + if i == 0: q.retType = branch.retType q.retNames = branch.retNames q.retTypeIsJson = branch.retTypeIsJson elif branch.retTypeIsJson != q.retTypeIsJson or not sameReturnShape(branch.retType, q.retType): - macros.error "all set operation branches must return the same types", n[i] + macros.error "all set operation branches must return the same types", branchNode + +proc buildSetOpQuery(n: NimNode; q: QueryBuilder) {.compileTime.} = + var branches: seq[NimNode] = @[] + for i in 1..= 4 + res.sort() + check res == @[1, 2, 4, 5] + + res = query: union( select person(id) where id <= 2, select person(id) where id >= 4 @@ -438,6 +445,13 @@ suite "query": test "intersect": var res = query: + select person(id) where id <= 3 + intersect + select person(id) where id >= 2 + res.sort() + check res == @[2, 3] + + res = query: intersect( select person(id) where id <= 3, select person(id) where id >= 2 @@ -447,6 +461,13 @@ suite "query": test "except": var res = query: + select person(id) where id <= 4 + `except` + select person(id) where id in {2, 3} + res.sort() + check res == @[1, 4] + + res = query: `except`( select person(id) where id <= 4, select person(id) where id in {2, 3} From c61f94aa2ec59aff89c8d0588c497e0dc849779d Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 16 Apr 2026 15:06:07 +0300 Subject: [PATCH 05/11] bump version --- ormin.nimble | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ormin.nimble b/ormin.nimble index 950a5e0..ba9c831 100644 --- a/ormin.nimble +++ b/ormin.nimble @@ -1,6 +1,6 @@ # Package -version = "0.6.0" +version = "0.7.0" author = "Araq" description = "Prepared SQL statement generator. A lightweight ORM." license = "MIT" From 403d27d715989d8e5ef5ccd09c52b1de613045b3 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 16 Apr 2026 15:28:09 +0300 Subject: [PATCH 06/11] add cte support --- README.md | 7 +- ormin/queries.nim | 201 ++++++++++++++++++++++++++++++++++++--------- tests/tfeature.nim | 21 +++++ 3 files changed, 189 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index b19edbd..716f3f6 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ let db {.global.} = open("localhost", "user", "password", "dbname") ## Query DSL -`query:` blocks are turned into prepared statements at compile time. Placeholders use `?` for Nim values and `%` for JSON values; Ormin chooses JSON instead of an ad-hoc variant type so your data can flow straight from/into `JsonNode` trees. `!!` splices vendor-specific SQL fragments. Typical clauses such as `where`, `join`, `orderby`, `groupby`, `limit`, `offset`, `exists`, `distinct`, `union`/`intersect`/`except` and `returning` are supported. Referring to columns from related tables can trigger **automatic join generation** based on foreign keys, reducing boilerplate joins. +`query:` blocks are turned into prepared statements at compile time. Placeholders use `?` for Nim values and `%` for JSON values; Ormin chooses JSON instead of an ad-hoc variant type so your data can flow straight from/into `JsonNode` trees. `!!` splices vendor-specific SQL fragments. Typical clauses such as `with`, `where`, `join`, `orderby`, `groupby`, `limit`, `offset`, `exists`, `distinct`, `union`/`intersect`/`except` and `returning` are supported. Referring to columns from related tables can trigger **automatic join generation** based on foreign keys, reducing boilerplate joins. Example snippets: @@ -116,6 +116,11 @@ let peopleWithPosts = query: select Person(id) where exists(select Post(id) where author == ?personId) +# CTEs use with cteName(select ...) +let recentAuthors = query: + with recent(select Post(id, author) where id <= 3) + select recent(author) + # Set operations can be written inline between select queries let mergedIds = query: select Person(id) where id <= 2 diff --git a/ormin/queries.nim b/ormin/queries.nim index 4ccdbad..69bf139 100644 --- a/ormin/queries.nim +++ b/ormin/queries.nim @@ -22,6 +22,15 @@ type arity: int # -1 for 'varargs' typ: DbTypeKind # if dbUnknown, use type of the last argument + CteColumn = object + name: string + typ: DbType + + CteDef = object + name: string + sql: string + cols: seq[CteColumn] + var functions {.compileTime.} = @[ Function(name: "count", arity: 1, typ: dbInt), @@ -65,7 +74,9 @@ proc typeNodeToDbKind(n: NimNode): DbTypeKind {.compileTime.} = n.strVal else: $n - return dbTypFromName(name) + result = dbTypFromName(name) + if result == dbUnknown and name.len > 4 and name.endsWith("Type"): + result = dbTypFromName(name[0..^5]) proc registerImportSqlFunction(name: string; arity: int; typ: DbTypeKind) {.compileTime.} = ## Adds or updates a Function descriptor for vendor specific SQL routines. @@ -128,6 +139,8 @@ type head, fromm, join, values, where, groupby, having, orderby: string limit, offset, returning: string env: Env + ctes: seq[CteDef] + cteBase: int kind: QueryKind retType: NimNode singleRow, retTypeIsJson: bool @@ -165,7 +178,7 @@ proc newQueryBuilder(): QueryBuilder {.compileTime.} = QueryBuilder(head: "", fromm: "", join: "", values: "", where: "", groupby: "", having: "", orderby: "", limit: "", offset: "", returning: "", - env: @[], kind: qkNone, params: @[], + env: @[], ctes: @[], cteBase: 0, kind: qkNone, params: @[], retType: newNimNode(nnkTupleTy), singleRow: false, retTypeIsJson: false, retNames: @[], coln: 0, qmark: 0, aliasGen: 1, colAliases: @[], @@ -182,27 +195,57 @@ proc placeholder(q: QueryBuilder): string = else: result = "?" -proc lookup(table, attr: string; env: Env; alias: var string): DbType = - var candidate = -1 - for i, m in attributes: - if cmpIgnoreCase(m.name, attr) == 0: - var inScope = false - for e in env: - if e[0] == m.tabIndex: +proc cteEnvIndex(i: int): int {.compileTime.} = tableNames.len + i +proc isCteEnvIndex(i: int): bool {.compileTime.} = i >= tableNames.len +proc fromCteEnvIndex(i: int): int {.compileTime.} = i - tableNames.len + +proc lookupCte(ctes: openArray[CteDef]; name: string): int {.compileTime.} = + result = -1 + for i, cte in ctes: + if cmpIgnoreCase(cte.name, name) == 0: + return i + +proc sourceName(q: QueryBuilder; source: int): string {.compileTime.} = + if isCteEnvIndex(source): + result = q.ctes[fromCteEnvIndex(source)].name + else: + result = tableNames[source] + +proc sourceColumns(q: QueryBuilder; source: int): seq[CteColumn] {.compileTime.} = + if isCteEnvIndex(source): + result = q.ctes[fromCteEnvIndex(source)].cols + else: + for a in attributes: + if a.tabIndex == source: + result.add CteColumn(name: a.name, typ: DbType(kind: a.typ)) + +proc sourceLookup(q: QueryBuilder; table: string): int {.compileTime.} = + for i, t in tableNames: + if cmpIgnoreCase(t, table) == 0: + return i + let cteIdx = lookupCte(q.ctes, table) + if cteIdx >= 0: + return cteEnvIndex(cteIdx) + result = -1 + +proc lookup(table, attr: string; qb: QueryBuilder; alias: var string): DbType = + var found = false + var foundSource = -1 + for e in qb.env: + if table.len == 0 or cmpIgnoreCase(sourceName(qb, e[0]), table) == 0: + for col in sourceColumns(qb, e[0]): + if cmpIgnoreCase(col.name, attr) == 0: + if found: + if foundSource != e[0] or alias != e[1]: + return DbType(kind: dbUnknown) + found = true + foundSource = e[0] alias = e[1] - inScope = true - break - if (inScope and table.len == 0) or - cmpIgnoreCase(tableNames[m.tabIndex], table) == 0: - # ambiguous match? - if candidate < 0: candidate = i - else: return DbType(kind: dbUnknown) - result = if candidate < 0: DbType(kind: dbUnknown) - else: DbType(kind: attributes[candidate].typ) - -proc lookup(table, attr: string; env: Env): DbType = + result = col.typ + +proc lookup(table, attr: string; qb: QueryBuilder): DbType = var alias: string - result = lookup(table, attr, env, alias) + result = lookup(table, attr, qb, alias) proc lookup(table: openArray[string], name: string): int = result = -1 @@ -283,7 +326,7 @@ proc nodeName(n: NimNode): string {.compileTime.} = proc isQueryClause(name: string): bool {.compileTime.} = case name.toLowerAscii() - of "select", "distinct", "insert", "update", "replace", "delete", + of "with", "select", "distinct", "insert", "update", "replace", "delete", "where", "join", "innerjoin", "outerjoin", "groupby", "orderby", "having", "limit", "offset", "returning", "produce": result = true @@ -377,7 +420,7 @@ proc lookupColumnInEnv(n: NimNode; q: var string; params: var Params; result = a[1] break checkAliases var alias: string - result = lookup("", name, qb.env, alias) + result = lookup("", name, qb, alias) if result.kind == dbUnknown: macros.error "unknown column name: " & name, n elif qb.kind in {qkSelect, qkJoin}: @@ -405,7 +448,7 @@ proc cond(n: NimNode; q: var string; params: var Params; escIdent(q, t) q.add '.' escIdent(q, a) - result = lookup(t, a, qb.env) + result = lookup(t, a, qb) of nnkPar, nnkStmtListExpr: if n.len == 1: q.add "(" @@ -731,18 +774,18 @@ proc selectAll(q: QueryBuilder; tabIndex: int; arg, lineInfo: NimNode) = template field(): untyped = fieldImpl(q, arg, a.name) case q.kind of qkSelect, qkJoin: - for a in attributes: - if a.tabIndex == tabIndex: + for a in sourceColumns(q, tabIndex): if q.coln > 0: q.head.add ", " inc q.coln - let t = a.typ - q.retType.add nnkIdentDefs.newTree(newIdentNode(a.name), toNimType(t), newEmptyNode()) + q.retType.add nnkIdentDefs.newTree(newIdentNode(a.name), toNimType(a.typ), newEmptyNode()) q.retNames.add a.name doAssert q.env.len > 0 q.head.add q.env[^1][1] q.head.add '.' escIdent(q.head, a.name) of qkInsert, qkInsertReturning, qkReplace: + if isCteEnvIndex(tabIndex): + macros.error "cannot insert into a CTE", lineInfo for a in attributes: # we do not set the primary key: if a.tabIndex == tabIndex and a.key != 1: @@ -753,6 +796,8 @@ proc selectAll(q: QueryBuilder; tabIndex: int; arg, lineInfo: NimNode) = if q.values.len > 0: q.values.add ", " q.values.add placeholder(q) of qkUpdate: + if isCteEnvIndex(tabIndex): + macros.error "cannot update a CTE", lineInfo for a in attributes: if a.tabIndex == tabIndex and a.key != 1: if q.coln > 0: q.head.add ", " @@ -769,11 +814,19 @@ proc tableSel(n: NimNode; q: QueryBuilder) = if n.kind == nnkCall and q.kind != qkDelete: let call = n let tab = $call[0] - let tabIndex = tableNames.lookup(tab) + let tabIndex = sourceLookup(q, tab) if tabIndex < 0: macros.error "unknown table name: " & tab & " from: " & fmtTableList(tableNames), n return - let alias = q.getAlias(tabIndex) + let alias = + if q.kind == qkJoin and q.env.len > 0 and q.env[^1][0] == tabIndex: + q.env[^1][1] + elif isCteEnvIndex(tabIndex): + let a = tab.toLowerAscii() & $q.aliasGen + inc q.aliasGen + a + else: + q.getAlias(tabIndex) if q.kind == qkSelect: escIdent(q.fromm, tab) q.fromm.add " as " & alias @@ -782,7 +835,8 @@ proc tableSel(n: NimNode; q: QueryBuilder) = if q.kind == qkUpdate: q.head.add " set " elif q.kind notin {qkSelect, qkJoin}: q.head.add "(" - q.env.add((tabindex, alias)) + if q.env.len == 0 or q.env[^1] != (tabIndex, alias): + q.env.add((tabindex, alias)) for i in 1.. 0: q.head.add ", " inc q.coln - let t = cond(col, q.head, q.params, DbType(kind: dbUnknown), q) + let t = + if col.kind in {nnkIdent, nnkSym}: + let colname = $col + var typ = DbType(kind: dbUnknown) + for srcCol in sourceColumns(q, tabIndex): + if cmpIgnoreCase(srcCol.name, colname) == 0: + typ = srcCol.typ + break + if typ.kind == dbUnknown: + macros.error "unknown column name: " & colname, col + q.head.add q.env[^1][1] + q.head.add '.' + escIdent(q.head, colname) + typ + else: + cond(col, q.head, q.params, DbType(kind: dbUnknown), q) q.retType.add nnkIdentDefs.newTree(newIdentNode(getColumnName(col)), toNimType(t), newEmptyNode()) q.retNames.add getColumnName(col) else: @@ -841,10 +910,12 @@ proc tableSel(n: NimNode; q: QueryBuilder) = if q.kind notin {qkUpdate, qkSelect, qkJoin}: q.head.add ")" elif n.kind in {nnkIdent, nnkAccQuoted, nnkSym} and q.kind == qkDelete: let tab = $n - let tabIndex = tableNames.lookup(tab) + let tabIndex = sourceLookup(q, tab) if tabIndex < 0: macros.error "unknown table name: " & tab & " from: " & fmtTableList(tableNames), n return + if isCteEnvIndex(tabIndex): + macros.error "cannot delete from a CTE", n escIdent(q.head, tab) q.env.add((tabindex, q.getAlias(tabIndex))) elif n.kind == nnkRStrLit: @@ -857,6 +928,37 @@ proc queryh(n: NimNode; q: QueryBuilder) = expectKind n, nnkCommand let kind = nodeName(n[0]) case kind + of "with": + expectLen n, 2 + expectKind n[1], nnkCall + if n[1].len != 2: + macros.error "with expects syntax like with cteName(select ...)", n[1] + let cteName = nodeName(n[1][0]) + if cteName.len == 0: + macros.error "with requires a CTE name", n[1][0] + if lookupCte(q.ctes, cteName) >= 0: + macros.error "duplicate CTE name: " & cteName, n[1][0] + var subq = newQueryBuilder() + subq.qmark = q.qmark + subq.aliasGen = q.aliasGen + subq.ctes = q.ctes + subq.cteBase = q.ctes.len + applyQueryNode(n[1][1], subq) + if subq.kind notin {qkSelect, qkJoin}: + macros.error "CTEs require a select-style query", n[1][1] + q.qmark = subq.qmark + q.aliasGen = subq.aliasGen + for p in subq.params: + q.params.add p + var cols: seq[CteColumn] = @[] + for i, name in subq.retNames: + let typNode = + if i < subq.retType.len and subq.retType[i].kind == nnkIdentDefs and subq.retType[i].len > 1: + subq.retType[i][1] + else: + newEmptyNode() + cols.add CteColumn(name: name, typ: DbType(kind: typeNodeToDbKind(typNode))) + q.ctes.add CteDef(name: cteName, sql: queryAsString(subq, n[1][1]), cols: cols) of "select": q.kind = qkSelect q.head = "select " @@ -905,12 +1007,13 @@ proc queryh(n: NimNode; q: QueryBuilder) = cmd[1].kind == nnkCommand and cmd[1].len == 2 and $cmd[1][0] == "on" and cmd[0].kind == nnkCall: let tab = $cmd[0][0] - let tabIndex = tableNames.lookup(tab) + let tabIndex = sourceLookup(q, tab) if tabIndex < 0: macros.error "unknown table name: " & tab & " from: " & fmtTableList(tableNames), n else: escIdent(q.join, tab) - let alias = q.getAlias(tabIndex) + let alias = if isCteEnvIndex(tabIndex): tab.toLowerAscii() & $q.aliasGen else: q.getAlias(tabIndex) + if isCteEnvIndex(tabIndex): inc q.aliasGen q.join.add " as " & alias var oldEnv = q.env q.env = @[(tabIndex, alias)] @@ -927,10 +1030,12 @@ proc queryh(n: NimNode; q: QueryBuilder) = elif cmd.kind == nnkCall: # auto join: let tab = $cmd[0] - let tabIndex = tableNames.lookup(tab) + let tabIndex = sourceLookup(q, tab) if tabIndex < 0: macros.error "unknown table name: " & tab & " from: " & fmtTableList(tableNames), n[1][0] else: + if isCteEnvIndex(tabIndex) or isCteEnvIndex(q.env[^1][0]): + macros.error "automatic joins are only supported for base tables", n let alias = q.getAlias(tabIndex) escIdent(q.join, tab) q.join.add " as " & alias @@ -1006,7 +1111,17 @@ proc queryh(n: NimNode; q: QueryBuilder) = macros.error "unknown query component " & repr(n), n proc queryAsString(q: QueryBuilder, n: NimNode): string = - result = q.head + if q.cteBase < q.ctes.len: + result.add "with " + for i in q.cteBase.. q.cteBase: + result.add ",\L" + escIdent(result, q.ctes[i].name) + result.add " as (\L" + result.add q.ctes[i].sql + result.add "\L)" + result.add "\L" + result.add q.head if q.fromm.len > 0: result.add "\Lfrom " result.add q.fromm @@ -1063,6 +1178,9 @@ proc buildSetOpQueryParts(op: string; branches: openArray[NimNode]; for i, branchNode in branches: var branch = newQueryBuilder() branch.qmark = q.qmark + branch.aliasGen = q.aliasGen + branch.ctes = q.ctes + branch.cteBase = q.ctes.len applyQueryNode(branchNode, branch) if branch.kind notin {qkSelect, qkJoin}: macros.error "set operations only support select-style queries", branchNode @@ -1077,6 +1195,7 @@ proc buildSetOpQueryParts(op: string; branches: openArray[NimNode]; q.head.add queryAsString(branch, branchNode) q.qmark = branch.qmark + q.aliasGen = branch.aliasGen for p in branch.params: q.params.add p @@ -1152,10 +1271,14 @@ proc renderInlineQuery(n: NimNode; params: var Params; qb: QueryBuilder): tuple[sql: string, typ: DbType] = var subq = newQueryBuilder() subq.qmark = qb.qmark + subq.aliasGen = qb.aliasGen + subq.ctes = qb.ctes + subq.cteBase = qb.ctes.len applyQueryNode(n, subq) if subq.kind notin {qkSelect, qkJoin}: macros.error "subqueries require a select-style query", n qb.qmark = subq.qmark + qb.aliasGen = subq.aliasGen for p in subq.params: params.add p result.sql = queryAsString(subq, n) diff --git a/tests/tfeature.nim b/tests/tfeature.nim index e676a96..e3e5324 100644 --- a/tests/tfeature.nim +++ b/tests/tfeature.nim @@ -427,6 +427,27 @@ suite "query": where not exists(select post(id) where author == ?missingAuthor) check res.sortedByIt(it) == persondata.mapIt(it.id) + test "with_cte": + let res = query: + with recent(select post(id, author) where id <= 3) + select recent(author) + orderby id + check res == postdata.filterIt(it.id <= 3).sortedByIt(it.id).mapIt(it.author) + + test "with_cte_chain": + let res = query: + with recent(select post(id, author) where id <= 3) + with authors(select recent(author)) + select authors(author) + check res.sortedByIt(it) == postdata.filterIt(it.id <= 3).mapIt(it.author).sortedByIt(it) + + test "with_cte_subquery": + let res = query: + with recent(select post(id, author) where id <= 3) + select person(id) + where id in (select recent(author)) + check res.sortedByIt(it) == postdata.filterIt(it.id <= 3).mapIt(it.author).deduplicate().sortedByIt(it) + test "union": var res = query: select person(id) where id <= 2 From d60fed01d659965b4ca0418a7ad8378185a0a8ed Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 16 Apr 2026 15:39:00 +0300 Subject: [PATCH 07/11] add support for outer joins, etc --- README.md | 6 +-- ormin/queries.nim | 101 +++++++++++++++++++++++++++++---------------- tests/tfeature.nim | 36 ++++++++++++++++ 3 files changed, 105 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 716f3f6..c570b3d 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ let db {.global.} = open("localhost", "user", "password", "dbname") ## Query DSL -`query:` blocks are turned into prepared statements at compile time. Placeholders use `?` for Nim values and `%` for JSON values; Ormin chooses JSON instead of an ad-hoc variant type so your data can flow straight from/into `JsonNode` trees. `!!` splices vendor-specific SQL fragments. Typical clauses such as `with`, `where`, `join`, `orderby`, `groupby`, `limit`, `offset`, `exists`, `distinct`, `union`/`intersect`/`except` and `returning` are supported. Referring to columns from related tables can trigger **automatic join generation** based on foreign keys, reducing boilerplate joins. +`query:` blocks are turned into prepared statements at compile time. Placeholders use `?` for Nim values and `%` for JSON values; Ormin chooses JSON instead of an ad-hoc variant type so your data can flow straight from/into `JsonNode` trees. `!!` splices vendor-specific SQL fragments. Typical clauses such as `with`, `where`, joins, `orderby`, `groupby`, `limit`, `offset`, `exists`, `distinct`, `union`/`intersect`/`except` and `returning` are supported. Referring to columns from related tables can trigger **automatic join generation** based on foreign keys, reducing boilerplate joins. Example snippets: @@ -91,7 +91,7 @@ query: # Explicit join with filter let rows = query: select Post(author) - join Person(name) on author == id + leftjoin Person(name) on author == id where id == ?postId # Automatic join generated from foreign keys @@ -149,7 +149,7 @@ Compile with `-d:debugOrminSql` to see the produced SQL at build time, which hel Selecting columns for the primary table is done using the syntax `select Post(title, author, ...)` where `Post` is the table and `title`, `author`, etc are columns of that table. This will return a tuple containing `(title, author, ...)`. Only one table can be selected and columns must be from that table. Unlike in SQL, columns for joined tables are selected directly in the `join` syntax. -Joins use the syntax `join Person(name, city) on author == id` where `Person` is the table and the columns `name`, and `city` are columns of that table. Often the join condition can be inferred from foreign keys and can be left out: `join Author(name, city)`. The columns listed in the joined tabled will be appended to the results tuple, i.e. `(title, author, name, city)`. Supported joins are: `join`, `innerjoin`, `outerjoin`. +Joins use the syntax `join Person(name, city) on author == id` where `Person` is the table and the columns `name`, and `city` are columns of that table. Often the join condition can be inferred from foreign keys and can be left out: `join Author(name, city)`. The columns listed in the joined tabled will be appended to the results tuple, i.e. `(title, author, name, city)`. Supported joins are: `join`, `innerjoin`, `leftjoin`, `leftouterjoin`, `rightjoin`, `rightouterjoin`, `fulljoin`, `fullouterjoin`, `crossjoin`, and the legacy `outerjoin`. Runtime support for `rightjoin` / `fulljoin` still depends on the SQL backend. The join syntax differs from SQL but simplifies selecting fields from multiple tables by making them more explicit while still maintaing SQL's full query capabilities. diff --git a/ormin/queries.nim b/ormin/queries.nim index 69bf139..f8afd0c 100644 --- a/ormin/queries.nim +++ b/ormin/queries.nim @@ -22,14 +22,14 @@ type arity: int # -1 for 'varargs' typ: DbTypeKind # if dbUnknown, use type of the last argument - CteColumn = object + SourceColumn = object name: string typ: DbType CteDef = object name: string sql: string - cols: seq[CteColumn] + cols: seq[SourceColumn] var functions {.compileTime.} = @[ @@ -211,13 +211,13 @@ proc sourceName(q: QueryBuilder; source: int): string {.compileTime.} = else: result = tableNames[source] -proc sourceColumns(q: QueryBuilder; source: int): seq[CteColumn] {.compileTime.} = +proc sourceColumns(q: QueryBuilder; source: int): seq[SourceColumn] {.compileTime.} = if isCteEnvIndex(source): result = q.ctes[fromCteEnvIndex(source)].cols else: for a in attributes: if a.tabIndex == source: - result.add CteColumn(name: a.name, typ: DbType(kind: a.typ)) + result.add SourceColumn(name: a.name, typ: DbType(kind: a.typ)) proc sourceLookup(q: QueryBuilder; table: string): int {.compileTime.} = for i, t in tableNames: @@ -228,6 +228,32 @@ proc sourceLookup(q: QueryBuilder; table: string): int {.compileTime.} = return cteEnvIndex(cteIdx) result = -1 +proc sourceAlias(q: QueryBuilder; source: int; sourceName: string): string {.compileTime.} = + if q.kind == qkJoin and q.env.len > 0 and q.env[^1][0] == source: + return q.env[^1][1] + if isCteEnvIndex(source): + result = sourceName.toLowerAscii() & $q.aliasGen + inc q.aliasGen + else: + result = q.getAlias(source) + +proc joinKeyword(kind: string): string {.compileTime.} = + case kind.toLowerAscii() + of "join", "innerjoin": + "inner join " + of "outerjoin": + "outer join " + of "leftjoin", "leftouterjoin": + "left outer join " + of "rightjoin", "rightouterjoin": + "right outer join " + of "fulljoin", "fullouterjoin": + "full outer join " + of "crossjoin": + "cross join " + else: + "" + proc lookup(table, attr: string; qb: QueryBuilder; alias: var string): DbType = var found = false var foundSource = -1 @@ -327,8 +353,9 @@ proc nodeName(n: NimNode): string {.compileTime.} = proc isQueryClause(name: string): bool {.compileTime.} = case name.toLowerAscii() of "with", "select", "distinct", "insert", "update", "replace", "delete", - "where", "join", "innerjoin", "outerjoin", "groupby", "orderby", - "having", "limit", "offset", "returning", "produce": + "where", "join", "innerjoin", "outerjoin", "leftjoin", "leftouterjoin", + "rightjoin", "rightouterjoin", "fulljoin", "fullouterjoin", "crossjoin", + "groupby", "orderby", "having", "limit", "offset", "returning", "produce": result = true else: result = false @@ -818,15 +845,7 @@ proc tableSel(n: NimNode; q: QueryBuilder) = if tabIndex < 0: macros.error "unknown table name: " & tab & " from: " & fmtTableList(tableNames), n return - let alias = - if q.kind == qkJoin and q.env.len > 0 and q.env[^1][0] == tabIndex: - q.env[^1][1] - elif isCteEnvIndex(tabIndex): - let a = tab.toLowerAscii() & $q.aliasGen - inc q.aliasGen - a - else: - q.getAlias(tabIndex) + let alias = sourceAlias(q, tabIndex, tab) if q.kind == qkSelect: escIdent(q.fromm, tab) q.fromm.add " as " & alias @@ -950,14 +969,14 @@ proc queryh(n: NimNode; q: QueryBuilder) = q.aliasGen = subq.aliasGen for p in subq.params: q.params.add p - var cols: seq[CteColumn] = @[] + var cols: seq[SourceColumn] = @[] for i, name in subq.retNames: let typNode = if i < subq.retType.len and subq.retType[i].kind == nnkIdentDefs and subq.retType[i].len > 1: subq.retType[i][1] else: newEmptyNode() - cols.add CteColumn(name: name, typ: DbType(kind: typeNodeToDbKind(typNode))) + cols.add SourceColumn(name: name, typ: DbType(kind: typeNodeToDbKind(typNode))) q.ctes.add CteDef(name: cteName, sql: queryAsString(subq, n[1][1]), cols: cols) of "select": q.kind = qkSelect @@ -998,11 +1017,14 @@ proc queryh(n: NimNode; q: QueryBuilder) = expectLen n, 2 let t = cond(n[1], q.where, q.params, DbType(kind: dbBool), q) checkBool(t, n) - of "join", "innerjoin", "outerjoin": - if kind == "outerjoin": q.join.add "\Louter join " - else: q.join.add "\Linner join " + of "join", "innerjoin", "outerjoin", "leftjoin", "leftouterjoin", + "rightjoin", "rightouterjoin", "fulljoin", "fullouterjoin", "crossjoin": + q.join.add "\L" & joinKeyword(kind) expectLen n, 2 let cmd = n[1] + if kind == "crossjoin" and cmd.kind == nnkCommand and cmd.len == 2 and + cmd[1].kind == nnkCommand and cmd[1].len == 2 and $cmd[1][0] == "on": + macros.error "crossjoin does not support an on clause", n if cmd.kind == nnkCommand and cmd.len == 2 and cmd[1].kind == nnkCommand and cmd[1].len == 2 and $cmd[1][0] == "on" and cmd[0].kind == nnkCall: @@ -1012,8 +1034,7 @@ proc queryh(n: NimNode; q: QueryBuilder) = macros.error "unknown table name: " & tab & " from: " & fmtTableList(tableNames), n else: escIdent(q.join, tab) - let alias = if isCteEnvIndex(tabIndex): tab.toLowerAscii() & $q.aliasGen else: q.getAlias(tabIndex) - if isCteEnvIndex(tabIndex): inc q.aliasGen + let alias = sourceAlias(q, tabIndex, tab) q.join.add " as " & alias var oldEnv = q.env q.env = @[(tabIndex, alias)] @@ -1028,24 +1049,34 @@ proc queryh(n: NimNode; q: QueryBuilder) = swap q.env, oldEnv checkBool(t, onn) elif cmd.kind == nnkCall: - # auto join: let tab = $cmd[0] let tabIndex = sourceLookup(q, tab) if tabIndex < 0: macros.error "unknown table name: " & tab & " from: " & fmtTableList(tableNames), n[1][0] else: - if isCteEnvIndex(tabIndex) or isCteEnvIndex(q.env[^1][0]): - macros.error "automatic joins are only supported for base tables", n - let alias = q.getAlias(tabIndex) - escIdent(q.join, tab) - q.join.add " as " & alias - if not autoJoin(q.join, q.env[^1], tabIndex, alias): - macros.error "cannot compute auto join from: " & tableNames[q.env[^1][0]] & " to: " & tab, n - var oldEnv = q.env - q.env = @[(tabIndex, alias)] - q.kind = qkJoin - tableSel(n[1], q) - swap q.env, oldEnv + if kind == "crossjoin": + let alias = sourceAlias(q, tabIndex, tab) + escIdent(q.join, tab) + q.join.add " as " & alias + var oldEnv = q.env + q.env = @[(tabIndex, alias)] + q.kind = qkJoin + tableSel(n[1], q) + swap q.env, oldEnv + else: + # auto join: + if isCteEnvIndex(tabIndex) or isCteEnvIndex(q.env[^1][0]): + macros.error "automatic joins are only supported for base tables", n + let alias = q.getAlias(tabIndex) + escIdent(q.join, tab) + q.join.add " as " & alias + if not autoJoin(q.join, q.env[^1], tabIndex, alias): + macros.error "cannot compute auto join from: " & tableNames[q.env[^1][0]] & " to: " & tab, n + var oldEnv = q.env + q.env = @[(tabIndex, alias)] + q.kind = qkJoin + tableSel(n[1], q) + swap q.env, oldEnv else: macros.error "unknown query component " & repr(n), n of "groupby": diff --git a/tests/tfeature.nim b/tests/tfeature.nim index e3e5324..2029349 100644 --- a/tests/tfeature.nim +++ b/tests/tfeature.nim @@ -66,6 +66,33 @@ suite "query": db.dropTable(sqlFilePath) db.createTable(sqlFilePath) + static: + doAssert compiles(block: + discard query: + select post(author) + crossjoin person(name) + ) + doAssert compiles(block: + discard query: + select post(author) + rightjoin person(name) on author == id + ) + doAssert compiles(block: + discard query: + select post(author) + rightouterjoin person(name) on author == id + ) + doAssert compiles(block: + discard query: + select post(author) + fulljoin person(name) on author == id + ) + doAssert compiles(block: + discard query: + select post(author) + fullouterjoin person(name) on author == id + ) + # prepare data to insert for i in 1..personcount: persondata.add((id: i, @@ -538,6 +565,15 @@ suite "query": let (author, name) = res[0] check name == persondata[author - 1].name + test "leftjoin_on": + let postid = 1 + let res = query: + select post(author) + leftjoin person(name) on author == id + where id == ?postid + let (author, name) = res[0] + check name == persondata[author - 1].name + test "casewhen": # need more test, only number # has problem with string or expression in condition From 056121ca86e06933b0c977583cc7614b4274cc4f Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 16 Apr 2026 15:44:52 +0300 Subject: [PATCH 08/11] add windowing functions --- README.md | 6 +++++- ormin/queries.nim | 37 +++++++++++++++++++++++++++++++++++++ tests/tfeature.nim | 22 ++++++++++++++++++++++ 3 files changed, 64 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c570b3d..2f31d15 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ let db {.global.} = open("localhost", "user", "password", "dbname") ## Query DSL -`query:` blocks are turned into prepared statements at compile time. Placeholders use `?` for Nim values and `%` for JSON values; Ormin chooses JSON instead of an ad-hoc variant type so your data can flow straight from/into `JsonNode` trees. `!!` splices vendor-specific SQL fragments. Typical clauses such as `with`, `where`, joins, `orderby`, `groupby`, `limit`, `offset`, `exists`, `distinct`, `union`/`intersect`/`except` and `returning` are supported. Referring to columns from related tables can trigger **automatic join generation** based on foreign keys, reducing boilerplate joins. +`query:` blocks are turned into prepared statements at compile time. Placeholders use `?` for Nim values and `%` for JSON values; Ormin chooses JSON instead of an ad-hoc variant type so your data can flow straight from/into `JsonNode` trees. `!!` splices vendor-specific SQL fragments. Typical clauses such as `with`, `where`, joins, `orderby`, `groupby`, `limit`, `offset`, `exists`, `distinct`, window expressions, `union`/`intersect`/`except` and `returning` are supported. Referring to columns from related tables can trigger **automatic join generation** based on foreign keys, reducing boilerplate joins. Example snippets: @@ -121,6 +121,10 @@ let recentAuthors = query: with recent(select Post(id, author) where id <= 3) select recent(author) +# Window functions use over(expr, ...) +let rankedPosts = query: + select Post(author, id, over(row_number(), partitionby(author), orderby(id)) as rn) + # Set operations can be written inline between select queries let mergedIds = query: select Person(id) where id <= 2 diff --git a/ormin/queries.nim b/ormin/queries.nim index f8afd0c..23594ab 100644 --- a/ormin/queries.nim +++ b/ormin/queries.nim @@ -39,6 +39,12 @@ var Function(name: "max", arity: 1, typ: dbUnknown), Function(name: "avg", arity: 1, typ: dbFloat), Function(name: "sum", arity: 1, typ: dbUnknown), + Function(name: "row_number", arity: 0, typ: dbInt), + Function(name: "rank", arity: 0, typ: dbInt), + Function(name: "dense_rank", arity: 0, typ: dbInt), + Function(name: "percent_rank", arity: 0, typ: dbFloat), + Function(name: "cume_dist", arity: 0, typ: dbFloat), + Function(name: "ntile", arity: 1, typ: dbInt), Function(name: "isnull", arity: 3, typ: dbUnknown), Function(name: "concat", arity: -1, typ: dbVarchar), Function(name: "abs", arity: 1, typ: dbUnknown), @@ -436,6 +442,25 @@ proc queryAsString(q: QueryBuilder, n: NimNode): string proc applyQueryNode(n: NimNode; q: QueryBuilder) proc renderInlineQuery(n: NimNode; params: var Params; qb: QueryBuilder): tuple[sql: string, typ: DbType] +proc cond(n: NimNode; q: var string; params: var Params; + expected: DbType, qb: QueryBuilder): DbType + +proc renderWindowClause(n: NimNode; q: var string; params: var Params; + qb: QueryBuilder) {.compileTime.} = + let op = nodeName(n[0]).toLowerAscii() + case op + of "partitionby": + q.add "partition by " + for i in 1.. 2: q.add " " + if n[i].kind notin nnkCallKinds: + macros.error "window clauses must be calls like partitionby(...) or orderby(...)", n[i] + renderWindowClause(n[i], q, params, qb) + q.add ")" + return if op == "exists": expectLen n, 2 let subq = renderInlineQuery(n[1], params, qb) diff --git a/tests/tfeature.nim b/tests/tfeature.nim index 2029349..6c5b975 100644 --- a/tests/tfeature.nim +++ b/tests/tfeature.nim @@ -279,6 +279,28 @@ suite "query": select post(count(distinct author)) check res == @[postdata.mapIt(it.author).deduplicate().len] + test "window_row_number": + let res = query: + select post(author, id, over(row_number(), partitionby(author), orderby(id)) as rn) + orderby author, id + var expected: seq[(int, int, int)] = @[] + var counters = initCountTable[int]() + for p in postdata.sortedByIt((it.author, it.id)): + counters.inc(p.author) + expected.add((p.author, p.id, counters[p.author])) + check res == expected + + test "window_running_sum": + let res = query: + select post(author, id, over(sum(id), partitionby(author), orderby(id)) as running) + orderby author, id + var expected: seq[(int, int, int)] = @[] + var sums = initTable[int, int]() + for p in postdata.sortedByIt((it.author, it.id)): + sums[p.author] = sums.getOrDefault(p.author) + p.id + expected.add((p.author, p.id, sums[p.author])) + check res == expected + test "limit": let id = 1 let res = query: From 9bf4643659915091979eb303ce7d7f8aaa36d4ca Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 16 Apr 2026 15:47:41 +0300 Subject: [PATCH 09/11] add predicate like / ilike --- README.md | 5 +++++ ormin.nimble | 2 +- ormin/queries.nim | 38 ++++++++++++++++++++++++++++++++++++++ tests/tfeature.nim | 21 +++++++++++++++++++++ 4 files changed, 65 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2f31d15..5dd3b1f 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,11 @@ let unassigned = query: select Ticket(id) where assignee == nil +# Pattern matching uses backticked infix operators +let matchingPeople = query: + select Person(id, name) + where name `like` ?"john%" + # EXISTS / NOT EXISTS subqueries let peopleWithPosts = query: select Person(id) diff --git a/ormin.nimble b/ormin.nimble index ba9c831..521174f 100644 --- a/ormin.nimble +++ b/ormin.nimble @@ -1,6 +1,6 @@ # Package -version = "0.7.0" +version = "0.8.0" author = "Araq" description = "Prepared SQL statement generator. A lightweight ORM." license = "MIT" diff --git a/ormin/queries.nim b/ormin/queries.nim index 23594ab..4841285 100644 --- a/ormin/queries.nim +++ b/ormin/queries.nim @@ -390,6 +390,10 @@ proc peelTrailingCommand(n: NimNode): tuple[core, tail: NimNode] {.compileTime.} nodeName(n[1][0]) == "on": return (copyNimTree(n), newEmptyNode()) + if n.kind == nnkCommand and n.len == 2 and n[1].kind == nnkCommand and + nodeName(n[1][0]).toLowerAscii() in ["like", "ilike"]: + return (copyNimTree(n), newEmptyNode()) + if n.kind == nnkCommand and n.len == 2 and n[1].kind == nnkCommand and not isQueryClause(nodeName(n[0])) and not isSetOpName(nodeName(n[0])): return (copyNimTree(n[0]), copyNimTree(n[1])) @@ -725,6 +729,40 @@ proc cond(n: NimNode; q: var string; params: var Params; let subq = renderInlineQuery(n, params, qb) q.add subq.sql result = subq.typ + elif n.len == 2 and n[1].kind == nnkCommand and n[1].len == 2: + let op = nodeName(n[1][0]).toLowerAscii() + if op in ["like", "ilike"]: + let env = qb.env + if env.len == 2: + qb.env = @[env[0]] + let a = + if op == "ilike" and dbBackend == DbBackend.sqlite: + q.add "lower(" + let t = cond(n[0], q, params, DbType(kind: dbUnknown), qb) + q.add ")" + t + else: + cond(n[0], q, params, DbType(kind: dbUnknown), qb) + q.add " " + if op == "ilike" and dbBackend != DbBackend.sqlite: + q.add "ilike" + else: + q.add "like" + q.add " " + if env.len == 2: + qb.env = @[env[1]] + let b = + if op == "ilike" and dbBackend == DbBackend.sqlite: + q.add "lower(" + let t = cond(n[1][1], q, params, a, qb) + q.add ")" + t + else: + cond(n[1][1], q, params, a, qb) + checkCompatible a, b, n + result = DbType(kind: dbBool) + else: + macros.error "construct not supported in condition: " & treeRepr n, n else: macros.error "construct not supported in condition: " & treeRepr n, n else: diff --git a/tests/tfeature.nim b/tests/tfeature.nim index 6c5b975..3f2fff1 100644 --- a/tests/tfeature.nim +++ b/tests/tfeature.nim @@ -267,6 +267,27 @@ suite "query": where id notin ?id1 .. ?id2 check res == persondata.filterIt(it.id < id1 or it.id > id2) + test "predicate_like": + let pattern = "john1%" + let res = query: + select person(id, name) + where name `like` ?pattern + check res == persondata.filterIt(it.name.startsWith("john1")).mapIt((it.id, it.name)) + + test "predicate_ilike": + let pattern = "JOHN1%" + let res = query: + select person(id, name) + where name `ilike` ?pattern + check res == persondata.filterIt(it.name.startsWith("john1")).mapIt((it.id, it.name)) + + test "predicate_not_like": + let pattern = "john1%" + let res = query: + select person(id, name) + where not (name `like` ?pattern) + check res == persondata.filterIt(not it.name.startsWith("john1")).mapIt((it.id, it.name)) + test "distinct": var res = query: select `distinct` post(author) From 3146145556e8cd6e568418c227a24fa657cd5734 Mon Sep 17 00:00:00 2001 From: Jaremy Creechley Date: Thu, 16 Apr 2026 15:50:53 +0300 Subject: [PATCH 10/11] add more join tests --- tests/tfeature.nim | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/tfeature.nim b/tests/tfeature.nim index 3f2fff1..b8f7428 100644 --- a/tests/tfeature.nim +++ b/tests/tfeature.nim @@ -72,6 +72,21 @@ suite "query": select post(author) crossjoin person(name) ) + doAssert compiles(block: + discard query: + select post(author) + outerjoin person(name) on author == id + ) + doAssert compiles(block: + discard query: + select post(author) + leftjoin person(name) on author == id + ) + doAssert compiles(block: + discard query: + select post(author) + leftouterjoin person(name) on author == id + ) doAssert compiles(block: discard query: select post(author) @@ -617,6 +632,15 @@ suite "query": let (author, name) = res[0] check name == persondata[author - 1].name + test "leftouterjoin_on": + let postid = 1 + let res = query: + select post(author) + leftouterjoin person(name) on author == id + where id == ?postid + let (author, name) = res[0] + check name == persondata[author - 1].name + test "casewhen": # need more test, only number # has problem with string or expression in condition From cb3c8145cf9f3bd3099c8be7f91c6f7f133faf9f Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Thu, 16 Apr 2026 16:57:06 +0200 Subject: [PATCH 11/11] Apply suggestions from code review Co-authored-by: Andreas Rumpf --- ormin/queries.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ormin/queries.nim b/ormin/queries.nim index 4841285..c35fb75 100644 --- a/ormin/queries.nim +++ b/ormin/queries.nim @@ -236,8 +236,8 @@ proc sourceLookup(q: QueryBuilder; table: string): int {.compileTime.} = proc sourceAlias(q: QueryBuilder; source: int; sourceName: string): string {.compileTime.} = if q.kind == qkJoin and q.env.len > 0 and q.env[^1][0] == source: - return q.env[^1][1] - if isCteEnvIndex(source): + result = q.env[^1][1] + elif isCteEnvIndex(source): result = sourceName.toLowerAscii() & $q.aliasGen inc q.aliasGen else: @@ -409,7 +409,7 @@ proc peelTrailingCommand(n: NimNode): tuple[core, tail: NimNode] {.compileTime.} result.core = copyNimTree(n) result.core[idx] = peeled.core result.tail = peeled.tail - return + return result result = (copyNimTree(n), newEmptyNode())